@socialgouv/fiches-travail-data-types 4.374.0 → 4.376.0

package/CHANGELOG.md

@@ -1,3 +1,23 @@
+# [4.376.0](https://github.com/SocialGouv/fiches-travail-data/compare/v4.375.0...v4.376.0) (2022-12-06)
+
+
+### Bug Fixes
+
+* **build:** ajout de la gestion build full typescript ✨ ([#386](https://github.com/SocialGouv/fiches-travail-data/issues/386)) ([46e5ade](https://github.com/SocialGouv/fiches-travail-data/commit/46e5ade139b31b3e4ffd220dff1136248cd3aeaa))
+* **picture:** ajout du scénario pour la balise picture + typescript / eslint ([#385](https://github.com/SocialGouv/fiches-travail-data/issues/385)) ([5dce100](https://github.com/SocialGouv/fiches-travail-data/commit/5dce100607fbfc938a96468ac6774e3924df559b))
+
+
+### Features
+
+* **data:** 20221206_1018 update ([8ee594e](https://github.com/SocialGouv/fiches-travail-data/commit/8ee594e4e29b30300a501030f37dbeeac83980a8))
+
+# [4.375.0](https://github.com/SocialGouv/fiches-travail-data/compare/v4.374.0...v4.375.0) (2022-12-01)
+
+
+### Features
+
+* **data:** 20221201_2211 update ([9bf8887](https://github.com/SocialGouv/fiches-travail-data/commit/9bf888781d07d7c8d1bdc87af5dc038fa1dcf9b3))
+
 # [4.374.0](https://github.com/SocialGouv/fiches-travail-data/compare/v4.373.0...v4.374.0) (2022-11-30)
 
 

package/build/checkRefs.d.ts

@@ -0,0 +1 @@
+export {};

package/build/checkRefs.js

@@ -0,0 +1,24 @@
+"use strict";
+const fiches = require("../data/fiches-travail.json");
+const UNDEFINED_KEY = "UNDEFINED";
+const undefinedReferences = fiches.filter((fiche) => {
+    const refErrors = fiche.sections.filter((section) => {
+        if (!section.references) {
+            console.log("no refs in " + fiche.title);
+        }
+        return section.references && UNDEFINED_KEY in section.references;
+    });
+    return refErrors.length > 0;
+});
+const printMissingRef = (fiche) => {
+    console.log(`#### [${fiche.title}](${fiche.url})`);
+    fiche.sections.forEach((section) => {
+        if (section.references && UNDEFINED_KEY in section.references) {
+            console.log(`- ${section.anchor}`);
+            const fmt = section.references[UNDEFINED_KEY].articles.map((ref) => ref.text);
+            console.log(`> ${Array.from(new Set(fmt)).join(" / ")}`);
+        }
+    });
+};
+console.log(`### ${undefinedReferences.length}/${fiches.length} fiches aux références non résolues.`);
+undefinedReferences.map((fiche) => printMissingRef(fiche));

package/build/email.d.ts

@@ -0,0 +1,3 @@
+declare function encode(str: string): string;
+declare function decode(str: string): string;
+export { decode, encode };

package/build/email.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.encode = exports.decode = void 0;
+const char = "_";
+function encode(str) {
+    return str.replace(/@/g, `${char}@`);
+}
+exports.encode = encode;
+function decode(str) {
+    return str.replace(new RegExp(`${char}@`, "g"), "@");
+}
+exports.decode = decode;

package/build/fetch-data/generateHeaders.d.ts

@@ -0,0 +1,4 @@
+/**
+ * Build the header for request with a specific token to bypass bot protection
+ */
+export function generateHeaders(extras: any): any;

package/build/fetch-data/generateHeaders.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateHeaders = void 0;
+/**
+ * Build the header for request with a specific token to bypass bot protection
+ */
+function generateHeaders(extras) {
+    if (!process.env.TOKEN_MT) {
+        throw Error("Token (cgtoken) is required to fetch the data. This token is provided by the travail-emploi.gouv.fr team.");
+    }
+    return {
+        ...extras,
+        Cookie: `cgtoken=${process.env.TOKEN_MT};`,
+    };
+}
+exports.generateHeaders = generateHeaders;

package/build/fetch-data/index.d.ts

@@ -0,0 +1,2 @@
+export function fetchFeed(url: any): Promise<any>;
+export function scrap(urls: any): Promise<any[]>;

package/build/fetch-data/index.js

@@ -0,0 +1,66 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scrap = exports.fetchFeed = void 0;
+const fs_1 = __importDefault(require("fs"));
+const got_1 = __importDefault(require("got"));
+const p_limit_1 = __importDefault(require("p-limit"));
+const path_1 = __importDefault(require("path"));
+const generateHeaders_1 = require("./generateHeaders");
+const scrapUrl_1 = require("./scrapUrl");
+const FEED_URL = "https://travail-emploi.gouv.fr/?page=oseo_json";
+const limit = (0, p_limit_1.default)(10);
+async function fetchFeed(url) {
+    const response = await got_1.default.post(url, {
+        headers: (0, generateHeaders_1.generateHeaders)({
+            "Content-Type": "application/json",
+        }),
+        http2: true,
+        retry: 3,
+    });
+    const { fiches: feed } = JSON.parse(response.body);
+    return feed;
+}
+exports.fetchFeed = fetchFeed;
+async function scrap(urls) {
+    const inputs = urls.map(({ id, url }) => limit(() => (0, scrapUrl_1.scrapUrl)(id, url)));
+    const results = await Promise.allSettled(inputs);
+    const failedPromise = results.filter(({ status }) => status === "rejected");
+    if (failedPromise.length > 0) {
+        console.error("scrap fail", failedPromise.map(({ reason }) => reason));
+        throw new Error("Error - fetching pages fail. Some pages are missing");
+    }
+    const resolvedPromise = results.flatMap(({ status, value }) => status === "fulfilled" ? [value] : []);
+    // ensure we not have duplicate url
+    let hasDuplicate = false;
+    for (const { pubId, url } of resolvedPromise) {
+        const count = resolvedPromise.filter((fiche) => fiche.pubId === pubId && pubId !== undefined).length;
+        if (count > 1) {
+            hasDuplicate = true;
+            console.error(`[error] la fiche ${url} est présente ${count} fois. Veuillez supprimer le doublon du datafiller`);
+        }
+    }
+    if (hasDuplicate) {
+        throw new Error(`[error] fiches en doublons. Veuillez supprimer les doublons du datafiller`);
+    }
+    return resolvedPromise;
+}
+exports.scrap = scrap;
+if (module === require.main) {
+    const t0 = Date.now();
+    fetchFeed(FEED_URL)
+        .then(scrap)
+        .then((fiches) => {
+        console.log(`done in ${Math.round((Date.now() - t0) / 1000)} sec`);
+        const dataFilePath = path_1.default.join(__dirname, "..", "..", "data", "fiches-travail.json");
+        fs_1.default.mkdirSync(path_1.default.dirname(dataFilePath), { recursive: true });
+        fs_1.default.writeFileSync(dataFilePath, JSON.stringify(fiches, null, 2));
+    })
+        .catch((error) => {
+        console.error(error);
+        console.error(`fail in ${Math.round((Date.now() - t0) / 1000)} sec`);
+        process.exit(1);
+    });
+}

package/build/fetch-data/parseDom.d.ts

@@ -0,0 +1,14 @@
+export function parseDom(dom: any, id: any, url: any): {
+    date: string;
+    description: any;
+    intro: any;
+    pubId: any;
+    sections: {
+        anchor: string;
+        html: string;
+        text: string;
+        title: any;
+    }[];
+    title: any;
+    url: any;
+};

package/build/fetch-data/parseDom.js

@@ -0,0 +1,211 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseDom = void 0;
+const cdtn_slugify_1 = __importDefault(require("@socialgouv/cdtn-slugify"));
+const got_1 = require("got");
+const email_1 = require("../email");
+const postProcess_1 = require("./postProcess");
+const referenceExtractor_1 = require("./referenceExtractor");
+const referenceResolver_1 = require("./referenceResolver");
+const $$ = (node, selector) => Array.from(node.querySelectorAll(selector));
+const $ = (node, selector) => node.querySelector(selector);
+function unwrapEmail(data = "") {
+    const [k, ...tokens] = Array.from({ length: data.length / 2 }, (_, i) => i * 2).map((val) => parseInt(data.slice(val, val + 2), 16));
+    const rawValue = tokens.map((v) => String.fromCharCode(v ^ k)).join("");
+    return (0, email_1.encode)(decodeURIComponent(escape(rawValue)));
+}
+const formatEmail = (node) => {
+    const value = unwrapEmail(node.getAttribute("data-cfemail"));
+    node.removeAttribute("data-cfemail");
+    node.textContent = value;
+};
+const formatPicture = (node) => {
+    const comment = node.parentElement.childNodes[0];
+    if (comment.nodeName !== "#comment") {
+        //upper sibbling node is not a comment so it's not a case we handle
+        return;
+    }
+    const [, src = ""] = comment.data.match(/src=["']([^'"]*)["']/);
+    if (src.length === 0) {
+        return;
+    }
+    let [srcClean] = src.split("?");
+    if (!srcClean.match(/^https?:\/\//)) {
+        if (srcClean.slice(0, 1) !== "/") {
+            srcClean = "/" + srcClean;
+        }
+        srcClean = `https://travail-emploi.gouv.fr${srcClean}`;
+    }
+    // we remove the ie comment that have timestamp in the url
+    comment.remove();
+    // we add e
+    const sourceNode = node.ownerDocument.createElement("source");
+    sourceNode.setAttribute("srcset", srcClean);
+    sourceNode.setAttribute("media", "(min-width: 300px)");
+    node.appendChild(sourceNode);
+    return node;
+};
+const formatAnchor = (node) => {
+    if (node.innerHTML.trim() === "") {
+        node.remove();
+        return;
+    }
+    if (node.getElementsByTagName("img").length) {
+        node.classList.add("no-after");
+    }
+    let href = node.getAttribute("href");
+    // remove ATTAg(...) on pdf link
+    node.removeAttribute("onclick");
+    if (!href)
+        return;
+    // unwrap link with href="javascript:"
+    if (/^javascript:/.test(href)) {
+        node.parentNode.innerHTML = node.textContent;
+    }
+    if (/email-protection/.test(href)) {
+        const [, data = ""] = href.split("#");
+        const value = unwrapEmail(data);
+        node.setAttribute("href", `mailto:${value}`);
+        return;
+    }
+    if (!href.match(/^https?:\/\//)) {
+        if (href.slice(0, 1) !== "/") {
+            href = "/" + href;
+        }
+        node.setAttribute("href", `https://travail-emploi.gouv.fr${href}`);
+        node.setAttribute("target", "_blank");
+        node.setAttribute("rel", "nofollow, noopener");
+    }
+};
+const flattenCsBlocs = (node) => {
+    node.insertAdjacentHTML("afterend", node.innerHTML);
+    node.parentNode.removeChild(node);
+};
+const getSectionTag = (article) => {
+    const h3 = $$(article, ".main-article__texte > h3").length && "h3";
+    const h4 = $$(article, ".main-article__texte > h4").length && "h4";
+    const h5 = $$(article, ".main-article__texte > h5").length && "h5";
+    return h3 || h4 || h5 || "sectionTag";
+};
+const getReferences = (text) => {
+    // first we extract the tokens referencing articles
+    const references = (0, referenceExtractor_1.extractReferences)(text);
+    // then we try to resolve the actual articles ids using legi-data
+    return (0, referenceResolver_1.resolveReferences)(references);
+};
+function parseDom(dom, id, url) {
+    const article = $(dom.window.document, "main");
+    if (!article) {
+        throw new got_1.ParseError("no <main>");
+    }
+    $$(article, "a").forEach(formatAnchor);
+    $$(article, "picture").forEach(formatPicture);
+    $$(article, "[data-cfemail]").forEach(formatEmail);
+    $$(article, ".cs_blocs").forEach(flattenCsBlocs);
+    const imgs = $$(article, "img");
+    imgs.forEach((node) => {
+        // remove adaptImgFix(this) on hero img
+        node.removeAttribute("onmousedown");
+    });
+    imgs
+        .filter((node) => node.getAttribute("src").indexOf("data:image") === -1)
+        .forEach((node) => {
+        let src = node.getAttribute("src");
+        if (!src.match(/^https?:\/\//)) {
+            if (src.slice(0, 1) !== "/") {
+                src = "/" + src;
+            }
+            src = `https://travail-emploi.gouv.fr${src}`;
+            node.setAttribute("src", src);
+        }
+    });
+    let titleElement = $(article, "h1");
+    if (!titleElement) {
+        titleElement = $(article, "h2");
+        if (!titleElement) {
+            throw new got_1.ParseError("No <h1> or <h2> element");
+        }
+    }
+    const title = titleElement.textContent.trim();
+    if (!id) {
+        throw new got_1.ParseError(`No id`);
+    }
+    const dateRaw = $(dom.window.document, "meta[property*=modified_time]") ||
+        $(dom.window.document, "meta[property$=published_time]");
+    const [year, month, day] = dateRaw.getAttribute("content").split("-");
+    let intro = $(article, ".main-article__chapo") || "";
+    intro =
+        intro && intro.innerHTML.replace(/\n/g, "").replace(/\s+/g, " ").trim();
+    const description = $(dom.window.document, "meta[name=description]").getAttribute("content");
+    const sections = [];
+    const sectionTag = getSectionTag(article);
+    // First pass is only to get a potential untitled section at the top of the article
+    // This section has neither anchor nor title
+    let nextArticleElement = $(article, ".main-article__texte > *");
+    const untitledSection = {
+        anchor: "",
+        html: "",
+        text: "",
+        title: title,
+    };
+    while (nextArticleElement &&
+        nextArticleElement.tagName.toLowerCase() !== sectionTag) {
+        if (nextArticleElement.textContent) {
+            if (!untitledSection.description) {
+                untitledSection.description = "temp description";
+            }
+            untitledSection.html += nextArticleElement.outerHTML
+                .replace(/\n+/g, "")
+                .replace(/>\s+</g, "><")
+                .replace(/\s+/g, " ");
+            untitledSection.text +=
+                " " + nextArticleElement.textContent.replace(/\s+/g, " ").trim();
+        }
+        nextArticleElement = nextArticleElement.nextElementSibling;
+    }
+    if (untitledSection.description) {
+        untitledSection.text.trim();
+        untitledSection.description = untitledSection.text.slice(0, 200).trim();
+        untitledSection.references = getReferences(untitledSection.text);
+        sections.push(untitledSection);
+    }
+    // Gets all the titled content
+    const articleChildren = $$(article, `.main-article__texte > ${sectionTag}`);
+    articleChildren.forEach(function (el) {
+        if (el.tagName.toLowerCase() === sectionTag) {
+            let nextEl = el.nextElementSibling;
+            let html = "";
+            while (nextEl && nextEl.tagName.toLowerCase() !== sectionTag) {
+                html += nextEl.outerHTML;
+                nextEl = nextEl.nextElementSibling;
+            }
+            const section = dom.window.document.createElement("div");
+            section.innerHTML = html;
+            const sectionText = section.textContent.replace(/\s+/g, " ").trim();
+            sections.push({
+                anchor: el.getAttribute("id") || (0, cdtn_slugify_1.default)(el.textContent),
+                description: sectionText.slice(0, 200).trim(),
+                html: (0, postProcess_1.htmlPostParser)(html.replace(/\n+/g, "").replace(/>\s+</g, "><").replace(/\s+/g, " ")),
+                references: getReferences(sectionText),
+                text: sectionText,
+                title: el.textContent.trim(),
+            });
+        }
+    });
+    if (sections.length === 0) {
+        throw new got_1.ParseError(`No sections`);
+    }
+    return {
+        date: `${day}/${month}/${year}`,
+        description,
+        intro,
+        pubId: id,
+        sections,
+        title,
+        url,
+    };
+}
+exports.parseDom = parseDom;

package/build/fetch-data/postProcess.d.ts

@@ -0,0 +1 @@
+export declare const htmlPostParser: (html: string) => string;

package/build/fetch-data/postProcess.js

@@ -0,0 +1,60 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.htmlPostParser = void 0;
+const cheerio = __importStar(require("cheerio"));
+const htmlPostParser = (html) => {
+    const $ = cheerio.load(html, null, false);
+    const arrImgSrc = [];
+    let currentIndex = 0;
+    $("style").remove();
+    // https://travail-emploi.gouv.fr/le-ministere-en-action/coronavirus-covid-19/questions-reponses-par-theme/article/mesures-de-prevention-dans-l-entreprise-contre-la-covid-19
+    $("button").remove();
+    $(".oembed-source").remove();
+    // https://travail-emploi.gouv.fr/emploi-et-insertion/accompagnement-des-mutations-economiques/activite-partielle-chomage-partiel/article/activite-partielle-chomage-partiel
+    $("*")
+        .contents()
+        .each(function () {
+        if (this.nodeType === 8) {
+            const regex = /src=['"](.*?)['"]/;
+            const result = regex.exec(this.nodeValue);
+            if (result) {
+                const src = result[0].slice(5, -1);
+                if (src)
+                    arrImgSrc.push(src);
+            }
+        }
+    });
+    $("picture").replaceWith(() => {
+        const src = arrImgSrc[currentIndex];
+        if (src) {
+            currentIndex++;
+            return `<img src="https://travail-emploi.gouv.fr/${src}" style="width:100%;height:auto;"/>`;
+        }
+        return $(this).html()?.toString() ?? "";
+    });
+    return $.html();
+};
+exports.htmlPostParser = htmlPostParser;

package/build/fetch-data/referenceExtractor.d.ts

@@ -0,0 +1,18 @@
+export function classifyTokens(tokens: any): any;
+export namespace CODE_SECU {
+    const id: string;
+    const name: string;
+}
+export namespace CODE_TRAVAIL {
+    const id_1: string;
+    export { id_1 as id };
+    const name_1: string;
+    export { name_1 as name };
+}
+export const codesFullNames: {
+    [x: string]: {
+        id: string;
+        name: string;
+    };
+};
+export function extractReferences(text: any): any;

package/build/fetch-data/referenceExtractor.js

@@ -0,0 +1,239 @@
+"use strict";
+/*
+Extracting references is done in several steps :
+1) classifyTokens : we identify valid article references (start with l/r/d then token of shape like 1234-12) => split text into sequence of tokens and give a label to each token
+2) identifyCode : we search for the associated code after the ref tokens (valid options are : code du travail / code de la scurite sociale)
+3) we group those to constitute structured reference of shape :
+  {
+    "article": "L. 2313-8",
+    "code": Object {
+      "id": "LEGITEXT000006072050",
+      "name": "code du travail",
+    },
+  }
+*/
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractReferences = exports.codesFullNames = exports.CODE_TRAVAIL = exports.CODE_SECU = exports.classifyTokens = void 0;
+const treebank_1 = __importDefault(require("talisman/tokenizers/words/treebank"));
+const NEGATIVE = "O";
+const ARTICLE = "B-ART";
+const CODE_PREFIX = "B-COD";
+// code du travail
+const CODE_TRA = CODE_PREFIX + "_TRA";
+// code sécurité sociale
+const CODE_SS = CODE_PREFIX + "_SS";
+// code any other
+const CODE_OTHER = CODE_PREFIX + "_O";
+const UNRECOGNIZED = "unrecognized";
+const CODE_TRAVAIL = {
+    id: "LEGITEXT000006072050",
+    name: "code du travail",
+};
+exports.CODE_TRAVAIL = CODE_TRAVAIL;
+const CODE_SECU = {
+    id: "LEGITEXT000006073189",
+    name: "code de la sécurité sociale",
+};
+exports.CODE_SECU = CODE_SECU;
+const codesFullNames = {
+    [CODE_SS]: CODE_SECU,
+    [CODE_TRA]: CODE_TRAVAIL,
+};
+exports.codesFullNames = codesFullNames;
+// maximum distance between code tokens and corresponding article ref
+const range = 20;
+const articleRegEx = new RegExp("^(\\d{1,4}(-\\d+){0,3})\\b"); //          nums        123 123-45 123-45-6 123-45-6-7
+function articleMatcher(token) {
+    return token.match(articleRegEx);
+}
+const validPrefix = ["l", "r", "d"];
+// returns :
+// 0 if not matching
+// 1 if matching prefix only (L.)
+// 2 if matching prefix and valid ref (L123.12)
+function prefixMatcher(token) {
+    const lowToken = token.toLowerCase();
+    // if starts with possible prefix
+    const matchingPrefix = validPrefix.filter((p) => lowToken.startsWith(p)).length > 0;
+    if (matchingPrefix) {
+        const residual = lowToken.slice(1);
+        // case only L
+        if (!residual.length) {
+            return 1;
+        }
+        else {
+            // case L.
+            if (residual == ".") {
+                return 1;
+            }
+            else {
+                // case L.123-12
+                if (residual.slice(0, 1) == "." && articleMatcher(residual.slice(1))) {
+                    return 2;
+                }
+                // case L.123-12
+                else if (articleMatcher(residual.slice(1))) {
+                    return 2;
+                }
+            }
+        }
+    }
+    // no match
+    return 0;
+}
+function infixMatcher(token) {
+    // this is quite subtle...
+    return ["à", "à"].includes(token);
+}
+// classify sequence of tokens to identify references to articles
+function classifyTokens(tokens) {
+    // step 1 : check for prefix matches or articles
+    const step1 = tokens.map((token) => {
+        const prefix = prefixMatcher(token);
+        const infix = infixMatcher(token);
+        const article = articleMatcher(token);
+        if (prefix > 0) {
+            return prefix;
+        }
+        else if (article) {
+            return 3;
+        }
+        else if (infix) {
+            return 4;
+        }
+        else {
+            return 0;
+        }
+    });
+    // step 2 : confirm valid sequences
+    // hack : we keep a buffer as last element of the accumulator
+    const predictions = step1.reduce((acc, e) => {
+        const buffer = acc[acc.length - 1];
+        const inSequence = buffer.length > 0;
+        const lastElement = buffer[buffer.length - 1];
+        // case continue existing
+        if (e >= 1 && inSequence) {
+            buffer.push(e);
+        }
+        // case finish existing
+        else if (e == 0 && inSequence && lastElement > 1) {
+            acc.pop();
+            // push buffer
+            buffer.forEach(() => acc.push(true));
+            // push for current
+            acc.push(false);
+            acc.push([]);
+        }
+        // case start (valid start are 1 or 2, as 3 is number only without prefix)
+        else if (e > 0 && e < 3 && !inSequence) {
+            buffer.push(e);
+        }
+        // other cases, flush buffer and append current
+        else {
+            acc.pop();
+            acc.push(...buffer.map(() => false));
+            acc.push(false);
+            acc.push([]);
+        }
+        return acc;
+    }, [[]]);
+    // conclude
+    const residual = predictions.pop();
+    // if ends with bigger than 1, then add residual as true
+    if (residual.length > 0 && residual[residual.length - 1] > 1) {
+        predictions.push(...residual.map(() => true));
+    }
+    else {
+        predictions.push(...residual.map(() => false));
+    }
+    return predictions.map((p) => (p ? ARTICLE : NEGATIVE));
+}
+exports.classifyTokens = classifyTokens;
+function identifyCodes(tokens, predicitions) {
+    // we look for "code" tokens (starting a code reference)
+    const matchCode = tokens.map((token, i) => {
+        return token.toLowerCase() == "code" ? CODE_PREFIX : predicitions[i];
+    });
+    // we search for entire code references
+    const resolvedCodePreds = matchCode.map((pred, i) => {
+        if (pred == CODE_PREFIX) {
+            const joinedNextTokens = tokens
+                .slice(i, i + 5)
+                .join(" ")
+                .toLowerCase();
+            if (joinedNextTokens.startsWith(codesFullNames[CODE_SS].name)) {
+                return CODE_SS;
+            }
+            else if (joinedNextTokens.startsWith(codesFullNames[CODE_TRA].name)) {
+                return CODE_TRA;
+            }
+            else {
+                return CODE_OTHER;
+            }
+        }
+        else {
+            return pred;
+        }
+    });
+    return resolvedCodePreds;
+}
+// extract references from free text : tokenize and classify
+function extractReferences(text) {
+    const tokens = (0, treebank_1.default)(text);
+    let predictions = classifyTokens(tokens);
+    predictions = identifyCodes(tokens, predictions);
+    // console.log(tokens);
+    // console.log(predictions);
+    // group continuous positives tokens and set code
+    // while continuous match, merge
+    // if code, then associate it to articles within range
+    return tokens
+        .map((token, index) => {
+        return { index, pred: predictions[index], token };
+    })
+        .reduce((acc, { token, index, pred }) => {
+        // case article : we start or merge
+        if (pred == ARTICLE) {
+            if (acc.length == 0) {
+                acc.push({ index, token });
+            }
+            else {
+                const last = acc[acc.length - 1];
+                // case continuous : we merge
+                if (last.index + 1 == index) {
+                    last.token = `${last.token} ${token}`;
+                    last.index = index;
+                }
+                else {
+                    acc.push({ index, token });
+                }
+            }
+        }
+        // case code, we associate it to articles within range
+        else if (pred.startsWith(CODE_PREFIX) && acc.length > 0) {
+            acc.forEach((match) => {
+                // if no code yet and in range
+                if (!match.code && match.index + range >= index) {
+                    if (pred in codesFullNames) {
+                        match.code = codesFullNames[pred];
+                    }
+                    else {
+                        match.code = UNRECOGNIZED;
+                    }
+                }
+            });
+        }
+        return acc;
+    }, [])
+        .filter(({ code }) => {
+        // valid cases are no code or code different than UNRECOGNIZED (for other codes : rural, education...)
+        return !code || (code && code != UNRECOGNIZED);
+    })
+        .map(({ token, code }) => {
+        return { code, text: token };
+    });
+}
+exports.extractReferences = extractReferences;

package/build/fetch-data/referenceResolver.d.ts

@@ -0,0 +1 @@
+export function resolveReferences(refs: any): any;

package/build/fetch-data/referenceResolver.js

@@ -0,0 +1,176 @@
+"use strict";
+/*
+Here we resolve the references :
+Given an article (or a range) and its code (code du travail ou securite sociale), we search for its
+actual id in the legi data corpus.
+*/
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveReferences = void 0;
+const unist_util_find_1 = __importDefault(require("unist-util-find"));
+const unist_util_visit_1 = __importDefault(require("unist-util-visit"));
+const referenceExtractor_1 = require("./referenceExtractor");
+const codes = {};
+Object.values(referenceExtractor_1.codesFullNames).forEach(({ id }) => {
+    const code = require(`@socialgouv/legi-data/data/${id}.json`);
+    codes[id] = code;
+});
+// duplicated in reference Extractor
+const rangeMarkers = ["à", "à"];
+const CODE_UNKNOWN = { id: "UNDEFINED" };
+// shall we use "code du travail" by default ?
+const DEFAULT_CODE = referenceExtractor_1.CODE_TRAVAIL;
+// dumb convert article.data.num as integer for comparison
+// each part up to MAX_DEPTH is padded with PAD_LENGTH
+const PAD_LENGTH = 5; // left pad numbers to X chars
+const MAX_DEPTH = 5; // max number of L432-1-1-1
+// padding numbers : 2 -> "0002"
+const leftPad = (num) => {
+    let padded = "" + num;
+    while (padded.length < PAD_LENGTH) {
+        padded = "0" + padded;
+    }
+    return padded;
+};
+// transform articles into comparable integers
+const asInt = (num) => {
+    const parts = num
+        .replace(/[^\d-]/g, "")
+        .split("-")
+        .map(leftPad);
+    while (parts.length < MAX_DEPTH) {
+        parts.push(leftPad(0));
+    }
+    const int = parseInt(parts.join(""));
+    return int;
+};
+function getLegiDataRange(code, start, end) {
+    // check if num is numerically after start. also check LRD prefix
+    const isAfterStart = (node) => asInt(node.data.num) >= asInt(start) &&
+        node.data.num.charAt(0) === start.charAt(0);
+    // check if num is numerically before end. also check LRD prefix
+    const isBeforeEnd = (node) => asInt(node.data.num) <= asInt(end) &&
+        node.data.num.charAt(0) === end.charAt(0);
+    const articles = [];
+    (0, unist_util_visit_1.default)(code, "article", (node) => {
+        if (isAfterStart(node) && isBeforeEnd(node)) {
+            articles.push(node);
+        }
+    });
+    return articles;
+}
+function formatStartEnd(startRaw, endRaw) {
+    // we need to identify special case where end ref is relative to start ref (e.g. L. 4733-9 à 11)
+    // if there's nothing in common between end and start, we consider being in this special case
+    const [startParts, endParts] = [startRaw, endRaw].map((a) => a
+        .replace(/\u2011/g, "-")
+        .replace()
+        .split("-")
+        .map((p) => p.trim()));
+    const letter = startParts[0].slice(0, 1);
+    const startNums = Array.from(startParts);
+    startNums[0] = startNums[0].replace(/\D/g, "");
+    let endNums = Array.from(endParts);
+    endNums[0] = endNums[0].replace(/\D/g, "");
+    if (endNums.length == 1 && /^\d+$/.test(endParts[0])) {
+        const endRange = endNums[0];
+        endNums = Array.from(startNums.slice(0, -1));
+        endNums.push(endRange);
+    }
+    return [letter + startNums.join("-"), letter + endNums.join("-")];
+}
+// in case of a range (like "L. 4733-9 à 4733-11"), we try to identify
+// the articles implicitly included within the range
+function unravelRange(range) {
+    const mark = rangeMarkers.filter((a) => range.text.includes(a))[0];
+    const rawParts = range.text.split(mark).map((p) => p.trim());
+    const code = range.code ? range.code : DEFAULT_CODE;
+    if (rawParts.length == 2 && code != CODE_UNKNOWN) {
+        // objective is to identify starting and ending articles (with the legi data correct format)
+        // then we can do a legi-data lookup
+        const [startRaw, endRaw] = rawParts;
+        const [startFMT, endFMT] = formatStartEnd(startRaw, endRaw);
+        const unraveled = getLegiDataRange(codes[code.id], startFMT, endFMT).map((a) => {
+            const fmt = a.data.num;
+            // keep original text for beginning and end
+            let text;
+            if (startFMT == fmt) {
+                text = startRaw;
+            }
+            else if (endFMT == fmt) {
+                text = endRaw;
+            }
+            return { ...(text && { text }), code, fmt };
+        });
+        if (unraveled.length > 0) {
+            return unraveled;
+        }
+    }
+    // default in case of error, note that we explicitly set code to unknown
+    // in order to identify range errors
+    return range.text.split(mark).map((a) => {
+        return { code: CODE_UNKNOWN, text: a.trim() };
+    });
+}
+function formatArticle(article) {
+    // remove dot and spaces + remove non digit trailing chars + replace unicode dash ‑ to standard -
+    return article
+        .toUpperCase()
+        .replace(".", "")
+        .replace(" ", "")
+        .replace(/\D*$/, "")
+        .replace(/\u2011/g, "-");
+}
+function resolveReference(ref) {
+    let toResolve = [ref];
+    if (rangeMarkers.filter((a) => ref.text.includes(a)).length != 0) {
+        toResolve = unravelRange(ref);
+    }
+    return toResolve.map((a) => {
+        // use default code if no defined
+        const code = (a.code == CODE_UNKNOWN) | (a.code == undefined) ? DEFAULT_CODE : a.code;
+        if (!a.fmt)
+            a.fmt = formatArticle(a.text);
+        if (code && code != CODE_UNKNOWN) {
+            const article = (0, unist_util_find_1.default)(codes[code.id], (node) => node.type === "article" && node.data.num === a.fmt);
+            if (article) {
+                a.cid = article.data.cid;
+                a.id = article.data.id;
+                a.code = code;
+            }
+            else {
+                // not found in code
+                a.code = CODE_UNKNOWN;
+            }
+        }
+        return a;
+    });
+}
+function resolveReferences(refs) {
+    const resolvedRefs = refs.map((ref) => resolveReference(ref)).flat();
+    const deduplicated = resolvedRefs.reduce((acc, art) => {
+        // drop duplicated references
+        const existing = acc
+            .map((a) => [a.text, a.fmt])
+            .flat()
+            .filter((v) => v);
+        if (!(existing.includes(art.fmt) || existing.includes(art.text))) {
+            acc.push(art);
+        }
+        return acc;
+    }, []);
+    // group by code
+    const grouped = deduplicated.reduce((acc, art) => {
+        const { code, ...rawArticle } = art;
+        const parsedCode = code ? code : CODE_UNKNOWN;
+        if (!Object.keys(acc).includes(parsedCode.id)) {
+            acc[parsedCode.id] = { articles: [], name: parsedCode.name };
+        }
+        acc[parsedCode.id].articles.push(rawArticle);
+        return acc;
+    }, {});
+    return grouped;
+}
+exports.resolveReferences = resolveReferences;

package/build/fetch-data/scrapUrl.d.ts

@@ -0,0 +1,14 @@
+export function scrapUrl(id: any, url: any): Promise<{
+    date: string;
+    description: any;
+    intro: any;
+    pubId: any;
+    sections: {
+        anchor: string;
+        html: string;
+        text: string;
+        title: any;
+    }[];
+    title: any;
+    url: any;
+}>;

package/build/fetch-data/scrapUrl.js

@@ -0,0 +1,52 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scrapUrl = void 0;
+const got_1 = __importDefault(require("got"));
+const jsdom_1 = require("jsdom");
+const generateHeaders_1 = require("./generateHeaders");
+const parseDom_1 = require("./parseDom");
+async function scrapUrl(id, url) {
+    const headers = (0, generateHeaders_1.generateHeaders)();
+    try {
+        let response = await (0, got_1.default)(url, {
+            followRedirect: true,
+            headers,
+            http2: true,
+            retry: 3,
+        });
+        if (/HTTP 30\d/.test(response.body)) {
+            const [, redirectUrl] = response.body.match(/href="(.*)"/);
+            try {
+                response = await (0, got_1.default)(redirectUrl, {
+                    followRedirect: true,
+                    headers,
+                    http2: true,
+                    retry: 3,
+                });
+            }
+            catch (error) {
+                throw new Error(`Wrong redirectUrl: ${url} => ${redirectUrl}`);
+            }
+        }
+        const dom = new jsdom_1.JSDOM(response.body, { url });
+        return (0, parseDom_1.parseDom)(dom, id, url);
+    }
+    catch (error) {
+        let err;
+        if (error instanceof got_1.default.ParseError) {
+            err = new Error(`Parsing Error: ${error.message}`);
+        }
+        else if (error instanceof got_1.default.HTTPError) {
+            err = new Error(`HTTP Error: ${error.response.statusCode} - ${error.options.url.href} - ${error.message}`);
+        }
+        else {
+            err = new Error(error.message);
+        }
+        err.url = url;
+        throw err;
+    }
+}
+exports.scrapUrl = scrapUrl;

package/build/index.d.ts

@@ -0,0 +1,3 @@
+import { decode } from "./email";
+export { decode };
+export * from "./types";

package/build/index.js

@@ -0,0 +1,20 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.decode = void 0;
+const email_1 = require("./email");
+Object.defineProperty(exports, "decode", { enumerable: true, get: function () { return email_1.decode; } });
+__exportStar(require("./types"), exports);

package/build/types.d.ts

@@ -0,0 +1,29 @@
+export type FicheTravailEmploi = {
+    date: string;
+    description: string;
+    intro: string;
+    pubId: string;
+    sections: Section[];
+    title: string;
+    url: string;
+};
+export type Section = {
+    anchor: string;
+    description: string;
+    html: string;
+    references: ReferencesMap;
+    text: string;
+    title: string;
+};
+export type ReferencesMap = {
+    [key: string]: {
+        name: string;
+        articles: ReferenceFTE[];
+    };
+};
+export type ReferenceFTE = {
+    id: string;
+    cid: string;
+    fmt: string;
+    text: string;
+};

package/build/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -1,16 +1,12 @@
 {
   "name": "@socialgouv/fiches-travail-data-types",
-  "version": "4.374.0",
-  "main": "index.js",
-  "module": "index.esm.js",
+  "version": "4.376.0",
+  "main": "build/index.js",
+  "module": "build/index.js",
   "files": [
-    "data",
-    "index.js",
-    "index.esm.js",
-    "index.d.ts",
-    "README.md"
+    "build"
   ],
-  "types": "index.d.ts",
+  "types": "build/index.d.ts",
   "repository": {
     "type": "git",
     "url": "https://github.com/SocialGouv/fiches-travail-data.git"
@@ -21,45 +17,26 @@
     "access": "public"
   },
   "scripts": {
-    "build": "rollup -c",
-    "start": "node -r esm src/fetch-data",
-    "checkRefs": "node -r esm src/checkRefs",
-    "lint": "eslint src --ext .js",
+    "build": "tsc",
+    "start": "node build/fetch-data",
+    "checkRefs": "node build/checkRefs",
+    "lint": "eslint \"./src/**/*.{js,ts}\"",
     "test": "jest"
   },
   "devDependencies": {
     "@babel/core": "^7.16.0",
     "@babel/preset-env": "^7.16.4",
-    "@socialgouv/cdtn-slugify": "^4.60.3",
-    "@socialgouv/eslint-config-recommended": "^1.100.0",
-    "@socialgouv/legi-data": "^2.100.0",
+    "@swc/core": "^1.3.21",
+    "@swc/jest": "^0.2.23",
+    "@typescript-eslint/eslint-plugin": "^5.45.0",
+    "@typescript-eslint/parser": "^5.45.0",
     "babel-jest": "^27.4.4",
-    "eslint": "^8.0.0",
-    "esm": "^3.2.25",
-    "got": "^11.8.3",
+    "eslint": "^8.28.0",
+    "eslint-plugin-jest": "^27.1.6",
     "husky": "^7.0.4",
-    "jest": "^27.4.4",
+    "jest": "^29.3.1",
     "jsdom": "^17.0.0",
-    "npm-run-all": "^4.1.5",
-    "p-limit": "^3.1.0",
     "prettier": "^2.5.1",
-    "rollup": "^2.60.2",
-    "superagent": "^6.1.0",
-    "talisman": "^1.1.4",
-    "unist-util-find": "^1.0.2",
-    "unist-util-visit": "^2.0.3"
-  },
-  "jest": {
-    "roots": [
-      "<rootDir>/src"
-    ],
-    "transform": {
-      "^.+\\.jsx?$": "babel-jest"
-    },
-    "testRegex": "(/__tests__/.*|(\\.|/)(test|spec))\\.jsx?$",
-    "moduleFileExtensions": [
-      "js",
-      "json"
-    ]
+    "typescript": "^4.9.3"
   }
 }

package/index.d.ts

@@ -1,32 +0,0 @@
-export type FicheTravailEmploi = {
-  date: string
-  description: string
-  intro: string
-  pubId: string
-  sections: Section[]
-  title: string
-  url: string
-}
-
-export type Section = {
-  anchor: string
-  description: string
-  html: string
-  references: ReferencesMap
-  text: string
-  title: string
-}
-
-export type ReferencesMap = {
-  [key: string]: {
-    name: string
-    articles: ReferenceFTE[]
-  }
-}
-
-export type ReferenceFTE = {
-  id: string
-  cid: string
-  fmt: string
-  text: string
-}

package/index.esm.js

@@ -1,7 +0,0 @@
-const char = "_";
-
-function decode(str) {
-  return str.replace(new RegExp(`${char}@`, "g"), "@");
-}
-
-export { decode };

package/index.js

@@ -1,11 +0,0 @@
-'use strict';
-
-Object.defineProperty(exports, '__esModule', { value: true });
-
-const char = "_";
-
-function decode(str) {
-  return str.replace(new RegExp(`${char}@`, "g"), "@");
-}
-
-exports.decode = decode;