@socialgouv/fiches-travail-data 4.374.0 → 4.376.0

package/CHANGELOG.md

@@ -1,3 +1,23 @@
+# [4.376.0](https://github.com/SocialGouv/fiches-travail-data/compare/v4.375.0...v4.376.0) (2022-12-06)
+
+
+### Bug Fixes
+
+* **build:** ajout de la gestion build full typescript ✨ ([#386](https://github.com/SocialGouv/fiches-travail-data/issues/386)) ([46e5ade](https://github.com/SocialGouv/fiches-travail-data/commit/46e5ade139b31b3e4ffd220dff1136248cd3aeaa))
+* **picture:** ajout du scénario pour la balise picture + typescript / eslint ([#385](https://github.com/SocialGouv/fiches-travail-data/issues/385)) ([5dce100](https://github.com/SocialGouv/fiches-travail-data/commit/5dce100607fbfc938a96468ac6774e3924df559b))
+
+
+### Features
+
+* **data:** 20221206_1018 update ([8ee594e](https://github.com/SocialGouv/fiches-travail-data/commit/8ee594e4e29b30300a501030f37dbeeac83980a8))
+
+# [4.375.0](https://github.com/SocialGouv/fiches-travail-data/compare/v4.374.0...v4.375.0) (2022-12-01)
+
+
+### Features
+
+* **data:** 20221201_2211 update ([9bf8887](https://github.com/SocialGouv/fiches-travail-data/commit/9bf888781d07d7c8d1bdc87af5dc038fa1dcf9b3))
+
 # [4.374.0](https://github.com/SocialGouv/fiches-travail-data/compare/v4.373.0...v4.374.0) (2022-11-30)
 
 

package/build/checkRefs.d.ts

@@ -0,0 +1 @@
+export {};

package/build/checkRefs.js

@@ -0,0 +1,24 @@
+"use strict";
+const fiches = require("../data/fiches-travail.json");
+const UNDEFINED_KEY = "UNDEFINED";
+const undefinedReferences = fiches.filter((fiche) => {
+    const refErrors = fiche.sections.filter((section) => {
+        if (!section.references) {
+            console.log("no refs in " + fiche.title);
+        }
+        return section.references && UNDEFINED_KEY in section.references;
+    });
+    return refErrors.length > 0;
+});
+const printMissingRef = (fiche) => {
+    console.log(`#### [${fiche.title}](${fiche.url})`);
+    fiche.sections.forEach((section) => {
+        if (section.references && UNDEFINED_KEY in section.references) {
+            console.log(`- ${section.anchor}`);
+            const fmt = section.references[UNDEFINED_KEY].articles.map((ref) => ref.text);
+            console.log(`> ${Array.from(new Set(fmt)).join(" / ")}`);
+        }
+    });
+};
+console.log(`### ${undefinedReferences.length}/${fiches.length} fiches aux références non résolues.`);
+undefinedReferences.map((fiche) => printMissingRef(fiche));

package/build/email.d.ts

@@ -0,0 +1,3 @@
+declare function encode(str: string): string;
+declare function decode(str: string): string;
+export { decode, encode };

package/build/email.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.encode = exports.decode = void 0;
+const char = "_";
+function encode(str) {
+    return str.replace(/@/g, `${char}@`);
+}
+exports.encode = encode;
+function decode(str) {
+    return str.replace(new RegExp(`${char}@`, "g"), "@");
+}
+exports.decode = decode;

package/build/fetch-data/generateHeaders.d.ts

@@ -0,0 +1,4 @@
+/**
+ * Build the header for request with a specific token to bypass bot protection
+ */
+export function generateHeaders(extras: any): any;

package/build/fetch-data/generateHeaders.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateHeaders = void 0;
+/**
+ * Build the header for request with a specific token to bypass bot protection
+ */
+function generateHeaders(extras) {
+    if (!process.env.TOKEN_MT) {
+        throw Error("Token (cgtoken) is required to fetch the data. This token is provided by the travail-emploi.gouv.fr team.");
+    }
+    return {
+        ...extras,
+        Cookie: `cgtoken=${process.env.TOKEN_MT};`,
+    };
+}
+exports.generateHeaders = generateHeaders;

package/build/fetch-data/index.d.ts

@@ -0,0 +1,2 @@
+export function fetchFeed(url: any): Promise<any>;
+export function scrap(urls: any): Promise<any[]>;

package/build/fetch-data/index.js

@@ -0,0 +1,66 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scrap = exports.fetchFeed = void 0;
+const fs_1 = __importDefault(require("fs"));
+const got_1 = __importDefault(require("got"));
+const p_limit_1 = __importDefault(require("p-limit"));
+const path_1 = __importDefault(require("path"));
+const generateHeaders_1 = require("./generateHeaders");
+const scrapUrl_1 = require("./scrapUrl");
+const FEED_URL = "https://travail-emploi.gouv.fr/?page=oseo_json";
+const limit = (0, p_limit_1.default)(10);
+async function fetchFeed(url) {
+    const response = await got_1.default.post(url, {
+        headers: (0, generateHeaders_1.generateHeaders)({
+            "Content-Type": "application/json",
+        }),
+        http2: true,
+        retry: 3,
+    });
+    const { fiches: feed } = JSON.parse(response.body);
+    return feed;
+}
+exports.fetchFeed = fetchFeed;
+async function scrap(urls) {
+    const inputs = urls.map(({ id, url }) => limit(() => (0, scrapUrl_1.scrapUrl)(id, url)));
+    const results = await Promise.allSettled(inputs);
+    const failedPromise = results.filter(({ status }) => status === "rejected");
+    if (failedPromise.length > 0) {
+        console.error("scrap fail", failedPromise.map(({ reason }) => reason));
+        throw new Error("Error - fetching pages fail. Some pages are missing");
+    }
+    const resolvedPromise = results.flatMap(({ status, value }) => status === "fulfilled" ? [value] : []);
+    // ensure we not have duplicate url
+    let hasDuplicate = false;
+    for (const { pubId, url } of resolvedPromise) {
+        const count = resolvedPromise.filter((fiche) => fiche.pubId === pubId && pubId !== undefined).length;
+        if (count > 1) {
+            hasDuplicate = true;
+            console.error(`[error] la fiche ${url} est présente ${count} fois. Veuillez supprimer le doublon du datafiller`);
+        }
+    }
+    if (hasDuplicate) {
+        throw new Error(`[error] fiches en doublons. Veuillez supprimer les doublons du datafiller`);
+    }
+    return resolvedPromise;
+}
+exports.scrap = scrap;
+if (module === require.main) {
+    const t0 = Date.now();
+    fetchFeed(FEED_URL)
+        .then(scrap)
+        .then((fiches) => {
+        console.log(`done in ${Math.round((Date.now() - t0) / 1000)} sec`);
+        const dataFilePath = path_1.default.join(__dirname, "..", "..", "data", "fiches-travail.json");
+        fs_1.default.mkdirSync(path_1.default.dirname(dataFilePath), { recursive: true });
+        fs_1.default.writeFileSync(dataFilePath, JSON.stringify(fiches, null, 2));
+    })
+        .catch((error) => {
+        console.error(error);
+        console.error(`fail in ${Math.round((Date.now() - t0) / 1000)} sec`);
+        process.exit(1);
+    });
+}

package/build/fetch-data/parseDom.d.ts

@@ -0,0 +1,14 @@
+export function parseDom(dom: any, id: any, url: any): {
+    date: string;
+    description: any;
+    intro: any;
+    pubId: any;
+    sections: {
+        anchor: string;
+        html: string;
+        text: string;
+        title: any;
+    }[];
+    title: any;
+    url: any;
+};

package/build/fetch-data/parseDom.js

@@ -0,0 +1,211 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseDom = void 0;
+const cdtn_slugify_1 = __importDefault(require("@socialgouv/cdtn-slugify"));
+const got_1 = require("got");
+const email_1 = require("../email");
+const postProcess_1 = require("./postProcess");
+const referenceExtractor_1 = require("./referenceExtractor");
+const referenceResolver_1 = require("./referenceResolver");
+const $$ = (node, selector) => Array.from(node.querySelectorAll(selector));
+const $ = (node, selector) => node.querySelector(selector);
+function unwrapEmail(data = "") {
+    const [k, ...tokens] = Array.from({ length: data.length / 2 }, (_, i) => i * 2).map((val) => parseInt(data.slice(val, val + 2), 16));
+    const rawValue = tokens.map((v) => String.fromCharCode(v ^ k)).join("");
+    return (0, email_1.encode)(decodeURIComponent(escape(rawValue)));
+}
+const formatEmail = (node) => {
+    const value = unwrapEmail(node.getAttribute("data-cfemail"));
+    node.removeAttribute("data-cfemail");
+    node.textContent = value;
+};
+const formatPicture = (node) => {
+    const comment = node.parentElement.childNodes[0];
+    if (comment.nodeName !== "#comment") {
+        //upper sibbling node is not a comment so it's not a case we handle
+        return;
+    }
+    const [, src = ""] = comment.data.match(/src=["']([^'"]*)["']/);
+    if (src.length === 0) {
+        return;
+    }
+    let [srcClean] = src.split("?");
+    if (!srcClean.match(/^https?:\/\//)) {
+        if (srcClean.slice(0, 1) !== "/") {
+            srcClean = "/" + srcClean;
+        }
+        srcClean = `https://travail-emploi.gouv.fr${srcClean}`;
+    }
+    // we remove the ie comment that have timestamp in the url
+    comment.remove();
+    // we add e
+    const sourceNode = node.ownerDocument.createElement("source");
+    sourceNode.setAttribute("srcset", srcClean);
+    sourceNode.setAttribute("media", "(min-width: 300px)");
+    node.appendChild(sourceNode);
+    return node;
+};
+const formatAnchor = (node) => {
+    if (node.innerHTML.trim() === "") {
+        node.remove();
+        return;
+    }
+    if (node.getElementsByTagName("img").length) {
+        node.classList.add("no-after");
+    }
+    let href = node.getAttribute("href");
+    // remove ATTAg(...) on pdf link
+    node.removeAttribute("onclick");
+    if (!href)
+        return;
+    // unwrap link with href="javascript:"
+    if (/^javascript:/.test(href)) {
+        node.parentNode.innerHTML = node.textContent;
+    }
+    if (/email-protection/.test(href)) {
+        const [, data = ""] = href.split("#");
+        const value = unwrapEmail(data);
+        node.setAttribute("href", `mailto:${value}`);
+        return;
+    }
+    if (!href.match(/^https?:\/\//)) {
+        if (href.slice(0, 1) !== "/") {
+            href = "/" + href;
+        }
+        node.setAttribute("href", `https://travail-emploi.gouv.fr${href}`);
+        node.setAttribute("target", "_blank");
+        node.setAttribute("rel", "nofollow, noopener");
+    }
+};
+const flattenCsBlocs = (node) => {
+    node.insertAdjacentHTML("afterend", node.innerHTML);
+    node.parentNode.removeChild(node);
+};
+const getSectionTag = (article) => {
+    const h3 = $$(article, ".main-article__texte > h3").length && "h3";
+    const h4 = $$(article, ".main-article__texte > h4").length && "h4";
+    const h5 = $$(article, ".main-article__texte > h5").length && "h5";
+    return h3 || h4 || h5 || "sectionTag";
+};
+const getReferences = (text) => {
+    // first we extract the tokens referencing articles
+    const references = (0, referenceExtractor_1.extractReferences)(text);
+    // then we try to resolve the actual articles ids using legi-data
+    return (0, referenceResolver_1.resolveReferences)(references);
+};
+function parseDom(dom, id, url) {
+    const article = $(dom.window.document, "main");
+    if (!article) {
+        throw new got_1.ParseError("no <main>");
+    }
+    $$(article, "a").forEach(formatAnchor);
+    $$(article, "picture").forEach(formatPicture);
+    $$(article, "[data-cfemail]").forEach(formatEmail);
+    $$(article, ".cs_blocs").forEach(flattenCsBlocs);
+    const imgs = $$(article, "img");
+    imgs.forEach((node) => {
+        // remove adaptImgFix(this) on hero img
+        node.removeAttribute("onmousedown");
+    });
+    imgs
+        .filter((node) => node.getAttribute("src").indexOf("data:image") === -1)
+        .forEach((node) => {
+        let src = node.getAttribute("src");
+        if (!src.match(/^https?:\/\//)) {
+            if (src.slice(0, 1) !== "/") {
+                src = "/" + src;
+            }
+            src = `https://travail-emploi.gouv.fr${src}`;
+            node.setAttribute("src", src);
+        }
+    });
+    let titleElement = $(article, "h1");
+    if (!titleElement) {
+        titleElement = $(article, "h2");
+        if (!titleElement) {
+            throw new got_1.ParseError("No <h1> or <h2> element");
+        }
+    }
+    const title = titleElement.textContent.trim();
+    if (!id) {
+        throw new got_1.ParseError(`No id`);
+    }
+    const dateRaw = $(dom.window.document, "meta[property*=modified_time]") ||
+        $(dom.window.document, "meta[property$=published_time]");
+    const [year, month, day] = dateRaw.getAttribute("content").split("-");
+    let intro = $(article, ".main-article__chapo") || "";
+    intro =
+        intro && intro.innerHTML.replace(/\n/g, "").replace(/\s+/g, " ").trim();
+    const description = $(dom.window.document, "meta[name=description]").getAttribute("content");
+    const sections = [];
+    const sectionTag = getSectionTag(article);
+    // First pass is only to get a potential untitled section at the top of the article
+    // This section has neither anchor nor title
+    let nextArticleElement = $(article, ".main-article__texte > *");
+    const untitledSection = {
+        anchor: "",
+        html: "",
+        text: "",
+        title: title,
+    };
+    while (nextArticleElement &&
+        nextArticleElement.tagName.toLowerCase() !== sectionTag) {
+        if (nextArticleElement.textContent) {
+            if (!untitledSection.description) {
+                untitledSection.description = "temp description";
+            }
+            untitledSection.html += nextArticleElement.outerHTML
+                .replace(/\n+/g, "")
+                .replace(/>\s+</g, "><")
+                .replace(/\s+/g, " ");
+            untitledSection.text +=
+                " " + nextArticleElement.textContent.replace(/\s+/g, " ").trim();
+        }
+        nextArticleElement = nextArticleElement.nextElementSibling;
+    }
+    if (untitledSection.description) {
+        untitledSection.text.trim();
+        untitledSection.description = untitledSection.text.slice(0, 200).trim();
+        untitledSection.references = getReferences(untitledSection.text);
+        sections.push(untitledSection);
+    }
+    // Gets all the titled content
+    const articleChildren = $$(article, `.main-article__texte > ${sectionTag}`);
+    articleChildren.forEach(function (el) {
+        if (el.tagName.toLowerCase() === sectionTag) {
+            let nextEl = el.nextElementSibling;
+            let html = "";
+            while (nextEl && nextEl.tagName.toLowerCase() !== sectionTag) {
+                html += nextEl.outerHTML;
+                nextEl = nextEl.nextElementSibling;
+            }
+            const section = dom.window.document.createElement("div");
+            section.innerHTML = html;
+            const sectionText = section.textContent.replace(/\s+/g, " ").trim();
+            sections.push({
+                anchor: el.getAttribute("id") || (0, cdtn_slugify_1.default)(el.textContent),
+                description: sectionText.slice(0, 200).trim(),
+                html: (0, postProcess_1.htmlPostParser)(html.replace(/\n+/g, "").replace(/>\s+</g, "><").replace(/\s+/g, " ")),
+                references: getReferences(sectionText),
+                text: sectionText,
+                title: el.textContent.trim(),
+            });
+        }
+    });
+    if (sections.length === 0) {
+        throw new got_1.ParseError(`No sections`);
+    }
+    return {
+        date: `${day}/${month}/${year}`,
+        description,
+        intro,
+        pubId: id,
+        sections,
+        title,
+        url,
+    };
+}
+exports.parseDom = parseDom;

package/build/fetch-data/postProcess.d.ts

@@ -0,0 +1 @@
+export declare const htmlPostParser: (html: string) => string;

package/build/fetch-data/postProcess.js

@@ -0,0 +1,60 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.htmlPostParser = void 0;
+const cheerio = __importStar(require("cheerio"));
+const htmlPostParser = (html) => {
+    const $ = cheerio.load(html, null, false);
+    const arrImgSrc = [];
+    let currentIndex = 0;
+    $("style").remove();
+    // https://travail-emploi.gouv.fr/le-ministere-en-action/coronavirus-covid-19/questions-reponses-par-theme/article/mesures-de-prevention-dans-l-entreprise-contre-la-covid-19
+    $("button").remove();
+    $(".oembed-source").remove();
+    // https://travail-emploi.gouv.fr/emploi-et-insertion/accompagnement-des-mutations-economiques/activite-partielle-chomage-partiel/article/activite-partielle-chomage-partiel
+    $("*")
+        .contents()
+        .each(function () {
+        if (this.nodeType === 8) {
+            const regex = /src=['"](.*?)['"]/;
+            const result = regex.exec(this.nodeValue);
+            if (result) {
+                const src = result[0].slice(5, -1);
+                if (src)
+                    arrImgSrc.push(src);
+            }
+        }
+    });
+    $("picture").replaceWith(() => {
+        const src = arrImgSrc[currentIndex];
+        if (src) {
+            currentIndex++;
+            return `<img src="https://travail-emploi.gouv.fr/${src}" style="width:100%;height:auto;"/>`;
+        }
+        return $(this).html()?.toString() ?? "";
+    });
+    return $.html();
+};
+exports.htmlPostParser = htmlPostParser;

package/build/fetch-data/referenceExtractor.d.ts

@@ -0,0 +1,18 @@
+export function classifyTokens(tokens: any): any;
+export namespace CODE_SECU {
+    const id: string;
+    const name: string;
+}
+export namespace CODE_TRAVAIL {
+    const id_1: string;
+    export { id_1 as id };
+    const name_1: string;
+    export { name_1 as name };
+}
+export const codesFullNames: {
+    [x: string]: {
+        id: string;
+        name: string;
+    };
+};
+export function extractReferences(text: any): any;