@tricoteuses/senat 2.20.21 → 2.20.23

package/lib/parsers/texte.js

@@ -0,0 +1,228 @@
+import { JSDOM } from "jsdom";
+import { AKN_IDENTIFICATION_STRUCTURE_REGEXP, AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP } from "../scripts/datautil";
+import { DivisionType, } from "../types/texte";
+function buildWorklow(metaElement) {
+    const stepElements = metaElement.querySelectorAll("workflow step");
+    const steps = [];
+    for (const stepElement of stepElements) {
+        const identification = stepElement.getAttribute("href") ?? "";
+        const identificationParts = AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP.exec(identification)?.groups;
+        steps.push({
+            eId: stepElement.getAttribute("eId"),
+            date: stepElement.getAttribute("date") ? new Date(stepElement.getAttribute("date") ?? "") : null,
+            type: identificationParts?.["type"] || null,
+            session: identificationParts?.["session"] || null,
+            numero: identificationParts?.["numTexte"] || null,
+            version: identificationParts?.["version"] ? identificationParts["version"] : null,
+            outcome: stepElement.getAttribute("outcome"),
+        });
+    }
+    return steps;
+}
+function buildDivision(node, index) {
+    const eId = node.getAttribute("eId");
+    const tag = node.nodeName;
+    const level = DivisionType[tag];
+    const titleNode = node.querySelector("num");
+    const subtitleNode = node.querySelector("heading");
+    const headings = [
+        ...(titleNode
+            ? [
+                {
+                    text: titleNode.textContent?.trim() ?? null,
+                    html: titleNode.innerHTML?.trim() ?? null,
+                },
+            ]
+            : []),
+        ...(subtitleNode
+            ? [
+                {
+                    text: subtitleNode.textContent?.trim() ?? null,
+                    html: subtitleNode.innerHTML?.trim() ?? null,
+                },
+            ]
+            : []),
+    ];
+    const division = {
+        index,
+        eId,
+        tag,
+        level,
+        headings,
+    };
+    if (tag === "article") {
+        ;
+        division.alineas = [];
+    }
+    return division;
+}
+function buildAlinea(contentNode, alineaNode) {
+    const eId = alineaNode.getAttribute("eId");
+    const heading = {
+        text: alineaNode.querySelector("num")?.textContent ?? null,
+    };
+    const pastille = alineaNode.getAttribute("data:pastille") ?? null;
+    return {
+        eId,
+        heading,
+        text: contentNode.textContent?.trim() ?? null,
+        html: contentNode.innerHTML?.trim() ?? null,
+        pastille,
+    };
+}
+function buildEmptyArticle(index) {
+    return {
+        index: index,
+        eId: "",
+        tag: "article",
+        level: DivisionType["article"],
+        headings: [],
+        alineas: [],
+    };
+}
+function flattenTexte(texteContentRoot) {
+    const divisions = [];
+    let divisionIndex = 0;
+    const iter = (node) => {
+        if (node.nodeName === "content") {
+            return;
+        }
+        switch (node.nodeName) {
+            case "tome":
+            case "part":
+            case "book":
+            case "title":
+            case "subtitle":
+            case "chapter":
+            case "section":
+            case "subsection":
+            case "paragraph":
+            case "article":
+                divisions.push(buildDivision(node, divisionIndex++));
+                break;
+        }
+        if (node.nodeName === "alinea") {
+            Array.from(node.childNodes)
+                // Find direct content children programmatically
+                // because `:scope` selector does not work
+                // https://github.com/jsdom/jsdom/issues/2998
+                .filter((alineaChildNode) => alineaChildNode.nodeName === "content")
+                .forEach((alineaContentNode) => {
+                // Hypothesis: alineas should always be enclosed in articles
+                let lastArticle = divisions.findLast((division) => division.tag === "article");
+                if (!lastArticle) {
+                    lastArticle = buildEmptyArticle(divisionIndex++);
+                    divisions.push(lastArticle);
+                }
+                lastArticle.alineas.push(buildAlinea(alineaContentNode, node));
+            });
+        }
+        if (node.hasChildNodes()) {
+            node.childNodes.forEach((childNode) => iter(childNode));
+        }
+    };
+    iter(texteContentRoot);
+    return divisions;
+}
+export function transformTexte(document) {
+    const metaElement = document.querySelector("meta");
+    const preambleElement = document.querySelector("preamble");
+    const identification = metaElement?.querySelector("FRBRExpression FRBRuri")?.getAttribute("value") ?? "";
+    const identificationParts = AKN_IDENTIFICATION_STRUCTURE_REGEXP.exec(identification)?.groups;
+    const bodyElement = document.querySelector("body");
+    const sessionYears = identificationParts?.["session"]?.split("-") || null;
+    const datePresentation = metaElement?.querySelector("FRBRdate[name='#presentation']")?.getAttribute("date");
+    const dateDepot = metaElement?.querySelector("FRBRdate[name='#depot']")?.getAttribute("date");
+    const datePublicationXml = metaElement?.querySelector("FRBRdate[name='#publication-xml']")?.getAttribute("date");
+    return {
+        titre: preambleElement?.querySelector("docTitle")?.textContent || null,
+        titreCourt: metaElement?.querySelector("FRBRalias[name='intitule-court']")?.getAttribute("value") || null,
+        signetDossier: metaElement?.querySelector("FRBRalias[name='signet-dossier-legislatif-senat']")?.getAttribute("value") || null,
+        urlDossierSenat: metaElement?.querySelector("FRBRalias[name='url-senat']")?.getAttribute("value") || null,
+        urlDossierAssemblee: metaElement?.querySelector("FRBRalias[name='url-AN']")?.getAttribute("value") || null,
+        type: identificationParts?.["type"] || null,
+        session: sessionYears && sessionYears.length > 0 ? sessionYears[0] : null,
+        numero: identificationParts?.["numTexte"] ? parseInt(identificationParts["numTexte"]) : null,
+        datePresentation: datePresentation ? new Date(datePresentation) : null,
+        dateDepot: dateDepot ? new Date(dateDepot) : null,
+        datePublicationXml: datePublicationXml ? new Date(datePublicationXml) : null,
+        version: identificationParts?.["version"] ? identificationParts["version"] : null,
+        workflow: metaElement ? buildWorklow(metaElement) : [],
+        divisions: bodyElement ? flattenTexte(bodyElement) : [],
+    };
+}
+export function transformExposeDesMotifs(document) {
+    const sectionElements = document.querySelectorAll("section");
+    const exposeDesMotifsRegexp = new RegExp("EXPOS.{1,2}[\\n\\s]DES[\\n\\s]MOTIFS");
+    for (const sectionElement of sectionElements) {
+        const firstParagraph = sectionElement.querySelector("p:first-of-type");
+        const secondParagraph = sectionElement.querySelector("p:nth-of-type(2)");
+        if (!firstParagraph) {
+            continue;
+        }
+        const firstParagraphContent = firstParagraph.textContent;
+        const secondParagraphContent = secondParagraph?.textContent;
+        if (!firstParagraphContent || !exposeDesMotifsRegexp.test(firstParagraphContent.toUpperCase())) {
+            if (!secondParagraphContent || !exposeDesMotifsRegexp.test(secondParagraphContent.toUpperCase())) {
+                continue;
+            }
+            else {
+                secondParagraph.remove();
+            }
+        }
+        firstParagraph.remove();
+        return {
+            text: sectionElement.textContent?.trim() ?? null,
+            html: sectionElement.innerHTML?.trim() ?? null,
+        };
+    }
+    return null;
+}
+export function parseTexte(texteXml) {
+    try {
+        const { document } = new JSDOM(texteXml, {
+            contentType: "text/xml",
+        }).window;
+        return transformTexte(document);
+    }
+    catch (error) {
+        console.error(`Could not parse texte with error ${error}`);
+    }
+    return null;
+}
+// Prevent from memory leak
+// https://github.com/jsdom/jsdom/issues/2583#issuecomment-559520814
+export async function parseTexteFromFile(xmlFilePath) {
+    try {
+        const { document } = (await JSDOM.fromFile(xmlFilePath, { contentType: "text/xml" })).window;
+        return transformTexte(document);
+    }
+    catch (error) {
+        console.error(`Could not parse texte with error ${error}`);
+    }
+    return null;
+}
+export function parseExposeDesMotifs(exposeDesMotifsHtml) {
+    try {
+        const { document } = new JSDOM(exposeDesMotifsHtml, {
+            contentType: "text/html",
+        }).window;
+        return transformExposeDesMotifs(document);
+    }
+    catch (error) {
+        console.error(`Could not parse exposé des motifs with error ${error}`);
+    }
+    return null;
+}
+// Prevent from memory leak
+// https://github.com/jsdom/jsdom/issues/2583#issuecomment-559520814
+export async function parseExposeDesMotifsFromFile(htmlFilePath) {
+    try {
+        const { document } = (await JSDOM.fromFile(htmlFilePath, { contentType: "text/html" })).window;
+        return transformExposeDesMotifs(document);
+    }
+    catch (error) {
+        console.error(`Could not parse exposé des motifs with error ${error}`);
+    }
+    return null;
+}

package/lib/scripts/convert_data.js

@@ -5,9 +5,10 @@ import path from "path";
 import pLimit from "p-limit";
 import * as git from "../git";
 import { datasets, EnabledDatasets, getEnabledDatasets } from "../datasets";
-import { DATA_ORIGINAL_FOLDER, DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, SCRUTINS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER, } from "../loaders";
+import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, SCRUTINS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER, } from "../loaders";
 import { findAllAmendements, findAllCirconscriptions, findAllDebats, findAllDossiers, findAllScrutins, findAllOrganismes, findAllQuestions, findAllSens, findSenatRapportUrls, findSenatTexteUrls, } from "../model";
-import { createActesLegislatifs } from "../model/dosleg";
+import { processRapport, processTexte } from "./retrieve_documents";
+import { buildActesLegislatifs } from "../model/dosleg";
 import { UNDEFINED_SESSION } from "../types/sessions";
 import { getSessionFromDate, getSessionFromSignet } from "./datautil";
 import { commonOptions } from "./shared/cli_helpers";
@@ -73,7 +74,7 @@ async function convertData() {
     }
     if (enabledDatasets & EnabledDatasets.Questions) {
         try {
-            await convertDatasetQuestions(dataDir);
+            await convertDatasetQuestions(dataDir, options);
             const questionsDir = path.join(dataDir, datasets.questions.database);
             exitCode = commitGit(questionsDir, options, exitCode);
         }
@@ -83,7 +84,7 @@ async function convertData() {
     }
     if (enabledDatasets & EnabledDatasets.Sens) {
         try {
-            await convertDatasetSens(dataDir);
+            await convertDatasetSens(dataDir, options);
             const sensDir = path.join(dataDir, datasets.sens.database);
             exitCode = commitGit(sensDir, options, exitCode);
         }
@@ -102,7 +103,9 @@ async function convertDatasetAmeli(dataDir, options) {
         console.log(`Converting database ${dataset.database} data into files…`);
     }
     const ameliReorganizedRootDir = path.join(dataDir, dataset.database);
-    await fs.ensureDir(ameliReorganizedRootDir);
+    if (!options.keepDir) {
+        ensureAndClearDir(ameliReorganizedRootDir);
+    }
     for await (const amendement of findAllAmendements(options["fromSession"])) {
         if (options["verbose"]) {
             console.log(`Converting ${amendement["numero"]} file…`);
@@ -110,11 +113,9 @@ async function convertDatasetAmeli(dataDir, options) {
         const session = String(amendement["session"]) || UNDEFINED_SESSION;
         const signetDossierLegislatif = amendement["signet_dossier_legislatif"] ||
             `${amendement["nature_texte"]}-${amendement["numero_texte"]}`.toLowerCase();
-        const ameliReorganizedDir = path.join(ameliReorganizedRootDir, String(session), signetDossierLegislatif);
-        await fs.ensureDir(ameliReorganizedDir);
         const amendementFileName = `${amendement["numero"]}.json`;
-        const filePath = path.join(ameliReorganizedDir, amendementFileName);
-        await fs.writeJSON(filePath, amendement, { spaces: 2 });
+        const filePath = path.join(ameliReorganizedRootDir, String(session), signetDossierLegislatif, amendementFileName);
+        await fs.outputJSON(filePath, amendement, { spaces: 2 });
     }
 }
 async function convertDatasetDebats(dataDir, options) {
@@ -123,7 +124,9 @@ async function convertDatasetDebats(dataDir, options) {
         console.log(`Converting database ${dataset.database} data into files…`);
     }
     const debatsReorganizedRootDir = path.join(dataDir, dataset.database);
-    ensureAndClearDir(debatsReorganizedRootDir);
+    if (!options.keepDir) {
+        ensureAndClearDir(debatsReorganizedRootDir);
+    }
     for await (const debat of findAllDebats()) {
         if (options["verbose"]) {
             console.log(`Converting ${debat.id} file…`);
@@ -132,11 +135,9 @@ async function convertDatasetDebats(dataDir, options) {
         if (options["fromSession"] && session < options["fromSession"]) {
             continue;
         }
-        const debatsReorganizedDir = path.join(debatsReorganizedRootDir, String(session));
-        await fs.ensureDir(debatsReorganizedDir);
         const debatFileName = `${debat.id}.json`;
-        const filePath = path.join(debatsReorganizedDir, debatFileName);
-        await fs.writeJSON(filePath, debat, { spaces: 2 });
+        const filePath = path.join(debatsReorganizedRootDir, String(session), debatFileName);
+        await fs.outputJSON(filePath, debat, { spaces: 2 });
     }
 }
 async function convertDatasetDosLeg(dataDir, options) {
@@ -146,28 +147,30 @@ async function convertDatasetDosLeg(dataDir, options) {
     }
     const doslegReorganizedRootDir = path.join(dataDir, dataset.database);
     const dossiersReorganizedDir = path.join(doslegReorganizedRootDir, DOSLEG_DOSSIERS_FOLDER);
-    ensureAndClearDir(doslegReorganizedRootDir);
-    ensureAndClearDir(dossiersReorganizedDir);
-    for await (const loi of findAllDossiers()) {
+    if (!options.keepDir) {
+        ensureAndClearDir(doslegReorganizedRootDir);
+        ensureAndClearDir(dossiersReorganizedDir);
+    }
+    for await (const dossier of findAllDossiers()) {
         if (options["verbose"]) {
-            console.log(`Converting ${loi["signet"]} file…`);
+            console.log(`Converting ${dossier["signet"]} file…`);
         }
-        let loiReorganizedDir = path.join(dossiersReorganizedDir, String(UNDEFINED_SESSION));
-        const session = getSessionFromSignet(loi["signet"]) || UNDEFINED_SESSION;
+        let dossierReorganizedDir = path.join(dossiersReorganizedDir, String(UNDEFINED_SESSION));
+        const session = getSessionFromSignet(dossier["signet"]) || UNDEFINED_SESSION;
         if (options["fromSession"] && session < options["fromSession"]) {
             continue;
         }
-        loiReorganizedDir = path.join(dossiersReorganizedDir, String(session));
-        await fs.ensureDir(loiReorganizedDir);
-        // Ajout des actes législatifs au dossier
-        const actesLegislatifs = createActesLegislatifs(loi);
-        const loiWithActes = { ...loi, actes_legislatifs: actesLegislatifs };
-        const dossierFile = `${loi["signet"]}.json`;
-        const filePath = path.join(loiReorganizedDir, dossierFile);
-        await fs.writeJSON(filePath, loiWithActes, { spaces: 2 });
+        dossierReorganizedDir = path.join(dossiersReorganizedDir, String(session));
+        const actesBrutsNormalises = buildActesLegislatifs(dossier);
+        const dossierWithActes = {
+            ...dossier,
+            actes_legislatifs: actesBrutsNormalises
+        };
+        const dossierFile = `${dossier["signet"]}.json`;
+        await fs.outputJSON(path.join(dossierReorganizedDir, dossierFile), dossierWithActes, { spaces: 2 });
     }
-    await convertTexteUrls(dataDir);
-    await convertRapportUrls(dataDir);
+    await convertTexteUrls(dataDir, options);
+    await convertRapportUrls(dataDir, options);
 }
 async function convertDatasetScrutins(dataDir, options) {
     const dataset = datasets.dosleg;
@@ -175,7 +178,9 @@ async function convertDatasetScrutins(dataDir, options) {
         console.log(`Converting database scrutins (${dataset.database}) data into files…`);
     }
     const scrutinsReorganizedDir = path.join(dataDir, SCRUTINS_FOLDER);
-    ensureAndClearDir(scrutinsReorganizedDir);
+    if (!options.keepDir) {
+        ensureAndClearDir(scrutinsReorganizedDir);
+    }
     for await (const scrutin of findAllScrutins(options["fromSession"])) {
         if (options["verbose"]) {
             console.log(`Converting ${scrutin["numero"]} file…`);
@@ -183,20 +188,21 @@ async function convertDatasetScrutins(dataDir, options) {
         let scrutinReorganizedDir = path.join(scrutinsReorganizedDir, String(UNDEFINED_SESSION));
         const session = scrutin["session"] || UNDEFINED_SESSION;
         scrutinReorganizedDir = path.join(scrutinsReorganizedDir, String(session));
-        await fs.ensureDir(scrutinReorganizedDir);
         const scrutinFileName = `${scrutin["numero"]}.json`;
-        await fs.writeJSON(path.join(scrutinReorganizedDir, scrutinFileName), scrutin, {
+        await fs.outputJSON(path.join(scrutinReorganizedDir, scrutinFileName), scrutin, {
             spaces: 2,
         });
     }
 }
-async function convertDatasetQuestions(dataDir) {
+async function convertDatasetQuestions(dataDir, options) {
     const dataset = datasets.questions;
     if (!options["silent"]) {
         console.log(`Converting database ${dataset.database} data into files…`);
     }
     const questionsReorganizedRootDir = path.join(dataDir, dataset.database);
-    ensureAndClearDir(questionsReorganizedRootDir);
+    if (!options.keepDir) {
+        ensureAndClearDir(questionsReorganizedRootDir);
+    }
     const limit = pLimit(10);
     const tasks = [];
     for await (const question of findAllQuestions()) {
@@ -205,22 +211,27 @@ async function convertDatasetQuestions(dataDir) {
                 console.log(`Converting ${question["reference"]} file…`);
             }
             const legislature = question["legislature"] ? question["legislature"] : 0;
-            const questionReorganizedDir = path.join(questionsReorganizedRootDir, String(legislature));
-            await fs.ensureDir(questionReorganizedDir);
             const questionFileName = `${question["reference"]}.json`;
-            await fs.writeJSON(path.join(questionReorganizedDir, questionFileName), question, { spaces: 2 });
+            await fs.outputJSON(path.join(questionsReorganizedRootDir, String(legislature), questionFileName), question, {
+                spaces: 2,
+            });
         }));
     }
     await Promise.all(tasks);
 }
-async function convertTexteUrls(dataDir) {
-    const textesDir = path.join(dataDir, TEXTE_FOLDER);
-    fs.ensureDirSync(textesDir);
-    const originalTextesDir = path.join(textesDir, DATA_ORIGINAL_FOLDER);
+async function convertTexteUrls(dataDir, options) {
+    const originalTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER);
+    const transformedTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_TRANSFORMED_FOLDER);
+    if (!options["silent"]) {
+        console.log(`Converting database textes data into files…`);
+    }
     for await (const texte of findSenatTexteUrls()) {
+        const session = texte.session ?? UNDEFINED_SESSION;
+        if (options["fromSession"] && session < options["fromSession"]) {
+            continue;
+        }
         const texteName = path.parse(texte.url).name;
-        const texteDir = path.join(originalTextesDir, `${texte.session ?? UNDEFINED_SESSION}`, texteName);
-        fs.ensureDirSync(texteDir);
+        const texteDir = path.join(originalTextesDir, `${session}`, texteName);
         const metadata = {
             name: texteName,
             session: texte.session,
@@ -232,20 +243,27 @@ async function convertTexteUrls(dataDir) {
             url_html: new URL(`${texteName}.html`, SENAT_TEXTE_BASE_URL),
             url_pdf: new URL(`${texteName}.pdf`, SENAT_TEXTE_BASE_URL),
         };
-        fs.writeJSONSync(path.join(texteDir, DOCUMENT_METADATA_FILE), metadata, {
+        fs.outputJSONSync(path.join(texteDir, DOCUMENT_METADATA_FILE), metadata, {
             spaces: 2,
         });
+        if (options.fetchDocuments) {
+            await processTexte(metadata, originalTextesDir, transformedTextesDir, options);
+        }
     }
 }
-async function convertRapportUrls(dataDir) {
-    const rapportsDir = path.join(dataDir, RAPPORT_FOLDER);
-    fs.ensureDirSync(rapportsDir);
-    const originalTextesDir = path.join(rapportsDir, DATA_ORIGINAL_FOLDER);
+async function convertRapportUrls(dataDir, options) {
+    const originalRapportsDir = path.join(dataDir, RAPPORT_FOLDER, DATA_ORIGINAL_FOLDER);
+    if (!options["silent"]) {
+        console.log(`Converting database rapports data into files…`);
+    }
     for await (const rapport of findSenatRapportUrls()) {
+        const session = rapport.session ?? UNDEFINED_SESSION;
+        if (options["fromSession"] && session < options["fromSession"]) {
+            continue;
+        }
         const parsedRapportUrl = path.parse(rapport.url);
         const rapportName = parsedRapportUrl.name;
-        const rapportDir = path.join(originalTextesDir, `${rapport.session ?? UNDEFINED_SESSION}`, rapportName);
-        fs.ensureDirSync(rapportDir);
+        const rapportDir = path.join(originalRapportsDir, `${session}`, rapportName);
         const rapportHtmlUrlBase = `${rapportName}_mono.html`;
         const rapportHtmlUrl = path.format({
             dir: parsedRapportUrl.dir,
@@ -263,12 +281,15 @@ async function convertRapportUrls(dataDir) {
             url_html: new URL(rapportHtmlUrl, SENAT_RAPPORT_BASE_URL),
             url_pdf: new URL(rapportPdfUrl, SENAT_RAPPORT_BASE_URL),
         };
-        fs.writeJSONSync(path.join(rapportDir, DOCUMENT_METADATA_FILE), metadata, {
+        fs.outputJSONSync(path.join(rapportDir, DOCUMENT_METADATA_FILE), metadata, {
             spaces: 2,
         });
+        if (options.fetchDocuments) {
+            await processRapport(metadata, originalRapportsDir, options);
+        }
     }
 }
-async function convertDatasetSens(dataDir) {
+async function convertDatasetSens(dataDir, options) {
     const dataset = datasets.sens;
     if (!options["silent"]) {
         console.log(`Converting database ${dataset.database} data into files…`);
@@ -277,16 +298,18 @@ async function convertDatasetSens(dataDir) {
     const senateursReorganizedDir = path.join(sensReorganizedRootDir, SENS_SENATEURS_FOLDER);
     const circonscriptionsReorganizedDir = path.join(sensReorganizedRootDir, SENS_CIRCONSCRIPTIONS_FOLDER);
     const organismesReorganizedDir = path.join(sensReorganizedRootDir, SENS_ORGANISMES_FOLDER);
-    ensureAndClearDir(sensReorganizedRootDir);
-    ensureAndClearDir(senateursReorganizedDir);
-    ensureAndClearDir(circonscriptionsReorganizedDir);
-    ensureAndClearDir(organismesReorganizedDir);
+    if (!options.keepDir) {
+        ensureAndClearDir(sensReorganizedRootDir);
+        ensureAndClearDir(senateursReorganizedDir);
+        ensureAndClearDir(circonscriptionsReorganizedDir);
+        ensureAndClearDir(organismesReorganizedDir);
+    }
     for await (const sen of findAllSens()) {
         if (options["verbose"]) {
             console.log(`Converting ${sen["matricule"]} file…`);
         }
         const senFileName = `${sen["matricule"]}.json`;
-        fs.writeJSONSync(path.join(senateursReorganizedDir, senFileName), sen, {
+        fs.outputJSONSync(path.join(senateursReorganizedDir, senFileName), sen, {
             spaces: 2,
         });
     }
@@ -295,16 +318,18 @@ async function convertDatasetSens(dataDir) {
             console.log(`Converting ${circonscription["identifiant"]} file…`);
         }
         const circonscriptionFileName = `${circonscription["identifiant"]}.json`;
-        fs.writeJSONSync(path.join(circonscriptionsReorganizedDir, circonscriptionFileName), circonscription, { spaces: 2 });
+        fs.outputJSONSync(path.join(circonscriptionsReorganizedDir, circonscriptionFileName), circonscription, {
+            spaces: 2,
+        });
     }
     for await (const organisme of findAllOrganismes()) {
         if (options["verbose"]) {
             console.log(`Converting ${organisme["code"]} file…`);
         }
         const organismeFileName = `${organisme["code"]}.json`;
-        const organismeDir = path.join(organismesReorganizedDir, organisme["type_code"]);
-        fs.ensureDirSync(organismeDir);
-        fs.writeJSONSync(path.join(organismeDir, organismeFileName), organisme, { spaces: 2 });
+        fs.outputJSONSync(path.join(organismesReorganizedDir, organisme["type_code"], organismeFileName), organisme, {
+            spaces: 2,
+        });
     }
 }
 convertData()

package/lib/scripts/data-download.js

@@ -5,5 +5,8 @@ try {
     execSync(`tsx src/scripts/convert_data.ts ${args}`, { stdio: "inherit" });
 }
 catch (error) {
-    process.exit(1);
+    if (error.status !== 10) {
+        console.error("Error during data retrieval:", error);
+        process.exit(error.status || 1);
+    }
 }

package/lib/scripts/retrieve_documents.d.ts

@@ -1 +1,2 @@
-export {};
+export declare function processTexte(texteMetadata: any, originalTextesDir: string, transformedTextesDir: string, options: any): Promise<void>;
+export declare function processRapport(rapportMetadata: any, originalRapportsDir: string, options: any): Promise<void>;