@tricoteuses/senat 2.20.20 → 2.20.22

package/README.md

@@ -43,7 +43,6 @@ npm run data:download ../senat-data
 
 - `npm run data:download <dir>`: Download, convert data to JSON
 - `npm run data:retrieve_documents <dir>`: Retrieval of textes and rapports from Sénat's website
-- `npm run data:parse_textes_lois <dir>`: Parse textes (requires xml files)
 - `npm run data:retrieve_agenda <dir>`: Retrieval of agenda from Sénat's website
 - `npm run data:retrieve_cr_seance <dir>`: Retrieval of comptes-rendus de séance from Sénat's data
 - `npm run data:retrieve_cr_commission <dir>`: Retrieval of comptes-rendus de commissions from Sénat's website

package/lib/loaders.d.ts

@@ -13,11 +13,11 @@ export declare const COMPTES_RENDUS_FOLDER = "seances";
 export declare const COMMISSION_FOLDER = "commissions";
 export declare const DOSLEG_DOSSIERS_FOLDER = "dossiers";
 export declare const SCRUTINS_FOLDER = "scrutins";
-export declare const RAPPORT_FOLDER = "rap";
 export declare const SENS_CIRCONSCRIPTIONS_FOLDER = "circonscriptions";
 export declare const SENS_ORGANISMES_FOLDER = "organismes";
 export declare const SENS_SENATEURS_FOLDER = "senateurs";
 export declare const TEXTE_FOLDER = "leg";
+export declare const RAPPORT_FOLDER = "rap";
 export declare const DATA_ORIGINAL_FOLDER = "original";
 export declare const DATA_TRANSFORMED_FOLDER = "transformed";
 export declare const DOCUMENT_METADATA_FILE = "metadata.json";
@@ -25,6 +25,7 @@ export type IterItem<T> = {
     item: T;
     filePathFromDataset?: string;
     legislature?: number;
+    gitStatus?: "A" | "M" | "D" | "R" | "C" | "T" | "U";
 };
 export interface TexteMetadata {
     name: string;

package/lib/loaders.js

@@ -1,6 +1,7 @@
 import fsex from "fs-extra";
 import fs from "fs";
 import path from "path";
+import * as git from "./git";
 import { datasets } from "./datasets";
 import { UNDEFINED_SESSION } from "./types/sessions";
 export { EnabledDatasets } from "./datasets";
@@ -9,11 +10,11 @@ export const COMPTES_RENDUS_FOLDER = "seances";
 export const COMMISSION_FOLDER = "commissions";
 export const DOSLEG_DOSSIERS_FOLDER = "dossiers";
 export const SCRUTINS_FOLDER = "scrutins";
-export const RAPPORT_FOLDER = "rap";
 export const SENS_CIRCONSCRIPTIONS_FOLDER = "circonscriptions";
 export const SENS_ORGANISMES_FOLDER = "organismes";
 export const SENS_SENATEURS_FOLDER = "senateurs";
 export const TEXTE_FOLDER = "leg";
+export const RAPPORT_FOLDER = "rap";
 export const DATA_ORIGINAL_FOLDER = "original";
 export const DATA_TRANSFORMED_FOLDER = "transformed";
 export const DOCUMENT_METADATA_FILE = "metadata.json";
@@ -30,7 +31,7 @@ export function* iterFilePaths(dirPath) {
         }
     }
 }
-function* iterLoadSenatItems(dataDir, dataName, legislatureOrSession, subDir, { log = false } = {}) {
+function* iterLoadSenatItems(dataDir, dataName, legislatureOrSession, subDir, { log = false, sinceCommit } = {}) {
     let itemsDir = path.join(dataDir, dataName);
     if (subDir) {
         itemsDir = path.join(itemsDir, subDir);
@@ -38,9 +39,26 @@ function* iterLoadSenatItems(dataDir, dataName, legislatureOrSession, subDir, {
     if (legislatureOrSession) {
         itemsDir = path.join(itemsDir, String(legislatureOrSession));
     }
+    // Get changed files if sinceCommit is specified (excluding deleted files)
+    const changedFiles = sinceCommit
+        ? git.getChangedFilesSinceCommit(itemsDir, sinceCommit, {
+            diffFilter: "AMR", // Added, Modified, Renamed
+        })
+        : null;
+    if (log && sinceCommit) {
+        console.log(`Filtering files changed since commit ${sinceCommit} in ${itemsDir}`);
+        console.log(`Found ${changedFiles?.size || 0} changed files (AMR)`);
+    }
     for (const filePath of iterFilePaths(itemsDir)) {
+        const relativePath = path.relative(path.join(dataDir, dataName), filePath);
+        const gitStatus = changedFiles?.get(relativePath);
+        // Filter by changed files if sinceCommit is specified
+        if (changedFiles && !gitStatus) {
+            // Skip files not in the change set
+            continue;
+        }
         if (log) {
-            console.log(`Loading file: ${filePath}…`);
+            console.log(`Loading file: ${filePath}…${gitStatus ? ` (${gitStatus})` : ""}`);
         }
         let item;
         try {
@@ -56,8 +74,35 @@ function* iterLoadSenatItems(dataDir, dataName, legislatureOrSession, subDir, {
             item,
             filePathFromDataset,
             legislature: legislatureOrSession,
+            ...(gitStatus && { gitStatus }), // Include gitStatus
         };
     }
+    // Yield deleted files at the end if sinceCommit is specified
+    if (sinceCommit) {
+        const deletedFiles = git.getChangedFilesSinceCommit(itemsDir, sinceCommit, {
+            diffFilter: "D", // Deleted
+        });
+        if (log) {
+            console.log(`Found ${deletedFiles.size || 0} deleted files (D)`);
+        }
+        for (const [relativePath, status] of deletedFiles.entries()) {
+            const deletedFilePath = path.join(itemsDir, relativePath);
+            if (log) {
+                console.log(`Deleted file: ${deletedFilePath}`);
+            }
+            // Extract UID from filename (remove extension) for the placeholder item
+            const fileExtension = path.extname(relativePath) || ".json"; // Assuming files use an extension like .json
+            const filename = path.basename(relativePath, fileExtension);
+            const fakeItem = { uid: filename }; // Placeholder item using uid constraint
+            const filePathFromDataset = deletedFilePath.substring(deletedFilePath.indexOf(dataName) + dataName.length);
+            yield {
+                item: fakeItem,
+                filePathFromDataset,
+                legislature: legislatureOrSession,
+                gitStatus: status,
+            };
+        }
+    }
 }
 export function* iterLoadSenatAmendements(dataDir, session, options = {}) {
     for (const amendementItem of iterLoadSenatItems(dataDir, datasets.ameli.database, session, undefined, options)) {

package/lib/model/dosleg.js

@@ -249,7 +249,7 @@ export function createActesLegislatifs(dossier) {
                     code_acte: `${codeParent}-DEBATS-SEANCE`,
                     date: lectureAss["dates_seances"][0]?.["date"],
                     id: lectureAss["id"],
-                    numero: lectureAss["numero"]
+                    numero: lectureAss["numero"],
                 });
             }
             const { textes, rapports, ...lectureAssWithoutTextes } = lectureAss;

package/lib/parsers/texte.d.ts

@@ -0,0 +1,7 @@
+import { ExposeDesMotifs, FlatTexte } from "../types/texte";
+export declare function transformTexte(document: Document): FlatTexte | null;
+export declare function transformExposeDesMotifs(document: Document): ExposeDesMotifs | null;
+export declare function parseTexte(texteXml: string): FlatTexte | null;
+export declare function parseTexteFromFile(xmlFilePath: string): Promise<FlatTexte | null>;
+export declare function parseExposeDesMotifs(exposeDesMotifsHtml: string): ExposeDesMotifs | null;
+export declare function parseExposeDesMotifsFromFile(htmlFilePath: string): Promise<ExposeDesMotifs | null>;

package/lib/parsers/texte.js

@@ -0,0 +1,228 @@
+import { JSDOM } from "jsdom";
+import { AKN_IDENTIFICATION_STRUCTURE_REGEXP, AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP } from "../scripts/datautil";
+import { DivisionType, } from "../types/texte";
+function buildWorklow(metaElement) {
+    const stepElements = metaElement.querySelectorAll("workflow step");
+    const steps = [];
+    for (const stepElement of stepElements) {
+        const identification = stepElement.getAttribute("href") ?? "";
+        const identificationParts = AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP.exec(identification)?.groups;
+        steps.push({
+            eId: stepElement.getAttribute("eId"),
+            date: stepElement.getAttribute("date") ? new Date(stepElement.getAttribute("date") ?? "") : null,
+            type: identificationParts?.["type"] || null,
+            session: identificationParts?.["session"] || null,
+            numero: identificationParts?.["numTexte"] || null,
+            version: identificationParts?.["version"] ? identificationParts["version"] : null,
+            outcome: stepElement.getAttribute("outcome"),
+        });
+    }
+    return steps;
+}
+function buildDivision(node, index) {
+    const eId = node.getAttribute("eId");
+    const tag = node.nodeName;
+    const level = DivisionType[tag];
+    const titleNode = node.querySelector("num");
+    const subtitleNode = node.querySelector("heading");
+    const headings = [
+        ...(titleNode
+            ? [
+                {
+                    text: titleNode.textContent?.trim() ?? null,
+                    html: titleNode.innerHTML?.trim() ?? null,
+                },
+            ]
+            : []),
+        ...(subtitleNode
+            ? [
+                {
+                    text: subtitleNode.textContent?.trim() ?? null,
+                    html: subtitleNode.innerHTML?.trim() ?? null,
+                },
+            ]
+            : []),
+    ];
+    const division = {
+        index,
+        eId,
+        tag,
+        level,
+        headings,
+    };
+    if (tag === "article") {
+        ;
+        division.alineas = [];
+    }
+    return division;
+}
+function buildAlinea(contentNode, alineaNode) {
+    const eId = alineaNode.getAttribute("eId");
+    const heading = {
+        text: alineaNode.querySelector("num")?.textContent ?? null,
+    };
+    const pastille = alineaNode.getAttribute("data:pastille") ?? null;
+    return {
+        eId,
+        heading,
+        text: contentNode.textContent?.trim() ?? null,
+        html: contentNode.innerHTML?.trim() ?? null,
+        pastille,
+    };
+}
+function buildEmptyArticle(index) {
+    return {
+        index: index,
+        eId: "",
+        tag: "article",
+        level: DivisionType["article"],
+        headings: [],
+        alineas: [],
+    };
+}
+function flattenTexte(texteContentRoot) {
+    const divisions = [];
+    let divisionIndex = 0;
+    const iter = (node) => {
+        if (node.nodeName === "content") {
+            return;
+        }
+        switch (node.nodeName) {
+            case "tome":
+            case "part":
+            case "book":
+            case "title":
+            case "subtitle":
+            case "chapter":
+            case "section":
+            case "subsection":
+            case "paragraph":
+            case "article":
+                divisions.push(buildDivision(node, divisionIndex++));
+                break;
+        }
+        if (node.nodeName === "alinea") {
+            Array.from(node.childNodes)
+                // Find direct content children programmatically
+                // because `:scope` selector does not work
+                // https://github.com/jsdom/jsdom/issues/2998
+                .filter((alineaChildNode) => alineaChildNode.nodeName === "content")
+                .forEach((alineaContentNode) => {
+                // Hypothesis: alineas should always be enclosed in articles
+                let lastArticle = divisions.findLast((division) => division.tag === "article");
+                if (!lastArticle) {
+                    lastArticle = buildEmptyArticle(divisionIndex++);
+                    divisions.push(lastArticle);
+                }
+                lastArticle.alineas.push(buildAlinea(alineaContentNode, node));
+            });
+        }
+        if (node.hasChildNodes()) {
+            node.childNodes.forEach((childNode) => iter(childNode));
+        }
+    };
+    iter(texteContentRoot);
+    return divisions;
+}
+export function transformTexte(document) {
+    const metaElement = document.querySelector("meta");
+    const preambleElement = document.querySelector("preamble");
+    const identification = metaElement?.querySelector("FRBRExpression FRBRuri")?.getAttribute("value") ?? "";
+    const identificationParts = AKN_IDENTIFICATION_STRUCTURE_REGEXP.exec(identification)?.groups;
+    const bodyElement = document.querySelector("body");
+    const sessionYears = identificationParts?.["session"]?.split("-") || null;
+    const datePresentation = metaElement?.querySelector("FRBRdate[name='#presentation']")?.getAttribute("date");
+    const dateDepot = metaElement?.querySelector("FRBRdate[name='#depot']")?.getAttribute("date");
+    const datePublicationXml = metaElement?.querySelector("FRBRdate[name='#publication-xml']")?.getAttribute("date");
+    return {
+        titre: preambleElement?.querySelector("docTitle")?.textContent || null,
+        titreCourt: metaElement?.querySelector("FRBRalias[name='intitule-court']")?.getAttribute("value") || null,
+        signetDossier: metaElement?.querySelector("FRBRalias[name='signet-dossier-legislatif-senat']")?.getAttribute("value") || null,
+        urlDossierSenat: metaElement?.querySelector("FRBRalias[name='url-senat']")?.getAttribute("value") || null,
+        urlDossierAssemblee: metaElement?.querySelector("FRBRalias[name='url-AN']")?.getAttribute("value") || null,
+        type: identificationParts?.["type"] || null,
+        session: sessionYears && sessionYears.length > 0 ? sessionYears[0] : null,
+        numero: identificationParts?.["numTexte"] ? parseInt(identificationParts["numTexte"]) : null,
+        datePresentation: datePresentation ? new Date(datePresentation) : null,
+        dateDepot: dateDepot ? new Date(dateDepot) : null,
+        datePublicationXml: datePublicationXml ? new Date(datePublicationXml) : null,
+        version: identificationParts?.["version"] ? identificationParts["version"] : null,
+        workflow: metaElement ? buildWorklow(metaElement) : [],
+        divisions: bodyElement ? flattenTexte(bodyElement) : [],
+    };
+}
+export function transformExposeDesMotifs(document) {
+    const sectionElements = document.querySelectorAll("section");
+    const exposeDesMotifsRegexp = new RegExp("EXPOS.{1,2}[\\n\\s]DES[\\n\\s]MOTIFS");
+    for (const sectionElement of sectionElements) {
+        const firstParagraph = sectionElement.querySelector("p:first-of-type");
+        const secondParagraph = sectionElement.querySelector("p:nth-of-type(2)");
+        if (!firstParagraph) {
+            continue;
+        }
+        const firstParagraphContent = firstParagraph.textContent;
+        const secondParagraphContent = secondParagraph?.textContent;
+        if (!firstParagraphContent || !exposeDesMotifsRegexp.test(firstParagraphContent.toUpperCase())) {
+            if (!secondParagraphContent || !exposeDesMotifsRegexp.test(secondParagraphContent.toUpperCase())) {
+                continue;
+            }
+            else {
+                secondParagraph.remove();
+            }
+        }
+        firstParagraph.remove();
+        return {
+            text: sectionElement.textContent?.trim() ?? null,
+            html: sectionElement.innerHTML?.trim() ?? null,
+        };
+    }
+    return null;
+}
+export function parseTexte(texteXml) {
+    try {
+        const { document } = new JSDOM(texteXml, {
+            contentType: "text/xml",
+        }).window;
+        return transformTexte(document);
+    }
+    catch (error) {
+        console.error(`Could not parse texte with error ${error}`);
+    }
+    return null;
+}
+// Prevent from memory leak
+// https://github.com/jsdom/jsdom/issues/2583#issuecomment-559520814
+export async function parseTexteFromFile(xmlFilePath) {
+    try {
+        const { document } = (await JSDOM.fromFile(xmlFilePath, { contentType: "text/xml" })).window;
+        return transformTexte(document);
+    }
+    catch (error) {
+        console.error(`Could not parse texte with error ${error}`);
+    }
+    return null;
+}
+export function parseExposeDesMotifs(exposeDesMotifsHtml) {
+    try {
+        const { document } = new JSDOM(exposeDesMotifsHtml, {
+            contentType: "text/html",
+        }).window;
+        return transformExposeDesMotifs(document);
+    }
+    catch (error) {
+        console.error(`Could not parse exposé des motifs with error ${error}`);
+    }
+    return null;
+}
+// Prevent from memory leak
+// https://github.com/jsdom/jsdom/issues/2583#issuecomment-559520814
+export async function parseExposeDesMotifsFromFile(htmlFilePath) {
+    try {
+        const { document } = (await JSDOM.fromFile(htmlFilePath, { contentType: "text/html" })).window;
+        return transformExposeDesMotifs(document);
+    }
+    catch (error) {
+        console.error(`Could not parse exposé des motifs with error ${error}`);
+    }
+    return null;
+}

package/lib/scripts/convert_data.js

@@ -5,8 +5,9 @@ import path from "path";
 import pLimit from "p-limit";
 import * as git from "../git";
 import { datasets, EnabledDatasets, getEnabledDatasets } from "../datasets";
-import { DATA_ORIGINAL_FOLDER, DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, SCRUTINS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER, } from "../loaders";
+import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, SCRUTINS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER, } from "../loaders";
 import { findAllAmendements, findAllCirconscriptions, findAllDebats, findAllDossiers, findAllScrutins, findAllOrganismes, findAllQuestions, findAllSens, findSenatRapportUrls, findSenatTexteUrls, } from "../model";
+import { processRapport, processTexte } from "./retrieve_documents";
 import { createActesLegislatifs } from "../model/dosleg";
 import { UNDEFINED_SESSION } from "../types/sessions";
 import { getSessionFromDate, getSessionFromSignet } from "./datautil";
@@ -73,7 +74,7 @@ async function convertData() {
     }
     if (enabledDatasets & EnabledDatasets.Questions) {
         try {
-            await convertDatasetQuestions(dataDir);
+            await convertDatasetQuestions(dataDir, options);
             const questionsDir = path.join(dataDir, datasets.questions.database);
             exitCode = commitGit(questionsDir, options, exitCode);
         }
@@ -83,7 +84,7 @@ async function convertData() {
     }
     if (enabledDatasets & EnabledDatasets.Sens) {
         try {
-            await convertDatasetSens(dataDir);
+            await convertDatasetSens(dataDir, options);
             const sensDir = path.join(dataDir, datasets.sens.database);
             exitCode = commitGit(sensDir, options, exitCode);
         }
@@ -102,7 +103,9 @@ async function convertDatasetAmeli(dataDir, options) {
         console.log(`Converting database ${dataset.database} data into files…`);
     }
     const ameliReorganizedRootDir = path.join(dataDir, dataset.database);
-    await fs.ensureDir(ameliReorganizedRootDir);
+    if (!options.keepDir) {
+        ensureAndClearDir(ameliReorganizedRootDir);
+    }
     for await (const amendement of findAllAmendements(options["fromSession"])) {
         if (options["verbose"]) {
             console.log(`Converting ${amendement["numero"]} file…`);
@@ -110,11 +113,9 @@ async function convertDatasetAmeli(dataDir, options) {
         const session = String(amendement["session"]) || UNDEFINED_SESSION;
         const signetDossierLegislatif = amendement["signet_dossier_legislatif"] ||
             `${amendement["nature_texte"]}-${amendement["numero_texte"]}`.toLowerCase();
-        const ameliReorganizedDir = path.join(ameliReorganizedRootDir, String(session), signetDossierLegislatif);
-        await fs.ensureDir(ameliReorganizedDir);
         const amendementFileName = `${amendement["numero"]}.json`;
-        const filePath = path.join(ameliReorganizedDir, amendementFileName);
-        await fs.writeJSON(filePath, amendement, { spaces: 2 });
+        const filePath = path.join(ameliReorganizedRootDir, String(session), signetDossierLegislatif, amendementFileName);
+        await fs.outputJSON(filePath, amendement, { spaces: 2 });
     }
 }
 async function convertDatasetDebats(dataDir, options) {
@@ -123,7 +124,9 @@ async function convertDatasetDebats(dataDir, options) {
         console.log(`Converting database ${dataset.database} data into files…`);
     }
     const debatsReorganizedRootDir = path.join(dataDir, dataset.database);
-    ensureAndClearDir(debatsReorganizedRootDir);
+    if (!options.keepDir) {
+        ensureAndClearDir(debatsReorganizedRootDir);
+    }
     for await (const debat of findAllDebats()) {
         if (options["verbose"]) {
             console.log(`Converting ${debat.id} file…`);
@@ -132,11 +135,9 @@ async function convertDatasetDebats(dataDir, options) {
         if (options["fromSession"] && session < options["fromSession"]) {
             continue;
         }
-        const debatsReorganizedDir = path.join(debatsReorganizedRootDir, String(session));
-        await fs.ensureDir(debatsReorganizedDir);
         const debatFileName = `${debat.id}.json`;
-        const filePath = path.join(debatsReorganizedDir, debatFileName);
-        await fs.writeJSON(filePath, debat, { spaces: 2 });
+        const filePath = path.join(debatsReorganizedRootDir, String(session), debatFileName);
+        await fs.outputJSON(filePath, debat, { spaces: 2 });
     }
 }
 async function convertDatasetDosLeg(dataDir, options) {
@@ -146,8 +147,10 @@ async function convertDatasetDosLeg(dataDir, options) {
     }
     const doslegReorganizedRootDir = path.join(dataDir, dataset.database);
     const dossiersReorganizedDir = path.join(doslegReorganizedRootDir, DOSLEG_DOSSIERS_FOLDER);
-    ensureAndClearDir(doslegReorganizedRootDir);
-    ensureAndClearDir(dossiersReorganizedDir);
+    if (!options.keepDir) {
+        ensureAndClearDir(doslegReorganizedRootDir);
+        ensureAndClearDir(dossiersReorganizedDir);
+    }
     for await (const loi of findAllDossiers()) {
         if (options["verbose"]) {
             console.log(`Converting ${loi["signet"]} file…`);
@@ -158,16 +161,14 @@ async function convertDatasetDosLeg(dataDir, options) {
             continue;
         }
         loiReorganizedDir = path.join(dossiersReorganizedDir, String(session));
-        await fs.ensureDir(loiReorganizedDir);
         // Ajout des actes législatifs au dossier
         const actesLegislatifs = createActesLegislatifs(loi);
         const loiWithActes = { ...loi, actes_legislatifs: actesLegislatifs };
         const dossierFile = `${loi["signet"]}.json`;
-        const filePath = path.join(loiReorganizedDir, dossierFile);
-        await fs.writeJSON(filePath, loiWithActes, { spaces: 2 });
+        await fs.outputJSON(path.join(loiReorganizedDir, dossierFile), loiWithActes, { spaces: 2 });
     }
-    await convertTexteUrls(dataDir);
-    await convertRapportUrls(dataDir);
+    await convertTexteUrls(dataDir, options);
+    await convertRapportUrls(dataDir, options);
 }
 async function convertDatasetScrutins(dataDir, options) {
     const dataset = datasets.dosleg;
@@ -175,7 +176,9 @@ async function convertDatasetScrutins(dataDir, options) {
         console.log(`Converting database scrutins (${dataset.database}) data into files…`);
     }
     const scrutinsReorganizedDir = path.join(dataDir, SCRUTINS_FOLDER);
-    ensureAndClearDir(scrutinsReorganizedDir);
+    if (!options.keepDir) {
+        ensureAndClearDir(scrutinsReorganizedDir);
+    }
     for await (const scrutin of findAllScrutins(options["fromSession"])) {
         if (options["verbose"]) {
             console.log(`Converting ${scrutin["numero"]} file…`);
@@ -183,20 +186,21 @@ async function convertDatasetScrutins(dataDir, options) {
         let scrutinReorganizedDir = path.join(scrutinsReorganizedDir, String(UNDEFINED_SESSION));
         const session = scrutin["session"] || UNDEFINED_SESSION;
         scrutinReorganizedDir = path.join(scrutinsReorganizedDir, String(session));
-        await fs.ensureDir(scrutinReorganizedDir);
         const scrutinFileName = `${scrutin["numero"]}.json`;
-        await fs.writeJSON(path.join(scrutinReorganizedDir, scrutinFileName), scrutin, {
+        await fs.outputJSON(path.join(scrutinReorganizedDir, scrutinFileName), scrutin, {
             spaces: 2,
         });
     }
 }
-async function convertDatasetQuestions(dataDir) {
+async function convertDatasetQuestions(dataDir, options) {
     const dataset = datasets.questions;
     if (!options["silent"]) {
         console.log(`Converting database ${dataset.database} data into files…`);
     }
     const questionsReorganizedRootDir = path.join(dataDir, dataset.database);
-    ensureAndClearDir(questionsReorganizedRootDir);
+    if (!options.keepDir) {
+        ensureAndClearDir(questionsReorganizedRootDir);
+    }
     const limit = pLimit(10);
     const tasks = [];
     for await (const question of findAllQuestions()) {
@@ -205,22 +209,27 @@ async function convertDatasetQuestions(dataDir) {
                 console.log(`Converting ${question["reference"]} file…`);
             }
             const legislature = question["legislature"] ? question["legislature"] : 0;
-            const questionReorganizedDir = path.join(questionsReorganizedRootDir, String(legislature));
-            await fs.ensureDir(questionReorganizedDir);
             const questionFileName = `${question["reference"]}.json`;
-            await fs.writeJSON(path.join(questionReorganizedDir, questionFileName), question, { spaces: 2 });
+            await fs.outputJSON(path.join(questionsReorganizedRootDir, String(legislature), questionFileName), question, {
+                spaces: 2,
+            });
         }));
     }
     await Promise.all(tasks);
 }
-async function convertTexteUrls(dataDir) {
-    const textesDir = path.join(dataDir, TEXTE_FOLDER);
-    fs.ensureDirSync(textesDir);
-    const originalTextesDir = path.join(textesDir, DATA_ORIGINAL_FOLDER);
+async function convertTexteUrls(dataDir, options) {
+    const originalTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER);
+    const transformedTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_TRANSFORMED_FOLDER);
+    if (!options["silent"]) {
+        console.log(`Converting database textes data into files…`);
+    }
     for await (const texte of findSenatTexteUrls()) {
+        const session = texte.session ?? UNDEFINED_SESSION;
+        if (options["fromSession"] && session < options["fromSession"]) {
+            continue;
+        }
         const texteName = path.parse(texte.url).name;
-        const texteDir = path.join(originalTextesDir, `${texte.session ?? UNDEFINED_SESSION}`, texteName);
-        fs.ensureDirSync(texteDir);
+        const texteDir = path.join(originalTextesDir, `${session}`, texteName);
         const metadata = {
             name: texteName,
             session: texte.session,
@@ -232,20 +241,27 @@ async function convertTexteUrls(dataDir) {
             url_html: new URL(`${texteName}.html`, SENAT_TEXTE_BASE_URL),
             url_pdf: new URL(`${texteName}.pdf`, SENAT_TEXTE_BASE_URL),
         };
-        fs.writeJSONSync(path.join(texteDir, DOCUMENT_METADATA_FILE), metadata, {
+        fs.outputJSONSync(path.join(texteDir, DOCUMENT_METADATA_FILE), metadata, {
             spaces: 2,
         });
+        if (options.fetchDocuments) {
+            await processTexte(metadata, originalTextesDir, transformedTextesDir, options);
+        }
     }
 }
-async function convertRapportUrls(dataDir) {
-    const rapportsDir = path.join(dataDir, RAPPORT_FOLDER);
-    fs.ensureDirSync(rapportsDir);
-    const originalTextesDir = path.join(rapportsDir, DATA_ORIGINAL_FOLDER);
+async function convertRapportUrls(dataDir, options) {
+    const originalRapportsDir = path.join(dataDir, RAPPORT_FOLDER, DATA_ORIGINAL_FOLDER);
+    if (!options["silent"]) {
+        console.log(`Converting database rapports data into files…`);
+    }
     for await (const rapport of findSenatRapportUrls()) {
+        const session = rapport.session ?? UNDEFINED_SESSION;
+        if (options["fromSession"] && session < options["fromSession"]) {
+            continue;
+        }
         const parsedRapportUrl = path.parse(rapport.url);
         const rapportName = parsedRapportUrl.name;
-        const rapportDir = path.join(originalTextesDir, `${rapport.session ?? UNDEFINED_SESSION}`, rapportName);
-        fs.ensureDirSync(rapportDir);
+        const rapportDir = path.join(originalRapportsDir, `${session}`, rapportName);
         const rapportHtmlUrlBase = `${rapportName}_mono.html`;
         const rapportHtmlUrl = path.format({
             dir: parsedRapportUrl.dir,
@@ -263,12 +279,15 @@ async function convertRapportUrls(dataDir) {
             url_html: new URL(rapportHtmlUrl, SENAT_RAPPORT_BASE_URL),
             url_pdf: new URL(rapportPdfUrl, SENAT_RAPPORT_BASE_URL),
         };
-        fs.writeJSONSync(path.join(rapportDir, DOCUMENT_METADATA_FILE), metadata, {
+        fs.outputJSONSync(path.join(rapportDir, DOCUMENT_METADATA_FILE), metadata, {
             spaces: 2,
         });
+        if (options.fetchDocuments) {
+            await processRapport(metadata, originalRapportsDir, options);
+        }
     }
 }
-async function convertDatasetSens(dataDir) {
+async function convertDatasetSens(dataDir, options) {
     const dataset = datasets.sens;
     if (!options["silent"]) {
         console.log(`Converting database ${dataset.database} data into files…`);
@@ -277,16 +296,18 @@ async function convertDatasetSens(dataDir) {
     const senateursReorganizedDir = path.join(sensReorganizedRootDir, SENS_SENATEURS_FOLDER);
     const circonscriptionsReorganizedDir = path.join(sensReorganizedRootDir, SENS_CIRCONSCRIPTIONS_FOLDER);
     const organismesReorganizedDir = path.join(sensReorganizedRootDir, SENS_ORGANISMES_FOLDER);
-    ensureAndClearDir(sensReorganizedRootDir);
-    ensureAndClearDir(senateursReorganizedDir);
-    ensureAndClearDir(circonscriptionsReorganizedDir);
-    ensureAndClearDir(organismesReorganizedDir);
+    if (!options.keepDir) {
+        ensureAndClearDir(sensReorganizedRootDir);
+        ensureAndClearDir(senateursReorganizedDir);
+        ensureAndClearDir(circonscriptionsReorganizedDir);
+        ensureAndClearDir(organismesReorganizedDir);
+    }
     for await (const sen of findAllSens()) {
         if (options["verbose"]) {
             console.log(`Converting ${sen["matricule"]} file…`);
         }
         const senFileName = `${sen["matricule"]}.json`;
-        fs.writeJSONSync(path.join(senateursReorganizedDir, senFileName), sen, {
+        fs.outputJSONSync(path.join(senateursReorganizedDir, senFileName), sen, {
             spaces: 2,
         });
     }
@@ -295,16 +316,18 @@ async function convertDatasetSens(dataDir) {
             console.log(`Converting ${circonscription["identifiant"]} file…`);
         }
         const circonscriptionFileName = `${circonscription["identifiant"]}.json`;
-        fs.writeJSONSync(path.join(circonscriptionsReorganizedDir, circonscriptionFileName), circonscription, { spaces: 2 });
+        fs.outputJSONSync(path.join(circonscriptionsReorganizedDir, circonscriptionFileName), circonscription, {
+            spaces: 2,
+        });
     }
     for await (const organisme of findAllOrganismes()) {
         if (options["verbose"]) {
             console.log(`Converting ${organisme["code"]} file…`);
         }
         const organismeFileName = `${organisme["code"]}.json`;
-        const organismeDir = path.join(organismesReorganizedDir, organisme["type_code"]);
-        fs.ensureDirSync(organismeDir);
-        fs.writeJSONSync(path.join(organismeDir, organismeFileName), organisme, { spaces: 2 });
+        fs.outputJSONSync(path.join(organismesReorganizedDir, organisme["type_code"], organismeFileName), organisme, {
+            spaces: 2,
+        });
     }
 }
 convertData()

package/lib/scripts/data-download.js

@@ -5,5 +5,8 @@ try {
     execSync(`tsx src/scripts/convert_data.ts ${args}`, { stdio: "inherit" });
 }
 catch (error) {
-    process.exit(1);
+    if (error.status !== 10) {
+        console.error("Error during data retrieval:", error);
+        process.exit(error.status || 1);
+    }
 }

package/lib/scripts/retrieve_documents.d.ts

@@ -1 +1,2 @@
-export {};
+export declare function processTexte(texteMetadata: any, originalTextesDir: string, transformedTextesDir: string, options: any): Promise<void>;
+export declare function processRapport(rapportMetadata: any, originalRapportsDir: string, options: any): Promise<void>;

package/lib/scripts/retrieve_documents.js

@@ -4,17 +4,12 @@ import fs from "fs-extra";
 import { DateTime } from "luxon";
 import path from "path";
 import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatDossiersLegislatifsRapportUrls, iterLoadSenatDossiersLegislatifsTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, } from "../loaders";
-import { parseExposeDesMotifs, parseTexte, parseTexteFromFile } from "../model/texte";
+import { parseExposeDesMotifs, parseTexte, parseTexteFromFile } from "../parsers/texte";
 import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions";
 import { commonOptions } from "./shared/cli_helpers";
 import { ensureAndClearDir, fetchWithRetry, isOptionEmptyOrHasValue } from "./shared/util";
 const optionsDefinitions = [
     ...commonOptions,
-    {
-        help: "parse and convert documents into JSON (textes only for now, requires format xml)",
-        name: "parseDocuments",
-        type: Boolean,
-    },
     {
         alias: "F",
         help: "formats of documents to retrieve (xml/html/pdf for textes, html/pdf for rapports); leave empty for all",
@@ -38,205 +33,142 @@ const options = commandLineArgs(optionsDefinitions);
 const textDecoder = new TextDecoder("utf8");
 const today = DateTime.now();
 function isDocumentRecent(documentDate, daysThreshold) {
-    if (!documentDate) {
+    if (!documentDate)
         return false;
-    }
     const docDate = DateTime.fromISO(documentDate);
-    if (!docDate.isValid) {
-        return false;
-    }
-    const daysDiff = today.diff(docDate, "days").days;
-    return daysDiff <= daysThreshold;
+    return docDate.isValid && today.diff(docDate, "days").days <= daysThreshold;
 }
-async function retrieveTextes(dataDir, sessions) {
-    const textesDir = path.join(dataDir, TEXTE_FOLDER);
-    fs.ensureDirSync(textesDir);
-    const originalTextesDir = path.join(textesDir, DATA_ORIGINAL_FOLDER);
-    const transformedTextesDir = path.join(textesDir, DATA_TRANSFORMED_FOLDER);
-    if (options["parseDocuments"]) {
-        ensureAndClearDir(transformedTextesDir);
+function shouldDownload(filePath, docDate, options) {
+    if (options.force)
+        return true;
+    if (!fs.existsSync(filePath))
+        return true;
+    if (options.onlyRecent !== undefined) {
+        return isDocumentRecent(docDate, options.onlyRecent);
+    }
+    return false;
+}
+async function downloadDocument(documentUrl, verbose) {
+    if (verbose) {
+        console.log(`Downloading document ${documentUrl}…`);
     }
-    let retrievedTextesCount = 0;
-    const texteUrlsNotFoundOrError = [];
-    const texteUrlsParseError = [];
-    for (const session of sessions) {
-        for (const { item: texteMetadata } of iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session)) {
-            const texteDir = path.join(originalTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name);
-            fs.ensureDirSync(texteDir);
-            let exposeDesMotifsContent = null;
-            if (texteMetadata.url_expose_des_motifs) {
-                exposeDesMotifsContent = await downloadExposeDesMotifs(texteDir, texteMetadata.name, String(texteMetadata.url_expose_des_motifs));
-            }
-            if (isOptionEmptyOrHasValue(options["formats"], "xml")) {
-                const textePath = path.join(texteDir, `${texteMetadata.name}.xml`);
-                let texteBuffer = null;
-                // Check if document should be skipped based on onlyRecent option
-                const shouldSkip = !options["force"] &&
-                    fs.existsSync(textePath) &&
-                    (options["only-recent"] === undefined || !isDocumentRecent(texteMetadata.date, options["only-recent"]));
-                if (shouldSkip) {
-                    if (!options["silent"]) {
-                        console.info(`Already downloaded texte ${textePath}…`);
-                    }
-                }
-                else {
-                    texteBuffer = await downloadDocument(texteMetadata.url_xml.toString());
-                    if (!texteBuffer) {
-                        texteUrlsNotFoundOrError.push(texteMetadata.url_xml);
-                        continue;
-                    }
-                    fs.writeFileSync(textePath, Buffer.from(texteBuffer));
-                    retrievedTextesCount++;
-                }
-                if (options["parseDocuments"]) {
-                    const parsedTexte = await parseDocument(texteMetadata.session, transformedTextesDir, textePath, texteMetadata.name, texteBuffer, exposeDesMotifsContent);
-                    if (!parsedTexte) {
-                        texteUrlsParseError.push(texteMetadata.url_xml);
-                    }
-                }
-            }
-            if (isOptionEmptyOrHasValue(options["formats"], "html")) {
-                const textePath = path.join(texteDir, `${texteMetadata.name}.html`);
-                // Check if document should be skipped based on onlyRecent option
-                const shouldSkip = !options["force"] &&
-                    fs.existsSync(textePath) &&
-                    (options["only-recent"] === undefined || !isDocumentRecent(texteMetadata.date, options["only-recent"]));
-                if (shouldSkip) {
-                    if (!options["silent"]) {
-                        console.info(`Already downloaded texte ${textePath}…`);
-                    }
-                }
-                else {
-                    const texteBuffer = await downloadDocument(texteMetadata.url_html.toString());
-                    if (!texteBuffer) {
-                        texteUrlsNotFoundOrError.push(texteMetadata.url_html);
-                        continue;
-                    }
-                    fs.writeFileSync(textePath, Buffer.from(texteBuffer));
-                    retrievedTextesCount++;
+    try {
+        const response = await fetchWithRetry(documentUrl);
+        if (!response.ok) {
+            if (response.status === 404) {
+                if (verbose) {
+                    console.warn(`Document ${documentUrl} not found`);
                 }
             }
-            if (isOptionEmptyOrHasValue(options["formats"], "pdf")) {
-                const textePath = path.join(texteDir, `${texteMetadata.name}.pdf`);
-                // Check if document should be skipped based on onlyRecent option
-                const shouldSkip = !options["force"] &&
-                    fs.existsSync(textePath) &&
-                    (options["only-recent"] === undefined || !isDocumentRecent(texteMetadata.date, options["only-recent"]));
-                if (shouldSkip) {
-                    if (!options["silent"]) {
-                        console.info(`Already downloaded texte ${textePath}…`);
-                    }
-                }
-                else {
-                    const texteBuffer = await downloadDocument(texteMetadata.url_pdf.toString());
-                    if (!texteBuffer) {
-                        texteUrlsNotFoundOrError.push(texteMetadata.url_pdf);
-                        continue;
-                    }
-                    fs.writeFileSync(textePath, Buffer.from(texteBuffer));
-                    retrievedTextesCount++;
+            else {
+                if (verbose) {
+                    console.error(`An error occurred while retrieving document ${documentUrl}: ${response.status}`);
                 }
             }
+            return null;
         }
+        return response.arrayBuffer();
     }
-    if (options["verbose"]) {
-        console.log(`${retrievedTextesCount} textes retrieved`);
-        console.log(`${texteUrlsNotFoundOrError.length} textes failed to be retrieved with URLs ${texteUrlsNotFoundOrError.join(", ")}`);
-        if (options["parseDocuments"]) {
-            console.log(`${texteUrlsParseError.length} textes failed to be parsed with URLs ${texteUrlsParseError.join(", ")}`);
-        }
+    catch (error) {
+        console.error(error.message);
+        return null;
     }
 }
-async function retrieveRapports(dataDir, sessions) {
-    const rapportsDir = path.join(dataDir, RAPPORT_FOLDER);
-    fs.ensureDirSync(rapportsDir);
-    const originalRapportsDir = path.join(rapportsDir, DATA_ORIGINAL_FOLDER);
-    let retrievedRapportsCount = 0;
-    const rapportUrlsNotFoundOrError = [];
-    for (const session of sessions) {
-        for (const { item: rapportMetadata } of iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session)) {
-            const rapportDir = path.join(originalRapportsDir, `${rapportMetadata.session ?? UNDEFINED_SESSION}`, rapportMetadata.name);
-            fs.ensureDirSync(rapportDir);
-            if (isOptionEmptyOrHasValue(options["formats"], "html")) {
-                const rapportPath = path.join(rapportDir, `${rapportMetadata.name}.html`);
-                // Check if document should be skipped based on onlyRecent option
-                const shouldSkip = !options["force"] &&
-                    fs.existsSync(rapportPath) &&
-                    (options["only-recent"] === undefined || !isDocumentRecent(rapportMetadata.date, options["only-recent"]));
-                if (shouldSkip) {
-                    if (!options["silent"]) {
-                        console.info(`Already downloaded rapport ${rapportPath}…`);
-                    }
-                    continue;
-                }
-                const rapportBuffer = await downloadDocument(rapportMetadata.url_html.toString());
-                if (!rapportBuffer) {
-                    rapportUrlsNotFoundOrError.push(rapportMetadata.url_html);
-                    continue;
-                }
-                fs.writeFileSync(rapportPath, Buffer.from(rapportBuffer));
-                retrievedRapportsCount++;
-            }
-            if (isOptionEmptyOrHasValue(options["formats"], "pdf")) {
-                const rapportPath = path.join(rapportDir, `${rapportMetadata.name}.pdf`);
-                // Check if document should be skipped based on onlyRecent option
-                const shouldSkip = !options["force"] &&
-                    fs.existsSync(rapportPath) &&
-                    (options["only-recent"] === undefined || !isDocumentRecent(rapportMetadata.date, options["only-recent"]));
-                if (shouldSkip) {
-                    if (!options["silent"]) {
-                        console.info(`Already downloaded rapport ${rapportPath}…`);
-                    }
-                    continue;
-                }
-                const rapportBuffer = await downloadDocument(rapportMetadata.url_pdf.toString());
-                if (!rapportBuffer) {
-                    rapportUrlsNotFoundOrError.push(rapportMetadata.url_pdf);
-                    continue;
-                }
-                fs.writeFileSync(rapportPath, Buffer.from(rapportBuffer));
-                retrievedRapportsCount++;
+async function processDocument(url, destPath, docDate, options) {
+    if (!shouldDownload(destPath, docDate, options)) {
+        if (options.verbose)
+            console.info(`Already downloaded ${destPath}…`);
+        return { success: true, skipped: true, buffer: null };
+    }
+    const arrayBuffer = await downloadDocument(url, options.verbose);
+    if (!arrayBuffer) {
+        return { success: false, skipped: false, buffer: null };
+    }
+    const buffer = Buffer.from(arrayBuffer);
+    await fs.outputFile(destPath, buffer);
+    return { success: true, skipped: false, buffer };
+}
+export async function processTexte(texteMetadata, originalTextesDir, transformedTextesDir, options) {
+    const texteDir = path.join(originalTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name);
+    let exposeDesMotifsContent = null;
+    if (texteMetadata.url_expose_des_motifs) {
+        const exposePath = path.join(texteDir, `${texteMetadata.name}-expose.html`);
+        const res = await processDocument(texteMetadata.url_expose_des_motifs.toString(), exposePath, texteMetadata.date, options);
+        if (res.buffer) {
+            exposeDesMotifsContent = res.buffer;
+        }
+        else if (res.skipped && options.parseDocuments) {
+            if (await fs.pathExists(exposePath)) {
+                exposeDesMotifsContent = await fs.readFile(exposePath);
             }
         }
     }
-    if (options["verbose"]) {
-        console.log(`${retrievedRapportsCount} rapports retrieved`);
-        console.log(`${rapportUrlsNotFoundOrError.length} rapports failed with URLs ${rapportUrlsNotFoundOrError.join(", ")}`);
+    const formats = [
+        { type: "xml", url: texteMetadata.url_xml, isParseTarget: true },
+        { type: "html", url: texteMetadata.url_html, isParseTarget: false },
+        { type: "pdf", url: texteMetadata.url_pdf, isParseTarget: false },
+    ];
+    for (const format of formats) {
+        if (!isOptionEmptyOrHasValue(options.formats, format.type))
+            continue;
+        const destPath = path.join(texteDir, `${texteMetadata.name}.${format.type}`);
+        const result = await processDocument(format.url.toString(), destPath, texteMetadata.date, options);
+        // Specific logic: Parsing (Only applies to XML)
+        if (format.isParseTarget && options.parseDocuments) {
+            await parseDocument(texteMetadata.session, transformedTextesDir, destPath, texteMetadata.name, result.buffer, exposeDesMotifsContent, options);
+        }
     }
 }
-async function downloadExposeDesMotifs(texteDir, texteName, url) {
-    const content = await downloadDocument(url);
-    if (!content) {
-        return null;
+export async function processRapport(rapportMetadata, originalRapportsDir, options) {
+    const rapportDir = path.join(originalRapportsDir, `${rapportMetadata.session ?? UNDEFINED_SESSION}`, rapportMetadata.name);
+    const formats = [
+        { type: "html", url: rapportMetadata.url_html },
+        { type: "pdf", url: rapportMetadata.url_pdf },
+    ];
+    for (const format of formats) {
+        if (!isOptionEmptyOrHasValue(options["formats"], format.type))
+            continue;
+        const destPath = path.join(rapportDir, `${rapportMetadata.name}.${format.type}`);
+        await processDocument(format.url.toString(), destPath, rapportMetadata.date, options);
     }
-    const exposeDesMotifsPath = path.join(texteDir, `${texteName}-expose.html`);
-    fs.writeFileSync(exposeDesMotifsPath, Buffer.from(content));
-    return content;
 }
-async function downloadDocument(documentUrl) {
-    if (!options["silent"]) {
-        console.log(`Downloading document ${documentUrl}…`);
+async function retrieveTextes(dataDir, sessions) {
+    const originalTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER);
+    const transformedTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_TRANSFORMED_FOLDER);
+    if (options["parseDocuments"]) {
+        ensureAndClearDir(transformedTextesDir);
     }
-    try {
-        const response = await fetchWithRetry(documentUrl);
-        if (!response.ok) {
-            if (response.status === 404) {
-                console.warn(`Texte ${documentUrl} not found`);
-            }
-            else {
-                console.error(`An error occurred while retrieving document ${documentUrl}: ${response.status}`);
-            }
-            return null;
+    const dlOptions = {
+        force: options["force"],
+        silent: options["silent"],
+        verbose: options["verbose"],
+        onlyRecent: options["only-recent"],
+        formats: options["formats"],
+        parseDocuments: options["parseDocuments"],
+    };
+    for (const session of sessions) {
+        for (const { item: texteMetadata } of iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session)) {
+            await processTexte(texteMetadata, originalTextesDir, transformedTextesDir, dlOptions);
         }
-        return response.arrayBuffer();
     }
-    catch (error) {
-        console.error(error.message);
-        return null;
+}
+async function retrieveRapports(dataDir, sessions) {
+    const originalRapportsDir = path.join(dataDir, RAPPORT_FOLDER, DATA_ORIGINAL_FOLDER);
+    const dlOptions = {
+        force: options["force"],
+        silent: options["silent"],
+        verbose: options["verbose"],
+        onlyRecent: options["only-recent"],
+        formats: options["formats"],
+    };
+    for (const session of sessions) {
+        for (const { item: rapportMetadata } of iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session)) {
+            await processRapport(rapportMetadata, originalRapportsDir, dlOptions);
+        }
     }
 }
-async function parseDocument(session, transformedTextesDir, textePath, texteName, texteBuffer, exposeDesMotifs = null) {
-    if (!options["silent"]) {
+async function parseDocument(session, transformedTextesDir, textePath, texteName, texteBuffer, exposeDesMotifs = null, options = {}) {
+    if (options.verbose) {
         console.log(`Parsing texte ${textePath}…`);
     }
     let parsedTexte;
@@ -247,19 +179,17 @@ async function parseDocument(session, transformedTextesDir, textePath, texteName
     else {
         parsedTexte = await parseTexteFromFile(textePath);
     }
-    if (!parsedTexte) {
+    if (!parsedTexte)
         return null;
-    }
     if (exposeDesMotifs) {
-        if (!options["silent"]) {
+        if (options.verbose) {
             console.log("Parsing exposé des motifs…");
         }
         const exposeDesMotifsHtml = textDecoder.decode(exposeDesMotifs);
         parsedTexte.exposeDesMotifs = parseExposeDesMotifs(exposeDesMotifsHtml);
     }
     const transformedTexteDir = path.join(transformedTextesDir, `${session ?? UNDEFINED_SESSION}`, texteName);
-    fs.ensureDirSync(transformedTexteDir);
-    fs.writeJSONSync(path.join(transformedTexteDir, `${texteName}.json`), parsedTexte, { spaces: 2 });
+    await fs.outputJSON(path.join(transformedTexteDir, `${texteName}.json`), parsedTexte, { spaces: 2 });
     return parsedTexte;
 }
 async function main() {
@@ -277,9 +207,11 @@ async function main() {
         console.timeEnd("documents processing time");
     }
 }
-main()
-    .then(() => process.exit(0))
-    .catch((error) => {
-    console.log(error);
-    process.exit(1);
-});
+if (process.argv[1].endsWith("retrieve_documents.ts")) {
+    main()
+        .then(() => process.exit(0))
+        .catch((error) => {
+        console.log(error);
+        process.exit(1);
+    });
+}

package/lib/scripts/retrieve_videos.js

@@ -8,7 +8,7 @@ import * as cheerio from "cheerio";
 import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatAgendas } from "../loaders";
 import { getSessionsFromStart } from "../types/sessions";
 import { commonOptions } from "./shared/cli_helpers";
-import { getFirstInterventionStartTimecode } from "../utils/nvs-timecode";
+import { getAgendaSegmentTimecodes } from "../utils/nvs-timecode";
 import { decodeHtmlEntities } from "../utils/string_cleaning";
 import { dice, normalize, scoreVideo } from "../utils/scoring";
 import { epochToParisDateTime, toFRDate, toTargetEpoch } from "../utils/date";
@@ -379,16 +379,22 @@ async function processGroupedReunion(agenda, session, dataDir) {
             console.warn(e);
         }
     }
-    let timecodeDebutVideo = null;
-    if (dataTxt && finalTxt) {
-        timecodeDebutVideo = getFirstInterventionStartTimecode(dataTxt, finalTxt);
-        if (timecodeDebutVideo === null) {
-            console.warn(`[warn] Cannot retrieve start video timecode from reunion` + reunionUid);
-        }
-    }
     // ==== 4) Update agenda file (only if accepted + m3u8) ====
     if ((accepted || skipDownload) && master) {
         const agendaJsonPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session), `${agenda.uid}.json`);
+        let timecodeDebutVideo = null;
+        let timecodeFinVideo = null;
+        if (dataTxt && finalTxt) {
+            const agendaKey = agenda.titre || agenda.objet || "";
+            const seg = getAgendaSegmentTimecodes(dataTxt, finalTxt, agendaKey);
+            if (!seg) {
+                console.warn(`[warn] Cannot retrieve agenda segment timecodes from reunion ${reunionUid}`);
+            }
+            else {
+                timecodeDebutVideo = seg.start;
+                timecodeFinVideo = seg.end;
+            }
+        }
         if (await fs.pathExists(agendaJsonPath)) {
             const raw = await fsp.readFile(agendaJsonPath, "utf-8");
             let obj;
@@ -403,6 +409,7 @@ async function processGroupedReunion(agenda, session, dataDir) {
                 const next = { ...obj, urlVideo: master };
                 if (timecodeDebutVideo != null) {
                     next.timecodeDebutVideo = timecodeDebutVideo;
+                    next.timecodeFinVideo = timecodeFinVideo;
                 }
                 await writeIfChanged(agendaJsonPath, JSON.stringify(next, null, 2));
                 if (!options["silent"]) {

package/lib/scripts/shared/cli_helpers.d.ts

@@ -64,6 +64,16 @@ export declare const pullOption: {
     name: string;
     type: BooleanConstructor;
 };
+export declare const fetchDocumentsOption: {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const parseDocumentsOption: {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
 export declare const commonOptions: ({
     defaultOption: boolean;
     help: string;

package/lib/scripts/shared/cli_helpers.js

@@ -64,6 +64,16 @@ export const pullOption = {
     name: "pull",
     type: Boolean,
 };
+export const fetchDocumentsOption = {
+    help: "download documents",
+    name: "fetchDocuments",
+    type: Boolean,
+};
+export const parseDocumentsOption = {
+    help: "parse documents",
+    name: "parseDocuments",
+    type: Boolean,
+};
 export const commonOptions = [
     categoriesOption,
     dataDirDefaultOption,
@@ -76,4 +86,6 @@ export const commonOptions = [
     commitOption,
     remoteOption,
     pullOption,
+    fetchDocumentsOption,
+    parseDocumentsOption,
 ];

package/lib/scripts/test_iter_load.js

@@ -1,29 +1,18 @@
-import { iterLoadSenatScrutins } from "../loaders";
+import { iterLoadSenatAmendements, iterLoadSenatDossiersLegislatifs } from "../loaders";
 import commandLineArgs from "command-line-args";
 import { dataDirDefaultOption } from "./shared/cli_helpers";
 const optionsDefinitions = [dataDirDefaultOption];
 const options = commandLineArgs(optionsDefinitions);
-const noValidation = false;
 const session = 2024;
-const s = new Set();
-for (const { item: scrutin } of iterLoadSenatScrutins(options["dataDir"], session, { noValidation: noValidation })) {
-    s.add(scrutin["lecture_libelle"]);
+const sinceCommit = undefined;
+for (const { item: amendement } of iterLoadSenatAmendements(options["dataDir"], session, {
+    log: true,
+    sinceCommit: sinceCommit,
+})) {
+    console.log(amendement["numero"]);
 }
-console.log(s);
-/*
-for (const { item: amendement } of iterLoadSenatAmendements(
-  options["dataDir"],
-  session,
-  { noValidation: noValidation },
-)) {
-  console.log(amendement["numero"])
+for (const { item: dossierLegislatif } of iterLoadSenatDossiersLegislatifs(options["dataDir"], session, {
+    sinceCommit: sinceCommit,
+})) {
+    console.log(dossierLegislatif["numero"]);
 }
-
-for (const { item: dossierLegislatif } of iterLoadSenatDossiersLegislatifs(
-  options["dataDir"],
-  session,
-  { noValidation: noValidation },
-)) {
-  console.log(dossierLegislatif["numero"])
-}
-*/ 

package/lib/types/agenda.d.ts

@@ -31,6 +31,7 @@ export interface Reunion {
     transcriptionRef?: string;
     urlVideo?: string;
     timecodeDebutVideo?: number;
+    timecodeFinVideo?: number;
     odj?: ReunionOdj;
 }
 export interface ReunionOdjPoint {

package/lib/utils/nvs-timecode.d.ts

@@ -1 +1,7 @@
-export declare function getFirstInterventionStartTimecode(dataNvs: string, finalPlayerNvs: string): number | null;
+export declare function getAgendaSegmentTimecodes(dataNvs: string, finalPlayerNvs: string, agendaTitleOrObjet: string): {
+    start: number;
+    end: number | null;
+    chapterId: string;
+    nextChapterId: string | null;
+    score: number;
+} | null;

package/lib/utils/nvs-timecode.js

@@ -1,4 +1,7 @@
 import { XMLParser } from "fast-xml-parser";
+import { dice, normalize } from "./scoring";
+import { decodeHtmlEntities } from "./string_cleaning";
+const CHAPTER_MATCH_THRESHOLD = 0.5;
 const xmlParser = new XMLParser({
     ignoreAttributes: false,
     attributeNamePrefix: "@_",
@@ -20,20 +23,57 @@ function getTimecodeForChapterId(finalPlayerNvs, chapterId) {
         return null;
     return Math.floor(ms / 1000);
 }
-export function getFirstInterventionStartTimecode(dataNvs, finalPlayerNvs) {
-    const firstChapterId = getFirstChapterId(dataNvs);
-    if (!firstChapterId)
-        return null;
-    return getTimecodeForChapterId(finalPlayerNvs, firstChapterId);
+function toArray(v) {
+    if (!v)
+        return [];
+    return Array.isArray(v) ? v : [v];
 }
-function getFirstChapterId(dataNvs) {
+function getLevel1Chapters(dataNvs) {
     const xml = xmlParser.parse(dataNvs);
-    const rootChapters = xml?.data?.chapters?.chapter;
-    if (!rootChapters)
+    const root = xml?.data?.chapters?.chapter ?? xml?.chapters?.chapter;
+    const roots = toArray(root);
+    return roots
+        .map((ch, i) => {
+        const id = ch?.id ?? ch?.["@_id"];
+        const labelRaw = ch?.label ?? ch?.["@_label"] ?? "";
+        return {
+            id: String(id),
+            label: decodeHtmlEntities(String(labelRaw)).trim(),
+            index: i,
+        };
+    })
+        .filter((c) => c.id && c.label);
+}
+function pickBestLevel1ChapterForAgenda(chapters, agendaTitle) {
+    const q = normalize(agendaTitle);
+    let best = null;
+    for (const ch of chapters) {
+        const s = dice(q, ch.label);
+        if (!best || s > best.score)
+            best = { chapter: ch, score: s };
+    }
+    if (!best || best.score < CHAPTER_MATCH_THRESHOLD)
+        return { chapter: chapters[0], score: 0 };
+    return best;
+}
+export function getAgendaSegmentTimecodes(dataNvs, finalPlayerNvs, agendaTitleOrObjet) {
+    const l1 = getLevel1Chapters(dataNvs);
+    if (!l1.length)
+        return null;
+    const best = pickBestLevel1ChapterForAgenda(l1, agendaTitleOrObjet);
+    if (!best)
         return null;
-    const chaptersArray = Array.isArray(rootChapters) ? rootChapters : [rootChapters];
-    const firstChapter = chaptersArray[0];
-    if (!firstChapter || !firstChapter["@_id"])
+    const chapter = best.chapter;
+    const next = l1[chapter.index + 1] ?? null;
+    const start = getTimecodeForChapterId(finalPlayerNvs, chapter.id);
+    if (start == null)
         return null;
-    return String(firstChapter["@_id"]);
+    const end = next ? getTimecodeForChapterId(finalPlayerNvs, next.id) : null;
+    return {
+        start,
+        end,
+        chapterId: chapter.id,
+        nextChapterId: next?.id ?? null,
+        score: best.score,
+    };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.20.20",
+  "version": "2.20.22",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",
@@ -52,7 +52,6 @@
     "data:retrieve_open_data": "tsx src/scripts/retrieve_open_data.ts --all",
     "data:retrieve_senateurs_photos": "tsx src/scripts/retrieve_senateurs_photos.ts --fetch",
     "data:retrieve_videos": "tsx src/scripts/retrieve_videos.ts",
-    "data:parse_textes_lois": "tsx src/scripts/parse_textes.ts",
     "prepare": "npm run build",
     "prepublishOnly": "npm run build",
     "prettier": "prettier --write 'src/**/*.ts' 'tests/**/*.test.ts'",