@tricoteuses/senat 2.20.21 → 2.20.23

package/lib/scripts/retrieve_documents.js

@@ -4,17 +4,12 @@ import fs from "fs-extra";
 import { DateTime } from "luxon";
 import path from "path";
 import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatDossiersLegislatifsRapportUrls, iterLoadSenatDossiersLegislatifsTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, } from "../loaders";
-import { parseExposeDesMotifs, parseTexte, parseTexteFromFile } from "../model/texte";
+import { parseExposeDesMotifs, parseTexte, parseTexteFromFile } from "../parsers/texte";
 import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions";
 import { commonOptions } from "./shared/cli_helpers";
 import { ensureAndClearDir, fetchWithRetry, isOptionEmptyOrHasValue } from "./shared/util";
 const optionsDefinitions = [
     ...commonOptions,
-    {
-        help: "parse and convert documents into JSON (textes only for now, requires format xml)",
-        name: "parseDocuments",
-        type: Boolean,
-    },
     {
         alias: "F",
         help: "formats of documents to retrieve (xml/html/pdf for textes, html/pdf for rapports); leave empty for all",
@@ -38,205 +33,142 @@ const options = commandLineArgs(optionsDefinitions);
 const textDecoder = new TextDecoder("utf8");
 const today = DateTime.now();
 function isDocumentRecent(documentDate, daysThreshold) {
-    if (!documentDate) {
+    if (!documentDate)
         return false;
-    }
     const docDate = DateTime.fromISO(documentDate);
-    if (!docDate.isValid) {
-        return false;
-    }
-    const daysDiff = today.diff(docDate, "days").days;
-    return daysDiff <= daysThreshold;
+    return docDate.isValid && today.diff(docDate, "days").days <= daysThreshold;
 }
-async function retrieveTextes(dataDir, sessions) {
-    const textesDir = path.join(dataDir, TEXTE_FOLDER);
-    fs.ensureDirSync(textesDir);
-    const originalTextesDir = path.join(textesDir, DATA_ORIGINAL_FOLDER);
-    const transformedTextesDir = path.join(textesDir, DATA_TRANSFORMED_FOLDER);
-    if (options["parseDocuments"]) {
-        ensureAndClearDir(transformedTextesDir);
+function shouldDownload(filePath, docDate, options) {
+    if (options.force)
+        return true;
+    if (!fs.existsSync(filePath))
+        return true;
+    if (options.onlyRecent !== undefined) {
+        return isDocumentRecent(docDate, options.onlyRecent);
+    }
+    return false;
+}
+async function downloadDocument(documentUrl, verbose) {
+    if (verbose) {
+        console.log(`Downloading document ${documentUrl}…`);
     }
-    let retrievedTextesCount = 0;
-    const texteUrlsNotFoundOrError = [];
-    const texteUrlsParseError = [];
-    for (const session of sessions) {
-        for (const { item: texteMetadata } of iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session)) {
-            const texteDir = path.join(originalTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name);
-            fs.ensureDirSync(texteDir);
-            let exposeDesMotifsContent = null;
-            if (texteMetadata.url_expose_des_motifs) {
-                exposeDesMotifsContent = await downloadExposeDesMotifs(texteDir, texteMetadata.name, String(texteMetadata.url_expose_des_motifs));
-            }
-            if (isOptionEmptyOrHasValue(options["formats"], "xml")) {
-                const textePath = path.join(texteDir, `${texteMetadata.name}.xml`);
-                let texteBuffer = null;
-                // Check if document should be skipped based on onlyRecent option
-                const shouldSkip = !options["force"] &&
-                    fs.existsSync(textePath) &&
-                    (options["only-recent"] === undefined || !isDocumentRecent(texteMetadata.date, options["only-recent"]));
-                if (shouldSkip) {
-                    if (!options["silent"]) {
-                        console.info(`Already downloaded texte ${textePath}…`);
-                    }
-                }
-                else {
-                    texteBuffer = await downloadDocument(texteMetadata.url_xml.toString());
-                    if (!texteBuffer) {
-                        texteUrlsNotFoundOrError.push(texteMetadata.url_xml);
-                        continue;
-                    }
-                    fs.writeFileSync(textePath, Buffer.from(texteBuffer));
-                    retrievedTextesCount++;
-                }
-                if (options["parseDocuments"]) {
-                    const parsedTexte = await parseDocument(texteMetadata.session, transformedTextesDir, textePath, texteMetadata.name, texteBuffer, exposeDesMotifsContent);
-                    if (!parsedTexte) {
-                        texteUrlsParseError.push(texteMetadata.url_xml);
-                    }
-                }
-            }
-            if (isOptionEmptyOrHasValue(options["formats"], "html")) {
-                const textePath = path.join(texteDir, `${texteMetadata.name}.html`);
-                // Check if document should be skipped based on onlyRecent option
-                const shouldSkip = !options["force"] &&
-                    fs.existsSync(textePath) &&
-                    (options["only-recent"] === undefined || !isDocumentRecent(texteMetadata.date, options["only-recent"]));
-                if (shouldSkip) {
-                    if (!options["silent"]) {
-                        console.info(`Already downloaded texte ${textePath}…`);
-                    }
-                }
-                else {
-                    const texteBuffer = await downloadDocument(texteMetadata.url_html.toString());
-                    if (!texteBuffer) {
-                        texteUrlsNotFoundOrError.push(texteMetadata.url_html);
-                        continue;
-                    }
-                    fs.writeFileSync(textePath, Buffer.from(texteBuffer));
-                    retrievedTextesCount++;
+    try {
+        const response = await fetchWithRetry(documentUrl);
+        if (!response.ok) {
+            if (response.status === 404) {
+                if (verbose) {
+                    console.warn(`Document ${documentUrl} not found`);
                 }
             }
-            if (isOptionEmptyOrHasValue(options["formats"], "pdf")) {
-                const textePath = path.join(texteDir, `${texteMetadata.name}.pdf`);
-                // Check if document should be skipped based on onlyRecent option
-                const shouldSkip = !options["force"] &&
-                    fs.existsSync(textePath) &&
-                    (options["only-recent"] === undefined || !isDocumentRecent(texteMetadata.date, options["only-recent"]));
-                if (shouldSkip) {
-                    if (!options["silent"]) {
-                        console.info(`Already downloaded texte ${textePath}…`);
-                    }
-                }
-                else {
-                    const texteBuffer = await downloadDocument(texteMetadata.url_pdf.toString());
-                    if (!texteBuffer) {
-                        texteUrlsNotFoundOrError.push(texteMetadata.url_pdf);
-                        continue;
-                    }
-                    fs.writeFileSync(textePath, Buffer.from(texteBuffer));
-                    retrievedTextesCount++;
+            else {
+                if (verbose) {
+                    console.error(`An error occurred while retrieving document ${documentUrl}: ${response.status}`);
                 }
             }
+            return null;
         }
+        return response.arrayBuffer();
     }
-    if (options["verbose"]) {
-        console.log(`${retrievedTextesCount} textes retrieved`);
-        console.log(`${texteUrlsNotFoundOrError.length} textes failed to be retrieved with URLs ${texteUrlsNotFoundOrError.join(", ")}`);
-        if (options["parseDocuments"]) {
-            console.log(`${texteUrlsParseError.length} textes failed to be parsed with URLs ${texteUrlsParseError.join(", ")}`);
-        }
+    catch (error) {
+        console.error(error.message);
+        return null;
     }
 }
-async function retrieveRapports(dataDir, sessions) {
-    const rapportsDir = path.join(dataDir, RAPPORT_FOLDER);
-    fs.ensureDirSync(rapportsDir);
-    const originalRapportsDir = path.join(rapportsDir, DATA_ORIGINAL_FOLDER);
-    let retrievedRapportsCount = 0;
-    const rapportUrlsNotFoundOrError = [];
-    for (const session of sessions) {
-        for (const { item: rapportMetadata } of iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session)) {
-            const rapportDir = path.join(originalRapportsDir, `${rapportMetadata.session ?? UNDEFINED_SESSION}`, rapportMetadata.name);
-            fs.ensureDirSync(rapportDir);
-            if (isOptionEmptyOrHasValue(options["formats"], "html")) {
-                const rapportPath = path.join(rapportDir, `${rapportMetadata.name}.html`);
-                // Check if document should be skipped based on onlyRecent option
-                const shouldSkip = !options["force"] &&
-                    fs.existsSync(rapportPath) &&
-                    (options["only-recent"] === undefined || !isDocumentRecent(rapportMetadata.date, options["only-recent"]));
-                if (shouldSkip) {
-                    if (!options["silent"]) {
-                        console.info(`Already downloaded rapport ${rapportPath}…`);
-                    }
-                    continue;
-                }
-                const rapportBuffer = await downloadDocument(rapportMetadata.url_html.toString());
-                if (!rapportBuffer) {
-                    rapportUrlsNotFoundOrError.push(rapportMetadata.url_html);
-                    continue;
-                }
-                fs.writeFileSync(rapportPath, Buffer.from(rapportBuffer));
-                retrievedRapportsCount++;
-            }
-            if (isOptionEmptyOrHasValue(options["formats"], "pdf")) {
-                const rapportPath = path.join(rapportDir, `${rapportMetadata.name}.pdf`);
-                // Check if document should be skipped based on onlyRecent option
-                const shouldSkip = !options["force"] &&
-                    fs.existsSync(rapportPath) &&
-                    (options["only-recent"] === undefined || !isDocumentRecent(rapportMetadata.date, options["only-recent"]));
-                if (shouldSkip) {
-                    if (!options["silent"]) {
-                        console.info(`Already downloaded rapport ${rapportPath}…`);
-                    }
-                    continue;
-                }
-                const rapportBuffer = await downloadDocument(rapportMetadata.url_pdf.toString());
-                if (!rapportBuffer) {
-                    rapportUrlsNotFoundOrError.push(rapportMetadata.url_pdf);
-                    continue;
-                }
-                fs.writeFileSync(rapportPath, Buffer.from(rapportBuffer));
-                retrievedRapportsCount++;
+async function processDocument(url, destPath, docDate, options) {
+    if (!shouldDownload(destPath, docDate, options)) {
+        if (options.verbose)
+            console.info(`Already downloaded ${destPath}…`);
+        return { success: true, skipped: true, buffer: null };
+    }
+    const arrayBuffer = await downloadDocument(url, options.verbose);
+    if (!arrayBuffer) {
+        return { success: false, skipped: false, buffer: null };
+    }
+    const buffer = Buffer.from(arrayBuffer);
+    await fs.outputFile(destPath, buffer);
+    return { success: true, skipped: false, buffer };
+}
+export async function processTexte(texteMetadata, originalTextesDir, transformedTextesDir, options) {
+    const texteDir = path.join(originalTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name);
+    let exposeDesMotifsContent = null;
+    if (texteMetadata.url_expose_des_motifs) {
+        const exposePath = path.join(texteDir, `${texteMetadata.name}-expose.html`);
+        const res = await processDocument(texteMetadata.url_expose_des_motifs.toString(), exposePath, texteMetadata.date, options);
+        if (res.buffer) {
+            exposeDesMotifsContent = res.buffer;
+        }
+        else if (res.skipped && options.parseDocuments) {
+            if (await fs.pathExists(exposePath)) {
+                exposeDesMotifsContent = await fs.readFile(exposePath);
             }
         }
     }
-    if (options["verbose"]) {
-        console.log(`${retrievedRapportsCount} rapports retrieved`);
-        console.log(`${rapportUrlsNotFoundOrError.length} rapports failed with URLs ${rapportUrlsNotFoundOrError.join(", ")}`);
+    const formats = [
+        { type: "xml", url: texteMetadata.url_xml, isParseTarget: true },
+        { type: "html", url: texteMetadata.url_html, isParseTarget: false },
+        { type: "pdf", url: texteMetadata.url_pdf, isParseTarget: false },
+    ];
+    for (const format of formats) {
+        if (!isOptionEmptyOrHasValue(options.formats, format.type))
+            continue;
+        const destPath = path.join(texteDir, `${texteMetadata.name}.${format.type}`);
+        const result = await processDocument(format.url.toString(), destPath, texteMetadata.date, options);
+        // Specific logic: Parsing (Only applies to XML)
+        if (format.isParseTarget && options.parseDocuments) {
+            await parseDocument(texteMetadata.session, transformedTextesDir, destPath, texteMetadata.name, result.buffer, exposeDesMotifsContent, options);
+        }
     }
 }
-async function downloadExposeDesMotifs(texteDir, texteName, url) {
-    const content = await downloadDocument(url);
-    if (!content) {
-        return null;
+export async function processRapport(rapportMetadata, originalRapportsDir, options) {
+    const rapportDir = path.join(originalRapportsDir, `${rapportMetadata.session ?? UNDEFINED_SESSION}`, rapportMetadata.name);
+    const formats = [
+        { type: "html", url: rapportMetadata.url_html },
+        { type: "pdf", url: rapportMetadata.url_pdf },
+    ];
+    for (const format of formats) {
+        if (!isOptionEmptyOrHasValue(options["formats"], format.type))
+            continue;
+        const destPath = path.join(rapportDir, `${rapportMetadata.name}.${format.type}`);
+        await processDocument(format.url.toString(), destPath, rapportMetadata.date, options);
     }
-    const exposeDesMotifsPath = path.join(texteDir, `${texteName}-expose.html`);
-    fs.writeFileSync(exposeDesMotifsPath, Buffer.from(content));
-    return content;
 }
-async function downloadDocument(documentUrl) {
-    if (!options["silent"]) {
-        console.log(`Downloading document ${documentUrl}…`);
+async function retrieveTextes(dataDir, sessions) {
+    const originalTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER);
+    const transformedTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_TRANSFORMED_FOLDER);
+    if (options["parseDocuments"]) {
+        ensureAndClearDir(transformedTextesDir);
     }
-    try {
-        const response = await fetchWithRetry(documentUrl);
-        if (!response.ok) {
-            if (response.status === 404) {
-                console.warn(`Texte ${documentUrl} not found`);
-            }
-            else {
-                console.error(`An error occurred while retrieving document ${documentUrl}: ${response.status}`);
-            }
-            return null;
+    const dlOptions = {
+        force: options["force"],
+        silent: options["silent"],
+        verbose: options["verbose"],
+        onlyRecent: options["only-recent"],
+        formats: options["formats"],
+        parseDocuments: options["parseDocuments"],
+    };
+    for (const session of sessions) {
+        for (const { item: texteMetadata } of iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session)) {
+            await processTexte(texteMetadata, originalTextesDir, transformedTextesDir, dlOptions);
         }
-        return response.arrayBuffer();
     }
-    catch (error) {
-        console.error(error.message);
-        return null;
+}
+async function retrieveRapports(dataDir, sessions) {
+    const originalRapportsDir = path.join(dataDir, RAPPORT_FOLDER, DATA_ORIGINAL_FOLDER);
+    const dlOptions = {
+        force: options["force"],
+        silent: options["silent"],
+        verbose: options["verbose"],
+        onlyRecent: options["only-recent"],
+        formats: options["formats"],
+    };
+    for (const session of sessions) {
+        for (const { item: rapportMetadata } of iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session)) {
+            await processRapport(rapportMetadata, originalRapportsDir, dlOptions);
+        }
     }
 }
-async function parseDocument(session, transformedTextesDir, textePath, texteName, texteBuffer, exposeDesMotifs = null) {
-    if (!options["silent"]) {
+async function parseDocument(session, transformedTextesDir, textePath, texteName, texteBuffer, exposeDesMotifs = null, options = {}) {
+    if (options.verbose) {
         console.log(`Parsing texte ${textePath}…`);
     }
     let parsedTexte;
@@ -247,19 +179,17 @@ async function parseDocument(session, transformedTextesDir, textePath, texteName
     else {
         parsedTexte = await parseTexteFromFile(textePath);
     }
-    if (!parsedTexte) {
+    if (!parsedTexte)
         return null;
-    }
     if (exposeDesMotifs) {
-        if (!options["silent"]) {
+        if (options.verbose) {
             console.log("Parsing exposé des motifs…");
         }
         const exposeDesMotifsHtml = textDecoder.decode(exposeDesMotifs);
         parsedTexte.exposeDesMotifs = parseExposeDesMotifs(exposeDesMotifsHtml);
     }
     const transformedTexteDir = path.join(transformedTextesDir, `${session ?? UNDEFINED_SESSION}`, texteName);
-    fs.ensureDirSync(transformedTexteDir);
-    fs.writeJSONSync(path.join(transformedTexteDir, `${texteName}.json`), parsedTexte, { spaces: 2 });
+    await fs.outputJSON(path.join(transformedTexteDir, `${texteName}.json`), parsedTexte, { spaces: 2 });
     return parsedTexte;
 }
 async function main() {
@@ -277,9 +207,11 @@ async function main() {
         console.timeEnd("documents processing time");
     }
 }
-main()
-    .then(() => process.exit(0))
-    .catch((error) => {
-    console.log(error);
-    process.exit(1);
-});
+if (process.argv[1].endsWith("retrieve_documents.ts")) {
+    main()
+        .then(() => process.exit(0))
+        .catch((error) => {
+        console.log(error);
+        process.exit(1);
+    });
+}

package/lib/scripts/shared/cli_helpers.d.ts

@@ -64,6 +64,16 @@ export declare const pullOption: {
     name: string;
     type: BooleanConstructor;
 };
+export declare const fetchDocumentsOption: {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const parseDocumentsOption: {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
 export declare const commonOptions: ({
     defaultOption: boolean;
     help: string;

package/lib/scripts/shared/cli_helpers.js

@@ -64,6 +64,16 @@ export const pullOption = {
     name: "pull",
     type: Boolean,
 };
+export const fetchDocumentsOption = {
+    help: "download documents",
+    name: "fetchDocuments",
+    type: Boolean,
+};
+export const parseDocumentsOption = {
+    help: "parse documents",
+    name: "parseDocuments",
+    type: Boolean,
+};
 export const commonOptions = [
     categoriesOption,
     dataDirDefaultOption,
@@ -76,4 +86,6 @@ export const commonOptions = [
     commitOption,
     remoteOption,
     pullOption,
+    fetchDocumentsOption,
+    parseDocumentsOption,
 ];

package/lib/scripts/test_iter_load.js

@@ -1,29 +1,18 @@
-import { iterLoadSenatScrutins } from "../loaders";
+import { iterLoadSenatAmendements, iterLoadSenatDossiersLegislatifs } from "../loaders";
 import commandLineArgs from "command-line-args";
 import { dataDirDefaultOption } from "./shared/cli_helpers";
 const optionsDefinitions = [dataDirDefaultOption];
 const options = commandLineArgs(optionsDefinitions);
-const noValidation = false;
 const session = 2024;
-const s = new Set();
-for (const { item: scrutin } of iterLoadSenatScrutins(options["dataDir"], session, { noValidation: noValidation })) {
-    s.add(scrutin["lecture_libelle"]);
+const sinceCommit = undefined;
+for (const { item: amendement, filePathFromDataset } of iterLoadSenatAmendements(options["dataDir"], session, {
+    log: true,
+    sinceCommit: sinceCommit,
+})) {
+    console.log(amendement["numero"]);
 }
-console.log(s);
-/*
-for (const { item: amendement } of iterLoadSenatAmendements(
-  options["dataDir"],
-  session,
-  { noValidation: noValidation },
-)) {
-  console.log(amendement["numero"])
+for (const { item: dossierLegislatif } of iterLoadSenatDossiersLegislatifs(options["dataDir"], session, {
+    sinceCommit: sinceCommit,
+})) {
+    console.log(dossierLegislatif["numero"]);
 }
-
-for (const { item: dossierLegislatif } of iterLoadSenatDossiersLegislatifs(
-  options["dataDir"],
-  session,
-  { noValidation: noValidation },
-)) {
-  console.log(dossierLegislatif["numero"])
-}
-*/ 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.20.21",
+  "version": "2.20.23",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",
@@ -52,7 +52,6 @@
     "data:retrieve_open_data": "tsx src/scripts/retrieve_open_data.ts --all",
     "data:retrieve_senateurs_photos": "tsx src/scripts/retrieve_senateurs_photos.ts --fetch",
     "data:retrieve_videos": "tsx src/scripts/retrieve_videos.ts",
-    "data:parse_textes_lois": "tsx src/scripts/parse_textes.ts",
     "prepare": "npm run build",
     "prepublishOnly": "npm run build",
     "prettier": "prettier --write 'src/**/*.ts' 'tests/**/*.test.ts'",
@@ -65,17 +64,17 @@
     "cheerio": "^1.1.2",
     "command-line-args": "^6.0.1",
     "dotenv": "^17.2.3",
-    "fast-xml-parser": "^5.3.2",
-    "fs-extra": "^11.3.2",
+    "fast-xml-parser": "^5.3.3",
+    "fs-extra": "^11.3.3",
     "jsdom": "^27.2.0",
-    "kysely": "^0.28.8",
+    "kysely": "^0.28.9",
     "luxon": "^3.7.2",
     "node-stream-zip": "^1.8.2",
     "p-limit": "^7.2.0",
     "pg": "^8.13.1",
     "pg-cursor": "^2.12.1",
     "slug": "^11.0.0",
-    "tsx": "^4.20.6",
+    "tsx": "^4.21.0",
     "windows-1252": "^3.0.4"
   },
   "devDependencies": {
@@ -93,7 +92,6 @@
     "@typescript-eslint/parser": "^8.46.0",
     "cross-env": "^10.1.0",
     "eslint": "^8.57.1",
-    "iconv-lite": "^0.7.0",
     "kysely-codegen": "^0.19.0",
     "prettier": "^3.5.3",
     "tslib": "^2.1.0",