@tricoteuses/senat 2.5.8 → 2.6.1

package/lib/scripts/retrieve_comptes_rendus.js

@@ -0,0 +1,106 @@
+import assert from "assert";
+import commandLineArgs from "command-line-args";
+import fs from "fs-extra";
+import path from "path";
+import { COMPTES_RENDUS_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatDebats } from "../loaders";
+import { parseCompteRenduFromFile } from "../model/compte_rendu";
+import { getSessionsFromStart } from "../types/sessions";
+import { commonOptions } from "./shared/cli_helpers";
+import { ensureAndClearDir } from "./shared/util";
+const optionsDefinitions = [
+    ...commonOptions,
+    {
+        help: "parse and convert comptes-rendus des débats into JSON",
+        name: "parseDebats",
+        type: Boolean,
+    },
+];
+const options = commandLineArgs(optionsDefinitions);
+const SENAT_COMPTE_RENDU_URL_ROOT = "https://www.senat.fr/seances";
+class CompteRenduError extends Error {
+    constructor(message, compteRenduUrl) {
+        super(`An error occurred while retrieving Compte-Rendu ${compteRenduUrl}: ${message}`);
+    }
+}
+async function retrieveComptesRendus(dataDir, sessions) {
+    const comptesRendusRootDir = path.join(dataDir, COMPTES_RENDUS_FOLDER);
+    ensureAndClearDir(comptesRendusRootDir);
+    const originalComptesRendusDir = path.join(comptesRendusRootDir, DATA_ORIGINAL_FOLDER);
+    fs.ensureDirSync(originalComptesRendusDir);
+    const transformedComptesRendusDir = path.join(comptesRendusRootDir, DATA_TRANSFORMED_FOLDER);
+    if (options["parseDebats"]) {
+        fs.ensureDirSync(transformedComptesRendusDir);
+    }
+    for (const session of sessions) {
+        const originalComptesRendusSessionDir = path.join(originalComptesRendusDir, `${session}`);
+        fs.ensureDirSync(originalComptesRendusSessionDir);
+        const transformedComptesRendusSessionDir = path.join(transformedComptesRendusDir, `${session}`);
+        if (options["parseDebats"]) {
+            fs.ensureDirSync(transformedComptesRendusSessionDir);
+        }
+        for (const { item: debat, } of iterLoadSenatDebats(dataDir, session)) {
+            if (!debat.url) {
+                continue;
+            }
+            try {
+                const debatMonoUrl = `${path.parse(debat.url).dir}/s${debat.id}_mono.html`;
+                const compteRenduPath = path.join(originalComptesRendusSessionDir, `${debat.id}.html`);
+                await downloadCompteRendu(debatMonoUrl, compteRenduPath);
+                if (options["parseDebats"]) {
+                    await parseCompteRendu(transformedComptesRendusSessionDir, compteRenduPath, debat);
+                }
+            }
+            catch (error) {
+                console.error(error);
+            }
+        }
+    }
+}
+async function downloadCompteRendu(debatUrl, compteRenduPath) {
+    const compteRenduUrl = `${SENAT_COMPTE_RENDU_URL_ROOT}/${debatUrl}`;
+    if (!options["silent"]) {
+        console.log(`Downloading Compte-Rendu ${compteRenduUrl}…`);
+    }
+    const response = await fetch(compteRenduUrl);
+    if (!response.ok) {
+        if (response.status === 404) {
+            console.warn(`Compte-Rendu ${compteRenduUrl} not found`);
+        }
+        else {
+            throw new CompteRenduError(String(response.status), compteRenduUrl);
+        }
+        return;
+    }
+    const compteRenduContent = await response.arrayBuffer();
+    if (!compteRenduContent) {
+        return;
+    }
+    fs.writeFileSync(compteRenduPath, Buffer.from(compteRenduContent));
+}
+async function parseCompteRendu(transformedComptesRendusSessionDir, compteRenduPath, debat) {
+    if (!options["silent"]) {
+        console.log(`Parsing compte-rendu ${compteRenduPath}…`);
+    }
+    const parsedCompteRendu = await parseCompteRenduFromFile(compteRenduPath, debat);
+    if (!parsedCompteRendu) {
+        return;
+    }
+    const parsedFilePath = path.parse(compteRenduPath);
+    fs.writeJSONSync(path.join(transformedComptesRendusSessionDir, `${parsedFilePath.name}.json`), parsedCompteRendu, { spaces: 2 });
+}
+async function main() {
+    const dataDir = options["dataDir"];
+    assert(dataDir, "Missing argument: data directory");
+    const sessions = getSessionsFromStart(options["fromSession"]);
+    console.time("comptes-rendus processing time");
+    await retrieveComptesRendus(dataDir, sessions);
+    if (!options["silent"]) {
+        console.timeEnd("comptes-rendus processing time");
+    }
+}
+main()
+    .then(() => process.exit(0))
+    .catch((error) => {
+    console.log(error);
+    process.exit(1);
+});

package/lib/scripts/retrieve_documents.js

@@ -2,11 +2,11 @@ import assert from "assert";
 import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 import path from "path";
-import { iterLoadSenatDossiersLegislatifsRapportUrls, iterLoadSenatDossiersLegislatifsTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, } from "../loaders";
-import { parseExposeDesMotifs, parseTexte, parseTexteFromFile, } from "../model/texte";
+import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatDossiersLegislatifsRapportUrls, iterLoadSenatDossiersLegislatifsTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, } from "../loaders";
+import { parseExposeDesMotifs, parseTexte, parseTexteFromFile } from "../model/texte";
 import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions";
 import { commonOptions } from "./shared/cli_helpers";
-import { ensureAndClearDir, fetchWithRetry, isOptionEmptyOrHasValue, } from "./shared/util";
+import { ensureAndClearDir, fetchWithRetry, isOptionEmptyOrHasValue } from "./shared/util";
 const optionsDefinitions = [
     ...commonOptions,
     {
@@ -35,28 +35,6 @@ const optionsDefinitions = [
 ];
 const options = commandLineArgs(optionsDefinitions);
 const textDecoder = new TextDecoder("utf8");
-async function retrieveDocument(documentUrl) {
-    if (!options["silent"]) {
-        console.log(`Retrieving document ${documentUrl}…`);
-    }
-    try {
-        const response = await fetchWithRetry(documentUrl);
-        if (!response.ok) {
-            if (response.status === 404) {
-                console.warn(`Texte ${documentUrl} not found`);
-            }
-            else {
-                console.error(`An error occurred while retrieving texte ${documentUrl}: ${response.status}`);
-            }
-            return null;
-        }
-        return response.arrayBuffer();
-    }
-    catch (error) {
-        console.error(error.message);
-        return null;
-    }
-}
 async function retrieveTextes(dataDir, sessions) {
     const textesDir = path.join(dataDir, TEXTE_FOLDER);
     fs.ensureDirSync(textesDir);
@@ -74,26 +52,18 @@ async function retrieveTextes(dataDir, sessions) {
             fs.ensureDirSync(texteDir);
             let exposeDesMotifsContent = null;
             if (texteMetadata.url_expose_des_motifs) {
-                if (!options["silent"]) {
-                    console.log("Retrieving exposé des motifs…");
-                }
-                const exposeDesMotifsPath = path.join(texteDir, `${texteMetadata.name}-expose.html`);
-                exposeDesMotifsContent = await retrieveDocument(texteMetadata.url_expose_des_motifs.toString());
-                if (!exposeDesMotifsContent) {
-                    continue;
-                }
-                fs.writeFileSync(exposeDesMotifsPath, Buffer.from(exposeDesMotifsContent));
+                exposeDesMotifsContent = await downloadExposeDesMotifs(texteDir, texteMetadata.name, String(texteMetadata.url_expose_des_motifs));
             }
             if (isOptionEmptyOrHasValue(options["formats"], "xml")) {
                 const textePath = path.join(texteDir, `${texteMetadata.name}.xml`);
                 let texteBuffer = null;
                 if (!options["force"] && fs.existsSync(textePath)) {
                     if (!options["silent"]) {
-                        console.info(`Already retrieved texte ${textePath}…`);
+                        console.info(`Already downloaded texte ${textePath}…`);
                     }
                 }
                 else {
-                    texteBuffer = await retrieveDocument(texteMetadata.url_xml.toString());
+                    texteBuffer = await downloadDocument(texteMetadata.url_xml.toString());
                     if (!texteBuffer) {
                         texteUrlsNotFoundOrError.push(texteMetadata.url_xml);
                         continue;
@@ -102,43 +72,21 @@ async function retrieveTextes(dataDir, sessions) {
                     retrievedTextesCount++;
                 }
                 if (options["parseDocuments"]) {
-                    if (!options["silent"]) {
-                        console.log(`Parsing texte ${texteMetadata.name}.xml…`);
-                    }
-                    let parsedTexte = null;
-                    if (texteBuffer) {
-                        const texteXml = textDecoder.decode(texteBuffer);
-                        parsedTexte = parseTexte(texteXml);
-                    }
-                    else {
-                        parsedTexte = await parseTexteFromFile(textePath);
-                    }
+                    const parsedTexte = await parseDocument(texteMetadata.session, transformedTextesDir, textePath, texteMetadata.name, texteBuffer, exposeDesMotifsContent);
                     if (!parsedTexte) {
                         texteUrlsParseError.push(texteMetadata.url_xml);
-                        continue;
                     }
-                    if (exposeDesMotifsContent) {
-                        if (!options["silent"]) {
-                            console.log("Parsing exposé des motifs…");
-                        }
-                        const exposeDesMotifsHtml = textDecoder.decode(exposeDesMotifsContent);
-                        parsedTexte.exposeDesMotifs =
-                            parseExposeDesMotifs(exposeDesMotifsHtml);
-                    }
-                    const transformedTexteDir = path.join(transformedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name);
-                    fs.ensureDirSync(transformedTexteDir);
-                    fs.writeJSONSync(path.join(transformedTexteDir, `${texteMetadata.name}.json`), parsedTexte, { spaces: 2 });
                 }
             }
             if (isOptionEmptyOrHasValue(options["formats"], "html")) {
                 const textePath = path.join(texteDir, `${texteMetadata.name}.html`);
                 if (!options["force"] && fs.existsSync(textePath)) {
                     if (!options["silent"]) {
-                        console.info(`Already retrieved texte ${textePath}…`);
+                        console.info(`Already downloaded texte ${textePath}…`);
                     }
                 }
                 else {
-                    const texteBuffer = await retrieveDocument(texteMetadata.url_html.toString());
+                    const texteBuffer = await downloadDocument(texteMetadata.url_html.toString());
                     if (!texteBuffer) {
                         texteUrlsNotFoundOrError.push(texteMetadata.url_html);
                         continue;
@@ -151,11 +99,11 @@ async function retrieveTextes(dataDir, sessions) {
                 const textePath = path.join(texteDir, `${texteMetadata.name}.pdf`);
                 if (!options["force"] && fs.existsSync(textePath)) {
                     if (!options["silent"]) {
-                        console.info(`Already retrieved texte ${textePath}…`);
+                        console.info(`Already downloaded texte ${textePath}…`);
                     }
                 }
                 else {
-                    const texteBuffer = await retrieveDocument(texteMetadata.url_pdf.toString());
+                    const texteBuffer = await downloadDocument(texteMetadata.url_pdf.toString());
                     if (!texteBuffer) {
                         texteUrlsNotFoundOrError.push(texteMetadata.url_pdf);
                         continue;
@@ -188,11 +136,11 @@ async function retrieveRapports(dataDir, sessions) {
                 const rapportPath = path.join(rapportDir, `${rapportMetadata.name}.html`);
                 if (!options["force"] && fs.existsSync(rapportPath)) {
                     if (!options["silent"]) {
-                        console.info(`Already retrieved rapport ${rapportPath}…`);
+                        console.info(`Already downloaded rapport ${rapportPath}…`);
                     }
                     continue;
                 }
-                const rapportBuffer = await retrieveDocument(rapportMetadata.url_html.toString());
+                const rapportBuffer = await downloadDocument(rapportMetadata.url_html.toString());
                 if (!rapportBuffer) {
                     rapportUrlsNotFoundOrError.push(rapportMetadata.url_html);
                     continue;
@@ -204,11 +152,11 @@ async function retrieveRapports(dataDir, sessions) {
                 const rapportPath = path.join(rapportDir, `${rapportMetadata.name}.pdf`);
                 if (!options["force"] && fs.existsSync(rapportPath)) {
                     if (!options["silent"]) {
-                        console.info(`Already retrieved rapport ${rapportPath}…`);
+                        console.info(`Already downloaded rapport ${rapportPath}…`);
                     }
                     continue;
                 }
-                const rapportBuffer = await retrieveDocument(rapportMetadata.url_pdf.toString());
+                const rapportBuffer = await downloadDocument(rapportMetadata.url_pdf.toString());
                 if (!rapportBuffer) {
                     rapportUrlsNotFoundOrError.push(rapportMetadata.url_pdf);
                     continue;
@@ -223,6 +171,65 @@ async function retrieveRapports(dataDir, sessions) {
         console.log(`${rapportUrlsNotFoundOrError.length} rapports failed with URLs ${rapportUrlsNotFoundOrError.join(", ")}`);
     }
 }
+async function downloadExposeDesMotifs(texteDir, texteName, url) {
+    const content = await downloadDocument(url);
+    if (!content) {
+        return null;
+    }
+    const exposeDesMotifsPath = path.join(texteDir, `${texteName}-expose.html`);
+    fs.writeFileSync(exposeDesMotifsPath, Buffer.from(content));
+    return content;
+}
+async function downloadDocument(documentUrl) {
+    if (!options["silent"]) {
+        console.log(`Downloading document ${documentUrl}…`);
+    }
+    try {
+        const response = await fetchWithRetry(documentUrl);
+        if (!response.ok) {
+            if (response.status === 404) {
+                console.warn(`Texte ${documentUrl} not found`);
+            }
+            else {
+                console.error(`An error occurred while retrieving document ${documentUrl}: ${response.status}`);
+            }
+            return null;
+        }
+        return response.arrayBuffer();
+    }
+    catch (error) {
+        console.error(error.message);
+        return null;
+    }
+}
+async function parseDocument(session, transformedTextesDir, textePath, texteName, texteBuffer, exposeDesMotifs = null) {
+    if (!options["silent"]) {
+        console.log(`Parsing texte ${textePath}…`);
+    }
+    let parsedTexte;
+    if (texteBuffer) {
+        const texteXml = textDecoder.decode(texteBuffer);
+        parsedTexte = parseTexte(texteXml);
+    }
+    else {
+        parsedTexte = await parseTexteFromFile(textePath);
+    }
+    if (!parsedTexte) {
+        return null;
+    }
+    if (exposeDesMotifs) {
+        if (!options["silent"]) {
+            console.log("Parsing exposé des motifs…");
+        }
+        const exposeDesMotifsHtml = textDecoder.decode(exposeDesMotifs);
+        parsedTexte.exposeDesMotifs =
+            parseExposeDesMotifs(exposeDesMotifsHtml);
+    }
+    const transformedTexteDir = path.join(transformedTextesDir, `${session ?? UNDEFINED_SESSION}`, texteName);
+    fs.ensureDirSync(transformedTexteDir);
+    fs.writeJSONSync(path.join(transformedTexteDir, `${texteName}.json`), parsedTexte, { spaces: 2 });
+    return parsedTexte;
+}
 async function main() {
     const dataDir = options["dataDir"];
     assert(dataDir, "Missing argument: data directory");

package/lib/types/compte_rendu.d.ts

@@ -0,0 +1,11 @@
+export interface CompteRendu {
+    sections: Section[];
+}
+export interface Section {
+    id: string | null;
+    interventions: Intervention[];
+}
+export interface Intervention {
+    id: string | null;
+    texteHtml: string | null;
+}

package/lib/types/compte_rendu.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.5.8",
+  "version": "2.6.1",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",
@@ -46,6 +46,7 @@
     "data:download": "bash -c 'npm run data:retrieve_open_data -- $@ && npm run data:convert_data -- $@' bash",
     "data:generate_schemas": "tsx src/scripts/retrieve_open_data.ts --schema",
     "data:retrieve_agenda": "TZ='Etc/UTC' tsx src/scripts/retrieve_agenda.ts",
+    "data:retrieve_comptes_rendus": "tsx src/scripts/retrieve_comptes_rendus.ts",
     "data:retrieve_documents": "tsx src/scripts/retrieve_documents.ts",
     "data:retrieve_open_data": "tsx src/scripts/retrieve_open_data.ts --all",
     "data:retrieve_senateurs_photos": "tsx src/scripts/retrieve_senateurs_photos.ts --fetch",
@@ -67,7 +68,6 @@
     "node-stream-zip": "^1.8.2",
     "pg": "^8.13.1",
     "pg-cursor": "^2.12.1",
-    "pg-promise": "^10.9.2",
     "slug": "^4.0.2",
     "tsx": "^4.19.4",
     "windows-1252": "^1.0.0"