@tricoteuses/senat 2.16.3 → 2.16.4

package/lib/raw_types/db.js

@@ -0,0 +1,5 @@
+/**
+ * This file was generated by kysely-codegen.
+ * Please do not edit it manually.
+ */
+export {};

package/lib/scripts/retrieve_comptes_rendus.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Needs to be run after retrieve_agenda.ts !
+ * - downloads the ZIP of comptes-rendus des débats (CRI) from data.senat.fr
+ * - extracts XML files, distributes them by session/year
+ */
+export declare function retrieveCriXmlDump(dataDir: string, options?: Record<string, any>): Promise<void>;

package/lib/scripts/retrieve_comptes_rendus.js

@@ -0,0 +1,274 @@
+/**
+ * Needs to be run after retrieve_agenda.ts !
+ * - downloads the ZIP of comptes-rendus des débats (CRI) from data.senat.fr
+ * - extracts XML files, distributes them by session/year
+ */
+import assert from "assert";
+import commandLineArgs from "command-line-args";
+import fs from "fs-extra";
+import path from "path";
+import StreamZip from "node-stream-zip";
+import * as cheerio from "cheerio";
+import { AGENDA_FOLDER, COMPTES_RENDUS_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, } from "../loaders";
+import { commonOptions } from "./shared/cli_helpers";
+import { deriveTitreObjetFromSommaire, parseCompteRenduSlotFromFile, parseYYYYMMDD, sessionStartYearFromDate } from "../model/compte_rendu";
+import { makeGroupUid } from "../utils/reunion_grouping";
+import { getSessionsFromStart } from "../types/sessions";
+import { ensureAndClearDir, fetchWithRetry } from "./shared/util";
+import { computeIntervalsBySlot } from "../utils/cr_spliting";
+const optionsDefinitions = [
+    ...commonOptions,
+    {
+        help: "parse and convert comptes-rendus des débats into JSON",
+        name: "parseDebats",
+        type: Boolean,
+    }
+];
+const options = commandLineArgs(optionsDefinitions);
+const CRI_ZIP_URL = "https://data.senat.fr/data/debats/cri.zip";
+const SLOT_ORDER = ["MATIN", "APRES-MIDI", "SOIR"];
+class CompteRenduError extends Error {
+    constructor(message, url) {
+        super(`An error occurred while retrieving ${url}: ${message}`);
+    }
+}
+function pickFirstSlotOfDay(slots) {
+    for (const s of SLOT_ORDER)
+        if (slots.includes(s))
+            return s;
+    return null;
+}
+function loadAgendaSPSlotsForDate(dataDir, yyyymmdd, session) {
+    const dirPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, session.toString());
+    if (!fs.existsSync(dirPath)) {
+        console.warn(`[AGENDA] Directory not found for session ${session} → ${dirPath}`);
+        return null;
+    }
+    const pattern = new RegExp(`^RUSN${yyyymmdd}IDS-(MATIN|APRES-MIDI|SOIR)\\.json$`);
+    const ALLOWED_SLOTS = new Set(["MATIN", "APRES-MIDI", "SOIR"]);
+    try {
+        const files = fs.readdirSync(dirPath);
+        const matched = files.filter((f) => pattern.test(f));
+        if (matched.length === 0) {
+            return null;
+        }
+        const found = new Set();
+        for (const name of matched) {
+            const m = name.match(pattern);
+            const raw = (m?.[1] ?? "");
+            if (ALLOWED_SLOTS.has(raw))
+                found.add(raw);
+        }
+        const slots = Array.from(found);
+        if (slots.length === 0) {
+            return null;
+        }
+        return { filePath: dirPath, slots };
+    }
+    catch {
+        return null;
+    }
+}
+async function downloadCriZip(zipPath) {
+    if (!options["silent"])
+        console.log(`Downloading CRI zip ${CRI_ZIP_URL}…`);
+    const response = await fetchWithRetry(CRI_ZIP_URL);
+    if (!response.ok) {
+        if (response.status === 404) {
+            console.warn(`CRI zip ${CRI_ZIP_URL} not found`);
+            return;
+        }
+        throw new CompteRenduError(String(response.status), CRI_ZIP_URL);
+    }
+    const buf = Buffer.from(await response.arrayBuffer());
+    await fs.writeFile(zipPath, buf);
+    if (!options["silent"]) {
+        const mb = (buf.length / (1024 * 1024)).toFixed(1);
+        console.log(`[CRI] Downloaded ${mb} MB → ${zipPath}`);
+    }
+}
+async function extractAndDistributeXmlBySession(zipPath, originalRoot) {
+    const zip = new StreamZip.async({ file: zipPath });
+    const entries = await zip.entries();
+    let count = 0;
+    for (const entryName of Object.keys(entries)) {
+        if (!entryName.toLowerCase().endsWith(".xml"))
+            continue;
+        // ex: d20231005.xml
+        const base = path.basename(entryName);
+        const m = base.match(/^d(\d{8})\.xml$/i);
+        if (!m)
+            continue;
+        const yyyymmdd = m[1];
+        const dt = parseYYYYMMDD(yyyymmdd);
+        if (!dt)
+            continue;
+        const session = sessionStartYearFromDate(dt);
+        const destDir = path.join(originalRoot, String(session));
+        await fs.ensureDir(destDir);
+        const outPath = path.join(destDir, base);
+        await zip.extract(entryName, outPath);
+        count++;
+    }
+    await zip.close();
+    return count;
+}
+export async function retrieveCriXmlDump(dataDir, options = {}) {
+    const root = path.join(dataDir, COMPTES_RENDUS_FOLDER);
+    ensureAndClearDir(root);
+    const originalRoot = path.join(root, DATA_ORIGINAL_FOLDER);
+    fs.ensureDirSync(originalRoot);
+    const transformedRoot = path.join(root, DATA_TRANSFORMED_FOLDER);
+    if (options["parseDebats"])
+        fs.ensureDirSync(transformedRoot);
+    const sessions = getSessionsFromStart(options["fromSession"]);
+    // 1) Download ZIP global + distribut by session
+    const zipPath = path.join(dataDir, "cri.zip");
+    console.log("[CRI] Downloading global CRI zip…");
+    await downloadCriZip(zipPath);
+    console.log("[CRI] Extracting + distributing XMLs by session…");
+    for (const session of sessions) {
+        const dir = path.join(originalRoot, String(session));
+        if (await fs.pathExists(dir)) {
+            for (const f of await fs.readdir(dir))
+                if (/\.xml$/i.test(f))
+                    await fs.remove(path.join(dir, f));
+        }
+    }
+    const n = await extractAndDistributeXmlBySession(zipPath, originalRoot);
+    if (n === 0) {
+        console.warn("[CRI] No XML extracted. Archive empty or layout changed?");
+    }
+    else {
+        console.log(`[CRI] Distributed ${n} XML file(s) into session folders.`);
+    }
+    if (!options["parseDebats"]) {
+        console.log("[CRI] parseDebats not requested → done.");
+        return;
+    }
+    for (const session of sessions) {
+        const originalSessionDir = path.join(originalRoot, String(session));
+        if (!(await fs.pathExists(originalSessionDir))) {
+            continue;
+        }
+        const xmlFiles = (await fs.readdir(originalSessionDir))
+            .filter((f) => /^d\d{8}\.xml$/i.test(f))
+            .sort();
+        const transformedSessionDir = path.join(transformedRoot, String(session));
+        if (options["parseDebats"])
+            await fs.ensureDir(transformedSessionDir);
+        for (const f of xmlFiles) {
+            const yyyymmdd = f.slice(1, 9);
+            const xmlPath = path.join(originalSessionDir, f);
+            // 1) Deduce slot(s) from agenda if it exsits
+            const agendaInfo = loadAgendaSPSlotsForDate(dataDir, yyyymmdd, session);
+            const firstSlotOfDay = pickFirstSlotOfDay(agendaInfo?.slots ?? []);
+            // 2) Detect slots from CRI content
+            let slotsInCri = [];
+            try {
+                const raw = await fs.readFile(xmlPath, "utf8");
+                const $ = cheerio.load(raw, { xml: false });
+                const order = $("body *").toArray();
+                const idx = new Map(order.map((el, i) => [el, i]));
+                const intervals = computeIntervalsBySlot($, idx, firstSlotOfDay ?? undefined);
+                const uniq = new Set();
+                for (const iv of intervals)
+                    if (iv.slot && iv.slot !== "UNKNOWN")
+                        uniq.add(iv.slot);
+                slotsInCri = Array.from(uniq);
+            }
+            catch (e) {
+                console.warn(`[CRI] [${session}] Cannot read/parse ${f}:`, e);
+                continue;
+            }
+            if (slotsInCri.length === 0) {
+                slotsInCri = [firstSlotOfDay ?? "MATIN"];
+            }
+            // 3) Parse & write each slot
+            for (const slot of slotsInCri) {
+                const outName = `CRSSN${yyyymmdd}-${slot}.json`;
+                const cr = await parseCompteRenduSlotFromFile(xmlPath, slot, firstSlotOfDay ?? slot);
+                if (!cr) {
+                    console.warn(`[CRI] [${session}] Empty or no points for ${yyyymmdd} (${slot}) → skip`);
+                    continue;
+                }
+                const outDir = transformedSessionDir;
+                await fs.ensureDir(outDir);
+                const outPath = path.join(outDir, outName);
+                await fs.writeJSON(outPath, cr, { spaces: 2 });
+                try {
+                    await linkCriSlotIntoAgendaGrouped(dataDir, yyyymmdd, slot, cr.uid, cr, session);
+                }
+                catch (e) {
+                    console.warn(`[AGENDA] [${session}] Could not link CR into grouped for ${yyyymmdd} ${slot}:`, e);
+                }
+            }
+        }
+    }
+}
+async function main() {
+    const dataDir = options["dataDir"];
+    assert(dataDir, "Missing argument: data directory");
+    console.time("CRI processing time");
+    await retrieveCriXmlDump(dataDir, options);
+    if (!options["silent"]) {
+        console.timeEnd("CRI processing time");
+    }
+}
+main()
+    .then(() => process.exit(0))
+    .catch((error) => {
+    console.error(error);
+    process.exit(1);
+});
+async function linkCriSlotIntoAgendaGrouped(dataDir, yyyymmdd, slot, crUid, cr, session) {
+    const groupedDir = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, session.toString());
+    fs.ensureDirSync(groupedDir);
+    const groupedPath = path.join(groupedDir, 'RUSN' + yyyymmdd + 'IDS-' + slot + '.json');
+    let groups = [];
+    if (fs.existsSync(groupedPath)) {
+        try {
+            groups = JSON.parse(fs.readFileSync(groupedPath, "utf8"));
+            if (!Array.isArray(groups))
+                groups = [];
+        }
+        catch (e) {
+            console.warn(`[AGENDA] unreadable grouped JSON → ${groupedPath} (${e}) → recreating`);
+            groups = [];
+        }
+    }
+    // find existing group with same slot
+    const sameSlot = groups.filter(g => g?.slot === slot);
+    let target = null;
+    if (sameSlot.length > 1) {
+        console.warn(`[AGENDA] multiple groups for ${yyyymmdd} ${slot} in ${groupedPath} → linking the first`);
+    }
+    target = sameSlot[0] ?? null;
+    const dateISO = `${yyyymmdd.slice(0, 4)}-${yyyymmdd.slice(4, 6)}-${yyyymmdd.slice(6, 8)}`;
+    const sommaire = cr?.metadonnees?.sommaire;
+    const { titre: dTitre, objet: dObjet } = deriveTitreObjetFromSommaire(sommaire, slot);
+    if (!target) {
+        const newGroup = {
+            uid: makeGroupUid(dateISO, slot),
+            chambre: "SN",
+            date: dateISO,
+            slot,
+            type: "Séance publique",
+            startTime: null,
+            endTime: null,
+            captationVideo: false,
+            titre: dTitre,
+            objet: dObjet || "",
+            events: [],
+            compteRenduRefUid: crUid,
+        };
+        groups.push(newGroup);
+    }
+    else {
+        target.compteRenduRefUid = crUid;
+    }
+    await fs.writeJSON(groupedPath, groups, { spaces: 2 });
+    if (!options["silent"]) {
+        console.log(`[AGENDA] Linked CR ${crUid} → ${path.basename(groupedPath)} [${slot}]`);
+    }
+}

package/lib/scripts/retrieve_open_data.js

@@ -1,13 +1,11 @@
 import assert from "assert";
-import { execSync } from "child_process";
+import { execSync, spawn } from "child_process";
 import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
-// import fetch from "node-fetch"
 import path from "path";
 import StreamZip from "node-stream-zip";
-import readline from "readline";
 import windows1252 from "windows-1252";
-import { pipeline } from "stream";
+import { pipeline, Transform } from "stream";
 import { promisify } from "util";
 import config from "../config";
 import { getChosenDatasets, getEnabledDatasets } from "../datasets";
@@ -69,60 +67,87 @@ async function downloadFile(url, dest) {
 }
 /**
  * Copy a dataset database to the main Senat database (overwriting its contents).
+ * Optimized to combine encoding repair and schema transformation in a single pass.
  */
 async function copyToSenat(dataset, dataDir, options) {
     if (!options["silent"]) {
         console.log(`Copying ${dataset.database} to Senat database...`);
     }
     const sqlFilePath = path.join(dataDir, `${dataset.database}.sql`);
-    const schemaDumpFile = path.join(dataDir, `${dataset.database}_schema_dump.sql`);
-    // Write the header and then stream the rest of the SQL file
-    const schemaSqlWriter = fs.createWriteStream(schemaDumpFile, { encoding: "utf8" });
-    // Add CREATE SCHEMA statement at the top
-    schemaSqlWriter.write(`CREATE SCHEMA IF NOT EXISTS ${dataset.database};\n`);
-    const lineReader = readline.createInterface({
-        input: fs.createReadStream(sqlFilePath, { encoding: "utf8" }),
-        crlfDelay: Infinity,
-    });
-    for await (const line of lineReader) {
-        let newLine = line;
-        // Replace 'public' schema outside single-quoted strings
-        function replacePublicOutsideStrings(line, schema) {
-            const parts = line.split(/(')/);
-            let inString = false;
-            for (let i = 0; i < parts.length; i++) {
-                if (parts[i] === "'") {
-                    inString = !inString;
-                }
-                else if (!inString) {
-                    // Only replace outside of strings, including before comma
-                    parts[i] = parts[i].replace(/\bpublic\b(?=(\s*\.|\s*[,;]|\s|$))/g, schema);
-                }
+    // Helper function to replace 'public' schema outside single-quoted strings
+    function replacePublicOutsideStrings(line, schema) {
+        const parts = line.split(/(')/);
+        let inString = false;
+        for (let i = 0; i < parts.length; i++) {
+            if (parts[i] === "'") {
+                inString = !inString;
+            }
+            else if (!inString) {
+                parts[i] = parts[i].replace(/\bpublic\b(?=(\s*\.|\s*[,;]|\s|$))/g, schema);
             }
-            return parts.join('');
         }
-        newLine = replacePublicOutsideStrings(line, dataset.database);
-        // Replace SET client_encoding to UTF8
-        newLine = newLine.replace(/SET client_encoding = 'LATIN1';/i, "SET client_encoding = 'UTF8';");
-        schemaSqlWriter.write(newLine + "\n");
+        return parts.join('');
     }
-    schemaSqlWriter.end();
-    await new Promise((resolve, reject) => {
-        schemaSqlWriter.on("finish", () => {
-            try {
-                execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -d senat -f ${schemaDumpFile}`, {
-                    env: process.env,
-                    encoding: "utf-8",
-                    stdio: ["ignore", "ignore", "pipe"],
-                });
+    // Spawn psql process
+    const psqlArgs = options["sudo"]
+        ? ["-u", options["sudo"], "psql", "--quiet", "-d", "senat"]
+        : ["--quiet", "-d", "senat"];
+    const psql = spawn(options["sudo"] ? "sudo" : "psql", psqlArgs, {
+        stdio: ["pipe", "ignore", "pipe"],
+        env: process.env,
+    });
+    psql.stdin.write(`CREATE SCHEMA IF NOT EXISTS ${dataset.database};\n`);
+    let buffer = '';
+    const combinedTransform = new Transform({
+        transform(chunk, encoding, callback) {
+            // Encoding repair if needed (decode from latin1 and fix Windows-1252 characters)
+            let data = dataset.repairEncoding
+                ? chunk.toString('latin1').replace(badWindows1252CharacterRegex, (match) => windows1252.decode(match, { mode: "fatal" }))
+                : chunk.toString();
+            buffer += data;
+            const lines = buffer.split('\n');
+            buffer = lines.pop() || '';
+            let processedData = '';
+            for (const line of lines) {
+                let newLine = replacePublicOutsideStrings(line, dataset.database);
+                newLine = newLine.replace(/SET client_encoding = 'LATIN1';/i, "SET client_encoding = 'UTF8';");
+                processedData += newLine + '\n';
+            }
+            callback(null, processedData);
+        },
+        flush(callback) {
+            // Process any remaining data in buffer
+            if (buffer) {
+                let newLine = replacePublicOutsideStrings(buffer, dataset.database);
+                newLine = newLine.replace(/SET client_encoding = 'LATIN1';/i, "SET client_encoding = 'UTF8';");
+                callback(null, newLine);
+            }
+            else {
+                callback();
+            }
+        }
+    });
+    let stderrData = '';
+    psql.stderr.on('data', (data) => {
+        stderrData += data.toString();
+    });
+    const pipelinePromise = streamPipeline(fs.createReadStream(sqlFilePath, {
+        encoding: dataset.repairEncoding ? undefined : "utf8",
+        highWaterMark: 4 * 1024 * 1024
+    }), combinedTransform, psql.stdin);
+    await pipelinePromise;
+    return new Promise((resolve, reject) => {
+        psql.on("close", (code) => {
+            if (code === 0) {
+                resolve();
             }
-            finally {
-                try { }
-                catch { }
+            else {
+                if (!options["silent"] && stderrData) {
+                    console.error(`psql stderr: ${stderrData}`);
+                }
+                reject(new Error(`psql exited with code ${code}`));
             }
-            resolve();
         });
-        schemaSqlWriter.on("error", reject);
     });
 }
 async function retrieveDataset(dataDir, dataset) {
@@ -176,31 +201,9 @@ async function retrieveDataset(dataDir, dataset) {
             dataset.repairZip(dataset, dataDir);
         }
     }
-    if ((options["all"] || options["repairEncoding"]) && dataset.repairEncoding) {
-        if (!options["silent"]) {
-            console.log(`Repairing Windows CP1252 encoding in ${dataset.title}: ${sqlFilename}…`);
-        }
-        const repairedSqlFilePath = sqlFilePath + ".repaired";
-        const repairedSqlWriter = fs.createWriteStream(repairedSqlFilePath, {
-            encoding: "utf8",
-        });
-        // Read the file as latin1 (ISO-8859-1/CP1252) and write as UTF-8
-        const lineReader = readline.createInterface({
-            input: fs.createReadStream(sqlFilePath, { encoding: "latin1" }),
-            crlfDelay: Infinity,
-        });
-        for await (const line of lineReader) {
-            // Optionally repair Windows-1252 control characters
-            let repairedLine = line.replace(badWindows1252CharacterRegex, (match) => windows1252.decode(match, { mode: "fatal" }));
-            repairedSqlWriter.write(repairedLine + "\n");
-        }
-        repairedSqlWriter.end();
-        await fs.move(repairedSqlFilePath, sqlFilePath, { overwrite: true });
-    }
+    // Encoding repair is now handled in copyToSenat for better performance (single-pass processing)
+    // The separate encoding repair step has been removed
     if (options["all"] || options["import"] || options["schema"]) {
-        if (!options["silent"]) {
-            console.log(`Importing ${dataset.title}: ${sqlFilename}…`);
-        }
         await copyToSenat(dataset, dataDir, options);
         // Create indexes programmatically after import
         if (dataset.indexes) {
@@ -270,7 +273,7 @@ async function retrieveOpenData() {
         process.env["PGUSER"] &&
         process.env["PGPASSWORD"], "Missing database configuration: environment variables PGHOST, PGPORT, PGUSER and PGPASSWORD or TRICOTEUSES_SENAT_DB_* in .env file");
     console.time("data extraction time");
-    execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -c "DROP DATABASE IF EXISTS senat"`, {
+    execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -c "DROP DATABASE IF EXISTS senat;"`, {
         cwd: dataDir,
         env: process.env,
         encoding: "utf-8",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.16.3",
+  "version": "2.16.4",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",