@tricoteuses/senat 2.22.11 → 2.22.13

package/lib/scripts/retrieve_cr_seance.js

@@ -0,0 +1,347 @@
+/**
+ * Needs to be ran after retrieve_agenda.ts script !
+ * - downloads the ZIP of comptes-rendus des débats (CRI) from data.senat.fr
+ * - extracts XML files, distributes them by session/year
+ */
+import assert from "assert";
+import commandLineArgs from "command-line-args";
+import fs, { ensureDirSync } from "fs-extra";
+import path from "path";
+import StreamZip from "node-stream-zip";
+import * as cheerio from "cheerio";
+import { AGENDA_FOLDER, COMPTES_RENDUS_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
+import { commonOptions } from "./shared/cli_helpers";
+import { parseCompteRenduIntervalFromFile, sessionStartYearFromDate } from "../model/seance";
+import { extractSommaireBlocks, makeReunionUid } from "../utils/reunion_parsing";
+import { getSessionsFromStart } from "../types/sessions";
+import { ensureAndClearDir, fetchWithRetry } from "./shared/util";
+import { isNoiseBlock, scoreSommaireBlockForEvent } from "../utils/scoring";
+import { parseYYYYMMDD } from "../utils/date";
+const optionsDefinitions = [
+    ...commonOptions,
+    {
+        help: "parse and convert comptes-rendus des débats into JSON",
+        name: "parseDebats",
+        type: Boolean,
+    },
+];
+const options = commandLineArgs(optionsDefinitions);
+const CRI_ZIP_URL = "https://data.senat.fr/data/debats/cri.zip";
+class CompteRenduError extends Error {
+    constructor(message, url) {
+        super(`An error occurred while retrieving ${url}: ${message}`);
+    }
+}
+async function downloadCriZip(zipPath) {
+    if (!options["silent"])
+        console.log(`Downloading CRI zip ${CRI_ZIP_URL}…`);
+    const response = await fetchWithRetry(CRI_ZIP_URL);
+    if (!response.ok) {
+        if (response.status === 404) {
+            console.warn(`CRI zip ${CRI_ZIP_URL} not found`);
+            return;
+        }
+        throw new CompteRenduError(String(response.status), CRI_ZIP_URL);
+    }
+    const buf = Buffer.from(await response.arrayBuffer());
+    await fs.writeFile(zipPath, buf);
+    if (!options["silent"]) {
+        const mb = (buf.length / (1024 * 1024)).toFixed(1);
+        console.log(`[CRI] Downloaded ${mb} MB → ${zipPath}`);
+    }
+}
+async function extractAndDistributeXmlBySession(zipPath, originalRoot) {
+    const zip = new StreamZip.async({ file: zipPath });
+    const entries = await zip.entries();
+    let count = 0;
+    for (const entryName of Object.keys(entries)) {
+        if (!entryName.toLowerCase().endsWith(".xml"))
+            continue;
+        // ex: d20231005.xml
+        const base = path.basename(entryName);
+        const m = base.match(/^d(\d{8})\.xml$/i);
+        if (!m)
+            continue;
+        const yyyymmdd = m[1];
+        const dt = parseYYYYMMDD(yyyymmdd);
+        if (!dt)
+            continue;
+        const session = sessionStartYearFromDate(dt);
+        const destDir = path.join(originalRoot, String(session));
+        await fs.ensureDir(destDir);
+        const outPath = path.join(destDir, base);
+        await zip.extract(entryName, outPath);
+        count++;
+    }
+    await zip.close();
+    return count;
+}
+export async function retrieveCriXmlDump(dataDir, options = {}) {
+    const root = path.join(dataDir, COMPTES_RENDUS_FOLDER);
+    ensureDirSync(root);
+    const originalRoot = path.join(root, DATA_ORIGINAL_FOLDER);
+    if (!options["keepDir"]) {
+        ensureAndClearDir(originalRoot);
+    }
+    else {
+        fs.ensureDirSync(originalRoot);
+    }
+    const transformedRoot = path.join(root, DATA_TRANSFORMED_FOLDER);
+    if (!options["keepDir"]) {
+        ensureAndClearDir(transformedRoot);
+    }
+    else {
+        fs.ensureDirSync(transformedRoot);
+    }
+    const sessions = getSessionsFromStart(options["fromSession"]);
+    // 1) Download ZIP global + distribut by session
+    const zipPath = path.join(dataDir, "cri.zip");
+    console.log("[CRI] Downloading global CRI zip…");
+    await downloadCriZip(zipPath);
+    console.log("[CRI] Extracting + distributing XMLs by session…");
+    for (const session of sessions) {
+        const dir = path.join(originalRoot, String(session));
+        if (await fs.pathExists(dir)) {
+            for (const f of await fs.readdir(dir))
+                if (/\.xml$/i.test(f))
+                    await fs.remove(path.join(dir, f));
+        }
+    }
+    const n = await extractAndDistributeXmlBySession(zipPath, originalRoot);
+    if (n === 0) {
+        console.warn("[CRI] No XML extracted. Archive empty or layout changed?");
+    }
+    else {
+        console.log(`[CRI] Distributed ${n} XML file(s) into session folders.`);
+    }
+    if (!options["parseDebats"]) {
+        console.log("[CRI] parseDebats not requested → done.");
+        return;
+    }
+    for (const session of sessions) {
+        const originalSessionDir = path.join(originalRoot, String(session));
+        if (!(await fs.pathExists(originalSessionDir))) {
+            continue;
+        }
+        const xmlFiles = (await fs.readdir(originalSessionDir)).filter((f) => /^d\d{8}\.xml$/i.test(f)).sort();
+        const transformedSessionDir = path.join(transformedRoot, String(session));
+        await fs.ensureDir(transformedSessionDir);
+        const now = Date.now();
+        for (const f of xmlFiles) {
+            const yyyymmdd = f.slice(1, 9);
+            const xmlPath = path.join(originalSessionDir, f);
+            // === ONLY-RECENT
+            if (options["only-recent"]) {
+                const cutoff = now - options["only-recent"] * 24 * 3600 * 1000;
+                const seanceTs = Date.parse(`${yyyymmdd.slice(0, 4)}-${yyyymmdd.slice(4, 6)}-${yyyymmdd.slice(6, 8)}`);
+                if (seanceTs < cutoff) {
+                    const files = await fs.readdir(transformedSessionDir);
+                    const dayFiles = files.filter((fn) => fn.startsWith(`CRSSN${yyyymmdd}E`) && fn.endsWith(".json"));
+                    if (dayFiles.length > 0) {
+                        for (const fn of dayFiles) {
+                            const match = fn.match(/^CRSSN(\d{8})E(.+)\.json$/);
+                            const eventId = match?.[2];
+                            if (!eventId)
+                                continue;
+                            const crPath = path.join(transformedSessionDir, fn);
+                            try {
+                                const cr = await fs.readJSON(crPath);
+                                await linkCriEventIntoAgenda(dataDir, yyyymmdd, eventId, cr.uid, cr, session);
+                            }
+                            catch (e) {
+                                console.warn(`[CR] [${session}] Could not relink existing CR into a reunion for ${yyyymmdd} event=${eventId}:`, e);
+                            }
+                        }
+                        continue;
+                    }
+                }
+            }
+            // === Charger les events SP du jour depuis les agendas groupés ===
+            const dayEvents = await loadAgendaSpEventsForDate(dataDir, yyyymmdd, session);
+            if (dayEvents.length === 0) {
+                console.warn(`[CRI] [${session}] No agenda SP events found for ${yyyymmdd} → skip split/link`);
+                continue;
+            }
+            // === Lire XML + construire index DOM ===
+            let raw;
+            let $;
+            let order;
+            let idx;
+            try {
+                raw = await fs.readFile(xmlPath, "utf8");
+                $ = cheerio.load(raw, { xml: false });
+                order = $("body *").toArray();
+                idx = new Map(order.map((el, i) => [el, i]));
+            }
+            catch (e) {
+                console.warn(`[CRI] [${session}] Cannot read/parse ${f}:`, e);
+                continue;
+            }
+            // === Extraire sommaire + matcher vers events agenda ===
+            const blocks = extractSommaireBlocks($, idx);
+            const intervals = buildIntervalsByAgendaEvents($, idx, order, blocks, dayEvents);
+            if (!intervals.length) {
+                console.warn(`[CRI] [${session}] No confident split intervals for ${yyyymmdd} → skip`);
+                continue;
+            }
+            // === Parser / écrire / linker chaque segment par event ===
+            for (const iv of intervals) {
+                const outName = `CRSSN${yyyymmdd}E${iv.agendaEventId}.json`;
+                const outPath = path.join(transformedSessionDir, outName);
+                const cr = await parseCompteRenduIntervalFromFile(xmlPath, iv.startIndex, iv.endIndex, iv.agendaEventId);
+                if (!cr) {
+                    console.warn(`[CRI] [${session}] Empty or no points for ${yyyymmdd} event=${iv.agendaEventId} → skip`);
+                    continue;
+                }
+                await fs.ensureDir(transformedSessionDir);
+                await fs.writeJSON(outPath, cr, { spaces: 2 });
+                try {
+                    await linkCriEventIntoAgenda(dataDir, yyyymmdd, iv.agendaEventId, cr.uid, cr, session);
+                }
+                catch (e) {
+                    console.warn(`[CR] [${session}] Could not link CR into agenda for ${yyyymmdd} event=${iv.agendaEventId}:`, e);
+                }
+            }
+        }
+    }
+}
+async function linkCriEventIntoAgenda(dataDir, yyyymmdd, agendaEventId, crUid, cr, session) {
+    const agendadDir = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, session.toString());
+    fs.ensureDirSync(agendadDir);
+    const dateISO = `${yyyymmdd.slice(0, 4)}-${yyyymmdd.slice(4, 6)}-${yyyymmdd.slice(6, 8)}`;
+    const agendaUid = makeReunionUid(dateISO, "SP", agendaEventId, null);
+    const agendaPath = path.join(agendadDir, `${agendaUid}.json`);
+    let agenda = null;
+    if (await fs.pathExists(agendaPath)) {
+        try {
+            agenda = await fs.readJSON(agendaPath);
+        }
+        catch (e) {
+            console.warn(`[CR] unreadable reunion JSON → ${agendaPath} (${e})`);
+            agenda = null;
+        }
+    }
+    if (!agenda) {
+        console.warn(`[CR] Missing reunion file for SP event=${agendaEventId}: ${agendaPath}`);
+        return;
+    }
+    ;
+    agenda.compteRenduRefUid = crUid;
+    await fs.writeJSON(agendaPath, agenda, { spaces: 2 });
+    console.log(`[CR] Linked CR ${crUid} → ${path.basename(agendaPath)} (event=${agendaEventId})`);
+}
+function buildIntervalsByAgendaEvents($, idx, order, blocks, dayEvents) {
+    const MIN_SCORE = 0.65;
+    const MIN_GAP = 0.08;
+    const firstIntervenant = $("div.intervenant").first()[0];
+    const firstIntervenantIdx = firstIntervenant ? (idx.get(firstIntervenant) ?? null) : null;
+    const pivots = [];
+    for (const b of blocks) {
+        if (isNoiseBlock(b.text))
+            continue;
+        let best = null;
+        let second = 0;
+        for (const ev of dayEvents) {
+            const s = scoreSommaireBlockForEvent(b.text, ev);
+            if (!best || s > best.score) {
+                second = best?.score ?? second;
+                best = { ev, score: s };
+            }
+            else if (s > second) {
+                second = s;
+            }
+        }
+        if (!best)
+            continue;
+        const resolved = resolveTargetIndex($, idx, b.targetId);
+        const contentStartIndex = resolved ?? b.startIndex;
+        if (firstIntervenantIdx != null && contentStartIndex < firstIntervenantIdx && resolved == null) {
+            continue;
+        }
+        if (best.score < MIN_SCORE)
+            continue;
+        if (best.score - second < MIN_GAP)
+            continue;
+        pivots.push({
+            agendaEventId: best.ev.id,
+            startIndex: contentStartIndex,
+            score: best.score,
+        });
+    }
+    if (pivots.length === 0)
+        return [];
+    // Dédupe par event (on garde le premier startIndex)
+    const byEvent = new Map();
+    for (const p of pivots.sort((a, b) => a.startIndex - b.startIndex)) {
+        if (!byEvent.has(p.agendaEventId)) {
+            byEvent.set(p.agendaEventId, {
+                startIndex: p.startIndex,
+                score: p.score,
+            });
+        }
+    }
+    const sorted = Array.from(byEvent.entries())
+        .map(([agendaEventId, v]) => ({
+        agendaEventId,
+        startIndex: v.startIndex,
+        score: v.score,
+    }))
+        .sort((a, b) => a.startIndex - b.startIndex);
+    // Construction des intervalles
+    const intervals = [];
+    for (let i = 0; i < sorted.length; i++) {
+        const cur = sorted[i];
+        const next = sorted[i + 1];
+        const endIndex = next ? next.startIndex - 1 : order.length - 1;
+        intervals.push({
+            agendaEventId: cur.agendaEventId,
+            startIndex: cur.startIndex,
+            endIndex,
+            score: cur.score,
+        });
+    }
+    return intervals;
+}
+async function loadAgendaSpEventsForDate(dataDir, yyyymmdd, session) {
+    const agendasDir = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, session.toString());
+    if (!(await fs.pathExists(agendasDir)))
+        return [];
+    const files = (await fs.readdir(agendasDir)).filter((fn) => fn.startsWith(`RUSN${yyyymmdd}IDS`) && fn.endsWith(".json"));
+    const events = [];
+    for (const fn of files) {
+        try {
+            const g = (await fs.readJSON(path.join(agendasDir, fn)));
+            const e = g?.events?.[0];
+            if (e && e.type === "Séance publique")
+                events.push(e);
+        }
+        catch { }
+    }
+    return events;
+}
+function cssEscapeIdent(s) {
+    return s.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
+}
+function resolveTargetIndex($, idx, targetId) {
+    if (!targetId)
+        return null;
+    const safe = cssEscapeIdent(targetId);
+    const el = $(`[id="${safe}"]`)[0] || $(`[name="${safe}"]`)[0];
+    if (!el)
+        return null;
+    const i = idx.get(el);
+    return i == null ? null : i;
+}
+async function main() {
+    const dataDir = options["dataDir"];
+    assert(dataDir, "Missing argument: data directory");
+    console.time("CRI processing time");
+    await retrieveCriXmlDump(dataDir, options);
+    console.timeEnd("CRI processing time");
+}
+main()
+    .then(() => process.exit(0))
+    .catch((error) => {
+    console.error(error);
+    process.exit(1);
+});

package/lib/scripts/retrieve_documents.d.ts

@@ -0,0 +1,3 @@
+import { DocumentMetadata } from "../types/texte";
+export declare function processTexte(texteMetadata: DocumentMetadata, originalTextesDir: string, transformedTextesDir: string, options: any): Promise<void>;
+export declare function processRapport(rapportMetadata: any, originalRapportsDir: string, options: any): Promise<void>;

package/lib/scripts/retrieve_documents.js

@@ -0,0 +1,219 @@
+import assert from "assert";
+import commandLineArgs from "command-line-args";
+import fs from "fs-extra";
+import { DateTime } from "luxon";
+import path from "path";
+import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatRapportUrls, iterLoadSenatTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, } from "../loaders";
+import { parseExposeDesMotifs, parseTexte, parseTexteFromFile } from "../parsers/texte";
+import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions";
+import { commonOptions } from "./shared/cli_helpers";
+import { ensureAndClearDir, fetchWithRetry, isOptionEmptyOrHasValue } from "./shared/util";
+const optionsDefinitions = [
+    ...commonOptions,
+    {
+        alias: "F",
+        help: "formats of documents to retrieve (xml/html/pdf for textes, html/pdf for rapports); leave empty for all",
+        multiple: true,
+        name: "formats",
+        type: String,
+    },
+    {
+        help: "types of documents to retrieve (textes/rapports); leave empty for all",
+        multiple: true,
+        name: "types",
+        type: String,
+    },
+    {
+        help: "force retrieve all documents, even already retrieved ones",
+        name: "force",
+        type: Boolean,
+    },
+];
+const options = commandLineArgs(optionsDefinitions);
+const textDecoder = new TextDecoder("utf8");
+const today = DateTime.now();
+function isDocumentRecent(documentDate, daysThreshold) {
+    if (!documentDate)
+        return false;
+    const docDate = DateTime.fromISO(documentDate);
+    return docDate.isValid && today.diff(docDate, "days").days <= daysThreshold;
+}
+function shouldDownload(filePath, docDate, options) {
+    if (options.force)
+        return true;
+    if (!fs.existsSync(filePath))
+        return true;
+    if (options.onlyRecent !== undefined) {
+        return isDocumentRecent(docDate, options.onlyRecent);
+    }
+    return false;
+}
+async function downloadDocument(documentUrl, verbose) {
+    if (verbose) {
+        console.log(`Downloading document ${documentUrl}…`);
+    }
+    try {
+        const response = await fetchWithRetry(documentUrl);
+        if (!response.ok) {
+            if (response.status === 404) {
+                if (verbose) {
+                    console.warn(`Document ${documentUrl} not found`);
+                }
+            }
+            else {
+                if (verbose) {
+                    console.error(`An error occurred while retrieving document ${documentUrl}: ${response.status}`);
+                }
+            }
+            return null;
+        }
+        return response.arrayBuffer();
+    }
+    catch (error) {
+        console.error(error.message);
+        return null;
+    }
+}
+async function processDocument(url, destPath, docDate, options) {
+    if (!shouldDownload(destPath, docDate, options)) {
+        if (options.verbose)
+            console.info(`Already downloaded ${destPath}…`);
+        return { success: true, skipped: true, buffer: null };
+    }
+    const arrayBuffer = await downloadDocument(url, options.verbose);
+    if (!arrayBuffer) {
+        return { success: false, skipped: false, buffer: null };
+    }
+    const buffer = Buffer.from(arrayBuffer);
+    await fs.outputFile(destPath, buffer);
+    return { success: true, skipped: false, buffer };
+}
+export async function processTexte(texteMetadata, originalTextesDir, transformedTextesDir, options) {
+    const texteDir = path.join(originalTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name);
+    let exposeDesMotifsContent = null;
+    if (texteMetadata.url_expose_des_motifs) {
+        const exposePath = path.join(texteDir, `${texteMetadata.name}-expose.html`);
+        const res = await processDocument(texteMetadata.url_expose_des_motifs.toString(), exposePath, texteMetadata.date, options);
+        if (res.buffer) {
+            exposeDesMotifsContent = res.buffer;
+        }
+        else if (res.skipped && options.parseDocuments) {
+            if (await fs.pathExists(exposePath)) {
+                exposeDesMotifsContent = await fs.readFile(exposePath);
+            }
+        }
+    }
+    const formats = [
+        { type: "xml", url: texteMetadata.url_xml, isParseTarget: true },
+        { type: "html", url: texteMetadata.url_html, isParseTarget: false },
+        { type: "pdf", url: texteMetadata.url_pdf, isParseTarget: false },
+    ];
+    for (const format of formats) {
+        if (!isOptionEmptyOrHasValue(options.formats, format.type))
+            continue;
+        if (!format.url)
+            continue;
+        const destPath = path.join(texteDir, `${texteMetadata.name}.${format.type}`);
+        const result = await processDocument(format.url.toString(), destPath, texteMetadata.date, options);
+        // Specific logic: Parsing (Only applies to XML)
+        if (format.isParseTarget && options.parseDocuments) {
+            await parseDocument(texteMetadata.session, transformedTextesDir, destPath, texteMetadata.name, result.buffer, exposeDesMotifsContent, options);
+        }
+    }
+}
+export async function processRapport(rapportMetadata, originalRapportsDir, options) {
+    const rapportDir = path.join(originalRapportsDir, `${rapportMetadata.session ?? UNDEFINED_SESSION}`, rapportMetadata.name);
+    const formats = [
+        { type: "html", url: rapportMetadata.url_html },
+        { type: "pdf", url: rapportMetadata.url_pdf },
+    ];
+    for (const format of formats) {
+        if (!isOptionEmptyOrHasValue(options["formats"], format.type))
+            continue;
+        const destPath = path.join(rapportDir, `${rapportMetadata.name}.${format.type}`);
+        await processDocument(format.url.toString(), destPath, rapportMetadata.date, options);
+    }
+}
+async function retrieveTextes(dataDir, sessions) {
+    const originalTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER);
+    const transformedTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_TRANSFORMED_FOLDER);
+    if (options["parseDocuments"]) {
+        ensureAndClearDir(transformedTextesDir);
+    }
+    const dlOptions = {
+        force: options["force"],
+        silent: options["silent"],
+        verbose: options["verbose"],
+        onlyRecent: options["only-recent"],
+        formats: options["formats"],
+        parseDocuments: options["parseDocuments"],
+    };
+    for (const session of sessions) {
+        for (const { item: texteMetadata } of iterLoadSenatTexteUrls(dataDir, session)) {
+            await processTexte(texteMetadata, originalTextesDir, transformedTextesDir, dlOptions);
+        }
+    }
+}
+async function retrieveRapports(dataDir, sessions) {
+    const originalRapportsDir = path.join(dataDir, RAPPORT_FOLDER, DATA_ORIGINAL_FOLDER);
+    const dlOptions = {
+        force: options["force"],
+        silent: options["silent"],
+        verbose: options["verbose"],
+        onlyRecent: options["only-recent"],
+        formats: options["formats"],
+    };
+    for (const session of sessions) {
+        for (const { item: rapportMetadata } of iterLoadSenatRapportUrls(dataDir, session)) {
+            await processRapport(rapportMetadata, originalRapportsDir, dlOptions);
+        }
+    }
+}
+async function parseDocument(session, transformedTextesDir, textePath, texteName, texteBuffer, exposeDesMotifs = null, options = {}) {
+    if (options.verbose) {
+        console.log(`Parsing texte ${textePath}…`);
+    }
+    let parsedTexte;
+    if (texteBuffer) {
+        const texteXml = textDecoder.decode(texteBuffer);
+        parsedTexte = parseTexte(texteXml);
+    }
+    else {
+        parsedTexte = await parseTexteFromFile(textePath);
+    }
+    if (!parsedTexte)
+        return null;
+    if (exposeDesMotifs) {
+        if (options.verbose) {
+            console.log("Parsing exposé des motifs…");
+        }
+        const exposeDesMotifsHtml = textDecoder.decode(exposeDesMotifs);
+        parsedTexte.exposeDesMotifs = parseExposeDesMotifs(exposeDesMotifsHtml);
+    }
+    const transformedTexteDir = path.join(transformedTextesDir, `${session ?? UNDEFINED_SESSION}`, texteName);
+    await fs.outputJSON(path.join(transformedTexteDir, `${texteName}.json`), parsedTexte, { spaces: 2 });
+    return parsedTexte;
+}
+async function main() {
+    const dataDir = options["dataDir"];
+    assert(dataDir, "Missing argument: data directory");
+    const sessions = getSessionsFromStart(options["fromSession"]);
+    console.time("documents processing time");
+    if (isOptionEmptyOrHasValue(options["types"], "textes")) {
+        await retrieveTextes(dataDir, sessions);
+    }
+    if (isOptionEmptyOrHasValue(options["types"], "rapports")) {
+        await retrieveRapports(dataDir, sessions);
+    }
+    if (!options["silent"]) {
+        console.timeEnd("documents processing time");
+    }
+}
+if (process.argv[1].endsWith("retrieve_documents.ts")) {
+    main()
+        .then(() => process.exit(0))
+        .catch((error) => {
+        console.log(error);
+        process.exit(1);
+    });
+}

package/lib/scripts/retrieve_open_data.d.ts

@@ -0,0 +1 @@
+export {};