@tricoteuses/senat 3.1.22 → 3.1.24

package/lib/scripts/retrieve_cr_seance.js

@@ -1,347 +0,0 @@
-/**
- * Needs to be ran after retrieve_agenda.ts script !
- * - downloads the ZIP of comptes-rendus des débats (CRI) from data.senat.fr
- * - extracts XML files, distributes them by session/year
- */
-import assert from "assert";
-import commandLineArgs from "command-line-args";
-import fs, { ensureDirSync } from "fs-extra";
-import path from "path";
-import StreamZip from "node-stream-zip";
-import * as cheerio from "cheerio";
-import { AGENDA_FOLDER, COMPTES_RENDUS_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
-import { commonOptions } from "./shared/cli_helpers";
-import { parseCompteRenduIntervalFromFile, sessionStartYearFromDate } from "../model/seance";
-import { extractSommaireBlocks, makeReunionUid } from "../utils/reunion_parsing";
-import { getSessionsFromStart } from "../types/sessions";
-import { ensureAndClearDir, fetchWithRetry } from "./shared/util";
-import { isNoiseBlock, scoreSommaireBlockForEvent } from "../utils/scoring";
-import { parseYYYYMMDD } from "../utils/date";
-const optionsDefinitions = [
-    ...commonOptions,
-    {
-        help: "parse and convert comptes-rendus des débats into JSON",
-        name: "parseDebats",
-        type: Boolean,
-    },
-];
-const options = commandLineArgs(optionsDefinitions);
-const CRI_ZIP_URL = "https://data.senat.fr/data/debats/cri.zip";
-class CompteRenduError extends Error {
-    constructor(message, url) {
-        super(`An error occurred while retrieving ${url}: ${message}`);
-    }
-}
-async function downloadCriZip(zipPath) {
-    if (!options["silent"])
-        console.log(`Downloading CRI zip ${CRI_ZIP_URL}…`);
-    const response = await fetchWithRetry(CRI_ZIP_URL);
-    if (!response.ok) {
-        if (response.status === 404) {
-            console.warn(`CRI zip ${CRI_ZIP_URL} not found`);
-            return;
-        }
-        throw new CompteRenduError(String(response.status), CRI_ZIP_URL);
-    }
-    const buf = Buffer.from(await response.arrayBuffer());
-    await fs.writeFile(zipPath, buf);
-    if (!options["silent"]) {
-        const mb = (buf.length / (1024 * 1024)).toFixed(1);
-        console.log(`[CRI] Downloaded ${mb} MB → ${zipPath}`);
-    }
-}
-async function extractAndDistributeXmlBySession(zipPath, originalRoot) {
-    const zip = new StreamZip.async({ file: zipPath });
-    const entries = await zip.entries();
-    let count = 0;
-    for (const entryName of Object.keys(entries)) {
-        if (!entryName.toLowerCase().endsWith(".xml"))
-            continue;
-        // ex: d20231005.xml
-        const base = path.basename(entryName);
-        const m = base.match(/^d(\d{8})\.xml$/i);
-        if (!m)
-            continue;
-        const yyyymmdd = m[1];
-        const dt = parseYYYYMMDD(yyyymmdd);
-        if (!dt)
-            continue;
-        const session = sessionStartYearFromDate(dt);
-        const destDir = path.join(originalRoot, String(session));
-        await fs.ensureDir(destDir);
-        const outPath = path.join(destDir, base);
-        await zip.extract(entryName, outPath);
-        count++;
-    }
-    await zip.close();
-    return count;
-}
-export async function retrieveCriXmlDump(dataDir, options = {}) {
-    const root = path.join(dataDir, COMPTES_RENDUS_FOLDER);
-    ensureDirSync(root);
-    const originalRoot = path.join(root, DATA_ORIGINAL_FOLDER);
-    if (!options["keepDir"]) {
-        ensureAndClearDir(originalRoot);
-    }
-    else {
-        fs.ensureDirSync(originalRoot);
-    }
-    const transformedRoot = path.join(root, DATA_TRANSFORMED_FOLDER);
-    if (!options["keepDir"]) {
-        ensureAndClearDir(transformedRoot);
-    }
-    else {
-        fs.ensureDirSync(transformedRoot);
-    }
-    const sessions = getSessionsFromStart(options["fromSession"]);
-    // 1) Download ZIP global + distribut by session
-    const zipPath = path.join(dataDir, "cri.zip");
-    console.log("[CRI] Downloading global CRI zip…");
-    await downloadCriZip(zipPath);
-    console.log("[CRI] Extracting + distributing XMLs by session…");
-    for (const session of sessions) {
-        const dir = path.join(originalRoot, String(session));
-        if (await fs.pathExists(dir)) {
-            for (const f of await fs.readdir(dir))
-                if (/\.xml$/i.test(f))
-                    await fs.remove(path.join(dir, f));
-        }
-    }
-    const n = await extractAndDistributeXmlBySession(zipPath, originalRoot);
-    if (n === 0) {
-        console.warn("[CRI] No XML extracted. Archive empty or layout changed?");
-    }
-    else {
-        console.log(`[CRI] Distributed ${n} XML file(s) into session folders.`);
-    }
-    if (!options["parseDebats"]) {
-        console.log("[CRI] parseDebats not requested → done.");
-        return;
-    }
-    for (const session of sessions) {
-        const originalSessionDir = path.join(originalRoot, String(session));
-        if (!(await fs.pathExists(originalSessionDir))) {
-            continue;
-        }
-        const xmlFiles = (await fs.readdir(originalSessionDir)).filter((f) => /^d\d{8}\.xml$/i.test(f)).sort();
-        const transformedSessionDir = path.join(transformedRoot, String(session));
-        await fs.ensureDir(transformedSessionDir);
-        const now = Date.now();
-        for (const f of xmlFiles) {
-            const yyyymmdd = f.slice(1, 9);
-            const xmlPath = path.join(originalSessionDir, f);
-            // === ONLY-RECENT
-            if (options["only-recent"]) {
-                const cutoff = now - options["only-recent"] * 24 * 3600 * 1000;
-                const seanceTs = Date.parse(`${yyyymmdd.slice(0, 4)}-${yyyymmdd.slice(4, 6)}-${yyyymmdd.slice(6, 8)}`);
-                if (seanceTs < cutoff) {
-                    const files = await fs.readdir(transformedSessionDir);
-                    const dayFiles = files.filter((fn) => fn.startsWith(`CRSSN${yyyymmdd}E`) && fn.endsWith(".json"));
-                    if (dayFiles.length > 0) {
-                        for (const fn of dayFiles) {
-                            const match = fn.match(/^CRSSN(\d{8})E(.+)\.json$/);
-                            const eventId = match?.[2];
-                            if (!eventId)
-                                continue;
-                            const crPath = path.join(transformedSessionDir, fn);
-                            try {
-                                const cr = await fs.readJSON(crPath);
-                                await linkCriEventIntoAgenda(dataDir, yyyymmdd, eventId, cr.uid, cr, session);
-                            }
-                            catch (e) {
-                                console.warn(`[CR] [${session}] Could not relink existing CR into a reunion for ${yyyymmdd} event=${eventId}:`, e);
-                            }
-                        }
-                        continue;
-                    }
-                }
-            }
-            // === Charger les events SP du jour depuis les agendas groupés ===
-            const dayEvents = await loadAgendaSpEventsForDate(dataDir, yyyymmdd, session);
-            if (dayEvents.length === 0) {
-                console.warn(`[CRI] [${session}] No agenda SP events found for ${yyyymmdd} → skip split/link`);
-                continue;
-            }
-            // === Lire XML + construire index DOM ===
-            let raw;
-            let $;
-            let order;
-            let idx;
-            try {
-                raw = await fs.readFile(xmlPath, "utf8");
-                $ = cheerio.load(raw, { xml: false });
-                order = $("body *").toArray();
-                idx = new Map(order.map((el, i) => [el, i]));
-            }
-            catch (e) {
-                console.warn(`[CRI] [${session}] Cannot read/parse ${f}:`, e);
-                continue;
-            }
-            // === Extraire sommaire + matcher vers events agenda ===
-            const blocks = extractSommaireBlocks($, idx);
-            const intervals = buildIntervalsByAgendaEvents($, idx, order, blocks, dayEvents);
-            if (!intervals.length) {
-                console.warn(`[CRI] [${session}] No confident split intervals for ${yyyymmdd} → skip`);
-                continue;
-            }
-            // === Parser / écrire / linker chaque segment par event ===
-            for (const iv of intervals) {
-                const outName = `CRSSN${yyyymmdd}E${iv.agendaEventId}.json`;
-                const outPath = path.join(transformedSessionDir, outName);
-                const cr = await parseCompteRenduIntervalFromFile(xmlPath, iv.startIndex, iv.endIndex, iv.agendaEventId);
-                if (!cr) {
-                    console.warn(`[CRI] [${session}] Empty or no points for ${yyyymmdd} event=${iv.agendaEventId} → skip`);
-                    continue;
-                }
-                await fs.ensureDir(transformedSessionDir);
-                await fs.writeJSON(outPath, cr, { spaces: 2 });
-                try {
-                    await linkCriEventIntoAgenda(dataDir, yyyymmdd, iv.agendaEventId, cr.uid, cr, session);
-                }
-                catch (e) {
-                    console.warn(`[CR] [${session}] Could not link CR into agenda for ${yyyymmdd} event=${iv.agendaEventId}:`, e);
-                }
-            }
-        }
-    }
-}
-async function linkCriEventIntoAgenda(dataDir, yyyymmdd, agendaEventId, crUid, cr, session) {
-    const agendadDir = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, session.toString());
-    fs.ensureDirSync(agendadDir);
-    const dateISO = `${yyyymmdd.slice(0, 4)}-${yyyymmdd.slice(4, 6)}-${yyyymmdd.slice(6, 8)}`;
-    const agendaUid = makeReunionUid(dateISO, "SP", agendaEventId, null);
-    const agendaPath = path.join(agendadDir, `${agendaUid}.json`);
-    let agenda = null;
-    if (await fs.pathExists(agendaPath)) {
-        try {
-            agenda = await fs.readJSON(agendaPath);
-        }
-        catch (e) {
-            console.warn(`[CR] unreadable reunion JSON → ${agendaPath} (${e})`);
-            agenda = null;
-        }
-    }
-    if (!agenda) {
-        console.warn(`[CR] Missing reunion file for SP event=${agendaEventId}: ${agendaPath}`);
-        return;
-    }
-    ;
-    agenda.compteRenduRefUid = crUid;
-    await fs.writeJSON(agendaPath, agenda, { spaces: 2 });
-    console.log(`[CR] Linked CR ${crUid} → ${path.basename(agendaPath)} (event=${agendaEventId})`);
-}
-function buildIntervalsByAgendaEvents($, idx, order, blocks, dayEvents) {
-    const MIN_SCORE = 0.65;
-    const MIN_GAP = 0.08;
-    const firstIntervenant = $("div.intervenant").first()[0];
-    const firstIntervenantIdx = firstIntervenant ? (idx.get(firstIntervenant) ?? null) : null;
-    const pivots = [];
-    for (const b of blocks) {
-        if (isNoiseBlock(b.text))
-            continue;
-        let best = null;
-        let second = 0;
-        for (const ev of dayEvents) {
-            const s = scoreSommaireBlockForEvent(b.text, ev);
-            if (!best || s > best.score) {
-                second = best?.score ?? second;
-                best = { ev, score: s };
-            }
-            else if (s > second) {
-                second = s;
-            }
-        }
-        if (!best)
-            continue;
-        const resolved = resolveTargetIndex($, idx, b.targetId);
-        const contentStartIndex = resolved ?? b.startIndex;
-        if (firstIntervenantIdx != null && contentStartIndex < firstIntervenantIdx && resolved == null) {
-            continue;
-        }
-        if (best.score < MIN_SCORE)
-            continue;
-        if (best.score - second < MIN_GAP)
-            continue;
-        pivots.push({
-            agendaEventId: best.ev.id,
-            startIndex: contentStartIndex,
-            score: best.score,
-        });
-    }
-    if (pivots.length === 0)
-        return [];
-    // Dédupe par event (on garde le premier startIndex)
-    const byEvent = new Map();
-    for (const p of pivots.sort((a, b) => a.startIndex - b.startIndex)) {
-        if (!byEvent.has(p.agendaEventId)) {
-            byEvent.set(p.agendaEventId, {
-                startIndex: p.startIndex,
-                score: p.score,
-            });
-        }
-    }
-    const sorted = Array.from(byEvent.entries())
-        .map(([agendaEventId, v]) => ({
-        agendaEventId,
-        startIndex: v.startIndex,
-        score: v.score,
-    }))
-        .sort((a, b) => a.startIndex - b.startIndex);
-    // Construction des intervalles
-    const intervals = [];
-    for (let i = 0; i < sorted.length; i++) {
-        const cur = sorted[i];
-        const next = sorted[i + 1];
-        const endIndex = next ? next.startIndex - 1 : order.length - 1;
-        intervals.push({
-            agendaEventId: cur.agendaEventId,
-            startIndex: cur.startIndex,
-            endIndex,
-            score: cur.score,
-        });
-    }
-    return intervals;
-}
-async function loadAgendaSpEventsForDate(dataDir, yyyymmdd, session) {
-    const agendasDir = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, session.toString());
-    if (!(await fs.pathExists(agendasDir)))
-        return [];
-    const files = (await fs.readdir(agendasDir)).filter((fn) => fn.startsWith(`RUSN${yyyymmdd}IDS`) && fn.endsWith(".json"));
-    const events = [];
-    for (const fn of files) {
-        try {
-            const g = (await fs.readJSON(path.join(agendasDir, fn)));
-            const e = g?.events?.[0];
-            if (e && e.type === "Séance publique")
-                events.push(e);
-        }
-        catch { }
-    }
-    return events;
-}
-function cssEscapeIdent(s) {
-    return s.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
-}
-function resolveTargetIndex($, idx, targetId) {
-    if (!targetId)
-        return null;
-    const safe = cssEscapeIdent(targetId);
-    const el = $(`[id="${safe}"]`)[0] || $(`[name="${safe}"]`)[0];
-    if (!el)
-        return null;
-    const i = idx.get(el);
-    return i == null ? null : i;
-}
-async function main() {
-    const dataDir = options["dataDir"];
-    assert(dataDir, "Missing argument: data directory");
-    console.time("CRI processing time");
-    await retrieveCriXmlDump(dataDir, options);
-    console.timeEnd("CRI processing time");
-}
-main()
-    .then(() => process.exit(0))
-    .catch((error) => {
-    console.error(error);
-    process.exit(1);
-});

package/lib/scripts/retrieve_documents.d.ts

@@ -1,3 +0,0 @@
-import { DocumentMetadata } from "../types/texte";
-export declare function processTexte(texteMetadata: DocumentMetadata, originalTextesDir: string, transformedTextesDir: string, options: any): Promise<void>;
-export declare function processRapport(rapportMetadata: any, originalRapportsDir: string, options: any): Promise<void>;

package/lib/scripts/retrieve_documents.js

@@ -1,219 +0,0 @@
-import assert from "assert";
-import commandLineArgs from "command-line-args";
-import fs from "fs-extra";
-import { DateTime } from "luxon";
-import path from "path";
-import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatRapportUrls, iterLoadSenatTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, } from "../loaders";
-import { parseExposeDesMotifs, parseTexte, parseTexteFromFile } from "../parsers/texte";
-import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions";
-import { commonOptions } from "./shared/cli_helpers";
-import { ensureAndClearDir, fetchWithRetry, isOptionEmptyOrHasValue } from "./shared/util";
-const optionsDefinitions = [
-    ...commonOptions,
-    {
-        alias: "F",
-        help: "formats of documents to retrieve (xml/html/pdf for textes, html/pdf for rapports); leave empty for all",
-        multiple: true,
-        name: "formats",
-        type: String,
-    },
-    {
-        help: "types of documents to retrieve (textes/rapports); leave empty for all",
-        multiple: true,
-        name: "types",
-        type: String,
-    },
-    {
-        help: "force retrieve all documents, even already retrieved ones",
-        name: "force",
-        type: Boolean,
-    },
-];
-const options = commandLineArgs(optionsDefinitions);
-const textDecoder = new TextDecoder("utf8");
-const today = DateTime.now();
-function isDocumentRecent(documentDate, daysThreshold) {
-    if (!documentDate)
-        return false;
-    const docDate = DateTime.fromISO(documentDate);
-    return docDate.isValid && today.diff(docDate, "days").days <= daysThreshold;
-}
-function shouldDownload(filePath, docDate, options) {
-    if (options.force)
-        return true;
-    if (!fs.existsSync(filePath))
-        return true;
-    if (options.onlyRecent !== undefined) {
-        return isDocumentRecent(docDate, options.onlyRecent);
-    }
-    return false;
-}
-async function downloadDocument(documentUrl, verbose) {
-    if (verbose) {
-        console.log(`Downloading document ${documentUrl}…`);
-    }
-    try {
-        const response = await fetchWithRetry(documentUrl);
-        if (!response.ok) {
-            if (response.status === 404) {
-                if (verbose) {
-                    console.warn(`Document ${documentUrl} not found`);
-                }
-            }
-            else {
-                if (verbose) {
-                    console.error(`An error occurred while retrieving document ${documentUrl}: ${response.status}`);
-                }
-            }
-            return null;
-        }
-        return response.arrayBuffer();
-    }
-    catch (error) {
-        console.error(error.message);
-        return null;
-    }
-}
-async function processDocument(url, destPath, docDate, options) {
-    if (!shouldDownload(destPath, docDate, options)) {
-        if (options.verbose)
-            console.info(`Already downloaded ${destPath}…`);
-        return { success: true, skipped: true, buffer: null };
-    }
-    const arrayBuffer = await downloadDocument(url, options.verbose);
-    if (!arrayBuffer) {
-        return { success: false, skipped: false, buffer: null };
-    }
-    const buffer = Buffer.from(arrayBuffer);
-    await fs.outputFile(destPath, buffer);
-    return { success: true, skipped: false, buffer };
-}
-export async function processTexte(texteMetadata, originalTextesDir, transformedTextesDir, options) {
-    const texteDir = path.join(originalTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name);
-    let exposeDesMotifsContent = null;
-    if (texteMetadata.url_expose_des_motifs) {
-        const exposePath = path.join(texteDir, `${texteMetadata.name}-expose.html`);
-        const res = await processDocument(texteMetadata.url_expose_des_motifs.toString(), exposePath, texteMetadata.date, options);
-        if (res.buffer) {
-            exposeDesMotifsContent = res.buffer;
-        }
-        else if (res.skipped && options.parseDocuments) {
-            if (await fs.pathExists(exposePath)) {
-                exposeDesMotifsContent = await fs.readFile(exposePath);
-            }
-        }
-    }
-    const formats = [
-        { type: "xml", url: texteMetadata.url_xml, isParseTarget: true },
-        { type: "html", url: texteMetadata.url_html, isParseTarget: false },
-        { type: "pdf", url: texteMetadata.url_pdf, isParseTarget: false },
-    ];
-    for (const format of formats) {
-        if (!isOptionEmptyOrHasValue(options.formats, format.type))
-            continue;
-        if (!format.url)
-            continue;
-        const destPath = path.join(texteDir, `${texteMetadata.name}.${format.type}`);
-        const result = await processDocument(format.url.toString(), destPath, texteMetadata.date, options);
-        // Specific logic: Parsing (Only applies to XML)
-        if (format.isParseTarget && options.parseDocuments) {
-            await parseDocument(texteMetadata.session, transformedTextesDir, destPath, texteMetadata.name, result.buffer, exposeDesMotifsContent, options);
-        }
-    }
-}
-export async function processRapport(rapportMetadata, originalRapportsDir, options) {
-    const rapportDir = path.join(originalRapportsDir, `${rapportMetadata.session ?? UNDEFINED_SESSION}`, rapportMetadata.name);
-    const formats = [
-        { type: "html", url: rapportMetadata.url_html },
-        { type: "pdf", url: rapportMetadata.url_pdf },
-    ];
-    for (const format of formats) {
-        if (!isOptionEmptyOrHasValue(options["formats"], format.type))
-            continue;
-        const destPath = path.join(rapportDir, `${rapportMetadata.name}.${format.type}`);
-        await processDocument(format.url.toString(), destPath, rapportMetadata.date, options);
-    }
-}
-async function retrieveTextes(dataDir, sessions) {
-    const originalTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER);
-    const transformedTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_TRANSFORMED_FOLDER);
-    if (options["parseDocuments"]) {
-        ensureAndClearDir(transformedTextesDir);
-    }
-    const dlOptions = {
-        force: options["force"],
-        silent: options["silent"],
-        verbose: options["verbose"],
-        onlyRecent: options["only-recent"],
-        formats: options["formats"],
-        parseDocuments: options["parseDocuments"],
-    };
-    for (const session of sessions) {
-        for (const { item: texteMetadata } of iterLoadSenatTexteUrls(dataDir, session)) {
-            await processTexte(texteMetadata, originalTextesDir, transformedTextesDir, dlOptions);
-        }
-    }
-}
-async function retrieveRapports(dataDir, sessions) {
-    const originalRapportsDir = path.join(dataDir, RAPPORT_FOLDER, DATA_ORIGINAL_FOLDER);
-    const dlOptions = {
-        force: options["force"],
-        silent: options["silent"],
-        verbose: options["verbose"],
-        onlyRecent: options["only-recent"],
-        formats: options["formats"],
-    };
-    for (const session of sessions) {
-        for (const { item: rapportMetadata } of iterLoadSenatRapportUrls(dataDir, session)) {
-            await processRapport(rapportMetadata, originalRapportsDir, dlOptions);
-        }
-    }
-}
-async function parseDocument(session, transformedTextesDir, textePath, texteName, texteBuffer, exposeDesMotifs = null, options = {}) {
-    if (options.verbose) {
-        console.log(`Parsing texte ${textePath}…`);
-    }
-    let parsedTexte;
-    if (texteBuffer) {
-        const texteXml = textDecoder.decode(texteBuffer);
-        parsedTexte = parseTexte(texteXml);
-    }
-    else {
-        parsedTexte = await parseTexteFromFile(textePath);
-    }
-    if (!parsedTexte)
-        return null;
-    if (exposeDesMotifs) {
-        if (options.verbose) {
-            console.log("Parsing exposé des motifs…");
-        }
-        const exposeDesMotifsHtml = textDecoder.decode(exposeDesMotifs);
-        parsedTexte.exposeDesMotifs = parseExposeDesMotifs(exposeDesMotifsHtml);
-    }
-    const transformedTexteDir = path.join(transformedTextesDir, `${session ?? UNDEFINED_SESSION}`, texteName);
-    await fs.outputJSON(path.join(transformedTexteDir, `${texteName}.json`), parsedTexte, { spaces: 2 });
-    return parsedTexte;
-}
-async function main() {
-    const dataDir = options["dataDir"];
-    assert(dataDir, "Missing argument: data directory");
-    const sessions = getSessionsFromStart(options["fromSession"]);
-    console.time("documents processing time");
-    if (isOptionEmptyOrHasValue(options["types"], "textes")) {
-        await retrieveTextes(dataDir, sessions);
-    }
-    if (isOptionEmptyOrHasValue(options["types"], "rapports")) {
-        await retrieveRapports(dataDir, sessions);
-    }
-    if (!options["silent"]) {
-        console.timeEnd("documents processing time");
-    }
-}
-if (process.argv[1].endsWith("retrieve_documents.ts")) {
-    main()
-        .then(() => process.exit(0))
-        .catch((error) => {
-        console.log(error);
-        process.exit(1);
-    });
-}