@tricoteuses/senat 2.10.1 → 2.10.2

package/lib/loaders.js

@@ -211,10 +211,10 @@ export function* iterLoadSenatAgendasGrouped(dataDir, session) {
             if (!g || typeof g !== "object")
                 continue;
             const gr = g;
-            if (!gr.date || !gr.slot)
+            if (!gr.date || !gr.uid || !gr.titre)
                 continue;
-            if (!Array.isArray(gr.reunions))
-                gr.reunions = [];
+            if (!Array.isArray(gr.events))
+                gr.events = [];
             yield { item: gr };
         }
     }

package/lib/scripts/retrieve_comptes_rendus.js

@@ -259,7 +259,7 @@ async function linkCriSlotIntoAgendaGrouped(dataDir, yyyymmdd, slot, crUid, cr,
             captationVideo: false,
             titre: dTitre,
             objet: dObjet || "",
-            reunions: [],
+            events: [],
             compteRenduRefUid: crUid,
         };
         groups.push(newGroup);

package/lib/scripts/retrieve_comptes_rendus_seance.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Needs to be run after retrieve_agenda.ts !
+ * - downloads the ZIP of comptes-rendus des débats (CRI) from data.senat.fr
+ * - extracts XML files, distributes them by session/year
+ */
+export {};

package/lib/scripts/retrieve_comptes_rendus_seance.js

@@ -0,0 +1,273 @@
+/**
+ * Needs to be run after retrieve_agenda.ts !
+ * - downloads the ZIP of comptes-rendus des débats (CRI) from data.senat.fr
+ * - extracts XML files, distributes them by session/year
+ */
+import assert from "assert";
+import commandLineArgs from "command-line-args";
+import fs from "fs-extra";
+import path from "path";
+import StreamZip from "node-stream-zip";
+import * as cheerio from "cheerio";
+import { AGENDA_FOLDER, COMPTES_RENDUS_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, } from "../loaders";
+import { commonOptions } from "./shared/cli_helpers";
+import { deriveTitreObjetFromSommaire, parseCompteRenduSlotFromFile, parseYYYYMMDD, sessionStartYearFromDate } from "../model/compte_rendu";
+import { makeGroupUid } from "../utils/reunion_grouping";
+import { getSessionsFromStart } from "../types/sessions";
+import { ensureAndClearDir, fetchWithRetry } from "./shared/util";
+import { computeIntervalsBySlot } from "../utils/cr_spliting";
+const optionsDefinitions = [
+    ...commonOptions,
+    {
+        help: "parse and convert comptes-rendus des débats into JSON",
+        name: "parseDebats",
+        type: Boolean,
+    }
+];
+const options = commandLineArgs(optionsDefinitions);
+const CRI_ZIP_URL = "https://data.senat.fr/data/debats/cri.zip";
+const SLOT_ORDER = ["MATIN", "APRES-MIDI", "SOIR"];
+class CompteRenduError extends Error {
+    constructor(message, url) {
+        super(`An error occurred while retrieving ${url}: ${message}`);
+    }
+}
+function pickFirstSlotOfDay(slots) {
+    for (const s of SLOT_ORDER)
+        if (slots.includes(s))
+            return s;
+    return null;
+}
+function loadAgendaSPSlotsForDate(dataDir, yyyymmdd, session) {
+    const dirPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, session.toString());
+    if (!fs.existsSync(dirPath)) {
+        console.warn(`[AGENDA] Directory not found for session ${session} → ${dirPath}`);
+        return null;
+    }
+    const pattern = new RegExp(`^RUSN${yyyymmdd}IDS-(MATIN|APRES-MIDI|SOIR)\\.json$`);
+    const ALLOWED_SLOTS = new Set(["MATIN", "APRES-MIDI", "SOIR"]);
+    try {
+        const files = fs.readdirSync(dirPath);
+        const matched = files.filter((f) => pattern.test(f));
+        if (matched.length === 0) {
+            return null;
+        }
+        const found = new Set();
+        for (const name of matched) {
+            const m = name.match(pattern);
+            const raw = (m?.[1] ?? "");
+            if (ALLOWED_SLOTS.has(raw))
+                found.add(raw);
+        }
+        const slots = Array.from(found);
+        if (slots.length === 0) {
+            return null;
+        }
+        return { filePath: dirPath, slots };
+    }
+    catch {
+        return null;
+    }
+}
+async function downloadCriZip(zipPath) {
+    if (!options["silent"])
+        console.log(`Downloading CRI zip ${CRI_ZIP_URL}…`);
+    const response = await fetchWithRetry(CRI_ZIP_URL);
+    if (!response.ok) {
+        if (response.status === 404) {
+            console.warn(`CRI zip ${CRI_ZIP_URL} not found`);
+            return;
+        }
+        throw new CompteRenduError(String(response.status), CRI_ZIP_URL);
+    }
+    const buf = Buffer.from(await response.arrayBuffer());
+    await fs.writeFile(zipPath, buf);
+    if (!options["silent"]) {
+        const mb = (buf.length / (1024 * 1024)).toFixed(1);
+        console.log(`[CRI] Downloaded ${mb} MB → ${zipPath}`);
+    }
+}
+async function extractAndDistributeXmlBySession(zipPath, originalRoot) {
+    const zip = new StreamZip.async({ file: zipPath });
+    const entries = await zip.entries();
+    let count = 0;
+    for (const entryName of Object.keys(entries)) {
+        if (!entryName.toLowerCase().endsWith(".xml"))
+            continue;
+        // ex: d20231005.xml
+        const base = path.basename(entryName);
+        const m = base.match(/^d(\d{8})\.xml$/i);
+        if (!m)
+            continue;
+        const yyyymmdd = m[1];
+        const dt = parseYYYYMMDD(yyyymmdd);
+        if (!dt)
+            continue;
+        const session = sessionStartYearFromDate(dt);
+        const destDir = path.join(originalRoot, String(session));
+        await fs.ensureDir(destDir);
+        const outPath = path.join(destDir, base);
+        await zip.extract(entryName, outPath);
+        count++;
+    }
+    await zip.close();
+    return count;
+}
+async function linkCriSlotIntoAgendaGrouped(dataDir, yyyymmdd, slot, crUid, cr, session) {
+    const groupedDir = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, session.toString());
+    fs.ensureDirSync(groupedDir);
+    const groupedPath = path.join(groupedDir, 'RUSN' + yyyymmdd + 'IDS-' + slot + '.json');
+    let groups = [];
+    if (fs.existsSync(groupedPath)) {
+        try {
+            groups = JSON.parse(fs.readFileSync(groupedPath, "utf8"));
+            if (!Array.isArray(groups))
+                groups = [];
+        }
+        catch (e) {
+            console.warn(`[AGENDA] unreadable grouped JSON → ${groupedPath} (${e}) → recreating`);
+            groups = [];
+        }
+    }
+    // find existing group with same slot
+    const sameSlot = groups.filter(g => g?.slot === slot);
+    let target = null;
+    if (sameSlot.length > 1) {
+        console.warn(`[AGENDA] multiple groups for ${yyyymmdd} ${slot} in ${groupedPath} → linking the first`);
+    }
+    target = sameSlot[0] ?? null;
+    const dateISO = `${yyyymmdd.slice(0, 4)}-${yyyymmdd.slice(4, 6)}-${yyyymmdd.slice(6, 8)}`;
+    const sommaire = cr?.metadonnees?.sommaire;
+    const { titre: dTitre, objet: dObjet } = deriveTitreObjetFromSommaire(sommaire, slot);
+    if (!target) {
+        const newGroup = {
+            uid: makeGroupUid(dateISO, slot),
+            chambre: "SN",
+            date: dateISO,
+            slot,
+            type: "Séance publique",
+            startTime: null,
+            endTime: null,
+            captationVideo: false,
+            titre: dTitre,
+            objet: dObjet || "",
+            events: [],
+            compteRenduRefUid: crUid,
+        };
+        groups.push(newGroup);
+    }
+    else {
+        target.compteRenduRefUid = crUid;
+    }
+    await fs.writeJSON(groupedPath, groups, { spaces: 2 });
+    console.log(`[AGENDA] Linked CR ${crUid} → ${path.basename(groupedPath)} [${slot}]`);
+}
+async function retrieveCriXmlDump(dataDir, options = {}) {
+    const root = path.join(dataDir, COMPTES_RENDUS_FOLDER);
+    ensureAndClearDir(root);
+    const originalRoot = path.join(root, DATA_ORIGINAL_FOLDER);
+    fs.ensureDirSync(originalRoot);
+    const transformedRoot = path.join(root, DATA_TRANSFORMED_FOLDER);
+    if (options["parseDebats"])
+        fs.ensureDirSync(transformedRoot);
+    const sessions = getSessionsFromStart(options["fromSession"]);
+    // 1) Download ZIP global + distribut by session
+    const zipPath = path.join(dataDir, "cri.zip");
+    console.log("[CRI] Downloading global CRI zip…");
+    await downloadCriZip(zipPath);
+    console.log("[CRI] Extracting + distributing XMLs by session…");
+    for (const session of sessions) {
+        const dir = path.join(originalRoot, String(session));
+        if (await fs.pathExists(dir)) {
+            for (const f of await fs.readdir(dir))
+                if (/\.xml$/i.test(f))
+                    await fs.remove(path.join(dir, f));
+        }
+    }
+    const n = await extractAndDistributeXmlBySession(zipPath, originalRoot);
+    if (n === 0) {
+        console.warn("[CRI] No XML extracted. Archive empty or layout changed?");
+    }
+    else {
+        console.log(`[CRI] Distributed ${n} XML file(s) into session folders.`);
+    }
+    if (!options["parseDebats"]) {
+        console.log("[CRI] parseDebats not requested → done.");
+        return;
+    }
+    for (const session of sessions) {
+        const originalSessionDir = path.join(originalRoot, String(session));
+        if (!(await fs.pathExists(originalSessionDir))) {
+            continue;
+        }
+        const xmlFiles = (await fs.readdir(originalSessionDir))
+            .filter((f) => /^d\d{8}\.xml$/i.test(f))
+            .sort();
+        const transformedSessionDir = path.join(transformedRoot, String(session));
+        if (options["parseDebats"])
+            await fs.ensureDir(transformedSessionDir);
+        for (const f of xmlFiles) {
+            const yyyymmdd = f.slice(1, 9);
+            const xmlPath = path.join(originalSessionDir, f);
+            // 1) Deduce slot(s) from agenda if it exsits
+            // TODO: A updater car maintenant les agendas sont déjà enregistrés par slot et plus à la journée seulement
+            const agendaInfo = loadAgendaSPSlotsForDate(dataDir, yyyymmdd, session);
+            const firstSlotOfDay = pickFirstSlotOfDay(agendaInfo?.slots ?? []);
+            // 2) Detect slots from CRI content
+            let slotsInCri = [];
+            try {
+                const raw = await fs.readFile(xmlPath, "utf8");
+                const $ = cheerio.load(raw, { xml: false });
+                const order = $("body *").toArray();
+                const idx = new Map(order.map((el, i) => [el, i]));
+                const intervals = computeIntervalsBySlot($, idx, firstSlotOfDay ?? undefined);
+                const uniq = new Set();
+                for (const iv of intervals)
+                    if (iv.slot && iv.slot !== "UNKNOWN")
+                        uniq.add(iv.slot);
+                slotsInCri = Array.from(uniq);
+            }
+            catch (e) {
+                console.warn(`[CRI] [${session}] Cannot read/parse ${f}:`, e);
+                continue;
+            }
+            if (slotsInCri.length === 0) {
+                slotsInCri = [firstSlotOfDay ?? "MATIN"];
+            }
+            // 3) Parse & write each slot
+            for (const slot of slotsInCri) {
+                const outName = `CRSSN${yyyymmdd}-${slot}.json`;
+                const cr = await parseCompteRenduSlotFromFile(xmlPath, slot, firstSlotOfDay ?? slot);
+                if (!cr) {
+                    console.warn(`[CRI] [${session}] Empty or no points for ${yyyymmdd} (${slot}) → skip`);
+                    continue;
+                }
+                const outDir = transformedSessionDir;
+                await fs.ensureDir(outDir);
+                const outPath = path.join(outDir, outName);
+                await fs.writeJSON(outPath, cr, { spaces: 2 });
+                try {
+                    await linkCriSlotIntoAgendaGrouped(dataDir, yyyymmdd, slot, cr.uid, cr, session);
+                }
+                catch (e) {
+                    console.warn(`[AGENDA] [${session}] Could not link CR into grouped for ${yyyymmdd} ${slot}:`, e);
+                }
+            }
+        }
+    }
+}
+async function main() {
+    const dataDir = options["dataDir"];
+    assert(dataDir, "Missing argument: data directory");
+    console.time("CRI processing time");
+    await retrieveCriXmlDump(dataDir, options);
+    if (!options["silent"]) {
+        console.timeEnd("CRI processing time");
+    }
+}
+main()
+    .then(() => process.exit(0))
+    .catch((error) => {
+    console.error(error);
+    process.exit(1);
+});

package/lib/types/agenda.d.ts

@@ -27,7 +27,7 @@ export interface GroupedReunion {
     organe?: string;
     objet?: string;
     lieu?: string;
-    reunions: AgendaEvent[];
+    events: AgendaEvent[];
     compteRenduRefUid?: string;
     urlVideo?: string;
     timecodeDebutVideo?: number;

package/lib/utils/reunion_grouping.js

@@ -64,7 +64,7 @@ export function groupNonSPByTypeOrganeHour(events) {
             captationVideo: enriched.some(x => x.ev.captationVideo === true),
             titre: compactTitleList(enriched.map(x => x.ev.titre || "").filter(Boolean), 8),
             objet: joinObjets(enriched.map(x => x.ev)),
-            reunions: enriched.map(x => x.ev),
+            events: enriched.map(x => x.ev),
         };
         out[suffix].push(group);
     }
@@ -142,7 +142,7 @@ export function groupSeancePubliqueBySlot(events) {
                 captationVideo,
                 titre: compactTitleList(titres, 5),
                 objet: joinObjets(sorted.map((x) => x.ev)),
-                reunions: sorted.map((x) => x.ev),
+                events: sorted.map((x) => x.ev),
             });
         }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.10.1",
+  "version": "2.10.2",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",