@tricoteuses/senat 2.15.7 → 2.16.2

package/lib/raw_types/db.js

@@ -1,5 +0,0 @@
-/**
- * This file was generated by kysely-codegen.
- * Please do not edit it manually.
- */
-export {};

package/lib/scripts/retrieve_comptes_rendus.d.ts

@@ -1,6 +0,0 @@
-/**
- * Needs to be run after retrieve_agenda.ts !
- * - downloads the ZIP of comptes-rendus des débats (CRI) from data.senat.fr
- * - extracts XML files, distributes them by session/year
- */
-export declare function retrieveCriXmlDump(dataDir: string, options?: Record<string, any>): Promise<void>;

package/lib/scripts/retrieve_comptes_rendus.js

@@ -1,274 +0,0 @@
-/**
- * Needs to be run after retrieve_agenda.ts !
- * - downloads the ZIP of comptes-rendus des débats (CRI) from data.senat.fr
- * - extracts XML files, distributes them by session/year
- */
-import assert from "assert";
-import commandLineArgs from "command-line-args";
-import fs from "fs-extra";
-import path from "path";
-import StreamZip from "node-stream-zip";
-import * as cheerio from "cheerio";
-import { AGENDA_FOLDER, COMPTES_RENDUS_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, } from "../loaders";
-import { commonOptions } from "./shared/cli_helpers";
-import { deriveTitreObjetFromSommaire, parseCompteRenduSlotFromFile, parseYYYYMMDD, sessionStartYearFromDate } from "../model/compte_rendu";
-import { makeGroupUid } from "../utils/reunion_grouping";
-import { getSessionsFromStart } from "../types/sessions";
-import { ensureAndClearDir, fetchWithRetry } from "./shared/util";
-import { computeIntervalsBySlot } from "../utils/cr_spliting";
-const optionsDefinitions = [
-    ...commonOptions,
-    {
-        help: "parse and convert comptes-rendus des débats into JSON",
-        name: "parseDebats",
-        type: Boolean,
-    }
-];
-const options = commandLineArgs(optionsDefinitions);
-const CRI_ZIP_URL = "https://data.senat.fr/data/debats/cri.zip";
-const SLOT_ORDER = ["MATIN", "APRES-MIDI", "SOIR"];
-class CompteRenduError extends Error {
-    constructor(message, url) {
-        super(`An error occurred while retrieving ${url}: ${message}`);
-    }
-}
-function pickFirstSlotOfDay(slots) {
-    for (const s of SLOT_ORDER)
-        if (slots.includes(s))
-            return s;
-    return null;
-}
-function loadAgendaSPSlotsForDate(dataDir, yyyymmdd, session) {
-    const dirPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, session.toString());
-    if (!fs.existsSync(dirPath)) {
-        console.warn(`[AGENDA] Directory not found for session ${session} → ${dirPath}`);
-        return null;
-    }
-    const pattern = new RegExp(`^RUSN${yyyymmdd}IDS-(MATIN|APRES-MIDI|SOIR)\\.json$`);
-    const ALLOWED_SLOTS = new Set(["MATIN", "APRES-MIDI", "SOIR"]);
-    try {
-        const files = fs.readdirSync(dirPath);
-        const matched = files.filter((f) => pattern.test(f));
-        if (matched.length === 0) {
-            return null;
-        }
-        const found = new Set();
-        for (const name of matched) {
-            const m = name.match(pattern);
-            const raw = (m?.[1] ?? "");
-            if (ALLOWED_SLOTS.has(raw))
-                found.add(raw);
-        }
-        const slots = Array.from(found);
-        if (slots.length === 0) {
-            return null;
-        }
-        return { filePath: dirPath, slots };
-    }
-    catch {
-        return null;
-    }
-}
-async function downloadCriZip(zipPath) {
-    if (!options["silent"])
-        console.log(`Downloading CRI zip ${CRI_ZIP_URL}…`);
-    const response = await fetchWithRetry(CRI_ZIP_URL);
-    if (!response.ok) {
-        if (response.status === 404) {
-            console.warn(`CRI zip ${CRI_ZIP_URL} not found`);
-            return;
-        }
-        throw new CompteRenduError(String(response.status), CRI_ZIP_URL);
-    }
-    const buf = Buffer.from(await response.arrayBuffer());
-    await fs.writeFile(zipPath, buf);
-    if (!options["silent"]) {
-        const mb = (buf.length / (1024 * 1024)).toFixed(1);
-        console.log(`[CRI] Downloaded ${mb} MB → ${zipPath}`);
-    }
-}
-async function extractAndDistributeXmlBySession(zipPath, originalRoot) {
-    const zip = new StreamZip.async({ file: zipPath });
-    const entries = await zip.entries();
-    let count = 0;
-    for (const entryName of Object.keys(entries)) {
-        if (!entryName.toLowerCase().endsWith(".xml"))
-            continue;
-        // ex: d20231005.xml
-        const base = path.basename(entryName);
-        const m = base.match(/^d(\d{8})\.xml$/i);
-        if (!m)
-            continue;
-        const yyyymmdd = m[1];
-        const dt = parseYYYYMMDD(yyyymmdd);
-        if (!dt)
-            continue;
-        const session = sessionStartYearFromDate(dt);
-        const destDir = path.join(originalRoot, String(session));
-        await fs.ensureDir(destDir);
-        const outPath = path.join(destDir, base);
-        await zip.extract(entryName, outPath);
-        count++;
-    }
-    await zip.close();
-    return count;
-}
-export async function retrieveCriXmlDump(dataDir, options = {}) {
-    const root = path.join(dataDir, COMPTES_RENDUS_FOLDER);
-    ensureAndClearDir(root);
-    const originalRoot = path.join(root, DATA_ORIGINAL_FOLDER);
-    fs.ensureDirSync(originalRoot);
-    const transformedRoot = path.join(root, DATA_TRANSFORMED_FOLDER);
-    if (options["parseDebats"])
-        fs.ensureDirSync(transformedRoot);
-    const sessions = getSessionsFromStart(options["fromSession"]);
-    // 1) Download ZIP global + distribut by session
-    const zipPath = path.join(dataDir, "cri.zip");
-    console.log("[CRI] Downloading global CRI zip…");
-    await downloadCriZip(zipPath);
-    console.log("[CRI] Extracting + distributing XMLs by session…");
-    for (const session of sessions) {
-        const dir = path.join(originalRoot, String(session));
-        if (await fs.pathExists(dir)) {
-            for (const f of await fs.readdir(dir))
-                if (/\.xml$/i.test(f))
-                    await fs.remove(path.join(dir, f));
-        }
-    }
-    const n = await extractAndDistributeXmlBySession(zipPath, originalRoot);
-    if (n === 0) {
-        console.warn("[CRI] No XML extracted. Archive empty or layout changed?");
-    }
-    else {
-        console.log(`[CRI] Distributed ${n} XML file(s) into session folders.`);
-    }
-    if (!options["parseDebats"]) {
-        console.log("[CRI] parseDebats not requested → done.");
-        return;
-    }
-    for (const session of sessions) {
-        const originalSessionDir = path.join(originalRoot, String(session));
-        if (!(await fs.pathExists(originalSessionDir))) {
-            continue;
-        }
-        const xmlFiles = (await fs.readdir(originalSessionDir))
-            .filter((f) => /^d\d{8}\.xml$/i.test(f))
-            .sort();
-        const transformedSessionDir = path.join(transformedRoot, String(session));
-        if (options["parseDebats"])
-            await fs.ensureDir(transformedSessionDir);
-        for (const f of xmlFiles) {
-            const yyyymmdd = f.slice(1, 9);
-            const xmlPath = path.join(originalSessionDir, f);
-            // 1) Deduce slot(s) from agenda if it exsits
-            const agendaInfo = loadAgendaSPSlotsForDate(dataDir, yyyymmdd, session);
-            const firstSlotOfDay = pickFirstSlotOfDay(agendaInfo?.slots ?? []);
-            // 2) Detect slots from CRI content
-            let slotsInCri = [];
-            try {
-                const raw = await fs.readFile(xmlPath, "utf8");
-                const $ = cheerio.load(raw, { xml: false });
-                const order = $("body *").toArray();
-                const idx = new Map(order.map((el, i) => [el, i]));
-                const intervals = computeIntervalsBySlot($, idx, firstSlotOfDay ?? undefined);
-                const uniq = new Set();
-                for (const iv of intervals)
-                    if (iv.slot && iv.slot !== "UNKNOWN")
-                        uniq.add(iv.slot);
-                slotsInCri = Array.from(uniq);
-            }
-            catch (e) {
-                console.warn(`[CRI] [${session}] Cannot read/parse ${f}:`, e);
-                continue;
-            }
-            if (slotsInCri.length === 0) {
-                slotsInCri = [firstSlotOfDay ?? "MATIN"];
-            }
-            // 3) Parse & write each slot
-            for (const slot of slotsInCri) {
-                const outName = `CRSSN${yyyymmdd}-${slot}.json`;
-                const cr = await parseCompteRenduSlotFromFile(xmlPath, slot, firstSlotOfDay ?? slot);
-                if (!cr) {
-                    console.warn(`[CRI] [${session}] Empty or no points for ${yyyymmdd} (${slot}) → skip`);
-                    continue;
-                }
-                const outDir = transformedSessionDir;
-                await fs.ensureDir(outDir);
-                const outPath = path.join(outDir, outName);
-                await fs.writeJSON(outPath, cr, { spaces: 2 });
-                try {
-                    await linkCriSlotIntoAgendaGrouped(dataDir, yyyymmdd, slot, cr.uid, cr, session);
-                }
-                catch (e) {
-                    console.warn(`[AGENDA] [${session}] Could not link CR into grouped for ${yyyymmdd} ${slot}:`, e);
-                }
-            }
-        }
-    }
-}
-async function main() {
-    const dataDir = options["dataDir"];
-    assert(dataDir, "Missing argument: data directory");
-    console.time("CRI processing time");
-    await retrieveCriXmlDump(dataDir, options);
-    if (!options["silent"]) {
-        console.timeEnd("CRI processing time");
-    }
-}
-main()
-    .then(() => process.exit(0))
-    .catch((error) => {
-    console.error(error);
-    process.exit(1);
-});
-async function linkCriSlotIntoAgendaGrouped(dataDir, yyyymmdd, slot, crUid, cr, session) {
-    const groupedDir = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, session.toString());
-    fs.ensureDirSync(groupedDir);
-    const groupedPath = path.join(groupedDir, 'RUSN' + yyyymmdd + 'IDS-' + slot + '.json');
-    let groups = [];
-    if (fs.existsSync(groupedPath)) {
-        try {
-            groups = JSON.parse(fs.readFileSync(groupedPath, "utf8"));
-            if (!Array.isArray(groups))
-                groups = [];
-        }
-        catch (e) {
-            console.warn(`[AGENDA] unreadable grouped JSON → ${groupedPath} (${e}) → recreating`);
-            groups = [];
-        }
-    }
-    // find existing group with same slot
-    const sameSlot = groups.filter(g => g?.slot === slot);
-    let target = null;
-    if (sameSlot.length > 1) {
-        console.warn(`[AGENDA] multiple groups for ${yyyymmdd} ${slot} in ${groupedPath} → linking the first`);
-    }
-    target = sameSlot[0] ?? null;
-    const dateISO = `${yyyymmdd.slice(0, 4)}-${yyyymmdd.slice(4, 6)}-${yyyymmdd.slice(6, 8)}`;
-    const sommaire = cr?.metadonnees?.sommaire;
-    const { titre: dTitre, objet: dObjet } = deriveTitreObjetFromSommaire(sommaire, slot);
-    if (!target) {
-        const newGroup = {
-            uid: makeGroupUid(dateISO, slot),
-            chambre: "SN",
-            date: dateISO,
-            slot,
-            type: "Séance publique",
-            startTime: null,
-            endTime: null,
-            captationVideo: false,
-            titre: dTitre,
-            objet: dObjet || "",
-            events: [],
-            compteRenduRefUid: crUid,
-        };
-        groups.push(newGroup);
-    }
-    else {
-        target.compteRenduRefUid = crUid;
-    }
-    await fs.writeJSON(groupedPath, groups, { spaces: 2 });
-    if (!options["silent"]) {
-        console.log(`[AGENDA] Linked CR ${crUid} → ${path.basename(groupedPath)} [${slot}]`);
-    }
-}