@tricoteuses/senat 2.11.0 → 2.11.2

package/lib/scripts/retrieve_cr_commission.js

@@ -0,0 +1,291 @@
+import fs from "fs-extra";
+import assert from "assert";
+import path from "path";
+import * as cheerio from "cheerio";
+import { COMMISSION_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
+import { createCommissionGroupIfMissing, loadCommissionAgendaForDate, parseCommissionMetadataFromHtml, } from "../utils/cr_spliting";
+import { parseCommissionCRFromFile } from "../model/commission";
+import commandLineArgs from "command-line-args";
+import { commonOptions } from "./shared/cli_helpers";
+import { sessionStartYearFromDate } from "../model/seance";
+import { getSessionsFromStart } from "../types/sessions";
+import { ensureAndClearDir, fetchWithRetry } from "./shared/util";
+class CommissionCRDownloadError extends Error {
+    constructor(message, url) {
+        super(`An error occurred while retrieving Commission CR ${url}: ${message}`);
+    }
+}
+const optionsDefinitions = [
+    ...commonOptions,
+    { name: "concurrency", type: Number, defaultValue: 6, help: "Max parallel downloads" },
+    { name: "politenessMs", type: Number, defaultValue: 150, help: "Delay per worker (ms)" },
+    {
+        help: "parse and convert comptes-rendus des débats into JSON",
+        name: "parseDebats",
+        type: Boolean,
+    },
+];
+const options = commandLineArgs(optionsDefinitions);
+const COMMISSION_HUBS = {
+    "affaires-etrangeres": [
+        "https://www.senat.fr/compte-rendu-commissions/affaires-etrangeres.html",
+        "https://www.senat.fr/compte-rendu-commissions/affaires-etrangeres_archives.html",
+    ],
+    "affaires-economiques": [
+        "https://www.senat.fr/compte-rendu-commissions/affaires-economiques.html",
+        "https://www.senat.fr/compte-rendu-commissions/affaires-economiques_archives.html",
+    ],
+    "amenagement-developpement-durable": [
+        "https://www.senat.fr/compte-rendu-commissions/cadre-de-vie-et-developpement-durable.html",
+        "https://www.senat.fr/compte-rendu-commissions/cadre-de-vie-et-developpement-durable_archives.html",
+    ],
+    culture: [
+        "https://www.senat.fr/compte-rendu-commissions/culture.html",
+        "https://www.senat.fr/compte-rendu-commissions/culture_archives.html",
+    ],
+    finances: [
+        "https://www.senat.fr/compte-rendu-commissions/finances.html",
+        "https://www.senat.fr/compte-rendu-commissions/finances_archives.html",
+    ],
+    lois: [
+        "https://www.senat.fr/compte-rendu-commissions/lois.html",
+        "https://www.senat.fr/compte-rendu-commissions/lois_archives.html",
+    ],
+    "affaires-sociales": [
+        "https://www.senat.fr/compte-rendu-commissions/affaires-sociales.html",
+        "https://www.senat.fr/compte-rendu-commissions/affaires-sociales_archives.html",
+    ],
+    "affaires-europeennes": [
+        "https://www.senat.fr/compte-rendu-commissions/affaires-europeennes.html",
+        "https://www.senat.fr/compte-rendu-commissions/affaires-europeennes_archives.html",
+    ],
+};
+async function harvestWeeklyLinksFromHub(hubUrl) {
+    const res = await fetchWithRetry(hubUrl);
+    if (!res.ok)
+        return [];
+    const html = await res.text();
+    const $ = cheerio.load(html);
+    const out = [];
+    $("a[href]").each((_, a) => {
+        const href = ($(a).attr("href") || "").trim();
+        const m = href.match(/\/compte-rendu-commissions\/(\d{8})\/([a-z0-9\-]+)\.html$/i);
+        if (m) {
+            const url = href.startsWith("http") ? href : new URL(href, hubUrl).toString();
+            out.push(url);
+        }
+    });
+    return Array.from(new Set(out));
+}
+async function discoverCommissionWeeklyPages(fromSession) {
+    const results = [];
+    for (const [commissionKey, hubs] of Object.entries(COMMISSION_HUBS)) {
+        for (const hubUrl of hubs) {
+            try {
+                const links = await harvestWeeklyLinksFromHub(hubUrl);
+                for (const url of links) {
+                    const m = url.match(/\/compte-rendu-commissions\/(\d{8})\/([a-z0-9\-]+)\.html$/i);
+                    if (!m)
+                        continue;
+                    const yyyymmdd = m[1];
+                    const year = Number(yyyymmdd.slice(0, 4));
+                    const month = Number(yyyymmdd.slice(4, 6));
+                    const session = month >= 10 ? year : year - 1;
+                    if (session < fromSession)
+                        continue;
+                    results.push({ url, yyyymmdd, commissionKey });
+                }
+            }
+            catch (e) {
+                console.warn(`[COM-CR][hub-fail] ${hubUrl} → ${e?.message ?? e}`);
+            }
+        }
+    }
+    return results.sort((a, b) => a.yyyymmdd.localeCompare(b.yyyymmdd));
+}
+function toHourShort(hhmm) {
+    if (!hhmm)
+        return null;
+    const m = hhmm.match(/^(\d{2}):(\d{2})$/);
+    return m ? `${m[1]}${m[2]}` : null;
+}
+function timeToMinutes(hhmm) {
+    const [h, m] = hhmm.split(":").map((n) => parseInt(n, 10));
+    return (h || 0) * 60 + (m || 0);
+}
+async function tryDownload(url) {
+    const res = await fetch(url, { redirect: "follow" });
+    if (res.status === 404)
+        return null;
+    if (!res.ok)
+        throw new CommissionCRDownloadError(String(res.status), url);
+    const ab = await res.arrayBuffer();
+    return Buffer.from(ab);
+}
+async function retrieveCommissionCRs(options = {}) {
+    const dataDir = options["dataDir"];
+    const fromSession = Number(options["fromSession"]);
+    const concurrency = Number(options["concurrency"] ?? 6);
+    const politenessMs = Number(options["politenessMs"] ?? 150);
+    const commissionsRootDir = path.join(dataDir, COMMISSION_FOLDER);
+    const originalRoot = path.join(commissionsRootDir, DATA_ORIGINAL_FOLDER);
+    ensureAndClearDir(originalRoot);
+    const discovered = await discoverCommissionWeeklyPages(fromSession);
+    console.log(`[COM-CR][discover] ${discovered.length} links (>= session ${fromSession})`);
+    const jobs = discovered.map(({ url, yyyymmdd }) => {
+        const d = new Date(Number(yyyymmdd.slice(0, 4)), Number(yyyymmdd.slice(4, 6)) - 1, Number(yyyymmdd.slice(6, 8)));
+        const session = sessionStartYearFromDate(d);
+        const dir = path.join(originalRoot, String(session));
+        fs.ensureDirSync(dir);
+        const slug = url.replace(/^.*\/(\d{8})\/([^\/]+)\.html$/i, "$2");
+        const outPath = path.join(dir, `${yyyymmdd}.${slug}.html`);
+        return { url, outPath, yyyymmdd };
+    });
+    console.log(`[COM-CR] Downloading ${jobs.length} links → ${path.relative(process.cwd(), originalRoot)}`);
+    let completed = 0, saved = 0, skipped = 0, notFound = 0;
+    const workers = Array.from({ length: Math.max(1, concurrency) }, async () => {
+        while (true) {
+            const job = jobs.shift();
+            if (!job)
+                break;
+            const { url, outPath, yyyymmdd } = job;
+            try {
+                if (await fs.pathExists(outPath)) {
+                    skipped++;
+                }
+                else {
+                    const buf = await tryDownload(url);
+                    if (!buf) {
+                        notFound++;
+                        console.warn(`[COM-CR][404] ${url} → week=${yyyymmdd}`);
+                    }
+                    else {
+                        await fs.writeFile(outPath, buf);
+                        saved++;
+                    }
+                }
+            }
+            catch (e) {
+                console.error(`[COM-CR][err] ${url} → ${e?.message || e}`);
+            }
+            finally {
+                completed++;
+                if (politenessMs > 0)
+                    await new Promise((r) => setTimeout(r, politenessMs));
+            }
+        }
+    });
+    await Promise.all(workers);
+    console.log(`[COM-CR] done: saved=${saved} | skipped=${skipped} | 404=${notFound} | total=${completed}`);
+    const sessions = getSessionsFromStart(options["fromSession"]);
+    const comRoot = path.join(dataDir, COMMISSION_FOLDER);
+    const transformedRoot = path.join(comRoot, DATA_TRANSFORMED_FOLDER);
+    if (options["parseDebats"])
+        ensureAndClearDir(transformedRoot);
+    for (const session of sessions) {
+        const originalSessionDir = path.join(originalRoot, String(session));
+        const transformedSessionDir = path.join(transformedRoot, String(session));
+        fs.ensureDirSync(transformedSessionDir);
+        if (!(await fs.pathExists(originalSessionDir)))
+            continue;
+        const htmlFiles = (await fs.readdir(originalSessionDir)).filter((f) => /\.html?$/i.test(f)).sort();
+        for (const f of htmlFiles) {
+            const htmlPath = path.join(originalSessionDir, f);
+            let meta;
+            try {
+                const raw = await fs.readFile(htmlPath, "utf8");
+                meta = parseCommissionMetadataFromHtml(raw, f);
+            }
+            catch (e) {
+                console.warn(`[COM-CR][PRE][${session}] Cannot read/parse ${f}:`, e);
+                continue;
+            }
+            const organeKeywords = (meta.organeDetected ?? meta.organeTitleRaw ?? "")
+                .toLowerCase()
+                .replace(/[’']/g, "'")
+                .split(/\W+/)
+                .filter((x) => x.length >= 3 && !["commission", "des", "de", "du", "d", "la", "le", "les", "et"].includes(x));
+            const MAX_TIME_DELTA_MIN = 120;
+            for (let i = 0; i < meta.days.length; i++) {
+                const day = meta.days[i];
+                const yyyymmdd = day.date.replace(/-/g, "");
+                const dt = new Date(Number(day.date.slice(0, 4)), Number(day.date.slice(5, 7)) - 1, Number(day.date.slice(8, 10)));
+                const daySession = sessionStartYearFromDate(dt);
+                const hits = await loadCommissionAgendaForDate(dataDir, yyyymmdd, daySession);
+                let best = null;
+                let reason = "fallback-none";
+                let deltaMin;
+                // a) score by title and organe keywords
+                if (organeKeywords.length && hits.length) {
+                    const scored = hits
+                        .map((h) => {
+                        const t = (h.titre ?? "").toLowerCase();
+                        const s = organeKeywords.reduce((acc, kw) => acc + (t.includes(kw) ? 1 : 0), 0);
+                        return { h, s };
+                    })
+                        .sort((a, b) => b.s - a.s);
+                    if (scored[0]?.s > 0) {
+                        best = scored[0].h;
+                        reason = "title";
+                    }
+                }
+                // b) otherwise score by time proximity
+                if (!best && day.openTime && hits.length) {
+                    const candidates = hits
+                        .map((h) => ({ h, hhmm: h.startTime ?? null }))
+                        .filter((x) => !!x.hhmm)
+                        .map((x) => ({
+                        h: x.h,
+                        d: Math.abs(timeToMinutes(x.hhmm) - timeToMinutes(day.openTime)),
+                    }))
+                        .sort((a, b) => a.d - b.d);
+                    if (candidates[0] && candidates[0].d <= MAX_TIME_DELTA_MIN) {
+                        best = candidates[0].h;
+                        reason = "time";
+                        deltaMin = candidates[0].d;
+                    }
+                }
+                if (best) {
+                    const cr = parseCommissionCRFromFile(htmlPath, best);
+                    if (!cr) {
+                        console.warn(`[COM-CR][TRANSFORM] parse failed for ${f} → ${best.uid}`);
+                    }
+                    else {
+                        const fileUid = cr.uid;
+                        const outPath = path.join(transformedSessionDir, `${fileUid}.json`);
+                        await fs.writeJSON(outPath, cr, { spaces: 2 });
+                        const npts = Array.isArray(cr.contenu.point) ? cr.contenu.point.length : cr.contenu.point ? 1 : 0;
+                        if (!options["silent"]) {
+                            console.log(`[COM-CR][TRANSFORM] saved ${path.basename(outPath)} (points=${npts})`);
+                        }
+                    }
+                }
+                else {
+                    const hourShort = toHourShort(day.openTime) ?? "NA";
+                    const titreGuess = meta.organeDetected || meta.organeTitleRaw || "Commission";
+                    const { uid, filePath } = await createCommissionGroupIfMissing(dataDir, day.date, meta.organeDetected ?? null, hourShort, titreGuess);
+                    if (!options["silent"]) {
+                        console.log(`[COM-CR][PRE-SPLIT][${session}] ${f} | ${day.date}` +
+                            (day.openTime ? ` ${day.openTime}` : ``) +
+                            ` → NO-MATCH → CREATED uid=${uid} file=${path.basename(filePath)}`);
+                    }
+                }
+            }
+        }
+    }
+}
+async function main() {
+    const dataDir = options["dataDir"];
+    assert(dataDir, "Missing argument: data directory");
+    console.time("CRI processing time");
+    await retrieveCommissionCRs(options);
+    if (!options["silent"]) {
+        console.timeEnd("CRI processing time");
+    }
+}
+main()
+    .then(() => process.exit(0))
+    .catch((error) => {
+    console.error(error);
+    process.exit(1);
+});

package/lib/scripts/{retrieve_comptes_rendus.js → retrieve_cr_seance.js}

@@ -11,7 +11,7 @@ import StreamZip from "node-stream-zip";
 import * as cheerio from "cheerio";
 import { AGENDA_FOLDER, COMPTES_RENDUS_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, } from "../loaders";
 import { commonOptions } from "./shared/cli_helpers";
-import { deriveTitreObjetFromSommaire, parseCompteRenduSlotFromFile, parseYYYYMMDD, sessionStartYearFromDate } from "../model/compte_rendu";
+import { deriveTitreObjetFromSommaire, parseCompteRenduSlotFromFile, parseYYYYMMDD, sessionStartYearFromDate } from "../model/seance";
 import { makeGroupUid } from "../utils/reunion_grouping";
 import { getSessionsFromStart } from "../types/sessions";
 import { ensureAndClearDir, fetchWithRetry } from "./shared/util";

package/lib/utils/cr_spliting.d.ts

@@ -1,7 +1,28 @@
-import { TimeSlot } from "../types/agenda";
+import { GroupedReunion, TimeSlot } from "../types/agenda";
 import * as cheerio from "cheerio";
 export declare function computeIntervalsBySlot($: cheerio.CheerioAPI, idx: Map<any, number>, firstSlotOfDay?: TimeSlot): {
     slot: TimeSlot;
     start: number;
     end: number;
 }[];
+export declare function parseCommissionMetadataFromHtml(html: string, sourceFileName?: string): {
+    sourceFile: string | null;
+    organeTitleRaw: string | null;
+    organeDetected: string | null;
+    organeCode: string | null;
+    weekStart: string | null;
+    days: {
+        date: string;
+        openTime?: string;
+        h2Index: number;
+    }[];
+};
+export declare function loadCommissionAgendaForDate(dataDir: string, yyyymmdd: string, session: number): Promise<GroupedReunion[]>;
+export declare function createCommissionGroupIfMissing(dataDir: string, dateISO: string, // "YYYY-MM-DD"
+organeDetected: string | null, // ex. "Commission des finances"
+hourShort: string | null, // "HHMM" | "NA"
+titreGuess?: string | null): Promise<{
+    uid: string;
+    filePath: string;
+    created: boolean;
+}>;

package/lib/utils/cr_spliting.js

@@ -1,7 +1,13 @@
+import path from "path";
+import * as cheerio from "cheerio";
+import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
+import fs from "fs-extra";
+import { makeTypeGroupUid } from "./reunion_grouping";
+import { sessionStartYearFromDate } from "../model/seance";
 export function computeIntervalsBySlot($, idx, firstSlotOfDay) {
     const all = $("body *").toArray();
     const cuts = [{ pos: 0, hhmm: undefined }];
-    $('a[name]').each((_, a) => {
+    $("a[name]").each((_, a) => {
         const name = (a.attribs?.["name"] || "").trim();
         if (!/^su/i.test(name))
             return;
@@ -30,7 +36,7 @@ export function computeIntervalsBySlot($, idx, firstSlotOfDay) {
             continue;
         // i=0  initialSlot
         // i>0 : if current cut has SU -> slotOfHHMM, otherwise lastSlot
-        const slot = i === 0 ? initialSlot : (cuts[i].hhmm ? slotOfHHMM(cuts[i].hhmm) : lastSlot);
+        const slot = i === 0 ? initialSlot : cuts[i].hhmm ? slotOfHHMM(cuts[i].hhmm) : lastSlot;
         intervals.push({ slot, start, end });
         lastSlot = slot;
     }
@@ -70,7 +76,11 @@ function extractOpeningHHMM($) {
 }
 // Convert "quinze heures trente", "15 heures 30", "dix-sept heures moins le quart", etc. en "HHMM"
 function parseFrenchClockToHHMM(input) {
-    const s = (input || "").toLowerCase().normalize("NFKD").replace(/[\u0300-\u036f]/g, "").trim();
+    const s = (input || "")
+        .toLowerCase()
+        .normalize("NFKD")
+        .replace(/[\u0300-\u036f]/g, "")
+        .trim();
     if (!s)
         return undefined;
     const digitMatch = s.match(/(\d{1,2})\s*heures?(?:\s*(\d{1,2}))?/);
@@ -80,12 +90,41 @@ function parseFrenchClockToHHMM(input) {
         return `${String(h).padStart(2, "0")}${String(m).padStart(2, "0")}`;
     }
     const NUM = new Map([
-        ["zero", 0], ["une", 1], ["un", 1], ["deux", 2], ["trois", 3], ["quatre", 4], ["cinq", 5], ["six", 6],
-        ["sept", 7], ["huit", 8], ["neuf", 9], ["dix", 10], ["onze", 11], ["douze", 12], ["treize", 13],
-        ["quatorze", 14], ["quinze", 15], ["seize", 16], ["dix-sept", 17], ["dix sept", 17], ["dix-huit", 18],
-        ["dix huit", 18], ["dix-neuf", 19], ["dix neuf", 19], ["vingt", 20], ["vingt et une", 21],
-        ["vingt-et-une", 21], ["vingt et un", 21], ["vingt-et-un", 21], ["vingt-deux", 22], ["vingt deux", 22],
-        ["vingt-trois", 23], ["vingt trois", 23], ["vingt-quatre", 24], ["vingt quatre", 24],
+        ["zero", 0],
+        ["une", 1],
+        ["un", 1],
+        ["deux", 2],
+        ["trois", 3],
+        ["quatre", 4],
+        ["cinq", 5],
+        ["six", 6],
+        ["sept", 7],
+        ["huit", 8],
+        ["neuf", 9],
+        ["dix", 10],
+        ["onze", 11],
+        ["douze", 12],
+        ["treize", 13],
+        ["quatorze", 14],
+        ["quinze", 15],
+        ["seize", 16],
+        ["dix-sept", 17],
+        ["dix sept", 17],
+        ["dix-huit", 18],
+        ["dix huit", 18],
+        ["dix-neuf", 19],
+        ["dix neuf", 19],
+        ["vingt", 20],
+        ["vingt et une", 21],
+        ["vingt-et-une", 21],
+        ["vingt et un", 21],
+        ["vingt-et-un", 21],
+        ["vingt-deux", 22],
+        ["vingt deux", 22],
+        ["vingt-trois", 23],
+        ["vingt trois", 23],
+        ["vingt-quatre", 24],
+        ["vingt quatre", 24],
     ]);
     const hourWordMatch = s.match(/([a-z\- ]+?)\s*heures?/);
     if (!hourWordMatch)
@@ -109,9 +148,21 @@ function parseFrenchClockToHHMM(input) {
     }
     else {
         const MIN = new Map([
-            ["cinq", 5], ["dix", 10], ["quinze", 15], ["vingt", 20], ["vingt-cinq", 25], ["vingt cinq", 25],
-            ["trente", 30], ["trente-cinq", 35], ["trente cinq", 35], ["quarante", 40], ["quarante-cinq", 45],
-            ["quarante cinq", 45], ["cinquante", 50], ["cinquante-cinq", 55], ["cinquante cinq", 55],
+            ["cinq", 5],
+            ["dix", 10],
+            ["quinze", 15],
+            ["vingt", 20],
+            ["vingt-cinq", 25],
+            ["vingt cinq", 25],
+            ["trente", 30],
+            ["trente-cinq", 35],
+            ["trente cinq", 35],
+            ["quarante", 40],
+            ["quarante-cinq", 45],
+            ["quarante cinq", 45],
+            ["cinquante", 50],
+            ["cinquante-cinq", 55],
+            ["cinquante cinq", 55],
         ]);
         const minWordMatch = s.match(/heures?\s+([a-z\- ]+?)(?:[).,;]|$)/);
         if (minWordMatch) {
@@ -123,3 +174,213 @@ function parseFrenchClockToHHMM(input) {
     }
     return `${String(hour).padStart(2, "0")}${String(minutes).padStart(2, "0")}`;
 }
+// Helpers locaux (autonomes)
+function frDateToISO(s) {
+    if (!s)
+        return;
+    const months = {
+        janvier: 1,
+        février: 2,
+        fevrier: 2,
+        mars: 3,
+        avril: 4,
+        mai: 5,
+        juin: 6,
+        juillet: 7,
+        août: 8,
+        aout: 8,
+        septembre: 9,
+        octobre: 10,
+        novembre: 11,
+        décembre: 12,
+        decembre: 12,
+    };
+    const m = s
+        .trim()
+        .replace(/\u00A0/g, " ")
+        .replace(/ +/g, " ")
+        .match(/^(\d{1,2})\s+([a-zéèêîïôûùç]+)\s+(\d{4})$/i);
+    if (!m)
+        return;
+    const d = String(parseInt(m[1], 10)).padStart(2, "0");
+    const mon = months[m[2].toLowerCase()];
+    if (!mon)
+        return;
+    const y = m[3];
+    return `${y}-${String(mon).padStart(2, "0")}-${d}`;
+}
+function extractWeekStartFromHead($) {
+    const og = $('meta[property="og:title"]').attr("content") || $("title").text();
+    const m = (og ?? "").toLowerCase().match(/semaine du\s+(\d{1,2}\s+\w+\s+\d{4})/i);
+    if (m)
+        return frDateToISO(m[1]);
+    return undefined;
+}
+function detectOrganeFromTitle(s) {
+    const t = (s ?? "").trim();
+    if (!t)
+        return { organeTitleRaw: undefined, organeDetected: undefined };
+    const lower = t.toLowerCase();
+    const m = lower.match(/commission(?:\s+des|\s+de|)\s+([^:]+)$/i);
+    let organeDetected;
+    if (m && m[1]) {
+        organeDetected = ("Commission " + m[1])
+            .replace(/\s+/g, " ")
+            .replace(/\s+:? comptes? rendus?$/i, "")
+            .trim();
+        organeDetected = organeDetected[0].toUpperCase() + organeDetected.slice(1);
+    }
+    return { organeTitleRaw: t, organeDetected };
+}
+function extractDaysAndOpenings($) {
+    const days = [];
+    const h2s = $("h2").toArray();
+    for (let i = 0; i < h2s.length; i++) {
+        const h = h2s[i];
+        const txt = $(h).text().trim();
+        const m = txt.match(/(?:Lundi|Mardi|Mercredi|Jeudi|Vendredi|Samedi|Dimanche)\s+(.+)$/i);
+        if (!m)
+            continue;
+        const iso = frDateToISO(m[1]);
+        if (!iso)
+            continue;
+        let openTime;
+        let cur = $(h).next();
+        while (cur.length && cur[0].tagName !== "h2") {
+            const t = cur.text().replace(/\s+/g, " ").trim();
+            const mt = t.match(/La réunion est ouverte à\s+(\d{1,2})(?:h(?:\s*(\d{2}))?)?/i);
+            if (mt) {
+                openTime = `${mt[1].padStart(2, "0")}:${(mt[2] ?? "00").padStart(2, "0")}`;
+                break;
+            }
+            cur = cur.next();
+        }
+        days.push({ date: iso, openTime, h2Index: i });
+    }
+    return days;
+}
+function extractOrganeCode($) {
+    const names = $("a[name]")
+        .toArray()
+        .map((a) => ($(a).attr("name") || "").trim());
+    return names.find((n) => /^[A-Z]{3,6}$/.test(n));
+}
+export function parseCommissionMetadataFromHtml(html, sourceFileName) {
+    const $ = cheerio.load(html);
+    const h1 = $("h1.page-title").first().text().trim() || undefined;
+    const headTitle = $('meta[property="og:title"]').attr("content") || $("title").text() || undefined;
+    const { organeTitleRaw, organeDetected } = detectOrganeFromTitle(h1 || headTitle);
+    let weekStart = extractWeekStartFromHead($);
+    const days = extractDaysAndOpenings($);
+    if (!weekStart && days.length > 0)
+        weekStart = days[0].date;
+    const organeCode = extractOrganeCode($);
+    return {
+        sourceFile: sourceFileName ?? null,
+        organeTitleRaw: organeTitleRaw ?? null,
+        organeDetected: organeDetected ?? null,
+        organeCode: organeCode ?? null,
+        weekStart: weekStart ?? null,
+        days, // [{date, openTime?, h2Index}]
+    };
+}
+function isGroupedReunion(o) {
+    return o && typeof o === "object" && typeof o.uid === "string" && typeof o.date === "string";
+}
+export async function loadCommissionAgendaForDate(dataDir, yyyymmdd, session) {
+    const baseDir = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session));
+    if (!(await fs.pathExists(baseDir)))
+        return [];
+    const files = (await fs.readdir(baseDir)).filter((f) => f.startsWith(`RUSN${yyyymmdd}IDC`) && f.toLowerCase().endsWith(".json"));
+    const out = [];
+    for (const f of files) {
+        const p = path.join(baseDir, f);
+        try {
+            const raw = await fs.readFile(p, "utf8");
+            const obj = JSON.parse(raw);
+            if (!isGroupedReunion(obj)) {
+                continue;
+            }
+            if (!obj.uid.startsWith(`RUSN${yyyymmdd}IDC`)) {
+                continue;
+            }
+            out.push(obj);
+        }
+        catch {
+            // ignore
+        }
+    }
+    return out;
+}
+function hourShortToStartTime(hourShort) {
+    if (!hourShort || hourShort === "NA")
+        return null;
+    if (!/^\d{4}$/.test(hourShort))
+        return null;
+    const hh = hourShort.slice(0, 2);
+    const mm = hourShort.slice(2, 4);
+    return `${hh}:${mm}`;
+}
+export async function createCommissionGroupIfMissing(dataDir, dateISO, // "YYYY-MM-DD"
+organeDetected, // ex. "Commission des finances"
+hourShort, // "HHMM" | "NA"
+titreGuess) {
+    const uid = makeTypeGroupUid(dateISO, "COM", hourShort ?? "NA", organeDetected ?? undefined);
+    const session = sessionStartYearFromDate(new Date(dateISO));
+    const dir = path.join(dataDir, "agenda", "transformed", String(session));
+    await fs.ensureDir(dir);
+    const filePath = path.join(dir, `${uid}.json`);
+    let groups = [];
+    let created = false;
+    if (await fs.pathExists(filePath)) {
+        try {
+            const raw = await fs.readFile(filePath, "utf8");
+            groups = JSON.parse(raw);
+            if (!Array.isArray(groups))
+                groups = [];
+        }
+        catch {
+            groups = [];
+        }
+        const exists = groups.some((g) => g?.uid === uid);
+        if (!exists) {
+            groups.push({
+                uid,
+                chambre: "SN",
+                date: dateISO,
+                slot: null,
+                type: organeDetected ?? "Commission",
+                startTime: hourShortToStartTime(hourShort),
+                endTime: null,
+                captationVideo: false,
+                titre: titreGuess ?? null,
+                objet: null,
+                reunions: [],
+                compteRenduRefUid: null,
+            });
+            await fs.writeJSON(filePath, groups, { spaces: 2 });
+            created = true;
+        }
+    }
+    else {
+        groups = [
+            {
+                uid,
+                chambre: "SN",
+                date: dateISO,
+                slot: null,
+                type: organeDetected ?? "Commission",
+                startTime: hourShortToStartTime(hourShort),
+                endTime: null,
+                captationVideo: false,
+                titre: titreGuess ?? null,
+                objet: null,
+                reunions: [],
+                compteRenduRefUid: null,
+            },
+        ];
+        await fs.writeJSON(filePath, groups, { spaces: 2 });
+        created = true;
+    }
+    return { uid, filePath, created };
+}

package/lib/utils/reunion_grouping.d.ts

@@ -1,6 +1,9 @@
 import { AgendaEvent, GroupedReunion, TimeSlot } from "../types/agenda";
+type KnownType = "SP" | "COM" | "MC" | "OD" | "ID";
 export declare function groupNonSPByTypeOrganeHour(events: AgendaEvent[]): Record<"IDC" | "IDM" | "IDO" | "IDI", GroupedReunion[]>;
 export declare function groupSeancePubliqueBySlot(events: AgendaEvent[]): GroupedReunion[];
+export declare function makeTypeGroupUid(dateISO: string, kind: KnownType, hourShort: string | null, organe?: string | null): string;
 export declare function makeGroupUid(date: string, slot: TimeSlot): string;
 export declare function formatYYYYMMDD(dateYYYYMMDD: string): string;
 export declare function makeReunionUid(agenda: AgendaEvent): string;
+export {};

package/lib/utils/reunion_grouping.js

@@ -243,7 +243,7 @@ function organeInitials(input, maxLen = 8) {
     const out = letters.join("");
     return out.slice(0, maxLen);
 }
-function makeTypeGroupUid(dateISO, kind, hourShort, organe) {
+export function makeTypeGroupUid(dateISO, kind, hourShort, organe) {
     const ymd = dateISO ? formatYYYYMMDD(dateISO) : "00000000";
     const suffix = typeToSuffixStrict(kind);
     const hh = hourShort ?? "NA";