@tricoteuses/senat 2.10.5 → 2.11.1

package/lib/utils/cr_spliting.js

@@ -1,7 +1,13 @@
+import path from "path";
+import * as cheerio from "cheerio";
+import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
+import fs from "fs-extra";
+import { makeTypeGroupUid } from "./reunion_grouping";
+import { sessionStartYearFromDate } from "../model/seance";
 export function computeIntervalsBySlot($, idx, firstSlotOfDay) {
     const all = $("body *").toArray();
     const cuts = [{ pos: 0, hhmm: undefined }];
-    $('a[name]').each((_, a) => {
+    $("a[name]").each((_, a) => {
         const name = (a.attribs?.["name"] || "").trim();
         if (!/^su/i.test(name))
             return;
@@ -30,7 +36,7 @@ export function computeIntervalsBySlot($, idx, firstSlotOfDay) {
             continue;
         // i=0  initialSlot
         // i>0 : if current cut has SU -> slotOfHHMM, otherwise lastSlot
-        const slot = i === 0 ? initialSlot : (cuts[i].hhmm ? slotOfHHMM(cuts[i].hhmm) : lastSlot);
+        const slot = i === 0 ? initialSlot : cuts[i].hhmm ? slotOfHHMM(cuts[i].hhmm) : lastSlot;
         intervals.push({ slot, start, end });
         lastSlot = slot;
     }
@@ -70,7 +76,11 @@ function extractOpeningHHMM($) {
 }
 // Convert "quinze heures trente", "15 heures 30", "dix-sept heures moins le quart", etc. en "HHMM"
 function parseFrenchClockToHHMM(input) {
-    const s = (input || "").toLowerCase().normalize("NFKD").replace(/[\u0300-\u036f]/g, "").trim();
+    const s = (input || "")
+        .toLowerCase()
+        .normalize("NFKD")
+        .replace(/[\u0300-\u036f]/g, "")
+        .trim();
     if (!s)
         return undefined;
     const digitMatch = s.match(/(\d{1,2})\s*heures?(?:\s*(\d{1,2}))?/);
@@ -80,12 +90,41 @@ function parseFrenchClockToHHMM(input) {
         return `${String(h).padStart(2, "0")}${String(m).padStart(2, "0")}`;
     }
     const NUM = new Map([
-        ["zero", 0], ["une", 1], ["un", 1], ["deux", 2], ["trois", 3], ["quatre", 4], ["cinq", 5], ["six", 6],
-        ["sept", 7], ["huit", 8], ["neuf", 9], ["dix", 10], ["onze", 11], ["douze", 12], ["treize", 13],
-        ["quatorze", 14], ["quinze", 15], ["seize", 16], ["dix-sept", 17], ["dix sept", 17], ["dix-huit", 18],
-        ["dix huit", 18], ["dix-neuf", 19], ["dix neuf", 19], ["vingt", 20], ["vingt et une", 21],
-        ["vingt-et-une", 21], ["vingt et un", 21], ["vingt-et-un", 21], ["vingt-deux", 22], ["vingt deux", 22],
-        ["vingt-trois", 23], ["vingt trois", 23], ["vingt-quatre", 24], ["vingt quatre", 24],
+        ["zero", 0],
+        ["une", 1],
+        ["un", 1],
+        ["deux", 2],
+        ["trois", 3],
+        ["quatre", 4],
+        ["cinq", 5],
+        ["six", 6],
+        ["sept", 7],
+        ["huit", 8],
+        ["neuf", 9],
+        ["dix", 10],
+        ["onze", 11],
+        ["douze", 12],
+        ["treize", 13],
+        ["quatorze", 14],
+        ["quinze", 15],
+        ["seize", 16],
+        ["dix-sept", 17],
+        ["dix sept", 17],
+        ["dix-huit", 18],
+        ["dix huit", 18],
+        ["dix-neuf", 19],
+        ["dix neuf", 19],
+        ["vingt", 20],
+        ["vingt et une", 21],
+        ["vingt-et-une", 21],
+        ["vingt et un", 21],
+        ["vingt-et-un", 21],
+        ["vingt-deux", 22],
+        ["vingt deux", 22],
+        ["vingt-trois", 23],
+        ["vingt trois", 23],
+        ["vingt-quatre", 24],
+        ["vingt quatre", 24],
     ]);
     const hourWordMatch = s.match(/([a-z\- ]+?)\s*heures?/);
     if (!hourWordMatch)
@@ -109,9 +148,21 @@ function parseFrenchClockToHHMM(input) {
     }
     else {
         const MIN = new Map([
-            ["cinq", 5], ["dix", 10], ["quinze", 15], ["vingt", 20], ["vingt-cinq", 25], ["vingt cinq", 25],
-            ["trente", 30], ["trente-cinq", 35], ["trente cinq", 35], ["quarante", 40], ["quarante-cinq", 45],
-            ["quarante cinq", 45], ["cinquante", 50], ["cinquante-cinq", 55], ["cinquante cinq", 55],
+            ["cinq", 5],
+            ["dix", 10],
+            ["quinze", 15],
+            ["vingt", 20],
+            ["vingt-cinq", 25],
+            ["vingt cinq", 25],
+            ["trente", 30],
+            ["trente-cinq", 35],
+            ["trente cinq", 35],
+            ["quarante", 40],
+            ["quarante-cinq", 45],
+            ["quarante cinq", 45],
+            ["cinquante", 50],
+            ["cinquante-cinq", 55],
+            ["cinquante cinq", 55],
         ]);
         const minWordMatch = s.match(/heures?\s+([a-z\- ]+?)(?:[).,;]|$)/);
         if (minWordMatch) {
@@ -123,3 +174,213 @@ function parseFrenchClockToHHMM(input) {
     }
     return `${String(hour).padStart(2, "0")}${String(minutes).padStart(2, "0")}`;
 }
+// Helpers locaux (autonomes)
+function frDateToISO(s) {
+    if (!s)
+        return;
+    const months = {
+        janvier: 1,
+        février: 2,
+        fevrier: 2,
+        mars: 3,
+        avril: 4,
+        mai: 5,
+        juin: 6,
+        juillet: 7,
+        août: 8,
+        aout: 8,
+        septembre: 9,
+        octobre: 10,
+        novembre: 11,
+        décembre: 12,
+        decembre: 12,
+    };
+    const m = s
+        .trim()
+        .replace(/\u00A0/g, " ")
+        .replace(/ +/g, " ")
+        .match(/^(\d{1,2})\s+([a-zéèêîïôûùç]+)\s+(\d{4})$/i);
+    if (!m)
+        return;
+    const d = String(parseInt(m[1], 10)).padStart(2, "0");
+    const mon = months[m[2].toLowerCase()];
+    if (!mon)
+        return;
+    const y = m[3];
+    return `${y}-${String(mon).padStart(2, "0")}-${d}`;
+}
+function extractWeekStartFromHead($) {
+    const og = $('meta[property="og:title"]').attr("content") || $("title").text();
+    const m = (og ?? "").toLowerCase().match(/semaine du\s+(\d{1,2}\s+\w+\s+\d{4})/i);
+    if (m)
+        return frDateToISO(m[1]);
+    return undefined;
+}
+function detectOrganeFromTitle(s) {
+    const t = (s ?? "").trim();
+    if (!t)
+        return { organeTitleRaw: undefined, organeDetected: undefined };
+    const lower = t.toLowerCase();
+    const m = lower.match(/commission(?:\s+des|\s+de|)\s+([^:]+)$/i);
+    let organeDetected;
+    if (m && m[1]) {
+        organeDetected = ("Commission " + m[1])
+            .replace(/\s+/g, " ")
+            .replace(/\s+:? comptes? rendus?$/i, "")
+            .trim();
+        organeDetected = organeDetected[0].toUpperCase() + organeDetected.slice(1);
+    }
+    return { organeTitleRaw: t, organeDetected };
+}
+function extractDaysAndOpenings($) {
+    const days = [];
+    const h2s = $("h2").toArray();
+    for (let i = 0; i < h2s.length; i++) {
+        const h = h2s[i];
+        const txt = $(h).text().trim();
+        const m = txt.match(/(?:Lundi|Mardi|Mercredi|Jeudi|Vendredi|Samedi|Dimanche)\s+(.+)$/i);
+        if (!m)
+            continue;
+        const iso = frDateToISO(m[1]);
+        if (!iso)
+            continue;
+        let openTime;
+        let cur = $(h).next();
+        while (cur.length && cur[0].tagName !== "h2") {
+            const t = cur.text().replace(/\s+/g, " ").trim();
+            const mt = t.match(/La réunion est ouverte à\s+(\d{1,2})(?:h(?:\s*(\d{2}))?)?/i);
+            if (mt) {
+                openTime = `${mt[1].padStart(2, "0")}:${(mt[2] ?? "00").padStart(2, "0")}`;
+                break;
+            }
+            cur = cur.next();
+        }
+        days.push({ date: iso, openTime, h2Index: i });
+    }
+    return days;
+}
+function extractOrganeCode($) {
+    const names = $("a[name]")
+        .toArray()
+        .map((a) => ($(a).attr("name") || "").trim());
+    return names.find((n) => /^[A-Z]{3,6}$/.test(n));
+}
+export function parseCommissionMetadataFromHtml(html, sourceFileName) {
+    const $ = cheerio.load(html);
+    const h1 = $("h1.page-title").first().text().trim() || undefined;
+    const headTitle = $('meta[property="og:title"]').attr("content") || $("title").text() || undefined;
+    const { organeTitleRaw, organeDetected } = detectOrganeFromTitle(h1 || headTitle);
+    let weekStart = extractWeekStartFromHead($);
+    const days = extractDaysAndOpenings($);
+    if (!weekStart && days.length > 0)
+        weekStart = days[0].date;
+    const organeCode = extractOrganeCode($);
+    return {
+        sourceFile: sourceFileName ?? null,
+        organeTitleRaw: organeTitleRaw ?? null,
+        organeDetected: organeDetected ?? null,
+        organeCode: organeCode ?? null,
+        weekStart: weekStart ?? null,
+        days, // [{date, openTime?, h2Index}]
+    };
+}
+function isGroupedReunion(o) {
+    return o && typeof o === "object" && typeof o.uid === "string" && typeof o.date === "string";
+}
+export async function loadCommissionAgendaForDate(dataDir, yyyymmdd, session) {
+    const baseDir = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session));
+    if (!(await fs.pathExists(baseDir)))
+        return [];
+    const files = (await fs.readdir(baseDir)).filter((f) => f.startsWith(`RUSN${yyyymmdd}IDC`) && f.toLowerCase().endsWith(".json"));
+    const out = [];
+    for (const f of files) {
+        const p = path.join(baseDir, f);
+        try {
+            const raw = await fs.readFile(p, "utf8");
+            const obj = JSON.parse(raw);
+            if (!isGroupedReunion(obj)) {
+                continue;
+            }
+            if (!obj.uid.startsWith(`RUSN${yyyymmdd}IDC`)) {
+                continue;
+            }
+            out.push(obj);
+        }
+        catch {
+            // ignore
+        }
+    }
+    return out;
+}
+function hourShortToStartTime(hourShort) {
+    if (!hourShort || hourShort === "NA")
+        return null;
+    if (!/^\d{4}$/.test(hourShort))
+        return null;
+    const hh = hourShort.slice(0, 2);
+    const mm = hourShort.slice(2, 4);
+    return `${hh}:${mm}`;
+}
+export async function createCommissionGroupIfMissing(dataDir, dateISO, // "YYYY-MM-DD"
+organeDetected, // ex. "Commission des finances"
+hourShort, // "HHMM" | "NA"
+titreGuess) {
+    const uid = makeTypeGroupUid(dateISO, "COM", hourShort ?? "NA", organeDetected ?? undefined);
+    const session = sessionStartYearFromDate(new Date(dateISO));
+    const dir = path.join(dataDir, "agenda", "transformed", String(session));
+    await fs.ensureDir(dir);
+    const filePath = path.join(dir, `${uid}.json`);
+    let groups = [];
+    let created = false;
+    if (await fs.pathExists(filePath)) {
+        try {
+            const raw = await fs.readFile(filePath, "utf8");
+            groups = JSON.parse(raw);
+            if (!Array.isArray(groups))
+                groups = [];
+        }
+        catch {
+            groups = [];
+        }
+        const exists = groups.some((g) => g?.uid === uid);
+        if (!exists) {
+            groups.push({
+                uid,
+                chambre: "SN",
+                date: dateISO,
+                slot: null,
+                type: organeDetected ?? "Commission",
+                startTime: hourShortToStartTime(hourShort),
+                endTime: null,
+                captationVideo: false,
+                titre: titreGuess ?? null,
+                objet: null,
+                reunions: [],
+                compteRenduRefUid: null,
+            });
+            await fs.writeJSON(filePath, groups, { spaces: 2 });
+            created = true;
+        }
+    }
+    else {
+        groups = [
+            {
+                uid,
+                chambre: "SN",
+                date: dateISO,
+                slot: null,
+                type: organeDetected ?? "Commission",
+                startTime: hourShortToStartTime(hourShort),
+                endTime: null,
+                captationVideo: false,
+                titre: titreGuess ?? null,
+                objet: null,
+                reunions: [],
+                compteRenduRefUid: null,
+            },
+        ];
+        await fs.writeJSON(filePath, groups, { spaces: 2 });
+        created = true;
+    }
+    return { uid, filePath, created };
+}

package/lib/utils/reunion_grouping.d.ts

@@ -1,6 +1,9 @@
 import { AgendaEvent, GroupedReunion, TimeSlot } from "../types/agenda";
+type KnownType = "SP" | "COM" | "MC" | "OD" | "ID";
 export declare function groupNonSPByTypeOrganeHour(events: AgendaEvent[]): Record<"IDC" | "IDM" | "IDO" | "IDI", GroupedReunion[]>;
 export declare function groupSeancePubliqueBySlot(events: AgendaEvent[]): GroupedReunion[];
+export declare function makeTypeGroupUid(dateISO: string, kind: KnownType, hourShort: string | null, organe?: string | null): string;
 export declare function makeGroupUid(date: string, slot: TimeSlot): string;
 export declare function formatYYYYMMDD(dateYYYYMMDD: string): string;
 export declare function makeReunionUid(agenda: AgendaEvent): string;
+export {};

package/lib/utils/reunion_grouping.js

@@ -243,7 +243,7 @@ function organeInitials(input, maxLen = 8) {
     const out = letters.join("");
     return out.slice(0, maxLen);
 }
-function makeTypeGroupUid(dateISO, kind, hourShort, organe) {
+export function makeTypeGroupUid(dateISO, kind, hourShort, organe) {
     const ymd = dateISO ? formatYYYYMMDD(dateISO) : "00000000";
     const suffix = typeToSuffixStrict(kind);
     const hh = hourShort ?? "NA";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.10.5",
+  "version": "2.11.1",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",
@@ -46,7 +46,8 @@
     "data:download": "tsx src/scripts/data-download.ts",
     "data:generate_schemas": "tsx src/scripts/retrieve_open_data.ts --schema",
     "data:retrieve_agenda": "cross-env TZ='Etc/UTC' tsx src/scripts/retrieve_agenda.ts",
-    "data:retrieve_comptes_rendus": "tsx src/scripts/retrieve_comptes_rendus.ts",
+    "data:retrieve_cr_seance": "tsx src/scripts/retrieve_cr_seance.ts",
+    "data:retrieve_cr_commission": "tsx src/scripts/retrieve_cr_commission.ts",
     "data:retrieve_documents": "tsx src/scripts/retrieve_documents.ts",
     "data:retrieve_open_data": "tsx src/scripts/retrieve_open_data.ts --all",
     "data:retrieve_senateurs_photos": "tsx src/scripts/retrieve_senateurs_photos.ts --fetch",
@@ -66,12 +67,12 @@
     "fs-extra": "^9.1.0",
     "jsdom": "^26.0.0",
     "kysely": "^0.27.4",
-    "luxon": "^3.5.0",
+    "luxon": "^3.7.2",
     "node-stream-zip": "^1.8.2",
     "pg": "^8.13.1",
     "pg-cursor": "^2.12.1",
     "slug": "^11.0.0",
-    "tsx": "^4.19.4",
+    "tsx": "^4.20.6",
     "windows-1252": "^1.0.0"
   },
   "devDependencies": {
@@ -80,19 +81,19 @@
     "@types/command-line-args": "^5.0.0",
     "@types/fs-extra": "^9.0.7",
     "@types/jsdom": "^21.1.7",
-    "@types/luxon": "^3.4.2",
+    "@types/luxon": "^3.7.1",
     "@types/node": "^20.17.6",
-    "@types/pg": "^8.11.10",
+    "@types/pg": "^8.15.5",
     "@types/pg-cursor": "^2.7.2",
     "@types/slug": "^5.0.9",
-    "@typescript-eslint/eslint-plugin": "^8.13.0",
-    "@typescript-eslint/parser": "^8.13.0",
-    "cross-env": "^10.0.0",
+    "@typescript-eslint/eslint-plugin": "^8.46.0",
+    "@typescript-eslint/parser": "^8.46.0",
+    "cross-env": "^10.1.0",
     "eslint": "^8.57.1",
     "iconv-lite": "^0.7.0",
-    "pg-to-ts": "^4.1.1",
+    "kysely-codegen": "^0.19.0",
     "prettier": "^3.5.3",
     "tslib": "^2.1.0",
-    "typescript": "^5.8.3"
+    "typescript": "^5.9.3"
   }
 }