@tricoteuses/senat 2.20.18 → 2.20.20

package/lib/index.d.ts

@@ -3,7 +3,7 @@ export type { DossierLegislatifResult } from "./model/dosleg";
 export type { ScrutinResult } from "./model/scrutins";
 export type { QuestionResult } from "./model/questions";
 export type { CirconscriptionResult, OrganismeResult, SenateurResult } from "./model/sens";
-export type { AgendaEvent, GroupedReunion, GroupedReunionOdjPoint } from "./types/agenda";
+export type { AgendaEvent, Reunion, ReunionOdjPoint } from "./types/agenda";
 export type { Ses, Sub, TxtAmeli } from "./types/ameli";
 export type { CompteRendu } from "./types/compte_rendu";
 export type { Debat, LecAssDeb } from "./types/debats";

package/lib/loaders.d.ts

@@ -4,7 +4,7 @@ import { DossierLegislatifResult } from "./model/dosleg";
 import { QuestionResult } from "./model/questions";
 import { ScrutinResult } from "./model/scrutins";
 import { CirconscriptionResult, OrganismeResult, SenateurResult } from "./model/sens";
-import { GroupedReunion } from "./types/agenda";
+import { Reunion } from "./types/agenda";
 import { FlatTexte } from "./types/texte";
 import { CompteRendu } from "./types/compte_rendu";
 export { EnabledDatasets } from "./datasets";
@@ -87,7 +87,7 @@ export declare function loadSenatTexteContent(dataDir: string, textePathFromData
 export declare function loadSenatCompteRenduContent(dataDir: string, session: number, debatId: string | number): {
     item: CompteRendu | null;
 };
-export declare function iterLoadSenatAgendas(dataDir: string, session: number | undefined): Generator<IterItem<GroupedReunion>>;
+export declare function iterLoadSenatAgendas(dataDir: string, session: number | undefined): Generator<IterItem<Reunion>>;
 export declare function iterLoadSenatCirconscriptions(dataDir: string, options?: {}): Generator<IterItem<CirconscriptionResult>>;
 export declare function iterLoadSenatOrganismes(dataDir: string, options?: {}): Generator<IterItem<OrganismeResult>>;
 export declare function iterLoadSenatSenateurs(dataDir: string, options?: {}): Generator<IterItem<SenateurResult>>;

package/lib/model/commission.d.ts

@@ -1,6 +1,6 @@
 import * as cheerio from "cheerio";
 import { CompteRendu } from "../types/compte_rendu";
-import { GroupedReunion } from "../types/agenda";
+import { Reunion } from "../types/agenda";
 export declare function getRemainingTextAfterSpeakerHeader($: cheerio.CheerioAPI, $p: cheerio.Cheerio<any>): string;
 export type DaySection = {
     title: string;
@@ -14,5 +14,5 @@ export declare function parseCommissionCRSectionFromDom($: cheerio.CheerioAPI, h
     hourShort: string | null;
     organe?: string | null;
     section: DaySection;
-    matched?: GroupedReunion;
+    matched?: Reunion;
 }): CompteRendu | null;

package/lib/model/commission.js

@@ -1,8 +1,9 @@
 import * as cheerio from "cheerio";
 import path from "path";
-import { norm, toCRDate } from "./util";
-import { makeTypeGroupUid } from "../utils/reunion_grouping";
-import { frDateToISO, hourShortToStartTime } from "../utils/cr_spliting";
+import { makeReunionUid } from "../utils/reunion_parsing";
+import { norm } from "../utils/string_cleaning";
+import { frDateToISO, hourShortToStartTime } from "../utils/date";
+import { toCRDate } from "./util";
 const PARA_h3_SEL = "p.sh_justify, p.sh_center, p.sh_marge, p[align], li, h3";
 function findDayRoot($, targetISO) {
     let $root = $();
@@ -208,7 +209,7 @@ export function extractDayH3Sections($, dateISO) {
 export function parseCommissionCRSectionFromDom($, htmlFilePath, opts) {
     try {
         const { dateISO, hourShort, organe, section, matched } = opts;
-        const seanceRef = matched?.uid ?? makeTypeGroupUid(dateISO, "COM", matched?.events[0].id ?? hourShort ?? "", organe ?? undefined);
+        const seanceRef = matched?.uid ?? makeReunionUid(dateISO, "COM", matched?.events[0].id ?? hourShort ?? "", organe ?? undefined);
         const uid = seanceRef.replace(/^RU/, "CRC");
         const dateSeance = toCRDate(dateISO, matched?.startTime ?? hourShortToStartTime(hourShort));
         const $dayRoot = findDayRoot($, dateISO);

package/lib/model/seance.d.ts

@@ -1,9 +1,3 @@
-import { CompteRendu, Sommaire } from "../types/compte_rendu";
-import { TimeSlot } from "../types/agenda";
-export declare function parseCompteRenduSlotFromFile(xmlFilePath: string, wantedSlot: TimeSlot, firstSlotOfDay?: TimeSlot): Promise<CompteRendu | null>;
+import { CompteRendu } from "../types/compte_rendu";
+export declare function parseCompteRenduIntervalFromFile(xmlFilePath: string, startIndex: number, endIndex: number, agendaEventId: string): Promise<CompteRendu | null>;
 export declare function sessionStartYearFromDate(d: Date): number;
-export declare function parseYYYYMMDD(yyyymmdd: string): Date | null;
-export declare function deriveTitreObjetFromSommaire(sommaire: Sommaire | undefined, slot: TimeSlot): {
-    titre: string;
-    objet: string;
-};

package/lib/model/seance.js

@@ -1,33 +1,29 @@
 import fs from "fs";
 import * as cheerio from "cheerio";
-import path from "path";
-import { computeIntervalsBySlot } from "../utils/cr_spliting";
-import { decodeHtmlEntities, norm, toCRDate } from "./util";
-const asArray = (x) => (x == null ? [] : Array.isArray(x) ? x : [x]);
-const toInt = (s) => (Number.isFinite(Number(s)) ? Number(s) : Number.POSITIVE_INFINITY);
-export async function parseCompteRenduSlotFromFile(xmlFilePath, wantedSlot, firstSlotOfDay) {
+import { toCRDate } from "./util";
+import { makeReunionUid } from "../utils/reunion_parsing";
+import { yyyymmddFromPath } from "../utils/date";
+import { decodeHtmlEntities, dedupeSpeaker, fixApostrophes, norm } from "../utils/string_cleaning";
+export async function parseCompteRenduIntervalFromFile(xmlFilePath, startIndex, endIndex, agendaEventId) {
     try {
         const raw = fs.readFileSync(xmlFilePath, "utf8");
         const $ = cheerio.load(raw, { xml: false });
         const metadonnees = extractMetadonnees($, xmlFilePath);
         const order = $("body *").toArray();
         const idx = new Map(order.map((el, i) => [el, i]));
-        const intervalsAll = computeIntervalsBySlot($, idx, firstSlotOfDay);
-        const intervals = intervalsAll.filter((iv) => iv.slot === wantedSlot);
-        if (intervals.length === 0) {
-            console.warn(`[CRI] no intervals for ${path.basename(xmlFilePath)} [${wantedSlot}]`);
-            return null;
-        }
+        const totalNodes = order.length;
+        const clampedStart = Math.max(0, Math.min(startIndex, totalNodes - 1));
+        const clampedEnd = Math.max(0, Math.min(endIndex, totalNodes - 1));
+        const intervals = [
+            {
+                start: clampedStart,
+                end: clampedEnd,
+            },
+        ];
         metadonnees.sommaire = extractSommaireForIntervals($, idx, intervals);
         const points = [];
         let ordre = 0;
         const addPoint = (p) => points.push({ ...p, ordre_absolu_seance: String(++ordre) });
-        // Titles removes because they are just listed at the top of the file and not linked to any ancre
-        // $("cri\\:titreS1 p.titre_S1").each((_, el) => {
-        //   if (!elementInAnyInterval(el, idx, intervals)) return
-        //   const t = normalizeTitle(norm($(el).text() || ""))
-        //   if (t) addPoint({ code_grammaire: "TITRE_TEXTE_DISCUSSION", texte: { _: t }, code_style: "Titre" })
-        // })
         // Interventions
         $("div.intervenant").each((_, block) => {
             if (!elementInAnyInterval(block, idx, intervals))
@@ -47,6 +43,8 @@ export async function parseCompteRenduSlotFromFile(xmlFilePath, wantedSlot, firs
             ].join(","))
                 .remove();
             const firstP = $block.find("p").first();
+            if (!firstP || firstP.length === 0)
+                return;
             const speakerLabelRaw = firstP.find(".orateur_nom").text() || firstP.find("a.lien_senfic").text() || "";
             const speakerLabel = dedupeSpeaker(speakerLabelRaw);
             const { mat, nom: nomCRI, qua: quaCRI } = readIntervenantMeta($block);
@@ -55,7 +53,8 @@ export async function parseCompteRenduSlotFromFile(xmlFilePath, wantedSlot, firs
             const canonicalName = dedupeSpeaker(nomCRI || speakerLabel);
             const role = roleForSpeaker(speakerLabel) || roleForSpeaker(qualite) || roleForSpeaker(quaCRI || "");
             const speechHtml = sanitizeInterventionHtml($, $block);
-            if (!norm(cheerio.load(speechHtml).text() || ""))
+            const speechText = norm(cheerio.load(speechHtml).text() || "");
+            if (!speechText)
                 return;
             addPoint({
                 code_grammaire: "PAROLE_GENERIQUE",
@@ -65,19 +64,25 @@ export async function parseCompteRenduSlotFromFile(xmlFilePath, wantedSlot, firs
             });
         });
         const contenu = {
-            quantiemes: { journee: metadonnees.dateSeance, session: metadonnees.session },
+            quantiemes: {
+                journee: metadonnees.dateSeance,
+                session: metadonnees.session,
+            },
             point: points,
         };
+        const yyyymmdd = yyyymmddFromPath(xmlFilePath);
+        const dateISO = `${yyyymmdd.slice(0, 4)}-${yyyymmdd.slice(4, 6)}-${yyyymmdd.slice(6, 8)}`;
+        const seanceRef = makeReunionUid(dateISO, "SP", agendaEventId, null);
         return {
-            uid: "CRSSN" + xmlFilePath.replace(/^.*?(\d{8}).*$/i, "$1") + `-${wantedSlot}`,
-            seanceRef: "RUSN" + xmlFilePath.replace(/^.*?(\d{8}).*$/i, "$1") + "IDS-" + wantedSlot,
+            uid: `CRSSN${yyyymmdd}E${agendaEventId}`,
+            seanceRef,
             sessionRef: metadonnees.session,
             metadonnees,
             contenu,
         };
     }
     catch (e) {
-        console.error(`[CRI] parseSlot error file=${xmlFilePath} slot=${wantedSlot}:`, e);
+        console.error(`[CRI] parseInterval error file=${xmlFilePath} interval=[${startIndex}..${endIndex}] event=${agendaEventId}:`, e);
         return null;
     }
 }
@@ -87,96 +92,6 @@ export function sessionStartYearFromDate(d) {
     const y = d.getFullYear();
     return m >= 9 ? y : y - 1;
 }
-export function parseYYYYMMDD(yyyymmdd) {
-    if (!/^\d{8}$/.test(yyyymmdd))
-        return null;
-    const y = Number(yyyymmdd.slice(0, 4));
-    const m = Number(yyyymmdd.slice(4, 6)) - 1;
-    const d = Number(yyyymmdd.slice(6, 8));
-    const dt = new Date(y, m, d);
-    return Number.isFinite(dt.getTime()) ? dt : null;
-}
-export function deriveTitreObjetFromSommaire(sommaire, slot) {
-    const items = extractLevel1Items(sommaire);
-    const meaningful = items.filter((it) => !isBoilerplate(it.label));
-    if (meaningful.length === 0) {
-        return {
-            titre: `Séance publique ${slotLabel(slot)}`,
-            objet: "",
-        };
-    }
-    const titre = meaningful[0].label;
-    const objet = meaningful
-        .slice(0, 3)
-        .map((it) => it.label)
-        .join(" ; ");
-    return { titre, objet };
-}
-function slotLabel(slot) {
-    switch (slot) {
-        case "MATIN":
-            return "du matin";
-        case "APRES-MIDI":
-            return "de l’après-midi";
-        case "SOIR":
-            return "du soir";
-        default:
-            return "";
-    }
-}
-const BOILERPLATE_PATTERNS = [
-    /proc(?:è|e)s-?verbal/i,
-    /hommages?/i,
-    /désignation des vice-?président/i,
-    /candidatures? aux?/i,
-    /ordre du jour/i,
-    /rappels? au règlement/i,
-    /communications?/i,
-    /dépôts?/i,
-    /proclamation/i,
-    /présidence de/i,
-    /questions? diverses?/i,
-    /ouverture de la séance/i,
-    /clo(?:t|̂)ure de la séance/i,
-];
-const isBoilerplate = (label) => !label?.trim() || BOILERPLATE_PATTERNS.some((rx) => rx.test(label));
-function extractLevel1Items(sommaire) {
-    const level1 = asArray(sommaire?.sommaire1);
-    return level1
-        .map((el) => ({
-        numero: toInt(el?.valeur_pts_odj),
-        label: String(el?.titreStruct?.intitule ?? "").trim(),
-    }))
-        .filter((it) => !!it.label)
-        .sort((a, b) => a.numero - b.numero);
-}
-function stripTrailingPunct(s) {
-    return s.replace(/\s*([:,.;])\s*$/u, "").trim();
-}
-function dedupeSpeaker(raw) {
-    let s = norm(raw);
-    s = stripTrailingPunct(s);
-    const dupPatterns = [/^(.+?)\s*[.]\s*\1$/u, /^(.+?)\s*,\s*\1,?$/u, /^(.+?)\s+\1$/u];
-    for (const re of dupPatterns) {
-        const m = s.match(re);
-        if (m) {
-            s = m[1];
-            break;
-        }
-    }
-    return s.replace(/\.\s*$/, "");
-}
-function fixApostrophes(s) {
-    let out = s;
-    out = out.replace(/\s*’\s*/g, "’");
-    out = out.replace(/\b([dljctmsn])\s*’/gi, (_, m) => m + "’");
-    out = out.replace(/’\s+([A-Za-zÀ-ÖØ-öø-ÿ])/g, "’$1");
-    out = out.replace(/\s+([,;:.!?])/g, "$1");
-    return out;
-}
-function normalizeTitle(text) {
-    return text.replace(/^PR[ÉE]SIDENCE DE\b/i, "Présidence de ");
-}
 function roleForSpeaker(labelOrQualite) {
     const s = (labelOrQualite || "").toLowerCase();
     if (/^(m\.|mme)?\s*(le|la)\s+pr[ée]sident(e)?\b/.test(s) || /\bpr[ée]sident[e]?\s+de\s+séance\b/.test(s))

package/lib/model/util.d.ts

@@ -6,8 +6,4 @@ export declare function removeSubstring(expr: Expression<string | null | undefin
 export declare function replace(expr: Expression<string | null | undefined>, pattern: Expression<string>, replacement: Expression<string>): import("kysely").RawBuilder<string>;
 export declare function rtrim(expr: Expression<string | null | undefined>): import("kysely").RawBuilder<string>;
 export declare function toDateString(expr: Expression<Date | null | undefined>, format?: Expression<string>): import("kysely").RawBuilder<string>;
-export declare function norm(s?: string | null): string;
 export declare function toCRDate(dateISO: string, startTime?: string | null): string;
-export declare function normalizeTitle(t: string): string;
-export declare function jaccardTokenSim(a: string, b: string): number;
-export declare function decodeHtmlEntities(s?: string | null): string;

package/lib/model/util.js

@@ -21,12 +21,6 @@ export function rtrim(expr) {
 export function toDateString(expr, format = sql.val(STANDARD_DATE_FORMAT)) {
     return sql `to_char(${expr}, ${format})`;
 }
-export function norm(s) {
-    return (s || "")
-        .replace(/\u00A0/g, " ")
-        .replace(/\s+/g, " ")
-        .trim();
-}
 export function toCRDate(dateISO, startTime) {
     const yyyymmdd = dateISO.replace(/-/g, ""); // "20250716"
     let hh = "00", mm = "00", ss = "00", SSS = "000";
@@ -42,35 +36,3 @@ export function toCRDate(dateISO, startTime) {
     }
     return `${yyyymmdd}${hh}${mm}${ss}${SSS}`;
 }
-export function normalizeTitle(t) {
-    return (t || "")
-        .toLowerCase()
-        .normalize("NFD")
-        .replace(/\p{Diacritic}/gu, "")
-        .replace(/[^a-z0-9\s]/g, " ")
-        .replace(/\s+/g, " ")
-        .trim();
-}
-export function jaccardTokenSim(a, b) {
-    const A = new Set(normalizeTitle(a).split(" ").filter(Boolean));
-    const B = new Set(normalizeTitle(b).split(" ").filter(Boolean));
-    if (A.size === 0 || B.size === 0)
-        return 0;
-    let inter = 0;
-    for (const x of A)
-        if (B.has(x))
-            inter++;
-    return inter / (A.size + B.size - inter);
-}
-export function decodeHtmlEntities(s) {
-    if (!s)
-        return "";
-    return s
-        .replace(/&#x([0-9a-fA-F]+);/g, (_, h) => String.fromCodePoint(parseInt(h, 16)))
-        .replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10)))
-        .replace(/&/g, "&")
-        .replace(/&lt;/g, "<")
-        .replace(/&gt;/g, ">")
-        .replace(/&quot;/g, '"')
-        .replace(/&apos;/g, "'");
-}

package/lib/scripts/retrieve_agenda.js

@@ -9,7 +9,7 @@ import { getSessionsFromStart } from "../types/sessions";
 import { ID_DATE_FORMAT } from "./datautil";
 import { commonOptions } from "./shared/cli_helpers";
 import { fetchWithRetry } from "./shared/util";
-import { groupNonSPByTypeOrganeHour, groupSeancePubliqueBySlot } from "../utils/reunion_grouping";
+import { buildReunionsByBucket } from "../utils/reunion_parsing";
 import { buildSenatDossierIndex } from "../utils/reunion_odj_building";
 const optionsDefinitions = [
     ...commonOptions,
@@ -103,25 +103,14 @@ async function parseAgenda(transformedAgendaSessionDir, agendaFileName, agendaPa
         return;
     const flatPath = path.join(transformedAgendaSessionDir, `${agendaFileName}.json`);
     fs.writeJSONSync(flatPath, parsedAgendaEvents, { spaces: 2 });
-    // 1) SP → grouped by (date, slot)
-    const spGrouped = groupSeancePubliqueBySlot(parsedAgendaEvents, dossierBySenatUrl);
-    // a) on a un Record<TimeSlot, GroupedReunion[]>, on le transforme en array
-    const spGroups = Object.values(spGrouped).flat();
-    // b) (reco) trier pour stabilité, comme pour les NON-SP
-    const PARIS = "Europe/Paris";
-    spGroups.sort((a, b) => {
-        const da = DateTime.fromISO(`${a.date}T${a.startTime || "00:00:00.000+02:00"}`, { zone: PARIS }).toMillis();
-        const db = DateTime.fromISO(`${b.date}T${b.startTime || "00:00:00.000+02:00"}`, { zone: PARIS }).toMillis();
-        // en cas d’égalité, ordre par slot pour stabilité
-        return da - db || (a.slot || "UNKNOWN").localeCompare(b.slot || "UNKNOWN");
-    });
-    if (spGroups.length > 0) {
-        writeGroupsAsFiles(transformedAgendaSessionDir, spGroups);
+    const byBucket = buildReunionsByBucket(parsedAgendaEvents, dossierBySenatUrl);
+    // SP
+    if (byBucket.IDS.length > 0) {
+        writeGroupsAsFiles(transformedAgendaSessionDir, byBucket.IDS);
     }
-    // 2) NON-SP → grouped by (date, organe, hour)
-    const groupedBySuffix = groupNonSPByTypeOrganeHour(parsedAgendaEvents, dossierBySenatUrl);
+    // NON-SP
     for (const suffix of ["IDC", "IDM", "IDO", "IDI"]) {
-        const groups = groupedBySuffix[suffix] || [];
+        const groups = byBucket[suffix];
         if (groups.length > 0) {
             writeGroupsAsFiles(transformedAgendaSessionDir, groups);
         }

package/lib/scripts/retrieve_cr_commission.js

@@ -10,7 +10,7 @@ import { commonOptions } from "./shared/cli_helpers";
 import { sessionStartYearFromDate } from "../model/seance";
 import { getSessionsFromStart } from "../types/sessions";
 import { ensureAndClearDir, fetchWithRetry } from "./shared/util";
-import { jaccardTokenSim } from "../model/util";
+import { jaccard, jaccardTokenSim } from "../utils/scoring";
 class CommissionCRDownloadError extends Error {
     constructor(message, url) {
         super(`An error occurred while retrieving Commission CR ${url}: ${message}`);
@@ -138,15 +138,6 @@ function toTokens(s) {
         .split(/\s+/)
         .filter((t) => t.length >= 3 && !["commission", "des", "de", "du", "d", "la", "le", "les", "et"].includes(t)));
 }
-function jaccard(a, b) {
-    if (!a.size || !b.size)
-        return 0;
-    let inter = 0;
-    for (const t of a)
-        if (b.has(t))
-            inter++;
-    return inter / (a.size + b.size - inter);
-}
 function reunionOrganeCandidates(h) {
     const any = h;
     const out = [any.organeSlug, any.organeKey, any.organe, h.titre].filter(Boolean);

package/lib/scripts/retrieve_cr_seance.d.ts

@@ -1,5 +1,5 @@
 /**
- * Needs to be run after retrieve_agenda.ts !
+ * Needs to be ran after retrieve_agenda.ts script !
  * - downloads the ZIP of comptes-rendus des débats (CRI) from data.senat.fr
  * - extracts XML files, distributes them by session/year
  */