@tricoteuses/senat 2.16.2 → 2.16.4

package/lib/model/commission.d.ts

@@ -5,6 +5,7 @@ export declare function getRemainingTextAfterSpeakerHeader($: cheerio.CheerioAPI
 export type DaySection = {
     title: string;
     $start: cheerio.Cheerio<any>;
+    time?: string;
 };
 export declare function cleanTitle(t: string): string;
 export declare function extractDayH3Sections($: cheerio.CheerioAPI, dateISO: string): DaySection[];

package/lib/model/commission.js

@@ -161,33 +161,48 @@ function buildPointsFromParagraphs($, paras) {
     flush();
     return points;
 }
+const TIME_RE = /(?:\b[àa]\s*)?(\d{1,2})\s*(?:h|heures?)\s*(?:([0-5]\d))?/i;
 export function cleanTitle(t) {
     return (t || "").replace(/\s+/g, " ").trim();
 }
+function parseTimeToHHmm(text) {
+    const m = normalizeSpaces(text).match(TIME_RE);
+    if (!m)
+        return undefined;
+    const hh = m[1]?.padStart(2, "0");
+    const mm = (m[2] ?? "00").padStart(2, "0");
+    const h = Number(hh);
+    if (h >= 0 && h <= 23)
+        return `${hh}:${mm}`;
+    return undefined;
+}
+function findNearbyTime($, $h3) {
+    let cur = $h3.prev();
+    for (let i = 0; i < 3 && cur.length; i++, cur = cur.prev()) {
+        const direct = parseTimeToHHmm(cur.text());
+        if (direct)
+            return direct;
+        const italic = parseTimeToHHmm(cur.find("i, em").first().text());
+        if (italic)
+            return italic;
+    }
+    return undefined;
+}
 export function extractDayH3Sections($, dateISO) {
     const sections = [];
     const $dayRoot = findDayRoot($, dateISO);
     if ($dayRoot.length === 0)
         return sections;
-    let $cursor = $dayRoot.next();
-    while ($cursor.length && !$cursor.is("h2")) {
-        if ($cursor.is("h3")) {
-            const title = cleanTitle($cursor.text());
-            if (title)
-                sections.push({ title, $start: $cursor });
-            $cursor = $cursor.next();
-            continue;
-        }
-        const $h3 = $cursor.find("h3").first();
-        if ($h3.length) {
-            const title = cleanTitle($h3.text());
-            if (title)
-                sections.push({ title, $start: $h3 });
-            $cursor = $cursor.next();
-            continue;
-        }
-        $cursor = $cursor.next();
-    }
+    const $range = $dayRoot.nextUntil("h2");
+    const $h3s = $range.filter("h3").add($range.find("h3"));
+    $h3s.each((_, el) => {
+        const $h3 = $(el);
+        const title = cleanTitle($h3.text());
+        if (!title)
+            return;
+        const time = findNearbyTime($, $h3);
+        sections.push({ title, $start: $h3, time });
+    });
     return sections;
 }
 export function parseCommissionCRSectionFromDom($, htmlFilePath, opts) {

package/lib/model/compte_rendu.d.ts

@@ -0,0 +1,9 @@
+import { CompteRendu, Sommaire } from "../types/compte_rendu";
+import { TimeSlot } from "../types/agenda";
+export declare function parseCompteRenduSlotFromFile(xmlFilePath: string, wantedSlot: TimeSlot, firstSlotOfDay?: TimeSlot): Promise<CompteRendu | null>;
+export declare function sessionStartYearFromDate(d: Date): number;
+export declare function parseYYYYMMDD(yyyymmdd: string): Date | null;
+export declare function deriveTitreObjetFromSommaire(sommaire: Sommaire | undefined, slot: TimeSlot): {
+    titre: string;
+    objet: string;
+};

package/lib/model/compte_rendu.js

@@ -0,0 +1,325 @@
+import fs from "fs";
+import * as cheerio from "cheerio";
+import path from "path";
+import { computeIntervalsBySlot } from "../utils/cr_spliting";
+import { norm } from "./util";
+const asArray = (x) => x == null ? [] : Array.isArray(x) ? x : [x];
+const toInt = (s) => Number.isFinite(Number(s)) ? Number(s) : Number.POSITIVE_INFINITY;
+export async function parseCompteRenduSlotFromFile(xmlFilePath, wantedSlot, firstSlotOfDay) {
+    try {
+        const raw = fs.readFileSync(xmlFilePath, "utf8");
+        const $ = cheerio.load(raw, { xml: false });
+        const metadonnees = extractMetadonnees($, xmlFilePath);
+        const order = $("body *").toArray();
+        const idx = new Map(order.map((el, i) => [el, i]));
+        const intervalsAll = computeIntervalsBySlot($, idx, firstSlotOfDay);
+        const intervals = intervalsAll.filter(iv => iv.slot === wantedSlot);
+        if (intervals.length === 0) {
+            console.warn(`[CRI] no intervals for ${path.basename(xmlFilePath)} [${wantedSlot}]`);
+            return null;
+        }
+        metadonnees.sommaire = extractSommaireForIntervals($, idx, intervals);
+        const points = [];
+        let ordre = 0;
+        const addPoint = (p) => points.push({ ...p, ordre_absolu_seance: String(++ordre) });
+        // Titles
+        $("cri\\:titreS1 p.titre_S1").each((_, el) => {
+            if (!elementInAnyInterval(el, idx, intervals))
+                return;
+            const t = normalizeTitle(norm($(el).text() || ""));
+            if (t)
+                addPoint({ code_grammaire: "TITRE_TEXTE_DISCUSSION", texte: { _: t }, code_style: "Titre" });
+        });
+        // Interventions
+        $("div.intervenant").each((_, block) => {
+            if (!elementInAnyInterval(block, idx, intervals))
+                return;
+            const $block = $(block);
+            $block.find([
+                "p[class^='titre_S']",
+                "p.mention_titre",
+                "p.intitule_titre",
+                "p.mention_chapitre",
+                "p.intitule_chapitre",
+                "p.mention_article",
+                "p.intitule_article",
+                "p.mention_section",
+                "p.intitule_section",
+            ].join(",")).remove();
+            const firstP = $block.find("p").first();
+            const speakerLabelRaw = firstP.find(".orateur_nom").text() || firstP.find("a.lien_senfic").text() || "";
+            const speakerLabel = dedupeSpeaker(speakerLabelRaw);
+            const { mat, nom: nomCRI, qua: quaCRI } = readIntervenantMeta($block);
+            const qualFromSpans = extractAndRemoveLeadingQualite($, $block);
+            const qualite = norm(decodeHtmlEntities(quaCRI || "")) || qualFromSpans;
+            const canonicalName = dedupeSpeaker(nomCRI || speakerLabel);
+            const role = roleForSpeaker(speakerLabel) || roleForSpeaker(qualite) || roleForSpeaker(quaCRI || "");
+            const speechHtml = sanitizeInterventionHtml($, $block);
+            if (!norm(cheerio.load(speechHtml).text() || ""))
+                return;
+            addPoint({
+                code_grammaire: "PAROLE_GENERIQUE",
+                roledebat: role,
+                orateurs: { orateur: { nom: canonicalName, id: mat || "", qualite } },
+                texte: { _: speechHtml },
+            });
+        });
+        const contenu = {
+            quantiemes: { journee: metadonnees.dateSeance, session: metadonnees.session },
+            point: points,
+        };
+        return {
+            uid: "CRSSN" + xmlFilePath.replace(/^.*?(\d{8}).*$/i, "$1") + `-${wantedSlot}`,
+            seanceRef: "RUSN" + xmlFilePath.replace(/^.*?(\d{8}).*$/i, "$1") + "IDS-" + wantedSlot,
+            sessionRef: metadonnees.session,
+            metadonnees,
+            contenu,
+        };
+    }
+    catch (e) {
+        console.error(`[CRI] parseSlot error file=${xmlFilePath} slot=${wantedSlot}:`, e);
+        return null;
+    }
+}
+export function sessionStartYearFromDate(d) {
+    // Session (1th oct N → 30 sept N+1)
+    const m = d.getMonth();
+    const y = d.getFullYear();
+    return m >= 9 ? y : y - 1;
+}
+export function parseYYYYMMDD(yyyymmdd) {
+    if (!/^\d{8}$/.test(yyyymmdd))
+        return null;
+    const y = Number(yyyymmdd.slice(0, 4));
+    const m = Number(yyyymmdd.slice(4, 6)) - 1;
+    const d = Number(yyyymmdd.slice(6, 8));
+    const dt = new Date(y, m, d);
+    return Number.isFinite(dt.getTime()) ? dt : null;
+}
+export function deriveTitreObjetFromSommaire(sommaire, slot) {
+    const items = extractLevel1Items(sommaire);
+    const meaningful = items.filter(it => !isBoilerplate(it.label));
+    if (meaningful.length === 0) {
+        return {
+            titre: `Séance publique ${slotLabel(slot)}`,
+            objet: "",
+        };
+    }
+    const titre = meaningful[0].label;
+    const objet = meaningful.slice(0, 3).map(it => it.label).join(" ; ");
+    return { titre, objet };
+}
+function slotLabel(slot) {
+    switch (slot) {
+        case "MATIN": return "du matin";
+        case "APRES-MIDI": return "de l’après-midi";
+        case "SOIR": return "du soir";
+        default: return "";
+    }
+}
+const BOILERPLATE_PATTERNS = [
+    /proc(?:è|e)s-?verbal/i,
+    /hommages?/i,
+    /désignation des vice-?président/i,
+    /candidatures? aux?/i,
+    /ordre du jour/i,
+    /rappels? au règlement/i,
+    /communications?/i,
+    /dépôts?/i,
+    /proclamation/i,
+    /présidence de/i,
+    /questions? diverses?/i,
+    /ouverture de la séance/i,
+    /clo(?:t|̂)ure de la séance/i,
+];
+const isBoilerplate = (label) => !label?.trim() || BOILERPLATE_PATTERNS.some(rx => rx.test(label));
+function extractLevel1Items(sommaire) {
+    const level1 = asArray(sommaire?.sommaire1);
+    return level1
+        .map(el => ({
+        numero: toInt(el?.valeur_pts_odj),
+        label: String(el?.titreStruct?.intitule ?? "").trim(),
+    }))
+        .filter(it => !!it.label)
+        .sort((a, b) => a.numero - b.numero);
+}
+function stripTrailingPunct(s) { return s.replace(/\s*([:,.;])\s*$/u, "").trim(); }
+function dedupeSpeaker(raw) {
+    let s = norm(raw);
+    s = stripTrailingPunct(s);
+    const dupPatterns = [/^(.+?)\s*[.]\s*\1$/u, /^(.+?)\s*,\s*\1,?$/u, /^(.+?)\s+\1$/u];
+    for (const re of dupPatterns) {
+        const m = s.match(re);
+        if (m) {
+            s = m[1];
+            break;
+        }
+    }
+    return s.replace(/\.\s*$/, "");
+}
+function decodeHtmlEntities(s) {
+    return s.replace(/&#(\d+);/g, (_, d) => String.fromCharCode(parseInt(d, 10)))
+        .replace(/&#x([0-9a-fA-F]+);/g, (_, h) => String.fromCharCode(parseInt(h, 16)));
+}
+function fixApostrophes(s) {
+    let out = s;
+    out = out.replace(/\s*’\s*/g, "’");
+    out = out.replace(/\b([dljctmsn])\s*’/gi, (_, m) => m + "’");
+    out = out.replace(/’\s+([A-Za-zÀ-ÖØ-öø-ÿ])/g, "’$1");
+    out = out.replace(/\s+([,;:.!?])/g, "$1");
+    return out;
+}
+function normalizeTitle(text) { return text.replace(/^PR[ÉE]SIDENCE DE\b/i, "Présidence de "); }
+function roleForSpeaker(labelOrQualite) {
+    const s = (labelOrQualite || "").toLowerCase();
+    if (/^(m\.|mme)?\s*(le|la)\s+pr[ée]sident(e)?\b/.test(s) || /\bpr[ée]sident[e]?\s+de\s+séance\b/.test(s))
+        return "président";
+    return "";
+}
+function readIntervenantMeta($block) {
+    const int = $block.find('cri\\:intervenant').first();
+    if (int.length)
+        return { mat: int.attr("mat") || undefined, nom: int.attr("nom") || undefined, qua: int.attr("qua") || undefined };
+    const html = $block.html() || "";
+    const m = html.match(/<!--\s*cri:intervenant\b([^>]+)-->/i);
+    if (!m)
+        return {};
+    const out = {};
+    const re = /(\w+)="([^"]*)"/g;
+    let a;
+    while ((a = re.exec(m[1])))
+        out[a[1]] = decodeHtmlEntities(a[2]);
+    return { mat: out["mat"], nom: out["nom"], qua: out["qua"] };
+}
+function extractAndRemoveLeadingQualite($, $block) {
+    const firstP = $block.find("p").first();
+    if (firstP.length === 0)
+        return "";
+    const parts = [];
+    let stop = false;
+    firstP.contents().each((_, node) => {
+        if (stop)
+            return;
+        if (node.type === "tag") {
+            const $node = $(node);
+            if ($node.hasClass("orateur_nom")) {
+                $node.remove();
+                return;
+            }
+            if ($node.hasClass("orateur_qualite")) {
+                parts.push($node.text() || "");
+                $node.remove();
+                return;
+            }
+            const t = norm($node.text() || "");
+            if (t)
+                stop = true;
+            else
+                $node.remove();
+        }
+        else if (node.type === "text") {
+            const t = norm(node.data || "");
+            if (!t || /^[:.,;–—-]+$/.test(t)) {
+                node.data = "";
+                return;
+            }
+            stop = true;
+        }
+    });
+    return fixApostrophes(norm(parts.join(" ")));
+}
+function sanitizeInterventionHtml($, $block) {
+    const $clone = $block.clone();
+    $clone.find('a[name]').remove();
+    $clone.find('div[align="right"]').remove();
+    $clone.find('a.link').remove();
+    $clone.find('img').remove();
+    $clone.find('a#ameli_amendement_cri_phrase, a#ameli_amendement_cra_contenu, a#ameli_amendement_cra_objet').remove();
+    $clone.find(".orateur_nom, .orateur_qualite").remove();
+    let html = $clone.html() || "";
+    html = html.replace(/<!--[\s\S]*?-->/g, "");
+    return html.trim();
+}
+function extractSommaireForIntervals($, idx, intervals) {
+    const inIv = (el) => elementInAnyInterval(el, idx, intervals);
+    const root = $("body");
+    const sommaire = { presidentSeance: { _: "" }, sommaire1: [] };
+    // (1) Présidence (tm2) — première ligne dans l’intervalle 
+    const pres = root.find("p.tm2").filter((_, el) => inIv(el)).first();
+    if (pres.length)
+        sommaire.presidentSeance = { _: norm(pres.text()) };
+    // (2) Paras tm5 présents dans l’intervalle 
+    const paras = [];
+    root.find("p.tm5").each((_, el) => {
+        if (!inIv(el))
+            return;
+        const t = norm($(el).text());
+        if (t)
+            paras.push({ _: t });
+    });
+    if (paras.length)
+        sommaire.para = paras.length === 1 ? paras[0] : paras;
+    // (3) Items de 1er niveau (tm3) présents dans l’intervalle 
+    const items = [];
+    root.find("p.tm3").each((_, el) => {
+        if (!inIv(el))
+            return;
+        const $p = $(el);
+        const full = norm($p.text() || "");
+        if (!full)
+            return;
+        const numMatch = full.match(/^(\d+)\s*[.\-–—]\s*/);
+        const valeur = numMatch ? numMatch[1] : undefined;
+        // prefere intitule in ancre <a> if present
+        const a = $p.find("a").first();
+        const intituleRaw = a.length ? a.text() : full.replace(/^(\d+)\s*[.\-–—]\s*/, "");
+        const intitule = norm(intituleRaw);
+        // id_syceron from href="#Niv1_SOMx"
+        const href = (a.attr("href") || "").trim();
+        const idSyceron = href.startsWith("#") ? href.slice(1) : href;
+        const titreStruct = { id_syceron: idSyceron || "", intitule };
+        items.push({ valeur_pts_odj: valeur, titreStruct });
+    });
+    if (items.length)
+        sommaire.sommaire1 = items;
+    return sommaire;
+}
+function extractMetadonnees($, filePath) {
+    let dateText = norm($("h1, h2, .page-title").first().text() || "");
+    if (!dateText)
+        dateText = norm($("p").first().text() || "");
+    const dateMatch = dateText.match(/\b(\d{1,2}\s+\w+\s+\d{4})\b/i);
+    const allText = norm($("body").text() || "");
+    const sessionMatch = allText.match(/\bsession\s+(\d{4}-\d{4})\b/i);
+    let dateSeance = dateMatch?.[1] || "";
+    if (!dateSeance) {
+        const m = filePath.match(/d(\d{4})(\d{2})(\d{2})\.xml$/i);
+        if (m)
+            dateSeance = `${m[1]}-${m[2]}-${m[3]}`;
+    }
+    return {
+        dateSeance,
+        dateSeanceJour: dateSeance,
+        numSeanceJour: "",
+        numSeance: "",
+        typeAssemblee: "SN",
+        legislature: "",
+        session: sessionMatch?.[1] || "",
+        nomFichierJo: "",
+        validite: "",
+        etat: "",
+        diffusion: "",
+        version: "1.0",
+        environnement: "",
+        heureGeneration: new Date()
+    };
+}
+function elementInAnyInterval(el, idx, intervals) {
+    const p = idx.get(el);
+    if (p == null)
+        return false;
+    for (const iv of intervals)
+        if (p >= iv.start && p < iv.end)
+            return true;
+    return false;
+}

package/lib/model/seance.js

@@ -237,16 +237,31 @@ function extractAndRemoveLeadingQualite($, $block) {
     return fixApostrophes(norm(parts.join(" ")));
 }
 function sanitizeInterventionHtml($, $block) {
-    const $clone = $block.clone();
-    $clone.find("a[name]").remove();
-    $clone.find('div[align="right"]').remove();
-    $clone.find("a.link").remove();
-    $clone.find("img").remove();
-    $clone.find("a#ameli_amendement_cri_phrase, a#ameli_amendement_cra_contenu, a#ameli_amendement_cra_objet").remove();
-    $clone.find(".orateur_nom, .orateur_qualite").remove();
-    let html = $clone.html() || "";
-    html = html.replace(/<!--[\s\S]*?-->/g, "");
-    return html.trim();
+    const ps = $block.find("p").toArray();
+    const cleaned = ps
+        .map((p) => {
+        const $p = $(p).clone();
+        $p.find(".orateur_nom, .orateur_qualite").remove();
+        $p.find("a").each((_, a) => {
+            const $a = $(a);
+            $a.replaceWith($a.text());
+        });
+        $p.find(".info_entre_parentheses").each((_, el) => {
+            const txt = $(el).text();
+            $(el).replaceWith($("<em/>").text(txt));
+        });
+        $p.find("span").each((_, span) => {
+            const $s = $(span);
+            if (!$s.text().trim())
+                $s.remove();
+        });
+        const inner = ($p.html() || "").trim();
+        if (!inner)
+            return null;
+        return `<p>${inner}</p>`;
+    })
+        .filter(Boolean);
+    return cleaned.join("<br/>");
 }
 function extractSommaireForIntervals($, idx, intervals) {
     const inIv = (el) => elementInAnyInterval(el, idx, intervals);