@tricoteuses/senat 2.14.6 → 2.14.7

package/lib/datasets.js

@@ -109,7 +109,7 @@ export const datasets = {
     },
     debats: {
         database: "debats",
-        repairEncoding: true,
+        repairEncoding: false,
         title: "Informations relatives aux comptes rendus intégraux de la séance publique",
         url: "https://data.senat.fr/data/debats/debats.zip",
         indexes: {
@@ -134,7 +134,7 @@ export const datasets = {
     },
     dosleg: {
         database: "dosleg",
-        repairEncoding: true,
+        repairEncoding: false,
         title: "Dossiers législatifs",
         url: "https://data.senat.fr/data/dosleg/dosleg.zip",
         indexes: {
@@ -229,7 +229,7 @@ export const datasets = {
     },
     questions: {
         database: "questions",
-        repairEncoding: true,
+        repairEncoding: false,
         title: "Questions écrites et orales posées par les sénateurs au Gouvernement",
         url: "https://data.senat.fr/data/questions/questions.zip",
         indexes: {
@@ -253,7 +253,7 @@ export const datasets = {
     },
     sens: {
         database: "sens",
-        repairEncoding: true,
+        repairEncoding: false,
         repairZip: (dataset, dataDir) => {
             const sqlFilename = `${dataset.database}.sql`;
             const sqlFilePath = path.join(dataDir, sqlFilename);

package/lib/model/compte_rendu.d.ts

@@ -0,0 +1,9 @@
+import { CompteRendu, Sommaire } from "../types/compte_rendu";
+import { TimeSlot } from "../types/agenda";
+export declare function parseCompteRenduSlotFromFile(xmlFilePath: string, wantedSlot: TimeSlot, firstSlotOfDay?: TimeSlot): Promise<CompteRendu | null>;
+export declare function sessionStartYearFromDate(d: Date): number;
+export declare function parseYYYYMMDD(yyyymmdd: string): Date | null;
+export declare function deriveTitreObjetFromSommaire(sommaire: Sommaire | undefined, slot: TimeSlot): {
+    titre: string;
+    objet: string;
+};

package/lib/model/compte_rendu.js

@@ -0,0 +1,325 @@
+import fs from "fs";
+import * as cheerio from "cheerio";
+import path from "path";
+import { computeIntervalsBySlot } from "../utils/cr_spliting";
+import { norm } from "./util";
+const asArray = (x) => x == null ? [] : Array.isArray(x) ? x : [x];
+const toInt = (s) => Number.isFinite(Number(s)) ? Number(s) : Number.POSITIVE_INFINITY;
+export async function parseCompteRenduSlotFromFile(xmlFilePath, wantedSlot, firstSlotOfDay) {
+    try {
+        const raw = fs.readFileSync(xmlFilePath, "utf8");
+        const $ = cheerio.load(raw, { xml: false });
+        const metadonnees = extractMetadonnees($, xmlFilePath);
+        const order = $("body *").toArray();
+        const idx = new Map(order.map((el, i) => [el, i]));
+        const intervalsAll = computeIntervalsBySlot($, idx, firstSlotOfDay);
+        const intervals = intervalsAll.filter(iv => iv.slot === wantedSlot);
+        if (intervals.length === 0) {
+            console.warn(`[CRI] no intervals for ${path.basename(xmlFilePath)} [${wantedSlot}]`);
+            return null;
+        }
+        metadonnees.sommaire = extractSommaireForIntervals($, idx, intervals);
+        const points = [];
+        let ordre = 0;
+        const addPoint = (p) => points.push({ ...p, ordre_absolu_seance: String(++ordre) });
+        // Titles
+        $("cri\\:titreS1 p.titre_S1").each((_, el) => {
+            if (!elementInAnyInterval(el, idx, intervals))
+                return;
+            const t = normalizeTitle(norm($(el).text() || ""));
+            if (t)
+                addPoint({ code_grammaire: "TITRE_TEXTE_DISCUSSION", texte: { _: t }, code_style: "Titre" });
+        });
+        // Interventions
+        $("div.intervenant").each((_, block) => {
+            if (!elementInAnyInterval(block, idx, intervals))
+                return;
+            const $block = $(block);
+            $block.find([
+                "p[class^='titre_S']",
+                "p.mention_titre",
+                "p.intitule_titre",
+                "p.mention_chapitre",
+                "p.intitule_chapitre",
+                "p.mention_article",
+                "p.intitule_article",
+                "p.mention_section",
+                "p.intitule_section",
+            ].join(",")).remove();
+            const firstP = $block.find("p").first();
+            const speakerLabelRaw = firstP.find(".orateur_nom").text() || firstP.find("a.lien_senfic").text() || "";
+            const speakerLabel = dedupeSpeaker(speakerLabelRaw);
+            const { mat, nom: nomCRI, qua: quaCRI } = readIntervenantMeta($block);
+            const qualFromSpans = extractAndRemoveLeadingQualite($, $block);
+            const qualite = norm(decodeHtmlEntities(quaCRI || "")) || qualFromSpans;
+            const canonicalName = dedupeSpeaker(nomCRI || speakerLabel);
+            const role = roleForSpeaker(speakerLabel) || roleForSpeaker(qualite) || roleForSpeaker(quaCRI || "");
+            const speechHtml = sanitizeInterventionHtml($, $block);
+            if (!norm(cheerio.load(speechHtml).text() || ""))
+                return;
+            addPoint({
+                code_grammaire: "PAROLE_GENERIQUE",
+                roledebat: role,
+                orateurs: { orateur: { nom: canonicalName, id: mat || "", qualite } },
+                texte: { _: speechHtml },
+            });
+        });
+        const contenu = {
+            quantiemes: { journee: metadonnees.dateSeance, session: metadonnees.session },
+            point: points,
+        };
+        return {
+            uid: "CRSSN" + xmlFilePath.replace(/^.*?(\d{8}).*$/i, "$1") + `-${wantedSlot}`,
+            seanceRef: "RUSN" + xmlFilePath.replace(/^.*?(\d{8}).*$/i, "$1") + "IDS-" + wantedSlot,
+            sessionRef: metadonnees.session,
+            metadonnees,
+            contenu,
+        };
+    }
+    catch (e) {
+        console.error(`[CRI] parseSlot error file=${xmlFilePath} slot=${wantedSlot}:`, e);
+        return null;
+    }
+}
+export function sessionStartYearFromDate(d) {
+    // Session (1th oct N → 30 sept N+1)
+    const m = d.getMonth();
+    const y = d.getFullYear();
+    return m >= 9 ? y : y - 1;
+}
+export function parseYYYYMMDD(yyyymmdd) {
+    if (!/^\d{8}$/.test(yyyymmdd))
+        return null;
+    const y = Number(yyyymmdd.slice(0, 4));
+    const m = Number(yyyymmdd.slice(4, 6)) - 1;
+    const d = Number(yyyymmdd.slice(6, 8));
+    const dt = new Date(y, m, d);
+    return Number.isFinite(dt.getTime()) ? dt : null;
+}
+export function deriveTitreObjetFromSommaire(sommaire, slot) {
+    const items = extractLevel1Items(sommaire);
+    const meaningful = items.filter(it => !isBoilerplate(it.label));
+    if (meaningful.length === 0) {
+        return {
+            titre: `Séance publique ${slotLabel(slot)}`,
+            objet: "",
+        };
+    }
+    const titre = meaningful[0].label;
+    const objet = meaningful.slice(0, 3).map(it => it.label).join(" ; ");
+    return { titre, objet };
+}
+function slotLabel(slot) {
+    switch (slot) {
+        case "MATIN": return "du matin";
+        case "APRES-MIDI": return "de l’après-midi";
+        case "SOIR": return "du soir";
+        default: return "";
+    }
+}
+const BOILERPLATE_PATTERNS = [
+    /proc(?:è|e)s-?verbal/i,
+    /hommages?/i,
+    /désignation des vice-?président/i,
+    /candidatures? aux?/i,
+    /ordre du jour/i,
+    /rappels? au règlement/i,
+    /communications?/i,
+    /dépôts?/i,
+    /proclamation/i,
+    /présidence de/i,
+    /questions? diverses?/i,
+    /ouverture de la séance/i,
+    /clo(?:t|̂)ure de la séance/i,
+];
+const isBoilerplate = (label) => !label?.trim() || BOILERPLATE_PATTERNS.some(rx => rx.test(label));
+function extractLevel1Items(sommaire) {
+    const level1 = asArray(sommaire?.sommaire1);
+    return level1
+        .map(el => ({
+        numero: toInt(el?.valeur_pts_odj),
+        label: String(el?.titreStruct?.intitule ?? "").trim(),
+    }))
+        .filter(it => !!it.label)
+        .sort((a, b) => a.numero - b.numero);
+}
+function stripTrailingPunct(s) { return s.replace(/\s*([:,.;])\s*$/u, "").trim(); }
+function dedupeSpeaker(raw) {
+    let s = norm(raw);
+    s = stripTrailingPunct(s);
+    const dupPatterns = [/^(.+?)\s*[.]\s*\1$/u, /^(.+?)\s*,\s*\1,?$/u, /^(.+?)\s+\1$/u];
+    for (const re of dupPatterns) {
+        const m = s.match(re);
+        if (m) {
+            s = m[1];
+            break;
+        }
+    }
+    return s.replace(/\.\s*$/, "");
+}
+function decodeHtmlEntities(s) {
+    return s.replace(/&#(\d+);/g, (_, d) => String.fromCharCode(parseInt(d, 10)))
+        .replace(/&#x([0-9a-fA-F]+);/g, (_, h) => String.fromCharCode(parseInt(h, 16)));
+}
+function fixApostrophes(s) {
+    let out = s;
+    out = out.replace(/\s*’\s*/g, "’");
+    out = out.replace(/\b([dljctmsn])\s*’/gi, (_, m) => m + "’");
+    out = out.replace(/’\s+([A-Za-zÀ-ÖØ-öø-ÿ])/g, "’$1");
+    out = out.replace(/\s+([,;:.!?])/g, "$1");
+    return out;
+}
+function normalizeTitle(text) { return text.replace(/^PR[ÉE]SIDENCE DE\b/i, "Présidence de "); }
+function roleForSpeaker(labelOrQualite) {
+    const s = (labelOrQualite || "").toLowerCase();
+    if (/^(m\.|mme)?\s*(le|la)\s+pr[ée]sident(e)?\b/.test(s) || /\bpr[ée]sident[e]?\s+de\s+séance\b/.test(s))
+        return "président";
+    return "";
+}
+function readIntervenantMeta($block) {
+    const int = $block.find('cri\\:intervenant').first();
+    if (int.length)
+        return { mat: int.attr("mat") || undefined, nom: int.attr("nom") || undefined, qua: int.attr("qua") || undefined };
+    const html = $block.html() || "";
+    const m = html.match(/<!--\s*cri:intervenant\b([^>]+)-->/i);
+    if (!m)
+        return {};
+    const out = {};
+    const re = /(\w+)="([^"]*)"/g;
+    let a;
+    while ((a = re.exec(m[1])))
+        out[a[1]] = decodeHtmlEntities(a[2]);
+    return { mat: out["mat"], nom: out["nom"], qua: out["qua"] };
+}
+function extractAndRemoveLeadingQualite($, $block) {
+    const firstP = $block.find("p").first();
+    if (firstP.length === 0)
+        return "";
+    const parts = [];
+    let stop = false;
+    firstP.contents().each((_, node) => {
+        if (stop)
+            return;
+        if (node.type === "tag") {
+            const $node = $(node);
+            if ($node.hasClass("orateur_nom")) {
+                $node.remove();
+                return;
+            }
+            if ($node.hasClass("orateur_qualite")) {
+                parts.push($node.text() || "");
+                $node.remove();
+                return;
+            }
+            const t = norm($node.text() || "");
+            if (t)
+                stop = true;
+            else
+                $node.remove();
+        }
+        else if (node.type === "text") {
+            const t = norm(node.data || "");
+            if (!t || /^[:.,;–—-]+$/.test(t)) {
+                node.data = "";
+                return;
+            }
+            stop = true;
+        }
+    });
+    return fixApostrophes(norm(parts.join(" ")));
+}
+function sanitizeInterventionHtml($, $block) {
+    const $clone = $block.clone();
+    $clone.find('a[name]').remove();
+    $clone.find('div[align="right"]').remove();
+    $clone.find('a.link').remove();
+    $clone.find('img').remove();
+    $clone.find('a#ameli_amendement_cri_phrase, a#ameli_amendement_cra_contenu, a#ameli_amendement_cra_objet').remove();
+    $clone.find(".orateur_nom, .orateur_qualite").remove();
+    let html = $clone.html() || "";
+    html = html.replace(/<!--[\s\S]*?-->/g, "");
+    return html.trim();
+}
+function extractSommaireForIntervals($, idx, intervals) {
+    const inIv = (el) => elementInAnyInterval(el, idx, intervals);
+    const root = $("body");
+    const sommaire = { presidentSeance: { _: "" }, sommaire1: [] };
+    // (1) Présidence (tm2) — première ligne dans l’intervalle 
+    const pres = root.find("p.tm2").filter((_, el) => inIv(el)).first();
+    if (pres.length)
+        sommaire.presidentSeance = { _: norm(pres.text()) };
+    // (2) Paras tm5 présents dans l’intervalle 
+    const paras = [];
+    root.find("p.tm5").each((_, el) => {
+        if (!inIv(el))
+            return;
+        const t = norm($(el).text());
+        if (t)
+            paras.push({ _: t });
+    });
+    if (paras.length)
+        sommaire.para = paras.length === 1 ? paras[0] : paras;
+    // (3) Items de 1er niveau (tm3) présents dans l’intervalle 
+    const items = [];
+    root.find("p.tm3").each((_, el) => {
+        if (!inIv(el))
+            return;
+        const $p = $(el);
+        const full = norm($p.text() || "");
+        if (!full)
+            return;
+        const numMatch = full.match(/^(\d+)\s*[.\-–—]\s*/);
+        const valeur = numMatch ? numMatch[1] : undefined;
+        // prefere intitule in ancre <a> if present
+        const a = $p.find("a").first();
+        const intituleRaw = a.length ? a.text() : full.replace(/^(\d+)\s*[.\-–—]\s*/, "");
+        const intitule = norm(intituleRaw);
+        // id_syceron from href="#Niv1_SOMx"
+        const href = (a.attr("href") || "").trim();
+        const idSyceron = href.startsWith("#") ? href.slice(1) : href;
+        const titreStruct = { id_syceron: idSyceron || "", intitule };
+        items.push({ valeur_pts_odj: valeur, titreStruct });
+    });
+    if (items.length)
+        sommaire.sommaire1 = items;
+    return sommaire;
+}
+function extractMetadonnees($, filePath) {
+    let dateText = norm($("h1, h2, .page-title").first().text() || "");
+    if (!dateText)
+        dateText = norm($("p").first().text() || "");
+    const dateMatch = dateText.match(/\b(\d{1,2}\s+\w+\s+\d{4})\b/i);
+    const allText = norm($("body").text() || "");
+    const sessionMatch = allText.match(/\bsession\s+(\d{4}-\d{4})\b/i);
+    let dateSeance = dateMatch?.[1] || "";
+    if (!dateSeance) {
+        const m = filePath.match(/d(\d{4})(\d{2})(\d{2})\.xml$/i);
+        if (m)
+            dateSeance = `${m[1]}-${m[2]}-${m[3]}`;
+    }
+    return {
+        dateSeance,
+        dateSeanceJour: dateSeance,
+        numSeanceJour: "",
+        numSeance: "",
+        typeAssemblee: "SN",
+        legislature: "",
+        session: sessionMatch?.[1] || "",
+        nomFichierJo: "",
+        validite: "",
+        etat: "",
+        diffusion: "",
+        version: "1.0",
+        environnement: "",
+        heureGeneration: new Date()
+    };
+}
+function elementInAnyInterval(el, idx, intervals) {
+    const p = idx.get(el);
+    if (p == null)
+        return false;
+    for (const iv of intervals)
+        if (p >= iv.start && p < iv.end)
+            return true;
+    return false;
+}

package/lib/model/dosleg.js

@@ -355,7 +355,7 @@ export function getCodeActeTexte(codeParent, texteOrigine) {
     if (texteOrigine === "retiré par l'auteur") {
         return `${codeParent}-RTRINI`;
     }
-    if (texteOrigine.includes("adopté") || texteOrigine.includes("rejeté") || texteOrigine.includes("modifié")) {
+    if (texteOrigine.includes("adopté") || texteOrigine.includes("rejeté") || texteOrigine.includes("modifié") || texteOrigine === "devenu résolution du Sénat") {
         return `${codeParent}-DEBATS-DEC`;
     }
     return null;