@tricoteuses/senat 2.15.1 → 2.15.2

package/lib/loaders.d.ts

@@ -2,6 +2,7 @@ import { AmendementResult } from "./model/ameli";
 import { DebatResult } from "./model/debats";
 import { DossierLegislatifResult } from "./model/dosleg";
 import { QuestionResult } from "./model/questions";
+import { ScrutinResult } from "./model/scrutins";
 import { CirconscriptionResult, OrganismeResult, SenateurResult } from "./model/sens";
 import { GroupedReunion } from "./types/agenda";
 import { FlatTexte } from "./types/texte";
@@ -88,3 +89,4 @@ export declare function iterLoadSenatCirconscriptions(dataDir: string, options?:
 export declare function iterLoadSenatOrganismes(dataDir: string, options?: {}): Generator<IterItem<OrganismeResult>>;
 export declare function iterLoadSenatSenateurs(dataDir: string, options?: {}): Generator<IterItem<SenateurResult>>;
 export declare function iterLoadSenatQuestions(dataDir: string, legislature: number, options?: {}): Generator<IterItem<QuestionResult>>;
+export declare function iterLoadSenatScrutins(dataDir: string, legislature: number, options?: {}): Generator<IterItem<ScrutinResult>>;

package/lib/loaders.js

@@ -227,3 +227,8 @@ export function* iterLoadSenatQuestions(dataDir, legislature, options = {}) {
         yield questionItem;
     }
 }
+export function* iterLoadSenatScrutins(dataDir, legislature, options = {}) {
+    for (const scrutinItem of iterLoadSenatItems(dataDir, datasets.dosleg.database, legislature, undefined, options)) {
+        yield scrutinItem;
+    }
+}

package/lib/model/compte_rendu.d.ts

@@ -0,0 +1,9 @@
+import { CompteRendu, Sommaire } from "../types/compte_rendu";
+import { TimeSlot } from "../types/agenda";
+export declare function parseCompteRenduSlotFromFile(xmlFilePath: string, wantedSlot: TimeSlot, firstSlotOfDay?: TimeSlot): Promise<CompteRendu | null>;
+export declare function sessionStartYearFromDate(d: Date): number;
+export declare function parseYYYYMMDD(yyyymmdd: string): Date | null;
+export declare function deriveTitreObjetFromSommaire(sommaire: Sommaire | undefined, slot: TimeSlot): {
+    titre: string;
+    objet: string;
+};

package/lib/model/compte_rendu.js

@@ -0,0 +1,325 @@
+import fs from "fs";
+import * as cheerio from "cheerio";
+import path from "path";
+import { computeIntervalsBySlot } from "../utils/cr_spliting";
+import { norm } from "./util";
+const asArray = (x) => x == null ? [] : Array.isArray(x) ? x : [x];
+const toInt = (s) => Number.isFinite(Number(s)) ? Number(s) : Number.POSITIVE_INFINITY;
+export async function parseCompteRenduSlotFromFile(xmlFilePath, wantedSlot, firstSlotOfDay) {
+    try {
+        const raw = fs.readFileSync(xmlFilePath, "utf8");
+        const $ = cheerio.load(raw, { xml: false });
+        const metadonnees = extractMetadonnees($, xmlFilePath);
+        const order = $("body *").toArray();
+        const idx = new Map(order.map((el, i) => [el, i]));
+        const intervalsAll = computeIntervalsBySlot($, idx, firstSlotOfDay);
+        const intervals = intervalsAll.filter(iv => iv.slot === wantedSlot);
+        if (intervals.length === 0) {
+            console.warn(`[CRI] no intervals for ${path.basename(xmlFilePath)} [${wantedSlot}]`);
+            return null;
+        }
+        metadonnees.sommaire = extractSommaireForIntervals($, idx, intervals);
+        const points = [];
+        let ordre = 0;
+        const addPoint = (p) => points.push({ ...p, ordre_absolu_seance: String(++ordre) });
+        // Titles
+        $("cri\\:titreS1 p.titre_S1").each((_, el) => {
+            if (!elementInAnyInterval(el, idx, intervals))
+                return;
+            const t = normalizeTitle(norm($(el).text() || ""));
+            if (t)
+                addPoint({ code_grammaire: "TITRE_TEXTE_DISCUSSION", texte: { _: t }, code_style: "Titre" });
+        });
+        // Interventions
+        $("div.intervenant").each((_, block) => {
+            if (!elementInAnyInterval(block, idx, intervals))
+                return;
+            const $block = $(block);
+            $block.find([
+                "p[class^='titre_S']",
+                "p.mention_titre",
+                "p.intitule_titre",
+                "p.mention_chapitre",
+                "p.intitule_chapitre",
+                "p.mention_article",
+                "p.intitule_article",
+                "p.mention_section",
+                "p.intitule_section",
+            ].join(",")).remove();
+            const firstP = $block.find("p").first();
+            const speakerLabelRaw = firstP.find(".orateur_nom").text() || firstP.find("a.lien_senfic").text() || "";
+            const speakerLabel = dedupeSpeaker(speakerLabelRaw);
+            const { mat, nom: nomCRI, qua: quaCRI } = readIntervenantMeta($block);
+            const qualFromSpans = extractAndRemoveLeadingQualite($, $block);
+            const qualite = norm(decodeHtmlEntities(quaCRI || "")) || qualFromSpans;
+            const canonicalName = dedupeSpeaker(nomCRI || speakerLabel);
+            const role = roleForSpeaker(speakerLabel) || roleForSpeaker(qualite) || roleForSpeaker(quaCRI || "");
+            const speechHtml = sanitizeInterventionHtml($, $block);
+            if (!norm(cheerio.load(speechHtml).text() || ""))
+                return;
+            addPoint({
+                code_grammaire: "PAROLE_GENERIQUE",
+                roledebat: role,
+                orateurs: { orateur: { nom: canonicalName, id: mat || "", qualite } },
+                texte: { _: speechHtml },
+            });
+        });
+        const contenu = {
+            quantiemes: { journee: metadonnees.dateSeance, session: metadonnees.session },
+            point: points,
+        };
+        return {
+            uid: "CRSSN" + xmlFilePath.replace(/^.*?(\d{8}).*$/i, "$1") + `-${wantedSlot}`,
+            seanceRef: "RUSN" + xmlFilePath.replace(/^.*?(\d{8}).*$/i, "$1") + "IDS-" + wantedSlot,
+            sessionRef: metadonnees.session,
+            metadonnees,
+            contenu,
+        };
+    }
+    catch (e) {
+        console.error(`[CRI] parseSlot error file=${xmlFilePath} slot=${wantedSlot}:`, e);
+        return null;
+    }
+}
+export function sessionStartYearFromDate(d) {
+    // Session (1th oct N → 30 sept N+1)
+    const m = d.getMonth();
+    const y = d.getFullYear();
+    return m >= 9 ? y : y - 1;
+}
+export function parseYYYYMMDD(yyyymmdd) {
+    if (!/^\d{8}$/.test(yyyymmdd))
+        return null;
+    const y = Number(yyyymmdd.slice(0, 4));
+    const m = Number(yyyymmdd.slice(4, 6)) - 1;
+    const d = Number(yyyymmdd.slice(6, 8));
+    const dt = new Date(y, m, d);
+    return Number.isFinite(dt.getTime()) ? dt : null;
+}
+export function deriveTitreObjetFromSommaire(sommaire, slot) {
+    const items = extractLevel1Items(sommaire);
+    const meaningful = items.filter(it => !isBoilerplate(it.label));
+    if (meaningful.length === 0) {
+        return {
+            titre: `Séance publique ${slotLabel(slot)}`,
+            objet: "",
+        };
+    }
+    const titre = meaningful[0].label;
+    const objet = meaningful.slice(0, 3).map(it => it.label).join(" ; ");
+    return { titre, objet };
+}
+function slotLabel(slot) {
+    switch (slot) {
+        case "MATIN": return "du matin";
+        case "APRES-MIDI": return "de l’après-midi";
+        case "SOIR": return "du soir";
+        default: return "";
+    }
+}
+const BOILERPLATE_PATTERNS = [
+    /proc(?:è|e)s-?verbal/i,
+    /hommages?/i,
+    /désignation des vice-?président/i,
+    /candidatures? aux?/i,
+    /ordre du jour/i,
+    /rappels? au règlement/i,
+    /communications?/i,
+    /dépôts?/i,
+    /proclamation/i,
+    /présidence de/i,
+    /questions? diverses?/i,
+    /ouverture de la séance/i,
+    /clo(?:t|̂)ure de la séance/i,
+];
+const isBoilerplate = (label) => !label?.trim() || BOILERPLATE_PATTERNS.some(rx => rx.test(label));
+function extractLevel1Items(sommaire) {
+    const level1 = asArray(sommaire?.sommaire1);
+    return level1
+        .map(el => ({
+        numero: toInt(el?.valeur_pts_odj),
+        label: String(el?.titreStruct?.intitule ?? "").trim(),
+    }))
+        .filter(it => !!it.label)
+        .sort((a, b) => a.numero - b.numero);
+}
+function stripTrailingPunct(s) { return s.replace(/\s*([:,.;])\s*$/u, "").trim(); }
+function dedupeSpeaker(raw) {
+    let s = norm(raw);
+    s = stripTrailingPunct(s);
+    const dupPatterns = [/^(.+?)\s*[.]\s*\1$/u, /^(.+?)\s*,\s*\1,?$/u, /^(.+?)\s+\1$/u];
+    for (const re of dupPatterns) {
+        const m = s.match(re);
+        if (m) {
+            s = m[1];
+            break;
+        }
+    }
+    return s.replace(/\.\s*$/, "");
+}
+function decodeHtmlEntities(s) {
+    return s.replace(/&#(\d+);/g, (_, d) => String.fromCharCode(parseInt(d, 10)))
+        .replace(/&#x([0-9a-fA-F]+);/g, (_, h) => String.fromCharCode(parseInt(h, 16)));
+}
+function fixApostrophes(s) {
+    let out = s;
+    out = out.replace(/\s*’\s*/g, "’");
+    out = out.replace(/\b([dljctmsn])\s*’/gi, (_, m) => m + "’");
+    out = out.replace(/’\s+([A-Za-zÀ-ÖØ-öø-ÿ])/g, "’$1");
+    out = out.replace(/\s+([,;:.!?])/g, "$1");
+    return out;
+}
+function normalizeTitle(text) { return text.replace(/^PR[ÉE]SIDENCE DE\b/i, "Présidence de "); }
+function roleForSpeaker(labelOrQualite) {
+    const s = (labelOrQualite || "").toLowerCase();
+    if (/^(m\.|mme)?\s*(le|la)\s+pr[ée]sident(e)?\b/.test(s) || /\bpr[ée]sident[e]?\s+de\s+séance\b/.test(s))
+        return "président";
+    return "";
+}
+function readIntervenantMeta($block) {
+    const int = $block.find('cri\\:intervenant').first();
+    if (int.length)
+        return { mat: int.attr("mat") || undefined, nom: int.attr("nom") || undefined, qua: int.attr("qua") || undefined };
+    const html = $block.html() || "";
+    const m = html.match(/<!--\s*cri:intervenant\b([^>]+)-->/i);
+    if (!m)
+        return {};
+    const out = {};
+    const re = /(\w+)="([^"]*)"/g;
+    let a;
+    while ((a = re.exec(m[1])))
+        out[a[1]] = decodeHtmlEntities(a[2]);
+    return { mat: out["mat"], nom: out["nom"], qua: out["qua"] };
+}
+function extractAndRemoveLeadingQualite($, $block) {
+    const firstP = $block.find("p").first();
+    if (firstP.length === 0)
+        return "";
+    const parts = [];
+    let stop = false;
+    firstP.contents().each((_, node) => {
+        if (stop)
+            return;
+        if (node.type === "tag") {
+            const $node = $(node);
+            if ($node.hasClass("orateur_nom")) {
+                $node.remove();
+                return;
+            }
+            if ($node.hasClass("orateur_qualite")) {
+                parts.push($node.text() || "");
+                $node.remove();
+                return;
+            }
+            const t = norm($node.text() || "");
+            if (t)
+                stop = true;
+            else
+                $node.remove();
+        }
+        else if (node.type === "text") {
+            const t = norm(node.data || "");
+            if (!t || /^[:.,;–—-]+$/.test(t)) {
+                node.data = "";
+                return;
+            }
+            stop = true;
+        }
+    });
+    return fixApostrophes(norm(parts.join(" ")));
+}
+function sanitizeInterventionHtml($, $block) {
+    const $clone = $block.clone();
+    $clone.find('a[name]').remove();
+    $clone.find('div[align="right"]').remove();
+    $clone.find('a.link').remove();
+    $clone.find('img').remove();
+    $clone.find('a#ameli_amendement_cri_phrase, a#ameli_amendement_cra_contenu, a#ameli_amendement_cra_objet').remove();
+    $clone.find(".orateur_nom, .orateur_qualite").remove();
+    let html = $clone.html() || "";
+    html = html.replace(/<!--[\s\S]*?-->/g, "");
+    return html.trim();
+}
+function extractSommaireForIntervals($, idx, intervals) {
+    const inIv = (el) => elementInAnyInterval(el, idx, intervals);
+    const root = $("body");
+    const sommaire = { presidentSeance: { _: "" }, sommaire1: [] };
+    // (1) Présidence (tm2) — première ligne dans l’intervalle 
+    const pres = root.find("p.tm2").filter((_, el) => inIv(el)).first();
+    if (pres.length)
+        sommaire.presidentSeance = { _: norm(pres.text()) };
+    // (2) Paras tm5 présents dans l’intervalle 
+    const paras = [];
+    root.find("p.tm5").each((_, el) => {
+        if (!inIv(el))
+            return;
+        const t = norm($(el).text());
+        if (t)
+            paras.push({ _: t });
+    });
+    if (paras.length)
+        sommaire.para = paras.length === 1 ? paras[0] : paras;
+    // (3) Items de 1er niveau (tm3) présents dans l’intervalle 
+    const items = [];
+    root.find("p.tm3").each((_, el) => {
+        if (!inIv(el))
+            return;
+        const $p = $(el);
+        const full = norm($p.text() || "");
+        if (!full)
+            return;
+        const numMatch = full.match(/^(\d+)\s*[.\-–—]\s*/);
+        const valeur = numMatch ? numMatch[1] : undefined;
+        // prefere intitule in ancre <a> if present
+        const a = $p.find("a").first();
+        const intituleRaw = a.length ? a.text() : full.replace(/^(\d+)\s*[.\-–—]\s*/, "");
+        const intitule = norm(intituleRaw);
+        // id_syceron from href="#Niv1_SOMx"
+        const href = (a.attr("href") || "").trim();
+        const idSyceron = href.startsWith("#") ? href.slice(1) : href;
+        const titreStruct = { id_syceron: idSyceron || "", intitule };
+        items.push({ valeur_pts_odj: valeur, titreStruct });
+    });
+    if (items.length)
+        sommaire.sommaire1 = items;
+    return sommaire;
+}
+function extractMetadonnees($, filePath) {
+    let dateText = norm($("h1, h2, .page-title").first().text() || "");
+    if (!dateText)
+        dateText = norm($("p").first().text() || "");
+    const dateMatch = dateText.match(/\b(\d{1,2}\s+\w+\s+\d{4})\b/i);
+    const allText = norm($("body").text() || "");
+    const sessionMatch = allText.match(/\bsession\s+(\d{4}-\d{4})\b/i);
+    let dateSeance = dateMatch?.[1] || "";
+    if (!dateSeance) {
+        const m = filePath.match(/d(\d{4})(\d{2})(\d{2})\.xml$/i);
+        if (m)
+            dateSeance = `${m[1]}-${m[2]}-${m[3]}`;
+    }
+    return {
+        dateSeance,
+        dateSeanceJour: dateSeance,
+        numSeanceJour: "",
+        numSeance: "",
+        typeAssemblee: "SN",
+        legislature: "",
+        session: sessionMatch?.[1] || "",
+        nomFichierJo: "",
+        validite: "",
+        etat: "",
+        diffusion: "",
+        version: "1.0",
+        environnement: "",
+        heureGeneration: new Date()
+    };
+}
+function elementInAnyInterval(el, idx, intervals) {
+    const p = idx.get(el);
+    if (p == null)
+        return false;
+    for (const iv of intervals)
+        if (p >= iv.start && p < iv.end)
+            return true;
+    return false;
+}

package/lib/model/scrutins.d.ts

@@ -1,3 +1,13 @@
+import { InferResult } from "kysely";
+export type ScrutinResult = InferResult<typeof findAllScrutinsQuery>[0];
+declare const findAllScrutinsQuery: import("kysely").SelectQueryBuilder<{
+    [x: string]: any;
+    [x: number]: any;
+    [x: symbol]: any;
+}, "date_seance" | "lecass" | "lecture" | "scr" | "typlec", {
+    [x: string]: any;
+}>;
 export declare function findAllScrutins(fromSession?: number): AsyncIterableIterator<{
     [x: string]: any;
 }>;
+export {};

package/lib/model/scrutins.js

@@ -1,13 +1,13 @@
 import { jsonArrayFrom } from "kysely/helpers/postgres";
 import { dbSenat } from "../databases";
-import { toDateString } from "./util";
+import { rtrim, toDateString } from "./util";
 function votes(scrutinNum, scrutinDate) {
     return jsonArrayFrom(dbSenat
         .selectFrom("dosleg.votsen")
         .leftJoin("dosleg.titsen", "dosleg.titsen.titsencod", "dosleg.votsen.titsencod")
         .leftJoin("dosleg.stavot", "dosleg.stavot.stavotidt", "dosleg.votsen.stavotidt")
         .leftJoin("dosleg.posvot", "dosleg.posvot.posvotcod", "dosleg.votsen.posvotcod")
-        .leftJoin("sens.sen", "sens.sen.senmat", "dosleg.votsen.senmat")
+        .leftJoin("sens.sen", "dosleg.votsen.senmat", "sens.sen.senmat")
         .leftJoin("sens.memgrppol", (join) => join
         .onRef("sens.memgrppol.senmat", "=", "dosleg.votsen.senmat")
         .onRef("sens.memgrppol.memgrppoldatdeb", "<=", scrutinDate)
@@ -16,6 +16,7 @@ function votes(scrutinNum, scrutinDate) {
         eb("sens.memgrppol.memgrppoldatfin", "is", null)
     ])))
         .where("dosleg.votsen.scrnum", "=", scrutinNum)
+        .where("sens.sen.etasencod", "=", "ACTIF")
         .select([
         "dosleg.votsen.senmat as matricule_votant",
         "dosleg.votsen.senmatdel as matricule_delegant",
@@ -23,7 +24,7 @@ function votes(scrutinNum, scrutinDate) {
         "dosleg.stavot.stavotlib as statut_votant",
         "dosleg.titsen.titsenlib as titre_votant",
         "sens.memgrppol.memgrppolid as groupe_politique_id",
-        "sens.memgrppol.evelib as groupe_politique_libelle",
+        "sens.memgrppol.grppolcod as groupe_politique_code",
     ]));
 }
 function misesAuPoint(scrutinNum) {
@@ -41,11 +42,15 @@ const findAllScrutinsQuery = dbSenat
     .withSchema("dosleg")
     .selectFrom("scr")
     .leftJoin("date_seance", "scr.code", "date_seance.code")
-    .leftJoin("lecture", "date_seance.lecidt", "lecture.lecidt")
+    .leftJoin("lecass", "date_seance.lecidt", "lecass.lecassidt")
+    .leftJoin("lecture", "lecture.lecidt", "lecass.lecidt")
+    .leftJoin("typlec", "typlec.typleccod", "lecture.typleccod")
     .select(({ eb, ref, val }) => [
     "scr.sesann as session",
     "scr.scrnum as numero",
-    "date_seance.lecidt as lecture_id",
+    rtrim(ref("date_seance.lecidt")).as("lecture_id"),
+    rtrim(ref("lecture.leccom")).as("lecture_libelle"),
+    rtrim(ref("typlec.typleclib")).as("type_lecture"),
     toDateString(ref("scr.scrdat")).as("date_scrutin"),
     toDateString(ref("scr.scrdateff")).as("date_scrutin_effective"),
     "scr.scrint as intitule",