@tricoteuses/senat 2.14.0 → 2.14.2

package/lib/loaders.d.ts

@@ -83,7 +83,7 @@ export declare function loadSenatTexteContent(dataDir: string, textePathFromData
 export declare function loadSenatCompteRenduContent(dataDir: string, session: number, debatId: string | number): {
     item: CompteRendu | null;
 };
-export declare function iterLoadSenatAgendasGrouped(dataDir: string, session: number | undefined): Generator<IterItem<GroupedReunion>>;
+export declare function iterLoadSenatAgendas(dataDir: string, session: number | undefined): Generator<IterItem<GroupedReunion>>;
 export declare function iterLoadSenatCirconscriptions(dataDir: string, options?: {}): Generator<IterItem<CirconscriptionResult>>;
 export declare function iterLoadSenatOrganismes(dataDir: string, options?: {}): Generator<IterItem<OrganismeResult>>;
 export declare function iterLoadSenatSenateurs(dataDir: string, options?: {}): Generator<IterItem<SenateurResult>>;

package/lib/loaders.js

@@ -183,7 +183,7 @@ export function loadSenatCompteRenduContent(dataDir, session, debatId) {
     const json = fs.readFileSync(fullPath, { encoding: "utf8" });
     return { item: JSON.parse(json) };
 }
-export function* iterLoadSenatAgendasGrouped(dataDir, session) {
+export function* iterLoadSenatAgendas(dataDir, session) {
     const baseDir = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session ?? ""));
     if (!fs.existsSync(baseDir))
         return;

package/lib/scripts/retrieve_videos.js

@@ -4,7 +4,7 @@ import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 import fsp from "fs/promises";
 import path from "path";
-import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatAgendasGrouped } from "../loaders";
+import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatAgendas } from "../loaders";
 import { getSessionsFromStart } from "../types/sessions";
 import { commonOptions } from "./shared/cli_helpers";
 // ===================== Constants =====================
@@ -398,7 +398,7 @@ async function processGroupedReunion(agenda, session, dataDir) {
 async function processAll(dataDir, sessions) {
     console.log("Process all Agendas and fetch video's url");
     for (const session of sessions) {
-        for (const { item: agenda } of iterLoadSenatAgendasGrouped(dataDir, session)) {
+        for (const { item: agenda } of iterLoadSenatAgendas(dataDir, session)) {
             try {
                 await processGroupedReunion(agenda, session, dataDir);
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.14.0",
+  "version": "2.14.2",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",

package/lib/model/compte_rendu.d.ts

@@ -1,9 +0,0 @@
-import { CompteRendu, Sommaire } from "../types/compte_rendu";
-import { TimeSlot } from "../types/agenda";
-export declare function parseCompteRenduSlotFromFile(xmlFilePath: string, wantedSlot: TimeSlot, firstSlotOfDay?: TimeSlot): Promise<CompteRendu | null>;
-export declare function sessionStartYearFromDate(d: Date): number;
-export declare function parseYYYYMMDD(yyyymmdd: string): Date | null;
-export declare function deriveTitreObjetFromSommaire(sommaire: Sommaire | undefined, slot: TimeSlot): {
-    titre: string;
-    objet: string;
-};

package/lib/model/compte_rendu.js

@@ -1,325 +0,0 @@
-import fs from "fs";
-import * as cheerio from "cheerio";
-import path from "path";
-import { computeIntervalsBySlot } from "../utils/cr_spliting";
-import { norm } from "./util";
-const asArray = (x) => x == null ? [] : Array.isArray(x) ? x : [x];
-const toInt = (s) => Number.isFinite(Number(s)) ? Number(s) : Number.POSITIVE_INFINITY;
-export async function parseCompteRenduSlotFromFile(xmlFilePath, wantedSlot, firstSlotOfDay) {
-    try {
-        const raw = fs.readFileSync(xmlFilePath, "utf8");
-        const $ = cheerio.load(raw, { xml: false });
-        const metadonnees = extractMetadonnees($, xmlFilePath);
-        const order = $("body *").toArray();
-        const idx = new Map(order.map((el, i) => [el, i]));
-        const intervalsAll = computeIntervalsBySlot($, idx, firstSlotOfDay);
-        const intervals = intervalsAll.filter(iv => iv.slot === wantedSlot);
-        if (intervals.length === 0) {
-            console.warn(`[CRI] no intervals for ${path.basename(xmlFilePath)} [${wantedSlot}]`);
-            return null;
-        }
-        metadonnees.sommaire = extractSommaireForIntervals($, idx, intervals);
-        const points = [];
-        let ordre = 0;
-        const addPoint = (p) => points.push({ ...p, ordre_absolu_seance: String(++ordre) });
-        // Titles
-        $("cri\\:titreS1 p.titre_S1").each((_, el) => {
-            if (!elementInAnyInterval(el, idx, intervals))
-                return;
-            const t = normalizeTitle(norm($(el).text() || ""));
-            if (t)
-                addPoint({ code_grammaire: "TITRE_TEXTE_DISCUSSION", texte: { _: t }, code_style: "Titre" });
-        });
-        // Interventions
-        $("div.intervenant").each((_, block) => {
-            if (!elementInAnyInterval(block, idx, intervals))
-                return;
-            const $block = $(block);
-            $block.find([
-                "p[class^='titre_S']",
-                "p.mention_titre",
-                "p.intitule_titre",
-                "p.mention_chapitre",
-                "p.intitule_chapitre",
-                "p.mention_article",
-                "p.intitule_article",
-                "p.mention_section",
-                "p.intitule_section",
-            ].join(",")).remove();
-            const firstP = $block.find("p").first();
-            const speakerLabelRaw = firstP.find(".orateur_nom").text() || firstP.find("a.lien_senfic").text() || "";
-            const speakerLabel = dedupeSpeaker(speakerLabelRaw);
-            const { mat, nom: nomCRI, qua: quaCRI } = readIntervenantMeta($block);
-            const qualFromSpans = extractAndRemoveLeadingQualite($, $block);
-            const qualite = norm(decodeHtmlEntities(quaCRI || "")) || qualFromSpans;
-            const canonicalName = dedupeSpeaker(nomCRI || speakerLabel);
-            const role = roleForSpeaker(speakerLabel) || roleForSpeaker(qualite) || roleForSpeaker(quaCRI || "");
-            const speechHtml = sanitizeInterventionHtml($, $block);
-            if (!norm(cheerio.load(speechHtml).text() || ""))
-                return;
-            addPoint({
-                code_grammaire: "PAROLE_GENERIQUE",
-                roledebat: role,
-                orateurs: { orateur: { nom: canonicalName, id: mat || "", qualite } },
-                texte: { _: speechHtml },
-            });
-        });
-        const contenu = {
-            quantiemes: { journee: metadonnees.dateSeance, session: metadonnees.session },
-            point: points,
-        };
-        return {
-            uid: "CRSSN" + xmlFilePath.replace(/^.*?(\d{8}).*$/i, "$1") + `-${wantedSlot}`,
-            seanceRef: "RUSN" + xmlFilePath.replace(/^.*?(\d{8}).*$/i, "$1") + "IDS-" + wantedSlot,
-            sessionRef: metadonnees.session,
-            metadonnees,
-            contenu,
-        };
-    }
-    catch (e) {
-        console.error(`[CRI] parseSlot error file=${xmlFilePath} slot=${wantedSlot}:`, e);
-        return null;
-    }
-}
-export function sessionStartYearFromDate(d) {
-    // Session (1th oct N → 30 sept N+1)
-    const m = d.getMonth();
-    const y = d.getFullYear();
-    return m >= 9 ? y : y - 1;
-}
-export function parseYYYYMMDD(yyyymmdd) {
-    if (!/^\d{8}$/.test(yyyymmdd))
-        return null;
-    const y = Number(yyyymmdd.slice(0, 4));
-    const m = Number(yyyymmdd.slice(4, 6)) - 1;
-    const d = Number(yyyymmdd.slice(6, 8));
-    const dt = new Date(y, m, d);
-    return Number.isFinite(dt.getTime()) ? dt : null;
-}
-export function deriveTitreObjetFromSommaire(sommaire, slot) {
-    const items = extractLevel1Items(sommaire);
-    const meaningful = items.filter(it => !isBoilerplate(it.label));
-    if (meaningful.length === 0) {
-        return {
-            titre: `Séance publique ${slotLabel(slot)}`,
-            objet: "",
-        };
-    }
-    const titre = meaningful[0].label;
-    const objet = meaningful.slice(0, 3).map(it => it.label).join(" ; ");
-    return { titre, objet };
-}
-function slotLabel(slot) {
-    switch (slot) {
-        case "MATIN": return "du matin";
-        case "APRES-MIDI": return "de l’après-midi";
-        case "SOIR": return "du soir";
-        default: return "";
-    }
-}
-const BOILERPLATE_PATTERNS = [
-    /proc(?:è|e)s-?verbal/i,
-    /hommages?/i,
-    /désignation des vice-?président/i,
-    /candidatures? aux?/i,
-    /ordre du jour/i,
-    /rappels? au règlement/i,
-    /communications?/i,
-    /dépôts?/i,
-    /proclamation/i,
-    /présidence de/i,
-    /questions? diverses?/i,
-    /ouverture de la séance/i,
-    /clo(?:t|̂)ure de la séance/i,
-];
-const isBoilerplate = (label) => !label?.trim() || BOILERPLATE_PATTERNS.some(rx => rx.test(label));
-function extractLevel1Items(sommaire) {
-    const level1 = asArray(sommaire?.sommaire1);
-    return level1
-        .map(el => ({
-        numero: toInt(el?.valeur_pts_odj),
-        label: String(el?.titreStruct?.intitule ?? "").trim(),
-    }))
-        .filter(it => !!it.label)
-        .sort((a, b) => a.numero - b.numero);
-}
-function stripTrailingPunct(s) { return s.replace(/\s*([:,.;])\s*$/u, "").trim(); }
-function dedupeSpeaker(raw) {
-    let s = norm(raw);
-    s = stripTrailingPunct(s);
-    const dupPatterns = [/^(.+?)\s*[.]\s*\1$/u, /^(.+?)\s*,\s*\1,?$/u, /^(.+?)\s+\1$/u];
-    for (const re of dupPatterns) {
-        const m = s.match(re);
-        if (m) {
-            s = m[1];
-            break;
-        }
-    }
-    return s.replace(/\.\s*$/, "");
-}
-function decodeHtmlEntities(s) {
-    return s.replace(/&#(\d+);/g, (_, d) => String.fromCharCode(parseInt(d, 10)))
-        .replace(/&#x([0-9a-fA-F]+);/g, (_, h) => String.fromCharCode(parseInt(h, 16)));
-}
-function fixApostrophes(s) {
-    let out = s;
-    out = out.replace(/\s*’\s*/g, "’");
-    out = out.replace(/\b([dljctmsn])\s*’/gi, (_, m) => m + "’");
-    out = out.replace(/’\s+([A-Za-zÀ-ÖØ-öø-ÿ])/g, "’$1");
-    out = out.replace(/\s+([,;:.!?])/g, "$1");
-    return out;
-}
-function normalizeTitle(text) { return text.replace(/^PR[ÉE]SIDENCE DE\b/i, "Présidence de "); }
-function roleForSpeaker(labelOrQualite) {
-    const s = (labelOrQualite || "").toLowerCase();
-    if (/^(m\.|mme)?\s*(le|la)\s+pr[ée]sident(e)?\b/.test(s) || /\bpr[ée]sident[e]?\s+de\s+séance\b/.test(s))
-        return "président";
-    return "";
-}
-function readIntervenantMeta($block) {
-    const int = $block.find('cri\\:intervenant').first();
-    if (int.length)
-        return { mat: int.attr("mat") || undefined, nom: int.attr("nom") || undefined, qua: int.attr("qua") || undefined };
-    const html = $block.html() || "";
-    const m = html.match(/<!--\s*cri:intervenant\b([^>]+)-->/i);
-    if (!m)
-        return {};
-    const out = {};
-    const re = /(\w+)="([^"]*)"/g;
-    let a;
-    while ((a = re.exec(m[1])))
-        out[a[1]] = decodeHtmlEntities(a[2]);
-    return { mat: out["mat"], nom: out["nom"], qua: out["qua"] };
-}
-function extractAndRemoveLeadingQualite($, $block) {
-    const firstP = $block.find("p").first();
-    if (firstP.length === 0)
-        return "";
-    const parts = [];
-    let stop = false;
-    firstP.contents().each((_, node) => {
-        if (stop)
-            return;
-        if (node.type === "tag") {
-            const $node = $(node);
-            if ($node.hasClass("orateur_nom")) {
-                $node.remove();
-                return;
-            }
-            if ($node.hasClass("orateur_qualite")) {
-                parts.push($node.text() || "");
-                $node.remove();
-                return;
-            }
-            const t = norm($node.text() || "");
-            if (t)
-                stop = true;
-            else
-                $node.remove();
-        }
-        else if (node.type === "text") {
-            const t = norm(node.data || "");
-            if (!t || /^[:.,;–—-]+$/.test(t)) {
-                node.data = "";
-                return;
-            }
-            stop = true;
-        }
-    });
-    return fixApostrophes(norm(parts.join(" ")));
-}
-function sanitizeInterventionHtml($, $block) {
-    const $clone = $block.clone();
-    $clone.find('a[name]').remove();
-    $clone.find('div[align="right"]').remove();
-    $clone.find('a.link').remove();
-    $clone.find('img').remove();
-    $clone.find('a#ameli_amendement_cri_phrase, a#ameli_amendement_cra_contenu, a#ameli_amendement_cra_objet').remove();
-    $clone.find(".orateur_nom, .orateur_qualite").remove();
-    let html = $clone.html() || "";
-    html = html.replace(/<!--[\s\S]*?-->/g, "");
-    return html.trim();
-}
-function extractSommaireForIntervals($, idx, intervals) {
-    const inIv = (el) => elementInAnyInterval(el, idx, intervals);
-    const root = $("body");
-    const sommaire = { presidentSeance: { _: "" }, sommaire1: [] };
-    // (1) Présidence (tm2) — première ligne dans l’intervalle 
-    const pres = root.find("p.tm2").filter((_, el) => inIv(el)).first();
-    if (pres.length)
-        sommaire.presidentSeance = { _: norm(pres.text()) };
-    // (2) Paras tm5 présents dans l’intervalle 
-    const paras = [];
-    root.find("p.tm5").each((_, el) => {
-        if (!inIv(el))
-            return;
-        const t = norm($(el).text());
-        if (t)
-            paras.push({ _: t });
-    });
-    if (paras.length)
-        sommaire.para = paras.length === 1 ? paras[0] : paras;
-    // (3) Items de 1er niveau (tm3) présents dans l’intervalle 
-    const items = [];
-    root.find("p.tm3").each((_, el) => {
-        if (!inIv(el))
-            return;
-        const $p = $(el);
-        const full = norm($p.text() || "");
-        if (!full)
-            return;
-        const numMatch = full.match(/^(\d+)\s*[.\-–—]\s*/);
-        const valeur = numMatch ? numMatch[1] : undefined;
-        // prefere intitule in ancre <a> if present
-        const a = $p.find("a").first();
-        const intituleRaw = a.length ? a.text() : full.replace(/^(\d+)\s*[.\-–—]\s*/, "");
-        const intitule = norm(intituleRaw);
-        // id_syceron from href="#Niv1_SOMx"
-        const href = (a.attr("href") || "").trim();
-        const idSyceron = href.startsWith("#") ? href.slice(1) : href;
-        const titreStruct = { id_syceron: idSyceron || "", intitule };
-        items.push({ valeur_pts_odj: valeur, titreStruct });
-    });
-    if (items.length)
-        sommaire.sommaire1 = items;
-    return sommaire;
-}
-function extractMetadonnees($, filePath) {
-    let dateText = norm($("h1, h2, .page-title").first().text() || "");
-    if (!dateText)
-        dateText = norm($("p").first().text() || "");
-    const dateMatch = dateText.match(/\b(\d{1,2}\s+\w+\s+\d{4})\b/i);
-    const allText = norm($("body").text() || "");
-    const sessionMatch = allText.match(/\bsession\s+(\d{4}-\d{4})\b/i);
-    let dateSeance = dateMatch?.[1] || "";
-    if (!dateSeance) {
-        const m = filePath.match(/d(\d{4})(\d{2})(\d{2})\.xml$/i);
-        if (m)
-            dateSeance = `${m[1]}-${m[2]}-${m[3]}`;
-    }
-    return {
-        dateSeance,
-        dateSeanceJour: dateSeance,
-        numSeanceJour: "",
-        numSeance: "",
-        typeAssemblee: "SN",
-        legislature: "",
-        session: sessionMatch?.[1] || "",
-        nomFichierJo: "",
-        validite: "",
-        etat: "",
-        diffusion: "",
-        version: "1.0",
-        environnement: "",
-        heureGeneration: new Date()
-    };
-}
-function elementInAnyInterval(el, idx, intervals) {
-    const p = idx.get(el);
-    if (p == null)
-        return false;
-    for (const iv of intervals)
-        if (p >= iv.start && p < iv.end)
-            return true;
-    return false;
-}