@tricoteuses/senat 2.9.5 → 2.9.6

package/lib/loaders.d.ts

@@ -5,6 +5,7 @@ import { QuestionResult } from "./model/questions";
 import { CirconscriptionResult, OrganismeResult, SenateurResult } from "./model/sens";
 import { AgendaEvent } from "./types/agenda";
 import { FlatTexte } from "./types/texte";
+import { CompteRendu } from "./types/compte_rendu";
 export { EnabledDatasets } from "./datasets";
 export declare const AGENDA_FOLDER = "agenda";
 export declare const COMPTES_RENDUS_FOLDER = "seances";
@@ -69,6 +70,9 @@ export declare function iterLoadSenatDossiersLegislatifsDocuments(dataDir: strin
 export declare function iterLoadSenatDossiersLegislatifsRapports(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DossierLegislatifDocumentResult>>;
 export declare function iterLoadSenatDossiersLegislatifsTextes(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DossierLegislatifDocumentResult>>;
 export declare function loadSenatTexteContent(dataDir: string, textePathFromDataset: string): IterItem<FlatTexte | null>;
+export declare function loadSenatCompteRenduContent(dataDir: string, session: number, debatId: string | number): {
+    item: CompteRendu | null;
+};
 export declare function iterLoadSenatAgendas(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<AgendaEvent[]>>;
 export declare function iterLoadSenatEvenements(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<AgendaEvent>>;
 export declare function iterLoadSenatCirconscriptions(dataDir: string, options?: {}): Generator<IterItem<CirconscriptionResult>>;

package/lib/loaders.js

@@ -144,6 +144,14 @@ export function loadSenatTexteContent(dataDir, textePathFromDataset) {
     const texteJson = fs.readFileSync(fullTextePath, { encoding: "utf8" });
     return { item: JSON.parse(texteJson) };
 }
+export function loadSenatCompteRenduContent(dataDir, session, debatId) {
+    const fullPath = path.join(dataDir, COMPTES_RENDUS_FOLDER, DATA_TRANSFORMED_FOLDER, String(session), `${debatId}.json`);
+    if (!fs.existsSync(fullPath)) {
+        return { item: null };
+    }
+    const json = fs.readFileSync(fullPath, { encoding: "utf8" });
+    return { item: JSON.parse(json) };
+}
 export function* iterLoadSenatAgendas(dataDir, session, options = {}) {
     for (const evenementsItem of iterLoadSenatItems(dataDir, AGENDA_FOLDER, session, DATA_TRANSFORMED_FOLDER, options)) {
         yield evenementsItem;

package/lib/model/compte_rendu.d.ts

@@ -1,3 +1,2 @@
 import { CompteRendu } from "../types/compte_rendu";
-import { DebatResult } from "./debats";
-export declare function parseCompteRenduFromFile(htmlFilePath: string, debat: DebatResult): Promise<CompteRendu | null>;
+export declare function parseCompteRenduFromFile(htmlFilePath: string): Promise<CompteRendu | null>;

package/lib/model/compte_rendu.js

@@ -1,32 +1,313 @@
 import { JSDOM } from "jsdom";
-function transformCompteRendu(document, debat) {
-    const compteRendu = {
-        sections: [],
+import * as cheerio from "cheerio";
+const norm = (s) => s.replace(/\u00A0/g, " ").replace(/\s+/g, " ").trim();
+const toTexte = (s) => ({ _: s });
+function extractSommaire($) {
+    const root = $("#wysiwyg").length ? $("#wysiwyg") : $("#cri");
+    const sommaire = {
+        presidentSeance: toTexte(""),
+        sommaire1: [],
     };
-    for (const section of debat.sections) {
-        const compteRenduSection = {
-            id: section.id,
-            interventions: [],
+    // (1) presidency line (e.g., "Présidence de Mme …")
+    const pres = root.find("p.tm2").filter((_, el) => /présidence/i.test($(el).text())).first();
+    if (pres.length) {
+        sommaire.presidentSeance = toTexte(norm(pres.text()));
+    }
+    // (2) extra info lines like "Secrétaires :" (tm5)
+    const paras = [];
+    root.find("p.tm5").each((_, el) => {
+        const t = norm($(el).text());
+        if (t)
+            paras.push(toTexte(t));
+    });
+    if (paras.length) {
+        sommaire.para = paras.length === 1 ? paras[0] : paras;
+    }
+    // (3) first-level items (tm3)
+    const items = [];
+    root.find("p.tm3").each((_, el) => {
+        const $p = $(el);
+        const full = norm($p.text());
+        // try to extract the numeric order at the start: "1. ..." or "2 – ..." etc.
+        const numMatch = full.match(/^(\d+)\s*[.\-–—]/);
+        const valeur = numMatch ? numMatch[1] : undefined;
+        // prefer the linked title text; fallback to full text
+        const a = $p.find("a").first();
+        const intitule = norm(a.text() || full.replace(/^(\d+)\s*[.\-–—]\s*/, ""));
+        // id_syceron = href target without '#' ? TODO verify
+        const href = a.attr("href") || "";
+        const idSyceron = href.startsWith("#") ? href.slice(1) : href;
+        const titreStruct = {
+            id_syceron: idSyceron || "",
+            intitule,
         };
-        compteRendu.sections.push(compteRenduSection);
-        for (const sectionIntervention of section.interventions) {
-            const interventionAnchor = document.querySelector(`a[name="int${sectionIntervention.id}"]`);
-            const interventionParent = interventionAnchor?.parentElement;
-            compteRenduSection.interventions.push({
-                id: sectionIntervention.id,
-                texteHtml: interventionParent?.outerHTML || null,
-            });
+        const elem = {
+            valeur_pts_odj: valeur,
+            titreStruct,
+            // sommaire2/3 undefined (first level only)
+        };
+        items.push(elem);
+    });
+    if (items.length) {
+        sommaire.sommaire1 = items;
+    }
+    return sommaire;
+}
+function stripTrailingPunct(s) {
+    return s.replace(/\s*([:,.;])\s*$/u, "").trim();
+}
+function dedupeSpeaker(raw) {
+    let s = norm(raw);
+    s = stripTrailingPunct(s);
+    const dupPatterns = [
+        /^(.+?)\s*[.]\s*\1$/u,
+        /^(.+?)\s*,\s*\1,?$/u,
+        /^(.+?)\s+\1$/u,
+    ];
+    for (const re of dupPatterns) {
+        const m = s.match(re);
+        if (m) {
+            s = m[1];
+            break;
         }
     }
-    return compteRendu;
+    return s.replace(/\.\s*$/, "");
 }
-export async function parseCompteRenduFromFile(htmlFilePath, debat) {
+function decodeHtmlEntities(s) {
+    return s
+        .replace(/&#(\d+);/g, (_, d) => String.fromCharCode(parseInt(d, 10)))
+        .replace(/&#x([0-9a-fA-F]+);/g, (_, h) => String.fromCharCode(parseInt(h, 16)));
+}
+function fixApostrophes(s) {
+    // Tighten spacing around French apostrophes and punctuation
+    let out = s;
+    out = out.replace(/\s*’\s*/g, "’");
+    out = out.replace(/\b([dljctmsn])\s*’/gi, (_, m) => m + "’");
+    out = out.replace(/’\s+([A-Za-zÀ-ÖØ-öø-ÿ])/g, "’$1");
+    out = out.replace(/\s+([,;:.!?])/g, "$1");
+    return out;
+}
+function normalizeTitle(text) {
+    return text.replace(/^PR[ÉE]SIDENCE DE\b/i, "Présidence de ");
+}
+function roleForSpeaker(labelOrQualite) {
+    const s = labelOrQualite.toLowerCase();
+    if (/^(m\.|mme)?\s*(le|la)\s+pr[ée]sident(e)?\b/.test(s) ||
+        /\bpr[ée]sident[e]?\s+de\s+séance\b/.test(s)) {
+        return "président";
+    }
+    return "";
+}
+// ---------------- DOM helpers ----------------
+function parseCriIntervenantComment(html) {
+    // From <!-- cri:intervenant mat="..." nom="..." qua="..." ... -->
+    const m = html.match(/<!--\s*cri:intervenant\b([^>]+)-->/i);
+    if (!m)
+        return {};
+    const attrs = m[1];
+    const out = {};
+    const re = /(\w+)="([^"]*)"/g;
+    let a;
+    while ((a = re.exec(attrs))) {
+        out[a[1]] = decodeHtmlEntities(a[2]);
+    }
+    return { mat: out["mat"], nom: out["nom"], qua: out["qua"] };
+}
+/**
+ * Extract leading .orateur_qualite chunks from the FIRST <p> only,
+ * concatenate them, clean punctuation/apostrophes, and REMOVE those nodes
+ * (and .orateur_nom) from the first paragraph so the speech starts cleanly.
+ */
+function extractAndRemoveLeadingQualite($, $block) {
+    const firstP = $block.find("p").first();
+    if (firstP.length === 0)
+        return "";
+    const parts = [];
+    // Iterate over the first <p>'s children from the start
+    let stop = false;
+    firstP.contents().each((_, node) => {
+        if (stop)
+            return;
+        if (node.type === "tag") {
+            const $node = $(node);
+            if ($node.hasClass("orateur_nom")) {
+                // speaker label node — remove it
+                $node.remove();
+                return;
+            }
+            if ($node.hasClass("orateur_qualite")) {
+                parts.push($node.text() || "");
+                $node.remove();
+                return;
+            }
+            // Non-qualite tag: if it has meaningful text, we reached the speech
+            const t = norm($node.text() || "");
+            if (t) {
+                stop = true;
+            }
+            else {
+                // empty-ish node; remove to avoid stray punctuation
+                $node.remove();
+            }
+        }
+        else if (node.type === "text") {
+            const t = norm(node.data || "");
+            if (!t) {
+                // whitespace only — drop it
+                ;
+                node.data = "";
+                return;
+            }
+            // boundary punctuation like ":" just after label — drop it
+            if (/^[:.,;–—-]+$/.test(t)) {
+                ;
+                node.data = "";
+                return;
+            }
+            // any other text means speech starts here
+            stop = true;
+        }
+        else {
+            // comment or others — ignore
+        }
+    });
+    const qual = fixApostrophes(norm(parts.join(" ")));
+    return qual;
+}
+function sanitizeInterventionHtml($, $block) {
+    // Clone to avoid mutating outer tree order
+    const $clone = $block.clone();
+    // Remove navigation / anchors / images
+    $clone.find('a[name]').remove();
+    $clone.find('div[align="right"]').remove();
+    $clone.find('a.link').remove();
+    $clone.find('img').remove();
+    // Remove technical anchors inside interventions
+    $clone.find('a#ameli_amendement_cri_phrase, a#ameli_amendement_cra_contenu, a#ameli_amendement_cra_objet').remove();
+    // Remove any remaining speaker label / quality spans anywhere
+    $clone.find(".orateur_nom").remove();
+    $clone.find(".orateur_qualite").remove();
+    // Strip HTML comments
+    let html = $clone.html() || "";
+    html = html.replace(/<!--[\s\S]*?-->/g, "");
+    return html.trim();
+}
+function extractMetadonnees($) {
+    const headerText = norm($("h1.page-title").text() || "");
+    const dateMatch = headerText.match(/\b(\d{1,2}\s+\w+\s+\d{4})\b/i);
+    const bodyText = norm($("#cri").text() || "");
+    const sessionMatch = bodyText.match(/\bsession\s+(\d{4}-\d{4})\b/i);
+    return {
+        dateSeance: dateMatch?.[1] || "",
+        dateSeanceJour: dateMatch?.[1] || "",
+        numSeanceJour: "",
+        numSeance: "",
+        typeAssemblee: "SN",
+        legislature: "",
+        session: sessionMatch?.[1] || "",
+        nomFichierJo: "",
+        validite: "",
+        etat: "",
+        diffusion: "",
+        version: "1.0",
+        environnement: "",
+        heureGeneration: new Date(),
+        sommaire: extractSommaire($)
+    };
+}
+// ---------------- main transform ----------------
+export async function parseCompteRenduFromFile(htmlFilePath) {
     try {
-        const { document } = (await JSDOM.fromFile(htmlFilePath, { contentType: "text/html" })).window;
-        return transformCompteRendu(document, debat);
+        const { window } = await JSDOM.fromFile(htmlFilePath, { contentType: "text/html" });
+        const $ = cheerio.load(window.document.documentElement.outerHTML);
+        const metadonnees = extractMetadonnees($);
+        const points = [];
+        let ordre = 0;
+        const addPoint = (p) => points.push({ ...p, ordre_absolu_seance: String(++ordre) });
+        // (1) Global section titles (common high-level headings)
+        let lastTitle = "";
+        $("#cri p[class^='titre_S']").each((_, el) => {
+            const t = normalizeTitle(norm($(el).text() || ""));
+            if (t && t !== lastTitle) {
+                addPoint({ code_grammaire: "TITRE_TEXTE_DISCUSSION", texte: { _: t }, code_style: "Titre" });
+                lastTitle = t;
+            }
+        });
+        // (2) Interventions
+        $("#cri div.intervenant").each((_, block) => {
+            const $block = $(block);
+            // (2.a) Extract internal structural titles inside this block (and remove them)
+            const structuralSel = [
+                "p[class^='titre_S']",
+                "p.mention_titre",
+                "p.intitule_titre",
+                "p.mention_chapitre",
+                "p.intitule_chapitre",
+                "p.mention_article",
+                "p.intitule_article",
+                "p.mention_section",
+                "p.intitule_section",
+            ].join(",");
+            $block.find(structuralSel).each((__, el) => {
+                const title = normalizeTitle(norm($(el).text() || ""));
+                if (title && title !== lastTitle) {
+                    addPoint({ code_grammaire: "TITRE_TEXTE_DISCUSSION", texte: { _: title }, code_style: "Titre" });
+                    lastTitle = title;
+                }
+                $(el).remove();
+            });
+            // (2.b) Speaker label & quality
+            const firstP = $block.find("p").first();
+            const speakerLabelRaw = firstP.find(".orateur_nom").text() ||
+                firstP.find("a.lien_senfic").text() ||
+                "";
+            const speakerLabel = dedupeSpeaker(speakerLabelRaw);
+            // Prefer <!--cri:intervenant ...--> for id/name/qualite when available
+            const rawHtml = $block.html() || "";
+            const { mat, nom: nomFromComment, qua: quaFromCommentRaw } = parseCriIntervenantComment(rawHtml);
+            // Extract and remove leading .orateur_qualite chunks from first <p>
+            const qualFromSpans = extractAndRemoveLeadingQualite($, $block);
+            const qualite = norm(decodeHtmlEntities(quaFromCommentRaw || "")) ||
+                qualFromSpans;
+            const canonicalName = dedupeSpeaker(nomFromComment || speakerLabel);
+            const role = roleForSpeaker(speakerLabel) ||
+                roleForSpeaker(qualite) ||
+                roleForSpeaker(quaFromCommentRaw || "");
+            // (2.c) Build cleaned speech HTML
+            let speechHtml = sanitizeInterventionHtml($, $block);
+            // If nothing meaningful remains, skip
+            if (!norm(cheerio.load(speechHtml).text() || ""))
+                return;
+            addPoint({
+                code_grammaire: "PAROLE_GENERIQUE",
+                roledebat: role,
+                orateurs: {
+                    orateur: {
+                        nom: canonicalName,
+                        id: mat || "",
+                        qualite: qualite,
+                    },
+                },
+                texte: { _: speechHtml },
+            });
+        });
+        const contenu = {
+            quantiemes: {
+                journee: metadonnees.dateSeance,
+                session: metadonnees.session,
+            },
+            point: points,
+        };
+        return {
+            uid: htmlFilePath.replace(/^.*?(\d{8}).*$/i, "$1"),
+            seanceRef: htmlFilePath.replace(/^.*?(\d{8}).*$/i, "$1"),
+            sessionRef: metadonnees.session,
+            metadonnees,
+            contenu,
+        };
     }
-    catch (error) {
-        console.error(`Could not parse compte-rendu with error ${error}`);
+    catch (e) {
+        console.error("Could not parse compte-rendu with error", e);
+        return null;
     }
-    return null;
 }

package/lib/scripts/retrieve_comptes_rendus.js

@@ -2,11 +2,11 @@ import assert from "assert";
 import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 import path from "path";
-import { COMPTES_RENDUS_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatDebats } from "../loaders";
-import { parseCompteRenduFromFile } from "../model/compte_rendu";
+import { COMPTES_RENDUS_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatDebats, } from "../loaders";
 import { getSessionsFromStart } from "../types/sessions";
 import { commonOptions } from "./shared/cli_helpers";
 import { ensureAndClearDir } from "./shared/util";
+import { parseCompteRenduFromFile } from "../model/compte_rendu";
 const optionsDefinitions = [
     ...commonOptions,
     {
@@ -22,6 +22,22 @@ class CompteRenduError extends Error {
         super(`An error occurred while retrieving Compte-Rendu ${compteRenduUrl}: ${message}`);
     }
 }
+async function fetchWithRetry(url, retries = 5, backoffMs = 1000) {
+    for (let attempt = 0; attempt < retries; attempt++) {
+        try {
+            return await fetch(url);
+        }
+        catch (e) {
+            if (attempt === retries)
+                break;
+            console.warn(`Fetch attempt ${attempt + 1} for ${url} failed. Retrying in ${backoffMs}ms...`);
+            await new Promise((resolve) => setTimeout(resolve, backoffMs));
+            backoffMs *= 2;
+        }
+    }
+    console.log(`Failed to fetch ${url} after ${retries} attempts.`);
+    return null;
+}
 async function retrieveComptesRendus(dataDir, sessions) {
     const comptesRendusRootDir = path.join(dataDir, COMPTES_RENDUS_FOLDER);
     ensureAndClearDir(comptesRendusRootDir);
@@ -38,16 +54,15 @@ async function retrieveComptesRendus(dataDir, sessions) {
         if (options["parseDebats"]) {
             fs.ensureDirSync(transformedComptesRendusSessionDir);
         }
-        for (const { item: debat, } of iterLoadSenatDebats(dataDir, session)) {
-            if (!debat.url) {
+        for (const { item: debat } of iterLoadSenatDebats(dataDir, session)) {
+            if (!debat.url)
                 continue;
-            }
             try {
                 const debatMonoUrl = `${path.parse(debat.url).dir}/s${debat.id}_mono.html`;
                 const compteRenduPath = path.join(originalComptesRendusSessionDir, `${debat.id}.html`);
                 await downloadCompteRendu(debatMonoUrl, compteRenduPath);
                 if (options["parseDebats"]) {
-                    await parseCompteRendu(transformedComptesRendusSessionDir, compteRenduPath, debat);
+                    await parseAndWriteJSON(transformedComptesRendusSessionDir, compteRenduPath, debat);
                 }
             }
             catch (error) {
@@ -61,7 +76,7 @@ async function downloadCompteRendu(debatUrl, compteRenduPath) {
     if (!options["silent"]) {
         console.log(`Downloading Compte-Rendu ${compteRenduUrl}…`);
     }
-    const response = await fetch(compteRenduUrl);
+    const response = await fetchWithRetry(compteRenduUrl);
     if (!response.ok) {
         if (response.status === 404) {
             console.warn(`Compte-Rendu ${compteRenduUrl} not found`);
@@ -72,21 +87,19 @@ async function downloadCompteRendu(debatUrl, compteRenduPath) {
         return;
     }
     const compteRenduContent = await response.arrayBuffer();
-    if (!compteRenduContent) {
+    if (!compteRenduContent)
         return;
-    }
     fs.writeFileSync(compteRenduPath, Buffer.from(compteRenduContent));
 }
-async function parseCompteRendu(transformedComptesRendusSessionDir, compteRenduPath, debat) {
+async function parseAndWriteJSON(transformedComptesRendusSessionDir, compteRenduPath, debat) {
     if (!options["silent"]) {
         console.log(`Parsing compte-rendu ${compteRenduPath}…`);
     }
-    const parsedCompteRendu = await parseCompteRenduFromFile(compteRenduPath, debat);
-    if (!parsedCompteRendu) {
+    const parsed = await parseCompteRenduFromFile(compteRenduPath);
+    if (!parsed)
         return;
-    }
     const parsedFilePath = path.parse(compteRenduPath);
-    fs.writeJSONSync(path.join(transformedComptesRendusSessionDir, `${parsedFilePath.name}.json`), parsedCompteRendu, { spaces: 2 });
+    fs.writeJSONSync(path.join(transformedComptesRendusSessionDir, `${parsedFilePath.name}.json`), parsed, { spaces: 2 });
 }
 async function main() {
     const dataDir = options["dataDir"];

package/lib/types/compte_rendu.d.ts

@@ -1,11 +1,76 @@
 export interface CompteRendu {
-    sections: Section[];
+    uid?: string;
+    seanceRef?: string;
+    sessionRef?: string;
+    metadonnees?: Metadonnees;
+    contenu?: Contenu;
 }
-export interface Section {
-    id: string | null;
-    interventions: Intervention[];
+export interface Metadonnees {
+    dateSeance: string;
+    dateSeanceJour: string;
+    numSeanceJour: string;
+    numSeance: string;
+    typeAssemblee: "AN" | "SN";
+    legislature: string;
+    session: string;
+    nomFichierJo: string;
+    validite: string;
+    etat: string;
+    diffusion: string;
+    version: string;
+    environnement: string;
+    heureGeneration: Date;
+    sommaire: Sommaire;
 }
-export interface Intervention {
-    id: string | null;
-    texteHtml: string | null;
+export interface Contenu {
+    quantiemes: {
+        journee: string;
+        session: string;
+    };
+    point: Point[];
+}
+export interface Point {
+    ordre_absolu_seance: string;
+    code_grammaire: string;
+    roledebat?: string;
+    orateurs?: {
+        orateur: {
+            nom: string;
+            id: string;
+            qualite: string;
+        };
+    };
+    texte: {
+        _: string;
+    };
+    code_style?: string;
+}
+export interface Texte {
+    _?: string;
+    id_syceron?: string;
+    stime?: string;
+    sup?: string;
+    lienAdt?: Texte[] | Texte;
+}
+export interface Sommaire {
+    presidentSeance: Texte;
+    sommaire1: SommaireElement[] | SommaireElement;
+    sommaire3?: SommaireElement[] | SommaireElement;
+    sommaire2?: SommaireElement[] | SommaireElement;
+    para?: Texte[] | Texte;
+}
+export interface SommaireElement {
+    valeur_pts_odj: string | undefined;
+    titreStruct: TitreStruct;
+    para?: Array<Texte | string> | Texte;
+    sommaire2?: SommaireElement[] | SommaireElement;
+    sommaire3?: SommaireElement[] | SommaireElement;
+    presidentSeance?: Texte[] | Texte;
+    type_debat?: string;
+}
+export interface TitreStruct {
+    id_syceron: string;
+    intitule?: string;
+    sousIntitule?: string;
+    type_debat?: string;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.9.5",
+  "version": "2.9.6",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",
@@ -60,6 +60,7 @@
   },
   "dependencies": {
     "@biryani/core": "^0.2.1",
+    "cheerio": "^1.1.2",
     "command-line-args": "^5.1.1",
     "dotenv": "^8.2.0",
     "fs-extra": "^9.1.0",