@tricoteuses/senat 3.1.4 → 3.1.5

package/lib/src/model/commission.js

@@ -0,0 +1,269 @@
+import * as cheerio from "cheerio";
+import path from "path";
+import { makeReunionUid } from "../utils/reunion_parsing.js";
+import { norm } from "../utils/string_cleaning.js";
+import { frDateToISO, hourShortToStartTime } from "../utils/date.js";
+import { toCRDate } from "./util.js";
+const PARA_h3_SEL = "p.sh_justify, p.sh_center, p.sh_marge, p[align], li, h3";
+function findDayRoot($, targetISO) {
+    let $root = $();
+    $("h2").each((_, el) => {
+        const txt = norm($(el).text());
+        const m = txt.match(/(?:Lundi|Mardi|Mercredi|Jeudi|Vendredi|Samedi|Dimanche)\s+(.+)$/i);
+        const iso = m ? frDateToISO(m[1]) : undefined;
+        if (iso === targetISO && $root.length === 0)
+            $root = $(el);
+    });
+    return $root;
+}
+function normalizeSpaces(s) {
+    return s.replace(/[\u00A0\u202F\u2009]/g, " ");
+}
+function stripIntroPunct(s) {
+    return s.replace(/^[\s]*[.:;]?\s*(?:[–—-]\s*)+/u, "");
+}
+function collectLeadingHeaderStrongEls($, $clone) {
+    const els = [];
+    const nodes = $clone.contents().toArray();
+    for (const node of nodes) {
+        if (node.type === "text") {
+            if (norm(node.data || ""))
+                break;
+            continue;
+        }
+        if (node.type === "tag") {
+            const $n = $(node);
+            if ($n.is("strong, b")) {
+                els.push(node);
+                continue;
+            }
+            if ($n.is("a") && $n.children("strong, b").length) {
+                $n.children("strong, b").each((_, el) => {
+                    els.push(el);
+                });
+                continue;
+            }
+            break;
+        }
+    }
+    return els;
+}
+// Remove orateur's name from text and clean intro punct
+export function getRemainingTextAfterSpeakerHeader($, $p) {
+    const $clone = $p.clone();
+    // 1) Remove <strong> at start
+    const headerStrongEls = collectLeadingHeaderStrongEls($, $clone);
+    for (const el of headerStrongEls)
+        $(el).remove();
+    // 2) normalize + clean intro punct
+    let remainingHtml = $clone.html() || "";
+    remainingHtml = normalizeSpaces(cheerio.load(remainingHtml).text());
+    remainingHtml = stripIntroPunct(remainingHtml);
+    const remainingText = norm(remainingHtml || "");
+    return remainingText;
+}
+function buildPointsFromParagraphs($, paras) {
+    const points = [];
+    let ordreAbsoluSeance = 0;
+    const normSpeaker = (s) => s
+        .normalize("NFKC")
+        .replace(/\s+/g, " ")
+        .replace(/[.:]\s*$/, "")
+        .trim();
+    const normQual = (s) => s
+        .normalize("NFKC")
+        .replace(/\s+/g, " ")
+        .replace(/^\s*,\s*|\s+$/g, "")
+        .replace(/[\s\u00A0]*[.,;:–—-]+$/u, "")
+        .trim();
+    let currentOrateur = null;
+    let currentQualite = "";
+    let currentTexte = "";
+    function isPresidentQual(qual) {
+        return /\bprésident(e)?\b/i.test(qual);
+    }
+    // Flush the buffered speaker’s text into points[] if any.
+    function flush() {
+        if (!currentOrateur || !currentTexte.trim())
+            return;
+        ordreAbsoluSeance++;
+        points.push({
+            code_grammaire: "PAROLE_GENERIQUE",
+            roledebat: isPresidentQual(currentQualite) ? "président" : "",
+            ordre_absolu_seance: String(ordreAbsoluSeance),
+            orateurs: { orateur: { nom: currentOrateur, id: "", qualite: currentQualite || "" } },
+            texte: { _: currentTexte.trim() },
+        });
+        currentOrateur = null;
+        currentQualite = "";
+        currentTexte = "";
+    }
+    function addPoint(payload) {
+        ordreAbsoluSeance++;
+        points.push({ ...payload, ordre_absolu_seance: String(ordreAbsoluSeance) });
+    }
+    for (const $p of paras) {
+        if ($p.closest("table").length)
+            continue;
+        const tagName = ($p.prop("tagName") || "").toString().toLowerCase();
+        const rawText = ($p.text() || "").replace(/\u00a0/g, " ").trim();
+        const text = norm(rawText);
+        if (!text || text.length <= 3)
+            continue;
+        const html = ($p.html() || "").trim();
+        const italicSpans = $p.find("i, em, span[style*='italic']");
+        const firstItalicOuter = italicSpans.length ? $(italicSpans[0]).prop("outerHTML") || "" : "";
+        const htmlBeforeFirstItalic = firstItalicOuter ? html.split(firstItalicOuter)[0].trim() : "";
+        const isPureItalic = italicSpans.length > 0 && italicSpans.length === $p.find("span,i,em").length && htmlBeforeFirstItalic === "";
+        if (tagName === "h3") {
+            flush();
+            addPoint({
+                code_style: "Titre",
+                code_grammaire: "TITRE_TEXTE_DISCUSSION",
+                texte: { _: text },
+            });
+            continue;
+        }
+        const boldSpans = $p.find("strong, b");
+        const joinedBold = norm(boldSpans
+            .map((_, el) => $(el).text() || "")
+            .get()
+            .join(""));
+        const [namePartRaw, qualPartRaw] = joinedBold.split(/\s*,\s+/, 2);
+        const namePart = namePartRaw ? normSpeaker(namePartRaw) : "";
+        const qualPart = qualPartRaw ? normQual(qualPartRaw) : "";
+        const looksLikeName = namePart.length > 3 && /^(M\.|Mme)[\s\u00A0\u202F]+/i.test(namePart);
+        const startsWithName = namePart && text.startsWith(namePart);
+        const isNewSpeaker = looksLikeName && startsWithName && namePart !== currentOrateur;
+        if (isNewSpeaker) {
+            flush();
+            currentOrateur = namePart;
+            currentQualite = qualPart;
+            const remainingText = getRemainingTextAfterSpeakerHeader($, $p);
+            currentTexte = remainingText;
+            continue;
+        }
+        if (isPureItalic || (!joinedBold && !currentOrateur && text)) {
+            flush();
+            addPoint({
+                code_style: "Info Italiques",
+                code_grammaire: "PAROLE_GENERIQUE",
+                texte: { _: "<i>" + text + "</i>" },
+            });
+            continue;
+        }
+        // concat text because same orateur
+        if (currentOrateur) {
+            const removeOrateurFromText = getRemainingTextAfterSpeakerHeader($, $p);
+            currentTexte += (currentTexte ? "<br/><br/>" : "") + removeOrateurFromText;
+            continue;
+        }
+    }
+    flush();
+    return points;
+}
+const TIME_RE = /(?:\b[àa]\s*)?(\d{1,2})\s*(?:h|heures?)\s*(?:([0-5]\d))?/i;
+export function cleanTitle(t) {
+    return (t || "").replace(/\s+/g, " ").trim();
+}
+function parseTimeToHHmm(text) {
+    const m = normalizeSpaces(text).match(TIME_RE);
+    if (!m)
+        return undefined;
+    const hh = m[1]?.padStart(2, "0");
+    const mm = (m[2] ?? "00").padStart(2, "0");
+    const h = Number(hh);
+    if (h >= 0 && h <= 23)
+        return `${hh}:${mm}`;
+    return undefined;
+}
+function findNearbyTime($, $h3) {
+    let cur = $h3.prev();
+    for (let i = 0; i < 3 && cur.length; i++, cur = cur.prev()) {
+        const direct = parseTimeToHHmm(cur.text());
+        if (direct)
+            return direct;
+        const italic = parseTimeToHHmm(cur.find("i, em").first().text());
+        if (italic)
+            return italic;
+    }
+    return undefined;
+}
+export function extractDayH3Sections($, dateISO) {
+    const sections = [];
+    const $dayRoot = findDayRoot($, dateISO);
+    if ($dayRoot.length === 0)
+        return sections;
+    const $range = $dayRoot.nextUntil("h2");
+    const $h3s = $range.filter("h3").add($range.find("h3"));
+    $h3s.each((_, el) => {
+        const $h3 = $(el);
+        const title = cleanTitle($h3.text());
+        if (!title)
+            return;
+        const time = findNearbyTime($, $h3);
+        sections.push({ title, $start: $h3, time });
+    });
+    return sections;
+}
+export function parseCommissionCRSectionFromDom($, htmlFilePath, opts) {
+    try {
+        const { dateISO, hourShort, organe, section, matched } = opts;
+        const seanceRef = matched?.uid ?? makeReunionUid(dateISO, "COM", matched?.events[0].id ?? hourShort ?? "", organe ?? undefined);
+        const uid = seanceRef.replace(/^RU/, "CRC");
+        const dateSeance = toCRDate(dateISO, matched?.startTime ?? hourShortToStartTime(hourShort));
+        const $dayRoot = findDayRoot($, dateISO);
+        if ($dayRoot.length === 0) {
+            console.warn(`[COM-CR][parse] day root not found for ${dateISO} in ${path.basename(htmlFilePath)}`);
+            return null;
+        }
+        const paras = [];
+        let $cursor = section.$start;
+        // Jump title if we do not want to add it to paragraphes
+        $cursor = $cursor.next();
+        while ($cursor.length && !$cursor.is("h2") && !$cursor.is("h3")) {
+            if ($cursor.is(PARA_h3_SEL)) {
+                paras.push($cursor);
+            }
+            else {
+                const $ps = $cursor.find(PARA_h3_SEL);
+                if ($ps.length)
+                    $ps.each((_, p) => {
+                        paras.push($(p));
+                    });
+            }
+            $cursor = $cursor.next();
+        }
+        const points = buildPointsFromParagraphs($, paras);
+        if (points.length < 4 || !points.some((pt) => pt.code_grammaire === "PAROLE_GENERIQUE" && pt.orateurs)) {
+            console.warn(`[COM-CR][parse] Insufficient points or no interventions found for a section in ${path.basename(htmlFilePath)}`);
+            return null;
+        }
+        const session = dateISO.slice(5, 7) >= "10" ? `${dateISO.slice(0, 4)}` : `${Number(dateISO.slice(0, 4)) - 1}`;
+        const contenu = {
+            quantiemes: { journee: dateISO, session },
+            point: points,
+        };
+        const metadonnees = {
+            dateSeance,
+            dateSeanceJour: dateISO,
+            numSeanceJour: "",
+            numSeance: "",
+            typeAssemblee: "SN",
+            legislature: "",
+            session,
+            nomFichierJo: path.basename(htmlFilePath),
+            validite: "non-certifie",
+            etat: "definitif",
+            diffusion: "publique",
+            version: "1",
+            environnement: "prod",
+            heureGeneration: new Date(),
+        };
+        return { uid, seanceRef, sessionRef: session, metadonnees, contenu };
+    }
+    catch (e) {
+        console.error(`[COM-CR][parse] error section file=${path.basename(htmlFilePath)}:`, e);
+        return null;
+    }
+}

package/lib/src/model/debats.d.ts

@@ -0,0 +1,39 @@
+export interface DebatAuteurRow {
+    code: string | null;
+    matricule: string | null;
+    nom: string | null;
+    prenom: string | null;
+}
+export interface DebatInterventionRow {
+    analyse: string | null;
+    auteur: DebatAuteurRow | null;
+    auteur_code: string;
+    fonction_intervenant: string | null;
+    id: string | null;
+    url: string | null;
+}
+export interface DebatSectionRow {
+    categorie: string | null;
+    id?: string | null;
+    interventions: DebatInterventionRow[];
+    lecture_id?: string | null;
+    libelle?: string | null;
+    numero?: string | null;
+    objet: string | null;
+    type: string | null;
+    url?: string | null;
+}
+export interface DebatLectureRow {
+    id: string;
+}
+export interface DebatResult {
+    date_seance: string | null;
+    etat_synchronisation: string | null;
+    id: string | null;
+    lectures: DebatLectureRow[];
+    numero: string | null;
+    sections: DebatSectionRow[];
+    sections_divers: DebatSectionRow[];
+    url: string | null;
+}
+export declare function findAll(): AsyncGenerator<DebatResult, void, unknown>;

package/lib/src/model/debats.js

@@ -0,0 +1,112 @@
+import { streamUnsafeQuery } from "../databases_postgres.js";
+function buildFindAllDebatsQuery() {
+    return {
+        params: [],
+        query: `
+      select
+        to_char(debats.datsea, 'YYYYMMDD') as id,
+        to_char(debats.datsea, 'YYYY-MM-DD') as date_seance,
+        debats.numero::text as numero,
+        debats.deburl as url,
+        debats.debsyn as etat_synchronisation,
+        (
+          select coalesce(json_agg(section_rows order by section_rows.section_order nulls last), '[]'::json)
+          from (
+            select
+              secdis.secdisordid::text as id,
+              secdis.secdisnum as numero,
+              secdis.secdisobj as objet,
+              secdis.secdisurl as url,
+              typsec.typseclib as type,
+              typsec.typseccat as categorie,
+              secdis.lecassidt as lecture_id,
+              secdis.secdisordid as section_order,
+              (
+                select coalesce(
+                  json_agg(intervention_rows order by intervention_rows.intervention_order nulls last),
+                  '[]'::json
+                )
+                from (
+                  select
+                    intpjl.intordid::text as id,
+                    intpjl.autcod as auteur_code,
+                    intpjl.intfon as fonction_intervenant,
+                    intpjl.inturl as url,
+                    intpjl.intana as analyse,
+                    json_build_object(
+                      'code', auteur.autcod,
+                      'nom', auteur.nomuse,
+                      'prenom', auteur.prenom,
+                      'matricule', auteur.autmat
+                    ) as auteur,
+                    intpjl.intordid as intervention_order
+                  from senat.debats_intpjl as intpjl
+                  left join senat.dosleg_auteur as auteur on intpjl.autcod = auteur.autcod
+                  where intpjl.secdiscle = secdis.secdiscle
+                ) as intervention_rows
+              ) as interventions
+            from senat.debats_secdis as secdis
+            left join senat.debats_typsec as typsec on secdis.typseccod = typsec.typseccod
+            where secdis.datsea = debats.datsea
+          ) as section_rows
+        ) as sections,
+        (
+          select coalesce(json_agg(section_rows), '[]'::json)
+          from (
+            select
+              secdivers.secdiverslibelle as libelle,
+              secdivers.secdiversobj as objet,
+              typsec.typseclib as type,
+              typsec.typseccat as categorie,
+              (
+                select coalesce(
+                  json_agg(intervention_rows order by intervention_rows.intervention_order nulls last),
+                  '[]'::json
+                )
+                from (
+                  select
+                    intdivers.intdiversordid::text as id,
+                    intdivers.autcod as auteur_code,
+                    intdivers.intfon as fonction_intervenant,
+                    intdivers.inturl as url,
+                    intdivers.intana as analyse,
+                    json_build_object(
+                      'code', auteur.autcod,
+                      'nom', auteur.nomuse,
+                      'prenom', auteur.prenom,
+                      'matricule', auteur.autmat
+                    ) as auteur,
+                    intdivers.intdiversordid as intervention_order
+                  from senat.debats_intdivers as intdivers
+                  left join senat.dosleg_auteur as auteur on intdivers.autcod = auteur.autcod
+                  where intdivers.secdiverscle = secdivers.secdiverscle
+                ) as intervention_rows
+              ) as interventions
+            from senat.debats_secdivers as secdivers
+            left join senat.debats_typsec as typsec on secdivers.typseccod = typsec.typseccod
+            where secdivers.datsea = debats.datsea
+          ) as section_rows
+        ) as sections_divers,
+        (
+          select coalesce(json_agg(lecture_rows), '[]'::json)
+          from (
+            select lecassdeb.lecassidt as id
+            from senat.debats_lecassdeb as lecassdeb
+            where lecassdeb.datsea = debats.datsea
+          ) as lecture_rows
+        ) as lectures
+      from senat.debats_debats as debats
+    `,
+    };
+}
+export async function* findAll() {
+    const { query, params } = buildFindAllDebatsQuery();
+    for await (const row of streamUnsafeQuery(query, params)) {
+        yield {
+            ...row,
+            lectures: row.lectures ?? [],
+            sections: row.sections ?? [],
+            sections_divers: row.sections_divers ?? [],
+        };
+    }
+}

package/lib/src/model/documents.d.ts

@@ -0,0 +1,32 @@
+export interface AuteurDocumentRow {
+    matricule: string | null;
+    nom_usuel: string;
+    ordre: string | null;
+    prenom: string | null;
+    qualite: string | null;
+    role: string | null;
+}
+export interface DocumentAnnexeRow {
+    type_document: string;
+    url: string | null;
+}
+export interface DocumentResult {
+    auteurs: AuteurDocumentRow[];
+    code_adoption?: string | null;
+    code_organisme: string | null;
+    date: string;
+    documents_annexes?: DocumentAnnexeRow[];
+    id: string | null;
+    modification?: string | null;
+    numero: number | string | null;
+    ordre_origine?: string | null;
+    origine?: string | null;
+    session: number | null;
+    signet_dossier?: string | null;
+    sous_titre?: string | null;
+    titre: string | null;
+    type: string | null;
+    url: string | null;
+}
+export declare function findAllTextes(): AsyncGenerator<DocumentResult, void, unknown>;
+export declare function findAllRapports(): AsyncGenerator<DocumentResult, void, unknown>;

package/lib/src/model/documents.js

@@ -0,0 +1,182 @@
+import { streamUnsafeQuery } from "../databases_postgres.js";
+function buildAuteursRapportSql(rapportIdSql) {
+    return `
+    (
+      select coalesce(json_agg(author_rows order by author_rows.ordre_num nulls last), '[]'::json)
+      from (
+        select
+          auteur.prenom as prenom,
+          auteur.nomuse as nom_usuel,
+          auteur.autmat as matricule,
+          ecr.ecrnumtri::text as ordre,
+          ecr.ecrnumtri as ordre_num,
+          rolsig.rolsiglib as role,
+          ecr.ecrqua as qualite
+        from senat.dosleg_auteur as auteur
+        left join senat.dosleg_ecr as ecr on ecr.autcod = auteur.autcod
+        left join senat.dosleg_rolsig as rolsig on rolsig.signataire = ecr.signataire
+        where ecr.rapcod = ${rapportIdSql}
+      ) as author_rows
+    )
+  `;
+}
+function buildDocumentsAttachesSql(rapportIdSql) {
+    return `
+    (
+      select coalesce(json_agg(attached_rows), '[]'::json)
+      from (
+        select
+          docatt.docatturl as url,
+          typatt.typattlib as type_document
+        from senat.dosleg_docatt as docatt
+        left join senat.dosleg_typatt as typatt on docatt.typattcod = typatt.typattcod
+        where docatt.rapcod = ${rapportIdSql}
+      ) as attached_rows
+    )
+  `;
+}
+function buildAuteursTexteSql(texteIdSql) {
+    return `
+    (
+      select coalesce(json_agg(author_rows order by author_rows.ordre_num nulls last), '[]'::json)
+      from (
+        select
+          auteur.prenom as prenom,
+          auteur.nomuse as nom_usuel,
+          auteur.autmat as matricule,
+          ecr.ecrnumtri::text as ordre,
+          ecr.ecrnumtri as ordre_num,
+          rolsig.rolsiglib as role,
+          ecr.ecrqua as qualite
+        from senat.dosleg_auteur as auteur
+        left join senat.dosleg_ecr as ecr on ecr.autcod = auteur.autcod
+        left join senat.dosleg_rolsig as rolsig on rolsig.signataire = ecr.signataire
+        where ecr.texcod = ${texteIdSql}
+      ) as author_rows
+    )
+  `;
+}
+function stripTrailingHashesSql(expr) {
+    return `regexp_replace(${expr}, '#+$', '')`;
+}
+function rapportIdSql() {
+    return `
+    case
+      when rap.rapurl is not null then
+        ${stripTrailingHashesSql("regexp_replace(trim(rap.rapurl), '^(.*/)?(.*?)(\\.html)?$', '\\2')")}
+      else null
+    end
+  `;
+}
+function rapportUrlSql() {
+    return `
+    case
+      when rap.typurl = 'I' then ${stripTrailingHashesSql("'https://www.senat.fr/rap/' || rtrim(rap.rapurl)")}
+      else ${stripTrailingHashesSql("rtrim(rap.rapurl)")}
+    end
+  `;
+}
+function texteIdSql() {
+    return `
+    case
+      when texte.texurl is not null then
+        ${stripTrailingHashesSql("regexp_replace(trim(texte.texurl), '^(.*/)?(.*?)(\\.html)?$', '\\2')")}
+      else null
+    end
+  `;
+}
+function texteUrlSql() {
+    return `
+    case
+      when texte.typurl = 'I' then ${stripTrailingHashesSql("'https://www.senat.fr/leg/' || rtrim(texte.texurl)")}
+      else ${stripTrailingHashesSql("rtrim(texte.texurl)")}
+    end
+  `;
+}
+function baseRapportsSelect(includeSignetDossier) {
+    const signet = includeSignetDossier ? "loi.signet as signet_dossier," : "";
+    return `
+    select
+      ${signet}
+      rap.rapnum as numero,
+      raporg.orgcod as code_organisme,
+      ${rapportIdSql()} as id,
+      ${rapportUrlSql()} as url,
+      rtrim(denrap.libdenrap) as type,
+      rtrim(rtrim(rap.raptil)) as titre,
+      rtrim(rtrim(rap.rapsoustit)) as sous_titre,
+      to_char(rap.date_depot, 'YYYY-MM-DD') as date,
+      rap.sesann::int as session,
+      ${buildAuteursRapportSql("rap.rapcod")} as auteurs,
+      ${buildDocumentsAttachesSql("rap.rapcod")} as documents_annexes
+  `;
+}
+function baseTextesSelect(includeSignetDossier) {
+    const signet = includeSignetDossier ? "loi.signet as signet_dossier," : "";
+    return `
+    select
+      ${signet}
+      texte.texnum as numero,
+      texte.orgcod as code_organisme,
+      ${texteIdSql()} as id,
+      ${texteUrlSql()} as url,
+      rtrim(oritxt.oritxtlib) as origine,
+      oritxt.oriordre as ordre_origine,
+      oritxt.oritxtado as code_adoption,
+      oritxt.oritxtmod as modification,
+      rtrim(typtxt.typtxtlib) as type,
+      to_char(texte.txtoritxtdat, 'YYYY-MM-DD') as date,
+      texte.sesann::int as session,
+      ${buildAuteursTexteSql("texte.texcod")} as auteurs,
+      null::text as titre
+  `;
+}
+function buildFindAllRapportsQuery() {
+    return {
+        params: [],
+        query: `
+      ${baseRapportsSelect(true)}
+      from senat.dosleg_rap as rap
+      left join senat.dosleg_raporg as raporg on raporg.rapcod = rap.rapcod
+      left join senat.dosleg_denrap as denrap on denrap.coddenrap = rap.coddenrap
+      left join senat.dosleg_lecassrap as lecassrap on lecassrap.rapcod = rap.rapcod
+      left join senat.dosleg_lecass as lecass on lecass.lecassidt = lecassrap.lecassidt
+      left join senat.dosleg_lecture as lecture on lecture.lecidt = lecass.lecidt
+      left join senat.dosleg_loi as loi on loi.loicod = lecture.loicod
+    `,
+    };
+}
+function buildFindAllTextesQuery() {
+    return {
+        params: [],
+        query: `
+      ${baseTextesSelect(true)}
+      from senat.dosleg_texte as texte
+      left join senat.dosleg_oritxt as oritxt on oritxt.oritxtcod = texte.oritxtcod
+      left join senat.dosleg_typtxt as typtxt on typtxt.typtxtcod = texte.typtxtcod
+      left join senat.dosleg_lecass as lecass on lecass.lecassidt = texte.lecassidt
+      left join senat.dosleg_lecture as lecture on lecture.lecidt = lecass.lecidt
+      left join senat.dosleg_loi as loi on loi.loicod = lecture.loicod
+      order by array_position(array['0','2','1'], oritxt.oriordre)
+    `,
+    };
+}
+export async function* findAllTextes() {
+    const { query, params } = buildFindAllTextesQuery();
+    for await (const row of streamUnsafeQuery(query, params)) {
+        yield {
+            ...row,
+            auteurs: row.auteurs ?? [],
+        };
+    }
+}
+export async function* findAllRapports() {
+    const { query, params } = buildFindAllRapportsQuery();
+    for await (const row of streamUnsafeQuery(query, params)) {
+        yield {
+            ...row,
+            auteurs: row.auteurs ?? [],
+            documents_annexes: row.documents_annexes ?? [],
+        };
+    }
+}