@tricoteuses/senat 3.0.2 → 3.1.0

package/lib/src/server/ameli.js

@@ -0,0 +1,153 @@
+import { sql } from "kysely";
+import { jsonArrayFrom } from "kysely/helpers/postgres";
+import { db, streamRawQuery } from "./databases_postgres.js";
+export async function* findAllAmendements(fromSession) {
+    const baseQuery = db
+        .withSchema("senat")
+        .selectFrom("ameli_amd as amd")
+        .leftJoin("ameli_sub as sub", "sub.id", "amd.subid")
+        .leftJoin("ameli_typsub as typsub", "typsub.id", "sub.typid")
+        .leftJoin("ameli_typrect as typrect", "typrect.id", "amd.typrectid")
+        .leftJoin("ameli_txt_ameli as txt_ameli", "txt_ameli.id", "amd.txtid")
+        .leftJoin("ameli_etatxt as etatxt", "etatxt.id", "txt_ameli.txtetaid")
+        .leftJoin("ameli_ses as ses", "ses.id", "txt_ameli.sesdepid")
+        .leftJoin("ameli_typses as typses", "typses.id", "ses.typid")
+        .leftJoin("ameli_nat as nat", "nat.id", "txt_ameli.natid")
+        .leftJoin("ameli_lec_ameli as lec_ameli", "lec_ameli.id", "txt_ameli.lecid")
+        .leftJoin("dosleg_texte as texte", (join) => join.onRef("ses.ann", "=", "texte.sesann").onRef("txt_ameli.numabs", "=", "texte.texnum"))
+        .leftJoin("dosleg_lecass as lecass", "lecass.lecassidt", "texte.lecassidt")
+        .leftJoin("ameli_mot as mot", "mot.id", "amd.motid")
+        .leftJoin("ameli_avicom as avicom", "avicom.id", "amd.avcid")
+        .leftJoin("ameli_avigvt as avigvt", "avigvt.id", "amd.avgid")
+        .leftJoin("ameli_sor as sor", "sor.id", "amd.sorid")
+        .leftJoin("ameli_irr as irr", "irr.id", "amd.irrid")
+        .leftJoin("ameli_grppol_ameli as grppol_ameli", "grppol_ameli.entid", "amd.nomentid")
+        .leftJoin("ameli_com_ameli as com_ameli", "com_ameli.entid", "amd.nomentid")
+        .leftJoin("ameli_cab as cab", "cab.entid", "amd.nomentid")
+        .select((eb) => [
+        sql `${eb.ref("ses.ann")}::text`.as("session"),
+        "ses.lil as session_libelle",
+        "typses.lib as type_session",
+        "txt_ameli.doslegsignet as signet_dossier_legislatif",
+        "nat.libcourt as nature_texte",
+        "nat.lib as nature_texte_libelle",
+        "txt_ameli.numabs as numero_texte",
+        "txt_ameli.numado as numero_adoption_texte",
+        "txt_ameli.int as intitule_texte",
+        "etatxt.lic as etat_texte",
+        "etatxt.lib as etat_texte_libelle",
+        "etatxt.txttyp as type_texte",
+        "lec_ameli.lib as lecture",
+        eb
+            .case()
+            .when("amd.typ", "=", "A")
+            .then("Amendement")
+            .when("amd.typ", "=", "M")
+            .then("Motion")
+            .when("amd.typ", "=", "S")
+            .then("Sous-amendement")
+            .else("")
+            .end()
+            .as("nature"),
+        "amd.id",
+        "amd.amdperid as parent_id",
+        "amd.ideid as identique_id",
+        "amd.discomid as discussion_commune_id",
+        sql `${eb.ref("amd.num")}`.as("numero"),
+        "amd.numabs as numero_absolu",
+        "amd.ord as ordre",
+        "amd.accgou as accepte_gouvernement",
+        "amd.txtid as texte_id",
+        "sub.lib as subdivision_libelle",
+        "sub.lic as subdivision_libelle_court",
+        "sub.pos as subdivision_position_texte",
+        "sub.posder as subdivision_position_discussion",
+        "sub.merid as subdivision_mere_id",
+        "sub.sig as subdivision_signet",
+        "sub.comdelid as subdivision_commission_id",
+        "sub.dupl as subdivision_dupliquee",
+        "typsub.lib as subdivision_type",
+        "amd.alinea",
+        "amd.obs as observations",
+        "amd.mot as observations_additionnelles",
+        sql `to_char(${eb.ref("amd.datdep")}, 'YYYY-MM-DD')`.as("date_depot"),
+        "amd.dis as dispositif",
+        "amd.obj as objet",
+        "typrect.lib as type_rectification",
+        "mot.lib as motion_libelle",
+        eb
+            .case()
+            .when("amd.etaid", "=", 7)
+            .then("Diffusé")
+            .when("amd.etaid", "=", 8)
+            .then("Retiré avant réunion ou séance")
+            .when("amd.etaid", "=", 9)
+            .then("Examiné en commission ou séance")
+            .when("amd.etaid", "=", 10)
+            .then("Irrecevable")
+            .when("amd.etaid", "=", 11)
+            .then("Irrecevable")
+            .else("")
+            .end()
+            .as("etat"),
+        "avicom.lib as avis_commission",
+        "avigvt.lib as avis_gouvernement",
+        eb.fn.coalesce("sor.lib", "irr.libirr").as("sort"),
+        "amd.rev as revision",
+        sql `
+        (
+          case
+            when ${eb.ref("amd.num")} like '%COM%' then 'https://www.senat.fr/amendements/commissions/'
+            else 'https://www.senat.fr/amendements/'
+          end ||
+          ${eb.ref("ses.ann")}::text || '-' ||
+          (${eb.ref("ses.ann")}::integer + 1)::text || '/' ||
+          ${eb.ref("txt_ameli.numabs")} || '/Amdt_' ||
+          ${eb.ref("amd.num")} || '.html'
+        )
+      `.as("url"),
+        "grppol_ameli.lilcou as au_nom_de_groupe_politique",
+        sql `rtrim(${eb.ref("com_ameli.lil")})`.as("au_nom_de_commission"),
+        sql `rtrim(${eb.ref("com_ameli.cod")})`.as("code_commission"),
+        sql `(${eb.ref("cab.entid")} is not null)`.as("auteur_est_gouvernement"),
+        // Sous-requête scalaire: numéro de scrutin
+        eb
+            .withSchema("senat")
+            .selectFrom("dosleg_amescr as amescr")
+            .leftJoin("dosleg_scr as scr", (join) => join.onRef("amescr.scrnum", "=", "scr.scrnum").onRef("amescr.sesann", "=", "scr.sesann"))
+            .leftJoin("dosleg_date_seance as date_seance", "date_seance.code", "scr.code")
+            .select(sql `amescr.scrnum::text`.as("scrnum"))
+            .whereRef("amescr.amescrnum", "=", "amd.num")
+            .whereRef("amescr.sesann", "=", "ses.ann")
+            .whereRef("date_seance.lecidt", "=", "texte.lecassidt")
+            .limit(1)
+            .as("scrutin_num"),
+        // Sous-requête pour les auteurs
+        jsonArrayFrom(eb
+            .withSchema("senat")
+            .selectFrom("ameli_amdsen as amdsen")
+            .leftJoin("ameli_sen_ameli as sen_ameli", "sen_ameli.entid", "amdsen.senid")
+            .leftJoin("ameli_grppol_ameli as grppol_ameli2", "grppol_ameli2.entid", "amdsen.grpid")
+            .select([
+            "amdsen.prenomuse as prenom",
+            "amdsen.hom as homonyme",
+            "amdsen.nomuse as nom",
+            "amdsen.qua as qualite",
+            sql `amdsen.rng::text`.as("rang"),
+            "sen_ameli.mat as matricule",
+            "amdsen.grpid as groupe_politique_id",
+            "grppol_ameli2.cod as group_politique_code",
+            "grppol_ameli2.libcou as groupe_politique_libelle_court",
+            "grppol_ameli2.lilcou as groupe_politique_libelle",
+        ])
+            .whereRef("amdsen.amdid", "=", "amd.id")
+            .orderBy("amdsen.rng", "asc")).as("auteurs"),
+    ]);
+    const compiledQuery = (fromSession === undefined ? baseQuery : baseQuery.where("ses.ann", ">=", String(fromSession))).compile();
+    for await (const row of streamRawQuery(compiledQuery)) {
+        yield {
+            ...row,
+            auteurs: row.auteurs ?? [],
+        };
+    }
+}

package/lib/src/server/commission.d.ts

@@ -0,0 +1,19 @@
+import * as cheerio from "cheerio";
+import type { AnyNode } from "domhandler";
+import { CompteRendu } from "../other_types/compte_rendu.js";
+import { Reunion } from "../other_types/agenda.js";
+export declare function getRemainingTextAfterSpeakerHeader($: cheerio.CheerioAPI, $p: cheerio.Cheerio<AnyNode>): string;
+export type DaySection = {
+    title: string;
+    $start: cheerio.Cheerio<AnyNode>;
+    time?: string;
+};
+export declare function cleanTitle(t: string): string;
+export declare function extractDayH3Sections($: cheerio.CheerioAPI, dateISO: string): DaySection[];
+export declare function parseCommissionCRSectionFromDom($: cheerio.CheerioAPI, htmlFilePath: string, opts: {
+    dateISO: string;
+    hourShort: string | null;
+    organe?: string | null;
+    section: DaySection;
+    matched?: Reunion;
+}): CompteRendu | null;

package/lib/src/server/commission.js

@@ -0,0 +1,269 @@
+import * as cheerio from "cheerio";
+import path from "path";
+import { makeReunionUid } from "../utils/reunion_parsing.js";
+import { norm } from "../utils/string_cleaning.js";
+import { frDateToISO, hourShortToStartTime } from "../utils/date.js";
+import { toCRDate } from "../util.js";
+const PARA_h3_SEL = "p.sh_justify, p.sh_center, p.sh_marge, p[align], li, h3";
+function findDayRoot($, targetISO) {
+    let $root = $();
+    $("h2").each((_, el) => {
+        const txt = norm($(el).text());
+        const m = txt.match(/(?:Lundi|Mardi|Mercredi|Jeudi|Vendredi|Samedi|Dimanche)\s+(.+)$/i);
+        const iso = m ? frDateToISO(m[1]) : undefined;
+        if (iso === targetISO && $root.length === 0)
+            $root = $(el);
+    });
+    return $root;
+}
+function normalizeSpaces(s) {
+    return s.replace(/[\u00A0\u202F\u2009]/g, " ");
+}
+function stripIntroPunct(s) {
+    return s.replace(/^[\s]*[.:;]?\s*(?:[–—-]\s*)+/u, "");
+}
+function collectLeadingHeaderStrongEls($, $clone) {
+    const els = [];
+    const nodes = $clone.contents().toArray();
+    for (const node of nodes) {
+        if (node.type === "text") {
+            if (norm(node.data || ""))
+                break;
+            continue;
+        }
+        if (node.type === "tag") {
+            const $n = $(node);
+            if ($n.is("strong, b")) {
+                els.push(node);
+                continue;
+            }
+            if ($n.is("a") && $n.children("strong, b").length) {
+                $n.children("strong, b").each((_, el) => {
+                    els.push(el);
+                });
+                continue;
+            }
+            break;
+        }
+    }
+    return els;
+}
+// Remove orateur's name from text and clean intro punct
+export function getRemainingTextAfterSpeakerHeader($, $p) {
+    const $clone = $p.clone();
+    // 1) Remove <strong> at start
+    const headerStrongEls = collectLeadingHeaderStrongEls($, $clone);
+    for (const el of headerStrongEls)
+        $(el).remove();
+    // 2) normalize + clean intro punct
+    let remainingHtml = $clone.html() || "";
+    remainingHtml = normalizeSpaces(cheerio.load(remainingHtml).text());
+    remainingHtml = stripIntroPunct(remainingHtml);
+    const remainingText = norm(remainingHtml || "");
+    return remainingText;
+}
+function buildPointsFromParagraphs($, paras) {
+    const points = [];
+    let ordreAbsoluSeance = 0;
+    const normSpeaker = (s) => s
+        .normalize("NFKC")
+        .replace(/\s+/g, " ")
+        .replace(/[.:]\s*$/, "")
+        .trim();
+    const normQual = (s) => s
+        .normalize("NFKC")
+        .replace(/\s+/g, " ")
+        .replace(/^\s*,\s*|\s+$/g, "")
+        .replace(/[\s\u00A0]*[.,;:–—-]+$/u, "")
+        .trim();
+    let currentOrateur = null;
+    let currentQualite = "";
+    let currentTexte = "";
+    function isPresidentQual(qual) {
+        return /\bprésident(e)?\b/i.test(qual);
+    }
+    // Flush the buffered speaker’s text into points[] if any.
+    function flush() {
+        if (!currentOrateur || !currentTexte.trim())
+            return;
+        ordreAbsoluSeance++;
+        points.push({
+            code_grammaire: "PAROLE_GENERIQUE",
+            roledebat: isPresidentQual(currentQualite) ? "président" : "",
+            ordre_absolu_seance: String(ordreAbsoluSeance),
+            orateurs: { orateur: { nom: currentOrateur, id: "", qualite: currentQualite || "" } },
+            texte: { _: currentTexte.trim() },
+        });
+        currentOrateur = null;
+        currentQualite = "";
+        currentTexte = "";
+    }
+    function addPoint(payload) {
+        ordreAbsoluSeance++;
+        points.push({ ...payload, ordre_absolu_seance: String(ordreAbsoluSeance) });
+    }
+    for (const $p of paras) {
+        if ($p.closest("table").length)
+            continue;
+        const tagName = ($p.prop("tagName") || "").toString().toLowerCase();
+        const rawText = ($p.text() || "").replace(/\u00a0/g, " ").trim();
+        const text = norm(rawText);
+        if (!text || text.length <= 3)
+            continue;
+        const html = ($p.html() || "").trim();
+        const italicSpans = $p.find("i, em, span[style*='italic']");
+        const firstItalicOuter = italicSpans.length ? $(italicSpans[0]).prop("outerHTML") || "" : "";
+        const htmlBeforeFirstItalic = firstItalicOuter ? html.split(firstItalicOuter)[0].trim() : "";
+        const isPureItalic = italicSpans.length > 0 && italicSpans.length === $p.find("span,i,em").length && htmlBeforeFirstItalic === "";
+        if (tagName === "h3") {
+            flush();
+            addPoint({
+                code_style: "Titre",
+                code_grammaire: "TITRE_TEXTE_DISCUSSION",
+                texte: { _: text },
+            });
+            continue;
+        }
+        const boldSpans = $p.find("strong, b");
+        const joinedBold = norm(boldSpans
+            .map((_, el) => $(el).text() || "")
+            .get()
+            .join(""));
+        const [namePartRaw, qualPartRaw] = joinedBold.split(/\s*,\s+/, 2);
+        const namePart = namePartRaw ? normSpeaker(namePartRaw) : "";
+        const qualPart = qualPartRaw ? normQual(qualPartRaw) : "";
+        const looksLikeName = namePart.length > 3 && /^(M\.|Mme)[\s\u00A0\u202F]+/i.test(namePart);
+        const startsWithName = namePart && text.startsWith(namePart);
+        const isNewSpeaker = looksLikeName && startsWithName && namePart !== currentOrateur;
+        if (isNewSpeaker) {
+            flush();
+            currentOrateur = namePart;
+            currentQualite = qualPart;
+            const remainingText = getRemainingTextAfterSpeakerHeader($, $p);
+            currentTexte = remainingText;
+            continue;
+        }
+        if (isPureItalic || (!joinedBold && !currentOrateur && text)) {
+            flush();
+            addPoint({
+                code_style: "Info Italiques",
+                code_grammaire: "PAROLE_GENERIQUE",
+                texte: { _: "<i>" + text + "</i>" },
+            });
+            continue;
+        }
+        // concat text because same orateur
+        if (currentOrateur) {
+            const removeOrateurFromText = getRemainingTextAfterSpeakerHeader($, $p);
+            currentTexte += (currentTexte ? "<br/><br/>" : "") + removeOrateurFromText;
+            continue;
+        }
+    }
+    flush();
+    return points;
+}
+const TIME_RE = /(?:\b[àa]\s*)?(\d{1,2})\s*(?:h|heures?)\s*(?:([0-5]\d))?/i;
+export function cleanTitle(t) {
+    return (t || "").replace(/\s+/g, " ").trim();
+}
+function parseTimeToHHmm(text) {
+    const m = normalizeSpaces(text).match(TIME_RE);
+    if (!m)
+        return undefined;
+    const hh = m[1]?.padStart(2, "0");
+    const mm = (m[2] ?? "00").padStart(2, "0");
+    const h = Number(hh);
+    if (h >= 0 && h <= 23)
+        return `${hh}:${mm}`;
+    return undefined;
+}
+function findNearbyTime($, $h3) {
+    let cur = $h3.prev();
+    for (let i = 0; i < 3 && cur.length; i++, cur = cur.prev()) {
+        const direct = parseTimeToHHmm(cur.text());
+        if (direct)
+            return direct;
+        const italic = parseTimeToHHmm(cur.find("i, em").first().text());
+        if (italic)
+            return italic;
+    }
+    return undefined;
+}
+export function extractDayH3Sections($, dateISO) {
+    const sections = [];
+    const $dayRoot = findDayRoot($, dateISO);
+    if ($dayRoot.length === 0)
+        return sections;
+    const $range = $dayRoot.nextUntil("h2");
+    const $h3s = $range.filter("h3").add($range.find("h3"));
+    $h3s.each((_, el) => {
+        const $h3 = $(el);
+        const title = cleanTitle($h3.text());
+        if (!title)
+            return;
+        const time = findNearbyTime($, $h3);
+        sections.push({ title, $start: $h3, time });
+    });
+    return sections;
+}
+export function parseCommissionCRSectionFromDom($, htmlFilePath, opts) {
+    try {
+        const { dateISO, hourShort, organe, section, matched } = opts;
+        const seanceRef = matched?.uid ?? makeReunionUid(dateISO, "COM", matched?.events[0].id ?? hourShort ?? "", organe ?? undefined);
+        const uid = seanceRef.replace(/^RU/, "CRC");
+        const dateSeance = toCRDate(dateISO, matched?.startTime ?? hourShortToStartTime(hourShort));
+        const $dayRoot = findDayRoot($, dateISO);
+        if ($dayRoot.length === 0) {
+            console.warn(`[COM-CR][parse] day root not found for ${dateISO} in ${path.basename(htmlFilePath)}`);
+            return null;
+        }
+        const paras = [];
+        let $cursor = section.$start;
+        // Jump title if we do not want to add it to paragraphes
+        $cursor = $cursor.next();
+        while ($cursor.length && !$cursor.is("h2") && !$cursor.is("h3")) {
+            if ($cursor.is(PARA_h3_SEL)) {
+                paras.push($cursor);
+            }
+            else {
+                const $ps = $cursor.find(PARA_h3_SEL);
+                if ($ps.length)
+                    $ps.each((_, p) => {
+                        paras.push($(p));
+                    });
+            }
+            $cursor = $cursor.next();
+        }
+        const points = buildPointsFromParagraphs($, paras);
+        if (points.length < 4 || !points.some((pt) => pt.code_grammaire === "PAROLE_GENERIQUE" && pt.orateurs)) {
+            console.warn(`[COM-CR][parse] Insufficient points or no interventions found for a section in ${path.basename(htmlFilePath)}`);
+            return null;
+        }
+        const session = dateISO.slice(5, 7) >= "10" ? `${dateISO.slice(0, 4)}` : `${Number(dateISO.slice(0, 4)) - 1}`;
+        const contenu = {
+            quantiemes: { journee: dateISO, session },
+            point: points,
+        };
+        const metadonnees = {
+            dateSeance,
+            dateSeanceJour: dateISO,
+            numSeanceJour: "",
+            numSeance: "",
+            typeAssemblee: "SN",
+            legislature: "",
+            session,
+            nomFichierJo: path.basename(htmlFilePath),
+            validite: "non-certifie",
+            etat: "definitif",
+            diffusion: "publique",
+            version: "1",
+            environnement: "prod",
+            heureGeneration: new Date(),
+        };
+        return { uid, seanceRef, sessionRef: session, metadonnees, contenu };
+    }
+    catch (e) {
+        console.error(`[COM-CR][parse] error section file=${path.basename(htmlFilePath)}:`, e);
+        return null;
+    }
+}

package/lib/src/server/config.d.ts

@@ -0,0 +1,43 @@
+import "dotenv/config";
+import { z } from "zod";
+export declare const dbSchema: z.ZodObject<{
+    host: z.ZodString;
+    name: z.ZodString;
+    password: z.ZodString;
+    port: z.ZodCoercedNumber<unknown>;
+    user: z.ZodString;
+}, z.core.$strip>;
+export type DbConfig = z.infer<typeof dbSchema>;
+export declare const configSchema: z.ZodObject<{
+    db: z.ZodObject<{
+        host: z.ZodString;
+        name: z.ZodString;
+        password: z.ZodString;
+        port: z.ZodCoercedNumber<unknown>;
+        user: z.ZodString;
+    }, z.core.$strip>;
+    stagingDb: z.ZodObject<{
+        host: z.ZodString;
+        name: z.ZodString;
+        password: z.ZodString;
+        port: z.ZodCoercedNumber<unknown>;
+        user: z.ZodString;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+declare const _default: {
+    db: {
+        host: string;
+        name: string;
+        password: string;
+        port: number;
+        user: string;
+    };
+    stagingDb: {
+        host: string;
+        name: string;
+        password: string;
+        port: number;
+        user: string;
+    };
+};
+export default _default;

package/lib/src/server/config.js

@@ -0,0 +1,37 @@
+import "dotenv/config";
+import { z } from "zod";
+export const dbSchema = z.object({
+    host: z.string().trim().min(1, "Must not be empty"),
+    name: z.string().trim().min(1, "Must not be empty"),
+    password: z.string().trim().min(1, "Must not be empty"),
+    port: z.coerce.number().int().min(0).max(65535),
+    user: z.string().trim().min(1, "Must not be empty"),
+});
+export const configSchema = z.object({
+    db: dbSchema,
+    stagingDb: dbSchema,
+});
+const config = {
+    db: {
+        host: process.env["DB_HOST"] || "localhost",
+        name: process.env["DB_NAME"] || "senat",
+        password: process.env["DB_PASSWORD"] || "opendata",
+        port: process.env["DB_PORT"] || 5432,
+        user: process.env["DB_USER"] || "opendata",
+    },
+    stagingDb: {
+        host: process.env["STAGING_DB_HOST"] || process.env["DB_HOST"] || "localhost",
+        name: process.env["STAGING_DB_NAME"] || "senat_staging",
+        password: process.env["STAGING_DB_PASSWORD"] || process.env["DB_PASSWORD"] || "opendata",
+        port: process.env["STAGING_DB_PORT"] || process.env["DB_PORT"] || 5432,
+        user: process.env["STAGING_DB_USER"] || process.env["DB_USER"] || "opendata",
+    },
+};
+const result = configSchema.safeParse(config);
+if (!result.success) {
+    const issues = JSON.stringify(result.error.issues, null, 2);
+    const serializedConfig = JSON.stringify(config, null, 2);
+    console.error(`Error in configuration:\n${serializedConfig}\nError:\n${issues}`);
+    process.exit(-1);
+}
+export default result.data;

package/lib/src/server/conversion_textes.d.ts

@@ -0,0 +1,11 @@
+export interface SenatMetadata {
+    number: string | null;
+    session: string | null;
+    date: string | null;
+    type: string | null;
+    authors: string | null;
+    title: string | null;
+    commission: string | null;
+}
+export declare function extractMetadata(xmlDoc: Document): SenatMetadata;
+export declare function convertSenatXmlToHtml(texteXml: string, outputFilePath: string): Promise<void>;