@tricoteuses/senat 3.0.2 → 3.1.1

package/lib/{loaders.js → src/server/loaders.js}

@@ -1,14 +1,15 @@
 import fsex from "fs-extra";
 import fs from "fs";
 import path from "path";
-import * as git from "./git";
-import { datasets } from "./datasets";
-import { UNDEFINED_SESSION } from "./types/sessions";
-export { EnabledDatasets } from "./datasets";
+import * as git from "./git.js";
+import { datasets } from "./datasets.js";
+import { UNDEFINED_SESSION } from "../other_types/sessions.js";
+export { EnabledDatasets } from "./datasets.js";
 export const AGENDA_FOLDER = "agenda";
 export const COMPTES_RENDUS_FOLDER = "seances";
 export const COMMISSION_FOLDER = "commissions";
 export const DOSLEG_DOSSIERS_FOLDER = "dossiers";
+export const ENRICHED_TEXTE_FOLDER = "leg_enrichi";
 export const SCRUTINS_FOLDER = "scrutins";
 export const SENS_CIRCONSCRIPTIONS_FOLDER = "circonscriptions";
 export const SENS_ORGANISMES_FOLDER = "organismes";
@@ -68,7 +69,8 @@ function* iterLoadSenatItems(dataDir, dataName, legislatureOrSession, subDir, {
             const itemJson = fs.readFileSync(filePath, { encoding: "utf8" });
             item = JSON.parse(itemJson);
         }
-        catch (err) {
+        catch (error) {
+            const err = error;
             console.warn(`[iterLoadSenatItems] skipped invalid JSON: ${filePath} (${err.message})`);
             continue;
         }
@@ -117,33 +119,15 @@ export function* iterLoadSenatDebats(dataDir, session, options = {}) {
         yield debatItem;
     }
 }
-function* iterLoadSenatComptesRendusGeneric(dataDir, session, subFolder) {
-    const basePath = path.join(dataDir, subFolder, DATA_TRANSFORMED_FOLDER, String(session));
-    if (!fs.existsSync(basePath)) {
-        return;
-    }
-    const files = (fs.readdirSync(basePath) || []).filter((f) => f.endsWith(".json")).sort();
-    for (const fileName of files) {
-        const filePath = path.join(basePath, fileName);
-        try {
-            const fileContent = fs.readFileSync(filePath, "utf-8");
-            const compteRendu = JSON.parse(fileContent);
-            if (!compteRendu?.uid) {
-                console.warn(`[SN] CR without uid → ${fileName}`);
-                continue;
-            }
-            yield { compteRendu, session };
-        }
-        catch (err) {
-            console.warn(`[SN] error reading CR → ${fileName}`, err);
-        }
+export function* iterLoadSenatComptesRendusSeances(dataDir, session, options = {}) {
+    for (const compteRenduItem of iterLoadSenatItems(dataDir, COMPTES_RENDUS_FOLDER, session, DATA_TRANSFORMED_FOLDER, options)) {
+        yield compteRenduItem;
     }
 }
-export function* iterLoadSenatComptesRendusSeances(dataDir, session) {
-    yield* iterLoadSenatComptesRendusGeneric(dataDir, session, COMPTES_RENDUS_FOLDER);
-}
-export function* iterLoadSenatComptesRendusCommissions(dataDir, session) {
-    yield* iterLoadSenatComptesRendusGeneric(dataDir, session, COMMISSION_FOLDER);
+export function* iterLoadSenatComptesRendusCommissions(dataDir, session, options = {}) {
+    for (const compteRenduItem of iterLoadSenatItems(dataDir, COMMISSION_FOLDER, session, DATA_TRANSFORMED_FOLDER, options)) {
+        yield compteRenduItem;
+    }
 }
 export function* iterLoadSenatDossiersLegislatifs(dataDir, session, options = {}) {
     for (const dossierLegislatifItem of iterLoadSenatItems(dataDir, datasets.dosleg.database, session, DOSLEG_DOSSIERS_FOLDER, options)) {
@@ -182,29 +166,6 @@ export function* iterLoadSenatTexteUrls(dataDir, session) {
         }
     }
 }
-export function* iterLoadSenatDocuments(dataDir, session, documentType, options = {}) {
-    for (const { item: dossierLegislatif } of iterLoadSenatDossiersLegislatifs(dataDir, session, options)) {
-        for (const lecture of dossierLegislatif["lectures"]) {
-            const lecturesSenat = lecture.lectures_assemblee.filter((lectureAssemblee) => lectureAssemblee.assemblee === "Sénat");
-            for (const lectureSenat of lecturesSenat) {
-                for (const document of lectureSenat[documentType]) {
-                    const enrichedDocument = {
-                        signet_dossier: dossierLegislatif["signet"],
-                        ...document,
-                    };
-                    const documentItem = {
-                        item: enrichedDocument
-                    };
-                    if (document.url) {
-                        const documentName = path.parse(document.url).name;
-                        documentItem.filePathFromDataset = path.join(`${document.session ?? UNDEFINED_SESSION}`, documentName, `${documentName}.pdf`);
-                    }
-                    yield documentItem;
-                }
-            }
-        }
-    }
-}
 export function* iterLoadSenatRapports(dataDir, session, options = {}) {
     for (const iterItem of iterLoadSenatItems(dataDir, RAPPORT_FOLDER, session, "original", options)) {
         if (iterItem.item?.["id"]) {
@@ -213,14 +174,25 @@ export function* iterLoadSenatRapports(dataDir, session, options = {}) {
     }
 }
 export function* iterLoadSenatTextes(dataDir, session, options = {}) {
-    for (const iterItem of iterLoadSenatDocuments(dataDir, session, "textes", options)) {
-        yield iterItem;
+    for (const iterItem of iterLoadSenatItems(dataDir, TEXTE_FOLDER, session, DATA_ORIGINAL_FOLDER, options)) {
+        if (!iterItem.item?.["id"]) {
+            continue;
+        }
+        const texteItem = iterItem;
+        const texte = texteItem.item;
+        const texteId = texte["id"];
+        if (!texteId) {
+            continue;
+        }
+        const { item: texteContent } = loadSenatTexteContent(dataDir, texte["session"], texteId);
+        if (texteContent) {
+            texteItem.item = { ...texteContent, ...texteItem.item };
+        }
+        yield texteItem;
     }
 }
-export function loadSenatTexteContent(dataDir, textePathFromDataset) {
-    const parsedTextePath = path.parse(textePathFromDataset);
-    const jsonTexteName = `${parsedTextePath.name}.json`;
-    const fullTextePath = path.join(dataDir, TEXTE_FOLDER, DATA_TRANSFORMED_FOLDER, parsedTextePath.dir, jsonTexteName);
+export function loadSenatTexteContent(dataDir, session, texteId) {
+    const fullTextePath = path.join(dataDir, TEXTE_FOLDER, DATA_TRANSFORMED_FOLDER, String(session ?? UNDEFINED_SESSION), texteId, `${texteId}.json`);
     if (!fs.existsSync(fullTextePath)) {
         return { item: null };
     }
@@ -254,9 +226,11 @@ export function* iterLoadSenatAgendas(dataDir, session) {
         const gr = raw;
         if (!gr.uid || !gr.date || !gr.titre)
             continue;
-        if (!Array.isArray(gr.events))
-            gr.events = [];
-        yield { item: gr };
+        const reunion = {
+            ...gr,
+            events: Array.isArray(gr.events) ? gr.events : [],
+        };
+        yield { item: reunion };
     }
 }
 export function* iterLoadSenatCirconscriptions(dataDir, options = {}) {

package/lib/src/server/questions.d.ts

@@ -0,0 +1,2 @@
+import type { QuestionResult } from "../rich_types/questions.js";
+export declare function findAll(): AsyncGenerator<QuestionResult, void, unknown>;

package/lib/src/server/questions.js

@@ -0,0 +1,89 @@
+import { sql } from "kysely";
+import { jsonArrayFrom } from "kysely/helpers/postgres";
+import { db, streamRawQuery } from "./databases_postgres.js";
+export async function* findAll() {
+    const compiledQuery = db
+        .withSchema("senat")
+        .selectFrom("questions_tam_questions as tam_questions")
+        .leftJoin("questions_tam_ministeres as tam_ministeres", "tam_ministeres.minid", "tam_questions.mindepotid")
+        .leftJoin("questions_sortquestion as sortquestion", (join) => join.on(sql `tam_questions.sorquecod::text = sortquestion.sorquecod`))
+        .leftJoin("questions_naturequestion as naturequestion", "naturequestion.natquecod", "tam_questions.natquecod")
+        .leftJoin("questions_etatquestion as etatquestion", (join) => join.on(sql `tam_questions.etaquecod::text = etatquestion.etaquecod`))
+        .select((eb) => [
+        sql `${eb.ref("tam_questions.repub")}::text`.as("republique"),
+        sql `${eb.ref("tam_questions.legislature")}::text`.as("legislature"),
+        "tam_questions.natquecod as nature",
+        "naturequestion.natquelib as nature_libelle",
+        "tam_questions.numero as numero",
+        "tam_questions.reference as reference",
+        "tam_questions.titre as titre",
+        "tam_questions.nom as nom",
+        "tam_questions.prenom as prenom",
+        "tam_questions.nomtechnique as nom_technique",
+        "tam_questions.codequalite as civilite",
+        "tam_questions.matricule as matricule",
+        sql `${eb.ref("tam_questions.cirnum")}::text`.as("circonscription_numero"),
+        "tam_questions.circonscription as circonscription",
+        "tam_questions.groupe as groupe",
+        "tam_questions.ratgrp as type_appartenance",
+        sql `to_char(${eb.ref("tam_questions.datejodepot")}, 'YYYY-MM-DD')`.as("date_publication_JO"),
+        sql `${eb.ref("tam_questions.pagejodepot")}::text`.as("page_JO"),
+        sql `rtrim(${eb.ref("tam_questions.mindepotlib")})`.as("ministere_depot"),
+        sql `${eb.ref("tam_questions.mindepotid")}::text`.as("ministere_depot_id"),
+        sql `to_char(${eb.ref("tam_ministeres.datedebut")}, 'YYYY-MM-DD')`.as("ministere_depot_date_debut"),
+        sql `to_char(${eb.ref("tam_questions.datejotran")}, 'YYYY-MM-DD')`.as("date_transmission"),
+        "tam_questions.mintranlib as ministere_transmission",
+        sql `to_char(${eb.ref("tam_questions.datejorep1")}, 'YYYY-MM-DD')`.as("date_reponse_JO"),
+        sql `to_char(${eb.ref("tam_questions.dateerr")}, 'YYYY-MM-DD')`.as("date_erratum_JO"),
+        sql `${eb.ref("tam_questions.pageerr")}::text`.as("page_erratum_JO"),
+        "tam_questions.minreplib1 as ministere_reponse",
+        sql `to_char(${eb.ref("tam_questions.datecloture")}, 'YYYY-MM-DD')`.as("date_cloture"),
+        sql `to_char(${eb.ref("tam_questions.datesignal")}, 'YYYY-MM-DD')`.as("date_signalement"),
+        "tam_questions.refquerappelee as reference_question_rappelee",
+        "tam_questions.txtque as texte",
+        "tam_questions.txterrque as texte_erratum",
+        "tam_questions.rubrique as rubrique",
+        "sortquestion.sorquelib as sort",
+        "sortquestion.sorquecod as sort_code",
+        sql `${eb.ref("sortquestion.sorquenumtri")}::text`.as("sort_tri"),
+        "etatquestion.etaquelib as etat_libelle",
+        "etatquestion.etaquecod as etat_code",
+        sql `${eb.ref("etatquestion.etaquenumtri")}::text`.as("etat_tri"),
+        // Thèmes (extraits par regexp depuis la colonne themes)
+        sql `(
+        select coalesce(json_agg(theme_rows.libelle order by theme_rows.theme_order nulls last), '[]'::json)
+        from (
+          select
+            the.thelib as libelle,
+            the.thenouidt as theme_order
+          from regexp_matches(coalesce(${eb.ref("tam_questions.themes")}, ''), '#(\\d+)', 'g') as matches(match)
+          join senat.questions_the as the on the.thenouidt::text = matches.match[1]
+          order by the.thenouidt asc
+        ) as theme_rows
+      )`.as("themes"),
+        // Réponses
+        jsonArrayFrom(eb
+            .withSchema("senat")
+            .selectFrom("questions_tam_reponses as tam_reponses")
+            .select([
+            sql `to_char(tam_reponses.datejorep, 'YYYY-MM-DD')`.as("date_reponse_JO"),
+            sql `tam_reponses.pagejorep::text`.as("page_JO"),
+            sql `to_char(tam_reponses.errdate, 'YYYY-MM-DD')`.as("date_erratum_JO"),
+            "tam_reponses.minreplib as ministere_reponse",
+            sql `tam_reponses.minrepid::text`.as("ministere_reponse_id"),
+            "tam_reponses.urlrep as url",
+            "tam_reponses.txtrep as texte",
+            "tam_reponses.txterrrep as texte_erratum",
+        ])
+            .whereRef("tam_reponses.idque", "=", "tam_questions.id")
+            .orderBy("tam_reponses.datejorep", "asc")).as("reponses"),
+    ])
+        .compile();
+    for await (const row of streamRawQuery(compiledQuery)) {
+        yield {
+            ...row,
+            reponses: row.reponses ?? [],
+            themes: row.themes ?? [],
+        };
+    }
+}

package/lib/src/server/scrutins.d.ts

@@ -0,0 +1,2 @@
+import type { ScrutinResult } from "../scrutins.js";
+export declare function findAllScrutins(fromSession?: number): AsyncGenerator<ScrutinResult, void, unknown>;

package/lib/src/server/scrutins.js

@@ -0,0 +1,114 @@
+import { sql } from "kysely";
+import { jsonArrayFrom } from "kysely/helpers/postgres";
+import { db, streamRawQuery } from "./databases_postgres.js";
+export async function* findAllScrutins(fromSession) {
+    const baseQuery = db
+        .withSchema("senat")
+        .selectFrom("dosleg_scr as scr")
+        .leftJoin("dosleg_date_seance as date_seance", "date_seance.code", "scr.code")
+        .leftJoin("dosleg_lecass as lecass", "lecass.lecassidt", "date_seance.lecidt")
+        .leftJoin("dosleg_lecture as lecture", "lecture.lecidt", "lecass.lecidt")
+        .leftJoin("dosleg_loi as loi", "loi.loicod", "lecture.loicod")
+        .leftJoin("dosleg_typlec as typlec", "typlec.typleccod", "lecture.typleccod")
+        .select((eb) => [
+        sql `${eb.ref("scr.sesann")}::text`.as("session"),
+        sql `${eb.ref("scr.scrnum")}::text`.as("numero"),
+        sql `rtrim(${eb.ref("date_seance.lecidt")})`.as("lecture_id"),
+        sql `rtrim(${eb.ref("lecture.leccom")})`.as("lecture_libelle"),
+        "loi.signet as signet_dossier",
+        sql `rtrim(${eb.ref("typlec.typleclib")})`.as("type_lecture"),
+        sql `to_char(${eb.ref("scr.scrdat")}, 'YYYY-MM-DD')`.as("date_scrutin"),
+        sql `to_char(${eb.ref("scr.scrdateff")}, 'YYYY-MM-DD')`.as("date_scrutin_effective"),
+        sql `replace(${eb.ref("scr.scrint")}, E'\\u0092', '''')`.as("intitule"),
+        "scr.scrbaspag as note",
+        sql `${eb.ref("scr.scrmaj")}::text`.as("nombre_majorite"),
+        sql `${eb.ref("scr.scrmajsea")}::text`.as("nombre_majorite_seance"),
+        sql `${eb.ref("scr.scrvot")}::text`.as("nombre_votants"),
+        sql `${eb.ref("scr.scrvotsea")}::text`.as("nombre_votants_seance"),
+        sql `${eb.ref("scr.scrsuf")}::text`.as("nombre_suffrages"),
+        sql `${eb.ref("scr.scrsufsea")}::text`.as("nombre_suffrages_seance"),
+        sql `${eb.ref("scr.scrcon")}::text`.as("nombre_contre"),
+        sql `${eb.ref("scr.scrconsea")}::text`.as("nombre_contre_seance"),
+        sql `${eb.ref("scr.scrpou")}::text`.as("nombre_pour"),
+        sql `${eb.ref("scr.scrpousea")}::text`.as("nombre_pour_seance"),
+        // Votes individuels
+        jsonArrayFrom(eb
+            .withSchema("senat")
+            .selectFrom("dosleg_votsen as votsen")
+            .leftJoin("dosleg_titsen as titsen", "titsen.titsencod", "votsen.titsencod")
+            .leftJoin("dosleg_stavot as stavot", "stavot.stavotidt", "votsen.stavotidt")
+            .leftJoin("dosleg_posvot as posvot", "posvot.posvotcod", "votsen.posvotcod")
+            .leftJoin("sens_memgrppol as memgrppol", (join) => join
+            .onRef("memgrppol.senmat", "=", "votsen.senmat")
+            .on(sql `memgrppol.memgrppoldatdeb <= ${eb.ref("scr.scrdat")}`)
+            .on(sql `(memgrppol.memgrppoldatfin >= ${eb.ref("scr.scrdat")} or memgrppol.memgrppoldatfin is null)`))
+            .select([
+            "votsen.senmat as matricule_votant",
+            "votsen.senmatdel as matricule_delegant",
+            "posvot.posvotlib as position",
+            "stavot.stavotlib as statut_votant",
+            "titsen.titsenlib as titre_votant",
+            sql `memgrppol.memgrppolid::text`.as("groupe_politique_id"),
+            "memgrppol.grppolcod as groupe_politique_code",
+            "memgrppol.memgrppoldatdeb as membership_start",
+        ])
+            .whereRef("votsen.scrnum", "=", "scr.scrnum")
+            .whereRef("votsen.sesann", "=", "scr.sesann")
+            .orderBy("memgrppol.memgrppoldatdeb", (ob) => ob.desc().nullsLast())).as("votes"),
+        // Groupes votants (avec déduplication via window function)
+        jsonArrayFrom(eb
+            .withSchema("senat")
+            .selectFrom(eb
+            .withSchema("senat")
+            .selectFrom("dosleg_votsen as votsen")
+            .leftJoin("dosleg_posvot as posvot", "posvot.posvotcod", "votsen.posvotcod")
+            .leftJoin("dosleg_stavot as stavot", "stavot.stavotidt", "votsen.stavotidt")
+            .leftJoin("sens_memgrppol as memgrppol", (join) => join
+            .onRef("memgrppol.senmat", "=", "votsen.senmat")
+            .on(sql `memgrppol.memgrppoldatdeb <= ${eb.ref("scr.scrdat")}`)
+            .on(sql `(memgrppol.memgrppoldatfin >= ${eb.ref("scr.scrdat")} or memgrppol.memgrppoldatfin is null)`))
+            .select([
+            "votsen.senmat",
+            "posvot.posvotlib",
+            "stavot.stavotlib",
+            "memgrppol.grppolcod",
+            sql `row_number() over (
+                  partition by votsen.senmat, votsen.scrnum, votsen.sesann
+                  order by memgrppol.memgrppoldatdeb desc nulls last
+                )`.as("vote_rank"),
+        ])
+            .whereRef("votsen.scrnum", "=", "scr.scrnum")
+            .whereRef("votsen.sesann", "=", "scr.sesann")
+            .as("unique_votes"))
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            .select((_eb2) => [
+            "unique_votes.grppolcod as groupe_politique_code",
+            sql `sum(case when unique_votes.posvotlib = 'pour' then 1 else 0 end)::text`.as("nombre_pour"),
+            sql `sum(case when unique_votes.posvotlib = 'contre' then 1 else 0 end)::text`.as("nombre_contre"),
+            sql `sum(case when unique_votes.posvotlib = 'abstention' then 1 else 0 end)::text`.as("nombre_abstentions"),
+            sql `sum(case when unique_votes.posvotlib = 'non-votant' then 1 else 0 end)::text`.as("nombre_non_votants"),
+            sql `count(unique_votes.senmat)::text`.as("nombre_votants"),
+        ])
+            .where("unique_votes.vote_rank", "=", 1)
+            .groupBy("unique_votes.grppolcod")).as("groupes_votants"),
+        // Mises au point (corrections)
+        jsonArrayFrom(eb
+            .withSchema("senat")
+            .selectFrom("dosleg_corscr as corscr")
+            .select([
+            "corscr.corscrtxt as texte",
+            sql `corscr.corscrord::text`.as("ordre"),
+            "corscr.corscrurl as url",
+        ])
+            .whereRef("corscr.scrnum", "=", "scr.scrnum")).as("mises_au_point"),
+    ]);
+    const compiledQuery = (fromSession === undefined ? baseQuery : baseQuery.where("scr.sesann", ">=", String(fromSession))).compile();
+    for await (const row of streamRawQuery(compiledQuery)) {
+        yield {
+            ...row,
+            groupes_votants: row.groupes_votants ?? [],
+            mises_au_point: row.mises_au_point ?? [],
+            votes: row.votes ?? [],
+        };
+    }
+}

package/lib/{model → src/server}/seance.d.ts

@@ -1,3 +1,3 @@
-import { CompteRendu } from "../types/compte_rendu";
+import { CompteRendu } from "../other_types/compte_rendu.js";
 export declare function parseCompteRenduIntervalFromFile(xmlFilePath: string, startIndex: number, endIndex: number, agendaEventId: string): Promise<CompteRendu | null>;
 export declare function sessionStartYearFromDate(d: Date): number;

package/lib/{model → src/server}/seance.js

@@ -1,9 +1,9 @@
 import fs from "fs";
 import * as cheerio from "cheerio";
-import { toCRDate } from "./util";
-import { makeReunionUid } from "../utils/reunion_parsing";
-import { yyyymmddFromPath } from "../utils/date";
-import { decodeHtmlEntities, dedupeSpeaker, fixApostrophes, norm } from "../utils/string_cleaning";
+import { toCRDate } from "../util.js";
+import { makeReunionUid } from "../utils/reunion_parsing.js";
+import { yyyymmddFromPath } from "../utils/date.js";
+import { decodeHtmlEntities, dedupeSpeaker, fixApostrophes, norm } from "../utils/string_cleaning.js";
 export async function parseCompteRenduIntervalFromFile(xmlFilePath, startIndex, endIndex, agendaEventId) {
     try {
         const raw = fs.readFileSync(xmlFilePath, "utf8");
@@ -140,10 +140,10 @@ function extractAndRemoveLeadingQualite($, $block) {
                 $node.remove();
         }
         else if (node.type === "text") {
-            const t = norm(node.data || "");
+            const textNode = node;
+            const t = norm(textNode.data || "");
             if (!t || /^[:.,;–—-]+$/.test(t)) {
-                ;
-                node.data = "";
+                textNode.data = "";
                 return;
             }
             stop = true;

package/lib/src/server/sens.d.ts

@@ -0,0 +1,9 @@
+import type { CirconscriptionResult, OrganismeResult, SenateurResult } from "../rich_types/sens.js";
+export declare function findAll(): AsyncGenerator<SenateurResult, void, unknown>;
+export declare function findAllCirconscriptions(): AsyncGenerator<CirconscriptionResult, void, unknown>;
+export declare function findAllOrganismes(): AsyncGenerator<OrganismeResult, void, unknown>;
+export declare function findActif(): AsyncGenerator<{
+    senmat: string;
+    sennomuse: string | null;
+    senprenomuse: string;
+}, void, unknown>;