@tricoteuses/senat 3.1.2 → 3.1.4

package/lib/src/rich_types/dosleg.js

@@ -39,7 +39,7 @@ function getDateSortValue(value) {
 function compareByDate(left, right) {
     return getDateSortValue(left.date) - getDateSortValue(right.date);
 }
-function getPhasePrefix(lecture, assemblee) {
+function getPhasePrefix(lecture, assemblee, codeNatureDossier) {
     if (assemblee !== "Sénat")
         return null;
     const typeLibelle = (lecture.type_lecture || "").toLowerCase();
@@ -51,6 +51,11 @@ function getPhasePrefix(lecture, assemblee) {
         return "SNLDEF";
     if (typeLibelle.includes("unique"))
         return "SNLUNI";
+    // Les propositions de résolution (ppr) suivent la procédure de lecture unique
+    // (art. 73 quinquies du Règlement du Sénat). Les données brutes encodent leur
+    // type_lecture comme "Première lecture", mais c'est toujours une lecture unique.
+    if (codeNatureDossier === "ppr")
+        return "SNLUNI";
     if (lecture.ordre_lecture) {
         return `SN${lecture.ordre_lecture}`;
     }
@@ -69,7 +74,7 @@ export function buildActesLegislatifs(dossier) {
         for (const lecAss of lecturesAssemblee) {
             if (lecAss.assemblee !== "Sénat")
                 continue;
-            const phasePrefix = getPhasePrefix(lecture, lecAss.assemblee);
+            const phasePrefix = getPhasePrefix(lecture, lecAss.assemblee, dossier.code_nature_dossier);
             if (!phasePrefix)
                 continue;
             const textes = lecAss.textes ?? [];
@@ -144,8 +149,13 @@ export function buildActesLegislatifs(dossier) {
                 else if (origine.includes("devenue résolution")) {
                     libelleStatut = "Adopté";
                 }
+                // "devenu résolution du Sénat" : adoptée en commission sans séance publique (art. 73 quinquies RSN)
+                // On utilise COM-CAE-DEC plutôt que DEBATS-DEC pour refléter l'absence de séance plénière
+                const codeActeDecision = (texteFinal.origine || "").toLowerCase().includes("devenu résolution")
+                    ? `${phasePrefix}-COM-CAE-DEC`
+                    : `${phasePrefix}-DEBATS-DEC`;
                 actes.push({
-                    code_acte: `${phasePrefix}-DEBATS-DEC`,
+                    code_acte: codeActeDecision,
                     date: texteFinal.date,
                     libelle: `${libelleStatut === "Adopté" ? "Adoption" : "Rejet"} (Texte n°${texteFinal.numero})`,
                     id: texteFinal.id,

package/lib/src/rich_types/sens.d.ts

@@ -68,9 +68,11 @@ export interface SenateurResult {
     fonctions_bureau: FonctionRow[];
     groupe_politique: string | null;
     groupes: MandatOrganismeRow[];
+    groupes_amitie: MandatOrganismeRow[];
     mandats_senateur: MandatSenateurRow[];
     matricule: string;
     nom_usuel: string | null;
+    organismes: MandatOrganismeRow[];
     points_contact: PointContactRow[];
     prenom_usuel: string;
     qualite: string;

package/lib/src/scripts/data-download.js

@@ -1,13 +1,20 @@
 import { execSync } from "child_process";
 const args = process.argv.slice(2).join(" ");
-try {
-    execSync(`tsx src/scripts/retrieve_open_data.ts --all ${args}`, { stdio: "inherit" });
-    execSync(`tsx src/scripts/convert_data.ts ${args}`, { stdio: "inherit" });
-}
-catch (error) {
-    const execError = error;
-    if (execError.status !== 10) {
-        console.error("Error during data retrieval:", error);
-        process.exit(execError.status || 1);
+function runScript(command) {
+    try {
+        execSync(command, { stdio: "inherit" });
+    }
+    catch (error) {
+        const execError = error;
+        if (execError.status !== 10) {
+            console.error(`Error during: ${command}`, error);
+            process.exit(execError.status || 1);
+        }
     }
 }
+runScript(`tsx src/scripts/retrieve_open_data.ts --all ${args}`);
+runScript(`tsx src/scripts/convert_data.ts ${args}`);
+runScript(`cross-env TZ='Etc/UTC' tsx src/scripts/retrieve_agenda.ts ${args} --parseAgenda --silent`);
+runScript(`tsx src/scripts/retrieve_cr_seance.ts ${args} --parseDebats --silent`);
+runScript(`tsx src/scripts/retrieve_cr_commission.ts ${args} --parseDebats --silent`);
+runScript(`tsx src/scripts/retrieve_videos.ts ${args} --silent`);

package/lib/src/scripts/retrieve_open_data.js

@@ -2,9 +2,11 @@ import assert from "assert";
 import { execFileSync } from "child_process";
 import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
-import { formatWithPrettier, makePgTsGenerator, markAsGenerated, processDatabase } from "kanel";
-import { makeKyselyHook } from "kanel-kysely";
+import { createRequire } from "module";
 import path from "path";
+const _require = createRequire(import.meta.url);
+const { formatWithPrettier, makePgTsGenerator, markAsGenerated, processDatabase } = _require("kanel");
+const { makeKyselyHook } = _require("kanel-kysely");
 import StreamZip from "node-stream-zip";
 import readline from "readline";
 import { pipeline, Readable } from "stream";

package/lib/src/scripts/shared/make_generate_zod_schemas.js

@@ -1,5 +1,9 @@
-import { escapeName, resolveType, useKanelContext, } from "kanel";
-import { defaultGetZodIdentifierMetadata, defaultGetZodSchemaMetadata, defaultZodTypeMap } from "kanel-zod";
+import { createRequire } from "module";
+import { useKanelContext, } from "kanel";
+import { defaultGetZodIdentifierMetadata, defaultGetZodSchemaMetadata } from "kanel-zod";
+const _require = createRequire(import.meta.url);
+const { escapeName, resolveType } = _require("kanel");
+const { defaultZodTypeMap } = _require("kanel-zod");
 const zImport = {
     asName: undefined,
     importAsType: false,

package/lib/src/server/databases_postgres.js

@@ -1,5 +1,6 @@
 import { Pool } from "pg";
 import { Kysely, PostgresDialect, sql as kyselySql } from "kysely";
+import Cursor from "pg-cursor";
 import config from "./config.js";
 // ---------------------------------------------------------------------------
 // Kysely instance (uses pg driver so .stream() is available)
@@ -19,7 +20,7 @@ const pool = new Pool({
  *   `db.withSchema("senat").selectFrom("sens_sen")`
  */
 export const db = new Kysely({
-    dialect: new PostgresDialect({ pool }),
+    dialect: new PostgresDialect({ pool, cursor: Cursor }),
 });
 // ---------------------------------------------------------------------------
 // Streaming helper

package/lib/src/server/documents.js

@@ -17,7 +17,7 @@ export async function* findAllTextes() {
         sql `
         case
           when ${eb.ref("texte.texurl")} is not null then
-            regexp_replace(regexp_replace(trim(${eb.ref("texte.texurl")}), '#+$', ''), '^(.*/)?(.*?)(\.html)?$', '\2')
+            regexp_replace(regexp_replace(trim(${eb.ref("texte.texurl")}), '#+$', ''), '^(.*/)?(.*?)(\\.html)?$', '\\2')
           else null
         end
       `.as("id"),
@@ -78,7 +78,7 @@ export async function* findAllRapports() {
         sql `
         case
           when ${eb.ref("rap.rapurl")} is not null then
-            regexp_replace(regexp_replace(trim(${eb.ref("rap.rapurl")}), '#+$', ''), '^(.*/)?(.*?)(\.html)?$', '\2')
+            regexp_replace(regexp_replace(trim(${eb.ref("rap.rapurl")}), '#+$', ''), '^(.*/)?(.*?)(\\.html)?$', '\\2')
           else null
         end
       `.as("id"),

package/lib/src/server/dosleg.js

@@ -101,7 +101,7 @@ export async function* findAllDossiers() {
                             when texte.texurl is not null then
                               regexp_replace(
                                 regexp_replace(trim(texte.texurl), '#+$', ''),
-                                '^(.*/)?(.*?)(\.html)?$', '\2')
+                                '^(.*/)?(.*?)(\.html)?$', '\\2')
                             else null
                           end
                         `.as("id"),
@@ -153,7 +153,7 @@ export async function* findAllDossiers() {
                             when rap.rapurl is not null then
                               regexp_replace(
                                 regexp_replace(trim(rap.rapurl), '#+$', ''),
-                                '^(.*/)?(.*?)(\.html)?$', '\2')
+                                '^(.*/)?(.*?)(\.html)?$', '\\2')
                             else null
                           end
                         `.as("id"),

package/lib/src/server/sens.js

@@ -163,6 +163,74 @@ export async function* findAll() {
         ])
             .whereRef("memgrppol.senmat", "=", "sen.senmat")
             .orderBy("memgrppol.memgrppoldatdeb", (ob) => ob.desc().nullsLast())).as("groupes"),
+        // Organismes divers (groupes d'études, missions, etc.)
+        jsonArrayFrom(eb
+            .withSchema("senat")
+            .selectFrom("sens_memorg as memorg")
+            .leftJoin("sens_org as org", "org.orgcod", "memorg.orgcod")
+            .leftJoin("sens_typorg as typorg4", "typorg4.typorgcod", "org.typorgcod")
+            .select((eb2) => [
+            "memorg.orgcod as code_organisme",
+            sql `to_char(memorg.memorgdatdeb, 'YYYY-MM-DD')`.as("date_debut"),
+            sql `to_char(memorg.memorgdatfin, 'YYYY-MM-DD')`.as("date_fin"),
+            "memorg.temvalcod as etat",
+            "org.evelib as libelle",
+            "org.typorgcod as type_code_organisme",
+            "typorg4.typorglib as type_organisme",
+            "memorg.memorgdatdeb as order_date",
+            // Fonctions dans l'organisme
+            jsonArrayFrom(eb2
+                .withSchema("senat")
+                .selectFrom("sens_fonmemorg as fonmemorg")
+                .leftJoin("sens_fonorg as fonorg", "fonorg.fonorgcod", "fonmemorg.fonorgcod")
+                .select([
+                sql `to_char(fonmemorg.fonmemorgdatdeb, 'YYYY-MM-DD')`.as("date_debut"),
+                sql `to_char(fonmemorg.fonmemorgdatfin, 'YYYY-MM-DD')`.as("date_fin"),
+                sql `coalesce(
+                    nullif(fonorg.fonorglib, ''),
+                    nullif(fonorg.fonorglil, ''),
+                    nullif(fonorg.fonorglic, ''))`.as("libelle"),
+                "fonmemorg.fonmemorgdatdeb as order_date",
+            ])
+                .whereRef("fonmemorg.memorgid", "=", "memorg.memorgid")
+                .orderBy("fonmemorg.fonmemorgdatdeb", (ob) => ob.desc().nullsLast())).as("fonctions"),
+        ])
+            .whereRef("memorg.senmat", "=", "sen.senmat")
+            .orderBy("memorg.memorgdatdeb", (ob) => ob.desc().nullsLast())).as("organismes"),
+        // Groupes sénatoriaux (groupes d'amitié)
+        jsonArrayFrom(eb
+            .withSchema("senat")
+            .selectFrom("sens_memgrpsen as memgrpsen")
+            .leftJoin("sens_grpsenami as grpsenami", "grpsenami.orgcod", "memgrpsen.orgcod")
+            .leftJoin("sens_typorg as typorg5", "typorg5.typorgcod", "grpsenami.typorgcod")
+            .select((eb2) => [
+            "memgrpsen.orgcod as code_organisme",
+            sql `to_char(memgrpsen.memgrpsendatent, 'YYYY-MM-DD')`.as("date_debut"),
+            sql `to_char(memgrpsen.memgrpsendatsor, 'YYYY-MM-DD')`.as("date_fin"),
+            "memgrpsen.temvalcod as etat",
+            "grpsenami.evelib as libelle",
+            "grpsenami.typorgcod as type_code_organisme",
+            "typorg5.typorglib as type_organisme",
+            "memgrpsen.memgrpsendatent as order_date",
+            // Fonctions dans le groupe sénatorial
+            jsonArrayFrom(eb2
+                .withSchema("senat")
+                .selectFrom("sens_fonmemgrpsen as fonmemgrpsen")
+                .leftJoin("sens_fongrpsen as fongrpsen", "fongrpsen.fongrpsencod", "fonmemgrpsen.fongrpsencod")
+                .select([
+                sql `to_char(fonmemgrpsen.fonmemgrpsendatdeb, 'YYYY-MM-DD')`.as("date_debut"),
+                sql `to_char(fonmemgrpsen.fonmemgrpsendatfin, 'YYYY-MM-DD')`.as("date_fin"),
+                sql `coalesce(
+                    nullif(fongrpsen.fongrpsenlib, ''),
+                    nullif(fongrpsen.fongrpsenlil, ''),
+                    nullif(fongrpsen.fongrpsenlic, ''))`.as("libelle"),
+                "fonmemgrpsen.fonmemgrpsendatdeb as order_date",
+            ])
+                .whereRef("fonmemgrpsen.memgrpsenid", "=", "memgrpsen.memgrpsenid")
+                .orderBy("fonmemgrpsen.fonmemgrpsendatdeb", (ob) => ob.desc().nullsLast())).as("fonctions"),
+        ])
+            .whereRef("memgrpsen.senmat", "=", "sen.senmat")
+            .orderBy("memgrpsen.memgrpsendatent", (ob) => ob.desc().nullsLast())).as("groupes_amitie"),
         // Fonctions au bureau du Sénat
         jsonArrayFrom(eb
             .withSchema("senat")
@@ -226,7 +294,9 @@ export async function* findAll() {
             delegations: row.delegations ?? [],
             fonctions_bureau: row.fonctions_bureau ?? [],
             groupes: row.groupes ?? [],
+            groupes_amitie: row.groupes_amitie ?? [],
             mandats_senateur: row.mandats_senateur ?? [],
+            organismes: row.organismes ?? [],
             points_contact: row.points_contact ?? [],
             urls: row.urls ?? [],
         };

package/lib/src/utils/reunion_parsing.js

@@ -50,7 +50,7 @@ export function buildReunionsByBucket(events, dossierBySenatUrl) {
     for (const e of events) {
         const kind = classifyAgendaType(e?.type);
         if (!kind) {
-            console.warn("Can't determine type of reunion");
+            console.warn("Can't determine type of reunion ", e.type, "for event", e.id, e.titre);
             continue;
         }
         const bucket = typeToSuffixStrict(kind);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "3.1.2",
+  "version": "3.1.4",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",
@@ -84,6 +84,7 @@
     "luxon": "^3.7.2",
     "node-stream-zip": "^1.8.2",
     "p-limit": "^7.2.0",
+    "pg-cursor": "^2.19.0",
     "postgres": "^3.4.8",
     "slug": "^11.0.0",
     "windows-1252": "^3.0.4",
@@ -97,6 +98,7 @@
     "@types/luxon": "^3.7.1",
     "@types/node": "^24.10.1",
     "@types/pg": "^8.20.0",
+    "@types/pg-cursor": "^2.7.2",
     "@types/slug": "^5.0.9",
     "@typescript-eslint/eslint-plugin": "^8.52.0",
     "@typescript-eslint/parser": "^8.52.0",

package/lib/src/config.d.ts

@@ -1,43 +0,0 @@
-import "dotenv/config";
-import { z } from "zod";
-export declare const dbSchema: z.ZodObject<{
-    host: z.ZodString;
-    name: z.ZodString;
-    password: z.ZodString;
-    port: z.ZodCoercedNumber<unknown>;
-    user: z.ZodString;
-}, z.core.$strip>;
-export type DbConfig = z.infer<typeof dbSchema>;
-export declare const configSchema: z.ZodObject<{
-    db: z.ZodObject<{
-        host: z.ZodString;
-        name: z.ZodString;
-        password: z.ZodString;
-        port: z.ZodCoercedNumber<unknown>;
-        user: z.ZodString;
-    }, z.core.$strip>;
-    stagingDb: z.ZodObject<{
-        host: z.ZodString;
-        name: z.ZodString;
-        password: z.ZodString;
-        port: z.ZodCoercedNumber<unknown>;
-        user: z.ZodString;
-    }, z.core.$strip>;
-}, z.core.$strip>;
-declare const _default: {
-    db: {
-        host: string;
-        name: string;
-        password: string;
-        port: number;
-        user: string;
-    };
-    stagingDb: {
-        host: string;
-        name: string;
-        password: string;
-        port: number;
-        user: string;
-    };
-};
-export default _default;

package/lib/src/config.js

@@ -1,37 +0,0 @@
-import "dotenv/config";
-import { z } from "zod";
-export const dbSchema = z.object({
-    host: z.string().trim().min(1, "Must not be empty"),
-    name: z.string().trim().min(1, "Must not be empty"),
-    password: z.string().trim().min(1, "Must not be empty"),
-    port: z.coerce.number().int().min(0).max(65535),
-    user: z.string().trim().min(1, "Must not be empty"),
-});
-export const configSchema = z.object({
-    db: dbSchema,
-    stagingDb: dbSchema,
-});
-const config = {
-    db: {
-        host: process.env["DB_HOST"] || "localhost",
-        name: process.env["DB_NAME"] || "senat",
-        password: process.env["DB_PASSWORD"] || "opendata",
-        port: process.env["DB_PORT"] || 5432,
-        user: process.env["DB_USER"] || "opendata",
-    },
-    stagingDb: {
-        host: process.env["STAGING_DB_HOST"] || process.env["DB_HOST"] || "localhost",
-        name: process.env["STAGING_DB_NAME"] || "senat_staging",
-        password: process.env["STAGING_DB_PASSWORD"] || process.env["DB_PASSWORD"] || "opendata",
-        port: process.env["STAGING_DB_PORT"] || process.env["DB_PORT"] || 5432,
-        user: process.env["STAGING_DB_USER"] || process.env["DB_USER"] || "opendata",
-    },
-};
-const result = configSchema.safeParse(config);
-if (!result.success) {
-    const issues = JSON.stringify(result.error.issues, null, 2);
-    const serializedConfig = JSON.stringify(config, null, 2);
-    console.error(`Error in configuration:\n${serializedConfig}\nError:\n${issues}`);
-    process.exit(-1);
-}
-export default result.data;

package/lib/src/conversion_textes.d.ts

@@ -1,11 +0,0 @@
-export interface SenatMetadata {
-    number: string | null;
-    session: string | null;
-    date: string | null;
-    type: string | null;
-    authors: string | null;
-    title: string | null;
-    commission: string | null;
-}
-export declare function extractMetadata(xmlDoc: Document): SenatMetadata;
-export declare function convertSenatXmlToHtml(texteXml: string, outputFilePath: string): Promise<void>;