@tricoteuses/senat 3.2.1 → 3.2.3

package/lib/src/raw_types_schemats/sens.js

@@ -0,0 +1,2 @@
+/* tslint:disable */
+export {};

package/lib/src/rich_types/ameli.d.ts

@@ -11,6 +11,14 @@ export interface AmendementAuteurRow {
     qualite: string | null;
     rang: string | null;
 }
+export interface LigneCreditPlf {
+    autorisation_engagement: string | null;
+    credit_paiement: string | null;
+    libelle: string;
+}
+export interface ProgrammePlf extends LigneCreditPlf {
+    lignes_credits?: LigneCreditPlf[];
+}
 export interface AmendementResult {
     accepte_gouvernement: string | null;
     alinea: number | null;
@@ -32,6 +40,9 @@ export interface AmendementResult {
     intitule_texte: string | null;
     lecture: string | null;
     motion_libelle: string | null;
+    mission_plf_id: number | null;
+    mission_plf_libelle: string | null;
+    mission_plf_libelle_court: string | null;
     nature: string;
     nature_texte: string | null;
     nature_texte_libelle: string | null;
@@ -44,6 +55,7 @@ export interface AmendementResult {
     observations_additionnelles: string | null;
     ordre: number | null;
     parent_id: number | null;
+    programmes_plf: ProgrammePlf[];
     revision: string | null;
     session: string;
     session_libelle: string;

package/lib/src/scripts/data-download.js

@@ -1,21 +1,21 @@
 import { execSync } from "child_process";
-const args = process.argv.slice(2).join(" ");
-function runScript(command) {
+import { runDataDownload } from "./shared/data_download.js";
+function exec(command) {
     try {
         execSync(command, { stdio: "inherit" });
+        return { status: 0 };
     }
     catch (error) {
         const execError = error;
-        if (execError.status !== 10) {
-            console.error(`Error during: ${command}`, error);
-            process.exit(execError.status || 1);
-        }
+        return { status: execError.status ?? null, error };
     }
 }
-runScript(`tsx src/scripts/retrieve_open_data.ts --all ${args}`);
-runScript(`tsx src/scripts/convert_data.ts ${args}`);
-runScript(`cross-env TZ='Etc/UTC' tsx src/scripts/retrieve_agenda.ts ${args} --parseAgenda --silent`);
-runScript(`tsx src/scripts/retrieve_cr_seance.ts ${args} --parseDebats --silent`);
-runScript(`tsx src/scripts/retrieve_cr_commission.ts ${args} --parseDebats --silent`);
-runScript(`tsx src/scripts/retrieve_videos.ts ${args} --silent`);
-runScript(`tsx src/scripts/retrieve_collaborateurs.ts ${args} --silent`);
+const exitCode = runDataDownload({
+    exec,
+    now: () => new Date(),
+    rawArgs: process.argv.slice(2),
+    log: console.log,
+    warn: console.warn,
+    error: (message, error) => error === undefined ? console.error(message) : console.error(message, error),
+});
+process.exit(exitCode);

package/lib/src/scripts/debug_dosleg_query.d.ts

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+/**
+ * Debug script to check the DosLeg query compilation
+ * Helps identify the "column undefined" error
+ */
+export {};

package/lib/src/scripts/debug_dosleg_query.js

@@ -0,0 +1,50 @@
+#!/usr/bin/env node
+/**
+ * Debug script to check the DosLeg query compilation
+ * Helps identify the "column undefined" error
+ */
+import { db } from "../server/databases_postgres.js";
+import { jsonArrayFrom } from "kysely/helpers/postgres";
+import { sql } from "kysely";
+async function debugDoslegQuery() {
+    console.log("=== Debugging DosLeg Query ===\n");
+    try {
+        // Build the same query as findAllDossiers but capture SQL
+        const query = db
+            .withSchema("senat")
+            .selectFrom("dosleg_loi as loi")
+            .leftJoin("dosleg_typloi as typloi", "typloi.typloicod", "loi.typloicod")
+            .leftJoin("dosleg_etaloi as etaloi", "etaloi.etaloicod", "loi.etaloicod")
+            .leftJoin("dosleg_deccoc as deccoc", "deccoc.deccoccod", "loi.deccoccod")
+            .select((eb) => [
+            sql `rtrim(${eb.ref("loi.loicod")})`.as("code"),
+            "loi.numero as numero",
+            "loi.signet as signet",
+        ]);
+        const compiled = query.compile();
+        console.log("✓ Query compiled successfully (basic columns)\n");
+        console.log("SQL:", compiled.sql);
+        console.log("\n");
+        // Try with simple lectures first
+        const queryWithLectures = db
+            .withSchema("senat")
+            .selectFrom("dosleg_loi as loi")
+            .select((eb) => [
+            "loi.signet as signet",
+            jsonArrayFrom(eb.withSchema("senat").selectFrom("dosleg_lecture as lecture").select(["lecture.lecidt as id"])).as("lectures"),
+        ]);
+        const compiledWithLectures = queryWithLectures.compile();
+        console.log("✓ Query with simple lectures compiled successfully\n");
+        console.log("SQL:", compiledWithLectures.sql);
+        console.log("\n");
+        console.log("✅ Basic query structures seem valid");
+        console.log("\nThe issue is likely in one of the nested subqueries with cross-references.");
+        console.log("Try running findAllDossiers with the verbose flag to see exactly where it fails.");
+    }
+    catch (error) {
+        console.error("❌ Error during query compilation:");
+        console.error(error instanceof Error ? error.message : String(error));
+        process.exit(1);
+    }
+}
+debugDoslegQuery().catch(console.error);

package/lib/src/scripts/retrieve_open_data.js

@@ -273,7 +273,8 @@ function ensureForeignStagingServer(dataDir, options, runtime) {
             "OPTIONS (",
             `host '${escapeSqlLiteral(runtime.staging.host)}', `,
             `dbname '${escapeSqlLiteral(runtime.staging.name)}', `,
-            `port '${escapeSqlLiteral(String(runtime.staging.port))}'`,
+            `port '${escapeSqlLiteral(String(runtime.staging.port))}', `,
+            "fetch_size '10000'",
             ")",
         ].join(""),
     ].join(" "), dataDir, options, runtime.target);

package/lib/src/scripts/shared/data_download.d.ts

@@ -0,0 +1,43 @@
+import { type Dataset } from "../../server/datasets.js";
+export declare const STATUS_FILENAME = "dataset_status.json";
+export type DatasetStatusEntry = {
+    status: "ok" | "failed";
+    lastRunAt: string;
+    lastSuccessAt?: string;
+    lastFailureAt?: string;
+    lastError?: string;
+    consecutiveFailures: number;
+    firstFailureAt?: string;
+};
+export type DatasetStatusFile = Record<string, DatasetStatusEntry>;
+export type ExecResult = {
+    status: number | null;
+    error?: unknown;
+};
+export type ExecFn = (command: string) => ExecResult;
+export type DataDownloadLogger = {
+    log?: (message: string) => void;
+    warn?: (message: string) => void;
+    error?: (message: string, error?: unknown) => void;
+};
+export type DataDownloadDeps = DataDownloadLogger & {
+    rawArgs: string[];
+    exec: ExecFn;
+    now: () => Date;
+};
+export declare const CATEGORY_DATASETS: Array<{
+    category: string;
+    dataset: Dataset;
+}>;
+export type ParsedArgs = {
+    categories: string[];
+    dataDir?: string;
+    rest: string[];
+};
+export declare function parseArgs(rawArgs: string[]): ParsedArgs;
+export declare function forwardArgs(parsedArgs: ParsedArgs, categories: string[]): string;
+export declare function readStatusFile(statusFilePath: string): DatasetStatusFile;
+export declare function writeStatusFile(statusFilePath: string, statuses: DatasetStatusFile): void;
+export declare function recordSuccess(statuses: DatasetStatusFile, datasetName: string, now: Date): void;
+export declare function recordFailure(statuses: DatasetStatusFile, datasetName: string, now: Date, errorMessage: string): void;
+export declare function runDataDownload(deps: DataDownloadDeps): number;

package/lib/src/scripts/shared/data_download.js

@@ -0,0 +1,175 @@
+import fs from "fs-extra";
+import path from "path";
+import { datasets } from "../../server/datasets.js";
+export const STATUS_FILENAME = "dataset_status.json";
+const MAX_ERROR_LENGTH = 2000;
+export const CATEGORY_DATASETS = [
+    { category: "Ameli", dataset: datasets.ameli },
+    { category: "Debats", dataset: datasets.debats },
+    { category: "DosLeg", dataset: datasets.dosleg },
+    { category: "Questions", dataset: datasets.questions },
+    { category: "Sens", dataset: datasets.sens },
+];
+// Options of retrieve_open_data whose next token is a value. They must not be
+// mistaken for the dataDir positional argument.
+const valueOptions = new Set([
+    "categories",
+    "k",
+    "sudo",
+    "clone",
+    "C",
+    "remote",
+    "r",
+    "fromSession",
+    "only-recent",
+]);
+export function parseArgs(rawArgs) {
+    const categories = [];
+    let dataDir;
+    const rest = [];
+    for (let index = 0; index < rawArgs.length; index++) {
+        const token = rawArgs[index];
+        const name = token.replace(/^-+/, "").split("=")[0];
+        if (name === "categories" || name === "k") {
+            const rawValues = token.includes("=") ? [token.slice(token.indexOf("=") + 1)] : [rawArgs[++index]];
+            for (const rawValue of rawValues) {
+                if (rawValue === undefined) {
+                    continue;
+                }
+                for (const value of rawValue.split(",")) {
+                    if (value.length > 0) {
+                        categories.push(value);
+                    }
+                }
+            }
+            continue;
+        }
+        if (token.startsWith("-") && valueOptions.has(name) && !token.includes("=")) {
+            rest.push(token);
+            const value = rawArgs[++index];
+            if (value !== undefined) {
+                rest.push(value);
+            }
+            continue;
+        }
+        if (!token.startsWith("-") && dataDir === undefined) {
+            dataDir = token;
+            continue;
+        }
+        rest.push(token);
+    }
+    return { categories: categories.length > 0 ? categories : ["All"], dataDir, rest };
+}
+export function forwardArgs(parsedArgs, categories) {
+    const parts = [...parsedArgs.rest];
+    if (parsedArgs.dataDir !== undefined) {
+        parts.push(parsedArgs.dataDir);
+    }
+    for (const category of categories) {
+        parts.push("--categories", category);
+    }
+    return parts.join(" ");
+}
+export function readStatusFile(statusFilePath) {
+    try {
+        const content = fs.readJsonSync(statusFilePath);
+        return content && typeof content === "object" ? content : {};
+    }
+    catch {
+        return {};
+    }
+}
+export function writeStatusFile(statusFilePath, statuses) {
+    fs.writeJsonSync(statusFilePath, statuses, { spaces: 2 });
+}
+export function recordSuccess(statuses, datasetName, now) {
+    statuses[datasetName] = {
+        consecutiveFailures: 0,
+        lastRunAt: now.toISOString(),
+        lastSuccessAt: now.toISOString(),
+        status: "ok",
+    };
+}
+export function recordFailure(statuses, datasetName, now, errorMessage) {
+    const previous = statuses[datasetName];
+    statuses[datasetName] = {
+        consecutiveFailures: (previous?.consecutiveFailures ?? 0) + 1,
+        firstFailureAt: previous?.firstFailureAt ?? now.toISOString(),
+        lastError: errorMessage.slice(0, MAX_ERROR_LENGTH),
+        lastFailureAt: now.toISOString(),
+        lastRunAt: now.toISOString(),
+        lastSuccessAt: previous?.lastSuccessAt,
+        status: "failed",
+    };
+}
+function execNormalized(deps, command) {
+    try {
+        return deps.exec(command);
+    }
+    catch (error) {
+        const execError = error;
+        return { status: execError.status ?? null, error };
+    }
+}
+function isSuccessfulStatus(result) {
+    return result.status === 0 || result.status === 10;
+}
+function runDownstreamScript(deps, command) {
+    const result = execNormalized(deps, command);
+    if (isSuccessfulStatus(result)) {
+        return null;
+    }
+    deps.error?.(`Error during: ${command}`, result.error);
+    return result.status || 1;
+}
+export function runDataDownload(deps) {
+    const parsedArgs = parseArgs(deps.rawArgs);
+    const chosen = parsedArgs.categories.includes("All")
+        ? CATEGORY_DATASETS
+        : CATEGORY_DATASETS.filter(({ category }) => parsedArgs.categories.includes(category));
+    if (chosen.length === 0) {
+        deps.error?.(`No known dataset among categories: ${parsedArgs.categories.join(", ")}`);
+        return 1;
+    }
+    const statusFilePath = parsedArgs.dataDir !== undefined ? path.join(parsedArgs.dataDir, STATUS_FILENAME) : null;
+    const statuses = statusFilePath !== null ? readStatusFile(statusFilePath) : {};
+    const okCategories = [];
+    const failedCategories = [];
+    for (const { category, dataset } of chosen) {
+        // --all: fetch, unzip, repair encoding and import (same as the original script).
+        const command = `tsx src/scripts/retrieve_open_data.ts --all ${forwardArgs(parsedArgs, [category])}`.trim();
+        deps.log?.(`\n=== Retrieving dataset ${dataset.title} (${category}) ===`);
+        const result = execNormalized(deps, command);
+        if (!isSuccessfulStatus(result)) {
+            const errorMessage = result.error instanceof Error ? result.error.message : String(result.error);
+            recordFailure(statuses, dataset.database, deps.now(), errorMessage);
+            failedCategories.push(category);
+            deps.error?.(`Error during: ${command}`, result.error);
+        }
+        else {
+            recordSuccess(statuses, dataset.database, deps.now());
+            okCategories.push(category);
+        }
+        if (statusFilePath !== null) {
+            writeStatusFile(statusFilePath, statuses);
+        }
+    }
+    if (failedCategories.length > 0) {
+        deps.warn?.(`\n⚠️  Warning: Some datasets failed to retrieve: ${failedCategories.join(", ")}. ` +
+            "Continuing with available datasets...\n");
+    }
+    if (okCategories.length === 0) {
+        deps.error?.("All datasets failed to retrieve; skipping data conversion.");
+        return 1;
+    }
+    const exitCode = runDownstreamScript(deps, `tsx src/scripts/convert_data.ts ${forwardArgs(parsedArgs, okCategories)}`) ??
+        runDownstreamScript(deps, `cross-env TZ='Etc/UTC' tsx src/scripts/retrieve_agenda.ts ${deps.rawArgs.join(" ")} --parseAgenda --silent`) ??
+        runDownstreamScript(deps, "cross-env TZ='Etc/UTC' tsx src/scripts/retrieve_cr_seance.ts " +
+            `${deps.rawArgs.join(" ")} --parseDebats --silent`) ??
+        runDownstreamScript(deps, "cross-env TZ='Etc/UTC' tsx src/scripts/retrieve_cr_commission.ts " +
+            `${deps.rawArgs.join(" ")} --parseDebats --silent`) ??
+        runDownstreamScript(deps, `tsx src/scripts/retrieve_videos.ts ${deps.rawArgs.join(" ")} --silent`) ??
+        runDownstreamScript(deps, `tsx src/scripts/retrieve_collaborateurs.ts ${deps.rawArgs.join(" ")} --silent`) ??
+        0;
+    return exitCode;
+}