@tricoteuses/senat 2.23.0 → 3.0.1

package/lib/src/scripts/convert_data.js

@@ -2,16 +2,16 @@ import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 import path from "path";
 import pLimit from "p-limit";
-import * as git from "../git";
-import { datasets, EnabledDatasets, getEnabledDatasets } from "../datasets";
-import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, SCRUTINS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER, ENRICHED_TEXTE_FOLDER, } from "../loaders";
-import { findAllAmendements, findAllCirconscriptions, findAllDebats, findAllDossiers, findAllScrutins, findAllOrganismes, findAllQuestions, findAllSens, findAllTextes, findAllRapports, } from "../model";
-import { processRapport, processTexte } from "./retrieve_documents";
-import { buildActesLegislatifs } from "../model/dosleg";
-import { UNDEFINED_SESSION } from "../types/sessions";
-import { getSessionFromDate, getSessionFromSignet } from "./datautil";
-import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers";
-import { ensureAndClearDir } from "./shared/util";
+import * as git from "../git.js";
+import { datasets, EnabledDatasets, getEnabledDatasets } from "../datasets.js";
+import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, SCRUTINS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER, ENRICHED_TEXTE_FOLDER, } from "../loaders.js";
+import { findAllAmendements, findAllCirconscriptions, findAllDebats, findAllDossiers, findAllScrutins, findAllOrganismes, findAllQuestions, findAllSens, findAllTextes, findAllRapports, } from "../model/index.js";
+import { processRapport, processTexte } from "./retrieve_documents.js";
+import { buildActesLegislatifs } from "../model/dosleg.js";
+import { UNDEFINED_SESSION } from "../types/sessions.js";
+import { getSessionFromDate, getSessionFromSignet } from "./datautil.js";
+import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers.js";
+import { ensureAndClearDir } from "./shared/util.js";
 let exitCode = 10; // 0: some data changed, 10: no modification
 const optionsDefinitions = [...commonOptions];
 const options = commandLineArgs(optionsDefinitions);

package/lib/src/scripts/convert_xml_to_html.js

@@ -1,7 +1,7 @@
 import fs from "fs-extra";
 import path from "path";
 import commandLineArgs from "command-line-args";
-import { convertSenatXmlToHtml } from "../conversion_textes";
+import { convertSenatXmlToHtml } from "../conversion_textes.js";
 const optionDefinitions = [
     { name: "input", alias: "i", type: String, defaultOption: true },
     { name: "output", alias: "o", type: String },

package/lib/src/scripts/datautil.d.ts

@@ -1,4 +1,4 @@
-import { Session } from "../types/sessions";
+import { Session } from "../types/sessions.js";
 export declare const STANDARD_DATE_FORMAT = "yyyy-MM-dd";
 export declare const ID_DATE_FORMAT = "yyyyMMdd";
 export declare const AKN_IDENTIFICATION_STRUCTURE_REGEXP: RegExp;

package/lib/src/scripts/retrieve_agenda.js

@@ -2,15 +2,15 @@ import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 import { DateTime } from "luxon";
 import path from "path";
-import * as git from "../git";
-import { AGENDA_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
-import { parseAgendaFromFile } from "../model/agenda";
-import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions";
-import { ID_DATE_FORMAT } from "./datautil";
-import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers";
-import { fetchWithRetry } from "./shared/util";
-import { buildReunionsByBucket } from "../utils/reunion_parsing";
-import { buildSenatDossierIndex } from "../utils/reunion_odj_building";
+import * as git from "../git.js";
+import { AGENDA_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders.js";
+import { parseAgendaFromFile } from "../model/agenda.js";
+import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions.js";
+import { ID_DATE_FORMAT } from "./datautil.js";
+import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers.js";
+import { fetchWithRetry } from "./shared/util.js";
+import { buildReunionsByBucket } from "../utils/reunion_parsing.js";
+import { buildSenatDossierIndex } from "../utils/reunion_odj_building.js";
 const optionsDefinitions = [
     ...commonOptions,
     {

package/lib/src/scripts/retrieve_cr_commission.js

@@ -1,15 +1,15 @@
 import fs, { ensureDir } from "fs-extra";
 import path from "path";
 import * as cheerio from "cheerio";
-import { COMMISSION_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
-import { loadAgendaForDate, parseCommissionMetadataFromHtml, linkCRtoCommissionGroup } from "../utils/cr_spliting";
-import { cleanTitle, extractDayH3Sections, parseCommissionCRSectionFromDom } from "../model/commission";
+import { COMMISSION_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders.js";
+import { loadAgendaForDate, parseCommissionMetadataFromHtml, linkCRtoCommissionGroup } from "../utils/cr_spliting.js";
+import { cleanTitle, extractDayH3Sections, parseCommissionCRSectionFromDom } from "../model/commission.js";
 import commandLineArgs from "command-line-args";
-import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers";
-import { sessionStartYearFromDate } from "../model/seance";
-import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions";
-import { ensureAndClearDir, fetchWithRetry } from "./shared/util";
-import { jaccard, jaccardTokenSim } from "../utils/scoring";
+import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers.js";
+import { sessionStartYearFromDate } from "../model/seance.js";
+import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions.js";
+import { ensureAndClearDir, fetchWithRetry } from "./shared/util.js";
+import { jaccard, jaccardTokenSim } from "../utils/scoring.js";
 import * as git from "../git.js";
 class CommissionCRDownloadError extends Error {
     constructor(message, url) {

package/lib/src/scripts/retrieve_cr_seance.d.ts

@@ -3,7 +3,7 @@
  * - downloads the ZIP of comptes-rendus des débats (CRI) from data.senat.fr
  * - extracts XML files, distributes them by session/year
  */
-import { Session } from "../types/sessions";
+import { Session } from "../types/sessions.js";
 import { CommandLineOptions } from "command-line-args";
 type RetrieveCriOptions = CommandLineOptions & {
     commit?: boolean;

package/lib/src/scripts/retrieve_cr_seance.js

@@ -8,14 +8,14 @@ import fs, { ensureDirSync } from "fs-extra";
 import path from "path";
 import StreamZip from "node-stream-zip";
 import * as cheerio from "cheerio";
-import { AGENDA_FOLDER, COMPTES_RENDUS_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
-import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers";
-import { parseCompteRenduIntervalFromFile, sessionStartYearFromDate } from "../model/seance";
-import { extractSommaireBlocks, makeReunionUid } from "../utils/reunion_parsing";
-import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions";
-import { ensureAndClearDir, fetchWithRetry } from "./shared/util";
-import { isNoiseBlock, scoreSommaireBlockForEvent } from "../utils/scoring";
-import { parseYYYYMMDD } from "../utils/date";
+import { AGENDA_FOLDER, COMPTES_RENDUS_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders.js";
+import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers.js";
+import { parseCompteRenduIntervalFromFile, sessionStartYearFromDate } from "../model/seance.js";
+import { extractSommaireBlocks, makeReunionUid } from "../utils/reunion_parsing.js";
+import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions.js";
+import { ensureAndClearDir, fetchWithRetry } from "./shared/util.js";
+import { isNoiseBlock, scoreSommaireBlockForEvent } from "../utils/scoring.js";
+import { parseYYYYMMDD } from "../utils/date.js";
 import * as git from "../git.js";
 const optionsDefinitions = [
     ...commonOptions,

package/lib/src/scripts/retrieve_documents.d.ts

@@ -1,4 +1,4 @@
-import { DocumentMetadata } from "../types/texte";
+import { DocumentMetadata } from "../types/texte.js";
 type DownloadOptions = {
     force?: boolean;
     formats?: string[];

package/lib/src/scripts/retrieve_documents.js

@@ -2,13 +2,13 @@ import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 import { DateTime } from "luxon";
 import path from "path";
-import { convertSenatXmlToHtml } from "../conversion_textes";
-import * as git from "../git";
-import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, ENRICHED_TEXTE_FOLDER, iterLoadSenatRapportUrls, iterLoadSenatTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, } from "../loaders";
-import { parseExposeDesMotifs, parseTexte, parseTexteFromFile } from "../parsers/texte";
-import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions";
-import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers";
-import { ensureAndClearDir, fetchWithRetry, isOptionEmptyOrHasValue } from "./shared/util";
+import { convertSenatXmlToHtml } from "../conversion_textes.js";
+import * as git from "../git.js";
+import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, ENRICHED_TEXTE_FOLDER, iterLoadSenatRapportUrls, iterLoadSenatTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, } from "../loaders.js";
+import { parseExposeDesMotifs, parseTexte, parseTexteFromFile } from "../parsers/texte.js";
+import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions.js";
+import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers.js";
+import { ensureAndClearDir, fetchWithRetry, isOptionEmptyOrHasValue } from "./shared/util.js";
 let exitCode = 10; // 0: some data changed, 10: no modification
 const optionsDefinitions = [
     ...commonOptions,

package/lib/src/scripts/retrieve_open_data.js

@@ -1,21 +1,23 @@
 import assert from "assert";
-import { execSync } from "child_process";
+import { execFileSync } from "child_process";
 import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
+import { formatWithPrettier, makePgTsGenerator, markAsGenerated, processDatabase } from "kanel";
+import { makeGenerateZodSchemas } from "kanel-zod";
 import path from "path";
 import StreamZip from "node-stream-zip";
 import readline from "readline";
 import { pipeline, Readable } from "stream";
 import { promisify } from "util";
 import * as windows1252 from "windows-1252";
-import config from "../config";
-import { getChosenDatasets, getEnabledDatasets } from "../datasets";
-import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers";
-import { buildIncrementalDatasetImportSql, buildNormalizeStagingSchemaSql } from "./shared/incremental_import_sql";
-import { normalizeGeneratedDefinition, prefixedName, senatSchemaName, stagingSchemaName, } from "./shared/prefixed_tables";
-import { buildEnsureSchemaVersionTableSql, buildIncrementSchemaVersionSql, buildSchemaStructureFingerprintQuery, } from "./shared/schema_version";
-import { buildExportStagingMetadataStatementsQuery } from "./shared/staging_metadata_sql";
-import { isCopyFromStdinLine, rewriteLineForStagingImport } from "./shared/staging_import";
+import config from "../config.js";
+import { getChosenDatasets, getEnabledDatasets } from "../datasets.js";
+import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers.js";
+import { buildIncrementalDatasetImportSql, buildNormalizeStagingSchemaSql } from "./shared/incremental_import_sql.js";
+import { buildGeneratedTableManifest, getGeneratedDefinitionPath, getGeneratedTableManifestPath, prefixedName, rawTypesDir, senatSchemaName, stagingSchemaName, stripDatasetPrefix, } from "./shared/prefixed_tables.js";
+import { buildEnsureSchemaVersionTableSql, buildIncrementSchemaVersionSql, buildSchemaStructureFingerprintQuery, } from "./shared/schema_version.js";
+import { buildExportStagingMetadataStatementsQuery } from "./shared/staging_metadata_sql.js";
+import { isCopyFromStdinLine, rewriteLineForStagingImport } from "./shared/staging_import.js";
 const badWindows1252CharacterRegex = /[\u0080-\u009f]/g;
 const optionsDefinitions = [
     ...commonOptions,
@@ -27,7 +29,7 @@ const optionsDefinitions = [
     },
     {
         alias: "c",
-        help: "create TypeScript interfaces from database schemas into src/raw_types_* directories",
+        help: "create TypeScript interfaces and Zod schemas from database tables into src/raw_types",
         name: "schema",
         type: Boolean,
     },
@@ -73,9 +75,6 @@ const stagingServerName = "staging_server";
 function isIncrementalImport(options) {
     return options["incremental"] === true;
 }
-function shellQuote(value) {
-    return `'${value.replace(/'/g, `'"'"'`)}'`;
-}
 function connectionEnv(connection) {
     return {
         ...process.env,
@@ -124,7 +123,8 @@ async function runWithRetry(operation, options, retryOptions) {
                 throw error;
             }
             if (!options["silent"]) {
-                console.warn(`${retryOptions.label} hit a transient PostgreSQL lock error on attempt ${attempt}/${retryOptions.attempts}; retrying in ${delayMs}ms...`);
+                console.warn(`${retryOptions.label} hit a transient PostgreSQL lock error ` +
+                    `on attempt ${attempt}/${retryOptions.attempts}; retrying in ${delayMs}ms...`);
             }
             await sleep(delayMs);
             attempt += 1;
@@ -132,18 +132,30 @@ async function runWithRetry(operation, options, retryOptions) {
         }
     }
 }
-function buildPsqlCommand(baseArgs, connection, options) {
-    const sudoPrefix = options["sudo"] ? `sudo -u ${options["sudo"]} ` : "";
-    return (`${sudoPrefix}psql --quiet ` +
-        `-h ${shellQuote(connection.host)} ` +
-        `-p ${shellQuote(String(connection.port))} ` +
-        `-U ${shellQuote(connection.user)} ` +
-        `-d ${shellQuote(connection.name)} ` +
-        baseArgs);
+function buildPsqlInvocation(baseArgs, connection, options) {
+    const psqlArgs = [
+        "--quiet",
+        "-h",
+        connection.host,
+        "-p",
+        String(connection.port),
+        "-U",
+        connection.user,
+        "-d",
+        connection.name,
+        ...baseArgs,
+    ];
+    if (!options["sudo"]) {
+        return { command: "psql", args: psqlArgs };
+    }
+    return {
+        command: "sudo",
+        args: ["-u", options["sudo"], "psql", ...psqlArgs],
+    };
 }
 function runPsqlFile(sqlFilePath, dataDir, options, connection, stopOnError = true) {
-    const onErrorFlag = stopOnError ? "-v ON_ERROR_STOP=1 " : "";
-    execSync(buildPsqlCommand(`${onErrorFlag}-f ${shellQuote(sqlFilePath)}`, connection, options), {
+    const { command, args } = buildPsqlInvocation([...(stopOnError ? ["-v", "ON_ERROR_STOP=1"] : []), "-f", sqlFilePath], connection, options);
+    execFileSync(command, args, {
         cwd: dataDir,
         env: connectionEnv(connection),
         encoding: "utf-8",
@@ -151,8 +163,9 @@ function runPsqlFile(sqlFilePath, dataDir, options, connection, stopOnError = tr
     });
 }
 function runPsqlCommand(command, dataDir, options, connection, stopOnError = true) {
-    const onErrorFlag = stopOnError ? "-v ON_ERROR_STOP=1 " : "";
-    execSync(buildPsqlCommand(`${onErrorFlag}-c ${shellQuote(command)}`, connection, options), {
+    const psqlCommand = command;
+    const { command: binary, args } = buildPsqlInvocation([...(stopOnError ? ["-v", "ON_ERROR_STOP=1"] : []), "-c", psqlCommand], connection, options);
+    execFileSync(binary, args, {
         cwd: dataDir,
         env: connectionEnv(connection),
         encoding: "utf-8",
@@ -160,8 +173,9 @@ function runPsqlCommand(command, dataDir, options, connection, stopOnError = tru
     });
 }
 function runPsqlQuery(command, dataDir, options, connection, stopOnError = true) {
-    const onErrorFlag = stopOnError ? "-v ON_ERROR_STOP=1 " : "";
-    return execSync(buildPsqlCommand(`${onErrorFlag}-At -c ${shellQuote(command)}`, connection, options), {
+    const psqlCommand = command;
+    const { command: binary, args } = buildPsqlInvocation([...(stopOnError ? ["-v", "ON_ERROR_STOP=1"] : []), "-At", "-c", psqlCommand], connection, options);
+    return execFileSync(binary, args, {
         cwd: dataDir,
         env: connectionEnv(connection),
         encoding: "utf-8",
@@ -171,7 +185,8 @@ function runPsqlQuery(command, dataDir, options, connection, stopOnError = true)
 function ensureStagingSchemaHasTables(dataset, dataDir, options, connection) {
     const stagingSchema = stagingSchemaName(dataset.database);
     const tableCount = Number.parseInt(runPsqlQuery(`SELECT count(*) FROM pg_tables WHERE schemaname = '${escapeSqlLiteral(stagingSchema)}'`, dataDir, options, connection).trim(), 10);
-    assert(tableCount > 0, `Staging schema ${stagingSchema} is empty after importing ${dataset.database}. Aborting incremental merge to protect ${senatSchemaName}.`);
+    assert(tableCount > 0, `Staging schema ${stagingSchema} is empty after importing ${dataset.database}. ` +
+        `Aborting incremental merge to protect ${senatSchemaName}.`);
 }
 function ensureStagingDatabase(dataDir, options, runtime) {
     const maintenanceDb = process.env["PGDATABASE"] || "postgres";
@@ -207,16 +222,27 @@ function ensureDatabaseExists(connection, dataDir, options) {
     runPsqlCommand(`CREATE DATABASE ${connection.name} WITH OWNER ${connection.user}`, dataDir, options, maintenanceConnection);
 }
 function ensureForeignStagingServer(dataDir, options, runtime) {
-    runPsqlCommand(`CREATE EXTENSION IF NOT EXISTS postgres_fdw`, dataDir, options, runtime.target);
+    runPsqlCommand("CREATE EXTENSION IF NOT EXISTS postgres_fdw", dataDir, options, runtime.target);
     runPsqlCommand(`DROP SERVER IF EXISTS ${stagingServerName} CASCADE`, dataDir, options, runtime.target, false);
     runPsqlCommand([
         `CREATE SERVER ${stagingServerName}`,
-        `FOREIGN DATA WRAPPER postgres_fdw`,
-        `OPTIONS (host '${escapeSqlLiteral(runtime.staging.host)}', dbname '${escapeSqlLiteral(runtime.staging.name)}', port '${escapeSqlLiteral(String(runtime.staging.port))}')`,
+        "FOREIGN DATA WRAPPER postgres_fdw",
+        [
+            "OPTIONS (",
+            `host '${escapeSqlLiteral(runtime.staging.host)}', `,
+            `dbname '${escapeSqlLiteral(runtime.staging.name)}', `,
+            `port '${escapeSqlLiteral(String(runtime.staging.port))}'`,
+            ")",
+        ].join(""),
     ].join(" "), dataDir, options, runtime.target);
     runPsqlCommand([
         `CREATE USER MAPPING FOR CURRENT_USER SERVER ${stagingServerName}`,
-        `OPTIONS (user '${escapeSqlLiteral(runtime.staging.user)}', password '${escapeSqlLiteral(runtime.staging.password)}')`,
+        [
+            "OPTIONS (",
+            `user '${escapeSqlLiteral(runtime.staging.user)}', `,
+            `password '${escapeSqlLiteral(runtime.staging.password)}'`,
+            ")",
+        ].join(""),
     ].join(" "), dataDir, options, runtime.target);
 }
 function cleanupForeignStagingServer(dataDir, options, runtime) {
@@ -224,7 +250,7 @@ function cleanupForeignStagingServer(dataDir, options, runtime) {
 }
 function mountForeignStagingSchema(dataset, dataDir, options, runtime) {
     const stagingSchema = stagingSchemaName(dataset.database);
-    const importForeignSchemaCommand = `IMPORT FOREIGN SCHEMA ${stagingSchema} FROM SERVER ${stagingServerName} INTO ${stagingSchema}`;
+    const importForeignSchemaCommand = `IMPORT FOREIGN SCHEMA ${stagingSchema} ` + `FROM SERVER ${stagingServerName} INTO ${stagingSchema}`;
     runPsqlCommand(`DROP SCHEMA IF EXISTS ${stagingSchema} CASCADE`, dataDir, options, runtime.target, false);
     runPsqlCommand(`CREATE SCHEMA ${stagingSchema}`, dataDir, options, runtime.target);
     try {
@@ -342,6 +368,97 @@ function listPrefixedTables(dataset, dataDir, options, runtime) {
         .map((tableName) => tableName.trim())
         .filter((tableName) => tableName.length > 0);
 }
+function toPascalCase(text) {
+    return text.replace(/(^|_)([a-z0-9])/gi, (_match, _separator, letter) => letter.toUpperCase());
+}
+function toIdentifierName(tableName, columnName) {
+    return `${toPascalCase(tableName)}${toPascalCase(columnName)}`;
+}
+function trimComment(comment) {
+    const trimmed = comment?.trim();
+    return trimmed ? trimmed : undefined;
+}
+async function generateRawTypes(dataset, runtime, prefixedTables) {
+    await fs.ensureDir(rawTypesDir);
+    const definitionFilePath = getGeneratedDefinitionPath(dataset.database);
+    const manifestFilePath = getGeneratedTableManifestPath(dataset.database);
+    const datasetPrefix = `${dataset.database}_`;
+    const datasetOutputPath = path.join(rawTypesDir, dataset.database);
+    const generateZodSchemas = makeGenerateZodSchemas({
+        castToSchema: true,
+        getZodIdentifierMetadata: (column, details) => ({
+            name: `${toIdentifierName(stripDatasetPrefix(details.name, dataset.database), column.name)}Schema`,
+        }),
+        getZodSchemaMetadata: (details, generateFor) => {
+            const baseName = toPascalCase(stripDatasetPrefix(details.name, dataset.database));
+            const suffix = generateFor === "selector" || generateFor === undefined ? "" : toPascalCase(generateFor);
+            return {
+                name: `${baseName}${suffix}Schema`,
+                path: datasetOutputPath,
+            };
+        },
+    });
+    await fs.remove(definitionFilePath);
+    await fs.remove(manifestFilePath);
+    await processDatabase({
+        connection: {
+            database: runtime.target.name,
+            host: runtime.target.host,
+            password: runtime.target.password,
+            port: runtime.target.port,
+            user: runtime.target.user,
+        },
+        filter: (pgType) => pgType.schemaName === senatSchemaName && pgType.name.startsWith(datasetPrefix),
+        generators: [
+            makePgTsGenerator({
+                filter: (pgType) => pgType.schemaName === senatSchemaName && pgType.name.startsWith(datasetPrefix),
+                generateIdentifierType: (column, details, builtinType) => {
+                    const tableName = stripDatasetPrefix(details.name, dataset.database);
+                    return {
+                        ...builtinType,
+                        comment: undefined,
+                        name: toIdentifierName(tableName, column.name),
+                        typeDefinition: builtinType.typeDefinition.map((typeDefinition) => typeDefinition.replace(/ & \{ __flavor\?: '[^']+' \}/g, "").replace(/ & \{ __brand: '[^']+' \}/g, "")),
+                    };
+                },
+                getMetadata: (details, generateFor, builtinMetadata) => {
+                    const baseName = toPascalCase(stripDatasetPrefix(details.name, dataset.database));
+                    const suffix = generateFor === "selector" || generateFor === undefined ? "" : toPascalCase(generateFor);
+                    const tableComment = trimComment(details.comment);
+                    return {
+                        ...builtinMetadata,
+                        comment: tableComment ? [tableComment] : undefined,
+                        exportAs: "named",
+                        name: `${baseName}${suffix}`,
+                        path: datasetOutputPath,
+                    };
+                },
+                getPropertyMetadata: (property, _details, generateFor, builtinMetadata) => {
+                    const comment = trimComment(property.comment);
+                    const defaultComment = generateFor === "initializer" && property.defaultValue !== null && property.defaultValue !== undefined
+                        ? `Default value: ${property.defaultValue}`
+                        : undefined;
+                    const comments = [comment, defaultComment].filter((value) => value !== undefined);
+                    return {
+                        ...builtinMetadata,
+                        comment: comments.length > 0 ? comments : undefined,
+                    };
+                },
+                preRenderHooks: [generateZodSchemas],
+            }),
+        ],
+        outputPath: rawTypesDir,
+        postRenderHooks: [markAsGenerated, formatWithPrettier],
+        schemaNames: [senatSchemaName],
+        typescriptConfig: {
+            enumStyle: "literal-union",
+            tsModuleFormat: "esm",
+        },
+    });
+    const generatedDefinition = await fs.readFile(definitionFilePath, { encoding: "utf8" });
+    await fs.writeFile(definitionFilePath, generatedDefinition.replace(/\r\n/g, "\n"));
+    await fs.writeFile(manifestFilePath, buildGeneratedTableManifest(dataset.database, prefixedTables));
+}
 async function downloadFile(url, dest) {
     const response = await fetch(url);
     if (!response.ok) {
@@ -474,7 +591,7 @@ async function retrieveDataset(dataDir, dataset, options, runtime) {
         repairedSqlWriter.end();
         await fs.move(repairedSqlFilePath, sqlFilePath, { overwrite: true });
     }
-    if (options["all"] || options["import"] || options["schema"]) {
+    if (options["all"] || options["import"]) {
         if (!options["silent"]) {
             console.log(`Merging ${dataset.title}: ${sqlFilename} into ${runtime.target.name}.${senatSchemaName}...`);
         }
@@ -485,24 +602,13 @@ async function retrieveDataset(dataDir, dataset, options, runtime) {
         }
     }
     if (options["schema"]) {
-        const definitionsDir = path.resolve("src", "raw_types_schemats");
-        assert(fs.statSync(definitionsDir).isDirectory());
+        await fs.ensureDir(rawTypesDir);
         if (!options["silent"]) {
             console.log(`Creating TypeScript definitions from prefixed ${senatSchemaName} tables ` +
                 `for '${dataset.database}' in database '${runtime.target.name}'...`);
         }
-        const dbConnectionString = `postgres://${runtime.target.user}:${runtime.target.password}` +
-            `@${runtime.target.host}:${runtime.target.port}/${runtime.target.name}`;
-        const definitionFilePath = path.join(definitionsDir, `${dataset.database}.ts`);
-        const tables = listPrefixedTables(dataset, dataDir, options, runtime);
-        const tableOptions = tables.map((tableName) => `-t ${tableName}`).join(" ");
-        execSync(`npx schemats generate -c ${dbConnectionString} -s ${senatSchemaName} ${tableOptions} -o ${definitionFilePath}`, {
-            env: process.env,
-            encoding: "utf-8",
-        });
-        const definition = fs.readFileSync(definitionFilePath, { encoding: "utf8" });
-        const definitionRepaired = normalizeGeneratedDefinition(definition, dataset.database);
-        fs.writeFileSync(definitionFilePath, definitionRepaired);
+        const prefixedTables = listPrefixedTables(dataset, dataDir, options, runtime);
+        await generateRawTypes(dataset, runtime, prefixedTables);
     }
 }
 function buildRuntimeContext() {

package/lib/src/scripts/retrieve_senateurs_photos.js

@@ -6,10 +6,10 @@ import path from "path";
 import { fileURLToPath } from "url";
 // import stream from "stream"
 // import util from "util"
-import * as git from "../git";
-import { findActif as findActifSenateurs } from "../model/sens";
-import { slugify } from "../strings";
-import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers";
+import * as git from "../git.js";
+import { findActif as findActifSenateurs } from "../model/sens.js";
+import { slugify } from "../strings.js";
+import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers.js";
 const optionsDefinitions = [
     ...commonOptions,
     {

package/lib/src/scripts/retrieve_videos.js

@@ -4,16 +4,16 @@ import fs from "fs-extra";
 import fsp from "fs/promises";
 import path from "path";
 import * as git from "../git.js";
-import { AGENDA_FOLDER, iterLoadSenatAgendas } from "../loaders";
-import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions";
-import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers";
-import { getAgendaSegmentTimecodes, buildSenatVodMasterM3u8FromNvs } from "../utils/nvs-parsing";
-import { epochToParisDateTime, isAmbiguousTimeOriginal, toTargetEpoch } from "../utils/date";
+import { AGENDA_FOLDER, iterLoadSenatAgendas } from "../loaders.js";
+import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions.js";
+import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers.js";
+import { getAgendaSegmentTimecodes, buildSenatVodMasterM3u8FromNvs } from "../utils/nvs-parsing.js";
+import { epochToParisDateTime, isAmbiguousTimeOriginal, toTargetEpoch } from "../utils/date.js";
 import { pathToFileURL } from "url";
-import { fetchCandidatesForAgenda, fetchText } from "../videos/search";
-import { matchAgendaToVideo } from "../videos/match";
-import { SENAT_DATAS_ROOT, STATS, VIDEOS_ROOT_FOLDER, weights } from "../videos/config";
-import { processBisIfNeeded, processOneReunionMatch, writeIfChanged } from "../videos";
+import { fetchCandidatesForAgenda, fetchText } from "../videos/search.js";
+import { matchAgendaToVideo } from "../videos/match.js";
+import { SENAT_DATAS_ROOT, STATS, VIDEOS_ROOT_FOLDER, weights } from "../videos/config.js";
+import { processBisIfNeeded, processOneReunionMatch, writeIfChanged } from "../videos/index.js";
 const optionsDefinitions = [...commonOptions];
 const options = commandLineArgs(optionsDefinitions);
 let exitCode = 10; // 0: some data changed, 10: no modification

package/lib/src/scripts/shared/incremental_import_sql.js

@@ -1,4 +1,4 @@
-import { prefixedName, senatSchemaName, stagingSchemaName } from "./prefixed_tables";
+import { prefixedName, senatSchemaName, stagingSchemaName } from "./prefixed_tables.js";
 function escapeSqlLiteral(value) {
     return value.replace(/'/g, "''");
 }

package/lib/src/scripts/shared/prefixed_tables.d.ts

@@ -1,7 +1,10 @@
 export declare const senatSchemaName = "senat";
+export declare const rawTypesDir: string;
 export declare function prefixedName(datasetName: string, relationName: string): string;
 export declare function stagingSchemaName(datasetName: string): string;
 export declare function escapeRegExp(text: string): string;
-export declare function normalizeGeneratedDefinition(definition: string, datasetName: string): string;
-export declare function extractTableNamesFromGeneratedDefinition(definition: string): string[];
-export declare function extractPrefixedTableNamesFromGeneratedDefinition(definition: string, datasetName: string): string[];
+export declare function stripDatasetPrefix(relationName: string, datasetName: string): string;
+export declare function getGeneratedDefinitionPath(datasetName: string): string;
+export declare function getGeneratedTableManifestPath(datasetName: string): string;
+export declare function buildGeneratedTableManifest(datasetName: string, prefixedTables: string[]): string;
+export declare function extractPrefixedTableNamesFromGeneratedManifest(manifest: string, datasetName: string): string[];

package/lib/src/scripts/shared/prefixed_tables.js

@@ -1,4 +1,6 @@
+import path from "path";
 export const senatSchemaName = "senat";
+export const rawTypesDir = path.resolve("src", "raw_types");
 export function prefixedName(datasetName, relationName) {
     return `${datasetName}_${relationName}`;
 }
@@ -8,23 +10,27 @@ export function stagingSchemaName(datasetName) {
 export function escapeRegExp(text) {
     return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
-export function normalizeGeneratedDefinition(definition, datasetName) {
+export function stripDatasetPrefix(relationName, datasetName) {
     const datasetPrefix = `${datasetName}_`;
-    return definition
-        .replace(/\r\n/g, "\n")
-        .replace(/AUTO-GENERATED FILE @ \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/, "AUTO-GENERATED FILE")
-        .replace(new RegExp(`\\b${escapeRegExp(datasetPrefix)}([A-Za-z0-9_]+Fields)\\b`, "g"), "$1")
-        .replace(new RegExp(`\\b${escapeRegExp(datasetPrefix)}([A-Za-z0-9_]+)\\b`, "g"), "$1");
-}
-export function extractTableNamesFromGeneratedDefinition(definition) {
-    const flattenedDefinition = definition.replace(/\n \*/g, " ");
-    const match = flattenedDefinition.match(/\$ schemats generate .*?((?: -t [^ ]+)+) -s [^\s]+/s);
-    if (!match) {
+    return relationName.startsWith(datasetPrefix) ? relationName.slice(datasetPrefix.length) : relationName;
+}
+export function getGeneratedDefinitionPath(datasetName) {
+    return path.join(rawTypesDir, `${datasetName}.ts`);
+}
+export function getGeneratedTableManifestPath(datasetName) {
+    return path.join(rawTypesDir, `${datasetName}.tables.json`);
+}
+export function buildGeneratedTableManifest(datasetName, prefixedTables) {
+    const manifest = {
+        dataset: datasetName,
+        prefixedTables: [...new Set(prefixedTables)],
+    };
+    return JSON.stringify(manifest, null, 2) + "\n";
+}
+export function extractPrefixedTableNamesFromGeneratedManifest(manifest, datasetName) {
+    const parsed = JSON.parse(manifest);
+    if (parsed.dataset !== datasetName || !Array.isArray(parsed.prefixedTables)) {
         return [];
     }
-    const tables = Array.from(match[1].matchAll(/-t ([^ ]+)/g), ([, tableName]) => tableName);
-    return [...new Set(tables)];
-}
-export function extractPrefixedTableNamesFromGeneratedDefinition(definition, datasetName) {
-    return extractTableNamesFromGeneratedDefinition(definition).map((tableName) => prefixedName(datasetName, tableName));
+    return [...new Set(parsed.prefixedTables.filter((tableName) => typeof tableName === "string"))];
 }

package/lib/src/scripts/shared/staging_import.d.ts

@@ -1,3 +1,3 @@
-import type { Dataset } from "../../datasets";
+import type { Dataset } from "../../datasets.js";
 export declare function isCopyFromStdinLine(line: string): boolean;
 export declare function rewriteLineForStagingImport(line: string, dataset: Dataset, stagingSchema: string, inCopyData?: boolean): string;

package/lib/src/scripts/validate_prefixed_tables.js

@@ -1,10 +1,9 @@
 import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
-import path from "path";
-import { sql } from "../databases_postgres";
-import { getChosenDatasets, getEnabledDatasets } from "../datasets";
-import { categoriesOption, silentOption } from "./shared/cli_helpers";
-import { extractPrefixedTableNamesFromGeneratedDefinition, senatSchemaName } from "./shared/prefixed_tables";
+import { sql } from "../databases_postgres.js";
+import { getChosenDatasets, getEnabledDatasets } from "../datasets.js";
+import { categoriesOption, silentOption } from "./shared/cli_helpers.js";
+import { extractPrefixedTableNamesFromGeneratedManifest, getGeneratedTableManifestPath, senatSchemaName, } from "./shared/prefixed_tables.js";
 const optionsDefinitions = [categoriesOption, silentOption];
 const options = commandLineArgs(optionsDefinitions);
 async function listPrefixedSenatTables(dataset) {
@@ -19,12 +18,12 @@ async function listPrefixedSenatTables(dataset) {
     return rows.map(({ relation_name }) => relation_name);
 }
 function getGeneratedDefinitionPath(dataset) {
-    return path.resolve("src", "raw_types_schemats", `${dataset.database}.ts`);
+    return getGeneratedTableManifestPath(dataset.database);
 }
 function listExpectedPrefixedSenatTables(dataset) {
-    const definitionFilePath = getGeneratedDefinitionPath(dataset);
-    const definition = fs.readFileSync(definitionFilePath, { encoding: "utf8" });
-    return extractPrefixedTableNamesFromGeneratedDefinition(definition, dataset.database);
+    const manifestFilePath = getGeneratedDefinitionPath(dataset);
+    const manifest = fs.readFileSync(manifestFilePath, { encoding: "utf8" });
+    return extractPrefixedTableNamesFromGeneratedManifest(manifest, dataset.database);
 }
 async function listColumns(schema, relationName) {
     const rows = await sql `