@tricoteuses/senat 2.22.16 → 2.23.0

package/lib/src/utils/cr_spliting.js

@@ -5,106 +5,6 @@ import fs from "fs-extra";
 import { sessionStartYearFromDate } from "../model/seance";
 import { frDateToISO, hourShortToStartTime } from "./date";
 import { normalizeSpaces } from "./string_cleaning";
-// Convert "quinze heures trente", "15 heures 30", "dix-sept heures moins le quart", etc. en "HHMM"
-function parseFrenchClockToHHMM(input) {
-    const s = (input || "")
-        .toLowerCase()
-        .normalize("NFKD")
-        .replace(/[\u0300-\u036f]/g, "")
-        .trim();
-    if (!s)
-        return undefined;
-    const digitMatch = s.match(/(\d{1,2})\s*heures?(?:\s*(\d{1,2}))?/);
-    if (digitMatch) {
-        const h = Math.min(24, Math.max(0, parseInt(digitMatch[1], 10)));
-        const m = digitMatch[2] ? Math.min(59, Math.max(0, parseInt(digitMatch[2], 10))) : 0;
-        return `${String(h).padStart(2, "0")}${String(m).padStart(2, "0")}`;
-    }
-    const NUM = new Map([
-        ["zero", 0],
-        ["une", 1],
-        ["un", 1],
-        ["deux", 2],
-        ["trois", 3],
-        ["quatre", 4],
-        ["cinq", 5],
-        ["six", 6],
-        ["sept", 7],
-        ["huit", 8],
-        ["neuf", 9],
-        ["dix", 10],
-        ["onze", 11],
-        ["douze", 12],
-        ["treize", 13],
-        ["quatorze", 14],
-        ["quinze", 15],
-        ["seize", 16],
-        ["dix-sept", 17],
-        ["dix sept", 17],
-        ["dix-huit", 18],
-        ["dix huit", 18],
-        ["dix-neuf", 19],
-        ["dix neuf", 19],
-        ["vingt", 20],
-        ["vingt et une", 21],
-        ["vingt-et-une", 21],
-        ["vingt et un", 21],
-        ["vingt-et-un", 21],
-        ["vingt-deux", 22],
-        ["vingt deux", 22],
-        ["vingt-trois", 23],
-        ["vingt trois", 23],
-        ["vingt-quatre", 24],
-        ["vingt quatre", 24],
-    ]);
-    const hourWordMatch = s.match(/([a-z\- ]+?)\s*heures?/);
-    if (!hourWordMatch)
-        return undefined;
-    const hourWord = hourWordMatch[1].trim();
-    let hour = NUM.get(hourWord);
-    if (hour == null) {
-        const cleaned = hourWord.replace(/\s+/g, " ");
-        hour = NUM.get(cleaned);
-    }
-    if (hour == null)
-        return undefined;
-    let minutes = 0;
-    if (/\bet (demie|demi)\b/.test(s))
-        minutes = 30;
-    else if (/\bet quart\b/.test(s))
-        minutes = 15;
-    else if (/\bmoins le quart\b/.test(s)) {
-        hour = (hour + 23) % 24;
-        minutes = 45;
-    }
-    else {
-        const MIN = new Map([
-            ["cinq", 5],
-            ["dix", 10],
-            ["quinze", 15],
-            ["vingt", 20],
-            ["vingt-cinq", 25],
-            ["vingt cinq", 25],
-            ["trente", 30],
-            ["trente-cinq", 35],
-            ["trente cinq", 35],
-            ["quarante", 40],
-            ["quarante-cinq", 45],
-            ["quarante cinq", 45],
-            ["cinquante", 50],
-            ["cinquante-cinq", 55],
-            ["cinquante cinq", 55],
-        ]);
-        const minWordMatch = s.match(/heures?\s+([a-z\- ]+?)(?:[).,;]|$)/);
-        if (minWordMatch) {
-            const mw = minWordMatch[1].trim();
-            const m1 = MIN.get(mw);
-            if (m1 != null)
-                minutes = m1;
-        }
-    }
-    return `${String(hour).padStart(2, "0")}${String(minutes).padStart(2, "0")}`;
-}
 function extractWeekStartFromHead($) {
     const og = $('meta[property="og:title"]').attr("content") || $("title").text();
     const m = (og ?? "").toLowerCase().match(/semaine du\s+(\d{1,2}\s+\w+\s+\d{4})/i);
@@ -181,7 +81,12 @@ export function parseCommissionMetadataFromHtml(html, sourceFileName) {
     };
 }
 function isGroupedReunion(o) {
-    return o && typeof o === "object" && typeof o.uid === "string" && typeof o.date === "string";
+    return (typeof o === "object" &&
+        o !== null &&
+        "uid" in o &&
+        typeof o.uid === "string" &&
+        "date" in o &&
+        typeof o.date === "string");
 }
 export async function loadAgendaForDate(dataDir, yyyymmdd, session) {
     const baseDir = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session));

package/lib/src/utils/reunion_odj_building.d.ts

@@ -1,5 +1,9 @@
 import commandLineArgs from "command-line-args";
-import { DossierLegislatifResult } from "../model/dosleg";
+import { ActeLegislatif, DossierLegislatifResult } from "../model/dosleg";
 import { AgendaEvent, ReunionOdj } from "../types/agenda";
-export declare function buildOdj(events: AgendaEvent[], dossierBySenatUrl: Record<string, DossierLegislatifResult>): ReunionOdj | undefined;
-export declare function buildSenatDossierIndex(options: commandLineArgs.CommandLineOptions): Record<string, DossierLegislatifResult>;
+type DossierWithActes = DossierLegislatifResult & {
+    actes_legislatifs?: ActeLegislatif[] | null;
+};
+export declare function buildOdj(events: AgendaEvent[], dossierBySenatUrl: Record<string, DossierWithActes>): ReunionOdj | undefined;
+export declare function buildSenatDossierIndex(options: commandLineArgs.CommandLineOptions): Record<string, DossierWithActes>;
+export {};

package/lib/src/utils/reunion_parsing.d.ts

@@ -1,4 +1,5 @@
 import { DateTime } from "luxon";
+import type { AnyNode } from "domhandler";
 import { AgendaEvent, Reunion } from "../types/agenda";
 import { DossierLegislatifResult } from "../model/dosleg";
 import * as cheerio from "cheerio";
@@ -18,6 +19,6 @@ export type SommaireBlock = {
     startIndex: number;
     targetId?: string | null;
 };
-export declare function extractSommaireBlocks($: cheerio.CheerioAPI, idx: Map<any, number>): SommaireBlock[];
+export declare function extractSommaireBlocks($: cheerio.CheerioAPI, idx: Map<AnyNode, number>): SommaireBlock[];
 export declare function parseISO(iso: string | null | undefined): DateTime | null;
 export {};

package/lib/src/utils/reunion_parsing.js

@@ -50,8 +50,8 @@ export function buildReunionsByBucket(events, dossierBySenatUrl) {
     for (const e of events) {
         const kind = classifyAgendaType(e?.type);
         if (!kind) {
-            continue;
             console.warn("Can't determine type of reunion");
+            continue;
         }
         const bucket = typeToSuffixStrict(kind);
         const uid = makeReunionUid(e.date, kind, e.id, e.organe ?? null);
@@ -83,7 +83,7 @@ function classifyAgendaType(typeLabel) {
         return "MC";
     if (/\boffices\b|\bdelegations\b/.test(s))
         return "OD";
-    if (/\instances\b|\decisionelles\b/.test(s))
+    if (/instances\b|decisionelles\b/.test(s))
         return "ID";
     return null;
 }

package/lib/src/videos/match.js

@@ -8,11 +8,14 @@ export async function matchOneReunion(args) {
     if (!options["silent"])
         console.log(`Matching video for reunion: ${agenda?.uid}`);
     const isSP = (agenda.type ?? "").toLowerCase().includes("séance publique");
-    const minAcceptBase = weights?.minAccept;
-    const margin = weights?.margin;
-    const titleDominance = weights?.titleDominance;
-    const orgUncertainPenalty = weights?.orgUncertainPenalty; // 1 disables
-    const orgSkipDice = weights?.orgSkipDice;
+    if (!weights) {
+        throw new Error("matchOneReunion: missing weights");
+    }
+    const minAcceptBase = weights.minAccept;
+    const margin = weights.margin;
+    const titleDominance = weights.titleDominance ?? 0;
+    const orgUncertainPenalty = weights.orgUncertainPenalty ?? 1; // 1 disables
+    const orgSkipDice = weights.orgSkipDice ?? 0;
     const minAccept = minAcceptBase + (candidates.length >= 20 ? 0.08 : candidates.length >= 10 ? 0.05 : 0);
     const vw = weights
         ? {

package/lib/src/videos/pipeline.d.ts

@@ -1,13 +1,16 @@
 import { Reunion } from "../types/agenda";
 import { BestMatch, LastForVideo, MatchContext } from "./types";
 import { CommandLineOptions } from "command-line-args";
+type VideoPipelineOptions = CommandLineOptions & {
+    silent?: boolean;
+};
 export declare function processOneReunionMatch(args: {
     agenda: Reunion;
     best: BestMatch | null;
     baseDir: string;
     dataDir: string;
     session: number;
-    options: Record<string, any>;
+    options: VideoPipelineOptions;
     writeIfChanged: (p: string, content: string) => Promise<void>;
     lastByVideo: Map<string, {
         agendaUid: string;
@@ -34,7 +37,7 @@ export declare function processBisIfNeeded(args: {
         baseDir: string;
         dataDir: string;
         session: number;
-        options: Record<string, any>;
+        options: VideoPipelineOptions;
         writeIfChanged: (p: string, content: string) => Promise<void>;
         lastByVideo: Map<string, LastForVideo>;
         getAgendaSegmentTimecodes: (dataNvs: string, finalNvs: string, agendaKey: string) => {
@@ -50,3 +53,4 @@ export declare function processBisIfNeeded(args: {
     buildSenatVodMasterM3u8FromNvs: (dataNvs: string) => string | null;
 }): Promise<void>;
 export declare function writeIfChanged(p: string, content: string): Promise<void>;
+export {};

package/lib/src/videos/pipeline.js

@@ -5,6 +5,9 @@ import { fetchText } from "./search";
 import fs from "fs-extra";
 import fsp from "fs/promises";
 import path from "path";
+function isReunion(value) {
+    return typeof value === "object" && value !== null && "uid" in value;
+}
 export async function processOneReunionMatch(args) {
     const { agenda, best, baseDir, dataDir, session, options, writeIfChanged, lastByVideo, getAgendaSegmentTimecodes, buildSenatVodMasterM3u8FromNvs, } = args;
     const reunionUid = agenda.uid;
@@ -14,7 +17,7 @@ export async function processOneReunionMatch(args) {
         dataTxt = await fsp.readFile(path.join(baseDir, "data.nvs"), "utf-8");
         finalTxt = await fsp.readFile(path.join(baseDir, "finalplayer.nvs"), "utf-8");
     }
-    catch (e) {
+    catch {
         console.warn(`[skip] Missing NVS files for reunion ${reunionUid}`);
         return;
     }
@@ -43,9 +46,10 @@ export async function processOneReunionMatch(args) {
         if (prev && prev.agendaJsonPath !== agendaJsonPath) {
             // micro-safety: do not close with an earlier timecode
             if (timecodeDebutVideo <= prev.start) {
-                console.warn(`[warn] timecode order inversion on same video: ` +
-                    `prev=${prev.agendaUid}(${prev.start}s) -> cur=${agenda.uid}(${timecodeDebutVideo}s). ` +
-                    `Skip closing prev to avoid negative segment.`);
+                console.warn("[warn] timecode order inversion on same video: " +
+                    `prev=${prev.agendaUid}(${prev.start}s) -> ` +
+                    `cur=${agenda.uid}(${timecodeDebutVideo}s). ` +
+                    "Skip closing prev to avoid negative segment.");
             }
             else {
                 await patchAgendaTimecodeFin({
@@ -63,10 +67,14 @@ export async function processOneReunionMatch(args) {
     try {
         obj = JSON.parse(raw);
     }
-    catch (e) {
+    catch {
         console.warn(`[warn] invalid JSON in ${agendaJsonPath}`);
         return;
     }
+    if (!isReunion(obj)) {
+        console.warn(`[warn] invalid reunion payload in ${agendaJsonPath}`);
+        return;
+    }
     const next = { ...obj, urlVideo: master, startTime: agenda.startTime, urlPageVideo: best?.pageUrl };
     if (timecodeDebutVideo != null) {
         next.timecodeDebutVideo = timecodeDebutVideo;
@@ -98,7 +106,7 @@ export async function processBisIfNeeded(args) {
     };
     const baseDirBis = path.join(path.dirname(ctx.baseDir), bisUid);
     await fs.ensureDir(baseDirBis);
-    await ensureBisAgendaJson({ agenda, agendaBis, dataDir: ctx.dataDir, session: ctx.session, options });
+    await ensureBisAgendaJson({ agenda, agendaBis, dataDir: ctx.dataDir, session: ctx.session });
     await downloadNvsForMatch(secondBest, baseDirBis);
     await processOneReunionMatch({
         agenda: agendaBis,
@@ -114,7 +122,7 @@ export async function processBisIfNeeded(args) {
     });
 }
 async function ensureBisAgendaJson(args) {
-    const { agenda, agendaBis, dataDir, session, options } = args;
+    const { agenda, agendaBis, dataDir, session } = args;
     const agendaJsonPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session), `${agenda.uid}.json`);
     const agendaBisJsonPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session), `${agendaBis.uid}.json`);
     if (!(await fs.pathExists(agendaJsonPath))) {
@@ -125,7 +133,8 @@ async function ensureBisAgendaJson(args) {
         const raw = await fsp.readFile(agendaJsonPath, "utf-8");
         const obj = JSON.parse(raw);
         if (!obj || typeof obj !== "object" || Array.isArray(obj)) {
-            console.warn(`[bis] cannot clone agenda json: expected object in ${agendaJsonPath}, got ${Array.isArray(obj) ? "array" : typeof obj}`);
+            console.warn(`[bis] cannot clone agenda json: expected object in ${agendaJsonPath}, ` +
+                `got ${Array.isArray(obj) ? "array" : typeof obj}`);
             return;
         }
         await writeIfChanged(agendaBisJsonPath, JSON.stringify(agendaBis, null, 2));
@@ -167,6 +176,10 @@ async function patchAgendaTimecodeFin(args) {
         console.warn(`[warn] invalid JSON in ${agendaJsonPath}`);
         return;
     }
+    if (typeof obj !== "object" || obj === null || Array.isArray(obj)) {
+        console.warn(`[warn] invalid reunion payload in ${agendaJsonPath}`);
+        return;
+    }
     const next = { ...obj, timecodeFinVideo };
     await writeIfChanged(agendaJsonPath, JSON.stringify(next, null, 2));
 }

package/lib/src/videos/search.js

@@ -55,7 +55,9 @@ export async function fetchCandidatesForAgenda(agenda, options) {
     const pages = await fetchAllSearchPages(searchParams);
     if (!pages.length) {
         if (!options["silent"]) {
-            console.log(`[miss] ${agenda.uid} no candidates (videotype=${searchParams.videotype}, organe=${searchParams.organe || "-"}, date=${searchParams.begin || "-"})`);
+            console.log(`[miss] ${agenda.uid} no candidates ` +
+                `(videotype=${searchParams.videotype}, organe=${searchParams.organe || "-"}, ` +
+                `date=${searchParams.begin || "-"})`);
         }
         return null;
     }
@@ -63,7 +65,9 @@ export async function fetchCandidatesForAgenda(agenda, options) {
     const candidates = extractCandidatesFromSearchHtml(combinedHtml).slice(0, MAX_CANDIDATES);
     if (!candidates.length) {
         if (!options["silent"]) {
-            console.log(`[miss] ${agenda.uid} no candidates after parse (videotype=${searchParams.videotype}, organe=${searchParams.organe || "-"}, date=${searchParams.begin || "-"})`);
+            console.log(`[miss] ${agenda.uid} no candidates after parse ` +
+                `(videotype=${searchParams.videotype}, organe=${searchParams.organe || "-"}, ` +
+                `date=${searchParams.begin || "-"})`);
         }
         return null;
     }

package/lib/strings.d.ts

@@ -0,0 +1 @@
+export declare function slugify(string: string, replacement?: string): string;

package/lib/strings.js

@@ -0,0 +1,18 @@
+import originalSlugify from "slug";
+const slugifyCharmap = {
+    ...originalSlugify.defaults.charmap,
+    "'": " ",
+    "@": " ",
+    ".": " ",
+    "-": "_",
+};
+export function slugify(string, replacement) {
+    const options = {
+        charmap: slugifyCharmap,
+        mode: "rfc3986",
+    };
+    if (replacement) {
+        options.replacement = replacement;
+    }
+    return originalSlugify(string, options);
+}

package/lib/strings.mjs

@@ -0,0 +1,18 @@
+import originalSlugify from "slug";
+const slugifyCharmap = {
+    ...originalSlugify.defaults.charmap,
+    "'": " ",
+    "@": " ",
+    ".": " ",
+    "-": "_",
+};
+export function slugify(string, replacement) {
+    const options = {
+        charmap: slugifyCharmap,
+        mode: "rfc3986",
+    };
+    if (replacement) {
+        options.replacement = replacement;
+    }
+    return originalSlugify(string, options);
+}

package/lib/strings.ts

@@ -0,0 +1,26 @@
+import originalSlugify from "slug"
+
+const slugifyCharmap = {
+  ...originalSlugify.defaults.charmap,
+  "'": " ",
+  "@": " ",
+  ".": " ",
+  "-": "_",
+}
+
+interface Options {
+  charmap: { [c: string]: string }
+  mode: string
+  replacement?: string
+}
+
+export function slugify(string: string, replacement?: string): string {
+  const options: Options = {
+    charmap: slugifyCharmap,
+    mode: "rfc3986",
+  }
+  if (replacement) {
+    options.replacement = replacement
+  }
+  return originalSlugify(string, options)
+}

package/lib/tests/incrementalImportSql.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/incrementalImportSql.test.js

@@ -0,0 +1,155 @@
+import { describe, expect, it } from "vitest";
+import { buildIncrementalDatasetImportSql, buildNormalizeStagingSchemaSql, } from "../src/scripts/shared/incremental_import_sql";
+import { stagingSchemaName } from "../src/scripts/shared/prefixed_tables";
+import { buildExportStagingMetadataStatementsQuery } from "../src/scripts/shared/staging_metadata_sql";
+import { isCopyFromStdinLine, rewriteLineForStagingImport } from "../src/scripts/shared/staging_import";
+describe("incremental import SQL", () => {
+    it("normalizes staging object names before merge", () => {
+        const sql = buildNormalizeStagingSchemaSql("dosleg");
+        expect(sql).toContain("WHERE n.nspname = 'dosleg_staging'");
+        expect(sql).toContain("AND c.relname NOT LIKE 'dosleg_' || '%'");
+        expect(sql).toContain("ALTER INDEX %I.%I RENAME TO %I");
+        expect(sql).toContain("ALTER SEQUENCE %I.%I RENAME TO %I");
+        expect(sql).toContain("ALTER TABLE %I.%I RENAME TO %I");
+    });
+    it("builds merge SQL for incremental synchronization into senat", () => {
+        const sql = buildIncrementalDatasetImportSql("dosleg", "opendata", {
+            corscr: ["sesann", "scrnum", "corscrord"],
+            docsea: ["evtseacle", "docseaord"],
+        }, ["amescr"]);
+        expect(sql).toContain("WHERE n.nspname = 'dosleg_staging'");
+        expect(sql).toContain("AND c.relkind IN ('r', 'p', 'f')");
+        expect(sql).toContain("CREATE TABLE senat.%I (LIKE %I.%I INCLUDING IDENTITY INCLUDING GENERATED INCLUDING DEFAULTS)");
+        expect(sql).toContain("ALTER TABLE senat.%I DROP COLUMN IF EXISTS %I CASCADE");
+        expect(sql).toContain("ALTER TABLE senat.%I ADD COLUMN %I %s");
+        expect(sql).toContain("ALTER TABLE senat.%I ALTER COLUMN %I TYPE %s%s USING %I::%s");
+        expect(sql).toContain("SELECT obj_description(c.oid, 'pg_class')");
+        expect(sql).toContain("COMMENT ON TABLE senat.%I IS %L");
+        expect(sql).toContain("col_description(staging_class.oid, staging_attribute.attnum) AS staging_comment");
+        expect(sql).toContain("COMMENT ON COLUMN senat.%I.%I IS %L");
+        expect(sql).toContain("SELECT 'dosleg_corscr' AS tablename, ARRAY['sesann', 'scrnum', 'corscrord']::text[] AS key_columns");
+        expect(sql).toContain("SELECT 'dosleg_docsea' AS tablename, ARRAY['evtseacle', 'docseaord']::text[] AS key_columns");
+        expect(sql).toContain("SELECT 'dosleg_amescr' AS tablename");
+        expect(sql).toContain("t.%1$I IS NOT DISTINCT FROM s.%1$I");
+        expect(sql).toContain("jsonb_build_array(");
+        expect(sql).toContain("Sync strategy for %.%: %");
+        expect(sql).toContain("Rejected sync key for %.% from % because only %/% columns are present in both staging and target: %");
+        expect(sql).toContain("WHERE table_namespace.nspname IN ('dosleg_staging', 'senat')");
+        expect(sql).toContain("CASE WHEN table_namespace.nspname = 'dosleg_staging' THEN 0 ELSE 1 END AS schema_priority");
+        expect(sql).toContain("No primary key or all-NOT-NULL unique key found for %.%; falling back to full refresh");
+        expect(sql).toContain("incremental_row_multiset");
+        expect(sql).toContain("LEFT JOIN staging_counts reference_row USING (row_signature)");
+        expect(sql).toContain("LEFT JOIN target_counts reference_row USING (row_signature)");
+        expect(sql).toContain("DELETE FROM senat.%1$I t WHERE NOT EXISTS (SELECT 1 FROM %2$I.%1$I s WHERE %3$s)");
+        expect(sql).toContain("UPDATE senat.%1$I t SET %2$s FROM %3$I.%1$I s WHERE %4$s AND (%5$s)");
+        expect(sql).toContain("INSERT INTO senat.%1$I (%2$s) SELECT %3$s FROM %4$I.%1$I s WHERE NOT EXISTS (SELECT 1 FROM senat.%1$I t WHERE %5$s)");
+        expect(sql).toContain("ALTER TABLE senat.%I ADD CONSTRAINT %I %s");
+        expect(sql).toContain("DROP SEQUENCE IF EXISTS senat.%I CASCADE");
+        expect(sql).not.toContain("DROP SCHEMA IF EXISTS dosleg_staging CASCADE;");
+    });
+    it("prefers row multiset strategy over configured keys when both are provided", () => {
+        const sql = buildIncrementalDatasetImportSql("questions", "opendata", {
+            tam_reponses: ["idque"],
+        }, ["tam_reponses"]);
+        expect(sql).toContain("IF use_row_multiset_merge THEN");
+        expect(sql).toContain("ELSIF key_columns IS NULL THEN");
+        expect(sql).toContain("SELECT 'questions_tam_reponses' AS tablename");
+        expect(sql).toContain("incremental_row_multiset");
+    });
+    it("escapes single quotes in SQL literals produced for staging names", () => {
+        const normalizeSql = buildNormalizeStagingSchemaSql("quo'te");
+        const incrementalSql = buildIncrementalDatasetImportSql("quo'te", "opendata", {
+            table: ["col'umn"],
+        }, ["ta'ble"]);
+        expect(normalizeSql).toContain("WHERE n.nspname = 'quo''te_staging'");
+        expect(normalizeSql).toContain("AND c.relname NOT LIKE 'quo''te_' || '%'");
+        expect(incrementalSql).toContain("WHERE n.nspname = 'quo''te_staging'");
+        expect(incrementalSql).toContain("tablename LIKE 'quo''te_' || '%'");
+        expect(incrementalSql).toContain("SELECT 'quo''te_table' AS tablename, ARRAY['col''umn']::text[] AS key_columns");
+        expect(incrementalSql).toContain("SELECT 'quo''te_ta''ble' AS tablename");
+    });
+    it("builds an unlogged staging schema name usable by import rewriting", () => {
+        expect(stagingSchemaName("sens")).toBe("sens_staging");
+    });
+    it("exports metadata synchronization statements from the real staging database", () => {
+        const sql = buildExportStagingMetadataStatementsQuery("sens_staging", "senat");
+        expect(sql).toContain("CREATE SEQUENCE IF NOT EXISTS %I.%I");
+        expect(sql).toContain("ALTER TABLE %I.%I ALTER COLUMN %I SET DEFAULT %s");
+        expect(sql).toContain("COMMENT ON TABLE %I.%I IS %L");
+        expect(sql).toContain("COMMENT ON COLUMN %I.%I.%I IS %L");
+        expect(sql).toContain("ALTER TABLE %I.%I ADD CONSTRAINT %I %s");
+        expect(sql).toContain("WHEN 'p' THEN 1");
+        expect(sql).toContain("WHEN 'f' THEN 4");
+        expect(sql).toContain("pg_get_indexdef(index_data.indexrelid)");
+        expect(sql).toContain("SELECT setval(%L, %s, %s)");
+        expect(sql).toContain("encode(convert_to(statement, 'UTF8'), 'hex')");
+        expect(sql).toContain("ORDER BY group_no, order_no, object_name");
+    });
+    it("rewrites staging DDL toward the staging schema and unlogged objects", () => {
+        const dataset = {
+            database: "dosleg",
+            repairEncoding: false,
+            title: "demo",
+            url: "https://example.test/demo.zip",
+        };
+        expect(rewriteLineForStagingImport("SET search_path = dosleg, pg_catalog;", dataset, "dosleg_staging")).toBe("SET search_path = dosleg_staging, pg_catalog;");
+        expect(rewriteLineForStagingImport("CREATE TABLE texte (id integer);", dataset, "dosleg_staging")).toBe("CREATE UNLOGGED TABLE texte (id integer);");
+        expect(rewriteLineForStagingImport("CREATE SEQUENCE texte_id_seq;", dataset, "dosleg_staging")).toBe("CREATE UNLOGGED SEQUENCE texte_id_seq;");
+        expect(rewriteLineForStagingImport("SELECT * FROM dosleg.texte;", dataset, "dosleg_staging")).toBe("SELECT * FROM dosleg_staging.texte;");
+        expect(rewriteLineForStagingImport("SELECT 'dosleg.texte'::text;", dataset, "dosleg_staging")).toBe("SELECT 'dosleg.texte'::text;");
+    });
+    it("keeps two-part COMMENT ON COLUMN statements when the table name matches the dataset name", () => {
+        const dataset = {
+            database: "debats",
+            repairEncoding: false,
+            title: "demo",
+            url: "https://example.test/demo.zip",
+        };
+        expect(rewriteLineForStagingImport("COMMENT ON COLUMN debats.datsea IS 'Date de la séance (clé)';", dataset, "debats_staging")).toBe("COMMENT ON COLUMN debats.datsea IS 'Date de la séance (clé)';");
+    });
+    it("rewrites sens dump references that still use the senat schema", () => {
+        const dataset = {
+            database: "sens",
+            repairEncoding: false,
+            title: "demo",
+            url: "https://example.test/demo.zip",
+        };
+        expect(rewriteLineForStagingImport("SET search_path = senat, pg_catalog;", dataset, "sens_staging")).toBe("SET search_path = sens_staging, pg_catalog;");
+        expect(rewriteLineForStagingImport("CREATE SCHEMA senat;", dataset, "sens_staging")).toBe("CREATE SCHEMA IF NOT EXISTS sens_staging;");
+        expect(rewriteLineForStagingImport("ALTER TABLE senat.senateur ADD COLUMN test integer;", dataset, "sens_staging")).toBe("ALTER TABLE sens_staging.senateur ADD COLUMN test integer;");
+    });
+    it("skips privilege statements during staging import", () => {
+        const dataset = {
+            database: "ameli",
+            repairEncoding: false,
+            title: "demo",
+            url: "https://example.test/demo.zip",
+        };
+        expect(rewriteLineForStagingImport("GRANT ALL ON TABLE amd TO opendata;", dataset, "ameli_staging")).toBe("-- skipped GRANT/REVOKE during staging import");
+        expect(rewriteLineForStagingImport("REVOKE ALL ON TABLE amd FROM opendata;", dataset, "ameli_staging")).toBe("-- skipped GRANT/REVOKE during staging import");
+    });
+    it("skips redundant dump cleanup statements during staging import", () => {
+        const dataset = {
+            database: "ameli",
+            repairEncoding: false,
+            title: "demo",
+            url: "https://example.test/demo.zip",
+        };
+        expect(rewriteLineForStagingImport("ALTER TABLE ONLY ameli_staging.txt_ameli DROP CONSTRAINT IF EXISTS txt_ameli_pkey;", dataset, "ameli_staging")).toBe("-- skipped redundant DROP CONSTRAINT during staging import");
+        expect(rewriteLineForStagingImport("DROP INDEX IF EXISTS ameli_staging.pk_txt;", dataset, "ameli_staging")).toBe("-- skipped redundant DROP INDEX during staging import");
+        expect(rewriteLineForStagingImport("DROP TABLE IF EXISTS ameli_staging.txt_ameli;", dataset, "ameli_staging")).toBe("-- skipped redundant DROP TABLE during staging import");
+        expect(rewriteLineForStagingImport("DROP SCHEMA public;", dataset, "ameli_staging")).toBe("-- skipped redundant DROP SCHEMA public during staging import");
+        expect(rewriteLineForStagingImport("CREATE SCHEMA public;", dataset, "ameli_staging")).toBe("-- skipped redundant CREATE SCHEMA public during staging import");
+        expect(rewriteLineForStagingImport("COMMENT ON SCHEMA public IS 'standard public schema';", dataset, "ameli_staging")).toBe("-- skipped redundant COMMENT ON SCHEMA public during staging import");
+    });
+    it("does not rewrite COPY payload lines", () => {
+        const dataset = {
+            database: "ameli",
+            repairEncoding: false,
+            title: "demo",
+            url: "https://example.test/demo.zip",
+        };
+        expect(isCopyFromStdinLine("COPY amd (id, obj) FROM stdin;")).toBe(true);
+        expect(rewriteLineForStagingImport("1\t<body><p>consultation du public.</p></body>\t<body><p>Mayotte public.</p></body>", dataset, "ameli_staging", true)).toBe("1\t<body><p>consultation du public.</p></body>\t<body><p>Mayotte public.</p></body>");
+    });
+});

package/lib/tests/prefixedTables.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/prefixedTables.test.js

@@ -0,0 +1,29 @@
+import { describe, expect, it } from "vitest";
+import { extractPrefixedTableNamesFromGeneratedDefinition, normalizeGeneratedDefinition, prefixedName, senatSchemaName, stagingSchemaName, } from "../src/scripts/shared/prefixed_tables";
+describe("prefixed table helpers", () => {
+    it("builds prefixed table and staging schema names", () => {
+        expect(prefixedName("dosleg", "texte")).toBe("dosleg_texte");
+        expect(stagingSchemaName("questions")).toBe("questions_staging");
+        expect(senatSchemaName).toBe("senat");
+    });
+    it("normalizes generated definitions by removing dataset prefixes", () => {
+        const definition = "/* tslint:disable */\n/**\n * AUTO-GENERATED FILE @ 2026-03-20 16:00:00\n" +
+            " * $ schemats generate -c postgres://demo -t ameli_amd -t ameli_ses -s senat\n" +
+            " */\nexport namespace ameli_amdFields {}\nexport interface ameli_amd {}\nexport interface ameli_ses {}\n";
+        const normalized = normalizeGeneratedDefinition(definition, "ameli");
+        expect(normalized).toContain("AUTO-GENERATED FILE");
+        expect(normalized).not.toContain("AUTO-GENERATED FILE @");
+        expect(normalized).toContain("export namespace amdFields");
+        expect(normalized).toContain("export interface amd");
+        expect(normalized).toContain("export interface ses");
+        expect(normalized).not.toContain("ameli_amd");
+    });
+    it("extracts expected prefixed public tables from generated definitions", () => {
+        const definition = "/**\n * $ schemats generate -c postgres://demo -t amd -t ses -t txt_ameli -s ameli\n */";
+        expect(extractPrefixedTableNamesFromGeneratedDefinition(definition, "ameli")).toEqual([
+            "ameli_amd",
+            "ameli_ses",
+            "ameli_txt_ameli",
+        ]);
+    });
+});

package/lib/tests/schemaVersion.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/schemaVersion.test.js

@@ -0,0 +1,23 @@
+import { describe, expect, it } from "vitest";
+import { buildEnsureSchemaVersionTableSql, buildIncrementSchemaVersionSql, buildSchemaStructureFingerprintQuery, } from "../src/scripts/shared/schema_version";
+describe("schema version SQL", () => {
+    it("creates the senat.version table and seeds version zero", () => {
+        const sql = buildEnsureSchemaVersionTableSql("senat");
+        expect(sql).toContain("CREATE TABLE IF NOT EXISTS senat.version(");
+        expect(sql).toContain("number integer PRIMARY KEY");
+        expect(sql).toContain("COMMENT ON TABLE senat.version IS 'version of database'");
+        expect(sql).toContain("INSERT INTO senat.version(number)");
+        expect(sql).toContain("SELECT 0");
+    });
+    it("builds a fingerprint query that ignores the version table itself", () => {
+        const sql = buildSchemaStructureFingerprintQuery("senat");
+        expect(sql).toContain("table_class.relname <> 'version'");
+        expect(sql).toContain("columns.table_name <> 'version'");
+        expect(sql).toContain("constraint_data.contype <> 'n'");
+        expect(sql).toContain("pg_get_indexdef(idx.indexrelid)");
+        expect(sql).toContain("SELECT coalesce(md5(string_agg(entry, E'\\n' ORDER BY entry)), '')");
+    });
+    it("increments the schema version with a simple update", () => {
+        expect(buildIncrementSchemaVersionSql("senat")).toBe("UPDATE senat.version SET number = number + 1");
+    });
+});

package/lib/tests/validatePrefixedTables.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/validatePrefixedTables.test.js

@@ -0,0 +1,14 @@
+import { readFileSync } from "fs";
+import { describe, expect, it } from "vitest";
+import { datasets } from "../src/datasets";
+import { extractPrefixedTableNamesFromGeneratedDefinition } from "../src/scripts/shared/prefixed_tables";
+describe("generated raw types coverage", () => {
+    it("maps each generated dataset definition to prefixed senat tables", () => {
+        for (const dataset of Object.values(datasets)) {
+            const definition = readFileSync(`src/raw_types_schemats/${dataset.database}.ts`, { encoding: "utf8" });
+            const tableNames = extractPrefixedTableNamesFromGeneratedDefinition(definition, dataset.database);
+            expect(tableNames.length).toBeGreaterThan(0);
+            expect(tableNames.every((tableName) => tableName.startsWith(`${dataset.database}_`))).toBe(true);
+        }
+    });
+});