@tricoteuses/senat 2.22.16 → 2.23.0

package/lib/src/scripts/retrieve_open_data.js

@@ -5,12 +5,17 @@ import fs from "fs-extra";
 import path from "path";
 import StreamZip from "node-stream-zip";
 import readline from "readline";
-import * as windows1252 from "windows-1252";
-import { pipeline } from "stream";
+import { pipeline, Readable } from "stream";
 import { promisify } from "util";
+import * as windows1252 from "windows-1252";
 import config from "../config";
 import { getChosenDatasets, getEnabledDatasets } from "../datasets";
-import { commonOptions } from "./shared/cli_helpers";
+import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers";
+import { buildIncrementalDatasetImportSql, buildNormalizeStagingSchemaSql } from "./shared/incremental_import_sql";
+import { normalizeGeneratedDefinition, prefixedName, senatSchemaName, stagingSchemaName, } from "./shared/prefixed_tables";
+import { buildEnsureSchemaVersionTableSql, buildIncrementSchemaVersionSql, buildSchemaStructureFingerprintQuery, } from "./shared/schema_version";
+import { buildExportStagingMetadataStatementsQuery } from "./shared/staging_metadata_sql";
+import { isCopyFromStdinLine, rewriteLineForStagingImport } from "./shared/staging_import";
 const badWindows1252CharacterRegex = /[\u0080-\u009f]/g;
 const optionsDefinitions = [
     ...commonOptions,
@@ -22,7 +27,7 @@ const optionsDefinitions = [
     },
     {
         alias: "c",
-        help: "create TypeScript interfaces from databases schemas into src/raw_types_* directories",
+        help: "create TypeScript interfaces from database schemas into src/raw_types_* directories",
         name: "schema",
         type: Boolean,
     },
@@ -38,15 +43,20 @@ const optionsDefinitions = [
         name: "fetch",
         type: Boolean,
     },
+    {
+        help: "use separate staging database and postgres_fdw for incremental merge into target schema",
+        name: "incremental",
+        type: Boolean,
+    },
     {
         alias: "i",
-        help: "import SQL dumps into a freshly (re-)created database",
+        help: "import SQL dumps into PostgreSQL",
         name: "import",
         type: Boolean,
     },
     {
         alias: "S",
-        help: "sudo psql commands with given user",
+        help: "sudo psql commands with given user (example: --sudo postgres)",
         name: "sudo",
         type: String,
     },
@@ -59,99 +69,358 @@ const optionsDefinitions = [
 ];
 const options = commandLineArgs(optionsDefinitions);
 const streamPipeline = promisify(pipeline);
+const stagingServerName = "staging_server";
+function isIncrementalImport(options) {
+    return options["incremental"] === true;
+}
+function shellQuote(value) {
+    return `'${value.replace(/'/g, `'"'"'`)}'`;
+}
+function connectionEnv(connection) {
+    return {
+        ...process.env,
+        PGDATABASE: connection.name,
+        PGHOST: connection.host,
+        PGPASSWORD: connection.password,
+        PGPORT: String(connection.port),
+        PGUSER: connection.user,
+    };
+}
+function escapeSqlLiteral(value) {
+    return value.replace(/'/g, "''");
+}
+function sleep(delayMs) {
+    return new Promise((resolve) => setTimeout(resolve, delayMs));
+}
+function getExecSyncErrorOutput(error) {
+    const execError = error;
+    return `${execError.stderr ?? ""}\n${execError.stdout ?? ""}`;
+}
+function isRetryablePostgresError(error) {
+    const output = getExecSyncErrorOutput(error);
+    return (output.includes("deadlock detected") ||
+        output.includes("could not obtain lock") ||
+        output.includes("canceling statement due to lock timeout") ||
+        output.includes("could not serialize access"));
+}
+function canReuseExistingStagingDatabase(error) {
+    const output = getExecSyncErrorOutput(error);
+    return (output.includes("permission denied to terminate process") ||
+        output.includes("database is being accessed by other users") ||
+        output.includes("cannot drop the currently open database"));
+}
+function isMissingForeignServerError(error, serverName) {
+    return getExecSyncErrorOutput(error).includes(`server "${serverName}" does not exist`);
+}
+async function runWithRetry(operation, options, retryOptions) {
+    let attempt = 1;
+    let delayMs = retryOptions.delayMs;
+    while (true) {
+        try {
+            return operation();
+        }
+        catch (error) {
+            if (!isRetryablePostgresError(error) || attempt >= retryOptions.attempts) {
+                throw error;
+            }
+            if (!options["silent"]) {
+                console.warn(`${retryOptions.label} hit a transient PostgreSQL lock error on attempt ${attempt}/${retryOptions.attempts}; retrying in ${delayMs}ms...`);
+            }
+            await sleep(delayMs);
+            attempt += 1;
+            delayMs *= 2;
+        }
+    }
+}
+function buildPsqlCommand(baseArgs, connection, options) {
+    const sudoPrefix = options["sudo"] ? `sudo -u ${options["sudo"]} ` : "";
+    return (`${sudoPrefix}psql --quiet ` +
+        `-h ${shellQuote(connection.host)} ` +
+        `-p ${shellQuote(String(connection.port))} ` +
+        `-U ${shellQuote(connection.user)} ` +
+        `-d ${shellQuote(connection.name)} ` +
+        baseArgs);
+}
+function runPsqlFile(sqlFilePath, dataDir, options, connection, stopOnError = true) {
+    const onErrorFlag = stopOnError ? "-v ON_ERROR_STOP=1 " : "";
+    execSync(buildPsqlCommand(`${onErrorFlag}-f ${shellQuote(sqlFilePath)}`, connection, options), {
+        cwd: dataDir,
+        env: connectionEnv(connection),
+        encoding: "utf-8",
+        stdio: ["ignore", "pipe", "pipe"],
+    });
+}
+function runPsqlCommand(command, dataDir, options, connection, stopOnError = true) {
+    const onErrorFlag = stopOnError ? "-v ON_ERROR_STOP=1 " : "";
+    execSync(buildPsqlCommand(`${onErrorFlag}-c ${shellQuote(command)}`, connection, options), {
+        cwd: dataDir,
+        env: connectionEnv(connection),
+        encoding: "utf-8",
+        stdio: ["ignore", "pipe", "pipe"],
+    });
+}
+function runPsqlQuery(command, dataDir, options, connection, stopOnError = true) {
+    const onErrorFlag = stopOnError ? "-v ON_ERROR_STOP=1 " : "";
+    return execSync(buildPsqlCommand(`${onErrorFlag}-At -c ${shellQuote(command)}`, connection, options), {
+        cwd: dataDir,
+        env: connectionEnv(connection),
+        encoding: "utf-8",
+        stdio: ["ignore", "pipe", "pipe"],
+    });
+}
+function ensureStagingSchemaHasTables(dataset, dataDir, options, connection) {
+    const stagingSchema = stagingSchemaName(dataset.database);
+    const tableCount = Number.parseInt(runPsqlQuery(`SELECT count(*) FROM pg_tables WHERE schemaname = '${escapeSqlLiteral(stagingSchema)}'`, dataDir, options, connection).trim(), 10);
+    assert(tableCount > 0, `Staging schema ${stagingSchema} is empty after importing ${dataset.database}. Aborting incremental merge to protect ${senatSchemaName}.`);
+}
+function ensureStagingDatabase(dataDir, options, runtime) {
+    const maintenanceDb = process.env["PGDATABASE"] || "postgres";
+    const maintenanceConnection = {
+        ...runtime.staging,
+        name: maintenanceDb,
+    };
+    try {
+        runPsqlCommand(`SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${runtime.staging.name.replace(/'/g, "''")}' AND pid <> pg_backend_pid();`, dataDir, options, maintenanceConnection, false);
+        runPsqlCommand(`DROP DATABASE IF EXISTS ${runtime.staging.name}`, dataDir, options, maintenanceConnection, false);
+        runPsqlCommand(`CREATE DATABASE ${runtime.staging.name} WITH OWNER ${runtime.staging.user}`, dataDir, options, maintenanceConnection);
+    }
+    catch (error) {
+        if (!canReuseExistingStagingDatabase(error)) {
+            throw error;
+        }
+        if (!options["silent"]) {
+            console.warn(`Could not recreate staging database ${runtime.staging.name}; reusing the existing database instead.`);
+        }
+        ensureDatabaseExists(runtime.staging, dataDir, options);
+    }
+}
+function ensureDatabaseExists(connection, dataDir, options) {
+    const maintenanceDb = process.env["PGDATABASE"] || "postgres";
+    const maintenanceConnection = {
+        ...connection,
+        name: maintenanceDb,
+    };
+    const exists = runPsqlQuery(`SELECT 1 FROM pg_database WHERE datname = '${escapeSqlLiteral(connection.name)}'`, dataDir, options, maintenanceConnection).trim();
+    if (exists === "1") {
+        return;
+    }
+    runPsqlCommand(`CREATE DATABASE ${connection.name} WITH OWNER ${connection.user}`, dataDir, options, maintenanceConnection);
+}
+function ensureForeignStagingServer(dataDir, options, runtime) {
+    runPsqlCommand(`CREATE EXTENSION IF NOT EXISTS postgres_fdw`, dataDir, options, runtime.target);
+    runPsqlCommand(`DROP SERVER IF EXISTS ${stagingServerName} CASCADE`, dataDir, options, runtime.target, false);
+    runPsqlCommand([
+        `CREATE SERVER ${stagingServerName}`,
+        `FOREIGN DATA WRAPPER postgres_fdw`,
+        `OPTIONS (host '${escapeSqlLiteral(runtime.staging.host)}', dbname '${escapeSqlLiteral(runtime.staging.name)}', port '${escapeSqlLiteral(String(runtime.staging.port))}')`,
+    ].join(" "), dataDir, options, runtime.target);
+    runPsqlCommand([
+        `CREATE USER MAPPING FOR CURRENT_USER SERVER ${stagingServerName}`,
+        `OPTIONS (user '${escapeSqlLiteral(runtime.staging.user)}', password '${escapeSqlLiteral(runtime.staging.password)}')`,
+    ].join(" "), dataDir, options, runtime.target);
+}
+function cleanupForeignStagingServer(dataDir, options, runtime) {
+    runPsqlCommand(`DROP SERVER IF EXISTS ${stagingServerName} CASCADE`, dataDir, options, runtime.target, false);
+}
+function mountForeignStagingSchema(dataset, dataDir, options, runtime) {
+    const stagingSchema = stagingSchemaName(dataset.database);
+    const importForeignSchemaCommand = `IMPORT FOREIGN SCHEMA ${stagingSchema} FROM SERVER ${stagingServerName} INTO ${stagingSchema}`;
+    runPsqlCommand(`DROP SCHEMA IF EXISTS ${stagingSchema} CASCADE`, dataDir, options, runtime.target, false);
+    runPsqlCommand(`CREATE SCHEMA ${stagingSchema}`, dataDir, options, runtime.target);
+    try {
+        runPsqlCommand(importForeignSchemaCommand, dataDir, options, runtime.target);
+    }
+    catch (error) {
+        if (!isMissingForeignServerError(error, stagingServerName)) {
+            throw error;
+        }
+        if (!options["silent"]) {
+            console.warn(`Foreign server ${stagingServerName} disappeared before schema import; recreating it.`);
+        }
+        ensureForeignStagingServer(dataDir, options, runtime);
+        runPsqlCommand(importForeignSchemaCommand, dataDir, options, runtime.target);
+    }
+}
+function ensureSchemaVersionTable(dataDir, options, runtime) {
+    runPsqlCommand(buildEnsureSchemaVersionTableSql(senatSchemaName), dataDir, options, runtime.target);
+}
+function getSchemaStructureFingerprint(dataDir, options, runtime) {
+    return runPsqlQuery(buildSchemaStructureFingerprintQuery(senatSchemaName), dataDir, options, runtime.target).trim();
+}
+function getSchemaVersionNumber(dataDir, options, runtime) {
+    const version = runPsqlQuery(`SELECT number FROM ${senatSchemaName}.version`, dataDir, options, runtime.target).trim();
+    return Number.parseInt(version, 10);
+}
+function bumpSchemaVersionIfNeeded(previousFingerprint, dataDir, options, runtime) {
+    const currentFingerprint = getSchemaStructureFingerprint(dataDir, options, runtime);
+    if (currentFingerprint !== previousFingerprint) {
+        runPsqlCommand(buildIncrementSchemaVersionSql(senatSchemaName), dataDir, options, runtime.target);
+    }
+    if (!options["silent"]) {
+        const versionNumber = getSchemaVersionNumber(dataDir, options, runtime);
+        if (currentFingerprint !== previousFingerprint) {
+            console.log(`Incremented ${senatSchemaName}.version to ${versionNumber} after schema structure change.`);
+        }
+        else {
+            console.log(`Current ${senatSchemaName}.version: ${versionNumber}.`);
+        }
+    }
+}
+function finalizeDatasetImport(dataset, dataDir, options, runtime) {
+    const postImportFilePath = path.join(dataDir, `${dataset.database}_post_import.sql`);
+    const postImportSql = buildIncrementalDatasetImportSql(dataset.database, runtime.target.user, dataset.mergeKeys, dataset.rowMultisetMergeTables);
+    const stagingSchema = stagingSchemaName(dataset.database);
+    fs.writeFileSync(postImportFilePath, postImportSql, { encoding: "utf8" });
+    if (isIncrementalImport(options)) {
+        mountForeignStagingSchema(dataset, dataDir, options, runtime);
+    }
+    runPsqlFile(postImportFilePath, dataDir, options, runtime.target);
+    runPsqlCommand(`DROP SCHEMA IF EXISTS ${stagingSchema} CASCADE`, dataDir, options, runtime.target, false);
+}
+function applyStagingMetadataToTarget(dataset, dataDir, options, runtime) {
+    const stagingSchema = stagingSchemaName(dataset.database);
+    const encodedStatements = runPsqlQuery(buildExportStagingMetadataStatementsQuery(stagingSchema, senatSchemaName), dataDir, options, runtime.staging)
+        .split("\n")
+        .map((line) => line.trim())
+        .filter((line) => line.length > 0);
+    for (const encodedStatement of encodedStatements) {
+        const statement = Buffer.from(encodedStatement, "hex").toString("utf8");
+        runPsqlCommand(statement, dataDir, options, runtime.target);
+    }
+}
+function listTablesInSchema(schemaName, dataDir, options, connection) {
+    const query = ["SELECT tablename", "FROM pg_tables", `WHERE schemaname = '${schemaName}'`, "ORDER BY tablename"].join("\n");
+    const output = runPsqlQuery(query, dataDir, options, connection);
+    return output
+        .split("\n")
+        .map((tableName) => tableName.trim())
+        .filter((tableName) => tableName.length > 0);
+}
+function createManagedIndexesInStaging(dataset, dataDir, options, runtime) {
+    if (!dataset.indexes) {
+        return;
+    }
+    const stagingSchema = stagingSchemaName(dataset.database);
+    const importedTables = new Set(listTablesInSchema(stagingSchema, dataDir, options, runtime.staging));
+    runPsqlCommand(`CREATE SCHEMA IF NOT EXISTS ${stagingSchema};`, dataDir, options, runtime.staging);
+    for (const [table, indexes] of Object.entries(dataset.indexes)) {
+        if (!importedTables.has(table)) {
+            if (!options["silent"]) {
+                console.warn(`Skipping managed indexes for missing table ${stagingSchema}.${table}`);
+            }
+            continue;
+        }
+        for (const index of indexes) {
+            const indexName = prefixedName(dataset.database, `${table}_${index.name}`);
+            const columns = index.columns.join(", ");
+            const sql = `CREATE INDEX IF NOT EXISTS ${indexName} ON ${stagingSchema}.${table} (${columns});`;
+            try {
+                runPsqlCommand(sql, dataDir, options, runtime.staging);
+            }
+            catch (err) {
+                console.error(`Failed to create managed index ${indexName} on ${stagingSchema}.${table}:`, err);
+                continue;
+            }
+            if (!options["silent"]) {
+                console.log(`Prepared managed index ${indexName} on ${stagingSchema}.${table}`);
+            }
+        }
+    }
+}
+function listPrefixedTables(dataset, dataDir, options, runtime) {
+    const prefix = `${dataset.database}_`;
+    const query = [
+        "SELECT tablename",
+        "FROM pg_tables",
+        `WHERE schemaname = '${senatSchemaName}'`,
+        `  AND tablename LIKE '${prefix.replace(/_/g, "\\_")}%' ESCAPE '\\'`,
+        "ORDER BY tablename",
+    ].join("\n");
+    const output = runPsqlQuery(query, dataDir, options, runtime.target);
+    return output
+        .split("\n")
+        .map((tableName) => tableName.trim())
+        .filter((tableName) => tableName.length > 0);
+}
 async function downloadFile(url, dest) {
     const response = await fetch(url);
     if (!response.ok) {
         throw new Error(`Download failed ${response.status} ${response.statusText} for ${url}`);
     }
-    await streamPipeline(response.body, fs.createWriteStream(dest));
+    assert(response.body, `Empty response body for ${url}`);
+    await streamPipeline(Readable.fromWeb(response.body), fs.createWriteStream(dest));
 }
-/**
- * Copy a dataset database to the main Senat database (overwriting its contents).
- */
-async function copyToSenat(dataset, dataDir, options) {
+async function importIntoStaging(dataset, dataDir, options, runtime) {
     if (!options["silent"]) {
-        console.log(`Copying ${dataset.database} to Senat database...`);
+        console.log(`Importing ${dataset.database} into staging database ${runtime.staging.name}...`);
     }
     const sqlFilePath = path.join(dataDir, `${dataset.database}.sql`);
     const schemaDumpFile = path.join(dataDir, `${dataset.database}_schema_dump.sql`);
-    // Write the header and then stream the rest of the SQL file
+    const normalizeSqlFile = path.join(dataDir, `${dataset.database}_normalize_staging.sql`);
+    const stagingSchema = stagingSchemaName(dataset.database);
     const schemaSqlWriter = fs.createWriteStream(schemaDumpFile, { encoding: "utf8" });
-    // Add CREATE SCHEMA statement at the top
-    schemaSqlWriter.write(`DROP SCHEMA IF EXISTS ${dataset.database} CASCADE;\n`);
-    schemaSqlWriter.write(`CREATE SCHEMA IF NOT EXISTS ${dataset.database};\n`);
-    schemaSqlWriter.write(`GRANT USAGE ON SCHEMA ${dataset.database} TO ${config.db.user};\n`);
-    schemaSqlWriter.write(`GRANT SELECT ON ALL TABLES IN SCHEMA ${dataset.database} TO ${config.db.user};\n`);
-    schemaSqlWriter.write(`ALTER DEFAULT PRIVILEGES IN SCHEMA ${dataset.database} GRANT SELECT ON TABLES TO ${config.db.user};\n`);
+    schemaSqlWriter.write(`DROP SCHEMA IF EXISTS ${stagingSchema} CASCADE;\n`);
+    schemaSqlWriter.write(`CREATE SCHEMA IF NOT EXISTS ${stagingSchema};\n`);
     const lineReader = readline.createInterface({
         input: fs.createReadStream(sqlFilePath, { encoding: "utf8" }),
         crlfDelay: Infinity,
     });
+    let inCopyData = false;
     for await (const line of lineReader) {
-        let newLine = line;
-        // Replace 'public' schema outside single-quoted strings
-        function replacePublicOutsideStrings(line, schema) {
-            const parts = line.split(/(')/);
-            let inString = false;
-            for (let i = 0; i < parts.length; i++) {
-                if (parts[i] === "'") {
-                    inString = !inString;
-                }
-                else if (!inString) {
-                    // Only replace outside of strings, including before comma
-                    parts[i] = parts[i].replace(/\bpublic\b(?=(\s*\.|\s*[,;]|\s|$))/g, schema);
-                }
+        let newLine = rewriteLineForStagingImport(line, dataset, stagingSchema, inCopyData);
+        if (!inCopyData) {
+            newLine = newLine.replace(/SET client_encoding = 'LATIN1';/i, "SET client_encoding = 'UTF8';");
+            if (isCopyFromStdinLine(newLine)) {
+                inCopyData = true;
             }
-            return parts.join("");
         }
-        newLine = replacePublicOutsideStrings(line, dataset.database);
-        // Replace SET client_encoding to UTF8
-        newLine = newLine.replace(/SET client_encoding = 'LATIN1';/i, "SET client_encoding = 'UTF8';");
+        else if (line === "\\.") {
+            inCopyData = false;
+        }
         schemaSqlWriter.write(newLine + "\n");
     }
     schemaSqlWriter.end();
     await new Promise((resolve, reject) => {
         schemaSqlWriter.on("finish", () => {
             try {
-                execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -d senat -f ${schemaDumpFile}`, {
-                    env: process.env,
-                    encoding: "utf-8",
-                    stdio: ["ignore", "pipe", "pipe"],
-                });
+                runPsqlFile(schemaDumpFile, dataDir, options, runtime.staging, false);
+                createManagedIndexesInStaging(dataset, dataDir, options, runtime);
+                fs.writeFileSync(normalizeSqlFile, buildNormalizeStagingSchemaSql(dataset.database), { encoding: "utf8" });
+                runWithRetry(() => runPsqlFile(normalizeSqlFile, dataDir, options, runtime.staging), options, {
+                    attempts: 4,
+                    delayMs: 500,
+                    label: `Staging normalization for ${dataset.database}`,
+                })
+                    .then(() => {
+                    ensureStagingSchemaHasTables(dataset, dataDir, options, runtime.staging);
+                    resolve();
+                })
+                    .catch(reject);
             }
             catch (error) {
+                const execError = error;
                 if (!options["silent"]) {
-                    console.error(`Failed to import ${dataset.database} schema:`);
-                    if (error.stderr) {
-                        console.error(error.stderr);
+                    console.error(`Failed to import ${dataset.database} data into staging:`);
+                    if (execError.stderr) {
+                        console.error(execError.stderr);
                     }
-                    if (error.stdout) {
-                        console.error(error.stdout);
+                    if (execError.stdout) {
+                        console.error(execError.stdout);
                     }
                 }
+                reject(error);
             }
-            resolve();
         });
         schemaSqlWriter.on("error", reject);
     });
 }
-async function retrieveDataset(dataDir, dataset) {
+async function retrieveDataset(dataDir, dataset, options, runtime) {
     const zipFilename = dataset.url.substring(dataset.url.lastIndexOf("/") + 1);
     const zipFilePath = path.join(dataDir, zipFilename);
     if (options["all"] || options["fetch"]) {
-        // Fetch & save ZIP file.
         if (!options["silent"]) {
-            console.log(`Loading ${dataset.title}: ${zipFilename}…`);
+            console.log(`Loading ${dataset.title}: ${zipFilename}...`);
         }
-        // Fetch fails with OpenSSL error: dh key too small.
-        // (so does "curl").
-        // const response = await fetch(dataset.url)
-        // if (!response.ok) {
-        //   console.error(response.status, response.statusText)
-        //   console.error(await response.text())
-        //   throw new Error(`Fetch failed: ${dataset.url}`)
-        // }
-        // await pipeline(response.body!, fs.createWriteStream(zipFilePath))
         fs.removeSync(zipFilePath);
         await downloadFile(dataset.url, zipFilePath);
     }
@@ -159,7 +428,7 @@ async function retrieveDataset(dataDir, dataset) {
     const sqlFilePath = path.join(dataDir, sqlFilename);
     if (options["all"] || options["unzip"]) {
         if (!options["silent"]) {
-            console.log(`Unzipping ${dataset.title}: ${zipFilename}…`);
+            console.log(`Unzipping ${dataset.title}: ${zipFilename}...`);
         }
         fs.removeSync(sqlFilePath);
         const zip = new StreamZip({
@@ -168,7 +437,7 @@ async function retrieveDataset(dataDir, dataset) {
         });
         await new Promise((resolve, reject) => {
             zip.on("ready", () => {
-                zip.extract(null, dataDir, (err, _count) => {
+                zip.extract(null, dataDir, (err) => {
                     zip.close();
                     if (err) {
                         reject(err);
@@ -181,27 +450,25 @@ async function retrieveDataset(dataDir, dataset) {
         });
         if (dataset.repairZip !== undefined) {
             if (!options["silent"]) {
-                console.log(`Repairing Zip path ${dataset.title}: ${sqlFilename}…`);
+                console.log(`Repairing Zip path ${dataset.title}: ${sqlFilename}...`);
             }
             dataset.repairZip(dataset, dataDir);
         }
     }
     if ((options["all"] || options["repairEncoding"]) && dataset.repairEncoding) {
         if (!options["silent"]) {
-            console.log(`Repairing Windows CP1252 encoding in ${dataset.title}: ${sqlFilename}…`);
+            console.log(`Repairing Windows CP1252 encoding in ${dataset.title}: ${sqlFilename}...`);
         }
         const repairedSqlFilePath = sqlFilePath + ".repaired";
         const repairedSqlWriter = fs.createWriteStream(repairedSqlFilePath, {
             encoding: "utf8",
         });
-        // Read the file as latin1 (ISO-8859-1/CP1252) and write as UTF-8
         const lineReader = readline.createInterface({
             input: fs.createReadStream(sqlFilePath, { encoding: "latin1" }),
             crlfDelay: Infinity,
         });
         for await (const line of lineReader) {
-            // Optionally repair Windows-1252 control characters
-            let repairedLine = line.replace(badWindows1252CharacterRegex, (match) => windows1252.decode(match, { mode: "fatal" }));
+            const repairedLine = line.replace(badWindows1252CharacterRegex, (match) => windows1252.decode(match, { mode: "fatal" }));
             repairedSqlWriter.write(repairedLine + "\n");
         }
         repairedSqlWriter.end();
@@ -209,103 +476,91 @@ async function retrieveDataset(dataDir, dataset) {
     }
     if (options["all"] || options["import"] || options["schema"]) {
         if (!options["silent"]) {
-            console.log(`Importing ${dataset.title}: ${sqlFilename}…`);
+            console.log(`Merging ${dataset.title}: ${sqlFilename} into ${runtime.target.name}.${senatSchemaName}...`);
         }
-        await copyToSenat(dataset, dataDir, options);
-        // Create indexes programmatically after import
-        if (dataset.indexes) {
-            for (const [table, indexes] of Object.entries(dataset.indexes)) {
-                for (const index of indexes) {
-                    const indexName = index.name;
-                    const columns = index.columns.join(", ");
-                    const schema = dataset.database;
-                    const sql = `CREATE INDEX IF NOT EXISTS ${indexName} ON ${schema}.${table} (${columns});`;
-                    try {
-                        execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -d senat -c "${sql}"`, {
-                            env: process.env,
-                            encoding: "utf-8",
-                            stdio: ["ignore", "ignore", "pipe"],
-                        });
-                        if (!options["silent"]) {
-                            console.log(`Created index: ${indexName} on ${schema}.${table} (${columns})`);
-                        }
-                    }
-                    catch (err) {
-                        console.error(`Failed to create index ${indexName} on ${schema}.${table}:`, err);
-                    }
-                }
-            }
+        await importIntoStaging(dataset, dataDir, options, runtime);
+        finalizeDatasetImport(dataset, dataDir, options, runtime);
+        if (isIncrementalImport(options)) {
+            applyStagingMetadataToTarget(dataset, dataDir, options, runtime);
         }
     }
     if (options["schema"]) {
-        let definitionsDir = path.resolve("src", "raw_types_schemats");
+        const definitionsDir = path.resolve("src", "raw_types_schemats");
         assert(fs.statSync(definitionsDir).isDirectory());
         if (!options["silent"]) {
-            console.log(`Creating TypeScript definitions from schema '${dataset.database}' in database 'senat'…`);
+            console.log(`Creating TypeScript definitions from prefixed ${senatSchemaName} tables ` +
+                `for '${dataset.database}' in database '${runtime.target.name}'...`);
         }
-        const dbConnectionString = `postgres://${process.env["PGUSER"]}:${process.env["PGPASSWORD"]}@${process.env["PGHOST"]}:${process.env["PGPORT"]}/senat`;
-        let definitionFilePath = path.join(definitionsDir, `${dataset.database}.ts`);
-        execSync(`npx schemats generate -c ${dbConnectionString} -s ${dataset.database} -o ${definitionFilePath}`, {
-            // cwd: dataDir,
+        const dbConnectionString = `postgres://${runtime.target.user}:${runtime.target.password}` +
+            `@${runtime.target.host}:${runtime.target.port}/${runtime.target.name}`;
+        const definitionFilePath = path.join(definitionsDir, `${dataset.database}.ts`);
+        const tables = listPrefixedTables(dataset, dataDir, options, runtime);
+        const tableOptions = tables.map((tableName) => `-t ${tableName}`).join(" ");
+        execSync(`npx schemats generate -c ${dbConnectionString} -s ${senatSchemaName} ${tableOptions} -o ${definitionFilePath}`, {
             env: process.env,
             encoding: "utf-8",
-            // stdio: ["ignore", "ignore", "pipe"],
         });
         const definition = fs.readFileSync(definitionFilePath, { encoding: "utf8" });
-        const definitionRepaired = definition
-            .replace(/\r\n/g, "\n")
-            .replace(/AUTO-GENERATED FILE @ \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/, "AUTO-GENERATED FILE");
+        const definitionRepaired = normalizeGeneratedDefinition(definition, dataset.database);
         fs.writeFileSync(definitionFilePath, definitionRepaired);
-        definitionsDir = path.resolve("src", "raw_types");
-        definitionFilePath = path.join(definitionsDir, `${dataset.database}.ts`);
-        execSync(`npx kysely-codegen --url '${dbConnectionString}' --default-schema ${dataset.database} --include-pattern '${dataset.database}.*' --out-file ${definitionFilePath}`, {
-            env: process.env,
-            encoding: "utf-8",
-            // stdio: ["ignore", "ignore", "pipe"],
-        });
     }
 }
+function buildRuntimeContext() {
+    const target = {
+        host: config.db.host,
+        name: config.db.name,
+        password: config.db.password,
+        port: config.db.port,
+        user: config.db.user,
+    };
+    if (!isIncrementalImport(options)) {
+        return {
+            staging: target,
+            target,
+        };
+    }
+    const staging = {
+        host: config.stagingDb.host,
+        name: config.stagingDb.name,
+        password: config.stagingDb.password,
+        port: config.stagingDb.port,
+        user: config.stagingDb.user,
+    };
+    return { staging, target };
+}
 async function retrieveOpenData() {
     const dataDir = options["dataDir"];
-    assert(dataDir, "Missing argument: data directory");
-    process.env = {
-        ...process.env,
-        PGHOST: process.env["PGHOST"] || config.db.host,
-        PGPORT: process.env["PGPORT"] || String(config.db.port),
-        PGDATABASE: process.env["PGDATABASE"] || config.db.name,
-        PGUSER: process.env["PGUSER"] || config.db.user,
-        PGPASSWORD: process.env["PGPASSWORD"] || config.db.password,
-    };
-    assert(process.env["PGHOST"] && process.env["PGPORT"] && process.env["PGUSER"] && process.env["PGPASSWORD"], "Missing database configuration: environment variables PGHOST, PGPORT, PGUSER and PGPASSWORD or TRICOTEUSES_SENAT_DB_* in .env file");
-    console.time("data extraction time");
-    // Create role 'opendata' if it does not exist
-    execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -c "CREATE ROLE opendata" || true`, {
-        cwd: dataDir,
-        env: process.env,
-        encoding: "utf-8",
-    });
-    execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -c "CREATE DATABASE senat WITH OWNER opendata" || true`, {
-        cwd: dataDir,
-        env: process.env,
-        encoding: "utf-8",
-    });
-    const enabledDatasets = getEnabledDatasets(options["categories"]);
-    const chosenDatasets = getChosenDatasets(enabledDatasets);
-    for (const dataset of chosenDatasets) {
-        await retrieveDataset(dataDir, dataset);
+    assertExistingDirectory(dataDir, "data directory");
+    assert(!options["sudo"] || !options["sudo"].startsWith("-"), "Option --sudo expects a Unix user name, for example: --sudo postgres");
+    const runtime = buildRuntimeContext();
+    assert(runtime.target.host && runtime.target.port && runtime.target.user && runtime.target.password, "Missing target database configuration: DB_* in .env file");
+    if (isIncrementalImport(options)) {
+        assert(runtime.staging.host && runtime.staging.port && runtime.staging.user && runtime.staging.password, "Missing staging database configuration: STAGING_DB_* in .env file");
+        assert(runtime.target.name !== runtime.staging.name, "Target and staging databases must be different");
     }
-    if (options["schema"]) {
-        const dbConnectionString = `postgres://${process.env["PGUSER"]}:${process.env["PGPASSWORD"]}@${process.env["PGHOST"]}:${process.env["PGPORT"]}/senat`;
-        const definitionsDir = path.resolve("src", "raw_types");
-        const definitionFilePath = path.join(definitionsDir, "senat.ts");
-        execSync(`npx kysely-codegen --url '${dbConnectionString}' --out-file ${definitionFilePath}`, {
-            env: process.env,
-            encoding: "utf-8",
-            // stdio: ["ignore", "ignore", "pipe"],
-        });
+    console.time("data extraction time");
+    try {
+        ensureDatabaseExists(runtime.target, dataDir, options);
+        if (isIncrementalImport(options)) {
+            ensureStagingDatabase(dataDir, options, runtime);
+            ensureForeignStagingServer(dataDir, options, runtime);
+        }
+        ensureSchemaVersionTable(dataDir, options, runtime);
+        const initialSchemaFingerprint = getSchemaStructureFingerprint(dataDir, options, runtime);
+        const enabledDatasets = getEnabledDatasets(options["categories"]);
+        const chosenDatasets = getChosenDatasets(enabledDatasets);
+        for (const dataset of chosenDatasets) {
+            await retrieveDataset(dataDir, dataset, options, runtime);
+        }
+        bumpSchemaVersionIfNeeded(initialSchemaFingerprint, dataDir, options, runtime);
     }
-    if (!options["silent"]) {
-        console.timeEnd("data extraction time");
+    finally {
+        if (isIncrementalImport(options)) {
+            cleanupForeignStagingServer(dataDir, options, runtime);
+        }
+        if (!options["silent"]) {
+            console.timeEnd("data extraction time");
+        }
     }
 }
 retrieveOpenData()