@tricoteuses/senat 2.9.9 → 2.9.10

package/lib/databases.d.ts

@@ -19,6 +19,7 @@ type QuestionsDb = {
 type SensDb = {
     [K in keyof SensTableTypes]: SensTableTypes[K]["select"];
 };
+export declare const dbSenat: Kysely<unknown>;
 export declare const dbAmeli: Kysely<AmeliDb>;
 export declare const dbDebats: Kysely<DebatsDb>;
 export declare const dbDosleg: Kysely<DoslegDb>;

package/lib/databases.js

@@ -9,25 +9,24 @@ import { datasets } from "./datasets";
 pg.types.setTypeParser(types.builtins.INT8, (val) => {
     return parseInt(val, 10);
 });
-export const dbAmeli = createDb(datasets.ameli.database, datasets.ameli.schema);
-export const dbDebats = createDb(datasets.debats.database, datasets.debats.schema);
-export const dbDosleg = createDb(datasets.dosleg.database, datasets.dosleg.schema);
-export const dbQuestions = createDb(datasets.questions.database, datasets.questions.schema);
-export const dbSens = createDb(datasets.sens.database, datasets.sens.schema);
-function createDb(database, schema) {
-    const dialect = new PostgresDialect({
-        pool: new pg.Pool({
-            database: database,
-            host: config.db.host,
-            user: config.db.user,
-            password: config.db.password,
-            port: config.db.port,
-            max: 10,
-        }),
-        cursor: Cursor,
-    });
-    return new Kysely({
-        log: ["error"],
-        dialect: dialect,
-    }).withSchema(schema);
-}
+const senatPool = new pg.Pool({
+    database: "senat",
+    host: config.db.host,
+    user: config.db.user,
+    password: config.db.password,
+    port: config.db.port,
+    max: 10,
+});
+const senatDialect = new PostgresDialect({
+    pool: senatPool,
+    cursor: Cursor,
+});
+export const dbSenat = new Kysely({
+    log: ["error"],
+    dialect: senatDialect,
+});
+export const dbAmeli = dbSenat.withSchema(datasets.ameli.database);
+export const dbDebats = dbSenat.withSchema(datasets.debats.database);
+export const dbDosleg = dbSenat.withSchema(datasets.dosleg.database);
+export const dbQuestions = dbSenat.withSchema(datasets.questions.database);
+export const dbSens = dbSenat.withSchema(datasets.sens.database);

package/lib/datasets.d.ts

@@ -2,7 +2,6 @@ export interface Dataset {
     database: string;
     repairEncoding: boolean;
     repairZip?: (dataset: Dataset, dataDir: string) => void;
-    schema: string;
     title: string;
     url: string;
 }

package/lib/datasets.js

@@ -16,35 +16,31 @@ export var EnabledDatasets;
 export const datasets = {
     ameli: {
         database: "ameli",
-        repairEncoding: false,
+        repairEncoding: true,
         repairZip: (dataset, dataDir) => {
             const sqlFilename = `${dataset.database}.sql`;
             const sqlFilePath = path.join(dataDir, sqlFilename);
             fs.removeSync(sqlFilePath);
             fs.moveSync(path.join(dataDir, "var", "opt", "opendata", sqlFilename), sqlFilePath);
         },
-        schema: "public",
         title: "Amendements",
         url: "https://data.senat.fr/data/ameli/ameli.zip",
     },
     debats: {
         database: "debats",
         repairEncoding: true,
-        schema: "public",
         title: "Informations relatives aux comptes rendus intégraux de la séance publique",
         url: "https://data.senat.fr/data/debats/debats.zip",
     },
     dosleg: {
         database: "dosleg",
         repairEncoding: true,
-        schema: "public",
         title: "Dossiers législatifs",
         url: "https://data.senat.fr/data/dosleg/dosleg.zip",
     },
     questions: {
         database: "questions",
         repairEncoding: true,
-        schema: "questions",
         title: "Questions écrites et orales posées par les sénateurs au Gouvernement",
         url: "https://data.senat.fr/data/questions/questions.zip",
     },
@@ -57,7 +53,6 @@ export const datasets = {
             fs.removeSync(sqlFilePath);
             fs.moveSync(path.join(dataDir, "export_sens.sql"), sqlFilePath);
         },
-        schema: "public",
         title: "Sénateurs (y compris organes et présence)",
         url: "https://data.senat.fr/data/senateurs/export_sens.zip",
     },

package/lib/scripts/retrieve_open_data.js

@@ -8,9 +8,10 @@ import StreamZip from "node-stream-zip";
 import readline from "readline";
 import windows1252 from "windows-1252";
 import { pipeline } from "stream";
+import iconv from "iconv-lite";
 import { promisify } from "util";
 import config from "../config";
-import { datasets, getChosenDatasets, getEnabledDatasets } from "../datasets";
+import { getChosenDatasets, getEnabledDatasets } from "../datasets";
 import { commonOptions } from "./shared/cli_helpers";
 const badWindows1252CharacterRegex = /[\u0080-\u009f]/g;
 const optionsDefinitions = [
@@ -67,6 +68,62 @@ async function downloadFile(url, dest) {
     }
     await streamPipeline(response.body, fs.createWriteStream(dest));
 }
+/**
+ * Copy a dataset database to the main Senat database (overwriting its contents).
+ */
+async function copyToSenat(dataset, dataDir, options) {
+    if (!options["silent"]) {
+        console.log(`Copying ${dataset.database} to Senat database...`);
+    }
+    const sqlFilePath = path.join(dataDir, `${dataset.database}.sql`);
+    const schemaDumpFile = path.join(dataDir, `${dataset.database}_schema_dump.sql`);
+    // Write the header and then stream the rest of the SQL file
+    const schemaSqlWriter = fs.createWriteStream(schemaDumpFile, { encoding: "utf8" });
+    const lineReader = readline.createInterface({
+        input: fs.createReadStream(sqlFilePath, { encoding: "utf8" }),
+        crlfDelay: Infinity,
+    });
+    for await (const line of lineReader) {
+        let newLine = line;
+        // Replace 'public' schema outside single-quoted strings
+        function replacePublicOutsideStrings(line, schema) {
+            const parts = line.split(/(')/);
+            let inString = false;
+            for (let i = 0; i < parts.length; i++) {
+                if (parts[i] === "'") {
+                    inString = !inString;
+                }
+                else if (!inString) {
+                    // Only replace outside of strings, including before comma
+                    parts[i] = parts[i].replace(/\bpublic\b(?=(\s*\.|\s*[,;]|\s|$))/g, schema);
+                }
+            }
+            return parts.join('');
+        }
+        newLine = replacePublicOutsideStrings(line, dataset.database);
+        // Convert to LATIN1, replacing unconvertible characters with '?'
+        const latin1Line = iconv.encode(newLine, 'latin1').toString('latin1');
+        schemaSqlWriter.write(latin1Line + "\n");
+    }
+    schemaSqlWriter.end();
+    await new Promise((resolve, reject) => {
+        schemaSqlWriter.on("finish", () => {
+            try {
+                execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -d senat -f ${schemaDumpFile}`, {
+                    cwd: dataDir,
+                    env: process.env,
+                    encoding: "utf-8",
+                });
+            }
+            finally {
+                try { }
+                catch { }
+            }
+            resolve();
+        });
+        schemaSqlWriter.on("error", reject);
+    });
+}
 async function retrieveDataset(dataDir, dataset) {
     const zipFilename = dataset.url.substring(dataset.url.lastIndexOf("/") + 1);
     const zipFilePath = path.join(dataDir, zipFilename);
@@ -120,7 +177,7 @@ async function retrieveDataset(dataDir, dataset) {
     }
     if ((options["all"] || options["repairEncoding"]) && dataset.repairEncoding) {
         if (!options["silent"]) {
-            console.log(`Repairing Windows CP1252 encoding of ${dataset.title}: ${sqlFilename}…`);
+            console.log(`Repairing Windows CP1252 encoding in ${dataset.title}: ${sqlFilename}…`);
         }
         const repairedSqlFilePath = sqlFilePath + ".repaired";
         const repairedSqlWriter = fs.createWriteStream(repairedSqlFilePath, {
@@ -131,7 +188,9 @@ async function retrieveDataset(dataDir, dataset) {
             crlfDelay: Infinity,
         });
         for await (const line of lineReader) {
-            repairedSqlWriter.write(line.replace(badWindows1252CharacterRegex, (match) => windows1252.decode(match, { mode: "fatal" })) + "\n");
+            // Only repair encoding
+            let repairedLine = line.replace(badWindows1252CharacterRegex, (match) => windows1252.decode(match, { mode: "fatal" }));
+            repairedSqlWriter.write(repairedLine + "\n");
         }
         repairedSqlWriter.end();
         await fs.move(repairedSqlFilePath, sqlFilePath, { overwrite: true });
@@ -140,22 +199,17 @@ async function retrieveDataset(dataDir, dataset) {
         if (!options["silent"]) {
             console.log(`Importing ${dataset.title}: ${sqlFilename}…`);
         }
-        execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -d ${dataset.database} -f ${sqlFilename}`, {
-            cwd: dataDir,
-            env: process.env,
-            encoding: "utf-8",
-            stdio: ["pipe", "ignore", "ignore"],
-        });
+        await copyToSenat(dataset, dataDir, options);
     }
     if (options["schema"]) {
         let definitionsDir = path.resolve("src", "raw_types_schemats");
         assert(fs.statSync(definitionsDir).isDirectory());
         if (!options["silent"]) {
-            console.log(`Creating TypeScript definitions from schema of database ${dataset.database}…`);
+            console.log(`Creating TypeScript definitions from schema '${dataset.database}' in database 'senat'…`);
         }
-        const dbConnectionString = `postgres://${process.env["PGUSER"]}:${process.env["PGPASSWORD"]}@${process.env["PGHOST"]}:${process.env["PGPORT"]}/${dataset.database}`;
+        const dbConnectionString = `postgres://${process.env["PGUSER"]}:${process.env["PGPASSWORD"]}@${process.env["PGHOST"]}:${process.env["PGPORT"]}/senat`;
         let definitionFilePath = path.join(definitionsDir, `${dataset.database}.ts`);
-        execSync(`npx schemats generate -c ${dbConnectionString} -s ${dataset.schema} -o ${definitionFilePath}`, {
+        execSync(`npx schemats generate -c ${dbConnectionString} -s ${dataset.database} -o ${definitionFilePath}`, {
             // cwd: dataDir,
             env: process.env,
             encoding: "utf-8",
@@ -168,7 +222,7 @@ async function retrieveDataset(dataDir, dataset) {
         fs.writeFileSync(definitionFilePath, definitionRepaired);
         definitionsDir = path.resolve("src", "raw_types");
         definitionFilePath = path.join(definitionsDir, `${dataset.database}.ts`);
-        execSync(`npx pg-to-ts generate -c '${dbConnectionString}' -s ${dataset.schema} -o ${definitionFilePath}`, {
+        execSync(`npx pg-to-ts generate -c '${dbConnectionString}' -s ${dataset.database} -o ${definitionFilePath}`, {
             env: process.env,
             encoding: "utf-8",
             // stdio: ["ignore", "ignore", "pipe"],
@@ -190,18 +244,16 @@ async function retrieveOpenData() {
         process.env["PGUSER"] &&
         process.env["PGPASSWORD"], "Missing database configuration: environment variables PGHOST, PGPORT, PGUSER and PGPASSWORD or TRICOTEUSES_SENAT_DB_* in .env file");
     console.time("data extraction time");
-    for (const [, dataset] of Object.entries(datasets)) {
-        execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -c "DROP DATABASE IF EXISTS ${dataset.database}"`, {
-            cwd: dataDir,
-            env: process.env,
-            encoding: "utf-8",
-        });
-        execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -c "CREATE DATABASE ${dataset.database} WITH OWNER opendata"`, {
-            cwd: dataDir,
-            env: process.env,
-            encoding: "utf-8",
-        });
-    }
+    execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -c "DROP DATABASE IF EXISTS senat"`, {
+        cwd: dataDir,
+        env: process.env,
+        encoding: "utf-8",
+    });
+    execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -c "CREATE DATABASE senat WITH OWNER opendata"`, {
+        cwd: dataDir,
+        env: process.env,
+        encoding: "utf-8",
+    });
     const enabledDatasets = getEnabledDatasets(options["categories"]);
     const chosenDatasets = getChosenDatasets(enabledDatasets);
     for (const dataset of chosenDatasets) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.9.9",
+  "version": "2.9.10",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",
@@ -89,6 +89,7 @@
     "@typescript-eslint/parser": "^8.13.0",
     "cross-env": "^10.0.0",
     "eslint": "^8.57.1",
+    "iconv-lite": "^0.7.0",
     "pg-to-ts": "^4.1.1",
     "prettier": "^3.5.3",
     "tslib": "^2.1.0",