@tricoteuses/senat 2.9.8 → 2.9.10

package/lib/raw_types_schemats/ameli.d.ts

@@ -1,7 +1,7 @@
 /**
  * AUTO-GENERATED FILE - DO NOT EDIT!
  *
- * This file was automatically generated by schemats v.2.8.1
+ * This file was automatically generated by schemats v.2.9.8
  * $ schemats generate -c postgres://username:password@localhost:5433/ameli -t amd -t amdsen -t avicom -t avigvt -t cab -t com_ameli -t ent -t etatxt -t fbu -t grppol_ameli -t gvt -t intora -t irr -t lec_ameli -t mot -t nat -t orarol -t sai -t saisen -t sea -t sen_ameli -t ses -t sor -t sub -t txt_ameli -t typrect -t typses -t typsub -t w_nivrec -s public
  *
  */

package/lib/raw_types_schemats/debats.d.ts

@@ -1,7 +1,7 @@
 /**
  * AUTO-GENERATED FILE - DO NOT EDIT!
  *
- * This file was automatically generated by schemats v.2.8.1
+ * This file was automatically generated by schemats v.2.9.8
  * $ schemats generate -c postgres://username:password@localhost:5433/debats -t debats -t intdivers -t intpjl -t lecassdeb -t secdis -t secdivers -t syndeb -t typsec -s public
  *
  */

package/lib/raw_types_schemats/dosleg.d.ts

@@ -1,7 +1,7 @@
 /**
  * AUTO-GENERATED FILE - DO NOT EDIT!
  *
- * This file was automatically generated by schemats v.2.8.1
+ * This file was automatically generated by schemats v.2.9.8
  * $ schemats generate -c postgres://username:password@localhost:5433/dosleg -t amescr -t ass -t aud -t auteur -t ble -t catrap -t corscr -t date_seance -t deccoc -t denrap -t doc -t docatt -t docsea -t ecr -t etaloi -t evtsea -t forpub -t gen -t lecass -t lecassrap -t lecture -t lnkrap -t loi -t loithe -t natloi -t org -t orgnomhis -t orippr -t oritxt -t posvot -t qua -t rap -t raporg -t rapthe -t rolsig -t scr -t ses -t stavot -t texte -t texte_ancien -t the -t titsen -t typatt -t typaut -t typdoc -t typevtsea -t typlec -t typloi -t typorg -t typrap -t typtxt -t typurl -t votsen -s public
  *
  */

package/lib/raw_types_schemats/questions.d.ts

@@ -1,7 +1,7 @@
 /**
  * AUTO-GENERATED FILE - DO NOT EDIT!
  *
- * This file was automatically generated by schemats v.2.8.1
+ * This file was automatically generated by schemats v.2.9.8
  * $ schemats generate -c postgres://username:password@localhost:5433/questions -t etatquestion -t legquestion -t naturequestion -t sortquestion -t tam_ministeres -t tam_questions -t tam_reponses -t the -s questions
  *
  */

package/lib/raw_types_schemats/sens.d.ts

@@ -1,7 +1,7 @@
 /**
  * AUTO-GENERATED FILE - DO NOT EDIT!
  *
- * This file was automatically generated by schemats v.2.8.1
+ * This file was automatically generated by schemats v.2.9.8
  * $ schemats generate -c postgres://username:password@localhost:5433/sens -t acr -t activite -t activite_audit -t activite_delegation -t activite_delegation_audit -t activite_loi -t activite_loi_audit -t activite_obligatoire -t activite_participant -t activite_participant_audit -t activite_senateur -t activite_senateur_audit -t activite_senateur_params -t activite_senateur_params_audit -t activites_liees -t activites_liees_audit -t actpro -t adhgrpsen -t adr -t adresse -t adrsen -t app -t assparint -t asster -t autgrpsen -t autorisation_profil -t autorisations -t avis_nomination_art13 -t basdes -t bur -t bur3r -t bur4r -t cad -t candid -t candidat -t candtodelete -t categorie_activite -t catpro -t catpro2e -t catterrit -t cible_categorie_periode -t cirdep -t com -t con -t cotgip -t csp -t cspfam -t databasechangelog -t databasechangeloglock -t delega -t derogation -t derogation_audit -t derogation_senateur -t derogation_senateur_audit -t design -t designoep -t designorg -t discou -t div -t dpt -t dpt_seuil_presence -t dptele -t dptele_files -t dptele_processing -t dptele_processing_type -t dpttypman -t droits_acces -t droits_acces_audit -t droits_type_derogation -t ele -t eleloc -t elucan -t eludep -t eludiv -t elueur -t elueur_apf -t elumet -t elureg -t elusen -t elusen2e -t elusen3r -t elusen4r -t elusencommu -t elusenpair -t eluter -t elutit -t eluvil -t etadebman -t etadebman3r -t etadebman4r -t etafinman -t etafinman3r -t etafinman4r -t etaprr -t etarpm -t etasen -t ext2e_bio -t ext2e_csp -t ext2e_mandats -t ext2e_minist -t extsencom_identite -t extsencom_mandat -t fonact_participant -t foncandid -t foncom -t fondelega -t fongrppol -t fongrpsen -t fonmemcom -t fonmemdelega -t fonmemextpar -t fonmemgrppol -t fonmemgrpsen -t fonmemorg -t fonorg -t grppol -t grppol4r -t grpsenami -t grpsenamiadh -t grpsenamiadhreq -t grpsenamiadhreqeta -t grpsenamiunadh -t grpsim -t gvt -t insee_pays2008 -t jhi_authority -t jhi_user -t jhi_user_authority -t lanetr -t libcom -t libdelega -t libgrppol -t libgrpsen -t liborg -t lisdptele -t mel -t memcom -t memcomsea -t memdelega -t memextpar -t memgrppol -t memgrpsen -t memorg -t met -t minind -t minist -t mis -t misetafin -t mismin -t misrapeta -t missen -t moddes -t mode_acces_elusenpair -t nation -t nationgrpsen -t nivlan -t org -t orgext -t orgextpres -t orgthe -t pairie_elusenpair -t parpol -t parpolglo -t participa -t pcs -t pcs24 -t pcs42 -t pcs8 -t pcscatpro -t per -t per_sen -t perapp -t periode_presence -t perpolglo -t perrol -t pj_justificatif -t pj_justificatif_audit -t plaind -t plan_table -t plsql_profiler_runs -t plsql_profiler_units -t poicon -t posvot -t presences_scrutin_surcharge -t presencesrevisionentity -t profil_applicatif -t qua -t rap_the -t reg -t reladr -t requetes_profil -t reslis -t resultat -t reu -t revchanges -t rne_mandat -t rne_mandat_diff -t rne_sen -t rne_sen_diff -t rne_type_mandat -t rol -t sal -t scr -t scrusoldelega -t sea -t sec -t sec2e -t secexe -t sen -t senbur -t senbur3r -t senbur4r -t sennom -t senpj -t sensim -t sentablenom -t senurl -t seuil_presence -t sirpas_elusen -t sirpas_fonmemcom -t sirpas_fonmemdelega -t sirpas_fonmemgrppol -t sirpas_memcom -t sirpas_memdelega -t sirpas_memgrppol -t sirpas_mvt -t sirpas_mvtcm -t sirpas_mvttri -t sirpas_sen -t sirpas_senbur -t sirpas_trf -t srv -t stajur -t stavot -t suspensiontravaux -t suspensiontravaux_audit -t sysage -t syscognos -t sysevt -t sysvar -t sysvar_sendev -t sysvar_senprod -t tapsenrevchanges -t tapsenrevisionentity -t telephone -t temval -t tenpol -t territ -t testoracle -t titele -t titelerne -t titmin -t titnob -t tmpsd -t toutes -t turelu -t typadr -t typapppol -t typbister -t typcandid -t type_activite -t type_activite_participant -t type_activite_rol -t type_activite_senateur -t type_categorie -t type_derogation -t type_droit_acces -t type_pj_justificatif -t type_rne_diff -t type_type_derogation -t typele -t typgrpsen -t typman -t typmin -t typmoddes -t typorg -t typorgext -t typparpol -t typpoicon -t typprs -t typprssta -t typscr -t typtel -t typurl -t typvoi -t uploaded_file -t uploaded_file_type -t validation -t validation_defview_profil -t validation_profil -t vercand -t verres -t votes -t zongeo -s public
  *
  */

package/lib/scripts/convert_data.js

@@ -108,13 +108,13 @@ async function convertDatasetDosLeg(dataDir) {
     ensureAndClearDir(dossiersReorganizedDir);
     for await (const loi of findAllDossiers()) {
         if (options["verbose"]) {
-            console.log(`Converting ${loi.signet} file…`);
+            console.log(`Converting ${loi["signet"]} file…`);
         }
         let loiReorganizedDir = path.join(dossiersReorganizedDir, String(UNDEFINED_SESSION));
-        const session = getSessionFromSignet(loi.signet) || UNDEFINED_SESSION;
+        const session = getSessionFromSignet(loi["signet"]) || UNDEFINED_SESSION;
         loiReorganizedDir = path.join(dossiersReorganizedDir, String(session));
         fs.ensureDirSync(loiReorganizedDir);
-        const scrutinFileName = `${loi.signet}.json`;
+        const scrutinFileName = `${loi["signet"]}.json`;
         fs.writeJSONSync(path.join(loiReorganizedDir, scrutinFileName), loi, {
             spaces: 2,
         });

package/lib/scripts/retrieve_open_data.js

@@ -8,9 +8,10 @@ import StreamZip from "node-stream-zip";
 import readline from "readline";
 import windows1252 from "windows-1252";
 import { pipeline } from "stream";
+import iconv from "iconv-lite";
 import { promisify } from "util";
 import config from "../config";
-import { datasets, getChosenDatasets, getEnabledDatasets } from "../datasets";
+import { getChosenDatasets, getEnabledDatasets } from "../datasets";
 import { commonOptions } from "./shared/cli_helpers";
 const badWindows1252CharacterRegex = /[\u0080-\u009f]/g;
 const optionsDefinitions = [
@@ -67,6 +68,62 @@ async function downloadFile(url, dest) {
     }
     await streamPipeline(response.body, fs.createWriteStream(dest));
 }
+/**
+ * Copy a dataset database to the main Senat database (overwriting its contents).
+ */
+async function copyToSenat(dataset, dataDir, options) {
+    if (!options["silent"]) {
+        console.log(`Copying ${dataset.database} to Senat database...`);
+    }
+    const sqlFilePath = path.join(dataDir, `${dataset.database}.sql`);
+    const schemaDumpFile = path.join(dataDir, `${dataset.database}_schema_dump.sql`);
+    // Write the header and then stream the rest of the SQL file
+    const schemaSqlWriter = fs.createWriteStream(schemaDumpFile, { encoding: "utf8" });
+    const lineReader = readline.createInterface({
+        input: fs.createReadStream(sqlFilePath, { encoding: "utf8" }),
+        crlfDelay: Infinity,
+    });
+    for await (const line of lineReader) {
+        let newLine = line;
+        // Replace 'public' schema outside single-quoted strings
+        function replacePublicOutsideStrings(line, schema) {
+            const parts = line.split(/(')/);
+            let inString = false;
+            for (let i = 0; i < parts.length; i++) {
+                if (parts[i] === "'") {
+                    inString = !inString;
+                }
+                else if (!inString) {
+                    // Only replace outside of strings, including before comma
+                    parts[i] = parts[i].replace(/\bpublic\b(?=(\s*\.|\s*[,;]|\s|$))/g, schema);
+                }
+            }
+            return parts.join('');
+        }
+        newLine = replacePublicOutsideStrings(line, dataset.database);
+        // Convert to LATIN1, replacing unconvertible characters with '?'
+        const latin1Line = iconv.encode(newLine, 'latin1').toString('latin1');
+        schemaSqlWriter.write(latin1Line + "\n");
+    }
+    schemaSqlWriter.end();
+    await new Promise((resolve, reject) => {
+        schemaSqlWriter.on("finish", () => {
+            try {
+                execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -d senat -f ${schemaDumpFile}`, {
+                    cwd: dataDir,
+                    env: process.env,
+                    encoding: "utf-8",
+                });
+            }
+            finally {
+                try { }
+                catch { }
+            }
+            resolve();
+        });
+        schemaSqlWriter.on("error", reject);
+    });
+}
 async function retrieveDataset(dataDir, dataset) {
     const zipFilename = dataset.url.substring(dataset.url.lastIndexOf("/") + 1);
     const zipFilePath = path.join(dataDir, zipFilename);
@@ -120,7 +177,7 @@ async function retrieveDataset(dataDir, dataset) {
     }
     if ((options["all"] || options["repairEncoding"]) && dataset.repairEncoding) {
         if (!options["silent"]) {
-            console.log(`Repairing Windows CP1252 encoding of ${dataset.title}: ${sqlFilename}…`);
+            console.log(`Repairing Windows CP1252 encoding in ${dataset.title}: ${sqlFilename}…`);
         }
         const repairedSqlFilePath = sqlFilePath + ".repaired";
         const repairedSqlWriter = fs.createWriteStream(repairedSqlFilePath, {
@@ -131,7 +188,9 @@ async function retrieveDataset(dataDir, dataset) {
             crlfDelay: Infinity,
         });
         for await (const line of lineReader) {
-            repairedSqlWriter.write(line.replace(badWindows1252CharacterRegex, (match) => windows1252.decode(match, { mode: "fatal" })) + "\n");
+            // Only repair encoding
+            let repairedLine = line.replace(badWindows1252CharacterRegex, (match) => windows1252.decode(match, { mode: "fatal" }));
+            repairedSqlWriter.write(repairedLine + "\n");
         }
         repairedSqlWriter.end();
         await fs.move(repairedSqlFilePath, sqlFilePath, { overwrite: true });
@@ -140,22 +199,17 @@ async function retrieveDataset(dataDir, dataset) {
         if (!options["silent"]) {
             console.log(`Importing ${dataset.title}: ${sqlFilename}…`);
         }
-        execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -d ${dataset.database} -f ${sqlFilename}`, {
-            cwd: dataDir,
-            env: process.env,
-            encoding: "utf-8",
-            stdio: ["pipe", "ignore", "ignore"],
-        });
+        await copyToSenat(dataset, dataDir, options);
     }
     if (options["schema"]) {
         let definitionsDir = path.resolve("src", "raw_types_schemats");
         assert(fs.statSync(definitionsDir).isDirectory());
         if (!options["silent"]) {
-            console.log(`Creating TypeScript definitions from schema of database ${dataset.database}…`);
+            console.log(`Creating TypeScript definitions from schema '${dataset.database}' in database 'senat'…`);
         }
-        const dbConnectionString = `postgres://${process.env["PGUSER"]}:${process.env["PGPASSWORD"]}@${process.env["PGHOST"]}:${process.env["PGPORT"]}/${dataset.database}`;
+        const dbConnectionString = `postgres://${process.env["PGUSER"]}:${process.env["PGPASSWORD"]}@${process.env["PGHOST"]}:${process.env["PGPORT"]}/senat`;
         let definitionFilePath = path.join(definitionsDir, `${dataset.database}.ts`);
-        execSync(`npx schemats generate -c ${dbConnectionString} -s ${dataset.schema} -o ${definitionFilePath}`, {
+        execSync(`npx schemats generate -c ${dbConnectionString} -s ${dataset.database} -o ${definitionFilePath}`, {
             // cwd: dataDir,
             env: process.env,
             encoding: "utf-8",
@@ -168,8 +222,7 @@ async function retrieveDataset(dataDir, dataset) {
         fs.writeFileSync(definitionFilePath, definitionRepaired);
         definitionsDir = path.resolve("src", "raw_types");
         definitionFilePath = path.join(definitionsDir, `${dataset.database}.ts`);
-        execSync(`kysely-codegen --url ${dbConnectionString} --default-schema=${dataset.schema} --out-file=${definitionFilePath}`, {
-            // cwd: dataDir,
+        execSync(`npx pg-to-ts generate -c '${dbConnectionString}' -s ${dataset.database} -o ${definitionFilePath}`, {
             env: process.env,
             encoding: "utf-8",
             // stdio: ["ignore", "ignore", "pipe"],
@@ -191,18 +244,16 @@ async function retrieveOpenData() {
         process.env["PGUSER"] &&
         process.env["PGPASSWORD"], "Missing database configuration: environment variables PGHOST, PGPORT, PGUSER and PGPASSWORD or TRICOTEUSES_SENAT_DB_* in .env file");
     console.time("data extraction time");
-    for (const [, dataset] of Object.entries(datasets)) {
-        execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -c "DROP DATABASE IF EXISTS ${dataset.database}"`, {
-            cwd: dataDir,
-            env: process.env,
-            encoding: "utf-8",
-        });
-        execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -c "CREATE DATABASE ${dataset.database} WITH OWNER opendata"`, {
-            cwd: dataDir,
-            env: process.env,
-            encoding: "utf-8",
-        });
-    }
+    execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -c "DROP DATABASE IF EXISTS senat"`, {
+        cwd: dataDir,
+        env: process.env,
+        encoding: "utf-8",
+    });
+    execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -c "CREATE DATABASE senat WITH OWNER opendata"`, {
+        cwd: dataDir,
+        env: process.env,
+        encoding: "utf-8",
+    });
     const enabledDatasets = getEnabledDatasets(options["categories"]);
     const chosenDatasets = getChosenDatasets(enabledDatasets);
     for (const dataset of chosenDatasets) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.9.8",
+  "version": "2.9.10",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",
@@ -89,7 +89,8 @@
     "@typescript-eslint/parser": "^8.13.0",
     "cross-env": "^10.0.0",
     "eslint": "^8.57.1",
-    "kysely-codegen": "^0.18.0",
+    "iconv-lite": "^0.7.0",
+    "pg-to-ts": "^4.1.1",
     "prettier": "^3.5.3",
     "tslib": "^2.1.0",
     "typescript": "^5.8.3"