@tricoteuses/senat 1.1.1 → 1.3.1

package/lib/scripts/retrieve_open_data.mjs

@@ -10,33 +10,27 @@ import readline from "readline";
 // import util from "util"
 import windows1252 from "windows-1252";
 import config from "../config";
-import { getChosenFromEnabledDatasets } from '../datasets';
+import { datasets, getChosenFromEnabledDatasets } from "../datasets";
+import { commonOptions } from "./shared/cli_helpers";
 const badWindows1252CharacterRegex = /[\u0080-\u009f]/g;
 const optionsDefinitions = [
-    {
-        alias: 'k',
-        defaultValue: ['All'],
-        help: 'categories of datasets to reorganize',
-        multiple: true,
-        name: 'categories',
-        type: String,
-    },
+    ...commonOptions,
     {
         alias: "a",
-        help: "all options: fetch, unzip, repair-encoding, import, schema",
+        help: "all options: fetch, unzip, repair-encoding, import",
         name: "all",
         type: Boolean,
     },
     {
         alias: "c",
-        help: "create TypeScript interfaces from databases schemas into src/raw_types directory",
+        help: "create TypeScript interfaces from databases schemas into src/raw_types_* directories",
         name: "schema",
         type: Boolean,
     },
     {
         alias: "e",
         help: "repair Windows CP 1252 encoding of SQL dumps",
-        name: "repair-encoding",
+        name: "repairEncoding",
         type: Boolean,
     },
     {
@@ -51,24 +45,18 @@ const optionsDefinitions = [
         name: "import",
         type: Boolean,
     },
+    {
+        alias: "S",
+        help: "sudo psql commands with given user",
+        name: "sudo",
+        type: String,
+    },
     {
         alias: "z",
         help: "unzip SQL files",
         name: "unzip",
         type: Boolean,
     },
-    {
-        alias: "s",
-        help: "don't log anything",
-        name: "silent",
-        type: Boolean,
-    },
-    {
-        defaultOption: true,
-        help: "directory containing Sénat open data files",
-        name: "dataDir",
-        type: String,
-    },
 ];
 const options = commandLineArgs(optionsDefinitions);
 // const pipeline = util.promisify(stream.pipeline)
@@ -128,7 +116,7 @@ async function retrieveDataset(dataDir, dataset) {
             dataset.repairZip(dataset, dataDir);
         }
     }
-    if ((options.all || options["repair-encoding"]) && dataset.repairEncoding) {
+    if ((options.all || options.repairEncoding) && dataset.repairEncoding) {
         if (!options.silent) {
             console.log(`Repairing Windows CP1252 encoding of ${dataset.title}: ${sqlFilename}…`);
         }
@@ -146,38 +134,26 @@ async function retrieveDataset(dataDir, dataset) {
         repairedSqlWriter.end();
         await fs.move(repairedSqlFilePath, sqlFilePath, { overwrite: true });
     }
-    if (options.all || options.import) {
+    if (options.all || options.import || options.schema) {
         if (!options.silent) {
             console.log(`Importing ${dataset.title}: ${sqlFilename}…`);
         }
-        execSync(`psql -c "DROP DATABASE IF EXISTS ${dataset.database}"`, {
-            cwd: dataDir,
-            env: process.env,
-            encoding: "utf-8",
-            // stdio: ["ignore", "ignore", "pipe"],
-        });
-        execSync(`psql -c "CREATE DATABASE ${dataset.database} WITH OWNER opendata"`, {
-            cwd: dataDir,
-            env: process.env,
-            encoding: "utf-8",
-            // stdio: ["ignore", "ignore", "pipe"],
-        });
-        execSync(`psql -f ${sqlFilename} ${dataset.database}`, {
+        execSync(`${options.sudo ? `sudo -u ${options.sudo} ` : ""}psql --quiet -d ${dataset.database} -f ${sqlFilename}`, {
             cwd: dataDir,
             env: process.env,
             encoding: "utf-8",
-            // stdio: ["ignore", "ignore", "pipe"],
+            stdio: ["pipe", "ignore", "ignore"],
         });
     }
     if (options.schema) {
-        const definitionsDir = path.resolve("src", "raw_types");
+        let definitionsDir = path.resolve("src", "raw_types_schemats");
         assert(fs.statSync(definitionsDir).isDirectory());
         if (!options.silent) {
             console.log(`Creating TypeScript definitions from schema of database ${dataset.database}…`);
         }
         const dbConnectionString = `postgres://${process.env.PGUSER}:${process.env.PGPASSWORD}@${process.env.PGHOST}:${process.env.PGPORT}/${dataset.database}`;
-        const definitionFilePath = path.join(definitionsDir, `${dataset.database}.ts`);
-        execSync(`npx schemats generate -c ${dbConnectionString} -s public -o ${definitionFilePath}`, {
+        let definitionFilePath = path.join(definitionsDir, `${dataset.database}.ts`);
+        execSync(`npx schemats generate -c ${dbConnectionString} -s ${dataset.schema} -o ${definitionFilePath}`, {
             // cwd: dataDir,
             env: process.env,
             encoding: "utf-8",
@@ -188,6 +164,14 @@ async function retrieveDataset(dataDir, dataset) {
             .replace(/\r\n/g, "\n")
             .replace(/AUTO-GENERATED FILE @ \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/, "AUTO-GENERATED FILE");
         fs.writeFileSync(definitionFilePath, definitionRepaired);
+        definitionsDir = path.resolve("src", "raw_types");
+        definitionFilePath = path.join(definitionsDir, `${dataset.database}.ts`);
+        execSync(`kysely-codegen --url ${dbConnectionString} --default-schema=${dataset.schema} --out-file=${definitionFilePath}`, {
+            // cwd: dataDir,
+            env: process.env,
+            encoding: "utf-8",
+            // stdio: ["ignore", "ignore", "pipe"],
+        });
     }
 }
 async function retrieveOpenData() {
@@ -198,17 +182,32 @@ async function retrieveOpenData() {
         PGHOST: process.env.PGHOST || config.db.host,
         PGPORT: process.env.PGPORT || config.db.port,
         PGUSER: process.env.PGUSER || config.db.user,
-        PGPASSWORD: process.env.PGPASSWORD || config.db.password
+        PGPASSWORD: process.env.PGPASSWORD || config.db.password,
     };
-    assert(process.env.PGHOST
-        && process.env.PGPORT
-        && process.env.PGUSER
-        && process.env.PGPASSWORD, 'Missing database configuration: environment variables PGHOST, PGPORT, PGUSER and PGPASSWORD or TRICOTEUSES_SENAT_DB_* in .env file');
+    assert(process.env.PGHOST &&
+        process.env.PGPORT &&
+        process.env.PGUSER &&
+        process.env.PGPASSWORD, "Missing database configuration: environment variables PGHOST, PGPORT, PGUSER and PGPASSWORD or TRICOTEUSES_SENAT_DB_* in .env file");
+    console.time("data extraction time");
+    for (const [, dataset] of Object.entries(datasets)) {
+        execSync(`${options.sudo ? `sudo -u ${options.sudo} ` : ""}psql --quiet -c "DROP DATABASE IF EXISTS ${dataset.database}"`, {
+            cwd: dataDir,
+            env: process.env,
+            encoding: "utf-8",
+        });
+        execSync(`${options.sudo ? `sudo -u ${options.sudo} ` : ""}psql --quiet -c "CREATE DATABASE ${dataset.database} WITH OWNER opendata"`, {
+            cwd: dataDir,
+            env: process.env,
+            encoding: "utf-8",
+        });
+    }
     const choosenDatasets = getChosenFromEnabledDatasets(options.categories);
-    // await Promise.all(choosenDatasets.map(dataset => retrieveDataset(dataDir, dataset)))
     for (const dataset of choosenDatasets) {
         await retrieveDataset(dataDir, dataset);
     }
+    if (!options.silent) {
+        console.timeEnd("data extraction time");
+    }
 }
 retrieveOpenData()
     .then(() => process.exit(0))

package/lib/scripts/retrieve_open_data.ts

@@ -11,38 +11,28 @@ import readline from "readline"
 import windows1252 from "windows-1252"
 
 import config from "../config"
-import {
-  Dataset,
-  getChosenFromEnabledDatasets
-} from '../datasets'
+import { Dataset, datasets, getChosenFromEnabledDatasets } from "../datasets"
+import { commonOptions } from "./shared/cli_helpers"
 
 const badWindows1252CharacterRegex = /[\u0080-\u009f]/g
 const optionsDefinitions = [
-  {
-    alias: 'k',
-    defaultValue: ['All'],
-    help: 'categories of datasets to reorganize',
-    multiple: true,
-    name: 'categories',
-    type: String,
-  },
+  ...commonOptions,
   {
     alias: "a",
-    help: "all options: fetch, unzip, repair-encoding, import, schema",
+    help: "all options: fetch, unzip, repair-encoding, import",
     name: "all",
     type: Boolean,
   },
   {
     alias: "c",
-    help:
-      "create TypeScript interfaces from databases schemas into src/raw_types directory",
+    help: "create TypeScript interfaces from databases schemas into src/raw_types_* directories",
     name: "schema",
     type: Boolean,
   },
   {
     alias: "e",
     help: "repair Windows CP 1252 encoding of SQL dumps",
-    name: "repair-encoding",
+    name: "repairEncoding",
     type: Boolean,
   },
   {
@@ -57,24 +47,18 @@ const optionsDefinitions = [
     name: "import",
     type: Boolean,
   },
+  {
+    alias: "S",
+    help: "sudo psql commands with given user",
+    name: "sudo",
+    type: String,
+  },
   {
     alias: "z",
     help: "unzip SQL files",
     name: "unzip",
     type: Boolean,
   },
-  {
-    alias: "s",
-    help: "don't log anything",
-    name: "silent",
-    type: Boolean,
-  },
-  {
-    defaultOption: true,
-    help: "directory containing Sénat open data files",
-    name: "dataDir",
-    type: String,
-  },
 ]
 const options = commandLineArgs(optionsDefinitions)
 
@@ -140,7 +124,7 @@ async function retrieveDataset(
     }
   }
 
-  if ((options.all || options["repair-encoding"]) && dataset.repairEncoding) {
+  if ((options.all || options.repairEncoding) && dataset.repairEncoding) {
     if (!options.silent) {
       console.log(
         `Repairing Windows CP1252 encoding of ${dataset.title}: ${sqlFilename}…`,
@@ -165,35 +149,25 @@ async function retrieveDataset(
     await fs.move(repairedSqlFilePath, sqlFilePath, { overwrite: true })
   }
 
-  if (options.all || options.import) {
+  if (options.all || options.import || options.schema) {
     if (!options.silent) {
       console.log(`Importing ${dataset.title}: ${sqlFilename}…`)
     }
-    execSync(`psql -c "DROP DATABASE IF EXISTS ${dataset.database}"`, {
-      cwd: dataDir,
-      env: process.env,
-      encoding: "utf-8",
-      // stdio: ["ignore", "ignore", "pipe"],
-    })
     execSync(
-      `psql -c "CREATE DATABASE ${dataset.database} WITH OWNER opendata"`,
+      `${options.sudo ? `sudo -u ${options.sudo} ` : ""}psql --quiet -d ${
+        dataset.database
+      } -f ${sqlFilename}`,
       {
         cwd: dataDir,
         env: process.env,
         encoding: "utf-8",
-        // stdio: ["ignore", "ignore", "pipe"],
+        stdio: ["pipe", "ignore", "ignore"],
       },
     )
-    execSync(`psql -f ${sqlFilename} ${dataset.database}`, {
-      cwd: dataDir,
-      env: process.env,
-      encoding: "utf-8",
-      // stdio: ["ignore", "ignore", "pipe"],
-    })
   }
 
   if (options.schema) {
-    const definitionsDir = path.resolve("src", "raw_types")
+    let definitionsDir = path.resolve("src", "raw_types_schemats")
     assert(fs.statSync(definitionsDir).isDirectory())
     if (!options.silent) {
       console.log(
@@ -201,12 +175,9 @@ async function retrieveDataset(
       )
     }
     const dbConnectionString = `postgres://${process.env.PGUSER}:${process.env.PGPASSWORD}@${process.env.PGHOST}:${process.env.PGPORT}/${dataset.database}`
-    const definitionFilePath = path.join(
-      definitionsDir,
-      `${dataset.database}.ts`,
-    )
+    let definitionFilePath = path.join(definitionsDir, `${dataset.database}.ts`)
     execSync(
-      `npx schemats generate -c ${dbConnectionString} -s public -o ${definitionFilePath}`,
+      `npx schemats generate -c ${dbConnectionString} -s ${dataset.schema} -o ${definitionFilePath}`,
       {
         // cwd: dataDir,
         env: process.env,
@@ -222,6 +193,18 @@ async function retrieveDataset(
         "AUTO-GENERATED FILE",
       )
     fs.writeFileSync(definitionFilePath, definitionRepaired)
+
+    definitionsDir = path.resolve("src", "raw_types")
+    definitionFilePath = path.join(definitionsDir, `${dataset.database}.ts`)
+    execSync(
+      `kysely-codegen --url ${dbConnectionString} --default-schema=${dataset.schema} --out-file=${definitionFilePath}`,
+      {
+        // cwd: dataDir,
+        env: process.env,
+        encoding: "utf-8",
+        // stdio: ["ignore", "ignore", "pipe"],
+      },
+    )
   }
 }
 
@@ -234,20 +217,53 @@ async function retrieveOpenData(): Promise<void> {
     PGHOST: process.env.PGHOST || config.db.host,
     PGPORT: process.env.PGPORT || config.db.port,
     PGUSER: process.env.PGUSER || config.db.user,
-    PGPASSWORD: process.env.PGPASSWORD || config.db.password
+    PGPASSWORD: process.env.PGPASSWORD || config.db.password,
   }
-  assert(process.env.PGHOST
-    && process.env.PGPORT
-    && process.env.PGUSER
-    && process.env.PGPASSWORD,
-    'Missing database configuration: environment variables PGHOST, PGPORT, PGUSER and PGPASSWORD or TRICOTEUSES_SENAT_DB_* in .env file'
+  assert(
+    process.env.PGHOST &&
+      process.env.PGPORT &&
+      process.env.PGUSER &&
+      process.env.PGPASSWORD,
+    "Missing database configuration: environment variables PGHOST, PGPORT, PGUSER and PGPASSWORD or TRICOTEUSES_SENAT_DB_* in .env file",
   )
 
-  const choosenDatasets: Dataset[] = getChosenFromEnabledDatasets(options.categories)
-  // await Promise.all(choosenDatasets.map(dataset => retrieveDataset(dataDir, dataset)))
+  console.time("data extraction time")
+
+  for (const [, dataset] of Object.entries(datasets)) {
+    execSync(
+      `${
+        options.sudo ? `sudo -u ${options.sudo} ` : ""
+      }psql --quiet -c "DROP DATABASE IF EXISTS ${dataset.database}"`,
+      {
+        cwd: dataDir,
+        env: process.env,
+        encoding: "utf-8",
+      },
+    )
+    execSync(
+      `${
+        options.sudo ? `sudo -u ${options.sudo} ` : ""
+      }psql --quiet -c "CREATE DATABASE ${
+        dataset.database
+      } WITH OWNER opendata"`,
+      {
+        cwd: dataDir,
+        env: process.env,
+        encoding: "utf-8",
+      },
+    )
+  }
+
+  const choosenDatasets: Dataset[] = getChosenFromEnabledDatasets(
+    options.categories,
+  )
   for (const dataset of choosenDatasets) {
     await retrieveDataset(dataDir, dataset)
   }
+
+  if (!options.silent) {
+    console.timeEnd("data extraction time")
+  }
 }
 
 retrieveOpenData()