@tricoteuses/senat 0.3.2 → 1.0.1

package/lib/scripts/retrieve_open_data.mjs

@@ -0,0 +1,215 @@
+import assert from "assert";
+import { execSync } from "child_process";
+import commandLineArgs from "command-line-args";
+import fs from "fs-extra";
+// import fetch from "node-fetch"
+import path from "path";
+// import stream from "stream"
+import StreamZip from "node-stream-zip";
+import readline from "readline";
+// import util from "util"
+import windows1252 from "windows-1252";
+import config from "../config";
+import { datasets, EnabledDatasets } from "../datasets";
+const badWindows1252CharacterRegex = /[\u0080-\u009f]/g;
+const optionsDefinitions = [
+    {
+        alias: "a",
+        help: "all options: fetch, unzip, repair-encoding, import, schema",
+        name: "all",
+        type: Boolean,
+    },
+    {
+        alias: "c",
+        help: "create TypeScript interfaces from databases schemas into src/raw_types directory",
+        name: "schema",
+        type: Boolean,
+    },
+    {
+        alias: "e",
+        help: "repair Windows CP 1252 encoding of SQL dumps",
+        name: "repair-encoding",
+        type: Boolean,
+    },
+    {
+        alias: "f",
+        help: "fetch datasets instead of retrieving them from files",
+        name: "fetch",
+        type: Boolean,
+    },
+    {
+        alias: "i",
+        help: "import SQL dumps into a freshly (re-)created database",
+        name: "import",
+        type: Boolean,
+    },
+    {
+        alias: "s",
+        help: "don't log anything",
+        name: "silent",
+        type: Boolean,
+    },
+    {
+        alias: "z",
+        help: "unzip SQL files",
+        name: "unzip",
+        type: Boolean,
+    },
+    {
+        defaultOption: true,
+        help: "directory containing Sénat open data files",
+        name: "dataDir",
+        type: String,
+    },
+];
+const options = commandLineArgs(optionsDefinitions);
+// const pipeline = util.promisify(stream.pipeline)
+async function retrieveDataset(dataDir, dataset) {
+    const zipFilename = dataset.url.substring(dataset.url.lastIndexOf("/") + 1);
+    const zipFilePath = path.join(dataDir, zipFilename);
+    if (options.all || options.fetch) {
+        // Fetch & save ZIP file.
+        if (!options.silent) {
+            console.log(`Loading ${dataset.title}: ${zipFilename}…`);
+        }
+        // Fetch fails with OpenSSL error: dh key too small.
+        // (so does "curl").
+        // const response = await fetch(dataset.url)
+        // if (!response.ok) {
+        //   console.error(response.status, response.statusText)
+        //   console.error(await response.text())
+        //   throw new Error(`Fetch failed: ${dataset.url}`)
+        // }
+        // await pipeline(response.body!, fs.createWriteStream(zipFilePath))
+        fs.removeSync(zipFilePath);
+        execSync(`wget --quiet ${dataset.url}`, {
+            cwd: dataDir,
+            env: process.env,
+            encoding: "utf-8",
+            // stdio: ["ignore", "ignore", "pipe"],
+        });
+    }
+    const sqlFilename = `${dataset.database}.sql`;
+    const sqlFilePath = path.join(dataDir, sqlFilename);
+    if (options.all || options.unzip) {
+        if (!options.silent) {
+            console.log(`Unzipping ${dataset.title}: ${zipFilename}…`);
+        }
+        fs.removeSync(sqlFilePath);
+        const zip = new StreamZip({
+            file: zipFilePath,
+            storeEntries: true,
+        });
+        await new Promise((resolve, reject) => {
+            zip.on("ready", () => {
+                zip.extract(null, dataDir, (err, _count) => {
+                    zip.close();
+                    if (err) {
+                        reject(err);
+                    }
+                    else {
+                        resolve(null);
+                    }
+                });
+            });
+        });
+        if (dataset.repairZip !== undefined) {
+            if (!options.silent) {
+                console.log(`Repairing Zip path ${dataset.title}: ${sqlFilename}…`);
+            }
+            dataset.repairZip(dataset, dataDir);
+        }
+    }
+    if ((options.all || options["repair-encoding"]) && dataset.repairEncoding) {
+        if (!options.silent) {
+            console.log(`Repairing Windows CP1252 encoding of ${dataset.title}: ${sqlFilename}…`);
+        }
+        const repairedSqlFilePath = sqlFilePath + ".repaired";
+        const repairedSqlWriter = fs.createWriteStream(repairedSqlFilePath, {
+            encoding: "utf8",
+        });
+        const lineReader = readline.createInterface({
+            input: fs.createReadStream(sqlFilePath, { encoding: "utf8" }),
+            crlfDelay: Infinity,
+        });
+        for await (const line of lineReader) {
+            repairedSqlWriter.write(line.replace(badWindows1252CharacterRegex, (match) => windows1252.decode(match, { mode: "fatal" })) + "\n");
+        }
+        repairedSqlWriter.end();
+        await fs.move(repairedSqlFilePath, sqlFilePath, { overwrite: true });
+    }
+    if (options.all || options.import) {
+        if (!options.silent) {
+            console.log(`Importing ${dataset.title}: ${sqlFilename}…`);
+        }
+        execSync(`psql -c "DROP DATABASE IF EXISTS ${dataset.database}"`, {
+            cwd: dataDir,
+            env: process.env,
+            encoding: "utf-8",
+            // stdio: ["ignore", "ignore", "pipe"],
+        });
+        execSync(`psql -c "CREATE DATABASE ${dataset.database} WITH OWNER opendata"`, {
+            cwd: dataDir,
+            env: process.env,
+            encoding: "utf-8",
+            // stdio: ["ignore", "ignore", "pipe"],
+        });
+        execSync(`psql -f ${sqlFilename} ${dataset.database}`, {
+            cwd: dataDir,
+            env: process.env,
+            encoding: "utf-8",
+            // stdio: ["ignore", "ignore", "pipe"],
+        });
+    }
+    if (options.schema) {
+        const definitionsDir = path.resolve("src", "raw_types");
+        assert(fs.statSync(definitionsDir).isDirectory());
+        if (!options.silent) {
+            console.log(`Creating TypeScript definitions from schema of database ${dataset.database}…`);
+        }
+        const dbConnectionString = `postgres://${process.env.PGUSER}:${process.env.PGPASSWORD}@${process.env.PGHOST}:${process.env.PGPORT}/${dataset.database}`;
+        const definitionFilePath = path.join(definitionsDir, `${dataset.database}.ts`);
+        execSync(`npx schemats generate -c ${dbConnectionString} -s public -o ${definitionFilePath}`, {
+            // cwd: dataDir,
+            env: process.env,
+            encoding: "utf-8",
+            // stdio: ["ignore", "ignore", "pipe"],
+        });
+        const definition = fs.readFileSync(definitionFilePath, { encoding: "utf8" });
+        const definitionRepaired = definition
+            .replace(/\r\n/g, "\n")
+            .replace(/AUTO-GENERATED FILE @ \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/, "AUTO-GENERATED FILE");
+        fs.writeFileSync(definitionFilePath, definitionRepaired);
+    }
+}
+async function retrieveOpenData(enabledDatasets) {
+    const dataDir = options.dataDir;
+    assert(dataDir, "Missing argument: data directory");
+    process.env = {
+        ...process.env,
+        PGHOST: process.env.PGHOST || config.db.host,
+        PGPORT: process.env.PGPORT || config.db.port,
+        PGUSER: process.env.PGUSER || config.db.user,
+        PGPASSWORD: process.env.PGPASSWORD || config.db.password
+    };
+    assert(process.env.PGHOST
+        && process.env.PGPORT
+        && process.env.PGUSER
+        && process.env.PGPASSWORD, 'Missing database configuration: environment variables PGHOST, PGPORT, PGUSER and PGPASSWORD or TRICOTEUSES_SENAT_DB_* in .env file');
+    const choosenDatasets = [
+        enabledDatasets & EnabledDatasets.Ameli ? datasets.ameli : null,
+        enabledDatasets & EnabledDatasets.Debats ? datasets.debats : null,
+        enabledDatasets & EnabledDatasets.DosLeg ? datasets.dosleg : null,
+        enabledDatasets & EnabledDatasets.Questions ? datasets.questions : null,
+        enabledDatasets & EnabledDatasets.Sens ? datasets.sens : null,
+    ].filter((dataset) => dataset !== null);
+    // await Promise.all(choosenDatasets.map(dataset => retrieveDataset(dataDir, dataset)))
+    for (const dataset of choosenDatasets) {
+        await retrieveDataset(dataDir, dataset);
+    }
+    process.exit(0);
+}
+retrieveOpenData(EnabledDatasets.All).catch((error) => {
+    console.log(error);
+    process.exit(1);
+});

package/lib/scripts/retrieve_open_data.ts

@@ -0,0 +1,254 @@
+import assert from "assert"
+import { execSync } from "child_process"
+import commandLineArgs from "command-line-args"
+import fs from "fs-extra"
+// import fetch from "node-fetch"
+import path from "path"
+// import stream from "stream"
+import StreamZip from "node-stream-zip"
+import readline from "readline"
+// import util from "util"
+import windows1252 from "windows-1252"
+
+import config from "../config"
+import { datasets, Dataset, EnabledDatasets } from "../datasets"
+
+const badWindows1252CharacterRegex = /[\u0080-\u009f]/g
+const optionsDefinitions = [
+  {
+    alias: "a",
+    help: "all options: fetch, unzip, repair-encoding, import, schema",
+    name: "all",
+    type: Boolean,
+  },
+  {
+    alias: "c",
+    help:
+      "create TypeScript interfaces from databases schemas into src/raw_types directory",
+    name: "schema",
+    type: Boolean,
+  },
+  {
+    alias: "e",
+    help: "repair Windows CP 1252 encoding of SQL dumps",
+    name: "repair-encoding",
+    type: Boolean,
+  },
+  {
+    alias: "f",
+    help: "fetch datasets instead of retrieving them from files",
+    name: "fetch",
+    type: Boolean,
+  },
+  {
+    alias: "i",
+    help: "import SQL dumps into a freshly (re-)created database",
+    name: "import",
+    type: Boolean,
+  },
+  {
+    alias: "s",
+    help: "don't log anything",
+    name: "silent",
+    type: Boolean,
+  },
+  {
+    alias: "z",
+    help: "unzip SQL files",
+    name: "unzip",
+    type: Boolean,
+  },
+  {
+    defaultOption: true,
+    help: "directory containing Sénat open data files",
+    name: "dataDir",
+    type: String,
+  },
+]
+const options = commandLineArgs(optionsDefinitions)
+
+// const pipeline = util.promisify(stream.pipeline)
+
+async function retrieveDataset(
+  dataDir: string,
+  dataset: Dataset,
+): Promise<void> {
+  const zipFilename = dataset.url.substring(dataset.url.lastIndexOf("/") + 1)
+  const zipFilePath = path.join(dataDir, zipFilename)
+  if (options.all || options.fetch) {
+    // Fetch & save ZIP file.
+    if (!options.silent) {
+      console.log(`Loading ${dataset.title}: ${zipFilename}…`)
+    }
+    // Fetch fails with OpenSSL error: dh key too small.
+    // (so does "curl").
+    // const response = await fetch(dataset.url)
+    // if (!response.ok) {
+    //   console.error(response.status, response.statusText)
+    //   console.error(await response.text())
+    //   throw new Error(`Fetch failed: ${dataset.url}`)
+    // }
+    // await pipeline(response.body!, fs.createWriteStream(zipFilePath))
+    fs.removeSync(zipFilePath)
+    execSync(`wget --quiet ${dataset.url}`, {
+      cwd: dataDir,
+      env: process.env,
+      encoding: "utf-8",
+      // stdio: ["ignore", "ignore", "pipe"],
+    })
+  }
+
+  const sqlFilename = `${dataset.database}.sql`
+  const sqlFilePath = path.join(dataDir, sqlFilename)
+  if (options.all || options.unzip) {
+    if (!options.silent) {
+      console.log(`Unzipping ${dataset.title}: ${zipFilename}…`)
+    }
+    fs.removeSync(sqlFilePath)
+    const zip = new StreamZip({
+      file: zipFilePath,
+      storeEntries: true,
+    })
+    await new Promise((resolve, reject) => {
+      zip.on("ready", () => {
+        zip.extract(null, dataDir, (err?: any, _count?: number) => {
+          zip.close()
+          if (err) {
+            reject(err)
+          } else {
+            resolve(null)
+          }
+        })
+      })
+    })
+    if (dataset.repairZip !== undefined) {
+      if (!options.silent) {
+        console.log(`Repairing Zip path ${dataset.title}: ${sqlFilename}…`)
+      }
+      dataset.repairZip(dataset, dataDir)
+    }
+  }
+
+  if ((options.all || options["repair-encoding"]) && dataset.repairEncoding) {
+    if (!options.silent) {
+      console.log(
+        `Repairing Windows CP1252 encoding of ${dataset.title}: ${sqlFilename}…`,
+      )
+    }
+    const repairedSqlFilePath = sqlFilePath + ".repaired"
+    const repairedSqlWriter = fs.createWriteStream(repairedSqlFilePath, {
+      encoding: "utf8",
+    })
+    const lineReader = readline.createInterface({
+      input: fs.createReadStream(sqlFilePath, { encoding: "utf8" }),
+      crlfDelay: Infinity,
+    })
+    for await (const line of lineReader) {
+      repairedSqlWriter.write(
+        line.replace(badWindows1252CharacterRegex, (match) =>
+          windows1252.decode(match, { mode: "fatal" }),
+        ) + "\n",
+      )
+    }
+    repairedSqlWriter.end()
+    await fs.move(repairedSqlFilePath, sqlFilePath, { overwrite: true })
+  }
+
+  if (options.all || options.import) {
+    if (!options.silent) {
+      console.log(`Importing ${dataset.title}: ${sqlFilename}…`)
+    }
+    execSync(`psql -c "DROP DATABASE IF EXISTS ${dataset.database}"`, {
+      cwd: dataDir,
+      env: process.env,
+      encoding: "utf-8",
+      // stdio: ["ignore", "ignore", "pipe"],
+    })
+    execSync(
+      `psql -c "CREATE DATABASE ${dataset.database} WITH OWNER opendata"`,
+      {
+        cwd: dataDir,
+        env: process.env,
+        encoding: "utf-8",
+        // stdio: ["ignore", "ignore", "pipe"],
+      },
+    )
+    execSync(`psql -f ${sqlFilename} ${dataset.database}`, {
+      cwd: dataDir,
+      env: process.env,
+      encoding: "utf-8",
+      // stdio: ["ignore", "ignore", "pipe"],
+    })
+  }
+
+  if (options.schema) {
+    const definitionsDir = path.resolve("src", "raw_types")
+    assert(fs.statSync(definitionsDir).isDirectory())
+    if (!options.silent) {
+      console.log(
+        `Creating TypeScript definitions from schema of database ${dataset.database}…`,
+      )
+    }
+    const dbConnectionString = `postgres://${process.env.PGUSER}:${process.env.PGPASSWORD}@${process.env.PGHOST}:${process.env.PGPORT}/${dataset.database}`
+    const definitionFilePath = path.join(
+      definitionsDir,
+      `${dataset.database}.ts`,
+    )
+    execSync(
+      `npx schemats generate -c ${dbConnectionString} -s public -o ${definitionFilePath}`,
+      {
+        // cwd: dataDir,
+        env: process.env,
+        encoding: "utf-8",
+        // stdio: ["ignore", "ignore", "pipe"],
+      },
+    )
+    const definition = fs.readFileSync(definitionFilePath, { encoding: "utf8" })
+    const definitionRepaired = definition
+      .replace(/\r\n/g, "\n")
+      .replace(
+        /AUTO-GENERATED FILE @ \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/,
+        "AUTO-GENERATED FILE",
+      )
+    fs.writeFileSync(definitionFilePath, definitionRepaired)
+  }
+}
+
+async function retrieveOpenData(
+  enabledDatasets: EnabledDatasets,
+): Promise<void> {
+  const dataDir = options.dataDir
+  assert(dataDir, "Missing argument: data directory")
+
+  process.env = {
+    ...process.env,
+    PGHOST: process.env.PGHOST || config.db.host,
+    PGPORT: process.env.PGPORT || config.db.port,
+    PGUSER: process.env.PGUSER || config.db.user,
+    PGPASSWORD: process.env.PGPASSWORD || config.db.password
+  }
+  assert(process.env.PGHOST
+    && process.env.PGPORT
+    && process.env.PGUSER
+    && process.env.PGPASSWORD,
+    'Missing database configuration: environment variables PGHOST, PGPORT, PGUSER and PGPASSWORD or TRICOTEUSES_SENAT_DB_* in .env file'
+  )
+
+  const choosenDatasets: Dataset[] = [
+    enabledDatasets & EnabledDatasets.Ameli ? datasets.ameli : null,
+    enabledDatasets & EnabledDatasets.Debats ? datasets.debats : null,
+    enabledDatasets & EnabledDatasets.DosLeg ? datasets.dosleg : null,
+    enabledDatasets & EnabledDatasets.Questions ? datasets.questions : null,
+    enabledDatasets & EnabledDatasets.Sens ? datasets.sens : null,
+  ].filter((dataset: Dataset | null) => dataset !== null) as Dataset[]
+  // await Promise.all(choosenDatasets.map(dataset => retrieveDataset(dataDir, dataset)))
+  for (const dataset of choosenDatasets) {
+    await retrieveDataset(dataDir, dataset)
+  }
+  process.exit(0)
+}
+
+retrieveOpenData(EnabledDatasets.All).catch((error) => {
+  console.log(error)
+  process.exit(1)
+})

package/lib/scripts/retrieve_senateurs_photos.d.ts

@@ -0,0 +1 @@
+export {};