@tricoteuses/senat 2.9.0 → 2.9.5

package/lib/scripts/retrieve_open_data.ts

@@ -1,261 +0,0 @@
-import assert from "assert"
-import { execSync } from "child_process"
-import commandLineArgs from "command-line-args"
-import fs from "fs-extra"
-// import fetch from "node-fetch"
-import path from "path"
-// import stream from "stream"
-import StreamZip from "node-stream-zip"
-import readline from "readline"
-// import util from "util"
-import windows1252 from "windows-1252"
-
-import config from "../config"
-import {
-  Dataset,
-  getChosenFromEnabledDatasets,
-} from "../datasets"
-import { commonOptions } from "./shared/cli_helpers"
-
-const badWindows1252CharacterRegex = /[\u0080-\u009f]/g
-const optionsDefinitions = [
-  ...commonOptions,
-  {
-    alias: "a",
-    help: "all options: fetch, unzip, repair-encoding, import",
-    name: "all",
-    type: Boolean,
-  },
-  {
-    alias: "f",
-    help: "fetch datasets instead of retrieving them from files",
-    name: "fetch",
-    type: Boolean,
-  },
-  {
-    alias: "z",
-    help: "unzip SQL files",
-    name: "unzip",
-    type: Boolean,
-  },
-  {
-    alias: "e",
-    help: "repair Windows CP 1252 encoding of SQL dumps",
-    name: "repairEncoding",
-    type: Boolean,
-  },
-  {
-    alias: "i",
-    help: "import SQL dumps into a freshly (re-)created database",
-    name: "import",
-    type: Boolean,
-  },
-  {
-    alias: "d",
-    help: "repair database (update schema and types)",
-    name: "repairDatabase",
-    type: Boolean,
-  },
-  {
-    alias: "c",
-    help:
-      "create TypeScript interfaces from databases schemas into src/raw_types_* directories",
-    name: "schema",
-    type: Boolean,
-  },
-]
-const options = commandLineArgs(optionsDefinitions)
-
-// const pipeline = util.promisify(stream.pipeline)
-
-async function retrieveDataset(
-  dataDir: string,
-  dataset: Dataset,
-): Promise<void> {
-  const zipFilename = dataset.url.substring(dataset.url.lastIndexOf("/") + 1)
-  const zipFilePath = path.join(dataDir, zipFilename)
-  if (options.all || options.fetch) {
-    // Fetch & save ZIP file.
-    if (!options.silent) {
-      console.log(`Loading ${dataset.title}: ${zipFilename}…`)
-    }
-    // Fetch fails with OpenSSL error: dh key too small.
-    // (so does "curl").
-    // const response = await fetch(dataset.url)
-    // if (!response.ok) {
-    //   console.error(response.status, response.statusText)
-    //   console.error(await response.text())
-    //   throw new Error(`Fetch failed: ${dataset.url}`)
-    // }
-    // await pipeline(response.body!, fs.createWriteStream(zipFilePath))
-    fs.removeSync(zipFilePath)
-    execSync(`wget --quiet ${dataset.url}`, {
-      cwd: dataDir,
-      env: process.env,
-      encoding: "utf-8",
-      // stdio: ["ignore", "ignore", "pipe"],
-    })
-  }
-
-  const sqlFilename = `${dataset.database}.sql`
-  const sqlFilePath = path.join(dataDir, sqlFilename)
-  if (options.all || options.unzip) {
-    if (!options.silent) {
-      console.log(`Unzipping ${dataset.title}: ${zipFilename}…`)
-    }
-    fs.removeSync(sqlFilePath)
-    const zip = new StreamZip({
-      file: zipFilePath,
-      storeEntries: true,
-    })
-    await new Promise((resolve, reject) => {
-      zip.on("ready", () => {
-        zip.extract(null, dataDir, (err?: any, _count?: number) => {
-          zip.close()
-          if (err) {
-            reject(err)
-          } else {
-            resolve(null)
-          }
-        })
-      })
-    })
-    if (dataset.repairZip !== undefined) {
-      if (!options.silent) {
-        console.log(`Repairing Zip path ${dataset.title}: ${sqlFilename}…`)
-      }
-      dataset.repairZip(dataset, dataDir)
-    }
-  }
-
-  if ((options.all || options.repairEncoding) && dataset.repairEncoding) {
-    if (!options.silent) {
-      console.log(
-        `Repairing Windows CP1252 encoding of ${dataset.title}: ${sqlFilename}…`,
-      )
-    }
-    const repairedSqlFilePath = sqlFilePath + ".repaired"
-    const repairedSqlWriter = fs.createWriteStream(repairedSqlFilePath, {
-      encoding: "utf8",
-    })
-    const lineReader = readline.createInterface({
-      input: fs.createReadStream(sqlFilePath, { encoding: "utf8" }),
-      crlfDelay: Infinity,
-    })
-    for await (const line of lineReader) {
-      repairedSqlWriter.write(
-        line.replace(badWindows1252CharacterRegex, (match) =>
-          windows1252.decode(match, { mode: "fatal" }),
-        ) + "\n",
-      )
-    }
-    repairedSqlWriter.end()
-    await fs.move(repairedSqlFilePath, sqlFilePath, { overwrite: true })
-  }
-
-  if (options.all || options.import) {
-    if (!options.silent) {
-      console.log(`Importing ${dataset.title}: ${sqlFilename}…`)
-    }
-    execSync(`psql -c "DROP DATABASE IF EXISTS ${dataset.database}"`, {
-      cwd: dataDir,
-      env: process.env,
-      encoding: "utf-8",
-      // stdio: ["ignore", "ignore", "pipe"],
-    })
-    execSync(
-      `psql -c "CREATE DATABASE ${dataset.database} WITH OWNER opendata"`,
-      {
-        cwd: dataDir,
-        env: process.env,
-        encoding: "utf-8",
-        // stdio: ["ignore", "ignore", "pipe"],
-      },
-    )
-    execSync(`psql -f ${sqlFilename} ${dataset.database}`, {
-      cwd: dataDir,
-      env: process.env,
-      encoding: "utf-8",
-      // stdio: ["ignore", "ignore", "pipe"],
-    })
-  }
-
-  if (options.schema) {
-    let definitionsDir = path.resolve("src", "raw_types_schemats")
-    assert(fs.statSync(definitionsDir).isDirectory())
-    if (!options.silent) {
-      console.log(
-        `Creating TypeScript definitions from schema of database ${dataset.database}…`,
-      )
-    }
-    const dbConnectionString = `postgres://${process.env.PGUSER}:${process.env.PGPASSWORD}@${process.env.PGHOST}:${process.env.PGPORT}/${dataset.database}`
-    let definitionFilePath = path.join(
-      definitionsDir,
-      `${dataset.database}.ts`,
-    )
-    execSync(
-      `npx schemats generate -c ${dbConnectionString} -s ${dataset.schema} -o ${definitionFilePath}`,
-      {
-        // cwd: dataDir,
-        env: process.env,
-        encoding: "utf-8",
-        // stdio: ["ignore", "ignore", "pipe"],
-      },
-    )
-    const definition = fs.readFileSync(definitionFilePath, { encoding: "utf8" })
-    const definitionRepaired = definition
-      .replace(/\r\n/g, "\n")
-      .replace(
-        /AUTO-GENERATED FILE @ \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/,
-        "AUTO-GENERATED FILE",
-      )
-    fs.writeFileSync(definitionFilePath, definitionRepaired)
-
-    definitionsDir = path.resolve("src", "raw_types_kysely")
-    definitionFilePath = path.join(
-      definitionsDir,
-      `${dataset.database}.ts`,
-    )
-    execSync(
-      `kysely-codegen --url ${dbConnectionString} --schema=${dataset.schema} --out-file=${definitionFilePath}`,
-      {
-        // cwd: dataDir,
-        env: process.env,
-        encoding: "utf-8",
-        // stdio: ["ignore", "ignore", "pipe"],
-      },
-    )
-  }
-}
-
-async function retrieveOpenData(): Promise<void> {
-  const dataDir = options.dataDir
-  assert(dataDir, "Missing argument: data directory")
-
-  process.env = {
-    ...process.env,
-    PGHOST: process.env.PGHOST || config.db.host,
-    PGPORT: process.env.PGPORT || config.db.port,
-    PGUSER: process.env.PGUSER || config.db.user,
-    PGPASSWORD: process.env.PGPASSWORD || config.db.password,
-  }
-  assert(process.env.PGHOST
-    && process.env.PGPORT
-    && process.env.PGUSER
-    && process.env.PGPASSWORD,
-    "Missing database configuration: environment variables PGHOST, PGPORT, PGUSER and PGPASSWORD or TRICOTEUSES_SENAT_DB_* in .env file"
-  )
-
-  const choosenDatasets: Dataset[] = getChosenFromEnabledDatasets(options.categories)
-  // await Promise.all(choosenDatasets.map(dataset => retrieveDataset(dataDir, dataset)))
-  for (const dataset of choosenDatasets) {
-    await retrieveDataset(dataDir, dataset)
-  }
-}
-
-retrieveOpenData()
-  .then(() => process.exit(0))
-  .catch((error) => {
-    console.log(error)
-    process.exit(1)
-  })

package/lib/scripts/retrieve_senateurs_photos.mjs

@@ -1,147 +0,0 @@
-import assert from "assert";
-import { execSync } from "child_process";
-import commandLineArgs from "command-line-args";
-import fs from "fs-extra";
-// import fetch from "node-fetch"
-import path from "path";
-// import stream from "stream"
-// import util from "util"
-import { findActif as findActifSenateurs } from "../model/sens";
-import { slugify } from "../strings";
-import { commonOptions } from "./shared/cli_helpers";
-const optionsDefinitions = [
-    ...commonOptions,
-    {
-        alias: "f",
-        help: "fetch sénateurs' pictures instead of retrieving them from files",
-        name: "fetch",
-        type: Boolean,
-    },
-];
-const options = commandLineArgs(optionsDefinitions);
-// const pipeline = util.promisify(stream.pipeline)
-async function retrievePhotosSenateurs() {
-    const dataDir = options.dataDir;
-    assert(dataDir, "Missing argument: data directory");
-    const photosDir = path.join(dataDir, "photos_senateurs");
-    const missingPhotoFilePath = path.resolve(__dirname, "images", "transparent_155x225.jpg");
-    const sens = await Array.fromAsync(findActifSenateurs());
-    // Download photos.
-    fs.ensureDirSync(photosDir);
-    if (options.fetch) {
-        for (const sen of sens) {
-            const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(sen.senprenomuse, "_")}${slugify(sen.senmat, "_")}`;
-            const photoFilename = photoStem + ".jpg";
-            const photoFilePath = path.join(photosDir, photoFilename);
-            const photoTempFilename = photoStem + "_temp.jpg";
-            const photoTempFilePath = path.join(photosDir, photoTempFilename);
-            const urlPhoto = `https://www.senat.fr/senimg/${photoFilename}`;
-            if (!options.silent) {
-                console.log(`Loading photo ${urlPhoto} for ${sen.senprenomuse} ${sen.sennomuse}…`);
-            }
-            // Fetch fails with OpenSSL error: dh key too small.
-            // (so does "curl").
-            // for (let retries = 0; retries < 3; retries++) {
-            //   const response = await fetch(urlPhoto)
-            //   if (response.ok) {
-            //     await pipeline(response.body, fs.createWriteStream(photoTempFilePath))
-            //     fs.renameSync(photoTempFilePath, photoFilePath)
-            //     break
-            //   }
-            //   if (retries >= 2) {
-            //     console.warn(`Fetch failed: ${urlPhoto} (${sen.senprenomuse} ${sen.sennomuse})`)
-            //     console.warn(response.status, response.statusText)
-            //     console.warn(await response.text())
-            //     if (fs.existsSync(photoFilePath)) {
-            //       console.warn("  => Reusing existing image")
-            //     } else {
-            //       console.warn("  => Using blank image")
-            //       fs.copyFileSync(missingPhotoFilePath, photoFilePath)
-            //     }
-            //     break
-            //   }
-            // }
-            try {
-                execSync(`wget --quiet -O ${photoTempFilename} ${urlPhoto}`, {
-                    cwd: photosDir,
-                    env: process.env,
-                    encoding: "utf-8",
-                    // stdio: ["ignore", "ignore", "pipe"],
-                });
-                fs.renameSync(photoTempFilePath, photoFilePath);
-            }
-            catch (error) {
-                if (typeof error === "object" && error && "status" in error && error.status === 8) {
-                    console.error(`Unable to load photo for ${sen.senprenomuse} ${sen.sennomuse}`);
-                    continue;
-                }
-                throw error;
-            }
-        }
-    }
-    // Resize photos to 155x225, because some haven't exactly this size.
-    for (const sen of sens) {
-        const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(sen.senprenomuse, "_")}${slugify(sen.senmat, "_")}`;
-        const photoFilename = photoStem + ".jpg";
-        const photoFilePath = path.join(photosDir, photoFilename);
-        if (fs.existsSync(photoFilePath)) {
-            if (!options.silent) {
-                console.log(`Resizing photo ${photoStem} for ${sen.senprenomuse} ${sen.sennomuse}…`);
-            }
-            execSync(`gm convert -resize 155x225! ${photoStem}.jpg ${photoStem}_155x225.jpg`, {
-                cwd: photosDir,
-            });
-        }
-        else {
-            if (!options.silent) {
-                console.warn(`Missing photo for ${sen.senprenomuse} ${sen.sennomuse}: using blank image`);
-            }
-            fs.copyFileSync(missingPhotoFilePath, path.join(photosDir, `${photoStem}_155x225.jpg`));
-        }
-    }
-    // Create a mosaic of photos.
-    if (!options.silent) {
-        console.log("Creating mosaic of photos…");
-    }
-    const photoBySenmat = {};
-    const rowsFilenames = [];
-    for (let senIndex = 0, rowIndex = 0; senIndex < sens.length; senIndex += 25, rowIndex++) {
-        const row = sens.slice(senIndex, senIndex + 25);
-        const photosFilenames = [];
-        for (const [columnIndex, sen] of row.entries()) {
-            const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(sen.senprenomuse, "_")}${slugify(sen.senmat, "_")}`;
-            const photoFilename = `${photoStem}_155x225.jpg`;
-            photosFilenames.push(photoFilename);
-            photoBySenmat[sen.senmat] = {
-                chemin: `photos_senateurs/${photoFilename}`,
-                cheminMosaique: "photos_senateurs/senateurs.jpg",
-                hauteur: 225,
-                largeur: 155,
-                xMosaique: columnIndex * 155,
-                yMosaique: rowIndex * 225,
-            };
-        }
-        const rowFilename = `row-${rowIndex}.jpg`;
-        execSync(`gm convert ${photosFilenames.join(" ")} +append ${rowFilename}`, {
-            cwd: photosDir,
-        });
-        rowsFilenames.push(rowFilename);
-    }
-    execSync(`gm convert ${rowsFilenames.join(" ")} -append senateurs.jpg`, {
-        cwd: photosDir,
-    });
-    for (const rowFilename of rowsFilenames) {
-        fs.unlinkSync(path.join(photosDir, rowFilename));
-    }
-    if (!options.silent) {
-        console.log("Creating JSON file containing informations on all pictures…");
-    }
-    const jsonFilePath = path.join(photosDir, "senateurs.json");
-    fs.writeFileSync(jsonFilePath, JSON.stringify(photoBySenmat, null, 2));
-}
-retrievePhotosSenateurs()
-    .then(() => process.exit(0))
-    .catch((error) => {
-    console.log(error);
-    process.exit(1);
-});

package/lib/scripts/retrieve_senateurs_photos.ts

@@ -1,177 +0,0 @@
-import assert from "assert"
-import { execSync } from "child_process"
-import commandLineArgs from "command-line-args"
-import fs from "fs-extra"
-// import fetch from "node-fetch"
-import path from "path"
-// import stream from "stream"
-// import util from "util"
-
-import { findActif as findActifSenateurs } from "../model/sens"
-import { slugify } from "../strings"
-import { Photo } from "../types/sens"
-import { commonOptions } from "./shared/cli_helpers"
-
-const optionsDefinitions = [
-  ...commonOptions,
-  {
-    alias: "f",
-    help: "fetch sénateurs' pictures instead of retrieving them from files",
-    name: "fetch",
-    type: Boolean,
-  },
-]
-const options = commandLineArgs(optionsDefinitions)
-// const pipeline = util.promisify(stream.pipeline)
-
-async function retrievePhotosSenateurs(): Promise<void> {
-  const dataDir = options.dataDir
-  assert(dataDir, "Missing argument: data directory")
-
-  const photosDir = path.join(dataDir, "photos_senateurs")
-  const missingPhotoFilePath = path.resolve(__dirname, "images", "transparent_155x225.jpg")
-
-  const sens = await Array.fromAsync(findActifSenateurs())
-
-  // Download photos.
-  fs.ensureDirSync(photosDir)
-  if (options.fetch) {
-    for (const sen of sens) {
-      const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(
-        sen.senprenomuse,
-        "_",
-      )}${slugify(sen.senmat, "_")}`
-      const photoFilename = photoStem + ".jpg"
-      const photoFilePath = path.join(photosDir, photoFilename)
-      const photoTempFilename = photoStem + "_temp.jpg"
-      const photoTempFilePath = path.join(photosDir, photoTempFilename)
-      const urlPhoto = `https://www.senat.fr/senimg/${photoFilename}`
-      if (!options.silent) {
-        console.log(
-          `Loading photo ${urlPhoto} for ${sen.senprenomuse} ${sen.sennomuse}…`,
-        )
-      }
-      // Fetch fails with OpenSSL error: dh key too small.
-      // (so does "curl").
-      // for (let retries = 0; retries < 3; retries++) {
-      //   const response = await fetch(urlPhoto)
-      //   if (response.ok) {
-      //     await pipeline(response.body, fs.createWriteStream(photoTempFilePath))
-      //     fs.renameSync(photoTempFilePath, photoFilePath)
-      //     break
-      //   }
-      //   if (retries >= 2) {
-      //     console.warn(`Fetch failed: ${urlPhoto} (${sen.senprenomuse} ${sen.sennomuse})`)
-      //     console.warn(response.status, response.statusText)
-      //     console.warn(await response.text())
-      //     if (fs.existsSync(photoFilePath)) {
-      //       console.warn("  => Reusing existing image")
-      //     } else {
-      //       console.warn("  => Using blank image")
-      //       fs.copyFileSync(missingPhotoFilePath, photoFilePath)
-      //     }
-      //     break
-      //   }
-      // }
-      try {
-        execSync(`wget --quiet -O ${photoTempFilename} ${urlPhoto}`, {
-          cwd: photosDir,
-          env: process.env,
-          encoding: "utf-8",
-          // stdio: ["ignore", "ignore", "pipe"],
-        })
-        fs.renameSync(photoTempFilePath, photoFilePath)
-      } catch (error) {
-        if (typeof error === "object" && error && "status" in error && error.status === 8) {
-          console.error(`Unable to load photo for ${sen.senprenomuse} ${sen.sennomuse}`)
-          continue
-        }
-        throw error
-      }
-    }
-  }
-
-  // Resize photos to 155x225, because some haven't exactly this size.
-  for (const sen of sens) {
-    const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(
-      sen.senprenomuse,
-      "_",
-    )}${slugify(sen.senmat, "_")}`
-    const photoFilename = photoStem + ".jpg"
-    const photoFilePath = path.join(photosDir, photoFilename)
-    if (fs.existsSync(photoFilePath)) {
-      if (!options.silent) {
-        console.log(
-          `Resizing photo ${photoStem} for ${sen.senprenomuse} ${sen.sennomuse}…`,
-        )
-      }
-
-      execSync(
-        `gm convert -resize 155x225! ${photoStem}.jpg ${photoStem}_155x225.jpg`,
-        {
-          cwd: photosDir,
-        },
-      )
-    } else {
-      if (!options.silent) {
-        console.warn(`Missing photo for ${sen.senprenomuse} ${sen.sennomuse}: using blank image`)
-      }
-      fs.copyFileSync(missingPhotoFilePath, path.join(photosDir, `${photoStem}_155x225.jpg`))
-    }
-  }
-
-  // Create a mosaic of photos.
-  if (!options.silent) {
-    console.log("Creating mosaic of photos…")
-  }
-  const photoBySenmat: { [senmat: string]: Photo } = {}
-  const rowsFilenames: string[] = []
-  for (
-    let senIndex = 0, rowIndex = 0;
-    senIndex < sens.length;
-    senIndex += 25, rowIndex++
-  ) {
-    const row = sens.slice(senIndex, senIndex + 25)
-    const photosFilenames: string[] = []
-    for (const [columnIndex, sen] of row.entries()) {
-      const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(
-        sen.senprenomuse,
-        "_",
-      )}${slugify(sen.senmat, "_")}`
-      const photoFilename = `${photoStem}_155x225.jpg`
-      photosFilenames.push(photoFilename)
-      photoBySenmat[sen.senmat] = {
-        chemin: `photos_senateurs/${photoFilename}`,
-        cheminMosaique: "photos_senateurs/senateurs.jpg",
-        hauteur: 225,
-        largeur: 155,
-        xMosaique: columnIndex * 155,
-        yMosaique: rowIndex * 225,
-      }
-    }
-    const rowFilename = `row-${rowIndex}.jpg`
-    execSync(`gm convert ${photosFilenames.join(" ")} +append ${rowFilename}`, {
-      cwd: photosDir,
-    })
-    rowsFilenames.push(rowFilename)
-  }
-  execSync(`gm convert ${rowsFilenames.join(" ")} -append senateurs.jpg`, {
-    cwd: photosDir,
-  })
-  for (const rowFilename of rowsFilenames) {
-    fs.unlinkSync(path.join(photosDir, rowFilename))
-  }
-
-  if (!options.silent) {
-    console.log("Creating JSON file containing informations on all pictures…")
-  }
-  const jsonFilePath = path.join(photosDir, "senateurs.json")
-  fs.writeFileSync(jsonFilePath, JSON.stringify(photoBySenmat, null, 2))
-}
-
-retrievePhotosSenateurs()
-  .then(() => process.exit(0))
-  .catch((error) => {
-    console.log(error)
-    process.exit(1)
-  })