@tricoteuses/senat 2.22.5 → 2.22.7

package/lib/scripts/retrieve_senateurs_photos.ts

@@ -1,177 +0,0 @@
-import assert from "assert"
-import { execSync } from "child_process"
-import commandLineArgs from "command-line-args"
-import fs from "fs-extra"
-// import fetch from "node-fetch"
-import path from "path"
-// import stream from "stream"
-// import util from "util"
-
-import { findActif as findActifSenateurs } from "../model/sens"
-import { slugify } from "../strings"
-import { Photo } from "../types/sens"
-import { commonOptions } from "./shared/cli_helpers"
-
-const optionsDefinitions = [
-  ...commonOptions,
-  {
-    alias: "f",
-    help: "fetch sénateurs' pictures instead of retrieving them from files",
-    name: "fetch",
-    type: Boolean,
-  },
-]
-const options = commandLineArgs(optionsDefinitions)
-// const pipeline = util.promisify(stream.pipeline)
-
-async function retrievePhotosSenateurs(): Promise<void> {
-  const dataDir = options.dataDir
-  assert(dataDir, "Missing argument: data directory")
-
-  const photosDir = path.join(dataDir, "photos_senateurs")
-  const missingPhotoFilePath = path.resolve(__dirname, "images", "transparent_155x225.jpg")
-
-  const sens = await Array.fromAsync(findActifSenateurs())
-
-  // Download photos.
-  fs.ensureDirSync(photosDir)
-  if (options.fetch) {
-    for (const sen of sens) {
-      const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(
-        sen.senprenomuse,
-        "_",
-      )}${slugify(sen.senmat, "_")}`
-      const photoFilename = photoStem + ".jpg"
-      const photoFilePath = path.join(photosDir, photoFilename)
-      const photoTempFilename = photoStem + "_temp.jpg"
-      const photoTempFilePath = path.join(photosDir, photoTempFilename)
-      const urlPhoto = `https://www.senat.fr/senimg/${photoFilename}`
-      if (!options.silent) {
-        console.log(
-          `Loading photo ${urlPhoto} for ${sen.senprenomuse} ${sen.sennomuse}…`,
-        )
-      }
-      // Fetch fails with OpenSSL error: dh key too small.
-      // (so does "curl").
-      // for (let retries = 0; retries < 3; retries++) {
-      //   const response = await fetch(urlPhoto)
-      //   if (response.ok) {
-      //     await pipeline(response.body, fs.createWriteStream(photoTempFilePath))
-      //     fs.renameSync(photoTempFilePath, photoFilePath)
-      //     break
-      //   }
-      //   if (retries >= 2) {
-      //     console.warn(`Fetch failed: ${urlPhoto} (${sen.senprenomuse} ${sen.sennomuse})`)
-      //     console.warn(response.status, response.statusText)
-      //     console.warn(await response.text())
-      //     if (fs.existsSync(photoFilePath)) {
-      //       console.warn("  => Reusing existing image")
-      //     } else {
-      //       console.warn("  => Using blank image")
-      //       fs.copyFileSync(missingPhotoFilePath, photoFilePath)
-      //     }
-      //     break
-      //   }
-      // }
-      try {
-        execSync(`wget --quiet -O ${photoTempFilename} ${urlPhoto}`, {
-          cwd: photosDir,
-          env: process.env,
-          encoding: "utf-8",
-          // stdio: ["ignore", "ignore", "pipe"],
-        })
-        fs.renameSync(photoTempFilePath, photoFilePath)
-      } catch (error) {
-        if (typeof error === "object" && error && "status" in error && error.status === 8) {
-          console.error(`Unable to load photo for ${sen.senprenomuse} ${sen.sennomuse}`)
-          continue
-        }
-        throw error
-      }
-    }
-  }
-
-  // Resize photos to 155x225, because some haven't exactly this size.
-  for (const sen of sens) {
-    const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(
-      sen.senprenomuse,
-      "_",
-    )}${slugify(sen.senmat, "_")}`
-    const photoFilename = photoStem + ".jpg"
-    const photoFilePath = path.join(photosDir, photoFilename)
-    if (fs.existsSync(photoFilePath)) {
-      if (!options.silent) {
-        console.log(
-          `Resizing photo ${photoStem} for ${sen.senprenomuse} ${sen.sennomuse}…`,
-        )
-      }
-
-      execSync(
-        `gm convert -resize 155x225! ${photoStem}.jpg ${photoStem}_155x225.jpg`,
-        {
-          cwd: photosDir,
-        },
-      )
-    } else {
-      if (!options.silent) {
-        console.warn(`Missing photo for ${sen.senprenomuse} ${sen.sennomuse}: using blank image`)
-      }
-      fs.copyFileSync(missingPhotoFilePath, path.join(photosDir, `${photoStem}_155x225.jpg`))
-    }
-  }
-
-  // Create a mosaic of photos.
-  if (!options.silent) {
-    console.log("Creating mosaic of photos…")
-  }
-  const photoBySenmat: { [senmat: string]: Photo } = {}
-  const rowsFilenames: string[] = []
-  for (
-    let senIndex = 0, rowIndex = 0;
-    senIndex < sens.length;
-    senIndex += 25, rowIndex++
-  ) {
-    const row = sens.slice(senIndex, senIndex + 25)
-    const photosFilenames: string[] = []
-    for (const [columnIndex, sen] of row.entries()) {
-      const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(
-        sen.senprenomuse,
-        "_",
-      )}${slugify(sen.senmat, "_")}`
-      const photoFilename = `${photoStem}_155x225.jpg`
-      photosFilenames.push(photoFilename)
-      photoBySenmat[sen.senmat] = {
-        chemin: `photos_senateurs/${photoFilename}`,
-        cheminMosaique: "photos_senateurs/senateurs.jpg",
-        hauteur: 225,
-        largeur: 155,
-        xMosaique: columnIndex * 155,
-        yMosaique: rowIndex * 225,
-      }
-    }
-    const rowFilename = `row-${rowIndex}.jpg`
-    execSync(`gm convert ${photosFilenames.join(" ")} +append ${rowFilename}`, {
-      cwd: photosDir,
-    })
-    rowsFilenames.push(rowFilename)
-  }
-  execSync(`gm convert ${rowsFilenames.join(" ")} -append senateurs.jpg`, {
-    cwd: photosDir,
-  })
-  for (const rowFilename of rowsFilenames) {
-    fs.unlinkSync(path.join(photosDir, rowFilename))
-  }
-
-  if (!options.silent) {
-    console.log("Creating JSON file containing informations on all pictures…")
-  }
-  const jsonFilePath = path.join(photosDir, "senateurs.json")
-  fs.writeFileSync(jsonFilePath, JSON.stringify(photoBySenmat, null, 2))
-}
-
-retrievePhotosSenateurs()
-  .then(() => process.exit(0))
-  .catch((error) => {
-    console.log(error)
-    process.exit(1)
-  })

package/lib/scripts/shared/cli_helpers.ts

@@ -1,36 +0,0 @@
-export const categoriesOption = {
-  alias: "k",
-  defaultValue: ["All"],
-  help: "categories of datasets to reorganize",
-  multiple: true,
-  name: "categories",
-  type: String,
-}
-
-export const dataDirDefaultOption = {
-  defaultOption: true,
-  help: "directory containing Sénat open data files",
-  name: "dataDir",
-  type: String,
-}
-
-export const silentOption = {
-  alias: "s",
-  help: "don't log anything",
-  name: "silent",
-  type: Boolean,
-}
-
-export const verboseOption = {
-  alias: "v",
-  help: "verbose logs",
-  name: "verbose",
-  type: Boolean,
-}
-
-export const commonOptions = [
-  categoriesOption,
-  dataDirDefaultOption,
-  silentOption,
-  verboseOption,
-]

package/lib/scripts/shared/util.ts

@@ -1,33 +0,0 @@
-import fs from "fs-extra"
-
-export function isOptionEmptyOrHasValue (option: string, value: string) {
-  return !option || option.length === 0 || option.includes(value)
-}
-
-export function ensureAndClearDir (path: string) {
-  if (!fs.existsSync(path)) {
-    fs.mkdirSync(path, { recursive: true })
-  } else {
-    fs.emptyDirSync(path)
-  }
-}
-
-export async function fetchWithRetry (
-  url: string,
-  retries: number = 3,
-  backoff: number = 300,
-): Promise<Response> {
-  for (let attempt = 0; attempt < retries; attempt++) {
-    try {
-      return await fetch(url)
-    } catch (error) {
-      if (attempt === retries - 1) {
-        throw error
-      }
-      console.warn(`Fetch attempt ${attempt + 1} for ${url} failed. Retrying in ${backoff}ms…`)
-      await new Promise((resolve) => setTimeout(resolve, backoff))
-      backoff *= 2
-    }
-  }
-  throw new Error(`Failed to fetch ${url} after ${retries} attempts`)
-}

package/lib/strings.mjs

@@ -1,18 +0,0 @@
-import originalSlugify from "slug";
-const slugifyCharmap = {
-    ...originalSlugify.defaults.charmap,
-    "'": " ",
-    "@": " ",
-    ".": " ",
-    "-": "_",
-};
-export function slugify(string, replacement) {
-    const options = {
-        charmap: slugifyCharmap,
-        mode: "rfc3986",
-    };
-    if (replacement) {
-        options.replacement = replacement;
-    }
-    return originalSlugify(string, options);
-}

package/lib/strings.ts

@@ -1,26 +0,0 @@
-import originalSlugify from "slug"
-
-const slugifyCharmap = {
-  ...originalSlugify.defaults.charmap,
-  "'": " ",
-  "@": " ",
-  ".": " ",
-  "-": "_",
-}
-
-interface Options {
-  charmap: { [c: string]: string }
-  mode: string
-  replacement?: string
-}
-
-export function slugify(string: string, replacement?: string): string {
-  const options: Options = {
-    charmap: slugifyCharmap,
-    mode: "rfc3986",
-  }
-  if (replacement) {
-    options.replacement = replacement
-  }
-  return originalSlugify(string, options)
-}

package/lib/types/ameli.mjs

@@ -1,13 +0,0 @@
-export const sesFieldsToParseInt = ["ann"];
-export const sesFieldsToTrim = ["lil"];
-export const subFieldsToParseInt = ["pos", "posder", "prires"];
-export const subFieldsToTrim = ["lic", "lib", "sig", "style"];
-export const txtAmeliFieldsToTrim = [
-    "num",
-    "int",
-    "inl",
-    "libdelim",
-    "libcplnat",
-    "doslegsignet",
-    "ordsnddelib",
-];

package/lib/types/ameli.ts

@@ -1,21 +0,0 @@
-import { ses as Ses, sub as Sub, subFields, txt_ameli } from "../raw_types_schemats/ameli"
-
-export { Ses, Sub }
-
-export interface TxtAmeli extends txt_ameli {
-  subids?: subFields.id[]
-}
-
-export const sesFieldsToParseInt = ["ann"]
-export const sesFieldsToTrim = ["lil"]
-export const subFieldsToParseInt = ["pos", "posder", "prires"]
-export const subFieldsToTrim = ["lic", "lib", "sig", "style"]
-export const txtAmeliFieldsToTrim = [
-  "num",
-  "int",
-  "inl",
-  "libdelim",
-  "libcplnat",
-  "doslegsignet",
-  "ordsnddelib",
-]

package/lib/types/debats.mjs

@@ -1,2 +0,0 @@
-export const debatsFieldsToTrim = ["deburl", "libspec"];
-export const lecassdebFieldsToTrim = ["lecassidt"];

package/lib/types/debats.ts

@@ -1,6 +0,0 @@
-import { debats as Debat, lecassdeb as LecAssDeb } from "../raw_types_schemats/debats"
-
-export { Debat, LecAssDeb }
-
-export const debatsFieldsToTrim = ["deburl", "libspec"]
-export const lecassdebFieldsToTrim = ["lecassidt"]

package/lib/types/dosleg.mjs

@@ -1,151 +0,0 @@
-export const assFieldsToTrim = ["libass"];
-export const audFieldsToTrim = ["lecassidt", "audtit", "audurl", "orgcod"];
-export const auteurFieldsToTrim = [
-    "autcod",
-    "quacod",
-    "typautcod",
-    "nomuse",
-    "prenom",
-    "nomtec",
-    "autmat",
-    "autfct",
-];
-export const dateSeanceFieldsToTrim = ["lecidt", "statut"];
-export const deccocFieldsToTrim = ["deccoccod", "deccoclib"];
-export const denrapFieldsToTrim = [
-    "coddenrap",
-    "typraprap",
-    "libdenrap",
-    "denrapmin",
-    "denraptit",
-    "denrapstymin",
-    "solnatrapcod",
-];
-export const docattFieldsToParseInt = ["docattcle", "rapcod"];
-export const docattFieldsToTrim = ["docatturl"];
-export const ecrFieldsToTrim = ["autcod", "ecrqua"];
-export const etaloiFieldsToTrim = ["etaloilib"];
-export const lecassFieldsToTrim = [
-    "lecassidt",
-    "lecidt",
-    "ptlurl",
-    "ptlnumcpl",
-    "ptlnot",
-    "ptlurl2",
-    "ptlnot2",
-    "ptlurl3",
-    "ptlnot3",
-    "ptlnumcpl2",
-    "ptlnumcpl3",
-    "lecassame",
-    "orgcod",
-    "loiintmod",
-    "reucom",
-    "debatsurl",
-    "libppr",
-    "ptlurlcom",
-    "aliasppr",
-    "lecassamecom",
-];
-export const lecassrapFieldsToTrim = ["lecassidt"];
-export const lectureFieldsToTrim = ["lecidt", "loicod", "typleccod", "leccom"];
-export const loiFieldsToTrim = [
-    "loicod",
-    "typloicod",
-    "deccoccod",
-    "numero",
-    "loient",
-    "motclef",
-    "loitit",
-    "loiint",
-    "url_jo",
-    "loinumjo",
-    "loititjo",
-    "url_jo2",
-    "loinumjo2",
-    "deccocurl",
-    "num_decision",
-    "loicodmai",
-    "loinoudelibcod",
-    "motionloiorigcod",
-    "url_ordonnance",
-    "saisine_par",
-    "loinumjo3",
-    "url_jo3",
-    "url_an",
-    "url_presart",
-    "signetalt",
-    "orgcod",
-    "doscocurl",
-    "loiintori",
-];
-export const orgFieldsToTrim = [
-    "orgcod",
-    "typorgcod",
-    "orgnom",
-    "orgliblon",
-    "orglibaff",
-    "orgurl",
-    "orglibcou",
-    "org_de",
-    "urltra",
-    "inttra",
-    "orgnomcouv",
-    "senorgcod",
-    "html_color",
-];
-export const oritxtFieldsToTrim = ["oritxtcod", "oritxtlib", "oritxtlibfem"];
-export const quaFieldsToTrim = ["quacod", "qualic", "quaabr", "quaabrplu"];
-export const rapFieldsToParseInt = ["rapcod", "sesann", "rapnum", "rapnuman"];
-export const rapFieldsToTrim = [
-    "coddenrap",
-    "blecod",
-    "raptitcou",
-    "raptil",
-    "rapurl",
-    "url2",
-    "url3",
-    "url4",
-    "url2txt",
-    "url3txt",
-    "url4txt",
-    "prix",
-    "numerobis",
-    "rapsoustit",
-    "rapres",
-    "forpubcod",
-];
-export const raporgFieldsToTrim = ["orgcod"];
-export const scrFieldsToTrim = ["scrint", "soslib"];
-export const texteFieldsToParseInt = ["texcod", "sesann", "texnum"];
-export const texteFieldsToTrim = [
-    "oritxtcod",
-    "typtxtcod",
-    "lecassidt",
-    "orgcod",
-    "texurl",
-    "url2",
-    "url3",
-    "url4",
-    "url2txt",
-    "url3txt",
-    "url4txt",
-    "prix",
-    "numerobis",
-    "reserve_comspe",
-];
-export const typattFieldsToTrim = ["typattlib"];
-export const typlecFieldsToTrim = ["typleccod", "typleclib"];
-export const typloiFieldsToTrim = [
-    "typloicod",
-    "typloilib",
-    "groupe",
-    "typloiden",
-    "typloigen",
-    "typloitit",
-    "typloidenplu",
-    "typloide",
-    "typloiabr",
-];
-export const typtxtFieldsToTrim = ["typtxtcod", "typtxtlib"];
-export const typurlFieldsToTrim = ["libtypurl"];