@tricoteuses/senat 2.8.1 → 2.8.4

package/lib/scripts/retrieve_documents.ts

@@ -0,0 +1,279 @@
+import assert from "assert"
+import commandLineArgs from "command-line-args"
+import fs from "fs-extra"
+import path from "path"
+
+import { findSenatRapportUrls, findSenatTexteUrls } from "../model/dosleg"
+import { parseTexte, parseTexteFromFile } from "../model/texte"
+import { UNDEFINED_SESSION } from "./datautil"
+import { commonOptions } from "./shared/cli_helpers"
+import { ensureAndClearDir, fetchWithRetry, isOptionEmptyOrHasValue } from "./shared/util"
+
+const optionsDefinitions = [
+  ...commonOptions,
+  {
+    help: "sessions of textes to retrieve; leave empty for all",
+    multiple: true,
+    name: "sessions",
+    type: String,
+  },
+  {
+    help: "parse and convert documents into JSON (textes only for now, requires format xml)",
+    name: "parseDocuments",
+    type: Boolean,
+  },
+  {
+    alias: "F",
+    help: "formats of documents to retrieve (xml/html/pdf for textes, html/pdf for rapports); leave empty for all",
+    multiple: true,
+    name: "formats",
+    type: String,
+  },
+  {
+    help: "types of documents to retrieve (textes/rapports); leave empty for all",
+    multiple: true,
+    name: "types",
+    type: String,
+  },
+  {
+    help: "force retrieve all documents, even already retrieved ones",
+    name: "force",
+    type: Boolean,
+  },
+]
+const options = commandLineArgs(optionsDefinitions)
+
+const SENAT_TEXTE_XML_BASE_URL = "https://www.senat.fr/akomantoso/"
+const SENAT_TEXTE_BASE_URL = "https://www.senat.fr/leg/"
+const SENAT_RAPPORT_BASE_URL = "https://www.senat.fr/rap/"
+
+const textDecoder = new TextDecoder("utf8")
+
+async function retrieveDocument (documentUrl: string): Promise<ArrayBuffer | null> {
+  if (!options.silent) {
+    console.log(`Retrieving document ${documentUrl}…`)
+  }
+
+  try {
+    const response = await fetchWithRetry(documentUrl)
+    if (!response.ok) {
+      if (response.status === 404) {
+        console.warn(`Texte ${documentUrl} not found`)
+      } else {
+        console.error(`An error occurred while retrieving texte ${documentUrl}: ${response.status}`)
+      }
+      return null
+    }
+    return response.arrayBuffer()
+  } catch (error: any) {
+    console.error(error.message)
+    return null
+  }
+}
+
+async function retrieveTextes (dataDir: string) {
+  const textesDir = path.join(dataDir, "leg")
+  fs.ensureDirSync(textesDir)
+  const originalTextesDir = path.join(textesDir, "original")
+  const transformedTextesDir = path.join(textesDir, "transformed")
+  ensureAndClearDir(transformedTextesDir)
+
+  let retrievedTextesCount = 0
+  const texteUrlsNotFoundOrError = []
+  const texteUrlsParseError = []
+
+  for await (const texte of findSenatTexteUrls(options.sessions)) {
+    const texteName = path.parse(texte.url).name
+    const texteDir = path.join(originalTextesDir, `${texte.session ?? UNDEFINED_SESSION}`, texteName)
+    fs.ensureDirSync(texteDir)
+
+    if (isOptionEmptyOrHasValue(options.formats, "xml")) {
+      const texteXmlUrl = `${texteName}.akn.xml`
+      const texteXmlAbsoluteUrl = new URL(texteXmlUrl, SENAT_TEXTE_XML_BASE_URL).toString()
+      const textePath = path.join(texteDir, texteXmlUrl)
+      let texteBuffer = null
+
+      if (!options.force && fs.existsSync(textePath)) {
+        if (!options.silent) {
+          console.info(`Already retrieved texte ${textePath}…`)
+        }
+      } else {
+        texteBuffer = await retrieveDocument(texteXmlAbsoluteUrl)
+        if (!texteBuffer) {
+          texteUrlsNotFoundOrError.push(texteXmlAbsoluteUrl)
+          continue
+        }
+        fs.writeFileSync(textePath, Buffer.from(texteBuffer))
+        retrievedTextesCount++
+      }
+
+      if (options.parseDocuments) {
+        if (!options.silent) {
+          console.log(`Parsing texte ${texteXmlUrl}…`)
+        }
+
+        let parsedTexte = null
+        if (texteBuffer) {
+          const texteXml = textDecoder.decode(texteBuffer)
+          parsedTexte = parseTexte(texteXml)
+        } else {
+          parsedTexte = await parseTexteFromFile(textePath)
+        }
+
+        if (!parsedTexte) {
+          texteUrlsParseError.push(texteXmlAbsoluteUrl)
+          continue
+        }
+
+        const transformedTexteDir
+          = path.join(transformedTextesDir, `${texte.session ?? UNDEFINED_SESSION}`, texteName)
+        fs.ensureDirSync(transformedTexteDir)
+        fs.writeJSONSync(path.join(transformedTexteDir, `${texteName}.akn.json`), parsedTexte, { spaces: 2 })
+      }
+    }
+
+    if (isOptionEmptyOrHasValue(options.formats, "html")) {
+      const texteHtmlUrl = `${texteName}.html`
+      const texteHtmlAbsoluteUrl = new URL(texteHtmlUrl, SENAT_TEXTE_BASE_URL).toString()
+      const textePath = path.join(texteDir, texteHtmlUrl)
+
+      if (!options.force && fs.existsSync(textePath)) {
+        if (!options.silent) {
+          console.info(`Already retrieved texte ${textePath}…`)
+        }
+        continue
+      }
+
+      const texteBuffer = await retrieveDocument(texteHtmlAbsoluteUrl)
+      if (!texteBuffer) {
+        texteUrlsNotFoundOrError.push(texteHtmlAbsoluteUrl)
+        continue
+      }
+      fs.writeFileSync(textePath, Buffer.from(texteBuffer))
+      retrievedTextesCount++
+    }
+
+    if (isOptionEmptyOrHasValue(options.formats, "pdf")) {
+      const textePdfUrl = `${texteName}.pdf`
+      const textePdfAbsoluteUrl = new URL(textePdfUrl, SENAT_TEXTE_BASE_URL).toString()
+      const textePath = path.join(texteDir, textePdfUrl)
+
+      if (!options.force && fs.existsSync(textePath)) {
+        if (!options.silent) {
+          console.info(`Already retrieved texte ${textePath}…`)
+        }
+        continue
+      }
+
+      const texteBuffer = await retrieveDocument(textePdfAbsoluteUrl)
+      if (!texteBuffer) {
+        texteUrlsNotFoundOrError.push(textePdfAbsoluteUrl)
+        continue
+      }
+      fs.writeFileSync(textePath, Buffer.from(texteBuffer))
+      retrievedTextesCount++
+    }
+  }
+
+  if (options.verbose) {
+    console.log(`${retrievedTextesCount} textes retrieved`)
+    console.log(
+      `${texteUrlsNotFoundOrError.length} textes failed to be retrieved with URLs ${texteUrlsNotFoundOrError.join(", ")}`
+    )
+    if (options.parseDocuments) {
+      console.log(`${texteUrlsParseError.length} textes failed to be parsed with URLs ${texteUrlsParseError.join(", ")}`)
+    }
+  }
+  // TODO retrieve exposé des motifs (/leg/exposes-des-motifs)
+}
+
+async function retrieveRapports (dataDir: string) {
+  const rapportsDir = path.join(dataDir, "rap")
+  fs.ensureDirSync(rapportsDir)
+
+  let retrievedRapportsCount = 0
+  const rapportUrlsNotFoundOrError = []
+
+  for await (const rapport of findSenatRapportUrls(options.sessions)) {
+    const parsedRapportUrl = path.parse(rapport.url)
+    const rapportName = parsedRapportUrl.name
+    const rapportDir = path.join(rapportsDir, `${rapport.session ?? UNDEFINED_SESSION}`, rapportName)
+    fs.ensureDirSync(rapportDir)
+
+    if (isOptionEmptyOrHasValue(options.formats, "html")) {
+      const rapportHtmlUrlBase = `${rapportName}_mono.html`
+      const rapportHtmlUrl = path.format({
+        dir: parsedRapportUrl.dir,
+        base: rapportHtmlUrlBase,
+      })
+      const rapportHtmlAbsoluteUrl = new URL(rapportHtmlUrl, SENAT_RAPPORT_BASE_URL).toString()
+      const rapportPath = path.join(rapportDir, rapportHtmlUrlBase)
+
+      if (!options.force && fs.existsSync(rapportPath)) {
+        if (!options.silent) {
+          console.info(`Already retrieved rapport ${rapportPath}…`)
+        }
+        continue
+      }
+
+      const rapportBuffer = await retrieveDocument(rapportHtmlAbsoluteUrl)
+      if (!rapportBuffer) {
+        rapportUrlsNotFoundOrError.push(rapportHtmlAbsoluteUrl)
+        continue
+      }
+      fs.writeFileSync(rapportPath, Buffer.from(rapportBuffer))
+      retrievedRapportsCount++
+    }
+
+    if (isOptionEmptyOrHasValue(options.formats, "pdf")) {
+      const rapportPdfUrlBase = `${rapportName}1.pdf`
+      const rapportPdfUrl = path.format({
+        dir: parsedRapportUrl.dir,
+        base: rapportPdfUrlBase,
+      })
+      const rapportPdfAbsoluteUrl = new URL(rapportPdfUrl, SENAT_RAPPORT_BASE_URL).toString()
+      const rapportPath = path.join(rapportDir, rapportPdfUrlBase)
+
+      if (!options.force && fs.existsSync(rapportPath)) {
+        if (!options.silent) {
+          console.info(`Already retrieved rapport ${rapportPath}…`)
+        }
+        continue
+      }
+
+      const rapportBuffer = await retrieveDocument(rapportPdfAbsoluteUrl)
+      if (!rapportBuffer) {
+        rapportUrlsNotFoundOrError.push(rapportPdfAbsoluteUrl)
+        continue
+      }
+      fs.writeFileSync(rapportPath, Buffer.from(rapportBuffer))
+      retrievedRapportsCount++
+    }
+  }
+
+  if (options.verbose) {
+    console.log(`${retrievedRapportsCount} rapports retrieved`)
+    console.log(
+      `${rapportUrlsNotFoundOrError.length} rapports failed with URLs ${rapportUrlsNotFoundOrError.join(", ")}`
+    )
+  }
+}
+
+async function main() {
+  const dataDir = options.dataDir
+  assert(dataDir, "Missing argument: data directory")
+
+  if (isOptionEmptyOrHasValue(options.types, "textes")) {
+    await retrieveTextes(dataDir)
+  }
+  if (isOptionEmptyOrHasValue(options.types, "rapports")) {
+    await retrieveRapports(dataDir)
+  }
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch((error) => {
+    console.log(error)
+    process.exit(1)
+  })

package/lib/scripts/retrieve_open_data.js

@@ -4,11 +4,11 @@ import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 // import fetch from "node-fetch"
 import path from "path";
-// import stream from "stream"
 import StreamZip from "node-stream-zip";
 import readline from "readline";
-// import util from "util"
 import windows1252 from "windows-1252";
+import { pipeline } from "stream";
+import { promisify } from "util";
 import config from "../config";
 import { datasets, getChosenDatasets, getEnabledDatasets } from "../datasets";
 import { commonOptions } from "./shared/cli_helpers";
@@ -59,7 +59,14 @@ const optionsDefinitions = [
     },
 ];
 const options = commandLineArgs(optionsDefinitions);
-// const pipeline = util.promisify(stream.pipeline)
+const streamPipeline = promisify(pipeline);
+async function downloadFile(url, dest) {
+    const response = await fetch(url);
+    if (!response.ok) {
+        throw new Error(`Download failed ${response.status} ${response.statusText} for ${url}`);
+    }
+    await streamPipeline(response.body, fs.createWriteStream(dest));
+}
 async function retrieveDataset(dataDir, dataset) {
     const zipFilename = dataset.url.substring(dataset.url.lastIndexOf("/") + 1);
     const zipFilePath = path.join(dataDir, zipFilename);
@@ -78,12 +85,7 @@ async function retrieveDataset(dataDir, dataset) {
         // }
         // await pipeline(response.body!, fs.createWriteStream(zipFilePath))
         fs.removeSync(zipFilePath);
-        execSync(`wget --quiet ${dataset.url}`, {
-            cwd: dataDir,
-            env: process.env,
-            encoding: "utf-8",
-            // stdio: ["ignore", "ignore", "pipe"],
-        });
+        await downloadFile(dataset.url, zipFilePath);
     }
     const sqlFilename = `${dataset.database}.sql`;
     const sqlFilePath = path.join(dataDir, sqlFilename);

package/lib/scripts/retrieve_open_data.mjs

@@ -0,0 +1,214 @@
+import assert from "assert";
+import { execSync } from "child_process";
+import commandLineArgs from "command-line-args";
+import fs from "fs-extra";
+// import fetch from "node-fetch"
+import path from "path";
+// import stream from "stream"
+import StreamZip from "node-stream-zip";
+import readline from "readline";
+// import util from "util"
+import windows1252 from "windows-1252";
+import config from "../config";
+import { getChosenFromEnabledDatasets, } from "../datasets";
+import { commonOptions } from "./shared/cli_helpers";
+const badWindows1252CharacterRegex = /[\u0080-\u009f]/g;
+const optionsDefinitions = [
+    ...commonOptions,
+    {
+        alias: "a",
+        help: "all options: fetch, unzip, repair-encoding, import",
+        name: "all",
+        type: Boolean,
+    },
+    {
+        alias: "f",
+        help: "fetch datasets instead of retrieving them from files",
+        name: "fetch",
+        type: Boolean,
+    },
+    {
+        alias: "z",
+        help: "unzip SQL files",
+        name: "unzip",
+        type: Boolean,
+    },
+    {
+        alias: "e",
+        help: "repair Windows CP 1252 encoding of SQL dumps",
+        name: "repairEncoding",
+        type: Boolean,
+    },
+    {
+        alias: "i",
+        help: "import SQL dumps into a freshly (re-)created database",
+        name: "import",
+        type: Boolean,
+    },
+    {
+        alias: "d",
+        help: "repair database (update schema and types)",
+        name: "repairDatabase",
+        type: Boolean,
+    },
+    {
+        alias: "c",
+        help: "create TypeScript interfaces from databases schemas into src/raw_types_* directories",
+        name: "schema",
+        type: Boolean,
+    },
+];
+const options = commandLineArgs(optionsDefinitions);
+// const pipeline = util.promisify(stream.pipeline)
+async function retrieveDataset(dataDir, dataset) {
+    const zipFilename = dataset.url.substring(dataset.url.lastIndexOf("/") + 1);
+    const zipFilePath = path.join(dataDir, zipFilename);
+    if (options.all || options.fetch) {
+        // Fetch & save ZIP file.
+        if (!options.silent) {
+            console.log(`Loading ${dataset.title}: ${zipFilename}…`);
+        }
+        // Fetch fails with OpenSSL error: dh key too small.
+        // (so does "curl").
+        // const response = await fetch(dataset.url)
+        // if (!response.ok) {
+        //   console.error(response.status, response.statusText)
+        //   console.error(await response.text())
+        //   throw new Error(`Fetch failed: ${dataset.url}`)
+        // }
+        // await pipeline(response.body!, fs.createWriteStream(zipFilePath))
+        fs.removeSync(zipFilePath);
+        execSync(`wget --quiet ${dataset.url}`, {
+            cwd: dataDir,
+            env: process.env,
+            encoding: "utf-8",
+            // stdio: ["ignore", "ignore", "pipe"],
+        });
+    }
+    const sqlFilename = `${dataset.database}.sql`;
+    const sqlFilePath = path.join(dataDir, sqlFilename);
+    if (options.all || options.unzip) {
+        if (!options.silent) {
+            console.log(`Unzipping ${dataset.title}: ${zipFilename}…`);
+        }
+        fs.removeSync(sqlFilePath);
+        const zip = new StreamZip({
+            file: zipFilePath,
+            storeEntries: true,
+        });
+        await new Promise((resolve, reject) => {
+            zip.on("ready", () => {
+                zip.extract(null, dataDir, (err, _count) => {
+                    zip.close();
+                    if (err) {
+                        reject(err);
+                    }
+                    else {
+                        resolve(null);
+                    }
+                });
+            });
+        });
+        if (dataset.repairZip !== undefined) {
+            if (!options.silent) {
+                console.log(`Repairing Zip path ${dataset.title}: ${sqlFilename}…`);
+            }
+            dataset.repairZip(dataset, dataDir);
+        }
+    }
+    if ((options.all || options.repairEncoding) && dataset.repairEncoding) {
+        if (!options.silent) {
+            console.log(`Repairing Windows CP1252 encoding of ${dataset.title}: ${sqlFilename}…`);
+        }
+        const repairedSqlFilePath = sqlFilePath + ".repaired";
+        const repairedSqlWriter = fs.createWriteStream(repairedSqlFilePath, {
+            encoding: "utf8",
+        });
+        const lineReader = readline.createInterface({
+            input: fs.createReadStream(sqlFilePath, { encoding: "utf8" }),
+            crlfDelay: Infinity,
+        });
+        for await (const line of lineReader) {
+            repairedSqlWriter.write(line.replace(badWindows1252CharacterRegex, (match) => windows1252.decode(match, { mode: "fatal" })) + "\n");
+        }
+        repairedSqlWriter.end();
+        await fs.move(repairedSqlFilePath, sqlFilePath, { overwrite: true });
+    }
+    if (options.all || options.import) {
+        if (!options.silent) {
+            console.log(`Importing ${dataset.title}: ${sqlFilename}…`);
+        }
+        execSync(`psql -c "DROP DATABASE IF EXISTS ${dataset.database}"`, {
+            cwd: dataDir,
+            env: process.env,
+            encoding: "utf-8",
+            // stdio: ["ignore", "ignore", "pipe"],
+        });
+        execSync(`psql -c "CREATE DATABASE ${dataset.database} WITH OWNER opendata"`, {
+            cwd: dataDir,
+            env: process.env,
+            encoding: "utf-8",
+            // stdio: ["ignore", "ignore", "pipe"],
+        });
+        execSync(`psql -f ${sqlFilename} ${dataset.database}`, {
+            cwd: dataDir,
+            env: process.env,
+            encoding: "utf-8",
+            // stdio: ["ignore", "ignore", "pipe"],
+        });
+    }
+    if (options.schema) {
+        let definitionsDir = path.resolve("src", "raw_types_schemats");
+        assert(fs.statSync(definitionsDir).isDirectory());
+        if (!options.silent) {
+            console.log(`Creating TypeScript definitions from schema of database ${dataset.database}…`);
+        }
+        const dbConnectionString = `postgres://${process.env.PGUSER}:${process.env.PGPASSWORD}@${process.env.PGHOST}:${process.env.PGPORT}/${dataset.database}`;
+        let definitionFilePath = path.join(definitionsDir, `${dataset.database}.ts`);
+        execSync(`npx schemats generate -c ${dbConnectionString} -s ${dataset.schema} -o ${definitionFilePath}`, {
+            // cwd: dataDir,
+            env: process.env,
+            encoding: "utf-8",
+            // stdio: ["ignore", "ignore", "pipe"],
+        });
+        const definition = fs.readFileSync(definitionFilePath, { encoding: "utf8" });
+        const definitionRepaired = definition
+            .replace(/\r\n/g, "\n")
+            .replace(/AUTO-GENERATED FILE @ \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/, "AUTO-GENERATED FILE");
+        fs.writeFileSync(definitionFilePath, definitionRepaired);
+        definitionsDir = path.resolve("src", "raw_types_kysely");
+        definitionFilePath = path.join(definitionsDir, `${dataset.database}.ts`);
+        execSync(`kysely-codegen --url ${dbConnectionString} --schema=${dataset.schema} --out-file=${definitionFilePath}`, {
+            // cwd: dataDir,
+            env: process.env,
+            encoding: "utf-8",
+            // stdio: ["ignore", "ignore", "pipe"],
+        });
+    }
+}
+async function retrieveOpenData() {
+    const dataDir = options.dataDir;
+    assert(dataDir, "Missing argument: data directory");
+    process.env = {
+        ...process.env,
+        PGHOST: process.env.PGHOST || config.db.host,
+        PGPORT: process.env.PGPORT || config.db.port,
+        PGUSER: process.env.PGUSER || config.db.user,
+        PGPASSWORD: process.env.PGPASSWORD || config.db.password,
+    };
+    assert(process.env.PGHOST
+        && process.env.PGPORT
+        && process.env.PGUSER
+        && process.env.PGPASSWORD, "Missing database configuration: environment variables PGHOST, PGPORT, PGUSER and PGPASSWORD or TRICOTEUSES_SENAT_DB_* in .env file");
+    const choosenDatasets = getChosenFromEnabledDatasets(options.categories);
+    // await Promise.all(choosenDatasets.map(dataset => retrieveDataset(dataDir, dataset)))
+    for (const dataset of choosenDatasets) {
+        await retrieveDataset(dataDir, dataset);
+    }
+}
+retrieveOpenData()
+    .then(() => process.exit(0))
+    .catch((error) => {
+    console.log(error);
+    process.exit(1);
+});