@tricoteuses/senat 1.1.2 → 1.3.1

package/lib/scripts/retrieve_documents.mjs

@@ -0,0 +1,249 @@
+import assert from "assert";
+import commandLineArgs from "command-line-args";
+import fs from "fs-extra";
+import path from "path";
+import { iterLoadSenatDossiersLegislatifsRapportUrls, iterLoadSenatDossiersLegislatifsTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER, TEXTE_TRANSFORMED_FOLDER, } from "../loaders";
+import { parseExposeDesMotifs, parseTexte, parseTexteFromFile } from "../model/texte";
+import { UNDEFINED_SESSION } from "./datautil";
+import { commonOptions } from "./shared/cli_helpers";
+import { ensureAndClearDir, fetchWithRetry, isOptionEmptyOrHasValue } from "./shared/util";
+const optionsDefinitions = [
+    ...commonOptions,
+    {
+        help: "sessions of textes to retrieve; leave empty for all",
+        multiple: true,
+        name: "sessions",
+        type: String,
+    },
+    {
+        help: "parse and convert documents into JSON (textes only for now, requires format xml)",
+        name: "parseDocuments",
+        type: Boolean,
+    },
+    {
+        alias: "F",
+        help: "formats of documents to retrieve (xml/html/pdf for textes, html/pdf for rapports); leave empty for all",
+        multiple: true,
+        name: "formats",
+        type: String,
+    },
+    {
+        help: "types of documents to retrieve (textes/rapports); leave empty for all",
+        multiple: true,
+        name: "types",
+        type: String,
+    },
+    {
+        help: "force retrieve all documents, even already retrieved ones",
+        name: "force",
+        type: Boolean,
+    },
+];
+const options = commandLineArgs(optionsDefinitions);
+const textDecoder = new TextDecoder("utf8");
+async function retrieveDocument(documentUrl) {
+    if (!options.silent) {
+        console.log(`Retrieving document ${documentUrl}…`);
+    }
+    try {
+        const response = await fetchWithRetry(documentUrl);
+        if (!response.ok) {
+            if (response.status === 404) {
+                console.warn(`Texte ${documentUrl} not found`);
+            }
+            else {
+                console.error(`An error occurred while retrieving texte ${documentUrl}: ${response.status}`);
+            }
+            return null;
+        }
+        return response.arrayBuffer();
+    }
+    catch (error) {
+        console.error(error.message);
+        return null;
+    }
+}
+async function retrieveTextes(dataDir) {
+    const textesDir = path.join(dataDir, TEXTE_FOLDER);
+    fs.ensureDirSync(textesDir);
+    const originalTextesDir = path.join(textesDir, TEXTE_ORIGINAL_FOLDER);
+    const transformedTextesDir = path.join(textesDir, TEXTE_TRANSFORMED_FOLDER);
+    if (options.parseDocuments) {
+        ensureAndClearDir(transformedTextesDir);
+    }
+    let retrievedTextesCount = 0;
+    const texteUrlsNotFoundOrError = [];
+    const texteUrlsParseError = [];
+    for (const session of options.sessions) {
+        for (const { item: texteMetadata } of iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session)) {
+            const texteDir = path.join(originalTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name);
+            fs.ensureDirSync(texteDir);
+            let exposeDesMotifsContent = null;
+            if (texteMetadata.url_expose_des_motifs) {
+                if (!options.silent) {
+                    console.log("Retrieving exposé des motifs…");
+                }
+                const exposeDesMotifsPath = path.join(texteDir, `${texteMetadata.name}-expose.html`);
+                exposeDesMotifsContent = await retrieveDocument(texteMetadata.url_expose_des_motifs.toString());
+                if (!exposeDesMotifsContent) {
+                    continue;
+                }
+                fs.writeFileSync(exposeDesMotifsPath, Buffer.from(exposeDesMotifsContent));
+            }
+            if (isOptionEmptyOrHasValue(options.formats, "xml")) {
+                const textePath = path.join(texteDir, `${texteMetadata.name}.xml`);
+                let texteBuffer = null;
+                if (!options.force && fs.existsSync(textePath)) {
+                    if (!options.silent) {
+                        console.info(`Already retrieved texte ${textePath}…`);
+                    }
+                }
+                else {
+                    texteBuffer = await retrieveDocument(texteMetadata.url_xml.toString());
+                    if (!texteBuffer) {
+                        texteUrlsNotFoundOrError.push(texteMetadata.url_xml);
+                        continue;
+                    }
+                    fs.writeFileSync(textePath, Buffer.from(texteBuffer));
+                    retrievedTextesCount++;
+                }
+                if (options.parseDocuments) {
+                    if (!options.silent) {
+                        console.log(`Parsing texte ${texteMetadata.name}.xml…`);
+                    }
+                    let parsedTexte = null;
+                    if (texteBuffer) {
+                        const texteXml = textDecoder.decode(texteBuffer);
+                        parsedTexte = parseTexte(texteXml);
+                    }
+                    else {
+                        parsedTexte = await parseTexteFromFile(textePath);
+                    }
+                    if (!parsedTexte) {
+                        texteUrlsParseError.push(texteMetadata.url_xml);
+                        continue;
+                    }
+                    if (exposeDesMotifsContent) {
+                        if (!options.silent) {
+                            console.log("Parsing exposé des motifs…");
+                        }
+                        const exposeDesMotifsHtml = textDecoder.decode(exposeDesMotifsContent);
+                        parsedTexte.exposeDesMotifs = parseExposeDesMotifs(exposeDesMotifsHtml);
+                    }
+                    const transformedTexteDir = path.join(transformedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name);
+                    fs.ensureDirSync(transformedTexteDir);
+                    fs.writeJSONSync(path.join(transformedTexteDir, `${texteMetadata.name}.json`), parsedTexte, { spaces: 2 });
+                }
+            }
+            if (isOptionEmptyOrHasValue(options.formats, "html")) {
+                const textePath = path.join(texteDir, `${texteMetadata.name}.html`);
+                if (!options.force && fs.existsSync(textePath)) {
+                    if (!options.silent) {
+                        console.info(`Already retrieved texte ${textePath}…`);
+                    }
+                }
+                else {
+                    const texteBuffer = await retrieveDocument(texteMetadata.url_html.toString());
+                    if (!texteBuffer) {
+                        texteUrlsNotFoundOrError.push(texteMetadata.url_html);
+                        continue;
+                    }
+                    fs.writeFileSync(textePath, Buffer.from(texteBuffer));
+                    retrievedTextesCount++;
+                }
+            }
+            if (isOptionEmptyOrHasValue(options.formats, "pdf")) {
+                const textePath = path.join(texteDir, `${texteMetadata.name}.pdf`);
+                if (!options.force && fs.existsSync(textePath)) {
+                    if (!options.silent) {
+                        console.info(`Already retrieved texte ${textePath}…`);
+                    }
+                }
+                else {
+                    const texteBuffer = await retrieveDocument(texteMetadata.url_pdf.toString());
+                    if (!texteBuffer) {
+                        texteUrlsNotFoundOrError.push(texteMetadata.url_pdf);
+                        continue;
+                    }
+                    fs.writeFileSync(textePath, Buffer.from(texteBuffer));
+                    retrievedTextesCount++;
+                }
+            }
+        }
+    }
+    if (options.verbose) {
+        console.log(`${retrievedTextesCount} textes retrieved`);
+        console.log(`${texteUrlsNotFoundOrError.length} textes failed to be retrieved with URLs ${texteUrlsNotFoundOrError.join(", ")}`);
+        if (options.parseDocuments) {
+            console.log(`${texteUrlsParseError.length} textes failed to be parsed with URLs ${texteUrlsParseError.join(", ")}`);
+        }
+    }
+}
+async function retrieveRapports(dataDir) {
+    const rapportsDir = path.join(dataDir, RAPPORT_FOLDER);
+    fs.ensureDirSync(rapportsDir);
+    let retrievedRapportsCount = 0;
+    const rapportUrlsNotFoundOrError = [];
+    for (const session of options.sessions) {
+        for (const { item: rapportMetadata } of iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session)) {
+            const rapportDir = path.join(rapportsDir, `${rapportMetadata.session ?? UNDEFINED_SESSION}`, rapportMetadata.name);
+            fs.ensureDirSync(rapportDir);
+            if (isOptionEmptyOrHasValue(options.formats, "html")) {
+                const rapportPath = path.join(rapportDir, `${rapportMetadata.name}.html`);
+                if (!options.force && fs.existsSync(rapportPath)) {
+                    if (!options.silent) {
+                        console.info(`Already retrieved rapport ${rapportPath}…`);
+                    }
+                    continue;
+                }
+                const rapportBuffer = await retrieveDocument(rapportMetadata.url_html.toString());
+                if (!rapportBuffer) {
+                    rapportUrlsNotFoundOrError.push(rapportMetadata.url_html);
+                    continue;
+                }
+                fs.writeFileSync(rapportPath, Buffer.from(rapportBuffer));
+                retrievedRapportsCount++;
+            }
+            if (isOptionEmptyOrHasValue(options.formats, "pdf")) {
+                const rapportPath = path.join(rapportDir, `${rapportMetadata.name}.pdf`);
+                if (!options.force && fs.existsSync(rapportPath)) {
+                    if (!options.silent) {
+                        console.info(`Already retrieved rapport ${rapportPath}…`);
+                    }
+                    continue;
+                }
+                const rapportBuffer = await retrieveDocument(rapportMetadata.url_pdf.toString());
+                if (!rapportBuffer) {
+                    rapportUrlsNotFoundOrError.push(rapportMetadata.url_pdf);
+                    continue;
+                }
+                fs.writeFileSync(rapportPath, Buffer.from(rapportBuffer));
+                retrievedRapportsCount++;
+            }
+        }
+    }
+    if (options.verbose) {
+        console.log(`${retrievedRapportsCount} rapports retrieved`);
+        console.log(`${rapportUrlsNotFoundOrError.length} rapports failed with URLs ${rapportUrlsNotFoundOrError.join(", ")}`);
+    }
+}
+async function main() {
+    const dataDir = options.dataDir;
+    assert(dataDir, "Missing argument: data directory");
+    console.time("documents processing time");
+    if (isOptionEmptyOrHasValue(options.types, "textes")) {
+        await retrieveTextes(dataDir);
+    }
+    if (isOptionEmptyOrHasValue(options.types, "rapports")) {
+        await retrieveRapports(dataDir);
+    }
+    if (!options.silent) {
+        console.timeEnd("documents processing time");
+    }
+}
+main()
+    .then(() => process.exit(0))
+    .catch((error) => {
+    console.log(error);
+    process.exit(1);
+});

package/lib/scripts/retrieve_documents.ts

@@ -0,0 +1,298 @@
+import assert from "assert"
+import commandLineArgs from "command-line-args"
+import fs from "fs-extra"
+import path from "path"
+
+import {
+  iterLoadSenatDossiersLegislatifsRapportUrls,
+  iterLoadSenatDossiersLegislatifsTexteUrls,
+  RAPPORT_FOLDER,
+  TEXTE_FOLDER,
+  TEXTE_ORIGINAL_FOLDER,
+  TEXTE_TRANSFORMED_FOLDER,
+} from "../loaders"
+import { parseExposeDesMotifs, parseTexte, parseTexteFromFile } from "../model/texte"
+import { UNDEFINED_SESSION } from "./datautil"
+import { commonOptions } from "./shared/cli_helpers"
+import { ensureAndClearDir, fetchWithRetry, isOptionEmptyOrHasValue } from "./shared/util"
+
+const optionsDefinitions = [
+  ...commonOptions,
+  {
+    help: "sessions of textes to retrieve; leave empty for all",
+    multiple: true,
+    name: "sessions",
+    type: String,
+  },
+  {
+    help: "parse and convert documents into JSON (textes only for now, requires format xml)",
+    name: "parseDocuments",
+    type: Boolean,
+  },
+  {
+    alias: "F",
+    help: "formats of documents to retrieve (xml/html/pdf for textes, html/pdf for rapports); leave empty for all",
+    multiple: true,
+    name: "formats",
+    type: String,
+  },
+  {
+    help: "types of documents to retrieve (textes/rapports); leave empty for all",
+    multiple: true,
+    name: "types",
+    type: String,
+  },
+  {
+    help: "force retrieve all documents, even already retrieved ones",
+    name: "force",
+    type: Boolean,
+  },
+]
+const options = commandLineArgs(optionsDefinitions)
+
+const textDecoder = new TextDecoder("utf8")
+
+async function retrieveDocument (documentUrl: string): Promise<ArrayBuffer | null> {
+  if (!options.silent) {
+    console.log(`Retrieving document ${documentUrl}…`)
+  }
+
+  try {
+    const response = await fetchWithRetry(documentUrl)
+    if (!response.ok) {
+      if (response.status === 404) {
+        console.warn(`Texte ${documentUrl} not found`)
+      } else {
+        console.error(`An error occurred while retrieving texte ${documentUrl}: ${response.status}`)
+      }
+      return null
+    }
+    return response.arrayBuffer()
+  } catch (error: any) {
+    console.error(error.message)
+    return null
+  }
+}
+
+async function retrieveTextes (dataDir: string) {
+  const textesDir = path.join(dataDir, TEXTE_FOLDER)
+  fs.ensureDirSync(textesDir)
+
+  const originalTextesDir = path.join(textesDir, TEXTE_ORIGINAL_FOLDER)
+  const transformedTextesDir = path.join(textesDir, TEXTE_TRANSFORMED_FOLDER)
+  if (options.parseDocuments) {
+    ensureAndClearDir(transformedTextesDir)
+  }
+
+  let retrievedTextesCount = 0
+  const texteUrlsNotFoundOrError = []
+  const texteUrlsParseError = []
+
+  for (const session of options.sessions) {
+    for (const { item: texteMetadata } of iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session)) {
+      const texteDir = path.join(originalTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name)
+      fs.ensureDirSync(texteDir)
+
+      let exposeDesMotifsContent = null
+      if (texteMetadata.url_expose_des_motifs) {
+        if (!options.silent) {
+          console.log("Retrieving exposé des motifs…")
+        }
+
+        const exposeDesMotifsPath = path.join(texteDir, `${texteMetadata.name}-expose.html`)
+
+        exposeDesMotifsContent = await retrieveDocument(texteMetadata.url_expose_des_motifs.toString())
+        if (!exposeDesMotifsContent) {
+          continue
+        }
+        fs.writeFileSync(exposeDesMotifsPath, Buffer.from(exposeDesMotifsContent))
+      }
+
+      if (isOptionEmptyOrHasValue(options.formats, "xml")) {
+        const textePath = path.join(texteDir, `${texteMetadata.name}.xml`)
+        let texteBuffer = null
+
+        if (!options.force && fs.existsSync(textePath)) {
+          if (!options.silent) {
+            console.info(`Already retrieved texte ${textePath}…`)
+          }
+        } else {
+          texteBuffer = await retrieveDocument(texteMetadata.url_xml.toString())
+          if (!texteBuffer) {
+            texteUrlsNotFoundOrError.push(texteMetadata.url_xml)
+            continue
+          }
+          fs.writeFileSync(textePath, Buffer.from(texteBuffer))
+          retrievedTextesCount++
+        }
+
+        if (options.parseDocuments) {
+          if (!options.silent) {
+            console.log(`Parsing texte ${texteMetadata.name}.xml…`)
+          }
+
+          let parsedTexte = null
+          if (texteBuffer) {
+            const texteXml = textDecoder.decode(texteBuffer)
+            parsedTexte = parseTexte(texteXml)
+          } else {
+            parsedTexte = await parseTexteFromFile(textePath)
+          }
+
+          if (!parsedTexte) {
+            texteUrlsParseError.push(texteMetadata.url_xml)
+            continue
+          }
+
+          if (exposeDesMotifsContent) {
+            if (!options.silent) {
+              console.log("Parsing exposé des motifs…")
+            }
+
+            const exposeDesMotifsHtml = textDecoder.decode(exposeDesMotifsContent)
+            parsedTexte.exposeDesMotifs = parseExposeDesMotifs(exposeDesMotifsHtml)
+          }
+
+          const transformedTexteDir
+            = path.join(transformedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name)
+          fs.ensureDirSync(transformedTexteDir)
+
+          fs.writeJSONSync(path.join(transformedTexteDir, `${texteMetadata.name}.json`), parsedTexte, { spaces: 2 })
+        }
+      }
+
+      if (isOptionEmptyOrHasValue(options.formats, "html")) {
+        const textePath = path.join(texteDir, `${texteMetadata.name}.html`)
+
+        if (!options.force && fs.existsSync(textePath)) {
+          if (!options.silent) {
+            console.info(`Already retrieved texte ${textePath}…`)
+          }
+        } else {
+
+          const texteBuffer = await retrieveDocument(texteMetadata.url_html.toString())
+          if (!texteBuffer) {
+            texteUrlsNotFoundOrError.push(texteMetadata.url_html)
+            continue
+          }
+          fs.writeFileSync(textePath, Buffer.from(texteBuffer))
+          retrievedTextesCount++
+        }
+      }
+
+      if (isOptionEmptyOrHasValue(options.formats, "pdf")) {
+        const textePath = path.join(texteDir, `${texteMetadata.name}.pdf`)
+
+        if (!options.force && fs.existsSync(textePath)) {
+          if (!options.silent) {
+            console.info(`Already retrieved texte ${textePath}…`)
+          }
+        } else {
+
+          const texteBuffer = await retrieveDocument(texteMetadata.url_pdf.toString())
+          if (!texteBuffer) {
+            texteUrlsNotFoundOrError.push(texteMetadata.url_pdf)
+            continue
+          }
+          fs.writeFileSync(textePath, Buffer.from(texteBuffer))
+          retrievedTextesCount++
+        }
+      }
+    }
+  }
+
+  if (options.verbose) {
+    console.log(`${retrievedTextesCount} textes retrieved`)
+    console.log(
+      `${texteUrlsNotFoundOrError.length} textes failed to be retrieved with URLs ${texteUrlsNotFoundOrError.join(", ")}`
+    )
+    if (options.parseDocuments) {
+      console.log(`${texteUrlsParseError.length} textes failed to be parsed with URLs ${texteUrlsParseError.join(", ")}`)
+    }
+  }
+}
+
+async function retrieveRapports (dataDir: string) {
+  const rapportsDir = path.join(dataDir, RAPPORT_FOLDER)
+  fs.ensureDirSync(rapportsDir)
+
+  let retrievedRapportsCount = 0
+  const rapportUrlsNotFoundOrError = []
+
+  for (const session of options.sessions) {
+    for (const { item: rapportMetadata } of iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session)) {
+      const rapportDir = path.join(rapportsDir, `${rapportMetadata.session ?? UNDEFINED_SESSION}`, rapportMetadata.name)
+      fs.ensureDirSync(rapportDir)
+
+      if (isOptionEmptyOrHasValue(options.formats, "html")) {
+        const rapportPath = path.join(rapportDir, `${rapportMetadata.name}.html`)
+
+        if (!options.force && fs.existsSync(rapportPath)) {
+          if (!options.silent) {
+            console.info(`Already retrieved rapport ${rapportPath}…`)
+          }
+          continue
+        }
+
+        const rapportBuffer = await retrieveDocument(rapportMetadata.url_html.toString())
+        if (!rapportBuffer) {
+          rapportUrlsNotFoundOrError.push(rapportMetadata.url_html)
+          continue
+        }
+        fs.writeFileSync(rapportPath, Buffer.from(rapportBuffer))
+        retrievedRapportsCount++
+      }
+
+      if (isOptionEmptyOrHasValue(options.formats, "pdf")) {
+        const rapportPath = path.join(rapportDir, `${rapportMetadata.name}.pdf`)
+
+        if (!options.force && fs.existsSync(rapportPath)) {
+          if (!options.silent) {
+            console.info(`Already retrieved rapport ${rapportPath}…`)
+          }
+          continue
+        }
+
+        const rapportBuffer = await retrieveDocument(rapportMetadata.url_pdf.toString())
+        if (!rapportBuffer) {
+          rapportUrlsNotFoundOrError.push(rapportMetadata.url_pdf)
+          continue
+        }
+        fs.writeFileSync(rapportPath, Buffer.from(rapportBuffer))
+        retrievedRapportsCount++
+      }
+    }
+  }
+
+  if (options.verbose) {
+    console.log(`${retrievedRapportsCount} rapports retrieved`)
+    console.log(
+      `${rapportUrlsNotFoundOrError.length} rapports failed with URLs ${rapportUrlsNotFoundOrError.join(", ")}`
+    )
+  }
+}
+
+async function main() {
+  const dataDir = options.dataDir
+  assert(dataDir, "Missing argument: data directory")
+
+  console.time("documents processing time")
+
+  if (isOptionEmptyOrHasValue(options.types, "textes")) {
+    await retrieveTextes(dataDir)
+  }
+  if (isOptionEmptyOrHasValue(options.types, "rapports")) {
+    await retrieveRapports(dataDir)
+  }
+
+  if (!options.silent) {
+      console.timeEnd("documents processing time")
+  }
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch((error) => {
+    console.log(error)
+    process.exit(1)
+  })