@tricoteuses/senat 1.3.1 → 1.3.4

package/lib/scripts/retrieve_documents.mjs

@@ -1,249 +0,0 @@
-import assert from "assert";
-import commandLineArgs from "command-line-args";
-import fs from "fs-extra";
-import path from "path";
-import { iterLoadSenatDossiersLegislatifsRapportUrls, iterLoadSenatDossiersLegislatifsTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER, TEXTE_TRANSFORMED_FOLDER, } from "../loaders";
-import { parseExposeDesMotifs, parseTexte, parseTexteFromFile } from "../model/texte";
-import { UNDEFINED_SESSION } from "./datautil";
-import { commonOptions } from "./shared/cli_helpers";
-import { ensureAndClearDir, fetchWithRetry, isOptionEmptyOrHasValue } from "./shared/util";
-const optionsDefinitions = [
-    ...commonOptions,
-    {
-        help: "sessions of textes to retrieve; leave empty for all",
-        multiple: true,
-        name: "sessions",
-        type: String,
-    },
-    {
-        help: "parse and convert documents into JSON (textes only for now, requires format xml)",
-        name: "parseDocuments",
-        type: Boolean,
-    },
-    {
-        alias: "F",
-        help: "formats of documents to retrieve (xml/html/pdf for textes, html/pdf for rapports); leave empty for all",
-        multiple: true,
-        name: "formats",
-        type: String,
-    },
-    {
-        help: "types of documents to retrieve (textes/rapports); leave empty for all",
-        multiple: true,
-        name: "types",
-        type: String,
-    },
-    {
-        help: "force retrieve all documents, even already retrieved ones",
-        name: "force",
-        type: Boolean,
-    },
-];
-const options = commandLineArgs(optionsDefinitions);
-const textDecoder = new TextDecoder("utf8");
-async function retrieveDocument(documentUrl) {
-    if (!options.silent) {
-        console.log(`Retrieving document ${documentUrl}…`);
-    }
-    try {
-        const response = await fetchWithRetry(documentUrl);
-        if (!response.ok) {
-            if (response.status === 404) {
-                console.warn(`Texte ${documentUrl} not found`);
-            }
-            else {
-                console.error(`An error occurred while retrieving texte ${documentUrl}: ${response.status}`);
-            }
-            return null;
-        }
-        return response.arrayBuffer();
-    }
-    catch (error) {
-        console.error(error.message);
-        return null;
-    }
-}
-async function retrieveTextes(dataDir) {
-    const textesDir = path.join(dataDir, TEXTE_FOLDER);
-    fs.ensureDirSync(textesDir);
-    const originalTextesDir = path.join(textesDir, TEXTE_ORIGINAL_FOLDER);
-    const transformedTextesDir = path.join(textesDir, TEXTE_TRANSFORMED_FOLDER);
-    if (options.parseDocuments) {
-        ensureAndClearDir(transformedTextesDir);
-    }
-    let retrievedTextesCount = 0;
-    const texteUrlsNotFoundOrError = [];
-    const texteUrlsParseError = [];
-    for (const session of options.sessions) {
-        for (const { item: texteMetadata } of iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session)) {
-            const texteDir = path.join(originalTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name);
-            fs.ensureDirSync(texteDir);
-            let exposeDesMotifsContent = null;
-            if (texteMetadata.url_expose_des_motifs) {
-                if (!options.silent) {
-                    console.log("Retrieving exposé des motifs…");
-                }
-                const exposeDesMotifsPath = path.join(texteDir, `${texteMetadata.name}-expose.html`);
-                exposeDesMotifsContent = await retrieveDocument(texteMetadata.url_expose_des_motifs.toString());
-                if (!exposeDesMotifsContent) {
-                    continue;
-                }
-                fs.writeFileSync(exposeDesMotifsPath, Buffer.from(exposeDesMotifsContent));
-            }
-            if (isOptionEmptyOrHasValue(options.formats, "xml")) {
-                const textePath = path.join(texteDir, `${texteMetadata.name}.xml`);
-                let texteBuffer = null;
-                if (!options.force && fs.existsSync(textePath)) {
-                    if (!options.silent) {
-                        console.info(`Already retrieved texte ${textePath}…`);
-                    }
-                }
-                else {
-                    texteBuffer = await retrieveDocument(texteMetadata.url_xml.toString());
-                    if (!texteBuffer) {
-                        texteUrlsNotFoundOrError.push(texteMetadata.url_xml);
-                        continue;
-                    }
-                    fs.writeFileSync(textePath, Buffer.from(texteBuffer));
-                    retrievedTextesCount++;
-                }
-                if (options.parseDocuments) {
-                    if (!options.silent) {
-                        console.log(`Parsing texte ${texteMetadata.name}.xml…`);
-                    }
-                    let parsedTexte = null;
-                    if (texteBuffer) {
-                        const texteXml = textDecoder.decode(texteBuffer);
-                        parsedTexte = parseTexte(texteXml);
-                    }
-                    else {
-                        parsedTexte = await parseTexteFromFile(textePath);
-                    }
-                    if (!parsedTexte) {
-                        texteUrlsParseError.push(texteMetadata.url_xml);
-                        continue;
-                    }
-                    if (exposeDesMotifsContent) {
-                        if (!options.silent) {
-                            console.log("Parsing exposé des motifs…");
-                        }
-                        const exposeDesMotifsHtml = textDecoder.decode(exposeDesMotifsContent);
-                        parsedTexte.exposeDesMotifs = parseExposeDesMotifs(exposeDesMotifsHtml);
-                    }
-                    const transformedTexteDir = path.join(transformedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name);
-                    fs.ensureDirSync(transformedTexteDir);
-                    fs.writeJSONSync(path.join(transformedTexteDir, `${texteMetadata.name}.json`), parsedTexte, { spaces: 2 });
-                }
-            }
-            if (isOptionEmptyOrHasValue(options.formats, "html")) {
-                const textePath = path.join(texteDir, `${texteMetadata.name}.html`);
-                if (!options.force && fs.existsSync(textePath)) {
-                    if (!options.silent) {
-                        console.info(`Already retrieved texte ${textePath}…`);
-                    }
-                }
-                else {
-                    const texteBuffer = await retrieveDocument(texteMetadata.url_html.toString());
-                    if (!texteBuffer) {
-                        texteUrlsNotFoundOrError.push(texteMetadata.url_html);
-                        continue;
-                    }
-                    fs.writeFileSync(textePath, Buffer.from(texteBuffer));
-                    retrievedTextesCount++;
-                }
-            }
-            if (isOptionEmptyOrHasValue(options.formats, "pdf")) {
-                const textePath = path.join(texteDir, `${texteMetadata.name}.pdf`);
-                if (!options.force && fs.existsSync(textePath)) {
-                    if (!options.silent) {
-                        console.info(`Already retrieved texte ${textePath}…`);
-                    }
-                }
-                else {
-                    const texteBuffer = await retrieveDocument(texteMetadata.url_pdf.toString());
-                    if (!texteBuffer) {
-                        texteUrlsNotFoundOrError.push(texteMetadata.url_pdf);
-                        continue;
-                    }
-                    fs.writeFileSync(textePath, Buffer.from(texteBuffer));
-                    retrievedTextesCount++;
-                }
-            }
-        }
-    }
-    if (options.verbose) {
-        console.log(`${retrievedTextesCount} textes retrieved`);
-        console.log(`${texteUrlsNotFoundOrError.length} textes failed to be retrieved with URLs ${texteUrlsNotFoundOrError.join(", ")}`);
-        if (options.parseDocuments) {
-            console.log(`${texteUrlsParseError.length} textes failed to be parsed with URLs ${texteUrlsParseError.join(", ")}`);
-        }
-    }
-}
-async function retrieveRapports(dataDir) {
-    const rapportsDir = path.join(dataDir, RAPPORT_FOLDER);
-    fs.ensureDirSync(rapportsDir);
-    let retrievedRapportsCount = 0;
-    const rapportUrlsNotFoundOrError = [];
-    for (const session of options.sessions) {
-        for (const { item: rapportMetadata } of iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session)) {
-            const rapportDir = path.join(rapportsDir, `${rapportMetadata.session ?? UNDEFINED_SESSION}`, rapportMetadata.name);
-            fs.ensureDirSync(rapportDir);
-            if (isOptionEmptyOrHasValue(options.formats, "html")) {
-                const rapportPath = path.join(rapportDir, `${rapportMetadata.name}.html`);
-                if (!options.force && fs.existsSync(rapportPath)) {
-                    if (!options.silent) {
-                        console.info(`Already retrieved rapport ${rapportPath}…`);
-                    }
-                    continue;
-                }
-                const rapportBuffer = await retrieveDocument(rapportMetadata.url_html.toString());
-                if (!rapportBuffer) {
-                    rapportUrlsNotFoundOrError.push(rapportMetadata.url_html);
-                    continue;
-                }
-                fs.writeFileSync(rapportPath, Buffer.from(rapportBuffer));
-                retrievedRapportsCount++;
-            }
-            if (isOptionEmptyOrHasValue(options.formats, "pdf")) {
-                const rapportPath = path.join(rapportDir, `${rapportMetadata.name}.pdf`);
-                if (!options.force && fs.existsSync(rapportPath)) {
-                    if (!options.silent) {
-                        console.info(`Already retrieved rapport ${rapportPath}…`);
-                    }
-                    continue;
-                }
-                const rapportBuffer = await retrieveDocument(rapportMetadata.url_pdf.toString());
-                if (!rapportBuffer) {
-                    rapportUrlsNotFoundOrError.push(rapportMetadata.url_pdf);
-                    continue;
-                }
-                fs.writeFileSync(rapportPath, Buffer.from(rapportBuffer));
-                retrievedRapportsCount++;
-            }
-        }
-    }
-    if (options.verbose) {
-        console.log(`${retrievedRapportsCount} rapports retrieved`);
-        console.log(`${rapportUrlsNotFoundOrError.length} rapports failed with URLs ${rapportUrlsNotFoundOrError.join(", ")}`);
-    }
-}
-async function main() {
-    const dataDir = options.dataDir;
-    assert(dataDir, "Missing argument: data directory");
-    console.time("documents processing time");
-    if (isOptionEmptyOrHasValue(options.types, "textes")) {
-        await retrieveTextes(dataDir);
-    }
-    if (isOptionEmptyOrHasValue(options.types, "rapports")) {
-        await retrieveRapports(dataDir);
-    }
-    if (!options.silent) {
-        console.timeEnd("documents processing time");
-    }
-}
-main()
-    .then(() => process.exit(0))
-    .catch((error) => {
-    console.log(error);
-    process.exit(1);
-});

package/lib/scripts/retrieve_documents.ts

@@ -1,298 +0,0 @@
-import assert from "assert"
-import commandLineArgs from "command-line-args"
-import fs from "fs-extra"
-import path from "path"
-
-import {
-  iterLoadSenatDossiersLegislatifsRapportUrls,
-  iterLoadSenatDossiersLegislatifsTexteUrls,
-  RAPPORT_FOLDER,
-  TEXTE_FOLDER,
-  TEXTE_ORIGINAL_FOLDER,
-  TEXTE_TRANSFORMED_FOLDER,
-} from "../loaders"
-import { parseExposeDesMotifs, parseTexte, parseTexteFromFile } from "../model/texte"
-import { UNDEFINED_SESSION } from "./datautil"
-import { commonOptions } from "./shared/cli_helpers"
-import { ensureAndClearDir, fetchWithRetry, isOptionEmptyOrHasValue } from "./shared/util"
-
-const optionsDefinitions = [
-  ...commonOptions,
-  {
-    help: "sessions of textes to retrieve; leave empty for all",
-    multiple: true,
-    name: "sessions",
-    type: String,
-  },
-  {
-    help: "parse and convert documents into JSON (textes only for now, requires format xml)",
-    name: "parseDocuments",
-    type: Boolean,
-  },
-  {
-    alias: "F",
-    help: "formats of documents to retrieve (xml/html/pdf for textes, html/pdf for rapports); leave empty for all",
-    multiple: true,
-    name: "formats",
-    type: String,
-  },
-  {
-    help: "types of documents to retrieve (textes/rapports); leave empty for all",
-    multiple: true,
-    name: "types",
-    type: String,
-  },
-  {
-    help: "force retrieve all documents, even already retrieved ones",
-    name: "force",
-    type: Boolean,
-  },
-]
-const options = commandLineArgs(optionsDefinitions)
-
-const textDecoder = new TextDecoder("utf8")
-
-async function retrieveDocument (documentUrl: string): Promise<ArrayBuffer | null> {
-  if (!options.silent) {
-    console.log(`Retrieving document ${documentUrl}…`)
-  }
-
-  try {
-    const response = await fetchWithRetry(documentUrl)
-    if (!response.ok) {
-      if (response.status === 404) {
-        console.warn(`Texte ${documentUrl} not found`)
-      } else {
-        console.error(`An error occurred while retrieving texte ${documentUrl}: ${response.status}`)
-      }
-      return null
-    }
-    return response.arrayBuffer()
-  } catch (error: any) {
-    console.error(error.message)
-    return null
-  }
-}
-
-async function retrieveTextes (dataDir: string) {
-  const textesDir = path.join(dataDir, TEXTE_FOLDER)
-  fs.ensureDirSync(textesDir)
-
-  const originalTextesDir = path.join(textesDir, TEXTE_ORIGINAL_FOLDER)
-  const transformedTextesDir = path.join(textesDir, TEXTE_TRANSFORMED_FOLDER)
-  if (options.parseDocuments) {
-    ensureAndClearDir(transformedTextesDir)
-  }
-
-  let retrievedTextesCount = 0
-  const texteUrlsNotFoundOrError = []
-  const texteUrlsParseError = []
-
-  for (const session of options.sessions) {
-    for (const { item: texteMetadata } of iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session)) {
-      const texteDir = path.join(originalTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name)
-      fs.ensureDirSync(texteDir)
-
-      let exposeDesMotifsContent = null
-      if (texteMetadata.url_expose_des_motifs) {
-        if (!options.silent) {
-          console.log("Retrieving exposé des motifs…")
-        }
-
-        const exposeDesMotifsPath = path.join(texteDir, `${texteMetadata.name}-expose.html`)
-
-        exposeDesMotifsContent = await retrieveDocument(texteMetadata.url_expose_des_motifs.toString())
-        if (!exposeDesMotifsContent) {
-          continue
-        }
-        fs.writeFileSync(exposeDesMotifsPath, Buffer.from(exposeDesMotifsContent))
-      }
-
-      if (isOptionEmptyOrHasValue(options.formats, "xml")) {
-        const textePath = path.join(texteDir, `${texteMetadata.name}.xml`)
-        let texteBuffer = null
-
-        if (!options.force && fs.existsSync(textePath)) {
-          if (!options.silent) {
-            console.info(`Already retrieved texte ${textePath}…`)
-          }
-        } else {
-          texteBuffer = await retrieveDocument(texteMetadata.url_xml.toString())
-          if (!texteBuffer) {
-            texteUrlsNotFoundOrError.push(texteMetadata.url_xml)
-            continue
-          }
-          fs.writeFileSync(textePath, Buffer.from(texteBuffer))
-          retrievedTextesCount++
-        }
-
-        if (options.parseDocuments) {
-          if (!options.silent) {
-            console.log(`Parsing texte ${texteMetadata.name}.xml…`)
-          }
-
-          let parsedTexte = null
-          if (texteBuffer) {
-            const texteXml = textDecoder.decode(texteBuffer)
-            parsedTexte = parseTexte(texteXml)
-          } else {
-            parsedTexte = await parseTexteFromFile(textePath)
-          }
-
-          if (!parsedTexte) {
-            texteUrlsParseError.push(texteMetadata.url_xml)
-            continue
-          }
-
-          if (exposeDesMotifsContent) {
-            if (!options.silent) {
-              console.log("Parsing exposé des motifs…")
-            }
-
-            const exposeDesMotifsHtml = textDecoder.decode(exposeDesMotifsContent)
-            parsedTexte.exposeDesMotifs = parseExposeDesMotifs(exposeDesMotifsHtml)
-          }
-
-          const transformedTexteDir
-            = path.join(transformedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name)
-          fs.ensureDirSync(transformedTexteDir)
-
-          fs.writeJSONSync(path.join(transformedTexteDir, `${texteMetadata.name}.json`), parsedTexte, { spaces: 2 })
-        }
-      }
-
-      if (isOptionEmptyOrHasValue(options.formats, "html")) {
-        const textePath = path.join(texteDir, `${texteMetadata.name}.html`)
-
-        if (!options.force && fs.existsSync(textePath)) {
-          if (!options.silent) {
-            console.info(`Already retrieved texte ${textePath}…`)
-          }
-        } else {
-
-          const texteBuffer = await retrieveDocument(texteMetadata.url_html.toString())
-          if (!texteBuffer) {
-            texteUrlsNotFoundOrError.push(texteMetadata.url_html)
-            continue
-          }
-          fs.writeFileSync(textePath, Buffer.from(texteBuffer))
-          retrievedTextesCount++
-        }
-      }
-
-      if (isOptionEmptyOrHasValue(options.formats, "pdf")) {
-        const textePath = path.join(texteDir, `${texteMetadata.name}.pdf`)
-
-        if (!options.force && fs.existsSync(textePath)) {
-          if (!options.silent) {
-            console.info(`Already retrieved texte ${textePath}…`)
-          }
-        } else {
-
-          const texteBuffer = await retrieveDocument(texteMetadata.url_pdf.toString())
-          if (!texteBuffer) {
-            texteUrlsNotFoundOrError.push(texteMetadata.url_pdf)
-            continue
-          }
-          fs.writeFileSync(textePath, Buffer.from(texteBuffer))
-          retrievedTextesCount++
-        }
-      }
-    }
-  }
-
-  if (options.verbose) {
-    console.log(`${retrievedTextesCount} textes retrieved`)
-    console.log(
-      `${texteUrlsNotFoundOrError.length} textes failed to be retrieved with URLs ${texteUrlsNotFoundOrError.join(", ")}`
-    )
-    if (options.parseDocuments) {
-      console.log(`${texteUrlsParseError.length} textes failed to be parsed with URLs ${texteUrlsParseError.join(", ")}`)
-    }
-  }
-}
-
-async function retrieveRapports (dataDir: string) {
-  const rapportsDir = path.join(dataDir, RAPPORT_FOLDER)
-  fs.ensureDirSync(rapportsDir)
-
-  let retrievedRapportsCount = 0
-  const rapportUrlsNotFoundOrError = []
-
-  for (const session of options.sessions) {
-    for (const { item: rapportMetadata } of iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session)) {
-      const rapportDir = path.join(rapportsDir, `${rapportMetadata.session ?? UNDEFINED_SESSION}`, rapportMetadata.name)
-      fs.ensureDirSync(rapportDir)
-
-      if (isOptionEmptyOrHasValue(options.formats, "html")) {
-        const rapportPath = path.join(rapportDir, `${rapportMetadata.name}.html`)
-
-        if (!options.force && fs.existsSync(rapportPath)) {
-          if (!options.silent) {
-            console.info(`Already retrieved rapport ${rapportPath}…`)
-          }
-          continue
-        }
-
-        const rapportBuffer = await retrieveDocument(rapportMetadata.url_html.toString())
-        if (!rapportBuffer) {
-          rapportUrlsNotFoundOrError.push(rapportMetadata.url_html)
-          continue
-        }
-        fs.writeFileSync(rapportPath, Buffer.from(rapportBuffer))
-        retrievedRapportsCount++
-      }
-
-      if (isOptionEmptyOrHasValue(options.formats, "pdf")) {
-        const rapportPath = path.join(rapportDir, `${rapportMetadata.name}.pdf`)
-
-        if (!options.force && fs.existsSync(rapportPath)) {
-          if (!options.silent) {
-            console.info(`Already retrieved rapport ${rapportPath}…`)
-          }
-          continue
-        }
-
-        const rapportBuffer = await retrieveDocument(rapportMetadata.url_pdf.toString())
-        if (!rapportBuffer) {
-          rapportUrlsNotFoundOrError.push(rapportMetadata.url_pdf)
-          continue
-        }
-        fs.writeFileSync(rapportPath, Buffer.from(rapportBuffer))
-        retrievedRapportsCount++
-      }
-    }
-  }
-
-  if (options.verbose) {
-    console.log(`${retrievedRapportsCount} rapports retrieved`)
-    console.log(
-      `${rapportUrlsNotFoundOrError.length} rapports failed with URLs ${rapportUrlsNotFoundOrError.join(", ")}`
-    )
-  }
-}
-
-async function main() {
-  const dataDir = options.dataDir
-  assert(dataDir, "Missing argument: data directory")
-
-  console.time("documents processing time")
-
-  if (isOptionEmptyOrHasValue(options.types, "textes")) {
-    await retrieveTextes(dataDir)
-  }
-  if (isOptionEmptyOrHasValue(options.types, "rapports")) {
-    await retrieveRapports(dataDir)
-  }
-
-  if (!options.silent) {
-      console.timeEnd("documents processing time")
-  }
-}
-
-main()
-  .then(() => process.exit(0))
-  .catch((error) => {
-    console.log(error)
-    process.exit(1)
-  })