@tricoteuses/senat 1.3.2 → 1.3.5

package/lib/scripts/convert_data.js

@@ -1,132 +1,127 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const assert_1 = __importDefault(require("assert"));
-const command_line_args_1 = __importDefault(require("command-line-args"));
-const fs_extra_1 = __importDefault(require("fs-extra"));
-const path_1 = __importDefault(require("path"));
-const datasets_1 = require("../datasets");
-const loaders_1 = require("../loaders");
-const model_1 = require("../model");
-const dosleg_1 = require("../model/dosleg");
-const datautil_1 = require("./datautil");
-const cli_helpers_1 = require("./shared/cli_helpers");
-const util_1 = require("./shared/util");
-const optionsDefinitions = [...cli_helpers_1.commonOptions];
-const options = (0, command_line_args_1.default)(optionsDefinitions);
+import assert from "assert";
+import commandLineArgs from "command-line-args";
+import fs from "fs-extra";
+import path from "path";
+import { datasets, EnabledDatasets, getEnabledDatasets } from "../datasets";
+import { DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER, } from "../loaders";
+import { findAllAmendements, findAllCirconscriptions, findAllLois, findAllOrganismes, findAllQuestions, findAllSens, } from "../model";
+import { findSenatRapportUrls, findSenatTexteUrls } from "../model/dosleg";
+import { formatToFourDigitSession, SIGNET_STRUCTURE_REGEXP, UNDEFINED_SESSION, } from "./datautil";
+import { commonOptions } from "./shared/cli_helpers";
+import { ensureAndClearDir } from "./shared/util";
+const optionsDefinitions = [...commonOptions];
+const options = commandLineArgs(optionsDefinitions);
 const SENAT_TEXTE_XML_BASE_URL = "https://www.senat.fr/akomantoso/";
 const SENAT_TEXTE_BASE_URL = "https://www.senat.fr/leg/";
 const SENAT_EXPOSE_DES_MOTIFS_BASE_URL = "https://www.senat.fr/leg/exposes-des-motifs/";
 const SENAT_RAPPORT_BASE_URL = "https://www.senat.fr/rap/";
 async function convertData() {
-    const enabledDatasets = (0, datasets_1.getEnabledDatasets)(options["categories"]);
+    const enabledDatasets = getEnabledDatasets(options["categories"]);
     const dataDir = options["dataDir"];
-    (0, assert_1.default)(dataDir, "Missing argument: data directory");
+    assert(dataDir, "Missing argument: data directory");
     console.time("data transformation time");
-    if (enabledDatasets & datasets_1.EnabledDatasets.Ameli) {
-        const dataset = datasets_1.datasets.ameli;
+    if (enabledDatasets & EnabledDatasets.Ameli) {
+        const dataset = datasets.ameli;
         if (!options["silent"]) {
             console.log(`Converting database ${dataset.database} data into files…`);
         }
-        const ameliReorganizedRootDir = path_1.default.join(dataDir, dataset.database);
-        (0, util_1.ensureAndClearDir)(ameliReorganizedRootDir);
-        for await (const amendement of (0, model_1.findAllAmendements)()) {
+        const ameliReorganizedRootDir = path.join(dataDir, dataset.database);
+        ensureAndClearDir(ameliReorganizedRootDir);
+        for await (const amendement of findAllAmendements()) {
             if (options["verbose"]) {
                 console.log(`Converting ${amendement.numero} file…`);
             }
-            const session = String(amendement.session) || datautil_1.UNDEFINED_SESSION;
+            const session = String(amendement.session) || UNDEFINED_SESSION;
             const signetDossierLegislatif = amendement.signet_dossier_legislatif ||
                 `${amendement.nature_texte}-${amendement.numero_texte}`.toLowerCase();
-            const ameliReorganizedDir = path_1.default.join(ameliReorganizedRootDir, session, signetDossierLegislatif);
-            fs_extra_1.default.ensureDirSync(ameliReorganizedDir);
+            const ameliReorganizedDir = path.join(ameliReorganizedRootDir, session, signetDossierLegislatif);
+            fs.ensureDirSync(ameliReorganizedDir);
             const amendementFileName = `${amendement.numero}.json`;
-            fs_extra_1.default.writeJSONSync(path_1.default.join(ameliReorganizedDir, amendementFileName), amendement, { spaces: 2 });
+            fs.writeJSONSync(path.join(ameliReorganizedDir, amendementFileName), amendement, { spaces: 2 });
         }
     }
-    if (enabledDatasets & datasets_1.EnabledDatasets.DosLeg) {
-        const dataset = datasets_1.datasets.dosleg;
+    if (enabledDatasets & EnabledDatasets.DosLeg) {
+        const dataset = datasets.dosleg;
         if (!options["silent"]) {
             console.log(`Converting database ${dataset.database} data into files…`);
         }
-        const doslegReorganizedRootDir = path_1.default.join(dataDir, dataset.database);
-        const dossiersReorganizedDir = path_1.default.join(doslegReorganizedRootDir, loaders_1.DOSLEG_DOSSIERS_FOLDER);
-        (0, util_1.ensureAndClearDir)(doslegReorganizedRootDir);
-        (0, util_1.ensureAndClearDir)(dossiersReorganizedDir);
-        for await (const loi of (0, model_1.findAllLois)()) {
+        const doslegReorganizedRootDir = path.join(dataDir, dataset.database);
+        const dossiersReorganizedDir = path.join(doslegReorganizedRootDir, DOSLEG_DOSSIERS_FOLDER);
+        ensureAndClearDir(doslegReorganizedRootDir);
+        ensureAndClearDir(dossiersReorganizedDir);
+        for await (const loi of findAllLois()) {
             if (options["verbose"]) {
                 console.log(`Converting ${loi.signet} file…`);
             }
-            let loiReorganizedDir = path_1.default.join(dossiersReorganizedDir, datautil_1.UNDEFINED_SESSION);
-            const signetParts = datautil_1.SIGNET_STRUCTURE_REGEXP.exec(loi.signet)?.groups;
+            let loiReorganizedDir = path.join(dossiersReorganizedDir, UNDEFINED_SESSION);
+            const signetParts = SIGNET_STRUCTURE_REGEXP.exec(loi.signet)?.groups;
             if (signetParts && "session" in signetParts) {
                 const { session } = signetParts;
-                const formattedSession = (0, datautil_1.formatToFourDigitSession)(session);
-                loiReorganizedDir = path_1.default.join(dossiersReorganizedDir, formattedSession);
+                const formattedSession = formatToFourDigitSession(session);
+                loiReorganizedDir = path.join(dossiersReorganizedDir, formattedSession);
             }
-            fs_extra_1.default.ensureDirSync(loiReorganizedDir);
+            fs.ensureDirSync(loiReorganizedDir);
             const loiFileName = `${loi.signet}.json`;
-            fs_extra_1.default.writeJSONSync(path_1.default.join(loiReorganizedDir, loiFileName), loi, {
+            fs.writeJSONSync(path.join(loiReorganizedDir, loiFileName), loi, {
                 spaces: 2,
             });
         }
         await convertTexteUrls(dataDir);
         await convertRapportUrls(dataDir);
     }
-    if (enabledDatasets & datasets_1.EnabledDatasets.Questions) {
-        const dataset = datasets_1.datasets.questions;
+    if (enabledDatasets & EnabledDatasets.Questions) {
+        const dataset = datasets.questions;
         if (!options["silent"]) {
             console.log(`Converting database ${dataset.database} data into files…`);
         }
-        const questionsReorganizedRootDir = path_1.default.join(dataDir, dataset.database);
-        (0, util_1.ensureAndClearDir)(questionsReorganizedRootDir);
-        for await (const question of (0, model_1.findAllQuestions)()) {
+        const questionsReorganizedRootDir = path.join(dataDir, dataset.database);
+        ensureAndClearDir(questionsReorganizedRootDir);
+        for await (const question of findAllQuestions()) {
             if (options["verbose"]) {
                 console.log(`Converting ${question.reference} file…`);
             }
             const legislature = question.legislature ? question.legislature : 0;
-            const questionReorganizedDir = path_1.default.join(questionsReorganizedRootDir, String(legislature));
-            fs_extra_1.default.ensureDirSync(questionReorganizedDir);
+            const questionReorganizedDir = path.join(questionsReorganizedRootDir, String(legislature));
+            fs.ensureDirSync(questionReorganizedDir);
             const questionFileName = `${question.reference}.json`;
-            fs_extra_1.default.writeJSONSync(path_1.default.join(questionReorganizedDir, questionFileName), question, { spaces: 2 });
+            fs.writeJSONSync(path.join(questionReorganizedDir, questionFileName), question, { spaces: 2 });
         }
     }
-    if (enabledDatasets & datasets_1.EnabledDatasets.Sens) {
-        const dataset = datasets_1.datasets.sens;
+    if (enabledDatasets & EnabledDatasets.Sens) {
+        const dataset = datasets.sens;
         if (!options["silent"]) {
             console.log(`Converting database ${dataset.database} data into files…`);
         }
-        const sensReorganizedRootDir = path_1.default.join(dataDir, dataset.database);
-        const senateursReorganizedDir = path_1.default.join(sensReorganizedRootDir, loaders_1.SENS_SENATEURS_FOLDER);
-        const circonscriptionsReorganizedDir = path_1.default.join(sensReorganizedRootDir, loaders_1.SENS_CIRCONSCRIPTIONS_FOLDER);
-        const organismesReorganizedDir = path_1.default.join(sensReorganizedRootDir, loaders_1.SENS_ORGANISMES_FOLDER);
-        (0, util_1.ensureAndClearDir)(sensReorganizedRootDir);
-        (0, util_1.ensureAndClearDir)(senateursReorganizedDir);
-        (0, util_1.ensureAndClearDir)(circonscriptionsReorganizedDir);
-        (0, util_1.ensureAndClearDir)(organismesReorganizedDir);
-        for await (const sen of (0, model_1.findAllSens)()) {
+        const sensReorganizedRootDir = path.join(dataDir, dataset.database);
+        const senateursReorganizedDir = path.join(sensReorganizedRootDir, SENS_SENATEURS_FOLDER);
+        const circonscriptionsReorganizedDir = path.join(sensReorganizedRootDir, SENS_CIRCONSCRIPTIONS_FOLDER);
+        const organismesReorganizedDir = path.join(sensReorganizedRootDir, SENS_ORGANISMES_FOLDER);
+        ensureAndClearDir(sensReorganizedRootDir);
+        ensureAndClearDir(senateursReorganizedDir);
+        ensureAndClearDir(circonscriptionsReorganizedDir);
+        ensureAndClearDir(organismesReorganizedDir);
+        for await (const sen of findAllSens()) {
             if (options["verbose"]) {
                 console.log(`Converting ${sen.matricule} file…`);
             }
             const senFileName = `${sen.matricule}.json`;
-            fs_extra_1.default.writeJSONSync(path_1.default.join(senateursReorganizedDir, senFileName), sen, {
+            fs.writeJSONSync(path.join(senateursReorganizedDir, senFileName), sen, {
                 spaces: 2,
             });
         }
-        for await (const circonscription of (0, model_1.findAllCirconscriptions)()) {
+        for await (const circonscription of findAllCirconscriptions()) {
             if (options["verbose"]) {
                 console.log(`Converting ${circonscription.identifiant} file…`);
             }
             const circonscriptionFileName = `${circonscription.identifiant}.json`;
-            fs_extra_1.default.writeJSONSync(path_1.default.join(circonscriptionsReorganizedDir, circonscriptionFileName), circonscription, { spaces: 2 });
+            fs.writeJSONSync(path.join(circonscriptionsReorganizedDir, circonscriptionFileName), circonscription, { spaces: 2 });
         }
-        for await (const organisme of (0, model_1.findAllOrganismes)()) {
+        for await (const organisme of findAllOrganismes()) {
             if (options["verbose"]) {
                 console.log(`Converting ${organisme.code} file…`);
             }
             const organismeFileName = `${organisme.code}.json`;
-            fs_extra_1.default.writeJSONSync(path_1.default.join(organismesReorganizedDir, organismeFileName), organisme, { spaces: 2 });
+            fs.writeJSONSync(path.join(organismesReorganizedDir, organismeFileName), organisme, { spaces: 2 });
         }
     }
     if (!options["silent"]) {
@@ -134,13 +129,13 @@ async function convertData() {
     }
 }
 async function convertTexteUrls(dataDir) {
-    const textesDir = path_1.default.join(dataDir, loaders_1.TEXTE_FOLDER);
-    fs_extra_1.default.ensureDirSync(textesDir);
-    const originalTextesDir = path_1.default.join(textesDir, loaders_1.TEXTE_ORIGINAL_FOLDER);
-    for await (const texte of (0, dosleg_1.findSenatTexteUrls)(options["sessions"])) {
-        const texteName = path_1.default.parse(texte.url).name;
-        const texteDir = path_1.default.join(originalTextesDir, `${texte.session ?? datautil_1.UNDEFINED_SESSION}`, texteName);
-        fs_extra_1.default.ensureDirSync(texteDir);
+    const textesDir = path.join(dataDir, TEXTE_FOLDER);
+    fs.ensureDirSync(textesDir);
+    const originalTextesDir = path.join(textesDir, TEXTE_ORIGINAL_FOLDER);
+    for await (const texte of findSenatTexteUrls(options["sessions"])) {
+        const texteName = path.parse(texte.url).name;
+        const texteDir = path.join(originalTextesDir, `${texte.session ?? UNDEFINED_SESSION}`, texteName);
+        fs.ensureDirSync(texteDir);
         const metadata = {
             name: texteName,
             session: texte.session,
@@ -151,26 +146,26 @@ async function convertTexteUrls(dataDir) {
             url_html: new URL(`${texteName}.html`, SENAT_TEXTE_BASE_URL),
             url_pdf: new URL(`${texteName}.pdf`, SENAT_TEXTE_BASE_URL),
         };
-        fs_extra_1.default.writeJSONSync(path_1.default.join(texteDir, loaders_1.DOCUMENT_METADATA_FILE), metadata, {
+        fs.writeJSONSync(path.join(texteDir, DOCUMENT_METADATA_FILE), metadata, {
             spaces: 2,
         });
     }
 }
 async function convertRapportUrls(dataDir) {
-    const rapportsDir = path_1.default.join(dataDir, loaders_1.RAPPORT_FOLDER);
-    fs_extra_1.default.ensureDirSync(rapportsDir);
-    for await (const rapport of (0, dosleg_1.findSenatRapportUrls)(options["sessions"])) {
-        const parsedRapportUrl = path_1.default.parse(rapport.url);
+    const rapportsDir = path.join(dataDir, RAPPORT_FOLDER);
+    fs.ensureDirSync(rapportsDir);
+    for await (const rapport of findSenatRapportUrls(options["sessions"])) {
+        const parsedRapportUrl = path.parse(rapport.url);
         const rapportName = parsedRapportUrl.name;
-        const rapportDir = path_1.default.join(rapportsDir, `${rapport.session ?? datautil_1.UNDEFINED_SESSION}`, rapportName);
-        fs_extra_1.default.ensureDirSync(rapportDir);
+        const rapportDir = path.join(rapportsDir, `${rapport.session ?? UNDEFINED_SESSION}`, rapportName);
+        fs.ensureDirSync(rapportDir);
         const rapportHtmlUrlBase = `${rapportName}_mono.html`;
-        const rapportHtmlUrl = path_1.default.format({
+        const rapportHtmlUrl = path.format({
             dir: parsedRapportUrl.dir,
             base: rapportHtmlUrlBase,
         });
         const rapportPdfUrlBase = `${rapportName}1.pdf`;
-        const rapportPdfUrl = path_1.default.format({
+        const rapportPdfUrl = path.format({
             dir: parsedRapportUrl.dir,
             base: rapportPdfUrlBase,
         });
@@ -180,7 +175,7 @@ async function convertRapportUrls(dataDir) {
             url_html: new URL(rapportHtmlUrl, SENAT_RAPPORT_BASE_URL),
             url_pdf: new URL(rapportPdfUrl, SENAT_RAPPORT_BASE_URL),
         };
-        fs_extra_1.default.writeJSONSync(path_1.default.join(rapportDir, loaders_1.DOCUMENT_METADATA_FILE), metadata, {
+        fs.writeJSONSync(path.join(rapportDir, DOCUMENT_METADATA_FILE), metadata, {
             spaces: 2,
         });
     }

package/lib/scripts/datautil.js

@@ -1,21 +1,17 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.UNDEFINED_SESSION = exports.AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP = exports.AKN_IDENTIFICATION_STRUCTURE_REGEXP = exports.SIGNET_STRUCTURE_REGEXP = void 0;
-exports.formatToFourDigitSession = formatToFourDigitSession;
-const luxon_1 = require("luxon");
-luxon_1.Settings.twoDigitCutoffYear = 50;
-exports.SIGNET_STRUCTURE_REGEXP = /^(?<type>[a-z]+)(?<session>\d{2,4})-?(?<numTexte>\d+)?/;
-exports.AKN_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{4}-\d{4})\/?(?<numTexte>\d+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/;
-exports.AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{2,4})\/?(?<numTexte>[a-zA-Z0-9]+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/;
-exports.UNDEFINED_SESSION = "0";
-function formatToFourDigitSession(session) {
+import { DateTime, Settings } from "luxon";
+Settings.twoDigitCutoffYear = 50;
+export const SIGNET_STRUCTURE_REGEXP = /^(?<type>[a-z]+)(?<session>\d{2,4})-?(?<numTexte>\d+)?/;
+export const AKN_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{4}-\d{4})\/?(?<numTexte>\d+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/;
+export const AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{2,4})\/?(?<numTexte>[a-zA-Z0-9]+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/;
+export const UNDEFINED_SESSION = "0";
+export function formatToFourDigitSession(session) {
     if (session.length >= 2) {
         const sessionFirstTwoDigits = session.substring(0, 2);
         const sessionLastTwoDigits = session.substring(session.length - 2);
         const twoDigitSession = parseInt(sessionFirstTwoDigits) === parseInt(sessionLastTwoDigits) - 1
             ? sessionFirstTwoDigits
             : sessionLastTwoDigits;
-        return luxon_1.DateTime.fromFormat(String(twoDigitSession), "yy").toFormat("yyyy");
+        return DateTime.fromFormat(String(twoDigitSession), "yy").toFormat("yyyy");
     }
-    return exports.UNDEFINED_SESSION;
+    return UNDEFINED_SESSION;
 }

package/lib/scripts/parse_textes.js

@@ -1,44 +1,39 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const assert_1 = __importDefault(require("assert"));
-const command_line_args_1 = __importDefault(require("command-line-args"));
-const fs_extra_1 = __importDefault(require("fs-extra"));
-const path_1 = __importDefault(require("path"));
-const loaders_1 = require("../loaders");
-const texte_1 = require("../model/texte");
-const cli_helpers_1 = require("./shared/cli_helpers");
-const util_1 = require("./shared/util");
-const optionsDefinitions = [...cli_helpers_1.commonOptions];
-const options = (0, command_line_args_1.default)(optionsDefinitions);
+import assert from "assert";
+import commandLineArgs from "command-line-args";
+import fs from "fs-extra";
+import path from "path";
+import { iterFilePaths, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER, TEXTE_TRANSFORMED_FOLDER, } from "../loaders";
+import { parseExposeDesMotifsFromFile, parseTexteFromFile, } from "../model/texte";
+import { commonOptions } from "./shared/cli_helpers";
+import { ensureAndClearDir } from "./shared/util";
+const optionsDefinitions = [...commonOptions];
+const options = commandLineArgs(optionsDefinitions);
 async function main() {
     const dataDir = options["dataDir"];
-    (0, assert_1.default)(dataDir, "Missing argument: data directory");
-    const transformedTextesDir = path_1.default.join(options["dataDir"], loaders_1.TEXTE_FOLDER, loaders_1.TEXTE_TRANSFORMED_FOLDER);
-    (0, util_1.ensureAndClearDir)(transformedTextesDir);
-    for (const filePath of (0, loaders_1.iterFilePaths)(path_1.default.join(dataDir, loaders_1.TEXTE_FOLDER, loaders_1.TEXTE_ORIGINAL_FOLDER))) {
-        const parsedFilePath = path_1.default.parse(filePath);
+    assert(dataDir, "Missing argument: data directory");
+    const transformedTextesDir = path.join(options["dataDir"], TEXTE_FOLDER, TEXTE_TRANSFORMED_FOLDER);
+    ensureAndClearDir(transformedTextesDir);
+    for (const filePath of iterFilePaths(path.join(dataDir, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER))) {
+        const parsedFilePath = path.parse(filePath);
         if (parsedFilePath.ext !== ".xml") {
             continue;
         }
-        const texteDirFromOriginal = parsedFilePath.dir.substring(filePath.indexOf(loaders_1.TEXTE_ORIGINAL_FOLDER) + loaders_1.TEXTE_ORIGINAL_FOLDER.length);
-        const transformedTexteDir = path_1.default.join(transformedTextesDir, texteDirFromOriginal);
-        fs_extra_1.default.ensureDirSync(transformedTexteDir);
+        const texteDirFromOriginal = parsedFilePath.dir.substring(filePath.indexOf(TEXTE_ORIGINAL_FOLDER) + TEXTE_ORIGINAL_FOLDER.length);
+        const transformedTexteDir = path.join(transformedTextesDir, texteDirFromOriginal);
+        fs.ensureDirSync(transformedTexteDir);
         if (!options["silent"]) {
             console.log(`Parsing texte ${parsedFilePath.name}.xml…`);
         }
-        const parsedTexte = await (0, texte_1.parseTexteFromFile)(filePath);
+        const parsedTexte = await parseTexteFromFile(filePath);
         const exposeDesMotifsFileName = `${parsedFilePath.name}-expose`;
-        const exposeDesMotifsFilePath = path_1.default.join(parsedFilePath.dir, `${exposeDesMotifsFileName}.html`);
-        if (parsedTexte && fs_extra_1.default.existsSync(exposeDesMotifsFilePath)) {
+        const exposeDesMotifsFilePath = path.join(parsedFilePath.dir, `${exposeDesMotifsFileName}.html`);
+        if (parsedTexte && fs.existsSync(exposeDesMotifsFilePath)) {
             if (!options["silent"]) {
                 console.log(`Parsing exposé des motifs ${exposeDesMotifsFileName}.html…`);
             }
-            parsedTexte.exposeDesMotifs = await (0, texte_1.parseExposeDesMotifsFromFile)(exposeDesMotifsFilePath);
+            parsedTexte.exposeDesMotifs = await parseExposeDesMotifsFromFile(exposeDesMotifsFilePath);
         }
-        fs_extra_1.default.writeJSONSync(path_1.default.join(transformedTexteDir, `${parsedFilePath.name}.json`), parsedTexte, { spaces: 2 });
+        fs.writeJSONSync(path.join(transformedTexteDir, `${parsedFilePath.name}.json`), parsedTexte, { spaces: 2 });
     }
 }
 main()

package/lib/scripts/retrieve_documents.js

@@ -1,19 +1,14 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const assert_1 = __importDefault(require("assert"));
-const command_line_args_1 = __importDefault(require("command-line-args"));
-const fs_extra_1 = __importDefault(require("fs-extra"));
-const path_1 = __importDefault(require("path"));
-const loaders_1 = require("../loaders");
-const texte_1 = require("../model/texte");
-const datautil_1 = require("./datautil");
-const cli_helpers_1 = require("./shared/cli_helpers");
-const util_1 = require("./shared/util");
+import assert from "assert";
+import commandLineArgs from "command-line-args";
+import fs from "fs-extra";
+import path from "path";
+import { iterLoadSenatDossiersLegislatifsRapportUrls, iterLoadSenatDossiersLegislatifsTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER, TEXTE_TRANSFORMED_FOLDER, } from "../loaders";
+import { parseExposeDesMotifs, parseTexte, parseTexteFromFile, } from "../model/texte";
+import { UNDEFINED_SESSION } from "./datautil";
+import { commonOptions } from "./shared/cli_helpers";
+import { ensureAndClearDir, fetchWithRetry, isOptionEmptyOrHasValue, } from "./shared/util";
 const optionsDefinitions = [
-    ...cli_helpers_1.commonOptions,
+    ...commonOptions,
     {
         help: "sessions of textes to retrieve; leave empty for all",
         multiple: true,
@@ -44,14 +39,14 @@ const optionsDefinitions = [
         type: Boolean,
     },
 ];
-const options = (0, command_line_args_1.default)(optionsDefinitions);
+const options = commandLineArgs(optionsDefinitions);
 const textDecoder = new TextDecoder("utf8");
 async function retrieveDocument(documentUrl) {
     if (!options["silent"]) {
         console.log(`Retrieving document ${documentUrl}…`);
     }
     try {
-        const response = await (0, util_1.fetchWithRetry)(documentUrl);
+        const response = await fetchWithRetry(documentUrl);
         if (!response.ok) {
             if (response.status === 404) {
                 console.warn(`Texte ${documentUrl} not found`);
@@ -69,36 +64,36 @@ async function retrieveDocument(documentUrl) {
     }
 }
 async function retrieveTextes(dataDir) {
-    const textesDir = path_1.default.join(dataDir, loaders_1.TEXTE_FOLDER);
-    fs_extra_1.default.ensureDirSync(textesDir);
-    const originalTextesDir = path_1.default.join(textesDir, loaders_1.TEXTE_ORIGINAL_FOLDER);
-    const transformedTextesDir = path_1.default.join(textesDir, loaders_1.TEXTE_TRANSFORMED_FOLDER);
+    const textesDir = path.join(dataDir, TEXTE_FOLDER);
+    fs.ensureDirSync(textesDir);
+    const originalTextesDir = path.join(textesDir, TEXTE_ORIGINAL_FOLDER);
+    const transformedTextesDir = path.join(textesDir, TEXTE_TRANSFORMED_FOLDER);
     if (options["parseDocuments"]) {
-        (0, util_1.ensureAndClearDir)(transformedTextesDir);
+        ensureAndClearDir(transformedTextesDir);
     }
     let retrievedTextesCount = 0;
     const texteUrlsNotFoundOrError = [];
     const texteUrlsParseError = [];
     for (const session of options["sessions"]) {
-        for (const { item: texteMetadata, } of (0, loaders_1.iterLoadSenatDossiersLegislatifsTexteUrls)(dataDir, session)) {
-            const texteDir = path_1.default.join(originalTextesDir, `${texteMetadata.session ?? datautil_1.UNDEFINED_SESSION}`, texteMetadata.name);
-            fs_extra_1.default.ensureDirSync(texteDir);
+        for (const { item: texteMetadata, } of iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session)) {
+            const texteDir = path.join(originalTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name);
+            fs.ensureDirSync(texteDir);
             let exposeDesMotifsContent = null;
             if (texteMetadata.url_expose_des_motifs) {
                 if (!options["silent"]) {
                     console.log("Retrieving exposé des motifs…");
                 }
-                const exposeDesMotifsPath = path_1.default.join(texteDir, `${texteMetadata.name}-expose.html`);
+                const exposeDesMotifsPath = path.join(texteDir, `${texteMetadata.name}-expose.html`);
                 exposeDesMotifsContent = await retrieveDocument(texteMetadata.url_expose_des_motifs.toString());
                 if (!exposeDesMotifsContent) {
                     continue;
                 }
-                fs_extra_1.default.writeFileSync(exposeDesMotifsPath, Buffer.from(exposeDesMotifsContent));
+                fs.writeFileSync(exposeDesMotifsPath, Buffer.from(exposeDesMotifsContent));
             }
-            if ((0, util_1.isOptionEmptyOrHasValue)(options["formats"], "xml")) {
-                const textePath = path_1.default.join(texteDir, `${texteMetadata.name}.xml`);
+            if (isOptionEmptyOrHasValue(options["formats"], "xml")) {
+                const textePath = path.join(texteDir, `${texteMetadata.name}.xml`);
                 let texteBuffer = null;
-                if (!options["force"] && fs_extra_1.default.existsSync(textePath)) {
+                if (!options["force"] && fs.existsSync(textePath)) {
                     if (!options["silent"]) {
                         console.info(`Already retrieved texte ${textePath}…`);
                     }
@@ -109,7 +104,7 @@ async function retrieveTextes(dataDir) {
                         texteUrlsNotFoundOrError.push(texteMetadata.url_xml);
                         continue;
                     }
-                    fs_extra_1.default.writeFileSync(textePath, Buffer.from(texteBuffer));
+                    fs.writeFileSync(textePath, Buffer.from(texteBuffer));
                     retrievedTextesCount++;
                 }
                 if (options["parseDocuments"]) {
@@ -119,10 +114,10 @@ async function retrieveTextes(dataDir) {
                     let parsedTexte = null;
                     if (texteBuffer) {
                         const texteXml = textDecoder.decode(texteBuffer);
-                        parsedTexte = (0, texte_1.parseTexte)(texteXml);
+                        parsedTexte = parseTexte(texteXml);
                     }
                     else {
-                        parsedTexte = await (0, texte_1.parseTexteFromFile)(textePath);
+                        parsedTexte = await parseTexteFromFile(textePath);
                     }
                     if (!parsedTexte) {
                         texteUrlsParseError.push(texteMetadata.url_xml);
@@ -134,16 +129,16 @@ async function retrieveTextes(dataDir) {
                         }
                         const exposeDesMotifsHtml = textDecoder.decode(exposeDesMotifsContent);
                         parsedTexte.exposeDesMotifs =
-                            (0, texte_1.parseExposeDesMotifs)(exposeDesMotifsHtml);
+                            parseExposeDesMotifs(exposeDesMotifsHtml);
                     }
-                    const transformedTexteDir = path_1.default.join(transformedTextesDir, `${texteMetadata.session ?? datautil_1.UNDEFINED_SESSION}`, texteMetadata.name);
-                    fs_extra_1.default.ensureDirSync(transformedTexteDir);
-                    fs_extra_1.default.writeJSONSync(path_1.default.join(transformedTexteDir, `${texteMetadata.name}.json`), parsedTexte, { spaces: 2 });
+                    const transformedTexteDir = path.join(transformedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name);
+                    fs.ensureDirSync(transformedTexteDir);
+                    fs.writeJSONSync(path.join(transformedTexteDir, `${texteMetadata.name}.json`), parsedTexte, { spaces: 2 });
                 }
             }
-            if ((0, util_1.isOptionEmptyOrHasValue)(options["formats"], "html")) {
-                const textePath = path_1.default.join(texteDir, `${texteMetadata.name}.html`);
-                if (!options["force"] && fs_extra_1.default.existsSync(textePath)) {
+            if (isOptionEmptyOrHasValue(options["formats"], "html")) {
+                const textePath = path.join(texteDir, `${texteMetadata.name}.html`);
+                if (!options["force"] && fs.existsSync(textePath)) {
                     if (!options["silent"]) {
                         console.info(`Already retrieved texte ${textePath}…`);
                     }
@@ -154,13 +149,13 @@ async function retrieveTextes(dataDir) {
                         texteUrlsNotFoundOrError.push(texteMetadata.url_html);
                         continue;
                     }
-                    fs_extra_1.default.writeFileSync(textePath, Buffer.from(texteBuffer));
+                    fs.writeFileSync(textePath, Buffer.from(texteBuffer));
                     retrievedTextesCount++;
                 }
             }
-            if ((0, util_1.isOptionEmptyOrHasValue)(options["formats"], "pdf")) {
-                const textePath = path_1.default.join(texteDir, `${texteMetadata.name}.pdf`);
-                if (!options["force"] && fs_extra_1.default.existsSync(textePath)) {
+            if (isOptionEmptyOrHasValue(options["formats"], "pdf")) {
+                const textePath = path.join(texteDir, `${texteMetadata.name}.pdf`);
+                if (!options["force"] && fs.existsSync(textePath)) {
                     if (!options["silent"]) {
                         console.info(`Already retrieved texte ${textePath}…`);
                     }
@@ -171,7 +166,7 @@ async function retrieveTextes(dataDir) {
                         texteUrlsNotFoundOrError.push(texteMetadata.url_pdf);
                         continue;
                     }
-                    fs_extra_1.default.writeFileSync(textePath, Buffer.from(texteBuffer));
+                    fs.writeFileSync(textePath, Buffer.from(texteBuffer));
                     retrievedTextesCount++;
                 }
             }
@@ -186,17 +181,17 @@ async function retrieveTextes(dataDir) {
     }
 }
 async function retrieveRapports(dataDir) {
-    const rapportsDir = path_1.default.join(dataDir, loaders_1.RAPPORT_FOLDER);
-    fs_extra_1.default.ensureDirSync(rapportsDir);
+    const rapportsDir = path.join(dataDir, RAPPORT_FOLDER);
+    fs.ensureDirSync(rapportsDir);
     let retrievedRapportsCount = 0;
     const rapportUrlsNotFoundOrError = [];
     for (const session of options["sessions"]) {
-        for (const { item: rapportMetadata, } of (0, loaders_1.iterLoadSenatDossiersLegislatifsRapportUrls)(dataDir, session)) {
-            const rapportDir = path_1.default.join(rapportsDir, `${rapportMetadata.session ?? datautil_1.UNDEFINED_SESSION}`, rapportMetadata.name);
-            fs_extra_1.default.ensureDirSync(rapportDir);
-            if ((0, util_1.isOptionEmptyOrHasValue)(options["formats"], "html")) {
-                const rapportPath = path_1.default.join(rapportDir, `${rapportMetadata.name}.html`);
-                if (!options["force"] && fs_extra_1.default.existsSync(rapportPath)) {
+        for (const { item: rapportMetadata, } of iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session)) {
+            const rapportDir = path.join(rapportsDir, `${rapportMetadata.session ?? UNDEFINED_SESSION}`, rapportMetadata.name);
+            fs.ensureDirSync(rapportDir);
+            if (isOptionEmptyOrHasValue(options["formats"], "html")) {
+                const rapportPath = path.join(rapportDir, `${rapportMetadata.name}.html`);
+                if (!options["force"] && fs.existsSync(rapportPath)) {
                     if (!options["silent"]) {
                         console.info(`Already retrieved rapport ${rapportPath}…`);
                     }
@@ -207,12 +202,12 @@ async function retrieveRapports(dataDir) {
                     rapportUrlsNotFoundOrError.push(rapportMetadata.url_html);
                     continue;
                 }
-                fs_extra_1.default.writeFileSync(rapportPath, Buffer.from(rapportBuffer));
+                fs.writeFileSync(rapportPath, Buffer.from(rapportBuffer));
                 retrievedRapportsCount++;
             }
-            if ((0, util_1.isOptionEmptyOrHasValue)(options["formats"], "pdf")) {
-                const rapportPath = path_1.default.join(rapportDir, `${rapportMetadata.name}.pdf`);
-                if (!options["force"] && fs_extra_1.default.existsSync(rapportPath)) {
+            if (isOptionEmptyOrHasValue(options["formats"], "pdf")) {
+                const rapportPath = path.join(rapportDir, `${rapportMetadata.name}.pdf`);
+                if (!options["force"] && fs.existsSync(rapportPath)) {
                     if (!options["silent"]) {
                         console.info(`Already retrieved rapport ${rapportPath}…`);
                     }
@@ -223,7 +218,7 @@ async function retrieveRapports(dataDir) {
                     rapportUrlsNotFoundOrError.push(rapportMetadata.url_pdf);
                     continue;
                 }
-                fs_extra_1.default.writeFileSync(rapportPath, Buffer.from(rapportBuffer));
+                fs.writeFileSync(rapportPath, Buffer.from(rapportBuffer));
                 retrievedRapportsCount++;
             }
         }
@@ -235,12 +230,12 @@ async function retrieveRapports(dataDir) {
 }
 async function main() {
     const dataDir = options["dataDir"];
-    (0, assert_1.default)(dataDir, "Missing argument: data directory");
+    assert(dataDir, "Missing argument: data directory");
     console.time("documents processing time");
-    if ((0, util_1.isOptionEmptyOrHasValue)(options["types"], "textes")) {
+    if (isOptionEmptyOrHasValue(options["types"], "textes")) {
         await retrieveTextes(dataDir);
     }
-    if ((0, util_1.isOptionEmptyOrHasValue)(options["types"], "rapports")) {
+    if (isOptionEmptyOrHasValue(options["types"], "rapports")) {
         await retrieveRapports(dataDir);
     }
     if (!options["silent"]) {