@tricoteuses/senat 1.1.2 → 1.3.2

package/lib/scripts/datautil.js

@@ -0,0 +1,21 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UNDEFINED_SESSION = exports.AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP = exports.AKN_IDENTIFICATION_STRUCTURE_REGEXP = exports.SIGNET_STRUCTURE_REGEXP = void 0;
+exports.formatToFourDigitSession = formatToFourDigitSession;
+const luxon_1 = require("luxon");
+luxon_1.Settings.twoDigitCutoffYear = 50;
+exports.SIGNET_STRUCTURE_REGEXP = /^(?<type>[a-z]+)(?<session>\d{2,4})-?(?<numTexte>\d+)?/;
+exports.AKN_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{4}-\d{4})\/?(?<numTexte>\d+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/;
+exports.AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{2,4})\/?(?<numTexte>[a-zA-Z0-9]+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/;
+exports.UNDEFINED_SESSION = "0";
+function formatToFourDigitSession(session) {
+    if (session.length >= 2) {
+        const sessionFirstTwoDigits = session.substring(0, 2);
+        const sessionLastTwoDigits = session.substring(session.length - 2);
+        const twoDigitSession = parseInt(sessionFirstTwoDigits) === parseInt(sessionLastTwoDigits) - 1
+            ? sessionFirstTwoDigits
+            : sessionLastTwoDigits;
+        return luxon_1.DateTime.fromFormat(String(twoDigitSession), "yy").toFormat("yyyy");
+    }
+    return exports.UNDEFINED_SESSION;
+}

package/lib/scripts/parse_textes.js

@@ -0,0 +1,49 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const assert_1 = __importDefault(require("assert"));
+const command_line_args_1 = __importDefault(require("command-line-args"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const path_1 = __importDefault(require("path"));
+const loaders_1 = require("../loaders");
+const texte_1 = require("../model/texte");
+const cli_helpers_1 = require("./shared/cli_helpers");
+const util_1 = require("./shared/util");
+const optionsDefinitions = [...cli_helpers_1.commonOptions];
+const options = (0, command_line_args_1.default)(optionsDefinitions);
+async function main() {
+    const dataDir = options["dataDir"];
+    (0, assert_1.default)(dataDir, "Missing argument: data directory");
+    const transformedTextesDir = path_1.default.join(options["dataDir"], loaders_1.TEXTE_FOLDER, loaders_1.TEXTE_TRANSFORMED_FOLDER);
+    (0, util_1.ensureAndClearDir)(transformedTextesDir);
+    for (const filePath of (0, loaders_1.iterFilePaths)(path_1.default.join(dataDir, loaders_1.TEXTE_FOLDER, loaders_1.TEXTE_ORIGINAL_FOLDER))) {
+        const parsedFilePath = path_1.default.parse(filePath);
+        if (parsedFilePath.ext !== ".xml") {
+            continue;
+        }
+        const texteDirFromOriginal = parsedFilePath.dir.substring(filePath.indexOf(loaders_1.TEXTE_ORIGINAL_FOLDER) + loaders_1.TEXTE_ORIGINAL_FOLDER.length);
+        const transformedTexteDir = path_1.default.join(transformedTextesDir, texteDirFromOriginal);
+        fs_extra_1.default.ensureDirSync(transformedTexteDir);
+        if (!options["silent"]) {
+            console.log(`Parsing texte ${parsedFilePath.name}.xml…`);
+        }
+        const parsedTexte = await (0, texte_1.parseTexteFromFile)(filePath);
+        const exposeDesMotifsFileName = `${parsedFilePath.name}-expose`;
+        const exposeDesMotifsFilePath = path_1.default.join(parsedFilePath.dir, `${exposeDesMotifsFileName}.html`);
+        if (parsedTexte && fs_extra_1.default.existsSync(exposeDesMotifsFilePath)) {
+            if (!options["silent"]) {
+                console.log(`Parsing exposé des motifs ${exposeDesMotifsFileName}.html…`);
+            }
+            parsedTexte.exposeDesMotifs = await (0, texte_1.parseExposeDesMotifsFromFile)(exposeDesMotifsFilePath);
+        }
+        fs_extra_1.default.writeJSONSync(path_1.default.join(transformedTexteDir, `${parsedFilePath.name}.json`), parsedTexte, { spaces: 2 });
+    }
+}
+main()
+    .then(() => process.exit(0))
+    .catch((error) => {
+    console.log(error);
+    process.exit(1);
+});

package/lib/scripts/retrieve_documents.js

@@ -0,0 +1,255 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const assert_1 = __importDefault(require("assert"));
+const command_line_args_1 = __importDefault(require("command-line-args"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const path_1 = __importDefault(require("path"));
+const loaders_1 = require("../loaders");
+const texte_1 = require("../model/texte");
+const datautil_1 = require("./datautil");
+const cli_helpers_1 = require("./shared/cli_helpers");
+const util_1 = require("./shared/util");
+const optionsDefinitions = [
+    ...cli_helpers_1.commonOptions,
+    {
+        help: "sessions of textes to retrieve; leave empty for all",
+        multiple: true,
+        name: "sessions",
+        type: String,
+    },
+    {
+        help: "parse and convert documents into JSON (textes only for now, requires format xml)",
+        name: "parseDocuments",
+        type: Boolean,
+    },
+    {
+        alias: "F",
+        help: "formats of documents to retrieve (xml/html/pdf for textes, html/pdf for rapports); leave empty for all",
+        multiple: true,
+        name: "formats",
+        type: String,
+    },
+    {
+        help: "types of documents to retrieve (textes/rapports); leave empty for all",
+        multiple: true,
+        name: "types",
+        type: String,
+    },
+    {
+        help: "force retrieve all documents, even already retrieved ones",
+        name: "force",
+        type: Boolean,
+    },
+];
+const options = (0, command_line_args_1.default)(optionsDefinitions);
+const textDecoder = new TextDecoder("utf8");
+async function retrieveDocument(documentUrl) {
+    if (!options["silent"]) {
+        console.log(`Retrieving document ${documentUrl}…`);
+    }
+    try {
+        const response = await (0, util_1.fetchWithRetry)(documentUrl);
+        if (!response.ok) {
+            if (response.status === 404) {
+                console.warn(`Texte ${documentUrl} not found`);
+            }
+            else {
+                console.error(`An error occurred while retrieving texte ${documentUrl}: ${response.status}`);
+            }
+            return null;
+        }
+        return response.arrayBuffer();
+    }
+    catch (error) {
+        console.error(error.message);
+        return null;
+    }
+}
+async function retrieveTextes(dataDir) {
+    const textesDir = path_1.default.join(dataDir, loaders_1.TEXTE_FOLDER);
+    fs_extra_1.default.ensureDirSync(textesDir);
+    const originalTextesDir = path_1.default.join(textesDir, loaders_1.TEXTE_ORIGINAL_FOLDER);
+    const transformedTextesDir = path_1.default.join(textesDir, loaders_1.TEXTE_TRANSFORMED_FOLDER);
+    if (options["parseDocuments"]) {
+        (0, util_1.ensureAndClearDir)(transformedTextesDir);
+    }
+    let retrievedTextesCount = 0;
+    const texteUrlsNotFoundOrError = [];
+    const texteUrlsParseError = [];
+    for (const session of options["sessions"]) {
+        for (const { item: texteMetadata, } of (0, loaders_1.iterLoadSenatDossiersLegislatifsTexteUrls)(dataDir, session)) {
+            const texteDir = path_1.default.join(originalTextesDir, `${texteMetadata.session ?? datautil_1.UNDEFINED_SESSION}`, texteMetadata.name);
+            fs_extra_1.default.ensureDirSync(texteDir);
+            let exposeDesMotifsContent = null;
+            if (texteMetadata.url_expose_des_motifs) {
+                if (!options["silent"]) {
+                    console.log("Retrieving exposé des motifs…");
+                }
+                const exposeDesMotifsPath = path_1.default.join(texteDir, `${texteMetadata.name}-expose.html`);
+                exposeDesMotifsContent = await retrieveDocument(texteMetadata.url_expose_des_motifs.toString());
+                if (!exposeDesMotifsContent) {
+                    continue;
+                }
+                fs_extra_1.default.writeFileSync(exposeDesMotifsPath, Buffer.from(exposeDesMotifsContent));
+            }
+            if ((0, util_1.isOptionEmptyOrHasValue)(options["formats"], "xml")) {
+                const textePath = path_1.default.join(texteDir, `${texteMetadata.name}.xml`);
+                let texteBuffer = null;
+                if (!options["force"] && fs_extra_1.default.existsSync(textePath)) {
+                    if (!options["silent"]) {
+                        console.info(`Already retrieved texte ${textePath}…`);
+                    }
+                }
+                else {
+                    texteBuffer = await retrieveDocument(texteMetadata.url_xml.toString());
+                    if (!texteBuffer) {
+                        texteUrlsNotFoundOrError.push(texteMetadata.url_xml);
+                        continue;
+                    }
+                    fs_extra_1.default.writeFileSync(textePath, Buffer.from(texteBuffer));
+                    retrievedTextesCount++;
+                }
+                if (options["parseDocuments"]) {
+                    if (!options["silent"]) {
+                        console.log(`Parsing texte ${texteMetadata.name}.xml…`);
+                    }
+                    let parsedTexte = null;
+                    if (texteBuffer) {
+                        const texteXml = textDecoder.decode(texteBuffer);
+                        parsedTexte = (0, texte_1.parseTexte)(texteXml);
+                    }
+                    else {
+                        parsedTexte = await (0, texte_1.parseTexteFromFile)(textePath);
+                    }
+                    if (!parsedTexte) {
+                        texteUrlsParseError.push(texteMetadata.url_xml);
+                        continue;
+                    }
+                    if (exposeDesMotifsContent) {
+                        if (!options["silent"]) {
+                            console.log("Parsing exposé des motifs…");
+                        }
+                        const exposeDesMotifsHtml = textDecoder.decode(exposeDesMotifsContent);
+                        parsedTexte.exposeDesMotifs =
+                            (0, texte_1.parseExposeDesMotifs)(exposeDesMotifsHtml);
+                    }
+                    const transformedTexteDir = path_1.default.join(transformedTextesDir, `${texteMetadata.session ?? datautil_1.UNDEFINED_SESSION}`, texteMetadata.name);
+                    fs_extra_1.default.ensureDirSync(transformedTexteDir);
+                    fs_extra_1.default.writeJSONSync(path_1.default.join(transformedTexteDir, `${texteMetadata.name}.json`), parsedTexte, { spaces: 2 });
+                }
+            }
+            if ((0, util_1.isOptionEmptyOrHasValue)(options["formats"], "html")) {
+                const textePath = path_1.default.join(texteDir, `${texteMetadata.name}.html`);
+                if (!options["force"] && fs_extra_1.default.existsSync(textePath)) {
+                    if (!options["silent"]) {
+                        console.info(`Already retrieved texte ${textePath}…`);
+                    }
+                }
+                else {
+                    const texteBuffer = await retrieveDocument(texteMetadata.url_html.toString());
+                    if (!texteBuffer) {
+                        texteUrlsNotFoundOrError.push(texteMetadata.url_html);
+                        continue;
+                    }
+                    fs_extra_1.default.writeFileSync(textePath, Buffer.from(texteBuffer));
+                    retrievedTextesCount++;
+                }
+            }
+            if ((0, util_1.isOptionEmptyOrHasValue)(options["formats"], "pdf")) {
+                const textePath = path_1.default.join(texteDir, `${texteMetadata.name}.pdf`);
+                if (!options["force"] && fs_extra_1.default.existsSync(textePath)) {
+                    if (!options["silent"]) {
+                        console.info(`Already retrieved texte ${textePath}…`);
+                    }
+                }
+                else {
+                    const texteBuffer = await retrieveDocument(texteMetadata.url_pdf.toString());
+                    if (!texteBuffer) {
+                        texteUrlsNotFoundOrError.push(texteMetadata.url_pdf);
+                        continue;
+                    }
+                    fs_extra_1.default.writeFileSync(textePath, Buffer.from(texteBuffer));
+                    retrievedTextesCount++;
+                }
+            }
+        }
+    }
+    if (options["verbose"]) {
+        console.log(`${retrievedTextesCount} textes retrieved`);
+        console.log(`${texteUrlsNotFoundOrError.length} textes failed to be retrieved with URLs ${texteUrlsNotFoundOrError.join(", ")}`);
+        if (options["parseDocuments"]) {
+            console.log(`${texteUrlsParseError.length} textes failed to be parsed with URLs ${texteUrlsParseError.join(", ")}`);
+        }
+    }
+}
+async function retrieveRapports(dataDir) {
+    const rapportsDir = path_1.default.join(dataDir, loaders_1.RAPPORT_FOLDER);
+    fs_extra_1.default.ensureDirSync(rapportsDir);
+    let retrievedRapportsCount = 0;
+    const rapportUrlsNotFoundOrError = [];
+    for (const session of options["sessions"]) {
+        for (const { item: rapportMetadata, } of (0, loaders_1.iterLoadSenatDossiersLegislatifsRapportUrls)(dataDir, session)) {
+            const rapportDir = path_1.default.join(rapportsDir, `${rapportMetadata.session ?? datautil_1.UNDEFINED_SESSION}`, rapportMetadata.name);
+            fs_extra_1.default.ensureDirSync(rapportDir);
+            if ((0, util_1.isOptionEmptyOrHasValue)(options["formats"], "html")) {
+                const rapportPath = path_1.default.join(rapportDir, `${rapportMetadata.name}.html`);
+                if (!options["force"] && fs_extra_1.default.existsSync(rapportPath)) {
+                    if (!options["silent"]) {
+                        console.info(`Already retrieved rapport ${rapportPath}…`);
+                    }
+                    continue;
+                }
+                const rapportBuffer = await retrieveDocument(rapportMetadata.url_html.toString());
+                if (!rapportBuffer) {
+                    rapportUrlsNotFoundOrError.push(rapportMetadata.url_html);
+                    continue;
+                }
+                fs_extra_1.default.writeFileSync(rapportPath, Buffer.from(rapportBuffer));
+                retrievedRapportsCount++;
+            }
+            if ((0, util_1.isOptionEmptyOrHasValue)(options["formats"], "pdf")) {
+                const rapportPath = path_1.default.join(rapportDir, `${rapportMetadata.name}.pdf`);
+                if (!options["force"] && fs_extra_1.default.existsSync(rapportPath)) {
+                    if (!options["silent"]) {
+                        console.info(`Already retrieved rapport ${rapportPath}…`);
+                    }
+                    continue;
+                }
+                const rapportBuffer = await retrieveDocument(rapportMetadata.url_pdf.toString());
+                if (!rapportBuffer) {
+                    rapportUrlsNotFoundOrError.push(rapportMetadata.url_pdf);
+                    continue;
+                }
+                fs_extra_1.default.writeFileSync(rapportPath, Buffer.from(rapportBuffer));
+                retrievedRapportsCount++;
+            }
+        }
+    }
+    if (options["verbose"]) {
+        console.log(`${retrievedRapportsCount} rapports retrieved`);
+        console.log(`${rapportUrlsNotFoundOrError.length} rapports failed with URLs ${rapportUrlsNotFoundOrError.join(", ")}`);
+    }
+}
+async function main() {
+    const dataDir = options["dataDir"];
+    (0, assert_1.default)(dataDir, "Missing argument: data directory");
+    console.time("documents processing time");
+    if ((0, util_1.isOptionEmptyOrHasValue)(options["types"], "textes")) {
+        await retrieveTextes(dataDir);
+    }
+    if ((0, util_1.isOptionEmptyOrHasValue)(options["types"], "rapports")) {
+        await retrieveRapports(dataDir);
+    }
+    if (!options["silent"]) {
+        console.timeEnd("documents processing time");
+    }
+}
+main()
+    .then(() => process.exit(0))
+    .catch((error) => {
+    console.log(error);
+    process.exit(1);
+});