@tricoteuses/senat 2.22.11 → 2.22.13

package/lib/raw_types_schemats/sens.js

@@ -0,0 +1,2 @@
+/* tslint:disable */
+export {};

package/lib/scripts/convert_data.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/scripts/convert_data.js

@@ -0,0 +1,354 @@
+import assert from "assert";
+import commandLineArgs from "command-line-args";
+import fs from "fs-extra";
+import path from "path";
+import pLimit from "p-limit";
+import * as git from "../git";
+import { datasets, EnabledDatasets, getEnabledDatasets } from "../datasets";
+import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, SCRUTINS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER, } from "../loaders";
+import { findAllAmendements, findAllCirconscriptions, findAllDebats, findAllDossiers, findAllScrutins, findAllOrganismes, findAllQuestions, findAllSens, findAllTextes, findAllRapports, } from "../model";
+import { processRapport, processTexte } from "./retrieve_documents";
+import { buildActesLegislatifs } from "../model/dosleg";
+import { UNDEFINED_SESSION } from "../types/sessions";
+import { getSessionFromDate, getSessionFromSignet } from "./datautil";
+import { commonOptions } from "./shared/cli_helpers";
+import { ensureAndClearDir } from "./shared/util";
+const optionsDefinitions = [...commonOptions];
+const options = commandLineArgs(optionsDefinitions);
+const SENAT_TEXTE_XML_BASE_URL = "https://www.senat.fr/akomantoso/";
+const SENAT_TEXTE_BASE_URL = "https://www.senat.fr/leg/";
+const SENAT_EXPOSE_DES_MOTIFS_BASE_URL = "https://www.senat.fr/leg/exposes-des-motifs/";
+const SENAT_RAPPORT_BASE_URL = "https://www.senat.fr/rap/";
+function commitGit(datasetDir, options, exitCode) {
+    if (options.commit) {
+        const errorCode = git.commitAndPush(datasetDir, "Nouvelle moisson", options.remote);
+        if ((exitCode === 10 && errorCode !== 10) || (exitCode === 0 && errorCode !== 0 && errorCode !== 10)) {
+            exitCode = errorCode;
+        }
+    }
+    return exitCode;
+}
+async function convertData() {
+    const dataDir = options["dataDir"];
+    assert(dataDir, "Missing argument: data directory");
+    const enabledDatasets = getEnabledDatasets(options["categories"]);
+    console.time("data transformation time");
+    let exitCode = 0;
+    if (enabledDatasets & EnabledDatasets.Ameli) {
+        try {
+            await convertDatasetAmeli(dataDir, options);
+            const ameliDir = path.join(dataDir, datasets.ameli.database);
+            exitCode = commitGit(ameliDir, options, exitCode);
+        }
+        catch (error) {
+            console.error(`Error converting Ameli dataset:`, error);
+        }
+    }
+    if (enabledDatasets & EnabledDatasets.Debats) {
+        try {
+            await convertDatasetDebats(dataDir, options);
+            const debatsDir = path.join(dataDir, datasets.debats.database);
+            exitCode = commitGit(debatsDir, options, exitCode);
+        }
+        catch (error) {
+            console.error(`Error converting Debats dataset:`, error);
+        }
+    }
+    if (enabledDatasets & EnabledDatasets.DosLeg) {
+        try {
+            await convertDatasetDosLeg(dataDir, options);
+            const doslegDir = path.join(dataDir, datasets.dosleg.database);
+            exitCode = commitGit(doslegDir, options, exitCode);
+        }
+        catch (error) {
+            console.error(`Error converting DosLeg dataset:`, error);
+        }
+        try {
+            await convertDatasetScrutins(dataDir, options);
+            const scrutinsDir = path.join(dataDir, SCRUTINS_FOLDER);
+            exitCode = commitGit(scrutinsDir, options, exitCode);
+        }
+        catch (error) {
+            console.error(`Error converting Scrutins dataset:`, error);
+        }
+    }
+    if (enabledDatasets & EnabledDatasets.Questions) {
+        try {
+            await convertDatasetQuestions(dataDir, options);
+            const questionsDir = path.join(dataDir, datasets.questions.database);
+            exitCode = commitGit(questionsDir, options, exitCode);
+        }
+        catch (error) {
+            console.error(`Error converting Questions dataset:`, error);
+        }
+    }
+    if (enabledDatasets & EnabledDatasets.Sens) {
+        try {
+            await convertDatasetSens(dataDir, options);
+            const sensDir = path.join(dataDir, datasets.sens.database);
+            exitCode = commitGit(sensDir, options, exitCode);
+        }
+        catch (error) {
+            console.error(`Error converting Sens dataset:`, error);
+        }
+    }
+    if (!options["silent"]) {
+        console.timeEnd("data transformation time");
+    }
+    return exitCode;
+}
+async function convertDatasetAmeli(dataDir, options) {
+    const dataset = datasets.ameli;
+    if (!options["silent"]) {
+        console.log(`Converting database ${dataset.database} data into files…`);
+    }
+    const ameliReorganizedRootDir = path.join(dataDir, dataset.database);
+    if (!options.keepDir) {
+        ensureAndClearDir(ameliReorganizedRootDir);
+    }
+    for await (const amendement of findAllAmendements(options["fromSession"])) {
+        if (options["verbose"]) {
+            console.log(`Converting ${amendement["numero"]} file…`);
+        }
+        const session = String(amendement["session"]) || UNDEFINED_SESSION;
+        const signetDossierLegislatif = amendement["signet_dossier_legislatif"] ||
+            `${amendement["nature_texte"]}-${amendement["numero_texte"]}`.toLowerCase();
+        const amendementFileName = `${amendement["numero"]}.json`;
+        const filePath = path.join(ameliReorganizedRootDir, String(session), signetDossierLegislatif, amendementFileName);
+        await fs.outputJSON(filePath, amendement, { spaces: 2 });
+    }
+}
+async function convertDatasetDebats(dataDir, options) {
+    const dataset = datasets.debats;
+    if (!options["silent"]) {
+        console.log(`Converting database ${dataset.database} data into files…`);
+    }
+    const debatsReorganizedRootDir = path.join(dataDir, dataset.database);
+    if (!options.keepDir) {
+        ensureAndClearDir(debatsReorganizedRootDir);
+    }
+    for await (const debat of findAllDebats()) {
+        if (options["verbose"]) {
+            console.log(`Converting ${debat.id} file…`);
+        }
+        const session = getSessionFromDate(debat.date_seance);
+        if (options["fromSession"] && session < options["fromSession"]) {
+            continue;
+        }
+        const debatFileName = `${debat.id}.json`;
+        const filePath = path.join(debatsReorganizedRootDir, String(session), debatFileName);
+        await fs.outputJSON(filePath, debat, { spaces: 2 });
+    }
+}
+async function convertDatasetDosLeg(dataDir, options) {
+    const dataset = datasets.dosleg;
+    if (!options["silent"]) {
+        console.log(`Converting database ${dataset.database} data into files…`);
+    }
+    const doslegReorganizedRootDir = path.join(dataDir, dataset.database);
+    const dossiersReorganizedDir = path.join(doslegReorganizedRootDir, DOSLEG_DOSSIERS_FOLDER);
+    if (!options.keepDir) {
+        ensureAndClearDir(doslegReorganizedRootDir);
+        ensureAndClearDir(dossiersReorganizedDir);
+    }
+    for await (const dossier of findAllDossiers()) {
+        if (options["verbose"]) {
+            console.log(`Converting ${dossier["signet"]} file…`);
+        }
+        let dossierReorganizedDir = path.join(dossiersReorganizedDir, String(UNDEFINED_SESSION));
+        const session = getSessionFromSignet(dossier["signet"]) || UNDEFINED_SESSION;
+        if (options["fromSession"] && session < options["fromSession"]) {
+            continue;
+        }
+        dossierReorganizedDir = path.join(dossiersReorganizedDir, String(session));
+        const actesBrutsNormalises = buildActesLegislatifs(dossier);
+        const dossierWithActes = {
+            ...dossier,
+            actes_legislatifs: actesBrutsNormalises,
+        };
+        const dossierFile = `${dossier["signet"]}.json`;
+        await fs.outputJSON(path.join(dossierReorganizedDir, dossierFile), dossierWithActes, { spaces: 2 });
+    }
+    await convertTextes(dataDir, options);
+    await convertRapports(dataDir, options);
+}
+async function convertDatasetScrutins(dataDir, options) {
+    const dataset = datasets.dosleg;
+    if (!options["silent"]) {
+        console.log(`Converting database scrutins (${dataset.database}) data into files…`);
+    }
+    const scrutinsReorganizedDir = path.join(dataDir, SCRUTINS_FOLDER);
+    if (!options.keepDir) {
+        ensureAndClearDir(scrutinsReorganizedDir);
+    }
+    for await (const scrutin of findAllScrutins(options["fromSession"])) {
+        if (options["verbose"]) {
+            console.log(`Converting ${scrutin["numero"]} file…`);
+        }
+        let scrutinReorganizedDir = path.join(scrutinsReorganizedDir, String(UNDEFINED_SESSION));
+        const session = scrutin["session"] || UNDEFINED_SESSION;
+        scrutinReorganizedDir = path.join(scrutinsReorganizedDir, String(session));
+        const scrutinFileName = `${scrutin["numero"]}.json`;
+        await fs.outputJSON(path.join(scrutinReorganizedDir, scrutinFileName), scrutin, {
+            spaces: 2,
+        });
+    }
+}
+async function convertDatasetQuestions(dataDir, options) {
+    const dataset = datasets.questions;
+    if (!options["silent"]) {
+        console.log(`Converting database ${dataset.database} data into files…`);
+    }
+    const questionsReorganizedRootDir = path.join(dataDir, dataset.database);
+    if (!options.keepDir) {
+        ensureAndClearDir(questionsReorganizedRootDir);
+    }
+    const limit = pLimit(10);
+    const tasks = [];
+    for await (const question of findAllQuestions()) {
+        tasks.push(limit(async () => {
+            if (options["verbose"]) {
+                console.log(`Converting ${question["reference"]} file…`);
+            }
+            const legislature = question["legislature"] ? question["legislature"] : 0;
+            const questionFileName = `${question["reference"]}.json`;
+            await fs.outputJSON(path.join(questionsReorganizedRootDir, String(legislature), questionFileName), question, {
+                spaces: 2,
+            });
+        }));
+    }
+    await Promise.all(tasks);
+}
+async function convertTextes(dataDir, options) {
+    const originalTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER);
+    const transformedTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_TRANSFORMED_FOLDER);
+    if (!options["silent"]) {
+        console.log(`Converting database textes data into files…`);
+    }
+    for await (const texte of findAllTextes()) {
+        const session = texte["session"] ?? UNDEFINED_SESSION;
+        if (options["fromSession"] && session < options["fromSession"]) {
+            continue;
+        }
+        if (!texte["url"]) {
+            continue;
+        }
+        const texteName = path.parse(texte["url"]).name;
+        const texteDir = path.join(originalTextesDir, `${session}`, texteName);
+        // oritxtcod = 1 corresponds to "Texte de loi déposé au Sénat"
+        const hasExposeDesMotifs = texte["origine"] === "Sénat" && texte["ordre_origine"] === "1";
+        const metadata = {
+            name: texteName,
+            session: texte["session"],
+            date: texte["date"],
+            url_expose_des_motifs: hasExposeDesMotifs
+                ? new URL(`${texteName}-expose.html`, SENAT_EXPOSE_DES_MOTIFS_BASE_URL)
+                : undefined,
+            url_xml: new URL(`${texteName}.akn.xml`, SENAT_TEXTE_XML_BASE_URL),
+            url_html: new URL(`${texteName}.html`, SENAT_TEXTE_BASE_URL),
+            url_pdf: new URL(`${texteName}.pdf`, SENAT_TEXTE_BASE_URL),
+        };
+        fs.outputJSONSync(path.join(texteDir, `${texteName}.json`), texte, {
+            spaces: 2,
+        });
+        fs.outputJSONSync(path.join(texteDir, DOCUMENT_METADATA_FILE), metadata, {
+            spaces: 2,
+        });
+        if (options.fetchDocuments) {
+            await processTexte(metadata, originalTextesDir, transformedTextesDir, options);
+        }
+    }
+}
+async function convertRapports(dataDir, options) {
+    const originalRapportsDir = path.join(dataDir, RAPPORT_FOLDER, DATA_ORIGINAL_FOLDER);
+    if (!options["silent"]) {
+        console.log(`Converting database rapports data into files…`);
+    }
+    for await (const rapport of findAllRapports()) {
+        const session = rapport["session"] ?? UNDEFINED_SESSION;
+        if (options["fromSession"] && session < options["fromSession"]) {
+            continue;
+        }
+        if (!rapport["url"]) {
+            continue;
+        }
+        const parsedRapportUrl = path.parse(rapport["url"]);
+        const rapportName = parsedRapportUrl.name;
+        const rapportDir = path.join(originalRapportsDir, `${session}`, rapportName);
+        const rapportHtmlUrlBase = `${rapportName}_mono.html`;
+        const rapportHtmlUrl = path.format({
+            dir: parsedRapportUrl.dir,
+            base: rapportHtmlUrlBase,
+        });
+        const rapportPdfUrlBase = `${rapportName}1.pdf`;
+        const rapportPdfUrl = path.format({
+            dir: parsedRapportUrl.dir,
+            base: rapportPdfUrlBase,
+        });
+        const metadata = {
+            name: rapportName,
+            session: rapport["session"],
+            date: rapport["date"],
+            url_html: new URL(rapportHtmlUrl, SENAT_RAPPORT_BASE_URL),
+            url_pdf: new URL(rapportPdfUrl, SENAT_RAPPORT_BASE_URL),
+        };
+        fs.outputJSONSync(path.join(rapportDir, `${rapportName}.json`), rapport, {
+            spaces: 2,
+        });
+        fs.outputJSONSync(path.join(rapportDir, DOCUMENT_METADATA_FILE), metadata, {
+            spaces: 2,
+        });
+        if (options.fetchDocuments) {
+            await processRapport(metadata, originalRapportsDir, options);
+        }
+    }
+}
+async function convertDatasetSens(dataDir, options) {
+    const dataset = datasets.sens;
+    if (!options["silent"]) {
+        console.log(`Converting database ${dataset.database} data into files…`);
+    }
+    const sensReorganizedRootDir = path.join(dataDir, dataset.database);
+    const senateursReorganizedDir = path.join(sensReorganizedRootDir, SENS_SENATEURS_FOLDER);
+    const circonscriptionsReorganizedDir = path.join(sensReorganizedRootDir, SENS_CIRCONSCRIPTIONS_FOLDER);
+    const organismesReorganizedDir = path.join(sensReorganizedRootDir, SENS_ORGANISMES_FOLDER);
+    if (!options.keepDir) {
+        ensureAndClearDir(sensReorganizedRootDir);
+        ensureAndClearDir(senateursReorganizedDir);
+        ensureAndClearDir(circonscriptionsReorganizedDir);
+        ensureAndClearDir(organismesReorganizedDir);
+    }
+    for await (const sen of findAllSens()) {
+        if (options["verbose"]) {
+            console.log(`Converting ${sen["matricule"]} file…`);
+        }
+        const senFileName = `${sen["matricule"]}.json`;
+        fs.outputJSONSync(path.join(senateursReorganizedDir, senFileName), sen, {
+            spaces: 2,
+        });
+    }
+    for await (const circonscription of findAllCirconscriptions()) {
+        if (options["verbose"]) {
+            console.log(`Converting ${circonscription["identifiant"]} file…`);
+        }
+        const circonscriptionFileName = `${circonscription["identifiant"]}.json`;
+        fs.outputJSONSync(path.join(circonscriptionsReorganizedDir, circonscriptionFileName), circonscription, {
+            spaces: 2,
+        });
+    }
+    for await (const organisme of findAllOrganismes()) {
+        if (options["verbose"]) {
+            console.log(`Converting ${organisme["code"]} file…`);
+        }
+        const organismeFileName = `${organisme["code"]}.json`;
+        fs.outputJSONSync(path.join(organismesReorganizedDir, organisme["type_code"], organismeFileName), organisme, {
+            spaces: 2,
+        });
+    }
+}
+convertData()
+    .then((exitCode) => process.exit(exitCode || 0))
+    .catch((error) => {
+    console.log(error);
+    process.exit(1);
+});

package/lib/scripts/data-download.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/scripts/data-download.js

@@ -0,0 +1,12 @@
+import { execSync } from "child_process";
+const args = process.argv.slice(2).join(" ");
+try {
+    execSync(`tsx src/scripts/retrieve_open_data.ts --all ${args}`, { stdio: "inherit" });
+    execSync(`tsx src/scripts/convert_data.ts ${args}`, { stdio: "inherit" });
+}
+catch (error) {
+    if (error.status !== 10) {
+        console.error("Error during data retrieval:", error);
+        process.exit(error.status || 1);
+    }
+}

package/lib/scripts/datautil.d.ts

@@ -0,0 +1,8 @@
+import { Session } from "../types/sessions";
+export declare const STANDARD_DATE_FORMAT = "yyyy-MM-dd";
+export declare const ID_DATE_FORMAT = "yyyyMMdd";
+export declare const AKN_IDENTIFICATION_STRUCTURE_REGEXP: RegExp;
+export declare const AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP: RegExp;
+export declare const SIGNET_STRUCTURE_REGEXP: RegExp;
+export declare function getSessionFromSignet(signet: string): Session | null;
+export declare function getSessionFromDate(date: string, format?: string): Session;

package/lib/scripts/datautil.js

@@ -0,0 +1,34 @@
+import { DateTime, Settings } from "luxon";
+Settings.twoDigitCutoffYear = 50;
+export const STANDARD_DATE_FORMAT = "yyyy-MM-dd";
+export const ID_DATE_FORMAT = "yyyyMMdd";
+export const AKN_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{4}-\d{4})\/?(?<numTexte>\d+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/;
+export const AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{2,4})\/?(?<numTexte>[a-zA-Z0-9]+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/;
+export const SIGNET_STRUCTURE_REGEXP = /^(?<type>[a-z]+)(?<session>\d{2,4})-?(?<numTexte>\d+)?/;
+export function getSessionFromSignet(signet) {
+    const signetParts = SIGNET_STRUCTURE_REGEXP.exec(signet)?.groups;
+    if (signetParts && "session" in signetParts) {
+        const { session } = signetParts;
+        return formatToFourDigitSession(session);
+    }
+    return null;
+}
+function formatToFourDigitSession(session) {
+    if (session.length >= 2) {
+        const sessionFirstTwoDigits = session.substring(0, 2);
+        const sessionLastTwoDigits = session.substring(session.length - 2);
+        const twoDigitSession = parseInt(sessionFirstTwoDigits) === parseInt(sessionLastTwoDigits) - 1
+            ? sessionFirstTwoDigits
+            : sessionLastTwoDigits;
+        return parseInt(DateTime.fromFormat(String(twoDigitSession), "yy").toFormat("yyyy"));
+    }
+    return null;
+}
+export function getSessionFromDate(date, format = STANDARD_DATE_FORMAT) {
+    const parsedDate = DateTime.fromFormat(date, format);
+    const endSessionDate = DateTime.fromObject({ year: parsedDate.year, month: 9, day: 30 });
+    if (parsedDate < endSessionDate) {
+        return (parsedDate.year - 1);
+    }
+    return parsedDate.year;
+}

package/lib/scripts/parse_textes.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/scripts/parse_textes.js

@@ -0,0 +1,44 @@
+import assert from "assert";
+import commandLineArgs from "command-line-args";
+import fs from "fs-extra";
+import path from "path";
+import { iterFilePaths, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
+import { parseExposeDesMotifsFromFile, parseTexteFromFile } from "../model/texte";
+import { commonOptions } from "./shared/cli_helpers";
+import { ensureAndClearDir } from "./shared/util";
+const optionsDefinitions = [...commonOptions];
+const options = commandLineArgs(optionsDefinitions);
+async function main() {
+    const dataDir = options["dataDir"];
+    assert(dataDir, "Missing argument: data directory");
+    const transformedTextesDir = path.join(options["dataDir"], TEXTE_FOLDER, DATA_TRANSFORMED_FOLDER);
+    ensureAndClearDir(transformedTextesDir);
+    for (const filePath of iterFilePaths(path.join(dataDir, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER))) {
+        const parsedFilePath = path.parse(filePath);
+        if (parsedFilePath.ext !== ".xml") {
+            continue;
+        }
+        const texteDirFromOriginal = parsedFilePath.dir.substring(filePath.indexOf(DATA_ORIGINAL_FOLDER) + DATA_ORIGINAL_FOLDER.length);
+        const transformedTexteDir = path.join(transformedTextesDir, texteDirFromOriginal);
+        fs.ensureDirSync(transformedTexteDir);
+        if (!options["silent"]) {
+            console.log(`Parsing texte ${parsedFilePath.name}.xml…`);
+        }
+        const parsedTexte = await parseTexteFromFile(filePath);
+        const exposeDesMotifsFileName = `${parsedFilePath.name}-expose`;
+        const exposeDesMotifsFilePath = path.join(parsedFilePath.dir, `${exposeDesMotifsFileName}.html`);
+        if (parsedTexte && fs.existsSync(exposeDesMotifsFilePath)) {
+            if (!options["silent"]) {
+                console.log(`Parsing exposé des motifs ${exposeDesMotifsFileName}.html…`);
+            }
+            parsedTexte.exposeDesMotifs = await parseExposeDesMotifsFromFile(exposeDesMotifsFilePath);
+        }
+        fs.writeJSONSync(path.join(transformedTexteDir, `${parsedFilePath.name}.json`), parsedTexte, { spaces: 2 });
+    }
+}
+main()
+    .then(() => process.exit(0))
+    .catch((error) => {
+    console.log(error);
+    process.exit(1);
+});

package/lib/scripts/retrieve_agenda.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/scripts/retrieve_agenda.js

@@ -0,0 +1,132 @@
+import assert from "assert";
+import commandLineArgs from "command-line-args";
+import fs from "fs-extra";
+import { DateTime } from "luxon";
+import path from "path";
+import { AGENDA_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
+import { parseAgendaFromFile } from "../model/agenda";
+import { getSessionsFromStart } from "../types/sessions";
+import { ID_DATE_FORMAT } from "./datautil";
+import { commonOptions } from "./shared/cli_helpers";
+import { fetchWithRetry } from "./shared/util";
+import { buildReunionsByBucket } from "../utils/reunion_parsing";
+import { buildSenatDossierIndex } from "../utils/reunion_odj_building";
+const optionsDefinitions = [
+    ...commonOptions,
+    {
+        help: "parse and convert agenda events into JSON",
+        name: "parseAgenda",
+        type: Boolean,
+    },
+];
+const options = commandLineArgs(optionsDefinitions);
+const SENAT_GLOBAL_AGENDA_URL_ROOT = "https://www.senat.fr/aglae/Global";
+const EVENT_DATE_FORMAT = "ddMMyyyy";
+class AgendaError extends Error {
+    constructor(message, agendaName) {
+        super(`An error occurred while retrieving Agenda ${agendaName}: ${message}`);
+    }
+}
+async function retrieveAgendas(options, sessions) {
+    console.log(`[AGENDA] Retrieving agendas for sessions ${sessions.join(", ")}`);
+    const agendaRootDir = path.join(options["dataDir"], AGENDA_FOLDER);
+    fs.ensureDirSync(agendaRootDir);
+    const originalAgendaDir = path.join(agendaRootDir, DATA_ORIGINAL_FOLDER);
+    fs.ensureDirSync(originalAgendaDir);
+    const transformedAgendaDir = path.join(agendaRootDir, DATA_TRANSFORMED_FOLDER);
+    if (options["parseAgenda"]) {
+        fs.ensureDirSync(transformedAgendaDir);
+    }
+    let dossierIndex = {};
+    dossierIndex = buildSenatDossierIndex(options);
+    for (const session of sessions) {
+        const originalAgendaSessionDir = path.join(originalAgendaDir, `${session}`);
+        fs.ensureDirSync(originalAgendaSessionDir);
+        if (!options["keepDir"])
+            fs.emptyDirSync(originalAgendaSessionDir);
+        const transformedAgendaSessionDir = path.join(transformedAgendaDir, `${session}`);
+        if (options["parseAgenda"]) {
+            fs.ensureDirSync(transformedAgendaSessionDir);
+            if (!options["keepDir"])
+                fs.emptyDirSync(transformedAgendaSessionDir);
+        }
+        const fifteenDaysFromNow = new Date();
+        fifteenDaysFromNow.setDate(fifteenDaysFromNow.getDate() + 15); // Don't download agendas more than 15 days in the future
+        for (const date = new Date(session, 9, 1); date <= new Date(session + 1, 8, 30) && date <= fifteenDaysFromNow; date.setDate(date.getDate() + 1)) {
+            const agendaName = DateTime.fromJSDate(date).toFormat(EVENT_DATE_FORMAT);
+            const agendaFileName = DateTime.fromJSDate(date).toFormat(ID_DATE_FORMAT);
+            const agendaPath = path.join(originalAgendaSessionDir, `${agendaFileName}.html`);
+            try {
+                await downloadAgenda(agendaName, agendaPath);
+                if (options["parseAgenda"]) {
+                    await parseAgenda(transformedAgendaSessionDir, agendaFileName, agendaPath, dossierIndex);
+                }
+            }
+            catch (error) {
+                console.error(error);
+            }
+        }
+    }
+}
+async function downloadAgenda(agendaName, agendaPath) {
+    const agendaUrl = `${SENAT_GLOBAL_AGENDA_URL_ROOT}/agl${agendaName}.html`;
+    if (!options["silent"]) {
+        console.log(`Downloading Agenda ${agendaUrl}…`);
+    }
+    const response = await fetchWithRetry(agendaUrl);
+    if (!response.ok) {
+        if (response.status === 404) {
+            console.warn(`Agenda ${agendaUrl} not found`);
+        }
+        else {
+            throw new AgendaError(String(response.status), agendaName);
+        }
+        return;
+    }
+    const agendaContent = await response.arrayBuffer();
+    if (!agendaContent) {
+        return;
+    }
+    fs.writeFileSync(agendaPath, Buffer.from(agendaContent));
+}
+function writeGroupsAsFiles(dir, groups) {
+    for (const g of groups) {
+        const outPath = path.join(dir, `${g.uid}.json`);
+        fs.writeJSONSync(outPath, g, { spaces: 2 });
+    }
+}
+async function parseAgenda(transformedAgendaSessionDir, agendaFileName, agendaPath, dossierBySenatUrl) {
+    if (!options["silent"])
+        console.log(`Parsing Agenda ${agendaPath}…`);
+    const parsedAgendaEvents = await parseAgendaFromFile(agendaPath);
+    if (!parsedAgendaEvents?.length)
+        return;
+    const flatPath = path.join(transformedAgendaSessionDir, `${agendaFileName}.json`);
+    fs.writeJSONSync(flatPath, parsedAgendaEvents, { spaces: 2 });
+    const byBucket = buildReunionsByBucket(parsedAgendaEvents, dossierBySenatUrl);
+    // SP
+    if (byBucket.IDS.length > 0) {
+        writeGroupsAsFiles(transformedAgendaSessionDir, byBucket.IDS);
+    }
+    // NON-SP
+    for (const suffix of ["IDC", "IDM", "IDO", "IDI"]) {
+        const groups = byBucket[suffix];
+        if (groups.length > 0) {
+            writeGroupsAsFiles(transformedAgendaSessionDir, groups);
+        }
+    }
+}
+async function main() {
+    const dataDir = options["dataDir"];
+    assert(dataDir, "Missing argument: data directory");
+    const sessions = getSessionsFromStart(options["fromSession"]);
+    console.time("agenda processing time");
+    await retrieveAgendas(options, sessions);
+    console.timeEnd("agenda processing time");
+}
+main()
+    .then(() => process.exit(0))
+    .catch((error) => {
+    console.log(error);
+    process.exit(1);
+});

package/lib/scripts/retrieve_cr_commission.d.ts

@@ -0,0 +1 @@
+export {};