@tricoteuses/senat 2.22.3 → 2.22.5

package/lib/scripts/convert_data.ts

@@ -0,0 +1,243 @@
+import assert from "assert"
+import commandLineArgs from "command-line-args"
+import fs from "fs-extra"
+import path from "path"
+
+import { datasets, EnabledDatasets, getEnabledDatasets } from "../datasets"
+import {
+  DOCUMENT_METADATA_FILE,
+  DOSLEG_DOSSIERS_FOLDER,
+  RAPPORT_FOLDER,
+  RapportMetadata,
+  SENS_CIRCONSCRIPTIONS_FOLDER,
+  SENS_ORGANISMES_FOLDER,
+  SENS_SENATEURS_FOLDER,
+  TEXTE_FOLDER,
+  TEXTE_ORIGINAL_FOLDER,
+  TexteMetadata,
+} from "../loaders"
+import {
+  findAllAmendements,
+  findAllCirconscriptions,
+  findAllLois,
+  findAllOrganismes,
+  findAllQuestions,
+  findAllSens,
+} from "../model"
+import { findSenatRapportUrls, findSenatTexteUrls } from "../model/dosleg"
+import { formatToFourDigitSession, SIGNET_STRUCTURE_REGEXP, UNDEFINED_SESSION } from "./datautil"
+import { commonOptions } from "./shared/cli_helpers"
+import { ensureAndClearDir } from "./shared/util"
+
+const optionsDefinitions = [
+  ...commonOptions,
+]
+const options = commandLineArgs(optionsDefinitions)
+
+const SENAT_TEXTE_XML_BASE_URL = "https://www.senat.fr/akomantoso/"
+const SENAT_TEXTE_BASE_URL = "https://www.senat.fr/leg/"
+const SENAT_EXPOSE_DES_MOTIFS_BASE_URL = "https://www.senat.fr/leg/exposes-des-motifs/"
+const SENAT_RAPPORT_BASE_URL = "https://www.senat.fr/rap/"
+
+async function convertData () {
+  const enabledDatasets = getEnabledDatasets(options.categories)
+  const dataDir: string = options.dataDir
+  assert(dataDir, "Missing argument: data directory")
+
+  console.time("data transformation time")
+
+  if (enabledDatasets & EnabledDatasets.Ameli) {
+    const dataset = datasets.ameli
+
+    if (!options.silent) {
+      console.log(`Converting database ${dataset.database} data into files…`)
+    }
+
+    const ameliReorganizedRootDir = path.join(dataDir, dataset.database)
+    ensureAndClearDir(ameliReorganizedRootDir)
+
+    for await (const amendement of findAllAmendements()) {
+      if (options.verbose) {
+        console.log(`Converting ${amendement.numero} file…`)
+      }
+
+      const session = String(amendement.session) || UNDEFINED_SESSION
+      const signetDossierLegislatif = amendement.signet_dossier_legislatif
+        || `${amendement.nature_texte}-${amendement.numero_texte}`.toLowerCase()
+      const ameliReorganizedDir = path.join(ameliReorganizedRootDir, session, signetDossierLegislatif)
+      fs.ensureDirSync(ameliReorganizedDir)
+      const amendementFileName = `${amendement.numero}.json`
+      fs.writeJSONSync(path.join(ameliReorganizedDir, amendementFileName), amendement, { spaces: 2 })
+    }
+  }
+
+  if (enabledDatasets & EnabledDatasets.DosLeg) {
+    const dataset = datasets.dosleg
+
+    if (!options.silent) {
+      console.log(`Converting database ${dataset.database} data into files…`)
+    }
+
+    const doslegReorganizedRootDir = path.join(dataDir, dataset.database)
+    const dossiersReorganizedDir = path.join(doslegReorganizedRootDir, DOSLEG_DOSSIERS_FOLDER)
+    ensureAndClearDir(doslegReorganizedRootDir)
+    ensureAndClearDir(dossiersReorganizedDir)
+
+    for await (const loi of findAllLois()) {
+      if (options.verbose) {
+        console.log(`Converting ${loi.signet} file…`)
+      }
+
+      let loiReorganizedDir = path.join(dossiersReorganizedDir, UNDEFINED_SESSION)
+      const signetParts = SIGNET_STRUCTURE_REGEXP.exec(loi.signet)?.groups
+      if (signetParts) {
+        const { session } = signetParts
+        const formattedSession = formatToFourDigitSession(session)
+        loiReorganizedDir = path.join(dossiersReorganizedDir, formattedSession)
+      }
+      fs.ensureDirSync(loiReorganizedDir)
+
+      const loiFileName = `${loi.signet}.json`
+      fs.writeJSONSync(path.join(loiReorganizedDir, loiFileName), loi, { spaces: 2 })
+    }
+
+    await convertTexteUrls(dataDir)
+    await convertRapportUrls(dataDir)
+  }
+
+  if (enabledDatasets & EnabledDatasets.Questions) {
+    const dataset = datasets.questions
+
+    if (!options.silent) {
+      console.log(`Converting database ${dataset.database} data into files…`)
+    }
+
+    const questionsReorganizedRootDir = path.join(dataDir, dataset.database)
+    ensureAndClearDir(questionsReorganizedRootDir)
+
+    for await (const question of findAllQuestions()) {
+      if (options.verbose) {
+        console.log(`Converting ${question.reference} file…`)
+      }
+
+      const legislature = question.legislature ? question.legislature : 0
+      const questionReorganizedDir = path.join(questionsReorganizedRootDir, String(legislature))
+      fs.ensureDirSync(questionReorganizedDir)
+      const questionFileName = `${question.reference}.json`
+      fs.writeJSONSync(path.join(questionReorganizedDir, questionFileName), question, { spaces: 2 })
+    }
+  }
+
+  if (enabledDatasets & EnabledDatasets.Sens) {
+    const dataset = datasets.sens
+
+    if (!options.silent) {
+      console.log(`Converting database ${dataset.database} data into files…`)
+    }
+
+    const sensReorganizedRootDir = path.join(dataDir, dataset.database)
+    const senateursReorganizedDir = path.join(sensReorganizedRootDir, SENS_SENATEURS_FOLDER)
+    const circonscriptionsReorganizedDir = path.join(sensReorganizedRootDir, SENS_CIRCONSCRIPTIONS_FOLDER)
+    const organismesReorganizedDir = path.join(sensReorganizedRootDir, SENS_ORGANISMES_FOLDER)
+    ensureAndClearDir(sensReorganizedRootDir)
+    ensureAndClearDir(senateursReorganizedDir)
+    ensureAndClearDir(circonscriptionsReorganizedDir)
+    ensureAndClearDir(organismesReorganizedDir)
+
+    for await (const sen of findAllSens()) {
+      if (options.verbose) {
+        console.log(`Converting ${sen.matricule} file…`)
+      }
+
+      const senFileName = `${sen.matricule}.json`
+      fs.writeJSONSync(path.join(senateursReorganizedDir, senFileName), sen, { spaces: 2 })
+    }
+
+    for await (const circonscription of findAllCirconscriptions()) {
+      if (options.verbose) {
+        console.log(`Converting ${circonscription.identifiant} file…`)
+      }
+
+      const circonscriptionFileName = `${circonscription.identifiant}.json`
+      fs.writeJSONSync(
+        path.join(circonscriptionsReorganizedDir, circonscriptionFileName), circonscription, { spaces: 2 }
+      )
+    }
+
+    for await (const organisme of findAllOrganismes()) {
+      if (options.verbose) {
+        console.log(`Converting ${organisme.code} file…`)
+      }
+
+      const organismeFileName = `${organisme.code}.json`
+      fs.writeJSONSync(path.join(organismesReorganizedDir, organismeFileName), organisme, { spaces: 2 })
+    }
+  }
+
+  if (!options.silent) {
+    console.timeEnd("data transformation time")
+  }
+}
+
+async function convertTexteUrls (dataDir: string) {
+  const textesDir = path.join(dataDir, TEXTE_FOLDER)
+  fs.ensureDirSync(textesDir)
+
+  const originalTextesDir = path.join(textesDir, TEXTE_ORIGINAL_FOLDER)
+
+  for await (const texte of findSenatTexteUrls(options.sessions)) {
+    const texteName = path.parse(texte.url).name
+    const texteDir = path.join(originalTextesDir, `${texte.session ?? UNDEFINED_SESSION}`, texteName)
+    fs.ensureDirSync(texteDir)
+
+    const metadata: TexteMetadata = {
+      name: texteName,
+      session: texte.session,
+      url_expose_des_motifs: texte.hasExposeDesMotifs ?
+        new URL(`${texteName}-expose.html`, SENAT_EXPOSE_DES_MOTIFS_BASE_URL) : undefined,
+      url_xml: new URL(`${texteName}.akn.xml`, SENAT_TEXTE_XML_BASE_URL),
+      url_html: new URL(`${texteName}.html`, SENAT_TEXTE_BASE_URL),
+      url_pdf: new URL(`${texteName}.pdf`, SENAT_TEXTE_BASE_URL),
+    }
+    fs.writeJSONSync(path.join(texteDir, DOCUMENT_METADATA_FILE), metadata, { spaces: 2 })
+  }
+}
+
+async function convertRapportUrls (dataDir: string) {
+  const rapportsDir = path.join(dataDir, RAPPORT_FOLDER)
+  fs.ensureDirSync(rapportsDir)
+
+  for await (const rapport of findSenatRapportUrls(options.sessions)) {
+    const parsedRapportUrl = path.parse(rapport.url)
+    const rapportName = parsedRapportUrl.name
+    const rapportDir = path.join(rapportsDir, `${rapport.session ?? UNDEFINED_SESSION}`, rapportName)
+    fs.ensureDirSync(rapportDir)
+
+    const rapportHtmlUrlBase = `${rapportName}_mono.html`
+    const rapportHtmlUrl = path.format({
+      dir: parsedRapportUrl.dir,
+      base: rapportHtmlUrlBase,
+    })
+
+    const rapportPdfUrlBase = `${rapportName}1.pdf`
+    const rapportPdfUrl = path.format({
+      dir: parsedRapportUrl.dir,
+      base: rapportPdfUrlBase,
+    })
+
+    const metadata: RapportMetadata = {
+      name: rapportName,
+      session: rapport.session,
+      url_html: new URL(rapportHtmlUrl, SENAT_RAPPORT_BASE_URL),
+      url_pdf: new URL(rapportPdfUrl, SENAT_RAPPORT_BASE_URL),
+    }
+    fs.writeJSONSync(path.join(rapportDir, DOCUMENT_METADATA_FILE), metadata, { spaces: 2 })
+  }
+}
+
+convertData()
+  .then(() => process.exit(0))
+  .catch((error) => {
+    console.log(error)
+    process.exit(1)
+  })

package/lib/scripts/data-download.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/scripts/data-download.js

@@ -0,0 +1,12 @@
+import { execSync } from "child_process";
+const args = process.argv.slice(2).join(" ");
+try {
+    execSync(`tsx src/scripts/retrieve_open_data.ts --all ${args}`, { stdio: "inherit" });
+    execSync(`tsx src/scripts/convert_data.ts ${args}`, { stdio: "inherit" });
+}
+catch (error) {
+    if (error.status !== 10) {
+        console.error("Error during data retrieval:", error);
+        process.exit(error.status || 1);
+    }
+}

package/lib/scripts/datautil.d.ts

@@ -0,0 +1,8 @@
+import { Session } from "../types/sessions";
+export declare const STANDARD_DATE_FORMAT = "yyyy-MM-dd";
+export declare const ID_DATE_FORMAT = "yyyyMMdd";
+export declare const AKN_IDENTIFICATION_STRUCTURE_REGEXP: RegExp;
+export declare const AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP: RegExp;
+export declare const SIGNET_STRUCTURE_REGEXP: RegExp;
+export declare function getSessionFromSignet(signet: string): Session | null;
+export declare function getSessionFromDate(date: string, format?: string): Session;

package/lib/scripts/datautil.js

@@ -0,0 +1,34 @@
+import { DateTime, Settings } from "luxon";
+Settings.twoDigitCutoffYear = 50;
+export const STANDARD_DATE_FORMAT = "yyyy-MM-dd";
+export const ID_DATE_FORMAT = "yyyyMMdd";
+export const AKN_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{4}-\d{4})\/?(?<numTexte>\d+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/;
+export const AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{2,4})\/?(?<numTexte>[a-zA-Z0-9]+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/;
+export const SIGNET_STRUCTURE_REGEXP = /^(?<type>[a-z]+)(?<session>\d{2,4})-?(?<numTexte>\d+)?/;
+export function getSessionFromSignet(signet) {
+    const signetParts = SIGNET_STRUCTURE_REGEXP.exec(signet)?.groups;
+    if (signetParts && "session" in signetParts) {
+        const { session } = signetParts;
+        return formatToFourDigitSession(session);
+    }
+    return null;
+}
+function formatToFourDigitSession(session) {
+    if (session.length >= 2) {
+        const sessionFirstTwoDigits = session.substring(0, 2);
+        const sessionLastTwoDigits = session.substring(session.length - 2);
+        const twoDigitSession = parseInt(sessionFirstTwoDigits) === parseInt(sessionLastTwoDigits) - 1
+            ? sessionFirstTwoDigits
+            : sessionLastTwoDigits;
+        return parseInt(DateTime.fromFormat(String(twoDigitSession), "yy").toFormat("yyyy"));
+    }
+    return null;
+}
+export function getSessionFromDate(date, format = STANDARD_DATE_FORMAT) {
+    const parsedDate = DateTime.fromFormat(date, format);
+    const endSessionDate = DateTime.fromObject({ year: parsedDate.year, month: 9, day: 30 });
+    if (parsedDate < endSessionDate) {
+        return (parsedDate.year - 1);
+    }
+    return parsedDate.year;
+}

package/lib/scripts/datautil.mjs

@@ -0,0 +1,16 @@
+import { DateTime, Settings } from "luxon";
+Settings.twoDigitCutoffYear = 50;
+export const SIGNET_STRUCTURE_REGEXP = /^(?<type>[a-z]+)(?<session>\d{2,4})-?(?<numTexte>\d+)?/;
+export const AKN_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{4}-\d{4})\/?(?<numTexte>\d+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/;
+export const AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{2,4})\/?(?<numTexte>[a-zA-Z0-9]+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/;
+export const UNDEFINED_SESSION = "0";
+export function formatToFourDigitSession(session) {
+    if (session.length >= 2) {
+        const sessionFirstTwoDigits = session.substring(0, 2);
+        const sessionLastTwoDigits = session.substring(session.length - 2);
+        const twoDigitSession = parseInt(sessionFirstTwoDigits) === parseInt(sessionLastTwoDigits) - 1 ?
+            sessionFirstTwoDigits : sessionLastTwoDigits;
+        return DateTime.fromFormat(String(twoDigitSession), "yy").toFormat("yyyy");
+    }
+    return UNDEFINED_SESSION;
+}

package/lib/scripts/datautil.ts

@@ -0,0 +1,19 @@
+import { DateTime, Settings } from "luxon"
+
+Settings.twoDigitCutoffYear = 50
+
+export const SIGNET_STRUCTURE_REGEXP = /^(?<type>[a-z]+)(?<session>\d{2,4})-?(?<numTexte>\d+)?/
+export const AKN_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{4}-\d{4})\/?(?<numTexte>\d+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/
+export const AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{2,4})\/?(?<numTexte>[a-zA-Z0-9]+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/
+export const UNDEFINED_SESSION = "0"
+
+export function formatToFourDigitSession (session: string) {
+  if (session.length >= 2) {
+    const sessionFirstTwoDigits = session.substring(0, 2)
+    const sessionLastTwoDigits = session.substring(session.length - 2)
+    const twoDigitSession = parseInt(sessionFirstTwoDigits) === parseInt(sessionLastTwoDigits) - 1 ?
+      sessionFirstTwoDigits : sessionLastTwoDigits
+    return DateTime.fromFormat(String(twoDigitSession), "yy").toFormat("yyyy")
+  }
+  return UNDEFINED_SESSION
+}

package/lib/scripts/parse_textes.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/scripts/parse_textes.js

@@ -0,0 +1,44 @@
+import assert from "assert";
+import commandLineArgs from "command-line-args";
+import fs from "fs-extra";
+import path from "path";
+import { iterFilePaths, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, } from "../loaders";
+import { parseExposeDesMotifsFromFile, parseTexteFromFile, } from "../model/texte";
+import { commonOptions } from "./shared/cli_helpers";
+import { ensureAndClearDir } from "./shared/util";
+const optionsDefinitions = [...commonOptions];
+const options = commandLineArgs(optionsDefinitions);
+async function main() {
+    const dataDir = options["dataDir"];
+    assert(dataDir, "Missing argument: data directory");
+    const transformedTextesDir = path.join(options["dataDir"], TEXTE_FOLDER, DATA_TRANSFORMED_FOLDER);
+    ensureAndClearDir(transformedTextesDir);
+    for (const filePath of iterFilePaths(path.join(dataDir, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER))) {
+        const parsedFilePath = path.parse(filePath);
+        if (parsedFilePath.ext !== ".xml") {
+            continue;
+        }
+        const texteDirFromOriginal = parsedFilePath.dir.substring(filePath.indexOf(DATA_ORIGINAL_FOLDER) + DATA_ORIGINAL_FOLDER.length);
+        const transformedTexteDir = path.join(transformedTextesDir, texteDirFromOriginal);
+        fs.ensureDirSync(transformedTexteDir);
+        if (!options["silent"]) {
+            console.log(`Parsing texte ${parsedFilePath.name}.xml…`);
+        }
+        const parsedTexte = await parseTexteFromFile(filePath);
+        const exposeDesMotifsFileName = `${parsedFilePath.name}-expose`;
+        const exposeDesMotifsFilePath = path.join(parsedFilePath.dir, `${exposeDesMotifsFileName}.html`);
+        if (parsedTexte && fs.existsSync(exposeDesMotifsFilePath)) {
+            if (!options["silent"]) {
+                console.log(`Parsing exposé des motifs ${exposeDesMotifsFileName}.html…`);
+            }
+            parsedTexte.exposeDesMotifs = await parseExposeDesMotifsFromFile(exposeDesMotifsFilePath);
+        }
+        fs.writeJSONSync(path.join(transformedTexteDir, `${parsedFilePath.name}.json`), parsedTexte, { spaces: 2 });
+    }
+}
+main()
+    .then(() => process.exit(0))
+    .catch((error) => {
+    console.log(error);
+    process.exit(1);
+});

package/lib/scripts/parse_textes.mjs

@@ -0,0 +1,46 @@
+import assert from "assert";
+import commandLineArgs from "command-line-args";
+import fs from "fs-extra";
+import path from "path";
+import { iterFilePaths, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER, TEXTE_TRANSFORMED_FOLDER } from "../loaders";
+import { parseExposeDesMotifsFromFile, parseTexteFromFile } from "../model/texte";
+import { commonOptions } from "./shared/cli_helpers";
+import { ensureAndClearDir } from "./shared/util";
+const optionsDefinitions = [
+    ...commonOptions,
+];
+const options = commandLineArgs(optionsDefinitions);
+async function main() {
+    const dataDir = options.dataDir;
+    assert(dataDir, "Missing argument: data directory");
+    const transformedTextesDir = path.join(options.dataDir, TEXTE_FOLDER, TEXTE_TRANSFORMED_FOLDER);
+    ensureAndClearDir(transformedTextesDir);
+    for (const filePath of iterFilePaths(path.join(dataDir, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER))) {
+        const parsedFilePath = path.parse(filePath);
+        if (parsedFilePath.ext !== ".xml") {
+            continue;
+        }
+        const texteDirFromOriginal = parsedFilePath.dir.substring(filePath.indexOf(TEXTE_ORIGINAL_FOLDER) + TEXTE_ORIGINAL_FOLDER.length);
+        const transformedTexteDir = path.join(transformedTextesDir, texteDirFromOriginal);
+        fs.ensureDirSync(transformedTexteDir);
+        if (!options.silent) {
+            console.log(`Parsing texte ${parsedFilePath.name}.xml…`);
+        }
+        const parsedTexte = await parseTexteFromFile(filePath);
+        const exposeDesMotifsFileName = `${parsedFilePath.name}-expose`;
+        const exposeDesMotifsFilePath = path.join(parsedFilePath.dir, `${exposeDesMotifsFileName}.html`);
+        if (parsedTexte && fs.existsSync(exposeDesMotifsFilePath)) {
+            if (!options.silent) {
+                console.log(`Parsing exposé des motifs ${exposeDesMotifsFileName}.html…`);
+            }
+            parsedTexte.exposeDesMotifs = await parseExposeDesMotifsFromFile(exposeDesMotifsFilePath);
+        }
+        fs.writeJSONSync(path.join(transformedTexteDir, `${parsedFilePath.name}.json`), parsedTexte, { spaces: 2 });
+    }
+}
+main()
+    .then(() => process.exit(0))
+    .catch((error) => {
+    console.log(error);
+    process.exit(1);
+});

package/lib/scripts/parse_textes.ts

@@ -0,0 +1,65 @@
+import assert from "assert"
+import commandLineArgs from "command-line-args"
+import fs from "fs-extra"
+import path from "path"
+
+import { iterFilePaths, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER, TEXTE_TRANSFORMED_FOLDER } from "../loaders"
+import { parseExposeDesMotifsFromFile, parseTexteFromFile } from "../model/texte"
+import { commonOptions } from "./shared/cli_helpers"
+import { ensureAndClearDir } from "./shared/util"
+
+const optionsDefinitions = [
+  ...commonOptions,
+]
+const options = commandLineArgs(optionsDefinitions)
+
+async function main() {
+  const dataDir = options.dataDir
+  assert(dataDir, "Missing argument: data directory")
+
+  const transformedTextesDir = path.join(options.dataDir, TEXTE_FOLDER, TEXTE_TRANSFORMED_FOLDER)
+  ensureAndClearDir(transformedTextesDir)
+
+  for (const filePath of iterFilePaths(path.join(dataDir, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER))) {
+    const parsedFilePath = path.parse(filePath)
+
+    if (parsedFilePath.ext !== ".xml") {
+      continue
+    }
+
+    const texteDirFromOriginal = parsedFilePath.dir.substring(
+      filePath.indexOf(TEXTE_ORIGINAL_FOLDER) + TEXTE_ORIGINAL_FOLDER.length
+    )
+    const transformedTexteDir = path.join(transformedTextesDir, texteDirFromOriginal)
+    fs.ensureDirSync(transformedTexteDir)
+
+    if (!options.silent) {
+      console.log(`Parsing texte ${parsedFilePath.name}.xml…`)
+    }
+
+    const parsedTexte = await parseTexteFromFile(filePath)
+
+    const exposeDesMotifsFileName = `${parsedFilePath.name}-expose`
+    const exposeDesMotifsFilePath = path.join(parsedFilePath.dir, `${exposeDesMotifsFileName}.html`)
+    if (parsedTexte && fs.existsSync(exposeDesMotifsFilePath)) {
+      if (!options.silent) {
+        console.log(`Parsing exposé des motifs ${exposeDesMotifsFileName}.html…`)
+      }
+
+      parsedTexte.exposeDesMotifs = await parseExposeDesMotifsFromFile(exposeDesMotifsFilePath)
+    }
+
+    fs.writeJSONSync(
+      path.join(transformedTexteDir, `${parsedFilePath.name}.json`),
+      parsedTexte,
+      { spaces: 2 }
+    )
+  }
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch((error) => {
+    console.log(error)
+    process.exit(1)
+  })

package/lib/scripts/retrieve_agenda.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/scripts/retrieve_agenda.js

@@ -0,0 +1,132 @@
+import assert from "assert";
+import commandLineArgs from "command-line-args";
+import fs from "fs-extra";
+import { DateTime } from "luxon";
+import path from "path";
+import { AGENDA_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
+import { parseAgendaFromFile } from "../model/agenda";
+import { getSessionsFromStart } from "../types/sessions";
+import { ID_DATE_FORMAT } from "./datautil";
+import { commonOptions } from "./shared/cli_helpers";
+import { fetchWithRetry } from "./shared/util";
+import { buildReunionsByBucket } from "../utils/reunion_parsing";
+import { buildSenatDossierIndex } from "../utils/reunion_odj_building";
+const optionsDefinitions = [
+    ...commonOptions,
+    {
+        help: "parse and convert agenda events into JSON",
+        name: "parseAgenda",
+        type: Boolean,
+    },
+];
+const options = commandLineArgs(optionsDefinitions);
+const SENAT_GLOBAL_AGENDA_URL_ROOT = "https://www.senat.fr/aglae/Global";
+const EVENT_DATE_FORMAT = "ddMMyyyy";
+class AgendaError extends Error {
+    constructor(message, agendaName) {
+        super(`An error occurred while retrieving Agenda ${agendaName}: ${message}`);
+    }
+}
+async function retrieveAgendas(options, sessions) {
+    console.log(`[AGENDA] Retrieving agendas for sessions ${sessions.join(", ")}`);
+    const agendaRootDir = path.join(options["dataDir"], AGENDA_FOLDER);
+    fs.ensureDirSync(agendaRootDir);
+    const originalAgendaDir = path.join(agendaRootDir, DATA_ORIGINAL_FOLDER);
+    fs.ensureDirSync(originalAgendaDir);
+    const transformedAgendaDir = path.join(agendaRootDir, DATA_TRANSFORMED_FOLDER);
+    if (options["parseAgenda"]) {
+        fs.ensureDirSync(transformedAgendaDir);
+    }
+    let dossierIndex = {};
+    dossierIndex = buildSenatDossierIndex(options);
+    for (const session of sessions) {
+        const originalAgendaSessionDir = path.join(originalAgendaDir, `${session}`);
+        fs.ensureDirSync(originalAgendaSessionDir);
+        if (!options["keepDir"])
+            fs.emptyDirSync(originalAgendaSessionDir);
+        const transformedAgendaSessionDir = path.join(transformedAgendaDir, `${session}`);
+        if (options["parseAgenda"]) {
+            fs.ensureDirSync(transformedAgendaSessionDir);
+            if (!options["keepDir"])
+                fs.emptyDirSync(transformedAgendaSessionDir);
+        }
+        const fifteenDaysFromNow = new Date();
+        fifteenDaysFromNow.setDate(fifteenDaysFromNow.getDate() + 15); // Don't download agendas more than 15 days in the future
+        for (const date = new Date(session, 9, 1); date <= new Date(session + 1, 8, 30) && date <= fifteenDaysFromNow; date.setDate(date.getDate() + 1)) {
+            const agendaName = DateTime.fromJSDate(date).toFormat(EVENT_DATE_FORMAT);
+            const agendaFileName = DateTime.fromJSDate(date).toFormat(ID_DATE_FORMAT);
+            const agendaPath = path.join(originalAgendaSessionDir, `${agendaFileName}.html`);
+            try {
+                await downloadAgenda(agendaName, agendaPath);
+                if (options["parseAgenda"]) {
+                    await parseAgenda(transformedAgendaSessionDir, agendaFileName, agendaPath, dossierIndex);
+                }
+            }
+            catch (error) {
+                console.error(error);
+            }
+        }
+    }
+}
+async function downloadAgenda(agendaName, agendaPath) {
+    const agendaUrl = `${SENAT_GLOBAL_AGENDA_URL_ROOT}/agl${agendaName}.html`;
+    if (!options["silent"]) {
+        console.log(`Downloading Agenda ${agendaUrl}…`);
+    }
+    const response = await fetchWithRetry(agendaUrl);
+    if (!response.ok) {
+        if (response.status === 404) {
+            console.warn(`Agenda ${agendaUrl} not found`);
+        }
+        else {
+            throw new AgendaError(String(response.status), agendaName);
+        }
+        return;
+    }
+    const agendaContent = await response.arrayBuffer();
+    if (!agendaContent) {
+        return;
+    }
+    fs.writeFileSync(agendaPath, Buffer.from(agendaContent));
+}
+function writeGroupsAsFiles(dir, groups) {
+    for (const g of groups) {
+        const outPath = path.join(dir, `${g.uid}.json`);
+        fs.writeJSONSync(outPath, g, { spaces: 2 });
+    }
+}
+async function parseAgenda(transformedAgendaSessionDir, agendaFileName, agendaPath, dossierBySenatUrl) {
+    if (!options["silent"])
+        console.log(`Parsing Agenda ${agendaPath}…`);
+    const parsedAgendaEvents = await parseAgendaFromFile(agendaPath);
+    if (!parsedAgendaEvents?.length)
+        return;
+    const flatPath = path.join(transformedAgendaSessionDir, `${agendaFileName}.json`);
+    fs.writeJSONSync(flatPath, parsedAgendaEvents, { spaces: 2 });
+    const byBucket = buildReunionsByBucket(parsedAgendaEvents, dossierBySenatUrl);
+    // SP
+    if (byBucket.IDS.length > 0) {
+        writeGroupsAsFiles(transformedAgendaSessionDir, byBucket.IDS);
+    }
+    // NON-SP
+    for (const suffix of ["IDC", "IDM", "IDO", "IDI"]) {
+        const groups = byBucket[suffix];
+        if (groups.length > 0) {
+            writeGroupsAsFiles(transformedAgendaSessionDir, groups);
+        }
+    }
+}
+async function main() {
+    const dataDir = options["dataDir"];
+    assert(dataDir, "Missing argument: data directory");
+    const sessions = getSessionsFromStart(options["fromSession"]);
+    console.time("agenda processing time");
+    await retrieveAgendas(options, sessions);
+    console.timeEnd("agenda processing time");
+}
+main()
+    .then(() => process.exit(0))
+    .catch((error) => {
+    console.log(error);
+    process.exit(1);
+});

package/lib/scripts/retrieve_cr_commission.d.ts

@@ -0,0 +1 @@
+export {};