@tricoteuses/senat 3.1.22 → 3.1.24

package/lib/scripts/convert_data.ts

@@ -1,243 +0,0 @@
-import assert from "assert"
-import commandLineArgs from "command-line-args"
-import fs from "fs-extra"
-import path from "path"
-
-import { datasets, EnabledDatasets, getEnabledDatasets } from "../datasets"
-import {
-  DOCUMENT_METADATA_FILE,
-  DOSLEG_DOSSIERS_FOLDER,
-  RAPPORT_FOLDER,
-  RapportMetadata,
-  SENS_CIRCONSCRIPTIONS_FOLDER,
-  SENS_ORGANISMES_FOLDER,
-  SENS_SENATEURS_FOLDER,
-  TEXTE_FOLDER,
-  TEXTE_ORIGINAL_FOLDER,
-  TexteMetadata,
-} from "../loaders"
-import {
-  findAllAmendements,
-  findAllCirconscriptions,
-  findAllLois,
-  findAllOrganismes,
-  findAllQuestions,
-  findAllSens,
-} from "../model"
-import { findSenatRapportUrls, findSenatTexteUrls } from "../model/dosleg"
-import { formatToFourDigitSession, SIGNET_STRUCTURE_REGEXP, UNDEFINED_SESSION } from "./datautil"
-import { commonOptions } from "./shared/cli_helpers"
-import { ensureAndClearDir } from "./shared/util"
-
-const optionsDefinitions = [
-  ...commonOptions,
-]
-const options = commandLineArgs(optionsDefinitions)
-
-const SENAT_TEXTE_XML_BASE_URL = "https://www.senat.fr/akomantoso/"
-const SENAT_TEXTE_BASE_URL = "https://www.senat.fr/leg/"
-const SENAT_EXPOSE_DES_MOTIFS_BASE_URL = "https://www.senat.fr/leg/exposes-des-motifs/"
-const SENAT_RAPPORT_BASE_URL = "https://www.senat.fr/rap/"
-
-async function convertData () {
-  const enabledDatasets = getEnabledDatasets(options.categories)
-  const dataDir: string = options.dataDir
-  assert(dataDir, "Missing argument: data directory")
-
-  console.time("data transformation time")
-
-  if (enabledDatasets & EnabledDatasets.Ameli) {
-    const dataset = datasets.ameli
-
-    if (!options.silent) {
-      console.log(`Converting database ${dataset.database} data into files…`)
-    }
-
-    const ameliReorganizedRootDir = path.join(dataDir, dataset.database)
-    ensureAndClearDir(ameliReorganizedRootDir)
-
-    for await (const amendement of findAllAmendements()) {
-      if (options.verbose) {
-        console.log(`Converting ${amendement.numero} file…`)
-      }
-
-      const session = String(amendement.session) || UNDEFINED_SESSION
-      const signetDossierLegislatif = amendement.signet_dossier_legislatif
-        || `${amendement.nature_texte}-${amendement.numero_texte}`.toLowerCase()
-      const ameliReorganizedDir = path.join(ameliReorganizedRootDir, session, signetDossierLegislatif)
-      fs.ensureDirSync(ameliReorganizedDir)
-      const amendementFileName = `${amendement.numero}.json`
-      fs.writeJSONSync(path.join(ameliReorganizedDir, amendementFileName), amendement, { spaces: 2 })
-    }
-  }
-
-  if (enabledDatasets & EnabledDatasets.DosLeg) {
-    const dataset = datasets.dosleg
-
-    if (!options.silent) {
-      console.log(`Converting database ${dataset.database} data into files…`)
-    }
-
-    const doslegReorganizedRootDir = path.join(dataDir, dataset.database)
-    const dossiersReorganizedDir = path.join(doslegReorganizedRootDir, DOSLEG_DOSSIERS_FOLDER)
-    ensureAndClearDir(doslegReorganizedRootDir)
-    ensureAndClearDir(dossiersReorganizedDir)
-
-    for await (const loi of findAllLois()) {
-      if (options.verbose) {
-        console.log(`Converting ${loi.signet} file…`)
-      }
-
-      let loiReorganizedDir = path.join(dossiersReorganizedDir, UNDEFINED_SESSION)
-      const signetParts = SIGNET_STRUCTURE_REGEXP.exec(loi.signet)?.groups
-      if (signetParts) {
-        const { session } = signetParts
-        const formattedSession = formatToFourDigitSession(session)
-        loiReorganizedDir = path.join(dossiersReorganizedDir, formattedSession)
-      }
-      fs.ensureDirSync(loiReorganizedDir)
-
-      const loiFileName = `${loi.signet}.json`
-      fs.writeJSONSync(path.join(loiReorganizedDir, loiFileName), loi, { spaces: 2 })
-    }
-
-    await convertTexteUrls(dataDir)
-    await convertRapportUrls(dataDir)
-  }
-
-  if (enabledDatasets & EnabledDatasets.Questions) {
-    const dataset = datasets.questions
-
-    if (!options.silent) {
-      console.log(`Converting database ${dataset.database} data into files…`)
-    }
-
-    const questionsReorganizedRootDir = path.join(dataDir, dataset.database)
-    ensureAndClearDir(questionsReorganizedRootDir)
-
-    for await (const question of findAllQuestions()) {
-      if (options.verbose) {
-        console.log(`Converting ${question.reference} file…`)
-      }
-
-      const legislature = question.legislature ? question.legislature : 0
-      const questionReorganizedDir = path.join(questionsReorganizedRootDir, String(legislature))
-      fs.ensureDirSync(questionReorganizedDir)
-      const questionFileName = `${question.reference}.json`
-      fs.writeJSONSync(path.join(questionReorganizedDir, questionFileName), question, { spaces: 2 })
-    }
-  }
-
-  if (enabledDatasets & EnabledDatasets.Sens) {
-    const dataset = datasets.sens
-
-    if (!options.silent) {
-      console.log(`Converting database ${dataset.database} data into files…`)
-    }
-
-    const sensReorganizedRootDir = path.join(dataDir, dataset.database)
-    const senateursReorganizedDir = path.join(sensReorganizedRootDir, SENS_SENATEURS_FOLDER)
-    const circonscriptionsReorganizedDir = path.join(sensReorganizedRootDir, SENS_CIRCONSCRIPTIONS_FOLDER)
-    const organismesReorganizedDir = path.join(sensReorganizedRootDir, SENS_ORGANISMES_FOLDER)
-    ensureAndClearDir(sensReorganizedRootDir)
-    ensureAndClearDir(senateursReorganizedDir)
-    ensureAndClearDir(circonscriptionsReorganizedDir)
-    ensureAndClearDir(organismesReorganizedDir)
-
-    for await (const sen of findAllSens()) {
-      if (options.verbose) {
-        console.log(`Converting ${sen.matricule} file…`)
-      }
-
-      const senFileName = `${sen.matricule}.json`
-      fs.writeJSONSync(path.join(senateursReorganizedDir, senFileName), sen, { spaces: 2 })
-    }
-
-    for await (const circonscription of findAllCirconscriptions()) {
-      if (options.verbose) {
-        console.log(`Converting ${circonscription.identifiant} file…`)
-      }
-
-      const circonscriptionFileName = `${circonscription.identifiant}.json`
-      fs.writeJSONSync(
-        path.join(circonscriptionsReorganizedDir, circonscriptionFileName), circonscription, { spaces: 2 }
-      )
-    }
-
-    for await (const organisme of findAllOrganismes()) {
-      if (options.verbose) {
-        console.log(`Converting ${organisme.code} file…`)
-      }
-
-      const organismeFileName = `${organisme.code}.json`
-      fs.writeJSONSync(path.join(organismesReorganizedDir, organismeFileName), organisme, { spaces: 2 })
-    }
-  }
-
-  if (!options.silent) {
-    console.timeEnd("data transformation time")
-  }
-}
-
-async function convertTexteUrls (dataDir: string) {
-  const textesDir = path.join(dataDir, TEXTE_FOLDER)
-  fs.ensureDirSync(textesDir)
-
-  const originalTextesDir = path.join(textesDir, TEXTE_ORIGINAL_FOLDER)
-
-  for await (const texte of findSenatTexteUrls(options.sessions)) {
-    const texteName = path.parse(texte.url).name
-    const texteDir = path.join(originalTextesDir, `${texte.session ?? UNDEFINED_SESSION}`, texteName)
-    fs.ensureDirSync(texteDir)
-
-    const metadata: TexteMetadata = {
-      name: texteName,
-      session: texte.session,
-      url_expose_des_motifs: texte.hasExposeDesMotifs ?
-        new URL(`${texteName}-expose.html`, SENAT_EXPOSE_DES_MOTIFS_BASE_URL) : undefined,
-      url_xml: new URL(`${texteName}.akn.xml`, SENAT_TEXTE_XML_BASE_URL),
-      url_html: new URL(`${texteName}.html`, SENAT_TEXTE_BASE_URL),
-      url_pdf: new URL(`${texteName}.pdf`, SENAT_TEXTE_BASE_URL),
-    }
-    fs.writeJSONSync(path.join(texteDir, DOCUMENT_METADATA_FILE), metadata, { spaces: 2 })
-  }
-}
-
-async function convertRapportUrls (dataDir: string) {
-  const rapportsDir = path.join(dataDir, RAPPORT_FOLDER)
-  fs.ensureDirSync(rapportsDir)
-
-  for await (const rapport of findSenatRapportUrls(options.sessions)) {
-    const parsedRapportUrl = path.parse(rapport.url)
-    const rapportName = parsedRapportUrl.name
-    const rapportDir = path.join(rapportsDir, `${rapport.session ?? UNDEFINED_SESSION}`, rapportName)
-    fs.ensureDirSync(rapportDir)
-
-    const rapportHtmlUrlBase = `${rapportName}_mono.html`
-    const rapportHtmlUrl = path.format({
-      dir: parsedRapportUrl.dir,
-      base: rapportHtmlUrlBase,
-    })
-
-    const rapportPdfUrlBase = `${rapportName}1.pdf`
-    const rapportPdfUrl = path.format({
-      dir: parsedRapportUrl.dir,
-      base: rapportPdfUrlBase,
-    })
-
-    const metadata: RapportMetadata = {
-      name: rapportName,
-      session: rapport.session,
-      url_html: new URL(rapportHtmlUrl, SENAT_RAPPORT_BASE_URL),
-      url_pdf: new URL(rapportPdfUrl, SENAT_RAPPORT_BASE_URL),
-    }
-    fs.writeJSONSync(path.join(rapportDir, DOCUMENT_METADATA_FILE), metadata, { spaces: 2 })
-  }
-}
-
-convertData()
-  .then(() => process.exit(0))
-  .catch((error) => {
-    console.log(error)
-    process.exit(1)
-  })

package/lib/scripts/data-download.d.ts

@@ -1 +0,0 @@
-export {};

package/lib/scripts/data-download.js

@@ -1,12 +0,0 @@
-import { execSync } from "child_process";
-const args = process.argv.slice(2).join(" ");
-try {
-    execSync(`tsx src/scripts/retrieve_open_data.ts --all ${args}`, { stdio: "inherit" });
-    execSync(`tsx src/scripts/convert_data.ts ${args}`, { stdio: "inherit" });
-}
-catch (error) {
-    if (error.status !== 10) {
-        console.error("Error during data retrieval:", error);
-        process.exit(error.status || 1);
-    }
-}

package/lib/scripts/datautil.d.ts

@@ -1,8 +0,0 @@
-import { Session } from "../types/sessions";
-export declare const STANDARD_DATE_FORMAT = "yyyy-MM-dd";
-export declare const ID_DATE_FORMAT = "yyyyMMdd";
-export declare const AKN_IDENTIFICATION_STRUCTURE_REGEXP: RegExp;
-export declare const AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP: RegExp;
-export declare const SIGNET_STRUCTURE_REGEXP: RegExp;
-export declare function getSessionFromSignet(signet: string): Session | null;
-export declare function getSessionFromDate(date: string, format?: string): Session;

package/lib/scripts/datautil.js

@@ -1,34 +0,0 @@
-import { DateTime, Settings } from "luxon";
-Settings.twoDigitCutoffYear = 50;
-export const STANDARD_DATE_FORMAT = "yyyy-MM-dd";
-export const ID_DATE_FORMAT = "yyyyMMdd";
-export const AKN_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{4}-\d{4})\/?(?<numTexte>\d+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/;
-export const AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{2,4})\/?(?<numTexte>[a-zA-Z0-9]+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/;
-export const SIGNET_STRUCTURE_REGEXP = /^(?<type>[a-z]+)(?<session>\d{2,4})-?(?<numTexte>\d+)?/;
-export function getSessionFromSignet(signet) {
-    const signetParts = SIGNET_STRUCTURE_REGEXP.exec(signet)?.groups;
-    if (signetParts && "session" in signetParts) {
-        const { session } = signetParts;
-        return formatToFourDigitSession(session);
-    }
-    return null;
-}
-function formatToFourDigitSession(session) {
-    if (session.length >= 2) {
-        const sessionFirstTwoDigits = session.substring(0, 2);
-        const sessionLastTwoDigits = session.substring(session.length - 2);
-        const twoDigitSession = parseInt(sessionFirstTwoDigits) === parseInt(sessionLastTwoDigits) - 1
-            ? sessionFirstTwoDigits
-            : sessionLastTwoDigits;
-        return parseInt(DateTime.fromFormat(String(twoDigitSession), "yy").toFormat("yyyy"));
-    }
-    return null;
-}
-export function getSessionFromDate(date, format = STANDARD_DATE_FORMAT) {
-    const parsedDate = DateTime.fromFormat(date, format);
-    const endSessionDate = DateTime.fromObject({ year: parsedDate.year, month: 9, day: 30 });
-    if (parsedDate < endSessionDate) {
-        return (parsedDate.year - 1);
-    }
-    return parsedDate.year;
-}

package/lib/scripts/datautil.mjs

@@ -1,16 +0,0 @@
-import { DateTime, Settings } from "luxon";
-Settings.twoDigitCutoffYear = 50;
-export const SIGNET_STRUCTURE_REGEXP = /^(?<type>[a-z]+)(?<session>\d{2,4})-?(?<numTexte>\d+)?/;
-export const AKN_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{4}-\d{4})\/?(?<numTexte>\d+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/;
-export const AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{2,4})\/?(?<numTexte>[a-zA-Z0-9]+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/;
-export const UNDEFINED_SESSION = "0";
-export function formatToFourDigitSession(session) {
-    if (session.length >= 2) {
-        const sessionFirstTwoDigits = session.substring(0, 2);
-        const sessionLastTwoDigits = session.substring(session.length - 2);
-        const twoDigitSession = parseInt(sessionFirstTwoDigits) === parseInt(sessionLastTwoDigits) - 1 ?
-            sessionFirstTwoDigits : sessionLastTwoDigits;
-        return DateTime.fromFormat(String(twoDigitSession), "yy").toFormat("yyyy");
-    }
-    return UNDEFINED_SESSION;
-}

package/lib/scripts/datautil.ts

@@ -1,19 +0,0 @@
-import { DateTime, Settings } from "luxon"
-
-Settings.twoDigitCutoffYear = 50
-
-export const SIGNET_STRUCTURE_REGEXP = /^(?<type>[a-z]+)(?<session>\d{2,4})-?(?<numTexte>\d+)?/
-export const AKN_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{4}-\d{4})\/?(?<numTexte>\d+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/
-export const AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP = /^\/akn\/fr\/(?<type>[a-z]+)\/(?<session>\d{2,4})\/?(?<numTexte>[a-zA-Z0-9]+)\/fr@(?<version>\b(?:RECT|RECT_BIS|RECT_TER|RECT_QUATER|RECT_QUINQUIES)\b)?/
-export const UNDEFINED_SESSION = "0"
-
-export function formatToFourDigitSession (session: string) {
-  if (session.length >= 2) {
-    const sessionFirstTwoDigits = session.substring(0, 2)
-    const sessionLastTwoDigits = session.substring(session.length - 2)
-    const twoDigitSession = parseInt(sessionFirstTwoDigits) === parseInt(sessionLastTwoDigits) - 1 ?
-      sessionFirstTwoDigits : sessionLastTwoDigits
-    return DateTime.fromFormat(String(twoDigitSession), "yy").toFormat("yyyy")
-  }
-  return UNDEFINED_SESSION
-}

package/lib/scripts/parse_textes.d.ts

@@ -1 +0,0 @@
-export {};

package/lib/scripts/parse_textes.js

@@ -1,44 +0,0 @@
-import assert from "assert";
-import commandLineArgs from "command-line-args";
-import fs from "fs-extra";
-import path from "path";
-import { iterFilePaths, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, } from "../loaders";
-import { parseExposeDesMotifsFromFile, parseTexteFromFile, } from "../model/texte";
-import { commonOptions } from "./shared/cli_helpers";
-import { ensureAndClearDir } from "./shared/util";
-const optionsDefinitions = [...commonOptions];
-const options = commandLineArgs(optionsDefinitions);
-async function main() {
-    const dataDir = options["dataDir"];
-    assert(dataDir, "Missing argument: data directory");
-    const transformedTextesDir = path.join(options["dataDir"], TEXTE_FOLDER, DATA_TRANSFORMED_FOLDER);
-    ensureAndClearDir(transformedTextesDir);
-    for (const filePath of iterFilePaths(path.join(dataDir, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER))) {
-        const parsedFilePath = path.parse(filePath);
-        if (parsedFilePath.ext !== ".xml") {
-            continue;
-        }
-        const texteDirFromOriginal = parsedFilePath.dir.substring(filePath.indexOf(DATA_ORIGINAL_FOLDER) + DATA_ORIGINAL_FOLDER.length);
-        const transformedTexteDir = path.join(transformedTextesDir, texteDirFromOriginal);
-        fs.ensureDirSync(transformedTexteDir);
-        if (!options["silent"]) {
-            console.log(`Parsing texte ${parsedFilePath.name}.xml…`);
-        }
-        const parsedTexte = await parseTexteFromFile(filePath);
-        const exposeDesMotifsFileName = `${parsedFilePath.name}-expose`;
-        const exposeDesMotifsFilePath = path.join(parsedFilePath.dir, `${exposeDesMotifsFileName}.html`);
-        if (parsedTexte && fs.existsSync(exposeDesMotifsFilePath)) {
-            if (!options["silent"]) {
-                console.log(`Parsing exposé des motifs ${exposeDesMotifsFileName}.html…`);
-            }
-            parsedTexte.exposeDesMotifs = await parseExposeDesMotifsFromFile(exposeDesMotifsFilePath);
-        }
-        fs.writeJSONSync(path.join(transformedTexteDir, `${parsedFilePath.name}.json`), parsedTexte, { spaces: 2 });
-    }
-}
-main()
-    .then(() => process.exit(0))
-    .catch((error) => {
-    console.log(error);
-    process.exit(1);
-});

package/lib/scripts/parse_textes.mjs

@@ -1,46 +0,0 @@
-import assert from "assert";
-import commandLineArgs from "command-line-args";
-import fs from "fs-extra";
-import path from "path";
-import { iterFilePaths, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER, TEXTE_TRANSFORMED_FOLDER } from "../loaders";
-import { parseExposeDesMotifsFromFile, parseTexteFromFile } from "../model/texte";
-import { commonOptions } from "./shared/cli_helpers";
-import { ensureAndClearDir } from "./shared/util";
-const optionsDefinitions = [
-    ...commonOptions,
-];
-const options = commandLineArgs(optionsDefinitions);
-async function main() {
-    const dataDir = options.dataDir;
-    assert(dataDir, "Missing argument: data directory");
-    const transformedTextesDir = path.join(options.dataDir, TEXTE_FOLDER, TEXTE_TRANSFORMED_FOLDER);
-    ensureAndClearDir(transformedTextesDir);
-    for (const filePath of iterFilePaths(path.join(dataDir, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER))) {
-        const parsedFilePath = path.parse(filePath);
-        if (parsedFilePath.ext !== ".xml") {
-            continue;
-        }
-        const texteDirFromOriginal = parsedFilePath.dir.substring(filePath.indexOf(TEXTE_ORIGINAL_FOLDER) + TEXTE_ORIGINAL_FOLDER.length);
-        const transformedTexteDir = path.join(transformedTextesDir, texteDirFromOriginal);
-        fs.ensureDirSync(transformedTexteDir);
-        if (!options.silent) {
-            console.log(`Parsing texte ${parsedFilePath.name}.xml…`);
-        }
-        const parsedTexte = await parseTexteFromFile(filePath);
-        const exposeDesMotifsFileName = `${parsedFilePath.name}-expose`;
-        const exposeDesMotifsFilePath = path.join(parsedFilePath.dir, `${exposeDesMotifsFileName}.html`);
-        if (parsedTexte && fs.existsSync(exposeDesMotifsFilePath)) {
-            if (!options.silent) {
-                console.log(`Parsing exposé des motifs ${exposeDesMotifsFileName}.html…`);
-            }
-            parsedTexte.exposeDesMotifs = await parseExposeDesMotifsFromFile(exposeDesMotifsFilePath);
-        }
-        fs.writeJSONSync(path.join(transformedTexteDir, `${parsedFilePath.name}.json`), parsedTexte, { spaces: 2 });
-    }
-}
-main()
-    .then(() => process.exit(0))
-    .catch((error) => {
-    console.log(error);
-    process.exit(1);
-});

package/lib/scripts/parse_textes.ts

@@ -1,65 +0,0 @@
-import assert from "assert"
-import commandLineArgs from "command-line-args"
-import fs from "fs-extra"
-import path from "path"
-
-import { iterFilePaths, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER, TEXTE_TRANSFORMED_FOLDER } from "../loaders"
-import { parseExposeDesMotifsFromFile, parseTexteFromFile } from "../model/texte"
-import { commonOptions } from "./shared/cli_helpers"
-import { ensureAndClearDir } from "./shared/util"
-
-const optionsDefinitions = [
-  ...commonOptions,
-]
-const options = commandLineArgs(optionsDefinitions)
-
-async function main() {
-  const dataDir = options.dataDir
-  assert(dataDir, "Missing argument: data directory")
-
-  const transformedTextesDir = path.join(options.dataDir, TEXTE_FOLDER, TEXTE_TRANSFORMED_FOLDER)
-  ensureAndClearDir(transformedTextesDir)
-
-  for (const filePath of iterFilePaths(path.join(dataDir, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER))) {
-    const parsedFilePath = path.parse(filePath)
-
-    if (parsedFilePath.ext !== ".xml") {
-      continue
-    }
-
-    const texteDirFromOriginal = parsedFilePath.dir.substring(
-      filePath.indexOf(TEXTE_ORIGINAL_FOLDER) + TEXTE_ORIGINAL_FOLDER.length
-    )
-    const transformedTexteDir = path.join(transformedTextesDir, texteDirFromOriginal)
-    fs.ensureDirSync(transformedTexteDir)
-
-    if (!options.silent) {
-      console.log(`Parsing texte ${parsedFilePath.name}.xml…`)
-    }
-
-    const parsedTexte = await parseTexteFromFile(filePath)
-
-    const exposeDesMotifsFileName = `${parsedFilePath.name}-expose`
-    const exposeDesMotifsFilePath = path.join(parsedFilePath.dir, `${exposeDesMotifsFileName}.html`)
-    if (parsedTexte && fs.existsSync(exposeDesMotifsFilePath)) {
-      if (!options.silent) {
-        console.log(`Parsing exposé des motifs ${exposeDesMotifsFileName}.html…`)
-      }
-
-      parsedTexte.exposeDesMotifs = await parseExposeDesMotifsFromFile(exposeDesMotifsFilePath)
-    }
-
-    fs.writeJSONSync(
-      path.join(transformedTexteDir, `${parsedFilePath.name}.json`),
-      parsedTexte,
-      { spaces: 2 }
-    )
-  }
-}
-
-main()
-  .then(() => process.exit(0))
-  .catch((error) => {
-    console.log(error)
-    process.exit(1)
-  })

package/lib/scripts/retrieve_agenda.d.ts

@@ -1 +0,0 @@
-export {};

package/lib/scripts/retrieve_agenda.js

@@ -1,132 +0,0 @@
-import assert from "assert";
-import commandLineArgs from "command-line-args";
-import fs from "fs-extra";
-import { DateTime } from "luxon";
-import path from "path";
-import { AGENDA_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
-import { parseAgendaFromFile } from "../model/agenda";
-import { getSessionsFromStart } from "../types/sessions";
-import { ID_DATE_FORMAT } from "./datautil";
-import { commonOptions } from "./shared/cli_helpers";
-import { fetchWithRetry } from "./shared/util";
-import { buildReunionsByBucket } from "../utils/reunion_parsing";
-import { buildSenatDossierIndex } from "../utils/reunion_odj_building";
-const optionsDefinitions = [
-    ...commonOptions,
-    {
-        help: "parse and convert agenda events into JSON",
-        name: "parseAgenda",
-        type: Boolean,
-    },
-];
-const options = commandLineArgs(optionsDefinitions);
-const SENAT_GLOBAL_AGENDA_URL_ROOT = "https://www.senat.fr/aglae/Global";
-const EVENT_DATE_FORMAT = "ddMMyyyy";
-class AgendaError extends Error {
-    constructor(message, agendaName) {
-        super(`An error occurred while retrieving Agenda ${agendaName}: ${message}`);
-    }
-}
-async function retrieveAgendas(options, sessions) {
-    console.log(`[AGENDA] Retrieving agendas for sessions ${sessions.join(", ")}`);
-    const agendaRootDir = path.join(options["dataDir"], AGENDA_FOLDER);
-    fs.ensureDirSync(agendaRootDir);
-    const originalAgendaDir = path.join(agendaRootDir, DATA_ORIGINAL_FOLDER);
-    fs.ensureDirSync(originalAgendaDir);
-    const transformedAgendaDir = path.join(agendaRootDir, DATA_TRANSFORMED_FOLDER);
-    if (options["parseAgenda"]) {
-        fs.ensureDirSync(transformedAgendaDir);
-    }
-    let dossierIndex = {};
-    dossierIndex = buildSenatDossierIndex(options);
-    for (const session of sessions) {
-        const originalAgendaSessionDir = path.join(originalAgendaDir, `${session}`);
-        fs.ensureDirSync(originalAgendaSessionDir);
-        if (!options["keepDir"])
-            fs.emptyDirSync(originalAgendaSessionDir);
-        const transformedAgendaSessionDir = path.join(transformedAgendaDir, `${session}`);
-        if (options["parseAgenda"]) {
-            fs.ensureDirSync(transformedAgendaSessionDir);
-            if (!options["keepDir"])
-                fs.emptyDirSync(transformedAgendaSessionDir);
-        }
-        const fifteenDaysFromNow = new Date();
-        fifteenDaysFromNow.setDate(fifteenDaysFromNow.getDate() + 15); // Don't download agendas more than 15 days in the future
-        for (const date = new Date(session, 9, 1); date <= new Date(session + 1, 8, 30) && date <= fifteenDaysFromNow; date.setDate(date.getDate() + 1)) {
-            const agendaName = DateTime.fromJSDate(date).toFormat(EVENT_DATE_FORMAT);
-            const agendaFileName = DateTime.fromJSDate(date).toFormat(ID_DATE_FORMAT);
-            const agendaPath = path.join(originalAgendaSessionDir, `${agendaFileName}.html`);
-            try {
-                await downloadAgenda(agendaName, agendaPath);
-                if (options["parseAgenda"]) {
-                    await parseAgenda(transformedAgendaSessionDir, agendaFileName, agendaPath, dossierIndex);
-                }
-            }
-            catch (error) {
-                console.error(error);
-            }
-        }
-    }
-}
-async function downloadAgenda(agendaName, agendaPath) {
-    const agendaUrl = `${SENAT_GLOBAL_AGENDA_URL_ROOT}/agl${agendaName}.html`;
-    if (!options["silent"]) {
-        console.log(`Downloading Agenda ${agendaUrl}…`);
-    }
-    const response = await fetchWithRetry(agendaUrl);
-    if (!response.ok) {
-        if (response.status === 404) {
-            console.warn(`Agenda ${agendaUrl} not found`);
-        }
-        else {
-            throw new AgendaError(String(response.status), agendaName);
-        }
-        return;
-    }
-    const agendaContent = await response.arrayBuffer();
-    if (!agendaContent) {
-        return;
-    }
-    fs.writeFileSync(agendaPath, Buffer.from(agendaContent));
-}
-function writeGroupsAsFiles(dir, groups) {
-    for (const g of groups) {
-        const outPath = path.join(dir, `${g.uid}.json`);
-        fs.writeJSONSync(outPath, g, { spaces: 2 });
-    }
-}
-async function parseAgenda(transformedAgendaSessionDir, agendaFileName, agendaPath, dossierBySenatUrl) {
-    if (!options["silent"])
-        console.log(`Parsing Agenda ${agendaPath}…`);
-    const parsedAgendaEvents = await parseAgendaFromFile(agendaPath);
-    if (!parsedAgendaEvents?.length)
-        return;
-    const flatPath = path.join(transformedAgendaSessionDir, `${agendaFileName}.json`);
-    fs.writeJSONSync(flatPath, parsedAgendaEvents, { spaces: 2 });
-    const byBucket = buildReunionsByBucket(parsedAgendaEvents, dossierBySenatUrl);
-    // SP
-    if (byBucket.IDS.length > 0) {
-        writeGroupsAsFiles(transformedAgendaSessionDir, byBucket.IDS);
-    }
-    // NON-SP
-    for (const suffix of ["IDC", "IDM", "IDO", "IDI"]) {
-        const groups = byBucket[suffix];
-        if (groups.length > 0) {
-            writeGroupsAsFiles(transformedAgendaSessionDir, groups);
-        }
-    }
-}
-async function main() {
-    const dataDir = options["dataDir"];
-    assert(dataDir, "Missing argument: data directory");
-    const sessions = getSessionsFromStart(options["fromSession"]);
-    console.time("agenda processing time");
-    await retrieveAgendas(options, sessions);
-    console.timeEnd("agenda processing time");
-}
-main()
-    .then(() => process.exit(0))
-    .catch((error) => {
-    console.log(error);
-    process.exit(1);
-});

package/lib/scripts/retrieve_cr_commission.d.ts

@@ -1 +0,0 @@
-export {};