@tricoteuses/senat 2.1.2 → 2.2.2

package/README.md

@@ -54,8 +54,8 @@ npm run data:retrieve_documents ../senat-data -- --fromSession 2023 --parseDocum
 # Parsing only
 npm run data:parse_textes_lois ../senat-data
 
-# Retrieval of agenda from Sénat's website
-npm run data:retrieve_agenda ../senat-data -- --fromSession 2023
+# Retrieval (& parsing) of agenda from Sénat's website
+npm run data:retrieve_agenda ../senat-data -- --fromSession 2023 [--parseAgenda]
 
 # Retrieval of sénateurs' pictures from Sénat's website
 npm run data:retrieve_senateurs_photos ../senat-data

package/lib/index.d.ts

@@ -2,6 +2,7 @@ export type { AmendementResult, } from "./model/ameli";
 export type { DossierLegislatifResult, } from "./model/dosleg";
 export type { QuestionResult, } from "./model/questions";
 export type { CirconscriptionResult, OrganismeResult, SenateurResult, } from "./model/sens";
+export type { AgendaEvent } from "./types/agenda";
 export type { Ses, Sub, TxtAmeli } from "./types/ameli";
 export type { Debat, LecAssDeb } from "./types/debats";
 export type { Ass, Aud, Auteur, DateSeance, DecCoc, DenRap, DocAtt, Ecr, EtaLoi, LecAss, LecAssRap, Lecture, Loi, Org, OriTxt, Qua, Rap, RapOrg, Scr, Texte, TypAtt, TypLec, TypLoi, TypTxt, TypUrl, } from "./types/dosleg";

package/lib/loaders.d.ts

@@ -2,6 +2,7 @@ import { AmendementResult } from "./model/ameli";
 import { DossierLegislatifResult } from "./model/dosleg";
 import { QuestionResult } from "./model/questions";
 import { CirconscriptionResult, OrganismeResult, SenateurResult } from "./model/sens";
+import { AgendaEvent } from "./types/agenda";
 import { FlatTexte } from "./types/texte";
 export { EnabledDatasets } from "./datasets";
 export declare const AGENDA_FOLDER = "agenda";
@@ -59,7 +60,8 @@ export declare function iterLoadSenatDossiersLegislatifsTexteUrls(dataDir: strin
 export declare function iterLoadSenatDossiersLegislatifsRapportUrls(dataDir: string, session: number | undefined): Generator<IterItem<RapportMetadata>>;
 export declare function iterLoadSenatDossiersLegislatifsTextes(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DossierLegislatifTexteResult>>;
 export declare function loadSenatTexteContent(dataDir: string, textePathFromDataset: string): IterItem<FlatTexte | null>;
+export declare function iterLoadSenatEvenements(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<AgendaEvent>>;
+export declare function iterLoadSenatCirconscriptions(dataDir: string, options?: {}): Generator<IterItem<CirconscriptionResult>>;
 export declare function iterLoadSenatOrganismes(dataDir: string, options?: {}): Generator<IterItem<OrganismeResult>>;
 export declare function iterLoadSenatSenateurs(dataDir: string, legislature: number, options?: {}): Generator<IterItem<SenateurResult>>;
-export declare function iterLoadSenatCirconscriptions(dataDir: string, options?: {}): Generator<IterItem<CirconscriptionResult>>;
 export declare function iterLoadSenatQuestions(dataDir: string, legislature: number, options?: {}): Generator<IterItem<QuestionResult>>;

package/lib/loaders.js

@@ -27,8 +27,8 @@ export function* iterFilePaths(dirPath) {
         }
     }
 }
-function* iterLoadSenatItems(dataDir, dataset, legislatureOrSession, subDir, { log = false } = {}) {
-    let itemsDir = path.join(dataDir, dataset.database);
+function* iterLoadSenatItems(dataDir, dataName, legislatureOrSession, subDir, { log = false } = {}) {
+    let itemsDir = path.join(dataDir, dataName);
     if (subDir) {
         itemsDir = path.join(itemsDir, subDir);
     }
@@ -41,7 +41,7 @@ function* iterLoadSenatItems(dataDir, dataset, legislatureOrSession, subDir, { l
         }
         const itemJson = fs.readFileSync(filePath, { encoding: "utf8" });
         const item = JSON.parse(itemJson);
-        const filePathFromDataset = filePath.substring(filePath.indexOf(dataset.database) + dataset.database.length);
+        const filePathFromDataset = filePath.substring(filePath.indexOf(dataName) + dataName.length);
         yield {
             item,
             filePathFromDataset,
@@ -50,12 +50,12 @@ function* iterLoadSenatItems(dataDir, dataset, legislatureOrSession, subDir, { l
     }
 }
 export function* iterLoadSenatAmendements(dataDir, session, options = {}) {
-    for (const amendementItem of iterLoadSenatItems(dataDir, datasets.ameli, session, undefined, options)) {
+    for (const amendementItem of iterLoadSenatItems(dataDir, datasets.ameli.database, session, undefined, options)) {
         yield amendementItem;
     }
 }
 export function* iterLoadSenatDossiersLegislatifs(dataDir, session, options = {}) {
-    for (const dossierLegislatifItem of iterLoadSenatItems(dataDir, datasets.dosleg, session, DOSLEG_DOSSIERS_FOLDER, options)) {
+    for (const dossierLegislatifItem of iterLoadSenatItems(dataDir, datasets.dosleg.database, session, DOSLEG_DOSSIERS_FOLDER, options)) {
         yield dossierLegislatifItem;
     }
 }
@@ -129,15 +129,27 @@ export function loadSenatTexteContent(dataDir, textePathFromDataset) {
     const texteJson = fs.readFileSync(fullTextePath, { encoding: "utf8" });
     return { item: JSON.parse(texteJson) };
 }
+export function* iterLoadSenatEvenements(dataDir, session, options = {}) {
+    for (const evenementsItem of iterLoadSenatItems(dataDir, AGENDA_FOLDER, session, DATA_TRANSFORMED_FOLDER, options)) {
+        for (const evenement of evenementsItem.item) {
+            yield { item: evenement };
+        }
+    }
+}
+export function* iterLoadSenatCirconscriptions(dataDir, options = {}) {
+    for (const circonscriptionItem of iterLoadSenatItems(dataDir, datasets.sens.database, undefined, SENS_CIRCONSCRIPTIONS_FOLDER, options)) {
+        yield circonscriptionItem;
+    }
+}
 export function* iterLoadSenatOrganismes(dataDir, options = {}) {
-    for (const organismeItem of iterLoadSenatItems(dataDir, datasets.sens, undefined, SENS_ORGANISMES_FOLDER, options)) {
+    for (const organismeItem of iterLoadSenatItems(dataDir, datasets.sens.database, undefined, SENS_ORGANISMES_FOLDER, options)) {
         yield organismeItem;
     }
 }
 export function* iterLoadSenatSenateurs(dataDir, legislature, options = {}) {
     const dateDebutLegislatureStr = legislatures.find((legislatureInfo) => legislatureInfo.numero === legislature)?.date_debut;
     const dateDebutLegislature = new Date(dateDebutLegislatureStr);
-    for (const senateurItem of iterLoadSenatItems(dataDir, datasets.sens, undefined, SENS_SENATEURS_FOLDER, options)) {
+    for (const senateurItem of iterLoadSenatItems(dataDir, datasets.sens.database, undefined, SENS_SENATEURS_FOLDER, options)) {
         const dateFinMandatSenateur = senateurItem.item.mandats_senateur[0]
             ?.date_fin
             ? new Date(senateurItem.item.mandats_senateur[0]?.date_fin)
@@ -148,13 +160,8 @@ export function* iterLoadSenatSenateurs(dataDir, legislature, options = {}) {
         yield senateurItem;
     }
 }
-export function* iterLoadSenatCirconscriptions(dataDir, options = {}) {
-    for (const circonscriptionItem of iterLoadSenatItems(dataDir, datasets.sens, undefined, SENS_CIRCONSCRIPTIONS_FOLDER, options)) {
-        yield circonscriptionItem;
-    }
-}
 export function* iterLoadSenatQuestions(dataDir, legislature, options = {}) {
-    for (const questionItem of iterLoadSenatItems(dataDir, datasets.questions, legislature, undefined, options)) {
+    for (const questionItem of iterLoadSenatItems(dataDir, datasets.questions.database, legislature, undefined, options)) {
         yield questionItem;
     }
 }

package/lib/model/agenda.js

@@ -1,5 +1,6 @@
 import { JSDOM } from "jsdom";
 import { DateTime } from "luxon";
+import path from "path";
 function getEventType(eventClasses) {
     const typeClass = [...eventClasses]
         .find(className => className.startsWith("evt-"))
@@ -23,52 +24,71 @@ function getUrlDossierSenat(lienElements) {
         .find(lienElement => lienElement.textContent?.includes("dossier législatif"));
     return urlElement ? urlElement.getAttribute("href") : null;
 }
-function getHeuresDebutFin(timeStr) {
-    const normalizedHeureDebut = timeStr
-        ?.replace(/^À l'issue de l'espace réservé .* et au plus tard\s/i, "") // Must be first
-        ?.replace(/^(?:le )?matin/i, "10h00")
-        ?.replace(/^(?:l')?après-midi/i, "16h00")
-        ?.replace(/^(?:le )?soir/i, "20h00")
-        ?.replace(/^(?:la )?nuit/i, "22h00")
+/**
+ * Normalize time string to become a simple start time ("H'h'mm") or a duration ("'de 'H'h'mm' à 'H'h'mm").
+ */
+function normalizeTime(timeStr) {
+    return timeStr
+        ?.replace(/^À l'issue de l'espace réservé .* et au plus tard\s/i, "") // Must be processed first
+        ?.replace(/^(?:le )?matin/i, "10h00") // We chose "matin" to mean 10h00
+        ?.replace(/^(?:l')?après-midi/i, "16h00") // We chose "après-midi" to mean 16h00
+        ?.replace(/^(?:le )?soir/i, "20h00") // We chose "soir" to mean 20h00
+        ?.replace(/^(?:la )?nuit/i, "22h00") // We chose "nuit" to mean 22h00
         ?.replace(/^à\s/ig, "")
         ?.replace(/heures/ig, "h00")
         ?.replace(/\set.*/i, "")
         ?.replace(/,.*/, "")
         ?.replace(/\s\(hors hémicycle\)/i, "")
         ?.replace(/\s*h\s*/ig, "h");
-    console.dir(`${timeStr};${normalizedHeureDebut}`);
-    const heureDebut = timeStr
-        ? DateTime.fromFormat(timeStr, "H'h'mm").toISOTime()
-        : null;
+}
+function getStartAndEndTimes(timeStr) {
+    const normalizedTime = normalizeTime(timeStr);
+    const timeMatches = normalizedTime
+        ?.match(/^de (?<startTime>\d{2}h\d{2}) à (?<endTime>\d{2}h\d{2})$/i);
+    if (timeMatches?.groups) {
+        const { startTime, endTime } = timeMatches.groups;
+        return {
+            startTime: startTime ? DateTime.fromFormat(startTime, "H'h'mm").toISOTime() : null,
+            endTime: endTime ? DateTime.fromFormat(endTime, "H'h'mm").toISOTime() : null,
+        };
+    }
     return {
-        heureDebut,
-        heureFin: null
+        startTime: normalizedTime ? DateTime.fromFormat(normalizedTime, "H'h'mm").toISOTime() : null,
+        endTime: null,
     };
 }
-function transformAgenda(document) {
+function transformAgenda(document, fileName) {
     const agendaEvents = [];
     const eventElements = document.querySelectorAll(".evt");
     for (const eventElement of eventElements) {
+        const id = eventElement.previousElementSibling?.getAttribute("name") || null;
+        if (!id) {
+            continue;
+        }
         const type = getEventType(eventElement.classList);
+        const date = DateTime.fromFormat(fileName, "yyyyMMdd").toFormat("yyyy-MM-dd");
         const timeOriginal = eventElement.querySelector(".time")?.textContent || null;
-        const { heureDebut, heureFin } = getHeuresDebutFin(timeOriginal);
-        const titre = eventElement.querySelector(".titre")?.textContent || null;
-        const organe = eventElement.querySelector(".organe")?.textContent || null;
-        const objet = eventElement.querySelector(".objet")?.textContent || null;
+        const { startTime, endTime } = getStartAndEndTimes(timeOriginal);
+        const titre = eventElement.querySelector(".titre")?.textContent?.trim() || null;
+        const organe = eventElement.querySelector(".organe")?.textContent?.trim() || null;
+        const objet = eventElement.querySelector(".objet")?.textContent
+            ?.trim()
+            ?.replace(/^- /, "")
+            || null;
         const lieu = eventElement.querySelector(".lieu")?.textContent || null;
         const url_dossier_senat = getUrlDossierSenat(eventElement.querySelectorAll(".lien a"));
-        const url_video = eventElement.querySelector(".video a")?.getAttribute("href") || null;
         agendaEvents.push({
+            id,
             type,
-            heureDebut,
-            heureFin,
+            date,
+            startTime,
+            endTime,
             timeOriginal,
             titre,
             organe,
             objet,
             lieu,
             url_dossier_senat,
-            url_video,
         });
     }
     return agendaEvents;
@@ -76,7 +96,8 @@ function transformAgenda(document) {
 export async function parseAgendaFromFile(htmlFilePath) {
     try {
         const { document } = (await JSDOM.fromFile(htmlFilePath, { contentType: "text/html" })).window;
-        return transformAgenda(document);
+        const fileName = path.parse(htmlFilePath).name;
+        return transformAgenda(document, fileName);
     }
     catch (error) {
         console.error(`Could not parse texte with error ${error}`);

package/lib/scripts/retrieve_agenda.js

@@ -3,35 +3,46 @@ import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 import { DateTime } from "luxon";
 import path from "path";
-import { AGENDA_FOLDER, DATA_ORIGINAL_FOLDER } from "../loaders";
+import { AGENDA_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
+import { parseAgendaFromFile } from "../model/agenda";
 import { getSessionsFromStart } from "../types/sessions";
 import { commonOptions } from "./shared/cli_helpers";
 import { ensureAndClearDir } from "./shared/util";
 const optionsDefinitions = [
     ...commonOptions,
     {
-        help: "parse and convert documents into JSON (textes only for now, requires format xml)",
-        name: "parseDocuments",
+        help: "parse and convert agenda events into JSON",
+        name: "parseAgenda",
         type: Boolean,
     },
 ];
 const options = commandLineArgs(optionsDefinitions);
 const SENAT_GLOBAL_AGENDA_URL_ROOT = "https://www.senat.fr/aglae/Global";
-async function retrieveAgenda(dataDir, sessions) {
+async function retrieveAgendas(dataDir, sessions) {
     const agendaRootDir = path.join(dataDir, AGENDA_FOLDER);
     ensureAndClearDir(agendaRootDir);
     const originalAgendaDir = path.join(agendaRootDir, DATA_ORIGINAL_FOLDER);
     fs.ensureDirSync(originalAgendaDir);
+    const transformedAgendaDir = path.join(agendaRootDir, DATA_TRANSFORMED_FOLDER);
+    if (options["parseAgenda"]) {
+        fs.ensureDirSync(transformedAgendaDir);
+    }
     for (const session of sessions) {
         if (!options["silent"]) {
             console.log(`Retrieving Agenda for session ${session}…`);
         }
-        const agendaSessionDir = path.join(originalAgendaDir, `${session}`);
-        fs.ensureDirSync(agendaSessionDir);
+        const originalAgendaSessionDir = path.join(originalAgendaDir, `${session}`);
+        fs.ensureDirSync(originalAgendaSessionDir);
+        const transformedAgendaSessionDir = path.join(transformedAgendaDir, `${session}`);
+        if (options["parseAgenda"]) {
+            fs.ensureDirSync(transformedAgendaSessionDir);
+        }
         const fifteenDaysFromNow = new Date();
         fifteenDaysFromNow.setDate(fifteenDaysFromNow.getDate() + 15);
         for (const date = new Date(session, 0, 1); date <= new Date(session, 11, 31) && date <= fifteenDaysFromNow; date.setDate(date.getDate() + 1)) {
             const agendaName = DateTime.fromJSDate(date).toFormat("ddMMyyyy");
+            const agendaFileName = DateTime.fromJSDate(date).toFormat("yyyyMMdd");
+            const agendaPath = path.join(originalAgendaSessionDir, agendaFileName);
             try {
                 const response = await fetch(`${SENAT_GLOBAL_AGENDA_URL_ROOT}/agl${agendaName}.html`);
                 if (!response.ok) {
@@ -47,11 +58,18 @@ async function retrieveAgenda(dataDir, sessions) {
                 if (!agendaContent) {
                     return;
                 }
-                fs.writeFileSync(path.join(agendaSessionDir, agendaName), Buffer.from(agendaContent));
+                fs.writeFileSync(agendaPath, Buffer.from(agendaContent));
             }
             catch (error) {
                 console.error(error.message);
             }
+            if (options["parseAgenda"]) {
+                const parsedAgendaEvents = await parseAgendaFromFile(agendaPath);
+                if (!parsedAgendaEvents || parsedAgendaEvents.length === 0) {
+                    continue;
+                }
+                fs.writeJSONSync(path.join(transformedAgendaSessionDir, `${agendaFileName}.json`), parsedAgendaEvents, { spaces: 2 });
+            }
         }
     }
 }
@@ -60,7 +78,7 @@ async function main() {
     assert(dataDir, "Missing argument: data directory");
     const sessions = getSessionsFromStart(options["fromSession"]);
     console.time("agenda processing time");
-    await retrieveAgenda(dataDir, sessions);
+    await retrieveAgendas(dataDir, sessions);
     if (!options["silent"]) {
         console.timeEnd("agenda processing time");
     }

package/lib/types/agenda.d.ts

@@ -1,12 +1,13 @@
 export interface AgendaEvent {
+    id: string;
     type: string | null;
-    heureDebut: string | null;
-    heureFin: string | null;
+    date: string | null;
+    startTime: string | null;
+    endTime: string | null;
     timeOriginal: string | null;
     titre: string | null;
     organe: string | null;
     objet: string | null;
     lieu: string | null;
     url_dossier_senat: string | null;
-    url_video: string | null;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.1.2",
+  "version": "2.2.2",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",