@tricoteuses/senat 3.2.2 → 3.3.0

package/lib/src/other_types/agenda.d.ts

@@ -1,3 +1,17 @@
+export interface AgendaDeadline {
+    title: string | null;
+    detail: string | null;
+    dateOriginal: string | null;
+    date: string | null;
+}
+export interface AgendaPresidency {
+    title: string | null;
+    content: string | null;
+}
+export interface AgendaLink {
+    label: string;
+    url: string;
+}
 export interface AgendaEvent {
     id: string;
     type: string | null;
@@ -12,6 +26,12 @@ export interface AgendaEvent {
     captationVideo: boolean;
     urlDossierSenat: string | null;
     quantieme: string | null;
+    public?: boolean;
+    forecast?: boolean;
+    contentHtml?: string | null;
+    presidency?: AgendaPresidency | null;
+    links?: AgendaLink[] | null;
+    deadlines?: AgendaDeadline[] | null;
 }
 export type TimeSlot = "MATIN" | "APRES-MIDI" | "SOIR" | "UNKNOWN";
 export interface Reunion {
@@ -34,6 +54,11 @@ export interface Reunion {
     timecodeDebutVideo?: number;
     timecodeFinVideo?: number;
     odj?: ReunionOdj;
+    deadlines?: AgendaDeadline[] | null;
+    presidency?: AgendaPresidency | null;
+    links?: AgendaLink[] | null;
+    forecast?: boolean;
+    public?: boolean;
 }
 export interface ReunionOdjPoint {
     objet: string | null;

package/lib/src/scripts/data-download.js

@@ -1,21 +1,21 @@
 import { execSync } from "child_process";
-const args = process.argv.slice(2).join(" ");
-function runScript(command) {
+import { runDataDownload } from "./shared/data_download.js";
+function exec(command) {
     try {
         execSync(command, { stdio: "inherit" });
+        return { status: 0 };
     }
     catch (error) {
         const execError = error;
-        if (execError.status !== 10) {
-            console.error(`Error during: ${command}`, error);
-            process.exit(execError.status || 1);
-        }
+        return { status: execError.status ?? null, error };
     }
 }
-runScript(`tsx src/scripts/retrieve_open_data.ts --all ${args}`);
-runScript(`tsx src/scripts/convert_data.ts ${args}`);
-runScript(`cross-env TZ='Etc/UTC' tsx src/scripts/retrieve_agenda.ts ${args} --parseAgenda --silent`);
-runScript(`tsx src/scripts/retrieve_cr_seance.ts ${args} --parseDebats --silent`);
-runScript(`tsx src/scripts/retrieve_cr_commission.ts ${args} --parseDebats --silent`);
-runScript(`tsx src/scripts/retrieve_videos.ts ${args} --silent`);
-runScript(`tsx src/scripts/retrieve_collaborateurs.ts ${args} --silent`);
+const exitCode = runDataDownload({
+    exec,
+    now: () => new Date(),
+    rawArgs: process.argv.slice(2),
+    log: console.log,
+    warn: console.warn,
+    error: (message, error) => error === undefined ? console.error(message) : console.error(message, error),
+});
+process.exit(exitCode);

package/lib/src/scripts/retrieve_agenda.js

@@ -3,12 +3,11 @@ import fs from "fs-extra";
 import { DateTime } from "luxon";
 import path from "path";
 import * as git from "../server/git.js";
-import { AGENDA_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../server/loaders.js";
-import { parseAgendaFromFile } from "../server/agenda.js";
+import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER } from "../server/loaders.js";
+import { fetchWeekAgendaEvents } from "../server/agenda_api.js";
 import { getSessionsFromStart, UNDEFINED_SESSION } from "../other_types/sessions.js";
 import { ID_DATE_FORMAT } from "./datautil.js";
 import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers.js";
-import { fetchWithRetry } from "./shared/util.js";
 import { buildReunionsByBucket } from "../utils/reunion_parsing.js";
 import { buildSenatDossierIndex, buildDoslegReunionIndex } from "../utils/reunion_odj_building.js";
 import { loadPreviousManifest, saveManifest, detectChanges } from "../utils/manifest.js";
@@ -23,13 +22,11 @@ const optionsDefinitions = [
 ];
 let exitCode = 10; // 0: some data changed, 10: no modification
 const options = commandLineArgs(optionsDefinitions);
-const SENAT_GLOBAL_AGENDA_URL_ROOT = "https://www.senat.fr/aglae/Global";
-const EVENT_DATE_FORMAT = "ddMMyyyy";
-class AgendaError extends Error {
-    constructor(message, agendaName) {
-        super(`An error occurred while retrieving Agenda ${agendaName}: ${message}`);
-    }
-}
+// The agenda.senat.fr API only serves dates after the launch of the new
+// website (~mid-December 2025). For earlier dates, files already downloaded
+// from the old agenda are kept as-is.
+const AGENDA_API_COVERAGE_START = "2025-12-13";
+const FR_TZ = "Europe/Paris";
 function commitAndPushGit(datasetDir, options) {
     if (options.commit) {
         const errorCode = git.commitAndPush(datasetDir, "Nouvelle moisson", options.remote);
@@ -43,78 +40,71 @@ async function retrieveAgendas(options, sessions) {
     const dataDir = assertExistingDirectory(options["dataDir"], "data directory");
     const agendaRootDir = path.join(dataDir, AGENDA_FOLDER);
     fs.ensureDirSync(agendaRootDir);
-    const originalAgendaDir = path.join(agendaRootDir, DATA_ORIGINAL_FOLDER);
-    fs.ensureDirSync(originalAgendaDir);
     const transformedAgendaDir = path.join(agendaRootDir, DATA_TRANSFORMED_FOLDER);
     if (options["parseAgenda"]) {
         fs.ensureDirSync(transformedAgendaDir);
     }
     const dossierIndex = buildSenatDossierIndex(options);
     const doslegReunionIndex = buildDoslegReunionIndex(options);
+    const coverageStart = DateTime.fromISO(AGENDA_API_COVERAGE_START, { zone: FR_TZ });
+    const now = DateTime.now().setZone(FR_TZ).startOf("day");
+    // Don't download agendas more than 15 days in the future.
+    const fifteenDaysFromNow = now.plus({ days: 15 });
     for (const session of sessions) {
-        const originalAgendaSessionDir = path.join(originalAgendaDir, `${session}`);
-        fs.ensureDirSync(originalAgendaSessionDir);
+        const sessionStart = DateTime.fromISO(`${session}-10-01`, { zone: FR_TZ });
+        const sessionEnd = DateTime.fromISO(`${session + 1}-09-30`, { zone: FR_TZ }).endOf("day");
+        if (sessionEnd < coverageStart || sessionStart > fifteenDaysFromNow)
+            continue;
         const transformedAgendaSessionDir = path.join(transformedAgendaDir, `${session}`);
         if (options["parseAgenda"]) {
             fs.ensureDirSync(transformedAgendaSessionDir);
         }
-        const fifteenDaysFromNow = new Date();
-        // Don't download agendas more than 15 days in the future.
-        fifteenDaysFromNow.setDate(fifteenDaysFromNow.getDate() + 15);
-        for (const date = new Date(session, 9, 1); date <= new Date(session + 1, 8, 30) && date <= fifteenDaysFromNow; date.setDate(date.getDate() + 1)) {
-            const agendaName = DateTime.fromJSDate(date).toFormat(EVENT_DATE_FORMAT);
-            const agendaFileName = DateTime.fromJSDate(date).toFormat(ID_DATE_FORMAT);
-            const agendaPath = path.join(originalAgendaSessionDir, `${agendaFileName}.html`);
+        const start = DateTime.max(sessionStart, coverageStart).startOf("week");
+        for (let date = start; date <= sessionEnd && date <= fifteenDaysFromNow; date = date.plus({ weeks: 1 })) {
+            let byDay;
             try {
-                await downloadAgenda(agendaName, agendaPath);
-                if (options["parseAgenda"]) {
-                    const createdUids = await parseAgenda(transformedAgendaSessionDir, agendaFileName, agendaPath, dossierIndex, doslegReunionIndex);
-                    await processManifestAndDeletions(transformedAgendaSessionDir, agendaFileName, session, createdUids);
-                }
+                byDay = await fetchWeekAgendaEvents(date.toISODate());
             }
             catch (error) {
-                console.error(error);
+                console.error(`Could not fetch agenda week ${date.toISODate()}:`, error);
+                continue;
+            }
+            for (const [dayISO, events] of byDay) {
+                const day = DateTime.fromISO(dayISO, { zone: FR_TZ });
+                if (day < sessionStart || day > sessionEnd || day > fifteenDaysFromNow)
+                    continue;
+                const agendaFileName = day.toFormat(ID_DATE_FORMAT);
+                try {
+                    if (options["parseAgenda"]) {
+                        // Day without any event on the API side: write and delete
+                        // nothing, to avoid corrupting existing data (partial API
+                        // coverage or events wrongly removed from the agenda).
+                        if (events.length === 0)
+                            continue;
+                        const createdUids = await processDayEvents(transformedAgendaSessionDir, agendaFileName, events, dossierIndex, doslegReunionIndex);
+                        await processManifestAndDeletions(transformedAgendaSessionDir, agendaFileName, session, createdUids);
+                    }
+                }
+                catch (error) {
+                    console.error(error);
+                }
             }
         }
     }
 }
-async function downloadAgenda(agendaName, agendaPath) {
-    const agendaUrl = `${SENAT_GLOBAL_AGENDA_URL_ROOT}/agl${agendaName}.html`;
-    if (!options["silent"]) {
-        console.log(`Downloading Agenda ${agendaUrl}…`);
-    }
-    const response = await fetchWithRetry(agendaUrl);
-    if (!response.ok) {
-        if (response.status === 404) {
-            console.warn(`Agenda ${agendaUrl} not found`);
-        }
-        else {
-            throw new AgendaError(String(response.status), agendaName);
-        }
-        return;
-    }
-    const agendaContent = await response.arrayBuffer();
-    if (!agendaContent) {
-        return;
-    }
-    fs.writeFileSync(agendaPath, Buffer.from(agendaContent));
-}
 function writeGroupsAsFiles(dir, groups) {
     for (const g of groups) {
         const outPath = path.join(dir, `${g.uid}.json`);
         fs.writeJSONSync(outPath, g, { spaces: 2 });
     }
 }
-async function parseAgenda(transformedAgendaSessionDir, agendaFileName, agendaPath, dossierBySenatUrl, doslegReunionIndex) {
+async function processDayEvents(transformedAgendaSessionDir, agendaFileName, events, dossierBySenatUrl, doslegReunionIndex) {
     if (!options["silent"])
-        console.log(`Parsing Agenda ${agendaPath}…`);
-    const parsedAgendaEvents = await parseAgendaFromFile(agendaPath);
-    if (!parsedAgendaEvents?.length)
-        return [];
+        console.log(`Processing agenda ${agendaFileName} (${events.length} event(s))…`);
     const flatPath = path.join(transformedAgendaSessionDir, `${agendaFileName}.json`);
-    fs.writeJSONSync(flatPath, parsedAgendaEvents, { spaces: 2 });
-    const byBucket = buildReunionsByBucket(parsedAgendaEvents, dossierBySenatUrl, doslegReunionIndex);
-    // Collecter tous les UIDs créés
+    fs.writeJSONSync(flatPath, events, { spaces: 2 });
+    const byBucket = buildReunionsByBucket(events, dossierBySenatUrl, doslegReunionIndex);
+    // Collect all created UIDs
     const createdUids = [];
     // SP
     if (byBucket.IDS.length > 0) {

package/lib/src/scripts/shared/data_download.d.ts

@@ -0,0 +1,43 @@
+import { type Dataset } from "../../server/datasets.js";
+export declare const STATUS_FILENAME = "dataset_status.json";
+export type DatasetStatusEntry = {
+    status: "ok" | "failed";
+    lastRunAt: string;
+    lastSuccessAt?: string;
+    lastFailureAt?: string;
+    lastError?: string;
+    consecutiveFailures: number;
+    firstFailureAt?: string;
+};
+export type DatasetStatusFile = Record<string, DatasetStatusEntry>;
+export type ExecResult = {
+    status: number | null;
+    error?: unknown;
+};
+export type ExecFn = (command: string) => ExecResult;
+export type DataDownloadLogger = {
+    log?: (message: string) => void;
+    warn?: (message: string) => void;
+    error?: (message: string, error?: unknown) => void;
+};
+export type DataDownloadDeps = DataDownloadLogger & {
+    rawArgs: string[];
+    exec: ExecFn;
+    now: () => Date;
+};
+export declare const CATEGORY_DATASETS: Array<{
+    category: string;
+    dataset: Dataset;
+}>;
+export type ParsedArgs = {
+    categories: string[];
+    dataDir?: string;
+    rest: string[];
+};
+export declare function parseArgs(rawArgs: string[]): ParsedArgs;
+export declare function forwardArgs(parsedArgs: ParsedArgs, categories: string[]): string;
+export declare function readStatusFile(statusFilePath: string): DatasetStatusFile;
+export declare function writeStatusFile(statusFilePath: string, statuses: DatasetStatusFile): void;
+export declare function recordSuccess(statuses: DatasetStatusFile, datasetName: string, now: Date): void;
+export declare function recordFailure(statuses: DatasetStatusFile, datasetName: string, now: Date, errorMessage: string): void;
+export declare function runDataDownload(deps: DataDownloadDeps): number;

package/lib/src/scripts/shared/data_download.js

@@ -0,0 +1,175 @@
+import fs from "fs-extra";
+import path from "path";
+import { datasets } from "../../server/datasets.js";
+export const STATUS_FILENAME = "dataset_status.json";
+const MAX_ERROR_LENGTH = 2000;
+export const CATEGORY_DATASETS = [
+    { category: "Ameli", dataset: datasets.ameli },
+    { category: "Debats", dataset: datasets.debats },
+    { category: "DosLeg", dataset: datasets.dosleg },
+    { category: "Questions", dataset: datasets.questions },
+    { category: "Sens", dataset: datasets.sens },
+];
+// Options of retrieve_open_data whose next token is a value. They must not be
+// mistaken for the dataDir positional argument.
+const valueOptions = new Set([
+    "categories",
+    "k",
+    "sudo",
+    "clone",
+    "C",
+    "remote",
+    "r",
+    "fromSession",
+    "only-recent",
+]);
+export function parseArgs(rawArgs) {
+    const categories = [];
+    let dataDir;
+    const rest = [];
+    for (let index = 0; index < rawArgs.length; index++) {
+        const token = rawArgs[index];
+        const name = token.replace(/^-+/, "").split("=")[0];
+        if (name === "categories" || name === "k") {
+            const rawValues = token.includes("=") ? [token.slice(token.indexOf("=") + 1)] : [rawArgs[++index]];
+            for (const rawValue of rawValues) {
+                if (rawValue === undefined) {
+                    continue;
+                }
+                for (const value of rawValue.split(",")) {
+                    if (value.length > 0) {
+                        categories.push(value);
+                    }
+                }
+            }
+            continue;
+        }
+        if (token.startsWith("-") && valueOptions.has(name) && !token.includes("=")) {
+            rest.push(token);
+            const value = rawArgs[++index];
+            if (value !== undefined) {
+                rest.push(value);
+            }
+            continue;
+        }
+        if (!token.startsWith("-") && dataDir === undefined) {
+            dataDir = token;
+            continue;
+        }
+        rest.push(token);
+    }
+    return { categories: categories.length > 0 ? categories : ["All"], dataDir, rest };
+}
+export function forwardArgs(parsedArgs, categories) {
+    const parts = [...parsedArgs.rest];
+    if (parsedArgs.dataDir !== undefined) {
+        parts.push(parsedArgs.dataDir);
+    }
+    for (const category of categories) {
+        parts.push("--categories", category);
+    }
+    return parts.join(" ");
+}
+export function readStatusFile(statusFilePath) {
+    try {
+        const content = fs.readJsonSync(statusFilePath);
+        return content && typeof content === "object" ? content : {};
+    }
+    catch {
+        return {};
+    }
+}
+export function writeStatusFile(statusFilePath, statuses) {
+    fs.writeJsonSync(statusFilePath, statuses, { spaces: 2 });
+}
+export function recordSuccess(statuses, datasetName, now) {
+    statuses[datasetName] = {
+        consecutiveFailures: 0,
+        lastRunAt: now.toISOString(),
+        lastSuccessAt: now.toISOString(),
+        status: "ok",
+    };
+}
+export function recordFailure(statuses, datasetName, now, errorMessage) {
+    const previous = statuses[datasetName];
+    statuses[datasetName] = {
+        consecutiveFailures: (previous?.consecutiveFailures ?? 0) + 1,
+        firstFailureAt: previous?.firstFailureAt ?? now.toISOString(),
+        lastError: errorMessage.slice(0, MAX_ERROR_LENGTH),
+        lastFailureAt: now.toISOString(),
+        lastRunAt: now.toISOString(),
+        lastSuccessAt: previous?.lastSuccessAt,
+        status: "failed",
+    };
+}
+function execNormalized(deps, command) {
+    try {
+        return deps.exec(command);
+    }
+    catch (error) {
+        const execError = error;
+        return { status: execError.status ?? null, error };
+    }
+}
+function isSuccessfulStatus(result) {
+    return result.status === 0 || result.status === 10;
+}
+function runDownstreamScript(deps, command) {
+    const result = execNormalized(deps, command);
+    if (isSuccessfulStatus(result)) {
+        return null;
+    }
+    deps.error?.(`Error during: ${command}`, result.error);
+    return result.status || 1;
+}
+export function runDataDownload(deps) {
+    const parsedArgs = parseArgs(deps.rawArgs);
+    const chosen = parsedArgs.categories.includes("All")
+        ? CATEGORY_DATASETS
+        : CATEGORY_DATASETS.filter(({ category }) => parsedArgs.categories.includes(category));
+    if (chosen.length === 0) {
+        deps.error?.(`No known dataset among categories: ${parsedArgs.categories.join(", ")}`);
+        return 1;
+    }
+    const statusFilePath = parsedArgs.dataDir !== undefined ? path.join(parsedArgs.dataDir, STATUS_FILENAME) : null;
+    const statuses = statusFilePath !== null ? readStatusFile(statusFilePath) : {};
+    const okCategories = [];
+    const failedCategories = [];
+    for (const { category, dataset } of chosen) {
+        // --all: fetch, unzip, repair encoding and import (same as the original script).
+        const command = `tsx src/scripts/retrieve_open_data.ts --all ${forwardArgs(parsedArgs, [category])}`.trim();
+        deps.log?.(`\n=== Retrieving dataset ${dataset.title} (${category}) ===`);
+        const result = execNormalized(deps, command);
+        if (!isSuccessfulStatus(result)) {
+            const errorMessage = result.error instanceof Error ? result.error.message : String(result.error);
+            recordFailure(statuses, dataset.database, deps.now(), errorMessage);
+            failedCategories.push(category);
+            deps.error?.(`Error during: ${command}`, result.error);
+        }
+        else {
+            recordSuccess(statuses, dataset.database, deps.now());
+            okCategories.push(category);
+        }
+        if (statusFilePath !== null) {
+            writeStatusFile(statusFilePath, statuses);
+        }
+    }
+    if (failedCategories.length > 0) {
+        deps.warn?.(`\n⚠️  Warning: Some datasets failed to retrieve: ${failedCategories.join(", ")}. ` +
+            "Continuing with available datasets...\n");
+    }
+    if (okCategories.length === 0) {
+        deps.error?.("All datasets failed to retrieve; skipping data conversion.");
+        return 1;
+    }
+    const exitCode = runDownstreamScript(deps, `tsx src/scripts/convert_data.ts ${forwardArgs(parsedArgs, okCategories)}`) ??
+        runDownstreamScript(deps, `cross-env TZ='Etc/UTC' tsx src/scripts/retrieve_agenda.ts ${deps.rawArgs.join(" ")} --parseAgenda --silent`) ??
+        runDownstreamScript(deps, "cross-env TZ='Etc/UTC' tsx src/scripts/retrieve_cr_seance.ts " +
+            `${deps.rawArgs.join(" ")} --parseDebats --silent`) ??
+        runDownstreamScript(deps, "cross-env TZ='Etc/UTC' tsx src/scripts/retrieve_cr_commission.ts " +
+            `${deps.rawArgs.join(" ")} --parseDebats --silent`) ??
+        runDownstreamScript(deps, `tsx src/scripts/retrieve_videos.ts ${deps.rawArgs.join(" ")} --silent`) ??
+        runDownstreamScript(deps, `tsx src/scripts/retrieve_collaborateurs.ts ${deps.rawArgs.join(" ")} --silent`) ??
+        0;
+    return exitCode;
+}

package/lib/src/server/agenda_api.d.ts

@@ -0,0 +1,9 @@
+import { AgendaEvent } from "../other_types/agenda.js";
+export declare const AGENDA_API_URL_ROOT = "https://www.senat.fr/api/v1/agenda";
+/**
+ * Récupère une semaine d'agenda (7 jours à compter de la semaine de dateISO,
+ * même comportement que le site agenda.senat.fr) puis le détail de chaque
+ * événement. Retourne un Map jour ISO → AgendaEvent[] (vide pour les jours
+ * sans événement).
+ */
+export declare function fetchWeekAgendaEvents(dateISO: string): Promise<Map<string, AgendaEvent[]>>;

package/lib/src/server/agenda_api.js

@@ -0,0 +1,32 @@
+import { fetchWithRetry } from "../scripts/shared/util.js";
+import { apiEventToAgendaEvent, computeQuantiemes, } from "./agenda_api_mapping.js";
+export const AGENDA_API_URL_ROOT = "https://www.senat.fr/api/v1/agenda";
+async function fetchJson(url) {
+    const response = await fetchWithRetry(url);
+    if (!response.ok) {
+        throw new Error(`Agenda API request failed (${response.status}): ${url}`);
+    }
+    return (await response.json());
+}
+async function fetchEventDetail(id) {
+    return fetchJson(`${AGENDA_API_URL_ROOT}/events/${id}`);
+}
+/**
+ * Récupère une semaine d'agenda (7 jours à compter de la semaine de dateISO,
+ * même comportement que le site agenda.senat.fr) puis le détail de chaque
+ * événement. Retourne un Map jour ISO → AgendaEvent[] (vide pour les jours
+ * sans événement).
+ */
+export async function fetchWeekAgendaEvents(dateISO) {
+    const url = `${AGENDA_API_URL_ROOT}/events?date=${dateISO}&week=1`;
+    const week = await fetchJson(url);
+    const byDay = new Map();
+    for (const [dayKey, dayValue] of Object.entries(week)) {
+        const day = dayValue?.day ?? dayKey;
+        const events = dayValue?.events ?? [];
+        const quantiemes = computeQuantiemes(events);
+        const details = await Promise.all(events.map((e) => fetchEventDetail(e.id)));
+        byDay.set(day, details.map((detail) => apiEventToAgendaEvent(detail, quantiemes[String(detail.id)] ?? null)));
+    }
+    return byDay;
+}

package/lib/src/server/agenda_api_mapping.d.ts

@@ -0,0 +1,56 @@
+import { AgendaDeadline, AgendaEvent } from "../other_types/agenda.js";
+export type AgendaApiEvent = {
+    id: number;
+    date: string;
+    hour: string | null;
+    title: string;
+    place: string | null;
+    instances: string[] | null;
+    forecast: boolean;
+    public: boolean;
+};
+export type AgendaApiEventDetail = AgendaApiEvent & {
+    content: string | null;
+    links: {
+        label: string;
+        to: string;
+        target?: string;
+    }[] | null;
+    presidency: {
+        title: string;
+        content: string;
+    } | null;
+    deadlines: {
+        title: string;
+        date: string;
+    }[] | null;
+    addToCalendar: {
+        startDate: string;
+        startTime: string;
+        endDate: string;
+        endTime: string;
+        description: string;
+    } | null;
+};
+export type KnownType = "SP" | "COM" | "MC" | "OD" | "ID";
+/**
+ * Classifies an instance from the new API into the 5 generic types of the
+ * old agenda (evt-seance, evt-instanz, evt-cemi, evt-deleg, evt-bureau).
+ */
+export declare function classifyInstance(instances: string[] | null | undefined): {
+    kind: KnownType;
+    type: string;
+};
+export declare function extractDossierUrl(contentHtml: string | null | undefined): string | null;
+export declare function cleanHtmlToText(contentHtml: string | null | undefined): string | null;
+export declare function parseDeadline(raw: {
+    title: string;
+    date: string;
+}): AgendaDeadline;
+/**
+ * Computes the session number ("Unique", "Première", …) of the public
+ * sittings of a day, same rule as the old parser (order of appearance
+ * in the agenda).
+ */
+export declare function computeQuantiemes(dayEvents: AgendaApiEvent[]): Record<string, string | null>;
+export declare function apiEventToAgendaEvent(detail: AgendaApiEventDetail, quantieme: string | null): AgendaEvent;

package/lib/src/server/agenda_api_mapping.js

@@ -0,0 +1,128 @@
+import { DateTime } from "luxon";
+import * as cheerio from "cheerio";
+import { getStartAndEndTimes } from "./agenda.js";
+const FR_TZ = "Europe/Paris";
+const GENERIC_TYPE = {
+    SP: "Séance publique",
+    COM: "Commissions",
+    MC: "Mission de contrôle",
+    OD: "Offices et délégations",
+    ID: "Instances décisionnelles",
+};
+/**
+ * Classifies an instance from the new API into the 5 generic types of the
+ * old agenda (evt-seance, evt-instanz, evt-cemi, evt-deleg, evt-bureau).
+ */
+export function classifyInstance(instances) {
+    const label = (instances?.[0] ?? "").trim();
+    const s = label
+        .normalize("NFKD")
+        .replace(/[\u0300-\u036f]/g, "")
+        .toLowerCase();
+    if (/\bseance\b.*\bpublique\b/.test(s))
+        return { kind: "SP", type: GENERIC_TYPE.SP };
+    if (/^commission/.test(s))
+        return { kind: "COM", type: GENERIC_TYPE.COM };
+    if (/^mi\b/.test(s) || /^mission\b/.test(s) || /^ce\b/.test(s))
+        return { kind: "MC", type: GENERIC_TYPE.MC };
+    if (/^(delegation|office)/.test(s))
+        return { kind: "OD", type: GENERIC_TYPE.OD };
+    return { kind: "ID", type: GENERIC_TYPE.ID };
+}
+export function extractDossierUrl(contentHtml) {
+    if (!contentHtml)
+        return null;
+    const $ = cheerio.load(contentHtml);
+    const href = $("a[href*='senat.fr/dossier-legislatif']").first().attr("href");
+    return href ?? null;
+}
+export function cleanHtmlToText(contentHtml) {
+    if (!contentHtml)
+        return null;
+    const $ = cheerio.load(contentHtml);
+    $("br").replaceWith("\n");
+    const text = $("body").text() ?? $.root().text();
+    const cleaned = text
+        .replace(/\u00a0/g, " ")
+        .replace(/[ \t]+/g, " ")
+        .replace(/\n\s*\n/g, "\n")
+        .trim();
+    return cleaned ? cleaned.replace(/^- /, "").trim() : null;
+}
+export function parseDeadline(raw) {
+    const [titlePart, ...restParts] = (raw.title ?? "").split(/<br\s*\/?>/i);
+    const title = cleanHtmlToText(titlePart ?? null) ?? null;
+    const detail = restParts.length > 0 ? (cleanHtmlToText(restParts.join("\n")) ?? null) : null;
+    const dateOriginal = (raw.date ?? "").replace(/\u00a0/g, " ").trim() || null;
+    let date = null;
+    if (dateOriginal) {
+        const parsed = DateTime.fromFormat(dateOriginal, "EEEE d MMMM yyyy HH'h'mm", {
+            locale: "fr",
+            zone: FR_TZ,
+        });
+        if (parsed.isValid)
+            date = parsed.toISO();
+    }
+    return { title, detail, dateOriginal, date };
+}
+function toAgendaLinks(links) {
+    if (!links?.length)
+        return null;
+    return links.map((l) => ({ label: l.label, url: l.to }));
+}
+function toAgendaPresidency(presidency) {
+    if (!presidency)
+        return null;
+    return { title: presidency.title ?? null, content: presidency.content ?? null };
+}
+/**
+ * Computes the session number ("Unique", "Première", …) of the public
+ * sittings of a day, same rule as the old parser (order of appearance
+ * in the agenda).
+ */
+export function computeQuantiemes(dayEvents) {
+    const result = {};
+    const seances = dayEvents.filter((e) => classifyInstance(e.instances).kind === "SP");
+    const labels = ["Unique", "Première", "Deuxième", "Troisième", "Quatrième", "Cinquième"];
+    for (const event of dayEvents) {
+        if (classifyInstance(event.instances).kind !== "SP") {
+            result[String(event.id)] = null;
+            continue;
+        }
+        const index = seances.indexOf(event);
+        result[String(event.id)] =
+            seances.length === 1 && index === 0 ? labels[0] : (labels[index + 1] ?? "Non défini");
+    }
+    return result;
+}
+export function apiEventToAgendaEvent(detail, quantieme) {
+    const { kind, type } = classifyInstance(detail.instances);
+    const hour = (detail.hour ?? "").trim();
+    const { startTime } = getStartAndEndTimes(hour, detail.date);
+    // The API provides no video capture flag; public sittings are always
+    // filmed, unlike other instances.
+    const captationVideo = kind === "SP";
+    return {
+        id: String(detail.id),
+        type,
+        date: detail.date,
+        startTime,
+        // The end time provided by the API (addToCalendar) is a heuristic
+        // (defaults to +2h): null is preferred to guarantee data reliability.
+        endTime: null,
+        timeOriginal: hour || null,
+        titre: (detail.title ?? "").trim(),
+        organe: kind === "SP" ? "Séance publique" : (detail.instances?.[0]?.trim() ?? null),
+        objet: cleanHtmlToText(detail.content),
+        lieu: (detail.place ?? "").trim() || null,
+        captationVideo,
+        urlDossierSenat: extractDossierUrl(detail.content),
+        quantieme,
+        public: detail.public,
+        forecast: detail.forecast,
+        contentHtml: detail.content ?? null,
+        presidency: toAgendaPresidency(detail.presidency),
+        links: toAgendaLinks(detail.links),
+        deadlines: detail.deadlines?.length ? detail.deadlines.map(parseDeadline) : null,
+    };
+}

package/lib/src/utils/reunion_parsing.js

@@ -41,6 +41,11 @@ function toReunion(e, dossierBySenatUrl, uid, doslegReunionIndex) {
         events: [e], // TODO remove
         odj: buildOdj([e], dossierBySenatUrl, doslegReunionIndex),
         lieu: e.lieu || undefined,
+        deadlines: e.deadlines ?? null,
+        presidency: e.presidency ?? null,
+        links: e.links ?? null,
+        forecast: e.forecast,
+        public: e.public,
     };
 }
 export function buildReunionsByBucket(events, dossierBySenatUrl, doslegReunionIndex) {
@@ -57,7 +62,7 @@ export function buildReunionsByBucket(events, dossierBySenatUrl, doslegReunionIn
         const uid = makeReunionUid(e.date, kind, e.id, e.organe ?? null);
         out[bucket].push(toReunion(e, dossierBySenatUrl, uid, doslegReunionIndex));
     }
-    // Tri stable par bucket (date + heure, inconnus à la fin)
+    // Stable sort by bucket (date + time, unknowns at the end)
     for (const k of Object.keys(out)) {
         out[k].sort((a, b) => {
             const da = DateTime.fromISO(`${a.date}T${a.startTime || "23:59:59.999+02:00"}`, { zone: PARIS }).toMillis();

package/lib/tests/agenda/agendaApi.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/agenda/agendaApi.test.js

@@ -0,0 +1,83 @@
+// Tests for the Sénat agenda API client (mocked fetch)
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { AGENDA_API_URL_ROOT, fetchWeekAgendaEvents } from "../../src/server/agenda_api.js";
+afterEach(() => {
+    vi.unstubAllGlobals();
+});
+function jsonResponse(body, status = 200) {
+    return new Response(JSON.stringify(body), { status, headers: { "Content-Type": "application/json" } });
+}
+describe("fetchWeekAgendaEvents", () => {
+    it("fetches the week then each event detail, and returns per-day AgendaEvents", async () => {
+        const liste = {
+            "2026-06-16": {
+                day: "2026-06-16",
+                events: [
+                    {
+                        id: 557355,
+                        date: "2026-06-16",
+                        hour: "9h30",
+                        title: "44 questions orales",
+                        place: "Hémicycle",
+                        instances: ["Séance publique"],
+                        forecast: false,
+                        public: true,
+                    },
+                    {
+                        id: 557356,
+                        date: "2026-06-16",
+                        hour: "14h30",
+                        title: "Séance 2",
+                        place: "Hémicycle",
+                        instances: ["Séance publique"],
+                        forecast: false,
+                        public: true,
+                    },
+                ],
+            },
+            "2026-06-17": { day: "2026-06-17", events: [] },
+        };
+        const detail = (id, title) => ({
+            id,
+            date: "2026-06-16",
+            hour: "14h30",
+            title,
+            place: "Hémicycle",
+            instances: ["Séance publique"],
+            forecast: false,
+            public: true,
+            content: `<p>- ${title}</p>`,
+            links: null,
+            presidency: null,
+            deadlines: null,
+            addToCalendar: null,
+        });
+        const fetchMock = vi.fn(async (url) => {
+            const u = String(url);
+            if (u === `${AGENDA_API_URL_ROOT}/events?date=2026-06-16&week=1`)
+                return jsonResponse(liste);
+            if (u === `${AGENDA_API_URL_ROOT}/events/557355`)
+                return jsonResponse(detail(557355, "44 questions orales"));
+            if (u === `${AGENDA_API_URL_ROOT}/events/557356`)
+                return jsonResponse(detail(557356, "Séance 2"));
+            throw new Error(`unexpected url ${u}`);
+        });
+        vi.stubGlobal("fetch", fetchMock);
+        const byDay = await fetchWeekAgendaEvents("2026-06-16");
+        expect([...byDay.keys()].sort()).toEqual(["2026-06-16", "2026-06-17"]);
+        expect(byDay.get("2026-06-17")).toEqual([]);
+        const day16 = byDay.get("2026-06-16");
+        expect(day16).toHaveLength(2);
+        expect(day16[0].id).toBe("557355");
+        expect(day16[0].objet).toBe("44 questions orales");
+        // 1 week request + 2 detail requests
+        expect(fetchMock).toHaveBeenCalledTimes(3);
+        // quantiemes computed from the order of sittings of the day
+        expect(day16[0].quantieme).toBe("Première");
+        expect(day16[1].quantieme).toBe("Deuxième");
+    });
+    it("propagates an error when the week request fails", async () => {
+        vi.stubGlobal("fetch", vi.fn(async () => jsonResponse({ error: "boom" }, 500)));
+        await expect(fetchWeekAgendaEvents("2026-06-16")).rejects.toThrow();
+    });
+});

package/lib/tests/agenda/agendaApiMapping.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/agenda/agendaApiMapping.test.js

@@ -0,0 +1,166 @@
+// Tests for the agenda.senat.fr API → AgendaEvent mapping (historical schema kept)
+import { describe, expect, it } from "vitest";
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import path from "node:path";
+import { apiEventToAgendaEvent, classifyInstance, computeQuantiemes, extractDossierUrl, parseDeadline, } from "../../src/server/agenda_api_mapping.js";
+import { makeReunionUid } from "../../src/utils/reunion_parsing.js";
+const fixturesDir = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "fixtures", "agenda_api");
+const loadFixture = (name) => JSON.parse(readFileSync(path.join(fixturesDir, name), "utf-8"));
+describe("classifyInstance", () => {
+    it("maps Séance publique to SP", () => {
+        expect(classifyInstance(["Séance publique"])).toEqual({ kind: "SP", type: "Séance publique" });
+    });
+    it("maps commissions (singular or plural) to COM", () => {
+        expect(classifyInstance(["Commission des lois"])).toEqual({ kind: "COM", type: "Commissions" });
+        expect(classifyInstance(["Commission aménagement du territoire / développement durable"])).toEqual({
+            kind: "COM",
+            type: "Commissions",
+        });
+    });
+    it("maps missions (MI) to MC", () => {
+        expect(classifyInstance(["MI Loi littoral Loi montagne"])).toEqual({ kind: "MC", type: "Mission de contrôle" });
+    });
+    it("maps commissions d'enquête (CE) to MC", () => {
+        expect(classifyInstance(["CE marges grande distribution"])).toEqual({ kind: "MC", type: "Mission de contrôle" });
+        // "CE Universités" = commission of inquiry on universities (like
+        // "CEMAGRDI", classified IDM by the old agenda)
+        expect(classifyInstance(["CE Universités"])).toEqual({ kind: "MC", type: "Mission de contrôle" });
+    });
+    it("maps délégations and offices to OD", () => {
+        expect(classifyInstance(["Délégation aux entreprises"])).toEqual({ kind: "OD", type: "Offices et délégations" });
+        expect(classifyInstance(["Office parlementaire d'évaluation des choix scient. tech."])).toEqual({
+            kind: "OD",
+            type: "Offices et délégations",
+        });
+    });
+    it("maps remaining instances (Présidence, instances, null) to ID", () => {
+        expect(classifyInstance(["Présidence"])).toEqual({ kind: "ID", type: "Instances décisionnelles" });
+        expect(classifyInstance(["Conférence des Présidents"])).toEqual({ kind: "ID", type: "Instances décisionnelles" });
+        expect(classifyInstance(null)).toEqual({ kind: "ID", type: "Instances décisionnelles" });
+    });
+});
+describe("extractDossierUrl", () => {
+    it("extracts the first dossier-legislatif link from content HTML", () => {
+        const detail = loadFixture("event_557372.json");
+        expect(extractDossierUrl(detail.content)).toBe("https://www.senat.fr/dossier-legislatif/ppl24-172.html");
+    });
+    it("returns null when content has no dossier link", () => {
+        expect(extractDossierUrl("<p>- 44 questions orales</p>")).toBeNull();
+        expect(extractDossierUrl(null)).toBeNull();
+    });
+});
+describe("parseDeadline", () => {
+    it("splits title/detail on <br> and normalizes the date to ISO", () => {
+        const detail = loadFixture("event_557372.json");
+        const parsed = parseDeadline(detail.deadlines[0]);
+        expect(parsed.title).toBe("Délai limite pour le dépôt des amendements de commission");
+        expect(parsed.detail).toBe("Proposition de loi portant diverses dispositions d'adaptation du droit des outre-mer");
+        expect(parsed.dateOriginal).toBe("Vendredi 29 mai 2026 12h00");
+        expect(parsed.date).toBe("2026-05-29T12:00:00.000+02:00");
+    });
+    it("handles deadline without <br> and unparseable date", () => {
+        const parsed = parseDeadline({ title: "Délai limite pour le dépôt des amendements", date: "date invalide" });
+        expect(parsed.title).toBe("Délai limite pour le dépôt des amendements");
+        expect(parsed.detail).toBeNull();
+        expect(parsed.dateOriginal).toBe("date invalide");
+        expect(parsed.date).toBeNull();
+    });
+});
+describe("computeQuantiemes", () => {
+    it("numbers séances publiques of a day in order", () => {
+        const week = JSON.parse(readFileSync(path.join(fixturesDir, "week_2026-06-15.json"), "utf-8"));
+        const day16 = week["2026-06-16"].events;
+        const quant = computeQuantiemes(day16);
+        expect(quant["557355"]).toBe("Première");
+        expect(quant["557356"]).toBe("Deuxième");
+        expect(quant["557358"]).toBe("Troisième");
+        expect(quant["557362"]).toBe("Quatrième");
+        expect(quant["557366"]).toBe("Cinquième");
+        // Non-SP events get null
+        expect(quant["557795"]).toBeNull();
+    });
+});
+describe("apiEventToAgendaEvent", () => {
+    it("maps a séance publique with deadlines, presidency and dossier link (557372)", () => {
+        const detail = loadFixture("event_557372.json");
+        const event = apiEventToAgendaEvent(detail, "Deuxième");
+        expect(event.id).toBe("557372");
+        expect(event.type).toBe("Séance publique");
+        expect(event.organe).toBe("Séance publique");
+        expect(event.date).toBe("2026-06-17");
+        expect(event.timeOriginal).toBe("16h30");
+        expect(event.startTime).toBe("14:30:00.000Z");
+        expect(event.endTime).toBeNull();
+        expect(event.titre).toBe("PPL Adaptation du droit des outre-mer");
+        expect(event.lieu).toBe("Hémicycle");
+        expect(event.captationVideo).toBe(true);
+        expect(event.quantieme).toBe("Deuxième");
+        expect(event.public).toBe(true);
+        expect(event.forecast).toBe(false);
+        expect(event.urlDossierSenat).toBe("https://www.senat.fr/dossier-legislatif/ppl24-172.html");
+        expect(event.objet).toBe("Proposition de loi portant diverses dispositions d'adaptation du droit des outre-mer, présentée par Mme Micheline JACQUES et plusieurs de ses collègues (texte de la commission, n° 691, 2025-2026)");
+        expect(event.deadlines).toHaveLength(3);
+        expect(event.deadlines[0].title).toBe("Délai limite pour le dépôt des amendements de commission");
+        expect(event.deadlines[0].date).toBe("2026-05-29T12:00:00.000+02:00");
+        expect(event.presidency).toEqual({
+            title: "Présidence de séance",
+            content: detail.presidency.content,
+        });
+        expect(event.links).toBeNull();
+        expect(event.contentHtml).toBe(detail.content);
+    });
+    it("maps a commission event (557828)", () => {
+        const detail = loadFixture("event_557828.json");
+        const event = apiEventToAgendaEvent(detail, null);
+        expect(event.type).toBe("Commissions");
+        expect(event.organe).toBe("Commission de la culture");
+        expect(event.lieu).toBe("Salle A245 - 2ème étage Ouest");
+        expect(event.startTime).toBe("13:30:00.000Z"); // 15h30 Paris (CEST)
+        expect(event.captationVideo).toBe(false);
+        expect(event.quantieme).toBeNull();
+        expect(makeReunionUid(event.date, "COM", event.id, event.organe)).toBe("RUSN20260616IDCCOCU557828");
+    });
+    it("maps a mission d'information (557825)", () => {
+        const detail = loadFixture("event_557825.json");
+        const event = apiEventToAgendaEvent(detail, null);
+        expect(event.type).toBe("Mission de contrôle");
+        expect(event.organe).toBe("MI Loi littoral Loi montagne");
+        expect(makeReunionUid(event.date, "MC", event.id, event.organe)).toBe("RUSN20260616IDMMILOLILO557825");
+    });
+    it("maps a délégation (557712) and an office (557752)", () => {
+        const deleg = apiEventToAgendaEvent(loadFixture("event_557712.json"), null);
+        expect(deleg.type).toBe("Offices et délégations");
+        expect(deleg.organe).toBe("Délégation aux entreprises");
+        const office = apiEventToAgendaEvent(loadFixture("event_557752.json"), null);
+        expect(office.type).toBe("Offices et délégations");
+    });
+    it("maps a Présidence event to Instances décisionnelles (557933)", () => {
+        const detail = loadFixture("event_557933.json");
+        const event = apiEventToAgendaEvent(detail, null);
+        expect(event.type).toBe("Instances décisionnelles");
+        expect(event.organe).toBe("Présidence");
+    });
+    it("maps textual hour 'Matin' like the old parser (557937)", () => {
+        const detail = loadFixture("event_557937.json");
+        const event = apiEventToAgendaEvent(detail, null);
+        expect(event.timeOriginal).toBe("Matin");
+        expect(event.startTime).toBe("08:00:00.000Z"); // 10h00 Paris (CEST)
+        expect(event.endTime).toBeNull();
+    });
+    it("maps an event without hour (557939)", () => {
+        const detail = loadFixture("event_557939.json");
+        const event = apiEventToAgendaEvent(detail, null);
+        expect(event.timeOriginal).toBeNull();
+        expect(event.startTime).toBeNull();
+        expect(event.endTime).toBeNull();
+    });
+    it("maps an event without place (Conférence des Présidents)", () => {
+        const detail = loadFixture("event_558859.json");
+        const event = apiEventToAgendaEvent(detail, null);
+        expect(event.lieu).toBeNull();
+        expect(event.organe).toBeNull();
+        expect(event.type).toBe("Instances décisionnelles");
+        expect(event.public).toBe(false);
+    });
+});

package/lib/tests/agenda/reunionApiFields.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/agenda/reunionApiFields.test.js

@@ -0,0 +1,44 @@
+// Checks that buildReunionsByBucket propagates the new API fields into Reunion
+import { describe, expect, it } from "vitest";
+import { buildReunionsByBucket } from "../../src/utils/reunion_parsing.js";
+describe("buildReunionsByBucket with API fields", () => {
+    it("copies deadlines, presidency, links, forecast and public into the Reunion", () => {
+        const event = {
+            id: "557372",
+            type: "Séance publique",
+            date: "2026-06-17",
+            startTime: "14:30:00.000Z",
+            endTime: null,
+            timeOriginal: "16h30",
+            titre: "PPL Adaptation du droit des outre-mer",
+            organe: "Séance publique",
+            objet: "Proposition de loi",
+            lieu: "Hémicycle",
+            captationVideo: true,
+            urlDossierSenat: "https://www.senat.fr/dossier-legislatif/ppl24-172.html",
+            quantieme: "Deuxième",
+            public: true,
+            forecast: false,
+            contentHtml: "<p>- Proposition de loi</p>",
+            presidency: { title: "Présidence de séance", content: "<p>M. X</p>" },
+            links: null,
+            deadlines: [
+                {
+                    title: "Délai limite pour le dépôt des amendements de séance",
+                    detail: "Proposition de loi portant diverses dispositions d'adaptation du droit des outre-mer",
+                    dateOriginal: "Vendredi 12 juin 2026 12h00",
+                    date: "2026-06-12T12:00:00.000+02:00",
+                },
+            ],
+        };
+        const buckets = buildReunionsByBucket([event], {});
+        expect(buckets.IDS).toHaveLength(1);
+        const reunion = buckets.IDS[0];
+        expect(reunion.uid).toBe("RUSN20260617IDS557372");
+        expect(reunion.deadlines).toEqual(event.deadlines);
+        expect(reunion.presidency).toEqual(event.presidency);
+        expect(reunion.links).toBeNull();
+        expect(reunion.forecast).toBe(false);
+        expect(reunion.public).toBe(true);
+    });
+});

package/lib/tests/dataDownload.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/dataDownload.test.js

@@ -0,0 +1,217 @@
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { forwardArgs, parseArgs, readStatusFile, recordFailure, recordSuccess, runDataDownload, STATUS_FILENAME, writeStatusFile, } from "../src/scripts/shared/data_download.js";
+let dataDir;
+afterEach(() => {
+    if (dataDir !== undefined) {
+        rmSync(dataDir, { force: true, recursive: true });
+    }
+});
+function createTempDataDir() {
+    dataDir = mkdtempSync(path.join(tmpdir(), "senat-data-download-"));
+    return dataDir;
+}
+const fixedDate = new Date("2026-09-08T10:00:00Z");
+function makeExec(respond) {
+    const commands = [];
+    return {
+        commands,
+        exec: (command) => {
+            commands.push(command);
+            return respond(command) ?? { status: 0 };
+        },
+    };
+}
+describe("parseArgs", () => {
+    it("defaults categories to All when none are provided", () => {
+        const parsed = parseArgs([]);
+        expect(parsed.categories).toEqual(["All"]);
+        expect(parsed.dataDir).toBeUndefined();
+        expect(parsed.rest).toEqual([]);
+    });
+    it("extracts the dataDir positional argument and forwards the other options", () => {
+        const rawArgs = [
+            "--all",
+            "/app/senat-data",
+            "--fromSession",
+            "2022",
+            "--fetchDocuments",
+            "--parseDocuments",
+            "--only-recent",
+            "30",
+            "--commit",
+            "--keepDir",
+        ];
+        const parsed = parseArgs(rawArgs);
+        expect(parsed.dataDir).toBe("/app/senat-data");
+        expect(parsed.rest).toEqual([
+            "--all",
+            "--fromSession",
+            "2022",
+            "--fetchDocuments",
+            "--parseDocuments",
+            "--only-recent",
+            "30",
+            "--commit",
+            "--keepDir",
+        ]);
+    });
+    it("collects --categories values including repeated occurrences", () => {
+        const parsed = parseArgs(["/data", "--categories", "Ameli", "--categories", "Sens"]);
+        expect(parsed.categories).toEqual(["Ameli", "Sens"]);
+        expect(parsed.dataDir).toBe("/data");
+        expect(parsed.rest).toEqual([]);
+    });
+    it("supports the -k alias and comma-separated values", () => {
+        const parsed = parseArgs(["-k", "Sens,Ameli", "/data"]);
+        expect(parsed.categories).toEqual(["Sens", "Ameli"]);
+    });
+    it("supports the --categories= form", () => {
+        const parsed = parseArgs(["--categories=Sens", "/data"]);
+        expect(parsed.categories).toEqual(["Sens"]);
+    });
+    it("does not mistake valued options for the dataDir positional", () => {
+        const parsed = parseArgs(["--sudo", "postgres", "--remote", "origin", "true-data", "--verbose"]);
+        expect(parsed.dataDir).toBe("true-data");
+        expect(parsed.rest).toEqual(["--sudo", "postgres", "--remote", "origin", "--verbose"]);
+    });
+});
+describe("forwardArgs", () => {
+    it("appends the dataDir then the requested categories to the forwarded options", () => {
+        const parsed = parseArgs(["--all", "/app/senat-data", "--categories", "All", "--fromSession", "2022"]);
+        const forwarded = forwardArgs(parsed, ["Sens"]);
+        expect(forwarded).toBe("--all --fromSession 2022 /app/senat-data --categories Sens");
+    });
+});
+describe("dataset status file", () => {
+    it("increments consecutiveFailures and keeps firstFailureAt across consecutive failures", () => {
+        const statuses = {};
+        recordFailure(statuses, "sens", new Date("2026-09-01T00:00:00Z"), "dump empty");
+        recordFailure(statuses, "sens", new Date("2026-09-02T00:00:00Z"), "dump empty again");
+        expect(statuses["sens"]).toMatchObject({
+            status: "failed",
+            consecutiveFailures: 2,
+            firstFailureAt: "2026-09-01T00:00:00.000Z",
+            lastFailureAt: "2026-09-02T00:00:00.000Z",
+            lastError: "dump empty again",
+            lastSuccessAt: undefined,
+        });
+    });
+    it("resets the failure streak on success while updating lastSuccessAt", () => {
+        const statuses = {};
+        recordFailure(statuses, "sens", new Date("2026-09-01T00:00:00Z"), "dump empty");
+        recordSuccess(statuses, "sens", new Date("2026-09-02T00:00:00Z"));
+        expect(statuses["sens"]).toEqual({
+            status: "ok",
+            consecutiveFailures: 0,
+            lastRunAt: "2026-09-02T00:00:00.000Z",
+            lastSuccessAt: "2026-09-02T00:00:00.000Z",
+        });
+    });
+    it("returns an empty state for a missing or corrupted status file", () => {
+        const dir = createTempDataDir();
+        expect(readStatusFile(path.join(dir, STATUS_FILENAME))).toEqual({});
+        const corruptedPath = path.join(dir, "corrupted.json");
+        writeFileSync(corruptedPath, "{not json", { encoding: "utf-8" });
+        expect(readStatusFile(corruptedPath)).toEqual({});
+    });
+    it("round-trips statuses through the status file", () => {
+        const dir = createTempDataDir();
+        const filePath = path.join(dir, STATUS_FILENAME);
+        const statuses = {};
+        recordFailure(statuses, "sens", fixedDate, "dump empty");
+        writeStatusFile(filePath, statuses);
+        expect(readStatusFile(filePath)).toEqual(statuses);
+    });
+});
+describe("runDataDownload", () => {
+    it("continues after a dataset failure, converts only successful categories and exits 0", () => {
+        const dir = createTempDataDir();
+        const { commands, exec } = makeExec((command) => command.includes("--categories Sens")
+            ? { status: 1, error: new Error("Staging schema sens_staging is empty") }
+            : undefined);
+        const exitCode = runDataDownload({
+            rawArgs: [dir, "--fromSession", "2022"],
+            exec,
+            now: () => fixedDate,
+        });
+        expect(exitCode).toBe(0);
+        const retrieveCommands = commands.filter((command) => command.includes("retrieve_open_data"));
+        expect(retrieveCommands).toHaveLength(5);
+        for (const command of retrieveCommands) {
+            expect(command).toContain("retrieve_open_data.ts --all ");
+        }
+        expect(retrieveCommands[0]).toContain("--categories Ameli");
+        expect(retrieveCommands[4]).toContain("--categories Sens");
+        const convertCommand = commands.find((command) => command.includes("convert_data"));
+        expect(convertCommand).toContain("--categories Ameli");
+        expect(convertCommand).toContain("--categories Debats");
+        expect(convertCommand).toContain("--categories DosLeg");
+        expect(convertCommand).toContain("--categories Questions");
+        expect(convertCommand).not.toContain("--categories Sens");
+        const agendaCommand = commands.find((command) => command.includes("retrieve_agenda"));
+        expect(agendaCommand).toBe(`cross-env TZ='Etc/UTC' tsx src/scripts/retrieve_agenda.ts ${dir} --fromSession 2022 --parseAgenda --silent`);
+        const statuses = JSON.parse(readFileSync(path.join(dir, STATUS_FILENAME), { encoding: "utf-8" }));
+        expect(statuses.sens).toMatchObject({ status: "failed", consecutiveFailures: 1 });
+        expect(statuses.ameli).toMatchObject({ status: "ok", consecutiveFailures: 0 });
+    });
+    it("exits 1 without conversion when all datasets fail", () => {
+        const dir = createTempDataDir();
+        const { commands, exec } = makeExec(() => ({ status: 1, error: new Error("boom") }));
+        const exitCode = runDataDownload({
+            rawArgs: [dir],
+            exec,
+            now: () => fixedDate,
+        });
+        expect(exitCode).toBe(1);
+        expect(commands.filter((command) => command.includes("retrieve_open_data"))).toHaveLength(5);
+        expect(commands.find((command) => command.includes("convert_data"))).toBeUndefined();
+        expect(commands.find((command) => command.includes("retrieve_agenda"))).toBeUndefined();
+        const statuses = JSON.parse(readFileSync(path.join(dir, STATUS_FILENAME), { encoding: "utf-8" }));
+        for (const dataset of ["ameli", "debats", "dosleg", "questions", "sens"]) {
+            expect(statuses[dataset]).toMatchObject({ status: "failed", consecutiveFailures: 1 });
+        }
+    });
+    it("runs every step when all datasets succeed", () => {
+        const dir = createTempDataDir();
+        const { commands, exec } = makeExec(() => undefined);
+        const exitCode = runDataDownload({
+            rawArgs: [dir, "--categories", "All"],
+            exec,
+            now: () => fixedDate,
+        });
+        expect(exitCode).toBe(0);
+        const convertCommand = commands.find((command) => command.includes("convert_data"));
+        for (const category of ["Ameli", "Debats", "DosLeg", "Questions", "Sens"]) {
+            expect(convertCommand).toContain(`--categories ${category}`);
+        }
+        expect(commands.find((command) => command.includes("retrieve_agenda"))).toBeDefined();
+        expect(commands.find((command) => command.includes("retrieve_cr_seance"))).toBeDefined();
+        expect(commands.find((command) => command.includes("retrieve_cr_commission"))).toBeDefined();
+        expect(commands.find((command) => command.includes("retrieve_videos"))).toBeDefined();
+        expect(commands.find((command) => command.includes("retrieve_collaborateurs"))).toBeDefined();
+    });
+    it("stops after convert_data when it fails and propagates its exit status", () => {
+        const dir = createTempDataDir();
+        const { commands, exec } = makeExec((command) => command.includes("convert_data") ? { status: 3, error: new Error("parse failure") } : undefined);
+        const exitCode = runDataDownload({
+            rawArgs: [dir],
+            exec,
+            now: () => fixedDate,
+        });
+        expect(exitCode).toBe(3);
+        expect(commands.find((command) => command.includes("retrieve_agenda"))).toBeUndefined();
+    });
+    it("exits 1 when no requested category matches a known dataset", () => {
+        const { commands, exec } = makeExec(() => undefined);
+        const exitCode = runDataDownload({
+            rawArgs: ["/data", "--categories", "Unknown"],
+            exec,
+            now: () => fixedDate,
+        });
+        expect(exitCode).toBe(1);
+        expect(commands).toHaveLength(0);
+    });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "3.2.2",
+  "version": "3.3.0",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",