@tricoteuses/senat 2.13.1 → 2.14.0

package/lib/raw_types/db.js

@@ -0,0 +1,5 @@
+/**
+ * This file was generated by kysely-codegen.
+ * Please do not edit it manually.
+ */
+export {};

package/lib/scripts/convert_data.js

@@ -5,7 +5,7 @@ import path from "path";
 import { datasets, EnabledDatasets, getEnabledDatasets } from "../datasets";
 import { DATA_ORIGINAL_FOLDER, DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, SCRUTINS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER, } from "../loaders";
 import { findAllAmendements, findAllCirconscriptions, findAllDebats, findAllDossiers, findAllScrutins, findAllOrganismes, findAllQuestions, findAllSens, findAuteurs, findSenatRapportUrls, findSenatTexteUrls, } from "../model";
-import { getCodeActeLecture, getCodeActeTexte } from "../model/dosleg";
+import { createActesLegislatifs } from "../model/dosleg";
 import { UNDEFINED_SESSION } from "../types/sessions";
 import { getSessionFromDate, getSessionFromSignet } from "./datautil";
 import { commonOptions } from "./shared/cli_helpers";
@@ -121,28 +121,11 @@ async function convertDatasetDosLeg(dataDir, options) {
         }
         loiReorganizedDir = path.join(dossiersReorganizedDir, String(session));
         fs.ensureDirSync(loiReorganizedDir);
-        // Ajout de code_acte à chaque lecture du dossier
-        const lecturesWithCodeActe = (loi["lectures"] || []).map((lecture) => {
-            const lecturesAssemblee = (lecture["lectures_assemblee"] || []).map((lectureAss) => {
-                const codeParent = getCodeActeLecture(loi["code_nature_dossier"], lecture["type_lecture"], lectureAss["assemblee"]);
-                const textesWithCodeActe = (lectureAss["textes"] || []).map((texte) => ({
-                    ...texte,
-                    code_acte: getCodeActeTexte(codeParent, texte["origine"])
-                }));
-                return {
-                    ...lectureAss,
-                    code_acte: codeParent,
-                    textes: textesWithCodeActe
-                };
-            });
-            return {
-                ...lecture,
-                lectures_assemblee: lecturesAssemblee
-            };
-        });
-        const loiWithCodeActe = { ...loi, lectures: lecturesWithCodeActe };
+        // Ajout des actes législatifs au dossier
+        const actesLegislatifs = createActesLegislatifs(loi);
+        const loiWithActes = { ...loi, actes_legislatifs: actesLegislatifs };
         const scrutinFileName = `${loi["signet"]}.json`;
-        fs.writeJSONSync(path.join(loiReorganizedDir, scrutinFileName), loiWithCodeActe, {
+        fs.writeJSONSync(path.join(loiReorganizedDir, scrutinFileName), loiWithActes, {
             spaces: 2,
         });
     }

package/lib/scripts/retrieve_comptes_rendus.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Needs to be run after retrieve_agenda.ts !
+ * - downloads the ZIP of comptes-rendus des débats (CRI) from data.senat.fr
+ * - extracts XML files, distributes them by session/year
+ */
+export declare function retrieveCriXmlDump(dataDir: string, options?: Record<string, any>): Promise<void>;

package/lib/scripts/retrieve_comptes_rendus.js

@@ -0,0 +1,274 @@
+/**
+ * Needs to be run after retrieve_agenda.ts !
+ * - downloads the ZIP of comptes-rendus des débats (CRI) from data.senat.fr
+ * - extracts XML files, distributes them by session/year
+ */
+import assert from "assert";
+import commandLineArgs from "command-line-args";
+import fs from "fs-extra";
+import path from "path";
+import StreamZip from "node-stream-zip";
+import * as cheerio from "cheerio";
+import { AGENDA_FOLDER, COMPTES_RENDUS_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, } from "../loaders";
+import { commonOptions } from "./shared/cli_helpers";
+import { deriveTitreObjetFromSommaire, parseCompteRenduSlotFromFile, parseYYYYMMDD, sessionStartYearFromDate } from "../model/compte_rendu";
+import { makeGroupUid } from "../utils/reunion_grouping";
+import { getSessionsFromStart } from "../types/sessions";
+import { ensureAndClearDir, fetchWithRetry } from "./shared/util";
+import { computeIntervalsBySlot } from "../utils/cr_spliting";
+const optionsDefinitions = [
+    ...commonOptions,
+    {
+        help: "parse and convert comptes-rendus des débats into JSON",
+        name: "parseDebats",
+        type: Boolean,
+    }
+];
+const options = commandLineArgs(optionsDefinitions);
+const CRI_ZIP_URL = "https://data.senat.fr/data/debats/cri.zip";
+const SLOT_ORDER = ["MATIN", "APRES-MIDI", "SOIR"];
+class CompteRenduError extends Error {
+    constructor(message, url) {
+        super(`An error occurred while retrieving ${url}: ${message}`);
+    }
+}
+function pickFirstSlotOfDay(slots) {
+    for (const s of SLOT_ORDER)
+        if (slots.includes(s))
+            return s;
+    return null;
+}
+function loadAgendaSPSlotsForDate(dataDir, yyyymmdd, session) {
+    const dirPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, session.toString());
+    if (!fs.existsSync(dirPath)) {
+        console.warn(`[AGENDA] Directory not found for session ${session} → ${dirPath}`);
+        return null;
+    }
+    const pattern = new RegExp(`^RUSN${yyyymmdd}IDS-(MATIN|APRES-MIDI|SOIR)\\.json$`);
+    const ALLOWED_SLOTS = new Set(["MATIN", "APRES-MIDI", "SOIR"]);
+    try {
+        const files = fs.readdirSync(dirPath);
+        const matched = files.filter((f) => pattern.test(f));
+        if (matched.length === 0) {
+            return null;
+        }
+        const found = new Set();
+        for (const name of matched) {
+            const m = name.match(pattern);
+            const raw = (m?.[1] ?? "");
+            if (ALLOWED_SLOTS.has(raw))
+                found.add(raw);
+        }
+        const slots = Array.from(found);
+        if (slots.length === 0) {
+            return null;
+        }
+        return { filePath: dirPath, slots };
+    }
+    catch {
+        return null;
+    }
+}
+async function downloadCriZip(zipPath) {
+    if (!options["silent"])
+        console.log(`Downloading CRI zip ${CRI_ZIP_URL}…`);
+    const response = await fetchWithRetry(CRI_ZIP_URL);
+    if (!response.ok) {
+        if (response.status === 404) {
+            console.warn(`CRI zip ${CRI_ZIP_URL} not found`);
+            return;
+        }
+        throw new CompteRenduError(String(response.status), CRI_ZIP_URL);
+    }
+    const buf = Buffer.from(await response.arrayBuffer());
+    await fs.writeFile(zipPath, buf);
+    if (!options["silent"]) {
+        const mb = (buf.length / (1024 * 1024)).toFixed(1);
+        console.log(`[CRI] Downloaded ${mb} MB → ${zipPath}`);
+    }
+}
+async function extractAndDistributeXmlBySession(zipPath, originalRoot) {
+    const zip = new StreamZip.async({ file: zipPath });
+    const entries = await zip.entries();
+    let count = 0;
+    for (const entryName of Object.keys(entries)) {
+        if (!entryName.toLowerCase().endsWith(".xml"))
+            continue;
+        // ex: d20231005.xml
+        const base = path.basename(entryName);
+        const m = base.match(/^d(\d{8})\.xml$/i);
+        if (!m)
+            continue;
+        const yyyymmdd = m[1];
+        const dt = parseYYYYMMDD(yyyymmdd);
+        if (!dt)
+            continue;
+        const session = sessionStartYearFromDate(dt);
+        const destDir = path.join(originalRoot, String(session));
+        await fs.ensureDir(destDir);
+        const outPath = path.join(destDir, base);
+        await zip.extract(entryName, outPath);
+        count++;
+    }
+    await zip.close();
+    return count;
+}
+export async function retrieveCriXmlDump(dataDir, options = {}) {
+    const root = path.join(dataDir, COMPTES_RENDUS_FOLDER);
+    ensureAndClearDir(root);
+    const originalRoot = path.join(root, DATA_ORIGINAL_FOLDER);
+    fs.ensureDirSync(originalRoot);
+    const transformedRoot = path.join(root, DATA_TRANSFORMED_FOLDER);
+    if (options["parseDebats"])
+        fs.ensureDirSync(transformedRoot);
+    const sessions = getSessionsFromStart(options["fromSession"]);
+    // 1) Download ZIP global + distribut by session
+    const zipPath = path.join(dataDir, "cri.zip");
+    console.log("[CRI] Downloading global CRI zip…");
+    await downloadCriZip(zipPath);
+    console.log("[CRI] Extracting + distributing XMLs by session…");
+    for (const session of sessions) {
+        const dir = path.join(originalRoot, String(session));
+        if (await fs.pathExists(dir)) {
+            for (const f of await fs.readdir(dir))
+                if (/\.xml$/i.test(f))
+                    await fs.remove(path.join(dir, f));
+        }
+    }
+    const n = await extractAndDistributeXmlBySession(zipPath, originalRoot);
+    if (n === 0) {
+        console.warn("[CRI] No XML extracted. Archive empty or layout changed?");
+    }
+    else {
+        console.log(`[CRI] Distributed ${n} XML file(s) into session folders.`);
+    }
+    if (!options["parseDebats"]) {
+        console.log("[CRI] parseDebats not requested → done.");
+        return;
+    }
+    for (const session of sessions) {
+        const originalSessionDir = path.join(originalRoot, String(session));
+        if (!(await fs.pathExists(originalSessionDir))) {
+            continue;
+        }
+        const xmlFiles = (await fs.readdir(originalSessionDir))
+            .filter((f) => /^d\d{8}\.xml$/i.test(f))
+            .sort();
+        const transformedSessionDir = path.join(transformedRoot, String(session));
+        if (options["parseDebats"])
+            await fs.ensureDir(transformedSessionDir);
+        for (const f of xmlFiles) {
+            const yyyymmdd = f.slice(1, 9);
+            const xmlPath = path.join(originalSessionDir, f);
+            // 1) Deduce slot(s) from agenda if it exsits
+            const agendaInfo = loadAgendaSPSlotsForDate(dataDir, yyyymmdd, session);
+            const firstSlotOfDay = pickFirstSlotOfDay(agendaInfo?.slots ?? []);
+            // 2) Detect slots from CRI content
+            let slotsInCri = [];
+            try {
+                const raw = await fs.readFile(xmlPath, "utf8");
+                const $ = cheerio.load(raw, { xml: false });
+                const order = $("body *").toArray();
+                const idx = new Map(order.map((el, i) => [el, i]));
+                const intervals = computeIntervalsBySlot($, idx, firstSlotOfDay ?? undefined);
+                const uniq = new Set();
+                for (const iv of intervals)
+                    if (iv.slot && iv.slot !== "UNKNOWN")
+                        uniq.add(iv.slot);
+                slotsInCri = Array.from(uniq);
+            }
+            catch (e) {
+                console.warn(`[CRI] [${session}] Cannot read/parse ${f}:`, e);
+                continue;
+            }
+            if (slotsInCri.length === 0) {
+                slotsInCri = [firstSlotOfDay ?? "MATIN"];
+            }
+            // 3) Parse & write each slot
+            for (const slot of slotsInCri) {
+                const outName = `CRSSN${yyyymmdd}-${slot}.json`;
+                const cr = await parseCompteRenduSlotFromFile(xmlPath, slot, firstSlotOfDay ?? slot);
+                if (!cr) {
+                    console.warn(`[CRI] [${session}] Empty or no points for ${yyyymmdd} (${slot}) → skip`);
+                    continue;
+                }
+                const outDir = transformedSessionDir;
+                await fs.ensureDir(outDir);
+                const outPath = path.join(outDir, outName);
+                await fs.writeJSON(outPath, cr, { spaces: 2 });
+                try {
+                    await linkCriSlotIntoAgendaGrouped(dataDir, yyyymmdd, slot, cr.uid, cr, session);
+                }
+                catch (e) {
+                    console.warn(`[AGENDA] [${session}] Could not link CR into grouped for ${yyyymmdd} ${slot}:`, e);
+                }
+            }
+        }
+    }
+}
+async function main() {
+    const dataDir = options["dataDir"];
+    assert(dataDir, "Missing argument: data directory");
+    console.time("CRI processing time");
+    await retrieveCriXmlDump(dataDir, options);
+    if (!options["silent"]) {
+        console.timeEnd("CRI processing time");
+    }
+}
+main()
+    .then(() => process.exit(0))
+    .catch((error) => {
+    console.error(error);
+    process.exit(1);
+});
+async function linkCriSlotIntoAgendaGrouped(dataDir, yyyymmdd, slot, crUid, cr, session) {
+    const groupedDir = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, session.toString());
+    fs.ensureDirSync(groupedDir);
+    const groupedPath = path.join(groupedDir, 'RUSN' + yyyymmdd + 'IDS-' + slot + '.json');
+    let groups = [];
+    if (fs.existsSync(groupedPath)) {
+        try {
+            groups = JSON.parse(fs.readFileSync(groupedPath, "utf8"));
+            if (!Array.isArray(groups))
+                groups = [];
+        }
+        catch (e) {
+            console.warn(`[AGENDA] unreadable grouped JSON → ${groupedPath} (${e}) → recreating`);
+            groups = [];
+        }
+    }
+    // find existing group with same slot
+    const sameSlot = groups.filter(g => g?.slot === slot);
+    let target = null;
+    if (sameSlot.length > 1) {
+        console.warn(`[AGENDA] multiple groups for ${yyyymmdd} ${slot} in ${groupedPath} → linking the first`);
+    }
+    target = sameSlot[0] ?? null;
+    const dateISO = `${yyyymmdd.slice(0, 4)}-${yyyymmdd.slice(4, 6)}-${yyyymmdd.slice(6, 8)}`;
+    const sommaire = cr?.metadonnees?.sommaire;
+    const { titre: dTitre, objet: dObjet } = deriveTitreObjetFromSommaire(sommaire, slot);
+    if (!target) {
+        const newGroup = {
+            uid: makeGroupUid(dateISO, slot),
+            chambre: "SN",
+            date: dateISO,
+            slot,
+            type: "Séance publique",
+            startTime: null,
+            endTime: null,
+            captationVideo: false,
+            titre: dTitre,
+            objet: dObjet || "",
+            events: [],
+            compteRenduRefUid: crUid,
+        };
+        groups.push(newGroup);
+    }
+    else {
+        target.compteRenduRefUid = crUid;
+    }
+    await fs.writeJSON(groupedPath, groups, { spaces: 2 });
+    if (!options["silent"]) {
+        console.log(`[AGENDA] Linked CR ${crUid} → ${path.basename(groupedPath)} [${slot}]`);
+    }
+}

package/lib/scripts/retrieve_videos.d.ts

@@ -1 +1,7 @@
-export {};
+import { GroupedReunion } from "../types/agenda";
+export declare function buildSenatVodMasterM3u8FromNvs(nvsText: string, finalText: string): string | null;
+export declare function score(agenda: GroupedReunion, agendaTs: number | null, videoTitle?: string, videoEpoch?: number): number;
+/**
+ * Build search strategies for senat's videos
+ */
+export declare function buildSearchStrategies(agenda: GroupedReunion): Array<Record<string, string>>;

package/lib/scripts/retrieve_videos.js

@@ -4,12 +4,11 @@ import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 import fsp from "fs/promises";
 import path from "path";
-import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatAgendas, } from "../loaders";
+import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatAgendasGrouped } from "../loaders";
 import { getSessionsFromStart } from "../types/sessions";
 import { commonOptions } from "./shared/cli_helpers";
-import { formatYYYYMMDD, makeReunionUid } from "../utils/reunion_grouping";
 // ===================== Constants =====================
-const MATCH_THRESHOLD = 0.60;
+const MATCH_THRESHOLD = 0.6;
 const MAX_CANDIDATES = 15;
 const MAX_PAGES = 3;
 const STATS = { total: 0, accepted: 0 };
@@ -18,9 +17,7 @@ const SENAT_VIDEOS_SEARCH_AJAX = "https://videos.senat.fr/senat_videos_search.ph
 const SENAT_DATAS_ROOT = "https://videos.senat.fr/Datas/senat";
 const SENAT_VOD_HOST = "https://vodsenat.akamaized.net";
 // ===================== CLI =====================
-const optionsDefinitions = [
-    ...commonOptions,
-];
+const optionsDefinitions = [...commonOptions];
 const options = commandLineArgs(optionsDefinitions);
 // ===================== Utils =====================
 function normalize(s) {
@@ -32,7 +29,9 @@ function normalize(s) {
         .replace(/\s+/g, " ")
         .trim();
 }
-function tokens(s) { return normalize(s).split(" ").filter(Boolean); }
+function tokens(s) {
+    return normalize(s).split(" ").filter(Boolean);
+}
 function dice(a, b) {
     const A = new Set(tokens(a)), B = new Set(tokens(b));
     if (!A.size || !B.size)
@@ -46,7 +45,7 @@ function dice(a, b) {
 // Heuristic for Europe/Paris DST: +02:00 ≈ April→October, +01:00 otherwise.
 function parisOffsetForDate(dateYYYYMMDD) {
     const m = Number(dateYYYYMMDD.split("-")[1] || "1");
-    return (m >= 4 && m <= 10) ? "+02:00" : "+01:00";
+    return m >= 4 && m <= 10 ? "+02:00" : "+01:00";
 }
 function epochToParisDateTime(epochSec) {
     if (!Number.isFinite(epochSec))
@@ -54,7 +53,7 @@ function epochToParisDateTime(epochSec) {
     const dUtc = new Date(epochSec * 1000);
     // Offset heuristic (same logique que parisOffsetForDate)
     const m = dUtc.getUTCMonth() + 1; // 1..12
-    const offsetHours = (m >= 4 && m <= 10) ? 2 : 1;
+    const offsetHours = m >= 4 && m <= 10 ? 2 : 1;
     const offsetStr = offsetHours === 2 ? "+02:00" : "+01:00";
     // Applique l'offset pour obtenir la date/heure locales Paris
     const localMs = dUtc.getTime() + offsetHours * 3600 * 1000;
@@ -149,7 +148,7 @@ function extractCandidatesFromSearchHtml(html) {
         out.push({ id, hash, pageUrl, title: t?.[1] });
     }
     const seen = new Set();
-    return out.filter(c => {
+    return out.filter((c) => {
         const k = `${c.id}_${c.hash}`;
         if (seen.has(k))
             return false;
@@ -162,46 +161,68 @@ function parseDataNvs(nvs) {
     const title = nvs.match(/<metadata\s+name="title"\s+value="([^"]+)"/i)?.[1];
     return { epoch: epoch ? Number(epoch) : undefined, title };
 }
-function buildSenatVodMasterM3u8FromNvs(xml, host = SENAT_VOD_HOST) {
-    if (!xml)
+// nvsText = contenu texte de data.nvs (utf-8)
+// finalText = contenu texte de finalplayer.nvs (utf-8)
+export function buildSenatVodMasterM3u8FromNvs(nvsText, finalText) {
+    // 1) Base Akamai depuis data.nvs (mp4 "serverfiles://senat/YYYY/MM/encoderX_YYYYMMDDHHMMSS_1.mp4")
+    const baseMatch = nvsText.match(/serverfiles:\/\/senat\/(\d{4})\/(\d{2})\/(encoder\d)_(\d{14})/i);
+    if (!baseMatch)
         return null;
-    // (a) Déjà un lien VOD complet en .smil/playlist.m3u8
-    const mVod = xml.match(/https?:\/\/[^"'<>]*vodsenat[^"'<>]*\.smil\/(?:playlist|master)\.m3u8/i);
-    if (mVod)
-        return mVod[0];
-    // (b) Chemin senat/YYYY/MM/<basename>.smil  -> normalise en playlist.m3u8
-    const mSmilPath = xml.match(/senat\/(\d{4})\/(\d{2})\/([^"'<>\/]+?)\.smil/i);
-    if (mSmilPath) {
-        const [, y, m, base] = mSmilPath;
-        return `${host}/senat/${y}/${m}/${base}.smil/playlist.m3u8`;
+    const [, yyyy, mm, encoder, stamp] = baseMatch;
+    const base = `https://vodsenat.akamaized.net/senat/${yyyy}/${mm}/${encoder}_${stamp}`;
+    // 2) start/end depuis finalplayer.nvs
+    let start = null, end = null;
+    const playerAttr = finalText.match(/player[^>]*\bstarttime="(\d+)"[^>]*\bendtime="(\d+)"/i);
+    if (playerAttr) {
+        start = parseInt(playerAttr[1], 10);
+        end = parseInt(playerAttr[2], 10);
+    }
+    else {
+        // fallback: prendre le plus petit timecode des <synchro timecode="...">
+        const tc = Array.from(finalText.matchAll(/timecode="(\d+)"/g)).map((m) => parseInt(m[1], 10));
+        if (tc.length)
+            start = Math.min(...tc);
+    }
+    // 3) si pas d'end, on peut déduire via "duree" (en secondes) de data.nvs
+    if (end == null) {
+        const durMeta = nvsText.match(/<metadata[^>]*\bname="duree"[^>]*\bvalue="(\d+)"[^>]*>/i);
+        if (durMeta && start != null) {
+            const durMs = parseInt(durMeta[1], 10) * 1000; // sec → ms
+            end = start + durMs;
+        }
     }
-    // (c) Chemin senat/YYYY/MM/<basename>.mp4  -> transforme en .smil/playlist.m3u8
-    const mMp4Path = xml.match(/senat\/(\d{4})\/(\d{2})\/([^"'<>\/]+?)\.mp4/i);
-    if (mMp4Path) {
-        const [, y, m, base] = mMp4Path;
-        return `${host}/senat/${y}/${m}/${base}.smil/playlist.m3u8`;
+    // 4) Construction de l’URL
+    //    - si on a start & end → utiliser ps/pd (robuste et conforme à ce que sert le Sénat)
+    //    - sinon fallback sans suffixe (souvent valide aussi)
+    if (start != null && end != null && end > start) {
+        const pd = end - start;
+        return `${base}_ps${start}_pd${pd}.smil/master.m3u8`;
     }
-    // (d) À défaut, n’importe quel .m3u8 présent (faible priorité — peut être du live)
-    const mAny = xml.match(/https?:\/\/[^"'<>]+\.m3u8/i);
-    return mAny ? mAny[0] : null;
+    return `${base}.smil/master.m3u8`;
 }
-function score(agenda, agendaTs, videoTitle, videoEpoch) {
+export function score(agenda, agendaTs, videoTitle, videoEpoch) {
     const titleScore = dice(agenda.titre || "", videoTitle || "");
     let timeScore = 0;
     if (agendaTs && videoEpoch) {
+        // second
         const deltaMin = Math.abs(videoEpoch - agendaTs) / 60;
-        timeScore = Math.max(0, 1 - (deltaMin / 180));
+        // delta : 180min
+        timeScore = Math.max(0, 1 - deltaMin / 180);
     }
     let orgBonus = 0;
     if (agenda.organe && videoTitle) {
         const o = normalize(agenda.organe);
         const t = normalize(videoTitle);
-        if (o && t.includes(o.split(" ")[0]))
+        const first = o.split(" ").filter(Boolean)[0];
+        if (first && t.includes(first))
             orgBonus = 0.15;
     }
-    return 0.3 * titleScore + 0.7 * timeScore + orgBonus;
+    return 0.3 * titleScore + 0.7 * timeScore + orgBonus; // Can be adjusted
 }
-function buildSearchStrategies(agenda) {
+/**
+ * Build search strategies for senat's videos
+ */
+export function buildSearchStrategies(agenda) {
     const fr = agenda.date ? toFRDate(agenda.date) : undefined;
     const kw = simplifyTitleForKeywords(agenda.titre || "");
     const commission = agenda.organe || undefined;
@@ -239,21 +260,23 @@ async function fetchAllSearchPages(args, baseDir, strategyIndex, maxPages = MAX_
     }
     return pages;
 }
-async function processAgenda(agenda, session, dataDir) {
+async function processGroupedReunion(agenda, session, dataDir) {
     if (!agenda)
         return;
+    // 1) Garde-fous
     if (!agenda.captationVideo) {
         if (!options["silent"])
-            console.log(`[skip] ${agenda.id} captationVideo=false`);
+            console.log(`[skip] ${agenda.uid} captationVideo=false`);
         return;
     }
     if (!agenda.date || !agenda.startTime) {
         if (!options["silent"])
-            console.log(`[skip] ${agenda.id} date/hour missing`);
+            console.log(`[skip] ${agenda.uid} date/hour missing`);
         return;
     }
     STATS.total++;
-    const reunionUid = makeReunionUid(agenda);
+    // 2) Dossier de sortie (utilise directement l'UID)
+    const reunionUid = agenda.uid;
     const baseDir = path.join(dataDir, VIDEOS_ROOT_FOLDER, String(session), reunionUid);
     await fs.ensureDir(baseDir);
     const agendaTs = toTargetEpoch(agenda.date, agenda.startTime);
@@ -276,8 +299,9 @@ async function processAgenda(agenda, session, dataDir) {
         }
     }
     if (usedStrategy === -1 || !candidates.length) {
-        if (!options["silent"])
-            console.log(`[miss] ${agenda.id} no candidates (triedStrategies=${strategies.length})`);
+        if (!options["silent"]) {
+            console.log(`[miss] ${agenda.uid} no candidates (triedStrategies=${strategies.length})`);
+        }
         return;
     }
     // ==== 2) Enrich via data.nvs + scoring; pick best ====
@@ -295,14 +319,14 @@ async function processAgenda(agenda, session, dataDir) {
     }
     if (!best) {
         if (!options["silent"])
-            console.log(`[miss] ${agenda.id} candidats without data.nvs`);
+            console.log(`[miss] ${agenda.uid} candidates without data.nvs`);
         return;
     }
     const accepted = best.score >= MATCH_THRESHOLD;
     if (accepted)
         STATS.accepted++;
     if (!options["silent"]) {
-        console.log(`[pick] ${agenda.id} best id=${best.id} hash=${best.hash} score=${best.score.toFixed(2)} accepted=${accepted} (strategy=${usedStrategy})`);
+        console.log(`[pick] ${agenda.uid} best id=${best.id} hash=${best.hash} score=${best.score.toFixed(2)} accepted=${accepted} (strategy=${usedStrategy})`);
     }
     // ==== 3) Write metadata + NVS of the best candidate (always) ====
     const bestDt = best?.epoch ? epochToParisDateTime(best.epoch) : null;
@@ -317,7 +341,7 @@ async function processAgenda(agenda, session, dataDir) {
             startTime: agenda.startTime,
             titre: agenda.titre,
             organe: agenda.organe ?? undefined,
-            id: agenda.id,
+            uid: agenda.uid,
         },
         best: {
             id: best.id,
@@ -340,37 +364,30 @@ async function processAgenda(agenda, session, dataDir) {
     if (finalTxt)
         await fsp.writeFile(path.join(baseDir, "finalplayer.nvs"), finalTxt, "utf-8");
     let master = null;
-    if (dataTxt)
-        master = buildSenatVodMasterM3u8FromNvs(dataTxt);
+    if (dataTxt && finalTxt)
+        master = buildSenatVodMasterM3u8FromNvs(dataTxt, finalTxt);
     // ==== 4) Update agenda file (only if accepted + m3u8) ====
     if (accepted && master) {
-        const agendaJsonPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session), `${formatYYYYMMDD(agenda.date)}.json`);
+        const agendaJsonPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session), `${agenda.uid}.json`);
         if (await fs.pathExists(agendaJsonPath)) {
             const raw = await fsp.readFile(agendaJsonPath, "utf-8");
-            let items;
+            let obj;
             try {
-                items = JSON.parse(raw);
+                obj = JSON.parse(raw);
             }
             catch (e) {
                 console.warn(`[warn] invalid JSON in ${agendaJsonPath}:`, e?.message);
-                items = null;
+                obj = null;
             }
-            if (Array.isArray(items)) {
-                const idx = items.findIndex((e) => String(e?.id) === String(agenda.id));
-                if (idx === -1) {
-                    console.warn(`[warn] agenda id ${agenda.id} not found in ${agendaJsonPath}`);
-                }
-                else {
-                    // add/update urlVideo on the matching item
-                    items[idx] = { ...items[idx], urlVideo: master };
-                    await writeIfChanged(agendaJsonPath, JSON.stringify(items, null, 2));
-                    if (!options["silent"]) {
-                        console.log(`[write] ${agenda.id} urlVideo ← ${master}`);
-                    }
+            if (obj && typeof obj === "object" && !Array.isArray(obj)) {
+                const next = { ...obj, urlVideo: master };
+                await writeIfChanged(agendaJsonPath, JSON.stringify(next, null, 2));
+                if (!options["silent"]) {
+                    console.log(`[write] ${agenda.uid} urlVideo ← ${master}`);
                 }
             }
             else {
-                console.warn(`[warn] expected an array in ${agendaJsonPath}, got ${typeof items}`);
+                console.warn(`[warn] expected an object in ${agendaJsonPath}, got ${Array.isArray(obj) ? "array" : typeof obj}`);
             }
         }
         else {
@@ -379,15 +396,14 @@ async function processAgenda(agenda, session, dataDir) {
     }
 }
 async function processAll(dataDir, sessions) {
+    console.log("Process all Agendas and fetch video's url");
     for (const session of sessions) {
-        for (const { item: agendas } of iterLoadSenatAgendas(dataDir, session, {})) {
-            for (const agenda of agendas) {
-                try {
-                    await processAgenda(agenda, session, dataDir);
-                }
-                catch (e) {
-                    console.error(`[error] ${agenda.id}:`, e?.message || e);
-                }
+        for (const { item: agenda } of iterLoadSenatAgendasGrouped(dataDir, session)) {
+            try {
+                await processGroupedReunion(agenda, session, dataDir);
+            }
+            catch (e) {
+                console.error(`[error] ${agenda?.uid ?? "unknown-uid"}:`, e?.message || e);
             }
         }
     }
@@ -396,17 +412,16 @@ async function main() {
     const dataDir = options["dataDir"];
     assert(dataDir, "Missing argument: data directory");
     const sessions = getSessionsFromStart(options["fromSession"]);
-    if (!options["silent"])
-        console.time("senat-agendas→videos start processing time");
+    console.time("senat-agendas→videos start processing time");
     await processAll(dataDir, sessions);
-    if (!options["silent"])
-        console.timeEnd("senat-agendas→videos processing time");
-    if (!options["silent"]) {
-        const { total, accepted } = STATS;
-        const ratio = total ? (accepted / total * 100).toFixed(1) : "0.0";
-        console.log(`[summary] accepted=${accepted} / total=${total} (${ratio}%)`);
-    }
+    console.timeEnd("senat-agendas→videos processing time");
+    const { total, accepted } = STATS;
+    const ratio = total ? ((accepted / total) * 100).toFixed(1) : "0.0";
+    console.log(`[summary] accepted=${accepted} / total=${total} (${ratio}%)`);
 }
 main()
     .then(() => process.exit(0))
-    .catch((err) => { console.error(err); process.exit(1); });
+    .catch((err) => {
+    console.error(err);
+    process.exit(1);
+});