@tricoteuses/senat 2.13.0 → 2.13.2

package/lib/loaders.d.ts

@@ -3,7 +3,7 @@ import { DebatResult } from "./model/debats";
 import { DossierLegislatifResult } from "./model/dosleg";
 import { QuestionResult } from "./model/questions";
 import { CirconscriptionResult, OrganismeResult, SenateurResult } from "./model/sens";
-import { AgendaEvent, GroupedReunion } from "./types/agenda";
+import { GroupedReunion } from "./types/agenda";
 import { FlatTexte } from "./types/texte";
 import { CompteRendu } from "./types/compte_rendu";
 export { EnabledDatasets } from "./datasets";
@@ -83,8 +83,6 @@ export declare function loadSenatTexteContent(dataDir: string, textePathFromData
 export declare function loadSenatCompteRenduContent(dataDir: string, session: number, debatId: string | number): {
     item: CompteRendu | null;
 };
-export declare function iterLoadSenatAgendas(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<AgendaEvent[]>>;
-export declare function iterLoadSenatEvenements(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<AgendaEvent>>;
 export declare function iterLoadSenatAgendasGrouped(dataDir: string, session: number | undefined): Generator<IterItem<GroupedReunion>>;
 export declare function iterLoadSenatCirconscriptions(dataDir: string, options?: {}): Generator<IterItem<CirconscriptionResult>>;
 export declare function iterLoadSenatOrganismes(dataDir: string, options?: {}): Generator<IterItem<OrganismeResult>>;

package/lib/loaders.js

@@ -183,18 +183,6 @@ export function loadSenatCompteRenduContent(dataDir, session, debatId) {
     const json = fs.readFileSync(fullPath, { encoding: "utf8" });
     return { item: JSON.parse(json) };
 }
-export function* iterLoadSenatAgendas(dataDir, session, options = {}) {
-    for (const evenementsItem of iterLoadSenatItems(dataDir, AGENDA_FOLDER, session, DATA_TRANSFORMED_FOLDER, options)) {
-        yield evenementsItem;
-    }
-}
-export function* iterLoadSenatEvenements(dataDir, session, options = {}) {
-    for (const evenementsItem of iterLoadSenatItems(dataDir, AGENDA_FOLDER, session, DATA_TRANSFORMED_FOLDER, options)) {
-        for (const evenement of evenementsItem.item) {
-            yield { item: evenement };
-        }
-    }
-}
 export function* iterLoadSenatAgendasGrouped(dataDir, session) {
     const baseDir = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session ?? ""));
     if (!fs.existsSync(baseDir))

package/lib/model/commission.js

@@ -200,21 +200,18 @@ export function parseCommissionCRFromFile(htmlFilePath, best, fallback) {
         }
         const raw = fs.readFileSync(htmlFilePath, "utf8");
         const $ = cheerio.load(raw, { xmlMode: false });
-        // --- champs déterminés depuis best OU fallback (aucun fallback via filename) ---
         const dateISO = best?.date ?? fallback.dateISO;
         const startTime = best?.startTime ?? hourShortToStartTime(fallback.hourShort);
         const organe = best?.organe ?? fallback?.organe ?? undefined;
-        // UIDs alignés sur makeTypeGroupUid (RUSN…) mais CR = RUSN → CRC
         const seanceRef = best?.uid ?? makeTypeGroupUid(dateISO, "COM", fallback.hourShort ?? "NA", organe);
         const uid = seanceRef.replace(/^RU/, "CRC");
         const dateSeance = toCRDate(dateISO, startTime);
-        // --- scope du jour ---
         const $dayRoot = findDayRoot($, dateISO);
         if ($dayRoot.length === 0) {
             console.warn(`[COM-CR][parse] day root not found for ${dateISO} in ${path.basename(htmlFilePath)}`);
             return null;
         }
-        // --- collecte des paragraphes/h3 jusqu’au prochain h2 ---
+        // --- Collect paragraphes/h3 until next h2 ---
         const dayParas = [];
         let $cursor = $dayRoot.next();
         while ($cursor.length && !$cursor.is("h2")) {
@@ -260,8 +257,8 @@ export function parseCommissionCRFromFile(htmlFilePath, best, fallback) {
             heureGeneration: new Date(),
         };
         return {
-            uid, // ex: CRC20240117IDC…-HHMM
-            seanceRef, // ex: RUSN20240117IDC…-HHMM
+            uid,
+            seanceRef,
             sessionRef: session,
             metadonnees,
             contenu,

package/lib/model/seance.js

@@ -22,14 +22,12 @@ export async function parseCompteRenduSlotFromFile(xmlFilePath, wantedSlot, firs
         const points = [];
         let ordre = 0;
         const addPoint = (p) => points.push({ ...p, ordre_absolu_seance: String(++ordre) });
-        // Titles
-        $("cri\\:titreS1 p.titre_S1").each((_, el) => {
-            if (!elementInAnyInterval(el, idx, intervals))
-                return;
-            const t = normalizeTitle(norm($(el).text() || ""));
-            if (t)
-                addPoint({ code_grammaire: "TITRE_TEXTE_DISCUSSION", texte: { _: t }, code_style: "Titre" });
-        });
+        // Titles removes because they are just listed at the top of the file and not linked to any ancre
+        // $("cri\\:titreS1 p.titre_S1").each((_, el) => {
+        //   if (!elementInAnyInterval(el, idx, intervals)) return
+        //   const t = normalizeTitle(norm($(el).text() || ""))
+        //   if (t) addPoint({ code_grammaire: "TITRE_TEXTE_DISCUSSION", texte: { _: t }, code_style: "Titre" })
+        // })
         // Interventions
         $("div.intervenant").each((_, block) => {
             if (!elementInAnyInterval(block, idx, intervals))

package/lib/scripts/retrieve_cr_commission.js

@@ -245,7 +245,7 @@ async function retrieveCommissionCRs(options = {}) {
                         deltaMin = candidates[0].d;
                     }
                 }
-                // Parse CR (avec ou sans best)
+                // Parse CR
                 const hourShort = toHourShort(day.openTime) ?? "NA";
                 const cr = parseCommissionCRFromFile(htmlPath, best ?? undefined, {
                     dateISO: day.date,

package/lib/scripts/retrieve_videos.d.ts

@@ -1 +1,7 @@
-export {};
+import { GroupedReunion } from "../types/agenda";
+export declare function buildSenatVodMasterM3u8FromNvs(nvsText: string, finalText: string): string | null;
+export declare function score(agenda: GroupedReunion, agendaTs: number | null, videoTitle?: string, videoEpoch?: number): number;
+/**
+ * Build search strategies for senat's videos
+ */
+export declare function buildSearchStrategies(agenda: GroupedReunion): Array<Record<string, string>>;

package/lib/scripts/retrieve_videos.js

@@ -4,12 +4,11 @@ import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 import fsp from "fs/promises";
 import path from "path";
-import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatAgendas, } from "../loaders";
+import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatAgendasGrouped } from "../loaders";
 import { getSessionsFromStart } from "../types/sessions";
 import { commonOptions } from "./shared/cli_helpers";
-import { formatYYYYMMDD, makeReunionUid } from "../utils/reunion_grouping";
 // ===================== Constants =====================
-const MATCH_THRESHOLD = 0.60;
+const MATCH_THRESHOLD = 0.6;
 const MAX_CANDIDATES = 15;
 const MAX_PAGES = 3;
 const STATS = { total: 0, accepted: 0 };
@@ -18,9 +17,7 @@ const SENAT_VIDEOS_SEARCH_AJAX = "https://videos.senat.fr/senat_videos_search.ph
 const SENAT_DATAS_ROOT = "https://videos.senat.fr/Datas/senat";
 const SENAT_VOD_HOST = "https://vodsenat.akamaized.net";
 // ===================== CLI =====================
-const optionsDefinitions = [
-    ...commonOptions,
-];
+const optionsDefinitions = [...commonOptions];
 const options = commandLineArgs(optionsDefinitions);
 // ===================== Utils =====================
 function normalize(s) {
@@ -32,7 +29,9 @@ function normalize(s) {
         .replace(/\s+/g, " ")
         .trim();
 }
-function tokens(s) { return normalize(s).split(" ").filter(Boolean); }
+function tokens(s) {
+    return normalize(s).split(" ").filter(Boolean);
+}
 function dice(a, b) {
     const A = new Set(tokens(a)), B = new Set(tokens(b));
     if (!A.size || !B.size)
@@ -46,7 +45,7 @@ function dice(a, b) {
 // Heuristic for Europe/Paris DST: +02:00 ≈ April→October, +01:00 otherwise.
 function parisOffsetForDate(dateYYYYMMDD) {
     const m = Number(dateYYYYMMDD.split("-")[1] || "1");
-    return (m >= 4 && m <= 10) ? "+02:00" : "+01:00";
+    return m >= 4 && m <= 10 ? "+02:00" : "+01:00";
 }
 function epochToParisDateTime(epochSec) {
     if (!Number.isFinite(epochSec))
@@ -54,7 +53,7 @@ function epochToParisDateTime(epochSec) {
     const dUtc = new Date(epochSec * 1000);
     // Offset heuristic (same logique que parisOffsetForDate)
     const m = dUtc.getUTCMonth() + 1; // 1..12
-    const offsetHours = (m >= 4 && m <= 10) ? 2 : 1;
+    const offsetHours = m >= 4 && m <= 10 ? 2 : 1;
     const offsetStr = offsetHours === 2 ? "+02:00" : "+01:00";
     // Applique l'offset pour obtenir la date/heure locales Paris
     const localMs = dUtc.getTime() + offsetHours * 3600 * 1000;
@@ -149,7 +148,7 @@ function extractCandidatesFromSearchHtml(html) {
         out.push({ id, hash, pageUrl, title: t?.[1] });
     }
     const seen = new Set();
-    return out.filter(c => {
+    return out.filter((c) => {
         const k = `${c.id}_${c.hash}`;
         if (seen.has(k))
             return false;
@@ -162,46 +161,68 @@ function parseDataNvs(nvs) {
     const title = nvs.match(/<metadata\s+name="title"\s+value="([^"]+)"/i)?.[1];
     return { epoch: epoch ? Number(epoch) : undefined, title };
 }
-function buildSenatVodMasterM3u8FromNvs(xml, host = SENAT_VOD_HOST) {
-    if (!xml)
+// nvsText = contenu texte de data.nvs (utf-8)
+// finalText = contenu texte de finalplayer.nvs (utf-8)
+export function buildSenatVodMasterM3u8FromNvs(nvsText, finalText) {
+    // 1) Base Akamai depuis data.nvs (mp4 "serverfiles://senat/YYYY/MM/encoderX_YYYYMMDDHHMMSS_1.mp4")
+    const baseMatch = nvsText.match(/serverfiles:\/\/senat\/(\d{4})\/(\d{2})\/(encoder\d)_(\d{14})/i);
+    if (!baseMatch)
         return null;
-    // (a) Déjà un lien VOD complet en .smil/playlist.m3u8
-    const mVod = xml.match(/https?:\/\/[^"'<>]*vodsenat[^"'<>]*\.smil\/(?:playlist|master)\.m3u8/i);
-    if (mVod)
-        return mVod[0];
-    // (b) Chemin senat/YYYY/MM/<basename>.smil  -> normalise en playlist.m3u8
-    const mSmilPath = xml.match(/senat\/(\d{4})\/(\d{2})\/([^"'<>\/]+?)\.smil/i);
-    if (mSmilPath) {
-        const [, y, m, base] = mSmilPath;
-        return `${host}/senat/${y}/${m}/${base}.smil/playlist.m3u8`;
+    const [, yyyy, mm, encoder, stamp] = baseMatch;
+    const base = `https://vodsenat.akamaized.net/senat/${yyyy}/${mm}/${encoder}_${stamp}`;
+    // 2) start/end depuis finalplayer.nvs
+    let start = null, end = null;
+    const playerAttr = finalText.match(/player[^>]*\bstarttime="(\d+)"[^>]*\bendtime="(\d+)"/i);
+    if (playerAttr) {
+        start = parseInt(playerAttr[1], 10);
+        end = parseInt(playerAttr[2], 10);
+    }
+    else {
+        // fallback: prendre le plus petit timecode des <synchro timecode="...">
+        const tc = Array.from(finalText.matchAll(/timecode="(\d+)"/g)).map((m) => parseInt(m[1], 10));
+        if (tc.length)
+            start = Math.min(...tc);
+    }
+    // 3) si pas d'end, on peut déduire via "duree" (en secondes) de data.nvs
+    if (end == null) {
+        const durMeta = nvsText.match(/<metadata[^>]*\bname="duree"[^>]*\bvalue="(\d+)"[^>]*>/i);
+        if (durMeta && start != null) {
+            const durMs = parseInt(durMeta[1], 10) * 1000; // sec → ms
+            end = start + durMs;
+        }
     }
-    // (c) Chemin senat/YYYY/MM/<basename>.mp4  -> transforme en .smil/playlist.m3u8
-    const mMp4Path = xml.match(/senat\/(\d{4})\/(\d{2})\/([^"'<>\/]+?)\.mp4/i);
-    if (mMp4Path) {
-        const [, y, m, base] = mMp4Path;
-        return `${host}/senat/${y}/${m}/${base}.smil/playlist.m3u8`;
+    // 4) Construction de l’URL
+    //    - si on a start & end → utiliser ps/pd (robuste et conforme à ce que sert le Sénat)
+    //    - sinon fallback sans suffixe (souvent valide aussi)
+    if (start != null && end != null && end > start) {
+        const pd = end - start;
+        return `${base}_ps${start}_pd${pd}.smil/master.m3u8`;
     }
-    // (d) À défaut, n’importe quel .m3u8 présent (faible priorité — peut être du live)
-    const mAny = xml.match(/https?:\/\/[^"'<>]+\.m3u8/i);
-    return mAny ? mAny[0] : null;
+    return `${base}.smil/master.m3u8`;
 }
-function score(agenda, agendaTs, videoTitle, videoEpoch) {
+export function score(agenda, agendaTs, videoTitle, videoEpoch) {
     const titleScore = dice(agenda.titre || "", videoTitle || "");
     let timeScore = 0;
     if (agendaTs && videoEpoch) {
+        // second
         const deltaMin = Math.abs(videoEpoch - agendaTs) / 60;
-        timeScore = Math.max(0, 1 - (deltaMin / 180));
+        // delta : 180min
+        timeScore = Math.max(0, 1 - deltaMin / 180);
     }
     let orgBonus = 0;
     if (agenda.organe && videoTitle) {
         const o = normalize(agenda.organe);
         const t = normalize(videoTitle);
-        if (o && t.includes(o.split(" ")[0]))
+        const first = o.split(" ").filter(Boolean)[0];
+        if (first && t.includes(first))
             orgBonus = 0.15;
     }
-    return 0.3 * titleScore + 0.7 * timeScore + orgBonus;
+    return 0.3 * titleScore + 0.7 * timeScore + orgBonus; // Can be adjusted
 }
-function buildSearchStrategies(agenda) {
+/**
+ * Build search strategies for senat's videos
+ */
+export function buildSearchStrategies(agenda) {
     const fr = agenda.date ? toFRDate(agenda.date) : undefined;
     const kw = simplifyTitleForKeywords(agenda.titre || "");
     const commission = agenda.organe || undefined;
@@ -239,21 +260,23 @@ async function fetchAllSearchPages(args, baseDir, strategyIndex, maxPages = MAX_
     }
     return pages;
 }
-async function processAgenda(agenda, session, dataDir) {
+async function processGroupedReunion(agenda, session, dataDir) {
     if (!agenda)
         return;
+    // 1) Garde-fous
     if (!agenda.captationVideo) {
         if (!options["silent"])
-            console.log(`[skip] ${agenda.id} captationVideo=false`);
+            console.log(`[skip] ${agenda.uid} captationVideo=false`);
         return;
     }
     if (!agenda.date || !agenda.startTime) {
         if (!options["silent"])
-            console.log(`[skip] ${agenda.id} date/hour missing`);
+            console.log(`[skip] ${agenda.uid} date/hour missing`);
         return;
     }
     STATS.total++;
-    const reunionUid = makeReunionUid(agenda);
+    // 2) Dossier de sortie (utilise directement l'UID)
+    const reunionUid = agenda.uid;
     const baseDir = path.join(dataDir, VIDEOS_ROOT_FOLDER, String(session), reunionUid);
     await fs.ensureDir(baseDir);
     const agendaTs = toTargetEpoch(agenda.date, agenda.startTime);
@@ -276,8 +299,9 @@ async function processAgenda(agenda, session, dataDir) {
         }
     }
     if (usedStrategy === -1 || !candidates.length) {
-        if (!options["silent"])
-            console.log(`[miss] ${agenda.id} no candidates (triedStrategies=${strategies.length})`);
+        if (!options["silent"]) {
+            console.log(`[miss] ${agenda.uid} no candidates (triedStrategies=${strategies.length})`);
+        }
         return;
     }
     // ==== 2) Enrich via data.nvs + scoring; pick best ====
@@ -295,14 +319,14 @@ async function processAgenda(agenda, session, dataDir) {
     }
     if (!best) {
         if (!options["silent"])
-            console.log(`[miss] ${agenda.id} candidats without data.nvs`);
+            console.log(`[miss] ${agenda.uid} candidates without data.nvs`);
         return;
     }
     const accepted = best.score >= MATCH_THRESHOLD;
     if (accepted)
         STATS.accepted++;
     if (!options["silent"]) {
-        console.log(`[pick] ${agenda.id} best id=${best.id} hash=${best.hash} score=${best.score.toFixed(2)} accepted=${accepted} (strategy=${usedStrategy})`);
+        console.log(`[pick] ${agenda.uid} best id=${best.id} hash=${best.hash} score=${best.score.toFixed(2)} accepted=${accepted} (strategy=${usedStrategy})`);
     }
     // ==== 3) Write metadata + NVS of the best candidate (always) ====
     const bestDt = best?.epoch ? epochToParisDateTime(best.epoch) : null;
@@ -317,7 +341,7 @@ async function processAgenda(agenda, session, dataDir) {
             startTime: agenda.startTime,
             titre: agenda.titre,
             organe: agenda.organe ?? undefined,
-            id: agenda.id,
+            uid: agenda.uid,
         },
         best: {
             id: best.id,
@@ -340,37 +364,30 @@ async function processAgenda(agenda, session, dataDir) {
     if (finalTxt)
         await fsp.writeFile(path.join(baseDir, "finalplayer.nvs"), finalTxt, "utf-8");
     let master = null;
-    if (dataTxt)
-        master = buildSenatVodMasterM3u8FromNvs(dataTxt);
+    if (dataTxt && finalTxt)
+        master = buildSenatVodMasterM3u8FromNvs(dataTxt, finalTxt);
     // ==== 4) Update agenda file (only if accepted + m3u8) ====
     if (accepted && master) {
-        const agendaJsonPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session), `${formatYYYYMMDD(agenda.date)}.json`);
+        const agendaJsonPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session), `${agenda.uid}.json`);
         if (await fs.pathExists(agendaJsonPath)) {
             const raw = await fsp.readFile(agendaJsonPath, "utf-8");
-            let items;
+            let obj;
             try {
-                items = JSON.parse(raw);
+                obj = JSON.parse(raw);
             }
             catch (e) {
                 console.warn(`[warn] invalid JSON in ${agendaJsonPath}:`, e?.message);
-                items = null;
+                obj = null;
             }
-            if (Array.isArray(items)) {
-                const idx = items.findIndex((e) => String(e?.id) === String(agenda.id));
-                if (idx === -1) {
-                    console.warn(`[warn] agenda id ${agenda.id} not found in ${agendaJsonPath}`);
-                }
-                else {
-                    // add/update urlVideo on the matching item
-                    items[idx] = { ...items[idx], urlVideo: master };
-                    await writeIfChanged(agendaJsonPath, JSON.stringify(items, null, 2));
-                    if (!options["silent"]) {
-                        console.log(`[write] ${agenda.id} urlVideo ← ${master}`);
-                    }
+            if (obj && typeof obj === "object" && !Array.isArray(obj)) {
+                const next = { ...obj, urlVideo: master };
+                await writeIfChanged(agendaJsonPath, JSON.stringify(next, null, 2));
+                if (!options["silent"]) {
+                    console.log(`[write] ${agenda.uid} urlVideo ← ${master}`);
                 }
             }
             else {
-                console.warn(`[warn] expected an array in ${agendaJsonPath}, got ${typeof items}`);
+                console.warn(`[warn] expected an object in ${agendaJsonPath}, got ${Array.isArray(obj) ? "array" : typeof obj}`);
             }
         }
         else {
@@ -379,15 +396,14 @@ async function processAgenda(agenda, session, dataDir) {
     }
 }
 async function processAll(dataDir, sessions) {
+    console.log("Process all Agendas and fetch video's url");
     for (const session of sessions) {
-        for (const { item: agendas } of iterLoadSenatAgendas(dataDir, session, {})) {
-            for (const agenda of agendas) {
-                try {
-                    await processAgenda(agenda, session, dataDir);
-                }
-                catch (e) {
-                    console.error(`[error] ${agenda.id}:`, e?.message || e);
-                }
+        for (const { item: agenda } of iterLoadSenatAgendasGrouped(dataDir, session)) {
+            try {
+                await processGroupedReunion(agenda, session, dataDir);
+            }
+            catch (e) {
+                console.error(`[error] ${agenda?.uid ?? "unknown-uid"}:`, e?.message || e);
             }
         }
     }
@@ -396,17 +412,16 @@ async function main() {
     const dataDir = options["dataDir"];
     assert(dataDir, "Missing argument: data directory");
     const sessions = getSessionsFromStart(options["fromSession"]);
-    if (!options["silent"])
-        console.time("senat-agendas→videos start processing time");
+    console.time("senat-agendas→videos start processing time");
     await processAll(dataDir, sessions);
-    if (!options["silent"])
-        console.timeEnd("senat-agendas→videos processing time");
-    if (!options["silent"]) {
-        const { total, accepted } = STATS;
-        const ratio = total ? (accepted / total * 100).toFixed(1) : "0.0";
-        console.log(`[summary] accepted=${accepted} / total=${total} (${ratio}%)`);
-    }
+    console.timeEnd("senat-agendas→videos processing time");
+    const { total, accepted } = STATS;
+    const ratio = total ? ((accepted / total) * 100).toFixed(1) : "0.0";
+    console.log(`[summary] accepted=${accepted} / total=${total} (${ratio}%)`);
 }
 main()
     .then(() => process.exit(0))
-    .catch((err) => { console.error(err); process.exit(1); });
+    .catch((err) => {
+    console.error(err);
+    process.exit(1);
+});

package/lib/utils/cr_spliting.js

@@ -337,6 +337,7 @@ export async function linkCRtoCommissionGroup(opts) {
         console.warn(`[AGENDA][COM] Unreadable JSON → ${filePath} (${e?.message}) → will recreate`);
     }
     if (!group) {
+        // FIX : fix the way groups are found because it creates doublons
         // group = {
         //   uid,
         //   chambre: "SN",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.13.0",
+  "version": "2.13.2",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",

package/lib/model/compte_rendu.d.ts

@@ -1,9 +0,0 @@
-import { CompteRendu, Sommaire } from "../types/compte_rendu";
-import { TimeSlot } from "../types/agenda";
-export declare function parseCompteRenduSlotFromFile(xmlFilePath: string, wantedSlot: TimeSlot, firstSlotOfDay?: TimeSlot): Promise<CompteRendu | null>;
-export declare function sessionStartYearFromDate(d: Date): number;
-export declare function parseYYYYMMDD(yyyymmdd: string): Date | null;
-export declare function deriveTitreObjetFromSommaire(sommaire: Sommaire | undefined, slot: TimeSlot): {
-    titre: string;
-    objet: string;
-};