@tricoteuses/senat 2.16.5 → 2.16.7

package/lib/loaders.d.ts

@@ -29,6 +29,7 @@ export type IterItem<T> = {
 export interface TexteMetadata {
     name: string;
     session: number | null | undefined;
+    date?: string | null;
     url_expose_des_motifs?: URL;
     url_xml: URL;
     url_html: URL;
@@ -37,6 +38,7 @@ export interface TexteMetadata {
 export interface RapportMetadata {
     name: string;
     session: number | null | undefined;
+    date?: string | null;
     url_html: URL;
     url_pdf: URL;
 }

package/lib/model/agenda.d.ts

@@ -1,2 +1,6 @@
 import { AgendaEvent } from "../types/agenda";
+export declare function getStartAndEndTimes(timeStr: string | null | undefined, dateISO: string): {
+    startTime: string | null;
+    endTime: string | null;
+};
 export declare function parseAgendaFromFile(htmlFilePath: string): Promise<AgendaEvent[] | null>;

package/lib/model/agenda.js

@@ -64,18 +64,37 @@ function normalizeTime(timeStr) {
         ?.replace(/\s\(hors hémicycle\)/i, "")
         ?.replace(/\s*h\s*/gi, "h");
 }
-function getStartAndEndTimes(timeStr) {
+export function getStartAndEndTimes(timeStr, dateISO) {
     const normalizedTime = normalizeTime(timeStr);
-    const timeMatches = normalizedTime?.match(/^de (?<startTime>\d{2}h\d{2}) à (?<endTime>\d{2}h\d{2})$/i);
-    if (timeMatches?.groups) {
-        const { startTime, endTime } = timeMatches.groups;
+    if (!normalizedTime) {
+        return { startTime: null, endTime: null };
+    }
+    const rangeMatch = normalizedTime.match(/^de (?<start>\d{1,2}h\d{2}) à (?<end>\d{1,2}h\d{2})$/i);
+    const toUtcIso = (value) => {
+        if (!value)
+            return null;
+        const time = DateTime.fromFormat(value, "H'h'mm", { zone: FR_TZ });
+        if (!time.isValid)
+            return null;
+        const local = DateTime.fromISO(dateISO, { zone: FR_TZ }).set({
+            hour: time.hour,
+            minute: time.minute,
+            second: 0,
+            millisecond: 0,
+        });
+        if (!local.isValid)
+            return null;
+        return local.toUTC().toISO();
+    };
+    if (rangeMatch?.groups) {
+        const { start, end } = rangeMatch.groups;
         return {
-            startTime: startTime ? DateTime.fromFormat(startTime, "H'h'mm", { zone: FR_TZ }).toISOTime() : null,
-            endTime: endTime ? DateTime.fromFormat(endTime, "H'h'mm", { zone: FR_TZ }).toISOTime() : null,
+            startTime: toUtcIso(start),
+            endTime: toUtcIso(end),
         };
     }
     return {
-        startTime: normalizedTime ? DateTime.fromFormat(normalizedTime, "H'h'mm", { zone: FR_TZ }).toISOTime() : null,
+        startTime: toUtcIso(normalizedTime),
         endTime: null,
     };
 }
@@ -91,7 +110,7 @@ function transformAgenda(document, fileName) {
         const type = getEventType(eventElement.classList);
         const date = DateTime.fromFormat(fileName, ID_DATE_FORMAT).toFormat(STANDARD_DATE_FORMAT);
         const timeOriginal = eventElement.querySelector(".time")?.textContent || null;
-        const { startTime, endTime } = getStartAndEndTimes(timeOriginal);
+        const { startTime, endTime } = getStartAndEndTimes(timeOriginal, date);
         const titre = eventElement.querySelector(".titre")?.textContent?.trim() || "";
         const organe = eventElement.querySelector(".organe")?.textContent?.trim() || null;
         const objet = eventElement.querySelector(".objet")?.textContent?.trim()?.replace(/^- /, "") || null;

package/lib/model/documents.js

@@ -1,5 +1,5 @@
 import { dbSenat } from "../databases";
-import { rtrim } from "./util";
+import { rtrim, toDateString } from "./util";
 export function findSenatTexteUrls(sessions = []) {
     return dbSenat
         .withSchema("dosleg")
@@ -10,13 +10,8 @@ export function findSenatTexteUrls(sessions = []) {
         .select(({ eb, ref }) => [
         "sesann as session",
         rtrim(ref("texurl")).as("url"),
-        eb
-            .case()
-            .when("oritxtcod", "=", "1")
-            .then(true)
-            .else(false)
-            .end()
-            .as("hasExposeDesMotifs"),
+        toDateString(ref("txtoritxtdat")).as("date"),
+        eb.case().when("oritxtcod", "=", "1").then(true).else(false).end().as("hasExposeDesMotifs"),
     ])
         .$narrowType()
         .stream();
@@ -31,6 +26,7 @@ export function findSenatRapportUrls(sessions = []) {
         .select(({ ref }) => [
         "sesann as session",
         rtrim(ref("rapurl")).as("url"),
+        toDateString(ref("date_depot")).as("date"),
     ])
         .$narrowType()
         .stream();

package/lib/scripts/convert_data.js

@@ -178,6 +178,7 @@ async function convertTexteUrls(dataDir) {
         const metadata = {
             name: texteName,
             session: texte.session,
+            date: texte.date,
             url_expose_des_motifs: texte.hasExposeDesMotifs
                 ? new URL(`${texteName}-expose.html`, SENAT_EXPOSE_DES_MOTIFS_BASE_URL)
                 : undefined,
@@ -212,6 +213,7 @@ async function convertRapportUrls(dataDir) {
         const metadata = {
             name: rapportName,
             session: rapport.session,
+            date: rapport.date,
             url_html: new URL(rapportHtmlUrl, SENAT_RAPPORT_BASE_URL),
             url_pdf: new URL(rapportPdfUrl, SENAT_RAPPORT_BASE_URL),
         };

package/lib/scripts/retrieve_documents.js

@@ -1,14 +1,16 @@
 import assert from "assert";
 import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
+import { DateTime } from "luxon";
 import path from "path";
 import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatDossiersLegislatifsRapportUrls, iterLoadSenatDossiersLegislatifsTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, } from "../loaders";
 import { parseExposeDesMotifs, parseTexte, parseTexteFromFile } from "../model/texte";
 import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions";
-import { commonOptions } from "./shared/cli_helpers";
+import { commonOptions, onlyRecentOption } from "./shared/cli_helpers";
 import { ensureAndClearDir, fetchWithRetry, isOptionEmptyOrHasValue } from "./shared/util";
 const optionsDefinitions = [
     ...commonOptions,
+    onlyRecentOption,
     {
         help: "parse and convert documents into JSON (textes only for now, requires format xml)",
         name: "parseDocuments",
@@ -35,6 +37,18 @@ const optionsDefinitions = [
 ];
 const options = commandLineArgs(optionsDefinitions);
 const textDecoder = new TextDecoder("utf8");
+const today = DateTime.now();
+function isDocumentRecent(documentDate, daysThreshold) {
+    if (!documentDate) {
+        return false;
+    }
+    const docDate = DateTime.fromISO(documentDate);
+    if (!docDate.isValid) {
+        return false;
+    }
+    const daysDiff = today.diff(docDate, "days").days;
+    return daysDiff <= daysThreshold;
+}
 async function retrieveTextes(dataDir, sessions) {
     const textesDir = path.join(dataDir, TEXTE_FOLDER);
     fs.ensureDirSync(textesDir);
@@ -47,7 +61,7 @@ async function retrieveTextes(dataDir, sessions) {
     const texteUrlsNotFoundOrError = [];
     const texteUrlsParseError = [];
     for (const session of sessions) {
-        for (const { item: texteMetadata, } of iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session)) {
+        for (const { item: texteMetadata } of iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session)) {
             const texteDir = path.join(originalTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name);
             fs.ensureDirSync(texteDir);
             let exposeDesMotifsContent = null;
@@ -57,7 +71,11 @@ async function retrieveTextes(dataDir, sessions) {
             if (isOptionEmptyOrHasValue(options["formats"], "xml")) {
                 const textePath = path.join(texteDir, `${texteMetadata.name}.xml`);
                 let texteBuffer = null;
-                if (!options["force"] && fs.existsSync(textePath)) {
+                // Check if document should be skipped based on onlyRecent option
+                const shouldSkip = !options["force"] &&
+                    fs.existsSync(textePath) &&
+                    (options["only-recent"] === undefined || !isDocumentRecent(texteMetadata.date, options["only-recent"]));
+                if (shouldSkip) {
                     if (!options["silent"]) {
                         console.info(`Already downloaded texte ${textePath}…`);
                     }
@@ -80,7 +98,11 @@ async function retrieveTextes(dataDir, sessions) {
             }
             if (isOptionEmptyOrHasValue(options["formats"], "html")) {
                 const textePath = path.join(texteDir, `${texteMetadata.name}.html`);
-                if (!options["force"] && fs.existsSync(textePath)) {
+                // Check if document should be skipped based on onlyRecent option
+                const shouldSkip = !options["force"] &&
+                    fs.existsSync(textePath) &&
+                    (options["only-recent"] === undefined || !isDocumentRecent(texteMetadata.date, options["only-recent"]));
+                if (shouldSkip) {
                     if (!options["silent"]) {
                         console.info(`Already downloaded texte ${textePath}…`);
                     }
@@ -97,7 +119,11 @@ async function retrieveTextes(dataDir, sessions) {
             }
             if (isOptionEmptyOrHasValue(options["formats"], "pdf")) {
                 const textePath = path.join(texteDir, `${texteMetadata.name}.pdf`);
-                if (!options["force"] && fs.existsSync(textePath)) {
+                // Check if document should be skipped based on onlyRecent option
+                const shouldSkip = !options["force"] &&
+                    fs.existsSync(textePath) &&
+                    (options["only-recent"] === undefined || !isDocumentRecent(texteMetadata.date, options["only-recent"]));
+                if (shouldSkip) {
                     if (!options["silent"]) {
                         console.info(`Already downloaded texte ${textePath}…`);
                     }
@@ -129,12 +155,16 @@ async function retrieveRapports(dataDir, sessions) {
     let retrievedRapportsCount = 0;
     const rapportUrlsNotFoundOrError = [];
     for (const session of sessions) {
-        for (const { item: rapportMetadata, } of iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session)) {
+        for (const { item: rapportMetadata } of iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session)) {
             const rapportDir = path.join(originalRapportsDir, `${rapportMetadata.session ?? UNDEFINED_SESSION}`, rapportMetadata.name);
             fs.ensureDirSync(rapportDir);
             if (isOptionEmptyOrHasValue(options["formats"], "html")) {
                 const rapportPath = path.join(rapportDir, `${rapportMetadata.name}.html`);
-                if (!options["force"] && fs.existsSync(rapportPath)) {
+                // Check if document should be skipped based on onlyRecent option
+                const shouldSkip = !options["force"] &&
+                    fs.existsSync(rapportPath) &&
+                    (options["only-recent"] === undefined || !isDocumentRecent(rapportMetadata.date, options["only-recent"]));
+                if (shouldSkip) {
                     if (!options["silent"]) {
                         console.info(`Already downloaded rapport ${rapportPath}…`);
                     }
@@ -150,7 +180,11 @@ async function retrieveRapports(dataDir, sessions) {
             }
             if (isOptionEmptyOrHasValue(options["formats"], "pdf")) {
                 const rapportPath = path.join(rapportDir, `${rapportMetadata.name}.pdf`);
-                if (!options["force"] && fs.existsSync(rapportPath)) {
+                // Check if document should be skipped based on onlyRecent option
+                const shouldSkip = !options["force"] &&
+                    fs.existsSync(rapportPath) &&
+                    (options["only-recent"] === undefined || !isDocumentRecent(rapportMetadata.date, options["only-recent"]));
+                if (shouldSkip) {
                     if (!options["silent"]) {
                         console.info(`Already downloaded rapport ${rapportPath}…`);
                     }
@@ -222,8 +256,7 @@ async function parseDocument(session, transformedTextesDir, textePath, texteName
             console.log("Parsing exposé des motifs…");
         }
         const exposeDesMotifsHtml = textDecoder.decode(exposeDesMotifs);
-        parsedTexte.exposeDesMotifs =
-            parseExposeDesMotifs(exposeDesMotifsHtml);
+        parsedTexte.exposeDesMotifs = parseExposeDesMotifs(exposeDesMotifsHtml);
     }
     const transformedTexteDir = path.join(transformedTextesDir, `${session ?? UNDEFINED_SESSION}`, texteName);
     fs.ensureDirSync(transformedTexteDir);

package/lib/scripts/retrieve_videos.d.ts

@@ -1,5 +1 @@
-import { GroupedReunion } from "../types/agenda";
-/**
- * Build search strategies for senat's videos
- */
-export declare function buildSearchStrategies(agenda: GroupedReunion): Array<Record<string, string>>;
+export {};

package/lib/scripts/retrieve_videos.js

@@ -8,10 +8,10 @@ import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatAgendas } from "..
 import { getSessionsFromStart } from "../types/sessions";
 import { commonOptions } from "./shared/cli_helpers";
 import { decodeHtmlEntities } from "../model/util";
+import { DateTime } from "luxon";
 // ===================== Constants =====================
-const MATCH_THRESHOLD = 0.5;
+const MATCH_THRESHOLD = 0.6;
 const MAX_CANDIDATES = 15;
-const MAX_PAGES = 3;
 const STATS = { total: 0, accepted: 0 };
 const VIDEOS_ROOT_FOLDER = "videos";
 const SENAT_VIDEOS_SEARCH_AJAX = "https://videos.senat.fr/senat_videos_search.php";
@@ -42,11 +42,6 @@ function dice(a, b) {
             inter++;
     return (2 * inter) / (A.size + B.size);
 }
-// Heuristic for Europe/Paris DST: +02:00 ≈ April→October, +01:00 otherwise.
-function parisOffsetForDate(dateYYYYMMDD) {
-    const m = Number(dateYYYYMMDD.split("-")[1] || "1");
-    return m >= 4 && m <= 10 ? "+02:00" : "+01:00";
-}
 function epochToParisDateTime(epochSec) {
     if (!Number.isFinite(epochSec))
         return null;
@@ -70,30 +65,13 @@ function epochToParisDateTime(epochSec) {
         startTime: `${hh}:${mi}:${ss}.${ms}${offsetStr}`,
     };
 }
-function toTargetEpoch(date, time) {
-    if (!date)
+function toTargetEpoch(time) {
+    if (!time)
         return null;
-    let t = (time ?? "00:00").trim();
-    // Si l'heure contient déjà un fuseau (Z ou ±HH:MM), on la fait simplement précéder de la date.
-    const hasTz = /(?:Z|[+-]\d{2}:\d{2})$/i.test(t);
-    let iso;
-    if (hasTz) {
-        // Exemple: 2022-10-04T18:00:00.000+02:00
-        iso = `${date}T${t}`;
-    }
-    else {
-        // Normalise pour avoir au moins HH:mm:ss
-        if (/^\d{1,2}$/.test(t)) {
-            t = `${t.padStart(2, "0")}:00:00`;
-        }
-        else if (/^\d{1,2}:\d{2}$/.test(t)) {
-            t = `${t}:00`;
-        } // sinon, on garde tel quel (gère HH:mm:ss et HH:mm:ss.SSS)
-        // Ajoute l’offset Paris (heuristique saisonnière)
-        iso = `${date}T${t}${parisOffsetForDate(date)}`;
-    }
-    const ms = Date.parse(iso);
-    return Number.isNaN(ms) ? null : Math.floor(ms / 1000);
+    const dtLocal = DateTime.fromISO(time, { zone: "Europe/Paris" });
+    if (!dtLocal.isValid)
+        return null;
+    return Math.floor(dtLocal.toUTC().toSeconds());
 }
 async function fetchText(url) {
     const res = await fetch(url);
@@ -122,16 +100,6 @@ function queryString(obj) {
         .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
         .join("&");
 }
-function simplifyTitleForKeywords(input) {
-    return (input || "")
-        .replace(/\baudition\s+de\b/gi, " ")
-        .replace(/\breunion\b/gi, " ")
-        .replace(/\bsur\b/gi, " ")
-        .replace(/\b(la|le|les|des|de|du|d’|d')\b/gi, " ")
-        .replace(/[–—-]/g, " ")
-        .replace(/\s+/g, " ")
-        .trim();
-}
 function toFRDate(dateYYYYMMDD) {
     const [y, m, d] = dateYYYYMMDD.split("-");
     return `${d}/${m}/${y}`; // DD/MM/YYYY
@@ -158,17 +126,24 @@ function extractCandidatesFromSearchHtml(html) {
         return true;
     });
 }
+function parseFinalNvs(nvs) {
+    const playerTag = nvs.match(/<player\b[^>]*>/i)?.[0];
+    if (!playerTag)
+        return {};
+    const sessionStartStr = playerTag.match(/\bsessionstart="(\d+)"/i)?.[1];
+    return {
+        sessionStart: sessionStartStr ? Number(sessionStartStr) : undefined,
+    };
+}
 function parseDataNvs(nvs) {
     const epochStr = nvs.match(/<metadata\s+name="date"\s+value="(\d+)"/i)?.[1];
     const epoch = epochStr ? Number(epochStr) : undefined;
     const organesTag = nvs.match(/<metadata\b[^>]*\bname="organes"[^>]*>/i)?.[0];
     let organeLabel;
-    let organeValue;
     if (organesTag) {
         organeLabel = organesTag.match(/\blabel="([^"]+)"/i)?.[1];
-        organeValue = organesTag.match(/\bvalue="([^"]+)"/i)?.[1];
     }
-    const organeRaw = organeLabel ?? organeValue;
+    const organeRaw = organeLabel ?? "Séance publique";
     const organe = decodeHtmlEntities(organeRaw)?.trim();
     const firstChapterLabel = decodeHtmlEntities(nvs.match(/<chapter\b[^>]*\blabel="([^"]+)"/i)[1]).trim();
     return { epoch, organe, firstChapterLabel };
@@ -207,50 +182,27 @@ function buildSenatVodMasterM3u8FromNvs(nvsText, finalText) {
     }
     return `${base}.smil/master.m3u8`;
 }
-function score(agenda, agendaTs, videoTitle, videoEpoch, videoOrgane) {
-    const titleScore = dice(agenda.titre || "", videoTitle || "");
+function score(agenda, agendaTs, sameOrg, videoTitle, videoEpoch, videoOrgane) {
+    const objetS = dice(agenda.objet || "", videoTitle || "");
+    const titleS = dice(agenda.titre || "", videoTitle || "");
+    const titleScore = Math.max(objetS, titleS);
     let timeScore = 0;
     if (agendaTs && videoEpoch) {
         // second
         const deltaMin = Math.abs(videoEpoch - agendaTs) / 60;
-        // delta : 180min
-        timeScore = Math.max(0, 1 - deltaMin / 180);
+        // delta : 60min
+        timeScore = Math.exp(-deltaMin / 60);
     }
     const orgScore = videoOrgane && agenda.organe ? dice(agenda.organe, videoOrgane) : 0;
-    if (orgScore === 0 && agenda.organe === "Séance publique") {
-        return 0.3 * titleScore + 0.7 * timeScore;
-    }
-    return 0.2 * titleScore + 0.4 * timeScore + orgScore * 0.4;
+    return 0.2 * titleScore + 0.4 * timeScore + (sameOrg ? 0.4 : orgScore * 0.4);
 }
-/**
- * Build search strategies for senat's videos
- */
-export function buildSearchStrategies(agenda) {
-    const fr = agenda.date ? toFRDate(agenda.date) : undefined;
-    const kw = simplifyTitleForKeywords(agenda.titre || "");
-    const commission = agenda.organe || undefined;
-    // common base
-    const base = { search: "true", videotype: "Commission" };
-    if (fr)
-        Object.assign(base, { period: "custom", begin: fr, end: fr });
-    const strategies = [];
-    // 1) keywords + commission
-    if (kw && commission)
-        strategies.push({ ...base, motscles: kw, commission });
-    // 2) keywords without commission
-    if (kw)
-        strategies.push({ ...base, motscles: kw });
-    // 3) full-text (AND) + commission
-    if (kw && commission)
-        strategies.push({ ...base, text: `AND${kw}`, commission });
-    // 4) full-text (AND) without commission
-    if (kw)
-        strategies.push({ ...base, text: `AND${kw}` });
-    // 5) no keywords (just type + period)
-    strategies.push({ ...base });
-    return strategies;
+function getAgendaType(agenda) {
+    const o = agenda.organe || "";
+    if (/séance publique/i.test(o))
+        return "Séance publique";
+    return "Commission";
 }
-async function fetchAllSearchPages(args, baseDir, strategyIndex, maxPages = MAX_PAGES) {
+async function fetchAllSearchPages(args, maxPages = 3) {
     const pages = [];
     for (let p = 1; p <= maxPages; p++) {
         const url = `${SENAT_VIDEOS_SEARCH_AJAX}?${queryString({ ...args, page: String(p) })}`;
@@ -263,9 +215,46 @@ async function fetchAllSearchPages(args, baseDir, strategyIndex, maxPages = MAX_
     }
     return pages;
 }
+function getOrgKey(norm) {
+    if (!norm)
+        return "autre";
+    if (norm.includes("seance publique"))
+        return "seance_publique";
+    if (norm.includes("culture"))
+        return "culture";
+    if (norm.includes("finances"))
+        return "finances";
+    if (norm.includes("sociales"))
+        return "affaires_sociales";
+    if (norm.includes("economiques"))
+        return "affaires_economiques";
+    if (norm.includes("europeennes"))
+        return "affaires_europeennes";
+    if (norm.includes("etrangeres") || norm.includes("forces armees") || norm.includes("defense")) {
+        return "affaires_etrangeres_defense";
+    }
+    if (norm.includes("territoire") || norm.includes("durable")) {
+        return "amenagement_territoire_dd";
+    }
+    if (norm.includes("commission des lois"))
+        return "lois";
+    if (norm.includes("delegation aux collectivites territoriales") || norm.includes("delegation a la decentralisation"))
+        return "delegation_collectivites";
+    if (norm.includes("delegation aux droits des femmes") ||
+        norm.includes("egalite des chances entre les hommes et les femmes"))
+        return "delegation_droits_femmes";
+    if (norm.includes("delegation aux entreprises"))
+        return "delegation_entreprises";
+    if (norm.includes("delegation senatoriale aux outre mer") || norm.includes("delegation aux outre mer"))
+        return "delegation_outre_mer";
+    if (norm.includes("delegation a la prospective"))
+        return "delegation_prospective";
+    if (norm.includes("office parlementaire d evaluation des choix scientifiques et technologiques") ||
+        norm.includes("opecst"))
+        return "opecst";
+    return "autre";
+}
 async function processGroupedReunion(agenda, session, dataDir) {
-    if (!agenda)
-        return;
     // 1) GuardRails
     if (!agenda.captationVideo) {
         // if (!options["silent"]) console.log(`[skip] ${agenda.uid} captationVideo=false`)
@@ -275,32 +264,40 @@ async function processGroupedReunion(agenda, session, dataDir) {
         // if (!options["silent"]) console.log(`[skip] ${agenda.uid} date/hour missing`)
         return;
     }
+    const agendaTs = toTargetEpoch(agenda.startTime);
+    const now = Date.now();
+    if (agendaTs && agendaTs * 1000 > now) {
+        return;
+    }
     STATS.total++;
     const reunionUid = agenda.uid;
     const baseDir = path.join(dataDir, VIDEOS_ROOT_FOLDER, String(session), reunionUid);
     await fs.ensureDir(baseDir);
-    const agendaTs = toTargetEpoch(agenda.date, agenda.startTime);
-    // ==== 1) Multi-strategy searches ====
-    const strategies = buildSearchStrategies(agenda);
-    let combinedHtml = "";
-    let usedStrategy = -1;
-    let candidates = [];
-    for (let i = 0; i < strategies.length; i++) {
-        const pages = await fetchAllSearchPages(strategies[i], baseDir, i + 1, MAX_PAGES);
-        if (pages.length === 0)
-            continue;
-        const combined = pages.join("\n<!-- PAGE SPLIT -->\n");
-        const cs = extractCandidatesFromSearchHtml(combined);
-        if (cs.length) {
-            combinedHtml = combined;
-            candidates = cs.slice(0, MAX_CANDIDATES);
-            usedStrategy = i + 1;
-            break;
+    const searchParams = {
+        search: "true",
+        videotype: getAgendaType(agenda),
+    };
+    if (agenda.date) {
+        const fr = toFRDate(agenda.date);
+        searchParams.period = "custom";
+        searchParams.begin = fr;
+        searchParams.end = fr;
+    }
+    if (agenda.organe) {
+        searchParams.organe = agenda.organe;
+    }
+    const pages = await fetchAllSearchPages(searchParams);
+    if (!pages.length) {
+        if (!options["silent"]) {
+            console.log(`[miss] ${agenda.uid} no candidates (videotype=${searchParams.videotype}, organe=${searchParams.organe || "-"}, date=${searchParams.begin || "-"})`);
         }
+        return;
     }
-    if (usedStrategy === -1 || !candidates.length) {
+    const combinedHtml = pages.join("\n<!-- PAGE SPLIT -->\n");
+    const candidates = extractCandidatesFromSearchHtml(combinedHtml).slice(0, MAX_CANDIDATES);
+    if (!candidates.length) {
         if (!options["silent"]) {
-            console.log(`[miss] ${agenda.uid} no candidates (triedStrategies=${strategies.length})`);
+            console.log(`[miss] ${agenda.uid} no candidates after parse (videotype=${searchParams.videotype}, organe=${searchParams.organe || "-"}, date=${searchParams.begin || "-"})`);
         }
         return;
     }
@@ -308,15 +305,32 @@ async function processGroupedReunion(agenda, session, dataDir) {
     let best = null;
     for (const c of candidates) {
         const dataUrl = `${SENAT_DATAS_ROOT}/${c.id}_${c.hash}/content/data.nvs`;
-        const buf = await fetchBuffer(dataUrl);
-        if (!buf)
+        const finalUrl = `${SENAT_DATAS_ROOT}/${c.id}_${c.hash}/content/finalplayer.nvs`;
+        const dataBuf = await fetchBuffer(dataUrl);
+        if (!dataBuf)
             continue;
-        const meta = parseDataNvs(buf.toString("utf-8"));
-        // If organes are different, go to next candidates
+        const meta = parseDataNvs(dataBuf.toString("utf-8"));
+        let sessionStart;
+        const finalBuf = await fetchBuffer(finalUrl);
+        if (finalBuf) {
+            const finalMeta = parseFinalNvs(finalBuf.toString("utf-8"));
+            sessionStart = finalMeta.sessionStart;
+        }
+        const videoEpoch = sessionStart ?? meta.epoch;
+        let sameOrg = false;
+        // If organes are too different, go to next candidates
         if (meta.organe && agenda.organe) {
             const videoOrgNorm = normalize(meta.organe);
             const agendaOrgNorm = normalize(agenda.organe);
-            if (dice(agendaOrgNorm, videoOrgNorm) < 0.5) {
+            const videoKey = getOrgKey(videoOrgNorm);
+            const agendaKey = getOrgKey(agendaOrgNorm);
+            const d = dice(agendaOrgNorm, videoOrgNorm);
+            if (videoKey === agendaKey && videoKey !== "autre") {
+                // same org we keep it
+                sameOrg = true;
+            }
+            else if (d < 0.7) {
+                // if diff org and dice too low we skip
                 continue;
             }
         }
@@ -324,7 +338,7 @@ async function processGroupedReunion(agenda, session, dataDir) {
         if (c.isSeancePublique && meta.firstChapterLabel) {
             videoTitle = meta.firstChapterLabel;
         }
-        const s = score(agenda, agendaTs, videoTitle, meta.epoch, meta.organe);
+        const s = score(agenda, agendaTs, sameOrg, videoTitle, videoEpoch, meta.organe);
         if (!best || s > best.score) {
             best = {
                 id: c.id,
@@ -339,17 +353,17 @@ async function processGroupedReunion(agenda, session, dataDir) {
     }
     if (!best) {
         if (!options["silent"])
-            console.log(`[miss] ${agenda.uid} candidates without data.nvs`);
+            console.log(`[miss] ${agenda.uid} No candidate found for this reunion`);
         return;
     }
     const accepted = best.score >= MATCH_THRESHOLD;
     if (accepted)
         STATS.accepted++;
     if (!options["silent"]) {
-        console.log(`[pick] ${agenda.uid} score=${best.score.toFixed(2)} 
-      agenda title="${agenda.titre ?? ""}" agenda organe="${agenda.organe ?? ""}"
+        console.log(`[pick] ${agenda.uid} score=${best.score.toFixed(2)}
+      agenda title="${agenda.titre ?? ""}" agenda organe="${agenda.organe ?? ""}" agenda heure=${agenda.startTime} 
       best title="${best.vtitle ?? ""}" best organe="${best.vorgane ?? ""}"
-      accepted=${accepted} (strategy=${usedStrategy})`);
+      accepted=${accepted}`);
     }
     // ==== 3) Write metadata + NVS of the best candidate (always) ====
     const bestDt = best?.epoch ? epochToParisDateTime(best.epoch) : null;
@@ -358,7 +372,6 @@ async function processGroupedReunion(agenda, session, dataDir) {
         session,
         accepted,
         threshold: MATCH_THRESHOLD,
-        strategy: usedStrategy,
         agenda: {
             date: agenda.date,
             startTime: agenda.startTime,

package/lib/scripts/shared/cli_helpers.d.ts

@@ -30,6 +30,11 @@ export declare const verboseOption: {
     name: string;
     type: BooleanConstructor;
 };
+export declare const onlyRecentOption: {
+    help: string;
+    name: string;
+    type: NumberConstructor;
+};
 export declare const commonOptions: ({
     alias: string;
     defaultValue: string[];

package/lib/scripts/shared/cli_helpers.js

@@ -30,10 +30,9 @@ export const verboseOption = {
     name: "verbose",
     type: Boolean,
 };
-export const commonOptions = [
-    categoriesOption,
-    dataDirDefaultOption,
-    fromSessionOption,
-    silentOption,
-    verboseOption,
-];
+export const onlyRecentOption = {
+    help: "retrieve only documents created within the last N days (that are not already downloaded)",
+    name: "only-recent",
+    type: Number,
+};
+export const commonOptions = [categoriesOption, dataDirDefaultOption, fromSessionOption, silentOption, verboseOption];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.16.5",
+  "version": "2.16.7",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",