@tricoteuses/senat 2.18.11 → 2.18.12

package/README.md

@@ -108,7 +108,10 @@ npm run data:generate_schemas ../senat-data
 To publish a new version of this package onto npm, bump the package version and publish.
 
 ```bash
-npm version x.y.z # Bumps version in package.json and creates a new tag x.y.z
+# Increment version and create a new Git tag automatically
+npm version patch   # +0.0.1 → small fixes
+npm version minor   # +0.1.0 → new features
+npm version major   # +1.0.0 → breaking changes
 npx tsc
 npm publish
 ```

package/lib/scripts/retrieve_videos.js

@@ -135,15 +135,6 @@ function extractCandidatesFromSearchHtml(html) {
         return true;
     });
 }
-function parseFinalNvs(nvs) {
-    const playerTag = nvs.match(/<player\b[^>]*>/i)?.[0];
-    if (!playerTag)
-        return {};
-    const sessionStartStr = playerTag.match(/\bsessionstart="(\d+)"/i)?.[1];
-    return {
-        sessionStart: sessionStartStr ? Number(sessionStartStr) : undefined,
-    };
-}
 function parseDataNvs(nvs) {
     const epochStr = nvs.match(/<metadata\s+name="date"\s+value="(\d+)"/i)?.[1];
     const epoch = epochStr ? Number(epochStr) : undefined;
@@ -207,8 +198,6 @@ function score(agenda, agendaTs, sameOrg, videoTitle, videoEpoch, videoOrganes)
     const titleScore = Math.max(objetS, titleS);
     let timeScore = 0;
     if (agendaTs && videoEpoch) {
-        console.log("agendaTs", agendaTs);
-        console.log("videoEpoch", videoEpoch);
         const deltaMin = Math.abs(videoEpoch - agendaTs) / 60;
         timeScore = Math.exp(-deltaMin / 60);
     }
@@ -292,142 +281,168 @@ async function processGroupedReunion(agenda, session, dataDir) {
     if (agendaTs && agendaTs * 1000 > now) {
         return;
     }
-    STATS.total++;
     const reunionUid = agenda.uid;
     const baseDir = path.join(dataDir, VIDEOS_ROOT_FOLDER, String(session), reunionUid);
     await fs.ensureDir(baseDir);
-    const searchParams = {
-        search: "true",
-        videotype: getAgendaType(agenda),
-    };
-    if (agenda.date) {
-        const fr = toFRDate(agenda.date);
-        searchParams.period = "custom";
-        searchParams.begin = fr;
-        searchParams.end = fr;
-    }
-    if (agenda.organe) {
-        searchParams.organe = agenda.organe;
-    }
-    const pages = await fetchAllSearchPages(searchParams);
-    if (!pages.length) {
-        if (!options["silent"]) {
-            console.log(`[miss] ${agenda.uid} no candidates (videotype=${searchParams.videotype}, organe=${searchParams.organe || "-"}, date=${searchParams.begin || "-"})`);
+    let skipDownload = false;
+    if (options["only-recent"]) {
+        const now = Date.now();
+        const cutoff = now - options["only-recent"] * 24 * 3600 * 1000;
+        const reunionTs = Date.parse(agenda.date);
+        if (reunionTs < cutoff) {
+            // Check if files already exist
+            const dataNvsPath = path.join(baseDir, "data.nvs");
+            const finalplayerNvsPath = path.join(baseDir, "finalplayer.nvs");
+            if (fs.existsSync(dataNvsPath) && fs.existsSync(finalplayerNvsPath)) {
+                skipDownload = true;
+            }
         }
-        return;
     }
-    const combinedHtml = pages.join("\n<!-- PAGE SPLIT -->\n");
-    const candidates = extractCandidatesFromSearchHtml(combinedHtml).slice(0, MAX_CANDIDATES);
-    if (!candidates.length) {
-        if (!options["silent"]) {
-            console.log(`[miss] ${agenda.uid} no candidates after parse (videotype=${searchParams.videotype}, organe=${searchParams.organe || "-"}, date=${searchParams.begin || "-"})`);
+    let master = null;
+    let accepted = false;
+    if (!skipDownload) {
+        STATS.total++;
+        const searchParams = {
+            search: "true",
+            videotype: getAgendaType(agenda),
+        };
+        if (agenda.date) {
+            const fr = toFRDate(agenda.date);
+            searchParams.period = "custom";
+            searchParams.begin = fr;
+            searchParams.end = fr;
         }
-        return;
-    }
-    // ==== 2) Enrich via data.nvs + scoring; pick best ====
-    let best = null;
-    for (const c of candidates) {
-        const dataUrl = `${SENAT_DATAS_ROOT}/${c.id}_${c.hash}/content/data.nvs`;
-        const finalUrl = `${SENAT_DATAS_ROOT}/${c.id}_${c.hash}/content/finalplayer.nvs`;
-        const dataBuf = await fetchBuffer(dataUrl);
-        if (!dataBuf)
-            continue;
-        const meta = parseDataNvs(dataBuf.toString("utf-8"));
-        let sameOrg = false;
-        // If organes are too different, go to next candidates
-        if (agenda.organe && meta.organes?.length) {
-            const agendaOrgNorm = normalize(agenda.organe);
-            const agendaKey = getOrgKey(agendaOrgNorm);
-            let bestDice = 0;
-            let hasSameKey = false;
-            for (const vo of meta.organes) {
-                const videoOrgNorm = normalize(vo);
-                const videoKey = getOrgKey(videoOrgNorm);
-                const d = dice(agendaOrgNorm, videoOrgNorm);
-                if (videoKey === agendaKey && videoKey !== "autre") {
-                    hasSameKey = true;
-                }
-                if (d > bestDice)
-                    bestDice = d;
+        if (agenda.organe) {
+            searchParams.organe = agenda.organe;
+        }
+        const pages = await fetchAllSearchPages(searchParams);
+        if (!pages.length) {
+            if (!options["silent"]) {
+                console.log(`[miss] ${agenda.uid} no candidates (videotype=${searchParams.videotype}, organe=${searchParams.organe || "-"}, date=${searchParams.begin || "-"})`);
             }
-            if (hasSameKey) {
-                sameOrg = true; // we are sure this is the same org
+            return;
+        }
+        const combinedHtml = pages.join("\n<!-- PAGE SPLIT -->\n");
+        const candidates = extractCandidatesFromSearchHtml(combinedHtml).slice(0, MAX_CANDIDATES);
+        if (!candidates.length) {
+            if (!options["silent"]) {
+                console.log(`[miss] ${agenda.uid} no candidates after parse (videotype=${searchParams.videotype}, organe=${searchParams.organe || "-"}, date=${searchParams.begin || "-"})`);
             }
-            else if (bestDice < 0.8) {
-                // if diff org and dice too low we skip
+            return;
+        }
+        // ==== 2) Enrich via data.nvs + scoring; pick best ====
+        let best = null;
+        for (const c of candidates) {
+            const dataUrl = `${SENAT_DATAS_ROOT}/${c.id}_${c.hash}/content/data.nvs`;
+            const finalUrl = `${SENAT_DATAS_ROOT}/${c.id}_${c.hash}/content/finalplayer.nvs`;
+            const dataBuf = await fetchBuffer(dataUrl);
+            if (!dataBuf)
                 continue;
+            const meta = parseDataNvs(dataBuf.toString("utf-8"));
+            let sameOrg = false;
+            // If organes are too different, go to next candidates
+            if (agenda.organe && meta.organes?.length) {
+                const agendaOrgNorm = normalize(agenda.organe);
+                const agendaKey = getOrgKey(agendaOrgNorm);
+                let bestDice = 0;
+                let hasSameKey = false;
+                for (const vo of meta.organes) {
+                    const videoOrgNorm = normalize(vo);
+                    const videoKey = getOrgKey(videoOrgNorm);
+                    const d = dice(agendaOrgNorm, videoOrgNorm);
+                    if (videoKey === agendaKey && videoKey !== "autre") {
+                        hasSameKey = true;
+                    }
+                    if (d > bestDice)
+                        bestDice = d;
+                }
+                if (hasSameKey) {
+                    sameOrg = true; // we are sure this is the same org
+                }
+                else if (bestDice < 0.8) {
+                    // if diff org and dice too low we skip
+                    continue;
+                }
+            }
+            let videoTitle = c.title;
+            if (c.isSeancePublique && meta.firstChapterLabel) {
+                videoTitle = meta.firstChapterLabel;
+            }
+            const s = score(agenda, agendaTs, sameOrg, videoTitle, meta.epoch, meta.organes);
+            if (!best || s > best.score) {
+                best = {
+                    id: c.id,
+                    hash: c.hash,
+                    pageUrl: c.pageUrl,
+                    epoch: meta.epoch,
+                    vtitle: videoTitle,
+                    score: s,
+                    vorgane: meta.organes[0],
+                };
             }
         }
-        let videoTitle = c.title;
-        if (c.isSeancePublique && meta.firstChapterLabel) {
-            videoTitle = meta.firstChapterLabel;
-        }
-        const s = score(agenda, agendaTs, sameOrg, videoTitle, meta.epoch, meta.organes);
-        if (!best || s > best.score) {
-            best = {
-                id: c.id,
-                hash: c.hash,
-                pageUrl: c.pageUrl,
-                epoch: meta.epoch,
-                vtitle: videoTitle,
-                score: s,
-                vorgane: meta.organes[0],
-            };
+        if (!best) {
+            if (!options["silent"])
+                console.log(`[miss] ${agenda.uid} No candidate found for this reunion`);
+            return;
         }
-    }
-    if (!best) {
-        if (!options["silent"])
-            console.log(`[miss] ${agenda.uid} No candidate found for this reunion`);
-        return;
-    }
-    const accepted = best.score >= MATCH_THRESHOLD;
-    if (accepted)
-        STATS.accepted++;
-    if (!options["silent"]) {
-        console.log(`[pick] ${agenda.uid} score=${best.score.toFixed(2)}
+        accepted = best.score >= MATCH_THRESHOLD;
+        if (accepted)
+            STATS.accepted++;
+        if (!options["silent"]) {
+            console.log(`[pick] ${agenda.uid} score=${best.score.toFixed(2)}
       agenda title="${agenda.titre ?? ""}" agenda organe="${agenda.organe ?? ""}" agenda heure=${agenda.startTime} 
       best title="${best.vtitle ?? ""}" best organe="${best.vorgane ?? ""}"
       accepted=${accepted}`);
+        }
+        // ==== 3) Write metadata + NVS of the best candidate (always) ====
+        const bestDt = best?.epoch ? epochToParisDateTime(best.epoch) : null;
+        const metadata = {
+            reunionUid,
+            session,
+            accepted,
+            threshold: MATCH_THRESHOLD,
+            agenda: {
+                date: agenda.date,
+                startTime: agenda.startTime,
+                titre: agenda.titre,
+                organe: agenda.organe ?? undefined,
+                uid: agenda.uid,
+            },
+            best: {
+                id: best.id,
+                hash: best.hash,
+                pageUrl: best.pageUrl,
+                epoch: best.epoch ?? null,
+                date: bestDt?.date ?? null,
+                startTime: bestDt?.startTime ?? null,
+                title: best.vtitle ?? null,
+                score: best.score,
+            },
+        };
+        await writeIfChanged(path.join(baseDir, "metadata.json"), JSON.stringify(metadata, null, 2));
+        const dataUrl = `${SENAT_DATAS_ROOT}/${best.id}_${best.hash}/content/data.nvs`;
+        const finalUrl = `${SENAT_DATAS_ROOT}/${best.id}_${best.hash}/content/finalplayer.nvs`;
+        const dataTxt = await fetchText(dataUrl);
+        const finalTxt = await fetchText(finalUrl);
+        if (dataTxt)
+            await fsp.writeFile(path.join(baseDir, "data.nvs"), dataTxt, "utf-8");
+        if (finalTxt)
+            await fsp.writeFile(path.join(baseDir, "finalplayer.nvs"), finalTxt, "utf-8");
+        if (dataTxt && finalTxt)
+            master = buildSenatVodMasterM3u8FromNvs(dataTxt, finalTxt);
+    }
+    else {
+        // Skipped download, but need to read data.nvs for urlVideo
+        try {
+            const dataTxt = await fsp.readFile(path.join(baseDir, "data.nvs"), "utf-8");
+            const finalTxt = await fsp.readFile(path.join(baseDir, "finalplayer.nvs"), "utf-8");
+            master = buildSenatVodMasterM3u8FromNvs(dataTxt, finalTxt);
+        }
+        catch { }
     }
-    // ==== 3) Write metadata + NVS of the best candidate (always) ====
-    const bestDt = best?.epoch ? epochToParisDateTime(best.epoch) : null;
-    const metadata = {
-        reunionUid,
-        session,
-        accepted,
-        threshold: MATCH_THRESHOLD,
-        agenda: {
-            date: agenda.date,
-            startTime: agenda.startTime,
-            titre: agenda.titre,
-            organe: agenda.organe ?? undefined,
-            uid: agenda.uid,
-        },
-        best: {
-            id: best.id,
-            hash: best.hash,
-            pageUrl: best.pageUrl,
-            epoch: best.epoch ?? null,
-            date: bestDt?.date ?? null,
-            startTime: bestDt?.startTime ?? null,
-            title: best.vtitle ?? null,
-            score: best.score,
-        },
-    };
-    await writeIfChanged(path.join(baseDir, "metadata.json"), JSON.stringify(metadata, null, 2));
-    const dataUrl = `${SENAT_DATAS_ROOT}/${best.id}_${best.hash}/content/data.nvs`;
-    const finalUrl = `${SENAT_DATAS_ROOT}/${best.id}_${best.hash}/content/finalplayer.nvs`;
-    const dataTxt = await fetchText(dataUrl);
-    const finalTxt = await fetchText(finalUrl);
-    if (dataTxt)
-        await fsp.writeFile(path.join(baseDir, "data.nvs"), dataTxt, "utf-8");
-    if (finalTxt)
-        await fsp.writeFile(path.join(baseDir, "finalplayer.nvs"), finalTxt, "utf-8");
-    let master = null;
-    if (dataTxt && finalTxt)
-        master = buildSenatVodMasterM3u8FromNvs(dataTxt, finalTxt);
     // ==== 4) Update agenda file (only if accepted + m3u8) ====
-    if (accepted && master) {
+    if ((accepted || skipDownload) && master) {
         const agendaJsonPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session), `${agenda.uid}.json`);
         if (await fs.pathExists(agendaJsonPath)) {
             const raw = await fsp.readFile(agendaJsonPath, "utf-8");

package/lib/scripts/shared/cli_helpers.d.ts

@@ -48,13 +48,12 @@ export declare const commonOptions: ({
     name: string;
     type: StringConstructor;
 } | {
-    defaultValue: number;
+    alias: string;
     help: string;
     name: string;
-    type: NumberConstructor;
+    type: BooleanConstructor;
 } | {
-    alias: string;
     help: string;
     name: string;
-    type: BooleanConstructor;
+    type: NumberConstructor;
 })[];

package/lib/scripts/shared/cli_helpers.js

@@ -35,4 +35,11 @@ export const onlyRecentOption = {
     name: "only-recent",
     type: Number,
 };
-export const commonOptions = [categoriesOption, dataDirDefaultOption, fromSessionOption, silentOption, verboseOption];
+export const commonOptions = [
+    categoriesOption,
+    dataDirDefaultOption,
+    fromSessionOption,
+    silentOption,
+    verboseOption,
+    onlyRecentOption,
+];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.18.11",
+  "version": "2.18.12",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",