@tricoteuses/senat 2.16.8 → 2.18.10

package/lib/scripts/retrieve_videos.js

@@ -10,7 +10,7 @@ import { commonOptions } from "./shared/cli_helpers";
 import { decodeHtmlEntities } from "../model/util";
 import { DateTime } from "luxon";
 // ===================== Constants =====================
-const MATCH_THRESHOLD = 0.6;
+const MATCH_THRESHOLD = 0.5;
 const MAX_CANDIDATES = 15;
 const STATS = { total: 0, accepted: 0 };
 const VIDEOS_ROOT_FOLDER = "videos";
@@ -138,15 +138,25 @@ function parseFinalNvs(nvs) {
 function parseDataNvs(nvs) {
     const epochStr = nvs.match(/<metadata\s+name="date"\s+value="(\d+)"/i)?.[1];
     const epoch = epochStr ? Number(epochStr) : undefined;
-    const organesTag = nvs.match(/<metadata\b[^>]*\bname="organes"[^>]*>/i)?.[0];
-    let organeLabel;
-    if (organesTag) {
-        organeLabel = organesTag.match(/\blabel="([^"]+)"/i)?.[1];
+    // There can be multiple organes for one video in meta
+    const organes = [];
+    const organesRegex = /<metadata\b[^>]*\bname="organes"[^>]*>/gi;
+    let m;
+    while ((m = organesRegex.exec(nvs)) !== null) {
+        const tag = m[0];
+        const label = tag.match(/\blabel="([^"]+)"/i)?.[1];
+        if (label) {
+            const decoded = decodeHtmlEntities(label).trim();
+            if (decoded)
+                organes.push(decoded);
+        }
+    }
+    if (organes.length === 0) {
+        organes.push("Séance publique");
     }
-    const organeRaw = organeLabel ?? "Séance publique";
-    const organe = decodeHtmlEntities(organeRaw)?.trim();
-    const firstChapterLabel = decodeHtmlEntities(nvs.match(/<chapter\b[^>]*\blabel="([^"]+)"/i)[1]).trim();
-    return { epoch, organe, firstChapterLabel };
+    const firstChapterLabelMatch = nvs.match(/<chapter\b[^>]*\blabel="([^"]+)"/i);
+    const firstChapterLabel = firstChapterLabelMatch ? decodeHtmlEntities(firstChapterLabelMatch[1]).trim() : undefined;
+    return { epoch, organes, firstChapterLabel };
 }
 function buildSenatVodMasterM3u8FromNvs(nvsText, finalText) {
     // 1) Base Akamai from data.nvs (mp4 "serverfiles://senat/YYYY/MM/encoderX_YYYYMMDDHHMMSS_1.mp4")
@@ -182,18 +192,20 @@ function buildSenatVodMasterM3u8FromNvs(nvsText, finalText) {
     }
     return `${base}.smil/master.m3u8`;
 }
-function score(agenda, agendaTs, sameOrg, videoTitle, videoEpoch, videoOrgane) {
+function score(agenda, agendaTs, sameOrg, videoTitle, videoEpoch, videoOrganes) {
     const objetS = dice(agenda.objet || "", videoTitle || "");
     const titleS = dice(agenda.titre || "", videoTitle || "");
     const titleScore = Math.max(objetS, titleS);
     let timeScore = 0;
     if (agendaTs && videoEpoch) {
-        // second
         const deltaMin = Math.abs(videoEpoch - agendaTs) / 60;
-        // delta : 60min
         timeScore = Math.exp(-deltaMin / 60);
     }
-    const orgScore = videoOrgane && agenda.organe ? dice(agenda.organe, videoOrgane) : 0;
+    let orgScore = 0;
+    if (agenda.organe && videoOrganes && videoOrganes.length) {
+        const agendaOrg = agenda.organe;
+        orgScore = Math.max(...videoOrganes.map((v) => dice(agendaOrg, v)));
+    }
     return 0.2 * titleScore + 0.4 * timeScore + (sameOrg ? 0.4 : orgScore * 0.4);
 }
 function getAgendaType(agenda) {
@@ -316,20 +328,28 @@ async function processGroupedReunion(agenda, session, dataDir) {
             const finalMeta = parseFinalNvs(finalBuf.toString("utf-8"));
             sessionStart = finalMeta.sessionStart;
         }
-        const videoEpoch = sessionStart ?? meta.epoch;
+        const videoEpoch = meta.epoch ?? sessionStart;
         let sameOrg = false;
         // If organes are too different, go to next candidates
-        if (meta.organe && agenda.organe) {
-            const videoOrgNorm = normalize(meta.organe);
+        if (agenda.organe && meta.organes?.length) {
             const agendaOrgNorm = normalize(agenda.organe);
-            const videoKey = getOrgKey(videoOrgNorm);
             const agendaKey = getOrgKey(agendaOrgNorm);
-            const d = dice(agendaOrgNorm, videoOrgNorm);
-            if (videoKey === agendaKey && videoKey !== "autre") {
-                // same org we keep it
-                sameOrg = true;
+            let bestDice = 0;
+            let hasSameKey = false;
+            for (const vo of meta.organes) {
+                const videoOrgNorm = normalize(vo);
+                const videoKey = getOrgKey(videoOrgNorm);
+                const d = dice(agendaOrgNorm, videoOrgNorm);
+                if (videoKey === agendaKey && videoKey !== "autre") {
+                    hasSameKey = true;
+                }
+                if (d > bestDice)
+                    bestDice = d;
+            }
+            if (hasSameKey) {
+                sameOrg = true; // we are sure this is the same org
             }
-            else if (d < 0.7) {
+            else if (bestDice < 0.8) {
                 // if diff org and dice too low we skip
                 continue;
             }
@@ -338,7 +358,7 @@ async function processGroupedReunion(agenda, session, dataDir) {
         if (c.isSeancePublique && meta.firstChapterLabel) {
             videoTitle = meta.firstChapterLabel;
         }
-        const s = score(agenda, agendaTs, sameOrg, videoTitle, videoEpoch, meta.organe);
+        const s = score(agenda, agendaTs, sameOrg, videoTitle, videoEpoch, meta.organes);
         if (!best || s > best.score) {
             best = {
                 id: c.id,
@@ -347,7 +367,7 @@ async function processGroupedReunion(agenda, session, dataDir) {
                 epoch: meta.epoch,
                 vtitle: videoTitle,
                 score: s,
-                vorgane: meta.organe,
+                vorgane: meta.organes[0],
             };
         }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.16.8",
+  "version": "2.18.10",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",