@tricoteuses/senat 2.16.7 → 2.18.9

package/lib/loaders.js

@@ -42,8 +42,15 @@ function* iterLoadSenatItems(dataDir, dataName, legislatureOrSession, subDir, {
         if (log) {
             console.log(`Loading file: ${filePath}…`);
         }
-        const itemJson = fs.readFileSync(filePath, { encoding: "utf8" });
-        const item = JSON.parse(itemJson);
+        let item;
+        try {
+            const itemJson = fs.readFileSync(filePath, { encoding: "utf8" });
+            item = JSON.parse(itemJson);
+        }
+        catch (err) {
+            console.warn(`[iterLoadSenatItems] skipped invalid JSON: ${filePath} (${err.message})`);
+            continue;
+        }
         const filePathFromDataset = filePath.substring(filePath.indexOf(dataName) + dataName.length);
         yield {
             item,

package/lib/model/ameli.js

@@ -114,20 +114,13 @@ const findAllAmendementsQuery = dbSenat
     concat(val("https://www.senat.fr/amendements/"), ref("ameli.ses.lil"), val("/"), ref("ameli.txt_ameli.numabs"), val("/Amdt_"), ref("ameli.amd.numabs"), val(".html")).as("url"),
     "ameli.grppol_ameli.lilcou as au_nom_de_groupe_politique",
     "ameli.com_ameli.lil as au_nom_de_commission",
-    eb
-        .case()
-        .when("ameli.cab.entid", "is not", null)
-        .then(true)
-        .else(false)
-        .end()
-        .as("auteur_est_gouvernement"),
-    auteurs(ref("ameli.amd.id")).as("auteurs")
-]);
+    eb.case().when("ameli.cab.entid", "is not", null).then(true).else(false).end().as("auteur_est_gouvernement"),
+    auteurs(ref("ameli.amd.id")).as("auteurs"),
+])
+    .distinctOn("ameli.amd.id");
 export function findAllAmendements(fromSession) {
     if (fromSession !== undefined) {
-        return findAllAmendementsQuery
-            .where("ameli.ses.ann", ">=", fromSession)
-            .stream();
+        return findAllAmendementsQuery.where("ameli.ses.ann", ">=", fromSession).stream();
     }
     return findAllAmendementsQuery.stream();
 }

package/lib/scripts/retrieve_videos.js

@@ -138,15 +138,25 @@ function parseFinalNvs(nvs) {
 function parseDataNvs(nvs) {
     const epochStr = nvs.match(/<metadata\s+name="date"\s+value="(\d+)"/i)?.[1];
     const epoch = epochStr ? Number(epochStr) : undefined;
-    const organesTag = nvs.match(/<metadata\b[^>]*\bname="organes"[^>]*>/i)?.[0];
-    let organeLabel;
-    if (organesTag) {
-        organeLabel = organesTag.match(/\blabel="([^"]+)"/i)?.[1];
+    // There can be multiple organes for one video in meta
+    const organes = [];
+    const organesRegex = /<metadata\b[^>]*\bname="organes"[^>]*>/gi;
+    let m;
+    while ((m = organesRegex.exec(nvs)) !== null) {
+        const tag = m[0];
+        const label = tag.match(/\blabel="([^"]+)"/i)?.[1];
+        if (label) {
+            const decoded = decodeHtmlEntities(label).trim();
+            if (decoded)
+                organes.push(decoded);
+        }
+    }
+    if (organes.length === 0) {
+        organes.push("Séance publique");
     }
-    const organeRaw = organeLabel ?? "Séance publique";
-    const organe = decodeHtmlEntities(organeRaw)?.trim();
-    const firstChapterLabel = decodeHtmlEntities(nvs.match(/<chapter\b[^>]*\blabel="([^"]+)"/i)[1]).trim();
-    return { epoch, organe, firstChapterLabel };
+    const firstChapterLabelMatch = nvs.match(/<chapter\b[^>]*\blabel="([^"]+)"/i);
+    const firstChapterLabel = firstChapterLabelMatch ? decodeHtmlEntities(firstChapterLabelMatch[1]).trim() : undefined;
+    return { epoch, organes, firstChapterLabel };
 }
 function buildSenatVodMasterM3u8FromNvs(nvsText, finalText) {
     // 1) Base Akamai from data.nvs (mp4 "serverfiles://senat/YYYY/MM/encoderX_YYYYMMDDHHMMSS_1.mp4")
@@ -182,18 +192,20 @@ function buildSenatVodMasterM3u8FromNvs(nvsText, finalText) {
     }
     return `${base}.smil/master.m3u8`;
 }
-function score(agenda, agendaTs, sameOrg, videoTitle, videoEpoch, videoOrgane) {
+function score(agenda, agendaTs, sameOrg, videoTitle, videoEpoch, videoOrganes) {
     const objetS = dice(agenda.objet || "", videoTitle || "");
     const titleS = dice(agenda.titre || "", videoTitle || "");
     const titleScore = Math.max(objetS, titleS);
     let timeScore = 0;
     if (agendaTs && videoEpoch) {
-        // second
         const deltaMin = Math.abs(videoEpoch - agendaTs) / 60;
-        // delta : 60min
         timeScore = Math.exp(-deltaMin / 60);
     }
-    const orgScore = videoOrgane && agenda.organe ? dice(agenda.organe, videoOrgane) : 0;
+    let orgScore = 0;
+    if (agenda.organe && videoOrganes && videoOrganes.length) {
+        const agendaOrg = agenda.organe;
+        orgScore = Math.max(...videoOrganes.map((v) => dice(agendaOrg, v)));
+    }
     return 0.2 * titleScore + 0.4 * timeScore + (sameOrg ? 0.4 : orgScore * 0.4);
 }
 function getAgendaType(agenda) {
@@ -319,17 +331,25 @@ async function processGroupedReunion(agenda, session, dataDir) {
         const videoEpoch = sessionStart ?? meta.epoch;
         let sameOrg = false;
         // If organes are too different, go to next candidates
-        if (meta.organe && agenda.organe) {
-            const videoOrgNorm = normalize(meta.organe);
+        if (agenda.organe && meta.organes?.length) {
             const agendaOrgNorm = normalize(agenda.organe);
-            const videoKey = getOrgKey(videoOrgNorm);
             const agendaKey = getOrgKey(agendaOrgNorm);
-            const d = dice(agendaOrgNorm, videoOrgNorm);
-            if (videoKey === agendaKey && videoKey !== "autre") {
-                // same org we keep it
-                sameOrg = true;
+            let bestDice = 0;
+            let hasSameKey = false;
+            for (const vo of meta.organes) {
+                const videoOrgNorm = normalize(vo);
+                const videoKey = getOrgKey(videoOrgNorm);
+                const d = dice(agendaOrgNorm, videoOrgNorm);
+                if (videoKey === agendaKey && videoKey !== "autre") {
+                    hasSameKey = true;
+                }
+                if (d > bestDice)
+                    bestDice = d;
+            }
+            if (hasSameKey) {
+                sameOrg = true; // we are sure this is the same org
             }
-            else if (d < 0.7) {
+            else if (bestDice < 0.7) {
                 // if diff org and dice too low we skip
                 continue;
             }
@@ -338,7 +358,7 @@ async function processGroupedReunion(agenda, session, dataDir) {
         if (c.isSeancePublique && meta.firstChapterLabel) {
             videoTitle = meta.firstChapterLabel;
         }
-        const s = score(agenda, agendaTs, sameOrg, videoTitle, videoEpoch, meta.organe);
+        const s = score(agenda, agendaTs, sameOrg, videoTitle, videoEpoch, meta.organes);
         if (!best || s > best.score) {
             best = {
                 id: c.id,
@@ -347,7 +367,7 @@ async function processGroupedReunion(agenda, session, dataDir) {
                 epoch: meta.epoch,
                 vtitle: videoTitle,
                 score: s,
-                vorgane: meta.organe,
+                vorgane: meta.organes[0],
             };
         }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.16.7",
+  "version": "2.18.9",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",