@tricoteuses/senat 2.20.19 → 2.20.21

package/lib/scripts/retrieve_videos.js

@@ -8,7 +8,7 @@ import * as cheerio from "cheerio";
 import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatAgendas } from "../loaders";
 import { getSessionsFromStart } from "../types/sessions";
 import { commonOptions } from "./shared/cli_helpers";
-import { getFirstInterventionStartTimecode } from "../utils/nvs-timecode";
+import { getAgendaSegmentTimecodes } from "../utils/nvs-timecode";
 import { decodeHtmlEntities } from "../utils/string_cleaning";
 import { dice, normalize, scoreVideo } from "../utils/scoring";
 import { epochToParisDateTime, toFRDate, toTargetEpoch } from "../utils/date";
@@ -379,16 +379,22 @@ async function processGroupedReunion(agenda, session, dataDir) {
             console.warn(e);
         }
     }
-    let timecodeDebutVideo = null;
-    if (dataTxt && finalTxt) {
-        timecodeDebutVideo = getFirstInterventionStartTimecode(dataTxt, finalTxt);
-        if (timecodeDebutVideo === null) {
-            console.warn(`[warn] Cannot retrieve start video timecode from reunion` + reunionUid);
-        }
-    }
     // ==== 4) Update agenda file (only if accepted + m3u8) ====
     if ((accepted || skipDownload) && master) {
         const agendaJsonPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session), `${agenda.uid}.json`);
+        let timecodeDebutVideo = null;
+        let timecodeFinVideo = null;
+        if (dataTxt && finalTxt) {
+            const agendaKey = agenda.titre || agenda.objet || "";
+            const seg = getAgendaSegmentTimecodes(dataTxt, finalTxt, agendaKey);
+            if (!seg) {
+                console.warn(`[warn] Cannot retrieve agenda segment timecodes from reunion ${reunionUid}`);
+            }
+            else {
+                timecodeDebutVideo = seg.start;
+                timecodeFinVideo = seg.end;
+            }
+        }
         if (await fs.pathExists(agendaJsonPath)) {
             const raw = await fsp.readFile(agendaJsonPath, "utf-8");
             let obj;
@@ -403,6 +409,7 @@ async function processGroupedReunion(agenda, session, dataDir) {
                 const next = { ...obj, urlVideo: master };
                 if (timecodeDebutVideo != null) {
                     next.timecodeDebutVideo = timecodeDebutVideo;
+                    next.timecodeFinVideo = timecodeFinVideo;
                 }
                 await writeIfChanged(agendaJsonPath, JSON.stringify(next, null, 2));
                 if (!options["silent"]) {

package/lib/types/agenda.d.ts

@@ -31,6 +31,7 @@ export interface Reunion {
     transcriptionRef?: string;
     urlVideo?: string;
     timecodeDebutVideo?: number;
+    timecodeFinVideo?: number;
     odj?: ReunionOdj;
 }
 export interface ReunionOdjPoint {

package/lib/utils/nvs-timecode.d.ts

@@ -1 +1,7 @@
-export declare function getFirstInterventionStartTimecode(dataNvs: string, finalPlayerNvs: string): number | null;
+export declare function getAgendaSegmentTimecodes(dataNvs: string, finalPlayerNvs: string, agendaTitleOrObjet: string): {
+    start: number;
+    end: number | null;
+    chapterId: string;
+    nextChapterId: string | null;
+    score: number;
+} | null;

package/lib/utils/nvs-timecode.js

@@ -1,4 +1,7 @@
 import { XMLParser } from "fast-xml-parser";
+import { dice, normalize } from "./scoring";
+import { decodeHtmlEntities } from "./string_cleaning";
+const CHAPTER_MATCH_THRESHOLD = 0.5;
 const xmlParser = new XMLParser({
     ignoreAttributes: false,
     attributeNamePrefix: "@_",
@@ -20,20 +23,57 @@ function getTimecodeForChapterId(finalPlayerNvs, chapterId) {
         return null;
     return Math.floor(ms / 1000);
 }
-export function getFirstInterventionStartTimecode(dataNvs, finalPlayerNvs) {
-    const firstChapterId = getFirstChapterId(dataNvs);
-    if (!firstChapterId)
-        return null;
-    return getTimecodeForChapterId(finalPlayerNvs, firstChapterId);
+function toArray(v) {
+    if (!v)
+        return [];
+    return Array.isArray(v) ? v : [v];
 }
-function getFirstChapterId(dataNvs) {
+function getLevel1Chapters(dataNvs) {
     const xml = xmlParser.parse(dataNvs);
-    const rootChapters = xml?.data?.chapters?.chapter;
-    if (!rootChapters)
+    const root = xml?.data?.chapters?.chapter ?? xml?.chapters?.chapter;
+    const roots = toArray(root);
+    return roots
+        .map((ch, i) => {
+        const id = ch?.id ?? ch?.["@_id"];
+        const labelRaw = ch?.label ?? ch?.["@_label"] ?? "";
+        return {
+            id: String(id),
+            label: decodeHtmlEntities(String(labelRaw)).trim(),
+            index: i,
+        };
+    })
+        .filter((c) => c.id && c.label);
+}
+function pickBestLevel1ChapterForAgenda(chapters, agendaTitle) {
+    const q = normalize(agendaTitle);
+    let best = null;
+    for (const ch of chapters) {
+        const s = dice(q, ch.label);
+        if (!best || s > best.score)
+            best = { chapter: ch, score: s };
+    }
+    if (!best || best.score < CHAPTER_MATCH_THRESHOLD)
+        return { chapter: chapters[0], score: 0 };
+    return best;
+}
+export function getAgendaSegmentTimecodes(dataNvs, finalPlayerNvs, agendaTitleOrObjet) {
+    const l1 = getLevel1Chapters(dataNvs);
+    if (!l1.length)
+        return null;
+    const best = pickBestLevel1ChapterForAgenda(l1, agendaTitleOrObjet);
+    if (!best)
         return null;
-    const chaptersArray = Array.isArray(rootChapters) ? rootChapters : [rootChapters];
-    const firstChapter = chaptersArray[0];
-    if (!firstChapter || !firstChapter["@_id"])
+    const chapter = best.chapter;
+    const next = l1[chapter.index + 1] ?? null;
+    const start = getTimecodeForChapterId(finalPlayerNvs, chapter.id);
+    if (start == null)
         return null;
-    return String(firstChapter["@_id"]);
+    const end = next ? getTimecodeForChapterId(finalPlayerNvs, next.id) : null;
+    return {
+        start,
+        end,
+        chapterId: chapter.id,
+        nextChapterId: next?.id ?? null,
+        score: best.score,
+    };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.20.19",
+  "version": "2.20.21",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",