@tricoteuses/senat 2.20.35 → 2.21.0

package/lib/src/scripts/retrieve_videos.d.ts

@@ -12,9 +12,17 @@ export declare const SENAT_DATAS_ROOT = "https://videos.senat.fr/Datas/senat";
 export declare function fetchText(url: string): Promise<string | null>;
 export declare function fetchBuffer(url: string): Promise<Buffer | null>;
 export declare function queryString(obj: Record<string, string>): string;
+export declare function parseDataNvs(nvs: string): {
+    epoch?: number;
+    organes: string[];
+    firstChapterLabel?: string;
+    salle?: string;
+};
+export declare function buildSenatVodMasterM3u8FromNvs(nvsText: string): string | null;
 export declare function isAmbiguousTimeOriginal(timeOriginal?: string | null): boolean;
 export declare function getAgendaType(agenda: Reunion): string;
 export declare function fetchAllSearchPages(args: SearchParams, maxPages?: number): Promise<string[]>;
+export declare function getOrgKey(norm: string): string;
 export type MatchResult = {
     reunionUid: string;
     picked: null | {

package/lib/src/scripts/retrieve_videos.js

@@ -8,7 +8,8 @@ import * as cheerio from "cheerio";
 import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatAgendas } from "../loaders";
 import { getSessionsFromStart } from "../types/sessions";
 import { commonOptions } from "./shared/cli_helpers";
-import { buildSenatVodMasterM3u8FromNvs, getAgendaSegmentTimecodes } from "../utils/nvs-parsing";
+import { getAgendaSegmentTimecodes } from "../utils/nvs-timecode";
+import { decodeHtmlEntities } from "../utils/string_cleaning";
 import { matchOneReunion } from "../utils/scoring";
 import { epochToParisDateTime, toFRDate, toTargetEpoch } from "../utils/date";
 import { pathToFileURL } from "url";
@@ -76,6 +77,39 @@ function extractCandidatesFromSearchHtml(html) {
         return true;
     });
 }
+export function parseDataNvs(nvs) {
+    const epochStr = nvs.match(/<metadata\s+name="date"\s+value="(\d+)"/i)?.[1];
+    const epoch = epochStr ? Number(epochStr) : undefined;
+    // There can be multiple organes for one video in meta
+    const organes = [];
+    const organesRegex = /<metadata\b[^>]*\bname="organes"[^>]*>/gi;
+    let m;
+    const salle = decodeHtmlEntities(nvs.match(/<metadata\s+name="salle"\s+value="([^"]+)"/i)?.[1]).trim();
+    while ((m = organesRegex.exec(nvs)) !== null) {
+        const tag = m[0];
+        const label = tag.match(/\blabel="([^"]+)"/i)?.[1];
+        if (label) {
+            const decoded = decodeHtmlEntities(label).trim();
+            if (decoded)
+                organes.push(decoded);
+        }
+    }
+    if (organes.length === 0) {
+        organes.push("Séance publique");
+    }
+    const firstChapterLabelMatch = nvs.match(/<chapter\b[^>]*\blabel="([^"]+)"/i);
+    const firstChapterLabel = firstChapterLabelMatch ? decodeHtmlEntities(firstChapterLabelMatch[1]).trim() : undefined;
+    return { epoch, organes, firstChapterLabel, salle };
+}
+export function buildSenatVodMasterM3u8FromNvs(nvsText) {
+    // serverfiles://senat/2025/10/encoder10_20251022084451_2.mp4
+    const m = nvsText.match(/serverfiles:\/\/senat\/(\d{4})\/(\d{2})\/(encoder\d+)_([0-9]{14})/i);
+    if (!m)
+        return null;
+    const [, yyyy, mm, encoder, stamp] = m;
+    const base = `https://vodsenat.akamaized.net/senat/${yyyy}/${mm}/${encoder}_${stamp}`;
+    return `${base}.smil/master.m3u8`;
+}
 export function isAmbiguousTimeOriginal(timeOriginal) {
     if (!timeOriginal)
         return false;
@@ -121,6 +155,45 @@ export async function fetchAllSearchPages(args, maxPages = 3) {
     }
     return pages;
 }
+export function getOrgKey(norm) {
+    if (!norm)
+        return "autre";
+    if (norm.includes("seance publique"))
+        return "seance_publique";
+    if (norm.includes("culture"))
+        return "culture";
+    if (norm.includes("finances"))
+        return "finances";
+    if (norm.includes("sociales"))
+        return "affaires_sociales";
+    if (norm.includes("economiques"))
+        return "affaires_economiques";
+    if (norm.includes("europeennes"))
+        return "affaires_europeennes";
+    if (norm.includes("etrangeres") || norm.includes("forces armees") || norm.includes("defense")) {
+        return "affaires_etrangeres_defense";
+    }
+    if (norm.includes("territoire") || norm.includes("durable")) {
+        return "amenagement_territoire_dd";
+    }
+    if (norm.includes("commission des lois"))
+        return "lois";
+    if (norm.includes("delegation aux collectivites territoriales") || norm.includes("delegation a la decentralisation"))
+        return "delegation_collectivites";
+    if (norm.includes("delegation aux droits des femmes") ||
+        norm.includes("egalite des chances entre les hommes et les femmes"))
+        return "delegation_droits_femmes";
+    if (norm.includes("delegation aux entreprises"))
+        return "delegation_entreprises";
+    if (norm.includes("delegation senatoriale aux outre mer") || norm.includes("delegation aux outre mer"))
+        return "delegation_outre_mer";
+    if (norm.includes("delegation a la prospective"))
+        return "delegation_prospective";
+    if (norm.includes("office parlementaire d evaluation des choix scientifiques et technologiques") ||
+        norm.includes("opecst"))
+        return "opecst";
+    return "autre";
+}
 async function processGroupedReunion(agenda, session, dataDir) {
     // 1) GuardRails
     if (!agenda.captationVideo) {

package/lib/src/utils/scoring.d.ts

@@ -1,5 +1,5 @@
 import { AgendaEvent, Reunion } from "../types/agenda";
-import { L1Chapter } from "./nvs-parsing";
+import { L1Chapter } from "./nvs-timecode";
 export type MatchWeights = {
     minAccept?: number;
     margin?: number;
@@ -60,7 +60,6 @@ export type BestMatch = {
     m3u8: string;
     signals: VideoScoreSignals;
 };
-export declare function getOrgKey(norm: string): string;
 export declare function dice(a: string, b: string): number;
 export declare function normalize(s?: string | null): string;
 export declare function normalizeSalle(s?: string | null): string | null;

package/lib/src/utils/scoring.js

@@ -1,4 +1,5 @@
-import { buildSenatVodMasterM3u8FromNvs, getLevel1Chapters, parseDataNvs } from "./nvs-parsing";
+import { buildSenatVodMasterM3u8FromNvs, getOrgKey, parseDataNvs } from "../scripts/retrieve_videos";
+import { getLevel1Chapters } from "./nvs-timecode";
 import { parseISO } from "./reunion_parsing";
 import { normalizeText } from "./string_cleaning";
 export function jaccard(a, b) {
@@ -103,45 +104,6 @@ function extractHourHints(text) {
         out.push({ h: Number(m[1]), m: Number(m[2]) });
     return out;
 }
-export function getOrgKey(norm) {
-    if (!norm)
-        return "autre";
-    if (norm.includes("seance publique"))
-        return "seance_publique";
-    if (norm.includes("culture"))
-        return "culture";
-    if (norm.includes("finances"))
-        return "finances";
-    if (norm.includes("sociales"))
-        return "affaires_sociales";
-    if (norm.includes("economiques"))
-        return "affaires_economiques";
-    if (norm.includes("europeennes"))
-        return "affaires_europeennes";
-    if (norm.includes("etrangeres") || norm.includes("forces armees") || norm.includes("defense")) {
-        return "affaires_etrangeres_defense";
-    }
-    if (norm.includes("territoire") || norm.includes("durable")) {
-        return "amenagement_territoire_dd";
-    }
-    if (norm.includes("commission des lois"))
-        return "lois";
-    if (norm.includes("delegation aux collectivites territoriales") || norm.includes("delegation a la decentralisation"))
-        return "delegation_collectivites";
-    if (norm.includes("delegation aux droits des femmes") ||
-        norm.includes("egalite des chances entre les hommes et les femmes"))
-        return "delegation_droits_femmes";
-    if (norm.includes("delegation aux entreprises"))
-        return "delegation_entreprises";
-    if (norm.includes("delegation senatoriale aux outre mer") || norm.includes("delegation aux outre mer"))
-        return "delegation_outre_mer";
-    if (norm.includes("delegation a la prospective"))
-        return "delegation_prospective";
-    if (norm.includes("office parlementaire d evaluation des choix scientifiques et technologiques") ||
-        norm.includes("opecst"))
-        return "opecst";
-    return "autre";
-}
 function tokensDice(s) {
     return normalize(s).split(" ").filter(Boolean);
 }

package/lib/tests/videoMatching.test.js

@@ -8,10 +8,10 @@ import { describe, it, expect } from "vitest";
 import * as fs from "node:fs/promises";
 import * as path from "node:path";
 import * as cheerio from "cheerio";
-import { fetchBuffer, fetchText, getAgendaType, isAmbiguousTimeOriginal, queryString, SENAT_DATAS_ROOT, SENAT_VIDEOS_SEARCH_AJAX, } from "../src/scripts/retrieve_videos";
+import { buildSenatVodMasterM3u8FromNvs, fetchBuffer, fetchText, getAgendaType, getOrgKey, isAmbiguousTimeOriginal, parseDataNvs, queryString, SENAT_DATAS_ROOT, SENAT_VIDEOS_SEARCH_AJAX, } from "../src/scripts/retrieve_videos";
 import { toFRDate } from "../src/utils/date";
-import { dice, getOrgKey, normalize, scoreVideo } from "../src/utils/scoring";
-import { buildSenatVodMasterM3u8FromNvs, getLevel1Chapters, parseDataNvs } from "../src/utils/nvs-parsing";
+import { dice, normalize, scoreVideo } from "../src/utils/scoring";
+import { getLevel1Chapters } from "../src/utils/nvs-timecode";
 const LIVE_CACHE_DIR = path.join(process.cwd(), "tests", ".cache", "video-matching-live");
 const FIXTURES_ROOT = path.join(process.cwd(), "tests", "fixtures", "data");
 const GOLD_PATH = path.join(FIXTURES_ROOT, "expected-video-matching.json");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.20.35",
+  "version": "2.21.0",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",
@@ -65,8 +65,7 @@
     "prettier": "prettier --write 'src/**/*.ts'",
     "test:iter_load": "tsx src/scripts/test_iter_load.ts",
     "type-check": "tsc --noEmit",
-    "type-check:watch": "npm run type-check -- --watch",
-    "test:video-matching": "vitest tests/videoMatching.test.ts"
+    "type-check:watch": "npm run type-check -- --watch"
   },
   "dependencies": {
     "cheerio": "^1.1.2",
@@ -83,9 +82,8 @@
     "pg-cursor": "^2.12.1",
     "slug": "^11.0.0",
     "tsx": "^4.21.0",
-    "zod": "^4.3.5",
-    "vitest": "^4.0.18",
-    "windows-1252": "^3.0.4"
+    "windows-1252": "^3.0.4",
+    "zod": "^4.3.5"
   },
   "devDependencies": {
     "@typed-code/schemats": "^5.0.1",

package/lib/src/utils/nvs-parsing.d.ts

@@ -1,24 +0,0 @@
-export type L1Chapter = {
-    id: string;
-    label: string;
-    index: number;
-};
-export declare function getLevel1Chapters(dataNvs: string): L1Chapter[];
-export declare function pickBestLevel1ChapterForAgenda(chapters: L1Chapter[], agendaTitle: string): {
-    chapter: L1Chapter;
-    score: number;
-} | null;
-export declare function getAgendaSegmentTimecodes(dataNvs: string, finalPlayerNvs: string, agendaTitleOrObjet: string): {
-    start: number;
-    end: number | null;
-    chapterId: string;
-    nextChapterId: string | null;
-    score: number;
-} | null;
-export declare function parseDataNvs(nvs: string): {
-    epoch?: number;
-    organes: string[];
-    firstChapterLabel?: string;
-    salle?: string;
-};
-export declare function buildSenatVodMasterM3u8FromNvs(nvsText: string): string | null;

package/lib/src/utils/nvs-parsing.js

@@ -1,112 +0,0 @@
-import { XMLParser } from "fast-xml-parser";
-import { dice, normalize } from "./scoring";
-import { decodeHtmlEntities } from "./string_cleaning";
-const CHAPTER_MATCH_THRESHOLD = 0.5;
-const xmlParser = new XMLParser({
-    ignoreAttributes: false,
-    attributeNamePrefix: "@_",
-});
-function getTimecodeForChapterId(finalPlayerNvs, chapterId) {
-    const xml = xmlParser.parse(finalPlayerNvs);
-    const synchros = xml?.player?.synchro;
-    if (!synchros)
-        return null;
-    const synchsArray = Array.isArray(synchros) ? synchros : [synchros];
-    const match = synchsArray.find((s) => String(s["@_id"]) === String(chapterId));
-    if (!match)
-        return null;
-    const rawTimecode = match["@_timecode"];
-    if (rawTimecode == null)
-        return null;
-    const ms = Number(rawTimecode);
-    if (Number.isNaN(ms))
-        return null;
-    return Math.floor(ms / 1000);
-}
-function toArray(v) {
-    if (!v)
-        return [];
-    return Array.isArray(v) ? v : [v];
-}
-export function getLevel1Chapters(dataNvs) {
-    const xml = xmlParser.parse(dataNvs);
-    const root = xml?.data?.chapters?.chapter ?? xml?.chapters?.chapter;
-    const roots = toArray(root);
-    return roots
-        .map((ch, i) => {
-        const id = ch?.id ?? ch?.["@_id"];
-        const labelRaw = ch?.label ?? ch?.["@_label"] ?? "";
-        return {
-            id: String(id),
-            label: decodeHtmlEntities(String(labelRaw)).trim(),
-            index: i,
-        };
-    })
-        .filter((c) => c.id && c.label);
-}
-export function pickBestLevel1ChapterForAgenda(chapters, agendaTitle) {
-    const q = normalize(agendaTitle);
-    let best = null;
-    for (const ch of chapters) {
-        const s = dice(q, ch.label);
-        if (!best || s > best.score)
-            best = { chapter: ch, score: s };
-    }
-    if (!best || best.score < CHAPTER_MATCH_THRESHOLD)
-        return { chapter: chapters[0], score: 0 };
-    return best;
-}
-export function getAgendaSegmentTimecodes(dataNvs, finalPlayerNvs, agendaTitleOrObjet) {
-    const l1 = getLevel1Chapters(dataNvs);
-    if (!l1.length)
-        return null;
-    const best = pickBestLevel1ChapterForAgenda(l1, agendaTitleOrObjet);
-    if (!best)
-        return null;
-    const chapter = best.chapter;
-    const next = l1[chapter.index + 1] ?? null;
-    const start = getTimecodeForChapterId(finalPlayerNvs, chapter.id);
-    if (start == null)
-        return null;
-    const end = next ? getTimecodeForChapterId(finalPlayerNvs, next.id) : null;
-    return {
-        start,
-        end,
-        chapterId: chapter.id,
-        nextChapterId: next?.id ?? null,
-        score: best.score,
-    };
-}
-export function parseDataNvs(nvs) {
-    const epochStr = nvs.match(/<metadata\s+name="date"\s+value="(\d+)"/i)?.[1];
-    const epoch = epochStr ? Number(epochStr) : undefined;
-    // There can be multiple organes for one video in meta
-    const organes = [];
-    const organesRegex = /<metadata\b[^>]*\bname="organes"[^>]*>/gi;
-    let m;
-    const salle = decodeHtmlEntities(nvs.match(/<metadata\s+name="salle"\s+value="([^"]+)"/i)?.[1]).trim();
-    while ((m = organesRegex.exec(nvs)) !== null) {
-        const tag = m[0];
-        const label = tag.match(/\blabel="([^"]+)"/i)?.[1];
-        if (label) {
-            const decoded = decodeHtmlEntities(label).trim();
-            if (decoded)
-                organes.push(decoded);
-        }
-    }
-    if (organes.length === 0) {
-        organes.push("Séance publique");
-    }
-    const firstChapterLabelMatch = nvs.match(/<chapter\b[^>]*\blabel="([^"]+)"/i);
-    const firstChapterLabel = firstChapterLabelMatch ? decodeHtmlEntities(firstChapterLabelMatch[1]).trim() : undefined;
-    return { epoch, organes, firstChapterLabel, salle };
-}
-export function buildSenatVodMasterM3u8FromNvs(nvsText) {
-    // serverfiles://senat/2025/10/encoder10_20251022084451_2.mp4
-    const m = nvsText.match(/serverfiles:\/\/senat\/(\d{4})\/(\d{2})\/(encoder\d+)_([0-9]{14})/i);
-    if (!m)
-        return null;
-    const [, yyyy, mm, encoder, stamp] = m;
-    const base = `https://vodsenat.akamaized.net/senat/${yyyy}/${mm}/${encoder}_${stamp}`;
-    return `${base}.smil/master.m3u8`;
-}