@tricoteuses/senat 2.20.17 → 2.20.19

package/lib/utils/reunion_parsing.d.ts

@@ -0,0 +1,23 @@
+import { DateTime } from "luxon";
+import { AgendaEvent, Reunion } from "../types/agenda";
+import { DossierLegislatifResult } from "../model/dosleg";
+import * as cheerio from "cheerio";
+type KnownType = "SP" | "COM" | "MC" | "OD" | "ID";
+type DossierBySenatUrl = Record<string, DossierLegislatifResult>;
+type ReunionBucket = "IDS" | "IDC" | "IDM" | "IDO" | "IDI";
+type ReunionsByBucket = Record<ReunionBucket, Reunion[]>;
+export declare function buildReunionsByBucket(events: AgendaEvent[], dossierBySenatUrl: DossierBySenatUrl): ReunionsByBucket;
+export declare function makeReunionUid(dateISO: string, kind: KnownType, agendaEventId: string, organe?: string | null): string;
+export declare function formatYYYYMMDD(dateYYYYMMDD: string): string;
+export declare function deriveTimesForEvent(ev: AgendaEvent): {
+    startISO: string | null;
+    endISO: string | null;
+};
+export type SommaireBlock = {
+    text: string;
+    startIndex: number;
+    targetId?: string | null;
+};
+export declare function extractSommaireBlocks($: cheerio.CheerioAPI, idx: Map<any, number>): SommaireBlock[];
+export declare function parseISO(iso: string | null | undefined): DateTime | null;
+export {};

package/lib/utils/reunion_parsing.js

@@ -0,0 +1,209 @@
+import { DateTime } from "luxon";
+import { buildOdj } from "./reunion_odj_building";
+import { norm } from "./string_cleaning";
+const PARIS = "Europe/Paris";
+const STOPWORDS = new Set([
+    "de",
+    "du",
+    "des",
+    "la",
+    "le",
+    "les",
+    "l",
+    "d",
+    "et",
+    "en",
+    "au",
+    "aux",
+    "pour",
+    "sur",
+    "sous",
+    "à",
+    "a",
+    "aux",
+]);
+function toReunion(e, dossierBySenatUrl, uid) {
+    const date = norm(e.date) ?? e.date;
+    const { startISO, endISO } = deriveTimesForEvent(e);
+    const startTime = startISO ?? e.startTime ?? null;
+    const endTime = endISO ?? e.endTime ?? null;
+    return {
+        uid,
+        chambre: "SN",
+        date,
+        type: e.type || "",
+        organe: e.organe || undefined,
+        startTime,
+        endTime,
+        captationVideo: e.captationVideo === true,
+        titre: e.titre,
+        objet: e.objet || "",
+        events: [e], // TODO remove
+        odj: buildOdj([e], dossierBySenatUrl),
+    };
+}
+export function buildReunionsByBucket(events, dossierBySenatUrl) {
+    const out = { IDS: [], IDC: [], IDM: [], IDO: [], IDI: [] };
+    if (!events?.length)
+        return out;
+    for (const e of events) {
+        const kind = classifyAgendaType(e?.type);
+        if (!kind) {
+            continue;
+            console.warn("Can't determine type of reunion");
+        }
+        const bucket = typeToSuffixStrict(kind);
+        const uid = makeReunionUid(e.date, kind, e.id, e.organe ?? null);
+        out[bucket].push(toReunion(e, dossierBySenatUrl, uid));
+    }
+    // Tri stable par bucket (date + heure, inconnus à la fin)
+    for (const k of Object.keys(out)) {
+        out[k].sort((a, b) => {
+            const da = DateTime.fromISO(`${a.date}T${a.startTime || "23:59:59.999+02:00"}`, { zone: PARIS }).toMillis();
+            const db = DateTime.fromISO(`${b.date}T${b.startTime || "23:59:59.999+02:00"}`, { zone: PARIS }).toMillis();
+            return da - db || (a.organe || "").localeCompare(b.organe || "") || (a.titre || "").localeCompare(b.titre || "");
+        });
+    }
+    return out;
+}
+function normalizeNoAccents(s) {
+    return (s || "")
+        .trim()
+        .normalize("NFKD")
+        .replace(/[\u0300-\u036f]/g, "");
+}
+function classifyAgendaType(typeLabel) {
+    const s = normalizeNoAccents(typeLabel || "").toLowerCase();
+    if (/\bseance\b.*\bpublique\b/.test(s))
+        return "SP";
+    if (/\bcommissions\b/.test(s))
+        return "COM";
+    if (/\bmission\b.*\bcontrole\b/.test(s))
+        return "MC";
+    if (/\boffices\b|\bdelegations\b/.test(s))
+        return "OD";
+    if (/\instances\b|\decisionelles\b/.test(s))
+        return "ID";
+    return null;
+}
+function typeToSuffixStrict(kind) {
+    switch (kind) {
+        case "SP":
+            return "IDS";
+        case "COM":
+            return "IDC";
+        case "MC":
+            return "IDM";
+        case "OD":
+            return "IDO";
+        case "ID":
+            return "IDI";
+    }
+}
+function organeInitials(input, maxLen = 8) {
+    if (!input)
+        return "";
+    const clean = normalizeNoAccents(input)
+        .replace(/['’]/g, " ")
+        .replace(/[^A-Za-z0-9\s]/g, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+    if (!clean)
+        return "";
+    const parts = clean.split(" ");
+    const letters = [];
+    for (const raw of parts) {
+        const w = raw.toLowerCase();
+        if (!w)
+            continue;
+        if (STOPWORDS.has(w))
+            continue;
+        // Take two first letter if alphanumeric
+        const two = raw.slice(0, 2);
+        if (/[A-Za-z0-9]/.test(two))
+            letters.push(two.toUpperCase());
+    }
+    const out = letters.join("");
+    return out.slice(0, maxLen);
+}
+export function makeReunionUid(dateISO, kind, agendaEventId, organe) {
+    const ymd = dateISO ? formatYYYYMMDD(dateISO) : "00000000";
+    const suffix = typeToSuffixStrict(kind);
+    const org = organe && suffix !== "IDS" ? organeInitials(organe) : "";
+    let base = `RUSN${ymd}${suffix}${org}${agendaEventId}`;
+    return base;
+}
+export function formatYYYYMMDD(dateYYYYMMDD) {
+    const [y, m, d] = dateYYYYMMDD.split("-");
+    return `${y}${m}${d}`;
+}
+// Extract hours/minutes from French text like "à 10 h 30", "de 10 h à 12 h", etc.
+function parseTimeOriginalFR(timeOriginal) {
+    if (!timeOriginal)
+        return { start: null, end: null };
+    const txt = (timeOriginal || "")
+        .replace(/\u00A0/g, " ") // nbsp → space
+        .replace(/\s+/g, " ") // espaces multiples
+        .toLowerCase()
+        .trim();
+    // 1) "de 10 h 30 à 12 heures", "de 10h30 à 12h", "de 9 h à 11 h 15", etc.
+    const reRange = /\bde\s+(\d{1,2})\s*(?:h|:)?\s*(\d{1,2})?\s*(?:heures?)?\s*à\s*(\d{1,2})\s*(?:h|:)?\s*(\d{1,2})?\s*(?:heures?)?/i;
+    const mRange = txt.match(reRange);
+    if (mRange) {
+        const h1 = clampHour(+mRange[1]), m1 = clampMinute(mRange[2] ? +mRange[2] : 0);
+        const h2 = clampHour(+mRange[3]), m2 = clampMinute(mRange[4] ? +mRange[4] : 0);
+        return { start: toIsoTime(h1, m1), end: toIsoTime(h2, m2) };
+    }
+    // 2) "à 10 h 30", "à 10h", "A 10h30", "A 9 heures", etc.
+    const reAt = /\b(?:a|à)\s*(\d{1,2})\s*(?:h|:)?\s*(\d{1,2})?\s*(?:heures?)?/i;
+    const mAt = txt.match(reAt);
+    if (mAt) {
+        const h = clampHour(+mAt[1]), m = clampMinute(mAt[2] ? +mAt[2] : 0);
+        return { start: toIsoTime(h, m), end: null };
+    }
+    // 3) "10 h 30", "15h", "9 heures" sans 'à' / 'de ... à ...'
+    const reBare = /\b(\d{1,2})\s*(?:h|:)?\s*(\d{1,2})?\s*(?:heures?)?\b/;
+    const mBare = txt.match(reBare);
+    if (mBare) {
+        const h = clampHour(+mBare[1]), m = clampMinute(mBare[2] ? +mBare[2] : 0);
+        return { start: toIsoTime(h, m), end: null };
+    }
+    return { start: null, end: null };
+}
+function clampHour(h) {
+    return Math.max(0, Math.min(23, h));
+}
+function clampMinute(m) {
+    return Math.max(0, Math.min(59, m));
+}
+function toIsoTime(h, m) {
+    return `${String(h).padStart(2, "0")}:${String(m).padStart(2, "0")}:00.000+02:00`;
+}
+export function deriveTimesForEvent(ev) {
+    const directStart = ev.startTime ?? null;
+    const directEnd = ev.endTime ?? null;
+    const fromText = parseTimeOriginalFR(ev.timeOriginal);
+    const startISO = directStart ?? fromText.start ?? null;
+    const endISO = directEnd ?? fromText.end ?? null;
+    return { startISO, endISO };
+}
+export function extractSommaireBlocks($, idx) {
+    const blocks = [];
+    // lignes du sommaire avec lien
+    $("cri\\:tm5 a[href^='#'], cri\\:tm3 a[href^='#'], p.tm5 a[href^='#'], p.tm3 a[href^='#']").each((_, a) => {
+        const href = $(a).attr("href") || "";
+        const targetId = href.startsWith("#") ? href.slice(1) : null;
+        const text = norm($(a).text() || "");
+        const startIndex = idx.get(a) ?? idx.get($(a).closest("p")[0]) ?? null;
+        if (!text || startIndex == null)
+            return;
+        blocks.push({ text, startIndex, targetId });
+    });
+    return blocks;
+}
+export function parseISO(iso) {
+    if (!iso)
+        return null;
+    const dt = DateTime.fromISO(iso, { setZone: true, zone: PARIS });
+    return dt.isValid ? dt : null;
+}

package/lib/utils/scoring.d.ts

@@ -0,0 +1,14 @@
+import { AgendaEvent, Reunion } from "../types/agenda";
+export declare function jaccard(a: Set<string>, b: Set<string>): number;
+export declare function jaccardTokenSim(a: string, b: string): number;
+export declare function isNoiseBlock(text: string): boolean;
+/**
+ * Score robuste pour "bloc court vs event long":
+ * - coverageBloc = |A∩B| / |A|  (critère principal)
+ * - jaccard en secondaire (utile quand les 2 sont longs)
+ * - bonus time optionnel (déjà chez toi)
+ */
+export declare function scoreSommaireBlockForEvent(blockText: string, ev: AgendaEvent): number;
+export declare function scoreVideo(agenda: Reunion, agendaTs: number | null, sameOrg: boolean, videoTitle?: string, videoEpoch?: number, videoOrganes?: string[], timeAmbigious?: boolean): number;
+export declare function dice(a: string, b: string): number;
+export declare function normalize(s?: string | null): string;

package/lib/utils/scoring.js

@@ -0,0 +1,147 @@
+import { parseISO } from "./reunion_parsing";
+import { normalizeText } from "./string_cleaning";
+export function jaccard(a, b) {
+    if (!a.size || !b.size)
+        return 0;
+    let inter = 0;
+    for (const t of a)
+        if (b.has(t))
+            inter++;
+    return inter / (a.size + b.size - inter);
+}
+export function jaccardTokenSim(a, b) {
+    const A = new Set(normalizeText(a).split(" ").filter(Boolean));
+    const B = new Set(normalizeText(b).split(" ").filter(Boolean));
+    if (A.size === 0 || B.size === 0)
+        return 0;
+    let inter = 0;
+    for (const x of A)
+        if (B.has(x))
+            inter++;
+    return inter / (A.size + B.size - inter);
+}
+export function isNoiseBlock(text) {
+    const t = normalizeText(text);
+    // très génériques / institutionnels / bruit
+    if (!t)
+        return true;
+    if (t.length < 6)
+        return true;
+    // Ex: "article 78", "ord re du jour", "organisation des travaux"
+    if (/^article\s+\d+/.test(t))
+        return true;
+    if (/\b(organisation des travaux|ordre du jour|suspension|reprise de la seance)\b/.test(t))
+        return true;
+    if (/\b(vice presidente|president|secretaire|ministre|rapporteur)\b/.test(t))
+        return true;
+    return false;
+}
+/**
+ * Score robuste pour "bloc court vs event long":
+ * - coverageBloc = |A∩B| / |A|  (critère principal)
+ * - jaccard en secondaire (utile quand les 2 sont longs)
+ * - bonus time optionnel (déjà chez toi)
+ */
+export function scoreSommaireBlockForEvent(blockText, ev) {
+    const evText = `${ev.titre ?? ""} ${ev.objet ?? ""}`;
+    const A = tokens(blockText); // tokens du bloc
+    const B = tokens(evText); // tokens de l'event
+    if (A.size === 0 || B.size === 0)
+        return 0;
+    const inter = intersectionSize(A, B);
+    const coverageBloc = inter / A.size; // 🔥 clé
+    const jac = jaccard(A, B);
+    // bonus inclusion brut (utile sur des expressions exactes)
+    const bNorm = normalizeText(blockText);
+    const eNorm = normalizeText(evText);
+    const inclusion = bNorm.length >= 10 && eNorm.includes(bNorm) ? 0.12 : 0;
+    // bonus heure (si tu l'as)
+    const timeBonus = timeProximityBonus(ev.startTime ?? null, blockText);
+    // combine: coverage domine
+    const score = Math.max(coverageBloc, jac) * 0.85 + Math.min(1, jac) * 0.1 + inclusion + timeBonus;
+    return Math.max(0, Math.min(1, score));
+}
+function intersectionSize(a, b) {
+    let inter = 0;
+    for (const x of a)
+        if (b.has(x))
+            inter++;
+    return inter;
+}
+function tokens(s) {
+    const stop = new Set(["de", "du", "des", "la", "le", "les", "et", "au", "aux", "sur", "en", "d", "l", "un", "une"]);
+    return new Set(normalizeText(s)
+        .split(" ")
+        .filter((w) => w.length >= 3 && !stop.has(w)));
+}
+function timeProximityBonus(eventStartISO, blockText) {
+    if (!eventStartISO)
+        return 0;
+    const dt = parseISO(eventStartISO);
+    if (!dt)
+        return 0;
+    const eventMin = dt.hour * 60 + dt.minute;
+    const hints = extractHourHints(blockText);
+    if (!hints.length)
+        return 0;
+    let best = Infinity;
+    for (const x of hints)
+        best = Math.min(best, Math.abs(x.h * 60 + x.m - eventMin));
+    if (best <= 30)
+        return 0.1;
+    if (best <= 60)
+        return 0.06;
+    if (best <= 120)
+        return 0.03;
+    return 0;
+}
+function extractHourHints(text) {
+    const t = (text || "").toLowerCase();
+    const out = [];
+    const re = /\b(\d{1,2})\s*h\s*(\d{2})\b/g;
+    let m;
+    while ((m = re.exec(t)))
+        out.push({ h: Number(m[1]), m: Number(m[2]) });
+    return out;
+}
+export function scoreVideo(agenda, agendaTs, sameOrg, videoTitle, videoEpoch, videoOrganes, timeAmbigious = false) {
+    const objetS = dice(agenda.objet || "", videoTitle || "");
+    const titleS = dice(agenda.titre || "", videoTitle || "");
+    const titleScore = Math.max(objetS, titleS);
+    let timeScore = 0;
+    if (agendaTs && videoEpoch) {
+        const deltaMin = Math.abs(videoEpoch - agendaTs) / 60;
+        timeScore = Math.exp(-deltaMin / 60);
+    }
+    let orgScore = 0;
+    if (agenda.organe && videoOrganes && videoOrganes.length) {
+        const agendaOrg = agenda.organe;
+        orgScore = Math.max(...videoOrganes.map((v) => dice(agendaOrg, v)));
+    }
+    if (timeAmbigious) {
+        return 0.6 * titleScore + (sameOrg ? 0.4 : orgScore * 0.4);
+    }
+    return 0.3 * titleScore + 0.3 * timeScore + (sameOrg ? 0.4 : orgScore * 0.4);
+}
+function tokensDice(s) {
+    return normalize(s).split(" ").filter(Boolean);
+}
+export function dice(a, b) {
+    const A = new Set(tokensDice(a)), B = new Set(tokensDice(b));
+    if (!A.size || !B.size)
+        return 0;
+    let inter = 0;
+    for (const t of A)
+        if (B.has(t))
+            inter++;
+    return (2 * inter) / (A.size + B.size);
+}
+export function normalize(s) {
+    return (s ?? "")
+        .toLowerCase()
+        .normalize("NFD")
+        .replace(/[\u0300-\u036f]/g, "")
+        .replace(/[^\p{L}\p{N}\s-]/gu, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+}

package/lib/utils/string_cleaning.d.ts

@@ -0,0 +1,7 @@
+export declare function normalizeText(t: string): string;
+export declare function decodeHtmlEntities(s?: string | null): string;
+export declare function stripTrailingPunct(s: string): string;
+export declare function dedupeSpeaker(raw: string): string;
+export declare function fixApostrophes(s: string): string;
+export declare function norm(s?: string | null): string;
+export declare function normalizeSpaces(s: string): string;

package/lib/utils/string_cleaning.js

@@ -0,0 +1,57 @@
+export function normalizeText(t) {
+    return (t || "")
+        .toLowerCase()
+        .normalize("NFD")
+        .replace(/\p{Diacritic}/gu, "")
+        .replace(/[^a-z0-9\s]/g, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+}
+export function decodeHtmlEntities(s) {
+    if (!s)
+        return "";
+    return s
+        .replace(/&#x([0-9a-fA-F]+);/g, (_, h) => String.fromCodePoint(parseInt(h, 16)))
+        .replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10)))
+        .replace(/&/g, "&")
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&quot;/g, '"')
+        .replace(/&apos;/g, "'");
+}
+export function stripTrailingPunct(s) {
+    return s.replace(/\s*([:,.;])\s*$/u, "").trim();
+}
+export function dedupeSpeaker(raw) {
+    let s = norm(raw);
+    s = stripTrailingPunct(s);
+    const dupPatterns = [/^(.+?)\s*[.]\s*\1$/u, /^(.+?)\s*,\s*\1,?$/u, /^(.+?)\s+\1$/u];
+    for (const re of dupPatterns) {
+        const m = s.match(re);
+        if (m) {
+            s = m[1];
+            break;
+        }
+    }
+    return s.replace(/\.\s*$/, "");
+}
+export function fixApostrophes(s) {
+    let out = s;
+    out = out.replace(/\s*’\s*/g, "’");
+    out = out.replace(/\b([dljctmsn])\s*’/gi, (_, m) => m + "’");
+    out = out.replace(/’\s+([A-Za-zÀ-ÖØ-öø-ÿ])/g, "’$1");
+    out = out.replace(/\s+([,;:.!?])/g, "$1");
+    return out;
+}
+export function norm(s) {
+    return (s || "")
+        .replace(/\u00A0/g, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+}
+export function normalizeSpaces(s) {
+    return s
+        .replace(/\u00A0/g, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.20.17",
+  "version": "2.20.19",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",