@tricoteuses/senat 2.22.9 → 2.22.10

package/lib/scripts/test_iter_load.js

@@ -0,0 +1,12 @@
+import { iterLoadSenatRapports, } from "../loaders";
+import commandLineArgs from "command-line-args";
+import { dataDirDefaultOption } from "./shared/cli_helpers";
+const optionsDefinitions = [dataDirDefaultOption];
+const options = commandLineArgs(optionsDefinitions);
+const session = 2024;
+const sinceCommit = undefined;
+for (const { item: rapport } of iterLoadSenatRapports(options["dataDir"], session, {
+    sinceCommit: sinceCommit,
+})) {
+    console.log(rapport);
+}

package/lib/src/utils/nvs-timecode.d.ts

@@ -0,0 +1,17 @@
+export type L1Chapter = {
+    id: string;
+    label: string;
+    index: number;
+};
+export declare function getLevel1Chapters(dataNvs: string): L1Chapter[];
+export declare function pickBestLevel1ChapterForAgenda(chapters: L1Chapter[], agendaTitle: string): {
+    chapter: L1Chapter;
+    score: number;
+} | null;
+export declare function getAgendaSegmentTimecodes(dataNvs: string, finalPlayerNvs: string, agendaTitleOrObjet: string): {
+    start: number;
+    end: number | null;
+    chapterId: string;
+    nextChapterId: string | null;
+    score: number;
+} | null;

package/lib/src/utils/nvs-timecode.js

@@ -0,0 +1,79 @@
+import { XMLParser } from "fast-xml-parser";
+import { dice, normalize } from "./scoring";
+import { decodeHtmlEntities } from "./string_cleaning";
+const CHAPTER_MATCH_THRESHOLD = 0.5;
+const xmlParser = new XMLParser({
+    ignoreAttributes: false,
+    attributeNamePrefix: "@_",
+});
+function getTimecodeForChapterId(finalPlayerNvs, chapterId) {
+    const xml = xmlParser.parse(finalPlayerNvs);
+    const synchros = xml?.player?.synchro;
+    if (!synchros)
+        return null;
+    const synchsArray = Array.isArray(synchros) ? synchros : [synchros];
+    const match = synchsArray.find((s) => String(s["@_id"]) === String(chapterId));
+    if (!match)
+        return null;
+    const rawTimecode = match["@_timecode"];
+    if (rawTimecode == null)
+        return null;
+    const ms = Number(rawTimecode);
+    if (Number.isNaN(ms))
+        return null;
+    return Math.floor(ms / 1000);
+}
+function toArray(v) {
+    if (!v)
+        return [];
+    return Array.isArray(v) ? v : [v];
+}
+export function getLevel1Chapters(dataNvs) {
+    const xml = xmlParser.parse(dataNvs);
+    const root = xml?.data?.chapters?.chapter ?? xml?.chapters?.chapter;
+    const roots = toArray(root);
+    return roots
+        .map((ch, i) => {
+        const id = ch?.id ?? ch?.["@_id"];
+        const labelRaw = ch?.label ?? ch?.["@_label"] ?? "";
+        return {
+            id: String(id),
+            label: decodeHtmlEntities(String(labelRaw)).trim(),
+            index: i,
+        };
+    })
+        .filter((c) => c.id && c.label);
+}
+export function pickBestLevel1ChapterForAgenda(chapters, agendaTitle) {
+    const q = normalize(agendaTitle);
+    let best = null;
+    for (const ch of chapters) {
+        const s = dice(q, ch.label);
+        if (!best || s > best.score)
+            best = { chapter: ch, score: s };
+    }
+    if (!best || best.score < CHAPTER_MATCH_THRESHOLD)
+        return { chapter: chapters[0], score: 0 };
+    return best;
+}
+export function getAgendaSegmentTimecodes(dataNvs, finalPlayerNvs, agendaTitleOrObjet) {
+    const l1 = getLevel1Chapters(dataNvs);
+    if (!l1.length)
+        return null;
+    const best = pickBestLevel1ChapterForAgenda(l1, agendaTitleOrObjet);
+    if (!best)
+        return null;
+    const chapter = best.chapter;
+    const next = l1[chapter.index + 1] ?? null;
+    const start = getTimecodeForChapterId(finalPlayerNvs, chapter.id);
+    if (start == null)
+        return null;
+    const end = next ? getTimecodeForChapterId(finalPlayerNvs, next.id) : null;
+    return {
+        start,
+        end,
+        chapterId: chapter.id,
+        nextChapterId: next?.id ?? null,
+        score: best.score,
+    };
+}

package/lib/src/utils/scoring.d.ts

@@ -13,6 +13,9 @@ export declare function isNoiseBlock(text: string): boolean;
 export declare function scoreSommaireBlockForEvent(blockText: string, ev: AgendaEvent): number;
 export declare function getOrgKey(norm: string): string;
 export declare function dice(a: string, b: string): number;
+export declare function coverage(reference?: string | null, candidate?: string | null): number;
+export declare function diceFiltered(a?: string | null, b?: string | null): number;
+export declare function similarityScore(a?: string | null, b?: string | null): number;
 export declare function normalize(s?: string | null): string;
 export declare function normalizeSalle(s?: string | null): string | null;
 export declare function scoreVideo(agenda: Reunion, agendaTs: number | null, sameOrg: boolean, w: VideoScoreWeights, videoTitle?: string, videoEpoch?: number, videoOrganes?: string[], timeAmbigious?: boolean, salle?: string, chapterTitles?: L1Chapter[]): {

package/lib/src/utils/scoring.js

@@ -154,6 +154,38 @@ export function dice(a, b) {
             inter++;
     return (2 * inter) / (A.size + B.size);
 }
+function tokenSet(s) {
+    if (!s)
+        return new Set();
+    return new Set(tokens(s));
+}
+export function coverage(reference, candidate) {
+    const A = tokenSet(reference);
+    const B = tokenSet(candidate);
+    if (!A.size || !B.size)
+        return 0;
+    let inter = 0;
+    for (const t of A)
+        if (B.has(t))
+            inter++;
+    return inter / A.size;
+}
+export function diceFiltered(a, b) {
+    const A = tokenSet(a);
+    const B = tokenSet(b);
+    if (!A.size || !B.size)
+        return 0;
+    let inter = 0;
+    for (const t of A)
+        if (B.has(t))
+            inter++;
+    return (2 * inter) / (A.size + B.size);
+}
+export function similarityScore(a, b) {
+    const cov = coverage(a, b);
+    const d = diceFiltered(a, b);
+    return 0.7 * cov + 0.3 * d;
+}
 export function normalize(s) {
     return (s ?? "")
         .toLowerCase()
@@ -174,13 +206,13 @@ export function normalizeSalle(s) {
 }
 export function scoreVideo(agenda, agendaTs, sameOrg, w, videoTitle, videoEpoch, videoOrganes, timeAmbigious = false, salle, chapterTitles) {
     const weights = w;
-    const objetS = dice(agenda.objet || "", videoTitle || "");
-    const titleS = dice(agenda.titre || "", videoTitle || "");
+    const objetS = similarityScore(agenda.objet || "", videoTitle || "");
+    const titleS = similarityScore(agenda.titre || "", videoTitle || "");
     let titleScore = Math.max(objetS, titleS);
     chapterTitles = chapterTitles || [];
     for (const ch of chapterTitles) {
-        const chObjetS = dice(agenda.objet || "", ch.label);
-        const chTitreS = dice(agenda.titre || "", ch.label);
+        const chObjetS = similarityScore(agenda.objet || "", ch.label);
+        const chTitreS = similarityScore(agenda.titre || "", ch.label);
         titleScore = Math.max(titleScore, Math.max(chObjetS, chTitreS));
     }
     let timeScore = 0;
@@ -190,8 +222,7 @@ export function scoreVideo(agenda, agendaTs, sameOrg, w, videoTitle, videoEpoch,
     }
     let orgScore = 0;
     if (agenda.organe && videoOrganes?.length) {
-        // NOTE: if you already normalize organes elsewhere, keep it there.
-        orgScore = Math.max(...videoOrganes.map((v) => dice(agenda.organe, v)));
+        orgScore = Math.max(...videoOrganes.map((v) => similarityScore(agenda.organe, v)));
     }
     // Salle: normalized "A263" matching
     let salleScore = 0;

package/lib/src/utils/weights_scoring_config.d.ts

@@ -0,0 +1,2 @@
+import { MatchWeights } from "./scoring";
+export declare const weights: MatchWeights;

package/lib/src/utils/weights_scoring_config.js

@@ -0,0 +1,15 @@
+// BASED ON TESTS RESULTS
+// these weights yield good results in the benchmark test suite
+// aiming at 0 WRONG matches while maximizing HITs
+export const weights = {
+    wTitle: 0.7,
+    wOrg: 0.1,
+    wSalle: 0,
+    wTime: 0.2,
+    sameOrgBonus: 0.2,
+    minAccept: 0.5,
+    margin: 0.1,
+    titleMin: 0.2,
+    titleDominance: 0,
+    orgUncertainPenalty: 0.8,
+};

package/lib/src/videos/types.d.ts

@@ -12,8 +12,8 @@ export type MatchResult = {
     reason?: "margin_ambiguous_time_sp";
 };
 export type MatchWeights = {
-    minAccept?: number;
-    margin?: number;
+    minAccept: number;
+    margin: number;
     titleDominance?: number;
     orgUncertainPenalty?: number;
     orgSkipDice?: number;

package/lib/strings.d.ts

@@ -0,0 +1 @@
+export declare function slugify(string: string, replacement?: string): string;

package/lib/strings.js

@@ -0,0 +1,18 @@
+import originalSlugify from "slug";
+const slugifyCharmap = {
+    ...originalSlugify.defaults.charmap,
+    "'": " ",
+    "@": " ",
+    ".": " ",
+    "-": "_",
+};
+export function slugify(string, replacement) {
+    const options = {
+        charmap: slugifyCharmap,
+        mode: "rfc3986",
+    };
+    if (replacement) {
+        options.replacement = replacement;
+    }
+    return originalSlugify(string, options);
+}

package/lib/types/agenda.d.ts

@@ -0,0 +1,44 @@
+export interface AgendaEvent {
+    id: string;
+    type: string | null;
+    date: string;
+    startTime: string | null;
+    endTime: string | null;
+    timeOriginal: string | null;
+    titre: string;
+    organe: string | null;
+    objet: string | null;
+    lieu: string | null;
+    captationVideo: boolean;
+    urlDossierSenat: string | null;
+    quantieme: string | null;
+}
+export type TimeSlot = "MATIN" | "APRES-MIDI" | "SOIR" | "UNKNOWN";
+export interface Reunion {
+    uid: string;
+    chambre: "SN";
+    date: string;
+    startTime: string | null;
+    endTime: string | null;
+    captationVideo: boolean;
+    titre: string;
+    type: string;
+    organe?: string;
+    objet?: string;
+    lieu?: string;
+    events: AgendaEvent[];
+    compteRenduRefUid?: string;
+    transcriptionRef?: string;
+    urlVideo?: string;
+    timecodeDebutVideo?: number;
+    timecodeFinVideo?: number;
+    odj?: ReunionOdj;
+}
+export interface ReunionOdjPoint {
+    objet: string | null;
+    dossierLegislatifRef: string | null;
+    codeEtape: string | null;
+}
+export interface ReunionOdj {
+    pointsOdj: ReunionOdjPoint[];
+}

package/lib/types/agenda.js

@@ -0,0 +1 @@
+export {};

package/lib/types/ameli.d.ts

@@ -0,0 +1,5 @@
+import { ses as Ses, sub as Sub, subFields, txt_ameli } from "../raw_types_schemats/ameli";
+export type { Ses, Sub };
+export interface TxtAmeli extends txt_ameli {
+    subids?: subFields.id[];
+}

package/lib/types/ameli.js

@@ -0,0 +1 @@
+export {};

package/lib/types/compte_rendu.d.ts

@@ -0,0 +1,83 @@
+export interface CompteRendu {
+    uid: string;
+    seanceRef: string;
+    sessionRef: string;
+    metadonnees: Metadonnees;
+    contenu: Contenu;
+}
+export interface Metadonnees {
+    dateSeance: string;
+    dateSeanceJour: string;
+    numSeanceJour: string;
+    numSeance: string;
+    typeAssemblee: "AN" | "SN";
+    legislature: string;
+    session: string;
+    nomFichierJo: string;
+    validite: string;
+    etat: string;
+    diffusion: string;
+    version: string;
+    environnement: string;
+    heureGeneration: Date;
+    sommaire?: Sommaire;
+}
+export interface Contenu {
+    quantiemes: Quantiemes;
+    ouvertureSeance?: Point[] | Point;
+    point: Point[] | Point;
+    finSeance?: FinSeance;
+    paragraphe?: Point[];
+}
+export interface FinSeance {
+    point: Point;
+}
+export interface Quantiemes {
+    journee: string;
+    session: string;
+}
+export interface Point {
+    ordre_absolu_seance: string;
+    code_grammaire: string;
+    roledebat?: string;
+    orateurs?: {
+        orateur: {
+            nom: string;
+            id: string;
+            qualite: string;
+        };
+    };
+    texte: {
+        _: string;
+    };
+    code_style?: string;
+}
+export interface Texte {
+    _?: string;
+    id_syceron?: string;
+    stime?: string;
+    sup?: string;
+    lienAdt?: Texte[] | Texte;
+}
+export interface Sommaire {
+    presidentSeance: Texte;
+    sommaire1: SommaireElement[] | SommaireElement;
+    sommaire3?: SommaireElement[] | SommaireElement;
+    sommaire2?: SommaireElement[] | SommaireElement;
+    para?: Texte[] | Texte;
+}
+export interface SommaireElement {
+    valeur_pts_odj: string | undefined;
+    titreStruct: TitreStruct;
+    para?: Array<Texte | string> | Texte;
+    sommaire2?: SommaireElement[] | SommaireElement;
+    sommaire3?: SommaireElement[] | SommaireElement;
+    presidentSeance?: Texte[] | Texte;
+    type_debat?: string;
+}
+export interface TitreStruct {
+    id_syceron: string;
+    intitule?: string;
+    sousIntitule?: string;
+    type_debat?: string;
+}

package/lib/types/compte_rendu.js

@@ -0,0 +1 @@
+export {};

package/lib/types/debats.d.ts

@@ -0,0 +1,2 @@
+import { debats as Debat, lecassdeb as LecAssDeb } from "../raw_types_schemats/debats";
+export type { Debat, LecAssDeb };

package/lib/types/debats.js

@@ -0,0 +1 @@
+export {};

package/lib/types/dosleg.d.ts

@@ -0,0 +1,70 @@
+import { txt_ameliFields } from "../raw_types_schemats/ameli";
+import { ass as Ass, aud, audFields, auteur, date_seance, date_seanceFields as dateSeanceFields, deccoc as DecCoc, denrap as DenRap, docatt, docattFields, ecr, ecrFields, etaloi as EtaLoi, lecass, lecassFields, lecassrap, lecture, lectureFields, loi, org as Org, oritxt as OriTxt, qua as Qua, rap, raporg as RapOrg, raporgFields, scr as Scr, texte, texteFields, typatt as TypAtt, typlec as TypLec, typloi as TypLoi, typtxt as TypTxt, typurl as TypUrl, typurlFields } from "../raw_types_schemats/dosleg";
+import { Debat } from "./debats";
+import { TxtAmeli } from "./ameli";
+export type { Ass, DecCoc, DenRap, EtaLoi, Org, OriTxt, Qua, RapOrg, Scr, TypAtt, TypLec, TypLoi, TypTxt, TypUrl };
+export interface Aud extends aud {
+    org?: Org;
+}
+export interface Auteur extends auteur {
+    qua?: Qua;
+}
+export interface DateSeance extends date_seance {
+    debat?: Debat;
+    scrids?: string[];
+    scrs?: Scr[];
+}
+export interface DocAtt extends docatt {
+    rap?: Rap;
+    typatt?: TypAtt;
+}
+export interface Ecr extends ecr {
+    aut?: Auteur;
+}
+export interface LecAss extends lecass {
+    ass?: Ass;
+    auds?: Aud[];
+    audcles?: audFields.audcle[];
+    datesSeances?: DateSeance[];
+    datesSeancesCodes?: dateSeanceFields.code[];
+    debatdatseas: Date[];
+    lecassraps?: LecAssRap[];
+    lecassrapids?: string[];
+    org?: Org;
+    texcods: texteFields.texcod[];
+    textes?: Texte[];
+}
+export interface LecAssRap extends lecassrap {
+    rap?: Rap;
+}
+export interface Lecture extends lecture {
+    typlec?: TypLec;
+    lecassidts?: lecassFields.lecassidt[];
+    lecasss?: LecAss[];
+}
+export interface Loi extends loi {
+    deccoc?: DecCoc;
+    etaloi?: EtaLoi;
+    lecidts?: lectureFields.lecidt[];
+    lectures?: Lecture[];
+    typloi?: TypLoi;
+}
+export interface Rap extends rap {
+    denrap?: DenRap;
+    docattcles?: docattFields.docattcle[];
+    docatts?: DocAtt[];
+    ecrnums?: ecrFields.ecrnum[];
+    ecrs?: Ecr[];
+    orgcods?: raporgFields.orgcod[];
+    orgs?: Org[];
+}
+export interface Texte extends texte {
+    ecrs?: Ecr[];
+    ecrnums?: ecrFields.ecrnum[];
+    libtypurl?: typurlFields.libtypurl;
+    org?: Org;
+    oritxt: OriTxt;
+    typtxt?: TypTxt;
+    txtAmeli?: TxtAmeli;
+    txtAmeliId: txt_ameliFields.id;
+}

package/lib/types/dosleg.js

@@ -0,0 +1 @@
+export {};

package/lib/types/questions.d.ts

@@ -0,0 +1,2 @@
+import { tam_questions as Question } from "../raw_types_schemats/questions";
+export type { Question };

package/lib/types/questions.js

@@ -0,0 +1 @@
+export {};

package/lib/types/sens.d.ts

@@ -0,0 +1,10 @@
+import { sen as Sen } from "../raw_types_schemats/sens";
+export type { Sen };
+export interface Photo {
+    chemin: string;
+    cheminMosaique: string;
+    hauteur: number;
+    largeur: number;
+    xMosaique: number;
+    yMosaique: number;
+}

package/lib/types/sens.js

@@ -0,0 +1 @@
+export {};

package/lib/types/sessions.d.ts

@@ -0,0 +1,5 @@
+export declare const UNDEFINED_SESSION = 0;
+declare const sessions: readonly [0, 1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026];
+export type Session = (typeof sessions)[number];
+export declare function getSessionsFromStart(startSession: Session): (0 | 1958 | 1959 | 1960 | 1961 | 1962 | 1963 | 1964 | 1965 | 1966 | 1967 | 1968 | 1969 | 1970 | 1971 | 1972 | 1973 | 1974 | 1975 | 1976 | 1977 | 1978 | 1979 | 1980 | 1981 | 1982 | 1983 | 1984 | 1985 | 1986 | 1987 | 1988 | 1989 | 1990 | 1991 | 1992 | 1993 | 1994 | 1995 | 1996 | 1997 | 1998 | 1999 | 2000 | 2001 | 2002 | 2003 | 2004 | 2005 | 2006 | 2007 | 2008 | 2009 | 2010 | 2011 | 2012 | 2013 | 2014 | 2015 | 2016 | 2017 | 2018 | 2019 | 2020 | 2021 | 2022 | 2023 | 2024 | 2025 | 2026)[];
+export {};

package/lib/types/sessions.js

@@ -0,0 +1,84 @@
+export const UNDEFINED_SESSION = 0;
+const sessions = [
+    UNDEFINED_SESSION,
+    1958,
+    1959,
+    1960,
+    1961,
+    1962,
+    1963,
+    1964,
+    1965,
+    1966,
+    1967,
+    1968,
+    1969,
+    1970,
+    1971,
+    1972,
+    1973,
+    1974,
+    1975,
+    1976,
+    1977,
+    1978,
+    1979,
+    1980,
+    1981,
+    1982,
+    1983,
+    1984,
+    1985,
+    1986,
+    1987,
+    1988,
+    1989,
+    1990,
+    1991,
+    1992,
+    1993,
+    1994,
+    1995,
+    1996,
+    1997,
+    1998,
+    1999,
+    2000,
+    2001,
+    2002,
+    2003,
+    2004,
+    2005,
+    2006,
+    2007,
+    2008,
+    2009,
+    2010,
+    2011,
+    2012,
+    2013,
+    2014,
+    2015,
+    2016,
+    2017,
+    2018,
+    2019,
+    2020,
+    2021,
+    2022,
+    2023,
+    2024,
+    2025,
+    2026,
+    // TO COMPLETE EVERY YEAR :)
+];
+export function getSessionsFromStart(startSession) {
+    if (startSession === 0) {
+        return Array.from(sessions);
+    }
+    const sessionIndex = sessions.findIndex((session) => startSession === session);
+    if (sessionIndex >= 0) {
+        return sessions.slice(sessionIndex);
+    }
+    return [];
+}

package/lib/types/texte.d.ts

@@ -0,0 +1,74 @@
+export declare enum DivisionType {
+    tome = 1,
+    part = 2,
+    book = 3,
+    title = 4,
+    subtitle = 5,
+    chapter = 6,
+    section = 7,
+    subsection = 8,
+    paragraph = 9,
+    article = 10,
+    alinea = 11,
+    division = 12
+}
+export type DivisionTag = keyof typeof DivisionType;
+export interface DocumentMetadata {
+    name: string;
+    session: number | null | undefined;
+    date?: string | null;
+    url_expose_des_motifs?: URL;
+    url_xml?: URL;
+    url_html: URL;
+    url_pdf: URL;
+}
+export interface FlatTexte {
+    titre: string | null;
+    titreCourt: string | null;
+    signetDossier: string | null;
+    urlDossierSenat: string | null;
+    urlDossierAssemblee: string | null;
+    type: string | null;
+    session: string | null;
+    numero: number | null;
+    datePresentation: Date | null;
+    dateDepot: Date | null;
+    datePublicationXml: Date | null;
+    version: Version | null;
+    workflow: Step[];
+    divisions: Division[];
+    exposeDesMotifs?: ExposeDesMotifs | null;
+}
+export type Version = "RECT" | "RECT_BIS" | "RECT_TER" | "RECT_QUATER" | "RECT_QUINQUIES";
+export interface Step {
+    eId: string;
+    date: Date | null;
+    type: string | null;
+    session: string | null;
+    numero: string | null;
+    version: Version | null;
+    outcome: string | null;
+}
+export interface Division {
+    index: number;
+    eId: string;
+    tag: DivisionTag;
+    level: number;
+    headings: DivisionContent[];
+}
+export interface Article extends Division {
+    alineas: Alinea[];
+}
+export interface DivisionContent {
+    text: string | null;
+    html?: string | null;
+}
+export interface Alinea extends DivisionContent {
+    eId: string;
+    heading: DivisionContent;
+    pastille: string | null;
+}
+export interface ExposeDesMotifs {
+    text: string | null;
+    html: string | null;
+}

package/lib/types/texte.js

@@ -0,0 +1,16 @@
+/* eslint no-use-before-define: 0 */
+export var DivisionType;
+(function (DivisionType) {
+    DivisionType[DivisionType["tome"] = 1] = "tome";
+    DivisionType[DivisionType["part"] = 2] = "part";
+    DivisionType[DivisionType["book"] = 3] = "book";
+    DivisionType[DivisionType["title"] = 4] = "title";
+    DivisionType[DivisionType["subtitle"] = 5] = "subtitle";
+    DivisionType[DivisionType["chapter"] = 6] = "chapter";
+    DivisionType[DivisionType["section"] = 7] = "section";
+    DivisionType[DivisionType["subsection"] = 8] = "subsection";
+    DivisionType[DivisionType["paragraph"] = 9] = "paragraph";
+    DivisionType[DivisionType["article"] = 10] = "article";
+    DivisionType[DivisionType["alinea"] = 11] = "alinea";
+    DivisionType[DivisionType["division"] = 12] = "division";
+})(DivisionType || (DivisionType = {}));