@tricoteuses/senat 2.20.17 → 2.20.19

package/README.md

@@ -29,40 +29,79 @@ docker run --name local-postgres -d -p 5432:5432 -e POSTGRES_PASSWORD=$YOUR_CUST
 
 ## Download data
 
+### Basic usage
+
 Create a folder where the data will be downloaded and run the following command to download the data and convert it into JSON files.
 
 ```bash
 mkdir ../senat-data/
 
-# Available options for optional `categories` parameter : All,  Ameli, Debats, DosLeg, Questions, Sens
-npm run data:download ../senat-data -- [--categories All]
+npm run data:download ../senat-data
 ```
 
-Data from other sources is also available :
+### Available Commands
+
+- `npm run data:download <dir>`: Download, convert data to JSON
+- `npm run data:retrieve_documents <dir>`: Retrieval of textes and rapports from Sénat's website
+- `npm run data:parse_textes_lois <dir>`: Parse textes (requires xml files)
+- `npm run data:retrieve_agenda <dir>`: Retrieval of agenda from Sénat's website
+- `npm run data:retrieve_cr_seance <dir>`: Retrieval of comptes-rendus de séance from Sénat's data
+- `npm run data:retrieve_cr_commission <dir>`: Retrieval of comptes-rendus de commissions from Sénat's website
+- `npm run data:retrieve_senateurs_photos <dir>`: Retrieval of sénateurs' pictures from Sénat's website
+
+### Filtering Options
+
+Downloading all the data is long and takes up a lot of disk space. It is possible to choose the type of data that you want to retrieve to reduce the load.
+
+Examples:
 
 ```bash
-# Retrieval of textes and rapports from Sénat's website
-# Available options for optional `formats` parameter : xml, html, pdf
-# Available options for optional `types` parameter : textes, rapports
-npm run data:retrieve_documents ../senat-data -- --fromSession 2022 [--formats xml pdf] [--types textes]
+# Only download amendments
+npm run data:download ../senat-data -- -k Ameli
 
-# Retrieval & parsing (textes in xml format only for now)
-npm run data:retrieve_documents ../senat-data -- --fromSession 2022 --parseDocuments
+# Only process data from session 2023 onwards
+npm run data:download ../senat-data -- --fromSession 2023
+```
 
-# Parsing only
-npm run data:parse_textes_lois ../senat-data
+### Common Options
 
-# Retrieval (& parsing) of agenda from Sénat's website
-npm run data:retrieve_agenda ../senat-data -- --fromSession 2022 [--parseAgenda]
+- `--categories` or `-k <name>`: Filter by dataset categories (Available options: `All`, `Ameli`, `Debats`, `DosLeg`, `Questions`, `Sens`)
+- `--fromSession <year>`: Specify the session year to retrieve data from (default: 2022)
+- `--dataDir <path>` (Mandatory): Path to the working directory where all data is stored (required)
+- `--silent` or `-s`: Disable logging
+- `--verbose` or `-v`: Enable verbose logging
+- `--commit` or `-c`: Automatically commit converted data
+- `--pull` or `-p`: Pull repositories before starting
+- `--clone` or `-C <url>`: Clone Git repositories from a remote group or organization
+- `--remote` or `-r <name>`: Push commits to specified Git remote(s)
+- `--keepDir`: Keep directories when cleaning data
+- `--only-recent <days>`: Retrieve only documents created within the last N days
+
+### Options for Retrieving Documents
+
+- `--formats <format>`: Specify document formats to retrieve (options: `xml`, `html`, `pdf`)
+- `--types <type>`: Specify document types to retrieve (options: `textes`, `rapports`)
+- `--parseDocuments`: Parse documents after retrieval
+- `--parseAgenda`: Parse agenda after retrieval
+- `--parseDebats`: Parse comptes-rendus after retrieval
+
+#### Examples
+
+```bash
+# Retrieval of textes and rapports in specific formats
+npm run data:retrieve_documents ../senat-data -- --fromSession 2022 --formats xml pdf --types textes
+
+# Retrieval & parsing (textes in xml format only for now)
+npm run data:retrieve_documents ../senat-data -- --fromSession 2022 --parseDocuments
 
-# Retrieval (& parsing) of comptes-rendus de séance from Sénat's data
-npm run data:retrieve_cr_seance ../senat-data -- [--parseDebats] [--keepDir]
+# Retrieval & parsing of agenda
+npm run data:retrieve_agenda ../senat-data -- --fromSession 2022 --parseAgenda
 
-# Retrieval (& parsing) of comptes-rendus de commissions from Sénat's website
-npm run data:retrieve_cr_commission ../senat-data -- [--parseDebats] [--keepDir]
+# Retrieval & parsing of comptes-rendus de séance
+npm run data:retrieve_cr_seance ../senat-data -- --parseDebats --keepDir
 
-# Retrieval of sénateurs' pictures from Sénat's website
-npm run data:retrieve_senateurs_photos ../senat-data
+# Retrieval & parsing of comptes-rendus de commissions
+npm run data:retrieve_cr_commission ../senat-data -- --parseDebats --keepDir
 ```
 
 ## Data download using Docker

package/lib/git.d.ts

@@ -0,0 +1,26 @@
+export declare function initRepo(repositoryDir: string): void;
+export declare function commit(repositoryDir: string, message: string): boolean;
+export declare function commitAndPush(repositoryDir: string, message: string, remotes?: string[]): number;
+export declare function resetAndPull(gitDir: string): boolean;
+export declare function clone(gitGroupUrl: string | undefined, gitName: string, workingDir: string): void;
+export declare function run(repositoryDir: string, args: string, verbose?: boolean): string;
+export declare function test(repositoryDir: string, args: string, verbose?: boolean): boolean;
+/**
+ * Information about a changed file in git
+ */
+export interface GitChangedFile {
+    path: string;
+    status: "A" | "M" | "D" | "R" | "C" | "T" | "U";
+}
+/**
+ * Get the list of files that have changed since a specific commit in a git repository.
+ * @param repositoryDir The directory of the git repository
+ * @param sinceCommit The commit hash to compare against (e.g., "HEAD~1", "abc123", etc.)
+ * @param options Options for filtering
+ * @param options.diffFilter Git diff-filter string (default: "AMR").
+ *        A=Added, M=Modified, D=Deleted, R=Renamed, C=Copied, T=Type changed, U=Unmerged
+ * @returns A Map of file paths to their git status
+ */
+export declare function getChangedFilesSinceCommit(repositoryDir: string, sinceCommit: string, options?: {
+    diffFilter?: string;
+}): Map<string, GitChangedFile["status"]>;

package/lib/git.js

@@ -0,0 +1,167 @@
+import { execSync } from "node:child_process";
+import fs from "fs-extra";
+import path from "node:path";
+const MAXBUFFER = 50 * 1024 * 1024;
+export function initRepo(repositoryDir) {
+    if (!fs.existsSync(path.join(repositoryDir, ".git"))) {
+        fs.ensureDirSync(repositoryDir);
+        execSync("git init", {
+            cwd: repositoryDir,
+            env: process.env,
+            encoding: "utf-8",
+            stdio: ["ignore", "ignore", "pipe"],
+        });
+    }
+}
+export function commit(repositoryDir, message) {
+    initRepo(repositoryDir);
+    execSync("git add .", {
+        cwd: repositoryDir,
+        env: process.env,
+        encoding: "utf-8",
+        stdio: ["ignore", "ignore", "pipe"],
+        maxBuffer: MAXBUFFER,
+    });
+    try {
+        execSync(`git commit -m "${message}" --quiet`, {
+            cwd: repositoryDir,
+            env: process.env,
+            encoding: "utf-8",
+            stdio: ["ignore", "pipe", "pipe"],
+        });
+        return true;
+    }
+    catch (childProcess) {
+        if (childProcess.stdout === null ||
+            !/nothing to commit|rien à valider/.test(childProcess.stdout)) {
+            console.error(childProcess.output);
+            throw childProcess;
+        }
+        return false;
+    }
+}
+export function commitAndPush(repositoryDir, message, remotes) {
+    let exitCode = 0;
+    if (commit(repositoryDir, message)) {
+        for (const remote of remotes || []) {
+            try {
+                execSync(`git push ${remote} master`, {
+                    cwd: repositoryDir,
+                    env: process.env,
+                    encoding: "utf-8",
+                    stdio: ["ignore", "ignore", "pipe"],
+                });
+            }
+            catch (childProcess) {
+                // Don't stop when push fails.
+                console.error(childProcess.output);
+                exitCode = childProcess.status;
+            }
+        }
+    }
+    else {
+        // There was nothing to commit.
+        exitCode = 10;
+    }
+    return exitCode;
+}
+export function resetAndPull(gitDir) {
+    execSync("git reset --hard origin/master", {
+        cwd: gitDir,
+        env: process.env,
+        encoding: "utf-8",
+        stdio: ["ignore", "ignore", "pipe"],
+    });
+    execSync("git pull --rebase", {
+        cwd: gitDir,
+        env: process.env,
+        encoding: "utf-8",
+        stdio: ["ignore", "ignore", "pipe"],
+    });
+    return true;
+}
+export function clone(gitGroupUrl, gitName, workingDir) {
+    if (gitGroupUrl !== undefined) {
+        execSync(`git clone ${gitGroupUrl}/${gitName}.git`, {
+            cwd: workingDir,
+            env: process.env,
+            encoding: "utf-8",
+            stdio: ["ignore", "ignore", "pipe"],
+        });
+    }
+}
+export function run(repositoryDir, args, verbose) {
+    try {
+        if (verbose)
+            console.log(`git -C ${repositoryDir} ${args}`);
+        const output = execSync(`git ${args}`, {
+            cwd: repositoryDir,
+            maxBuffer: MAXBUFFER,
+        })
+            .toString()
+            .trim();
+        if (verbose)
+            console.log(output);
+        return output;
+    }
+    catch (childProcess) {
+        for (const output of ["stdout", "stderr"])
+            console.error(`${output}: ${childProcess[output]}`);
+        throw childProcess;
+    }
+}
+export function test(repositoryDir, args, verbose) {
+    try {
+        if (verbose)
+            console.log(`git -C ${repositoryDir} ${args}`);
+        const output = execSync(`git ${args}`, {
+            cwd: repositoryDir,
+            stdio: ["ignore", "pipe", "pipe"],
+            maxBuffer: MAXBUFFER,
+        })
+            .toString()
+            .trim();
+        if (verbose)
+            console.log(output);
+        return true;
+    }
+    catch (childProcess) {
+        if (childProcess.status != 0)
+            return false;
+        throw childProcess;
+    }
+}
+/**
+ * Get the list of files that have changed since a specific commit in a git repository.
+ * @param repositoryDir The directory of the git repository
+ * @param sinceCommit The commit hash to compare against (e.g., "HEAD~1", "abc123", etc.)
+ * @param options Options for filtering
+ * @param options.diffFilter Git diff-filter string (default: "AMR").
+ *        A=Added, M=Modified, D=Deleted, R=Renamed, C=Copied, T=Type changed, U=Unmerged
+ * @returns A Map of file paths to their git status
+ */
+export function getChangedFilesSinceCommit(repositoryDir, sinceCommit, options = {}) {
+    const { diffFilter } = options;
+    try {
+        // Using diff-filter: A = Added, M = Modified, R = Renamed, D = Deleted, etc.
+        // Default to AMR (excludes deleted files to prevent loading errors)
+        const filter = diffFilter ?? "AMR";
+        const output = run(repositoryDir, `diff --name-status --diff-filter=${filter} ${sinceCommit}`, false);
+        const changedFiles = new Map();
+        for (const line of output.split("\n")) {
+            if (line.trim().length === 0)
+                continue;
+            const parts = line.split("\t");
+            if (parts.length >= 2) {
+                const status = parts[0].charAt(0);
+                const path = parts[1];
+                changedFiles.set(path, status);
+            }
+        }
+        return changedFiles;
+    }
+    catch (error) {
+        console.error(`Error getting changed files since commit ${sinceCommit}:`, error);
+        return new Map();
+    }
+}

package/lib/index.d.ts

@@ -3,7 +3,7 @@ export type { DossierLegislatifResult } from "./model/dosleg";
 export type { ScrutinResult } from "./model/scrutins";
 export type { QuestionResult } from "./model/questions";
 export type { CirconscriptionResult, OrganismeResult, SenateurResult } from "./model/sens";
-export type { AgendaEvent, GroupedReunion, GroupedReunionOdjPoint } from "./types/agenda";
+export type { AgendaEvent, Reunion, ReunionOdjPoint } from "./types/agenda";
 export type { Ses, Sub, TxtAmeli } from "./types/ameli";
 export type { CompteRendu } from "./types/compte_rendu";
 export type { Debat, LecAssDeb } from "./types/debats";

package/lib/loaders.d.ts

@@ -4,7 +4,7 @@ import { DossierLegislatifResult } from "./model/dosleg";
 import { QuestionResult } from "./model/questions";
 import { ScrutinResult } from "./model/scrutins";
 import { CirconscriptionResult, OrganismeResult, SenateurResult } from "./model/sens";
-import { GroupedReunion } from "./types/agenda";
+import { Reunion } from "./types/agenda";
 import { FlatTexte } from "./types/texte";
 import { CompteRendu } from "./types/compte_rendu";
 export { EnabledDatasets } from "./datasets";
@@ -49,6 +49,7 @@ export interface DossierLegislatifDocumentResult {
     type_lecture: string;
     libelle_lecture: string;
     libelle_organisme: string | null;
+    code_organisme: string | null;
     numero: number | null;
     id: string | null;
     url: string;
@@ -86,7 +87,7 @@ export declare function loadSenatTexteContent(dataDir: string, textePathFromData
 export declare function loadSenatCompteRenduContent(dataDir: string, session: number, debatId: string | number): {
     item: CompteRendu | null;
 };
-export declare function iterLoadSenatAgendas(dataDir: string, session: number | undefined): Generator<IterItem<GroupedReunion>>;
+export declare function iterLoadSenatAgendas(dataDir: string, session: number | undefined): Generator<IterItem<Reunion>>;
 export declare function iterLoadSenatCirconscriptions(dataDir: string, options?: {}): Generator<IterItem<CirconscriptionResult>>;
 export declare function iterLoadSenatOrganismes(dataDir: string, options?: {}): Generator<IterItem<OrganismeResult>>;
 export declare function iterLoadSenatSenateurs(dataDir: string, options?: {}): Generator<IterItem<SenateurResult>>;

package/lib/model/commission.d.ts

@@ -1,6 +1,6 @@
 import * as cheerio from "cheerio";
 import { CompteRendu } from "../types/compte_rendu";
-import { GroupedReunion } from "../types/agenda";
+import { Reunion } from "../types/agenda";
 export declare function getRemainingTextAfterSpeakerHeader($: cheerio.CheerioAPI, $p: cheerio.Cheerio<any>): string;
 export type DaySection = {
     title: string;
@@ -14,5 +14,5 @@ export declare function parseCommissionCRSectionFromDom($: cheerio.CheerioAPI, h
     hourShort: string | null;
     organe?: string | null;
     section: DaySection;
-    matched?: GroupedReunion;
+    matched?: Reunion;
 }): CompteRendu | null;

package/lib/model/commission.js

@@ -1,8 +1,9 @@
 import * as cheerio from "cheerio";
 import path from "path";
-import { norm, toCRDate } from "./util";
-import { makeTypeGroupUid } from "../utils/reunion_grouping";
-import { frDateToISO, hourShortToStartTime } from "../utils/cr_spliting";
+import { makeReunionUid } from "../utils/reunion_parsing";
+import { norm } from "../utils/string_cleaning";
+import { frDateToISO, hourShortToStartTime } from "../utils/date";
+import { toCRDate } from "./util";
 const PARA_h3_SEL = "p.sh_justify, p.sh_center, p.sh_marge, p[align], li, h3";
 function findDayRoot($, targetISO) {
     let $root = $();
@@ -208,7 +209,7 @@ export function extractDayH3Sections($, dateISO) {
 export function parseCommissionCRSectionFromDom($, htmlFilePath, opts) {
     try {
         const { dateISO, hourShort, organe, section, matched } = opts;
-        const seanceRef = matched?.uid ?? makeTypeGroupUid(dateISO, "COM", matched?.events[0].id ?? hourShort ?? "", organe ?? undefined);
+        const seanceRef = matched?.uid ?? makeReunionUid(dateISO, "COM", matched?.events[0].id ?? hourShort ?? "", organe ?? undefined);
         const uid = seanceRef.replace(/^RU/, "CRC");
         const dateSeance = toCRDate(dateISO, matched?.startTime ?? hourShortToStartTime(hourShort));
         const $dayRoot = findDayRoot($, dateISO);

package/lib/model/seance.d.ts

@@ -1,9 +1,3 @@
-import { CompteRendu, Sommaire } from "../types/compte_rendu";
-import { TimeSlot } from "../types/agenda";
-export declare function parseCompteRenduSlotFromFile(xmlFilePath: string, wantedSlot: TimeSlot, firstSlotOfDay?: TimeSlot): Promise<CompteRendu | null>;
+import { CompteRendu } from "../types/compte_rendu";
+export declare function parseCompteRenduIntervalFromFile(xmlFilePath: string, startIndex: number, endIndex: number, agendaEventId: string): Promise<CompteRendu | null>;
 export declare function sessionStartYearFromDate(d: Date): number;
-export declare function parseYYYYMMDD(yyyymmdd: string): Date | null;
-export declare function deriveTitreObjetFromSommaire(sommaire: Sommaire | undefined, slot: TimeSlot): {
-    titre: string;
-    objet: string;
-};

package/lib/model/seance.js

@@ -1,33 +1,29 @@
 import fs from "fs";
 import * as cheerio from "cheerio";
-import path from "path";
-import { computeIntervalsBySlot } from "../utils/cr_spliting";
-import { decodeHtmlEntities, norm, toCRDate } from "./util";
-const asArray = (x) => (x == null ? [] : Array.isArray(x) ? x : [x]);
-const toInt = (s) => (Number.isFinite(Number(s)) ? Number(s) : Number.POSITIVE_INFINITY);
-export async function parseCompteRenduSlotFromFile(xmlFilePath, wantedSlot, firstSlotOfDay) {
+import { toCRDate } from "./util";
+import { makeReunionUid } from "../utils/reunion_parsing";
+import { yyyymmddFromPath } from "../utils/date";
+import { decodeHtmlEntities, dedupeSpeaker, fixApostrophes, norm } from "../utils/string_cleaning";
+export async function parseCompteRenduIntervalFromFile(xmlFilePath, startIndex, endIndex, agendaEventId) {
     try {
         const raw = fs.readFileSync(xmlFilePath, "utf8");
         const $ = cheerio.load(raw, { xml: false });
         const metadonnees = extractMetadonnees($, xmlFilePath);
         const order = $("body *").toArray();
         const idx = new Map(order.map((el, i) => [el, i]));
-        const intervalsAll = computeIntervalsBySlot($, idx, firstSlotOfDay);
-        const intervals = intervalsAll.filter((iv) => iv.slot === wantedSlot);
-        if (intervals.length === 0) {
-            console.warn(`[CRI] no intervals for ${path.basename(xmlFilePath)} [${wantedSlot}]`);
-            return null;
-        }
+        const totalNodes = order.length;
+        const clampedStart = Math.max(0, Math.min(startIndex, totalNodes - 1));
+        const clampedEnd = Math.max(0, Math.min(endIndex, totalNodes - 1));
+        const intervals = [
+            {
+                start: clampedStart,
+                end: clampedEnd,
+            },
+        ];
         metadonnees.sommaire = extractSommaireForIntervals($, idx, intervals);
         const points = [];
         let ordre = 0;
         const addPoint = (p) => points.push({ ...p, ordre_absolu_seance: String(++ordre) });
-        // Titles removes because they are just listed at the top of the file and not linked to any ancre
-        // $("cri\\:titreS1 p.titre_S1").each((_, el) => {
-        //   if (!elementInAnyInterval(el, idx, intervals)) return
-        //   const t = normalizeTitle(norm($(el).text() || ""))
-        //   if (t) addPoint({ code_grammaire: "TITRE_TEXTE_DISCUSSION", texte: { _: t }, code_style: "Titre" })
-        // })
         // Interventions
         $("div.intervenant").each((_, block) => {
             if (!elementInAnyInterval(block, idx, intervals))
@@ -47,6 +43,8 @@ export async function parseCompteRenduSlotFromFile(xmlFilePath, wantedSlot, firs
             ].join(","))
                 .remove();
             const firstP = $block.find("p").first();
+            if (!firstP || firstP.length === 0)
+                return;
             const speakerLabelRaw = firstP.find(".orateur_nom").text() || firstP.find("a.lien_senfic").text() || "";
             const speakerLabel = dedupeSpeaker(speakerLabelRaw);
             const { mat, nom: nomCRI, qua: quaCRI } = readIntervenantMeta($block);
@@ -55,7 +53,8 @@ export async function parseCompteRenduSlotFromFile(xmlFilePath, wantedSlot, firs
             const canonicalName = dedupeSpeaker(nomCRI || speakerLabel);
             const role = roleForSpeaker(speakerLabel) || roleForSpeaker(qualite) || roleForSpeaker(quaCRI || "");
             const speechHtml = sanitizeInterventionHtml($, $block);
-            if (!norm(cheerio.load(speechHtml).text() || ""))
+            const speechText = norm(cheerio.load(speechHtml).text() || "");
+            if (!speechText)
                 return;
             addPoint({
                 code_grammaire: "PAROLE_GENERIQUE",
@@ -65,19 +64,25 @@ export async function parseCompteRenduSlotFromFile(xmlFilePath, wantedSlot, firs
             });
         });
         const contenu = {
-            quantiemes: { journee: metadonnees.dateSeance, session: metadonnees.session },
+            quantiemes: {
+                journee: metadonnees.dateSeance,
+                session: metadonnees.session,
+            },
             point: points,
         };
+        const yyyymmdd = yyyymmddFromPath(xmlFilePath);
+        const dateISO = `${yyyymmdd.slice(0, 4)}-${yyyymmdd.slice(4, 6)}-${yyyymmdd.slice(6, 8)}`;
+        const seanceRef = makeReunionUid(dateISO, "SP", agendaEventId, null);
         return {
-            uid: "CRSSN" + xmlFilePath.replace(/^.*?(\d{8}).*$/i, "$1") + `-${wantedSlot}`,
-            seanceRef: "RUSN" + xmlFilePath.replace(/^.*?(\d{8}).*$/i, "$1") + "IDS-" + wantedSlot,
+            uid: `CRSSN${yyyymmdd}E${agendaEventId}`,
+            seanceRef,
             sessionRef: metadonnees.session,
             metadonnees,
             contenu,
         };
     }
     catch (e) {
-        console.error(`[CRI] parseSlot error file=${xmlFilePath} slot=${wantedSlot}:`, e);
+        console.error(`[CRI] parseInterval error file=${xmlFilePath} interval=[${startIndex}..${endIndex}] event=${agendaEventId}:`, e);
         return null;
     }
 }
@@ -87,96 +92,6 @@ export function sessionStartYearFromDate(d) {
     const y = d.getFullYear();
     return m >= 9 ? y : y - 1;
 }
-export function parseYYYYMMDD(yyyymmdd) {
-    if (!/^\d{8}$/.test(yyyymmdd))
-        return null;
-    const y = Number(yyyymmdd.slice(0, 4));
-    const m = Number(yyyymmdd.slice(4, 6)) - 1;
-    const d = Number(yyyymmdd.slice(6, 8));
-    const dt = new Date(y, m, d);
-    return Number.isFinite(dt.getTime()) ? dt : null;
-}
-export function deriveTitreObjetFromSommaire(sommaire, slot) {
-    const items = extractLevel1Items(sommaire);
-    const meaningful = items.filter((it) => !isBoilerplate(it.label));
-    if (meaningful.length === 0) {
-        return {
-            titre: `Séance publique ${slotLabel(slot)}`,
-            objet: "",
-        };
-    }
-    const titre = meaningful[0].label;
-    const objet = meaningful
-        .slice(0, 3)
-        .map((it) => it.label)
-        .join(" ; ");
-    return { titre, objet };
-}
-function slotLabel(slot) {
-    switch (slot) {
-        case "MATIN":
-            return "du matin";
-        case "APRES-MIDI":
-            return "de l’après-midi";
-        case "SOIR":
-            return "du soir";
-        default:
-            return "";
-    }
-}
-const BOILERPLATE_PATTERNS = [
-    /proc(?:è|e)s-?verbal/i,
-    /hommages?/i,
-    /désignation des vice-?président/i,
-    /candidatures? aux?/i,
-    /ordre du jour/i,
-    /rappels? au règlement/i,
-    /communications?/i,
-    /dépôts?/i,
-    /proclamation/i,
-    /présidence de/i,
-    /questions? diverses?/i,
-    /ouverture de la séance/i,
-    /clo(?:t|̂)ure de la séance/i,
-];
-const isBoilerplate = (label) => !label?.trim() || BOILERPLATE_PATTERNS.some((rx) => rx.test(label));
-function extractLevel1Items(sommaire) {
-    const level1 = asArray(sommaire?.sommaire1);
-    return level1
-        .map((el) => ({
-        numero: toInt(el?.valeur_pts_odj),
-        label: String(el?.titreStruct?.intitule ?? "").trim(),
-    }))
-        .filter((it) => !!it.label)
-        .sort((a, b) => a.numero - b.numero);
-}
-function stripTrailingPunct(s) {
-    return s.replace(/\s*([:,.;])\s*$/u, "").trim();
-}
-function dedupeSpeaker(raw) {
-    let s = norm(raw);
-    s = stripTrailingPunct(s);
-    const dupPatterns = [/^(.+?)\s*[.]\s*\1$/u, /^(.+?)\s*,\s*\1,?$/u, /^(.+?)\s+\1$/u];
-    for (const re of dupPatterns) {
-        const m = s.match(re);
-        if (m) {
-            s = m[1];
-            break;
-        }
-    }
-    return s.replace(/\.\s*$/, "");
-}
-function fixApostrophes(s) {
-    let out = s;
-    out = out.replace(/\s*’\s*/g, "’");
-    out = out.replace(/\b([dljctmsn])\s*’/gi, (_, m) => m + "’");
-    out = out.replace(/’\s+([A-Za-zÀ-ÖØ-öø-ÿ])/g, "’$1");
-    out = out.replace(/\s+([,;:.!?])/g, "$1");
-    return out;
-}
-function normalizeTitle(text) {
-    return text.replace(/^PR[ÉE]SIDENCE DE\b/i, "Présidence de ");
-}
 function roleForSpeaker(labelOrQualite) {
     const s = (labelOrQualite || "").toLowerCase();
     if (/^(m\.|mme)?\s*(le|la)\s+pr[ée]sident(e)?\b/.test(s) || /\bpr[ée]sident[e]?\s+de\s+séance\b/.test(s))

package/lib/model/util.d.ts

@@ -6,8 +6,4 @@ export declare function removeSubstring(expr: Expression<string | null | undefin
 export declare function replace(expr: Expression<string | null | undefined>, pattern: Expression<string>, replacement: Expression<string>): import("kysely").RawBuilder<string>;
 export declare function rtrim(expr: Expression<string | null | undefined>): import("kysely").RawBuilder<string>;
 export declare function toDateString(expr: Expression<Date | null | undefined>, format?: Expression<string>): import("kysely").RawBuilder<string>;
-export declare function norm(s?: string | null): string;
 export declare function toCRDate(dateISO: string, startTime?: string | null): string;
-export declare function normalizeTitle(t: string): string;
-export declare function jaccardTokenSim(a: string, b: string): number;
-export declare function decodeHtmlEntities(s?: string | null): string;

package/lib/model/util.js

@@ -21,12 +21,6 @@ export function rtrim(expr) {
 export function toDateString(expr, format = sql.val(STANDARD_DATE_FORMAT)) {
     return sql `to_char(${expr}, ${format})`;
 }
-export function norm(s) {
-    return (s || "")
-        .replace(/\u00A0/g, " ")
-        .replace(/\s+/g, " ")
-        .trim();
-}
 export function toCRDate(dateISO, startTime) {
     const yyyymmdd = dateISO.replace(/-/g, ""); // "20250716"
     let hh = "00", mm = "00", ss = "00", SSS = "000";
@@ -42,35 +36,3 @@ export function toCRDate(dateISO, startTime) {
     }
     return `${yyyymmdd}${hh}${mm}${ss}${SSS}`;
 }
-export function normalizeTitle(t) {
-    return (t || "")
-        .toLowerCase()
-        .normalize("NFD")
-        .replace(/\p{Diacritic}/gu, "")
-        .replace(/[^a-z0-9\s]/g, " ")
-        .replace(/\s+/g, " ")
-        .trim();
-}
-export function jaccardTokenSim(a, b) {
-    const A = new Set(normalizeTitle(a).split(" ").filter(Boolean));
-    const B = new Set(normalizeTitle(b).split(" ").filter(Boolean));
-    if (A.size === 0 || B.size === 0)
-        return 0;
-    let inter = 0;
-    for (const x of A)
-        if (B.has(x))
-            inter++;
-    return inter / (A.size + B.size - inter);
-}
-export function decodeHtmlEntities(s) {
-    if (!s)
-        return "";
-    return s
-        .replace(/&#x([0-9a-fA-F]+);/g, (_, h) => String.fromCodePoint(parseInt(h, 16)))
-        .replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10)))
-        .replace(/&/g, "&")
-        .replace(/&lt;/g, "<")
-        .replace(/&gt;/g, ">")
-        .replace(/&quot;/g, '"')
-        .replace(/&apos;/g, "'");
-}