@tricoteuses/senat 2.16.2 → 2.16.3

package/lib/model/commission.d.ts

@@ -5,6 +5,7 @@ export declare function getRemainingTextAfterSpeakerHeader($: cheerio.CheerioAPI
 export type DaySection = {
     title: string;
     $start: cheerio.Cheerio<any>;
+    time?: string;
 };
 export declare function cleanTitle(t: string): string;
 export declare function extractDayH3Sections($: cheerio.CheerioAPI, dateISO: string): DaySection[];

package/lib/model/commission.js

@@ -161,33 +161,48 @@ function buildPointsFromParagraphs($, paras) {
     flush();
     return points;
 }
+const TIME_RE = /(?:\b[àa]\s*)?(\d{1,2})\s*(?:h|heures?)\s*(?:([0-5]\d))?/i;
 export function cleanTitle(t) {
     return (t || "").replace(/\s+/g, " ").trim();
 }
+function parseTimeToHHmm(text) {
+    const m = normalizeSpaces(text).match(TIME_RE);
+    if (!m)
+        return undefined;
+    const hh = m[1]?.padStart(2, "0");
+    const mm = (m[2] ?? "00").padStart(2, "0");
+    const h = Number(hh);
+    if (h >= 0 && h <= 23)
+        return `${hh}:${mm}`;
+    return undefined;
+}
+function findNearbyTime($, $h3) {
+    let cur = $h3.prev();
+    for (let i = 0; i < 3 && cur.length; i++, cur = cur.prev()) {
+        const direct = parseTimeToHHmm(cur.text());
+        if (direct)
+            return direct;
+        const italic = parseTimeToHHmm(cur.find("i, em").first().text());
+        if (italic)
+            return italic;
+    }
+    return undefined;
+}
 export function extractDayH3Sections($, dateISO) {
     const sections = [];
     const $dayRoot = findDayRoot($, dateISO);
     if ($dayRoot.length === 0)
         return sections;
-    let $cursor = $dayRoot.next();
-    while ($cursor.length && !$cursor.is("h2")) {
-        if ($cursor.is("h3")) {
-            const title = cleanTitle($cursor.text());
-            if (title)
-                sections.push({ title, $start: $cursor });
-            $cursor = $cursor.next();
-            continue;
-        }
-        const $h3 = $cursor.find("h3").first();
-        if ($h3.length) {
-            const title = cleanTitle($h3.text());
-            if (title)
-                sections.push({ title, $start: $h3 });
-            $cursor = $cursor.next();
-            continue;
-        }
-        $cursor = $cursor.next();
-    }
+    const $range = $dayRoot.nextUntil("h2");
+    const $h3s = $range.filter("h3").add($range.find("h3"));
+    $h3s.each((_, el) => {
+        const $h3 = $(el);
+        const title = cleanTitle($h3.text());
+        if (!title)
+            return;
+        const time = findNearbyTime($, $h3);
+        sections.push({ title, $start: $h3, time });
+    });
     return sections;
 }
 export function parseCommissionCRSectionFromDom($, htmlFilePath, opts) {

package/lib/model/seance.js

@@ -237,16 +237,31 @@ function extractAndRemoveLeadingQualite($, $block) {
     return fixApostrophes(norm(parts.join(" ")));
 }
 function sanitizeInterventionHtml($, $block) {
-    const $clone = $block.clone();
-    $clone.find("a[name]").remove();
-    $clone.find('div[align="right"]').remove();
-    $clone.find("a.link").remove();
-    $clone.find("img").remove();
-    $clone.find("a#ameli_amendement_cri_phrase, a#ameli_amendement_cra_contenu, a#ameli_amendement_cra_objet").remove();
-    $clone.find(".orateur_nom, .orateur_qualite").remove();
-    let html = $clone.html() || "";
-    html = html.replace(/<!--[\s\S]*?-->/g, "");
-    return html.trim();
+    const ps = $block.find("p").toArray();
+    const cleaned = ps
+        .map((p) => {
+        const $p = $(p).clone();
+        $p.find(".orateur_nom, .orateur_qualite").remove();
+        $p.find("a").each((_, a) => {
+            const $a = $(a);
+            $a.replaceWith($a.text());
+        });
+        $p.find(".info_entre_parentheses").each((_, el) => {
+            const txt = $(el).text();
+            $(el).replaceWith($("<em/>").text(txt));
+        });
+        $p.find("span").each((_, span) => {
+            const $s = $(span);
+            if (!$s.text().trim())
+                $s.remove();
+        });
+        const inner = ($p.html() || "").trim();
+        if (!inner)
+            return null;
+        return `<p>${inner}</p>`;
+    })
+        .filter(Boolean);
+    return cleaned.join("<br/>");
 }
 function extractSommaireForIntervals($, idx, intervals) {
     const inIv = (el) => elementInAnyInterval(el, idx, intervals);

package/lib/scripts/convert_data.js

@@ -9,7 +9,7 @@ import { createActesLegislatifs } from "../model/dosleg";
 import { UNDEFINED_SESSION } from "../types/sessions";
 import { getSessionFromDate, getSessionFromSignet } from "./datautil";
 import { commonOptions } from "./shared/cli_helpers";
-import { ensureAndClearDir } from "./shared/util";
+import { ensureAndClearDir, ensureAndClearDirSync } from "./shared/util";
 const optionsDefinitions = [...commonOptions];
 const options = commandLineArgs(optionsDefinitions);
 const SENAT_TEXTE_XML_BASE_URL = "https://www.senat.fr/akomantoso/";
@@ -47,7 +47,7 @@ async function convertDatasetAmeli(dataDir, options) {
         console.log(`Converting database ${dataset.database} data into files…`);
     }
     const ameliReorganizedRootDir = path.join(dataDir, dataset.database);
-    ensureAndClearDir(ameliReorganizedRootDir);
+    ensureAndClearDirSync(ameliReorganizedRootDir);
     for await (const amendement of findAllAmendements(options["fromSession"])) {
         if (options["verbose"]) {
             console.log(`Converting ${amendement["numero"]} file…`);
@@ -88,18 +88,20 @@ async function enrichDebat(debat, auteurs) {
     const enrichedDebat = { ...debat };
     for (const section of enrichedDebat.sections) {
         for (const intervention of section.interventions) {
+            ;
             intervention.auteur = findAuteur(intervention["auteur_code"], auteurs);
         }
     }
     for (const section of enrichedDebat.sections_divers) {
         for (const intervention of section.interventions) {
+            ;
             intervention.auteur = findAuteur(intervention["auteur_code"], auteurs);
         }
     }
     return enrichedDebat;
 }
 function findAuteur(auteurCode, auteurs) {
-    return auteurs.find(auteur => auteur.code === auteurCode);
+    return auteurs.find((auteur) => auteur.code === auteurCode);
 }
 async function convertDatasetDosLeg(dataDir, options) {
     const dataset = datasets.dosleg;

package/lib/scripts/retrieve_cr_commission.js

@@ -281,9 +281,9 @@ async function retrieveCommissionCRs(options = {}) {
                     const MAX_TIME_DELTA_MIN = 120;
                     const ORGANE_GATE = 0.55;
                     const TITLE_GATE = 0.2;
-                    const W_ORG = 0.5;
-                    const W_TIM = 0.2;
-                    const W_TIT = 0.3;
+                    const W_ORG = 0.4;
+                    const W_TIM = 0.4;
+                    const W_TIT = 0.2;
                     for (let sIdx = 0; sIdx < sections.length; sIdx++) {
                         const sec = sections[sIdx];
                         let best = null;
@@ -292,7 +292,7 @@ async function retrieveCommissionCRs(options = {}) {
                             const scored = hits
                                 .map((h) => {
                                 const sOrg = organeSimilarity(h, commissionKey); // 0..1
-                                const sTim = timeProximityScore(h, day.openTime ?? null, MAX_TIME_DELTA_MIN); // 0..1
+                                const sTim = timeProximityScore(h, sec.time ?? day.openTime ?? null, MAX_TIME_DELTA_MIN); // 0..1
                                 const sTit = titleSimilarity(h, sec.title); // 0..1
                                 const total = W_ORG * sOrg + W_TIM * sTim + W_TIT * sTit;
                                 return { h, sOrg, sTim, sTit, total };

package/lib/scripts/retrieve_videos.js

@@ -9,14 +9,13 @@ import { getSessionsFromStart } from "../types/sessions";
 import { commonOptions } from "./shared/cli_helpers";
 import { decodeHtmlEntities } from "../model/util";
 // ===================== Constants =====================
-const MATCH_THRESHOLD = 0.56;
+const MATCH_THRESHOLD = 0.5;
 const MAX_CANDIDATES = 15;
 const MAX_PAGES = 3;
 const STATS = { total: 0, accepted: 0 };
 const VIDEOS_ROOT_FOLDER = "videos";
 const SENAT_VIDEOS_SEARCH_AJAX = "https://videos.senat.fr/senat_videos_search.php";
 const SENAT_DATAS_ROOT = "https://videos.senat.fr/Datas/senat";
-const SENAT_VOD_HOST = "https://vodsenat.akamaized.net";
 // ===================== CLI =====================
 const optionsDefinitions = [...commonOptions];
 const options = commandLineArgs(optionsDefinitions);
@@ -219,9 +218,9 @@ function score(agenda, agendaTs, videoTitle, videoEpoch, videoOrgane) {
     }
     const orgScore = videoOrgane && agenda.organe ? dice(agenda.organe, videoOrgane) : 0;
     if (orgScore === 0 && agenda.organe === "Séance publique") {
-        return 0.5 * titleScore + 0.5 * timeScore;
+        return 0.3 * titleScore + 0.7 * timeScore;
     }
-    return 0.4 * titleScore + 0.3 * timeScore + orgScore * 0.3;
+    return 0.2 * titleScore + 0.4 * timeScore + orgScore * 0.4;
 }
 /**
  * Build search strategies for senat's videos

package/lib/scripts/shared/util.d.ts

@@ -1,3 +1,4 @@
 export declare function isOptionEmptyOrHasValue(option: string, value: string): boolean;
+export declare function ensureAndClearDirSync(dir: string): void;
 export declare function ensureAndClearDir(path: string): void;
 export declare function fetchWithRetry(url: string, retries?: number, backoff?: number): Promise<Response>;

package/lib/scripts/shared/util.js

@@ -1,7 +1,14 @@
 import fs from "fs-extra";
+import path from "path";
 export function isOptionEmptyOrHasValue(option, value) {
     return !option || option.length === 0 || option.includes(value);
 }
+export function ensureAndClearDirSync(dir) {
+    fs.ensureDirSync(dir);
+    for (const name of fs.readdirSync(dir)) {
+        fs.rmSync(path.join(dir, name), { recursive: true, force: true });
+    }
+}
 export function ensureAndClearDir(path) {
     if (!fs.existsSync(path)) {
         fs.mkdirSync(path, { recursive: true });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.16.2",
+  "version": "2.16.3",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",