@tricoteuses/senat 2.22.9 → 2.22.11

package/lib/src/scripts/retrieve_documents.js

@@ -101,6 +101,9 @@ async function processDocument(url, destPath, docDate, options) {
 }
 export async function processTexte(texteMetadata, originalTextesDir, transformedTextesDir, enrichedTextesDir, options) {
     const texteDir = path.join(originalTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name);
+    // Pre-compute whether the parsed JSON output already exists, to avoid re-parsing unchanged files
+    const parsedJsonPath = path.join(transformedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name, `${texteMetadata.name}.json`);
+    const parsedOutputExists = options.parseDocuments ? await fs.pathExists(parsedJsonPath) : false;
     let exposeDesMotifsContent = null;
     if (texteMetadata.url_expose_des_motifs) {
         const exposePath = path.join(texteDir, `${texteMetadata.name}-expose.html`);
@@ -108,7 +111,8 @@ export async function processTexte(texteMetadata, originalTextesDir, transformed
         if (res.buffer) {
             exposeDesMotifsContent = res.buffer;
         }
-        else if (res.skipped && options.parseDocuments) {
+        else if (res.skipped && options.parseDocuments && !parsedOutputExists) {
+            // Only load expose from disk if we'll actually need to re-parse
             if (await fs.pathExists(exposePath)) {
                 exposeDesMotifsContent = await fs.readFile(exposePath);
             }
@@ -128,23 +132,28 @@ export async function processTexte(texteMetadata, originalTextesDir, transformed
         const result = await processDocument(format.url.toString(), destPath, texteMetadata.date, options);
         // Specific logic: Parsing (Only applies to XML)
         if (format.isParseTarget && options.parseDocuments) {
-            if (result.buffer !== null || (await fs.pathExists(destPath))) {
+            // Skip re-parsing if the XML was not newly downloaded AND the parsed output already exists
+            const needsParsing = !result.skipped || !parsedOutputExists;
+            if (needsParsing && (result.buffer !== null || (await fs.pathExists(destPath)))) {
                 await parseDocument(texteMetadata.session, transformedTextesDir, destPath, texteMetadata.name, result.buffer, exposeDesMotifsContent, options);
-            }
-            let texteXmlContent = null;
-            if (result.buffer === null && (await fs.pathExists(destPath))) {
-                texteXmlContent = await fs.readFile(destPath, "utf-8");
-            }
-            else if (result.buffer !== null) {
-                texteXmlContent = textDecoder.decode(result.buffer);
-            }
-            if (texteXmlContent !== null) {
-                try {
-                    await convertSenatXmlToHtml(texteXmlContent, path.join(enrichedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name, `${texteMetadata.name}.html`));
+                let texteXmlContent = null;
+                if (result.buffer === null && (await fs.pathExists(destPath))) {
+                    texteXmlContent = await fs.readFile(destPath, "utf-8");
                 }
-                catch (error) {
-                    console.error(`Error converting ${texteMetadata.name} to HTML: ${error.message}`);
+                else if (result.buffer !== null) {
+                    texteXmlContent = textDecoder.decode(result.buffer);
                 }
+                if (texteXmlContent !== null) {
+                    try {
+                        await convertSenatXmlToHtml(texteXmlContent, path.join(enrichedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name, `${texteMetadata.name}.html`));
+                    }
+                    catch (error) {
+                        console.error(`Error converting ${texteMetadata.name} to HTML: ${error.message}`);
+                    }
+                }
+            }
+            else if (options.verbose) {
+                console.info(`Skipping parse for already processed texte ${texteMetadata.name}…`);
             }
         }
     }

package/lib/src/utils/scoring.d.ts

@@ -13,6 +13,9 @@ export declare function isNoiseBlock(text: string): boolean;
 export declare function scoreSommaireBlockForEvent(blockText: string, ev: AgendaEvent): number;
 export declare function getOrgKey(norm: string): string;
 export declare function dice(a: string, b: string): number;
+export declare function coverage(reference?: string | null, candidate?: string | null): number;
+export declare function diceFiltered(a?: string | null, b?: string | null): number;
+export declare function similarityScore(a?: string | null, b?: string | null): number;
 export declare function normalize(s?: string | null): string;
 export declare function normalizeSalle(s?: string | null): string | null;
 export declare function scoreVideo(agenda: Reunion, agendaTs: number | null, sameOrg: boolean, w: VideoScoreWeights, videoTitle?: string, videoEpoch?: number, videoOrganes?: string[], timeAmbigious?: boolean, salle?: string, chapterTitles?: L1Chapter[]): {

package/lib/src/utils/scoring.js

@@ -154,6 +154,38 @@ export function dice(a, b) {
             inter++;
     return (2 * inter) / (A.size + B.size);
 }
+function tokenSet(s) {
+    if (!s)
+        return new Set();
+    return new Set(tokens(s));
+}
+export function coverage(reference, candidate) {
+    const A = tokenSet(reference);
+    const B = tokenSet(candidate);
+    if (!A.size || !B.size)
+        return 0;
+    let inter = 0;
+    for (const t of A)
+        if (B.has(t))
+            inter++;
+    return inter / A.size;
+}
+export function diceFiltered(a, b) {
+    const A = tokenSet(a);
+    const B = tokenSet(b);
+    if (!A.size || !B.size)
+        return 0;
+    let inter = 0;
+    for (const t of A)
+        if (B.has(t))
+            inter++;
+    return (2 * inter) / (A.size + B.size);
+}
+export function similarityScore(a, b) {
+    const cov = coverage(a, b);
+    const d = diceFiltered(a, b);
+    return 0.7 * cov + 0.3 * d;
+}
 export function normalize(s) {
     return (s ?? "")
         .toLowerCase()
@@ -174,13 +206,13 @@ export function normalizeSalle(s) {
 }
 export function scoreVideo(agenda, agendaTs, sameOrg, w, videoTitle, videoEpoch, videoOrganes, timeAmbigious = false, salle, chapterTitles) {
     const weights = w;
-    const objetS = dice(agenda.objet || "", videoTitle || "");
-    const titleS = dice(agenda.titre || "", videoTitle || "");
+    const objetS = similarityScore(agenda.objet || "", videoTitle || "");
+    const titleS = similarityScore(agenda.titre || "", videoTitle || "");
     let titleScore = Math.max(objetS, titleS);
     chapterTitles = chapterTitles || [];
     for (const ch of chapterTitles) {
-        const chObjetS = dice(agenda.objet || "", ch.label);
-        const chTitreS = dice(agenda.titre || "", ch.label);
+        const chObjetS = similarityScore(agenda.objet || "", ch.label);
+        const chTitreS = similarityScore(agenda.titre || "", ch.label);
         titleScore = Math.max(titleScore, Math.max(chObjetS, chTitreS));
     }
     let timeScore = 0;
@@ -190,8 +222,7 @@ export function scoreVideo(agenda, agendaTs, sameOrg, w, videoTitle, videoEpoch,
     }
     let orgScore = 0;
     if (agenda.organe && videoOrganes?.length) {
-        // NOTE: if you already normalize organes elsewhere, keep it there.
-        orgScore = Math.max(...videoOrganes.map((v) => dice(agenda.organe, v)));
+        orgScore = Math.max(...videoOrganes.map((v) => similarityScore(agenda.organe, v)));
     }
     // Salle: normalized "A263" matching
     let salleScore = 0;

package/lib/src/videos/types.d.ts

@@ -12,8 +12,8 @@ export type MatchResult = {
     reason?: "margin_ambiguous_time_sp";
 };
 export type MatchWeights = {
-    minAccept?: number;
-    margin?: number;
+    minAccept: number;
+    margin: number;
     titleDominance?: number;
     orgUncertainPenalty?: number;
     orgSkipDice?: number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.22.9",
+  "version": "2.22.11",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",