@tricoteuses/senat 3.1.18 → 3.1.20

package/lib/src/scripts/retrieve_videos.js

@@ -42,7 +42,11 @@ function shouldSkipDownload(agenda, baseDir) {
     const cutoff = now - options["only-recent"] * 24 * 3600 * 1000;
     const reunionTs = Date.parse(agenda.date);
     if (reunionTs >= cutoff)
-        return false;
+        return false; // recent: always try
+    // Old meeting with no captation: will never match (minAccept=0.9), skip
+    if (!agenda.captationVideo)
+        return true;
+    // Old meeting with captation: skip only if already downloaded
     const dataNvsPath = path.join(baseDir, "data.nvs");
     const finalplayerNvsPath = path.join(baseDir, "finalplayer.nvs");
     return fs.existsSync(dataNvsPath) && fs.existsSync(finalplayerNvsPath);

package/lib/src/scripts/test_iter_load.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/src/scripts/test_iter_load.js

@@ -0,0 +1,12 @@
+import { iterLoadSenatTextes, } from "../loaders";
+import commandLineArgs from "command-line-args";
+import { dataDirDefaultOption } from "./shared/cli_helpers";
+const optionsDefinitions = [dataDirDefaultOption];
+const options = commandLineArgs(optionsDefinitions);
+const session = 2025;
+const sinceCommit = undefined;
+for (const { item: texte } of iterLoadSenatTextes(options["dataDir"], session, {
+    sinceCommit: sinceCommit,
+})) {
+    console.log(texte);
+}

package/lib/src/types/ameli.d.ts

@@ -1,5 +1,5 @@
-import { TxtAmeli } from "../raw_types/ameli.js";
-import { Sub } from "../raw_types/ameli.js";
-export interface TxtAmeliCustom extends TxtAmeli {
-    subids?: Sub["id"][];
+import { ses as Ses, sub as Sub, subFields, txt_ameli } from "../raw_types_schemats/ameli";
+export type { Ses, Sub };
+export interface TxtAmeli extends txt_ameli {
+    subids?: subFields.id[];
 }

package/lib/src/types/dosleg.d.ts

@@ -1,70 +1,70 @@
-import { Ass, Aud as AudRaw, Auteur as AuteurRaw, DateSeance as DateSeanceRaw, Deccoc, Denrap, Docatt as DocattRaw, Ecr as EcrRaw, Etaloi, Lecass as LecassRaw, Lecassrap as LecassrapRaw, Lecture as LectureRaw, Loi as LoiRaw, Org, Oritxt, Qua, Rap as RapRaw, Raporg, Scr, Texte as TexteRaw, Typatt, Typlec, Typloi, Typtxt, Typurl } from "../raw_types/dosleg.js";
-import { TxtAmeli } from "../raw_types/ameli.js";
-import { Debats } from "../raw_types/debats.js";
-import { TxtAmeliCustom } from "./ameli.js";
-export type { Deccoc as DecCoc, Denrap as DenRap, Etaloi as EtaLoi, Oritxt as OriTxt, Raporg as RapOrg, Typatt as TypAtt, Typlec as TypLec, Typloi as TypLoi, Typtxt as TypTxt, Typurl as TypUrl, };
-export interface Aud extends AudRaw {
+import { txt_ameliFields } from "../raw_types_schemats/ameli";
+import { ass as Ass, aud, audFields, auteur, date_seance, date_seanceFields as dateSeanceFields, deccoc as DecCoc, denrap as DenRap, docatt, docattFields, ecr, ecrFields, etaloi as EtaLoi, lecass, lecassFields, lecassrap, lecture, lectureFields, loi, org as Org, oritxt as OriTxt, qua as Qua, rap, raporg as RapOrg, raporgFields, scr as Scr, texte, texteFields, typatt as TypAtt, typlec as TypLec, typloi as TypLoi, typtxt as TypTxt, typurl as TypUrl, typurlFields } from "../raw_types_schemats/dosleg";
+import { Debat } from "./debats";
+import { TxtAmeli } from "./ameli";
+export type { Ass, DecCoc, DenRap, EtaLoi, Org, OriTxt, Qua, RapOrg, Scr, TypAtt, TypLec, TypLoi, TypTxt, TypUrl };
+export interface Aud extends aud {
     org?: Org;
 }
-export interface Auteur extends AuteurRaw {
+export interface Auteur extends auteur {
     qua?: Qua;
 }
-export interface DateSeance extends DateSeanceRaw {
-    debat?: Debats;
+export interface DateSeance extends date_seance {
+    debat?: Debat;
     scrids?: string[];
     scrs?: Scr[];
 }
-export interface DocAtt extends DocattRaw {
-    rap?: RapRaw;
-    typatt?: Typatt;
+export interface DocAtt extends docatt {
+    rap?: Rap;
+    typatt?: TypAtt;
 }
-export interface Ecr extends EcrRaw {
+export interface Ecr extends ecr {
     aut?: Auteur;
 }
-export interface LecAss extends LecassRaw {
+export interface LecAss extends lecass {
     ass?: Ass;
     auds?: Aud[];
-    audcles?: AudRaw["audcle"][];
+    audcles?: audFields.audcle[];
     datesSeances?: DateSeance[];
-    datesSeancesCodes?: DateSeanceRaw["code"][];
+    datesSeancesCodes?: dateSeanceFields.code[];
     debatdatseas: Date[];
     lecassraps?: LecAssRap[];
     lecassrapids?: string[];
     org?: Org;
-    texcods: TexteRaw["texcod"][];
+    texcods: texteFields.texcod[];
     textes?: Texte[];
 }
-export interface LecAssRap extends LecassrapRaw {
-    rap?: RapRaw;
+export interface LecAssRap extends lecassrap {
+    rap?: Rap;
 }
-export interface Lecture extends LectureRaw {
-    typlec?: Typlec;
-    lecassidts?: LecassRaw["lecassidt"][];
+export interface Lecture extends lecture {
+    typlec?: TypLec;
+    lecassidts?: lecassFields.lecassidt[];
     lecasss?: LecAss[];
 }
-export interface Loi extends LoiRaw {
-    deccoc?: Deccoc;
-    etaloi?: Etaloi;
-    lecidts?: LectureRaw["lecidt"][];
+export interface Loi extends loi {
+    deccoc?: DecCoc;
+    etaloi?: EtaLoi;
+    lecidts?: lectureFields.lecidt[];
     lectures?: Lecture[];
-    typloi?: Typloi;
+    typloi?: TypLoi;
 }
-export interface Rap extends RapRaw {
-    denrap?: Denrap;
-    docattcles?: DocattRaw["docattcle"][];
+export interface Rap extends rap {
+    denrap?: DenRap;
+    docattcles?: docattFields.docattcle[];
     docatts?: DocAtt[];
-    ecrnums?: EcrRaw["ecrnum"][];
+    ecrnums?: ecrFields.ecrnum[];
     ecrs?: Ecr[];
-    orgcods?: Raporg["orgcod"][];
+    orgcods?: raporgFields.orgcod[];
     orgs?: Org[];
 }
-export interface Texte extends TexteRaw {
+export interface Texte extends texte {
     ecrs?: Ecr[];
-    ecrnums?: EcrRaw["ecrnum"][];
-    libtypurl?: Typurl["libtypurl"];
+    ecrnums?: ecrFields.ecrnum[];
+    libtypurl?: typurlFields.libtypurl;
     org?: Org;
-    oritxt: Oritxt;
-    typtxt?: Typtxt;
-    txtAmeli?: TxtAmeliCustom;
-    txtAmeliId: TxtAmeli["id"];
+    oritxt: OriTxt;
+    typtxt?: TypTxt;
+    txtAmeli?: TxtAmeli;
+    txtAmeliId: txt_ameliFields.id;
 }

package/lib/src/types/questions.d.ts

@@ -1,2 +1,2 @@
-import { TamQuestions } from "../raw_types/questions.js";
-export type { TamQuestions as Question };
+import { tam_questions as Question } from "../raw_types_schemats/questions";
+export type { Question };

package/lib/src/types/sens.d.ts

@@ -1,3 +1,5 @@
+import { sen as Sen } from "../raw_types_schemats/sens";
+export type { Sen };
 export interface Photo {
     chemin: string;
     cheminMosaique: string;

package/lib/src/types/texte.d.ts

@@ -1,4 +1,4 @@
-import { DocumentResult } from "../loaders.js";
+import { DocumentResult } from "../loaders";
 export declare enum DivisionType {
     tome = 1,
     part = 2,
@@ -23,7 +23,7 @@ export interface DocumentMetadata {
     url_html: URL;
     url_pdf: URL;
 }
-export interface FlatTexte extends Partial<DocumentResult> {
+export interface FlatTexte extends DocumentResult {
     titre: string | null;
     titre_court: string | null;
     url_dossier_senat: string | null;

package/lib/src/utils/reunion_odj_building.js

@@ -14,16 +14,32 @@ function buildReunionLookupKey(date, organe) {
     const o = normalizeOrganeForMatch(organe || "");
     return `${d}|${o}`;
 }
+function organeWordOverlap(a, b) {
+    const wordsA = new Set(a.split(/\s+/).filter((w) => w.length >= 5));
+    const wordsB = new Set(b.split(/\s+/).filter((w) => w.length >= 5));
+    if (wordsA.size === 0 || wordsB.size === 0)
+        return 0;
+    let overlap = 0;
+    for (const w of wordsA) {
+        if (wordsB.has(w))
+            overlap++;
+    }
+    return overlap / Math.min(wordsA.size, wordsB.size);
+}
 export function buildOdj(events, dossierBySenatUrl, doslegReunionIndex) {
     const byObjet = new Map();
     let lastCodeEtape = null;
     let lastFlatActes;
     for (const ev of events) {
         const objetKey = (ev.objet ?? "").trim();
-        const url = normalizeSenatUrl(ev.urlDossierSenat) ?? undefined;
-        let dossier = url ? (dossierBySenatUrl[url] ?? null) : null;
-        if (!dossier && doslegReunionIndex) {
-            dossier = lookupDossierFromDoslegReunion(ev, doslegReunionIndex);
+        // Prefer the dosleg reunion index (source officielle, dosleg_aud) over the URL scraped
+        // from the agenda HTML, which can contain wrong links (senat.fr bug).
+        let dossier = doslegReunionIndex
+            ? lookupDossierFromDoslegReunion(ev, doslegReunionIndex)
+            : null;
+        if (!dossier) {
+            const url = normalizeSenatUrl(ev.urlDossierSenat) ?? undefined;
+            dossier = url ? (dossierBySenatUrl[url] ?? null) : null;
         }
         const dossierUid = dossier ? pickDossierUid(dossier) : undefined;
         if (dossier) {
@@ -59,7 +75,9 @@ function scoreCandidate(c, evText) {
     if (evText.includes(dosSignet))
         score += 10;
     if (c.reunionTitre) {
-        const reunionWords = normalizeOrganeForMatch(c.reunionTitre).split(/\s+/).filter((w) => w.length >= 5);
+        const reunionWords = normalizeOrganeForMatch(c.reunionTitre)
+            .split(/\s+/)
+            .filter((w) => w.length >= 5);
         score += reunionWords.filter((w) => evText.includes(w)).length;
     }
     const dossierTitre = normalizeOrganeForMatch(c.dossier.titre ?? "");
@@ -91,7 +109,9 @@ function lookupDossierFromDoslegReunion(ev, index) {
     for (const [k, v] of index) {
         if (k.startsWith(evDate + "|") && k !== key) {
             const kOrgane = k.split("|")[1] || "";
-            if (evOrgane && kOrgane && (evOrgane.includes(kOrgane) || kOrgane.includes(evOrgane))) {
+            if (evOrgane &&
+                kOrgane &&
+                (evOrgane.includes(kOrgane) || kOrgane.includes(evOrgane) || organeWordOverlap(evOrgane, kOrgane) >= 0.6)) {
                 dateOnlyCandidates.push(...v);
             }
         }
@@ -191,11 +211,7 @@ function computeCodeEtape(ev, dossier, flatActes) {
     const cleanCode = (code) => code.replace(/-SEANCE$/, "");
     const lecture = detectLecture(ev.objet ?? "");
     const organeLower = (ev.organe ?? "").toLowerCase();
-    const nature = organeLower.includes("commission")
-        ? "COM"
-        : organeLower.includes("séance publique")
-            ? "DEBATS"
-            : "";
+    const nature = organeLower.includes("commission") ? "COM" : organeLower.includes("séance publique") ? "DEBATS" : "";
     const evDate = ev.date.split("T")[0];
     const flat = flatActes ?? buildFlatActes(dossier);
     // 1) Strict matching: same date + same nature

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "3.1.18",
+  "version": "3.1.20",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",

package/lib/src/databases_postgres.d.ts

@@ -1,4 +0,0 @@
-import postgres from "postgres";
-export type SqlValue = postgres.ParameterOrJSON<never>;
-export declare const sql: postgres.Sql<{}>;
-export declare function streamUnsafeQuery<T>(query: string, params?: readonly unknown[], batchSize?: number): AsyncGenerator<T, void, unknown>;

package/lib/src/databases_postgres.js

@@ -1,23 +0,0 @@
-import postgres from "postgres";
-import config from "./config.js";
-export const sql = postgres({
-    database: config.db.name,
-    host: config.db.host,
-    password: config.db.password,
-    port: config.db.port,
-    transform: {
-        undefined: null,
-    },
-    user: config.db.user,
-});
-function toSqlValue(value) {
-    return value;
-}
-export async function* streamUnsafeQuery(query, params = [], batchSize = 100) {
-    const values = params.map(toSqlValue);
-    for await (const rows of sql.unsafe(query, values).cursor(batchSize)) {
-        for (const row of rows) {
-            yield row;
-        }
-    }
-}

package/lib/src/scripts/debug_dosleg_query.d.ts

@@ -1,6 +0,0 @@
-#!/usr/bin/env node
-/**
- * Debug script to check the DosLeg query compilation
- * Helps identify the "column undefined" error
- */
-export {};

package/lib/src/scripts/debug_dosleg_query.js

@@ -1,50 +0,0 @@
-#!/usr/bin/env node
-/**
- * Debug script to check the DosLeg query compilation
- * Helps identify the "column undefined" error
- */
-import { db } from "../server/databases_postgres.js";
-import { jsonArrayFrom } from "kysely/helpers/postgres";
-import { sql } from "kysely";
-async function debugDoslegQuery() {
-    console.log("=== Debugging DosLeg Query ===\n");
-    try {
-        // Build the same query as findAllDossiers but capture SQL
-        const query = db
-            .withSchema("senat")
-            .selectFrom("dosleg_loi as loi")
-            .leftJoin("dosleg_typloi as typloi", "typloi.typloicod", "loi.typloicod")
-            .leftJoin("dosleg_etaloi as etaloi", "etaloi.etaloicod", "loi.etaloicod")
-            .leftJoin("dosleg_deccoc as deccoc", "deccoc.deccoccod", "loi.deccoccod")
-            .select((eb) => [
-            sql `rtrim(${eb.ref("loi.loicod")})`.as("code"),
-            "loi.numero as numero",
-            "loi.signet as signet",
-        ]);
-        const compiled = query.compile();
-        console.log("✓ Query compiled successfully (basic columns)\n");
-        console.log("SQL:", compiled.sql);
-        console.log("\n");
-        // Try with simple lectures first
-        const queryWithLectures = db
-            .withSchema("senat")
-            .selectFrom("dosleg_loi as loi")
-            .select((eb) => [
-            "loi.signet as signet",
-            jsonArrayFrom(eb.withSchema("senat").selectFrom("dosleg_lecture as lecture").select(["lecture.lecidt as id"])).as("lectures"),
-        ]);
-        const compiledWithLectures = queryWithLectures.compile();
-        console.log("✓ Query with simple lectures compiled successfully\n");
-        console.log("SQL:", compiledWithLectures.sql);
-        console.log("\n");
-        console.log("✅ Basic query structures seem valid");
-        console.log("\nThe issue is likely in one of the nested subqueries with cross-references.");
-        console.log("Try running findAllDossiers with the verbose flag to see exactly where it fails.");
-    }
-    catch (error) {
-        console.error("❌ Error during query compilation:");
-        console.error(error instanceof Error ? error.message : String(error));
-        process.exit(1);
-    }
-}
-debugDoslegQuery().catch(console.error);