@tricoteuses/senat 2.20.9 → 2.20.11

package/README.md

@@ -56,10 +56,10 @@ npm run data:parse_textes_lois ../senat-data
 npm run data:retrieve_agenda ../senat-data -- --fromSession 2022 [--parseAgenda]
 
 # Retrieval (& parsing) of comptes-rendus de séance from Sénat's data
-npm run data:retrieve_cr_seance ../senat-data -- [--parseDebats]
+npm run data:retrieve_cr_seance ../senat-data -- [--parseDebats] [--keepDir]
 
 # Retrieval (& parsing) of comptes-rendus de commissions from Sénat's website
-npm run data:retrieve_cr_commission ../senat-data -- [--parseDebats]
+npm run data:retrieve_cr_commission ../senat-data -- [--parseDebats] [--keepDir]
 
 # Retrieval of sénateurs' pictures from Sénat's website
 npm run data:retrieve_senateurs_photos ../senat-data

package/lib/model/ameli.js

@@ -123,7 +123,7 @@ const findAllAmendementsQuery = dbSenat
     "ameli.avigvt.lib as avis_gouvernement",
     eb.fn.coalesce("ameli.sor.lib", "ameli.irr.libirr").as("sort"),
     "ameli.amd.rev as revision",
-    concat(val("https://www.senat.fr/amendements/"), ref("ameli.ses.ann"), val("-"), sql `(ameli.ses.ann + 1)`, val("/"), ref("ameli.txt_ameli.numabs"), val("/Amdt_"), ref("ameli.amd.numabs"), val(".html")).as("url"),
+    concat(val("https://www.senat.fr/amendements/"), ref("ameli.ses.ann"), val("-"), sql `(ameli.ses.ann + 1)`, val("/"), ref("ameli.txt_ameli.numabs"), val("/Amdt_"), ref("ameli.amd.num"), val(".html")).as("url"),
     "ameli.grppol_ameli.lilcou as au_nom_de_groupe_politique",
     "ameli.com_ameli.lil as au_nom_de_commission",
     eb.case().when("ameli.cab.entid", "is not", null).then(true).else(false).end().as("auteur_est_gouvernement"),

package/lib/model/dosleg.js

@@ -34,10 +34,12 @@ function rapports(lectureAssembleeId) {
         .withSchema("dosleg")
         .selectFrom("rap")
         .leftJoin("lecassrap", "lecassrap.rapcod", "rap.rapcod")
+        .leftJoin("raporg", "raporg.rapcod", "rap.rapcod")
         .leftJoin("denrap", "denrap.coddenrap", "rap.coddenrap")
         .where("lecassrap.lecassidt", "=", lectureAssembleeId)
         .select(({ eb, ref, val }) => [
         "rap.rapnum as numero",
+        "raporg.orgcod as code_organisme",
         eb
             .case()
             .when("rap.typurl", "=", "I")
@@ -84,6 +86,7 @@ function textes(lectureAssembleeId) {
         .where("texte.lecassidt", "=", lectureAssembleeId)
         .select(({ eb, ref, val }) => [
         "texte.texnum as numero",
+        "texte.orgcod as code_organisme",
         eb
             .case()
             .when("texte.typurl", "=", "I")

package/lib/scripts/retrieve_agenda.js

@@ -10,6 +10,7 @@ import { ID_DATE_FORMAT } from "./datautil";
 import { commonOptions } from "./shared/cli_helpers";
 import { fetchWithRetry } from "./shared/util";
 import { groupNonSPByTypeOrganeHour, groupSeancePubliqueBySlot } from "../utils/reunion_grouping";
+import { buildSenatDossierIndex } from "../utils/reunion_odj_building";
 const optionsDefinitions = [
     ...commonOptions,
     {
@@ -26,9 +27,9 @@ class AgendaError extends Error {
         super(`An error occurred while retrieving Agenda ${agendaName}: ${message}`);
     }
 }
-async function retrieveAgendas(dataDir, sessions) {
+async function retrieveAgendas(options, sessions) {
     console.log(`[AGENDA] Retrieving agendas for sessions ${sessions.join(", ")}`);
-    const agendaRootDir = path.join(dataDir, AGENDA_FOLDER);
+    const agendaRootDir = path.join(options["dataDir"], AGENDA_FOLDER);
     fs.ensureDirSync(agendaRootDir);
     const originalAgendaDir = path.join(agendaRootDir, DATA_ORIGINAL_FOLDER);
     fs.ensureDirSync(originalAgendaDir);
@@ -36,14 +37,18 @@ async function retrieveAgendas(dataDir, sessions) {
     if (options["parseAgenda"]) {
         fs.ensureDirSync(transformedAgendaDir);
     }
+    let dossierIndex = {};
+    dossierIndex = buildSenatDossierIndex(options);
     for (const session of sessions) {
         const originalAgendaSessionDir = path.join(originalAgendaDir, `${session}`);
         fs.ensureDirSync(originalAgendaSessionDir);
-        fs.emptyDirSync(originalAgendaSessionDir);
+        if (!options["keepDir"])
+            fs.emptyDirSync(originalAgendaSessionDir);
         const transformedAgendaSessionDir = path.join(transformedAgendaDir, `${session}`);
         if (options["parseAgenda"]) {
             fs.ensureDirSync(transformedAgendaSessionDir);
-            fs.emptyDirSync(transformedAgendaSessionDir);
+            if (!options["keepDir"])
+                fs.emptyDirSync(transformedAgendaSessionDir);
         }
         const fifteenDaysFromNow = new Date();
         fifteenDaysFromNow.setDate(fifteenDaysFromNow.getDate() + 15); // Don't download agendas more than 15 days in the future
@@ -54,7 +59,7 @@ async function retrieveAgendas(dataDir, sessions) {
             try {
                 await downloadAgenda(agendaName, agendaPath);
                 if (options["parseAgenda"]) {
-                    await parseAgenda(transformedAgendaSessionDir, agendaFileName, agendaPath);
+                    await parseAgenda(transformedAgendaSessionDir, agendaFileName, agendaPath, dossierIndex);
                 }
             }
             catch (error) {
@@ -90,7 +95,7 @@ function writeGroupsAsFiles(dir, groups) {
         fs.writeJSONSync(outPath, g, { spaces: 2 });
     }
 }
-async function parseAgenda(transformedAgendaSessionDir, agendaFileName, agendaPath) {
+async function parseAgenda(transformedAgendaSessionDir, agendaFileName, agendaPath, dossierBySenatUrl) {
     if (!options["silent"])
         console.log(`Parsing Agenda ${agendaPath}…`);
     const parsedAgendaEvents = await parseAgendaFromFile(agendaPath);
@@ -99,7 +104,7 @@ async function parseAgenda(transformedAgendaSessionDir, agendaFileName, agendaPa
     const flatPath = path.join(transformedAgendaSessionDir, `${agendaFileName}.json`);
     fs.writeJSONSync(flatPath, parsedAgendaEvents, { spaces: 2 });
     // 1) SP → grouped by (date, slot)
-    const spGrouped = groupSeancePubliqueBySlot(parsedAgendaEvents);
+    const spGrouped = groupSeancePubliqueBySlot(parsedAgendaEvents, dossierBySenatUrl);
     // a) on a un Record<TimeSlot, GroupedReunion[]>, on le transforme en array
     const spGroups = Object.values(spGrouped).flat();
     // b) (reco) trier pour stabilité, comme pour les NON-SP
@@ -114,7 +119,7 @@ async function parseAgenda(transformedAgendaSessionDir, agendaFileName, agendaPa
         writeGroupsAsFiles(transformedAgendaSessionDir, spGroups);
     }
     // 2) NON-SP → grouped by (date, organe, hour)
-    const groupedBySuffix = groupNonSPByTypeOrganeHour(parsedAgendaEvents);
+    const groupedBySuffix = groupNonSPByTypeOrganeHour(parsedAgendaEvents, dossierBySenatUrl);
     for (const suffix of ["IDC", "IDM", "IDO", "IDI"]) {
         const groups = groupedBySuffix[suffix] || [];
         if (groups.length > 0) {
@@ -127,7 +132,7 @@ async function main() {
     assert(dataDir, "Missing argument: data directory");
     const sessions = getSessionsFromStart(options["fromSession"]);
     console.time("agenda processing time");
-    await retrieveAgendas(dataDir, sessions);
+    await retrieveAgendas(options, sessions);
     console.timeEnd("agenda processing time");
 }
 main()

package/lib/scripts/retrieve_cr_commission.js

@@ -1,4 +1,4 @@
-import fs from "fs-extra";
+import fs, { ensureDir } from "fs-extra";
 import assert from "assert";
 import path from "path";
 import * as cheerio from "cheerio";
@@ -187,7 +187,12 @@ async function retrieveCommissionCRs(options = {}) {
     const politenessMs = Number(options["politenessMs"] ?? 150);
     const commissionsRootDir = path.join(dataDir, COMMISSION_FOLDER);
     const originalRoot = path.join(commissionsRootDir, DATA_ORIGINAL_FOLDER);
-    ensureAndClearDir(originalRoot);
+    if (!options["keepDir"]) {
+        ensureAndClearDir(originalRoot);
+    }
+    else {
+        ensureDir(originalRoot);
+    }
     const discovered = await discoverCommissionWeeklyPages(fromSession);
     console.log(`[COM-CR][discover] ${discovered.length} links (>= session ${fromSession})`);
     const jobs = discovered.map(({ url, yyyymmdd, commissionKey }) => {
@@ -238,7 +243,9 @@ async function retrieveCommissionCRs(options = {}) {
     const sessions = getSessionsFromStart(options["fromSession"]);
     const comRoot = path.join(dataDir, COMMISSION_FOLDER);
     const transformedRoot = path.join(comRoot, DATA_TRANSFORMED_FOLDER);
-    if (options["parseDebats"])
+    if (options["keepDir"])
+        ensureDir(transformedRoot);
+    else
         ensureAndClearDir(transformedRoot);
     for (const session of sessions) {
         const originalSessionDir = path.join(originalRoot, String(session));

package/lib/scripts/shared/cli_helpers.d.ts

@@ -35,6 +35,11 @@ export declare const onlyRecentOption: {
     name: string;
     type: NumberConstructor;
 };
+export declare const keepDirOption: {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
 export declare const commonOptions: ({
     alias: string;
     defaultValue: string[];
@@ -48,12 +53,11 @@ export declare const commonOptions: ({
     name: string;
     type: StringConstructor;
 } | {
-    alias: string;
     help: string;
     name: string;
-    type: BooleanConstructor;
+    type: NumberConstructor;
 } | {
     help: string;
     name: string;
-    type: NumberConstructor;
+    type: BooleanConstructor;
 })[];

package/lib/scripts/shared/cli_helpers.js

@@ -35,6 +35,11 @@ export const onlyRecentOption = {
     name: "only-recent",
     type: Number,
 };
+export const keepDirOption = {
+    help: "keep directories when cleaning data",
+    name: "keepDir",
+    type: Boolean,
+};
 export const commonOptions = [
     categoriesOption,
     dataDirDefaultOption,
@@ -42,4 +47,5 @@ export const commonOptions = [
     silentOption,
     verboseOption,
     onlyRecentOption,
+    keepDirOption,
 ];

package/lib/types/agenda.d.ts

@@ -32,4 +32,13 @@ export interface GroupedReunion {
     transcriptionRef?: string;
     urlVideo?: string;
     timecodeDebutVideo?: number;
+    odj?: GroupedReunionOdj;
+}
+export interface GroupedReunionOdjPoint {
+    objet: string | null;
+    dossierLegislatifRef: string | null;
+    codeEtape: string | null;
+}
+export interface GroupedReunionOdj {
+    pointsOdj: GroupedReunionOdjPoint[];
 }

package/lib/utils/reunion_grouping.d.ts

@@ -1,7 +1,9 @@
 import { AgendaEvent, GroupedReunion, TimeSlot } from "../types/agenda";
+import { DossierLegislatifResult } from "../model/dosleg";
 type KnownType = "SP" | "COM" | "MC" | "OD" | "ID";
-export declare function groupNonSPByTypeOrganeHour(events: AgendaEvent[]): Record<"IDC" | "IDM" | "IDO" | "IDI", GroupedReunion[]>;
-export declare function groupSeancePubliqueBySlot(events: AgendaEvent[]): Record<TimeSlot, GroupedReunion[]>;
+type DossierBySenatUrl = Record<string, DossierLegislatifResult>;
+export declare function groupNonSPByTypeOrganeHour(events: AgendaEvent[], DossierBySenatUrl: DossierBySenatUrl): Record<"IDC" | "IDM" | "IDO" | "IDI", GroupedReunion[]>;
+export declare function groupSeancePubliqueBySlot(events: AgendaEvent[], dossierBySenatUrl: DossierBySenatUrl): Record<TimeSlot, GroupedReunion[]>;
 export declare function makeTypeGroupUid(dateISO: string, kind: KnownType, agendaEventId: string, organe?: string | null): string;
 export declare function makeGroupUid(date: string, slot: TimeSlot): string;
 export declare function formatYYYYMMDD(dateYYYYMMDD: string): string;

package/lib/utils/reunion_grouping.js

@@ -1,5 +1,6 @@
 import { DateTime } from "luxon";
 import { norm } from "../model/util";
+import { buildOdj } from "./reunion_odj_building";
 const PARIS = "Europe/Paris";
 const STOPWORDS = new Set([
     "de",
@@ -21,7 +22,7 @@ const STOPWORDS = new Set([
     "a",
     "aux",
 ]);
-export function groupNonSPByTypeOrganeHour(events) {
+export function groupNonSPByTypeOrganeHour(events, DossierBySenatUrl) {
     const out = { IDC: [], IDM: [], IDO: [], IDI: [] };
     if (!events?.length)
         return out;
@@ -49,6 +50,7 @@ export function groupNonSPByTypeOrganeHour(events) {
             titre: e.titre,
             objet: e.objet || "",
             events: [e],
+            odj: buildOdj([e], DossierBySenatUrl),
         };
         out[suffix].push(group);
     }
@@ -61,8 +63,7 @@ export function groupNonSPByTypeOrganeHour(events) {
     }
     return out;
 }
-export function groupSeancePubliqueBySlot(events) {
-    // Résultat à la manière de groupNonSPByTypeOrganeHour : objet de listes, ici indexé par créneau
+export function groupSeancePubliqueBySlot(events, dossierBySenatUrl) {
     const out = {};
     const ensureBucket = (slot) => (out[slot] ??= []);
     if (!events?.length)
@@ -70,7 +71,6 @@ export function groupSeancePubliqueBySlot(events) {
     const sp = events.filter((e) => isSeancePublique(e?.type));
     if (sp.length === 0)
         return out;
-    // Regroupement par date
     const byDate = new Map();
     for (const e of sp) {
         const d = norm(e.date);
@@ -124,6 +124,7 @@ export function groupSeancePubliqueBySlot(events) {
             }, null);
             const titres = sorted.map((x) => x.ev.titre || "").filter(Boolean);
             const captationVideo = sorted.some((x) => x.ev.captationVideo === true);
+            const eventsForSlot = sorted.map((x) => x.ev);
             ensureBucket(slot).push({
                 uid: makeGroupUid(date, slot),
                 chambre: "SN",
@@ -137,6 +138,7 @@ export function groupSeancePubliqueBySlot(events) {
                 titre: compactTitleList(titres, 5),
                 objet: joinObjets(sorted.map((x) => x.ev)),
                 events: sorted.map((x) => x.ev),
+                odj: buildOdj(eventsForSlot, dossierBySenatUrl),
             });
         }
     }
@@ -189,32 +191,6 @@ function typeToSuffixStrict(kind) {
             return "IDI";
     }
 }
-function hourShortFromISO(iso) {
-    if (!iso)
-        return null;
-    const dt = parseISO(iso);
-    if (!dt)
-        return null;
-    const z = DateTime.fromISO(iso, { zone: PARIS });
-    const H = String(z.hour);
-    const mm = String(z.minute).padStart(2, "0");
-    return `${H}${mm}`;
-}
-function hourShortFromOriginal(s) {
-    if (!s)
-        return null;
-    const clean = normalizeNoAccents(s).toLowerCase();
-    const m = clean.match(/(\d{1,2})\s*[h:]\s*(\d{2})/);
-    if (m) {
-        const H = String(parseInt(m[1], 10));
-        const mm = m[2].padStart(2, "0");
-        return `${H}${mm}`;
-    }
-    const m2 = clean.match(/(\d{1,2})\s*h\b/);
-    if (m2)
-        return `${parseInt(m2[1], 10)}00`;
-    return null;
-}
 function organeInitials(input, maxLen = 8) {
     if (!input)
         return "";

package/lib/utils/reunion_odj_building.d.ts

@@ -0,0 +1,5 @@
+import commandLineArgs from "command-line-args";
+import { DossierLegislatifResult } from "../model/dosleg";
+import { AgendaEvent, GroupedReunionOdj } from "../types/agenda";
+export declare function buildOdj(events: AgendaEvent[], dossierBySenatUrl: Record<string, DossierLegislatifResult>): GroupedReunionOdj | undefined;
+export declare function buildSenatDossierIndex(options: commandLineArgs.CommandLineOptions): Record<string, DossierLegislatifResult>;

package/lib/utils/reunion_odj_building.js

@@ -0,0 +1,162 @@
+import { getSessionsFromStart } from "../types/sessions";
+import { iterLoadSenatDossiersLegislatifs } from "../loaders";
+export function buildOdj(events, dossierBySenatUrl) {
+    const byObjet = new Map(); // objet -> set de dossier uids
+    let codeEtape = null;
+    let dossier = null;
+    for (const ev of events) {
+        const objetKey = (ev.objet ?? "").trim();
+        const url = normalizeSenatUrl(ev.urlDossierSenat) ?? undefined;
+        if (url)
+            console.log(` urlDossierSenat: ${url}`);
+        dossier = url ? dossierBySenatUrl[url] : null;
+        const dossierUid = dossier ? pickDossierUid(dossier) : undefined;
+        if (url)
+            console.log(`  → matched dossier uid: ${dossierUid}`);
+        codeEtape = dossier ? computeCodeEtape(ev, dossier) : null;
+        // si on n’a ni objet ni dossier, ça ne sert à rien de créer un point
+        if (!objetKey && !dossierUid)
+            continue;
+        if (!byObjet.has(objetKey) && dossierUid) {
+            byObjet.set(objetKey, dossierUid);
+        }
+    }
+    if (byObjet.size === 0)
+        return undefined;
+    const pointsOdj = [];
+    for (const [objetKey, dossierUid] of byObjet) {
+        pointsOdj.push({
+            objet: objetKey || null,
+            dossierLegislatifRef: dossierUid || null,
+            codeEtape,
+        });
+    }
+    return { pointsOdj };
+}
+function pickDossierUid(d) {
+    if (d["signet"] && d["signet"].trim())
+        return d["signet"].trim();
+    if (d["code"] && String(d["code"]).trim())
+        return String(d["code"]).trim();
+    return undefined;
+}
+function normalizeSenatUrl(url) {
+    if (!url)
+        return null;
+    let u = url.trim();
+    if (!u)
+        return null;
+    if (!/^https?:\/\//i.test(u))
+        return u;
+    // force https://
+    u = u.replace(/^http:\/\//i, "https://");
+    u = u.replace(/\/+$/, "");
+    return u;
+}
+export function buildSenatDossierIndex(options) {
+    const index = {};
+    const sessions = getSessionsFromStart(2015);
+    for (const session of sessions) {
+        for (const item of iterLoadSenatDossiersLegislatifs(options["dataDir"], session)) {
+            const dossier = item.item;
+            const url = dossier["url"] ? normalizeSenatUrl(dossier["url"]) : undefined;
+            if (url)
+                index[url] = dossier;
+        }
+    }
+    return index;
+}
+function detectLecture(objet) {
+    objet = objet.toLowerCase();
+    if (objet.includes("première lecture"))
+        return 1;
+    if (objet.includes("deuxième lecture") || objet.includes("2ème"))
+        return 2;
+    if (objet.includes("troisième lecture") || objet.includes("3ème"))
+        return 3;
+    return undefined;
+}
+function computeCodeEtape(ev, dossier) {
+    const lecture = detectLecture(ev.objet ?? "");
+    const organe = ev.organe ?? "";
+    const nature = organe.toLowerCase().includes("commission")
+        ? "COM"
+        : organe.toLowerCase().includes("séance publique")
+            ? "DEBATS"
+            : "";
+    const evDate = ev.date.split("T")[0];
+    const flat = buildFlatActes(dossier);
+    // 1) Match strict : même date + nature (COM / DEBATS si connue)
+    let candidates = flat.filter((a) => {
+        if (a.date !== evDate)
+            return false;
+        if (nature && !a.codeActe.includes(nature))
+            return false;
+        return true;
+    });
+    console.log(`  → candidats STRICT (date==${evDate} & nature=${nature || "ANY"}): ${candidates.length}`);
+    candidates.forEach((c) => console.log(`    STRICT MATCH: ${c.codeActe} (date=${c.date}, lecture=${c.ordreLecture})`));
+    // Si lecture détectée → on filtre si ça garde des candidats
+    if (lecture !== undefined && candidates.length > 0) {
+        const withLecture = candidates.filter((c) => c.ordreLecture === lecture);
+        if (withLecture.length > 0) {
+            console.log(`  → filtre lecture=${lecture} : ${withLecture.length} candidats`);
+            candidates = withLecture;
+        }
+    }
+    // Multiple candidates : we take the longest ?
+    if (candidates.length > 0) {
+        candidates.sort((a, b) => b.codeActe.length - a.codeActe.length);
+        console.log(`  ✔ match FINAL (intervalle) : ${candidates[0].codeActe}`);
+        return candidates[0].codeActe;
+    }
+    // 2) fallback COM : dernier acte COM avant la date
+    if (nature === "COM") {
+        let comActs = flat.filter((a) => a.codeActe.includes("COM") && a.date <= evDate);
+        console.log(`  → fallback COM: actes COM <= date : ${comActs.length}`);
+        if (lecture !== undefined) {
+            const byLecture = comActs.filter((a) => a.ordreLecture === lecture);
+            if (byLecture.length > 0) {
+                comActs = byLecture;
+                console.log(`  → filtrés lecture=${lecture}: ${comActs.length}`);
+            }
+        }
+        if (comActs.length > 0) {
+            comActs.sort((a, b) => b.date.localeCompare(a.date) || b.codeActe.length - a.codeActe.length);
+            console.log(`  ✔ match FINAL (fallback COM): ${comActs[0].codeActe}`);
+            return comActs[0].codeActe;
+        }
+    }
+    // 3) Fallback lecture générale (SN1, SN2…) depuis la structure d’origine
+    if (lecture !== undefined) {
+        const lectures = dossier["actes_legislatifs"] ?? [];
+        const lectureNode = lectures.find((l) => l.ordre_lecture === lecture);
+        const rootCode = lectureNode?.code_acte;
+        if (rootCode && typeof rootCode === "string") {
+            console.log(`  ✔ FALLBACK LECTURE: ${rootCode}`);
+            return rootCode;
+        }
+    }
+    console.log(`  ✖ aucun code d’étape trouvé pour ev=${ev.id}`);
+    return null;
+}
+function buildFlatActes(dossier) {
+    const lectures = dossier["actes_legislatifs"] ?? [];
+    const res = [];
+    for (const lectureActe of lectures) {
+        const assemblee = lectureActe["assemblee"];
+        if (assemblee !== "Sénat")
+            continue;
+        const ordreLecture = lectureActe.ordre_lecture;
+        const sub = lectureActe.actes_legislatifs;
+        const actes = Array.isArray(sub) && sub.length > 0 ? sub : [lectureActe];
+        for (const acte of actes) {
+            const codeActe = acte.code_acte;
+            const dateActe = acte.date?.split("T")[0];
+            if (!codeActe || !dateActe)
+                continue;
+            res.push({ codeActe, ordreLecture, date: dateActe });
+        }
+    }
+    return res;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.20.9",
+  "version": "2.20.11",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",