@tricoteuses/senat 2.20.28 → 2.20.30

package/lib/loaders.d.ts

@@ -5,9 +5,11 @@ import { QuestionResult } from "./model/questions";
 import { ScrutinResult } from "./model/scrutins";
 import { CirconscriptionResult, OrganismeResult, SenateurResult } from "./model/sens";
 import { Reunion } from "./types/agenda";
-import { FlatTexte } from "./types/texte";
+import { FlatTexte, DocumentMetadata } from "./types/texte";
 import { CompteRendu } from "./types/compte_rendu";
+import { DocumentResult } from "./model/documents";
 export { EnabledDatasets } from "./datasets";
+export type { DocumentResult } from "./model/documents";
 export declare const AGENDA_FOLDER = "agenda";
 export declare const COMPTES_RENDUS_FOLDER = "seances";
 export declare const COMMISSION_FOLDER = "commissions";
@@ -27,46 +29,6 @@ export type IterItem<T> = {
     legislature?: number;
     gitStatus?: "A" | "M" | "D" | "R" | "C" | "T" | "U";
 };
-export interface TexteMetadata {
-    name: string;
-    session: number | null | undefined;
-    date?: string | null;
-    url_expose_des_motifs?: URL;
-    url_xml: URL;
-    url_html: URL;
-    url_pdf: URL;
-}
-export interface RapportMetadata {
-    name: string;
-    session: number | null | undefined;
-    date?: string | null;
-    url_html: URL;
-    url_pdf: URL;
-}
-export interface DossierLegislatifDocumentResult {
-    signet_dossier: string;
-    url_dossier_senat: string;
-    url_dossier_assemblee_nationale: string | null;
-    type_lecture: string;
-    libelle_lecture: string;
-    libelle_organisme: string | null;
-    code_organisme: string | null;
-    numero: number | null;
-    id: string | null;
-    url: string;
-    origine?: string | null | undefined;
-    type: string;
-    date: string;
-    session: number | null;
-    auteurs: {
-        prenom: string | null;
-        nom_usuel: string;
-        matricule: string | null;
-    }[];
-    organismes?: {
-        libelle: string;
-    }[] | undefined;
-}
 export declare function iterFilePaths(dirPath: string): Generator<string>;
 export declare function iterLoadSenatAmendements(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<AmendementResult>>;
 export declare function iterLoadSenatDebats(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DebatResult>>;
@@ -79,11 +41,11 @@ export declare function iterLoadSenatComptesRendusCommissions(dataDir: string, s
     session: number;
 }>;
 export declare function iterLoadSenatDossiersLegislatifs(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DossierLegislatifResult>>;
-export declare function iterLoadSenatDossiersLegislatifsRapportUrls(dataDir: string, session: number | undefined): Generator<IterItem<RapportMetadata>>;
-export declare function iterLoadSenatDossiersLegislatifsTexteUrls(dataDir: string, session: number | undefined): Generator<IterItem<TexteMetadata>>;
-export declare function iterLoadSenatDossiersLegislatifsDocuments(dataDir: string, session: number | undefined, documentType: "textes" | "rapports", options?: {}): Generator<IterItem<DossierLegislatifDocumentResult>>;
-export declare function iterLoadSenatDossiersLegislatifsRapports(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DossierLegislatifDocumentResult>>;
-export declare function iterLoadSenatDossiersLegislatifsTextes(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DossierLegislatifDocumentResult>>;
+export declare function iterLoadSenatDossiersLegislatifsRapportUrls(dataDir: string, session: number | undefined): Generator<IterItem<DocumentMetadata>>;
+export declare function iterLoadSenatDossiersLegislatifsTexteUrls(dataDir: string, session: number | undefined): Generator<IterItem<DocumentMetadata>>;
+export declare function iterLoadSenatDossiersLegislatifsDocuments(dataDir: string, session: number | undefined, documentType: "textes" | "rapports", options?: {}): Generator<IterItem<DocumentResult>>;
+export declare function iterLoadSenatDossiersLegislatifsRapports(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DocumentResult>>;
+export declare function iterLoadSenatDossiersLegislatifsTextes(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DocumentResult>>;
 export declare function loadSenatTexteContent(dataDir: string, textePathFromDataset: string): IterItem<FlatTexte | null>;
 export declare function loadSenatCompteRenduContent(dataDir: string, session: number, debatId: string | number): {
     item: CompteRendu | null;

package/lib/model/documents.d.ts

@@ -1,2 +1,12 @@
-export declare function findSenatTexteUrls(sessions?: number[]): any;
-export declare function findSenatRapportUrls(sessions?: number[]): any;
+import { Expression, InferResult, SelectQueryBuilder } from "kysely";
+export declare function rapports(lectureAssembleeId: Expression<string>): import("kysely").RawBuilder<{
+    [x: string]: any;
+}[]>;
+declare const queryTextes: SelectQueryBuilder<any, any, any>;
+export declare function textes(lectureAssembleeId: Expression<string>): import("kysely").RawBuilder<{
+    [x: string]: any;
+}[]>;
+export declare function findAllTextes(): AsyncIterableIterator<DocumentResult>;
+export declare function findAllRapports(): AsyncIterableIterator<DocumentResult>;
+export type DocumentResult = InferResult<typeof queryTextes>[0];
+export {};

package/lib/model/documents.js

@@ -1,33 +1,128 @@
+import { sql } from "kysely";
 import { dbSenat } from "../databases";
-import { rtrim, toDateString } from "./util";
-export function findSenatTexteUrls(sessions = []) {
-    return dbSenat
+import { concat, rtrim, toDateString } from "./util";
+import { jsonArrayFrom } from "kysely/helpers/postgres";
+function orderOrdreOrigineTexte(expr) {
+    return sql `array_position(array['0','2','1'], ${expr})`;
+}
+function auteursRapport(rapportId) {
+    return jsonArrayFrom(dbSenat
         .withSchema("dosleg")
-        .selectFrom("texte")
-        .where("texurl", "is not", null)
-        .where("typurl", "=", "I")
-        .$if(sessions.length > 0, (qb) => qb.where("sesann", "in", sessions))
-        .select(({ eb, ref }) => [
-        "sesann as session",
-        rtrim(ref("texurl")).as("url"),
-        toDateString(ref("txtoritxtdat")).as("date"),
-        eb.case().when("oritxtcod", "=", "1").then(true).else(false).end().as("hasExposeDesMotifs"),
+        .selectFrom("dosleg.auteur")
+        .leftJoin("dosleg.ecr", "dosleg.ecr.autcod", "dosleg.auteur.autcod")
+        .leftJoin("dosleg.rolsig", "dosleg.rolsig.signataire", "dosleg.ecr.signataire")
+        .where("dosleg.ecr.rapcod", "=", rapportId)
+        .select([
+        "dosleg.auteur.prenom as prenom",
+        "dosleg.auteur.nomuse as nom_usuel",
+        "dosleg.auteur.autmat as matricule",
+        "dosleg.ecr.ecrnumtri as ordre",
+        "dosleg.rolsig.rolsiglib as role",
+        "dosleg.ecr.ecrqua as qualite",
     ])
-        .$narrowType()
-        .stream();
+        .orderBy("dosleg.ecr.ecrnumtri", "asc"));
+}
+function documentsAttaches(rapportId) {
+    return jsonArrayFrom(dbSenat
+        .withSchema("dosleg")
+        .selectFrom("docatt")
+        .leftJoin("typatt", "docatt.typattcod", "typatt.typattcod")
+        .where("docatt.rapcod", "=", rapportId)
+        .select([
+        "docatt.docatturl as url",
+        "typatt.typattlib as type_document"
+    ]));
+}
+const queryRapports = dbSenat
+    .withSchema("dosleg")
+    .selectFrom("rap")
+    .leftJoin("raporg", "raporg.rapcod", "rap.rapcod")
+    .leftJoin("denrap", "denrap.coddenrap", "rap.coddenrap")
+    .select(({ eb, ref, val }) => [
+    "rap.rapnum as numero",
+    "raporg.orgcod as code_organisme",
+    eb
+        .case()
+        .when("rap.rapurl", "is not", null)
+        .then(sql `regexp_replace(trim(${ref("rap.rapurl")}), '^(.*/)?(.*?)(\\.html)?$', '\\2')`)
+        .else(null)
+        .end()
+        .as("id"),
+    eb
+        .case()
+        .when("rap.typurl", "=", "I")
+        .then(concat(val("https://www.senat.fr/rap/"), rtrim(ref("rap.rapurl"))))
+        .else(rtrim(ref("rap.rapurl")))
+        .end()
+        .as("url"),
+    rtrim(ref("denrap.libdenrap")).as("type"),
+    rtrim(ref("rap.raptil")).as("titre"),
+    rtrim(ref("rap.rapsoustit")).as("sous_titre"),
+    toDateString(ref("rap.date_depot")).as("date"),
+    "sesann as session",
+    auteursRapport(ref("rap.rapcod")).as("auteurs"),
+    documentsAttaches(ref("rap.rapcod")).as("documents_annexes"),
+]);
+export function rapports(lectureAssembleeId) {
+    return jsonArrayFrom(queryRapports
+        .leftJoin("lecassrap", "lecassrap.rapcod", "rap.rapcod")
+        .where("lecassrap.lecassidt", "=", lectureAssembleeId));
 }
-export function findSenatRapportUrls(sessions = []) {
-    return dbSenat
+function auteursTexte(texteId) {
+    return jsonArrayFrom(dbSenat
         .withSchema("dosleg")
-        .selectFrom("rap")
-        .where("rapurl", "is not", null)
-        .where("typurl", "=", "I")
-        .$if(sessions.length > 0, (qb) => qb.where("sesann", "in", sessions))
-        .select(({ ref }) => [
-        "sesann as session",
-        rtrim(ref("rapurl")).as("url"),
-        toDateString(ref("date_depot")).as("date"),
+        .selectFrom("auteur")
+        .leftJoin("ecr", "ecr.autcod", "auteur.autcod")
+        .leftJoin("rolsig", "rolsig.signataire", "ecr.signataire")
+        .where("ecr.texcod", "=", texteId)
+        .select([
+        "auteur.prenom as prenom",
+        "auteur.nomuse as nom_usuel",
+        "auteur.autmat as matricule",
+        "ecr.ecrnumtri as ordre",
+        "rolsig.rolsiglib as role",
+        "ecr.ecrqua as qualite",
     ])
-        .$narrowType()
-        .stream();
+        .orderBy("ecr.ecrnumtri", "asc"));
+}
+const queryTextes = dbSenat
+    .withSchema("dosleg")
+    .selectFrom("texte")
+    .leftJoin("oritxt", "oritxt.oritxtcod", "texte.oritxtcod")
+    .leftJoin("typtxt", "typtxt.typtxtcod", "texte.typtxtcod")
+    .select(({ eb, ref, val }) => [
+    "texte.texnum as numero",
+    "texte.orgcod as code_organisme",
+    eb
+        .case()
+        .when("texte.texurl", "is not", null)
+        .then(sql `regexp_replace(trim(${ref("texte.texurl")}), '^(.*/)?(.*?)(\\.html)?$', '\\2')`)
+        .else(null)
+        .end()
+        .as("id"),
+    eb
+        .case()
+        .when("texte.typurl", "=", "I")
+        .then(concat(val("https://www.senat.fr/leg/"), rtrim(ref("texte.texurl"))))
+        .else(rtrim(ref("texte.texurl")))
+        .end()
+        .as("url"),
+    rtrim(ref("oritxt.oritxtlib")).as("origine"),
+    "oritxt.oriordre as ordre_origine",
+    "oritxt.oritxtado as code_adoption",
+    "oritxt.oritxtmod as modification",
+    rtrim(ref("typtxt.typtxtlib")).as("type"),
+    toDateString(ref("texte.txtoritxtdat")).as("date"),
+    "sesann as session",
+    auteursTexte(ref("texte.texcod")).as("auteurs"),
+])
+    .orderBy(({ ref }) => orderOrdreOrigineTexte(ref("oritxt.oriordre")));
+export function textes(lectureAssembleeId) {
+    return jsonArrayFrom(queryTextes.where("texte.lecassidt", "=", lectureAssembleeId));
+}
+export function findAllTextes() {
+    return queryTextes.stream();
+}
+export function findAllRapports() {
+    return queryRapports.stream();
 }

package/lib/model/dosleg.js

@@ -1,10 +1,8 @@
 import { sql } from "kysely";
 import { jsonArrayFrom } from "kysely/helpers/postgres";
 import { dbSenat } from "../databases";
-import { concat, removeSubstring, rtrim, toDateString } from "./util";
-function orderOrdreOrigineTexte(expr) {
-    return sql `array_position(array['0','2','1'], ${expr})`;
-}
+import { concat, rtrim, toDateString } from "./util";
+import { textes, rapports } from "./documents";
 function datesSeances(lectureAssembleeId) {
     return jsonArrayFrom(dbSenat
         .withSchema("dosleg")
@@ -12,120 +10,6 @@ function datesSeances(lectureAssembleeId) {
         .where("dosleg.date_seance.lecidt", "=", lectureAssembleeId)
         .select(({ ref }) => [toDateString(ref("dosleg.date_seance.date_s")).as("date")]));
 }
-function auteursRapport(rapportId) {
-    return jsonArrayFrom(dbSenat
-        .withSchema("dosleg")
-        .selectFrom("dosleg.auteur")
-        .leftJoin("dosleg.ecr", "dosleg.ecr.autcod", "dosleg.auteur.autcod")
-        .leftJoin("dosleg.rolsig", "dosleg.rolsig.signataire", "dosleg.ecr.signataire")
-        .where("dosleg.ecr.rapcod", "=", rapportId)
-        .select([
-        "dosleg.auteur.prenom as prenom",
-        "dosleg.auteur.nomuse as nom_usuel",
-        "dosleg.auteur.autmat as matricule",
-        "dosleg.ecr.ecrnumtri as ordre",
-        "dosleg.rolsig.rolsiglib as role",
-        "dosleg.ecr.ecrqua as qualite",
-    ])
-        .orderBy("dosleg.ecr.ecrnumtri", "asc"));
-}
-function documentsAttaches(rapportId) {
-    return jsonArrayFrom(dbSenat
-        .withSchema("dosleg")
-        .selectFrom("docatt")
-        .leftJoin("typatt", "docatt.typattcod", "typatt.typattcod")
-        .where("docatt.rapcod", "=", rapportId)
-        .select([
-        "docatt.docatturl as url",
-        "typatt.typattlib as type_document"
-    ]));
-}
-function rapports(lectureAssembleeId) {
-    return jsonArrayFrom(dbSenat
-        .withSchema("dosleg")
-        .selectFrom("rap")
-        .leftJoin("lecassrap", "lecassrap.rapcod", "rap.rapcod")
-        .leftJoin("raporg", "raporg.rapcod", "rap.rapcod")
-        .leftJoin("denrap", "denrap.coddenrap", "rap.coddenrap")
-        .where("lecassrap.lecassidt", "=", lectureAssembleeId)
-        .select(({ eb, ref, val }) => [
-        "rap.rapnum as numero",
-        "raporg.orgcod as code_organisme",
-        eb
-            .case()
-            .when("rap.typurl", "=", "I")
-            .then(removeSubstring(ref("rap.rapurl"), val(".html")))
-            .else(null)
-            .end()
-            .as("id"),
-        eb
-            .case()
-            .when("rap.typurl", "=", "I")
-            .then(concat(val("https://www.senat.fr/rap/"), rtrim(ref("rap.rapurl"))))
-            .else(rtrim(ref("rap.rapurl")))
-            .end()
-            .as("url"),
-        rtrim(ref("denrap.libdenrap")).as("type"),
-        rtrim(ref("rap.raptil")).as("titre"),
-        rtrim(ref("rap.rapsoustit")).as("sous_titre"),
-        toDateString(ref("rap.date_depot")).as("date"),
-        "sesann as session",
-        auteursRapport(ref("rap.rapcod")).as("auteurs"),
-        documentsAttaches(ref("rap.rapcod")).as("documents_annexes"),
-    ]));
-}
-function auteursTexte(texteId) {
-    return jsonArrayFrom(dbSenat
-        .withSchema("dosleg")
-        .selectFrom("auteur")
-        .leftJoin("ecr", "ecr.autcod", "auteur.autcod")
-        .leftJoin("rolsig", "rolsig.signataire", "ecr.signataire")
-        .where("ecr.texcod", "=", texteId)
-        .select([
-        "auteur.prenom as prenom",
-        "auteur.nomuse as nom_usuel",
-        "auteur.autmat as matricule",
-        "ecr.ecrnumtri as ordre",
-        "rolsig.rolsiglib as role",
-        "ecr.ecrqua as qualite",
-    ])
-        .orderBy("ecr.ecrnumtri", "asc"));
-}
-function textes(lectureAssembleeId) {
-    return jsonArrayFrom(dbSenat
-        .withSchema("dosleg")
-        .selectFrom("texte")
-        .leftJoin("oritxt", "oritxt.oritxtcod", "texte.oritxtcod")
-        .leftJoin("typtxt", "typtxt.typtxtcod", "texte.typtxtcod")
-        .where("texte.lecassidt", "=", lectureAssembleeId)
-        .select(({ eb, ref, val }) => [
-        "texte.texnum as numero",
-        "texte.orgcod as code_organisme",
-        eb
-            .case()
-            .when("texte.typurl", "=", "I")
-            .then(removeSubstring(ref("texte.texurl"), val(".html")))
-            .else(null)
-            .end()
-            .as("id"),
-        eb
-            .case()
-            .when("texte.typurl", "=", "I")
-            .then(concat(val("https://www.senat.fr/leg/"), rtrim(ref("texte.texurl"))))
-            .else(rtrim(ref("texte.texurl")))
-            .end()
-            .as("url"),
-        rtrim(ref("oritxt.oritxtlib")).as("origine"),
-        "oritxt.oriordre as ordre_origine",
-        "oritxt.oritxtado as code_adoption",
-        "oritxt.oritxtmod as modification",
-        rtrim(ref("typtxt.typtxtlib")).as("type"),
-        toDateString(ref("texte.txtoritxtdat")).as("date"),
-        "sesann as session",
-        auteursTexte(ref("texte.texcod")).as("auteurs"),
-    ])
-        .orderBy(({ ref }) => orderOrdreOrigineTexte(ref("oritxt.oriordre"))));
-}
 function lecturesAssemblee(lectureId) {
     return jsonArrayFrom(dbSenat
         .withSchema("dosleg")

package/lib/model/index.d.ts

@@ -1,7 +1,7 @@
 export { findAllAmendements } from "./ameli";
 export { findAll as findAllDebats } from "./debats";
 export { findAllDossiers } from "./dosleg";
-export { findSenatTexteUrls, findSenatRapportUrls } from "./documents";
+export { findAllTextes, findAllRapports } from "./documents";
 export { findAllScrutins } from "./scrutins";
 export { findAll as findAllQuestions } from "./questions";
 export { findAll as findAllSens, findAllCirconscriptions, findAllOrganismes } from "./sens";

package/lib/model/index.js

@@ -1,7 +1,7 @@
 export { findAllAmendements } from "./ameli";
 export { findAll as findAllDebats } from "./debats";
 export { findAllDossiers } from "./dosleg";
-export { findSenatTexteUrls, findSenatRapportUrls } from "./documents";
+export { findAllTextes, findAllRapports } from "./documents";
 export { findAllScrutins } from "./scrutins";
 export { findAll as findAllQuestions } from "./questions";
 export { findAll as findAllSens, findAllCirconscriptions, findAllOrganismes } from "./sens";

package/lib/scripts/convert_data.js

@@ -5,8 +5,8 @@ import path from "path";
 import pLimit from "p-limit";
 import * as git from "../git";
 import { datasets, EnabledDatasets, getEnabledDatasets } from "../datasets";
-import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, SCRUTINS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER, } from "../loaders";
-import { findAllAmendements, findAllCirconscriptions, findAllDebats, findAllDossiers, findAllScrutins, findAllOrganismes, findAllQuestions, findAllSens, findSenatRapportUrls, findSenatTexteUrls, } from "../model";
+import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, SCRUTINS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER } from "../loaders";
+import { findAllAmendements, findAllCirconscriptions, findAllDebats, findAllDossiers, findAllScrutins, findAllOrganismes, findAllQuestions, findAllSens, findAllTextes, findAllRapports, } from "../model";
 import { processRapport, processTexte } from "./retrieve_documents";
 import { buildActesLegislatifs } from "../model/dosleg";
 import { UNDEFINED_SESSION } from "../types/sessions";
@@ -169,8 +169,8 @@ async function convertDatasetDosLeg(dataDir, options) {
         const dossierFile = `${dossier["signet"]}.json`;
         await fs.outputJSON(path.join(dossierReorganizedDir, dossierFile), dossierWithActes, { spaces: 2 });
     }
-    await convertTexteUrls(dataDir, options);
-    await convertRapportUrls(dataDir, options);
+    await convertTextes(dataDir, options);
+    await convertRapports(dataDir, options);
 }
 async function convertDatasetScrutins(dataDir, options) {
     const dataset = datasets.dosleg;
@@ -219,30 +219,38 @@ async function convertDatasetQuestions(dataDir, options) {
     }
     await Promise.all(tasks);
 }
-async function convertTexteUrls(dataDir, options) {
+async function convertTextes(dataDir, options) {
     const originalTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER);
     const transformedTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_TRANSFORMED_FOLDER);
     if (!options["silent"]) {
         console.log(`Converting database textes data into files…`);
     }
-    for await (const texte of findSenatTexteUrls()) {
-        const session = texte.session ?? UNDEFINED_SESSION;
+    for await (const texte of findAllTextes()) {
+        const session = texte["session"] ?? UNDEFINED_SESSION;
         if (options["fromSession"] && session < options["fromSession"]) {
             continue;
         }
-        const texteName = path.parse(texte.url).name;
+        if (!texte["url"]) {
+            continue;
+        }
+        const texteName = path.parse(texte["url"]).name;
         const texteDir = path.join(originalTextesDir, `${session}`, texteName);
+        // oritxtcod = 1 corresponds to "Texte de loi déposé au Sénat"
+        const hasExposeDesMotifs = texte["origine"] === 'Sénat' && texte["ordre_origine"] === '1';
         const metadata = {
             name: texteName,
-            session: texte.session,
-            date: texte.date,
-            url_expose_des_motifs: texte.hasExposeDesMotifs
+            session: texte["session"],
+            date: texte["date"],
+            url_expose_des_motifs: hasExposeDesMotifs
                 ? new URL(`${texteName}-expose.html`, SENAT_EXPOSE_DES_MOTIFS_BASE_URL)
                 : undefined,
             url_xml: new URL(`${texteName}.akn.xml`, SENAT_TEXTE_XML_BASE_URL),
             url_html: new URL(`${texteName}.html`, SENAT_TEXTE_BASE_URL),
             url_pdf: new URL(`${texteName}.pdf`, SENAT_TEXTE_BASE_URL),
         };
+        fs.outputJSONSync(path.join(texteDir, `${texteName}.json`), texte, {
+            spaces: 2,
+        });
         fs.outputJSONSync(path.join(texteDir, DOCUMENT_METADATA_FILE), metadata, {
             spaces: 2,
         });
@@ -251,17 +259,20 @@ async function convertTexteUrls(dataDir, options) {
         }
     }
 }
-async function convertRapportUrls(dataDir, options) {
+async function convertRapports(dataDir, options) {
     const originalRapportsDir = path.join(dataDir, RAPPORT_FOLDER, DATA_ORIGINAL_FOLDER);
     if (!options["silent"]) {
         console.log(`Converting database rapports data into files…`);
     }
-    for await (const rapport of findSenatRapportUrls()) {
-        const session = rapport.session ?? UNDEFINED_SESSION;
+    for await (const rapport of findAllRapports()) {
+        const session = rapport["session"] ?? UNDEFINED_SESSION;
         if (options["fromSession"] && session < options["fromSession"]) {
             continue;
         }
-        const parsedRapportUrl = path.parse(rapport.url);
+        if (!rapport["url"]) {
+            continue;
+        }
+        const parsedRapportUrl = path.parse(rapport["url"]);
         const rapportName = parsedRapportUrl.name;
         const rapportDir = path.join(originalRapportsDir, `${session}`, rapportName);
         const rapportHtmlUrlBase = `${rapportName}_mono.html`;
@@ -276,11 +287,14 @@ async function convertRapportUrls(dataDir, options) {
         });
         const metadata = {
             name: rapportName,
-            session: rapport.session,
-            date: rapport.date,
+            session: rapport["session"],
+            date: rapport["date"],
             url_html: new URL(rapportHtmlUrl, SENAT_RAPPORT_BASE_URL),
             url_pdf: new URL(rapportPdfUrl, SENAT_RAPPORT_BASE_URL),
         };
+        fs.outputJSONSync(path.join(rapportDir, `${rapportName}.json`), rapport, {
+            spaces: 2,
+        });
         fs.outputJSONSync(path.join(rapportDir, DOCUMENT_METADATA_FILE), metadata, {
             spaces: 2,
         });

package/lib/scripts/retrieve_documents.d.ts

@@ -1,2 +1,3 @@
-export declare function processTexte(texteMetadata: any, originalTextesDir: string, transformedTextesDir: string, options: any): Promise<void>;
+import { DocumentMetadata } from "../types/texte";
+export declare function processTexte(texteMetadata: DocumentMetadata, originalTextesDir: string, transformedTextesDir: string, options: any): Promise<void>;
 export declare function processRapport(rapportMetadata: any, originalRapportsDir: string, options: any): Promise<void>;

package/lib/scripts/retrieve_documents.js

@@ -111,6 +111,8 @@ export async function processTexte(texteMetadata, originalTextesDir, transformed
     for (const format of formats) {
         if (!isOptionEmptyOrHasValue(options.formats, format.type))
             continue;
+        if (!format.url)
+            continue;
         const destPath = path.join(texteDir, `${texteMetadata.name}.${format.type}`);
         const result = await processDocument(format.url.toString(), destPath, texteMetadata.date, options);
         // Specific logic: Parsing (Only applies to XML)

package/lib/scripts/test_iter_load.js

@@ -1,18 +1,14 @@
-import { iterLoadSenatAmendements, iterLoadSenatDossiersLegislatifs } from "../loaders";
+import { iterLoadSenatDossiersLegislatifsRapports } from "../loaders";
 import commandLineArgs from "command-line-args";
 import { dataDirDefaultOption } from "./shared/cli_helpers";
 const optionsDefinitions = [dataDirDefaultOption];
 const options = commandLineArgs(optionsDefinitions);
 const session = 2024;
 const sinceCommit = undefined;
-for (const { item: amendement, filePathFromDataset } of iterLoadSenatAmendements(options["dataDir"], session, {
-    log: true,
+for (const { item: dossierLegislatif } of iterLoadSenatDossiersLegislatifsRapports(options["dataDir"], session, {
     sinceCommit: sinceCommit,
 })) {
-    console.log(amendement["numero"]);
-}
-for (const { item: dossierLegislatif } of iterLoadSenatDossiersLegislatifs(options["dataDir"], session, {
-    sinceCommit: sinceCommit,
-})) {
-    console.log(dossierLegislatif["numero"]);
+    if (!dossierLegislatif["id"]?.includes("r24")) {
+        console.log(dossierLegislatif["id"]);
+    }
 }

package/lib/types/texte.d.ts

@@ -13,6 +13,15 @@ export declare enum DivisionType {
     division = 12
 }
 export type DivisionTag = keyof typeof DivisionType;
+export interface DocumentMetadata {
+    name: string;
+    session: number | null | undefined;
+    date?: string | null;
+    url_expose_des_motifs?: URL;
+    url_xml?: URL;
+    url_html: URL;
+    url_pdf: URL;
+}
 export interface FlatTexte {
     titre: string | null;
     titreCourt: string | null;

package/lib/utils/reunion_odj_building.js

@@ -73,7 +73,7 @@ function detectLecture(objet) {
     return undefined;
 }
 function computeCodeEtape(ev, dossier) {
-    // In order to match with etape, we need to remove the '-SEANCE' suffix from the codeActe
+    // In order to match with stage, we need to remove the '-SEANCE' suffix from the codeActe
     const cleanCode = (code) => code.replace(/-SEANCE$/, "");
     const lecture = detectLecture(ev.objet ?? "");
     const organe = ev.organe ?? "";
@@ -84,7 +84,7 @@ function computeCodeEtape(ev, dossier) {
             : "";
     const evDate = ev.date.split("T")[0];
     const flat = buildFlatActes(dossier);
-    // 1) Strict matching : same date + nature
+    // 1) Strict matching: same date + same nature
     let candidates = flat.filter((a) => {
         if (a.date !== evDate)
             return false;
@@ -92,67 +92,63 @@ function computeCodeEtape(ev, dossier) {
             return false;
         return true;
     });
-    // If lecture is specified, filter by it
+    // If a specific lecture is detected in the agenda event, refine the candidates
     if (lecture !== undefined && candidates.length > 0) {
         const withLecture = candidates.filter((c) => c.ordreLecture === lecture);
         if (withLecture.length > 0) {
             candidates = withLecture;
         }
     }
-    // Multiple candidates : we take the longest ?
     if (candidates.length > 0) {
+        // Multiple candidates: pick the most specific one (longest code string)
         candidates.sort((a, b) => b.codeActe.length - a.codeActe.length);
         return cleanCode(candidates[0].codeActe);
     }
-    // 2) fallback COM
+    // 2) Fallback COM: If no exact date match for a commission event,
+    // take the latest commission act for this lecture on or before the event date.
     if (nature === "COM") {
         let comActs = flat.filter((a) => a.codeActe.includes("COM") && a.date <= evDate);
         if (lecture !== undefined) {
             const byLecture = comActs.filter((a) => a.ordreLecture === lecture);
-            if (byLecture.length > 0) {
+            if (byLecture.length > 0)
                 comActs = byLecture;
-            }
         }
         if (comActs.length > 0) {
             comActs.sort((a, b) => b.date.localeCompare(a.date) || b.codeActe.length - a.codeActe.length);
             return cleanCode(comActs[0].codeActe);
         }
     }
-    // 3) Fallback lecture générale
+    // 3) Fallback general lecture: if nothing else worked but a lecture is identified,
+    // find any act belonging to that lecture (e.g., SN1-DEPOT).
     if (lecture !== undefined) {
-        const lectures = dossier["actes_legislatifs"] ?? [];
-        const lectureNode = lectures.find((l) => l.ordre_lecture === lecture);
-        const rootCode = lectureNode?.code_acte;
-        if (rootCode && typeof rootCode === "string") {
-            return cleanCode(rootCode);
+        const genericActe = flat.find((a) => a.ordreLecture === lecture);
+        if (genericActe) {
+            return cleanCode(genericActe.codeActe);
         }
     }
-    console.log(`Détails technique pour ev=${ev.id}:`, {
-        nbLectures: dossier["actes_legislatifs"]?.length || 0,
-        clesDossier: Object.keys(dossier).slice(0, 10),
-        structurePremierActe: dossier["actes_legislatifs"]?.[0]
-            ? JSON.stringify(dossier["actes_legislatifs"][0]).substring(0, 200)
-            : "AUCUN_ACTE",
+    console.log(`✖ No stage code found for ev=${ev.id} (Date: ${evDate}, Nature: ${nature}, Lecture: ${lecture})`, {
+        totalActsInDossier: dossier["actes_legislatifs"]?.length || 0,
+        firstActDate: flat[0]?.date,
     });
     return null;
 }
 function buildFlatActes(dossier) {
-    const lectures = dossier["actes_legislatifs"] ?? [];
+    const actes = dossier["actes_legislatifs"] ?? [];
     const res = [];
-    for (const lectureActe of lectures) {
-        const assemblee = lectureActe["assemblee"] || "";
-        if (!assemblee.toLowerCase().includes("sénat") && !assemblee.toLowerCase().includes("senat"))
+    for (const acte of actes) {
+        if (acte["chambre"] !== "SN")
             continue;
-        const ordreLecture = lectureActe.ordre_lecture;
-        const sub = lectureActe.actes_legislatifs;
-        const actes = Array.isArray(sub) && sub.length > 0 ? sub : [lectureActe];
-        for (const acte of actes) {
-            const codeActe = acte.code_acte;
-            const dateActe = acte.date?.split("T")[0];
-            if (!codeActe || !dateActe)
-                continue;
-            res.push({ codeActe, ordreLecture, date: dateActe });
-        }
+        const codeActe = acte.code_acte;
+        const dateActe = acte.date?.split("T")[0];
+        if (!codeActe || !dateActe)
+            continue;
+        const match = codeActe.match(/^(?:SN|AN)(\d+)/);
+        const ordreLecture = match ? parseInt(match[1], 10) : undefined;
+        res.push({
+            codeActe,
+            date: dateActe,
+            ordreLecture,
+        });
     }
     return res;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.20.28",
+  "version": "2.20.30",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",