@tricoteuses/senat 2.20.21 → 2.20.23

package/README.md

@@ -43,7 +43,6 @@ npm run data:download ../senat-data
 
 - `npm run data:download <dir>`: Download, convert data to JSON
 - `npm run data:retrieve_documents <dir>`: Retrieval of textes and rapports from Sénat's website
-- `npm run data:parse_textes_lois <dir>`: Parse textes (requires xml files)
 - `npm run data:retrieve_agenda <dir>`: Retrieval of agenda from Sénat's website
 - `npm run data:retrieve_cr_seance <dir>`: Retrieval of comptes-rendus de séance from Sénat's data
 - `npm run data:retrieve_cr_commission <dir>`: Retrieval of comptes-rendus de commissions from Sénat's website

package/lib/loaders.d.ts

@@ -13,11 +13,11 @@ export declare const COMPTES_RENDUS_FOLDER = "seances";
 export declare const COMMISSION_FOLDER = "commissions";
 export declare const DOSLEG_DOSSIERS_FOLDER = "dossiers";
 export declare const SCRUTINS_FOLDER = "scrutins";
-export declare const RAPPORT_FOLDER = "rap";
 export declare const SENS_CIRCONSCRIPTIONS_FOLDER = "circonscriptions";
 export declare const SENS_ORGANISMES_FOLDER = "organismes";
 export declare const SENS_SENATEURS_FOLDER = "senateurs";
 export declare const TEXTE_FOLDER = "leg";
+export declare const RAPPORT_FOLDER = "rap";
 export declare const DATA_ORIGINAL_FOLDER = "original";
 export declare const DATA_TRANSFORMED_FOLDER = "transformed";
 export declare const DOCUMENT_METADATA_FILE = "metadata.json";
@@ -25,6 +25,7 @@ export type IterItem<T> = {
     item: T;
     filePathFromDataset?: string;
     legislature?: number;
+    gitStatus?: "A" | "M" | "D" | "R" | "C" | "T" | "U";
 };
 export interface TexteMetadata {
     name: string;

package/lib/loaders.js

@@ -1,6 +1,7 @@
 import fsex from "fs-extra";
 import fs from "fs";
 import path from "path";
+import * as git from "./git";
 import { datasets } from "./datasets";
 import { UNDEFINED_SESSION } from "./types/sessions";
 export { EnabledDatasets } from "./datasets";
@@ -9,11 +10,11 @@ export const COMPTES_RENDUS_FOLDER = "seances";
 export const COMMISSION_FOLDER = "commissions";
 export const DOSLEG_DOSSIERS_FOLDER = "dossiers";
 export const SCRUTINS_FOLDER = "scrutins";
-export const RAPPORT_FOLDER = "rap";
 export const SENS_CIRCONSCRIPTIONS_FOLDER = "circonscriptions";
 export const SENS_ORGANISMES_FOLDER = "organismes";
 export const SENS_SENATEURS_FOLDER = "senateurs";
 export const TEXTE_FOLDER = "leg";
+export const RAPPORT_FOLDER = "rap";
 export const DATA_ORIGINAL_FOLDER = "original";
 export const DATA_TRANSFORMED_FOLDER = "transformed";
 export const DOCUMENT_METADATA_FILE = "metadata.json";
@@ -30,7 +31,7 @@ export function* iterFilePaths(dirPath) {
         }
     }
 }
-function* iterLoadSenatItems(dataDir, dataName, legislatureOrSession, subDir, { log = false } = {}) {
+function* iterLoadSenatItems(dataDir, dataName, legislatureOrSession, subDir, { log = false, sinceCommit } = {}) {
     let itemsDir = path.join(dataDir, dataName);
     if (subDir) {
         itemsDir = path.join(itemsDir, subDir);
@@ -38,9 +39,26 @@ function* iterLoadSenatItems(dataDir, dataName, legislatureOrSession, subDir, {
     if (legislatureOrSession) {
         itemsDir = path.join(itemsDir, String(legislatureOrSession));
     }
+    // Get changed files if sinceCommit is specified (excluding deleted files)
+    const changedFiles = sinceCommit
+        ? git.getChangedFilesSinceCommit(itemsDir, sinceCommit, {
+            diffFilter: "AMR", // Added, Modified, Renamed
+        })
+        : null;
+    if (log && sinceCommit) {
+        console.log(`Filtering files changed since commit ${sinceCommit} in ${itemsDir}`);
+        console.log(`Found ${changedFiles?.size || 0} changed files (AMR)`);
+    }
     for (const filePath of iterFilePaths(itemsDir)) {
+        const relativePath = path.relative(path.join(dataDir, dataName), filePath);
+        const gitStatus = changedFiles?.get(relativePath);
+        // Filter by changed files if sinceCommit is specified
+        if (changedFiles && !gitStatus) {
+            // Skip files not in the change set
+            continue;
+        }
         if (log) {
-            console.log(`Loading file: ${filePath}…`);
+            console.log(`Loading file: ${filePath}…${gitStatus ? ` (${gitStatus})` : ""}`);
         }
         let item;
         try {
@@ -56,8 +74,35 @@ function* iterLoadSenatItems(dataDir, dataName, legislatureOrSession, subDir, {
             item,
             filePathFromDataset,
             legislature: legislatureOrSession,
+            ...(gitStatus && { gitStatus }), // Include gitStatus
         };
     }
+    // Yield deleted files at the end if sinceCommit is specified
+    if (sinceCommit) {
+        const deletedFiles = git.getChangedFilesSinceCommit(itemsDir, sinceCommit, {
+            diffFilter: "D", // Deleted
+        });
+        if (log) {
+            console.log(`Found ${deletedFiles.size || 0} deleted files (D)`);
+        }
+        for (const [relativePath, status] of deletedFiles.entries()) {
+            const deletedFilePath = path.join(itemsDir, relativePath);
+            if (log) {
+                console.log(`Deleted file: ${deletedFilePath}`);
+            }
+            // Extract UID from filename (remove extension) for the placeholder item
+            const fileExtension = path.extname(relativePath) || ".json"; // Assuming files use an extension like .json
+            const filename = path.basename(relativePath, fileExtension);
+            const fakeItem = { uid: filename }; // Placeholder item using uid constraint
+            const filePathFromDataset = deletedFilePath.substring(deletedFilePath.indexOf(dataName) + dataName.length);
+            yield {
+                item: fakeItem,
+                filePathFromDataset,
+                legislature: legislatureOrSession,
+                gitStatus: status,
+            };
+        }
+    }
 }
 export function* iterLoadSenatAmendements(dataDir, session, options = {}) {
     for (const amendementItem of iterLoadSenatItems(dataDir, datasets.ameli.database, session, undefined, options)) {

package/lib/model/dosleg.d.ts

@@ -1,8 +1,7 @@
 import { InferResult, SelectQueryBuilder } from "kysely";
 declare const findAllDossiersQuery: SelectQueryBuilder<any, any, any>;
 export declare function findAllDossiers(): AsyncIterableIterator<DossierLegislatifResult>;
-export declare function createActesLegislatifs(dossier: DossierLegislatifResult): any;
 export declare function getCodeActeLecture(codeNatureDossier: string, typeLecture: string, assemblee: string): string | null;
-export declare function getCodeActeTexte(codeParent: string | null, texteOrigine: string): string | null;
 export type DossierLegislatifResult = InferResult<typeof findAllDossiersQuery>[0];
+export declare function buildActesLegislatifs(dossier: any): any[];
 export {};

package/lib/model/dosleg.js

@@ -29,6 +29,17 @@ function auteursRapport(rapportId) {
     ])
         .orderBy("dosleg.ecr.ecrnumtri", "asc"));
 }
+function documentsAttaches(rapportId) {
+    return jsonArrayFrom(dbSenat
+        .withSchema("dosleg")
+        .selectFrom("docatt")
+        .leftJoin("typatt", "docatt.typattcod", "typatt.typattcod")
+        .where("docatt.rapcod", "=", rapportId)
+        .select([
+        "docatt.docatturl as url",
+        "typatt.typattlib as type_document"
+    ]));
+}
 function rapports(lectureAssembleeId) {
     return jsonArrayFrom(dbSenat
         .withSchema("dosleg")
@@ -55,9 +66,12 @@ function rapports(lectureAssembleeId) {
             .end()
             .as("url"),
         rtrim(ref("denrap.libdenrap")).as("type"),
+        rtrim(ref("rap.raptil")).as("titre"),
+        rtrim(ref("rap.rapsoustit")).as("sous_titre"),
         toDateString(ref("rap.date_depot")).as("date"),
         "sesann as session",
         auteursRapport(ref("rap.rapcod")).as("auteurs"),
+        documentsAttaches(ref("rap.rapcod")).as("documents_annexes"),
     ]));
 }
 function auteursTexte(texteId) {
@@ -221,90 +235,6 @@ const findAllDossiersQuery = dbSenat
 export function findAllDossiers() {
     return findAllDossiersQuery.stream();
 }
-export function createActesLegislatifs(dossier) {
-    const actesLegislatifs = (dossier["lectures"] || []).flatMap((lecture) => {
-        const lecturesAssemblee = (lecture["lectures_assemblee"] || []).map((lectureAss) => {
-            const codeParent = getCodeActeLecture(dossier["code_nature_dossier"], lecture["type_lecture"], lectureAss["assemblee"]);
-            const textesWithCodeActe = (lectureAss["textes"] || []).map((texte) => ({
-                code_acte: getCodeActeTexte(codeParent, texte["origine"]),
-                ...texte,
-            }));
-            // Ajout étape -COM-FOND après chaque -DEPOT
-            let acteLegislatifsLecture = [];
-            for (let i = 0; i < textesWithCodeActe.length; i++) {
-                const t = textesWithCodeActe[i];
-                acteLegislatifsLecture.push(t);
-                if (t.code_acte?.endsWith("-DEPOT") && t.type === "texte de loi") {
-                    acteLegislatifsLecture.push({
-                        ...t,
-                        code_acte: t.code_acte.replace("-DEPOT", "-COM-FOND"),
-                    });
-                }
-            }
-            if (lectureAss["dates_seances"]?.length > 0) {
-                acteLegislatifsLecture.push({
-                    session: lectureAss["session"],
-                    type_lecture: lecture["type_lecture"],
-                    libelle_lecture: "Discussion en séance publique",
-                    code_acte: `${codeParent}-DEBATS-SEANCE`,
-                    date: lectureAss["dates_seances"][0]?.["date"],
-                    id: lectureAss["id"],
-                    numero: lectureAss["numero"]
-                });
-            }
-            const { textes, rapports, ...lectureAssWithoutTextes } = lectureAss;
-            return {
-                type_lecture: lecture["type_lecture"],
-                ordre_lecture: lecture["ordre_lecture"],
-                libelle_lecture: lecture["libelle"],
-                code_acte: codeParent,
-                actes_legislatifs: acteLegislatifsLecture,
-                ...lectureAssWithoutTextes,
-            };
-        });
-        return lecturesAssemblee;
-    });
-    if (dossier["date_decision_CoC"]) {
-        actesLegislatifs.push({
-            type_lecture: "Conseil constitutionnel",
-            ordre_lecture: null,
-            libelle_lecture: "Conseil constitutionnel",
-            code_acte: "CC",
-            actes_legislatifs: [
-                {
-                    code_acte: "CC-SAISIE",
-                    date: dossier["date_saisine_CoC"],
-                    libelle_decision_CoC: dossier["libelle_decision_CoC"],
-                    date_decision_CoC: dossier["date_decision_CoC"],
-                    num_decision_CoC: dossier["num_decision_CoC"],
-                    url_decision_CoC: dossier["url_decision_CoC"],
-                    url_dossier_CoC: dossier["url_dossier_CoC"],
-                    date_saisine_CoC: dossier["date_saisine_CoC"],
-                    condition_saisine_CoC: dossier["condition_saisine_CoC"],
-                },
-            ],
-        });
-    }
-    if (dossier["date_publication_JO"]) {
-        actesLegislatifs.push({
-            type_lecture: "Promulgation",
-            ordre_lecture: null,
-            libelle_lecture: "Promulgation",
-            code_acte: "PROM",
-            actes_legislatifs: [
-                {
-                    code_acte: "PROM-PUB",
-                    date: dossier["date_publication_JO"],
-                    titre_JO: dossier["titre_JO"],
-                    date_publication_JO: dossier["date_publication_JO"],
-                    numero_JO: dossier["numero_JO"],
-                    url_JO: dossier["url_JO"],
-                },
-            ],
-        });
-    }
-    return actesLegislatifs;
-}
 export function getCodeActeLecture(codeNatureDossier, typeLecture, assemblee) {
     const codeAssemblee = assemblee === "Sénat" ? "SN" : assemblee === "Assemblée nationale" ? "AN" : null;
     if (typeLecture === "Commission mixte paritaire") {
@@ -336,38 +266,177 @@ export function getCodeActeLecture(codeNatureDossier, typeLecture, assemblee) {
     }
     return null;
 }
-export function getCodeActeTexte(codeParent, texteOrigine) {
-    if (codeParent === "CMP") {
-        if (texteOrigine === "de la commission") {
-            return "CMP-DEBATS-AN";
-        }
-        else if (texteOrigine === "adopté par l'Assemblée Nationale" ||
-            texteOrigine === "adopté par l'Assemblée nationale") {
-            return "CMP-DEBATS-SN";
-        }
-    }
-    if (texteOrigine === "transmis au Sénat" ||
-        texteOrigine === "déposé au Sénat" ||
-        texteOrigine === "transmis à l'Assemblée nationale" ||
-        texteOrigine === "déposé à l'Assemblée nationale" ||
-        texteOrigine === "transmis à l'Assemblée Nationale" ||
-        texteOrigine === "déposé à l'Assemblée Nationale" ||
-        texteOrigine === "transmis en application de l'article 47-1, alinéa 2, de la Constitution") {
-        return `${codeParent}-DEPOT`;
+// Helper pour déterminer le code de phase (SN1, SN2, CMP...)
+function getPhasePrefix(lecture, assemblee) {
+    if (assemblee !== "Sénat")
+        return null;
+    const typeLibelle = (lecture.type_lecture || "").toLowerCase();
+    if (typeLibelle.includes("cmp") || typeLibelle.includes("mixte"))
+        return "CMP";
+    if (typeLibelle.includes("nouvelle lecture"))
+        return "SNNLEC";
+    if (typeLibelle.includes("d\u00e9finitive"))
+        return "SNLDEF";
+    if (typeLibelle.includes("unique"))
+        return "SNLUNI";
+    if (lecture.ordre_lecture) {
+        return `SN${lecture.ordre_lecture}`;
     }
-    if (texteOrigine === "de la commission" ||
-        texteOrigine === "de la commission (AN)" ||
-        texteOrigine === "résultat des travaux de la commission") {
-        return `${codeParent}-DEBATS-SEANCE`;
+    if (typeLibelle.includes("premi\u00e8re"))
+        return "SN1";
+    if (typeLibelle.includes("deuxi\u00e8me") || typeLibelle.includes("seconde"))
+        return "SN2";
+    return "SN1";
+}
+export function buildActesLegislatifs(dossier) {
+    const actes = [];
+    const loiSignet = dossier.signet;
+    const lectures = dossier.lectures || [];
+    for (const lecture of lectures) {
+        const lecturesAssemblee = lecture.lectures_assemblee || [];
+        for (const lecAss of lecturesAssemblee) {
+            // On ne traite que la partie SÉNAT
+            if (lecAss.assemblee !== "Sénat")
+                continue;
+            const phasePrefix = getPhasePrefix(lecture, lecAss.assemblee);
+            if (!phasePrefix)
+                continue;
+            // Préparation des textes (tri chronologique)
+            const textes = lecAss.textes || [];
+            const textesTries = [...textes].sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime());
+            // =================================================================
+            // A. DÉPÔT
+            // =================================================================
+            const depotTexte = textesTries.find((t) => (t.origine || "").toLowerCase().includes("déposé") ||
+                (t.origine || "").toLowerCase().includes("transmis") ||
+                t.ordre_origine === "0");
+            if (depotTexte && depotTexte.date) {
+                actes.push({
+                    code_acte: `${phasePrefix}-DEPOT`,
+                    date: depotTexte.date,
+                    libelle: `Dépôt du texte n°${depotTexte.numero}`,
+                    id: depotTexte.id,
+                    numero: depotTexte.numero,
+                    uid: `${loiSignet}-${phasePrefix}-DEPOT`,
+                    session: lecAss.session,
+                    chambre: 'SN',
+                    signet_dossier: loiSignet,
+                    texte_url: depotTexte.url,
+                    code_organisme: null
+                });
+            }
+            // =================================================================
+            // B. COMMISSION (Rapports)
+            // =================================================================
+            const rapports = lecAss.rapports || [];
+            for (const rap of rapports) {
+                if (rap.date) {
+                    actes.push({
+                        code_acte: `${phasePrefix}-COM-FOND`,
+                        date: rap.date,
+                        libelle: `Rapport n°${rap.numero} de la commission`,
+                        id: rap.id,
+                        numero: rap.numero,
+                        code_organisme: rap.code_organisme,
+                        adoption: rap.adoption,
+                        uid: `${loiSignet}-${phasePrefix}-COM`,
+                        session: lecAss.session,
+                        chambre: 'SN',
+                        signet_dossier: loiSignet,
+                        texte_url: rap.url,
+                    });
+                }
+            }
+            // =================================================================
+            // C. SÉANCE PUBLIQUE
+            // =================================================================
+            const datesSeances = lecAss.dates_seances || [];
+            if (datesSeances.length > 0) {
+                // Tri des objets dates
+                datesSeances.sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime());
+                const premiereSeance = datesSeances[0];
+                if (premiereSeance && premiereSeance.date) {
+                    actes.push({
+                        // Champs pour buildParlementActeLegislatif
+                        code_acte: `${phasePrefix}-DEBATS-SEANCE`,
+                        date: premiereSeance.date,
+                        libelle: `Discussion en séance publique`,
+                        uid: `${loiSignet}-${phasePrefix}-DEBATS-SEANCE`,
+                        session: lecAss.session,
+                        chambre: 'SN',
+                        signet_dossier: loiSignet,
+                        code_organisme: null
+                    });
+                }
+            }
+            // =================================================================
+            // D. DÉCISION / VOTE
+            // =================================================================
+            const texteFinal = [...textesTries].reverse().find((t) => {
+                const origine = (t.origine || "").toLowerCase();
+                return (origine.includes("adopté") ||
+                    origine.includes("rejeté") ||
+                    origine.includes("devenu résolution") ||
+                    t.code_adoption === "O");
+            });
+            if (texteFinal && texteFinal.date) {
+                const origine = (texteFinal.origine || "").toLowerCase();
+                let libelleStatut = "Adopté";
+                if (origine.includes("rejeté")) {
+                    libelleStatut = "Rejeté";
+                }
+                else if (origine.includes("devenue résolution")) {
+                    libelleStatut = "Adopté";
+                }
+                actes.push({
+                    code_acte: `${phasePrefix}-DEBATS-DEC`,
+                    date: texteFinal.date,
+                    libelle: `${libelleStatut === "Adopté" ? "Adoption" : "Rejet"} (Texte n°${texteFinal.numero})`,
+                    id: texteFinal.id,
+                    numero: texteFinal.numero,
+                    adoption: libelleStatut,
+                    uid: `${loiSignet}-DEC-${texteFinal.numero}`,
+                    session: lecAss.session,
+                    chambre: 'SN',
+                    signet_dossier: loiSignet,
+                    texte_url: texteFinal.url,
+                    code_organisme: null
+                });
+            }
+        }
     }
-    if (texteOrigine === "retiré par l'auteur") {
-        return `${codeParent}-RTRINI`;
+    // =================================================================
+    // E. HORS LECTURE (CC & PROMULGATION)
+    // =================================================================
+    if (dossier.date_decision_CoC) {
+        actes.push({
+            code_acte: 'CC',
+            date: dossier.date_decision_CoC,
+            libelle: `Décision du Conseil constitutionnel`,
+            id: dossier.url_decision_CoC,
+            uid: `${loiSignet}-CC`,
+            chambre: 'AN',
+            signet_dossier: loiSignet,
+            texte_url: dossier.url_decision_CoC || dossier.url_dossier_CoC,
+        });
     }
-    if (texteOrigine.includes("adopté") ||
-        texteOrigine.includes("rejeté") ||
-        texteOrigine.includes("modifié") ||
-        texteOrigine === "devenu résolution du Sénat") {
-        return `${codeParent}-DEBATS-DEC`;
+    if (dossier.date_promulgation) {
+        actes.push({
+            code_acte: 'PROM',
+            date: dossier.date_promulgation,
+            libelle: `Promulgation de la loi`,
+            date_publication_JO: dossier.date_publication_JO,
+            numero_JO: dossier.numero_JO,
+            url_legifrance: dossier.url_JO,
+            id: dossier.url_JO,
+            uid: `${loiSignet}-PROM`,
+            chambre: 'AN',
+            signet_dossier: loiSignet,
+        });
     }
-    return null;
+    return actes.sort((a, b) => {
+        const dateA = new Date(a.date).getTime();
+        const dateB = new Date(b.date).getTime();
+        return dateA - dateB;
+    });
 }

package/lib/parsers/texte.d.ts

@@ -0,0 +1,7 @@
+import { ExposeDesMotifs, FlatTexte } from "../types/texte";
+export declare function transformTexte(document: Document): FlatTexte | null;
+export declare function transformExposeDesMotifs(document: Document): ExposeDesMotifs | null;
+export declare function parseTexte(texteXml: string): FlatTexte | null;
+export declare function parseTexteFromFile(xmlFilePath: string): Promise<FlatTexte | null>;
+export declare function parseExposeDesMotifs(exposeDesMotifsHtml: string): ExposeDesMotifs | null;
+export declare function parseExposeDesMotifsFromFile(htmlFilePath: string): Promise<ExposeDesMotifs | null>;