@tricoteuses/senat 2.20.30 → 2.20.32

package/lib/loaders.d.ts

@@ -41,11 +41,11 @@ export declare function iterLoadSenatComptesRendusCommissions(dataDir: string, s
     session: number;
 }>;
 export declare function iterLoadSenatDossiersLegislatifs(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DossierLegislatifResult>>;
-export declare function iterLoadSenatDossiersLegislatifsRapportUrls(dataDir: string, session: number | undefined): Generator<IterItem<DocumentMetadata>>;
-export declare function iterLoadSenatDossiersLegislatifsTexteUrls(dataDir: string, session: number | undefined): Generator<IterItem<DocumentMetadata>>;
-export declare function iterLoadSenatDossiersLegislatifsDocuments(dataDir: string, session: number | undefined, documentType: "textes" | "rapports", options?: {}): Generator<IterItem<DocumentResult>>;
-export declare function iterLoadSenatDossiersLegislatifsRapports(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DocumentResult>>;
-export declare function iterLoadSenatDossiersLegislatifsTextes(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DocumentResult>>;
+export declare function iterLoadSenatRapportUrls(dataDir: string, session: number | undefined): Generator<IterItem<DocumentMetadata>>;
+export declare function iterLoadSenatTexteUrls(dataDir: string, session: number | undefined): Generator<IterItem<DocumentMetadata>>;
+export declare function iterLoadSenatDocuments(dataDir: string, session: number | undefined, documentType: "textes" | "rapports", options?: {}): Generator<IterItem<DocumentResult>>;
+export declare function iterLoadSenatRapports(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DocumentResult>>;
+export declare function iterLoadSenatTextes(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DocumentResult>>;
 export declare function loadSenatTexteContent(dataDir: string, textePathFromDataset: string): IterItem<FlatTexte | null>;
 export declare function loadSenatCompteRenduContent(dataDir: string, session: number, debatId: string | number): {
     item: CompteRendu | null;

package/lib/loaders.js

@@ -50,6 +50,9 @@ function* iterLoadSenatItems(dataDir, dataName, legislatureOrSession, subDir, {
         console.log(`Found ${changedFiles?.size || 0} changed files (AMR)`);
     }
     for (const filePath of iterFilePaths(itemsDir)) {
+        if (!filePath.endsWith(".json")) {
+            continue;
+        }
         const relativePath = path.relative(path.join(dataDir, dataName), filePath);
         const gitStatus = changedFiles?.get(relativePath);
         // Filter by changed files if sinceCommit is specified
@@ -147,7 +150,7 @@ export function* iterLoadSenatDossiersLegislatifs(dataDir, session, options = {}
         yield dossierLegislatifItem;
     }
 }
-export function* iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session) {
+export function* iterLoadSenatRapportUrls(dataDir, session) {
     let itemsDir = path.join(dataDir, RAPPORT_FOLDER, DATA_ORIGINAL_FOLDER);
     if (session) {
         itemsDir = path.join(itemsDir, session.toString());
@@ -163,7 +166,7 @@ export function* iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session) {
         }
     }
 }
-export function* iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session) {
+export function* iterLoadSenatTexteUrls(dataDir, session) {
     let itemsDir = path.join(dataDir, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER);
     if (session) {
         itemsDir = path.join(itemsDir, session.toString());
@@ -179,23 +182,14 @@ export function* iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session) {
         }
     }
 }
-export function* iterLoadSenatDossiersLegislatifsDocuments(dataDir, session, documentType, options = {}) {
+export function* iterLoadSenatDocuments(dataDir, session, documentType, options = {}) {
     for (const { item: dossierLegislatif } of iterLoadSenatDossiersLegislatifs(dataDir, session, options)) {
         for (const lecture of dossierLegislatif["lectures"]) {
             const lecturesSenat = lecture.lectures_assemblee.filter((lectureAssemblee) => lectureAssemblee.assemblee === "Sénat");
             for (const lectureSenat of lecturesSenat) {
                 for (const document of lectureSenat[documentType]) {
-                    const enrichedDocument = {
-                        signet_dossier: dossierLegislatif["signet"],
-                        url_dossier_senat: dossierLegislatif["url"],
-                        url_dossier_assemblee_nationale: dossierLegislatif["url_dossier_assemblee_nationale"],
-                        type_lecture: lecture.type_lecture,
-                        libelle_lecture: lecture.libelle,
-                        libelle_organisme: lectureSenat.libelle_organisme,
-                        ...document,
-                    };
                     const documentItem = {
-                        item: enrichedDocument,
+                        item: document,
                     };
                     if (document.url) {
                         const documentName = path.parse(document.url).name;
@@ -207,13 +201,15 @@ export function* iterLoadSenatDossiersLegislatifsDocuments(dataDir, session, doc
         }
     }
 }
-export function* iterLoadSenatDossiersLegislatifsRapports(dataDir, session, options = {}) {
-    for (const iterItem of iterLoadSenatDossiersLegislatifsDocuments(dataDir, session, "rapports", options)) {
-        yield iterItem;
+export function* iterLoadSenatRapports(dataDir, session, options = {}) {
+    for (const iterItem of iterLoadSenatItems(dataDir, RAPPORT_FOLDER, session, "original", options)) {
+        if (iterItem.item?.["id"]) {
+            yield iterItem;
+        }
     }
 }
-export function* iterLoadSenatDossiersLegislatifsTextes(dataDir, session, options = {}) {
-    for (const iterItem of iterLoadSenatDossiersLegislatifsDocuments(dataDir, session, "textes", options)) {
+export function* iterLoadSenatTextes(dataDir, session, options = {}) {
+    for (const iterItem of iterLoadSenatDocuments(dataDir, session, "textes", options)) {
         yield iterItem;
     }
 }

package/lib/model/documents.js

@@ -33,39 +33,50 @@ function documentsAttaches(rapportId) {
         "typatt.typattlib as type_document"
     ]));
 }
-const queryRapports = dbSenat
+function selectRapportAttributes({ eb, ref, val }) {
+    return [
+        "rap.rapnum as numero",
+        "raporg.orgcod as code_organisme",
+        eb
+            .case()
+            .when("rap.rapurl", "is not", null)
+            .then(sql `regexp_replace(trim(${ref("rap.rapurl")}), '^(.*/)?(.*?)(\\.html)?$', '\\2')`)
+            .else(null)
+            .end()
+            .as("id"),
+        eb
+            .case()
+            .when("rap.typurl", "=", "I")
+            .then(concat(val("https://www.senat.fr/rap/"), rtrim(ref("rap.rapurl"))))
+            .else(rtrim(ref("rap.rapurl")))
+            .end()
+            .as("url"),
+        rtrim(ref("denrap.libdenrap")).as("type"),
+        rtrim(rtrim(ref("rap.raptil"))).as("titre"),
+        rtrim(rtrim(ref("rap.rapsoustit"))).as("sous_titre"),
+        toDateString(ref("rap.date_depot")).as("date"),
+        "rap.sesann as session",
+        auteursRapport(ref("rap.rapcod")).as("auteurs"),
+        documentsAttaches(ref("rap.rapcod")).as("documents_annexes"),
+    ];
+}
+const baseQueryRapports = dbSenat
     .withSchema("dosleg")
     .selectFrom("rap")
     .leftJoin("raporg", "raporg.rapcod", "rap.rapcod")
     .leftJoin("denrap", "denrap.coddenrap", "rap.coddenrap")
-    .select(({ eb, ref, val }) => [
-    "rap.rapnum as numero",
-    "raporg.orgcod as code_organisme",
-    eb
-        .case()
-        .when("rap.rapurl", "is not", null)
-        .then(sql `regexp_replace(trim(${ref("rap.rapurl")}), '^(.*/)?(.*?)(\\.html)?$', '\\2')`)
-        .else(null)
-        .end()
-        .as("id"),
-    eb
-        .case()
-        .when("rap.typurl", "=", "I")
-        .then(concat(val("https://www.senat.fr/rap/"), rtrim(ref("rap.rapurl"))))
-        .else(rtrim(ref("rap.rapurl")))
-        .end()
-        .as("url"),
-    rtrim(ref("denrap.libdenrap")).as("type"),
-    rtrim(ref("rap.raptil")).as("titre"),
-    rtrim(ref("rap.rapsoustit")).as("sous_titre"),
-    toDateString(ref("rap.date_depot")).as("date"),
-    "sesann as session",
-    auteursRapport(ref("rap.rapcod")).as("auteurs"),
-    documentsAttaches(ref("rap.rapcod")).as("documents_annexes"),
+    .leftJoin("lecassrap", "lecassrap.rapcod", "rap.rapcod");
+const queryRapports = baseQueryRapports
+    .leftJoin("lecass", "lecass.lecassidt", "lecassrap.lecassidt")
+    .leftJoin("lecture", "lecture.lecidt", "lecass.lecidt")
+    .leftJoin("loi", "loi.loicod", "lecture.loicod")
+    .select((args) => [
+    "loi.signet as signet_dossier",
+    ...selectRapportAttributes(args),
 ]);
 export function rapports(lectureAssembleeId) {
-    return jsonArrayFrom(queryRapports
-        .leftJoin("lecassrap", "lecassrap.rapcod", "rap.rapcod")
+    return jsonArrayFrom(baseQueryRapports
+        .select(selectRapportAttributes)
         .where("lecassrap.lecassidt", "=", lectureAssembleeId));
 }
 function auteursTexte(texteId) {
@@ -85,40 +96,52 @@ function auteursTexte(texteId) {
     ])
         .orderBy("ecr.ecrnumtri", "asc"));
 }
-const queryTextes = dbSenat
+function selectTexteAttributes({ eb, ref, val }) {
+    return [
+        "texte.texnum as numero",
+        "texte.orgcod as code_organisme",
+        eb
+            .case()
+            .when("texte.texurl", "is not", null)
+            .then(sql `regexp_replace(trim(${ref("texte.texurl")}), '^(.*/)?(.*?)(\\.html)?$', '\\2')`)
+            .else(null)
+            .end()
+            .as("id"),
+        eb
+            .case()
+            .when("texte.typurl", "=", "I")
+            .then(concat(val("https://www.senat.fr/leg/"), rtrim(ref("texte.texurl"))))
+            .else(rtrim(ref("texte.texurl")))
+            .end()
+            .as("url"),
+        rtrim(ref("oritxt.oritxtlib")).as("origine"),
+        "oritxt.oriordre as ordre_origine",
+        "oritxt.oritxtado as code_adoption",
+        "oritxt.oritxtmod as modification",
+        rtrim(ref("typtxt.typtxtlib")).as("type"),
+        toDateString(ref("texte.txtoritxtdat")).as("date"),
+        "texte.sesann as session",
+        auteursTexte(ref("texte.texcod")).as("auteurs"),
+    ];
+}
+const baseQueryTextes = dbSenat
     .withSchema("dosleg")
     .selectFrom("texte")
     .leftJoin("oritxt", "oritxt.oritxtcod", "texte.oritxtcod")
     .leftJoin("typtxt", "typtxt.typtxtcod", "texte.typtxtcod")
-    .select(({ eb, ref, val }) => [
-    "texte.texnum as numero",
-    "texte.orgcod as code_organisme",
-    eb
-        .case()
-        .when("texte.texurl", "is not", null)
-        .then(sql `regexp_replace(trim(${ref("texte.texurl")}), '^(.*/)?(.*?)(\\.html)?$', '\\2')`)
-        .else(null)
-        .end()
-        .as("id"),
-    eb
-        .case()
-        .when("texte.typurl", "=", "I")
-        .then(concat(val("https://www.senat.fr/leg/"), rtrim(ref("texte.texurl"))))
-        .else(rtrim(ref("texte.texurl")))
-        .end()
-        .as("url"),
-    rtrim(ref("oritxt.oritxtlib")).as("origine"),
-    "oritxt.oriordre as ordre_origine",
-    "oritxt.oritxtado as code_adoption",
-    "oritxt.oritxtmod as modification",
-    rtrim(ref("typtxt.typtxtlib")).as("type"),
-    toDateString(ref("texte.txtoritxtdat")).as("date"),
-    "sesann as session",
-    auteursTexte(ref("texte.texcod")).as("auteurs"),
-])
     .orderBy(({ ref }) => orderOrdreOrigineTexte(ref("oritxt.oriordre")));
+const queryTextes = baseQueryTextes
+    .leftJoin("lecass", "lecass.lecassidt", "texte.lecassidt")
+    .leftJoin("lecture", "lecture.lecidt", "lecass.lecidt")
+    .leftJoin("loi", "loi.loicod", "lecture.loicod")
+    .select((args) => [
+    "loi.signet as signet_dossier",
+    ...selectTexteAttributes(args),
+]);
 export function textes(lectureAssembleeId) {
-    return jsonArrayFrom(queryTextes.where("texte.lecassidt", "=", lectureAssembleeId));
+    return jsonArrayFrom(baseQueryTextes
+        .select(selectTexteAttributes)
+        .where("texte.lecassidt", "=", lectureAssembleeId));
 }
 export function findAllTextes() {
     return queryTextes.stream();

package/lib/scripts/retrieve_documents.js

@@ -3,7 +3,7 @@ import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 import { DateTime } from "luxon";
 import path from "path";
-import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatDossiersLegislatifsRapportUrls, iterLoadSenatDossiersLegislatifsTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, } from "../loaders";
+import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatRapportUrls, iterLoadSenatTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, } from "../loaders";
 import { parseExposeDesMotifs, parseTexte, parseTexteFromFile } from "../parsers/texte";
 import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions";
 import { commonOptions } from "./shared/cli_helpers";
@@ -149,7 +149,7 @@ async function retrieveTextes(dataDir, sessions) {
         parseDocuments: options["parseDocuments"],
     };
     for (const session of sessions) {
-        for (const { item: texteMetadata } of iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session)) {
+        for (const { item: texteMetadata } of iterLoadSenatTexteUrls(dataDir, session)) {
             await processTexte(texteMetadata, originalTextesDir, transformedTextesDir, dlOptions);
         }
     }
@@ -164,7 +164,7 @@ async function retrieveRapports(dataDir, sessions) {
         formats: options["formats"],
     };
     for (const session of sessions) {
-        for (const { item: rapportMetadata } of iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session)) {
+        for (const { item: rapportMetadata } of iterLoadSenatRapportUrls(dataDir, session)) {
             await processRapport(rapportMetadata, originalRapportsDir, dlOptions);
         }
     }

package/lib/scripts/test_iter_load.js

@@ -1,14 +1,12 @@
-import { iterLoadSenatDossiersLegislatifsRapports } from "../loaders";
+import { iterLoadSenatRapports } from "../loaders";
 import commandLineArgs from "command-line-args";
 import { dataDirDefaultOption } from "./shared/cli_helpers";
 const optionsDefinitions = [dataDirDefaultOption];
 const options = commandLineArgs(optionsDefinitions);
 const session = 2024;
 const sinceCommit = undefined;
-for (const { item: dossierLegislatif } of iterLoadSenatDossiersLegislatifsRapports(options["dataDir"], session, {
+for (const { item: rapport } of iterLoadSenatRapports(options["dataDir"], session, {
     sinceCommit: sinceCommit,
 })) {
-    if (!dossierLegislatif["id"]?.includes("r24")) {
-        console.log(dossierLegislatif["id"]);
-    }
+    console.log(rapport);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.20.30",
+  "version": "2.20.32",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",