@tricoteuses/senat 2.20.33 → 2.20.34

package/LICENSE.md

@@ -2,9 +2,19 @@
 
 ## _Handle French Sénat's open data_
 
-By: Emmanuel Raviart <mailto:emmanuel@raviart.com>
+By:
 
-Copyright (C) 2019, 2020, 2021 Emmanuel Raviart
+- Henry Boisgibault <mailto:henry.boisgibault@proton.me>
+- Pierre Drege <mailto:dregop@proton.me>
+- Hélène Jonin <mailto:helene.jonin@gmail.com>
+- Paul-Henry Ngounou <mailto:ngpaulhenry@gmail.com>
+- Emmanuel Raviart <mailto:emmanuel@raviart.com>
+
+Copyright:
+
+- © 2019, 2021, 2022 Emmanuel Raviart
+- © 2024 Logora
+- © 2025, 2026 Logora & Emmanuel Raviart
 
 https://git.tricoteuses.fr/logiciels/tricoteuses-senat
 

package/lib/loaders.d.ts

@@ -5,11 +5,9 @@ import { QuestionResult } from "./model/questions";
 import { ScrutinResult } from "./model/scrutins";
 import { CirconscriptionResult, OrganismeResult, SenateurResult } from "./model/sens";
 import { Reunion } from "./types/agenda";
-import { FlatTexte, DocumentMetadata } from "./types/texte";
+import { FlatTexte } from "./types/texte";
 import { CompteRendu } from "./types/compte_rendu";
-import { DocumentResult } from "./model/documents";
 export { EnabledDatasets } from "./datasets";
-export type { DocumentResult } from "./model/documents";
 export declare const AGENDA_FOLDER = "agenda";
 export declare const COMPTES_RENDUS_FOLDER = "seances";
 export declare const COMMISSION_FOLDER = "commissions";
@@ -29,6 +27,46 @@ export type IterItem<T> = {
     legislature?: number;
     gitStatus?: "A" | "M" | "D" | "R" | "C" | "T" | "U";
 };
+export interface TexteMetadata {
+    name: string;
+    session: number | null | undefined;
+    date?: string | null;
+    url_expose_des_motifs?: URL;
+    url_xml: URL;
+    url_html: URL;
+    url_pdf: URL;
+}
+export interface RapportMetadata {
+    name: string;
+    session: number | null | undefined;
+    date?: string | null;
+    url_html: URL;
+    url_pdf: URL;
+}
+export interface DossierLegislatifDocumentResult {
+    signet_dossier: string;
+    url_dossier_senat: string;
+    url_dossier_assemblee_nationale: string | null;
+    type_lecture: string;
+    libelle_lecture: string;
+    libelle_organisme: string | null;
+    code_organisme: string | null;
+    numero: number | null;
+    id: string | null;
+    url: string;
+    origine?: string | null | undefined;
+    type: string;
+    date: string;
+    session: number | null;
+    auteurs: {
+        prenom: string | null;
+        nom_usuel: string;
+        matricule: string | null;
+    }[];
+    organismes?: {
+        libelle: string;
+    }[] | undefined;
+}
 export declare function iterFilePaths(dirPath: string): Generator<string>;
 export declare function iterLoadSenatAmendements(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<AmendementResult>>;
 export declare function iterLoadSenatDebats(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DebatResult>>;
@@ -41,11 +79,11 @@ export declare function iterLoadSenatComptesRendusCommissions(dataDir: string, s
     session: number;
 }>;
 export declare function iterLoadSenatDossiersLegislatifs(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DossierLegislatifResult>>;
-export declare function iterLoadSenatRapportUrls(dataDir: string, session: number | undefined): Generator<IterItem<DocumentMetadata>>;
-export declare function iterLoadSenatTexteUrls(dataDir: string, session: number | undefined): Generator<IterItem<DocumentMetadata>>;
-export declare function iterLoadSenatDocuments(dataDir: string, session: number | undefined, documentType: "textes" | "rapports", options?: {}): Generator<IterItem<DocumentResult>>;
-export declare function iterLoadSenatRapports(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DocumentResult>>;
-export declare function iterLoadSenatTextes(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DocumentResult>>;
+export declare function iterLoadSenatDossiersLegislatifsRapportUrls(dataDir: string, session: number | undefined): Generator<IterItem<RapportMetadata>>;
+export declare function iterLoadSenatDossiersLegislatifsTexteUrls(dataDir: string, session: number | undefined): Generator<IterItem<TexteMetadata>>;
+export declare function iterLoadSenatDossiersLegislatifsDocuments(dataDir: string, session: number | undefined, documentType: "textes" | "rapports", options?: {}): Generator<IterItem<DossierLegislatifDocumentResult>>;
+export declare function iterLoadSenatDossiersLegislatifsRapports(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DossierLegislatifDocumentResult>>;
+export declare function iterLoadSenatDossiersLegislatifsTextes(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DossierLegislatifDocumentResult>>;
 export declare function loadSenatTexteContent(dataDir: string, textePathFromDataset: string): IterItem<FlatTexte | null>;
 export declare function loadSenatCompteRenduContent(dataDir: string, session: number, debatId: string | number): {
     item: CompteRendu | null;

package/lib/loaders.js

@@ -50,9 +50,6 @@ function* iterLoadSenatItems(dataDir, dataName, legislatureOrSession, subDir, {
         console.log(`Found ${changedFiles?.size || 0} changed files (AMR)`);
     }
     for (const filePath of iterFilePaths(itemsDir)) {
-        if (!filePath.endsWith(".json")) {
-            continue;
-        }
         const relativePath = path.relative(path.join(dataDir, dataName), filePath);
         const gitStatus = changedFiles?.get(relativePath);
         // Filter by changed files if sinceCommit is specified
@@ -150,7 +147,7 @@ export function* iterLoadSenatDossiersLegislatifs(dataDir, session, options = {}
         yield dossierLegislatifItem;
     }
 }
-export function* iterLoadSenatRapportUrls(dataDir, session) {
+export function* iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session) {
     let itemsDir = path.join(dataDir, RAPPORT_FOLDER, DATA_ORIGINAL_FOLDER);
     if (session) {
         itemsDir = path.join(itemsDir, session.toString());
@@ -166,7 +163,7 @@ export function* iterLoadSenatRapportUrls(dataDir, session) {
         }
     }
 }
-export function* iterLoadSenatTexteUrls(dataDir, session) {
+export function* iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session) {
     let itemsDir = path.join(dataDir, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER);
     if (session) {
         itemsDir = path.join(itemsDir, session.toString());
@@ -182,7 +179,7 @@ export function* iterLoadSenatTexteUrls(dataDir, session) {
         }
     }
 }
-export function* iterLoadSenatDocuments(dataDir, session, documentType, options = {}) {
+export function* iterLoadSenatDossiersLegislatifsDocuments(dataDir, session, documentType, options = {}) {
     for (const { item: dossierLegislatif } of iterLoadSenatDossiersLegislatifs(dataDir, session, options)) {
         for (const lecture of dossierLegislatif["lectures"]) {
             const lecturesSenat = lecture.lectures_assemblee.filter((lectureAssemblee) => lectureAssemblee.assemblee === "Sénat");
@@ -190,10 +187,15 @@ export function* iterLoadSenatDocuments(dataDir, session, documentType, options
                 for (const document of lectureSenat[documentType]) {
                     const enrichedDocument = {
                         signet_dossier: dossierLegislatif["signet"],
+                        url_dossier_senat: dossierLegislatif["url"],
+                        url_dossier_assemblee_nationale: dossierLegislatif["url_dossier_assemblee_nationale"],
+                        type_lecture: lecture.type_lecture,
+                        libelle_lecture: lecture.libelle,
+                        libelle_organisme: lectureSenat.libelle_organisme,
                         ...document,
                     };
                     const documentItem = {
-                        item: enrichedDocument
+                        item: enrichedDocument,
                     };
                     if (document.url) {
                         const documentName = path.parse(document.url).name;
@@ -205,15 +207,13 @@ export function* iterLoadSenatDocuments(dataDir, session, documentType, options
         }
     }
 }
-export function* iterLoadSenatRapports(dataDir, session, options = {}) {
-    for (const iterItem of iterLoadSenatItems(dataDir, RAPPORT_FOLDER, session, "original", options)) {
-        if (iterItem.item?.["id"]) {
-            yield iterItem;
-        }
+export function* iterLoadSenatDossiersLegislatifsRapports(dataDir, session, options = {}) {
+    for (const iterItem of iterLoadSenatDossiersLegislatifsDocuments(dataDir, session, "rapports", options)) {
+        yield iterItem;
     }
 }
-export function* iterLoadSenatTextes(dataDir, session, options = {}) {
-    for (const iterItem of iterLoadSenatDocuments(dataDir, session, "textes", options)) {
+export function* iterLoadSenatDossiersLegislatifsTextes(dataDir, session, options = {}) {
+    for (const iterItem of iterLoadSenatDossiersLegislatifsDocuments(dataDir, session, "textes", options)) {
         yield iterItem;
     }
 }

package/lib/model/documents.d.ts

@@ -1,12 +1,2 @@
-import { Expression, InferResult, SelectQueryBuilder } from "kysely";
-export declare function rapports(lectureAssembleeId: Expression<string>): import("kysely").RawBuilder<{
-    [x: string]: any;
-}[]>;
-declare const queryTextes: SelectQueryBuilder<any, any, any>;
-export declare function textes(lectureAssembleeId: Expression<string>): import("kysely").RawBuilder<{
-    [x: string]: any;
-}[]>;
-export declare function findAllTextes(): AsyncIterableIterator<DocumentResult>;
-export declare function findAllRapports(): AsyncIterableIterator<DocumentResult>;
-export type DocumentResult = InferResult<typeof queryTextes>[0];
-export {};
+export declare function findSenatTexteUrls(sessions?: number[]): any;
+export declare function findSenatRapportUrls(sessions?: number[]): any;

package/lib/model/documents.js

@@ -1,151 +1,33 @@
-import { sql } from "kysely";
 import { dbSenat } from "../databases";
-import { concat, rtrim, toDateString } from "./util";
-import { jsonArrayFrom } from "kysely/helpers/postgres";
-function orderOrdreOrigineTexte(expr) {
-    return sql `array_position(array['0','2','1'], ${expr})`;
-}
-function auteursRapport(rapportId) {
-    return jsonArrayFrom(dbSenat
+import { rtrim, toDateString } from "./util";
+export function findSenatTexteUrls(sessions = []) {
+    return dbSenat
         .withSchema("dosleg")
-        .selectFrom("dosleg.auteur")
-        .leftJoin("dosleg.ecr", "dosleg.ecr.autcod", "dosleg.auteur.autcod")
-        .leftJoin("dosleg.rolsig", "dosleg.rolsig.signataire", "dosleg.ecr.signataire")
-        .where("dosleg.ecr.rapcod", "=", rapportId)
-        .select([
-        "dosleg.auteur.prenom as prenom",
-        "dosleg.auteur.nomuse as nom_usuel",
-        "dosleg.auteur.autmat as matricule",
-        "dosleg.ecr.ecrnumtri as ordre",
-        "dosleg.rolsig.rolsiglib as role",
-        "dosleg.ecr.ecrqua as qualite",
+        .selectFrom("texte")
+        .where("texurl", "is not", null)
+        .where("typurl", "=", "I")
+        .$if(sessions.length > 0, (qb) => qb.where("sesann", "in", sessions))
+        .select(({ eb, ref }) => [
+        "sesann as session",
+        rtrim(ref("texurl")).as("url"),
+        toDateString(ref("txtoritxtdat")).as("date"),
+        eb.case().when("oritxtcod", "=", "1").then(true).else(false).end().as("hasExposeDesMotifs"),
     ])
-        .orderBy("dosleg.ecr.ecrnumtri", "asc"));
-}
-function documentsAttaches(rapportId) {
-    return jsonArrayFrom(dbSenat
-        .withSchema("dosleg")
-        .selectFrom("docatt")
-        .leftJoin("typatt", "docatt.typattcod", "typatt.typattcod")
-        .where("docatt.rapcod", "=", rapportId)
-        .select([
-        "docatt.docatturl as url",
-        "typatt.typattlib as type_document"
-    ]));
-}
-function selectRapportAttributes({ eb, ref, val }) {
-    return [
-        "rap.rapnum as numero",
-        "raporg.orgcod as code_organisme",
-        eb
-            .case()
-            .when("rap.rapurl", "is not", null)
-            .then(sql `regexp_replace(trim(${ref("rap.rapurl")}), '^(.*/)?(.*?)(\\.html)?$', '\\2')`)
-            .else(null)
-            .end()
-            .as("id"),
-        eb
-            .case()
-            .when("rap.typurl", "=", "I")
-            .then(concat(val("https://www.senat.fr/rap/"), rtrim(ref("rap.rapurl"))))
-            .else(rtrim(ref("rap.rapurl")))
-            .end()
-            .as("url"),
-        rtrim(ref("denrap.libdenrap")).as("type"),
-        rtrim(rtrim(ref("rap.raptil"))).as("titre"),
-        rtrim(rtrim(ref("rap.rapsoustit"))).as("sous_titre"),
-        toDateString(ref("rap.date_depot")).as("date"),
-        "rap.sesann as session",
-        auteursRapport(ref("rap.rapcod")).as("auteurs"),
-        documentsAttaches(ref("rap.rapcod")).as("documents_annexes"),
-    ];
-}
-const baseQueryRapports = dbSenat
-    .withSchema("dosleg")
-    .selectFrom("rap")
-    .leftJoin("raporg", "raporg.rapcod", "rap.rapcod")
-    .leftJoin("denrap", "denrap.coddenrap", "rap.coddenrap")
-    .leftJoin("lecassrap", "lecassrap.rapcod", "rap.rapcod");
-const queryRapports = baseQueryRapports
-    .leftJoin("lecass", "lecass.lecassidt", "lecassrap.lecassidt")
-    .leftJoin("lecture", "lecture.lecidt", "lecass.lecidt")
-    .leftJoin("loi", "loi.loicod", "lecture.loicod")
-    .select((args) => [
-    "loi.signet as signet_dossier",
-    ...selectRapportAttributes(args),
-]);
-export function rapports(lectureAssembleeId) {
-    return jsonArrayFrom(baseQueryRapports
-        .select(selectRapportAttributes)
-        .where("lecassrap.lecassidt", "=", lectureAssembleeId));
+        .$narrowType()
+        .stream();
 }
-function auteursTexte(texteId) {
-    return jsonArrayFrom(dbSenat
+export function findSenatRapportUrls(sessions = []) {
+    return dbSenat
         .withSchema("dosleg")
-        .selectFrom("auteur")
-        .leftJoin("ecr", "ecr.autcod", "auteur.autcod")
-        .leftJoin("rolsig", "rolsig.signataire", "ecr.signataire")
-        .where("ecr.texcod", "=", texteId)
-        .select([
-        "auteur.prenom as prenom",
-        "auteur.nomuse as nom_usuel",
-        "auteur.autmat as matricule",
-        "ecr.ecrnumtri as ordre",
-        "rolsig.rolsiglib as role",
-        "ecr.ecrqua as qualite",
+        .selectFrom("rap")
+        .where("rapurl", "is not", null)
+        .where("typurl", "=", "I")
+        .$if(sessions.length > 0, (qb) => qb.where("sesann", "in", sessions))
+        .select(({ ref }) => [
+        "sesann as session",
+        rtrim(ref("rapurl")).as("url"),
+        toDateString(ref("date_depot")).as("date"),
     ])
-        .orderBy("ecr.ecrnumtri", "asc"));
-}
-function selectTexteAttributes({ eb, ref, val }) {
-    return [
-        "texte.texnum as numero",
-        "texte.orgcod as code_organisme",
-        eb
-            .case()
-            .when("texte.texurl", "is not", null)
-            .then(sql `regexp_replace(trim(${ref("texte.texurl")}), '^(.*/)?(.*?)(\\.html)?$', '\\2')`)
-            .else(null)
-            .end()
-            .as("id"),
-        eb
-            .case()
-            .when("texte.typurl", "=", "I")
-            .then(concat(val("https://www.senat.fr/leg/"), rtrim(ref("texte.texurl"))))
-            .else(rtrim(ref("texte.texurl")))
-            .end()
-            .as("url"),
-        rtrim(ref("oritxt.oritxtlib")).as("origine"),
-        "oritxt.oriordre as ordre_origine",
-        "oritxt.oritxtado as code_adoption",
-        "oritxt.oritxtmod as modification",
-        rtrim(ref("typtxt.typtxtlib")).as("type"),
-        toDateString(ref("texte.txtoritxtdat")).as("date"),
-        "texte.sesann as session",
-        auteursTexte(ref("texte.texcod")).as("auteurs"),
-    ];
-}
-const baseQueryTextes = dbSenat
-    .withSchema("dosleg")
-    .selectFrom("texte")
-    .leftJoin("oritxt", "oritxt.oritxtcod", "texte.oritxtcod")
-    .leftJoin("typtxt", "typtxt.typtxtcod", "texte.typtxtcod")
-    .orderBy(({ ref }) => orderOrdreOrigineTexte(ref("oritxt.oriordre")));
-const queryTextes = baseQueryTextes
-    .leftJoin("lecass", "lecass.lecassidt", "texte.lecassidt")
-    .leftJoin("lecture", "lecture.lecidt", "lecass.lecidt")
-    .leftJoin("loi", "loi.loicod", "lecture.loicod")
-    .select((args) => [
-    "loi.signet as signet_dossier",
-    ...selectTexteAttributes(args),
-]);
-export function textes(lectureAssembleeId) {
-    return jsonArrayFrom(baseQueryTextes
-        .select(selectTexteAttributes)
-        .where("texte.lecassidt", "=", lectureAssembleeId));
-}
-export function findAllTextes() {
-    return queryTextes.stream();
-}
-export function findAllRapports() {
-    return queryRapports.stream();
+        .$narrowType()
+        .stream();
 }

package/lib/model/dosleg.js

@@ -1,8 +1,10 @@
 import { sql } from "kysely";
 import { jsonArrayFrom } from "kysely/helpers/postgres";
 import { dbSenat } from "../databases";
-import { concat, rtrim, toDateString } from "./util";
-import { textes, rapports } from "./documents";
+import { concat, removeSubstring, rtrim, toDateString } from "./util";
+function orderOrdreOrigineTexte(expr) {
+    return sql `array_position(array['0','2','1'], ${expr})`;
+}
 function datesSeances(lectureAssembleeId) {
     return jsonArrayFrom(dbSenat
         .withSchema("dosleg")
@@ -10,6 +12,120 @@ function datesSeances(lectureAssembleeId) {
         .where("dosleg.date_seance.lecidt", "=", lectureAssembleeId)
         .select(({ ref }) => [toDateString(ref("dosleg.date_seance.date_s")).as("date")]));
 }
+function auteursRapport(rapportId) {
+    return jsonArrayFrom(dbSenat
+        .withSchema("dosleg")
+        .selectFrom("dosleg.auteur")
+        .leftJoin("dosleg.ecr", "dosleg.ecr.autcod", "dosleg.auteur.autcod")
+        .leftJoin("dosleg.rolsig", "dosleg.rolsig.signataire", "dosleg.ecr.signataire")
+        .where("dosleg.ecr.rapcod", "=", rapportId)
+        .select([
+        "dosleg.auteur.prenom as prenom",
+        "dosleg.auteur.nomuse as nom_usuel",
+        "dosleg.auteur.autmat as matricule",
+        "dosleg.ecr.ecrnumtri as ordre",
+        "dosleg.rolsig.rolsiglib as role",
+        "dosleg.ecr.ecrqua as qualite",
+    ])
+        .orderBy("dosleg.ecr.ecrnumtri", "asc"));
+}
+function documentsAttaches(rapportId) {
+    return jsonArrayFrom(dbSenat
+        .withSchema("dosleg")
+        .selectFrom("docatt")
+        .leftJoin("typatt", "docatt.typattcod", "typatt.typattcod")
+        .where("docatt.rapcod", "=", rapportId)
+        .select([
+        "docatt.docatturl as url",
+        "typatt.typattlib as type_document"
+    ]));
+}
+function rapports(lectureAssembleeId) {
+    return jsonArrayFrom(dbSenat
+        .withSchema("dosleg")
+        .selectFrom("rap")
+        .leftJoin("lecassrap", "lecassrap.rapcod", "rap.rapcod")
+        .leftJoin("raporg", "raporg.rapcod", "rap.rapcod")
+        .leftJoin("denrap", "denrap.coddenrap", "rap.coddenrap")
+        .where("lecassrap.lecassidt", "=", lectureAssembleeId)
+        .select(({ eb, ref, val }) => [
+        "rap.rapnum as numero",
+        "raporg.orgcod as code_organisme",
+        eb
+            .case()
+            .when("rap.typurl", "=", "I")
+            .then(removeSubstring(ref("rap.rapurl"), val(".html")))
+            .else(null)
+            .end()
+            .as("id"),
+        eb
+            .case()
+            .when("rap.typurl", "=", "I")
+            .then(concat(val("https://www.senat.fr/rap/"), rtrim(ref("rap.rapurl"))))
+            .else(rtrim(ref("rap.rapurl")))
+            .end()
+            .as("url"),
+        rtrim(ref("denrap.libdenrap")).as("type"),
+        rtrim(ref("rap.raptil")).as("titre"),
+        rtrim(ref("rap.rapsoustit")).as("sous_titre"),
+        toDateString(ref("rap.date_depot")).as("date"),
+        "sesann as session",
+        auteursRapport(ref("rap.rapcod")).as("auteurs"),
+        documentsAttaches(ref("rap.rapcod")).as("documents_annexes"),
+    ]));
+}
+function auteursTexte(texteId) {
+    return jsonArrayFrom(dbSenat
+        .withSchema("dosleg")
+        .selectFrom("auteur")
+        .leftJoin("ecr", "ecr.autcod", "auteur.autcod")
+        .leftJoin("rolsig", "rolsig.signataire", "ecr.signataire")
+        .where("ecr.texcod", "=", texteId)
+        .select([
+        "auteur.prenom as prenom",
+        "auteur.nomuse as nom_usuel",
+        "auteur.autmat as matricule",
+        "ecr.ecrnumtri as ordre",
+        "rolsig.rolsiglib as role",
+        "ecr.ecrqua as qualite",
+    ])
+        .orderBy("ecr.ecrnumtri", "asc"));
+}
+function textes(lectureAssembleeId) {
+    return jsonArrayFrom(dbSenat
+        .withSchema("dosleg")
+        .selectFrom("texte")
+        .leftJoin("oritxt", "oritxt.oritxtcod", "texte.oritxtcod")
+        .leftJoin("typtxt", "typtxt.typtxtcod", "texte.typtxtcod")
+        .where("texte.lecassidt", "=", lectureAssembleeId)
+        .select(({ eb, ref, val }) => [
+        "texte.texnum as numero",
+        "texte.orgcod as code_organisme",
+        eb
+            .case()
+            .when("texte.typurl", "=", "I")
+            .then(removeSubstring(ref("texte.texurl"), val(".html")))
+            .else(null)
+            .end()
+            .as("id"),
+        eb
+            .case()
+            .when("texte.typurl", "=", "I")
+            .then(concat(val("https://www.senat.fr/leg/"), rtrim(ref("texte.texurl"))))
+            .else(rtrim(ref("texte.texurl")))
+            .end()
+            .as("url"),
+        rtrim(ref("oritxt.oritxtlib")).as("origine"),
+        "oritxt.oriordre as ordre_origine",
+        "oritxt.oritxtado as code_adoption",
+        "oritxt.oritxtmod as modification",
+        rtrim(ref("typtxt.typtxtlib")).as("type"),
+        toDateString(ref("texte.txtoritxtdat")).as("date"),
+        "sesann as session",
+        auteursTexte(ref("texte.texcod")).as("auteurs"),
+    ])
+        .orderBy(({ ref }) => orderOrdreOrigineTexte(ref("oritxt.oriordre"))));
+}
 function lecturesAssemblee(lectureId) {
     return jsonArrayFrom(dbSenat
         .withSchema("dosleg")

package/lib/model/index.d.ts

@@ -1,7 +1,7 @@
 export { findAllAmendements } from "./ameli";
 export { findAll as findAllDebats } from "./debats";
 export { findAllDossiers } from "./dosleg";
-export { findAllTextes, findAllRapports } from "./documents";
+export { findSenatTexteUrls, findSenatRapportUrls } from "./documents";
 export { findAllScrutins } from "./scrutins";
 export { findAll as findAllQuestions } from "./questions";
 export { findAll as findAllSens, findAllCirconscriptions, findAllOrganismes } from "./sens";

package/lib/model/index.js

@@ -1,7 +1,7 @@
 export { findAllAmendements } from "./ameli";
 export { findAll as findAllDebats } from "./debats";
 export { findAllDossiers } from "./dosleg";
-export { findAllTextes, findAllRapports } from "./documents";
+export { findSenatTexteUrls, findSenatRapportUrls } from "./documents";
 export { findAllScrutins } from "./scrutins";
 export { findAll as findAllQuestions } from "./questions";
 export { findAll as findAllSens, findAllCirconscriptions, findAllOrganismes } from "./sens";

package/lib/scripts/convert_data.js

@@ -5,8 +5,8 @@ import path from "path";
 import pLimit from "p-limit";
 import * as git from "../git";
 import { datasets, EnabledDatasets, getEnabledDatasets } from "../datasets";
-import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, SCRUTINS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER } from "../loaders";
-import { findAllAmendements, findAllCirconscriptions, findAllDebats, findAllDossiers, findAllScrutins, findAllOrganismes, findAllQuestions, findAllSens, findAllTextes, findAllRapports, } from "../model";
+import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, SCRUTINS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER, } from "../loaders";
+import { findAllAmendements, findAllCirconscriptions, findAllDebats, findAllDossiers, findAllScrutins, findAllOrganismes, findAllQuestions, findAllSens, findSenatRapportUrls, findSenatTexteUrls, } from "../model";
 import { processRapport, processTexte } from "./retrieve_documents";
 import { buildActesLegislatifs } from "../model/dosleg";
 import { UNDEFINED_SESSION } from "../types/sessions";
@@ -169,8 +169,8 @@ async function convertDatasetDosLeg(dataDir, options) {
         const dossierFile = `${dossier["signet"]}.json`;
         await fs.outputJSON(path.join(dossierReorganizedDir, dossierFile), dossierWithActes, { spaces: 2 });
     }
-    await convertTextes(dataDir, options);
-    await convertRapports(dataDir, options);
+    await convertTexteUrls(dataDir, options);
+    await convertRapportUrls(dataDir, options);
 }
 async function convertDatasetScrutins(dataDir, options) {
     const dataset = datasets.dosleg;
@@ -219,38 +219,30 @@ async function convertDatasetQuestions(dataDir, options) {
     }
     await Promise.all(tasks);
 }
-async function convertTextes(dataDir, options) {
+async function convertTexteUrls(dataDir, options) {
     const originalTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER);
     const transformedTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_TRANSFORMED_FOLDER);
     if (!options["silent"]) {
         console.log(`Converting database textes data into files…`);
     }
-    for await (const texte of findAllTextes()) {
-        const session = texte["session"] ?? UNDEFINED_SESSION;
+    for await (const texte of findSenatTexteUrls()) {
+        const session = texte.session ?? UNDEFINED_SESSION;
         if (options["fromSession"] && session < options["fromSession"]) {
             continue;
         }
-        if (!texte["url"]) {
-            continue;
-        }
-        const texteName = path.parse(texte["url"]).name;
+        const texteName = path.parse(texte.url).name;
         const texteDir = path.join(originalTextesDir, `${session}`, texteName);
-        // oritxtcod = 1 corresponds to "Texte de loi déposé au Sénat"
-        const hasExposeDesMotifs = texte["origine"] === 'Sénat' && texte["ordre_origine"] === '1';
         const metadata = {
             name: texteName,
-            session: texte["session"],
-            date: texte["date"],
-            url_expose_des_motifs: hasExposeDesMotifs
+            session: texte.session,
+            date: texte.date,
+            url_expose_des_motifs: texte.hasExposeDesMotifs
                 ? new URL(`${texteName}-expose.html`, SENAT_EXPOSE_DES_MOTIFS_BASE_URL)
                 : undefined,
             url_xml: new URL(`${texteName}.akn.xml`, SENAT_TEXTE_XML_BASE_URL),
             url_html: new URL(`${texteName}.html`, SENAT_TEXTE_BASE_URL),
             url_pdf: new URL(`${texteName}.pdf`, SENAT_TEXTE_BASE_URL),
         };
-        fs.outputJSONSync(path.join(texteDir, `${texteName}.json`), texte, {
-            spaces: 2,
-        });
         fs.outputJSONSync(path.join(texteDir, DOCUMENT_METADATA_FILE), metadata, {
             spaces: 2,
         });
@@ -259,20 +251,17 @@ async function convertTextes(dataDir, options) {
         }
     }
 }
-async function convertRapports(dataDir, options) {
+async function convertRapportUrls(dataDir, options) {
     const originalRapportsDir = path.join(dataDir, RAPPORT_FOLDER, DATA_ORIGINAL_FOLDER);
     if (!options["silent"]) {
         console.log(`Converting database rapports data into files…`);
     }
-    for await (const rapport of findAllRapports()) {
-        const session = rapport["session"] ?? UNDEFINED_SESSION;
+    for await (const rapport of findSenatRapportUrls()) {
+        const session = rapport.session ?? UNDEFINED_SESSION;
         if (options["fromSession"] && session < options["fromSession"]) {
             continue;
         }
-        if (!rapport["url"]) {
-            continue;
-        }
-        const parsedRapportUrl = path.parse(rapport["url"]);
+        const parsedRapportUrl = path.parse(rapport.url);
         const rapportName = parsedRapportUrl.name;
         const rapportDir = path.join(originalRapportsDir, `${session}`, rapportName);
         const rapportHtmlUrlBase = `${rapportName}_mono.html`;
@@ -287,14 +276,11 @@ async function convertRapports(dataDir, options) {
         });
         const metadata = {
             name: rapportName,
-            session: rapport["session"],
-            date: rapport["date"],
+            session: rapport.session,
+            date: rapport.date,
             url_html: new URL(rapportHtmlUrl, SENAT_RAPPORT_BASE_URL),
             url_pdf: new URL(rapportPdfUrl, SENAT_RAPPORT_BASE_URL),
         };
-        fs.outputJSONSync(path.join(rapportDir, `${rapportName}.json`), rapport, {
-            spaces: 2,
-        });
         fs.outputJSONSync(path.join(rapportDir, DOCUMENT_METADATA_FILE), metadata, {
             spaces: 2,
         });

package/lib/scripts/retrieve_documents.d.ts

@@ -1,3 +1,2 @@
-import { DocumentMetadata } from "../types/texte";
-export declare function processTexte(texteMetadata: DocumentMetadata, originalTextesDir: string, transformedTextesDir: string, options: any): Promise<void>;
+export declare function processTexte(texteMetadata: any, originalTextesDir: string, transformedTextesDir: string, options: any): Promise<void>;
 export declare function processRapport(rapportMetadata: any, originalRapportsDir: string, options: any): Promise<void>;