@tricoteuses/senat 2.20.29 → 2.20.31

package/lib/loaders.d.ts

@@ -5,9 +5,11 @@ import { QuestionResult } from "./model/questions";
 import { ScrutinResult } from "./model/scrutins";
 import { CirconscriptionResult, OrganismeResult, SenateurResult } from "./model/sens";
 import { Reunion } from "./types/agenda";
-import { FlatTexte } from "./types/texte";
+import { FlatTexte, DocumentMetadata } from "./types/texte";
 import { CompteRendu } from "./types/compte_rendu";
+import { DocumentResult } from "./model/documents";
 export { EnabledDatasets } from "./datasets";
+export type { DocumentResult } from "./model/documents";
 export declare const AGENDA_FOLDER = "agenda";
 export declare const COMPTES_RENDUS_FOLDER = "seances";
 export declare const COMMISSION_FOLDER = "commissions";
@@ -27,46 +29,6 @@ export type IterItem<T> = {
     legislature?: number;
     gitStatus?: "A" | "M" | "D" | "R" | "C" | "T" | "U";
 };
-export interface TexteMetadata {
-    name: string;
-    session: number | null | undefined;
-    date?: string | null;
-    url_expose_des_motifs?: URL;
-    url_xml: URL;
-    url_html: URL;
-    url_pdf: URL;
-}
-export interface RapportMetadata {
-    name: string;
-    session: number | null | undefined;
-    date?: string | null;
-    url_html: URL;
-    url_pdf: URL;
-}
-export interface DossierLegislatifDocumentResult {
-    signet_dossier: string;
-    url_dossier_senat: string;
-    url_dossier_assemblee_nationale: string | null;
-    type_lecture: string;
-    libelle_lecture: string;
-    libelle_organisme: string | null;
-    code_organisme: string | null;
-    numero: number | null;
-    id: string | null;
-    url: string;
-    origine?: string | null | undefined;
-    type: string;
-    date: string;
-    session: number | null;
-    auteurs: {
-        prenom: string | null;
-        nom_usuel: string;
-        matricule: string | null;
-    }[];
-    organismes?: {
-        libelle: string;
-    }[] | undefined;
-}
 export declare function iterFilePaths(dirPath: string): Generator<string>;
 export declare function iterLoadSenatAmendements(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<AmendementResult>>;
 export declare function iterLoadSenatDebats(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DebatResult>>;
@@ -79,11 +41,11 @@ export declare function iterLoadSenatComptesRendusCommissions(dataDir: string, s
     session: number;
 }>;
 export declare function iterLoadSenatDossiersLegislatifs(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DossierLegislatifResult>>;
-export declare function iterLoadSenatDossiersLegislatifsRapportUrls(dataDir: string, session: number | undefined): Generator<IterItem<RapportMetadata>>;
-export declare function iterLoadSenatDossiersLegislatifsTexteUrls(dataDir: string, session: number | undefined): Generator<IterItem<TexteMetadata>>;
-export declare function iterLoadSenatDossiersLegislatifsDocuments(dataDir: string, session: number | undefined, documentType: "textes" | "rapports", options?: {}): Generator<IterItem<DossierLegislatifDocumentResult>>;
-export declare function iterLoadSenatDossiersLegislatifsRapports(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DossierLegislatifDocumentResult>>;
-export declare function iterLoadSenatDossiersLegislatifsTextes(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DossierLegislatifDocumentResult>>;
+export declare function iterLoadSenatRapportUrls(dataDir: string, session: number | undefined): Generator<IterItem<DocumentMetadata>>;
+export declare function iterLoadSenatTexteUrls(dataDir: string, session: number | undefined): Generator<IterItem<DocumentMetadata>>;
+export declare function iterLoadSenatDocuments(dataDir: string, session: number | undefined, documentType: "textes" | "rapports", options?: {}): Generator<IterItem<DocumentResult>>;
+export declare function iterLoadSenatRapports(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DocumentResult>>;
+export declare function iterLoadSenatTextes(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DocumentResult>>;
 export declare function loadSenatTexteContent(dataDir: string, textePathFromDataset: string): IterItem<FlatTexte | null>;
 export declare function loadSenatCompteRenduContent(dataDir: string, session: number, debatId: string | number): {
     item: CompteRendu | null;

package/lib/loaders.js

@@ -147,7 +147,7 @@ export function* iterLoadSenatDossiersLegislatifs(dataDir, session, options = {}
         yield dossierLegislatifItem;
     }
 }
-export function* iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session) {
+export function* iterLoadSenatRapportUrls(dataDir, session) {
     let itemsDir = path.join(dataDir, RAPPORT_FOLDER, DATA_ORIGINAL_FOLDER);
     if (session) {
         itemsDir = path.join(itemsDir, session.toString());
@@ -163,7 +163,7 @@ export function* iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session) {
         }
     }
 }
-export function* iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session) {
+export function* iterLoadSenatTexteUrls(dataDir, session) {
     let itemsDir = path.join(dataDir, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER);
     if (session) {
         itemsDir = path.join(itemsDir, session.toString());
@@ -179,23 +179,14 @@ export function* iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session) {
         }
     }
 }
-export function* iterLoadSenatDossiersLegislatifsDocuments(dataDir, session, documentType, options = {}) {
+export function* iterLoadSenatDocuments(dataDir, session, documentType, options = {}) {
     for (const { item: dossierLegislatif } of iterLoadSenatDossiersLegislatifs(dataDir, session, options)) {
         for (const lecture of dossierLegislatif["lectures"]) {
             const lecturesSenat = lecture.lectures_assemblee.filter((lectureAssemblee) => lectureAssemblee.assemblee === "Sénat");
             for (const lectureSenat of lecturesSenat) {
                 for (const document of lectureSenat[documentType]) {
-                    const enrichedDocument = {
-                        signet_dossier: dossierLegislatif["signet"],
-                        url_dossier_senat: dossierLegislatif["url"],
-                        url_dossier_assemblee_nationale: dossierLegislatif["url_dossier_assemblee_nationale"],
-                        type_lecture: lecture.type_lecture,
-                        libelle_lecture: lecture.libelle,
-                        libelle_organisme: lectureSenat.libelle_organisme,
-                        ...document,
-                    };
                     const documentItem = {
-                        item: enrichedDocument,
+                        item: document,
                     };
                     if (document.url) {
                         const documentName = path.parse(document.url).name;
@@ -207,13 +198,13 @@ export function* iterLoadSenatDossiersLegislatifsDocuments(dataDir, session, doc
         }
     }
 }
-export function* iterLoadSenatDossiersLegislatifsRapports(dataDir, session, options = {}) {
-    for (const iterItem of iterLoadSenatDossiersLegislatifsDocuments(dataDir, session, "rapports", options)) {
+export function* iterLoadSenatRapports(dataDir, session, options = {}) {
+    for (const iterItem of iterLoadSenatItems(dataDir, RAPPORT_FOLDER, session, "original", options)) {
         yield iterItem;
     }
 }
-export function* iterLoadSenatDossiersLegislatifsTextes(dataDir, session, options = {}) {
-    for (const iterItem of iterLoadSenatDossiersLegislatifsDocuments(dataDir, session, "textes", options)) {
+export function* iterLoadSenatTextes(dataDir, session, options = {}) {
+    for (const iterItem of iterLoadSenatDocuments(dataDir, session, "textes", options)) {
         yield iterItem;
     }
 }

package/lib/model/documents.d.ts

@@ -1,2 +1,12 @@
-export declare function findSenatTexteUrls(sessions?: number[]): any;
-export declare function findSenatRapportUrls(sessions?: number[]): any;
+import { Expression, InferResult, SelectQueryBuilder } from "kysely";
+export declare function rapports(lectureAssembleeId: Expression<string>): import("kysely").RawBuilder<{
+    [x: string]: any;
+}[]>;
+declare const queryTextes: SelectQueryBuilder<any, any, any>;
+export declare function textes(lectureAssembleeId: Expression<string>): import("kysely").RawBuilder<{
+    [x: string]: any;
+}[]>;
+export declare function findAllTextes(): AsyncIterableIterator<DocumentResult>;
+export declare function findAllRapports(): AsyncIterableIterator<DocumentResult>;
+export type DocumentResult = InferResult<typeof queryTextes>[0];
+export {};

package/lib/model/documents.js

@@ -1,33 +1,151 @@
+import { sql } from "kysely";
 import { dbSenat } from "../databases";
-import { rtrim, toDateString } from "./util";
-export function findSenatTexteUrls(sessions = []) {
-    return dbSenat
+import { concat, rtrim, toDateString } from "./util";
+import { jsonArrayFrom } from "kysely/helpers/postgres";
+function orderOrdreOrigineTexte(expr) {
+    return sql `array_position(array['0','2','1'], ${expr})`;
+}
+function auteursRapport(rapportId) {
+    return jsonArrayFrom(dbSenat
         .withSchema("dosleg")
-        .selectFrom("texte")
-        .where("texurl", "is not", null)
-        .where("typurl", "=", "I")
-        .$if(sessions.length > 0, (qb) => qb.where("sesann", "in", sessions))
-        .select(({ eb, ref }) => [
-        "sesann as session",
-        rtrim(ref("texurl")).as("url"),
-        toDateString(ref("txtoritxtdat")).as("date"),
-        eb.case().when("oritxtcod", "=", "1").then(true).else(false).end().as("hasExposeDesMotifs"),
+        .selectFrom("dosleg.auteur")
+        .leftJoin("dosleg.ecr", "dosleg.ecr.autcod", "dosleg.auteur.autcod")
+        .leftJoin("dosleg.rolsig", "dosleg.rolsig.signataire", "dosleg.ecr.signataire")
+        .where("dosleg.ecr.rapcod", "=", rapportId)
+        .select([
+        "dosleg.auteur.prenom as prenom",
+        "dosleg.auteur.nomuse as nom_usuel",
+        "dosleg.auteur.autmat as matricule",
+        "dosleg.ecr.ecrnumtri as ordre",
+        "dosleg.rolsig.rolsiglib as role",
+        "dosleg.ecr.ecrqua as qualite",
     ])
-        .$narrowType()
-        .stream();
+        .orderBy("dosleg.ecr.ecrnumtri", "asc"));
+}
+function documentsAttaches(rapportId) {
+    return jsonArrayFrom(dbSenat
+        .withSchema("dosleg")
+        .selectFrom("docatt")
+        .leftJoin("typatt", "docatt.typattcod", "typatt.typattcod")
+        .where("docatt.rapcod", "=", rapportId)
+        .select([
+        "docatt.docatturl as url",
+        "typatt.typattlib as type_document"
+    ]));
+}
+function selectRapportAttributes({ eb, ref, val }) {
+    return [
+        "rap.rapnum as numero",
+        "raporg.orgcod as code_organisme",
+        eb
+            .case()
+            .when("rap.rapurl", "is not", null)
+            .then(sql `regexp_replace(trim(${ref("rap.rapurl")}), '^(.*/)?(.*?)(\\.html)?$', '\\2')`)
+            .else(null)
+            .end()
+            .as("id"),
+        eb
+            .case()
+            .when("rap.typurl", "=", "I")
+            .then(concat(val("https://www.senat.fr/rap/"), rtrim(ref("rap.rapurl"))))
+            .else(rtrim(ref("rap.rapurl")))
+            .end()
+            .as("url"),
+        rtrim(ref("denrap.libdenrap")).as("type"),
+        rtrim(rtrim(ref("rap.raptil"))).as("titre"),
+        rtrim(rtrim(ref("rap.rapsoustit"))).as("sous_titre"),
+        toDateString(ref("rap.date_depot")).as("date"),
+        "rap.sesann as session",
+        auteursRapport(ref("rap.rapcod")).as("auteurs"),
+        documentsAttaches(ref("rap.rapcod")).as("documents_annexes"),
+    ];
+}
+const baseQueryRapports = dbSenat
+    .withSchema("dosleg")
+    .selectFrom("rap")
+    .leftJoin("raporg", "raporg.rapcod", "rap.rapcod")
+    .leftJoin("denrap", "denrap.coddenrap", "rap.coddenrap")
+    .leftJoin("lecassrap", "lecassrap.rapcod", "rap.rapcod");
+const queryRapports = baseQueryRapports
+    .leftJoin("lecass", "lecass.lecassidt", "lecassrap.lecassidt")
+    .leftJoin("lecture", "lecture.lecidt", "lecass.lecidt")
+    .leftJoin("loi", "loi.loicod", "lecture.loicod")
+    .select((args) => [
+    "loi.signet as signet_dossier",
+    ...selectRapportAttributes(args),
+]);
+export function rapports(lectureAssembleeId) {
+    return jsonArrayFrom(baseQueryRapports
+        .select(selectRapportAttributes)
+        .where("lecassrap.lecassidt", "=", lectureAssembleeId));
 }
-export function findSenatRapportUrls(sessions = []) {
-    return dbSenat
+function auteursTexte(texteId) {
+    return jsonArrayFrom(dbSenat
         .withSchema("dosleg")
-        .selectFrom("rap")
-        .where("rapurl", "is not", null)
-        .where("typurl", "=", "I")
-        .$if(sessions.length > 0, (qb) => qb.where("sesann", "in", sessions))
-        .select(({ ref }) => [
-        "sesann as session",
-        rtrim(ref("rapurl")).as("url"),
-        toDateString(ref("date_depot")).as("date"),
+        .selectFrom("auteur")
+        .leftJoin("ecr", "ecr.autcod", "auteur.autcod")
+        .leftJoin("rolsig", "rolsig.signataire", "ecr.signataire")
+        .where("ecr.texcod", "=", texteId)
+        .select([
+        "auteur.prenom as prenom",
+        "auteur.nomuse as nom_usuel",
+        "auteur.autmat as matricule",
+        "ecr.ecrnumtri as ordre",
+        "rolsig.rolsiglib as role",
+        "ecr.ecrqua as qualite",
     ])
-        .$narrowType()
-        .stream();
+        .orderBy("ecr.ecrnumtri", "asc"));
+}
+function selectTexteAttributes({ eb, ref, val }) {
+    return [
+        "texte.texnum as numero",
+        "texte.orgcod as code_organisme",
+        eb
+            .case()
+            .when("texte.texurl", "is not", null)
+            .then(sql `regexp_replace(trim(${ref("texte.texurl")}), '^(.*/)?(.*?)(\\.html)?$', '\\2')`)
+            .else(null)
+            .end()
+            .as("id"),
+        eb
+            .case()
+            .when("texte.typurl", "=", "I")
+            .then(concat(val("https://www.senat.fr/leg/"), rtrim(ref("texte.texurl"))))
+            .else(rtrim(ref("texte.texurl")))
+            .end()
+            .as("url"),
+        rtrim(ref("oritxt.oritxtlib")).as("origine"),
+        "oritxt.oriordre as ordre_origine",
+        "oritxt.oritxtado as code_adoption",
+        "oritxt.oritxtmod as modification",
+        rtrim(ref("typtxt.typtxtlib")).as("type"),
+        toDateString(ref("texte.txtoritxtdat")).as("date"),
+        "texte.sesann as session",
+        auteursTexte(ref("texte.texcod")).as("auteurs"),
+    ];
+}
+const baseQueryTextes = dbSenat
+    .withSchema("dosleg")
+    .selectFrom("texte")
+    .leftJoin("oritxt", "oritxt.oritxtcod", "texte.oritxtcod")
+    .leftJoin("typtxt", "typtxt.typtxtcod", "texte.typtxtcod")
+    .orderBy(({ ref }) => orderOrdreOrigineTexte(ref("oritxt.oriordre")));
+const queryTextes = baseQueryTextes
+    .leftJoin("lecass", "lecass.lecassidt", "texte.lecassidt")
+    .leftJoin("lecture", "lecture.lecidt", "lecass.lecidt")
+    .leftJoin("loi", "loi.loicod", "lecture.loicod")
+    .select((args) => [
+    "loi.signet as signet_dossier",
+    ...selectTexteAttributes(args),
+]);
+export function textes(lectureAssembleeId) {
+    return jsonArrayFrom(baseQueryTextes
+        .select(selectTexteAttributes)
+        .where("texte.lecassidt", "=", lectureAssembleeId));
+}
+export function findAllTextes() {
+    return queryTextes.stream();
+}
+export function findAllRapports() {
+    return queryRapports.stream();
 }

package/lib/model/dosleg.js

@@ -1,10 +1,8 @@
 import { sql } from "kysely";
 import { jsonArrayFrom } from "kysely/helpers/postgres";
 import { dbSenat } from "../databases";
-import { concat, removeSubstring, rtrim, toDateString } from "./util";
-function orderOrdreOrigineTexte(expr) {
-    return sql `array_position(array['0','2','1'], ${expr})`;
-}
+import { concat, rtrim, toDateString } from "./util";
+import { textes, rapports } from "./documents";
 function datesSeances(lectureAssembleeId) {
     return jsonArrayFrom(dbSenat
         .withSchema("dosleg")
@@ -12,120 +10,6 @@ function datesSeances(lectureAssembleeId) {
         .where("dosleg.date_seance.lecidt", "=", lectureAssembleeId)
         .select(({ ref }) => [toDateString(ref("dosleg.date_seance.date_s")).as("date")]));
 }
-function auteursRapport(rapportId) {
-    return jsonArrayFrom(dbSenat
-        .withSchema("dosleg")
-        .selectFrom("dosleg.auteur")
-        .leftJoin("dosleg.ecr", "dosleg.ecr.autcod", "dosleg.auteur.autcod")
-        .leftJoin("dosleg.rolsig", "dosleg.rolsig.signataire", "dosleg.ecr.signataire")
-        .where("dosleg.ecr.rapcod", "=", rapportId)
-        .select([
-        "dosleg.auteur.prenom as prenom",
-        "dosleg.auteur.nomuse as nom_usuel",
-        "dosleg.auteur.autmat as matricule",
-        "dosleg.ecr.ecrnumtri as ordre",
-        "dosleg.rolsig.rolsiglib as role",
-        "dosleg.ecr.ecrqua as qualite",
-    ])
-        .orderBy("dosleg.ecr.ecrnumtri", "asc"));
-}
-function documentsAttaches(rapportId) {
-    return jsonArrayFrom(dbSenat
-        .withSchema("dosleg")
-        .selectFrom("docatt")
-        .leftJoin("typatt", "docatt.typattcod", "typatt.typattcod")
-        .where("docatt.rapcod", "=", rapportId)
-        .select([
-        "docatt.docatturl as url",
-        "typatt.typattlib as type_document"
-    ]));
-}
-function rapports(lectureAssembleeId) {
-    return jsonArrayFrom(dbSenat
-        .withSchema("dosleg")
-        .selectFrom("rap")
-        .leftJoin("lecassrap", "lecassrap.rapcod", "rap.rapcod")
-        .leftJoin("raporg", "raporg.rapcod", "rap.rapcod")
-        .leftJoin("denrap", "denrap.coddenrap", "rap.coddenrap")
-        .where("lecassrap.lecassidt", "=", lectureAssembleeId)
-        .select(({ eb, ref, val }) => [
-        "rap.rapnum as numero",
-        "raporg.orgcod as code_organisme",
-        eb
-            .case()
-            .when("rap.typurl", "=", "I")
-            .then(removeSubstring(ref("rap.rapurl"), val(".html")))
-            .else(null)
-            .end()
-            .as("id"),
-        eb
-            .case()
-            .when("rap.typurl", "=", "I")
-            .then(concat(val("https://www.senat.fr/rap/"), rtrim(ref("rap.rapurl"))))
-            .else(rtrim(ref("rap.rapurl")))
-            .end()
-            .as("url"),
-        rtrim(ref("denrap.libdenrap")).as("type"),
-        rtrim(ref("rap.raptil")).as("titre"),
-        rtrim(ref("rap.rapsoustit")).as("sous_titre"),
-        toDateString(ref("rap.date_depot")).as("date"),
-        "sesann as session",
-        auteursRapport(ref("rap.rapcod")).as("auteurs"),
-        documentsAttaches(ref("rap.rapcod")).as("documents_annexes"),
-    ]));
-}
-function auteursTexte(texteId) {
-    return jsonArrayFrom(dbSenat
-        .withSchema("dosleg")
-        .selectFrom("auteur")
-        .leftJoin("ecr", "ecr.autcod", "auteur.autcod")
-        .leftJoin("rolsig", "rolsig.signataire", "ecr.signataire")
-        .where("ecr.texcod", "=", texteId)
-        .select([
-        "auteur.prenom as prenom",
-        "auteur.nomuse as nom_usuel",
-        "auteur.autmat as matricule",
-        "ecr.ecrnumtri as ordre",
-        "rolsig.rolsiglib as role",
-        "ecr.ecrqua as qualite",
-    ])
-        .orderBy("ecr.ecrnumtri", "asc"));
-}
-function textes(lectureAssembleeId) {
-    return jsonArrayFrom(dbSenat
-        .withSchema("dosleg")
-        .selectFrom("texte")
-        .leftJoin("oritxt", "oritxt.oritxtcod", "texte.oritxtcod")
-        .leftJoin("typtxt", "typtxt.typtxtcod", "texte.typtxtcod")
-        .where("texte.lecassidt", "=", lectureAssembleeId)
-        .select(({ eb, ref, val }) => [
-        "texte.texnum as numero",
-        "texte.orgcod as code_organisme",
-        eb
-            .case()
-            .when("texte.typurl", "=", "I")
-            .then(removeSubstring(ref("texte.texurl"), val(".html")))
-            .else(null)
-            .end()
-            .as("id"),
-        eb
-            .case()
-            .when("texte.typurl", "=", "I")
-            .then(concat(val("https://www.senat.fr/leg/"), rtrim(ref("texte.texurl"))))
-            .else(rtrim(ref("texte.texurl")))
-            .end()
-            .as("url"),
-        rtrim(ref("oritxt.oritxtlib")).as("origine"),
-        "oritxt.oriordre as ordre_origine",
-        "oritxt.oritxtado as code_adoption",
-        "oritxt.oritxtmod as modification",
-        rtrim(ref("typtxt.typtxtlib")).as("type"),
-        toDateString(ref("texte.txtoritxtdat")).as("date"),
-        "sesann as session",
-        auteursTexte(ref("texte.texcod")).as("auteurs"),
-    ])
-        .orderBy(({ ref }) => orderOrdreOrigineTexte(ref("oritxt.oriordre"))));
-}
 function lecturesAssemblee(lectureId) {
     return jsonArrayFrom(dbSenat
         .withSchema("dosleg")

package/lib/model/index.d.ts

@@ -1,7 +1,7 @@
 export { findAllAmendements } from "./ameli";
 export { findAll as findAllDebats } from "./debats";
 export { findAllDossiers } from "./dosleg";
-export { findSenatTexteUrls, findSenatRapportUrls } from "./documents";
+export { findAllTextes, findAllRapports } from "./documents";
 export { findAllScrutins } from "./scrutins";
 export { findAll as findAllQuestions } from "./questions";
 export { findAll as findAllSens, findAllCirconscriptions, findAllOrganismes } from "./sens";

package/lib/model/index.js

@@ -1,7 +1,7 @@
 export { findAllAmendements } from "./ameli";
 export { findAll as findAllDebats } from "./debats";
 export { findAllDossiers } from "./dosleg";
-export { findSenatTexteUrls, findSenatRapportUrls } from "./documents";
+export { findAllTextes, findAllRapports } from "./documents";
 export { findAllScrutins } from "./scrutins";
 export { findAll as findAllQuestions } from "./questions";
 export { findAll as findAllSens, findAllCirconscriptions, findAllOrganismes } from "./sens";

package/lib/scripts/convert_data.js

@@ -5,8 +5,8 @@ import path from "path";
 import pLimit from "p-limit";
 import * as git from "../git";
 import { datasets, EnabledDatasets, getEnabledDatasets } from "../datasets";
-import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, SCRUTINS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER, } from "../loaders";
-import { findAllAmendements, findAllCirconscriptions, findAllDebats, findAllDossiers, findAllScrutins, findAllOrganismes, findAllQuestions, findAllSens, findSenatRapportUrls, findSenatTexteUrls, } from "../model";
+import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, SCRUTINS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER } from "../loaders";
+import { findAllAmendements, findAllCirconscriptions, findAllDebats, findAllDossiers, findAllScrutins, findAllOrganismes, findAllQuestions, findAllSens, findAllTextes, findAllRapports, } from "../model";
 import { processRapport, processTexte } from "./retrieve_documents";
 import { buildActesLegislatifs } from "../model/dosleg";
 import { UNDEFINED_SESSION } from "../types/sessions";
@@ -169,8 +169,8 @@ async function convertDatasetDosLeg(dataDir, options) {
         const dossierFile = `${dossier["signet"]}.json`;
         await fs.outputJSON(path.join(dossierReorganizedDir, dossierFile), dossierWithActes, { spaces: 2 });
     }
-    await convertTexteUrls(dataDir, options);
-    await convertRapportUrls(dataDir, options);
+    await convertTextes(dataDir, options);
+    await convertRapports(dataDir, options);
 }
 async function convertDatasetScrutins(dataDir, options) {
     const dataset = datasets.dosleg;
@@ -219,30 +219,38 @@ async function convertDatasetQuestions(dataDir, options) {
     }
     await Promise.all(tasks);
 }
-async function convertTexteUrls(dataDir, options) {
+async function convertTextes(dataDir, options) {
     const originalTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER);
     const transformedTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_TRANSFORMED_FOLDER);
     if (!options["silent"]) {
         console.log(`Converting database textes data into files…`);
     }
-    for await (const texte of findSenatTexteUrls()) {
-        const session = texte.session ?? UNDEFINED_SESSION;
+    for await (const texte of findAllTextes()) {
+        const session = texte["session"] ?? UNDEFINED_SESSION;
         if (options["fromSession"] && session < options["fromSession"]) {
             continue;
         }
-        const texteName = path.parse(texte.url).name;
+        if (!texte["url"]) {
+            continue;
+        }
+        const texteName = path.parse(texte["url"]).name;
         const texteDir = path.join(originalTextesDir, `${session}`, texteName);
+        // oritxtcod = 1 corresponds to "Texte de loi déposé au Sénat"
+        const hasExposeDesMotifs = texte["origine"] === 'Sénat' && texte["ordre_origine"] === '1';
         const metadata = {
             name: texteName,
-            session: texte.session,
-            date: texte.date,
-            url_expose_des_motifs: texte.hasExposeDesMotifs
+            session: texte["session"],
+            date: texte["date"],
+            url_expose_des_motifs: hasExposeDesMotifs
                 ? new URL(`${texteName}-expose.html`, SENAT_EXPOSE_DES_MOTIFS_BASE_URL)
                 : undefined,
             url_xml: new URL(`${texteName}.akn.xml`, SENAT_TEXTE_XML_BASE_URL),
             url_html: new URL(`${texteName}.html`, SENAT_TEXTE_BASE_URL),
             url_pdf: new URL(`${texteName}.pdf`, SENAT_TEXTE_BASE_URL),
         };
+        fs.outputJSONSync(path.join(texteDir, `${texteName}.json`), texte, {
+            spaces: 2,
+        });
         fs.outputJSONSync(path.join(texteDir, DOCUMENT_METADATA_FILE), metadata, {
             spaces: 2,
         });
@@ -251,17 +259,20 @@ async function convertTexteUrls(dataDir, options) {
         }
     }
 }
-async function convertRapportUrls(dataDir, options) {
+async function convertRapports(dataDir, options) {
     const originalRapportsDir = path.join(dataDir, RAPPORT_FOLDER, DATA_ORIGINAL_FOLDER);
     if (!options["silent"]) {
         console.log(`Converting database rapports data into files…`);
     }
-    for await (const rapport of findSenatRapportUrls()) {
-        const session = rapport.session ?? UNDEFINED_SESSION;
+    for await (const rapport of findAllRapports()) {
+        const session = rapport["session"] ?? UNDEFINED_SESSION;
         if (options["fromSession"] && session < options["fromSession"]) {
             continue;
         }
-        const parsedRapportUrl = path.parse(rapport.url);
+        if (!rapport["url"]) {
+            continue;
+        }
+        const parsedRapportUrl = path.parse(rapport["url"]);
         const rapportName = parsedRapportUrl.name;
         const rapportDir = path.join(originalRapportsDir, `${session}`, rapportName);
         const rapportHtmlUrlBase = `${rapportName}_mono.html`;
@@ -276,11 +287,14 @@ async function convertRapportUrls(dataDir, options) {
         });
         const metadata = {
             name: rapportName,
-            session: rapport.session,
-            date: rapport.date,
+            session: rapport["session"],
+            date: rapport["date"],
             url_html: new URL(rapportHtmlUrl, SENAT_RAPPORT_BASE_URL),
             url_pdf: new URL(rapportPdfUrl, SENAT_RAPPORT_BASE_URL),
         };
+        fs.outputJSONSync(path.join(rapportDir, `${rapportName}.json`), rapport, {
+            spaces: 2,
+        });
         fs.outputJSONSync(path.join(rapportDir, DOCUMENT_METADATA_FILE), metadata, {
             spaces: 2,
         });

package/lib/scripts/retrieve_documents.d.ts

@@ -1,2 +1,3 @@
-export declare function processTexte(texteMetadata: any, originalTextesDir: string, transformedTextesDir: string, options: any): Promise<void>;
+import { DocumentMetadata } from "../types/texte";
+export declare function processTexte(texteMetadata: DocumentMetadata, originalTextesDir: string, transformedTextesDir: string, options: any): Promise<void>;
 export declare function processRapport(rapportMetadata: any, originalRapportsDir: string, options: any): Promise<void>;

package/lib/scripts/retrieve_documents.js

@@ -3,7 +3,7 @@ import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 import { DateTime } from "luxon";
 import path from "path";
-import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatDossiersLegislatifsRapportUrls, iterLoadSenatDossiersLegislatifsTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, } from "../loaders";
+import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatRapportUrls, iterLoadSenatTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, } from "../loaders";
 import { parseExposeDesMotifs, parseTexte, parseTexteFromFile } from "../parsers/texte";
 import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions";
 import { commonOptions } from "./shared/cli_helpers";
@@ -111,6 +111,8 @@ export async function processTexte(texteMetadata, originalTextesDir, transformed
     for (const format of formats) {
         if (!isOptionEmptyOrHasValue(options.formats, format.type))
             continue;
+        if (!format.url)
+            continue;
         const destPath = path.join(texteDir, `${texteMetadata.name}.${format.type}`);
         const result = await processDocument(format.url.toString(), destPath, texteMetadata.date, options);
         // Specific logic: Parsing (Only applies to XML)
@@ -147,7 +149,7 @@ async function retrieveTextes(dataDir, sessions) {
         parseDocuments: options["parseDocuments"],
     };
     for (const session of sessions) {
-        for (const { item: texteMetadata } of iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session)) {
+        for (const { item: texteMetadata } of iterLoadSenatTexteUrls(dataDir, session)) {
             await processTexte(texteMetadata, originalTextesDir, transformedTextesDir, dlOptions);
         }
     }
@@ -162,7 +164,7 @@ async function retrieveRapports(dataDir, sessions) {
         formats: options["formats"],
     };
     for (const session of sessions) {
-        for (const { item: rapportMetadata } of iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session)) {
+        for (const { item: rapportMetadata } of iterLoadSenatRapportUrls(dataDir, session)) {
             await processRapport(rapportMetadata, originalRapportsDir, dlOptions);
         }
     }

package/lib/scripts/test_iter_load.js

@@ -1,18 +1,12 @@
-import { iterLoadSenatAmendements, iterLoadSenatDossiersLegislatifs } from "../loaders";
+import { iterLoadSenatRapports } from "../loaders";
 import commandLineArgs from "command-line-args";
 import { dataDirDefaultOption } from "./shared/cli_helpers";
 const optionsDefinitions = [dataDirDefaultOption];
 const options = commandLineArgs(optionsDefinitions);
 const session = 2024;
 const sinceCommit = undefined;
-for (const { item: amendement, filePathFromDataset } of iterLoadSenatAmendements(options["dataDir"], session, {
-    log: true,
+for (const { item: rapport } of iterLoadSenatRapports(options["dataDir"], session, {
     sinceCommit: sinceCommit,
 })) {
-    console.log(amendement["numero"]);
-}
-for (const { item: dossierLegislatif } of iterLoadSenatDossiersLegislatifs(options["dataDir"], session, {
-    sinceCommit: sinceCommit,
-})) {
-    console.log(dossierLegislatif["numero"]);
+    console.log(rapport);
 }

package/lib/types/texte.d.ts

@@ -13,6 +13,15 @@ export declare enum DivisionType {
     division = 12
 }
 export type DivisionTag = keyof typeof DivisionType;
+export interface DocumentMetadata {
+    name: string;
+    session: number | null | undefined;
+    date?: string | null;
+    url_expose_des_motifs?: URL;
+    url_xml?: URL;
+    url_html: URL;
+    url_pdf: URL;
+}
 export interface FlatTexte {
     titre: string | null;
     titreCourt: string | null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.20.29",
+  "version": "2.20.31",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",