@tricoteuses/senat 1.1.1 → 1.3.1

package/lib/loaders.mjs

@@ -0,0 +1,158 @@
+import fs from "fs";
+import path from "path";
+import legislatures from "./legislatures.json";
+import { datasets } from "./datasets";
+import { UNDEFINED_SESSION } from "./scripts/datautil";
+export { EnabledDatasets } from "./datasets";
+export const DOSLEG_DOSSIERS_FOLDER = "dossiers";
+export const SENS_CIRCONSCRIPTIONS_FOLDER = "circonscriptions";
+export const SENS_ORGANISMES_FOLDER = "organismes";
+export const SENS_SENATEURS_FOLDER = "senateurs";
+export const TEXTE_FOLDER = "leg";
+export const TEXTE_ORIGINAL_FOLDER = "original";
+export const TEXTE_TRANSFORMED_FOLDER = "transformed";
+export const DOCUMENT_METADATA_FILE = "metadata.json";
+export const RAPPORT_FOLDER = "rap";
+export function* iterFilePaths(dirPath) {
+    if (dirPath && fs.existsSync(dirPath)) {
+        const files = fs.readdirSync(dirPath, { withFileTypes: true, recursive: true });
+        for (const file of files) {
+            if (file.isFile()) {
+                yield path.join(file.parentPath, file.name);
+            }
+        }
+    }
+}
+function* iterLoadSenatItems(dataDir, dataset, legislatureOrSession, subDir, { log = false } = {}) {
+    let itemsDir = path.join(dataDir, dataset.database);
+    if (subDir) {
+        itemsDir = path.join(itemsDir, subDir);
+    }
+    if (legislatureOrSession) {
+        itemsDir = path.join(itemsDir, String(legislatureOrSession));
+    }
+    for (const filePath of iterFilePaths(itemsDir)) {
+        if (log) {
+            console.log(`Loading file: ${filePath}…`);
+        }
+        const itemJson = fs.readFileSync(filePath, { encoding: "utf8" });
+        const item = JSON.parse(itemJson);
+        const filePathFromDataset = filePath.substring(filePath.indexOf(dataset.database) + dataset.database.length);
+        yield {
+            item,
+            filePathFromDataset,
+            legislature: legislatureOrSession,
+        };
+    }
+}
+export function* iterLoadSenatAmendements(dataDir, session, options = {}) {
+    for (const amendementItem of iterLoadSenatItems(dataDir, datasets.ameli, session, undefined, options)) {
+        yield amendementItem;
+    }
+}
+export function* iterLoadSenatDossiersLegislatifs(dataDir, session, options = {}) {
+    for (const dossierLegislatifItem of iterLoadSenatItems(dataDir, datasets.dosleg, session, DOSLEG_DOSSIERS_FOLDER, options)) {
+        yield dossierLegislatifItem;
+    }
+}
+export function* iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session) {
+    let itemsDir = path.join(dataDir, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER);
+    if (session) {
+        itemsDir = path.join(itemsDir, session.toString());
+    }
+    for (const filePath of iterFilePaths(itemsDir)) {
+        const parsedFilePath = path.parse(filePath);
+        if (parsedFilePath.base === DOCUMENT_METADATA_FILE) {
+            const itemJson = fs.readFileSync(filePath, { encoding: "utf8" });
+            const item = JSON.parse(itemJson);
+            yield {
+                item,
+            };
+        }
+    }
+}
+export function* iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session) {
+    let itemsDir = path.join(dataDir, RAPPORT_FOLDER);
+    if (session) {
+        itemsDir = path.join(itemsDir, session.toString());
+    }
+    for (const filePath of iterFilePaths(itemsDir)) {
+        const parsedFilePath = path.parse(filePath);
+        if (parsedFilePath.base === DOCUMENT_METADATA_FILE) {
+            const itemJson = fs.readFileSync(filePath, { encoding: "utf8" });
+            const item = JSON.parse(itemJson);
+            yield {
+                item,
+            };
+        }
+    }
+}
+export function* iterLoadSenatDossiersLegislatifsTextes(dataDir, session, options = {}) {
+    for (const { item: dossierLegislatif } of iterLoadSenatDossiersLegislatifs(dataDir, session, options)) {
+        for (const lecture of dossierLegislatif.lectures) {
+            const lecturesSenat = lecture.lectures_assemblee
+                .filter(lectureAssemblee => lectureAssemblee.assemblee === "Sénat");
+            for (const lectureSenat of lecturesSenat) {
+                for (const texte of lectureSenat.textes) {
+                    const enrichedTexte = {
+                        signet_dossier: dossierLegislatif.signet,
+                        url_dossier_senat: dossierLegislatif.url,
+                        url_dossier_assemblee_nationale: dossierLegislatif.url_dossier_assemblee_nationale,
+                        type_lecture: lecture.type_lecture,
+                        libelle_lecture: lecture.libelle,
+                        libelle_organisme: lectureSenat.libelle_organisme,
+                        ...texte,
+                    };
+                    const texteItem = {
+                        item: enrichedTexte,
+                    };
+                    if (texte.url) {
+                        const texteName = path.parse(texte.url).name;
+                        texteItem.filePathFromDataset
+                            = path.join(`${texte.session ?? UNDEFINED_SESSION}`, texteName, `${texteName}.pdf`);
+                    }
+                    yield texteItem;
+                }
+            }
+        }
+    }
+}
+export function loadSenatTexteContent(dataDir, textePathFromDataset) {
+    const parsedTextePath = path.parse(textePathFromDataset);
+    const jsonTexteName = `${parsedTextePath.name}.json`;
+    const fullTextePath = path.join(dataDir, TEXTE_FOLDER, TEXTE_TRANSFORMED_FOLDER, parsedTextePath.dir, jsonTexteName);
+    if (!fs.existsSync(fullTextePath)) {
+        return { item: null };
+    }
+    const texteJson = fs.readFileSync(fullTextePath, { encoding: "utf8" });
+    return { item: JSON.parse(texteJson) };
+}
+export function* iterLoadSenatOrganismes(dataDir, options = {}) {
+    for (const organismeItem of iterLoadSenatItems(dataDir, datasets.sens, undefined, SENS_ORGANISMES_FOLDER, options)) {
+        yield organismeItem;
+    }
+}
+export function* iterLoadSenatSenateurs(dataDir, legislature, options = {}) {
+    const dateDebutLegislatureStr = legislatures
+        .find(legislatureInfo => legislatureInfo.numero === legislature)
+        ?.date_debut;
+    const dateDebutLegislature = new Date(dateDebutLegislatureStr);
+    for (const senateurItem of iterLoadSenatItems(dataDir, datasets.sens, undefined, SENS_SENATEURS_FOLDER, options)) {
+        const dateFinMandatSenateur = senateurItem.item.mandats_senateur[0]?.date_fin ?
+            new Date(senateurItem.item.mandats_senateur[0]?.date_fin) : null;
+        if (dateFinMandatSenateur && dateFinMandatSenateur < dateDebutLegislature) {
+            continue;
+        }
+        yield senateurItem;
+    }
+}
+export function* iterLoadSenatCirconscriptions(dataDir, options = {}) {
+    for (const circonscriptionItem of iterLoadSenatItems(dataDir, datasets.sens, undefined, SENS_CIRCONSCRIPTIONS_FOLDER, options)) {
+        yield circonscriptionItem;
+    }
+}
+export function* iterLoadSenatQuestions(dataDir, legislature, options = {}) {
+    for (const questionItem of iterLoadSenatItems(dataDir, datasets.questions, legislature, undefined, options)) {
+        yield questionItem;
+    }
+}

package/lib/loaders.ts

@@ -0,0 +1,271 @@
+import fs from "fs"
+import path from "path"
+
+import legislatures from "./legislatures.json"
+
+import { Dataset, datasets } from "./datasets"
+import { AmendementResult } from "./model/ameli"
+import { DossierLegislatifResult } from "./model/dosleg"
+import { QuestionResult } from "./model/questions"
+import { CirconscriptionResult, OrganismeResult, SenateurResult } from "./model/sens"
+import { UNDEFINED_SESSION } from "./scripts/datautil"
+import { FlatTexte } from "./types/texte"
+
+export { EnabledDatasets } from "./datasets"
+
+export const DOSLEG_DOSSIERS_FOLDER = "dossiers"
+export const SENS_CIRCONSCRIPTIONS_FOLDER = "circonscriptions"
+export const SENS_ORGANISMES_FOLDER = "organismes"
+export const SENS_SENATEURS_FOLDER = "senateurs"
+export const TEXTE_FOLDER = "leg"
+export const TEXTE_ORIGINAL_FOLDER = "original"
+export const TEXTE_TRANSFORMED_FOLDER = "transformed"
+export const DOCUMENT_METADATA_FILE = "metadata.json"
+export const RAPPORT_FOLDER = "rap"
+
+type IterItem<T> = {
+  item: T
+  filePathFromDataset?: string,
+  legislature?: number
+}
+
+export interface TexteMetadata {
+  name: string,
+  session: string | null | undefined,
+  url_expose_des_motifs?: URL,
+  url_xml: URL,
+  url_html: URL,
+  url_pdf: URL,
+}
+
+export interface RapportMetadata {
+  name: string,
+  session: string | null | undefined,
+  url_html: URL,
+  url_pdf: URL,
+}
+
+export interface DossierLegislatifTexteResult {
+  signet_dossier: string,
+  url_dossier_senat: string,
+  url_dossier_assemblee_nationale: string | null,
+  type_lecture: string
+  libelle_lecture: string
+  libelle_organisme: string | null
+  numero: string | null
+  url: string
+  origine_texte: string
+  type_texte: string
+  date: string
+  session: string | null
+  auteurs: {
+    prenom: string | null
+    nom_usuel: string
+    matricule: string | null
+  }[]
+}
+
+export function * iterFilePaths (dirPath: string): Generator<string> {
+  if (dirPath && fs.existsSync(dirPath)) {
+    const files = fs.readdirSync(dirPath, { withFileTypes: true, recursive: true })
+    for (const file of files) {
+      if (file.isFile()) {
+        yield path.join(file.parentPath, file.name)
+      }
+    }
+  }
+}
+
+function * iterLoadSenatItems<T> (
+  dataDir: string,
+  dataset: Dataset,
+  legislatureOrSession?: number,
+  subDir?: string,
+  { log = false } = {},
+): Generator<IterItem<T>> {
+  let itemsDir = path.join(dataDir, dataset.database)
+  if (subDir) {
+    itemsDir = path.join(itemsDir, subDir)
+  }
+  if (legislatureOrSession) {
+    itemsDir = path.join(itemsDir, String(legislatureOrSession))
+  }
+  for (const filePath of iterFilePaths(itemsDir)) {
+    if (log) {
+      console.log(`Loading file: ${filePath}…`)
+    }
+
+    const itemJson = fs.readFileSync(filePath, { encoding: "utf8" })
+    const item: T = JSON.parse(itemJson)
+
+    const filePathFromDataset = filePath.substring(filePath.indexOf(dataset.database) + dataset.database.length)
+
+    yield {
+      item,
+      filePathFromDataset,
+      legislature: legislatureOrSession,
+    }
+  }
+}
+
+export function * iterLoadSenatAmendements (
+  dataDir: string,
+  session: number | undefined,
+  options = {},
+): Generator<IterItem<AmendementResult>> {
+  for (const amendementItem
+    of iterLoadSenatItems<AmendementResult>(dataDir, datasets.ameli, session, undefined, options)
+    ) {
+    yield amendementItem
+  }
+}
+
+export function * iterLoadSenatDossiersLegislatifs (
+  dataDir: string,
+  session: number | undefined,
+  options = {},
+): Generator<IterItem<DossierLegislatifResult>> {
+  for (const dossierLegislatifItem
+    of iterLoadSenatItems<DossierLegislatifResult>(dataDir, datasets.dosleg, session, DOSLEG_DOSSIERS_FOLDER, options)
+    ) {
+    yield dossierLegislatifItem
+  }
+}
+
+export function * iterLoadSenatDossiersLegislatifsTexteUrls (
+  dataDir: string,
+  session: number | undefined,
+): Generator<IterItem<TexteMetadata>> {
+  let itemsDir = path.join(dataDir, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER)
+  if (session) {
+    itemsDir = path.join(itemsDir, session.toString())
+  }
+  for (const filePath of iterFilePaths(itemsDir)) {
+    const parsedFilePath = path.parse(filePath)
+    if (parsedFilePath.base === DOCUMENT_METADATA_FILE) {
+      const itemJson = fs.readFileSync(filePath, { encoding: "utf8" })
+      const item = JSON.parse(itemJson)
+      yield {
+        item,
+      }
+    }
+  }
+}
+
+export function * iterLoadSenatDossiersLegislatifsRapportUrls (
+  dataDir: string,
+  session: number | undefined,
+): Generator<IterItem<RapportMetadata>> {
+  let itemsDir = path.join(dataDir, RAPPORT_FOLDER)
+  if (session) {
+    itemsDir = path.join(itemsDir, session.toString())
+  }
+  for (const filePath of iterFilePaths(itemsDir)) {
+    const parsedFilePath = path.parse(filePath)
+    if (parsedFilePath.base === DOCUMENT_METADATA_FILE) {
+      const itemJson = fs.readFileSync(filePath, { encoding: "utf8" })
+      const item = JSON.parse(itemJson)
+      yield {
+        item,
+      }
+    }
+  }
+}
+
+export function * iterLoadSenatDossiersLegislatifsTextes (
+  dataDir: string,
+  session: number | undefined,
+  options = {},
+): Generator<IterItem<DossierLegislatifTexteResult>> {
+  for (const { item: dossierLegislatif } of iterLoadSenatDossiersLegislatifs(dataDir, session, options)) {
+    for (const lecture of dossierLegislatif.lectures) {
+      const lecturesSenat = lecture.lectures_assemblee
+        .filter(lectureAssemblee => lectureAssemblee.assemblee === "Sénat")
+      for (const lectureSenat of lecturesSenat) {
+        for (const texte of lectureSenat.textes) {
+          const enrichedTexte = {
+            signet_dossier: dossierLegislatif.signet,
+            url_dossier_senat: dossierLegislatif.url,
+            url_dossier_assemblee_nationale: dossierLegislatif.url_dossier_assemblee_nationale,
+            type_lecture: lecture.type_lecture,
+            libelle_lecture: lecture.libelle,
+            libelle_organisme: lectureSenat.libelle_organisme,
+            ...texte,
+          }
+          const texteItem: IterItem<DossierLegislatifTexteResult> = {
+            item: enrichedTexte,
+          }
+          if (texte.url) {
+            const texteName = path.parse(texte.url).name
+            texteItem.filePathFromDataset
+              = path.join(`${texte.session ?? UNDEFINED_SESSION}`, texteName, `${texteName}.pdf`)
+          }
+          yield texteItem
+        }
+      }
+    }
+  }
+}
+
+export function loadSenatTexteContent (
+  dataDir: string,
+  textePathFromDataset: string,
+): IterItem<FlatTexte | null> {
+  const parsedTextePath = path.parse(textePathFromDataset)
+  const jsonTexteName = `${parsedTextePath.name}.json`
+  const fullTextePath = path.join(dataDir, TEXTE_FOLDER, TEXTE_TRANSFORMED_FOLDER, parsedTextePath.dir, jsonTexteName)
+  if (!fs.existsSync(fullTextePath)) {
+    return { item: null }
+  }
+  const texteJson = fs.readFileSync(fullTextePath, { encoding: "utf8" })
+  return { item: JSON.parse(texteJson) }
+}
+
+export function * iterLoadSenatOrganismes (
+  dataDir: string,
+  options = {},
+): Generator<IterItem<OrganismeResult>> {
+  for (const organismeItem of iterLoadSenatItems<OrganismeResult>(dataDir, datasets.sens, undefined, SENS_ORGANISMES_FOLDER, options)) {
+    yield organismeItem
+  }
+}
+
+export function * iterLoadSenatSenateurs (
+  dataDir: string,
+  legislature: number,
+  options = {},
+): Generator<IterItem<SenateurResult>> {
+  const dateDebutLegislatureStr = legislatures
+    .find(legislatureInfo => legislatureInfo.numero === legislature)
+    ?.date_debut
+  const dateDebutLegislature = new Date(dateDebutLegislatureStr!)
+  for (const senateurItem of iterLoadSenatItems<SenateurResult>(dataDir, datasets.sens, undefined, SENS_SENATEURS_FOLDER, options)) {
+    const dateFinMandatSenateur = senateurItem.item.mandats_senateur[0]?.date_fin ?
+      new Date(senateurItem.item.mandats_senateur[0]?.date_fin) : null
+    if (dateFinMandatSenateur && dateFinMandatSenateur < dateDebutLegislature) {
+      continue
+    }
+    yield senateurItem
+  }
+}
+
+export function * iterLoadSenatCirconscriptions (
+  dataDir: string,
+  options = {},
+): Generator<IterItem<CirconscriptionResult>> {
+  for (const circonscriptionItem of iterLoadSenatItems<CirconscriptionResult>(dataDir, datasets.sens, undefined, SENS_CIRCONSCRIPTIONS_FOLDER, options)) {
+    yield circonscriptionItem
+  }
+}
+
+export function * iterLoadSenatQuestions (
+  dataDir: string,
+  legislature: number,
+  options = {},
+): Generator<IterItem<QuestionResult>> {
+  for (const questionItem
+    of iterLoadSenatItems<QuestionResult>(dataDir, datasets.questions, legislature, undefined, options)
+    ) {
+    yield questionItem
+  }
+}

package/lib/model/ameli.d.ts

@@ -1,4 +1,105 @@
-import { Ses, Sub, TxtAmeli } from '../types/ameli';
-export declare const getSessFromAnns: (ids: string[]) => Promise<Ses[]>;
-export declare const getSubsFromTxtids: (ids: string[]) => Promise<Sub[]>;
-export declare const getTxtsAmeliBySesannNumJoins: (ids: string[]) => Promise<TxtAmeli[]>;
+import { InferResult } from "kysely";
+declare const findAllQuery: import("kysely").SelectQueryBuilder<{
+    sub: import("kysely").Nullable<import("../raw_types/ameli").Sub>;
+    amdsen: import("../raw_types/ameli").Amdsen;
+    amd: import("../raw_types/ameli").Amd;
+    avicom: import("kysely").Nullable<import("../raw_types/ameli").Avicom>;
+    avigvt: import("kysely").Nullable<import("../raw_types/ameli").Avigvt>;
+    cab: import("kysely").Nullable<import("../raw_types/ameli").Cab>;
+    com_ameli: import("kysely").Nullable<import("../raw_types/ameli").ComAmeli>;
+    ent: import("../raw_types/ameli").Ent;
+    etatxt: import("../raw_types/ameli").Etatxt;
+    fbu: import("../raw_types/ameli").Fbu;
+    grppol_ameli: import("kysely").Nullable<import("../raw_types/ameli").GrppolAmeli>;
+    gvt: import("../raw_types/ameli").Gvt;
+    intora: import("../raw_types/ameli").Intora;
+    irr: import("kysely").Nullable<import("../raw_types/ameli").Irr>;
+    lec_ameli: import("kysely").Nullable<import("../raw_types/ameli").LecAmeli>;
+    mot: import("../raw_types/ameli").Mot;
+    nat: import("kysely").Nullable<import("../raw_types/ameli").Nat>;
+    orarol: import("../raw_types/ameli").Orarol;
+    sai: import("../raw_types/ameli").Sai;
+    saisen: import("../raw_types/ameli").Saisen;
+    sea: import("../raw_types/ameli").Sea;
+    sen_ameli: import("../raw_types/ameli").SenAmeli;
+    ses: import("kysely").Nullable<import("../raw_types/ameli").Ses>;
+    sor: import("kysely").Nullable<import("../raw_types/ameli").Sor>;
+    txt_ameli: import("kysely").Nullable<import("../raw_types/ameli").TxtAmeli>;
+    typrect: import("../raw_types/ameli").Typrect;
+    typses: import("../raw_types/ameli").Typses;
+    typsub: import("kysely").Nullable<import("../raw_types/ameli").Typsub>;
+    w_nivrec: import("../raw_types/ameli").WNivrec;
+}, "sub" | "amd" | "avicom" | "avigvt" | "cab" | "com_ameli" | "grppol_ameli" | "irr" | "lec_ameli" | "nat" | "ses" | "sor" | "txt_ameli" | "typsub", {
+    session: string | null;
+    signet_dossier_legislatif: string | null;
+    nature_texte: string | null;
+    numero_texte: number | null;
+    intitule_texte: string | null;
+    lecture: string | null;
+    nature: string;
+    id: number;
+    parent_id: number | null;
+    numero: string | null;
+    numero_absolu: string | null;
+    ordre: string | null;
+    subdivision_libelle: string | null;
+    subdivision_libelle_court: string | null;
+    subdivision_type: string | null;
+    alinea: number | null;
+    date_depot: string;
+    dispositif: string | null;
+    objet: string | null;
+    etat_id: number;
+    avis_commission: string | null;
+    avis_gouvernement: string | null;
+    sort: string | null;
+    revision: string;
+    url: string;
+    au_nom_de_groupe_politique: string | null;
+    au_nom_de_commission: string | null;
+    auteur_est_gouvernement: boolean;
+} & {
+    auteurs: {
+        prenom: string | null;
+        nom: string | null;
+        matricule: string | null;
+    }[];
+}>;
+export type AmendementResult = InferResult<typeof findAllQuery>[0];
+export declare function findAll(): AsyncIterableIterator<{
+    session: string | null;
+    signet_dossier_legislatif: string | null;
+    nature_texte: string | null;
+    numero_texte: number | null;
+    intitule_texte: string | null;
+    lecture: string | null;
+    nature: string;
+    id: number;
+    parent_id: number | null;
+    numero: string | null;
+    numero_absolu: string | null;
+    ordre: string | null;
+    subdivision_libelle: string | null;
+    subdivision_libelle_court: string | null;
+    subdivision_type: string | null;
+    alinea: number | null;
+    date_depot: string;
+    dispositif: string | null;
+    objet: string | null;
+    etat_id: number;
+    avis_commission: string | null;
+    avis_gouvernement: string | null;
+    sort: string | null;
+    revision: string;
+    url: string;
+    au_nom_de_groupe_politique: string | null;
+    au_nom_de_commission: string | null;
+    auteur_est_gouvernement: boolean;
+} & {
+    auteurs: {
+        prenom: string | null;
+        nom: string | null;
+        matricule: string | null;
+    }[];
+}>;
+export {};