@tricoteuses/senat 2.7.1 → 2.8.4

package/lib/loaders.ts

@@ -0,0 +1,173 @@
+import fs from "fs"
+import path from "path"
+
+import { Dataset, datasets } from "./datasets"
+import { DossierLegislatifResult } from "./model/dosleg"
+import { QuestionResult } from "./model/questions"
+import { CirconscriptionResult, SenateurResult } from "./model/sens"
+import { UNDEFINED_SESSION } from "./scripts/datautil"
+import { FlatTexte } from "./types/texte"
+
+export { EnabledDatasets } from "./datasets"
+
+type IterItem<T> = {
+  item: T
+  filePathFromDataset?: string,
+  legislature?: number
+}
+
+export interface DossierLegislatifTexteResult {
+  signet_dossier: string,
+  url_dossier_senat: string,
+  url_dossier_assemblee_nationale: string | null,
+  type_lecture: string
+  libelle_lecture: string
+  libelle_organisme: string | null
+  numero: string | null
+  url: string
+  origine_texte: string
+  type_texte: string
+  date: Date
+  session: string | null
+  auteurs: {
+    prenom: string | null
+    nom_usuel: string
+    matricule: string | null
+  }[]
+}
+
+export function * iterFilePaths (dirPath: string): Generator<string> {
+  if (dirPath && fs.existsSync(dirPath)) {
+    const files = fs.readdirSync(dirPath, { withFileTypes: true, recursive: true })
+    for (const file of files) {
+      if (file.isFile()) {
+        yield path.join(file.parentPath, file.name)
+      }
+    }
+  }
+}
+
+function * iterLoadSenatItems<T> (
+  dataDir: string,
+  dataset: Dataset,
+  legislatureOrSession?: number,
+  subDir?: string,
+  { log = false } = {},
+): Generator<IterItem<T>> {
+  let itemsDir = path.join(dataDir, dataset.database)
+  if (legislatureOrSession) {
+    itemsDir = path.join(itemsDir, String(legislatureOrSession))
+  }
+  if (subDir) {
+    itemsDir = path.join(itemsDir, subDir)
+  }
+  for (const filePath of iterFilePaths(itemsDir)) {
+    if (log) {
+      console.log(`Loading file: ${filePath}…`)
+    }
+
+    const itemJson = fs.readFileSync(filePath, { encoding: "utf8" })
+    const item: T = JSON.parse(itemJson)
+
+    const filePathFromDataset = filePath.substring(filePath.indexOf(dataset.database) + dataset.database.length)
+
+    yield {
+      item,
+      filePathFromDataset,
+      legislature: legislatureOrSession,
+    }
+  }
+}
+
+export function * iterLoadSenatDossiersLegislatifs (
+  dataDir: string,
+  session: number | undefined,
+  options = {},
+): Generator<IterItem<DossierLegislatifResult>> {
+  for (const dossierLegislatifItem
+    of iterLoadSenatItems<DossierLegislatifResult>(dataDir, datasets.dosleg, session, undefined, options)
+    ) {
+    yield dossierLegislatifItem
+  }
+}
+
+export function * iterLoadSenatDossiersLegislatifsTextes (
+  dataDir: string,
+  session: number | undefined,
+  options = {},
+): Generator<IterItem<DossierLegislatifTexteResult>> {
+  for (const { item: dossierLegislatif } of iterLoadSenatDossiersLegislatifs(dataDir, session, options)) {
+    for (const lecture of dossierLegislatif.lectures) {
+      const lecturesSenat = lecture.lectures_assemblee
+        .filter(lectureAssemblee => lectureAssemblee.assemblee === "Sénat")
+      for (const lectureSenat of lecturesSenat) {
+        for (const texte of lectureSenat.textes) {
+          const enrichedTexte = {
+            signet_dossier: dossierLegislatif.signet,
+            url_dossier_senat: dossierLegislatif.url,
+            url_dossier_assemblee_nationale: dossierLegislatif.url_dossier_assemblee_nationale,
+            type_lecture: lecture.type_lecture,
+            libelle_lecture: lecture.libelle,
+            libelle_organisme: lectureSenat.libelle_organisme,
+            ...texte,
+          }
+          const texteItem: IterItem<DossierLegislatifTexteResult> = {
+            item: enrichedTexte,
+          }
+          if (texte.url) {
+            const texteName = path.parse(texte.url).name
+            texteItem.filePathFromDataset
+              = path.join(`${texte.session ?? UNDEFINED_SESSION}`, texteName, `${texteName}.pdf`)
+          }
+          yield texteItem
+        }
+      }
+    }
+  }
+}
+
+export function loadSenatTexteContent (
+  dataDir: string,
+  textePathFromDataset: string,
+): IterItem<FlatTexte | null> {
+  const parsedTextePath = path.parse(textePathFromDataset)
+  const jsonTexteName = `${parsedTextePath.name}.akn.json`
+  const fullTextePath = path.join(dataDir, "leg", "transformed", parsedTextePath.dir, jsonTexteName)
+  if (!fs.existsSync(fullTextePath)) {
+    return { item: null }
+  }
+  const texteJson = fs.readFileSync(fullTextePath, { encoding: "utf8" })
+  return { item: JSON.parse(texteJson) }
+}
+
+export function * iterLoadSenatSenateurs (
+  dataDir: string,
+  options = {},
+): Generator<IterItem<SenateurResult>> {
+  for (const senateurItem of iterLoadSenatItems<SenateurResult>(dataDir, datasets.sens, undefined, "sens", options)) {
+    yield senateurItem
+  }
+}
+
+export function * iterLoadSenatCirconscriptions (
+  dataDir: string,
+  options = {},
+): Generator<IterItem<CirconscriptionResult>> {
+  for (const circonscriptionItem
+    of iterLoadSenatItems<CirconscriptionResult>(dataDir, datasets.sens, undefined, "circonscriptions", options)
+    ) {
+    yield circonscriptionItem
+  }
+}
+
+export function * iterLoadSenatQuestions (
+  dataDir: string,
+  legislature: number,
+  options = {},
+): Generator<IterItem<QuestionResult>> {
+  for (const questionItem
+    of iterLoadSenatItems<QuestionResult>(dataDir, datasets.questions, legislature, undefined, options)
+    ) {
+    yield questionItem
+  }
+}

package/lib/model/agenda.js

@@ -3,6 +3,9 @@ import { DateTime } from "luxon";
 import path from "path";
 import { ID_DATE_FORMAT, STANDARD_DATE_FORMAT } from "../scripts/datautil";
 const FR_TZ = "Europe/Paris";
+function eventIsSeance(eventElement) {
+    return eventElement.classList.contains("evt-seance");
+}
 function getEventType(eventClasses) {
     const typeClass = [...eventClasses]
         .find(className => className.startsWith("evt-"))
@@ -27,27 +30,25 @@ function getUrlDossierSenat(lienElements) {
     return urlElement ? urlElement.getAttribute("href") : null;
 }
 function getQuantieme(eventElement, seancesElements) {
-    if (eventElement.classList.contains("evt-seance")) {
-        const seanceIndex = seancesElements.indexOf(eventElement);
-        if (seancesElements.length === 1 && seanceIndex === 0) {
-            return "Unique";
-        }
-        else {
-            switch (seanceIndex) {
-                case 0:
-                    return "Première";
-                case 1:
-                    return "Deuxième";
-                case 2:
-                    return "Troisième";
-                case 3:
-                    return "Quatrième";
-                case 4:
-                    return "Cinquième";
-            }
+    const seanceIndex = seancesElements.indexOf(eventElement);
+    if (seancesElements.length === 1 && seanceIndex === 0) {
+        return "Unique";
+    }
+    else {
+        switch (seanceIndex) {
+            case 0:
+                return "Première";
+            case 1:
+                return "Deuxième";
+            case 2:
+                return "Troisième";
+            case 3:
+                return "Quatrième";
+            case 4:
+                return "Cinquième";
         }
     }
-    return null;
+    return "Non défini";
 }
 /**
  * Normalize time string to become a simple start time ("H'h'mm") or a duration ("'de 'H'h'mm' à 'H'h'mm").
@@ -85,8 +86,7 @@ function getStartAndEndTimes(timeStr) {
 function transformAgenda(document, fileName) {
     const agendaEvents = [];
     const eventElements = document.querySelectorAll(".evt");
-    const seanceElements = Array.from(eventElements)
-        .filter(eventElement => eventElement.classList.contains("evt-seance"));
+    const seanceElements = Array.from(eventElements).filter(eventElement => eventIsSeance(eventElement));
     for (const eventElement of eventElements) {
         const id = eventElement.previousElementSibling?.getAttribute("name") || null;
         if (!id) {
@@ -118,7 +118,7 @@ function transformAgenda(document, fileName) {
             lieu,
             captationVideo: videoElement !== null,
             urlDossierSenat: urlDossierSenat,
-            quantieme: getQuantieme(eventElement, seanceElements),
+            quantieme: eventIsSeance(eventElement) ? getQuantieme(eventElement, seanceElements) : null,
         });
     }
     return agendaEvents;

package/lib/model/ameli.mjs

@@ -0,0 +1,57 @@
+import { sesFieldsToParseInt, sesFieldsToTrim, subFieldsToParseInt, subFieldsToTrim, txtAmeliFieldsToTrim, } from "../types/ameli";
+import { dbByName } from "../databases";
+import { parseIntFields, trimFieldsRight } from "../fields";
+export const getSessFromAnns = async (ids) => {
+    if (ids.length === 0) {
+        return [];
+    }
+    return (await dbByName.ameli.any(`
+      SELECT *
+      FROM ses
+      WHERE ann IN ($<ids:list>)
+    `, {
+        ids,
+    })).map((ses) => parseIntFields(sesFieldsToParseInt, trimFieldsRight(sesFieldsToTrim, ses)));
+};
+export const getSubsFromTxtids = async (ids) => {
+    if (ids.length === 0) {
+        return [];
+    }
+    return (await dbByName.ameli.any(`
+      SELECT *
+      FROM sub
+      WHERE txtid IN($<ids:list>)
+    `, {
+        ids,
+    })).map((sub) => parseIntFields(subFieldsToParseInt, trimFieldsRight(subFieldsToTrim, sub)));
+};
+export const getTxtsAmeliBySesannNumJoins = async (ids) => {
+    if (ids.length === 0) {
+        return [];
+    }
+    const txtsAmelis = [];
+    for (const id of ids) {
+        const idSplit = id.split(" ");
+        const sesann = parseInt(idSplit[0]);
+        const num = idSplit[1];
+        const txtAmeli = trimFieldsRight(txtAmeliFieldsToTrim, await dbByName.ameli.oneOrNone(`
+          SELECT *
+          FROM txt_ameli
+          WHERE
+            sesinsid in (
+              SELECT id
+              FROM ses
+              WHERE ann = $<sesann>
+            ) AND
+            num = $<num>
+          LIMIT 1
+        `, {
+            num,
+            sesann,
+        }));
+        if (txtAmeli !== null) {
+            txtsAmelis.push(txtAmeli);
+        }
+    }
+    return txtsAmelis;
+};

package/lib/model/ameli.ts

@@ -0,0 +1,86 @@
+import {
+  Ses, sesFieldsToParseInt, sesFieldsToTrim,
+  Sub, subFieldsToParseInt, subFieldsToTrim,
+  TxtAmeli, txtAmeliFieldsToTrim,
+} from "../types/ameli"
+import { dbByName } from "../databases"
+import { parseIntFields, trimFieldsRight } from "../fields"
+
+export const getSessFromAnns = async (ids: string[]): Promise<Ses[]> => {
+  if (ids.length === 0) {
+    return []
+  }
+  return (
+    await dbByName.ameli.any(
+      `
+      SELECT *
+      FROM ses
+      WHERE ann IN ($<ids:list>)
+    `,
+      {
+        ids,
+      },
+    )
+  ).map((ses: Ses) =>
+    parseIntFields(sesFieldsToParseInt, trimFieldsRight(sesFieldsToTrim, ses)),
+  )
+}
+
+export const getSubsFromTxtids = async (ids: string[]): Promise<Sub[]> => {
+  if (ids.length === 0) {
+    return []
+  }
+  return (
+    await dbByName.ameli.any(
+      `
+      SELECT *
+      FROM sub
+      WHERE txtid IN($<ids:list>)
+    `,
+      {
+        ids,
+      },
+    )
+  ).map((sub: Sub) =>
+    parseIntFields(subFieldsToParseInt, trimFieldsRight(subFieldsToTrim, sub)),
+  )
+}
+
+export const getTxtsAmeliBySesannNumJoins = async (
+  ids: string[],
+): Promise<TxtAmeli[]> => {
+  if (ids.length === 0) {
+    return []
+  }
+  const txtsAmelis = []
+  for (const id of ids) {
+    const idSplit = id.split(" ")
+    const sesann = parseInt(idSplit[0])
+    const num = idSplit[1]
+    const txtAmeli = trimFieldsRight(
+      txtAmeliFieldsToTrim,
+      await dbByName.ameli.oneOrNone(
+        `
+          SELECT *
+          FROM txt_ameli
+          WHERE
+            sesinsid in (
+              SELECT id
+              FROM ses
+              WHERE ann = $<sesann>
+            ) AND
+            num = $<num>
+          LIMIT 1
+        `,
+        {
+          num,
+          sesann,
+        },
+      ),
+    ) as TxtAmeli
+    if (txtAmeli !== null) {
+      txtsAmelis.push(txtAmeli)
+    }
+  }
+  return txtsAmelis
+}

package/lib/model/debats.mjs

@@ -0,0 +1,43 @@
+import { debatsFieldsToTrim, lecassdebFieldsToTrim, } from "../types/debats";
+import { dbByName } from "../databases";
+import { trimFieldsRight } from "../fields";
+export const getDebats = async (ids) => {
+    if (ids.length === 0) {
+        return [];
+    }
+    return (await dbByName.debats.any(`
+      SELECT *
+      FROM debats
+      WHERE datsea IN ($<ids:list>)
+    `, {
+        ids,
+    })).map((debat) => trimFieldsRight(debatsFieldsToTrim, debat));
+};
+export const getDebatsFromLecassidts = async (ids) => {
+    if (ids.length === 0) {
+        return [];
+    }
+    return (await dbByName.debats.any(`
+      SELECT *
+      FROM debats
+      WHERE datsea IN (
+        SELECT datsea
+        FROM lecassdeb
+        WHERE lecassidt IN ($<ids:list>)
+      )
+    `, {
+        ids,
+    })).map((debat) => trimFieldsRight(debatsFieldsToTrim, debat));
+};
+export const getLecAssDebsFromDatseas = async (ids) => {
+    if (ids.length === 0) {
+        return [];
+    }
+    return (await dbByName.debats.any(`
+      SELECT *
+      FROM lecassdeb
+      WHERE datsea IN ($<ids:list>)
+    `, {
+        ids,
+    })).map((lecassdeb) => trimFieldsRight(lecassdebFieldsToTrim, lecassdeb));
+};

package/lib/model/debats.ts

@@ -0,0 +1,68 @@
+import {
+  Debat, debatsFieldsToTrim,
+  LecAssDeb, lecassdebFieldsToTrim,
+} from "../types/debats"
+import { dbByName } from "../databases"
+import { trimFieldsRight } from "../fields"
+
+export const getDebats = async (ids: string[]): Promise<Debat[]> => {
+  if (ids.length === 0) {
+    return []
+  }
+  return (
+    await dbByName.debats.any(
+      `
+      SELECT *
+      FROM debats
+      WHERE datsea IN ($<ids:list>)
+    `,
+      {
+        ids,
+      },
+    )
+  ).map((debat: Debat) => trimFieldsRight(debatsFieldsToTrim, debat))
+}
+
+export const getDebatsFromLecassidts = async (
+  ids: string[],
+): Promise<Debat[]> => {
+  if (ids.length === 0) {
+    return []
+  }
+  return (
+    await dbByName.debats.any(
+      `
+      SELECT *
+      FROM debats
+      WHERE datsea IN (
+        SELECT datsea
+        FROM lecassdeb
+        WHERE lecassidt IN ($<ids:list>)
+      )
+    `,
+      {
+        ids,
+      },
+    )
+  ).map((debat: Debat) => trimFieldsRight(debatsFieldsToTrim, debat))
+}
+
+export const getLecAssDebsFromDatseas = async (
+  ids: string[],
+): Promise<LecAssDeb[]> => {
+  if (ids.length === 0) {
+    return []
+  }
+  return (
+    await dbByName.debats.any(
+      `
+      SELECT *
+      FROM lecassdeb
+      WHERE datsea IN ($<ids:list>)
+    `,
+      {
+        ids,
+      },
+    )
+  ).map((lecassdeb: LecAssDeb) => trimFieldsRight(lecassdebFieldsToTrim, lecassdeb))
+}

package/lib/model/dosleg.mjs

@@ -0,0 +1,163 @@
+import { sql } from "kysely";
+import { dbDosleg } from "../databases";
+import { jsonArrayFrom } from "kysely/helpers/postgres";
+function datesSeances(lectureAssembleeId) {
+    return jsonArrayFrom(dbDosleg.selectFrom("date_seance")
+        .where("date_seance.lecidt", "=", lectureAssembleeId)
+        .select([
+        "date_seance.date_s as date",
+    ]));
+}
+function organismesRapport(rapportId) {
+    return jsonArrayFrom(dbDosleg.selectFrom("org")
+        .leftJoin("raporg", "raporg.orgcod", "org.orgcod")
+        .where("raporg.rapcod", "=", rapportId)
+        .select([
+        "org.orgnom as libelle",
+    ]));
+}
+function auteursRapport(rapportId) {
+    return jsonArrayFrom(dbDosleg.selectFrom("auteur")
+        .leftJoin("ecr", "ecr.autcod", "auteur.autcod")
+        .where("ecr.rapcod", "=", rapportId)
+        .select([
+        "auteur.prenom as prenom",
+        "auteur.nomuse as nom_usuel",
+        "auteur.autmat as matricule",
+    ])
+        .orderBy("ecr.ecrnumtri", "asc"));
+}
+function rapports(lectureAssembleeId) {
+    return jsonArrayFrom(dbDosleg.selectFrom("rap")
+        .leftJoin("lecassrap", "lecassrap.rapcod", "rap.rapcod")
+        .leftJoin("denrap", "denrap.coddenrap", "rap.coddenrap")
+        .where("lecassrap.lecassidt", "=", lectureAssembleeId)
+        .select((eb) => [
+        "rap.rapnum as numero",
+        sql `rtrim(denrap.libdenrap)`.as("type_rapport"),
+        eb.case()
+            .when("rap.typurl", "=", "I")
+            .then(sql `concat('https://www.senat.fr/rap/', rtrim(rap.rapurl))`)
+            .else(sql `rtrim(rap.rapurl)`)
+            .end()
+            .as("url"),
+        "rap.date_depot as date_depot",
+        auteursRapport(eb.ref("rap.rapcod")).as("auteurs"),
+        organismesRapport(eb.ref("rap.rapcod")).as("organismes"),
+    ]));
+}
+function auteursTextes(texteId) {
+    return jsonArrayFrom(dbDosleg.selectFrom("auteur")
+        .leftJoin("ecr", "ecr.autcod", "auteur.autcod")
+        .where("ecr.texcod", "=", texteId)
+        .select([
+        "auteur.prenom as prenom",
+        "auteur.nomuse as nom_usuel",
+        "auteur.autmat as matricule",
+    ])
+        .orderBy("ecr.ecrnumtri", "asc"));
+}
+function textes(lectureAssembleeId) {
+    return jsonArrayFrom(dbDosleg.selectFrom("texte")
+        .leftJoin("oritxt", "oritxt.oritxtcod", "texte.oritxtcod")
+        .leftJoin("typtxt", "typtxt.typtxtcod", "texte.typtxtcod")
+        .where("texte.lecassidt", "=", lectureAssembleeId)
+        .select((eb) => [
+        "texte.texnum as numero",
+        eb.case()
+            .when("texte.typurl", "=", "I")
+            .then(sql `concat('https://www.senat.fr/leg/', rtrim(texte.texurl))`)
+            .else(sql `rtrim(texte.texurl)`)
+            .end()
+            .as("url"),
+        sql `rtrim(oritxt.oritxtlib)`.as("origine_texte"),
+        sql `rtrim(typtxt.typtxtlib)`.as("type_texte"),
+        "texte.txtoritxtdat as date",
+        "sesann as session",
+        auteursTextes(eb.ref("texte.texcod")).as("auteurs"),
+    ])
+        .orderBy(sql `array_position(array['0','2','1'], oritxt.oriordre)`));
+}
+function lecturesAssemblee(lectureId) {
+    return jsonArrayFrom(dbDosleg.selectFrom("lecass")
+        .where("lecass.lecidt", "=", lectureId)
+        .leftJoin("ass", "ass.codass", "lecass.codass")
+        .leftJoin("org", "org.orgcod", "lecass.orgcod")
+        .select(({ ref }) => [
+        sql `rtrim(ass.libass)`.as("assemblee"),
+        "org.orgnom as libelle_organisme",
+        textes(ref("lecass.lecassidt")).as("textes"),
+        rapports(ref("lecass.lecassidt")).as("rapports"),
+        datesSeances(ref("lecass.lecassidt")).as("dates_seances"),
+    ])
+        .orderBy("lecass.ordreass", "asc"));
+}
+function lectures(loiId) {
+    return jsonArrayFrom(dbDosleg.selectFrom("lecture")
+        .leftJoin("typlec", "typlec.typleccod", "lecture.typleccod")
+        .where("lecture.loicod", "=", loiId)
+        .select(({ ref }) => [
+        sql `rtrim(typlec.typleclib)`.as("type_lecture"),
+        sql `rtrim(lecture.leccom)`.as("libelle"),
+        lecturesAssemblee(ref("lecture.lecidt")).as("lectures_assemblee"),
+    ])
+        .orderBy("typlec.typlecord", "asc"));
+}
+function themes(loiId) {
+    return jsonArrayFrom(dbDosleg.selectFrom("the")
+        .leftJoin("loithe", "loithe.thecle", "the.thecle")
+        .where("loithe.loicod", "=", loiId)
+        .select([
+        "the.thelib as libelle",
+    ]));
+}
+const findAllQuery = dbDosleg
+    .selectFrom("loi")
+    .leftJoin("typloi", "typloi.typloicod", "loi.typloicod")
+    .leftJoin("etaloi", "etaloi.etaloicod", "loi.etaloicod")
+    .select(({ ref }) => [
+    "loi.signet as signet",
+    sql `concat('https://www.senat.fr/dossier-legislatif/', loi.signet, '.html')`.as("url"),
+    sql `concat(rtrim(typloi.typloilib), ' ', rtrim(loi.loitit))`.as("titre"),
+    sql `rtrim(typloi.typloilib)`.as("type_dossier"),
+    sql `rtrim(etaloi.etaloilib)`.as("etat_dossier"),
+    "loi.url_an as url_dossier_assemblee_nationale",
+    //" as decision_conseil_constitutionnel", // TODO
+    //" as date_decision_conseil_constitutionnel", // TODO
+    "loi.date_loi as date_promulgation",
+    "loi.numero as numero_loi",
+    lectures(ref("loi.loicod")).as("lectures"),
+    themes(ref("loi.loicod")).as("themes"),
+])
+    .$narrowType();
+export function findAll() {
+    return findAllQuery.stream();
+}
+export function findSenatTexteUrls(sessions = []) {
+    return dbDosleg
+        .selectFrom("texte")
+        .where("texurl", "is not", null)
+        .where("typurl", "=", "I")
+        .$if(sessions.length > 0, (qb) => qb
+        .where("sesann", "in", sessions))
+        .select([
+        sql `rtrim(texurl)`.as("url"),
+        "sesann as session",
+    ])
+        .$narrowType()
+        .stream();
+}
+export function findSenatRapportUrls(sessions = []) {
+    return dbDosleg
+        .selectFrom("rap")
+        .where("rapurl", "is not", null)
+        .where("typurl", "=", "I")
+        .$if(sessions.length > 0, (qb) => qb
+        .where("sesann", "in", sessions))
+        .select([
+        sql `rtrim(rapurl)`.as("url"),
+        "sesann as session",
+    ])
+        .$narrowType()
+        .stream();
+}