@tricoteuses/senat 2.16.4 → 2.16.5

package/lib/model/debats.d.ts

@@ -1,6 +1,6 @@
 import { InferResult } from "kysely";
 export type DebatResult = InferResult<typeof findAllQuery>[0];
-declare const findAllQuery: import("kysely").SelectQueryBuilder<any, "debats", {
+declare const findAllQuery: import("kysely").SelectQueryBuilder<any, "debats.debats", {
     [x: string]: any;
     id: string;
     date_seance: string;
@@ -8,19 +8,31 @@ declare const findAllQuery: import("kysely").SelectQueryBuilder<any, "debats", {
         [x: string]: any;
         interventions: {
             [x: string]: any;
+            auteur: {
+                code: any;
+                nom: any;
+                prenom: any;
+                matricule: any;
+            };
         }[];
     }[];
     sections_divers: {
         [x: string]: any;
         interventions: {
             [x: string]: any;
+            auteur: {
+                code: any;
+                nom: any;
+                prenom: any;
+                matricule: any;
+            };
         }[];
     }[];
     lectures: {
         id: any;
     }[];
 }>;
-export declare function findAll(): AsyncIterableIterator<{
+export declare function findAll(fromSession?: number): AsyncIterableIterator<{
     [x: string]: any;
     id: string;
     date_seance: string;
@@ -28,12 +40,24 @@ export declare function findAll(): AsyncIterableIterator<{
         [x: string]: any;
         interventions: {
             [x: string]: any;
+            auteur: {
+                code: any;
+                nom: any;
+                prenom: any;
+                matricule: any;
+            };
         }[];
     }[];
     sections_divers: {
         [x: string]: any;
         interventions: {
             [x: string]: any;
+            auteur: {
+                code: any;
+                nom: any;
+                prenom: any;
+                matricule: any;
+            };
         }[];
     }[];
     lectures: {

package/lib/model/debats.js

@@ -1,87 +1,95 @@
-import { jsonArrayFrom } from "kysely/helpers/postgres";
+import { jsonArrayFrom, jsonBuildObject } from "kysely/helpers/postgres";
 import { dbSenat } from "../databases";
 import { ID_DATE_FORMAT } from "../scripts/datautil";
 import { toDateString } from "./util";
 function sectionsLegislatives(dateSeance) {
     return jsonArrayFrom(dbSenat
-        .withSchema("debats")
-        .selectFrom("secdis")
-        .leftJoin("typsec", "secdis.typseccod", "typsec.typseccod")
-        .where("secdis.datsea", "=", dateSeance)
+        .selectFrom("debats.secdis")
+        .leftJoin("debats.typsec", "debats.secdis.typseccod", "debats.typsec.typseccod")
+        .where("debats.secdis.datsea", "=", dateSeance)
         .select(({ ref }) => [
-        "secdis.secdisordid as id",
-        "secdis.secdisnum as numero",
-        "secdis.secdisobj as objet",
-        "secdis.secdisurl as url",
-        "typsec.typseclib as type",
-        "typsec.typseccat as categorie",
-        interventionsLegislatives(ref("secdis.secdiscle")).as("interventions"),
-        "secdis.lecassidt as lecture_id"
+        "debats.secdis.secdisordid as id",
+        "debats.secdis.secdisnum as numero",
+        "debats.secdis.secdisobj as objet",
+        "debats.secdis.secdisurl as url",
+        "debats.typsec.typseclib as type",
+        "debats.typsec.typseccat as categorie",
+        interventionsLegislatives(ref("debats.secdis.secdiscle")).as("interventions"),
+        "debats.secdis.lecassidt as lecture_id"
     ])
-        .orderBy("secdis.secdisordid asc"));
+        .orderBy("debats.secdis.secdisordid asc"));
 }
 function interventionsLegislatives(sectionId) {
     return jsonArrayFrom(dbSenat
-        .withSchema("debats")
-        .selectFrom("intpjl")
-        .where("intpjl.secdiscle", "=", sectionId)
-        .select(({ ref, val }) => [
-        "intpjl.intordid as id",
-        "intpjl.autcod as auteur_code",
-        "intpjl.intfon as fonction_intervenant",
-        "intpjl.inturl as url",
-        "intpjl.intana as analyse",
+        .selectFrom("debats.intpjl")
+        .leftJoin("dosleg.auteur", "debats.intpjl.autcod", "dosleg.auteur.autcod")
+        .where("debats.intpjl.secdiscle", "=", sectionId)
+        .select(({ ref, val, fn }) => [
+        "debats.intpjl.intordid as id",
+        "debats.intpjl.autcod as auteur_code",
+        "debats.intpjl.intfon as fonction_intervenant",
+        "debats.intpjl.inturl as url",
+        "debats.intpjl.intana as analyse",
+        jsonBuildObject({
+            code: ref("dosleg.auteur.autcod"),
+            nom: ref("dosleg.auteur.nomuse"),
+            prenom: ref("dosleg.auteur.prenom"),
+            matricule: ref("dosleg.auteur.autmat")
+        }).as("auteur")
     ])
-        .orderBy("intpjl.intordid asc"));
+        .orderBy("debats.intpjl.intordid asc"));
 }
 function sectionsNonLegislatives(dateSeance) {
     return jsonArrayFrom(dbSenat
-        .withSchema("debats")
-        .selectFrom("secdivers")
-        .leftJoin("typsec", "secdivers.typseccod", "typsec.typseccod")
-        .where("secdivers.datsea", "=", dateSeance)
+        .selectFrom("debats.secdivers")
+        .leftJoin("debats.typsec", "debats.secdivers.typseccod", "debats.typsec.typseccod")
+        .where("debats.secdivers.datsea", "=", dateSeance)
         .select(({ ref }) => [
-        "secdivers.secdiverslibelle as libelle",
-        "secdivers.secdiversobj as objet",
-        "typsec.typseclib as type",
-        "typsec.typseccat as categorie",
-        interventionsNonLegislatives(ref("secdivers.secdiverscle")).as("interventions"),
+        "debats.secdivers.secdiverslibelle as libelle",
+        "debats.secdivers.secdiversobj as objet",
+        "debats.typsec.typseclib as type",
+        "debats.typsec.typseccat as categorie",
+        interventionsNonLegislatives(ref("debats.secdivers.secdiverscle")).as("interventions"),
     ]));
 }
 function interventionsNonLegislatives(sectionId) {
     return jsonArrayFrom(dbSenat
-        .withSchema("debats")
-        .selectFrom("intdivers")
-        .where("intdivers.intdiverscle", "=", sectionId)
+        .selectFrom("debats.intdivers")
+        .leftJoin("dosleg.auteur", "debats.intdivers.autcod", "dosleg.auteur.autcod")
+        .where("debats.intdivers.intdiverscle", "=", sectionId)
         .select(({ ref, val }) => [
-        "intdivers.intdiversordid as id",
-        "intdivers.autcod as auteur_code",
-        "intdivers.intfon as fonction_intervenant",
-        "intdivers.inturl as url",
-        "intdivers.intana as analyse",
+        "debats.intdivers.intdiversordid as id",
+        "debats.intdivers.autcod as auteur_code",
+        "debats.intdivers.intfon as fonction_intervenant",
+        "debats.intdivers.inturl as url",
+        "debats.intdivers.intana as analyse",
+        jsonBuildObject({
+            code: ref("dosleg.auteur.autcod"),
+            nom: ref("dosleg.auteur.nomuse"),
+            prenom: ref("dosleg.auteur.prenom"),
+            matricule: ref("dosleg.auteur.autmat")
+        }).as("auteur")
     ])
-        .orderBy("intdivers.intdiversordid asc"));
+        .orderBy("debats.intdivers.intdiversordid asc"));
 }
 function lecturesAssemblee(dateSeance) {
     return jsonArrayFrom(dbSenat
-        .withSchema("debats")
-        .selectFrom("lecassdeb")
-        .where("lecassdeb.datsea", "=", dateSeance)
-        .select("lecassdeb.lecassidt as id"));
+        .selectFrom("debats.lecassdeb")
+        .where("debats.lecassdeb.datsea", "=", dateSeance)
+        .select("debats.lecassdeb.lecassidt as id"));
 }
 const findAllQuery = dbSenat
-    .withSchema("debats")
-    .selectFrom("debats")
+    .selectFrom("debats.debats")
     .select(({ ref, val }) => [
-    toDateString(ref("debats.datsea"), val(ID_DATE_FORMAT)).as("id"),
-    toDateString(ref("debats.datsea")).as("date_seance"),
-    "debats.numero as numero",
-    "debats.deburl as url",
-    "debats.debsyn as etat_synchronisation",
-    sectionsLegislatives(ref("debats.datsea")).as("sections"),
-    sectionsNonLegislatives(ref("debats.datsea")).as("sections_divers"),
-    lecturesAssemblee(ref("debats.datsea")).as("lectures"),
+    toDateString(ref("debats.debats.datsea"), val(ID_DATE_FORMAT)).as("id"),
+    toDateString(ref("debats.debats.datsea")).as("date_seance"),
+    "debats.debats.numero as numero",
+    "debats.debats.deburl as url",
+    "debats.debats.debsyn as etat_synchronisation",
+    sectionsLegislatives(ref("debats.debats.datsea")).as("sections"),
+    sectionsNonLegislatives(ref("debats.debats.datsea")).as("sections_divers"),
+    lecturesAssemblee(ref("debats.debats.datsea")).as("lectures"),
 ]);
-export function findAll() {
+export function findAll(fromSession) {
     return findAllQuery.stream();
 }

package/lib/model/dosleg.d.ts

@@ -1,21 +1,8 @@
 import { InferResult, SelectQueryBuilder } from "kysely";
 declare const findAllDossiersQuery: SelectQueryBuilder<any, any, any>;
 export declare function findAllDossiers(): AsyncIterableIterator<DossierLegislatifResult>;
-declare const findAuteursQuery: SelectQueryBuilder<any, "dosleg.auteur", {
-    code: any;
-    nom: any;
-    prenom: any;
-    matricule: any;
-}>;
-export declare function findAuteurs(): Promise<{
-    code: any;
-    nom: any;
-    prenom: any;
-    matricule: any;
-}[]>;
 export declare function createActesLegislatifs(dossier: DossierLegislatifResult): any;
 export declare function getCodeActeLecture(codeNatureDossier: string, typeLecture: string, assemblee: string): string | null;
 export declare function getCodeActeTexte(codeParent: string | null, texteOrigine: string): string | null;
 export type DossierLegislatifResult = InferResult<typeof findAllDossiersQuery>[0];
-export type AuteurResult = InferResult<typeof findAuteursQuery>[0];
 export {};

package/lib/model/dosleg.js

@@ -222,19 +222,6 @@ const findAllDossiersQuery = dbSenat
 export function findAllDossiers() {
     return findAllDossiersQuery.stream();
 }
-const findAuteursQuery = dbSenat
-    .withSchema("dosleg")
-    .selectFrom("dosleg.auteur")
-    .select([
-    "autcod as code",
-    "nomuse as nom",
-    "prenom as prenom",
-    "autmat as matricule",
-]);
-export async function findAuteurs() {
-    return findAuteursQuery
-        .execute();
-}
 export function createActesLegislatifs(dossier) {
     const actesLegislatifs = (dossier["lectures"] || []).map((lecture) => {
         const lecturesAssemblee = (lecture["lectures_assemblee"] || []).map((lectureAss) => {

package/lib/model/index.d.ts

@@ -1,6 +1,6 @@
 export { findAllAmendements } from "./ameli";
 export { findAll as findAllDebats } from "./debats";
-export { findAllDossiers, findAuteurs, } from "./dosleg";
+export { findAllDossiers, } from "./dosleg";
 export { findSenatTexteUrls, findSenatRapportUrls } from "./documents";
 export { findAllScrutins } from "./scrutins";
 export { findAll as findAllQuestions } from "./questions";

package/lib/model/index.js

@@ -1,6 +1,6 @@
 export { findAllAmendements } from "./ameli";
 export { findAll as findAllDebats } from "./debats";
-export { findAllDossiers, findAuteurs, } from "./dosleg";
+export { findAllDossiers, } from "./dosleg";
 export { findSenatTexteUrls, findSenatRapportUrls } from "./documents";
 export { findAllScrutins } from "./scrutins";
 export { findAll as findAllQuestions } from "./questions";

package/lib/scripts/convert_data.js

@@ -2,14 +2,15 @@ import assert from "assert";
 import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 import path from "path";
+import pLimit from "p-limit";
 import { datasets, EnabledDatasets, getEnabledDatasets } from "../datasets";
 import { DATA_ORIGINAL_FOLDER, DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, SCRUTINS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER, } from "../loaders";
-import { findAllAmendements, findAllCirconscriptions, findAllDebats, findAllDossiers, findAllScrutins, findAllOrganismes, findAllQuestions, findAllSens, findAuteurs, findSenatRapportUrls, findSenatTexteUrls, } from "../model";
+import { findAllAmendements, findAllCirconscriptions, findAllDebats, findAllDossiers, findAllScrutins, findAllOrganismes, findAllQuestions, findAllSens, findSenatRapportUrls, findSenatTexteUrls, } from "../model";
 import { createActesLegislatifs } from "../model/dosleg";
 import { UNDEFINED_SESSION } from "../types/sessions";
 import { getSessionFromDate, getSessionFromSignet } from "./datautil";
 import { commonOptions } from "./shared/cli_helpers";
-import { ensureAndClearDir, ensureAndClearDirSync } from "./shared/util";
+import { ensureAndClearDir } from "./shared/util";
 const optionsDefinitions = [...commonOptions];
 const options = commandLineArgs(optionsDefinitions);
 const SENAT_TEXTE_XML_BASE_URL = "https://www.senat.fr/akomantoso/";
@@ -47,19 +48,24 @@ async function convertDatasetAmeli(dataDir, options) {
         console.log(`Converting database ${dataset.database} data into files…`);
     }
     const ameliReorganizedRootDir = path.join(dataDir, dataset.database);
-    ensureAndClearDirSync(ameliReorganizedRootDir);
+    await fs.ensureDir(ameliReorganizedRootDir);
+    const limit = pLimit(10);
+    const tasks = [];
     for await (const amendement of findAllAmendements(options["fromSession"])) {
-        if (options["verbose"]) {
-            console.log(`Converting ${amendement["numero"]} file…`);
-        }
-        const session = String(amendement["session"]) || UNDEFINED_SESSION;
-        const signetDossierLegislatif = amendement["signet_dossier_legislatif"] ||
-            `${amendement["nature_texte"]}-${amendement["numero_texte"]}`.toLowerCase();
-        const ameliReorganizedDir = path.join(ameliReorganizedRootDir, String(session), signetDossierLegislatif);
-        fs.ensureDirSync(ameliReorganizedDir);
-        const amendementFileName = `${amendement["numero"]}.json`;
-        fs.writeJSONSync(path.join(ameliReorganizedDir, amendementFileName), amendement, { spaces: 2 });
+        tasks.push(limit(async () => {
+            if (options["verbose"]) {
+                console.log(`Converting ${amendement["numero"]} file…`);
+            }
+            const session = String(amendement["session"]) || UNDEFINED_SESSION;
+            const signetDossierLegislatif = amendement["signet_dossier_legislatif"] ||
+                `${amendement["nature_texte"]}-${amendement["numero_texte"]}`.toLowerCase();
+            const ameliReorganizedDir = path.join(ameliReorganizedRootDir, String(session), signetDossierLegislatif);
+            await fs.ensureDir(ameliReorganizedDir);
+            const amendementFileName = `${amendement["numero"]}.json`;
+            await fs.writeJSON(path.join(ameliReorganizedDir, amendementFileName), amendement, { spaces: 2 });
+        }));
     }
+    await Promise.all(tasks);
 }
 async function convertDatasetDebats(dataDir, options) {
     const dataset = datasets.debats;
@@ -68,40 +74,19 @@ async function convertDatasetDebats(dataDir, options) {
     }
     const debatsReorganizedRootDir = path.join(dataDir, dataset.database);
     ensureAndClearDir(debatsReorganizedRootDir);
-    const allAuteurs = await findAuteurs();
     for await (const debat of findAllDebats()) {
         if (options["verbose"]) {
             console.log(`Converting ${debat.id} file…`);
         }
-        const enrichedDebat = await enrichDebat(debat, allAuteurs);
-        const session = getSessionFromDate(enrichedDebat.date_seance);
+        const session = getSessionFromDate(debat.date_seance);
         if (options["fromSession"] && session < options["fromSession"]) {
             continue;
         }
         const debatsReorganizedDir = path.join(debatsReorganizedRootDir, String(session));
         fs.ensureDirSync(debatsReorganizedDir);
-        const debatFileName = `${enrichedDebat.id}.json`;
-        fs.writeJSONSync(path.join(debatsReorganizedDir, debatFileName), enrichedDebat, { spaces: 2 });
-    }
-}
-async function enrichDebat(debat, auteurs) {
-    const enrichedDebat = { ...debat };
-    for (const section of enrichedDebat.sections) {
-        for (const intervention of section.interventions) {
-            ;
-            intervention.auteur = findAuteur(intervention["auteur_code"], auteurs);
-        }
-    }
-    for (const section of enrichedDebat.sections_divers) {
-        for (const intervention of section.interventions) {
-            ;
-            intervention.auteur = findAuteur(intervention["auteur_code"], auteurs);
-        }
+        const debatFileName = `${debat.id}.json`;
+        fs.writeJSONSync(path.join(debatsReorganizedDir, debatFileName), debat, { spaces: 2 });
     }
-    return enrichedDebat;
-}
-function findAuteur(auteurCode, auteurs) {
-    return auteurs.find((auteur) => auteur.code === auteurCode);
 }
 async function convertDatasetDosLeg(dataDir, options) {
     const dataset = datasets.dosleg;
@@ -141,18 +126,22 @@ async function convertDatasetScrutins(dataDir, options) {
     }
     const scrutinsReorganizedDir = path.join(dataDir, SCRUTINS_FOLDER);
     ensureAndClearDir(scrutinsReorganizedDir);
+    const limit = pLimit(10);
+    const tasks = [];
     for await (const scrutin of findAllScrutins(options["fromSession"])) {
-        if (options["verbose"]) {
-            console.log(`Converting ${scrutin["numero"]} file…`);
-        }
-        let scrutinReorganizedDir = path.join(scrutinsReorganizedDir, String(UNDEFINED_SESSION));
-        const session = scrutin["session"] || UNDEFINED_SESSION;
-        scrutinReorganizedDir = path.join(scrutinsReorganizedDir, String(session));
-        fs.ensureDirSync(scrutinReorganizedDir);
-        const scrutinFileName = `${scrutin["numero"]}.json`;
-        fs.writeJSONSync(path.join(scrutinReorganizedDir, scrutinFileName), scrutin, {
-            spaces: 2,
-        });
+        tasks.push(limit(async () => {
+            if (options["verbose"]) {
+                console.log(`Converting ${scrutin["numero"]} file…`);
+            }
+            let scrutinReorganizedDir = path.join(scrutinsReorganizedDir, String(UNDEFINED_SESSION));
+            const session = scrutin["session"] || UNDEFINED_SESSION;
+            scrutinReorganizedDir = path.join(scrutinsReorganizedDir, String(session));
+            await fs.ensureDir(scrutinReorganizedDir);
+            const scrutinFileName = `${scrutin["numero"]}.json`;
+            await fs.writeJSON(path.join(scrutinReorganizedDir, scrutinFileName), scrutin, {
+                spaces: 2,
+            });
+        }));
     }
 }
 async function convertDatasetQuestions(dataDir) {
@@ -162,16 +151,21 @@ async function convertDatasetQuestions(dataDir) {
     }
     const questionsReorganizedRootDir = path.join(dataDir, dataset.database);
     ensureAndClearDir(questionsReorganizedRootDir);
+    const limit = pLimit(10);
+    const tasks = [];
     for await (const question of findAllQuestions()) {
-        if (options["verbose"]) {
-            console.log(`Converting ${question["reference"]} file…`);
-        }
-        const legislature = question["legislature"] ? question["legislature"] : 0;
-        const questionReorganizedDir = path.join(questionsReorganizedRootDir, String(legislature));
-        fs.ensureDirSync(questionReorganizedDir);
-        const questionFileName = `${question["reference"]}.json`;
-        fs.writeJSONSync(path.join(questionReorganizedDir, questionFileName), question, { spaces: 2 });
+        tasks.push(limit(async () => {
+            if (options["verbose"]) {
+                console.log(`Converting ${question["reference"]} file…`);
+            }
+            const legislature = question["legislature"] ? question["legislature"] : 0;
+            const questionReorganizedDir = path.join(questionsReorganizedRootDir, String(legislature));
+            await fs.ensureDir(questionReorganizedDir);
+            const questionFileName = `${question["reference"]}.json`;
+            await fs.writeJSON(path.join(questionReorganizedDir, questionFileName), question, { spaces: 2 });
+        }));
     }
+    await Promise.all(tasks);
 }
 async function convertTexteUrls(dataDir) {
     const textesDir = path.join(dataDir, TEXTE_FOLDER);

package/lib/scripts/retrieve_open_data.js

@@ -96,6 +96,7 @@ async function copyToSenat(dataset, dataDir, options) {
         stdio: ["pipe", "ignore", "pipe"],
         env: process.env,
     });
+    psql.stdin.write(`DROP SCHEMA IF EXISTS ${dataset.database} CASCADE;\n`);
     psql.stdin.write(`CREATE SCHEMA IF NOT EXISTS ${dataset.database};\n`);
     let buffer = '';
     const combinedTransform = new Transform({
@@ -273,12 +274,7 @@ async function retrieveOpenData() {
         process.env["PGUSER"] &&
         process.env["PGPASSWORD"], "Missing database configuration: environment variables PGHOST, PGPORT, PGUSER and PGPASSWORD or TRICOTEUSES_SENAT_DB_* in .env file");
     console.time("data extraction time");
-    execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -c "DROP DATABASE IF EXISTS senat;"`, {
-        cwd: dataDir,
-        env: process.env,
-        encoding: "utf-8",
-    });
-    execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -c "CREATE DATABASE senat WITH OWNER opendata"`, {
+    execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -c "CREATE DATABASE senat WITH OWNER opendata" || true`, {
         cwd: dataDir,
         env: process.env,
         encoding: "utf-8",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.16.4",
+  "version": "2.16.5",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",
@@ -72,6 +72,7 @@
     "node-stream-zip": "^1.8.2",
     "pg": "^8.13.1",
     "pg-cursor": "^2.12.1",
+    "p-limit": "^7.2.0",
     "slug": "^11.0.0",
     "tsx": "^4.20.6",
     "windows-1252": "^1.0.0"