@tricoteuses/senat 3.1.18 → 3.1.19

package/lib/src/config.d.ts

@@ -1,13 +1,5 @@
 import "dotenv/config";
 import { z } from "zod";
-export declare const dbSchema: z.ZodObject<{
-    host: z.ZodString;
-    name: z.ZodString;
-    password: z.ZodString;
-    port: z.ZodCoercedNumber<unknown>;
-    user: z.ZodString;
-}, z.core.$strip>;
-export type DbConfig = z.infer<typeof dbSchema>;
 export declare const configSchema: z.ZodObject<{
     db: z.ZodObject<{
         host: z.ZodString;
@@ -16,13 +8,6 @@ export declare const configSchema: z.ZodObject<{
         port: z.ZodCoercedNumber<unknown>;
         user: z.ZodString;
     }, z.core.$strip>;
-    stagingDb: z.ZodObject<{
-        host: z.ZodString;
-        name: z.ZodString;
-        password: z.ZodString;
-        port: z.ZodCoercedNumber<unknown>;
-        user: z.ZodString;
-    }, z.core.$strip>;
 }, z.core.$strip>;
 declare const _default: {
     db: {
@@ -32,12 +17,5 @@ declare const _default: {
         port: number;
         user: string;
     };
-    stagingDb: {
-        host: string;
-        name: string;
-        password: string;
-        port: number;
-        user: string;
-    };
 };
 export default _default;

package/lib/src/config.js

@@ -1,6 +1,6 @@
 import "dotenv/config";
 import { z } from "zod";
-export const dbSchema = z.object({
+const dbSchema = z.object({
     host: z.string().trim().min(1, "Must not be empty"),
     name: z.string().trim().min(1, "Must not be empty"),
     password: z.string().trim().min(1, "Must not be empty"),
@@ -9,29 +9,19 @@ export const dbSchema = z.object({
 });
 export const configSchema = z.object({
     db: dbSchema,
-    stagingDb: dbSchema,
 });
 const config = {
     db: {
-        host: process.env["DB_HOST"] || "localhost",
-        name: process.env["DB_NAME"] || "senat",
-        password: process.env["DB_PASSWORD"] || "opendata",
-        port: process.env["DB_PORT"] || 5432,
-        user: process.env["DB_USER"] || "opendata",
-    },
-    stagingDb: {
-        host: process.env["STAGING_DB_HOST"] || process.env["DB_HOST"] || "localhost",
-        name: process.env["STAGING_DB_NAME"] || "senat_staging",
-        password: process.env["STAGING_DB_PASSWORD"] || process.env["DB_PASSWORD"] || "opendata",
-        port: process.env["STAGING_DB_PORT"] || process.env["DB_PORT"] || 5432,
-        user: process.env["STAGING_DB_USER"] || process.env["DB_USER"] || "opendata",
+        host: process.env["TRICOTEUSES_SENAT_DB_HOST"] || "localhost",
+        name: process.env["TRICOTEUSES_SENAT_DB_NAME"] || "postgres",
+        password: process.env["TRICOTEUSES_SENAT_DB_PASSWORD"] || "opendata",
+        port: process.env["TRICOTEUSES_SENAT_DB_PORT"] || 5432,
+        user: process.env["TRICOTEUSES_SENAT_DB_USER"] || "opendata",
     },
 };
 const result = configSchema.safeParse(config);
 if (!result.success) {
-    const issues = JSON.stringify(result.error.issues, null, 2);
-    const serializedConfig = JSON.stringify(config, null, 2);
-    console.error(`Error in configuration:\n${serializedConfig}\nError:\n${issues}`);
+    console.error(`Error in configuration:\n${JSON.stringify(config, null, 2)}\nError:\n${JSON.stringify(result.error.issues, null, 2)}`);
     process.exit(-1);
 }
 export default result.data;

package/lib/src/conversion_textes.js

@@ -186,11 +186,7 @@ p.has-alinea {
     <div class="header-type">${metadata.type || ""}</div>
     <div class="header-authors">${metadata.authors || ""}</div>
     ${metadata.commission
-        ? [
-            `<div class="header-commission">Envoyée à la ${metadata.commission.toLowerCase()},`,
-            "sous réserve de la constitution éventuelle d'une commission spéciale dans les conditions prévues",
-            "par le Règlement.</div>",
-        ].join(" ")
+        ? `<div class="header-commission">Envoyée à la ${metadata.commission.toLowerCase()}, sous réserve de la constitution éventuelle d'une commission spéciale dans les conditions prévues par le Règlement.</div>`
         : ""}
   </div>
   <h1>${metadata.title || ""}</h1>

package/lib/src/databases.d.ts

@@ -0,0 +1,2 @@
+import { Kysely } from "kysely";
+export declare const dbSenat: Kysely<any>;

package/lib/src/databases.js

@@ -0,0 +1,26 @@
+import { Kysely, PostgresDialect } from "kysely";
+import * as pg from "pg";
+import { types } from "pg";
+import Cursor from "pg-cursor";
+import config from "./config";
+// Map int8 to number instead of string
+// https://github.com/brianc/node-pg-types#use
+pg.types.setTypeParser(types.builtins.INT8, (val) => {
+    return parseInt(val, 10);
+});
+const senatPool = new pg.Pool({
+    database: "senat",
+    host: config.db.host,
+    user: config.db.user,
+    password: config.db.password,
+    port: config.db.port,
+    max: 10,
+});
+const senatDialect = new PostgresDialect({
+    pool: senatPool,
+    cursor: Cursor,
+});
+export const dbSenat = new Kysely({
+    log: ["error"],
+    dialect: senatDialect,
+});

package/lib/src/datasets.d.ts

@@ -10,10 +10,6 @@ export interface Dataset {
             columns: string[];
         }>;
     };
-    mergeKeys?: {
-        [table: string]: string[];
-    };
-    rowMultisetMergeTables?: string[];
 }
 export interface Datasets {
     ameli: Dataset;

package/lib/src/datasets.js

@@ -167,12 +167,6 @@ export const datasets = {
                 { name: "idx_senmat", columns: ["senmat"] },
             ],
         },
-        mergeKeys: {
-            corscr: ["sesann", "scrnum", "corscrord"],
-            docsea: ["evtseacle", "docseaord"],
-            gen: ["gencod"],
-        },
-        rowMultisetMergeTables: ["amescr"],
     },
     questions: {
         database: "questions",
@@ -189,7 +183,6 @@ export const datasets = {
             sortquestion: [{ name: "idx_sorquecod", columns: ["sorquecod"] }],
             etatquestion: [{ name: "idx_etaquecod", columns: ["etaquecod"] }],
         },
-        rowMultisetMergeTables: ["tam_reponses"],
     },
     sens: {
         database: "sens",
@@ -220,19 +213,12 @@ export const datasets = {
                 { name: "idx_typmancod", columns: ["typmancod"] },
             ],
         },
-        mergeKeys: {
-            activite: ["actid"],
-        },
     },
 };
-function getEnabledDatasetByName(datasetName) {
-    const enabledDataset = EnabledDatasets[datasetName];
-    return typeof enabledDataset === "number" ? enabledDataset : undefined;
-}
 export function getEnabledDatasets(categories) {
-    categories.map((datasetName) => assert.notStrictEqual(getEnabledDatasetByName(datasetName), undefined, `Unknown name of dataset: ${datasetName}`));
+    categories.map((datasetName) => assert.notStrictEqual(EnabledDatasets[datasetName], undefined, `Unknown name of dataset: ${datasetName}`));
     return categories.reduce((enabledDatasets, datasetName) => {
-        const enabledDataset = getEnabledDatasetByName(datasetName);
+        const enabledDataset = EnabledDatasets[datasetName];
         return enabledDatasets | (enabledDataset || EnabledDatasets.None);
     }, EnabledDatasets.None);
 }

package/lib/src/git.d.ts

@@ -1,7 +1,6 @@
 export declare function initRepo(repositoryDir: string): void;
 export declare function commit(repositoryDir: string, message: string): boolean;
 export declare function commitAndPush(repositoryDir: string, message: string, remotes?: string[]): number;
-export declare function pull(repositoryDir: string, remote?: string, branch?: string): boolean;
 export declare function resetAndPull(gitDir: string): boolean;
 export declare function clone(gitGroupUrl: string | undefined, gitName: string, workingDir: string): void;
 export declare function run(repositoryDir: string, args: string, verbose?: boolean): string;

package/lib/src/git.js

@@ -5,15 +5,6 @@ const MAXBUFFER = 50 * 1024 * 1024;
 const GIT_LOCK_RETRY_DELAY_MS = 1000;
 const GIT_LOCK_RETRY_COUNT = 5;
 const GIT_LOCK_STALE_AFTER_MS = 2 * 60 * 1000;
-function asExecSyncError(error) {
-    return error;
-}
-function outputToString(output) {
-    if (output == null) {
-        return "";
-    }
-    return Buffer.isBuffer(output) ? output.toString("utf-8") : output;
-}
 function sleep(ms) {
     Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 }
@@ -21,7 +12,7 @@ function getIndexLockPath(repositoryDir) {
     return path.join(repositoryDir, ".git", "index.lock");
 }
 function isIndexLockError(error) {
-    const stderr = error instanceof Error ? outputToString(asExecSyncError(error).stderr) : "";
+    const stderr = String(error?.stderr || "");
     return /index\.lock': File exists\./.test(stderr);
 }
 function removeStaleIndexLock(repositoryDir) {
@@ -90,9 +81,9 @@ export function commit(repositoryDir, message) {
         });
         return true;
     }
-    catch (error) {
-        const childProcess = asExecSyncError(error);
-        if (childProcess.stdout === null || !/nothing to commit|rien à valider/.test(outputToString(childProcess.stdout))) {
+    catch (childProcess) {
+        if (childProcess.stdout === null ||
+            !/nothing to commit|rien à valider/.test(childProcess.stdout)) {
             console.error(childProcess.output);
             throw childProcess;
         }
@@ -111,11 +102,10 @@ export function commitAndPush(repositoryDir, message, remotes) {
                     stdio: ["ignore", "ignore", "pipe"],
                 });
             }
-            catch (error) {
-                const childProcess = asExecSyncError(error);
+            catch (childProcess) {
                 // Don't stop when push fails.
                 console.error(childProcess.output);
-                exitCode = childProcess.status ?? 1;
+                exitCode = childProcess.status;
             }
         }
     }
@@ -125,27 +115,6 @@ export function commitAndPush(repositoryDir, message, remotes) {
     }
     return exitCode;
 }
-export function pull(repositoryDir, remote = "origin", branch = "master") {
-    initRepo(repositoryDir);
-    const statusOutput = execSync("git status --porcelain", {
-        cwd: repositoryDir,
-        env: process.env,
-        encoding: "utf-8",
-        stdio: ["ignore", "pipe", "pipe"],
-        maxBuffer: MAXBUFFER,
-    }).trim();
-    if (statusOutput.length > 0) {
-        return false;
-    }
-    execSync(`git pull --rebase ${remote} ${branch}`, {
-        cwd: repositoryDir,
-        env: process.env,
-        encoding: "utf-8",
-        stdio: ["ignore", "ignore", "pipe"],
-        maxBuffer: MAXBUFFER,
-    });
-    return true;
-}
 export function resetAndPull(gitDir) {
     execSync("git reset --hard origin/master", {
         cwd: gitDir,
@@ -185,11 +154,9 @@ export function run(repositoryDir, args, verbose) {
             console.log(output);
         return output;
     }
-    catch (error) {
-        const childProcess = asExecSyncError(error);
-        for (const output of ["stdout", "stderr"]) {
-            console.error(`${output}: ${outputToString(childProcess[output])}`);
-        }
+    catch (childProcess) {
+        for (const output of ["stdout", "stderr"])
+            console.error(`${output}: ${childProcess[output]}`);
         throw childProcess;
     }
 }
@@ -208,9 +175,8 @@ export function test(repositoryDir, args, verbose) {
             console.log(output);
         return true;
     }
-    catch (error) {
-        const childProcess = asExecSyncError(error);
-        if (childProcess.status !== 0)
+    catch (childProcess) {
+        if (childProcess.status != 0)
             return false;
         throw childProcess;
     }

package/lib/src/loaders.d.ts

@@ -1,15 +1,15 @@
-import { AmendementResult } from "./model/ameli.js";
-import { DebatResult } from "./model/debats.js";
-import { DossierLegislatifResult } from "./model/dosleg.js";
-import { QuestionResult } from "./model/questions.js";
-import { ScrutinResult } from "./model/scrutins.js";
-import { CirconscriptionResult, OrganismeResult, SenateurResult } from "./model/sens.js";
-import { Reunion } from "./types/agenda.js";
-import { FlatTexte, DocumentMetadata } from "./types/texte.js";
-import { CompteRendu } from "./types/compte_rendu.js";
-import { DocumentResult } from "./model/documents.js";
-export { EnabledDatasets } from "./datasets.js";
-export type { DocumentResult } from "./model/documents.js";
+import { AmendementResult } from "./model/ameli";
+import { DebatResult } from "./model/debats";
+import { DossierLegislatifResult } from "./model/dosleg";
+import { QuestionResult } from "./model/questions";
+import { ScrutinResult } from "./model/scrutins";
+import { CirconscriptionResult, OrganismeResult, SenateurResult } from "./model/sens";
+import { Reunion } from "./types/agenda";
+import { FlatTexte, DocumentMetadata } from "./types/texte";
+import { CompteRendu } from "./types/compte_rendu";
+import { DocumentResult } from "./model/documents";
+export { EnabledDatasets } from "./datasets";
+export type { DocumentResult } from "./model/documents";
 export declare const AGENDA_FOLDER = "agenda";
 export declare const COMPTES_RENDUS_FOLDER = "seances";
 export declare const COMMISSION_FOLDER = "commissions";

package/lib/src/loaders.js

@@ -1,10 +1,10 @@
 import fsex from "fs-extra";
 import fs from "fs";
 import path from "path";
-import * as git from "./git.js";
-import { datasets } from "./datasets.js";
-import { UNDEFINED_SESSION } from "./types/sessions.js";
-export { EnabledDatasets } from "./datasets.js";
+import * as git from "./git";
+import { datasets } from "./datasets";
+import { UNDEFINED_SESSION } from "./types/sessions";
+export { EnabledDatasets } from "./datasets";
 export const AGENDA_FOLDER = "agenda";
 export const COMPTES_RENDUS_FOLDER = "seances";
 export const COMMISSION_FOLDER = "commissions";
@@ -69,8 +69,7 @@ function* iterLoadSenatItems(dataDir, dataName, legislatureOrSession, subDir, {
             const itemJson = fs.readFileSync(filePath, { encoding: "utf8" });
             item = JSON.parse(itemJson);
         }
-        catch (error) {
-            const err = error;
+        catch (err) {
             console.warn(`[iterLoadSenatItems] skipped invalid JSON: ${filePath} (${err.message})`);
             continue;
         }
@@ -181,9 +180,6 @@ export function* iterLoadSenatTextes(dataDir, session, options = {}) {
         const texteItem = iterItem;
         const texte = texteItem.item;
         const texteId = texte["id"];
-        if (!texteId) {
-            continue;
-        }
         const { item: texteContent } = loadSenatTexteContent(dataDir, texte["session"], texteId);
         if (texteContent) {
             texteItem.item = { ...texteContent, ...texteItem.item };
@@ -226,11 +222,9 @@ export function* iterLoadSenatAgendas(dataDir, session) {
         const gr = raw;
         if (!gr.uid || !gr.date || !gr.titre)
             continue;
-        const reunion = {
-            ...gr,
-            events: Array.isArray(gr.events) ? gr.events : [],
-        };
-        yield { item: reunion };
+        if (!Array.isArray(gr.events))
+            gr.events = [];
+        yield { item: gr };
     }
 }
 export function* iterLoadSenatCirconscriptions(dataDir, options = {}) {

package/lib/src/model/agenda.d.ts

@@ -1,4 +1,4 @@
-import { AgendaEvent } from "../types/agenda.js";
+import { AgendaEvent } from "../types/agenda";
 export declare function getStartAndEndTimes(timeStr: string | null | undefined, dateISO: string): {
     startTime: string | null;
     endTime: string | null;

package/lib/src/model/agenda.js

@@ -1,7 +1,7 @@
 import { JSDOM } from "jsdom";
 import { DateTime } from "luxon";
 import path from "path";
-import { ID_DATE_FORMAT, STANDARD_DATE_FORMAT } from "../scripts/datautil.js";
+import { ID_DATE_FORMAT, STANDARD_DATE_FORMAT } from "../scripts/datautil";
 const FR_TZ = "Europe/Paris";
 function eventIsSeance(eventElement) {
     return eventElement.classList.contains("evt-seance");
@@ -113,7 +113,7 @@ function transformAgenda(document, fileName) {
         const { startTime, endTime } = getStartAndEndTimes(timeOriginal, date);
         const titre = eventElement.querySelector(".titre")?.textContent?.trim() || "";
         const organe = eventElement.querySelector(".organe")?.textContent?.trim() || null;
-        const objet = eventElement.querySelector(".objet")?.textContent?.trim()?.replace(/^- /, "") || null;
+        const objet = eventElement.querySelector(".objet")?.textContent?.trim()?.replace(/^- /, "") || null;
         const lieu = eventElement.querySelector(".lieu")?.textContent || null;
         const videoElement = eventElement.querySelector(".video");
         const urlDossierSenat = getUrlDossierSenat(eventElement.querySelectorAll(".lien a"));
@@ -142,7 +142,7 @@ export async function parseAgendaFromFile(htmlFilePath) {
         return transformAgenda(document, fileName);
     }
     catch (error) {
-        console.error(`Could not parse texte with error ${error.message}`);
+        console.error(`Could not parse texte with error ${error}`);
     }
     return null;
 }

package/lib/src/model/ameli.d.ts

@@ -1,67 +1,55 @@
-export interface AmendementAuteurRow {
-    group_politique_code: string | null;
-    groupe_politique_id: number | null;
-    groupe_politique_libelle: string | null;
-    groupe_politique_libelle_court: string | null;
-    homonyme: string | null;
-    matricule: string | null;
-    nom: string | null;
-    prenom: string | null;
-    qualite: string | null;
-    rang: string | null;
-}
-export interface AmendementResult {
-    accepte_gouvernement: string | null;
-    alinea: number | null;
-    auteur_est_gouvernement: boolean;
-    au_nom_de_commission: string | null;
-    au_nom_de_groupe_politique: string | null;
-    auteurs: AmendementAuteurRow[];
-    avis_commission: string | null;
-    avis_gouvernement: string | null;
-    code_commission: string | null;
-    date_depot: string | null;
-    discussion_commune_id: number | null;
-    dispositif: string | null;
+import { InferResult } from "kysely";
+export type AmendementResult = InferResult<typeof findAllAmendementsQuery>[0];
+declare const findAllAmendementsQuery: import("kysely").SelectQueryBuilder<{
+    [x: string]: any;
+    [x: number]: any;
+    [x: symbol]: any;
+}, "ameli.amd" | "ameli.sub" | "ameli.typsub" | "ameli.typrect" | "ameli.txt_ameli" | "ameli.etatxt" | "ameli.ses" | "ameli.typses" | "ameli.nat" | "ameli.lec_ameli" | "dosleg.texte" | "dosleg.lecass" | "ameli.mot" | "ameli.avicom" | "ameli.avigvt" | "ameli.sor" | "ameli.irr" | "ameli.grppol_ameli" | "ameli.com_ameli" | "ameli.cab", {
+    [x: string]: any;
+    nature: string;
+    date_depot: string;
     etat: string;
-    etat_texte: string | null;
-    etat_texte_libelle: string | null;
-    id: number;
-    identique_id: number | null;
-    intitule_texte: string | null;
-    lecture: string | null;
-    motion_libelle: string | null;
+    sort: any;
+    url: string;
+    au_nom_de_commission: string;
+    code_commission: string;
+    auteur_est_gouvernement: boolean;
+    scrutin_num: any;
+    auteurs: {
+        prenom: any;
+        homonyme: any;
+        nom: any;
+        qualite: any;
+        rang: any;
+        matricule: any;
+        groupe_politique_id: any;
+        group_politique_code: any;
+        groupe_politique_libelle_court: any;
+        groupe_politique_libelle: any;
+    }[];
+}>;
+export declare function findAllAmendements(fromSession?: number): AsyncIterableIterator<{
+    [x: string]: any;
     nature: string;
-    nature_texte: string | null;
-    nature_texte_libelle: string | null;
-    numero: string;
-    numero_absolu: string | null;
-    numero_adoption_texte: string | null;
-    numero_texte: string | null;
-    objet: string | null;
-    observations: string | null;
-    observations_additionnelles: string | null;
-    ordre: number | null;
-    parent_id: number | null;
-    revision: string | null;
-    session: string;
-    session_libelle: string;
-    signet_dossier_legislatif: string | null;
-    sort: string | null;
-    scrutin_num: string | null;
-    subdivision_commission_id: number | null;
-    subdivision_dupliquee: string | null;
-    subdivision_libelle: string | null;
-    subdivision_libelle_court: string | null;
-    subdivision_mere_id: number | null;
-    subdivision_position_discussion: number | null;
-    subdivision_position_texte: number | null;
-    subdivision_signet: string | null;
-    subdivision_type: string | null;
-    texte_id: number | null;
-    type_rectification: string | null;
-    type_session: string | null;
-    type_texte: string | null;
+    date_depot: string;
+    etat: string;
+    sort: any;
     url: string;
-}
-export declare function findAllAmendements(fromSession?: number): AsyncGenerator<AmendementResult, void, unknown>;
+    au_nom_de_commission: string;
+    code_commission: string;
+    auteur_est_gouvernement: boolean;
+    scrutin_num: any;
+    auteurs: {
+        prenom: any;
+        homonyme: any;
+        nom: any;
+        qualite: any;
+        rang: any;
+        matricule: any;
+        groupe_politique_id: any;
+        group_politique_code: any;
+        groupe_politique_libelle_court: any;
+        groupe_politique_libelle: any;
+    }[];
+}>;
+export {};