@tricoteuses/senat 2.22.15 → 2.23.0

package/lib/scripts/shared/cli_helpers.d.ts

@@ -0,0 +1,95 @@
+export declare const categoriesOption: {
+    alias: string;
+    defaultValue: string[];
+    help: string;
+    multiple: boolean;
+    name: string;
+    type: StringConstructor;
+};
+export declare const dataDirDefaultOption: {
+    defaultOption: boolean;
+    help: string;
+    name: string;
+    type: StringConstructor;
+};
+export declare const fromSessionOption: {
+    defaultValue: number;
+    help: string;
+    name: string;
+    type: NumberConstructor;
+};
+export declare const silentOption: {
+    alias: string;
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const verboseOption: {
+    alias: string;
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const onlyRecentOption: {
+    help: string;
+    name: string;
+    type: NumberConstructor;
+};
+export declare const keepDirOption: {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const cloneOption: {
+    alias: string;
+    help: string;
+    name: string;
+    type: StringConstructor;
+};
+export declare const commitOption: {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const remoteOption: {
+    alias: string;
+    help: string;
+    multiple: boolean;
+    name: string;
+    type: StringConstructor;
+};
+export declare const pullOption: {
+    alias: string;
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const fetchDocumentsOption: {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const parseDocumentsOption: {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const commonOptions: ({
+    defaultOption: boolean;
+    help: string;
+    name: string;
+    type: StringConstructor;
+} | {
+    help: string;
+    name: string;
+    type: NumberConstructor;
+} | {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+} | {
+    alias: string;
+    help: string;
+    name: string;
+    type: StringConstructor;
+})[];

package/lib/scripts/shared/cli_helpers.js

@@ -0,0 +1,91 @@
+export const categoriesOption = {
+    alias: "k",
+    defaultValue: ["All"],
+    help: "categories of datasets to reorganize; default All",
+    multiple: true,
+    name: "categories",
+    type: String,
+};
+export const dataDirDefaultOption = {
+    defaultOption: true,
+    help: "directory containing Sénat open data files",
+    name: "dataDir",
+    type: String,
+};
+export const fromSessionOption = {
+    defaultValue: 2022,
+    help: "session year to retrieve data from; default 2022",
+    name: "fromSession",
+    type: Number,
+};
+export const silentOption = {
+    alias: "s",
+    help: "don't log anything",
+    name: "silent",
+    type: Boolean,
+};
+export const verboseOption = {
+    alias: "v",
+    help: "verbose logs",
+    name: "verbose",
+    type: Boolean,
+};
+export const onlyRecentOption = {
+    help: "retrieve only documents created within the last N days (that are not already downloaded)",
+    name: "only-recent",
+    type: Number,
+};
+export const keepDirOption = {
+    help: "keep directories when cleaning data",
+    name: "keepDir",
+    type: Boolean,
+};
+export const cloneOption = {
+    alias: "C",
+    help: "clone repositories from given group (or organization) git URL",
+    name: "clone",
+    type: String,
+};
+export const commitOption = {
+    help: "commit clean files",
+    name: "commit",
+    type: Boolean,
+};
+export const remoteOption = {
+    alias: "r",
+    help: "push commit to given remote",
+    multiple: true,
+    name: "remote",
+    type: String,
+};
+export const pullOption = {
+    alias: "p",
+    help: "pull repositories before proceeding",
+    name: "pull",
+    type: Boolean,
+};
+export const fetchDocumentsOption = {
+    help: "download documents",
+    name: "fetchDocuments",
+    type: Boolean,
+};
+export const parseDocumentsOption = {
+    help: "parse documents",
+    name: "parseDocuments",
+    type: Boolean,
+};
+export const commonOptions = [
+    categoriesOption,
+    dataDirDefaultOption,
+    fromSessionOption,
+    silentOption,
+    verboseOption,
+    onlyRecentOption,
+    keepDirOption,
+    cloneOption,
+    commitOption,
+    remoteOption,
+    pullOption,
+    fetchDocumentsOption,
+    parseDocumentsOption,
+];

package/lib/scripts/shared/cli_helpers.ts

@@ -0,0 +1,36 @@
+export const categoriesOption = {
+  alias: "k",
+  defaultValue: ["All"],
+  help: "categories of datasets to reorganize",
+  multiple: true,
+  name: "categories",
+  type: String,
+}
+
+export const dataDirDefaultOption = {
+  defaultOption: true,
+  help: "directory containing Sénat open data files",
+  name: "dataDir",
+  type: String,
+}
+
+export const silentOption = {
+  alias: "s",
+  help: "don't log anything",
+  name: "silent",
+  type: Boolean,
+}
+
+export const verboseOption = {
+  alias: "v",
+  help: "verbose logs",
+  name: "verbose",
+  type: Boolean,
+}
+
+export const commonOptions = [
+  categoriesOption,
+  dataDirDefaultOption,
+  silentOption,
+  verboseOption,
+]

package/lib/scripts/shared/util.d.ts

@@ -0,0 +1,4 @@
+export declare function isOptionEmptyOrHasValue(option: string, value: string): boolean;
+export declare function ensureAndClearDirSync(dir: string): void;
+export declare function ensureAndClearDir(path: string): void;
+export declare function fetchWithRetry(url: string, retries?: number, backoff?: number): Promise<Response>;

package/lib/scripts/shared/util.js

@@ -0,0 +1,35 @@
+import fs from "fs-extra";
+import path from "path";
+export function isOptionEmptyOrHasValue(option, value) {
+    return !option || option.length === 0 || option.includes(value);
+}
+export function ensureAndClearDirSync(dir) {
+    fs.ensureDirSync(dir);
+    for (const name of fs.readdirSync(dir)) {
+        fs.rmSync(path.join(dir, name), { recursive: true, force: true });
+    }
+}
+export function ensureAndClearDir(path) {
+    if (!fs.existsSync(path)) {
+        fs.mkdirSync(path, { recursive: true });
+    }
+    else {
+        fs.emptyDirSync(path);
+    }
+}
+export async function fetchWithRetry(url, retries = 3, backoff = 300) {
+    for (let attempt = 0; attempt < retries; attempt++) {
+        try {
+            return await fetch(url);
+        }
+        catch (error) {
+            if (attempt === retries - 1) {
+                throw error;
+            }
+            console.warn(`Fetch attempt ${attempt + 1} for ${url} failed. Retrying in ${backoff}ms…`);
+            await new Promise((resolve) => setTimeout(resolve, backoff));
+            backoff *= 2;
+        }
+    }
+    throw new Error(`Failed to fetch ${url} after ${retries} attempts`);
+}

package/lib/scripts/shared/util.ts

@@ -0,0 +1,33 @@
+import fs from "fs-extra"
+
+export function isOptionEmptyOrHasValue (option: string, value: string) {
+  return !option || option.length === 0 || option.includes(value)
+}
+
+export function ensureAndClearDir (path: string) {
+  if (!fs.existsSync(path)) {
+    fs.mkdirSync(path, { recursive: true })
+  } else {
+    fs.emptyDirSync(path)
+  }
+}
+
+export async function fetchWithRetry (
+  url: string,
+  retries: number = 3,
+  backoff: number = 300,
+): Promise<Response> {
+  for (let attempt = 0; attempt < retries; attempt++) {
+    try {
+      return await fetch(url)
+    } catch (error) {
+      if (attempt === retries - 1) {
+        throw error
+      }
+      console.warn(`Fetch attempt ${attempt + 1} for ${url} failed. Retrying in ${backoff}ms…`)
+      await new Promise((resolve) => setTimeout(resolve, backoff))
+      backoff *= 2
+    }
+  }
+  throw new Error(`Failed to fetch ${url} after ${retries} attempts`)
+}

package/lib/scripts/test_iter_load.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/scripts/test_iter_load.js

@@ -0,0 +1,12 @@
+import { iterLoadSenatRapports } from "../loaders";
+import commandLineArgs from "command-line-args";
+import { dataDirDefaultOption } from "./shared/cli_helpers";
+const optionsDefinitions = [dataDirDefaultOption];
+const options = commandLineArgs(optionsDefinitions);
+const session = 2024;
+const sinceCommit = undefined;
+for (const { item: rapport } of iterLoadSenatRapports(options["dataDir"], session, {
+    sinceCommit: sinceCommit,
+})) {
+    console.log(rapport);
+}

package/lib/src/config.d.ts

@@ -1,5 +1,13 @@
 import "dotenv/config";
 import { z } from "zod";
+export declare const dbSchema: z.ZodObject<{
+    host: z.ZodString;
+    name: z.ZodString;
+    password: z.ZodString;
+    port: z.ZodCoercedNumber<unknown>;
+    user: z.ZodString;
+}, z.core.$strip>;
+export type DbConfig = z.infer<typeof dbSchema>;
 export declare const configSchema: z.ZodObject<{
     db: z.ZodObject<{
         host: z.ZodString;
@@ -8,6 +16,13 @@ export declare const configSchema: z.ZodObject<{
         port: z.ZodCoercedNumber<unknown>;
         user: z.ZodString;
     }, z.core.$strip>;
+    stagingDb: z.ZodObject<{
+        host: z.ZodString;
+        name: z.ZodString;
+        password: z.ZodString;
+        port: z.ZodCoercedNumber<unknown>;
+        user: z.ZodString;
+    }, z.core.$strip>;
 }, z.core.$strip>;
 declare const _default: {
     db: {
@@ -17,5 +32,12 @@ declare const _default: {
         port: number;
         user: string;
     };
+    stagingDb: {
+        host: string;
+        name: string;
+        password: string;
+        port: number;
+        user: string;
+    };
 };
 export default _default;

package/lib/src/config.js

@@ -1,6 +1,6 @@
 import "dotenv/config";
 import { z } from "zod";
-const dbSchema = z.object({
+export const dbSchema = z.object({
     host: z.string().trim().min(1, "Must not be empty"),
     name: z.string().trim().min(1, "Must not be empty"),
     password: z.string().trim().min(1, "Must not be empty"),
@@ -9,19 +9,29 @@ const dbSchema = z.object({
 });
 export const configSchema = z.object({
     db: dbSchema,
+    stagingDb: dbSchema,
 });
 const config = {
     db: {
-        host: process.env["TRICOTEUSES_SENAT_DB_HOST"] || "localhost",
-        name: process.env["TRICOTEUSES_SENAT_DB_NAME"] || "postgres",
-        password: process.env["TRICOTEUSES_SENAT_DB_PASSWORD"] || "opendata",
-        port: process.env["TRICOTEUSES_SENAT_DB_PORT"] || 5432,
-        user: process.env["TRICOTEUSES_SENAT_DB_USER"] || "opendata",
+        host: process.env["DB_HOST"] || "localhost",
+        name: process.env["DB_NAME"] || "senat",
+        password: process.env["DB_PASSWORD"] || "opendata",
+        port: process.env["DB_PORT"] || 5432,
+        user: process.env["DB_USER"] || "opendata",
+    },
+    stagingDb: {
+        host: process.env["STAGING_DB_HOST"] || process.env["DB_HOST"] || "localhost",
+        name: process.env["STAGING_DB_NAME"] || "senat_staging",
+        password: process.env["STAGING_DB_PASSWORD"] || process.env["DB_PASSWORD"] || "opendata",
+        port: process.env["STAGING_DB_PORT"] || process.env["DB_PORT"] || 5432,
+        user: process.env["STAGING_DB_USER"] || process.env["DB_USER"] || "opendata",
     },
 };
 const result = configSchema.safeParse(config);
 if (!result.success) {
-    console.error(`Error in configuration:\n${JSON.stringify(config, null, 2)}\nError:\n${JSON.stringify(result.error.issues, null, 2)}`);
+    const issues = JSON.stringify(result.error.issues, null, 2);
+    const serializedConfig = JSON.stringify(config, null, 2);
+    console.error(`Error in configuration:\n${serializedConfig}\nError:\n${issues}`);
     process.exit(-1);
 }
 export default result.data;

package/lib/src/conversion_textes.js

@@ -186,7 +186,11 @@ p.has-alinea {
     <div class="header-type">${metadata.type || ""}</div>
     <div class="header-authors">${metadata.authors || ""}</div>
     ${metadata.commission
-        ? `<div class="header-commission">Envoyée à la ${metadata.commission.toLowerCase()}, sous réserve de la constitution éventuelle d'une commission spéciale dans les conditions prévues par le Règlement.</div>`
+        ? [
+            `<div class="header-commission">Envoyée à la ${metadata.commission.toLowerCase()},`,
+            "sous réserve de la constitution éventuelle d'une commission spéciale dans les conditions prévues",
+            "par le Règlement.</div>",
+        ].join(" ")
         : ""}
   </div>
   <h1>${metadata.title || ""}</h1>

package/lib/src/databases.d.ts

@@ -1,2 +1,3 @@
 import { Kysely } from "kysely";
-export declare const dbSenat: Kysely<any>;
+import type { LooseKyselyDatabase } from "./typings/kysely-loose";
+export declare const dbSenat: Kysely<LooseKyselyDatabase>;

package/lib/src/databases_postgres.d.ts

@@ -0,0 +1,4 @@
+import postgres from "postgres";
+export type SqlValue = postgres.ParameterOrJSON<never>;
+export declare const sql: postgres.Sql<{}>;
+export declare function streamUnsafeQuery<T>(query: string, params?: readonly unknown[], batchSize?: number): AsyncGenerator<T, void, unknown>;

package/lib/src/databases_postgres.js

@@ -0,0 +1,23 @@
+import postgres from "postgres";
+import config from "./config";
+export const sql = postgres({
+    database: config.db.name,
+    host: config.db.host,
+    password: config.db.password,
+    port: config.db.port,
+    transform: {
+        undefined: null,
+    },
+    user: config.db.user,
+});
+function toSqlValue(value) {
+    return value;
+}
+export async function* streamUnsafeQuery(query, params = [], batchSize = 100) {
+    const values = params.map(toSqlValue);
+    for await (const rows of sql.unsafe(query, values).cursor(batchSize)) {
+        for (const row of rows) {
+            yield row;
+        }
+    }
+}

package/lib/src/datasets.d.ts

@@ -10,6 +10,10 @@ export interface Dataset {
             columns: string[];
         }>;
     };
+    mergeKeys?: {
+        [table: string]: string[];
+    };
+    rowMultisetMergeTables?: string[];
 }
 export interface Datasets {
     ameli: Dataset;

package/lib/src/datasets.js

@@ -167,6 +167,12 @@ export const datasets = {
                 { name: "idx_senmat", columns: ["senmat"] },
             ],
         },
+        mergeKeys: {
+            corscr: ["sesann", "scrnum", "corscrord"],
+            docsea: ["evtseacle", "docseaord"],
+            gen: ["gencod"],
+        },
+        rowMultisetMergeTables: ["amescr"],
     },
     questions: {
         database: "questions",
@@ -183,6 +189,7 @@ export const datasets = {
             sortquestion: [{ name: "idx_sorquecod", columns: ["sorquecod"] }],
             etatquestion: [{ name: "idx_etaquecod", columns: ["etaquecod"] }],
         },
+        rowMultisetMergeTables: ["tam_reponses"],
     },
     sens: {
         database: "sens",
@@ -213,12 +220,19 @@ export const datasets = {
                 { name: "idx_typmancod", columns: ["typmancod"] },
             ],
         },
+        mergeKeys: {
+            activite: ["actid"],
+        },
     },
 };
+function getEnabledDatasetByName(datasetName) {
+    const enabledDataset = EnabledDatasets[datasetName];
+    return typeof enabledDataset === "number" ? enabledDataset : undefined;
+}
 export function getEnabledDatasets(categories) {
-    categories.map((datasetName) => assert.notStrictEqual(EnabledDatasets[datasetName], undefined, `Unknown name of dataset: ${datasetName}`));
+    categories.map((datasetName) => assert.notStrictEqual(getEnabledDatasetByName(datasetName), undefined, `Unknown name of dataset: ${datasetName}`));
     return categories.reduce((enabledDatasets, datasetName) => {
-        const enabledDataset = EnabledDatasets[datasetName];
+        const enabledDataset = getEnabledDatasetByName(datasetName);
         return enabledDatasets | (enabledDataset || EnabledDatasets.None);
     }, EnabledDatasets.None);
 }

package/lib/src/git.d.ts

@@ -1,6 +1,7 @@
 export declare function initRepo(repositoryDir: string): void;
 export declare function commit(repositoryDir: string, message: string): boolean;
 export declare function commitAndPush(repositoryDir: string, message: string, remotes?: string[]): number;
+export declare function pull(repositoryDir: string, remote?: string, branch?: string): boolean;
 export declare function resetAndPull(gitDir: string): boolean;
 export declare function clone(gitGroupUrl: string | undefined, gitName: string, workingDir: string): void;
 export declare function run(repositoryDir: string, args: string, verbose?: boolean): string;

package/lib/src/git.js

@@ -5,6 +5,15 @@ const MAXBUFFER = 50 * 1024 * 1024;
 const GIT_LOCK_RETRY_DELAY_MS = 1000;
 const GIT_LOCK_RETRY_COUNT = 5;
 const GIT_LOCK_STALE_AFTER_MS = 2 * 60 * 1000;
+function asExecSyncError(error) {
+    return error;
+}
+function outputToString(output) {
+    if (output == null) {
+        return "";
+    }
+    return Buffer.isBuffer(output) ? output.toString("utf-8") : output;
+}
 function sleep(ms) {
     Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 }
@@ -12,7 +21,7 @@ function getIndexLockPath(repositoryDir) {
     return path.join(repositoryDir, ".git", "index.lock");
 }
 function isIndexLockError(error) {
-    const stderr = String(error?.stderr || "");
+    const stderr = error instanceof Error ? outputToString(asExecSyncError(error).stderr) : "";
     return /index\.lock': File exists\./.test(stderr);
 }
 function removeStaleIndexLock(repositoryDir) {
@@ -81,9 +90,9 @@ export function commit(repositoryDir, message) {
         });
         return true;
     }
-    catch (childProcess) {
-        if (childProcess.stdout === null ||
-            !/nothing to commit|rien à valider/.test(childProcess.stdout)) {
+    catch (error) {
+        const childProcess = asExecSyncError(error);
+        if (childProcess.stdout === null || !/nothing to commit|rien à valider/.test(outputToString(childProcess.stdout))) {
             console.error(childProcess.output);
             throw childProcess;
         }
@@ -102,10 +111,11 @@ export function commitAndPush(repositoryDir, message, remotes) {
                     stdio: ["ignore", "ignore", "pipe"],
                 });
             }
-            catch (childProcess) {
+            catch (error) {
+                const childProcess = asExecSyncError(error);
                 // Don't stop when push fails.
                 console.error(childProcess.output);
-                exitCode = childProcess.status;
+                exitCode = childProcess.status ?? 1;
             }
         }
     }
@@ -115,6 +125,27 @@ export function commitAndPush(repositoryDir, message, remotes) {
     }
     return exitCode;
 }
+export function pull(repositoryDir, remote = "origin", branch = "master") {
+    initRepo(repositoryDir);
+    const statusOutput = execSync("git status --porcelain", {
+        cwd: repositoryDir,
+        env: process.env,
+        encoding: "utf-8",
+        stdio: ["ignore", "pipe", "pipe"],
+        maxBuffer: MAXBUFFER,
+    }).trim();
+    if (statusOutput.length > 0) {
+        return false;
+    }
+    execSync(`git pull --rebase ${remote} ${branch}`, {
+        cwd: repositoryDir,
+        env: process.env,
+        encoding: "utf-8",
+        stdio: ["ignore", "ignore", "pipe"],
+        maxBuffer: MAXBUFFER,
+    });
+    return true;
+}
 export function resetAndPull(gitDir) {
     execSync("git reset --hard origin/master", {
         cwd: gitDir,
@@ -154,9 +185,11 @@ export function run(repositoryDir, args, verbose) {
             console.log(output);
         return output;
     }
-    catch (childProcess) {
-        for (const output of ["stdout", "stderr"])
-            console.error(`${output}: ${childProcess[output]}`);
+    catch (error) {
+        const childProcess = asExecSyncError(error);
+        for (const output of ["stdout", "stderr"]) {
+            console.error(`${output}: ${outputToString(childProcess[output])}`);
+        }
         throw childProcess;
     }
 }
@@ -175,8 +208,9 @@ export function test(repositoryDir, args, verbose) {
             console.log(output);
         return true;
     }
-    catch (childProcess) {
-        if (childProcess.status != 0)
+    catch (error) {
+        const childProcess = asExecSyncError(error);
+        if (childProcess.status !== 0)
             return false;
         throw childProcess;
     }

package/lib/src/loaders.js

@@ -69,7 +69,8 @@ function* iterLoadSenatItems(dataDir, dataName, legislatureOrSession, subDir, {
             const itemJson = fs.readFileSync(filePath, { encoding: "utf8" });
             item = JSON.parse(itemJson);
         }
-        catch (err) {
+        catch (error) {
+            const err = error;
             console.warn(`[iterLoadSenatItems] skipped invalid JSON: ${filePath} (${err.message})`);
             continue;
         }
@@ -180,6 +181,9 @@ export function* iterLoadSenatTextes(dataDir, session, options = {}) {
         const texteItem = iterItem;
         const texte = texteItem.item;
         const texteId = texte["id"];
+        if (!texteId) {
+            continue;
+        }
         const { item: texteContent } = loadSenatTexteContent(dataDir, texte["session"], texteId);
         if (texteContent) {
             texteItem.item = { ...texteContent, ...texteItem.item };
@@ -222,9 +226,11 @@ export function* iterLoadSenatAgendas(dataDir, session) {
         const gr = raw;
         if (!gr.uid || !gr.date || !gr.titre)
             continue;
-        if (!Array.isArray(gr.events))
-            gr.events = [];
-        yield { item: gr };
+        const reunion = {
+            ...gr,
+            events: Array.isArray(gr.events) ? gr.events : [],
+        };
+        yield { item: reunion };
     }
 }
 export function* iterLoadSenatCirconscriptions(dataDir, options = {}) {

package/lib/src/model/agenda.js

@@ -113,7 +113,7 @@ function transformAgenda(document, fileName) {
         const { startTime, endTime } = getStartAndEndTimes(timeOriginal, date);
         const titre = eventElement.querySelector(".titre")?.textContent?.trim() || "";
         const organe = eventElement.querySelector(".organe")?.textContent?.trim() || null;
-        const objet = eventElement.querySelector(".objet")?.textContent?.trim()?.replace(/^- /, "") || null;
+        const objet = eventElement.querySelector(".objet")?.textContent?.trim()?.replace(/^- /, "") || null;
         const lieu = eventElement.querySelector(".lieu")?.textContent || null;
         const videoElement = eventElement.querySelector(".video");
         const urlDossierSenat = getUrlDossierSenat(eventElement.querySelectorAll(".lien a"));
@@ -142,7 +142,7 @@ export async function parseAgendaFromFile(htmlFilePath) {
         return transformAgenda(document, fileName);
     }
     catch (error) {
-        console.error(`Could not parse texte with error ${error}`);
+        console.error(`Could not parse texte with error ${error.message}`);
     }
     return null;
 }