@tricoteuses/senat 3.1.21 → 3.1.22

package/lib/scripts/shared/cli_helpers.d.ts

@@ -0,0 +1,95 @@
+export declare const categoriesOption: {
+    alias: string;
+    defaultValue: string[];
+    help: string;
+    multiple: boolean;
+    name: string;
+    type: StringConstructor;
+};
+export declare const dataDirDefaultOption: {
+    defaultOption: boolean;
+    help: string;
+    name: string;
+    type: StringConstructor;
+};
+export declare const fromSessionOption: {
+    defaultValue: number;
+    help: string;
+    name: string;
+    type: NumberConstructor;
+};
+export declare const silentOption: {
+    alias: string;
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const verboseOption: {
+    alias: string;
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const onlyRecentOption: {
+    help: string;
+    name: string;
+    type: NumberConstructor;
+};
+export declare const keepDirOption: {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const cloneOption: {
+    alias: string;
+    help: string;
+    name: string;
+    type: StringConstructor;
+};
+export declare const commitOption: {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const remoteOption: {
+    alias: string;
+    help: string;
+    multiple: boolean;
+    name: string;
+    type: StringConstructor;
+};
+export declare const pullOption: {
+    alias: string;
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const fetchDocumentsOption: {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const parseDocumentsOption: {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const commonOptions: ({
+    defaultOption: boolean;
+    help: string;
+    name: string;
+    type: StringConstructor;
+} | {
+    help: string;
+    name: string;
+    type: NumberConstructor;
+} | {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+} | {
+    alias: string;
+    help: string;
+    name: string;
+    type: StringConstructor;
+})[];

package/lib/scripts/shared/cli_helpers.js

@@ -0,0 +1,91 @@
+export const categoriesOption = {
+    alias: "k",
+    defaultValue: ["All"],
+    help: "categories of datasets to reorganize; default All",
+    multiple: true,
+    name: "categories",
+    type: String,
+};
+export const dataDirDefaultOption = {
+    defaultOption: true,
+    help: "directory containing Sénat open data files",
+    name: "dataDir",
+    type: String,
+};
+export const fromSessionOption = {
+    defaultValue: 2022,
+    help: "session year to retrieve data from; default 2022",
+    name: "fromSession",
+    type: Number,
+};
+export const silentOption = {
+    alias: "s",
+    help: "don't log anything",
+    name: "silent",
+    type: Boolean,
+};
+export const verboseOption = {
+    alias: "v",
+    help: "verbose logs",
+    name: "verbose",
+    type: Boolean,
+};
+export const onlyRecentOption = {
+    help: "retrieve only documents created within the last N days (that are not already downloaded)",
+    name: "only-recent",
+    type: Number,
+};
+export const keepDirOption = {
+    help: "keep directories when cleaning data",
+    name: "keepDir",
+    type: Boolean,
+};
+export const cloneOption = {
+    alias: "C",
+    help: "clone repositories from given group (or organization) git URL",
+    name: "clone",
+    type: String,
+};
+export const commitOption = {
+    help: "commit clean files",
+    name: "commit",
+    type: Boolean,
+};
+export const remoteOption = {
+    alias: "r",
+    help: "push commit to given remote",
+    multiple: true,
+    name: "remote",
+    type: String,
+};
+export const pullOption = {
+    alias: "p",
+    help: "pull repositories before proceeding",
+    name: "pull",
+    type: Boolean,
+};
+export const fetchDocumentsOption = {
+    help: "download documents",
+    name: "fetchDocuments",
+    type: Boolean,
+};
+export const parseDocumentsOption = {
+    help: "parse documents",
+    name: "parseDocuments",
+    type: Boolean,
+};
+export const commonOptions = [
+    categoriesOption,
+    dataDirDefaultOption,
+    fromSessionOption,
+    silentOption,
+    verboseOption,
+    onlyRecentOption,
+    keepDirOption,
+    cloneOption,
+    commitOption,
+    remoteOption,
+    pullOption,
+    fetchDocumentsOption,
+    parseDocumentsOption,
+];

package/lib/scripts/shared/cli_helpers.ts

@@ -0,0 +1,36 @@
+export const categoriesOption = {
+  alias: "k",
+  defaultValue: ["All"],
+  help: "categories of datasets to reorganize",
+  multiple: true,
+  name: "categories",
+  type: String,
+}
+
+export const dataDirDefaultOption = {
+  defaultOption: true,
+  help: "directory containing Sénat open data files",
+  name: "dataDir",
+  type: String,
+}
+
+export const silentOption = {
+  alias: "s",
+  help: "don't log anything",
+  name: "silent",
+  type: Boolean,
+}
+
+export const verboseOption = {
+  alias: "v",
+  help: "verbose logs",
+  name: "verbose",
+  type: Boolean,
+}
+
+export const commonOptions = [
+  categoriesOption,
+  dataDirDefaultOption,
+  silentOption,
+  verboseOption,
+]

package/lib/scripts/shared/util.d.ts

@@ -0,0 +1,4 @@
+export declare function isOptionEmptyOrHasValue(option: string, value: string): boolean;
+export declare function ensureAndClearDirSync(dir: string): void;
+export declare function ensureAndClearDir(path: string): void;
+export declare function fetchWithRetry(url: string, retries?: number, backoff?: number): Promise<Response>;

package/lib/scripts/shared/util.js

@@ -0,0 +1,35 @@
+import fs from "fs-extra";
+import path from "path";
+export function isOptionEmptyOrHasValue(option, value) {
+    return !option || option.length === 0 || option.includes(value);
+}
+export function ensureAndClearDirSync(dir) {
+    fs.ensureDirSync(dir);
+    for (const name of fs.readdirSync(dir)) {
+        fs.rmSync(path.join(dir, name), { recursive: true, force: true });
+    }
+}
+export function ensureAndClearDir(path) {
+    if (!fs.existsSync(path)) {
+        fs.mkdirSync(path, { recursive: true });
+    }
+    else {
+        fs.emptyDirSync(path);
+    }
+}
+export async function fetchWithRetry(url, retries = 3, backoff = 300) {
+    for (let attempt = 0; attempt < retries; attempt++) {
+        try {
+            return await fetch(url);
+        }
+        catch (error) {
+            if (attempt === retries - 1) {
+                throw error;
+            }
+            console.warn(`Fetch attempt ${attempt + 1} for ${url} failed. Retrying in ${backoff}ms…`);
+            await new Promise((resolve) => setTimeout(resolve, backoff));
+            backoff *= 2;
+        }
+    }
+    throw new Error(`Failed to fetch ${url} after ${retries} attempts`);
+}

package/lib/scripts/shared/util.ts

@@ -0,0 +1,33 @@
+import fs from "fs-extra"
+
+export function isOptionEmptyOrHasValue (option: string, value: string) {
+  return !option || option.length === 0 || option.includes(value)
+}
+
+export function ensureAndClearDir (path: string) {
+  if (!fs.existsSync(path)) {
+    fs.mkdirSync(path, { recursive: true })
+  } else {
+    fs.emptyDirSync(path)
+  }
+}
+
+export async function fetchWithRetry (
+  url: string,
+  retries: number = 3,
+  backoff: number = 300,
+): Promise<Response> {
+  for (let attempt = 0; attempt < retries; attempt++) {
+    try {
+      return await fetch(url)
+    } catch (error) {
+      if (attempt === retries - 1) {
+        throw error
+      }
+      console.warn(`Fetch attempt ${attempt + 1} for ${url} failed. Retrying in ${backoff}ms…`)
+      await new Promise((resolve) => setTimeout(resolve, backoff))
+      backoff *= 2
+    }
+  }
+  throw new Error(`Failed to fetch ${url} after ${retries} attempts`)
+}

package/lib/scripts/test_iter_load.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/scripts/test_iter_load.js

@@ -0,0 +1,12 @@
+import { iterLoadSenatRapports } from "../loaders";
+import commandLineArgs from "command-line-args";
+import { dataDirDefaultOption } from "./shared/cli_helpers";
+const optionsDefinitions = [dataDirDefaultOption];
+const options = commandLineArgs(optionsDefinitions);
+const session = 2024;
+const sinceCommit = undefined;
+for (const { item: rapport } of iterLoadSenatRapports(options["dataDir"], session, {
+    sinceCommit: sinceCommit,
+})) {
+    console.log(rapport);
+}

package/lib/src/ameli.d.ts

@@ -0,0 +1,66 @@
+export interface AmendementAuteurRow {
+    group_politique_code: string | null;
+    groupe_politique_id: number | null;
+    groupe_politique_libelle: string | null;
+    groupe_politique_libelle_court: string | null;
+    homonyme: string | null;
+    matricule: string | null;
+    nom: string | null;
+    prenom: string | null;
+    qualite: string | null;
+    rang: string | null;
+}
+export interface AmendementResult {
+    accepte_gouvernement: string | null;
+    alinea: number | null;
+    auteur_est_gouvernement: boolean;
+    au_nom_de_commission: string | null;
+    au_nom_de_groupe_politique: string | null;
+    auteurs: AmendementAuteurRow[];
+    avis_commission: string | null;
+    avis_gouvernement: string | null;
+    code_commission: string | null;
+    date_depot: string | null;
+    discussion_commune_id: number | null;
+    dispositif: string | null;
+    etat: string;
+    etat_texte: string | null;
+    etat_texte_libelle: string | null;
+    id: number;
+    identique_id: number | null;
+    intitule_texte: string | null;
+    lecture: string | null;
+    motion_libelle: string | null;
+    nature: string;
+    nature_texte: string | null;
+    nature_texte_libelle: string | null;
+    numero: string;
+    numero_absolu: string | null;
+    numero_adoption_texte: string | null;
+    numero_texte: string | null;
+    objet: string | null;
+    observations: string | null;
+    observations_additionnelles: string | null;
+    ordre: number | null;
+    parent_id: number | null;
+    revision: string | null;
+    session: string;
+    session_libelle: string;
+    signet_dossier_legislatif: string | null;
+    sort: string | null;
+    scrutin_num: string | null;
+    subdivision_commission_id: number | null;
+    subdivision_dupliquee: string | null;
+    subdivision_libelle: string | null;
+    subdivision_libelle_court: string | null;
+    subdivision_mere_id: number | null;
+    subdivision_position_discussion: number | null;
+    subdivision_position_texte: number | null;
+    subdivision_signet: string | null;
+    subdivision_type: string | null;
+    texte_id: number | null;
+    type_rectification: string | null;
+    type_session: string | null;
+    type_texte: string | null;
+    url: string;
+}

package/lib/src/ameli.js

@@ -0,0 +1 @@
+export {};

package/lib/src/config.d.ts

@@ -0,0 +1,43 @@
+import "dotenv/config";
+import { z } from "zod";
+export declare const dbSchema: z.ZodObject<{
+    host: z.ZodString;
+    name: z.ZodString;
+    password: z.ZodString;
+    port: z.ZodCoercedNumber<unknown>;
+    user: z.ZodString;
+}, z.core.$strip>;
+export type DbConfig = z.infer<typeof dbSchema>;
+export declare const configSchema: z.ZodObject<{
+    db: z.ZodObject<{
+        host: z.ZodString;
+        name: z.ZodString;
+        password: z.ZodString;
+        port: z.ZodCoercedNumber<unknown>;
+        user: z.ZodString;
+    }, z.core.$strip>;
+    stagingDb: z.ZodObject<{
+        host: z.ZodString;
+        name: z.ZodString;
+        password: z.ZodString;
+        port: z.ZodCoercedNumber<unknown>;
+        user: z.ZodString;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+declare const _default: {
+    db: {
+        host: string;
+        name: string;
+        password: string;
+        port: number;
+        user: string;
+    };
+    stagingDb: {
+        host: string;
+        name: string;
+        password: string;
+        port: number;
+        user: string;
+    };
+};
+export default _default;

package/lib/src/config.js

@@ -0,0 +1,37 @@
+import "dotenv/config";
+import { z } from "zod";
+export const dbSchema = z.object({
+    host: z.string().trim().min(1, "Must not be empty"),
+    name: z.string().trim().min(1, "Must not be empty"),
+    password: z.string().trim().min(1, "Must not be empty"),
+    port: z.coerce.number().int().min(0).max(65535),
+    user: z.string().trim().min(1, "Must not be empty"),
+});
+export const configSchema = z.object({
+    db: dbSchema,
+    stagingDb: dbSchema,
+});
+const config = {
+    db: {
+        host: process.env["DB_HOST"] || "localhost",
+        name: process.env["DB_NAME"] || "senat",
+        password: process.env["DB_PASSWORD"] || "opendata",
+        port: process.env["DB_PORT"] || 5432,
+        user: process.env["DB_USER"] || "opendata",
+    },
+    stagingDb: {
+        host: process.env["STAGING_DB_HOST"] || process.env["DB_HOST"] || "localhost",
+        name: process.env["STAGING_DB_NAME"] || "senat_staging",
+        password: process.env["STAGING_DB_PASSWORD"] || process.env["DB_PASSWORD"] || "opendata",
+        port: process.env["STAGING_DB_PORT"] || process.env["DB_PORT"] || 5432,
+        user: process.env["STAGING_DB_USER"] || process.env["DB_USER"] || "opendata",
+    },
+};
+const result = configSchema.safeParse(config);
+if (!result.success) {
+    const issues = JSON.stringify(result.error.issues, null, 2);
+    const serializedConfig = JSON.stringify(config, null, 2);
+    console.error(`Error in configuration:\n${serializedConfig}\nError:\n${issues}`);
+    process.exit(-1);
+}
+export default result.data;

package/lib/src/conversion_textes.d.ts

@@ -0,0 +1,11 @@
+export interface SenatMetadata {
+    number: string | null;
+    session: string | null;
+    date: string | null;
+    type: string | null;
+    authors: string | null;
+    title: string | null;
+    commission: string | null;
+}
+export declare function extractMetadata(xmlDoc: Document): SenatMetadata;
+export declare function convertSenatXmlToHtml(texteXml: string, outputFilePath: string): Promise<void>;