@tricoteuses/senat 1.1.1 → 1.3.1

package/lib/src/config.d.ts

@@ -0,0 +1,2 @@
+declare const validConfig: any;
+export default validConfig;

package/lib/src/config.mjs

@@ -0,0 +1,16 @@
+require("dotenv").config();
+import { validateConfig } from "./validators/config";
+const config = {
+    db: {
+        host: process.env.TRICOTEUSES_SENAT_DB_HOST || "localhost",
+        password: process.env.TRICOTEUSES_SENAT_DB_PASSWORD || "opendata",
+        port: process.env.TRICOTEUSES_SENAT_DB_PORT || 5432,
+        user: process.env.TRICOTEUSES_SENAT_DB_USER || "opendata",
+    },
+};
+const [validConfig, error] = validateConfig(config);
+if (error !== null) {
+    console.error(`Error in configuration:\n${JSON.stringify(validConfig, null, 2)}\nError:\n${JSON.stringify(error, null, 2)}`);
+    process.exit(-1);
+}
+export default validConfig;

package/lib/src/databases.d.ts

@@ -0,0 +1,18 @@
+import { Kysely } from "kysely";
+import PgPromise from "pg-promise";
+import { DB as DoslegDatabase } from "./raw_types_kysely/dosleg";
+import { DB as QuestionsDatabase } from "./raw_types_kysely/questions";
+import { DB as SensDatabase } from "./raw_types_kysely/sens";
+export declare const pgPromise: PgPromise.IMain<{}, import("pg-promise/typescript/pg-subset").IClient>;
+export declare const dbNames: string[];
+export declare const dbByName: {
+    [name: string]: any;
+};
+export declare const dbDosleg: Kysely<DoslegDatabase>;
+export declare const dbQuestions: Kysely<QuestionsDatabase>;
+export declare const dbSens: Kysely<SensDatabase>;
+export declare let dbSharedConnectionObjectByName: {
+    [name: string]: PgPromise.IConnected<unknown, any>;
+};
+export declare function checkDatabase(name: string): Promise<any>;
+export declare function checkDatabases(): Promise<void>;

package/lib/src/databases.mjs

@@ -0,0 +1,55 @@
+import { Kysely, PostgresDialect } from "kysely";
+import * as pg from "pg";
+import Cursor from "pg-cursor";
+import PgPromise from "pg-promise";
+import config from "./config";
+import { datasets } from "./datasets";
+const int8TypeId = 20;
+// Map int8 to number.
+pg.types.setTypeParser(int8TypeId, (val) => {
+    return parseInt(val, 10);
+});
+export const pgPromise = PgPromise();
+export const dbNames = ["ameli", "debats", "dosleg"];
+export const dbByName = {};
+for (const name of dbNames) {
+    dbByName[name] = pgPromise({
+        database: name,
+        host: config.db.host,
+        password: config.db.password,
+        port: config.db.port,
+        user: config.db.user,
+    });
+}
+function createDb(database, schema) {
+    const dialect = new PostgresDialect({
+        pool: new pg.Pool({
+            database: database,
+            host: config.db.host,
+            user: config.db.user,
+            password: config.db.password,
+            port: config.db.port,
+            max: 10,
+        }),
+        cursor: Cursor,
+    });
+    return new Kysely({
+        log: ["error"],
+        dialect: dialect,
+    }).withSchema(schema);
+}
+export const dbDosleg = createDb(datasets.dosleg.database, datasets.dosleg.schema);
+export const dbQuestions = createDb(datasets.questions.database, datasets.questions.schema);
+export const dbSens = createDb(datasets.sens.database, datasets.sens.schema);
+export let dbSharedConnectionObjectByName = {};
+export async function checkDatabase(name) {
+    // Check that database exists.
+    const db = dbByName[name];
+    dbSharedConnectionObjectByName[name] = await db.connect();
+    return db;
+}
+export async function checkDatabases() {
+    for (const name of dbNames) {
+        await checkDatabase(name);
+    }
+}

package/lib/src/datasets.d.ts

@@ -0,0 +1,28 @@
+export interface Dataset {
+    database: string;
+    repairEncoding: boolean;
+    repairZip?: (dataset: Dataset, dataDir: string) => void;
+    schema: string;
+    title: string;
+    url: string;
+}
+export interface Datasets {
+    ameli: Dataset;
+    debats: Dataset;
+    dosleg: Dataset;
+    questions: Dataset;
+    sens: Dataset;
+}
+export declare enum EnabledDatasets {
+    None = 0,
+    Ameli = 1,
+    Debats = 2,
+    DosLeg = 4,
+    Questions = 8,
+    Sens = 16,
+    PhotosSenateurs = 32,
+    All = 63
+}
+export declare const datasets: Datasets;
+export declare function getEnabledDatasets(categories: any): EnabledDatasets;
+export declare function getChosenFromEnabledDatasets(categories: any): Dataset[];

package/lib/src/datasets.mjs

@@ -0,0 +1,78 @@
+import fs from "fs-extra";
+import path from "path";
+import assert from "assert";
+export var EnabledDatasets;
+(function (EnabledDatasets) {
+    EnabledDatasets[EnabledDatasets["None"] = 0] = "None";
+    EnabledDatasets[EnabledDatasets["Ameli"] = 1] = "Ameli";
+    EnabledDatasets[EnabledDatasets["Debats"] = 2] = "Debats";
+    EnabledDatasets[EnabledDatasets["DosLeg"] = 4] = "DosLeg";
+    EnabledDatasets[EnabledDatasets["Questions"] = 8] = "Questions";
+    EnabledDatasets[EnabledDatasets["Sens"] = 16] = "Sens";
+    EnabledDatasets[EnabledDatasets["PhotosSenateurs"] = 32] = "PhotosSenateurs";
+    EnabledDatasets[EnabledDatasets["All"] = 63] = "All";
+})(EnabledDatasets || (EnabledDatasets = {}));
+export const datasets = {
+    ameli: {
+        database: "ameli",
+        repairEncoding: false,
+        repairZip: (dataset, dataDir) => {
+            const sqlFilename = `${dataset.database}.sql`;
+            const sqlFilePath = path.join(dataDir, sqlFilename);
+            fs.removeSync(sqlFilePath);
+            fs.moveSync(path.join(dataDir, "var", "opt", "opendata", sqlFilename), sqlFilePath);
+        },
+        schema: "public",
+        title: "Amendements",
+        url: "https://data.senat.fr/data/ameli/ameli.zip",
+    },
+    debats: {
+        database: "debats",
+        repairEncoding: true,
+        schema: "public",
+        title: "Informations relatives aux comptes rendus intégraux de la séance publique",
+        url: "https://data.senat.fr/data/debats/debats.zip",
+    },
+    dosleg: {
+        database: "dosleg",
+        repairEncoding: true,
+        schema: "public",
+        title: "Dossiers législatifs",
+        url: "https://data.senat.fr/data/dosleg/dosleg.zip",
+    },
+    questions: {
+        database: "questions",
+        repairEncoding: true,
+        schema: "questions",
+        title: "Questions écrites et orales posées par les sénateurs au Gouvernement",
+        url: "https://data.senat.fr/data/questions/questions.zip",
+    },
+    sens: {
+        database: "sens",
+        repairEncoding: true,
+        repairZip: (dataset, dataDir) => {
+            const sqlFilename = `${dataset.database}.sql`;
+            const sqlFilePath = path.join(dataDir, sqlFilename);
+            fs.removeSync(sqlFilePath);
+            fs.moveSync(path.join(dataDir, "export_sens.sql"), sqlFilePath);
+        },
+        schema: "public",
+        title: "Sénateurs (y compris organes et présence)",
+        url: "https://data.senat.fr/data/senateurs/export_sens.zip",
+    },
+};
+export function getEnabledDatasets(categories) {
+    categories.map((datasetName) => assert.notStrictEqual(EnabledDatasets[datasetName], undefined, `Unknown name of dataset: ${datasetName}`));
+    return categories.reduce((enabledDatasets, datasetName) => enabledDatasets |
+        EnabledDatasets[datasetName], EnabledDatasets.None);
+}
+export function getChosenFromEnabledDatasets(categories) {
+    const enabledDatasets = getEnabledDatasets(categories);
+    return [
+        enabledDatasets & EnabledDatasets.Ameli ? datasets.ameli : null,
+        enabledDatasets & EnabledDatasets.Debats ? datasets.debats : null,
+        enabledDatasets & EnabledDatasets.DosLeg ? datasets.dosleg : null,
+        enabledDatasets & EnabledDatasets.Questions ? datasets.questions : null,
+        enabledDatasets & EnabledDatasets.Sens ? datasets.sens : null,
+    ].filter((dataset) => dataset !== null);
+}

package/lib/src/fields.d.ts

@@ -0,0 +1,10 @@
+export declare function parseIntFields(fieldsNames: string[], entry: {
+    [fieldName: string]: any;
+} | null): {
+    [fieldName: string]: any;
+} | null;
+export declare function trimFieldsRight(fieldsNames: string[], entry: {
+    [fieldName: string]: any;
+} | null): {
+    [fieldName: string]: any;
+} | null;

package/lib/src/fields.mjs

@@ -0,0 +1,22 @@
+export function parseIntFields(fieldsNames, entry) {
+    if (entry === null) {
+        return null;
+    }
+    for (const fieldName of fieldsNames) {
+        if (entry[fieldName] !== null) {
+            entry[fieldName] = parseInt(entry[fieldName]);
+        }
+    }
+    return entry;
+}
+export function trimFieldsRight(fieldsNames, entry) {
+    if (entry === null) {
+        return null;
+    }
+    for (const fieldName of fieldsNames) {
+        if (entry[fieldName] != null) {
+            entry[fieldName] = entry[fieldName].trimEnd();
+        }
+    }
+    return entry;
+}

package/lib/src/index.d.ts

@@ -0,0 +1,8 @@
+export { allFollows, Aggregator, } from "./aggregates";
+export { insertAudReferences, insertDateSeanceReferences, insertAuteurReferences, insertDocAttReferences, insertEcrReferences, insertLecassReferences, insertLecassrapReferences, insertLectureReferences, insertLoiReferences, insertRapReferences, insertTexteReferences, OutputData, } from "./inserters";
+export { Ses, sesFieldsToParseInt, sesFieldsToTrim, Sub, subFieldsToParseInt, subFieldsToTrim, TxtAmeli, txtAmeliFieldsToTrim, } from "./types/ameli";
+export { Debat, debatsFieldsToTrim, LecAssDeb, lecassdebFieldsToTrim, } from "./types/debats";
+export { Ass, assFieldsToTrim, Aud, audFieldsToTrim, Auteur, auteurFieldsToTrim, DateSeance, dateSeanceFieldsToTrim, DecCoc, deccocFieldsToTrim, DenRap, denrapFieldsToTrim, DocAtt, docattFieldsToParseInt, docattFieldsToTrim, Ecr, ecrFieldsToTrim, EtaLoi, etaloiFieldsToTrim, LecAss, lecassFieldsToTrim, LecAssRap, lecassrapFieldsToTrim, Lecture, lectureFieldsToTrim, Loi, loiFieldsToTrim, Org, orgFieldsToTrim, OriTxt, oritxtFieldsToTrim, Qua, quaFieldsToTrim, Rap, rapFieldsToParseInt, rapFieldsToTrim, RapOrg, raporgFieldsToTrim, Scr, scrFieldsToTrim, Texte, texteFieldsToParseInt, texteFieldsToTrim, TypAtt, typattFieldsToTrim, TypLec, typlecFieldsToTrim, TypLoi, typloiFieldsToTrim, TypTxt, typtxtFieldsToTrim, TypUrl, typurlFieldsToTrim, } from "./types/dosleg";
+export { Photo, Sen, } from "./types/sens";
+export { Session } from "./types/sessions";
+export { parseIntFields, trimFieldsRight, } from "./fields";

package/lib/src/index.mjs

@@ -0,0 +1,7 @@
+export { allFollows, Aggregator, } from "./aggregates";
+export { insertAudReferences, insertDateSeanceReferences, insertAuteurReferences, insertDocAttReferences, insertEcrReferences, insertLecassReferences, insertLecassrapReferences, insertLectureReferences, insertLoiReferences, insertRapReferences, insertTexteReferences, } from "./inserters";
+export { sesFieldsToParseInt, sesFieldsToTrim, subFieldsToParseInt, subFieldsToTrim, txtAmeliFieldsToTrim, } from "./types/ameli";
+export { debatsFieldsToTrim, lecassdebFieldsToTrim, } from "./types/debats";
+export { assFieldsToTrim, audFieldsToTrim, auteurFieldsToTrim, dateSeanceFieldsToTrim, deccocFieldsToTrim, denrapFieldsToTrim, docattFieldsToParseInt, docattFieldsToTrim, ecrFieldsToTrim, etaloiFieldsToTrim, lecassFieldsToTrim, lecassrapFieldsToTrim, lectureFieldsToTrim, loiFieldsToTrim, orgFieldsToTrim, oritxtFieldsToTrim, quaFieldsToTrim, rapFieldsToParseInt, rapFieldsToTrim, raporgFieldsToTrim, scrFieldsToTrim, texteFieldsToParseInt, texteFieldsToTrim, typattFieldsToTrim, typlecFieldsToTrim, typloiFieldsToTrim, typtxtFieldsToTrim, typurlFieldsToTrim, } from "./types/dosleg";
+export { Session } from "./types/sessions";
+export { parseIntFields, trimFieldsRight, } from "./fields";

package/lib/src/inserters.d.ts

@@ -0,0 +1,98 @@
+import { TxtAmeli } from "./types/ameli";
+import { Debat } from "./types/debats";
+import { Ass, Aud, Auteur, DateSeance, DecCoc, DenRap, DocAtt, Ecr, EtaLoi, LecAss, LecAssRap, Lecture, Loi, Org, OriTxt, Qua, Rap, Scr, Texte, TypAtt, TypLec, TypLoi, TypTxt, TypUrl } from "./types/dosleg";
+export interface OutputData {
+    readonly ass?: {
+        [id: string]: Ass;
+    };
+    readonly aud?: {
+        [id: string]: Aud;
+    };
+    readonly auteur?: {
+        [id: string]: Auteur;
+    };
+    readonly date_seance?: {
+        [id: string]: DateSeance;
+    };
+    readonly debats?: {
+        [id: string]: Debat;
+    };
+    readonly deccoc?: {
+        [id: string]: DecCoc;
+    };
+    readonly denrap?: {
+        [id: string]: DenRap;
+    };
+    readonly docatt?: {
+        [id: string]: DocAtt;
+    };
+    readonly ecr?: {
+        [id: string]: Ecr;
+    };
+    readonly etaloi?: {
+        [id: string]: EtaLoi;
+    };
+    readonly lecass?: {
+        [id: string]: LecAss;
+    };
+    readonly lecassrap?: {
+        [id: string]: LecAssRap;
+    };
+    readonly lecture?: {
+        [id: string]: Lecture;
+    };
+    readonly loi?: {
+        [id: string]: Loi;
+    };
+    readonly org?: {
+        [id: string]: Org;
+    };
+    readonly oritxt?: {
+        [id: string]: OriTxt;
+    };
+    readonly qua?: {
+        [id: string]: Qua;
+    };
+    readonly rap?: {
+        [id: string]: Rap;
+    };
+    readonly scr?: {
+        [id: string]: Scr;
+    };
+    readonly texte?: {
+        [id: string]: Texte;
+    };
+    readonly typatt?: {
+        [id: string]: TypAtt;
+    };
+    readonly typlec?: {
+        [id: string]: TypLec;
+    };
+    readonly typloi?: {
+        [id: string]: TypLoi;
+    };
+    readonly typtxt?: {
+        [id: string]: TypTxt;
+    };
+    readonly typurl?: {
+        [id: string]: TypUrl;
+    };
+    readonly txt_ameli?: {
+        [id: string]: TxtAmeli;
+    };
+}
+type VisitedIdsByTableName = {
+    [tableName: string]: Set<number | string>;
+};
+export declare function insertAudReferences(aud: Aud, data: OutputData, visitedIdsByTableName: VisitedIdsByTableName): Aud;
+export declare function insertAuteurReferences(auteur: Auteur, data: OutputData, visitedIdsByTableName: VisitedIdsByTableName): Auteur;
+export declare function insertDateSeanceReferences(dateSeance: DateSeance, data: OutputData, visitedIdsByTableName: VisitedIdsByTableName): DateSeance;
+export declare function insertDocAttReferences(docatt: DocAtt, data: OutputData, visitedIdsByTableName: VisitedIdsByTableName, performInsertRapReferences?: boolean): DocAtt;
+export declare function insertEcrReferences(ecr: Ecr, data: OutputData, visitedIdsByTableName: VisitedIdsByTableName): Ecr;
+export declare function insertLecassrapReferences(lecassrap: LecAssRap, data: OutputData, visitedIdsByTableName: VisitedIdsByTableName): LecAssRap;
+export declare function insertLecassReferences(lecass: LecAss, data: OutputData, visitedIdsByTableName: VisitedIdsByTableName): LecAss;
+export declare function insertLectureReferences(lecture: Lecture, data: OutputData, visitedIdsByTableName: VisitedIdsByTableName): Lecture;
+export declare function insertLoiReferences(loi: Loi, data: OutputData, visitedIdsByTableName: VisitedIdsByTableName): Loi;
+export declare function insertRapReferences(rap: Rap, data: OutputData, visitedIdsByTableName: VisitedIdsByTableName): Rap;
+export declare function insertTexteReferences(texte: Texte, data: OutputData, visitedIdsByTableName: VisitedIdsByTableName): Texte;
+export {};