@tricoteuses/senat 2.22.16 → 3.0.0

package/README.md

@@ -138,6 +138,174 @@ for (const { item: question } of iterLoadSenatQuestions("../senat-data", 17)) {
 npm run data:generate_schemas ../senat-data
 ```
 
+## PostgreSQL import modes
+
+`retrieve_open_data.ts` supports two import modes:
+
+- default mode: direct import in a single database
+- `--incremental`: separate staging database plus `postgres_fdw` bridge into the target database
+
+The default mode is the simplest one and is the one used by the Docker image. The incremental mode is intended for replicated environments where heavy transient work should stay outside the target database.
+
+## Incremental PostgreSQL import architecture
+
+When `--incremental` is enabled, the open-data SQL import is designed to keep heavy transient work out of the replicated target database.
+
+```text
+                       +-----------------------------------+
+                       |  Source Open Data Senat           |
+                       |  ZIP -> SQL dumps                 |
+                       +----------------+------------------+
+                                        |
+                                        v
+                       +-----------------------------------+
+                       |  PostgreSQL staging instance      |
+                       |  non-replicated                   |
+                       |  database: senat_staging          |
+                       |                                   |
+                       |  schemas:                         |
+                       |  - ameli_staging                  |
+                       |  - debats_staging                 |
+                       |  - dosleg_staging                 |
+                       |  - questions_staging              |
+                       |  - sens_staging                   |
+                       |                                   |
+                       |  heavy operations:                |
+                       |  - raw dump import                |
+                       |  - table renaming/prefixing       |
+                       |  - staging indexes                |
+                       +----------------+------------------+
+                                        |
+                             postgres_fdw (read-only bridge)
+                                        |
+                                        v
++----------------------------------------------------------------------------+
+| PostgreSQL target instance, replicated                                     |
+| database: canutes                                                           |
+|                                                                            |
+| preserved schemas: assemblee, legifrance, ...                              |
+| updated schema: senat                                                      |
+|                                                                            |
+| lightweight temporary FDW schemas:                                         |
+| - ameli_staging                                                            |
+| - debats_staging                                                           |
+| - dosleg_staging                                                           |
+| - questions_staging                                                        |
+| - sens_staging                                                             |
+|                                                                            |
+| final operations only:                                                     |
+| - read staging data through postgres_fdw                                   |
+| - incremental merge into canutes.senat                                     |
+| - optional schema alignment when source structure changes                  |
++----------------------------------------------------------------------------+
+```
+
+At the end of an incremental import:
+
+- the target database is `canutes`
+- the target schema is `senat`
+- other schemas in `canutes` are left untouched
+- bulk transient data stays in the staging database
+
+## Direct single-database mode
+
+Without `--incremental`, the script works directly in the configured target database:
+
+- dumps are imported into temporary `*_staging` schemas in that same database
+- the final tables are merged into schema `senat`
+- the temporary staging schemas are dropped at the end
+
+This mode is appropriate for:
+
+- local development
+- disposable databases
+- the Docker image workflow
+- environments where replication cost is not a concern
+
+## Preparing postgres_fdw and database rights
+
+This section only applies when using `--incremental`.
+
+The target database must be able to connect to the separate staging database with `postgres_fdw`.
+
+### Environment variables
+
+Set the target connection in `.env`. Add the staging connection only if you use `--incremental`:
+
+```bash
+# Target database
+DB_HOST="localhost"
+DB_PORT=5432
+DB_USER="postgres"
+DB_PASSWORD="PASSWORD"
+DB_NAME="canutes"
+
+# Separate non-replicated staging database or instance, only for --incremental
+STAGING_DB_HOST="localhost"
+STAGING_DB_PORT=5433
+STAGING_DB_USER="postgres"
+STAGING_DB_PASSWORD="PASSWORD"
+STAGING_DB_NAME="senat_staging"
+```
+
+### Typical commands
+
+Default direct mode:
+
+```bash
+npm run data:retrieve_open_data -- ../senat-data --all
+```
+
+Incremental mode with separate staging database:
+
+```bash
+npm run data:retrieve_open_data -- ../senat-data --all --incremental
+```
+
+### Target database prerequisites
+
+On `canutes`, the import role must be able to:
+
+- connect to the database
+- create the `postgres_fdw` extension, or reuse it if already installed
+- create and drop FDW servers and user mappings
+- create and drop temporary `*_staging` schemas used for foreign tables
+- create, alter and drop objects inside schema `senat`
+
+Typical one-time setup on the target database:
+
+```sql
+CREATE EXTENSION IF NOT EXISTS postgres_fdw;
+```
+
+### Staging database prerequisites
+
+The staging PostgreSQL instance should ideally be outside the replicated cluster.
+
+The staging role must be able to:
+
+- recreate the `senat_staging` database
+- create schemas and import dumps there
+- create the technical staging indexes
+
+The target instance must also be allowed to connect to the staging instance over the network. In practice:
+
+- open the staging host and port from the target server
+- allow the staging user in `pg_hba.conf`
+- ensure the staging credentials used in `.env` are valid
+
+### Operational note
+
+The replicated target still sees a small amount of temporary DDL for FDW objects, but the large dump imports and staging tables remain outside the replicated database. Most replicated volume should therefore come from the real incremental changes applied to `canutes.senat`.
+
+## Validation of prefixed SQL imports
+
+After importing datasets with prefixed tables in schema `senat`, you can verify that the expected renamed tables exist and still match the generated definitions:
+
+```bash
+npm run data:validate_prefixed_tables -- --categories All
+```
+
 ## Publishing
 
 To publish a new version of this package onto npm, bump the package version and publish.

package/lib/aggregates.d.ts

@@ -0,0 +1,52 @@
+import { Ses, Sub, TxtAmeli } from "./types/ameli";
+import { Debat } from "./types/debats";
+import { Ass, Aud, Auteur, DateSeance, DecCoc, DenRap, DocAtt, Ecr, EtaLoi, LecAss, LecAssRap, Lecture, Loi, Org, OriTxt, Qua, Rap, Scr, Texte, TypAtt, TypLec, TypLoi, TypTxt, TypUrl } from "./types/dosleg";
+import { Question } from "./types/questions";
+export declare const allFollows: string[];
+export declare class Aggregator {
+    follow: Set<string>;
+    objectByIdByTableName: {
+        [name: string]: {
+            [id: string]: object;
+        };
+    };
+    requestedIdsByTableName: {
+        [name: string]: Set<string>;
+    };
+    visitedIdsByTableName: {
+        [name: string]: Set<string>;
+    };
+    constructor(follow: Set<string>);
+    addAss(ass: Ass): void;
+    addAud(aud: Aud): void;
+    addAuteur(auteur: Auteur): void;
+    addDateSeance(dateSeance: DateSeance): void;
+    addDebat(debat: Debat): void;
+    addDeccoc(deccoc: DecCoc): void;
+    addDenrap(denrap: DenRap): void;
+    addDocatt(docatt: DocAtt): void;
+    addEcr(ecr: Ecr): void;
+    addEtaloi(etaloi: EtaLoi): void;
+    addLecass(lecass: LecAss): void;
+    addLecassrap(lecassrap: LecAssRap): void;
+    addLecture(lecture: Lecture): void;
+    addLoi(loi: Loi): void;
+    addOrg(org: Org): void;
+    addOritxt(oritxt: OriTxt): void;
+    addQua(qua: Qua): void;
+    addQuestion(question: Question): void;
+    addRap(rap: Rap): void;
+    addScr(scr: Scr): void;
+    addSes(ses: Ses): void;
+    addSub(sub: Sub): void;
+    addTexte(texte: Texte): void;
+    addTxtAmeli(txtAmeli: TxtAmeli): void;
+    addTypatt(typatt: TypAtt): void;
+    addTyplec(typlec: TypLec): void;
+    addTyploi(typloi: TypLoi): void;
+    addTyptxt(typtxt: TypTxt): void;
+    addTypurl(typurl: TypUrl): void;
+    getAll(): Promise<void>;
+    request(tableName: string, id: string): void;
+    toJson(): any;
+}