@tricoteuses/senat 1.1.2 → 1.3.1

package/README.md

@@ -1,6 +1,10 @@
 # Tricoteuses-Senat
 
-## _Handle French Sénat's open data_
+## _Retrieve, clean up & handle  French Sénat's open data_
+
+## Requirements
+
+- Node >= 22
 
 ## Installation
 
@@ -15,7 +19,7 @@ Create a `.env` file to set PostgreSQL database informations and other configura
 npm install
 ```
 
-### Database creation
+### Database creation (not needed if downloading with Docker image)
 
 #### Using Docker
 
@@ -26,33 +30,67 @@ docker run --name local-postgres -d -p 5432:5432 -e POSTGRES_PASSWORD=$YOUR_CUST
 docker exec -it local-postgres psql -U postgres -c "CREATE ROLE opendata;"
 ```
 
-## Usage
+## Download data
 
-### Retrieval, cleaning & import of open data from Sénat
+Create a folder where the data will be downloaded and run the following command to download the data and convert it into JSON files.
 
 ```bash
 mkdir ../senat-data/
-npx babel-node --extensions ".ts" --max-old-space-size=4096 -- src/scripts/retrieve_open_data.ts --fetch --unzip --repair-encoding ../senat-data/
-npx babel-node --extensions ".ts" -- src/scripts/retrieve_open_data.ts --import ../senat-data/
-npx babel-node --extensions ".ts" -- src/scripts/fix_db.ts --updateSchema
-npx babel-node --extensions ".ts" -- src/scripts/convert_data.ts ../senat-data/ [--categories DosLeg]
+
+# Available options for optional `categories` parameter : All,  Ameli, Debats, DosLeg, Questions, Sens
+npm run data:download ../senat-data -- [--categories All]
+```
+
+Data from other sources is also available :
+```bash
+# Retrieval of sénateurs' pictures from Sénat's website
+npm run data:retrieve_senateurs_photos ../senat-data
+
+# Retrieval of textes and rapports from Sénat's website
+# Available options for optional `formats` parameter : xml, html, pdf
+# Available options for optional `types` parameter : textes, rapports
+npm run data:retrieve_documents ../senat-data -- --sessions 2024 2025 [--formats xml pdf] [--types textes]
+
+# Retrieval & parsing (textes in xml format only for now)
+npm run data:retrieve_documents ../senat-data -- --sessions 2024 2025 --parseDocuments
+
+# Parsing only
+npm run data:parse_textes_lois ../senat-data
 ```
 
-## Retrieval of sénateurs' pictures from Sénat's website
+## Data download using Docker
+
+A Docker image that downloads and cleans the data all at once is available. Create and run it locally from the container registry:
 
 ```bash
-npx babel-node --extensions ".ts" -- src/scripts/retrieve_senateurs_photos.ts --fetch ../senat-data/
+docker volume create senat-data # Create a volume to download the data
+docker run --name tricoteuses-senat -v senat-data:/app/senat-data -d registry.en-root.org/tricoteuses/tricoteuses-senat:latest
 ```
 
-### Retrieval & cleaning of bills from Sénat HTML pages
+Use the environment variable `CATEGORIES` and `SESSIONS` if needed.
+
+## Using the data
+
+Once the data is downloaded, you can use loaders to retrieve it.
+To use loaders in your project, you can install the _@tricoteuses/senat_ package, and import the iterator functions that you need.
+
+_Only iterator for Questions, Sénateurs and Circonscriptions available for now_
 
 ```bash
-npx babel-node --extensions ".ts" -- src/scripts/retrieve_textes.ts ../senat-data/
+npm install @tricoteuses/senat
+```
+
+```js
+import { iterLoadSenatQuestions } from "@tricoteuses/senat/lib/loaders"
+
+// Pass data directory and legislature as arguments
+for (const { item: question } of iterLoadSenatQuestions("../senat-data", 17)) {
+  console.log(question.id)
+}
 ```
 
-### Generation of raw types from SQL schema.
+## Generation of raw types from SQL schema (for contributors only)
 
-To be run after database has been fixed (`src/scripts/fix_db.ts --updateSchema`).
 ```bash
-npx babel-node --extensions ".ts" -- src/scripts/retrieve_open_data.ts --schema ../senat-data/
+npm run data:generate_schemas ../senat-data
 ```

package/lib/aggregates.d.ts

@@ -1,8 +1,7 @@
-import { Ses, Sub, TxtAmeli } from './types/ameli';
-import { Debat } from './types/debats';
-import { Ass, Aud, Auteur, DateSeance, DecCoc, DenRap, DocAtt, Ecr, EtaLoi, LecAss, LecAssRap, Lecture, Loi, Org, OriTxt, Qua, Rap, Scr, Texte, TypAtt, TypLec, TypLoi, TypTxt, TypUrl } from './types/dosleg';
-import { Question } from './types/questions';
-import { Sen } from './types/sens';
+import { Ses, Sub, TxtAmeli } from "./types/ameli";
+import { Debat } from "./types/debats";
+import { Ass, Aud, Auteur, DateSeance, DecCoc, DenRap, DocAtt, Ecr, EtaLoi, LecAss, LecAssRap, Lecture, Loi, Org, OriTxt, Qua, Rap, Scr, Texte, TypAtt, TypLec, TypLoi, TypTxt, TypUrl } from "./types/dosleg";
+import { Question } from "./types/questions";
 export declare const allFollows: string[];
 export declare class Aggregator {
     follow: Set<string>;
@@ -38,7 +37,6 @@ export declare class Aggregator {
     addQuestion(question: Question): void;
     addRap(rap: Rap): void;
     addScr(scr: Scr): void;
-    addSen(sen: Sen): void;
     addSes(ses: Ses): void;
     addSub(sub: Sub): void;
     addTexte(texte: Texte): void;