@tricoteuses/senat 2.10.5 → 2.11.1

package/lib/raw_types_schemats/ameli.d.ts

@@ -1,8 +1,8 @@
 /**
  * AUTO-GENERATED FILE - DO NOT EDIT!
  *
- * This file was automatically generated by schemats v.2.9.8
- * $ schemats generate -c postgres://username:password@localhost:5433/ameli -t amd -t amdsen -t avicom -t avigvt -t cab -t com_ameli -t ent -t etatxt -t fbu -t grppol_ameli -t gvt -t intora -t irr -t lec_ameli -t mot -t nat -t orarol -t sai -t saisen -t sea -t sen_ameli -t ses -t sor -t sub -t txt_ameli -t typrect -t typses -t typsub -t w_nivrec -s public
+ * This file was automatically generated by schemats v.2.10.5
+ * $ schemats generate -c postgres://username:password@localhost:5433/senat -t amd -t amdsen -t avicom -t avigvt -t cab -t com_ameli -t ent -t etatxt -t fbu -t grppol_ameli -t gvt -t intora -t irr -t lec_ameli -t mot -t nat -t orarol -t sai -t saisen -t sea -t sen_ameli -t ses -t sor -t sub -t txt_ameli -t typrect -t typses -t typsub -t w_nivrec -s ameli
  *
  */
 export declare namespace amdFields {

package/lib/raw_types_schemats/debats.d.ts

@@ -1,8 +1,8 @@
 /**
  * AUTO-GENERATED FILE - DO NOT EDIT!
  *
- * This file was automatically generated by schemats v.2.9.8
- * $ schemats generate -c postgres://username:password@localhost:5433/debats -t debats -t intdivers -t intpjl -t lecassdeb -t secdis -t secdivers -t syndeb -t typsec -s public
+ * This file was automatically generated by schemats v.2.10.5
+ * $ schemats generate -c postgres://username:password@localhost:5433/senat -t debats -t intdivers -t intpjl -t lecassdeb -t secdis -t secdivers -t syndeb -t typsec -s debats
  *
  */
 export declare namespace debatsFields {

package/lib/raw_types_schemats/dosleg.d.ts

@@ -1,8 +1,8 @@
 /**
  * AUTO-GENERATED FILE - DO NOT EDIT!
  *
- * This file was automatically generated by schemats v.2.9.8
- * $ schemats generate -c postgres://username:password@localhost:5433/dosleg -t amescr -t ass -t aud -t auteur -t ble -t catrap -t corscr -t date_seance -t deccoc -t denrap -t doc -t docatt -t docsea -t ecr -t etaloi -t evtsea -t forpub -t gen -t lecass -t lecassrap -t lecture -t lnkrap -t loi -t loithe -t natloi -t org -t orgnomhis -t orippr -t oritxt -t posvot -t qua -t rap -t raporg -t rapthe -t rolsig -t scr -t ses -t stavot -t texte -t texte_ancien -t the -t titsen -t typatt -t typaut -t typdoc -t typevtsea -t typlec -t typloi -t typorg -t typrap -t typtxt -t typurl -t votsen -s public
+ * This file was automatically generated by schemats v.2.10.5
+ * $ schemats generate -c postgres://username:password@localhost:5433/senat -t amescr -t ass -t aud -t auteur -t ble -t catrap -t corscr -t date_seance -t deccoc -t denrap -t doc -t docatt -t docsea -t ecr -t etaloi -t evtsea -t forpub -t gen -t lecass -t lecassrap -t lecture -t lnkrap -t loi -t loithe -t natloi -t org -t orgnomhis -t orippr -t oritxt -t posvot -t qua -t rap -t raporg -t rapthe -t rolsig -t scr -t ses -t stavot -t texte -t texte_ancien -t the -t titsen -t typatt -t typaut -t typdoc -t typevtsea -t typlec -t typloi -t typorg -t typrap -t typtxt -t typurl -t votsen -s dosleg
  *
  */
 export declare namespace amescrFields {

package/lib/raw_types_schemats/questions.d.ts

@@ -1,8 +1,8 @@
 /**
  * AUTO-GENERATED FILE - DO NOT EDIT!
  *
- * This file was automatically generated by schemats v.2.9.8
- * $ schemats generate -c postgres://username:password@localhost:5433/questions -t etatquestion -t legquestion -t naturequestion -t sortquestion -t tam_ministeres -t tam_questions -t tam_reponses -t the -s questions
+ * This file was automatically generated by schemats v.2.10.5
+ * $ schemats generate -c postgres://username:password@localhost:5433/senat -t etatquestion -t legquestion -t naturequestion -t sortquestion -t tam_ministeres -t tam_questions -t tam_reponses -t the -s questions
  *
  */
 export declare namespace etatquestionFields {

package/lib/raw_types_schemats/sens.d.ts

@@ -1,8 +1,8 @@
 /**
  * AUTO-GENERATED FILE - DO NOT EDIT!
  *
- * This file was automatically generated by schemats v.2.9.8
- * $ schemats generate -c postgres://username:password@localhost:5433/sens -t acr -t activite -t activite_audit -t activite_delegation -t activite_delegation_audit -t activite_loi -t activite_loi_audit -t activite_obligatoire -t activite_participant -t activite_participant_audit -t activite_senateur -t activite_senateur_audit -t activite_senateur_params -t activite_senateur_params_audit -t activites_liees -t activites_liees_audit -t actpro -t adhgrpsen -t adr -t adresse -t adrsen -t app -t assparint -t asster -t autgrpsen -t autorisation_profil -t autorisations -t avis_nomination_art13 -t basdes -t bur -t bur3r -t bur4r -t cad -t candid -t candidat -t candtodelete -t categorie_activite -t catpro -t catpro2e -t catterrit -t cible_categorie_periode -t cirdep -t com -t con -t cotgip -t csp -t cspfam -t databasechangelog -t databasechangeloglock -t delega -t derogation -t derogation_audit -t derogation_senateur -t derogation_senateur_audit -t design -t designoep -t designorg -t discou -t div -t dpt -t dpt_seuil_presence -t dptele -t dptele_files -t dptele_processing -t dptele_processing_type -t dpttypman -t droits_acces -t droits_acces_audit -t droits_type_derogation -t ele -t eleloc -t elucan -t eludep -t eludiv -t elueur -t elueur_apf -t elumet -t elureg -t elusen -t elusen2e -t elusen3r -t elusen4r -t elusencommu -t elusenpair -t eluter -t elutit -t eluvil -t etadebman -t etadebman3r -t etadebman4r -t etafinman -t etafinman3r -t etafinman4r -t etaprr -t etarpm -t etasen -t ext2e_bio -t ext2e_csp -t ext2e_mandats -t ext2e_minist -t extsencom_identite -t extsencom_mandat -t fonact_participant -t foncandid -t foncom -t fondelega -t fongrppol -t fongrpsen -t fonmemcom -t fonmemdelega -t fonmemextpar -t fonmemgrppol -t fonmemgrpsen -t fonmemorg -t fonorg -t grppol -t grppol4r -t grpsenami -t grpsenamiadh -t grpsenamiadhreq -t grpsenamiadhreqeta -t grpsenamiunadh -t grpsim -t gvt -t insee_pays2008 -t jhi_authority -t jhi_user -t jhi_user_authority -t lanetr -t libcom -t libdelega -t libgrppol -t libgrpsen -t liborg -t lisdptele -t mel -t memcom -t memcomsea -t memdelega -t memextpar -t memgrppol -t memgrpsen -t memorg -t met -t minind -t minist -t mis -t misetafin -t mismin -t misrapeta -t missen -t moddes -t mode_acces_elusenpair -t nation -t nationgrpsen -t nivlan -t org -t orgext -t orgextpres -t orgthe -t pairie_elusenpair -t parpol -t parpolglo -t participa -t pcs -t pcs24 -t pcs42 -t pcs8 -t pcscatpro -t per -t per_sen -t perapp -t periode_presence -t perpolglo -t perrol -t pj_justificatif -t pj_justificatif_audit -t plaind -t plan_table -t plsql_profiler_runs -t plsql_profiler_units -t poicon -t posvot -t presences_scrutin_surcharge -t presencesrevisionentity -t profil_applicatif -t qua -t rap_the -t reg -t reladr -t requetes_profil -t reslis -t resultat -t reu -t revchanges -t rne_mandat -t rne_mandat_diff -t rne_sen -t rne_sen_diff -t rne_type_mandat -t rol -t sal -t scr -t scrusoldelega -t sea -t sec -t sec2e -t secexe -t sen -t senbur -t senbur3r -t senbur4r -t sennom -t senpj -t sensim -t sentablenom -t senurl -t seuil_presence -t sirpas_elusen -t sirpas_fonmemcom -t sirpas_fonmemdelega -t sirpas_fonmemgrppol -t sirpas_memcom -t sirpas_memdelega -t sirpas_memgrppol -t sirpas_mvt -t sirpas_mvtcm -t sirpas_mvttri -t sirpas_sen -t sirpas_senbur -t sirpas_trf -t srv -t stajur -t stavot -t suspensiontravaux -t suspensiontravaux_audit -t sysage -t syscognos -t sysevt -t sysvar -t sysvar_sendev -t sysvar_senprod -t tapsenrevchanges -t tapsenrevisionentity -t telephone -t temval -t tenpol -t territ -t testoracle -t titele -t titelerne -t titmin -t titnob -t tmpsd -t toutes -t turelu -t typadr -t typapppol -t typbister -t typcandid -t type_activite -t type_activite_participant -t type_activite_rol -t type_activite_senateur -t type_categorie -t type_derogation -t type_droit_acces -t type_pj_justificatif -t type_rne_diff -t type_type_derogation -t typele -t typgrpsen -t typman -t typmin -t typmoddes -t typorg -t typorgext -t typparpol -t typpoicon -t typprs -t typprssta -t typscr -t typtel -t typurl -t typvoi -t uploaded_file -t uploaded_file_type -t validation -t validation_defview_profil -t validation_profil -t vercand -t verres -t votes -t zongeo -s public
+ * This file was automatically generated by schemats v.2.10.5
+ * $ schemats generate -c postgres://username:password@localhost:5433/senat -t acr -t activite -t activite_audit -t activite_delegation -t activite_delegation_audit -t activite_loi -t activite_loi_audit -t activite_obligatoire -t activite_participant -t activite_participant_audit -t activite_senateur -t activite_senateur_audit -t activite_senateur_params -t activite_senateur_params_audit -t activites_liees -t activites_liees_audit -t actpro -t adhgrpsen -t adr -t adresse -t adrsen -t app -t assparint -t asster -t autgrpsen -t autorisation_profil -t autorisations -t avis_nomination_art13 -t basdes -t bur -t bur3r -t bur4r -t cad -t candid -t candidat -t candtodelete -t categorie_activite -t catpro -t catpro2e -t catterrit -t cible_categorie_periode -t cirdep -t com -t con -t cotgip -t csp -t cspfam -t databasechangelog -t databasechangeloglock -t delega -t derogation -t derogation_audit -t derogation_senateur -t derogation_senateur_audit -t design -t designoep -t designorg -t discou -t div -t dpt -t dpt_seuil_presence -t dptele -t dptele_files -t dptele_processing -t dptele_processing_type -t dpttypman -t droits_acces -t droits_acces_audit -t droits_type_derogation -t ele -t eleloc -t elucan -t eludep -t eludiv -t elueur -t elueur_apf -t elumet -t elureg -t elusen -t elusen2e -t elusen3r -t elusen4r -t elusencommu -t elusenpair -t eluter -t elutit -t eluvil -t etadebman -t etadebman3r -t etadebman4r -t etafinman -t etafinman3r -t etafinman4r -t etaprr -t etarpm -t etasen -t ext2e_bio -t ext2e_csp -t ext2e_mandats -t ext2e_minist -t extsencom_identite -t extsencom_mandat -t fonact_participant -t foncandid -t foncom -t fondelega -t fongrppol -t fongrpsen -t fonmemcom -t fonmemdelega -t fonmemextpar -t fonmemgrppol -t fonmemgrpsen -t fonmemorg -t fonorg -t grppol -t grppol4r -t grpsenami -t grpsenamiadh -t grpsenamiadhreq -t grpsenamiadhreqeta -t grpsenamiunadh -t grpsim -t gvt -t insee_pays2008 -t jhi_authority -t jhi_user -t jhi_user_authority -t lanetr -t libcom -t libdelega -t libgrppol -t libgrpsen -t liborg -t lisdptele -t mel -t memcom -t memcomsea -t memdelega -t memextpar -t memgrppol -t memgrpsen -t memorg -t met -t minind -t minist -t mis -t misetafin -t mismin -t misrapeta -t missen -t moddes -t mode_acces_elusenpair -t nation -t nationgrpsen -t nivlan -t org -t orgext -t orgextpres -t orgthe -t pairie_elusenpair -t parpol -t parpolglo -t participa -t pcs -t pcs24 -t pcs42 -t pcs8 -t pcscatpro -t per -t per_sen -t perapp -t periode_presence -t perpolglo -t perrol -t pj_justificatif -t pj_justificatif_audit -t plaind -t plan_table -t plsql_profiler_runs -t plsql_profiler_units -t poicon -t posvot -t presences_scrutin_surcharge -t presencesrevisionentity -t profil_applicatif -t qua -t rap_the -t reg -t reladr -t requetes_profil -t reslis -t resultat -t reu -t revchanges -t rne_mandat -t rne_mandat_diff -t rne_sen -t rne_sen_diff -t rne_type_mandat -t rol -t sal -t scr -t scrusoldelega -t sea -t sec -t sec2e -t secexe -t sen -t senbur -t senbur3r -t senbur4r -t sennom -t senpj -t sensim -t sentablenom -t senurl -t seuil_presence -t sirpas_elusen -t sirpas_fonmemcom -t sirpas_fonmemdelega -t sirpas_fonmemgrppol -t sirpas_memcom -t sirpas_memdelega -t sirpas_memgrppol -t sirpas_mvt -t sirpas_mvtcm -t sirpas_mvttri -t sirpas_sen -t sirpas_senbur -t sirpas_trf -t srv -t stajur -t stavot -t suspensiontravaux -t suspensiontravaux_audit -t sysage -t syscognos -t sysevt -t sysvar -t sysvar_sendev -t sysvar_senprod -t tapsenrevchanges -t tapsenrevisionentity -t telephone -t temval -t tenpol -t territ -t testoracle -t titele -t titelerne -t titmin -t titnob -t tmpsd -t toutes -t turelu -t typadr -t typapppol -t typbister -t typcandid -t type_activite -t type_activite_participant -t type_activite_rol -t type_activite_senateur -t type_categorie -t type_derogation -t type_droit_acces -t type_pj_justificatif -t type_rne_diff -t type_type_derogation -t typele -t typgrpsen -t typman -t typmin -t typmoddes -t typorg -t typorgext -t typparpol -t typpoicon -t typprs -t typprssta -t typscr -t typtel -t typurl -t typvoi -t uploaded_file -t uploaded_file_type -t validation -t validation_defview_profil -t validation_profil -t vercand -t verres -t votes -t zongeo -s sens
  *
  */
 export declare namespace acrFields {

package/lib/scripts/convert_data.js

@@ -21,14 +21,14 @@ async function convertData() {
     const enabledDatasets = getEnabledDatasets(options["categories"]);
     console.time("data transformation time");
     if (enabledDatasets & EnabledDatasets.Ameli) {
-        await convertDatasetAmeli(dataDir);
+        await convertDatasetAmeli(dataDir, options);
     }
     if (enabledDatasets & EnabledDatasets.Debats) {
-        await convertDatasetDebats(dataDir);
+        await convertDatasetDebats(dataDir, options);
     }
     if (enabledDatasets & EnabledDatasets.DosLeg) {
-        await convertDatasetDosLeg(dataDir);
-        await convertDatasetScrutins(dataDir);
+        await convertDatasetDosLeg(dataDir, options);
+        await convertDatasetScrutins(dataDir, options);
     }
     if (enabledDatasets & EnabledDatasets.Questions) {
         await convertDatasetQuestions(dataDir);
@@ -40,27 +40,27 @@ async function convertData() {
         console.timeEnd("data transformation time");
     }
 }
-async function convertDatasetAmeli(dataDir) {
+async function convertDatasetAmeli(dataDir, options) {
     const dataset = datasets.ameli;
     if (!options["silent"]) {
         console.log(`Converting database ${dataset.database} data into files…`);
     }
     const ameliReorganizedRootDir = path.join(dataDir, dataset.database);
     ensureAndClearDir(ameliReorganizedRootDir);
-    for await (const amendement of findAllAmendements()) {
+    for await (const amendement of findAllAmendements(options["fromSession"])) {
         if (options["verbose"]) {
-            console.log(`Converting ${amendement.numero} file…`);
+            console.log(`Converting ${amendement["numero"]} file…`);
         }
-        const session = String(amendement.session) || UNDEFINED_SESSION;
-        const signetDossierLegislatif = amendement.signet_dossier_legislatif ||
-            `${amendement.nature_texte}-${amendement.numero_texte}`.toLowerCase();
+        const session = String(amendement["session"]) || UNDEFINED_SESSION;
+        const signetDossierLegislatif = amendement["signet_dossier_legislatif"] ||
+            `${amendement["nature_texte"]}-${amendement["numero_texte"]}`.toLowerCase();
         const ameliReorganizedDir = path.join(ameliReorganizedRootDir, String(session), signetDossierLegislatif);
         fs.ensureDirSync(ameliReorganizedDir);
-        const amendementFileName = `${amendement.numero}.json`;
+        const amendementFileName = `${amendement["numero"]}.json`;
         fs.writeJSONSync(path.join(ameliReorganizedDir, amendementFileName), amendement, { spaces: 2 });
     }
 }
-async function convertDatasetDebats(dataDir) {
+async function convertDatasetDebats(dataDir, options) {
     const dataset = datasets.debats;
     if (!options["silent"]) {
         console.log(`Converting database ${dataset.database} data into files…`);
@@ -74,6 +74,9 @@ async function convertDatasetDebats(dataDir) {
         }
         const enrichedDebat = await enrichDebat(debat, allAuteurs);
         const session = getSessionFromDate(enrichedDebat.date_seance);
+        if (options["fromSession"] && session < options["fromSession"]) {
+            continue;
+        }
         const debatsReorganizedDir = path.join(debatsReorganizedRootDir, String(session));
         fs.ensureDirSync(debatsReorganizedDir);
         const debatFileName = `${enrichedDebat.id}.json`;
@@ -84,12 +87,12 @@ async function enrichDebat(debat, auteurs) {
     const enrichedDebat = { ...debat };
     for (const section of enrichedDebat.sections) {
         for (const intervention of section.interventions) {
-            intervention.auteur = findAuteur(intervention.auteur_code, auteurs);
+            intervention.auteur = findAuteur(intervention["auteur_code"], auteurs);
         }
     }
     for (const section of enrichedDebat.sections_divers) {
         for (const intervention of section.interventions) {
-            intervention.auteur = findAuteur(intervention.auteur_code, auteurs);
+            intervention.auteur = findAuteur(intervention["auteur_code"], auteurs);
         }
     }
     return enrichedDebat;
@@ -97,7 +100,7 @@ async function enrichDebat(debat, auteurs) {
 function findAuteur(auteurCode, auteurs) {
     return auteurs.find(auteur => auteur.code === auteurCode);
 }
-async function convertDatasetDosLeg(dataDir) {
+async function convertDatasetDosLeg(dataDir, options) {
     const dataset = datasets.dosleg;
     if (!options["silent"]) {
         console.log(`Converting database ${dataset.database} data into files…`);
@@ -112,6 +115,9 @@ async function convertDatasetDosLeg(dataDir) {
         }
         let loiReorganizedDir = path.join(dossiersReorganizedDir, String(UNDEFINED_SESSION));
         const session = getSessionFromSignet(loi["signet"]) || UNDEFINED_SESSION;
+        if (options["fromSession"] && session < options["fromSession"]) {
+            continue;
+        }
         loiReorganizedDir = path.join(dossiersReorganizedDir, String(session));
         fs.ensureDirSync(loiReorganizedDir);
         const scrutinFileName = `${loi["signet"]}.json`;
@@ -122,22 +128,22 @@ async function convertDatasetDosLeg(dataDir) {
     await convertTexteUrls(dataDir);
     await convertRapportUrls(dataDir);
 }
-async function convertDatasetScrutins(dataDir) {
+async function convertDatasetScrutins(dataDir, options) {
     const dataset = datasets.dosleg;
     if (!options["silent"]) {
-        console.log(`Converting database ${dataset.database} scrutins data into files…`);
+        console.log(`Converting database scrutins (${dataset.database}) data into files…`);
     }
     const scrutinsReorganizedDir = path.join(dataDir, SCRUTINS_FOLDER);
     ensureAndClearDir(scrutinsReorganizedDir);
-    for await (const scrutin of findAllScrutins()) {
+    for await (const scrutin of findAllScrutins(options["fromSession"])) {
         if (options["verbose"]) {
-            console.log(`Converting ${scrutin.numero} file…`);
+            console.log(`Converting ${scrutin["numero"]} file…`);
         }
         let scrutinReorganizedDir = path.join(scrutinsReorganizedDir, String(UNDEFINED_SESSION));
-        const session = scrutin.session || UNDEFINED_SESSION;
+        const session = scrutin["session"] || UNDEFINED_SESSION;
         scrutinReorganizedDir = path.join(scrutinsReorganizedDir, String(session));
         fs.ensureDirSync(scrutinReorganizedDir);
-        const scrutinFileName = `${scrutin.numero}.json`;
+        const scrutinFileName = `${scrutin["numero"]}.json`;
         fs.writeJSONSync(path.join(scrutinReorganizedDir, scrutinFileName), scrutin, {
             spaces: 2,
         });
@@ -152,12 +158,12 @@ async function convertDatasetQuestions(dataDir) {
     ensureAndClearDir(questionsReorganizedRootDir);
     for await (const question of findAllQuestions()) {
         if (options["verbose"]) {
-            console.log(`Converting ${question.reference} file…`);
+            console.log(`Converting ${question["reference"]} file…`);
         }
-        const legislature = question.legislature ? question.legislature : 0;
+        const legislature = question["legislature"] ? question["legislature"] : 0;
         const questionReorganizedDir = path.join(questionsReorganizedRootDir, String(legislature));
         fs.ensureDirSync(questionReorganizedDir);
-        const questionFileName = `${question.reference}.json`;
+        const questionFileName = `${question["reference"]}.json`;
         fs.writeJSONSync(path.join(questionReorganizedDir, questionFileName), question, { spaces: 2 });
     }
 }
@@ -229,26 +235,26 @@ async function convertDatasetSens(dataDir) {
     ensureAndClearDir(organismesReorganizedDir);
     for await (const sen of findAllSens()) {
         if (options["verbose"]) {
-            console.log(`Converting ${sen.matricule} file…`);
+            console.log(`Converting ${sen["matricule"]} file…`);
         }
-        const senFileName = `${sen.matricule}.json`;
+        const senFileName = `${sen["matricule"]}.json`;
         fs.writeJSONSync(path.join(senateursReorganizedDir, senFileName), sen, {
             spaces: 2,
         });
     }
     for await (const circonscription of findAllCirconscriptions()) {
         if (options["verbose"]) {
-            console.log(`Converting ${circonscription.identifiant} file…`);
+            console.log(`Converting ${circonscription["identifiant"]} file…`);
         }
-        const circonscriptionFileName = `${circonscription.identifiant}.json`;
+        const circonscriptionFileName = `${circonscription["identifiant"]}.json`;
         fs.writeJSONSync(path.join(circonscriptionsReorganizedDir, circonscriptionFileName), circonscription, { spaces: 2 });
     }
     for await (const organisme of findAllOrganismes()) {
         if (options["verbose"]) {
-            console.log(`Converting ${organisme.code} file…`);
+            console.log(`Converting ${organisme["code"]} file…`);
         }
-        const organismeFileName = `${organisme.code}.json`;
-        const organismeDir = path.join(organismesReorganizedDir, organisme.type_code);
+        const organismeFileName = `${organisme["code"]}.json`;
+        const organismeDir = path.join(organismesReorganizedDir, organisme["type_code"]);
         fs.ensureDirSync(organismeDir);
         fs.writeJSONSync(path.join(organismeDir, organismeFileName), organisme, { spaces: 2 });
     }

package/lib/scripts/retrieve_cr_commission.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/scripts/retrieve_cr_commission.js

@@ -0,0 +1,291 @@
+import fs from "fs-extra";
+import assert from "assert";
+import path from "path";
+import * as cheerio from "cheerio";
+import { COMMISSION_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
+import { createCommissionGroupIfMissing, loadCommissionAgendaForDate, parseCommissionMetadataFromHtml, } from "../utils/cr_spliting";
+import { parseCommissionCRFromFile } from "../model/commission";
+import commandLineArgs from "command-line-args";
+import { commonOptions } from "./shared/cli_helpers";
+import { sessionStartYearFromDate } from "../model/seance";
+import { getSessionsFromStart } from "../types/sessions";
+import { ensureAndClearDir, fetchWithRetry } from "./shared/util";
+class CommissionCRDownloadError extends Error {
+    constructor(message, url) {
+        super(`An error occurred while retrieving Commission CR ${url}: ${message}`);
+    }
+}
+const optionsDefinitions = [
+    ...commonOptions,
+    { name: "concurrency", type: Number, defaultValue: 6, help: "Max parallel downloads" },
+    { name: "politenessMs", type: Number, defaultValue: 150, help: "Delay per worker (ms)" },
+    {
+        help: "parse and convert comptes-rendus des débats into JSON",
+        name: "parseDebats",
+        type: Boolean,
+    },
+];
+const options = commandLineArgs(optionsDefinitions);
+const COMMISSION_HUBS = {
+    "affaires-etrangeres": [
+        "https://www.senat.fr/compte-rendu-commissions/affaires-etrangeres.html",
+        "https://www.senat.fr/compte-rendu-commissions/affaires-etrangeres_archives.html",
+    ],
+    "affaires-economiques": [
+        "https://www.senat.fr/compte-rendu-commissions/affaires-economiques.html",
+        "https://www.senat.fr/compte-rendu-commissions/affaires-economiques_archives.html",
+    ],
+    "amenagement-developpement-durable": [
+        "https://www.senat.fr/compte-rendu-commissions/cadre-de-vie-et-developpement-durable.html",
+        "https://www.senat.fr/compte-rendu-commissions/cadre-de-vie-et-developpement-durable_archives.html",
+    ],
+    culture: [
+        "https://www.senat.fr/compte-rendu-commissions/culture.html",
+        "https://www.senat.fr/compte-rendu-commissions/culture_archives.html",
+    ],
+    finances: [
+        "https://www.senat.fr/compte-rendu-commissions/finances.html",
+        "https://www.senat.fr/compte-rendu-commissions/finances_archives.html",
+    ],
+    lois: [
+        "https://www.senat.fr/compte-rendu-commissions/lois.html",
+        "https://www.senat.fr/compte-rendu-commissions/lois_archives.html",
+    ],
+    "affaires-sociales": [
+        "https://www.senat.fr/compte-rendu-commissions/affaires-sociales.html",
+        "https://www.senat.fr/compte-rendu-commissions/affaires-sociales_archives.html",
+    ],
+    "affaires-europeennes": [
+        "https://www.senat.fr/compte-rendu-commissions/affaires-europeennes.html",
+        "https://www.senat.fr/compte-rendu-commissions/affaires-europeennes_archives.html",
+    ],
+};
+async function harvestWeeklyLinksFromHub(hubUrl) {
+    const res = await fetchWithRetry(hubUrl);
+    if (!res.ok)
+        return [];
+    const html = await res.text();
+    const $ = cheerio.load(html);
+    const out = [];
+    $("a[href]").each((_, a) => {
+        const href = ($(a).attr("href") || "").trim();
+        const m = href.match(/\/compte-rendu-commissions\/(\d{8})\/([a-z0-9\-]+)\.html$/i);
+        if (m) {
+            const url = href.startsWith("http") ? href : new URL(href, hubUrl).toString();
+            out.push(url);
+        }
+    });
+    return Array.from(new Set(out));
+}
+async function discoverCommissionWeeklyPages(fromSession) {
+    const results = [];
+    for (const [commissionKey, hubs] of Object.entries(COMMISSION_HUBS)) {
+        for (const hubUrl of hubs) {
+            try {
+                const links = await harvestWeeklyLinksFromHub(hubUrl);
+                for (const url of links) {
+                    const m = url.match(/\/compte-rendu-commissions\/(\d{8})\/([a-z0-9\-]+)\.html$/i);
+                    if (!m)
+                        continue;
+                    const yyyymmdd = m[1];
+                    const year = Number(yyyymmdd.slice(0, 4));
+                    const month = Number(yyyymmdd.slice(4, 6));
+                    const session = month >= 10 ? year : year - 1;
+                    if (session < fromSession)
+                        continue;
+                    results.push({ url, yyyymmdd, commissionKey });
+                }
+            }
+            catch (e) {
+                console.warn(`[COM-CR][hub-fail] ${hubUrl} → ${e?.message ?? e}`);
+            }
+        }
+    }
+    return results.sort((a, b) => a.yyyymmdd.localeCompare(b.yyyymmdd));
+}
+function toHourShort(hhmm) {
+    if (!hhmm)
+        return null;
+    const m = hhmm.match(/^(\d{2}):(\d{2})$/);
+    return m ? `${m[1]}${m[2]}` : null;
+}
+function timeToMinutes(hhmm) {
+    const [h, m] = hhmm.split(":").map((n) => parseInt(n, 10));
+    return (h || 0) * 60 + (m || 0);
+}
+async function tryDownload(url) {
+    const res = await fetch(url, { redirect: "follow" });
+    if (res.status === 404)
+        return null;
+    if (!res.ok)
+        throw new CommissionCRDownloadError(String(res.status), url);
+    const ab = await res.arrayBuffer();
+    return Buffer.from(ab);
+}
+async function retrieveCommissionCRs(options = {}) {
+    const dataDir = options["dataDir"];
+    const fromSession = Number(options["fromSession"]);
+    const concurrency = Number(options["concurrency"] ?? 6);
+    const politenessMs = Number(options["politenessMs"] ?? 150);
+    const commissionsRootDir = path.join(dataDir, COMMISSION_FOLDER);
+    const originalRoot = path.join(commissionsRootDir, DATA_ORIGINAL_FOLDER);
+    ensureAndClearDir(originalRoot);
+    const discovered = await discoverCommissionWeeklyPages(fromSession);
+    console.log(`[COM-CR][discover] ${discovered.length} links (>= session ${fromSession})`);
+    const jobs = discovered.map(({ url, yyyymmdd }) => {
+        const d = new Date(Number(yyyymmdd.slice(0, 4)), Number(yyyymmdd.slice(4, 6)) - 1, Number(yyyymmdd.slice(6, 8)));
+        const session = sessionStartYearFromDate(d);
+        const dir = path.join(originalRoot, String(session));
+        fs.ensureDirSync(dir);
+        const slug = url.replace(/^.*\/(\d{8})\/([^\/]+)\.html$/i, "$2");
+        const outPath = path.join(dir, `${yyyymmdd}.${slug}.html`);
+        return { url, outPath, yyyymmdd };
+    });
+    console.log(`[COM-CR] Downloading ${jobs.length} links → ${path.relative(process.cwd(), originalRoot)}`);
+    let completed = 0, saved = 0, skipped = 0, notFound = 0;
+    const workers = Array.from({ length: Math.max(1, concurrency) }, async () => {
+        while (true) {
+            const job = jobs.shift();
+            if (!job)
+                break;
+            const { url, outPath, yyyymmdd } = job;
+            try {
+                if (await fs.pathExists(outPath)) {
+                    skipped++;
+                }
+                else {
+                    const buf = await tryDownload(url);
+                    if (!buf) {
+                        notFound++;
+                        console.warn(`[COM-CR][404] ${url} → week=${yyyymmdd}`);
+                    }
+                    else {
+                        await fs.writeFile(outPath, buf);
+                        saved++;
+                    }
+                }
+            }
+            catch (e) {
+                console.error(`[COM-CR][err] ${url} → ${e?.message || e}`);
+            }
+            finally {
+                completed++;
+                if (politenessMs > 0)
+                    await new Promise((r) => setTimeout(r, politenessMs));
+            }
+        }
+    });
+    await Promise.all(workers);
+    console.log(`[COM-CR] done: saved=${saved} | skipped=${skipped} | 404=${notFound} | total=${completed}`);
+    const sessions = getSessionsFromStart(options["fromSession"]);
+    const comRoot = path.join(dataDir, COMMISSION_FOLDER);
+    const transformedRoot = path.join(comRoot, DATA_TRANSFORMED_FOLDER);
+    if (options["parseDebats"])
+        ensureAndClearDir(transformedRoot);
+    for (const session of sessions) {
+        const originalSessionDir = path.join(originalRoot, String(session));
+        const transformedSessionDir = path.join(transformedRoot, String(session));
+        fs.ensureDirSync(transformedSessionDir);
+        if (!(await fs.pathExists(originalSessionDir)))
+            continue;
+        const htmlFiles = (await fs.readdir(originalSessionDir)).filter((f) => /\.html?$/i.test(f)).sort();
+        for (const f of htmlFiles) {
+            const htmlPath = path.join(originalSessionDir, f);
+            let meta;
+            try {
+                const raw = await fs.readFile(htmlPath, "utf8");
+                meta = parseCommissionMetadataFromHtml(raw, f);
+            }
+            catch (e) {
+                console.warn(`[COM-CR][PRE][${session}] Cannot read/parse ${f}:`, e);
+                continue;
+            }
+            const organeKeywords = (meta.organeDetected ?? meta.organeTitleRaw ?? "")
+                .toLowerCase()
+                .replace(/[’']/g, "'")
+                .split(/\W+/)
+                .filter((x) => x.length >= 3 && !["commission", "des", "de", "du", "d", "la", "le", "les", "et"].includes(x));
+            const MAX_TIME_DELTA_MIN = 120;
+            for (let i = 0; i < meta.days.length; i++) {
+                const day = meta.days[i];
+                const yyyymmdd = day.date.replace(/-/g, "");
+                const dt = new Date(Number(day.date.slice(0, 4)), Number(day.date.slice(5, 7)) - 1, Number(day.date.slice(8, 10)));
+                const daySession = sessionStartYearFromDate(dt);
+                const hits = await loadCommissionAgendaForDate(dataDir, yyyymmdd, daySession);
+                let best = null;
+                let reason = "fallback-none";
+                let deltaMin;
+                // a) score by title and organe keywords
+                if (organeKeywords.length && hits.length) {
+                    const scored = hits
+                        .map((h) => {
+                        const t = (h.titre ?? "").toLowerCase();
+                        const s = organeKeywords.reduce((acc, kw) => acc + (t.includes(kw) ? 1 : 0), 0);
+                        return { h, s };
+                    })
+                        .sort((a, b) => b.s - a.s);
+                    if (scored[0]?.s > 0) {
+                        best = scored[0].h;
+                        reason = "title";
+                    }
+                }
+                // b) otherwise score by time proximity
+                if (!best && day.openTime && hits.length) {
+                    const candidates = hits
+                        .map((h) => ({ h, hhmm: h.startTime ?? null }))
+                        .filter((x) => !!x.hhmm)
+                        .map((x) => ({
+                        h: x.h,
+                        d: Math.abs(timeToMinutes(x.hhmm) - timeToMinutes(day.openTime)),
+                    }))
+                        .sort((a, b) => a.d - b.d);
+                    if (candidates[0] && candidates[0].d <= MAX_TIME_DELTA_MIN) {
+                        best = candidates[0].h;
+                        reason = "time";
+                        deltaMin = candidates[0].d;
+                    }
+                }
+                if (best) {
+                    const cr = parseCommissionCRFromFile(htmlPath, best);
+                    if (!cr) {
+                        console.warn(`[COM-CR][TRANSFORM] parse failed for ${f} → ${best.uid}`);
+                    }
+                    else {
+                        const fileUid = cr.uid;
+                        const outPath = path.join(transformedSessionDir, `${fileUid}.json`);
+                        await fs.writeJSON(outPath, cr, { spaces: 2 });
+                        const npts = Array.isArray(cr.contenu.point) ? cr.contenu.point.length : cr.contenu.point ? 1 : 0;
+                        if (!options["silent"]) {
+                            console.log(`[COM-CR][TRANSFORM] saved ${path.basename(outPath)} (points=${npts})`);
+                        }
+                    }
+                }
+                else {
+                    const hourShort = toHourShort(day.openTime) ?? "NA";
+                    const titreGuess = meta.organeDetected || meta.organeTitleRaw || "Commission";
+                    const { uid, filePath } = await createCommissionGroupIfMissing(dataDir, day.date, meta.organeDetected ?? null, hourShort, titreGuess);
+                    if (!options["silent"]) {
+                        console.log(`[COM-CR][PRE-SPLIT][${session}] ${f} | ${day.date}` +
+                            (day.openTime ? ` ${day.openTime}` : ``) +
+                            ` → NO-MATCH → CREATED uid=${uid} file=${path.basename(filePath)}`);
+                    }
+                }
+            }
+        }
+    }
+}
+async function main() {
+    const dataDir = options["dataDir"];
+    assert(dataDir, "Missing argument: data directory");
+    console.time("CRI processing time");
+    await retrieveCommissionCRs(options);
+    if (!options["silent"]) {
+        console.timeEnd("CRI processing time");
+    }
+}
+main()
+    .then(() => process.exit(0))
+    .catch((error) => {
+    console.error(error);
+    process.exit(1);
+});

package/lib/scripts/{retrieve_comptes_rendus.js → retrieve_cr_seance.js}

@@ -11,7 +11,7 @@ import StreamZip from "node-stream-zip";
 import * as cheerio from "cheerio";
 import { AGENDA_FOLDER, COMPTES_RENDUS_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, } from "../loaders";
 import { commonOptions } from "./shared/cli_helpers";
-import { deriveTitreObjetFromSommaire, parseCompteRenduSlotFromFile, parseYYYYMMDD, sessionStartYearFromDate } from "../model/compte_rendu";
+import { deriveTitreObjetFromSommaire, parseCompteRenduSlotFromFile, parseYYYYMMDD, sessionStartYearFromDate } from "../model/seance";
 import { makeGroupUid } from "../utils/reunion_grouping";
 import { getSessionsFromStart } from "../types/sessions";
 import { ensureAndClearDir, fetchWithRetry } from "./shared/util";

package/lib/scripts/retrieve_open_data.js

@@ -202,6 +202,30 @@ async function retrieveDataset(dataDir, dataset) {
             console.log(`Importing ${dataset.title}: ${sqlFilename}…`);
         }
         await copyToSenat(dataset, dataDir, options);
+        // Create indexes programmatically after import
+        if (dataset.indexes) {
+            for (const [table, indexes] of Object.entries(dataset.indexes)) {
+                for (const index of indexes) {
+                    const indexName = index.name;
+                    const columns = index.columns.join(", ");
+                    const schema = dataset.database;
+                    const sql = `CREATE INDEX IF NOT EXISTS ${indexName} ON ${schema}.${table} (${columns});`;
+                    try {
+                        execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -d senat -c "${sql}"`, {
+                            env: process.env,
+                            encoding: "utf-8",
+                            stdio: ["ignore", "ignore", "pipe"],
+                        });
+                        if (!options["silent"]) {
+                            console.log(`Created index: ${indexName} on ${schema}.${table} (${columns})`);
+                        }
+                    }
+                    catch (err) {
+                        console.error(`Failed to create index ${indexName} on ${schema}.${table}:`, err);
+                    }
+                }
+            }
+        }
     }
     if (options["schema"]) {
         let definitionsDir = path.resolve("src", "raw_types_schemats");
@@ -224,7 +248,7 @@ async function retrieveDataset(dataDir, dataset) {
         fs.writeFileSync(definitionFilePath, definitionRepaired);
         definitionsDir = path.resolve("src", "raw_types");
         definitionFilePath = path.join(definitionsDir, `${dataset.database}.ts`);
-        execSync(`npx pg-to-ts generate -c '${dbConnectionString}' -s ${dataset.database} -o ${definitionFilePath}`, {
+        execSync(`npx kysely-codegen --url '${dbConnectionString}' --default-schema ${dataset.database} --include-pattern '${dataset.database}.*' --out-file ${definitionFilePath}`, {
             env: process.env,
             encoding: "utf-8",
             // stdio: ["ignore", "ignore", "pipe"],
@@ -261,6 +285,16 @@ async function retrieveOpenData() {
     for (const dataset of chosenDatasets) {
         await retrieveDataset(dataDir, dataset);
     }
+    if (options["schema"]) {
+        const dbConnectionString = `postgres://${process.env["PGUSER"]}:${process.env["PGPASSWORD"]}@${process.env["PGHOST"]}:${process.env["PGPORT"]}/senat`;
+        const definitionsDir = path.resolve("src", "raw_types");
+        const definitionFilePath = path.join(definitionsDir, `senat.ts`);
+        execSync(`npx kysely-codegen --url '${dbConnectionString}' --out-file ${definitionFilePath}`, {
+            env: process.env,
+            encoding: "utf-8",
+            // stdio: ["ignore", "ignore", "pipe"],
+        });
+    }
     if (!options["silent"]) {
         console.timeEnd("data extraction time");
     }

package/lib/utils/cr_spliting.d.ts

@@ -1,7 +1,28 @@
-import { TimeSlot } from "../types/agenda";
+import { GroupedReunion, TimeSlot } from "../types/agenda";
 import * as cheerio from "cheerio";
 export declare function computeIntervalsBySlot($: cheerio.CheerioAPI, idx: Map<any, number>, firstSlotOfDay?: TimeSlot): {
     slot: TimeSlot;
     start: number;
     end: number;
 }[];
+export declare function parseCommissionMetadataFromHtml(html: string, sourceFileName?: string): {
+    sourceFile: string | null;
+    organeTitleRaw: string | null;
+    organeDetected: string | null;
+    organeCode: string | null;
+    weekStart: string | null;
+    days: {
+        date: string;
+        openTime?: string;
+        h2Index: number;
+    }[];
+};
+export declare function loadCommissionAgendaForDate(dataDir: string, yyyymmdd: string, session: number): Promise<GroupedReunion[]>;
+export declare function createCommissionGroupIfMissing(dataDir: string, dateISO: string, // "YYYY-MM-DD"
+organeDetected: string | null, // ex. "Commission des finances"
+hourShort: string | null, // "HHMM" | "NA"
+titreGuess?: string | null): Promise<{
+    uid: string;
+    filePath: string;
+    created: boolean;
+}>;