@tricoteuses/senat 3.0.2 → 3.1.1

package/lib/src/server/documents.js

@@ -0,0 +1,132 @@
+import { sql } from "kysely";
+import { jsonArrayFrom } from "kysely/helpers/postgres";
+import { db, streamRawQuery } from "./databases_postgres.js";
+export async function* findAllTextes() {
+    const compiledQuery = db
+        .withSchema("senat")
+        .selectFrom("dosleg_texte as texte")
+        .leftJoin("dosleg_oritxt as oritxt", "oritxt.oritxtcod", "texte.oritxtcod")
+        .leftJoin("dosleg_typtxt as typtxt", "typtxt.typtxtcod", "texte.typtxtcod")
+        .leftJoin("dosleg_lecass as lecass", "lecass.lecassidt", "texte.lecassidt")
+        .leftJoin("dosleg_lecture as lecture", "lecture.lecidt", "lecass.lecidt")
+        .leftJoin("dosleg_loi as loi", "loi.loicod", "lecture.loicod")
+        .select((eb) => [
+        "loi.signet as signet_dossier",
+        "texte.texnum as numero",
+        "texte.orgcod as code_organisme",
+        sql `
+        case
+          when ${eb.ref("texte.texurl")} is not null then
+            regexp_replace(regexp_replace(trim(${eb.ref("texte.texurl")}), '#+$', ''), '^(.*/)?(.*?)(\.html)?$', '\2')
+          else null
+        end
+      `.as("id"),
+        sql `
+        case
+          when ${eb.ref("texte.typurl")} = 'I' then regexp_replace('https://www.senat.fr/leg/' || rtrim(${eb.ref("texte.texurl")}), '#+$', '')
+          else regexp_replace(rtrim(${eb.ref("texte.texurl")}), '#+$', '')
+        end
+      `.as("url"),
+        sql `rtrim(${eb.ref("oritxt.oritxtlib")})`.as("origine"),
+        "oritxt.oriordre as ordre_origine",
+        "oritxt.oritxtado as code_adoption",
+        "oritxt.oritxtmod as modification",
+        sql `rtrim(${eb.ref("typtxt.typtxtlib")})`.as("type"),
+        sql `to_char(${eb.ref("texte.txtoritxtdat")}, 'YYYY-MM-DD')`.as("date"),
+        sql `${eb.ref("texte.sesann")}::int`.as("session"),
+        // Auteurs
+        jsonArrayFrom(eb
+            .withSchema("senat")
+            .selectFrom("dosleg_auteur as auteur")
+            .leftJoin("dosleg_ecr as ecr", "ecr.autcod", "auteur.autcod")
+            .leftJoin("dosleg_rolsig as rolsig", "rolsig.signataire", "ecr.signataire")
+            .select([
+            "auteur.prenom as prenom",
+            "auteur.nomuse as nom_usuel",
+            "auteur.autmat as matricule",
+            sql `ecr.ecrnumtri::text`.as("ordre"),
+            "ecr.ecrnumtri as ordre_num",
+            "rolsig.rolsiglib as role",
+            "ecr.ecrqua as qualite",
+        ])
+            .whereRef("ecr.texcod", "=", "texte.texcod")).as("auteurs"),
+        sql `null::text`.as("titre"),
+    ])
+        .orderBy(sql `array_position(array['0','2','1'], oritxt.oriordre)`)
+        .compile();
+    for await (const row of streamRawQuery(compiledQuery)) {
+        yield {
+            ...row,
+            auteurs: row.auteurs ?? [],
+        };
+    }
+}
+export async function* findAllRapports() {
+    const compiledQuery = db
+        .withSchema("senat")
+        .selectFrom("dosleg_rap as rap")
+        .leftJoin("dosleg_raporg as raporg", "raporg.rapcod", "rap.rapcod")
+        .leftJoin("dosleg_denrap as denrap", "denrap.coddenrap", "rap.coddenrap")
+        .leftJoin("dosleg_lecassrap as lecassrap", "lecassrap.rapcod", "rap.rapcod")
+        .leftJoin("dosleg_lecass as lecass", "lecass.lecassidt", "lecassrap.lecassidt")
+        .leftJoin("dosleg_lecture as lecture", "lecture.lecidt", "lecass.lecidt")
+        .leftJoin("dosleg_loi as loi", "loi.loicod", "lecture.loicod")
+        .select((eb) => [
+        "loi.signet as signet_dossier",
+        "rap.rapnum as numero",
+        "raporg.orgcod as code_organisme",
+        sql `
+        case
+          when ${eb.ref("rap.rapurl")} is not null then
+            regexp_replace(regexp_replace(trim(${eb.ref("rap.rapurl")}), '#+$', ''), '^(.*/)?(.*?)(\.html)?$', '\2')
+          else null
+        end
+      `.as("id"),
+        sql `
+        case
+          when ${eb.ref("rap.typurl")} = 'I' then regexp_replace('https://www.senat.fr/rap/' || rtrim(${eb.ref("rap.rapurl")}), '#+$', '')
+          else regexp_replace(rtrim(${eb.ref("rap.rapurl")}), '#+$', '')
+        end
+      `.as("url"),
+        sql `rtrim(${eb.ref("denrap.libdenrap")})`.as("type"),
+        sql `rtrim(rtrim(${eb.ref("rap.raptil")}))`.as("titre"),
+        sql `rtrim(rtrim(${eb.ref("rap.rapsoustit")}))`.as("sous_titre"),
+        sql `to_char(${eb.ref("rap.date_depot")}, 'YYYY-MM-DD')`.as("date"),
+        sql `${eb.ref("rap.sesann")}::int`.as("session"),
+        // Auteurs
+        jsonArrayFrom(eb
+            .withSchema("senat")
+            .selectFrom("dosleg_auteur as auteur")
+            .leftJoin("dosleg_ecr as ecr", "ecr.autcod", "auteur.autcod")
+            .leftJoin("dosleg_rolsig as rolsig", "rolsig.signataire", "ecr.signataire")
+            .select([
+            "auteur.prenom as prenom",
+            "auteur.nomuse as nom_usuel",
+            "auteur.autmat as matricule",
+            sql `ecr.ecrnumtri::text`.as("ordre"),
+            "ecr.ecrnumtri as ordre_num",
+            "rolsig.rolsiglib as role",
+            "ecr.ecrqua as qualite",
+        ])
+            .whereRef("ecr.rapcod", "=", "rap.rapcod")).as("auteurs"),
+        // Documents annexes
+        jsonArrayFrom(eb
+            .withSchema("senat")
+            .selectFrom("dosleg_docatt as docatt")
+            .leftJoin("dosleg_typatt as typatt", "typatt.typattcod", "docatt.typattcod")
+            .select(["docatt.docatturl as url", "typatt.typattlib as type_document"])
+            .whereRef("docatt.rapcod", "=", "rap.rapcod")).as("documents_annexes"),
+        sql `null::text`.as("origine"),
+        sql `null::text`.as("ordre_origine"),
+        sql `null::text`.as("code_adoption"),
+        sql `null::text`.as("modification"),
+    ])
+        .compile();
+    for await (const row of streamRawQuery(compiledQuery)) {
+        yield {
+            ...row,
+            auteurs: row.auteurs ?? [],
+            documents_annexes: row.documents_annexes ?? [],
+        };
+    }
+}

package/lib/src/server/dosleg.d.ts

@@ -0,0 +1,2 @@
+import type { DossierLegislatifResult } from "../rich_types/dosleg.js";
+export declare function findAllDossiers(): AsyncGenerator<DossierLegislatifResult, void, unknown>;

package/lib/src/server/dosleg.js

@@ -0,0 +1,228 @@
+import { sql } from "kysely";
+import { jsonArrayFrom } from "kysely/helpers/postgres";
+import { db, streamRawQuery } from "./databases_postgres.js";
+export async function* findAllDossiers() {
+    const compiledQuery = db
+        .withSchema("senat")
+        .selectFrom("dosleg_loi as loi")
+        .leftJoin("dosleg_typloi as typloi", "typloi.typloicod", "loi.typloicod")
+        .leftJoin("dosleg_etaloi as etaloi", "etaloi.etaloicod", "loi.etaloicod")
+        .leftJoin("dosleg_deccoc as deccoc", "deccoc.deccoccod", "loi.deccoccod")
+        .select((eb) => [
+        sql `rtrim(${eb.ref("loi.loicod")})`.as("code"),
+        "loi.numero as numero",
+        "loi.signet as signet",
+        sql `nullif(regexp_replace(${eb.ref("loi.url_an")}, '^.*\\/(DL[^\\/]+)\\.asp$', '\\1'), '')`.as("code_dossier_an"),
+        "loi.signetalt as signet_alternatif",
+        sql `rtrim(${eb.ref("loi.motclef")})`.as("mot_cle"),
+        sql `rtrim(${eb.ref("loi.loient")})`.as("titre_court"),
+        sql `rtrim(${eb.ref("typloi.typloiden")}) || ' ' || rtrim(${eb.ref("loi.loitit")})`.as("titre"),
+        sql `rtrim(${eb.ref("typloi.typloiden")}) || ' ' || rtrim(${eb.ref("loi.loiint")})`.as("titre_long"),
+        sql `rtrim(${eb.ref("typloi.typloiden")}) || ' ' || rtrim(${eb.ref("loi.loiintori")})`.as("titre_long_original"),
+        sql `'https://www.senat.fr/dossier-legislatif/' || ${eb.ref("loi.signet")} || '.html'`.as("url"),
+        "loi.urgence as urgence",
+        sql `rtrim(${eb.ref("typloi.groupe")})`.as("code_nature_dossier"),
+        sql `rtrim(${eb.ref("typloi.typloilib")})`.as("libelle_type_dossier"),
+        sql `rtrim(${eb.ref("etaloi.etaloilib")})`.as("etat_dossier"),
+        "loi.url_an as url_dossier_assemblee_nationale",
+        "loi.url_presart as url_presentation_articles",
+        "loi.url_ordonnance as url_ordonnance",
+        "loi.orgcod as code_organisme_resolution",
+        "deccoc.deccoclib as libelle_decision_CoC",
+        sql `to_char(${eb.ref("loi.date_decision")}, 'YYYY-MM-DD')`.as("date_decision_CoC"),
+        "loi.num_decision as num_decision_CoC",
+        "loi.deccocurl as url_decision_CoC",
+        "loi.doscocurl as url_dossier_CoC",
+        sql `to_char(${eb.ref("loi.saisine_date")}, 'YYYY-MM-DD')`.as("date_saisine_CoC"),
+        "loi.saisine_par as condition_saisine_CoC",
+        sql `to_char(${eb.ref("loi.proaccdat")}, 'YYYY-MM-DD')`.as("date_procedure_acceleree"),
+        sql `to_char(${eb.ref("loi.proaccoppdat")}, 'YYYY-MM-DD')`.as("date_opposition_procedure_acceleree"),
+        sql `to_char(${eb.ref("loi.retproaccdat")}, 'YYYY-MM-DD')`.as("date_retrait_procedure_acceleree"),
+        sql `to_char(${eb.ref("loi.date_loi")}, 'YYYY-MM-DD')`.as("date_promulgation"),
+        sql `
+        case
+          when ${eb.ref("loi.loititjo")} is not null
+            then rtrim(${eb.ref("typloi.typloiden")})
+              || ' ' || rtrim(${eb.ref("loi.loititjo")})
+          else ''
+        end
+      `.as("titre_JO"),
+        sql `to_char(${eb.ref("loi.loidatjo")}, 'YYYY-MM-DD')`.as("date_publication_JO"),
+        "loi.loinumjo as numero_JO",
+        "loi.url_jo as url_JO",
+        sql `to_char(${eb.ref("loi.loidatjo2")}, 'YYYY-MM-DD')`.as("date_publication_JO_correctif_1"),
+        "loi.loinumjo2 as numero_JO_correctif_1",
+        "loi.url_jo2 as url_JO_correctif_1",
+        sql `to_char(${eb.ref("loi.loidatjo3")}, 'YYYY-MM-DD')`.as("date_publication_JO_correctif_2"),
+        "loi.loinumjo3 as numero_JO_correctif_2",
+        "loi.url_jo3 as url_JO_correctif_2",
+        // Lectures
+        jsonArrayFrom(eb
+            .withSchema("senat")
+            .selectFrom("dosleg_lecture as lecture")
+            .leftJoin("dosleg_typlec as typlec", "typlec.typleccod", "lecture.typleccod")
+            .select((eb2) => [
+            sql `rtrim(typlec.typleclib)`.as("type_lecture"),
+            sql `rtrim(lecture.leccom)`.as("libelle"),
+            sql `typlec.typlecord::text`.as("ordre_lecture"),
+            "typlec.typlecord as ordre_lecture_num",
+            // Lectures par assemblée
+            jsonArrayFrom(eb2
+                .withSchema("senat")
+                .selectFrom("dosleg_lecass as lecass")
+                .leftJoin("dosleg_ass as ass", "ass.codass", "lecass.codass")
+                .leftJoin("dosleg_org as org", "org.orgcod", "lecass.orgcod")
+                .leftJoin("dosleg_orippr as orippr", "orippr.oripprcod", "lecass.orippr")
+                .select((eb3) => [
+                sql `rtrim(ass.libass)`.as("assemblee"),
+                "org.orgnom as libelle_organisme",
+                "org.senorgcod as code_organisme",
+                sql `lecass.ordreass::text`.as("ordre_lecture_assemblee"),
+                "lecass.ordreass as ordre_lecture_assemblee_num",
+                sql `lecass.sesann::int`.as("session"),
+                "orippr.oripprlib as origine_proposition",
+                "lecass.ptlnum as numero_petite_loi",
+                "lecass.ptlurl as url_petite_loi",
+                "lecass.loiintmod as loi_intitule_modifie",
+                "lecass.debatsurl as url_cr_debats",
+                sql `to_char(lecass.lecassamecomdat, 'YYYY-MM-DD')`.as("date_publication_amendements_commission"),
+                sql `to_char(lecass.lecassamedat, 'YYYY-MM-DD')`.as("date_publication_amendements_seance"),
+                // Textes
+                jsonArrayFrom(eb3
+                    .withSchema("senat")
+                    .selectFrom("dosleg_texte as texte")
+                    .leftJoin("dosleg_oritxt as oritxt", "oritxt.oritxtcod", "texte.oritxtcod")
+                    .leftJoin("dosleg_typtxt as typtxt", "typtxt.typtxtcod", "texte.typtxtcod")
+                    .select((eb4) => [
+                    "texte.texnum as numero",
+                    "texte.orgcod as code_organisme",
+                    sql `
+                          case
+                            when texte.texurl is not null then
+                              regexp_replace(
+                                regexp_replace(trim(texte.texurl), '#+$', ''),
+                                '^(.*/)?(.*?)(\.html)?$', '\2')
+                            else null
+                          end
+                        `.as("id"),
+                    sql `
+                          case
+                            when texte.typurl = 'I' then regexp_replace('https://www.senat.fr/leg/' || rtrim(texte.texurl), '#+$', '')
+                            else regexp_replace(rtrim(texte.texurl), '#+$', '')
+                          end
+                        `.as("url"),
+                    sql `rtrim(oritxt.oritxtlib)`.as("origine"),
+                    "oritxt.oriordre as ordre_origine",
+                    "oritxt.oritxtado as code_adoption",
+                    "oritxt.oritxtmod as modification",
+                    sql `rtrim(typtxt.typtxtlib)`.as("type"),
+                    sql `to_char(texte.txtoritxtdat, 'YYYY-MM-DD')`.as("date"),
+                    sql `texte.sesann::int`.as("session"),
+                    // Auteurs du texte
+                    jsonArrayFrom(eb4
+                        .withSchema("senat")
+                        .selectFrom("dosleg_auteur as auteur")
+                        .leftJoin("dosleg_ecr as ecr", "ecr.autcod", "auteur.autcod")
+                        .leftJoin("dosleg_rolsig as rolsig", "rolsig.signataire", "ecr.signataire")
+                        .select([
+                        "auteur.prenom as prenom",
+                        "auteur.nomuse as nom_usuel",
+                        "auteur.autmat as matricule",
+                        sql `ecr.ecrnumtri::text`.as("ordre"),
+                        "ecr.ecrnumtri as ordre_num",
+                        "rolsig.rolsiglib as role",
+                        "ecr.ecrqua as qualite",
+                    ])
+                        .whereRef("ecr.texcod", "=", "texte.texcod")).as("auteurs"),
+                    sql `array_position(array['0','2','1'], oritxt.oriordre)`.as("ordre_tri"),
+                ])
+                    .whereRef("texte.lecassidt", "=", "lecass.lecassidt")
+                    .orderBy(sql `array_position(array['0','2','1'], oritxt.oriordre)`)).as("textes"),
+                // Rapports
+                jsonArrayFrom(eb3
+                    .withSchema("senat")
+                    .selectFrom("dosleg_rap as rap")
+                    .leftJoin("dosleg_raporg as raporg", "raporg.rapcod", "rap.rapcod")
+                    .leftJoin("dosleg_denrap as denrap", "denrap.coddenrap", "rap.coddenrap")
+                    .leftJoin("dosleg_lecassrap as lecassrap", "lecassrap.rapcod", "rap.rapcod")
+                    .select((eb4) => [
+                    "rap.rapnum as numero",
+                    "raporg.orgcod as code_organisme",
+                    sql `
+                          case
+                            when rap.rapurl is not null then
+                              regexp_replace(
+                                regexp_replace(trim(rap.rapurl), '#+$', ''),
+                                '^(.*/)?(.*?)(\.html)?$', '\2')
+                            else null
+                          end
+                        `.as("id"),
+                    sql `
+                          case
+                            when rap.typurl = 'I' then regexp_replace('https://www.senat.fr/rap/' || rtrim(rap.rapurl), '#+$', '')
+                            else regexp_replace(rtrim(rap.rapurl), '#+$', '')
+                          end
+                        `.as("url"),
+                    sql `rtrim(denrap.libdenrap)`.as("type"),
+                    sql `rtrim(rtrim(rap.raptil))`.as("titre"),
+                    sql `rtrim(rtrim(rap.rapsoustit))`.as("sous_titre"),
+                    sql `to_char(rap.date_depot, 'YYYY-MM-DD')`.as("date"),
+                    sql `rap.sesann::int`.as("session"),
+                    // Auteurs du rapport
+                    jsonArrayFrom(eb4
+                        .withSchema("senat")
+                        .selectFrom("dosleg_auteur as auteur")
+                        .leftJoin("dosleg_ecr as ecr", "ecr.autcod", "auteur.autcod")
+                        .leftJoin("dosleg_rolsig as rolsig", "rolsig.signataire", "ecr.signataire")
+                        .select([
+                        "auteur.prenom as prenom",
+                        "auteur.nomuse as nom_usuel",
+                        "auteur.autmat as matricule",
+                        sql `ecr.ecrnumtri::text`.as("ordre"),
+                        "ecr.ecrnumtri as ordre_num",
+                        "rolsig.rolsiglib as role",
+                        "ecr.ecrqua as qualite",
+                    ])
+                        .whereRef("ecr.rapcod", "=", "rap.rapcod")).as("auteurs"),
+                    // Documents annexes
+                    jsonArrayFrom(eb4
+                        .withSchema("senat")
+                        .selectFrom("dosleg_docatt as docatt")
+                        .leftJoin("dosleg_typatt as typatt", "typatt.typattcod", "docatt.typattcod")
+                        .select(["docatt.docatturl as url", "typatt.typattlib as type_document"])
+                        .whereRef("docatt.rapcod", "=", "rap.rapcod")).as("documents_annexes"),
+                ])
+                    .whereRef("lecassrap.lecassidt", "=", "lecass.lecassidt")).as("rapports"),
+                // Dates de séances
+                jsonArrayFrom(eb3
+                    .withSchema("senat")
+                    .selectFrom("dosleg_date_seance as date_seance")
+                    .select([sql `to_char(date_seance.date_s, 'YYYY-MM-DD')`.as("date")])
+                    .whereRef("date_seance.lecidt", "=", "lecass.lecassidt")).as("dates_seances"),
+            ])
+                .whereRef("lecass.lecidt", "=", "lecture.lecidt")
+                .orderBy("lecass.ordreass", "asc")).as("lectures_assemblee"),
+        ])
+            .whereRef("lecture.loicod", "=", "loi.loicod")
+            .orderBy("typlec.typlecord", "asc")).as("lectures"),
+        // Thèmes
+        jsonArrayFrom(eb
+            .withSchema("senat")
+            .selectFrom("dosleg_the as the")
+            .leftJoin("dosleg_loithe as loithe", "loithe.thecle", "the.thecle")
+            .select(["the.thelib as libelle"])
+            .whereRef("loithe.loicod", "=", "loi.loicod")).as("themes"),
+    ])
+        .compile();
+    for await (const row of streamRawQuery(compiledQuery)) {
+        if (!row.signet) {
+            continue;
+        }
+        yield {
+            ...row,
+            signet: row.signet,
+            lectures: row.lectures ?? [],
+            themes: row.themes ?? [],
+        };
+    }
+}

package/lib/{git.d.ts → src/server/git.d.ts}

@@ -1,6 +1,7 @@
 export declare function initRepo(repositoryDir: string): void;
 export declare function commit(repositoryDir: string, message: string): boolean;
 export declare function commitAndPush(repositoryDir: string, message: string, remotes?: string[]): number;
+export declare function pull(repositoryDir: string, remote?: string, branch?: string): boolean;
 export declare function resetAndPull(gitDir: string): boolean;
 export declare function clone(gitGroupUrl: string | undefined, gitName: string, workingDir: string): void;
 export declare function run(repositoryDir: string, args: string, verbose?: boolean): string;

package/lib/{git.js → src/server/git.js}

@@ -2,6 +2,66 @@ import { execSync } from "node:child_process";
 import fs from "fs-extra";
 import path from "node:path";
 const MAXBUFFER = 50 * 1024 * 1024;
+const GIT_LOCK_RETRY_DELAY_MS = 1000;
+const GIT_LOCK_RETRY_COUNT = 5;
+const GIT_LOCK_STALE_AFTER_MS = 2 * 60 * 1000;
+function asExecSyncError(error) {
+    return error;
+}
+function outputToString(output) {
+    if (output == null) {
+        return "";
+    }
+    return Buffer.isBuffer(output) ? output.toString("utf-8") : output;
+}
+function sleep(ms) {
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+function getIndexLockPath(repositoryDir) {
+    return path.join(repositoryDir, ".git", "index.lock");
+}
+function isIndexLockError(error) {
+    const stderr = error instanceof Error ? outputToString(asExecSyncError(error).stderr) : "";
+    return /index\.lock': File exists\./.test(stderr);
+}
+function removeStaleIndexLock(repositoryDir) {
+    const lockPath = getIndexLockPath(repositoryDir);
+    if (!fs.existsSync(lockPath)) {
+        return false;
+    }
+    const stats = fs.statSync(lockPath);
+    const ageMs = Date.now() - stats.mtimeMs;
+    if (ageMs < GIT_LOCK_STALE_AFTER_MS) {
+        return false;
+    }
+    fs.removeSync(lockPath);
+    return true;
+}
+function execGitWithIndexLockRecovery(command, repositoryDir, options) {
+    let lockRemoved = false;
+    for (let attempt = 1; attempt <= GIT_LOCK_RETRY_COUNT; attempt++) {
+        try {
+            execSync(command, {
+                cwd: repositoryDir,
+                ...options,
+            });
+            return;
+        }
+        catch (error) {
+            if (!isIndexLockError(error)) {
+                throw error;
+            }
+            if (!lockRemoved && removeStaleIndexLock(repositoryDir)) {
+                lockRemoved = true;
+                continue;
+            }
+            if (attempt === GIT_LOCK_RETRY_COUNT) {
+                throw error;
+            }
+            sleep(GIT_LOCK_RETRY_DELAY_MS);
+        }
+    }
+}
 export function initRepo(repositoryDir) {
     if (!fs.existsSync(path.join(repositoryDir, ".git"))) {
         fs.ensureDirSync(repositoryDir);
@@ -15,8 +75,7 @@ export function initRepo(repositoryDir) {
 }
 export function commit(repositoryDir, message) {
     initRepo(repositoryDir);
-    execSync("git add .", {
-        cwd: repositoryDir,
+    execGitWithIndexLockRecovery("git add .", repositoryDir, {
         env: process.env,
         encoding: "utf-8",
         stdio: ["ignore", "ignore", "pipe"],
@@ -31,9 +90,9 @@ export function commit(repositoryDir, message) {
         });
         return true;
     }
-    catch (childProcess) {
-        if (childProcess.stdout === null ||
-            !/nothing to commit|rien à valider/.test(childProcess.stdout)) {
+    catch (error) {
+        const childProcess = asExecSyncError(error);
+        if (childProcess.stdout === null || !/nothing to commit|rien à valider/.test(outputToString(childProcess.stdout))) {
             console.error(childProcess.output);
             throw childProcess;
         }
@@ -52,10 +111,11 @@ export function commitAndPush(repositoryDir, message, remotes) {
                     stdio: ["ignore", "ignore", "pipe"],
                 });
             }
-            catch (childProcess) {
+            catch (error) {
+                const childProcess = asExecSyncError(error);
                 // Don't stop when push fails.
                 console.error(childProcess.output);
-                exitCode = childProcess.status;
+                exitCode = childProcess.status ?? 1;
             }
         }
     }
@@ -65,6 +125,27 @@ export function commitAndPush(repositoryDir, message, remotes) {
     }
     return exitCode;
 }
+export function pull(repositoryDir, remote = "origin", branch = "master") {
+    initRepo(repositoryDir);
+    const statusOutput = execSync("git status --porcelain", {
+        cwd: repositoryDir,
+        env: process.env,
+        encoding: "utf-8",
+        stdio: ["ignore", "pipe", "pipe"],
+        maxBuffer: MAXBUFFER,
+    }).trim();
+    if (statusOutput.length > 0) {
+        return false;
+    }
+    execSync(`git pull --rebase ${remote} ${branch}`, {
+        cwd: repositoryDir,
+        env: process.env,
+        encoding: "utf-8",
+        stdio: ["ignore", "ignore", "pipe"],
+        maxBuffer: MAXBUFFER,
+    });
+    return true;
+}
 export function resetAndPull(gitDir) {
     execSync("git reset --hard origin/master", {
         cwd: gitDir,
@@ -104,9 +185,11 @@ export function run(repositoryDir, args, verbose) {
             console.log(output);
         return output;
     }
-    catch (childProcess) {
-        for (const output of ["stdout", "stderr"])
-            console.error(`${output}: ${childProcess[output]}`);
+    catch (error) {
+        const childProcess = asExecSyncError(error);
+        for (const output of ["stdout", "stderr"]) {
+            console.error(`${output}: ${outputToString(childProcess[output])}`);
+        }
         throw childProcess;
     }
 }
@@ -125,8 +208,9 @@ export function test(repositoryDir, args, verbose) {
             console.log(output);
         return true;
     }
-    catch (childProcess) {
-        if (childProcess.status != 0)
+    catch (error) {
+        const childProcess = asExecSyncError(error);
+        if (childProcess.status !== 0)
             return false;
         throw childProcess;
     }

package/lib/src/server/index.d.ts

@@ -0,0 +1,18 @@
+export { AGENDA_FOLDER, COMPTES_RENDUS_FOLDER, COMMISSION_FOLDER, DOSLEG_DOSSIERS_FOLDER, ENRICHED_TEXTE_FOLDER, SCRUTINS_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER, RAPPORT_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, DOCUMENT_METADATA_FILE, iterFilePaths, iterLoadSenatAmendements, iterLoadSenatDebats, iterLoadSenatComptesRendusSeances, iterLoadSenatComptesRendusCommissions, iterLoadSenatDossiersLegislatifs, iterLoadSenatRapportUrls, iterLoadSenatTexteUrls, iterLoadSenatRapports, iterLoadSenatTextes, loadSenatTexteContent, loadSenatCompteRenduContent, iterLoadSenatAgendas, iterLoadSenatCirconscriptions, iterLoadSenatOrganismes, iterLoadSenatSenateurs, iterLoadSenatQuestions, iterLoadSenatScrutins, type DocumentResult, type IterItem, } from "./loaders.js";
+export { datasets, getChosenDatasets, getEnabledDatasets, EnabledDatasets } from "./datasets.js";
+export { default as config, configSchema, dbSchema } from "./config.js";
+export type { DbConfig } from "./config.js";
+export { db, sql, streamRawQuery } from "./databases_postgres.js";
+export type { SenatDatabase } from "./databases_postgres.js";
+export { commit, commitAndPush, clone, getChangedFilesSinceCommit, initRepo, pull, resetAndPull, run } from "./git.js";
+export { convertSenatXmlToHtml, extractMetadata } from "./conversion_textes.js";
+export { parseAgendaFromFile, getStartAndEndTimes } from "./agenda.js";
+export { cleanTitle, extractDayH3Sections, getRemainingTextAfterSpeakerHeader, parseCommissionCRSectionFromDom, } from "./commission.js";
+export { findAllAmendements } from "./ameli.js";
+export { findAll as findAllDebats } from "./debats.js";
+export { findAllDossiers } from "./dosleg.js";
+export { findAllTextes, findAllRapports } from "./documents.js";
+export { findAll as findAllQuestions } from "./questions.js";
+export { findAllScrutins } from "./scrutins.js";
+export { parseCompteRenduIntervalFromFile, sessionStartYearFromDate } from "./seance.js";
+export { findAll as findAllSens, findAllCirconscriptions, findAllOrganismes, findActif } from "./sens.js";

package/lib/src/server/index.js

@@ -0,0 +1,16 @@
+export { AGENDA_FOLDER, COMPTES_RENDUS_FOLDER, COMMISSION_FOLDER, DOSLEG_DOSSIERS_FOLDER, ENRICHED_TEXTE_FOLDER, SCRUTINS_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER, RAPPORT_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, DOCUMENT_METADATA_FILE, iterFilePaths, iterLoadSenatAmendements, iterLoadSenatDebats, iterLoadSenatComptesRendusSeances, iterLoadSenatComptesRendusCommissions, iterLoadSenatDossiersLegislatifs, iterLoadSenatRapportUrls, iterLoadSenatTexteUrls, iterLoadSenatRapports, iterLoadSenatTextes, loadSenatTexteContent, loadSenatCompteRenduContent, iterLoadSenatAgendas, iterLoadSenatCirconscriptions, iterLoadSenatOrganismes, iterLoadSenatSenateurs, iterLoadSenatQuestions, iterLoadSenatScrutins, } from "./loaders.js";
+export { datasets, getChosenDatasets, getEnabledDatasets, EnabledDatasets } from "./datasets.js";
+export { default as config, configSchema, dbSchema } from "./config.js";
+export { db, sql, streamRawQuery } from "./databases_postgres.js";
+export { commit, commitAndPush, clone, getChangedFilesSinceCommit, initRepo, pull, resetAndPull, run } from "./git.js";
+export { convertSenatXmlToHtml, extractMetadata } from "./conversion_textes.js";
+export { parseAgendaFromFile, getStartAndEndTimes } from "./agenda.js";
+export { cleanTitle, extractDayH3Sections, getRemainingTextAfterSpeakerHeader, parseCommissionCRSectionFromDom, } from "./commission.js";
+export { findAllAmendements } from "./ameli.js";
+export { findAll as findAllDebats } from "./debats.js";
+export { findAllDossiers } from "./dosleg.js";
+export { findAllTextes, findAllRapports } from "./documents.js";
+export { findAll as findAllQuestions } from "./questions.js";
+export { findAllScrutins } from "./scrutins.js";
+export { parseCompteRenduIntervalFromFile, sessionStartYearFromDate } from "./seance.js";
+export { findAll as findAllSens, findAllCirconscriptions, findAllOrganismes, findActif } from "./sens.js";

package/lib/{loaders.d.ts → src/server/loaders.d.ts}

@@ -1,19 +1,20 @@
-import { AmendementResult } from "./model/ameli";
-import { DebatResult } from "./model/debats";
-import { DossierLegislatifResult } from "./model/dosleg";
-import { QuestionResult } from "./model/questions";
-import { ScrutinResult } from "./model/scrutins";
-import { CirconscriptionResult, OrganismeResult, SenateurResult } from "./model/sens";
-import { Reunion } from "./types/agenda";
-import { FlatTexte, DocumentMetadata } from "./types/texte";
-import { CompteRendu } from "./types/compte_rendu";
-import { DocumentResult } from "./model/documents";
-export { EnabledDatasets } from "./datasets";
-export type { DocumentResult } from "./model/documents";
+import { AmendementResult } from "../rich_types/ameli.js";
+import { DebatResult } from "../rich_types/debats.js";
+import { DossierLegislatifResult } from "../rich_types/dosleg.js";
+import { QuestionResult } from "../rich_types/questions.js";
+import { ScrutinResult } from "../scrutins.js";
+import { CirconscriptionResult, OrganismeResult, SenateurResult } from "../rich_types/sens.js";
+import { Reunion } from "../other_types/agenda.js";
+import { FlatTexte, DocumentMetadata } from "../other_types/texte.js";
+import { CompteRendu } from "../other_types/compte_rendu.js";
+import { DocumentResult } from "../documents.js";
+export { EnabledDatasets } from "./datasets.js";
+export type { DocumentResult } from "../documents.js";
 export declare const AGENDA_FOLDER = "agenda";
 export declare const COMPTES_RENDUS_FOLDER = "seances";
 export declare const COMMISSION_FOLDER = "commissions";
 export declare const DOSLEG_DOSSIERS_FOLDER = "dossiers";
+export declare const ENRICHED_TEXTE_FOLDER = "leg_enrichi";
 export declare const SCRUTINS_FOLDER = "scrutins";
 export declare const SENS_CIRCONSCRIPTIONS_FOLDER = "circonscriptions";
 export declare const SENS_ORGANISMES_FOLDER = "organismes";
@@ -32,21 +33,14 @@ export type IterItem<T> = {
 export declare function iterFilePaths(dirPath: string): Generator<string>;
 export declare function iterLoadSenatAmendements(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<AmendementResult>>;
 export declare function iterLoadSenatDebats(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DebatResult>>;
-export declare function iterLoadSenatComptesRendusSeances(dataDir: string, session: number): Generator<{
-    compteRendu: CompteRendu;
-    session: number;
-}>;
-export declare function iterLoadSenatComptesRendusCommissions(dataDir: string, session: number): Generator<{
-    compteRendu: CompteRendu;
-    session: number;
-}>;
+export declare function iterLoadSenatComptesRendusSeances(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<CompteRendu>>;
+export declare function iterLoadSenatComptesRendusCommissions(dataDir: string, session: number, options?: {}): Generator<IterItem<CompteRendu>>;
 export declare function iterLoadSenatDossiersLegislatifs(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DossierLegislatifResult>>;
 export declare function iterLoadSenatRapportUrls(dataDir: string, session: number | undefined): Generator<IterItem<DocumentMetadata>>;
 export declare function iterLoadSenatTexteUrls(dataDir: string, session: number | undefined): Generator<IterItem<DocumentMetadata>>;
-export declare function iterLoadSenatDocuments(dataDir: string, session: number | undefined, documentType: "textes" | "rapports", options?: {}): Generator<IterItem<DocumentResult>>;
 export declare function iterLoadSenatRapports(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DocumentResult>>;
-export declare function iterLoadSenatTextes(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DocumentResult>>;
-export declare function loadSenatTexteContent(dataDir: string, textePathFromDataset: string): IterItem<FlatTexte | null>;
+export declare function iterLoadSenatTextes(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DocumentResult & Partial<FlatTexte>>>;
+export declare function loadSenatTexteContent(dataDir: string, session: number | string | null | undefined, texteId: string): IterItem<FlatTexte | null>;
 export declare function loadSenatCompteRenduContent(dataDir: string, session: number, debatId: string | number): {
     item: CompteRendu | null;
 };