@tricoteuses/senat 3.1.3 → 3.1.5

package/lib/src/raw_types_schemats/sens.js

@@ -0,0 +1,2 @@
+/* tslint:disable */
+export {};

package/lib/src/scripts/data-download.js

@@ -1,13 +1,20 @@
 import { execSync } from "child_process";
 const args = process.argv.slice(2).join(" ");
-try {
-    execSync(`tsx src/scripts/retrieve_open_data.ts --all ${args}`, { stdio: "inherit" });
-    execSync(`tsx src/scripts/convert_data.ts ${args}`, { stdio: "inherit" });
-}
-catch (error) {
-    const execError = error;
-    if (execError.status !== 10) {
-        console.error("Error during data retrieval:", error);
-        process.exit(execError.status || 1);
+function runScript(command) {
+    try {
+        execSync(command, { stdio: "inherit" });
+    }
+    catch (error) {
+        const execError = error;
+        if (execError.status !== 10) {
+            console.error(`Error during: ${command}`, error);
+            process.exit(execError.status || 1);
+        }
     }
 }
+runScript(`tsx src/scripts/retrieve_open_data.ts --all ${args}`);
+runScript(`tsx src/scripts/convert_data.ts ${args}`);
+runScript(`cross-env TZ='Etc/UTC' tsx src/scripts/retrieve_agenda.ts ${args} --parseAgenda --silent`);
+runScript(`tsx src/scripts/retrieve_cr_seance.ts ${args} --parseDebats --silent`);
+runScript(`tsx src/scripts/retrieve_cr_commission.ts ${args} --parseDebats --silent`);
+runScript(`tsx src/scripts/retrieve_videos.ts ${args} --silent`);

package/lib/src/scripts/debug_dosleg_query.d.ts

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+/**
+ * Debug script to check the DosLeg query compilation
+ * Helps identify the "column undefined" error
+ */
+export {};

package/lib/src/scripts/debug_dosleg_query.js

@@ -0,0 +1,50 @@
+#!/usr/bin/env node
+/**
+ * Debug script to check the DosLeg query compilation
+ * Helps identify the "column undefined" error
+ */
+import { db } from "../server/databases_postgres.js";
+import { jsonArrayFrom } from "kysely/helpers/postgres";
+import { sql } from "kysely";
+async function debugDoslegQuery() {
+    console.log("=== Debugging DosLeg Query ===\n");
+    try {
+        // Build the same query as findAllDossiers but capture SQL
+        const query = db
+            .withSchema("senat")
+            .selectFrom("dosleg_loi as loi")
+            .leftJoin("dosleg_typloi as typloi", "typloi.typloicod", "loi.typloicod")
+            .leftJoin("dosleg_etaloi as etaloi", "etaloi.etaloicod", "loi.etaloicod")
+            .leftJoin("dosleg_deccoc as deccoc", "deccoc.deccoccod", "loi.deccoccod")
+            .select((eb) => [
+            sql `rtrim(${eb.ref("loi.loicod")})`.as("code"),
+            "loi.numero as numero",
+            "loi.signet as signet",
+        ]);
+        const compiled = query.compile();
+        console.log("✓ Query compiled successfully (basic columns)\n");
+        console.log("SQL:", compiled.sql);
+        console.log("\n");
+        // Try with simple lectures first
+        const queryWithLectures = db
+            .withSchema("senat")
+            .selectFrom("dosleg_loi as loi")
+            .select((eb) => [
+            "loi.signet as signet",
+            jsonArrayFrom(eb.withSchema("senat").selectFrom("dosleg_lecture as lecture").select(["lecture.lecidt as id"])).as("lectures"),
+        ]);
+        const compiledWithLectures = queryWithLectures.compile();
+        console.log("✓ Query with simple lectures compiled successfully\n");
+        console.log("SQL:", compiledWithLectures.sql);
+        console.log("\n");
+        console.log("✅ Basic query structures seem valid");
+        console.log("\nThe issue is likely in one of the nested subqueries with cross-references.");
+        console.log("Try running findAllDossiers with the verbose flag to see exactly where it fails.");
+    }
+    catch (error) {
+        console.error("❌ Error during query compilation:");
+        console.error(error instanceof Error ? error.message : String(error));
+        process.exit(1);
+    }
+}
+debugDoslegQuery().catch(console.error);

package/lib/src/scripts/retrieve_agenda.js

@@ -52,13 +52,11 @@ async function retrieveAgendas(options, sessions) {
     for (const session of sessions) {
         const originalAgendaSessionDir = path.join(originalAgendaDir, `${session}`);
         fs.ensureDirSync(originalAgendaSessionDir);
-        if (!options["keepDir"])
-            fs.emptyDirSync(originalAgendaSessionDir);
+        fs.emptyDirSync(originalAgendaSessionDir);
         const transformedAgendaSessionDir = path.join(transformedAgendaDir, `${session}`);
         if (options["parseAgenda"]) {
             fs.ensureDirSync(transformedAgendaSessionDir);
-            if (!options["keepDir"])
-                fs.emptyDirSync(transformedAgendaSessionDir);
+            fs.emptyDirSync(transformedAgendaSessionDir);
         }
         const fifteenDaysFromNow = new Date();
         // Don't download agendas more than 15 days in the future.

package/lib/src/types/agenda.d.ts

@@ -0,0 +1,45 @@
+export interface AgendaEvent {
+    id: string;
+    type: string | null;
+    date: string;
+    startTime: string | null;
+    endTime: string | null;
+    timeOriginal: string | null;
+    titre: string;
+    organe: string | null;
+    objet: string | null;
+    lieu: string | null;
+    captationVideo: boolean;
+    urlDossierSenat: string | null;
+    quantieme: string | null;
+}
+export type TimeSlot = "MATIN" | "APRES-MIDI" | "SOIR" | "UNKNOWN";
+export interface Reunion {
+    uid: string;
+    chambre: "SN";
+    date: string;
+    startTime: string | null;
+    endTime: string | null;
+    captationVideo: boolean;
+    titre: string;
+    type: string;
+    organe?: string;
+    objet?: string;
+    lieu?: string;
+    events: AgendaEvent[];
+    compteRenduRefUid?: string;
+    transcriptionRef?: string;
+    urlVideo?: string;
+    urlPageVideo?: string;
+    timecodeDebutVideo?: number;
+    timecodeFinVideo?: number;
+    odj?: ReunionOdj;
+}
+export interface ReunionOdjPoint {
+    objet: string | null;
+    dossierLegislatifRef: string | null;
+    codeEtape: string | null;
+}
+export interface ReunionOdj {
+    pointsOdj: ReunionOdjPoint[];
+}

package/lib/src/types/agenda.js

@@ -0,0 +1 @@
+export {};

package/lib/src/types/ameli.d.ts

@@ -0,0 +1,5 @@
+import { TxtAmeli } from "../raw_types/ameli.js";
+import { Sub } from "../raw_types/ameli.js";
+export interface TxtAmeliCustom extends TxtAmeli {
+    subids?: Sub["id"][];
+}

package/lib/src/types/ameli.js

@@ -0,0 +1 @@
+export {};

package/lib/src/types/compte_rendu.d.ts

@@ -0,0 +1,83 @@
+export interface CompteRendu {
+    uid: string;
+    seanceRef: string;
+    sessionRef: string;
+    metadonnees: Metadonnees;
+    contenu: Contenu;
+}
+export interface Metadonnees {
+    dateSeance: string;
+    dateSeanceJour: string;
+    numSeanceJour: string;
+    numSeance: string;
+    typeAssemblee: "AN" | "SN";
+    legislature: string;
+    session: string;
+    nomFichierJo: string;
+    validite: string;
+    etat: string;
+    diffusion: string;
+    version: string;
+    environnement: string;
+    heureGeneration: Date;
+    sommaire?: Sommaire;
+}
+export interface Contenu {
+    quantiemes: Quantiemes;
+    ouvertureSeance?: Point[] | Point;
+    point: Point[] | Point;
+    finSeance?: FinSeance;
+    paragraphe?: Point[];
+}
+export interface FinSeance {
+    point: Point;
+}
+export interface Quantiemes {
+    journee: string;
+    session: string;
+}
+export interface Point {
+    ordre_absolu_seance: string;
+    code_grammaire: string;
+    roledebat?: string;
+    orateurs?: {
+        orateur: {
+            nom: string;
+            id: string;
+            qualite: string;
+        };
+    };
+    texte: {
+        _: string;
+    };
+    code_style?: string;
+}
+export interface Texte {
+    _?: string;
+    id_syceron?: string;
+    stime?: string;
+    sup?: string;
+    lienAdt?: Texte[] | Texte;
+}
+export interface Sommaire {
+    presidentSeance: Texte;
+    sommaire1: SommaireElement[] | SommaireElement;
+    sommaire3?: SommaireElement[] | SommaireElement;
+    sommaire2?: SommaireElement[] | SommaireElement;
+    para?: Texte[] | Texte;
+}
+export interface SommaireElement {
+    valeur_pts_odj: string | undefined;
+    titreStruct: TitreStruct;
+    para?: Array<Texte | string> | Texte;
+    sommaire2?: SommaireElement[] | SommaireElement;
+    sommaire3?: SommaireElement[] | SommaireElement;
+    presidentSeance?: Texte[] | Texte;
+    type_debat?: string;
+}
+export interface TitreStruct {
+    id_syceron: string;
+    intitule?: string;
+    sousIntitule?: string;
+    type_debat?: string;
+}

package/lib/src/types/compte_rendu.js

@@ -0,0 +1 @@
+export {};

package/lib/src/types/debats.d.ts

@@ -0,0 +1,2 @@
+import { debats as Debat, lecassdeb as LecAssDeb } from "../raw_types_schemats/debats";
+export type { Debat, LecAssDeb };

package/lib/src/types/debats.js

@@ -0,0 +1 @@
+export {};

package/lib/src/types/dosleg.d.ts

@@ -0,0 +1,70 @@
+import { Ass, Aud as AudRaw, Auteur as AuteurRaw, DateSeance as DateSeanceRaw, Deccoc, Denrap, Docatt as DocattRaw, Ecr as EcrRaw, Etaloi, Lecass as LecassRaw, Lecassrap as LecassrapRaw, Lecture as LectureRaw, Loi as LoiRaw, Org, Oritxt, Qua, Rap as RapRaw, Raporg, Scr, Texte as TexteRaw, Typatt, Typlec, Typloi, Typtxt, Typurl } from "../raw_types/dosleg.js";
+import { TxtAmeli } from "../raw_types/ameli.js";
+import { Debats } from "../raw_types/debats.js";
+import { TxtAmeliCustom } from "./ameli.js";
+export type { Deccoc as DecCoc, Denrap as DenRap, Etaloi as EtaLoi, Oritxt as OriTxt, Raporg as RapOrg, Typatt as TypAtt, Typlec as TypLec, Typloi as TypLoi, Typtxt as TypTxt, Typurl as TypUrl, };
+export interface Aud extends AudRaw {
+    org?: Org;
+}
+export interface Auteur extends AuteurRaw {
+    qua?: Qua;
+}
+export interface DateSeance extends DateSeanceRaw {
+    debat?: Debats;
+    scrids?: string[];
+    scrs?: Scr[];
+}
+export interface DocAtt extends DocattRaw {
+    rap?: RapRaw;
+    typatt?: Typatt;
+}
+export interface Ecr extends EcrRaw {
+    aut?: Auteur;
+}
+export interface LecAss extends LecassRaw {
+    ass?: Ass;
+    auds?: Aud[];
+    audcles?: AudRaw["audcle"][];
+    datesSeances?: DateSeance[];
+    datesSeancesCodes?: DateSeanceRaw["code"][];
+    debatdatseas: Date[];
+    lecassraps?: LecAssRap[];
+    lecassrapids?: string[];
+    org?: Org;
+    texcods: TexteRaw["texcod"][];
+    textes?: Texte[];
+}
+export interface LecAssRap extends LecassrapRaw {
+    rap?: RapRaw;
+}
+export interface Lecture extends LectureRaw {
+    typlec?: Typlec;
+    lecassidts?: LecassRaw["lecassidt"][];
+    lecasss?: LecAss[];
+}
+export interface Loi extends LoiRaw {
+    deccoc?: Deccoc;
+    etaloi?: Etaloi;
+    lecidts?: LectureRaw["lecidt"][];
+    lectures?: Lecture[];
+    typloi?: Typloi;
+}
+export interface Rap extends RapRaw {
+    denrap?: Denrap;
+    docattcles?: DocattRaw["docattcle"][];
+    docatts?: DocAtt[];
+    ecrnums?: EcrRaw["ecrnum"][];
+    ecrs?: Ecr[];
+    orgcods?: Raporg["orgcod"][];
+    orgs?: Org[];
+}
+export interface Texte extends TexteRaw {
+    ecrs?: Ecr[];
+    ecrnums?: EcrRaw["ecrnum"][];
+    libtypurl?: Typurl["libtypurl"];
+    org?: Org;
+    oritxt: Oritxt;
+    typtxt?: Typtxt;
+    txtAmeli?: TxtAmeliCustom;
+    txtAmeliId: TxtAmeli["id"];
+}

package/lib/src/types/dosleg.js

@@ -0,0 +1 @@
+export {};

package/lib/src/types/questions.d.ts

@@ -0,0 +1,2 @@
+import { TamQuestions } from "../raw_types/questions.js";
+export type { TamQuestions as Question };

package/lib/src/types/questions.js

@@ -0,0 +1 @@
+export {};

package/lib/src/types/sens.d.ts

@@ -0,0 +1,8 @@
+export interface Photo {
+    chemin: string;
+    cheminMosaique: string;
+    hauteur: number;
+    largeur: number;
+    xMosaique: number;
+    yMosaique: number;
+}

package/lib/src/types/sens.js

@@ -0,0 +1 @@
+export {};

package/lib/src/types/sessions.d.ts

@@ -0,0 +1,6 @@
+export declare const UNDEFINED_SESSION = 0;
+export declare const sessionOptions: readonly [1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026];
+export declare const sessionOptionsOrAll: readonly [0, 1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026];
+export type Session = (typeof sessionOptions)[number];
+export type SessionOrAll = (typeof sessionOptionsOrAll)[number];
+export declare function getSessionsFromStart(startSession: SessionOrAll): (1958 | 1959 | 1960 | 1961 | 1962 | 1963 | 1964 | 1965 | 1966 | 1967 | 1968 | 1969 | 1970 | 1971 | 1972 | 1973 | 1974 | 1975 | 1976 | 1977 | 1978 | 1979 | 1980 | 1981 | 1982 | 1983 | 1984 | 1985 | 1986 | 1987 | 1988 | 1989 | 1990 | 1991 | 1992 | 1993 | 1994 | 1995 | 1996 | 1997 | 1998 | 1999 | 2000 | 2001 | 2002 | 2003 | 2004 | 2005 | 2006 | 2007 | 2008 | 2009 | 2010 | 2011 | 2012 | 2013 | 2014 | 2015 | 2016 | 2017 | 2018 | 2019 | 2020 | 2021 | 2022 | 2023 | 2024 | 2025 | 2026)[];

package/lib/src/types/sessions.js

@@ -0,0 +1,19 @@
+export const UNDEFINED_SESSION = 0;
+export const sessionOptions = [
+    1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976,
+    1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995,
+    1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014,
+    2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026,
+    // TO COMPLETE EVERY YEAR :)
+];
+export const sessionOptionsOrAll = [UNDEFINED_SESSION, ...sessionOptions];
+export function getSessionsFromStart(startSession) {
+    if (startSession === UNDEFINED_SESSION) {
+        return Array.from(sessionOptions);
+    }
+    const sessionIndex = sessionOptions.findIndex((session) => startSession === session);
+    if (sessionIndex >= 0) {
+        return sessionOptions.slice(sessionIndex);
+    }
+    return [];
+}

package/lib/src/types/texte.d.ts

@@ -0,0 +1,72 @@
+import { DocumentResult } from "../loaders.js";
+export declare enum DivisionType {
+    tome = 1,
+    part = 2,
+    book = 3,
+    title = 4,
+    subtitle = 5,
+    chapter = 6,
+    section = 7,
+    subsection = 8,
+    paragraph = 9,
+    article = 10,
+    alinea = 11,
+    division = 12
+}
+export type DivisionTag = keyof typeof DivisionType;
+export interface DocumentMetadata {
+    name: string;
+    session: number | null | undefined;
+    date?: string | null;
+    url_expose_des_motifs?: URL;
+    url_xml?: URL;
+    url_html: URL;
+    url_pdf: URL;
+}
+export interface FlatTexte extends Partial<DocumentResult> {
+    titre: string | null;
+    titre_court: string | null;
+    url_dossier_senat: string | null;
+    url_dossier_assemblee: string | null;
+    date_presentation: Date | null;
+    date_depot: Date | null;
+    date_publication_xml: Date | null;
+    version: Version | null;
+    divisions: Division[];
+    expose_motifs?: ExposeDesMotifs | null;
+}
+export type Version = "RECT" | "RECT_BIS" | "RECT_TER" | "RECT_QUATER" | "RECT_QUINQUIES";
+export interface Step {
+    eId: string;
+    date: Date | null;
+    type: string | null;
+    session: string | null;
+    numero: string | null;
+    version: Version | null;
+    outcome: string | null;
+}
+export interface Division {
+    index: number;
+    eId: string;
+    tag: DivisionTag;
+    level: number;
+    headings: DivisionContent[];
+}
+export interface Article extends Division {
+    alineas: Alinea[];
+}
+export interface DivisionContent {
+    text: string | null;
+    html?: string | null;
+}
+export interface Alinea extends DivisionContent {
+    eId: string;
+    heading: DivisionContent;
+    pastille: string | null;
+    text_avec_liens?: string | null;
+    html_avec_liens?: string | null;
+}
+export interface ExposeDesMotifs {
+    text: string | null;
+    html: string | null;
+}

package/lib/src/types/texte.js

@@ -0,0 +1,15 @@
+export var DivisionType;
+(function (DivisionType) {
+    DivisionType[DivisionType["tome"] = 1] = "tome";
+    DivisionType[DivisionType["part"] = 2] = "part";
+    DivisionType[DivisionType["book"] = 3] = "book";
+    DivisionType[DivisionType["title"] = 4] = "title";
+    DivisionType[DivisionType["subtitle"] = 5] = "subtitle";
+    DivisionType[DivisionType["chapter"] = 6] = "chapter";
+    DivisionType[DivisionType["section"] = 7] = "section";
+    DivisionType[DivisionType["subsection"] = 8] = "subsection";
+    DivisionType[DivisionType["paragraph"] = 9] = "paragraph";
+    DivisionType[DivisionType["article"] = 10] = "article";
+    DivisionType[DivisionType["alinea"] = 11] = "alinea";
+    DivisionType[DivisionType["division"] = 12] = "division";
+})(DivisionType || (DivisionType = {}));

package/lib/src/validators/config.d.ts

@@ -0,0 +1,9 @@
+import { z } from "zod";
+export declare const configSchema: z.ZodObject<{
+    db: z.ZodObject<{
+        host: z.ZodString;
+        password: z.ZodString;
+        user: z.ZodString;
+        port: z.ZodCoercedNumber<unknown>;
+    }, z.core.$strip>;
+}, z.core.$strip>;

package/lib/src/validators/config.js

@@ -0,0 +1,10 @@
+import { z } from "zod";
+const dbSchema = z.object({
+    host: z.string().trim().min(1, "Must not be empty"),
+    password: z.string().trim().min(1, "Must not be empty"),
+    user: z.string().trim().min(1, "Must not be empty"),
+    port: z.coerce.number().int().min(0).max(65535),
+});
+export const configSchema = z.object({
+    db: dbSchema,
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "3.1.3",
+  "version": "3.1.5",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",