@tricoteuses/senat 3.1.21 → 3.1.22

package/lib/src/raw_types_schemats/sens.js

@@ -0,0 +1,2 @@
+/* tslint:disable */
+export {};

package/lib/src/rich_types/agenda.d.ts

@@ -0,0 +1,45 @@
+export interface AgendaEvent {
+    id: string;
+    type: string | null;
+    date: string;
+    startTime: string | null;
+    endTime: string | null;
+    timeOriginal: string | null;
+    titre: string;
+    organe: string | null;
+    objet: string | null;
+    lieu: string | null;
+    captationVideo: boolean;
+    urlDossierSenat: string | null;
+    quantieme: string | null;
+}
+export type TimeSlot = "MATIN" | "APRES-MIDI" | "SOIR" | "UNKNOWN";
+export interface Reunion {
+    uid: string;
+    chambre: "SN";
+    date: string;
+    startTime: string | null;
+    endTime: string | null;
+    captationVideo: boolean;
+    titre: string;
+    type: string;
+    organe?: string;
+    objet?: string;
+    lieu?: string;
+    events: AgendaEvent[];
+    compteRenduRefUid?: string;
+    transcriptionRef?: string;
+    urlVideo?: string;
+    urlPageVideo?: string;
+    timecodeDebutVideo?: number;
+    timecodeFinVideo?: number;
+    odj?: ReunionOdj;
+}
+export interface ReunionOdjPoint {
+    objet: string | null;
+    dossierLegislatifRef: string | null;
+    codeEtape: string | null;
+}
+export interface ReunionOdj {
+    pointsOdj: ReunionOdjPoint[];
+}

package/lib/src/rich_types/agenda.js

@@ -0,0 +1 @@
+export {};

package/lib/src/rich_types/compte_rendu.d.ts

@@ -0,0 +1,83 @@
+export interface CompteRendu {
+    uid: string;
+    seanceRef: string;
+    sessionRef: string;
+    metadonnees: Metadonnees;
+    contenu: Contenu;
+}
+export interface Metadonnees {
+    dateSeance: string;
+    dateSeanceJour: string;
+    numSeanceJour: string;
+    numSeance: string;
+    typeAssemblee: "AN" | "SN";
+    legislature: string;
+    session: string;
+    nomFichierJo: string;
+    validite: string;
+    etat: string;
+    diffusion: string;
+    version: string;
+    environnement: string;
+    heureGeneration: Date;
+    sommaire?: Sommaire;
+}
+export interface Contenu {
+    quantiemes: Quantiemes;
+    ouvertureSeance?: Point[] | Point;
+    point: Point[] | Point;
+    finSeance?: FinSeance;
+    paragraphe?: Point[];
+}
+export interface FinSeance {
+    point: Point;
+}
+export interface Quantiemes {
+    journee: string;
+    session: string;
+}
+export interface Point {
+    ordre_absolu_seance: string;
+    code_grammaire: string;
+    roledebat?: string;
+    orateurs?: {
+        orateur: {
+            nom: string;
+            id: string;
+            qualite: string;
+        };
+    };
+    texte: {
+        _: string;
+    };
+    code_style?: string;
+}
+export interface Texte {
+    _?: string;
+    id_syceron?: string;
+    stime?: string;
+    sup?: string;
+    lienAdt?: Texte[] | Texte;
+}
+export interface Sommaire {
+    presidentSeance: Texte;
+    sommaire1: SommaireElement[] | SommaireElement;
+    sommaire3?: SommaireElement[] | SommaireElement;
+    sommaire2?: SommaireElement[] | SommaireElement;
+    para?: Texte[] | Texte;
+}
+export interface SommaireElement {
+    valeur_pts_odj: string | undefined;
+    titreStruct: TitreStruct;
+    para?: Array<Texte | string> | Texte;
+    sommaire2?: SommaireElement[] | SommaireElement;
+    sommaire3?: SommaireElement[] | SommaireElement;
+    presidentSeance?: Texte[] | Texte;
+    type_debat?: string;
+}
+export interface TitreStruct {
+    id_syceron: string;
+    intitule?: string;
+    sousIntitule?: string;
+    type_debat?: string;
+}

package/lib/src/rich_types/compte_rendu.js

@@ -0,0 +1 @@
+export {};

package/lib/src/rich_types/dosleg.js

@@ -39,13 +39,6 @@ function getDateSortValue(value) {
 function compareByDate(left, right) {
     return getDateSortValue(left.date) - getDateSortValue(right.date);
 }
-function getCommissionCodeActe(phasePrefix, rapportType) {
-    const normalizedType = (rapportType || "").toLowerCase();
-    if (normalizedType.includes("avis") || normalizedType.includes("information")) {
-        return `${phasePrefix}-COM-AVIS`;
-    }
-    return `${phasePrefix}-COM-FOND`;
-}
 function getPhasePrefix(lecture, assemblee, codeNatureDossier) {
     if (assemblee !== "Sénat")
         return null;

package/lib/src/rich_types/sessions.d.ts

@@ -0,0 +1,6 @@
+export declare const UNDEFINED_SESSION = 0;
+export declare const sessionOptions: readonly [1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026];
+export declare const sessionOptionsOrAll: readonly [0, 1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026];
+export type Session = (typeof sessionOptions)[number];
+export type SessionOrAll = (typeof sessionOptionsOrAll)[number];
+export declare function getSessionsFromStart(startSession: SessionOrAll): (1958 | 1959 | 1960 | 1961 | 1962 | 1963 | 1964 | 1965 | 1966 | 1967 | 1968 | 1969 | 1970 | 1971 | 1972 | 1973 | 1974 | 1975 | 1976 | 1977 | 1978 | 1979 | 1980 | 1981 | 1982 | 1983 | 1984 | 1985 | 1986 | 1987 | 1988 | 1989 | 1990 | 1991 | 1992 | 1993 | 1994 | 1995 | 1996 | 1997 | 1998 | 1999 | 2000 | 2001 | 2002 | 2003 | 2004 | 2005 | 2006 | 2007 | 2008 | 2009 | 2010 | 2011 | 2012 | 2013 | 2014 | 2015 | 2016 | 2017 | 2018 | 2019 | 2020 | 2021 | 2022 | 2023 | 2024 | 2025 | 2026)[];

package/lib/src/rich_types/sessions.js

@@ -0,0 +1,19 @@
+export const UNDEFINED_SESSION = 0;
+export const sessionOptions = [
+    1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976,
+    1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995,
+    1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014,
+    2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026,
+    // TO COMPLETE EVERY YEAR :)
+];
+export const sessionOptionsOrAll = [UNDEFINED_SESSION, ...sessionOptions];
+export function getSessionsFromStart(startSession) {
+    if (startSession === UNDEFINED_SESSION) {
+        return Array.from(sessionOptions);
+    }
+    const sessionIndex = sessionOptions.findIndex((session) => startSession === session);
+    if (sessionIndex >= 0) {
+        return sessionOptions.slice(sessionIndex);
+    }
+    return [];
+}

package/lib/src/rich_types/texte.d.ts

@@ -0,0 +1,72 @@
+import { DocumentResult } from "../documents.js";
+export declare enum DivisionType {
+    tome = 1,
+    part = 2,
+    book = 3,
+    title = 4,
+    subtitle = 5,
+    chapter = 6,
+    section = 7,
+    subsection = 8,
+    paragraph = 9,
+    article = 10,
+    alinea = 11,
+    division = 12
+}
+export type DivisionTag = keyof typeof DivisionType;
+export interface DocumentMetadata {
+    name: string;
+    session: number | null | undefined;
+    date?: string | null;
+    url_expose_des_motifs?: URL;
+    url_xml?: URL;
+    url_html: URL;
+    url_pdf: URL;
+}
+export interface FlatTexte extends Partial<DocumentResult> {
+    titre: string | null;
+    titre_court: string | null;
+    url_dossier_senat: string | null;
+    url_dossier_assemblee: string | null;
+    date_presentation: Date | null;
+    date_depot: Date | null;
+    date_publication_xml: Date | null;
+    version: Version | null;
+    divisions: Division[];
+    expose_motifs?: ExposeDesMotifs | null;
+}
+export type Version = "RECT" | "RECT_BIS" | "RECT_TER" | "RECT_QUATER" | "RECT_QUINQUIES";
+export interface Step {
+    eId: string;
+    date: Date | null;
+    type: string | null;
+    session: string | null;
+    numero: string | null;
+    version: Version | null;
+    outcome: string | null;
+}
+export interface Division {
+    index: number;
+    eId: string;
+    tag: DivisionTag;
+    level: number;
+    headings: DivisionContent[];
+}
+export interface Article extends Division {
+    alineas: Alinea[];
+}
+export interface DivisionContent {
+    text: string | null;
+    html?: string | null;
+}
+export interface Alinea extends DivisionContent {
+    eId: string;
+    heading: DivisionContent;
+    pastille: string | null;
+    text_avec_liens?: string | null;
+    html_avec_liens?: string | null;
+}
+export interface ExposeDesMotifs {
+    text: string | null;
+    html: string | null;
+}

package/lib/src/rich_types/texte.js

@@ -0,0 +1,15 @@
+export var DivisionType;
+(function (DivisionType) {
+    DivisionType[DivisionType["tome"] = 1] = "tome";
+    DivisionType[DivisionType["part"] = 2] = "part";
+    DivisionType[DivisionType["book"] = 3] = "book";
+    DivisionType[DivisionType["title"] = 4] = "title";
+    DivisionType[DivisionType["subtitle"] = 5] = "subtitle";
+    DivisionType[DivisionType["chapter"] = 6] = "chapter";
+    DivisionType[DivisionType["section"] = 7] = "section";
+    DivisionType[DivisionType["subsection"] = 8] = "subsection";
+    DivisionType[DivisionType["paragraph"] = 9] = "paragraph";
+    DivisionType[DivisionType["article"] = 10] = "article";
+    DivisionType[DivisionType["alinea"] = 11] = "alinea";
+    DivisionType[DivisionType["division"] = 12] = "division";
+})(DivisionType || (DivisionType = {}));

package/lib/src/scripts/convert_data.js

@@ -154,12 +154,11 @@ async function convertDatasetDosLeg(dataDir, options) {
         if (options["verbose"]) {
             console.log(`Converting ${dossier["signet"]} file…`);
         }
-        let dossierReorganizedDir = path.join(dossiersReorganizedDir, String(UNDEFINED_SESSION));
         const session = getSessionFromSignet(dossier["signet"]) || UNDEFINED_SESSION;
         if (options["fromSession"] && session < options["fromSession"]) {
             continue;
         }
-        dossierReorganizedDir = path.join(dossiersReorganizedDir, String(session));
+        const dossierReorganizedDir = path.join(dossiersReorganizedDir, String(session));
         const actesBrutsNormalises = buildActesLegislatifs(dossier);
         const dossierWithActes = {
             ...dossier,
@@ -184,9 +183,8 @@ async function convertDatasetScrutins(dataDir, options) {
         if (options["verbose"]) {
             console.log(`Converting ${scrutin["numero"]} file…`);
         }
-        let scrutinReorganizedDir = path.join(scrutinsReorganizedDir, String(UNDEFINED_SESSION));
         const session = scrutin["session"] || UNDEFINED_SESSION;
-        scrutinReorganizedDir = path.join(scrutinsReorganizedDir, String(session));
+        const scrutinReorganizedDir = path.join(scrutinsReorganizedDir, String(session));
         const scrutinFileName = `${scrutin["numero"]}.json`;
         await fs.outputJSON(path.join(scrutinReorganizedDir, scrutinFileName), scrutin, {
             spaces: 2,
@@ -339,7 +337,8 @@ async function convertDatasetSens(dataDir, options) {
     for await (const organisme of findAllOrganismes()) {
         const sanitizedCode = organisme.code.trim().replace(/\s+/g, "-");
         if (sanitizedCode !== organisme.code) {
-            console.warn(`Warning: code "${organisme.code}" contains leading/trailing or multiple consecutive whitespace characters. It has been sanitized to "${sanitizedCode}". Please check the original code for potential issues.`);
+            console.warn(`Warning: code "${organisme.code}" contains leading/trailing or multiple consecutive whitespace characters. ` +
+                `It has been sanitized to "${sanitizedCode}". Please check the original code for potential issues.`);
         }
         if (options["verbose"]) {
             console.log(`Converting ${sanitizedCode} file…`);

package/lib/src/scripts/retrieve_cr_commission.js

@@ -263,7 +263,7 @@ async function retrieveCommissionCRs(options = {}) {
             for (const f of htmlFiles) {
                 const htmlPath = path.join(commissionDir, f);
                 let meta;
-                let raw = "";
+                let raw;
                 try {
                     raw = await fs.readFile(htmlPath, "utf8");
                     meta = parseCommissionMetadataFromHtml(raw, f);

package/lib/src/scripts/retrieve_open_data.js

@@ -185,10 +185,49 @@ function runPsqlQuery(command, dataDir, options, connection, stopOnError = true)
         stdio: ["ignore", "pipe", "pipe"],
     });
 }
+function sourceSchemaNames(dataset) {
+    if (dataset.database === "sens") {
+        return ["sens", "senat"];
+    }
+    return [dataset.database];
+}
+function inspectSourceDump(dataset, dataDir) {
+    const sqlFilePath = path.join(dataDir, `${dataset.database}.sql`);
+    if (!fs.existsSync(sqlFilePath)) {
+        return {
+            copyStatements: 0,
+            createTableStatements: 0,
+            fileSizeBytes: null,
+            schemaQualifiedTableStatements: 0,
+            sqlFilePath,
+        };
+    }
+    const sqlContent = fs.readFileSync(sqlFilePath, { encoding: "utf8" });
+    const schemaPattern = sourceSchemaNames(dataset)
+        .map((schemaName) => schemaName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))
+        .join("|");
+    return {
+        copyStatements: (sqlContent.match(/^COPY\s+/gim) ?? []).length,
+        createTableStatements: (sqlContent.match(/^CREATE\s+TABLE\s+/gim) ?? []).length,
+        fileSizeBytes: fs.statSync(sqlFilePath).size,
+        schemaQualifiedTableStatements: (sqlContent.match(new RegExp(`\\b(?:${schemaPattern})\\.`, "g")) ?? []).length,
+        sqlFilePath,
+    };
+}
 function ensureStagingSchemaHasTables(dataset, dataDir, options, connection) {
     const stagingSchema = stagingSchemaName(dataset.database);
     const tableCount = Number.parseInt(runPsqlQuery(`SELECT count(*) FROM pg_tables WHERE schemaname = '${escapeSqlLiteral(stagingSchema)}'`, dataDir, options, connection).trim(), 10);
+    if (tableCount === 0) {
+        const dumpInspection = inspectSourceDump(dataset, dataDir);
+        const fileSizeLabel = dumpInspection.fileSizeBytes === null ? "missing" : `${dumpInspection.fileSizeBytes} bytes`;
+        console.error(`Source dump diagnostics for ${dataset.database}: file=${dumpInspection.sqlFilePath}, ` +
+            `size=${fileSizeLabel}, create_table=${dumpInspection.createTableStatements}, ` +
+            `copy=${dumpInspection.copyStatements}, ` +
+            `schema_refs=${dumpInspection.schemaQualifiedTableStatements}`);
+    }
     assert(tableCount > 0, `Staging schema ${stagingSchema} is empty after importing ${dataset.database}. ` +
+        "Source dump may be empty or malformed. " +
+        `Check /home/gitlab-runner/senat-data/${dataset.database}.sql and the upstream archive before retrying. ` +
         `Aborting incremental merge to protect ${senatSchemaName}.`);
 }
 function ensureStagingDatabase(dataDir, options, runtime) {

package/lib/src/scripts/retrieve_videos.js

@@ -106,7 +106,6 @@ async function processGroupedReunion(agenda, session, dataDir, lastByVideo) {
         return;
     const ctx = await computeContext(agenda, session, dataDir);
     const skipDownload = shouldSkipDownload(agenda, ctx.baseDir);
-    let match = null;
     let best = null;
     let secondBest = null;
     // 2) Match + download artifacts (only if not skipped)
@@ -117,7 +116,7 @@ async function processGroupedReunion(agenda, session, dataDir, lastByVideo) {
             console.log(`[warn] ${agenda.uid} No candidate found for this reunion. Probably VOD not published yet.`);
             return;
         }
-        match = await matchAgendaToVideo({ agenda, agendaTs: ctx.agendaTs, candidates, options });
+        const match = await matchAgendaToVideo({ agenda, agendaTs: ctx.agendaTs, candidates, options });
         if (!match) {
             console.log(`[miss] ${agenda.uid} No match found for this reunion`);
             return;