@tricoteuses/senat 2.19.6 → 2.19.8

package/lib/scripts/convert_data.js

@@ -49,23 +49,19 @@ async function convertDatasetAmeli(dataDir, options) {
     }
     const ameliReorganizedRootDir = path.join(dataDir, dataset.database);
     await fs.ensureDir(ameliReorganizedRootDir);
-    const limit = pLimit(10);
-    const tasks = [];
     for await (const amendement of findAllAmendements(options["fromSession"])) {
-        tasks.push(limit(async () => {
-            if (options["verbose"]) {
-                console.log(`Converting ${amendement["numero"]} file…`);
-            }
-            const session = String(amendement["session"]) || UNDEFINED_SESSION;
-            const signetDossierLegislatif = amendement["signet_dossier_legislatif"] ||
-                `${amendement["nature_texte"]}-${amendement["numero_texte"]}`.toLowerCase();
-            const ameliReorganizedDir = path.join(ameliReorganizedRootDir, String(session), signetDossierLegislatif);
-            await fs.ensureDir(ameliReorganizedDir);
-            const amendementFileName = `${amendement["numero"]}.json`;
-            await fs.writeJSON(path.join(ameliReorganizedDir, amendementFileName), amendement, { spaces: 2 });
-        }));
+        if (options["verbose"]) {
+            console.log(`Converting ${amendement["numero"]} file…`);
+        }
+        const session = String(amendement["session"]) || UNDEFINED_SESSION;
+        const signetDossierLegislatif = amendement["signet_dossier_legislatif"] ||
+            `${amendement["nature_texte"]}-${amendement["numero_texte"]}`.toLowerCase();
+        const ameliReorganizedDir = path.join(ameliReorganizedRootDir, String(session), signetDossierLegislatif);
+        await fs.ensureDir(ameliReorganizedDir);
+        const amendementFileName = `${amendement["numero"]}.json`;
+        const filePath = path.join(ameliReorganizedDir, amendementFileName);
+        await fs.writeJSON(filePath, amendement, { spaces: 2 });
     }
-    await Promise.all(tasks);
 }
 async function convertDatasetDebats(dataDir, options) {
     const dataset = datasets.debats;
@@ -83,9 +79,10 @@ async function convertDatasetDebats(dataDir, options) {
             continue;
         }
         const debatsReorganizedDir = path.join(debatsReorganizedRootDir, String(session));
-        fs.ensureDirSync(debatsReorganizedDir);
+        await fs.ensureDir(debatsReorganizedDir);
         const debatFileName = `${debat.id}.json`;
-        fs.writeJSONSync(path.join(debatsReorganizedDir, debatFileName), debat, { spaces: 2 });
+        const filePath = path.join(debatsReorganizedDir, debatFileName);
+        await fs.writeJSON(filePath, debat, { spaces: 2 });
     }
 }
 async function convertDatasetDosLeg(dataDir, options) {
@@ -107,14 +104,13 @@ async function convertDatasetDosLeg(dataDir, options) {
             continue;
         }
         loiReorganizedDir = path.join(dossiersReorganizedDir, String(session));
-        fs.ensureDirSync(loiReorganizedDir);
+        await fs.ensureDir(loiReorganizedDir);
         // Ajout des actes législatifs au dossier
         const actesLegislatifs = createActesLegislatifs(loi);
         const loiWithActes = { ...loi, actes_legislatifs: actesLegislatifs };
-        const scrutinFileName = `${loi["signet"]}.json`;
-        fs.writeJSONSync(path.join(loiReorganizedDir, scrutinFileName), loiWithActes, {
-            spaces: 2,
-        });
+        const dossierFile = `${loi["signet"]}.json`;
+        const filePath = path.join(loiReorganizedDir, dossierFile);
+        await fs.writeJSON(filePath, loiWithActes, { spaces: 2 });
     }
     await convertTexteUrls(dataDir);
     await convertRapportUrls(dataDir);

package/lib/scripts/datautil.js

@@ -28,7 +28,7 @@ export function getSessionFromDate(date, format = STANDARD_DATE_FORMAT) {
     const parsedDate = DateTime.fromFormat(date, format);
     const endSessionDate = DateTime.fromObject({ year: parsedDate.year, month: 9, day: 30 });
     if (parsedDate < endSessionDate) {
-        return parsedDate.year - 1;
+        return (parsedDate.year - 1);
     }
     return parsedDate.year;
 }

package/lib/scripts/parse_textes.js

@@ -2,8 +2,8 @@ import assert from "assert";
 import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 import path from "path";
-import { iterFilePaths, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, } from "../loaders";
-import { parseExposeDesMotifsFromFile, parseTexteFromFile, } from "../model/texte";
+import { iterFilePaths, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
+import { parseExposeDesMotifsFromFile, parseTexteFromFile } from "../model/texte";
 import { commonOptions } from "./shared/cli_helpers";
 import { ensureAndClearDir } from "./shared/util";
 const optionsDefinitions = [...commonOptions];

package/lib/scripts/retrieve_agenda.js

@@ -128,9 +128,7 @@ async function main() {
     const sessions = getSessionsFromStart(options["fromSession"]);
     console.time("agenda processing time");
     await retrieveAgendas(dataDir, sessions);
-    if (!options["silent"]) {
-        console.timeEnd("agenda processing time");
-    }
+    console.timeEnd("agenda processing time");
 }
 main()
     .then(() => process.exit(0))

package/lib/scripts/retrieve_cr_commission.js

@@ -356,9 +356,7 @@ async function main() {
     assert(dataDir, "Missing argument: data directory");
     console.time("CRI processing time");
     await retrieveCommissionCRs(options);
-    if (!options["silent"]) {
-        console.timeEnd("CRI processing time");
-    }
+    console.timeEnd("CRI processing time");
 }
 main()
     .then(() => process.exit(0))

package/lib/scripts/retrieve_cr_seance.js

@@ -5,7 +5,7 @@
  */
 import assert from "assert";
 import commandLineArgs from "command-line-args";
-import fs from "fs-extra";
+import fs, { ensureDirSync } from "fs-extra";
 import path from "path";
 import StreamZip from "node-stream-zip";
 import * as cheerio from "cheerio";
@@ -14,7 +14,7 @@ import { commonOptions } from "./shared/cli_helpers";
 import { deriveTitreObjetFromSommaire, parseCompteRenduSlotFromFile, parseYYYYMMDD, sessionStartYearFromDate, } from "../model/seance";
 import { makeGroupUid } from "../utils/reunion_grouping";
 import { getSessionsFromStart } from "../types/sessions";
-import { ensureAndClearDir, fetchWithRetry } from "./shared/util";
+import { fetchWithRetry } from "./shared/util";
 import { computeIntervalsBySlot } from "../utils/cr_spliting";
 const optionsDefinitions = [
     ...commonOptions,
@@ -115,7 +115,7 @@ async function extractAndDistributeXmlBySession(zipPath, originalRoot) {
 }
 export async function retrieveCriXmlDump(dataDir, options = {}) {
     const root = path.join(dataDir, COMPTES_RENDUS_FOLDER);
-    ensureAndClearDir(root);
+    ensureDirSync(root);
     const originalRoot = path.join(root, DATA_ORIGINAL_FOLDER);
     fs.ensureDirSync(originalRoot);
     const transformedRoot = path.join(root, DATA_TRANSFORMED_FOLDER);
@@ -153,11 +153,22 @@ export async function retrieveCriXmlDump(dataDir, options = {}) {
         }
         const xmlFiles = (await fs.readdir(originalSessionDir)).filter((f) => /^d\d{8}\.xml$/i.test(f)).sort();
         const transformedSessionDir = path.join(transformedRoot, String(session));
-        if (options["parseDebats"])
-            await fs.ensureDir(transformedSessionDir);
+        await fs.ensureDir(transformedSessionDir);
+        const now = Date.now();
         for (const f of xmlFiles) {
             const yyyymmdd = f.slice(1, 9);
             const xmlPath = path.join(originalSessionDir, f);
+            if (options["only-recent"]) {
+                const cutoff = now - options["only-recent"] * 24 * 3600 * 1000;
+                const seanceTs = Date.parse(yyyymmdd.slice(0, 4) + "-" + yyyymmdd.slice(4, 6) + "-" + yyyymmdd.slice(6, 8));
+                if (seanceTs < cutoff) {
+                    // Check if some file exists sarting with CRSSN{yyyymmdd} in transformed dir
+                    const someFile = (await fs.readdir(transformedSessionDir)).find((fn) => fn.startsWith(`CRSSN${yyyymmdd}`));
+                    if (someFile) {
+                        continue;
+                    }
+                }
+            }
             // 1) Deduce slot(s) from agenda if it exsits
             const agendaInfo = loadAgendaSPSlotsForDate(dataDir, yyyymmdd, session);
             const firstSlotOfDay = pickFirstSlotOfDay(agendaInfo?.slots ?? []);
@@ -209,9 +220,7 @@ async function main() {
     assert(dataDir, "Missing argument: data directory");
     console.time("CRI processing time");
     await retrieveCriXmlDump(dataDir, options);
-    if (!options["silent"]) {
-        console.timeEnd("CRI processing time");
-    }
+    console.timeEnd("CRI processing time");
 }
 main()
     .then(() => process.exit(0))

package/lib/scripts/retrieve_open_data.js

@@ -5,7 +5,7 @@ import fs from "fs-extra";
 import path from "path";
 import StreamZip from "node-stream-zip";
 import readline from "readline";
-import windows1252 from "windows-1252";
+import * as windows1252 from "windows-1252";
 import { pipeline } from "stream";
 import { promisify } from "util";
 import config from "../config";
@@ -102,7 +102,7 @@ async function copyToSenat(dataset, dataDir, options) {
                     parts[i] = parts[i].replace(/\bpublic\b(?=(\s*\.|\s*[,;]|\s|$))/g, schema);
                 }
             }
-            return parts.join('');
+            return parts.join("");
         }
         newLine = replacePublicOutsideStrings(line, dataset.database);
         // Replace SET client_encoding to UTF8
@@ -120,7 +120,8 @@ async function copyToSenat(dataset, dataDir, options) {
                 });
             }
             finally {
-                try { }
+                try {
+                }
                 catch { }
             }
             resolve();
@@ -268,10 +269,7 @@ async function retrieveOpenData() {
         PGUSER: process.env["PGUSER"] || config.db.user,
         PGPASSWORD: process.env["PGPASSWORD"] || config.db.password,
     };
-    assert(process.env["PGHOST"] &&
-        process.env["PGPORT"] &&
-        process.env["PGUSER"] &&
-        process.env["PGPASSWORD"], "Missing database configuration: environment variables PGHOST, PGPORT, PGUSER and PGPASSWORD or TRICOTEUSES_SENAT_DB_* in .env file");
+    assert(process.env["PGHOST"] && process.env["PGPORT"] && process.env["PGUSER"] && process.env["PGPASSWORD"], "Missing database configuration: environment variables PGHOST, PGPORT, PGUSER and PGPASSWORD or TRICOTEUSES_SENAT_DB_* in .env file");
     console.time("data extraction time");
     // Create role 'opendata' if it does not exist
     execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -c "CREATE ROLE opendata" || true`, {

package/lib/scripts/retrieve_senateurs_photos.js

@@ -71,10 +71,7 @@ async function retrievePhotosSenateurs() {
                 fs.renameSync(photoTempFilePath, photoFilePath);
             }
             catch (error) {
-                if (typeof error === "object" &&
-                    error &&
-                    "status" in error &&
-                    error.status === 8) {
+                if (typeof error === "object" && error && "status" in error && error.status === 8) {
                     console.error(`Unable to load photo for ${sen.senprenomuse} ${sen.sennomuse}`);
                     continue;
                 }

package/lib/scripts/test_iter_load.js

@@ -1,4 +1,4 @@
-import { iterLoadSenatScrutins } from "../loaders";
+import { iterLoadSenatScrutins, iterLoadSenatAmendements } from "../loaders";
 import commandLineArgs from "command-line-args";
 import { dataDirDefaultOption } from "./shared/cli_helpers";
 const optionsDefinitions = [dataDirDefaultOption];
@@ -8,3 +8,6 @@ const session = 2024;
 for (const { item: scrutin } of iterLoadSenatScrutins(options["dataDir"], session, { noValidation: noValidation })) {
     console.log(scrutin["numero"]);
 }
+for (const { item: amendement } of iterLoadSenatAmendements(options["dataDir"], session, { noValidation: noValidation })) {
+    console.log(amendement["numero"]);
+}

package/lib/types/dosleg.d.ts

@@ -2,7 +2,7 @@ import { txt_ameliFields } from "../raw_types_schemats/ameli";
 import { ass as Ass, aud, audFields, auteur, date_seance, date_seanceFields as dateSeanceFields, deccoc as DecCoc, denrap as DenRap, docatt, docattFields, ecr, ecrFields, etaloi as EtaLoi, lecass, lecassFields, lecassrap, lecture, lectureFields, loi, org as Org, oritxt as OriTxt, qua as Qua, rap, raporg as RapOrg, raporgFields, scr as Scr, texte, texteFields, typatt as TypAtt, typlec as TypLec, typloi as TypLoi, typtxt as TypTxt, typurl as TypUrl, typurlFields } from "../raw_types_schemats/dosleg";
 import { Debat } from "./debats";
 import { TxtAmeli } from "./ameli";
-export type { Ass, DecCoc, DenRap, EtaLoi, Org, OriTxt, Qua, RapOrg, Scr, TypAtt, TypLec, TypLoi, TypTxt, TypUrl, };
+export type { Ass, DecCoc, DenRap, EtaLoi, Org, OriTxt, Qua, RapOrg, Scr, TypAtt, TypLec, TypLoi, TypTxt, TypUrl };
 export interface Aud extends aud {
     org?: Org;
 }

package/lib/types/sessions.d.ts

@@ -1,5 +1,5 @@
 export declare const UNDEFINED_SESSION = 0;
 declare const sessions: readonly [0, 1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026];
-export type Session = typeof sessions[number];
+export type Session = (typeof sessions)[number];
 export declare function getSessionsFromStart(startSession: Session): (0 | 1958 | 1959 | 1960 | 1961 | 1962 | 1963 | 1964 | 1965 | 1966 | 1967 | 1968 | 1969 | 1970 | 1971 | 1972 | 1973 | 1974 | 1975 | 1976 | 1977 | 1978 | 1979 | 1980 | 1981 | 1982 | 1983 | 1984 | 1985 | 1986 | 1987 | 1988 | 1989 | 1990 | 1991 | 1992 | 1993 | 1994 | 1995 | 1996 | 1997 | 1998 | 1999 | 2000 | 2001 | 2002 | 2003 | 2004 | 2005 | 2006 | 2007 | 2008 | 2009 | 2010 | 2011 | 2012 | 2013 | 2014 | 2015 | 2016 | 2017 | 2018 | 2019 | 2020 | 2021 | 2022 | 2023 | 2024 | 2025 | 2026)[];
 export {};

package/lib/types/sessions.js

@@ -76,7 +76,7 @@ export function getSessionsFromStart(startSession) {
     if (startSession === 0) {
         return Array.from(sessions);
     }
-    const sessionIndex = sessions.findIndex(session => startSession === session);
+    const sessionIndex = sessions.findIndex((session) => startSession === session);
     if (sessionIndex >= 0) {
         return sessions.slice(sessionIndex);
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.19.6",
+  "version": "2.19.8",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",
@@ -63,28 +63,28 @@
   "dependencies": {
     "@biryani/core": "^0.2.1",
     "cheerio": "^1.1.2",
-    "command-line-args": "^5.1.1",
-    "dotenv": "^8.2.0",
-    "fs-extra": "^9.1.0",
-    "jsdom": "^26.0.0",
-    "kysely": "^0.27.4",
+    "command-line-args": "^6.0.1",
+    "dotenv": "^17.2.3",
+    "fs-extra": "^11.3.2",
+    "jsdom": "^27.2.0",
+    "kysely": "^0.28.8",
     "luxon": "^3.7.2",
     "node-stream-zip": "^1.8.2",
+    "p-limit": "^7.2.0",
     "pg": "^8.13.1",
     "pg-cursor": "^2.12.1",
-    "p-limit": "^7.2.0",
     "slug": "^11.0.0",
     "tsx": "^4.20.6",
-    "windows-1252": "^1.0.0"
+    "windows-1252": "^3.0.4"
   },
   "devDependencies": {
     "@typed-code/schemats": "^5.0.1",
     "@types/cheerio": "^1.0.0",
     "@types/command-line-args": "^5.0.0",
-    "@types/fs-extra": "^9.0.7",
-    "@types/jsdom": "^21.1.7",
+    "@types/fs-extra": "^11.0.4",
+    "@types/jsdom": "^27.0.0",
     "@types/luxon": "^3.7.1",
-    "@types/node": "^20.17.6",
+    "@types/node": "^24.10.1",
     "@types/pg": "^8.15.5",
     "@types/pg-cursor": "^2.7.2",
     "@types/slug": "^5.0.9",