@tricoteuses/senat 2.19.7 → 2.19.8

package/lib/scripts/retrieve_agenda.js

@@ -128,9 +128,7 @@ async function main() {
     const sessions = getSessionsFromStart(options["fromSession"]);
     console.time("agenda processing time");
     await retrieveAgendas(dataDir, sessions);
-    if (!options["silent"]) {
-        console.timeEnd("agenda processing time");
-    }
+    console.timeEnd("agenda processing time");
 }
 main()
     .then(() => process.exit(0))

package/lib/scripts/retrieve_cr_commission.js

@@ -356,9 +356,7 @@ async function main() {
     assert(dataDir, "Missing argument: data directory");
     console.time("CRI processing time");
     await retrieveCommissionCRs(options);
-    if (!options["silent"]) {
-        console.timeEnd("CRI processing time");
-    }
+    console.timeEnd("CRI processing time");
 }
 main()
     .then(() => process.exit(0))

package/lib/scripts/retrieve_cr_seance.js

@@ -5,7 +5,7 @@
  */
 import assert from "assert";
 import commandLineArgs from "command-line-args";
-import fs from "fs-extra";
+import fs, { ensureDirSync } from "fs-extra";
 import path from "path";
 import StreamZip from "node-stream-zip";
 import * as cheerio from "cheerio";
@@ -14,7 +14,7 @@ import { commonOptions } from "./shared/cli_helpers";
 import { deriveTitreObjetFromSommaire, parseCompteRenduSlotFromFile, parseYYYYMMDD, sessionStartYearFromDate, } from "../model/seance";
 import { makeGroupUid } from "../utils/reunion_grouping";
 import { getSessionsFromStart } from "../types/sessions";
-import { ensureAndClearDir, fetchWithRetry } from "./shared/util";
+import { fetchWithRetry } from "./shared/util";
 import { computeIntervalsBySlot } from "../utils/cr_spliting";
 const optionsDefinitions = [
     ...commonOptions,
@@ -115,7 +115,7 @@ async function extractAndDistributeXmlBySession(zipPath, originalRoot) {
 }
 export async function retrieveCriXmlDump(dataDir, options = {}) {
     const root = path.join(dataDir, COMPTES_RENDUS_FOLDER);
-    ensureAndClearDir(root);
+    ensureDirSync(root);
     const originalRoot = path.join(root, DATA_ORIGINAL_FOLDER);
     fs.ensureDirSync(originalRoot);
     const transformedRoot = path.join(root, DATA_TRANSFORMED_FOLDER);
@@ -153,11 +153,22 @@ export async function retrieveCriXmlDump(dataDir, options = {}) {
         }
         const xmlFiles = (await fs.readdir(originalSessionDir)).filter((f) => /^d\d{8}\.xml$/i.test(f)).sort();
         const transformedSessionDir = path.join(transformedRoot, String(session));
-        if (options["parseDebats"])
-            await fs.ensureDir(transformedSessionDir);
+        await fs.ensureDir(transformedSessionDir);
+        const now = Date.now();
         for (const f of xmlFiles) {
             const yyyymmdd = f.slice(1, 9);
             const xmlPath = path.join(originalSessionDir, f);
+            if (options["only-recent"]) {
+                const cutoff = now - options["only-recent"] * 24 * 3600 * 1000;
+                const seanceTs = Date.parse(yyyymmdd.slice(0, 4) + "-" + yyyymmdd.slice(4, 6) + "-" + yyyymmdd.slice(6, 8));
+                if (seanceTs < cutoff) {
+                    // Check if some file exists sarting with CRSSN{yyyymmdd} in transformed dir
+                    const someFile = (await fs.readdir(transformedSessionDir)).find((fn) => fn.startsWith(`CRSSN${yyyymmdd}`));
+                    if (someFile) {
+                        continue;
+                    }
+                }
+            }
             // 1) Deduce slot(s) from agenda if it exsits
             const agendaInfo = loadAgendaSPSlotsForDate(dataDir, yyyymmdd, session);
             const firstSlotOfDay = pickFirstSlotOfDay(agendaInfo?.slots ?? []);
@@ -209,9 +220,7 @@ async function main() {
     assert(dataDir, "Missing argument: data directory");
     console.time("CRI processing time");
     await retrieveCriXmlDump(dataDir, options);
-    if (!options["silent"]) {
-        console.timeEnd("CRI processing time");
-    }
+    console.timeEnd("CRI processing time");
 }
 main()
     .then(() => process.exit(0))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.19.7",
+  "version": "2.19.8",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",