@tricoteuses/senat 2.22.12 → 2.22.14

package/lib/src/loaders.d.ts

@@ -33,14 +33,8 @@ export type IterItem<T> = {
 export declare function iterFilePaths(dirPath: string): Generator<string>;
 export declare function iterLoadSenatAmendements(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<AmendementResult>>;
 export declare function iterLoadSenatDebats(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DebatResult>>;
-export declare function iterLoadSenatComptesRendusSeances(dataDir: string, session: number): Generator<{
-    compteRendu: CompteRendu;
-    session: number;
-}>;
-export declare function iterLoadSenatComptesRendusCommissions(dataDir: string, session: number): Generator<{
-    compteRendu: CompteRendu;
-    session: number;
-}>;
+export declare function iterLoadSenatComptesRendusSeances(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<CompteRendu>>;
+export declare function iterLoadSenatComptesRendusCommissions(dataDir: string, session: number, options?: {}): Generator<IterItem<CompteRendu>>;
 export declare function iterLoadSenatDossiersLegislatifs(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DossierLegislatifResult>>;
 export declare function iterLoadSenatRapportUrls(dataDir: string, session: number | undefined): Generator<IterItem<DocumentMetadata>>;
 export declare function iterLoadSenatTexteUrls(dataDir: string, session: number | undefined): Generator<IterItem<DocumentMetadata>>;

package/lib/src/loaders.js

@@ -118,33 +118,15 @@ export function* iterLoadSenatDebats(dataDir, session, options = {}) {
         yield debatItem;
     }
 }
-function* iterLoadSenatComptesRendusGeneric(dataDir, session, subFolder) {
-    const basePath = path.join(dataDir, subFolder, DATA_TRANSFORMED_FOLDER, String(session));
-    if (!fs.existsSync(basePath)) {
-        return;
-    }
-    const files = (fs.readdirSync(basePath) || []).filter((f) => f.endsWith(".json")).sort();
-    for (const fileName of files) {
-        const filePath = path.join(basePath, fileName);
-        try {
-            const fileContent = fs.readFileSync(filePath, "utf-8");
-            const compteRendu = JSON.parse(fileContent);
-            if (!compteRendu?.uid) {
-                console.warn(`[SN] CR without uid → ${fileName}`);
-                continue;
-            }
-            yield { compteRendu, session };
-        }
-        catch (err) {
-            console.warn(`[SN] error reading CR → ${fileName}`, err);
-        }
+export function* iterLoadSenatComptesRendusSeances(dataDir, session, options = {}) {
+    for (const compteRenduItem of iterLoadSenatItems(dataDir, COMPTES_RENDUS_FOLDER, session, DATA_TRANSFORMED_FOLDER, options)) {
+        yield compteRenduItem;
     }
 }
-export function* iterLoadSenatComptesRendusSeances(dataDir, session) {
-    yield* iterLoadSenatComptesRendusGeneric(dataDir, session, COMPTES_RENDUS_FOLDER);
-}
-export function* iterLoadSenatComptesRendusCommissions(dataDir, session) {
-    yield* iterLoadSenatComptesRendusGeneric(dataDir, session, COMMISSION_FOLDER);
+export function* iterLoadSenatComptesRendusCommissions(dataDir, session, options = {}) {
+    for (const compteRenduItem of iterLoadSenatItems(dataDir, COMMISSION_FOLDER, session, DATA_TRANSFORMED_FOLDER, options)) {
+        yield compteRenduItem;
+    }
 }
 export function* iterLoadSenatDossiersLegislatifs(dataDir, session, options = {}) {
     for (const dossierLegislatifItem of iterLoadSenatItems(dataDir, datasets.dosleg.database, session, DOSLEG_DOSSIERS_FOLDER, options)) {

package/lib/src/scripts/retrieve_cr_commission.js

@@ -11,6 +11,7 @@ import { sessionStartYearFromDate } from "../model/seance";
 import { getSessionsFromStart } from "../types/sessions";
 import { ensureAndClearDir, fetchWithRetry } from "./shared/util";
 import { jaccard, jaccardTokenSim } from "../utils/scoring";
+import * as git from "../git.js";
 class CommissionCRDownloadError extends Error {
     constructor(message, url) {
         super(`An error occurred while retrieving Commission CR ${url}: ${message}`);
@@ -104,6 +105,15 @@ async function discoverCommissionWeeklyPages(fromSession) {
     }
     return results.sort((a, b) => a.yyyymmdd.localeCompare(b.yyyymmdd));
 }
+function commitAndPushGit(datasetDir, options) {
+    let exitCode = 10; // 0: some data changed, 10: no modification
+    if (options.commit) {
+        const errorCode = git.commitAndPush(datasetDir, "Nouvelle moisson", options.remote);
+        if ((exitCode === 10 && errorCode !== 10) || (exitCode === 0 && errorCode !== 0 && errorCode !== 10)) {
+            exitCode = errorCode;
+        }
+    }
+}
 function toHourShort(hhmm) {
     if (!hhmm)
         return null;
@@ -348,6 +358,8 @@ async function retrieveCommissionCRs(options = {}) {
             }
         }
     }
+    const debatsDir = path.join(dataDir, COMMISSION_FOLDER, DATA_TRANSFORMED_FOLDER);
+    commitAndPushGit(debatsDir, options);
 }
 async function main() {
     const dataDir = options["dataDir"];

package/lib/src/scripts/retrieve_cr_seance.js

@@ -17,6 +17,7 @@ import { getSessionsFromStart } from "../types/sessions";
 import { ensureAndClearDir, fetchWithRetry } from "./shared/util";
 import { isNoiseBlock, scoreSommaireBlockForEvent } from "../utils/scoring";
 import { parseYYYYMMDD } from "../utils/date";
+import * as git from "../git.js";
 const optionsDefinitions = [
     ...commonOptions,
     {
@@ -204,6 +205,17 @@ export async function retrieveCriXmlDump(dataDir, options = {}) {
             }
         }
     }
+    const debatsDir = path.join(dataDir, COMPTES_RENDUS_FOLDER, DATA_TRANSFORMED_FOLDER);
+    commitAndPushGit(debatsDir, options);
+}
+function commitAndPushGit(datasetDir, options) {
+    let exitCode = 10; // 0: some data changed, 10: no modification
+    if (options.commit) {
+        const errorCode = git.commitAndPush(datasetDir, "Nouvelle moisson", options.remote);
+        if ((exitCode === 10 && errorCode !== 10) || (exitCode === 0 && errorCode !== 0 && errorCode !== 10)) {
+            exitCode = errorCode;
+        }
+    }
 }
 async function linkCriEventIntoAgenda(dataDir, yyyymmdd, agendaEventId, crUid, cr, session) {
     const agendadDir = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, session.toString());

package/lib/src/scripts/retrieve_videos.js

@@ -4,7 +4,8 @@ import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 import fsp from "fs/promises";
 import path from "path";
-import { iterLoadSenatAgendas } from "../loaders";
+import * as git from "../git.js";
+import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatAgendas } from "../loaders";
 import { getSessionsFromStart } from "../types/sessions";
 import { commonOptions } from "./shared/cli_helpers";
 import { getAgendaSegmentTimecodes, buildSenatVodMasterM3u8FromNvs } from "../utils/nvs-parsing";
@@ -168,6 +169,17 @@ async function processAll(dataDir, sessions) {
             }
         }
     }
+    const debatsDir = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER);
+    commitAndPushGit(debatsDir, options);
+}
+function commitAndPushGit(datasetDir, options) {
+    let exitCode = 10; // 0: some data changed, 10: no modification
+    if (options.commit) {
+        const errorCode = git.commitAndPush(datasetDir, "Nouvelle moisson", options.remote);
+        if ((exitCode === 10 && errorCode !== 10) || (exitCode === 0 && errorCode !== 0 && errorCode !== 10)) {
+            exitCode = errorCode;
+        }
+    }
 }
 async function main() {
     const dataDir = options["dataDir"];

package/lib/tests/test_iter_load.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/test_iter_load.test.js

@@ -0,0 +1,17 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { describe, it, expect } from "vitest";
+import { COMPTES_RENDUS_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatComptesRendusSeances } from "../src/loaders";
+describe("iterLoadSenatComptesRendusSeances", () => {
+    it("loads comptes rendus from transformed/session folder", () => {
+        const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "senat-cr-"));
+        const session = 2025;
+        const targetDir = path.join(tmpDir, COMPTES_RENDUS_FOLDER, DATA_TRANSFORMED_FOLDER, String(session));
+        fs.mkdirSync(targetDir, { recursive: true });
+        fs.writeFileSync(path.join(targetDir, "cr-1.json"), JSON.stringify({ uid: "cr-1", titre: "CR 1" }), "utf8");
+        const results = Array.from(iterLoadSenatComptesRendusSeances(tmpDir, session));
+        expect(results).toHaveLength(1);
+        expect(results[0].item.uid).toBe("cr-1");
+    });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.22.12",
+  "version": "2.22.14",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",
@@ -63,7 +63,7 @@
     "prepare": "npm run build",
     "prepublishOnly": "npm run build",
     "prettier": "prettier --write 'src/**/*.ts'",
-    "test:iter_load": "tsx src/scripts/test_iter_load.ts",
+    "test:iter_load": "vitest tests/test_iter_load.test.ts",
     "type-check": "tsc --noEmit",
     "type-check:watch": "npm run type-check -- --watch",
     "test:video-matching": "vitest tests/videoMatching.test.ts"