@tricoteuses/senat 2.22.16 → 2.23.0

package/lib/src/scripts/retrieve_senateurs_photos.js

@@ -1,14 +1,15 @@
-import assert from "assert";
 import { execSync } from "child_process";
 import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 // import fetch from "node-fetch"
 import path from "path";
+import { fileURLToPath } from "url";
 // import stream from "stream"
 // import util from "util"
+import * as git from "../git";
 import { findActif as findActifSenateurs } from "../model/sens";
 import { slugify } from "../strings";
-import { commonOptions } from "./shared/cli_helpers";
+import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers";
 const optionsDefinitions = [
     ...commonOptions,
     {
@@ -18,22 +19,32 @@ const optionsDefinitions = [
         type: Boolean,
     },
 ];
+let exitCode = 10; // 0: some data changed, 10: no modification
 const options = commandLineArgs(optionsDefinitions);
 // const pipeline = util.promisify(stream.pipeline)
+const scriptDir = path.dirname(fileURLToPath(import.meta.url));
+function commitAndPushGit(datasetDir, options) {
+    if (options.commit) {
+        const errorCode = git.commitAndPush(datasetDir, "Nouvelle moisson", options.remote);
+        if ((exitCode === 10 && errorCode !== 10) || (exitCode === 0 && errorCode !== 0 && errorCode !== 10)) {
+            exitCode = errorCode;
+        }
+    }
+}
 async function retrievePhotosSenateurs() {
-    const dataDir = options["dataDir"];
-    assert(dataDir, "Missing argument: data directory");
+    const dataDir = assertExistingDirectory(options["dataDir"], "data directory");
     const photosDir = path.join(dataDir, "photos_senateurs");
-    const missingPhotoFilePath = path.resolve(__dirname, "images", "transparent_155x225.jpg");
+    const missingPhotoFilePath = path.resolve(scriptDir, "images", "transparent_155x225.jpg");
     const sens = await Array.fromAsync(findActifSenateurs());
     // Download photos.
     fs.ensureDirSync(photosDir);
     if (options["fetch"]) {
         for (const sen of sens) {
-            const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(sen.senprenomuse, "_")}${slugify(sen.senmat, "_")}`;
-            const photoFilename = photoStem + ".jpg";
+            const nom = sen.sennomuse ?? "";
+            const safePhotoStem = `${slugify(nom, "_")}_${slugify(sen.senprenomuse, "_")}${slugify(sen.senmat, "_")}`;
+            const photoFilename = safePhotoStem + ".jpg";
             const photoFilePath = path.join(photosDir, photoFilename);
-            const photoTempFilename = photoStem + "_temp.jpg";
+            const photoTempFilename = safePhotoStem + "_temp.jpg";
             const photoTempFilePath = path.join(photosDir, photoTempFilename);
             const urlPhoto = `https://www.senat.fr/senimg/${photoFilename}`;
             if (!options["silent"]) {
@@ -81,7 +92,8 @@ async function retrievePhotosSenateurs() {
     }
     // Resize photos to 155x225, because some haven't exactly this size.
     for (const sen of sens) {
-        const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(sen.senprenomuse, "_")}${slugify(sen.senmat, "_")}`;
+        const nom = sen.sennomuse ?? "";
+        const photoStem = `${slugify(nom, "_")}_${slugify(sen.senprenomuse, "_")}${slugify(sen.senmat, "_")}`;
         const photoFilename = photoStem + ".jpg";
         const photoFilePath = path.join(photosDir, photoFilename);
         if (fs.existsSync(photoFilePath)) {
@@ -109,7 +121,8 @@ async function retrievePhotosSenateurs() {
         const row = sens.slice(senIndex, senIndex + 25);
         const photosFilenames = [];
         for (const [columnIndex, sen] of row.entries()) {
-            const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(sen.senprenomuse, "_")}${slugify(sen.senmat, "_")}`;
+            const nom = sen.sennomuse ?? "";
+            const photoStem = `${slugify(nom, "_")}_${slugify(sen.senprenomuse, "_")}${slugify(sen.senmat, "_")}`;
             const photoFilename = `${photoStem}_155x225.jpg`;
             photosFilenames.push(photoFilename);
             photoBySenmat[sen.senmat] = {
@@ -138,9 +151,10 @@ async function retrievePhotosSenateurs() {
     }
     const jsonFilePath = path.join(photosDir, "senateurs.json");
     fs.writeFileSync(jsonFilePath, JSON.stringify(photoBySenmat, null, 2));
+    commitAndPushGit(photosDir, options);
 }
 retrievePhotosSenateurs()
-    .then(() => process.exit(0))
+    .then(() => process.exit(exitCode))
     .catch((error) => {
     console.log(error);
     process.exit(1);

package/lib/src/scripts/retrieve_videos.js

@@ -1,13 +1,12 @@
 // scripts/retrieve_senat_videos_from_agendas.ts
-import assert from "assert";
 import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 import fsp from "fs/promises";
 import path from "path";
 import * as git from "../git.js";
 import { AGENDA_FOLDER, iterLoadSenatAgendas } from "../loaders";
-import { getSessionsFromStart } from "../types/sessions";
-import { commonOptions } from "./shared/cli_helpers";
+import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions";
+import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers";
 import { getAgendaSegmentTimecodes, buildSenatVodMasterM3u8FromNvs } from "../utils/nvs-parsing";
 import { epochToParisDateTime, isAmbiguousTimeOriginal, toTargetEpoch } from "../utils/date";
 import { pathToFileURL } from "url";
@@ -17,6 +16,7 @@ import { SENAT_DATAS_ROOT, STATS, VIDEOS_ROOT_FOLDER, weights } from "../videos/
 import { processBisIfNeeded, processOneReunionMatch, writeIfChanged } from "../videos";
 const optionsDefinitions = [...commonOptions];
 const options = commandLineArgs(optionsDefinitions);
+let exitCode = 10; // 0: some data changed, 10: no modification
 function shouldSkipAgenda(agenda) {
     if (!agenda.captationVideo)
         return true;
@@ -164,8 +164,9 @@ async function processAll(dataDir, sessions) {
             try {
                 await processGroupedReunion(agenda, session, dataDir, lastByVideo);
             }
-            catch (e) {
-                console.error(`[error] ${agenda?.uid ?? "unknown-uid"}:`, e?.message || e);
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                console.error(`[error] ${agenda?.uid ?? "unknown-uid"}:`, message);
             }
         }
     }
@@ -173,18 +174,16 @@ async function processAll(dataDir, sessions) {
     commitAndPushGit(debatsDir, options);
 }
 function commitAndPushGit(datasetDir, options) {
-    let exitCode = 10; // 0: some data changed, 10: no modification
     if (options.commit) {
-        const errorCode = git.commitAndPush(datasetDir, "Nouvelle moisson", options.remote);
+        const errorCode = git.commitAndPush(datasetDir, "Nouvelle moisson", options["remote"]);
         if ((exitCode === 10 && errorCode !== 10) || (exitCode === 0 && errorCode !== 0 && errorCode !== 10)) {
             exitCode = errorCode;
         }
     }
 }
 async function main() {
-    const dataDir = options["dataDir"];
-    assert(dataDir, "Missing argument: data directory");
-    const sessions = getSessionsFromStart(options["fromSession"]);
+    const dataDir = assertExistingDirectory(options["dataDir"], "data directory");
+    const sessions = getSessionsFromStart((options["fromSession"] ?? UNDEFINED_SESSION));
     const TIMER = "senat-agendas→videos processing time";
     console.time(TIMER);
     await processAll(dataDir, sessions);
@@ -194,7 +193,9 @@ async function main() {
     console.log(`[summary] accepted=${accepted} / total=${total} (${ratio}%)`);
 }
 if (import.meta.url === pathToFileURL(process.argv[1]).href) {
-    main().catch((err) => {
+    main()
+        .then(() => process.exit(exitCode))
+        .catch((err) => {
         console.error(err);
         process.exit(1);
     });

package/lib/src/scripts/shared/cli_helpers.d.ts

@@ -58,12 +58,6 @@ export declare const remoteOption: {
     name: string;
     type: StringConstructor;
 };
-export declare const pullOption: {
-    alias: string;
-    help: string;
-    name: string;
-    type: BooleanConstructor;
-};
 export declare const fetchDocumentsOption: {
     help: string;
     name: string;
@@ -93,3 +87,4 @@ export declare const commonOptions: ({
     name: string;
     type: StringConstructor;
 })[];
+export declare function assertExistingDirectory(directoryPath: string | undefined, description: string): string;

package/lib/src/scripts/shared/cli_helpers.js

@@ -1,3 +1,5 @@
+import assert from "assert";
+import fs from "fs-extra";
 export const categoriesOption = {
     alias: "k",
     defaultValue: ["All"],
@@ -53,17 +55,11 @@ export const commitOption = {
 };
 export const remoteOption = {
     alias: "r",
-    help: "push commit to given remote",
+    help: "pull before proceeding and push commit to given remote",
     multiple: true,
     name: "remote",
     type: String,
 };
-export const pullOption = {
-    alias: "p",
-    help: "pull repositories before proceeding",
-    name: "pull",
-    type: Boolean,
-};
 export const fetchDocumentsOption = {
     help: "download documents",
     name: "fetchDocuments",
@@ -85,7 +81,12 @@ export const commonOptions = [
     cloneOption,
     commitOption,
     remoteOption,
-    pullOption,
     fetchDocumentsOption,
     parseDocumentsOption,
 ];
+export function assertExistingDirectory(directoryPath, description) {
+    assert(directoryPath, `Missing argument: ${description}`);
+    assert(fs.existsSync(directoryPath), `${description} does not exist: ${directoryPath}`);
+    assert(fs.statSync(directoryPath).isDirectory(), `${description} is not a directory: ${directoryPath}`);
+    return directoryPath;
+}

package/lib/src/scripts/shared/incremental_import_sql.d.ts

@@ -0,0 +1,2 @@
+export declare function buildNormalizeStagingSchemaSql(datasetName: string): string;
+export declare function buildIncrementalDatasetImportSql(datasetName: string, dbUser: string, mergeKeys?: Record<string, string[]>, rowMultisetMergeTables?: string[]): string;