@tricoteuses/senat 2.22.16 → 3.0.0

package/lib/src/scripts/convert_data.js

@@ -1,4 +1,3 @@
-import assert from "assert";
 import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 import path from "path";
@@ -11,7 +10,7 @@ import { processRapport, processTexte } from "./retrieve_documents";
 import { buildActesLegislatifs } from "../model/dosleg";
 import { UNDEFINED_SESSION } from "../types/sessions";
 import { getSessionFromDate, getSessionFromSignet } from "./datautil";
-import { commonOptions } from "./shared/cli_helpers";
+import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers";
 import { ensureAndClearDir } from "./shared/util";
 let exitCode = 10; // 0: some data changed, 10: no modification
 const optionsDefinitions = [...commonOptions];
@@ -29,8 +28,7 @@ function commitAndPushGit(datasetDir, options) {
     }
 }
 async function convertData() {
-    const dataDir = options["dataDir"];
-    assert(dataDir, "Missing argument: data directory");
+    const dataDir = assertExistingDirectory(options["dataDir"], "data directory");
     const enabledDatasets = getEnabledDatasets(options["categories"]);
     console.time("data transformation time");
     if (enabledDatasets & EnabledDatasets.Ameli) {
@@ -129,6 +127,9 @@ async function convertDatasetDebats(dataDir, options) {
         if (options["verbose"]) {
             console.log(`Converting ${debat.id} file…`);
         }
+        if (!debat.date_seance) {
+            continue;
+        }
         const session = getSessionFromDate(debat.date_seance);
         if (options["fromSession"] && session < options["fromSession"]) {
             continue;
@@ -238,7 +239,7 @@ async function convertTextes(dataDir, options) {
         const hasExposeDesMotifs = texte["origine"] === "déposé au Sénat" || texte["origine"] === "transmis au Sénat";
         const metadata = {
             name: texteName,
-            session: texte["session"],
+            session,
             date: texte["date"],
             url_expose_des_motifs: hasExposeDesMotifs
                 ? new URL(`${texteName}-expose.html`, SENAT_EXPOSE_DES_MOTIFS_BASE_URL)
@@ -286,7 +287,7 @@ async function convertRapports(dataDir, options) {
         });
         const metadata = {
             name: rapportName,
-            session: rapport["session"],
+            session,
             date: rapport["date"],
             url_html: new URL(rapportHtmlUrl, SENAT_RAPPORT_BASE_URL),
             url_pdf: new URL(rapportPdfUrl, SENAT_RAPPORT_BASE_URL),

package/lib/src/scripts/convert_xml_to_html.js

@@ -12,8 +12,8 @@ async function main() {
     try {
         options = commandLineArgs(optionDefinitions, { stopAtFirstUnknown: true });
     }
-    catch (err) {
-        console.error(`Error: ${err.message}`);
+    catch (error) {
+        console.error(`Error: ${error.message}`);
         process.exit(1);
     }
     // Handle positional arguments if not using flags

package/lib/src/scripts/data-download.js

@@ -5,8 +5,9 @@ try {
     execSync(`tsx src/scripts/convert_data.ts ${args}`, { stdio: "inherit" });
 }
 catch (error) {
-    if (error.status !== 10) {
+    const execError = error;
+    if (execError.status !== 10) {
         console.error("Error during data retrieval:", error);
-        process.exit(error.status || 1);
+        process.exit(execError.status || 1);
     }
 }

package/lib/src/scripts/retrieve_agenda.js

@@ -1,13 +1,13 @@
-import assert from "assert";
 import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 import { DateTime } from "luxon";
 import path from "path";
+import * as git from "../git";
 import { AGENDA_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
 import { parseAgendaFromFile } from "../model/agenda";
-import { getSessionsFromStart } from "../types/sessions";
+import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions";
 import { ID_DATE_FORMAT } from "./datautil";
-import { commonOptions } from "./shared/cli_helpers";
+import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers";
 import { fetchWithRetry } from "./shared/util";
 import { buildReunionsByBucket } from "../utils/reunion_parsing";
 import { buildSenatDossierIndex } from "../utils/reunion_odj_building";
@@ -19,6 +19,7 @@ const optionsDefinitions = [
         type: Boolean,
     },
 ];
+let exitCode = 10; // 0: some data changed, 10: no modification
 const options = commandLineArgs(optionsDefinitions);
 const SENAT_GLOBAL_AGENDA_URL_ROOT = "https://www.senat.fr/aglae/Global";
 const EVENT_DATE_FORMAT = "ddMMyyyy";
@@ -27,9 +28,18 @@ class AgendaError extends Error {
         super(`An error occurred while retrieving Agenda ${agendaName}: ${message}`);
     }
 }
+function commitAndPushGit(datasetDir, options) {
+    if (options.commit) {
+        const errorCode = git.commitAndPush(datasetDir, "Nouvelle moisson", options.remote);
+        if ((exitCode === 10 && errorCode !== 10) || (exitCode === 0 && errorCode !== 0 && errorCode !== 10)) {
+            exitCode = errorCode;
+        }
+    }
+}
 async function retrieveAgendas(options, sessions) {
     console.log(`[AGENDA] Retrieving agendas for sessions ${sessions.join(", ")}`);
-    const agendaRootDir = path.join(options["dataDir"], AGENDA_FOLDER);
+    const dataDir = assertExistingDirectory(options["dataDir"], "data directory");
+    const agendaRootDir = path.join(dataDir, AGENDA_FOLDER);
     fs.ensureDirSync(agendaRootDir);
     const originalAgendaDir = path.join(agendaRootDir, DATA_ORIGINAL_FOLDER);
     fs.ensureDirSync(originalAgendaDir);
@@ -51,7 +61,8 @@ async function retrieveAgendas(options, sessions) {
                 fs.emptyDirSync(transformedAgendaSessionDir);
         }
         const fifteenDaysFromNow = new Date();
-        fifteenDaysFromNow.setDate(fifteenDaysFromNow.getDate() + 15); // Don't download agendas more than 15 days in the future
+        // Don't download agendas more than 15 days in the future.
+        fifteenDaysFromNow.setDate(fifteenDaysFromNow.getDate() + 15);
         for (const date = new Date(session, 9, 1); date <= new Date(session + 1, 8, 30) && date <= fifteenDaysFromNow; date.setDate(date.getDate() + 1)) {
             const agendaName = DateTime.fromJSDate(date).toFormat(EVENT_DATE_FORMAT);
             const agendaFileName = DateTime.fromJSDate(date).toFormat(ID_DATE_FORMAT);
@@ -117,15 +128,16 @@ async function parseAgenda(transformedAgendaSessionDir, agendaFileName, agendaPa
     }
 }
 async function main() {
-    const dataDir = options["dataDir"];
-    assert(dataDir, "Missing argument: data directory");
-    const sessions = getSessionsFromStart(options["fromSession"]);
+    const dataDir = assertExistingDirectory(options["dataDir"], "data directory");
+    const sessions = getSessionsFromStart((options["fromSession"] ?? UNDEFINED_SESSION));
     console.time("agenda processing time");
     await retrieveAgendas(options, sessions);
+    const agendaDir = path.join(dataDir, AGENDA_FOLDER);
+    commitAndPushGit(agendaDir, options);
     console.timeEnd("agenda processing time");
 }
 main()
-    .then(() => process.exit(0))
+    .then(() => process.exit(exitCode))
     .catch((error) => {
     console.log(error);
     process.exit(1);

package/lib/src/scripts/retrieve_cr_commission.js

@@ -1,14 +1,13 @@
 import fs, { ensureDir } from "fs-extra";
-import assert from "assert";
 import path from "path";
 import * as cheerio from "cheerio";
 import { COMMISSION_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
 import { loadAgendaForDate, parseCommissionMetadataFromHtml, linkCRtoCommissionGroup } from "../utils/cr_spliting";
 import { cleanTitle, extractDayH3Sections, parseCommissionCRSectionFromDom } from "../model/commission";
 import commandLineArgs from "command-line-args";
-import { commonOptions } from "./shared/cli_helpers";
+import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers";
 import { sessionStartYearFromDate } from "../model/seance";
-import { getSessionsFromStart } from "../types/sessions";
+import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions";
 import { ensureAndClearDir, fetchWithRetry } from "./shared/util";
 import { jaccard, jaccardTokenSim } from "../utils/scoring";
 import * as git from "../git.js";
@@ -28,6 +27,7 @@ const optionsDefinitions = [
     },
 ];
 const options = commandLineArgs(optionsDefinitions);
+let exitCode = 10; // 0: some data changed, 10: no modification
 const COMMISSION_HUBS = {
     "Commission des affaires étrangères": [
         "https://www.senat.fr/compte-rendu-commissions/affaires-etrangeres.html",
@@ -71,7 +71,7 @@ async function harvestWeeklyLinksFromHub(hubUrl) {
     const out = [];
     $("a[href]").each((_, a) => {
         const href = ($(a).attr("href") || "").trim();
-        const m = href.match(/\/compte-rendu-commissions\/(\d{8})\/([a-z0-9\-]+)\.html$/i);
+        const m = href.match(/\/compte-rendu-commissions\/(\d{8})\/([a-z0-9-]+)\.html$/i);
         if (m) {
             const url = href.startsWith("http") ? href : new URL(href, hubUrl).toString();
             out.push(url);
@@ -86,7 +86,7 @@ async function discoverCommissionWeeklyPages(fromSession) {
             try {
                 const links = await harvestWeeklyLinksFromHub(hubUrl);
                 for (const url of links) {
-                    const m = url.match(/\/compte-rendu-commissions\/(\d{8})\/([a-z0-9\-]+)\.html$/i);
+                    const m = url.match(/\/compte-rendu-commissions\/(\d{8})\/([a-z0-9-]+)\.html$/i);
                     if (!m)
                         continue;
                     const yyyymmdd = m[1];
@@ -106,7 +106,6 @@ async function discoverCommissionWeeklyPages(fromSession) {
     return results.sort((a, b) => a.yyyymmdd.localeCompare(b.yyyymmdd));
 }
 function commitAndPushGit(datasetDir, options) {
-    let exitCode = 10; // 0: some data changed, 10: no modification
     if (options.commit) {
         const errorCode = git.commitAndPush(datasetDir, "Nouvelle moisson", options.remote);
         if ((exitCode === 10 && errorCode !== 10) || (exitCode === 0 && errorCode !== 0 && errorCode !== 10)) {
@@ -149,8 +148,8 @@ function toTokens(s) {
         .filter((t) => t.length >= 3 && !["commission", "des", "de", "du", "d", "la", "le", "les", "et"].includes(t)));
 }
 function reunionOrganeCandidates(h) {
-    const any = h;
-    const out = [any.organeSlug, any.organeKey, any.organe, h.titre].filter(Boolean);
+    const extended = h;
+    const out = [extended.organeSlug, extended.organeKey, extended.organe, h.titre].filter(Boolean);
     return Array.from(new Set(out.map(normOrgane)));
 }
 function organeSimilarity(h, commissionKey) {
@@ -182,7 +181,7 @@ function titleSimilarity(reunion, sectionTitle) {
     return Math.max(sTit, sObj);
 }
 async function retrieveCommissionCRs(options = {}) {
-    const dataDir = options["dataDir"];
+    const dataDir = assertExistingDirectory(options["dataDir"], "data directory");
     const fromSession = Number(options["fromSession"]);
     const concurrency = Number(options["concurrency"] ?? 6);
     const politenessMs = Number(options["politenessMs"] ?? 150);
@@ -201,7 +200,7 @@ async function retrieveCommissionCRs(options = {}) {
         const session = sessionStartYearFromDate(d);
         const dir = path.join(originalRoot, String(session), commissionKey);
         fs.ensureDirSync(dir);
-        const slug = url.replace(/^.*\/(\d{8})\/([^\/]+)\.html$/i, "$2");
+        const slug = url.replace(/^.*\/(\d{8})\/([^/]+)\.html$/i, "$2");
         const outPath = path.join(dir, `${yyyymmdd}.${slug}.html`);
         return { url, outPath, yyyymmdd, commissionKey };
     });
@@ -229,8 +228,9 @@ async function retrieveCommissionCRs(options = {}) {
                     }
                 }
             }
-            catch (e) {
-                console.error(`[COM-CR][err] ${url} → ${e?.message || e}`);
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                console.error(`[COM-CR][err] ${url} → ${message}`);
             }
             finally {
                 completed++;
@@ -241,7 +241,7 @@ async function retrieveCommissionCRs(options = {}) {
     });
     await Promise.all(workers);
     console.log(`[COM-CR] done: saved=${saved} | skipped=${skipped} | 404=${notFound} | total=${completed}`);
-    const sessions = getSessionsFromStart(options["fromSession"]);
+    const sessions = getSessionsFromStart((options["fromSession"] ?? UNDEFINED_SESSION));
     const comRoot = path.join(dataDir, COMMISSION_FOLDER);
     const transformedRoot = path.join(comRoot, DATA_TRANSFORMED_FOLDER);
     if (options["keepDir"])
@@ -354,7 +354,8 @@ async function retrieveCommissionCRs(options = {}) {
                 }
             }
             if (!options["silent"]) {
-                console.log(`[COM-CR][SESSION ${session}][${commissionKey}] Processed ${totalFiles} CR files, linked to agenda: ${linkedFiles}`);
+                console.log(`[COM-CR][SESSION ${session}][${commissionKey}] Processed ${totalFiles} CR files`);
+                console.log(`[COM-CR][SESSION ${session}][${commissionKey}] Linked to agenda: ${linkedFiles}`);
             }
         }
     }
@@ -362,14 +363,13 @@ async function retrieveCommissionCRs(options = {}) {
     commitAndPushGit(debatsDir, options);
 }
 async function main() {
-    const dataDir = options["dataDir"];
-    assert(dataDir, "Missing argument: data directory");
+    const dataDir = assertExistingDirectory(options["dataDir"], "data directory");
     console.time("CRI processing time");
     await retrieveCommissionCRs(options);
     console.timeEnd("CRI processing time");
 }
 main()
-    .then(() => process.exit(0))
+    .then(() => process.exit(exitCode))
     .catch((error) => {
     console.error(error);
     process.exit(1);

package/lib/src/scripts/retrieve_cr_seance.d.ts

@@ -3,4 +3,17 @@
  * - downloads the ZIP of comptes-rendus des débats (CRI) from data.senat.fr
  * - extracts XML files, distributes them by session/year
  */
-export declare function retrieveCriXmlDump(dataDir: string, options?: Record<string, any>): Promise<void>;
+import { Session } from "../types/sessions";
+import { CommandLineOptions } from "command-line-args";
+type RetrieveCriOptions = CommandLineOptions & {
+    commit?: boolean;
+    dataDir?: string;
+    fromSession?: Session;
+    keepDir?: boolean;
+    "only-recent"?: number;
+    parseDebats?: boolean;
+    remote?: string[];
+    silent?: boolean;
+};
+export declare function retrieveCriXmlDump(dataDir: string, options?: RetrieveCriOptions): Promise<void>;
+export {};

package/lib/src/scripts/retrieve_cr_seance.js

@@ -3,17 +3,16 @@
  * - downloads the ZIP of comptes-rendus des débats (CRI) from data.senat.fr
  * - extracts XML files, distributes them by session/year
  */
-import assert from "assert";
 import commandLineArgs from "command-line-args";
 import fs, { ensureDirSync } from "fs-extra";
 import path from "path";
 import StreamZip from "node-stream-zip";
 import * as cheerio from "cheerio";
 import { AGENDA_FOLDER, COMPTES_RENDUS_FOLDER, DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
-import { commonOptions } from "./shared/cli_helpers";
+import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers";
 import { parseCompteRenduIntervalFromFile, sessionStartYearFromDate } from "../model/seance";
 import { extractSommaireBlocks, makeReunionUid } from "../utils/reunion_parsing";
-import { getSessionsFromStart } from "../types/sessions";
+import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions";
 import { ensureAndClearDir, fetchWithRetry } from "./shared/util";
 import { isNoiseBlock, scoreSommaireBlockForEvent } from "../utils/scoring";
 import { parseYYYYMMDD } from "../utils/date";
@@ -28,6 +27,7 @@ const optionsDefinitions = [
 ];
 const options = commandLineArgs(optionsDefinitions);
 const CRI_ZIP_URL = "https://data.senat.fr/data/debats/cri.zip";
+let exitCode = 10; // 0: some data changed, 10: no modification
 class CompteRenduError extends Error {
     constructor(message, url) {
         super(`An error occurred while retrieving ${url}: ${message}`);
@@ -94,7 +94,7 @@ export async function retrieveCriXmlDump(dataDir, options = {}) {
     else {
         fs.ensureDirSync(transformedRoot);
     }
-    const sessions = getSessionsFromStart(options["fromSession"]);
+    const sessions = getSessionsFromStart((options["fromSession"] ?? UNDEFINED_SESSION));
     // 1) Download ZIP global + distribut by session
     const zipPath = path.join(dataDir, "cri.zip");
     console.log("[CRI] Downloading global CRI zip…");
@@ -209,9 +209,8 @@ export async function retrieveCriXmlDump(dataDir, options = {}) {
     commitAndPushGit(debatsDir, options);
 }
 function commitAndPushGit(datasetDir, options) {
-    let exitCode = 10; // 0: some data changed, 10: no modification
     if (options.commit) {
-        const errorCode = git.commitAndPush(datasetDir, "Nouvelle moisson", options.remote);
+        const errorCode = git.commitAndPush(datasetDir, "Nouvelle moisson", options["remote"]);
         if ((exitCode === 10 && errorCode !== 10) || (exitCode === 0 && errorCode !== 0 && errorCode !== 10)) {
             exitCode = errorCode;
         }
@@ -237,7 +236,6 @@ async function linkCriEventIntoAgenda(dataDir, yyyymmdd, agendaEventId, crUid, c
         console.warn(`[CR] Missing reunion file for SP event=${agendaEventId}: ${agendaPath}`);
         return;
     }
-    ;
     agenda.compteRenduRefUid = crUid;
     await fs.writeJSON(agendaPath, agenda, { spaces: 2 });
     console.log(`[CR] Linked CR ${crUid} → ${path.basename(agendaPath)} (event=${agendaEventId})`);
@@ -327,7 +325,9 @@ async function loadAgendaSpEventsForDate(dataDir, yyyymmdd, session) {
             if (e && e.type === "Séance publique")
                 events.push(e);
         }
-        catch { }
+        catch {
+            continue;
+        }
     }
     return events;
 }
@@ -345,14 +345,13 @@ function resolveTargetIndex($, idx, targetId) {
     return i == null ? null : i;
 }
 async function main() {
-    const dataDir = options["dataDir"];
-    assert(dataDir, "Missing argument: data directory");
+    const dataDir = assertExistingDirectory(options["dataDir"], "data directory");
     console.time("CRI processing time");
     await retrieveCriXmlDump(dataDir, options);
     console.timeEnd("CRI processing time");
 }
 main()
-    .then(() => process.exit(0))
+    .then(() => process.exit(exitCode))
     .catch((error) => {
     console.error(error);
     process.exit(1);

package/lib/src/scripts/retrieve_documents.d.ts

@@ -1,3 +1,12 @@
 import { DocumentMetadata } from "../types/texte";
-export declare function processTexte(texteMetadata: DocumentMetadata, originalTextesDir: string, transformedTextesDir: string, enrichedTextesDir: string, options: any): Promise<void>;
-export declare function processRapport(rapportMetadata: any, originalRapportsDir: string, options: any): Promise<void>;
+type DownloadOptions = {
+    force?: boolean;
+    formats?: string[];
+    "only-recent"?: number;
+    parseDocuments?: boolean;
+    silent?: boolean;
+    verbose?: boolean;
+};
+export declare function processTexte(texteMetadata: DocumentMetadata, originalTextesDir: string, transformedTextesDir: string, enrichedTextesDir: string, options: DownloadOptions): Promise<void>;
+export declare function processRapport(rapportMetadata: DocumentMetadata, originalRapportsDir: string, options: DownloadOptions): Promise<void>;
+export {};

package/lib/src/scripts/retrieve_documents.js

@@ -1,4 +1,3 @@
-import assert from "assert";
 import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 import { DateTime } from "luxon";
@@ -8,7 +7,7 @@ import * as git from "../git";
 import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, ENRICHED_TEXTE_FOLDER, iterLoadSenatRapportUrls, iterLoadSenatTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, } from "../loaders";
 import { parseExposeDesMotifs, parseTexte, parseTexteFromFile } from "../parsers/texte";
 import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions";
-import { commonOptions } from "./shared/cli_helpers";
+import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers";
 import { ensureAndClearDir, fetchWithRetry, isOptionEmptyOrHasValue } from "./shared/util";
 let exitCode = 10; // 0: some data changed, 10: no modification
 const optionsDefinitions = [
@@ -54,8 +53,8 @@ function shouldDownload(filePath, docDate, options) {
         return true;
     if (!fs.existsSync(filePath))
         return true;
-    if (options.onlyRecent !== undefined) {
-        return isDocumentRecent(docDate, options.onlyRecent);
+    if (options["only-recent"] !== undefined) {
+        return isDocumentRecent(docDate, options["only-recent"]);
     }
     return false;
 }
@@ -124,7 +123,7 @@ export async function processTexte(texteMetadata, originalTextesDir, transformed
         { type: "pdf", url: texteMetadata.url_pdf, isParseTarget: false },
     ];
     for (const format of formats) {
-        if (!isOptionEmptyOrHasValue(options.formats, format.type))
+        if (!isOptionEmptyOrHasValue((options.formats ?? []).join(","), format.type))
             continue;
         if (format.url === undefined || format.url.toString().includes("#"))
             continue;
@@ -165,7 +164,7 @@ export async function processRapport(rapportMetadata, originalRapportsDir, optio
         { type: "pdf", url: rapportMetadata.url_pdf },
     ];
     for (const format of formats) {
-        if (!isOptionEmptyOrHasValue(options["formats"], format.type))
+        if (!isOptionEmptyOrHasValue((options.formats ?? []).join(","), format.type))
             continue;
         const destPath = path.join(rapportDir, `${rapportMetadata.name}.${format.type}`);
         await processDocument(format.url.toString(), destPath, rapportMetadata.date, options);
@@ -176,7 +175,7 @@ async function processTextes(dataDir, sessions) {
     const originalTextesDir = path.join(textesDir, DATA_ORIGINAL_FOLDER);
     const transformedTextesDir = path.join(textesDir, DATA_TRANSFORMED_FOLDER);
     const enrichedTextesDir = path.join(dataDir, ENRICHED_TEXTE_FOLDER);
-    if (options["parseDocuments"] && options["only-recent"] === undefined) {
+    if (options.parseDocuments && options["only-recent"] === undefined) {
         ensureAndClearDir(transformedTextesDir);
         ensureAndClearDir(enrichedTextesDir);
     }
@@ -184,7 +183,7 @@ async function processTextes(dataDir, sessions) {
         force: options["force"],
         silent: options["silent"],
         verbose: options["verbose"],
-        onlyRecent: options["only-recent"],
+        "only-recent": options["only-recent"],
         formats: options["formats"],
         parseDocuments: options["parseDocuments"],
     };
@@ -203,7 +202,7 @@ async function processRapports(dataDir, sessions) {
         force: options["force"],
         silent: options["silent"],
         verbose: options["verbose"],
-        onlyRecent: options["only-recent"],
+        "only-recent": options["only-recent"],
         formats: options["formats"],
     };
     for (const session of sessions) {
@@ -245,14 +244,26 @@ async function parseDocument(session, transformedTextesDir, textePath, texteName
     return parsedTexte;
 }
 async function main() {
-    const dataDir = options["dataDir"];
-    assert(dataDir, "Missing argument: data directory");
-    const sessions = getSessionsFromStart(options["fromSession"]);
+    const dataDir = assertExistingDirectory(options["dataDir"], "data directory");
+    const sessions = getSessionsFromStart((options.fromSession ?? UNDEFINED_SESSION));
+    const textesDir = path.join(dataDir, TEXTE_FOLDER);
+    const enrichedTextesDir = path.join(dataDir, ENRICHED_TEXTE_FOLDER);
+    const rapportsDir = path.join(dataDir, RAPPORT_FOLDER);
+    const syncRemote = options["remote"]?.[0];
     console.time("documents processing time");
-    if (isOptionEmptyOrHasValue(options["types"], "textes")) {
+    if (syncRemote) {
+        if (isOptionEmptyOrHasValue(options.types?.join(",") ?? "", "textes")) {
+            git.pull(textesDir, syncRemote);
+            git.pull(enrichedTextesDir, syncRemote);
+        }
+        if (isOptionEmptyOrHasValue(options.types?.join(",") ?? "", "rapports")) {
+            git.pull(rapportsDir, syncRemote);
+        }
+    }
+    if (isOptionEmptyOrHasValue(options.types?.join(",") ?? "", "textes")) {
         await processTextes(dataDir, sessions);
     }
-    if (isOptionEmptyOrHasValue(options["types"], "rapports")) {
+    if (isOptionEmptyOrHasValue(options.types?.join(",") ?? "", "rapports")) {
         await processRapports(dataDir, sessions);
     }
     if (!options["silent"]) {