@tricoteuses/senat 2.20.16 → 2.20.18

package/README.md

@@ -29,40 +29,79 @@ docker run --name local-postgres -d -p 5432:5432 -e POSTGRES_PASSWORD=$YOUR_CUST
 
 ## Download data
 
+### Basic usage
+
 Create a folder where the data will be downloaded and run the following command to download the data and convert it into JSON files.
 
 ```bash
 mkdir ../senat-data/
 
-# Available options for optional `categories` parameter : All,  Ameli, Debats, DosLeg, Questions, Sens
-npm run data:download ../senat-data -- [--categories All]
+npm run data:download ../senat-data
 ```
 
-Data from other sources is also available :
+### Available Commands
+
+- `npm run data:download <dir>`: Download, convert data to JSON
+- `npm run data:retrieve_documents <dir>`: Retrieval of textes and rapports from Sénat's website
+- `npm run data:parse_textes_lois <dir>`: Parse textes (requires xml files)
+- `npm run data:retrieve_agenda <dir>`: Retrieval of agenda from Sénat's website
+- `npm run data:retrieve_cr_seance <dir>`: Retrieval of comptes-rendus de séance from Sénat's data
+- `npm run data:retrieve_cr_commission <dir>`: Retrieval of comptes-rendus de commissions from Sénat's website
+- `npm run data:retrieve_senateurs_photos <dir>`: Retrieval of sénateurs' pictures from Sénat's website
+
+### Filtering Options
+
+Downloading all the data is long and takes up a lot of disk space. It is possible to choose the type of data that you want to retrieve to reduce the load.
+
+Examples:
 
 ```bash
-# Retrieval of textes and rapports from Sénat's website
-# Available options for optional `formats` parameter : xml, html, pdf
-# Available options for optional `types` parameter : textes, rapports
-npm run data:retrieve_documents ../senat-data -- --fromSession 2022 [--formats xml pdf] [--types textes]
+# Only download amendments
+npm run data:download ../senat-data -- -k Ameli
 
-# Retrieval & parsing (textes in xml format only for now)
-npm run data:retrieve_documents ../senat-data -- --fromSession 2022 --parseDocuments
+# Only process data from session 2023 onwards
+npm run data:download ../senat-data -- --fromSession 2023
+```
 
-# Parsing only
-npm run data:parse_textes_lois ../senat-data
+### Common Options
 
-# Retrieval (& parsing) of agenda from Sénat's website
-npm run data:retrieve_agenda ../senat-data -- --fromSession 2022 [--parseAgenda]
+- `--categories` or `-k <name>`: Filter by dataset categories (Available options: `All`, `Ameli`, `Debats`, `DosLeg`, `Questions`, `Sens`)
+- `--fromSession <year>`: Specify the session year to retrieve data from (default: 2022)
+- `--dataDir <path>` (Mandatory): Path to the working directory where all data is stored (required)
+- `--silent` or `-s`: Disable logging
+- `--verbose` or `-v`: Enable verbose logging
+- `--commit` or `-c`: Automatically commit converted data
+- `--pull` or `-p`: Pull repositories before starting
+- `--clone` or `-C <url>`: Clone Git repositories from a remote group or organization
+- `--remote` or `-r <name>`: Push commits to specified Git remote(s)
+- `--keepDir`: Keep directories when cleaning data
+- `--only-recent <days>`: Retrieve only documents created within the last N days
+
+### Options for Retrieving Documents
+
+- `--formats <format>`: Specify document formats to retrieve (options: `xml`, `html`, `pdf`)
+- `--types <type>`: Specify document types to retrieve (options: `textes`, `rapports`)
+- `--parseDocuments`: Parse documents after retrieval
+- `--parseAgenda`: Parse agenda after retrieval
+- `--parseDebats`: Parse comptes-rendus after retrieval
+
+#### Examples
+
+```bash
+# Retrieval of textes and rapports in specific formats
+npm run data:retrieve_documents ../senat-data -- --fromSession 2022 --formats xml pdf --types textes
+
+# Retrieval & parsing (textes in xml format only for now)
+npm run data:retrieve_documents ../senat-data -- --fromSession 2022 --parseDocuments
 
-# Retrieval (& parsing) of comptes-rendus de séance from Sénat's data
-npm run data:retrieve_cr_seance ../senat-data -- [--parseDebats] [--keepDir]
+# Retrieval & parsing of agenda
+npm run data:retrieve_agenda ../senat-data -- --fromSession 2022 --parseAgenda
 
-# Retrieval (& parsing) of comptes-rendus de commissions from Sénat's website
-npm run data:retrieve_cr_commission ../senat-data -- [--parseDebats] [--keepDir]
+# Retrieval & parsing of comptes-rendus de séance
+npm run data:retrieve_cr_seance ../senat-data -- --parseDebats --keepDir
 
-# Retrieval of sénateurs' pictures from Sénat's website
-npm run data:retrieve_senateurs_photos ../senat-data
+# Retrieval & parsing of comptes-rendus de commissions
+npm run data:retrieve_cr_commission ../senat-data -- --parseDebats --keepDir
 ```
 
 ## Data download using Docker

package/lib/git.d.ts

@@ -0,0 +1,26 @@
+export declare function initRepo(repositoryDir: string): void;
+export declare function commit(repositoryDir: string, message: string): boolean;
+export declare function commitAndPush(repositoryDir: string, message: string, remotes?: string[]): number;
+export declare function resetAndPull(gitDir: string): boolean;
+export declare function clone(gitGroupUrl: string | undefined, gitName: string, workingDir: string): void;
+export declare function run(repositoryDir: string, args: string, verbose?: boolean): string;
+export declare function test(repositoryDir: string, args: string, verbose?: boolean): boolean;
+/**
+ * Information about a changed file in git
+ */
+export interface GitChangedFile {
+    path: string;
+    status: "A" | "M" | "D" | "R" | "C" | "T" | "U";
+}
+/**
+ * Get the list of files that have changed since a specific commit in a git repository.
+ * @param repositoryDir The directory of the git repository
+ * @param sinceCommit The commit hash to compare against (e.g., "HEAD~1", "abc123", etc.)
+ * @param options Options for filtering
+ * @param options.diffFilter Git diff-filter string (default: "AMR").
+ *        A=Added, M=Modified, D=Deleted, R=Renamed, C=Copied, T=Type changed, U=Unmerged
+ * @returns A Map of file paths to their git status
+ */
+export declare function getChangedFilesSinceCommit(repositoryDir: string, sinceCommit: string, options?: {
+    diffFilter?: string;
+}): Map<string, GitChangedFile["status"]>;

package/lib/git.js

@@ -0,0 +1,167 @@
+import { execSync } from "node:child_process";
+import fs from "fs-extra";
+import path from "node:path";
+const MAXBUFFER = 50 * 1024 * 1024;
+export function initRepo(repositoryDir) {
+    if (!fs.existsSync(path.join(repositoryDir, ".git"))) {
+        fs.ensureDirSync(repositoryDir);
+        execSync("git init", {
+            cwd: repositoryDir,
+            env: process.env,
+            encoding: "utf-8",
+            stdio: ["ignore", "ignore", "pipe"],
+        });
+    }
+}
+export function commit(repositoryDir, message) {
+    initRepo(repositoryDir);
+    execSync("git add .", {
+        cwd: repositoryDir,
+        env: process.env,
+        encoding: "utf-8",
+        stdio: ["ignore", "ignore", "pipe"],
+        maxBuffer: MAXBUFFER,
+    });
+    try {
+        execSync(`git commit -m "${message}" --quiet`, {
+            cwd: repositoryDir,
+            env: process.env,
+            encoding: "utf-8",
+            stdio: ["ignore", "pipe", "pipe"],
+        });
+        return true;
+    }
+    catch (childProcess) {
+        if (childProcess.stdout === null ||
+            !/nothing to commit|rien à valider/.test(childProcess.stdout)) {
+            console.error(childProcess.output);
+            throw childProcess;
+        }
+        return false;
+    }
+}
+export function commitAndPush(repositoryDir, message, remotes) {
+    let exitCode = 0;
+    if (commit(repositoryDir, message)) {
+        for (const remote of remotes || []) {
+            try {
+                execSync(`git push ${remote} master`, {
+                    cwd: repositoryDir,
+                    env: process.env,
+                    encoding: "utf-8",
+                    stdio: ["ignore", "ignore", "pipe"],
+                });
+            }
+            catch (childProcess) {
+                // Don't stop when push fails.
+                console.error(childProcess.output);
+                exitCode = childProcess.status;
+            }
+        }
+    }
+    else {
+        // There was nothing to commit.
+        exitCode = 10;
+    }
+    return exitCode;
+}
+export function resetAndPull(gitDir) {
+    execSync("git reset --hard origin/master", {
+        cwd: gitDir,
+        env: process.env,
+        encoding: "utf-8",
+        stdio: ["ignore", "ignore", "pipe"],
+    });
+    execSync("git pull --rebase", {
+        cwd: gitDir,
+        env: process.env,
+        encoding: "utf-8",
+        stdio: ["ignore", "ignore", "pipe"],
+    });
+    return true;
+}
+export function clone(gitGroupUrl, gitName, workingDir) {
+    if (gitGroupUrl !== undefined) {
+        execSync(`git clone ${gitGroupUrl}/${gitName}.git`, {
+            cwd: workingDir,
+            env: process.env,
+            encoding: "utf-8",
+            stdio: ["ignore", "ignore", "pipe"],
+        });
+    }
+}
+export function run(repositoryDir, args, verbose) {
+    try {
+        if (verbose)
+            console.log(`git -C ${repositoryDir} ${args}`);
+        const output = execSync(`git ${args}`, {
+            cwd: repositoryDir,
+            maxBuffer: MAXBUFFER,
+        })
+            .toString()
+            .trim();
+        if (verbose)
+            console.log(output);
+        return output;
+    }
+    catch (childProcess) {
+        for (const output of ["stdout", "stderr"])
+            console.error(`${output}: ${childProcess[output]}`);
+        throw childProcess;
+    }
+}
+export function test(repositoryDir, args, verbose) {
+    try {
+        if (verbose)
+            console.log(`git -C ${repositoryDir} ${args}`);
+        const output = execSync(`git ${args}`, {
+            cwd: repositoryDir,
+            stdio: ["ignore", "pipe", "pipe"],
+            maxBuffer: MAXBUFFER,
+        })
+            .toString()
+            .trim();
+        if (verbose)
+            console.log(output);
+        return true;
+    }
+    catch (childProcess) {
+        if (childProcess.status != 0)
+            return false;
+        throw childProcess;
+    }
+}
+/**
+ * Get the list of files that have changed since a specific commit in a git repository.
+ * @param repositoryDir The directory of the git repository
+ * @param sinceCommit The commit hash to compare against (e.g., "HEAD~1", "abc123", etc.)
+ * @param options Options for filtering
+ * @param options.diffFilter Git diff-filter string (default: "AMR").
+ *        A=Added, M=Modified, D=Deleted, R=Renamed, C=Copied, T=Type changed, U=Unmerged
+ * @returns A Map of file paths to their git status
+ */
+export function getChangedFilesSinceCommit(repositoryDir, sinceCommit, options = {}) {
+    const { diffFilter } = options;
+    try {
+        // Using diff-filter: A = Added, M = Modified, R = Renamed, D = Deleted, etc.
+        // Default to AMR (excludes deleted files to prevent loading errors)
+        const filter = diffFilter ?? "AMR";
+        const output = run(repositoryDir, `diff --name-status --diff-filter=${filter} ${sinceCommit}`, false);
+        const changedFiles = new Map();
+        for (const line of output.split("\n")) {
+            if (line.trim().length === 0)
+                continue;
+            const parts = line.split("\t");
+            if (parts.length >= 2) {
+                const status = parts[0].charAt(0);
+                const path = parts[1];
+                changedFiles.set(path, status);
+            }
+        }
+        return changedFiles;
+    }
+    catch (error) {
+        console.error(`Error getting changed files since commit ${sinceCommit}:`, error);
+        return new Map();
+    }
+}

package/lib/loaders.d.ts

@@ -49,6 +49,7 @@ export interface DossierLegislatifDocumentResult {
     type_lecture: string;
     libelle_lecture: string;
     libelle_organisme: string | null;
+    code_organisme: string | null;
     numero: number | null;
     id: string | null;
     url: string;

package/lib/scripts/convert_data.js

@@ -3,6 +3,7 @@ import commandLineArgs from "command-line-args";
 import fs from "fs-extra";
 import path from "path";
 import pLimit from "p-limit";
+import * as git from "../git";
 import { datasets, EnabledDatasets, getEnabledDatasets } from "../datasets";
 import { DATA_ORIGINAL_FOLDER, DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, SCRUTINS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER, } from "../loaders";
 import { findAllAmendements, findAllCirconscriptions, findAllDebats, findAllDossiers, findAllScrutins, findAllOrganismes, findAllQuestions, findAllSens, findSenatRapportUrls, findSenatTexteUrls, } from "../model";
@@ -17,14 +18,26 @@ const SENAT_TEXTE_XML_BASE_URL = "https://www.senat.fr/akomantoso/";
 const SENAT_TEXTE_BASE_URL = "https://www.senat.fr/leg/";
 const SENAT_EXPOSE_DES_MOTIFS_BASE_URL = "https://www.senat.fr/leg/exposes-des-motifs/";
 const SENAT_RAPPORT_BASE_URL = "https://www.senat.fr/rap/";
+function commitGit(datasetDir, options, exitCode) {
+    if (options.commit) {
+        const errorCode = git.commitAndPush(datasetDir, "Nouvelle moisson", options.remote);
+        if ((exitCode === 10 && errorCode !== 10) || (exitCode === 0 && errorCode !== 0 && errorCode !== 10)) {
+            exitCode = errorCode;
+        }
+    }
+    return exitCode;
+}
 async function convertData() {
     const dataDir = options["dataDir"];
     assert(dataDir, "Missing argument: data directory");
     const enabledDatasets = getEnabledDatasets(options["categories"]);
     console.time("data transformation time");
+    let exitCode = 0;
     if (enabledDatasets & EnabledDatasets.Ameli) {
         try {
             await convertDatasetAmeli(dataDir, options);
+            const ameliDir = path.join(dataDir, datasets.ameli.database);
+            exitCode = commitGit(ameliDir, options, exitCode);
         }
         catch (error) {
             console.error(`Error converting Ameli dataset:`, error);
@@ -33,6 +46,8 @@ async function convertData() {
     if (enabledDatasets & EnabledDatasets.Debats) {
         try {
             await convertDatasetDebats(dataDir, options);
+            const debatsDir = path.join(dataDir, datasets.debats.database);
+            exitCode = commitGit(debatsDir, options, exitCode);
         }
         catch (error) {
             console.error(`Error converting Debats dataset:`, error);
@@ -41,12 +56,16 @@ async function convertData() {
     if (enabledDatasets & EnabledDatasets.DosLeg) {
         try {
             await convertDatasetDosLeg(dataDir, options);
+            const doslegDir = path.join(dataDir, datasets.dosleg.database);
+            exitCode = commitGit(doslegDir, options, exitCode);
         }
         catch (error) {
             console.error(`Error converting DosLeg dataset:`, error);
         }
         try {
             await convertDatasetScrutins(dataDir, options);
+            const scrutinsDir = path.join(dataDir, SCRUTINS_FOLDER);
+            exitCode = commitGit(scrutinsDir, options, exitCode);
         }
         catch (error) {
             console.error(`Error converting Scrutins dataset:`, error);
@@ -55,6 +74,8 @@ async function convertData() {
     if (enabledDatasets & EnabledDatasets.Questions) {
         try {
             await convertDatasetQuestions(dataDir);
+            const questionsDir = path.join(dataDir, datasets.questions.database);
+            exitCode = commitGit(questionsDir, options, exitCode);
         }
         catch (error) {
             console.error(`Error converting Questions dataset:`, error);
@@ -63,6 +84,8 @@ async function convertData() {
     if (enabledDatasets & EnabledDatasets.Sens) {
         try {
             await convertDatasetSens(dataDir);
+            const sensDir = path.join(dataDir, datasets.sens.database);
+            exitCode = commitGit(sensDir, options, exitCode);
         }
         catch (error) {
             console.error(`Error converting Sens dataset:`, error);
@@ -71,6 +94,7 @@ async function convertData() {
     if (!options["silent"]) {
         console.timeEnd("data transformation time");
     }
+    return exitCode;
 }
 async function convertDatasetAmeli(dataDir, options) {
     const dataset = datasets.ameli;
@@ -284,7 +308,7 @@ async function convertDatasetSens(dataDir) {
     }
 }
 convertData()
-    .then(() => process.exit(0))
+    .then((exitCode) => process.exit(exitCode || 0))
     .catch((error) => {
     console.log(error);
     process.exit(1);

package/lib/scripts/shared/cli_helpers.d.ts

@@ -40,14 +40,31 @@ export declare const keepDirOption: {
     name: string;
     type: BooleanConstructor;
 };
-export declare const commonOptions: ({
+export declare const cloneOption: {
+    alias: string;
+    help: string;
+    name: string;
+    type: StringConstructor;
+};
+export declare const commitOption: {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const remoteOption: {
     alias: string;
-    defaultValue: string[];
     help: string;
     multiple: boolean;
     name: string;
     type: StringConstructor;
-} | {
+};
+export declare const pullOption: {
+    alias: string;
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
+export declare const commonOptions: ({
     defaultOption: boolean;
     help: string;
     name: string;
@@ -60,4 +77,9 @@ export declare const commonOptions: ({
     help: string;
     name: string;
     type: BooleanConstructor;
+} | {
+    alias: string;
+    help: string;
+    name: string;
+    type: StringConstructor;
 })[];

package/lib/scripts/shared/cli_helpers.js

@@ -40,6 +40,30 @@ export const keepDirOption = {
     name: "keepDir",
     type: Boolean,
 };
+export const cloneOption = {
+    alias: "C",
+    help: "clone repositories from given group (or organization) git URL",
+    name: "clone",
+    type: String,
+};
+export const commitOption = {
+    help: "commit clean files",
+    name: "commit",
+    type: Boolean,
+};
+export const remoteOption = {
+    alias: "r",
+    help: "push commit to given remote",
+    multiple: true,
+    name: "remote",
+    type: String,
+};
+export const pullOption = {
+    alias: "p",
+    help: "pull repositories before proceeding",
+    name: "pull",
+    type: Boolean,
+};
 export const commonOptions = [
     categoriesOption,
     dataDirDefaultOption,
@@ -48,4 +72,8 @@ export const commonOptions = [
     verboseOption,
     onlyRecentOption,
     keepDirOption,
+    cloneOption,
+    commitOption,
+    remoteOption,
+    pullOption,
 ];

package/lib/utils/nvs-timecode.js

@@ -3,40 +3,6 @@ const xmlParser = new XMLParser({
     ignoreAttributes: false,
     attributeNamePrefix: "@_",
 });
-function getFirstInterventionChapterId(dataNvs) {
-    const xml = xmlParser.parse(dataNvs);
-    const rootChapters = xml?.data?.chapters?.chapter;
-    if (!rootChapters)
-        return null;
-    const chaptersArray = Array.isArray(rootChapters) ? rootChapters : [rootChapters];
-    let foundId = null;
-    function dfsChapter(chapter) {
-        if (foundId)
-            return;
-        const metas = chapter.metadata ? (Array.isArray(chapter.metadata) ? chapter.metadata : [chapter.metadata]) : [];
-        const isIntervention = metas.some((m) => m?.["@_name"] === "type" && (m?.["@_value"] === "IN" || m?.["@_label"] === "Intervention"));
-        const hasSpeaker = !!chapter.speaker;
-        if (isIntervention && hasSpeaker && chapter["@_id"]) {
-            foundId = String(chapter["@_id"]);
-            return;
-        }
-        const children = chapter.chapter;
-        if (!children)
-            return;
-        const childArray = Array.isArray(children) ? children : [children];
-        for (const child of childArray) {
-            dfsChapter(child);
-            if (foundId)
-                return;
-        }
-    }
-    for (const ch of chaptersArray) {
-        dfsChapter(ch);
-        if (foundId)
-            break;
-    }
-    return foundId;
-}
 function getTimecodeForChapterId(finalPlayerNvs, chapterId) {
     const xml = xmlParser.parse(finalPlayerNvs);
     const synchros = xml?.player?.synchro;
@@ -55,8 +21,19 @@ function getTimecodeForChapterId(finalPlayerNvs, chapterId) {
     return Math.floor(ms / 1000);
 }
 export function getFirstInterventionStartTimecode(dataNvs, finalPlayerNvs) {
-    const firstChapterId = getFirstInterventionChapterId(dataNvs);
+    const firstChapterId = getFirstChapterId(dataNvs);
     if (!firstChapterId)
         return null;
     return getTimecodeForChapterId(finalPlayerNvs, firstChapterId);
 }
+function getFirstChapterId(dataNvs) {
+    const xml = xmlParser.parse(dataNvs);
+    const rootChapters = xml?.data?.chapters?.chapter;
+    if (!rootChapters)
+        return null;
+    const chaptersArray = Array.isArray(rootChapters) ? rootChapters : [rootChapters];
+    const firstChapter = chaptersArray[0];
+    if (!firstChapter || !firstChapter["@_id"])
+        return null;
+    return String(firstChapter["@_id"]);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.20.16",
+  "version": "2.20.18",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",