@tricoteuses/senat 3.1.24 → 3.2.1

package/lib/src/scripts/retrieve_documents.js

@@ -5,7 +5,7 @@ import path from "path";
 import { convertSenatXmlToHtml } from "../server/conversion_textes.js";
 import * as git from "../server/git.js";
 import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, ENRICHED_TEXTE_FOLDER, iterLoadSenatRapportUrls, iterLoadSenatTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, } from "../server/loaders.js";
-import { parseExposeDesMotifs, parseTexte, parseTexteFromFile } from "../parsers/texte.js";
+import { parseDocument } from "../server/parse_document.js";
 import { getSessionsFromStart, UNDEFINED_SESSION } from "../other_types/sessions.js";
 import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers.js";
 import { ensureAndClearDir, fetchWithRetry, isOptionEmptyOrHasValue } from "./shared/util.js";
@@ -134,21 +134,13 @@ export async function processTexte(texteMetadata, originalTextesDir, transformed
             // Skip re-parsing if the XML was not newly downloaded AND the parsed output already exists
             const needsParsing = !result.skipped || !parsedOutputExists;
             if (needsParsing && (result.buffer !== null || (await fs.pathExists(destPath)))) {
-                await parseDocument(texteMetadata.session, transformedTextesDir, destPath, texteMetadata.name, result.buffer, exposeDesMotifsContent, options);
-                let texteXmlContent = null;
-                if (result.buffer === null && (await fs.pathExists(destPath))) {
-                    texteXmlContent = await fs.readFile(destPath, "utf-8");
+                const texteXml = result.buffer !== null ? textDecoder.decode(result.buffer) : await fs.readFile(destPath, "utf-8");
+                await parseDocument(texteMetadata.session, transformedTextesDir, texteMetadata.name, texteXml, exposeDesMotifsContent ? textDecoder.decode(exposeDesMotifsContent) : null, options);
+                try {
+                    await convertSenatXmlToHtml(texteXml, path.join(enrichedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name, `${texteMetadata.name}.html`));
                 }
-                else if (result.buffer !== null) {
-                    texteXmlContent = textDecoder.decode(result.buffer);
-                }
-                if (texteXmlContent !== null) {
-                    try {
-                        await convertSenatXmlToHtml(texteXmlContent, path.join(enrichedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name, `${texteMetadata.name}.html`));
-                    }
-                    catch (error) {
-                        console.error(`Error converting ${texteMetadata.name} to HTML: ${error.message}`);
-                    }
+                catch (error) {
+                    console.error(`Error converting ${texteMetadata.name} to HTML: ${error.message}`);
                 }
             }
             else if (options.verbose) {
@@ -212,37 +204,6 @@ async function processRapports(dataDir, sessions) {
     }
     commitAndPushGit(rapportsDir);
 }
-async function parseDocument(session, transformedTextesDir, textePath, texteName, texteBuffer, exposeDesMotifs = null, options = {}) {
-    if (options.verbose) {
-        console.log(`Parsing texte ${textePath}…`);
-    }
-    let parsedTexte;
-    if (texteBuffer) {
-        const texteXml = textDecoder.decode(texteBuffer);
-        parsedTexte = parseTexte(texteXml);
-    }
-    else {
-        if (!(await fs.pathExists(textePath))) {
-            if (options.verbose) {
-                console.warn(`Skipping parse for missing XML file: ${textePath}`);
-            }
-            return null;
-        }
-        parsedTexte = await parseTexteFromFile(textePath);
-    }
-    if (!parsedTexte)
-        return null;
-    if (exposeDesMotifs) {
-        if (options.verbose) {
-            console.log("Parsing exposé des motifs…");
-        }
-        const exposeDesMotifsHtml = textDecoder.decode(exposeDesMotifs);
-        parsedTexte.expose_motifs = parseExposeDesMotifs(exposeDesMotifsHtml);
-    }
-    const transformedTexteDir = path.join(transformedTextesDir, `${session ?? UNDEFINED_SESSION}`, texteName);
-    await fs.outputJSON(path.join(transformedTexteDir, `${texteName}.json`), parsedTexte, { spaces: 2 });
-    return parsedTexte;
-}
 async function main() {
     const dataDir = assertExistingDirectory(options["dataDir"], "data directory");
     const sessions = getSessionsFromStart((options.fromSession ?? UNDEFINED_SESSION));

package/lib/src/server/parse_document.d.ts

@@ -0,0 +1,5 @@
+import { FlatTexte } from "../other_types/texte.js";
+export type ParseDocumentOptions = {
+    verbose?: boolean;
+};
+export declare function parseDocument(session: number | null | undefined, transformedTextesDir: string, texteName: string, texteXml: string, exposeHtml?: string | null, options?: ParseDocumentOptions): Promise<FlatTexte | null>;

package/lib/src/server/parse_document.js

@@ -0,0 +1,47 @@
+import fs from "fs-extra";
+import { JSDOM } from "jsdom";
+import path from "path";
+import { UNDEFINED_SESSION } from "../other_types/sessions.js";
+import { parseExposeDesMotifs, parseTexte } from "../parsers/texte.js";
+import { parsePlfCredits } from "../parsers/plf/index.js";
+export async function parseDocument(session, transformedTextesDir, texteName, texteXml, exposeHtml = null, options = {}) {
+    if (options.verbose) {
+        console.log(`Parsing texte ${texteName}…`);
+    }
+    let document;
+    try {
+        document = new JSDOM(texteXml, { contentType: "text/xml" }).window.document;
+    }
+    catch (error) {
+        console.error(`Could not parse texte with error ${error.message}`);
+        return null;
+    }
+    const parsedTexte = parseTexte(texteXml, document);
+    if (!parsedTexte)
+        return null;
+    if (exposeHtml) {
+        if (options.verbose) {
+            console.log("Parsing exposé des motifs…");
+        }
+        parsedTexte.expose_motifs = parseExposeDesMotifs(exposeHtml);
+    }
+    const transformedTexteDir = path.join(transformedTextesDir, `${session ?? UNDEFINED_SESSION}`, texteName);
+    await fs.outputJSON(path.join(transformedTexteDir, `${texteName}.json`), parsedTexte, { spaces: 2 });
+    // Parse PLF credits
+    try {
+        const credits = parsePlfCredits(document);
+        if (credits.length > 0) {
+            await fs.outputJSON(path.join(transformedTexteDir, `${texteName}_plf_credits.json`), credits, { spaces: 2 });
+            if (options.verbose) {
+                console.log(`  Parsed ${credits.length} PLF credit section(s) for ${texteName}`);
+            }
+        }
+        else {
+            await fs.remove(path.join(transformedTexteDir, `${texteName}_plf_credits.json`));
+        }
+    }
+    catch {
+        // PLF parsing is best-effort; failures should not block the main text parse
+    }
+    return parsedTexte;
+}

package/lib/tests/parsers/parseDocument.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/parsers/parseDocument.test.js

@@ -0,0 +1,73 @@
+import fs from "fs-extra";
+import os from "os";
+import path from "path";
+import { afterEach, describe, expect, it } from "vitest";
+import { parseDocument } from "../../src/server/parse_document.js";
+import { article, articleAlinea, xmlDoc } from "./texte.test_utils.js";
+import { buildArticle, etatTable3Col, missionRow, totalRow } from "./plf/test_utils.js";
+const tmpDirs = [];
+function makeTmpDir(prefix) {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+    tmpDirs.push(dir);
+    return dir;
+}
+afterEach(() => {
+    for (const dir of tmpDirs.splice(0)) {
+        fs.rmSync(dir, { recursive: true, force: true });
+    }
+});
+describe("parseDocument", () => {
+    const session = 2024;
+    const texteName = "pjl-test";
+    const body = article("art_1er", articleAlinea("al_1__art_1er", "<p>Un article de test.</p>"));
+    it("writes the parsed FlatTexte as JSON in the session/name folder and returns it", async () => {
+        const transformedDir = makeTmpDir("senat-parse-doc-");
+        const texteXml = xmlDoc(body);
+        const result = await parseDocument(session, transformedDir, texteName, texteXml);
+        const jsonPath = path.join(transformedDir, `${session}`, texteName, `${texteName}.json`);
+        expect(fs.existsSync(jsonPath)).toBe(true);
+        const written = fs.readJsonSync(jsonPath);
+        expect(written.titre).toBe("Projet de loi de finances pour 2026");
+        expect(written.divisions).toHaveLength(1);
+        expect(result?.titre).toBe(written.titre);
+    });
+    it("parses and injects the exposé des motifs when exposeHtml is provided", async () => {
+        const transformedDir = makeTmpDir("senat-parse-expose-");
+        const exposeHtml = "<section><p>EXPOSÉ DES MOTIFS</p><p>Voici les motifs du texte.</p></section>";
+        const result = await parseDocument(session, transformedDir, texteName, xmlDoc(body), exposeHtml);
+        expect(result?.expose_motifs?.text).toContain("Voici les motifs du texte.");
+    });
+    it("does not write a PLF credits file when the texte has no credits", async () => {
+        const transformedDir = makeTmpDir("senat-parse-nocredits-");
+        await parseDocument(session, transformedDir, texteName, xmlDoc(body));
+        expect(fs.existsSync(path.join(transformedDir, `${session}`, texteName, `${texteName}_plf_credits.json`))).toBe(false);
+    });
+    it("writes a PLF credits file when the texte contains an 'etat' article", async () => {
+        const transformedDir = makeTmpDir("senat-parse-credits-");
+        const plfBody = buildArticle("etat_B", "État B", etatTable3Col("B", missionRow("Action extérieure de l'État", "3 449 452 613", "3 454 425 325") +
+            totalRow("620 249 232 626", "593 890 071 649")));
+        await parseDocument(session, transformedDir, texteName, xmlDoc(plfBody));
+        const creditsPath = path.join(transformedDir, `${session}`, texteName, `${texteName}_plf_credits.json`);
+        expect(fs.existsSync(creditsPath)).toBe(true);
+        const credits = fs.readJsonSync(creditsPath);
+        expect(credits).toHaveLength(1);
+        expect(credits[0].libelle).toBe("Budget général");
+    });
+    it("removes a stale PLF credits file when a re-parse produces no credits", async () => {
+        const transformedDir = makeTmpDir("senat-parse-stalecredits-");
+        const creditsPath = path.join(transformedDir, `${session}`, texteName, `${texteName}_plf_credits.json`);
+        const nbCreditsDir = path.join(transformedDir, `${session}`, texteName);
+        const plfBody = buildArticle("etat_B", "État B", etatTable3Col("B", missionRow("Action extérieure de l'État", "3 449 452 613", "3 454 425 325") +
+            totalRow("620 249 232 626", "593 890 071 649")));
+        await parseDocument(session, transformedDir, texteName, xmlDoc(plfBody));
+        expect(fs.existsSync(creditsPath)).toBe(true);
+        await parseDocument(session, transformedDir, texteName, xmlDoc(body));
+        expect(fs.existsSync(creditsPath)).toBe(false);
+        expect(fs.existsSync(nbCreditsDir)).toBe(true);
+    });
+    it("works for a session-less texte using the undefined session", async () => {
+        const transformedDir = makeTmpDir("senat-parse-nosession-");
+        await parseDocument(null, transformedDir, texteName, xmlDoc(body));
+        expect(fs.existsSync(path.join(transformedDir, "0", texteName, `${texteName}.json`))).toBe(true);
+    });
+});

package/lib/tests/parsers/plf/credits.integration.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/tests/parsers/plf/credits.integration.test.js

@@ -0,0 +1,118 @@
+import { beforeAll, describe, expect, it } from "vitest";
+import { parsePlfCredits } from "../../../src/parsers/plf/index.js";
+import { expectValidCreditSections, loadPlfFixture, sectionByLibelle, sectionLibelles, } from "./integration.test_utils.js";
+// Real Sénat AKN XML, extracted from the actual senat-data dumps (only the
+// État B, C and D articles, trimmed to a representative row sample). Regenerate/
+// extend with:
+//   npx tsx src/scripts/extract_plf_fixtures.ts --src <senat .xml> --name <name> --only-parsed
+const fixtures = {
+    plf2026Depot: { name: "plf2026-depot", label: "PLF 2026 (pjl25-138) — dépôt" },
+    plf2026Avance: { name: "plf2026-avance", label: "PLF 2026 (pjl25-308) — stade avancé" },
+    plf2025: { name: "plf2025", label: "PLF 2025 (pjl24-143) — dépôt" },
+};
+describe("parsePlfCredits — documents réels (integration)", () => {
+    let parsed;
+    beforeAll(() => {
+        parsed = Object.fromEntries(Object.values(fixtures).map(({ name }) => [
+            name,
+            parsePlfCredits(loadPlfFixture(name)),
+        ]));
+    }, 30_000);
+    const loader = (name) => parsed[name];
+    describe("invariants", () => {
+        it.each(Object.values(fixtures))("$label — produit des sections cohérentes", ({ name }) => {
+            const sections = loader(name);
+            expect(sections.length).toBeGreaterThan(0);
+            expectValidCreditSections(sections);
+        });
+    });
+    describe("section count et libellés", () => {
+        it.each([
+            {
+                fixture: fixtures.plf2026Depot,
+                expected: [
+                    "Budget général",
+                    "Budgets annexes",
+                    "Comptes d'affectation spéciale",
+                    "Comptes de concours financiers",
+                ],
+            },
+            {
+                fixture: fixtures.plf2025,
+                expected: [
+                    "Budget général",
+                    "Budgets annexes",
+                    "Comptes d'affectation spéciale",
+                    "Comptes de concours financiers",
+                ],
+            },
+            {
+                // L'État C est « (Conforme) » à ce stade (aucun tableau de crédits)
+                fixture: fixtures.plf2026Avance,
+                expected: [
+                    "Budget général",
+                    "Comptes d'affectation spéciale",
+                    "Comptes de concours financiers",
+                ],
+            },
+        ])("$fixture.label — expose les sections attendues", ({ fixture, expected }) => {
+            expect(sectionLibelles(loader(fixture.name))).toEqual(expected);
+        });
+    });
+    describe("totaux", () => {
+        it.each([
+            { fixture: fixtures.plf2026Depot, section: "Budget général", total: 613_009_460_112 },
+            { fixture: fixtures.plf2026Depot, section: "Budgets annexes", total: 2_691_230_585 },
+            {
+                fixture: fixtures.plf2026Depot,
+                section: "Comptes d'affectation spéciale",
+                total: 77_423_204_686,
+            },
+            {
+                fixture: fixtures.plf2026Depot,
+                section: "Comptes de concours financiers",
+                total: 149_661_603_151,
+            },
+        ])("$fixture.label / $section — total AE correct", ({ fixture, section, total }) => {
+            expect(sectionByLibelle(loader(fixture.name), section)?.total?.autorisationEngagement).toBe(total);
+        });
+    });
+    describe("missions", () => {
+        it.each([
+            {
+                fixture: fixtures.plf2026Depot,
+                section: "Budget général",
+                mission: "Action extérieure de l’État",
+                ae: 3_452_452_613,
+                programmes: 3,
+            },
+            {
+                fixture: fixtures.plf2026Avance,
+                section: "Budget général",
+                mission: "Action extérieure de l’État",
+                ae: 3_449_452_613,
+                programmes: 3,
+            },
+            {
+                fixture: fixtures.plf2025,
+                section: "Budget général",
+                mission: "Action extérieure de l’État",
+                ae: 3_527_537_701,
+                programmes: 3,
+            },
+            {
+                fixture: fixtures.plf2026Depot,
+                section: "Comptes d'affectation spéciale",
+                mission: "Contrôle de la circulation et du stationnement routiers",
+                ae: 1_928_700_107,
+            },
+        ])("$fixture.label / $mission — première mission et son AE", ({ fixture, section, mission, ae, programmes }) => {
+            const sections = loader(fixture.name);
+            const first = sectionByLibelle(sections, section)?.missions[0];
+            expect(first?.libelle).toBe(mission);
+            expect(first?.autorisationEngagement).toBe(ae);
+            if (programmes !== undefined)
+                expect(first?.programmes).toHaveLength(programmes);
+        });
+    });
+});

package/lib/tests/parsers/plf/credits.test.d.ts

@@ -0,0 +1 @@
+export {};