@tricoteuses/senat 3.1.23 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/src/index.d.ts +2 -0
- package/lib/src/other_types/plf.d.ts +21 -0
- package/lib/src/other_types/plf.js +1 -0
- package/lib/src/parsers/collaborateurs.d.ts +24 -24
- package/lib/src/parsers/collaborateurs.js +76 -76
- package/lib/src/parsers/plf/credits.d.ts +2 -0
- package/lib/src/parsers/plf/credits.js +171 -0
- package/lib/src/parsers/plf/index.d.ts +1 -0
- package/lib/src/parsers/plf/index.js +1 -0
- package/lib/src/parsers/plf/parser_utils.d.ts +18 -0
- package/lib/src/parsers/plf/parser_utils.js +172 -0
- package/lib/src/parsers/texte.d.ts +1 -3
- package/lib/src/parsers/texte.js +63 -30
- package/lib/src/rich_types/sens.d.ts +5 -0
- package/lib/src/scripts/data-download.js +1 -0
- package/lib/src/scripts/extract_plf_fixtures.d.ts +1 -0
- package/lib/src/scripts/extract_plf_fixtures.js +101 -0
- package/lib/src/scripts/retrieve_collaborateurs.js +93 -93
- package/lib/src/scripts/retrieve_documents.js +7 -46
- package/lib/src/server/parse_document.d.ts +5 -0
- package/lib/src/server/parse_document.js +47 -0
- package/lib/tests/collaborateurs.test.js +52 -52
- package/lib/tests/parsers/parseDocument.test.d.ts +1 -0
- package/lib/tests/parsers/parseDocument.test.js +73 -0
- package/lib/tests/parsers/plf/credits.integration.test.d.ts +1 -0
- package/lib/tests/parsers/plf/credits.integration.test.js +118 -0
- package/lib/tests/parsers/plf/credits.test.d.ts +1 -0
- package/lib/tests/parsers/plf/credits.test.js +467 -0
- package/lib/tests/parsers/plf/integration.test_utils.d.ts +8 -0
- package/lib/tests/parsers/plf/integration.test_utils.js +73 -0
- package/lib/tests/parsers/plf/test_utils.d.ts +20 -0
- package/lib/tests/parsers/plf/test_utils.js +85 -0
- package/lib/tests/parsers/texte.test.d.ts +1 -0
- package/lib/tests/parsers/texte.test.js +71 -0
- package/lib/tests/parsers/texte.test_utils.d.ts +15 -0
- package/lib/tests/parsers/texte.test_utils.js +73 -0
- package/lib/tests/plf/credits.test.d.ts +1 -0
- package/lib/tests/plf/credits.test.js +458 -0
- package/lib/tests/plf/test_utils.d.ts +20 -0
- package/lib/tests/plf/test_utils.js +85 -0
- package/package.json +5 -1
|
@@ -1,51 +1,51 @@
|
|
|
1
1
|
import fs from "fs-extra";
|
|
2
2
|
import path from "path";
|
|
3
3
|
import { getDocumentProxy } from "unpdf";
|
|
4
|
-
import {
|
|
4
|
+
import { buildCollaboratorRows, extractEditionDate, normalizeForComparison, } from "../parsers/collaborateurs.js";
|
|
5
5
|
import { assertExistingDirectory } from "./shared/cli_helpers.js";
|
|
6
6
|
import { iterFilePaths } from "../server/loaders.js";
|
|
7
|
-
const
|
|
7
|
+
const DEFAULT_PDF_URL = "https://www.senat.fr/pubagas/liste_senateurs_collaborateurs.pdf";
|
|
8
8
|
const USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36";
|
|
9
|
-
async function
|
|
9
|
+
async function downloadPdf(url) {
|
|
10
10
|
const cookies = new Map();
|
|
11
|
-
let
|
|
11
|
+
let current = url;
|
|
12
12
|
for (let i = 0; i < 8; i += 1) {
|
|
13
13
|
const headers = { "User-Agent": USER_AGENT };
|
|
14
14
|
if (cookies.size > 0) {
|
|
15
15
|
headers["Cookie"] = [...cookies].map(([k, v]) => `${k}=${v}`).join("; ");
|
|
16
16
|
}
|
|
17
|
-
const
|
|
18
|
-
for (const
|
|
19
|
-
const [
|
|
20
|
-
const idx =
|
|
17
|
+
const response = await fetch(current, { redirect: "manual", headers });
|
|
18
|
+
for (const raw of response.headers.getSetCookie?.() ?? []) {
|
|
19
|
+
const [pair] = raw.split(";");
|
|
20
|
+
const idx = pair.indexOf("=");
|
|
21
21
|
if (idx > 0)
|
|
22
|
-
cookies.set(
|
|
22
|
+
cookies.set(pair.slice(0, idx).trim(), pair.slice(idx + 1).trim());
|
|
23
23
|
}
|
|
24
|
-
if (
|
|
25
|
-
const location =
|
|
24
|
+
if (response.status >= 300 && response.status < 400) {
|
|
25
|
+
const location = response.headers.get("location");
|
|
26
26
|
if (!location)
|
|
27
|
-
throw new Error(`
|
|
28
|
-
|
|
27
|
+
throw new Error(`Redirect without Location header (HTTP ${response.status})`);
|
|
28
|
+
current = new URL(location, current).toString();
|
|
29
29
|
continue;
|
|
30
30
|
}
|
|
31
|
-
if (!
|
|
32
|
-
throw new Error(`
|
|
33
|
-
const buffer = Buffer.from(await
|
|
31
|
+
if (!response.ok)
|
|
32
|
+
throw new Error(`PDF download failed: HTTP ${response.status}`);
|
|
33
|
+
const buffer = Buffer.from(await response.arrayBuffer());
|
|
34
34
|
if (buffer.subarray(0, 5).toString("latin1") !== "%PDF-") {
|
|
35
|
-
throw new Error("
|
|
35
|
+
throw new Error("Downloaded content is not a PDF");
|
|
36
36
|
}
|
|
37
37
|
return buffer;
|
|
38
38
|
}
|
|
39
|
-
throw new Error("
|
|
39
|
+
throw new Error("Too many redirects during HR PDF download");
|
|
40
40
|
}
|
|
41
|
-
async function
|
|
41
|
+
async function extractPdfItems(buffer) {
|
|
42
42
|
const pdf = await getDocumentProxy(new Uint8Array(buffer));
|
|
43
43
|
const items = [];
|
|
44
44
|
const strings = [];
|
|
45
45
|
for (let p = 1; p <= pdf.numPages; p += 1) {
|
|
46
46
|
const page = await pdf.getPage(p);
|
|
47
|
-
const
|
|
48
|
-
for (const item of
|
|
47
|
+
const content = await page.getTextContent();
|
|
48
|
+
for (const item of content.items) {
|
|
49
49
|
if (typeof item.str !== "string" || !item.str.trim() || !item.transform)
|
|
50
50
|
continue;
|
|
51
51
|
items.push({ str: item.str, x: item.transform[4], y: item.transform[5], page: p });
|
|
@@ -54,110 +54,110 @@ async function extraireItemsPdf(buffer) {
|
|
|
54
54
|
}
|
|
55
55
|
return { items, strings };
|
|
56
56
|
}
|
|
57
|
-
function
|
|
58
|
-
const
|
|
57
|
+
function loadSenators(dataDir) {
|
|
58
|
+
const senatorsDir = path.join(dataDir, "sens", "senateurs");
|
|
59
59
|
const files = new Map();
|
|
60
|
-
const
|
|
61
|
-
if (!fs.existsSync(
|
|
62
|
-
return { files,
|
|
63
|
-
for (const filePath of iterFilePaths(
|
|
60
|
+
const indexByName = new Map();
|
|
61
|
+
if (!fs.existsSync(senatorsDir))
|
|
62
|
+
return { files, indexByName };
|
|
63
|
+
for (const filePath of iterFilePaths(senatorsDir)) {
|
|
64
64
|
const senator = JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
|
65
65
|
files.set(senator.matricule, senator);
|
|
66
|
-
const
|
|
67
|
-
const
|
|
68
|
-
if (
|
|
69
|
-
|
|
66
|
+
const key = normalizeForComparison(senator.nom_usuel);
|
|
67
|
+
const list = indexByName.get(key);
|
|
68
|
+
if (list)
|
|
69
|
+
list.push(senator);
|
|
70
70
|
else
|
|
71
|
-
|
|
71
|
+
indexByName.set(key, [senator]);
|
|
72
72
|
}
|
|
73
|
-
return { files,
|
|
73
|
+
return { files, indexByName };
|
|
74
74
|
}
|
|
75
75
|
async function main() {
|
|
76
76
|
const dataDir = process.argv[2];
|
|
77
77
|
assertExistingDirectory(dataDir, "dataDir");
|
|
78
|
-
// 1.
|
|
79
|
-
console.log(`
|
|
80
|
-
const buffer = await
|
|
81
|
-
const { items, strings } = await
|
|
82
|
-
const
|
|
83
|
-
const
|
|
84
|
-
console.log(`PDF
|
|
85
|
-
if (
|
|
86
|
-
throw new Error("
|
|
78
|
+
// 1. Download + parse PDF
|
|
79
|
+
console.log(`Downloading HR PDF: ${DEFAULT_PDF_URL}`);
|
|
80
|
+
const buffer = await downloadPdf(DEFAULT_PDF_URL);
|
|
81
|
+
const { items, strings } = await extractPdfItems(buffer);
|
|
82
|
+
const rows = buildCollaboratorRows(items);
|
|
83
|
+
const editionDate = extractEditionDate(strings);
|
|
84
|
+
console.log(`PDF parsed: ${rows.length} collaborator rows, edition ${editionDate?.toISOString().slice(0, 10) ?? "unknown"}`);
|
|
85
|
+
if (rows.length === 0) {
|
|
86
|
+
throw new Error("No collaborator rows extracted from PDF: unexpected format, aborting without changes.");
|
|
87
87
|
}
|
|
88
|
-
// 2.
|
|
89
|
-
const {
|
|
90
|
-
if (
|
|
91
|
-
throw new Error("
|
|
88
|
+
// 2. Load existing senator files
|
|
89
|
+
const { indexByName } = loadSenators(dataDir);
|
|
90
|
+
if (indexByName.size === 0) {
|
|
91
|
+
throw new Error("No senator files found. Run data:download first to generate the files.");
|
|
92
92
|
}
|
|
93
|
-
// 3.
|
|
94
|
-
let
|
|
95
|
-
let
|
|
96
|
-
//
|
|
97
|
-
const
|
|
98
|
-
const
|
|
99
|
-
for (const
|
|
100
|
-
const
|
|
101
|
-
const
|
|
102
|
-
if (!
|
|
103
|
-
|
|
104
|
-
|
|
93
|
+
// 3. Resolve PDF senators → files
|
|
94
|
+
let nbEnriched = 0;
|
|
95
|
+
let nbUnresolved = 0;
|
|
96
|
+
// Group collaborators by senator matricule
|
|
97
|
+
const collabsByMatricule = new Map();
|
|
98
|
+
const unresolvedNames = new Set();
|
|
99
|
+
for (const row of rows) {
|
|
100
|
+
const nameKey = normalizeForComparison(row.senateur.nom);
|
|
101
|
+
const candidates = indexByName.get(nameKey);
|
|
102
|
+
if (!candidates || candidates.length === 0) {
|
|
103
|
+
unresolvedNames.add(`${row.senateur.civilite} ${row.senateur.nom} ${row.senateur.prenom}`.trim());
|
|
104
|
+
nbUnresolved += 1;
|
|
105
105
|
continue;
|
|
106
106
|
}
|
|
107
|
-
//
|
|
108
|
-
const
|
|
109
|
-
const
|
|
107
|
+
// Disambiguation by first name prefix
|
|
108
|
+
const targetFirstName = normalizeForComparison(row.senateur.prenom);
|
|
109
|
+
const byFirstName = candidates.filter((c) => normalizeForComparison(c.prenom_usuel).startsWith(targetFirstName));
|
|
110
110
|
let senator = null;
|
|
111
|
-
if (
|
|
112
|
-
senator =
|
|
111
|
+
if (candidates.length === 1) {
|
|
112
|
+
senator = candidates[0];
|
|
113
113
|
}
|
|
114
|
-
else if (
|
|
115
|
-
senator =
|
|
114
|
+
else if (byFirstName.length === 1) {
|
|
115
|
+
senator = byFirstName[0];
|
|
116
116
|
}
|
|
117
117
|
else {
|
|
118
|
-
|
|
119
|
-
|
|
118
|
+
unresolvedNames.add(`${row.senateur.civilite} ${row.senateur.nom} ${row.senateur.prenom}`.trim());
|
|
119
|
+
nbUnresolved += 1;
|
|
120
120
|
continue;
|
|
121
121
|
}
|
|
122
|
-
let collabs =
|
|
122
|
+
let collabs = collabsByMatricule.get(senator.matricule);
|
|
123
123
|
if (!collabs) {
|
|
124
124
|
collabs = [];
|
|
125
|
-
|
|
125
|
+
collabsByMatricule.set(senator.matricule, collabs);
|
|
126
126
|
}
|
|
127
|
-
//
|
|
128
|
-
const
|
|
129
|
-
const
|
|
130
|
-
if (!collabs.some((c) =>
|
|
127
|
+
// Deduplication
|
|
128
|
+
const nName = normalizeForComparison(row.collaborateur.nom);
|
|
129
|
+
const nFirstName = normalizeForComparison(row.collaborateur.prenom);
|
|
130
|
+
if (!collabs.some((c) => normalizeForComparison(c.nom) === nName && normalizeForComparison(c.prenom) === nFirstName)) {
|
|
131
131
|
collabs.push({
|
|
132
|
-
civilite:
|
|
133
|
-
nom:
|
|
134
|
-
prenom:
|
|
132
|
+
civilite: row.collaborateur.civilite,
|
|
133
|
+
nom: row.collaborateur.nom,
|
|
134
|
+
prenom: row.collaborateur.prenom,
|
|
135
135
|
});
|
|
136
136
|
}
|
|
137
137
|
}
|
|
138
|
-
// 4.
|
|
139
|
-
for (const [matricule, collaborateurs] of
|
|
140
|
-
const
|
|
141
|
-
if (!fs.existsSync(
|
|
138
|
+
// 4. Write collaborators into senator files
|
|
139
|
+
for (const [matricule, collaborateurs] of collabsByMatricule) {
|
|
140
|
+
const filePath = path.join(dataDir, "sens", "senateurs", `${matricule}.json`);
|
|
141
|
+
if (!fs.existsSync(filePath))
|
|
142
142
|
continue;
|
|
143
|
-
const senator = JSON.parse(fs.readFileSync(
|
|
143
|
+
const senator = JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
|
144
144
|
senator.collaborateurs = collaborateurs;
|
|
145
|
-
fs.writeFileSync(
|
|
146
|
-
|
|
145
|
+
fs.writeFileSync(filePath, JSON.stringify(senator, null, 2) + "\n");
|
|
146
|
+
nbEnriched += 1;
|
|
147
147
|
}
|
|
148
|
-
// 5.
|
|
149
|
-
const
|
|
150
|
-
const { files:
|
|
151
|
-
for (const [matricule, senator] of
|
|
152
|
-
if (!
|
|
148
|
+
// 5. Clean collaborators from senators absent from the PDF
|
|
149
|
+
const enrichedMatricules = new Set(collabsByMatricule.keys());
|
|
150
|
+
const { files: allFiles } = loadSenators(dataDir);
|
|
151
|
+
for (const [matricule, senator] of allFiles) {
|
|
152
|
+
if (!enrichedMatricules.has(matricule) && senator.collaborateurs) {
|
|
153
153
|
delete senator.collaborateurs;
|
|
154
|
-
const
|
|
155
|
-
fs.writeFileSync(
|
|
154
|
+
const filePath = path.join(dataDir, "sens", "senateurs", `${matricule}.json`);
|
|
155
|
+
fs.writeFileSync(filePath, JSON.stringify(senator, null, 2) + "\n");
|
|
156
156
|
}
|
|
157
157
|
}
|
|
158
|
-
console.log(`${
|
|
159
|
-
if (
|
|
160
|
-
console.warn(`${
|
|
158
|
+
console.log(`${nbEnriched} senator(s) enriched with their collaborators`);
|
|
159
|
+
if (unresolvedNames.size > 0) {
|
|
160
|
+
console.warn(`${nbUnresolved} PDF senator(s) could not be matched: ${[...unresolvedNames].slice(0, 10).join(", ")}${unresolvedNames.size > 10 ? "…" : ""}`);
|
|
161
161
|
}
|
|
162
162
|
}
|
|
163
163
|
main().catch((error) => {
|
|
@@ -5,7 +5,7 @@ import path from "path";
|
|
|
5
5
|
import { convertSenatXmlToHtml } from "../server/conversion_textes.js";
|
|
6
6
|
import * as git from "../server/git.js";
|
|
7
7
|
import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, ENRICHED_TEXTE_FOLDER, iterLoadSenatRapportUrls, iterLoadSenatTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, } from "../server/loaders.js";
|
|
8
|
-
import {
|
|
8
|
+
import { parseDocument } from "../server/parse_document.js";
|
|
9
9
|
import { getSessionsFromStart, UNDEFINED_SESSION } from "../other_types/sessions.js";
|
|
10
10
|
import { assertExistingDirectory, commonOptions } from "./shared/cli_helpers.js";
|
|
11
11
|
import { ensureAndClearDir, fetchWithRetry, isOptionEmptyOrHasValue } from "./shared/util.js";
|
|
@@ -134,21 +134,13 @@ export async function processTexte(texteMetadata, originalTextesDir, transformed
|
|
|
134
134
|
// Skip re-parsing if the XML was not newly downloaded AND the parsed output already exists
|
|
135
135
|
const needsParsing = !result.skipped || !parsedOutputExists;
|
|
136
136
|
if (needsParsing && (result.buffer !== null || (await fs.pathExists(destPath)))) {
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
137
|
+
const texteXml = result.buffer !== null ? textDecoder.decode(result.buffer) : await fs.readFile(destPath, "utf-8");
|
|
138
|
+
await parseDocument(texteMetadata.session, transformedTextesDir, texteMetadata.name, texteXml, exposeDesMotifsContent ? textDecoder.decode(exposeDesMotifsContent) : null, options);
|
|
139
|
+
try {
|
|
140
|
+
await convertSenatXmlToHtml(texteXml, path.join(enrichedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name, `${texteMetadata.name}.html`));
|
|
141
141
|
}
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
}
|
|
145
|
-
if (texteXmlContent !== null) {
|
|
146
|
-
try {
|
|
147
|
-
await convertSenatXmlToHtml(texteXmlContent, path.join(enrichedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name, `${texteMetadata.name}.html`));
|
|
148
|
-
}
|
|
149
|
-
catch (error) {
|
|
150
|
-
console.error(`Error converting ${texteMetadata.name} to HTML: ${error.message}`);
|
|
151
|
-
}
|
|
142
|
+
catch (error) {
|
|
143
|
+
console.error(`Error converting ${texteMetadata.name} to HTML: ${error.message}`);
|
|
152
144
|
}
|
|
153
145
|
}
|
|
154
146
|
else if (options.verbose) {
|
|
@@ -212,37 +204,6 @@ async function processRapports(dataDir, sessions) {
|
|
|
212
204
|
}
|
|
213
205
|
commitAndPushGit(rapportsDir);
|
|
214
206
|
}
|
|
215
|
-
async function parseDocument(session, transformedTextesDir, textePath, texteName, texteBuffer, exposeDesMotifs = null, options = {}) {
|
|
216
|
-
if (options.verbose) {
|
|
217
|
-
console.log(`Parsing texte ${textePath}…`);
|
|
218
|
-
}
|
|
219
|
-
let parsedTexte;
|
|
220
|
-
if (texteBuffer) {
|
|
221
|
-
const texteXml = textDecoder.decode(texteBuffer);
|
|
222
|
-
parsedTexte = parseTexte(texteXml);
|
|
223
|
-
}
|
|
224
|
-
else {
|
|
225
|
-
if (!(await fs.pathExists(textePath))) {
|
|
226
|
-
if (options.verbose) {
|
|
227
|
-
console.warn(`Skipping parse for missing XML file: ${textePath}`);
|
|
228
|
-
}
|
|
229
|
-
return null;
|
|
230
|
-
}
|
|
231
|
-
parsedTexte = await parseTexteFromFile(textePath);
|
|
232
|
-
}
|
|
233
|
-
if (!parsedTexte)
|
|
234
|
-
return null;
|
|
235
|
-
if (exposeDesMotifs) {
|
|
236
|
-
if (options.verbose) {
|
|
237
|
-
console.log("Parsing exposé des motifs…");
|
|
238
|
-
}
|
|
239
|
-
const exposeDesMotifsHtml = textDecoder.decode(exposeDesMotifs);
|
|
240
|
-
parsedTexte.expose_motifs = parseExposeDesMotifs(exposeDesMotifsHtml);
|
|
241
|
-
}
|
|
242
|
-
const transformedTexteDir = path.join(transformedTextesDir, `${session ?? UNDEFINED_SESSION}`, texteName);
|
|
243
|
-
await fs.outputJSON(path.join(transformedTexteDir, `${texteName}.json`), parsedTexte, { spaces: 2 });
|
|
244
|
-
return parsedTexte;
|
|
245
|
-
}
|
|
246
207
|
async function main() {
|
|
247
208
|
const dataDir = assertExistingDirectory(options["dataDir"], "data directory");
|
|
248
209
|
const sessions = getSessionsFromStart((options.fromSession ?? UNDEFINED_SESSION));
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { FlatTexte } from "../other_types/texte.js";
|
|
2
|
+
export type ParseDocumentOptions = {
|
|
3
|
+
verbose?: boolean;
|
|
4
|
+
};
|
|
5
|
+
export declare function parseDocument(session: number | null | undefined, transformedTextesDir: string, texteName: string, texteXml: string, exposeHtml?: string | null, options?: ParseDocumentOptions): Promise<FlatTexte | null>;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import fs from "fs-extra";
|
|
2
|
+
import { JSDOM } from "jsdom";
|
|
3
|
+
import path from "path";
|
|
4
|
+
import { UNDEFINED_SESSION } from "../other_types/sessions.js";
|
|
5
|
+
import { parseExposeDesMotifs, parseTexte } from "../parsers/texte.js";
|
|
6
|
+
import { parsePlfCredits } from "../parsers/plf/index.js";
|
|
7
|
+
export async function parseDocument(session, transformedTextesDir, texteName, texteXml, exposeHtml = null, options = {}) {
|
|
8
|
+
if (options.verbose) {
|
|
9
|
+
console.log(`Parsing texte ${texteName}…`);
|
|
10
|
+
}
|
|
11
|
+
let document;
|
|
12
|
+
try {
|
|
13
|
+
document = new JSDOM(texteXml, { contentType: "text/xml" }).window.document;
|
|
14
|
+
}
|
|
15
|
+
catch (error) {
|
|
16
|
+
console.error(`Could not parse texte with error ${error.message}`);
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
const parsedTexte = parseTexte(texteXml, document);
|
|
20
|
+
if (!parsedTexte)
|
|
21
|
+
return null;
|
|
22
|
+
if (exposeHtml) {
|
|
23
|
+
if (options.verbose) {
|
|
24
|
+
console.log("Parsing exposé des motifs…");
|
|
25
|
+
}
|
|
26
|
+
parsedTexte.expose_motifs = parseExposeDesMotifs(exposeHtml);
|
|
27
|
+
}
|
|
28
|
+
const transformedTexteDir = path.join(transformedTextesDir, `${session ?? UNDEFINED_SESSION}`, texteName);
|
|
29
|
+
await fs.outputJSON(path.join(transformedTexteDir, `${texteName}.json`), parsedTexte, { spaces: 2 });
|
|
30
|
+
// Parse PLF credits
|
|
31
|
+
try {
|
|
32
|
+
const credits = parsePlfCredits(document);
|
|
33
|
+
if (credits.length > 0) {
|
|
34
|
+
await fs.outputJSON(path.join(transformedTexteDir, `${texteName}_plf_credits.json`), credits, { spaces: 2 });
|
|
35
|
+
if (options.verbose) {
|
|
36
|
+
console.log(` Parsed ${credits.length} PLF credit section(s) for ${texteName}`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
else {
|
|
40
|
+
await fs.remove(path.join(transformedTexteDir, `${texteName}_plf_credits.json`));
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
// PLF parsing is best-effort; failures should not block the main text parse
|
|
45
|
+
}
|
|
46
|
+
return parsedTexte;
|
|
47
|
+
}
|
|
@@ -1,84 +1,84 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
|
-
import {
|
|
3
|
-
//
|
|
4
|
-
const
|
|
5
|
-
const
|
|
6
|
-
/**
|
|
7
|
-
function pageItems(
|
|
2
|
+
import { buildCollaboratorRows, extractEditionDate, normalizeForComparison, parsePersonCell, } from "../src/parsers/collaborateurs.js";
|
|
3
|
+
// X bands measured on the real PDF: employer ≈ 147, collaborator ≈ 285.
|
|
4
|
+
const X_EMPLOYER = 147;
|
|
5
|
+
const X_COLLABORATOR = 285;
|
|
6
|
+
/** Builds a synthetic page: for each row, [employer?, collaborator]. */
|
|
7
|
+
function pageItems(rows, page = 1) {
|
|
8
8
|
const items = [];
|
|
9
9
|
let y = 700;
|
|
10
|
-
for (const [
|
|
11
|
-
if (
|
|
12
|
-
items.push({ str:
|
|
13
|
-
items.push({ str:
|
|
10
|
+
for (const [employer, collaborator] of rows) {
|
|
11
|
+
if (employer)
|
|
12
|
+
items.push({ str: employer, x: X_EMPLOYER, y, page });
|
|
13
|
+
items.push({ str: collaborator, x: X_COLLABORATOR, y, page });
|
|
14
14
|
y -= 15;
|
|
15
15
|
}
|
|
16
16
|
return items;
|
|
17
17
|
}
|
|
18
|
-
describe("
|
|
19
|
-
it("
|
|
20
|
-
expect(
|
|
21
|
-
expect(
|
|
18
|
+
describe("parsePersonCell", () => {
|
|
19
|
+
it("splits civility, last name and first name", () => {
|
|
20
|
+
expect(parsePersonCell("M. CHAREF Dahmane")).toEqual({ civilite: "M.", nom: "CHAREF", prenom: "Dahmane" });
|
|
21
|
+
expect(parsePersonCell("Mme OLIVIER Béatrice")).toEqual({
|
|
22
22
|
civilite: "Mme",
|
|
23
23
|
nom: "OLIVIER",
|
|
24
24
|
prenom: "Béatrice",
|
|
25
25
|
});
|
|
26
26
|
});
|
|
27
|
-
it("
|
|
28
|
-
expect(
|
|
27
|
+
it("handles compound names (multiple uppercase tokens)", () => {
|
|
28
|
+
expect(parsePersonCell("Mme MAZENOT CHAPPUY Sophie")).toEqual({
|
|
29
29
|
civilite: "Mme",
|
|
30
30
|
nom: "MAZENOT CHAPPUY",
|
|
31
31
|
prenom: "Sophie",
|
|
32
32
|
});
|
|
33
|
-
expect(
|
|
33
|
+
expect(parsePersonCell("M. DI FOLCO Camille")).toEqual({ civilite: "M.", nom: "DI FOLCO", prenom: "Camille" });
|
|
34
34
|
});
|
|
35
|
-
it("
|
|
36
|
-
expect(
|
|
37
|
-
expect(
|
|
35
|
+
it("handles nobiliary particles at the start of the name (de, de la)", () => {
|
|
36
|
+
expect(parsePersonCell("Mme de CIDRAC Marta")).toEqual({ civilite: "Mme", nom: "de CIDRAC", prenom: "Marta" });
|
|
37
|
+
expect(parsePersonCell("Mme de LA GONTRIE Marie-Pierre")).toEqual({
|
|
38
38
|
civilite: "Mme",
|
|
39
39
|
nom: "de LA GONTRIE",
|
|
40
40
|
prenom: "Marie-Pierre",
|
|
41
41
|
});
|
|
42
|
-
expect(
|
|
42
|
+
expect(parsePersonCell("M. de LEGGE Dominique")).toEqual({
|
|
43
43
|
civilite: "M.",
|
|
44
44
|
nom: "de LEGGE",
|
|
45
45
|
prenom: "Dominique",
|
|
46
46
|
});
|
|
47
|
-
expect(
|
|
47
|
+
expect(parsePersonCell("Mme APOURCEAU-POLY Cathy")).toEqual({
|
|
48
48
|
civilite: "Mme",
|
|
49
49
|
nom: "APOURCEAU-POLY",
|
|
50
50
|
prenom: "Cathy",
|
|
51
51
|
});
|
|
52
52
|
});
|
|
53
|
-
it("
|
|
54
|
-
expect(
|
|
53
|
+
it("keeps a truncated first name as-is", () => {
|
|
54
|
+
expect(parsePersonCell("Mme AESCHLIMANN Marie-Do")).toEqual({
|
|
55
55
|
civilite: "Mme",
|
|
56
56
|
nom: "AESCHLIMANN",
|
|
57
57
|
prenom: "Marie-Do",
|
|
58
58
|
});
|
|
59
59
|
});
|
|
60
|
-
it("
|
|
61
|
-
expect(
|
|
60
|
+
it("ignores the (*) unpaid leave marker", () => {
|
|
61
|
+
expect(parsePersonCell("M. DURAND (*) Paul")).toEqual({ civilite: "M.", nom: "DURAND", prenom: "Paul" });
|
|
62
62
|
});
|
|
63
|
-
it("
|
|
64
|
-
expect(
|
|
65
|
-
expect(
|
|
63
|
+
it("returns null for an empty cell or civility-only", () => {
|
|
64
|
+
expect(parsePersonCell("")).toBeNull();
|
|
65
|
+
expect(parsePersonCell("M.")).toBeNull();
|
|
66
66
|
});
|
|
67
67
|
});
|
|
68
|
-
describe("
|
|
69
|
-
it("
|
|
68
|
+
describe("buildCollaboratorRows", () => {
|
|
69
|
+
it("attaches multiple collaborators to the current senator (carry-forward when employer is empty)", () => {
|
|
70
70
|
const items = pageItems([
|
|
71
71
|
["Mme AESCHLIMANN Marie-Do", "M. CHAREF Dahmane"],
|
|
72
72
|
[null, "M. POURQUERY Alexandre"],
|
|
73
73
|
[null, "M. SEBRIER Valentin"],
|
|
74
74
|
["M. ALLIZARD Pascal", "Mme OLIVIER Béatrice"],
|
|
75
75
|
]);
|
|
76
|
-
const
|
|
77
|
-
expect(
|
|
78
|
-
expect(
|
|
79
|
-
expect(
|
|
76
|
+
const rows = buildCollaboratorRows(items);
|
|
77
|
+
expect(rows).toHaveLength(4);
|
|
78
|
+
expect(rows.slice(0, 3).every((r) => r.senateur.nom === "AESCHLIMANN")).toBe(true);
|
|
79
|
+
expect(rows[3].senateur.nom === "ALLIZARD" && rows[3].collaborateur.nom === "OLIVIER").toBe(true);
|
|
80
80
|
});
|
|
81
|
-
it("
|
|
81
|
+
it("ignores headers, footers and page numbers, and handles multiple pages", () => {
|
|
82
82
|
const items = [
|
|
83
83
|
{ str: "Liste des collaborateurs par Sénateur", x: 208, y: 760, page: 1 },
|
|
84
84
|
{ str: "Employeur", x: 193, y: 740, page: 1 },
|
|
@@ -88,28 +88,28 @@ describe("construireLignesCollaborateurs", () => {
|
|
|
88
88
|
{ str: "- 1 -", x: 296, y: 30, page: 1 },
|
|
89
89
|
...pageItems([["Mme BÉLIM Audrey", "M. BOUCHER Thibault"]], 2),
|
|
90
90
|
];
|
|
91
|
-
const
|
|
92
|
-
expect(
|
|
93
|
-
expect(
|
|
94
|
-
expect(
|
|
91
|
+
const rows = buildCollaboratorRows(items);
|
|
92
|
+
expect(rows).toHaveLength(2);
|
|
93
|
+
expect(rows[0].senateur.nom).toBe("BACCI");
|
|
94
|
+
expect(rows[1].senateur.nom).toBe("BÉLIM");
|
|
95
95
|
});
|
|
96
|
-
it("
|
|
97
|
-
const
|
|
98
|
-
expect(
|
|
96
|
+
it("emits nothing until an employer has been encountered", () => {
|
|
97
|
+
const orphan = [{ str: "M. ORPHELIN Jean", x: X_COLLABORATOR, y: 700, page: 1 }];
|
|
98
|
+
expect(buildCollaboratorRows(orphan)).toHaveLength(0);
|
|
99
99
|
});
|
|
100
100
|
});
|
|
101
|
-
describe("
|
|
102
|
-
it("
|
|
103
|
-
const date =
|
|
101
|
+
describe("extractEditionDate", () => {
|
|
102
|
+
it("extracts the edition date from the footer", () => {
|
|
103
|
+
const date = extractEditionDate(["…", "Edition du 24/06/2026", "(*) Congé non rémunéré"]);
|
|
104
104
|
expect(date?.toISOString().slice(0, 10)).toBe("2026-06-24");
|
|
105
105
|
});
|
|
106
|
-
it("
|
|
107
|
-
expect(
|
|
106
|
+
it("returns null if absent", () => {
|
|
107
|
+
expect(extractEditionDate(["nothing here"])).toBeNull();
|
|
108
108
|
});
|
|
109
109
|
});
|
|
110
|
-
describe("
|
|
111
|
-
it("
|
|
112
|
-
expect(
|
|
113
|
-
expect(
|
|
110
|
+
describe("normalizeForComparison", () => {
|
|
111
|
+
it("uppercases, removes accents and compacts spaces", () => {
|
|
112
|
+
expect(normalizeForComparison(" Béatrice ")).toBe("BEATRICE");
|
|
113
|
+
expect(normalizeForComparison("Bélim")).toBe("BELIM");
|
|
114
114
|
});
|
|
115
115
|
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|