@tricoteuses/senat 3.2.2 → 3.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
import { execSync } from "child_process";
|
|
2
|
-
|
|
3
|
-
function
|
|
2
|
+
import { runDataDownload } from "./shared/data_download.js";
|
|
3
|
+
function exec(command) {
|
|
4
4
|
try {
|
|
5
5
|
execSync(command, { stdio: "inherit" });
|
|
6
|
+
return { status: 0 };
|
|
6
7
|
}
|
|
7
8
|
catch (error) {
|
|
8
9
|
const execError = error;
|
|
9
|
-
|
|
10
|
-
console.error(`Error during: ${command}`, error);
|
|
11
|
-
process.exit(execError.status || 1);
|
|
12
|
-
}
|
|
10
|
+
return { status: execError.status ?? null, error };
|
|
13
11
|
}
|
|
14
12
|
}
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
13
|
+
const exitCode = runDataDownload({
|
|
14
|
+
exec,
|
|
15
|
+
now: () => new Date(),
|
|
16
|
+
rawArgs: process.argv.slice(2),
|
|
17
|
+
log: console.log,
|
|
18
|
+
warn: console.warn,
|
|
19
|
+
error: (message, error) => error === undefined ? console.error(message) : console.error(message, error),
|
|
20
|
+
});
|
|
21
|
+
process.exit(exitCode);
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { type Dataset } from "../../server/datasets.js";
|
|
2
|
+
export declare const STATUS_FILENAME = "dataset_status.json";
|
|
3
|
+
export type DatasetStatusEntry = {
|
|
4
|
+
status: "ok" | "failed";
|
|
5
|
+
lastRunAt: string;
|
|
6
|
+
lastSuccessAt?: string;
|
|
7
|
+
lastFailureAt?: string;
|
|
8
|
+
lastError?: string;
|
|
9
|
+
consecutiveFailures: number;
|
|
10
|
+
firstFailureAt?: string;
|
|
11
|
+
};
|
|
12
|
+
export type DatasetStatusFile = Record<string, DatasetStatusEntry>;
|
|
13
|
+
export type ExecResult = {
|
|
14
|
+
status: number | null;
|
|
15
|
+
error?: unknown;
|
|
16
|
+
};
|
|
17
|
+
export type ExecFn = (command: string) => ExecResult;
|
|
18
|
+
export type DataDownloadLogger = {
|
|
19
|
+
log?: (message: string) => void;
|
|
20
|
+
warn?: (message: string) => void;
|
|
21
|
+
error?: (message: string, error?: unknown) => void;
|
|
22
|
+
};
|
|
23
|
+
export type DataDownloadDeps = DataDownloadLogger & {
|
|
24
|
+
rawArgs: string[];
|
|
25
|
+
exec: ExecFn;
|
|
26
|
+
now: () => Date;
|
|
27
|
+
};
|
|
28
|
+
export declare const CATEGORY_DATASETS: Array<{
|
|
29
|
+
category: string;
|
|
30
|
+
dataset: Dataset;
|
|
31
|
+
}>;
|
|
32
|
+
export type ParsedArgs = {
|
|
33
|
+
categories: string[];
|
|
34
|
+
dataDir?: string;
|
|
35
|
+
rest: string[];
|
|
36
|
+
};
|
|
37
|
+
export declare function parseArgs(rawArgs: string[]): ParsedArgs;
|
|
38
|
+
export declare function forwardArgs(parsedArgs: ParsedArgs, categories: string[]): string;
|
|
39
|
+
export declare function readStatusFile(statusFilePath: string): DatasetStatusFile;
|
|
40
|
+
export declare function writeStatusFile(statusFilePath: string, statuses: DatasetStatusFile): void;
|
|
41
|
+
export declare function recordSuccess(statuses: DatasetStatusFile, datasetName: string, now: Date): void;
|
|
42
|
+
export declare function recordFailure(statuses: DatasetStatusFile, datasetName: string, now: Date, errorMessage: string): void;
|
|
43
|
+
export declare function runDataDownload(deps: DataDownloadDeps): number;
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import fs from "fs-extra";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import { datasets } from "../../server/datasets.js";
|
|
4
|
+
export const STATUS_FILENAME = "dataset_status.json";
|
|
5
|
+
const MAX_ERROR_LENGTH = 2000;
|
|
6
|
+
export const CATEGORY_DATASETS = [
|
|
7
|
+
{ category: "Ameli", dataset: datasets.ameli },
|
|
8
|
+
{ category: "Debats", dataset: datasets.debats },
|
|
9
|
+
{ category: "DosLeg", dataset: datasets.dosleg },
|
|
10
|
+
{ category: "Questions", dataset: datasets.questions },
|
|
11
|
+
{ category: "Sens", dataset: datasets.sens },
|
|
12
|
+
];
|
|
13
|
+
// Options of retrieve_open_data whose next token is a value. They must not be
|
|
14
|
+
// mistaken for the dataDir positional argument.
|
|
15
|
+
const valueOptions = new Set([
|
|
16
|
+
"categories",
|
|
17
|
+
"k",
|
|
18
|
+
"sudo",
|
|
19
|
+
"clone",
|
|
20
|
+
"C",
|
|
21
|
+
"remote",
|
|
22
|
+
"r",
|
|
23
|
+
"fromSession",
|
|
24
|
+
"only-recent",
|
|
25
|
+
]);
|
|
26
|
+
export function parseArgs(rawArgs) {
|
|
27
|
+
const categories = [];
|
|
28
|
+
let dataDir;
|
|
29
|
+
const rest = [];
|
|
30
|
+
for (let index = 0; index < rawArgs.length; index++) {
|
|
31
|
+
const token = rawArgs[index];
|
|
32
|
+
const name = token.replace(/^-+/, "").split("=")[0];
|
|
33
|
+
if (name === "categories" || name === "k") {
|
|
34
|
+
const rawValues = token.includes("=") ? [token.slice(token.indexOf("=") + 1)] : [rawArgs[++index]];
|
|
35
|
+
for (const rawValue of rawValues) {
|
|
36
|
+
if (rawValue === undefined) {
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
for (const value of rawValue.split(",")) {
|
|
40
|
+
if (value.length > 0) {
|
|
41
|
+
categories.push(value);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
if (token.startsWith("-") && valueOptions.has(name) && !token.includes("=")) {
|
|
48
|
+
rest.push(token);
|
|
49
|
+
const value = rawArgs[++index];
|
|
50
|
+
if (value !== undefined) {
|
|
51
|
+
rest.push(value);
|
|
52
|
+
}
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
55
|
+
if (!token.startsWith("-") && dataDir === undefined) {
|
|
56
|
+
dataDir = token;
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
rest.push(token);
|
|
60
|
+
}
|
|
61
|
+
return { categories: categories.length > 0 ? categories : ["All"], dataDir, rest };
|
|
62
|
+
}
|
|
63
|
+
export function forwardArgs(parsedArgs, categories) {
|
|
64
|
+
const parts = [...parsedArgs.rest];
|
|
65
|
+
if (parsedArgs.dataDir !== undefined) {
|
|
66
|
+
parts.push(parsedArgs.dataDir);
|
|
67
|
+
}
|
|
68
|
+
for (const category of categories) {
|
|
69
|
+
parts.push("--categories", category);
|
|
70
|
+
}
|
|
71
|
+
return parts.join(" ");
|
|
72
|
+
}
|
|
73
|
+
export function readStatusFile(statusFilePath) {
|
|
74
|
+
try {
|
|
75
|
+
const content = fs.readJsonSync(statusFilePath);
|
|
76
|
+
return content && typeof content === "object" ? content : {};
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
return {};
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
export function writeStatusFile(statusFilePath, statuses) {
|
|
83
|
+
fs.writeJsonSync(statusFilePath, statuses, { spaces: 2 });
|
|
84
|
+
}
|
|
85
|
+
export function recordSuccess(statuses, datasetName, now) {
|
|
86
|
+
statuses[datasetName] = {
|
|
87
|
+
consecutiveFailures: 0,
|
|
88
|
+
lastRunAt: now.toISOString(),
|
|
89
|
+
lastSuccessAt: now.toISOString(),
|
|
90
|
+
status: "ok",
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
export function recordFailure(statuses, datasetName, now, errorMessage) {
|
|
94
|
+
const previous = statuses[datasetName];
|
|
95
|
+
statuses[datasetName] = {
|
|
96
|
+
consecutiveFailures: (previous?.consecutiveFailures ?? 0) + 1,
|
|
97
|
+
firstFailureAt: previous?.firstFailureAt ?? now.toISOString(),
|
|
98
|
+
lastError: errorMessage.slice(0, MAX_ERROR_LENGTH),
|
|
99
|
+
lastFailureAt: now.toISOString(),
|
|
100
|
+
lastRunAt: now.toISOString(),
|
|
101
|
+
lastSuccessAt: previous?.lastSuccessAt,
|
|
102
|
+
status: "failed",
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
function execNormalized(deps, command) {
|
|
106
|
+
try {
|
|
107
|
+
return deps.exec(command);
|
|
108
|
+
}
|
|
109
|
+
catch (error) {
|
|
110
|
+
const execError = error;
|
|
111
|
+
return { status: execError.status ?? null, error };
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
function isSuccessfulStatus(result) {
|
|
115
|
+
return result.status === 0 || result.status === 10;
|
|
116
|
+
}
|
|
117
|
+
function runDownstreamScript(deps, command) {
|
|
118
|
+
const result = execNormalized(deps, command);
|
|
119
|
+
if (isSuccessfulStatus(result)) {
|
|
120
|
+
return null;
|
|
121
|
+
}
|
|
122
|
+
deps.error?.(`Error during: ${command}`, result.error);
|
|
123
|
+
return result.status || 1;
|
|
124
|
+
}
|
|
125
|
+
export function runDataDownload(deps) {
|
|
126
|
+
const parsedArgs = parseArgs(deps.rawArgs);
|
|
127
|
+
const chosen = parsedArgs.categories.includes("All")
|
|
128
|
+
? CATEGORY_DATASETS
|
|
129
|
+
: CATEGORY_DATASETS.filter(({ category }) => parsedArgs.categories.includes(category));
|
|
130
|
+
if (chosen.length === 0) {
|
|
131
|
+
deps.error?.(`No known dataset among categories: ${parsedArgs.categories.join(", ")}`);
|
|
132
|
+
return 1;
|
|
133
|
+
}
|
|
134
|
+
const statusFilePath = parsedArgs.dataDir !== undefined ? path.join(parsedArgs.dataDir, STATUS_FILENAME) : null;
|
|
135
|
+
const statuses = statusFilePath !== null ? readStatusFile(statusFilePath) : {};
|
|
136
|
+
const okCategories = [];
|
|
137
|
+
const failedCategories = [];
|
|
138
|
+
for (const { category, dataset } of chosen) {
|
|
139
|
+
// --all: fetch, unzip, repair encoding and import (same as the original script).
|
|
140
|
+
const command = `tsx src/scripts/retrieve_open_data.ts --all ${forwardArgs(parsedArgs, [category])}`.trim();
|
|
141
|
+
deps.log?.(`\n=== Retrieving dataset ${dataset.title} (${category}) ===`);
|
|
142
|
+
const result = execNormalized(deps, command);
|
|
143
|
+
if (!isSuccessfulStatus(result)) {
|
|
144
|
+
const errorMessage = result.error instanceof Error ? result.error.message : String(result.error);
|
|
145
|
+
recordFailure(statuses, dataset.database, deps.now(), errorMessage);
|
|
146
|
+
failedCategories.push(category);
|
|
147
|
+
deps.error?.(`Error during: ${command}`, result.error);
|
|
148
|
+
}
|
|
149
|
+
else {
|
|
150
|
+
recordSuccess(statuses, dataset.database, deps.now());
|
|
151
|
+
okCategories.push(category);
|
|
152
|
+
}
|
|
153
|
+
if (statusFilePath !== null) {
|
|
154
|
+
writeStatusFile(statusFilePath, statuses);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
if (failedCategories.length > 0) {
|
|
158
|
+
deps.warn?.(`\n⚠️ Warning: Some datasets failed to retrieve: ${failedCategories.join(", ")}. ` +
|
|
159
|
+
"Continuing with available datasets...\n");
|
|
160
|
+
}
|
|
161
|
+
if (okCategories.length === 0) {
|
|
162
|
+
deps.error?.("All datasets failed to retrieve; skipping data conversion.");
|
|
163
|
+
return 1;
|
|
164
|
+
}
|
|
165
|
+
const exitCode = runDownstreamScript(deps, `tsx src/scripts/convert_data.ts ${forwardArgs(parsedArgs, okCategories)}`) ??
|
|
166
|
+
runDownstreamScript(deps, `cross-env TZ='Etc/UTC' tsx src/scripts/retrieve_agenda.ts ${deps.rawArgs.join(" ")} --parseAgenda --silent`) ??
|
|
167
|
+
runDownstreamScript(deps, "cross-env TZ='Etc/UTC' tsx src/scripts/retrieve_cr_seance.ts " +
|
|
168
|
+
`${deps.rawArgs.join(" ")} --parseDebats --silent`) ??
|
|
169
|
+
runDownstreamScript(deps, "cross-env TZ='Etc/UTC' tsx src/scripts/retrieve_cr_commission.ts " +
|
|
170
|
+
`${deps.rawArgs.join(" ")} --parseDebats --silent`) ??
|
|
171
|
+
runDownstreamScript(deps, `tsx src/scripts/retrieve_videos.ts ${deps.rawArgs.join(" ")} --silent`) ??
|
|
172
|
+
runDownstreamScript(deps, `tsx src/scripts/retrieve_collaborateurs.ts ${deps.rawArgs.join(" ")} --silent`) ??
|
|
173
|
+
0;
|
|
174
|
+
return exitCode;
|
|
175
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { afterEach, describe, expect, it } from "vitest";
|
|
5
|
+
import { forwardArgs, parseArgs, readStatusFile, recordFailure, recordSuccess, runDataDownload, STATUS_FILENAME, writeStatusFile, } from "../src/scripts/shared/data_download.js";
|
|
6
|
+
let dataDir;
|
|
7
|
+
afterEach(() => {
|
|
8
|
+
if (dataDir !== undefined) {
|
|
9
|
+
rmSync(dataDir, { force: true, recursive: true });
|
|
10
|
+
}
|
|
11
|
+
});
|
|
12
|
+
function createTempDataDir() {
|
|
13
|
+
dataDir = mkdtempSync(path.join(tmpdir(), "senat-data-download-"));
|
|
14
|
+
return dataDir;
|
|
15
|
+
}
|
|
16
|
+
const fixedDate = new Date("2026-09-08T10:00:00Z");
|
|
17
|
+
function makeExec(respond) {
|
|
18
|
+
const commands = [];
|
|
19
|
+
return {
|
|
20
|
+
commands,
|
|
21
|
+
exec: (command) => {
|
|
22
|
+
commands.push(command);
|
|
23
|
+
return respond(command) ?? { status: 0 };
|
|
24
|
+
},
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
describe("parseArgs", () => {
|
|
28
|
+
it("defaults categories to All when none are provided", () => {
|
|
29
|
+
const parsed = parseArgs([]);
|
|
30
|
+
expect(parsed.categories).toEqual(["All"]);
|
|
31
|
+
expect(parsed.dataDir).toBeUndefined();
|
|
32
|
+
expect(parsed.rest).toEqual([]);
|
|
33
|
+
});
|
|
34
|
+
it("extracts the dataDir positional argument and forwards the other options", () => {
|
|
35
|
+
const rawArgs = [
|
|
36
|
+
"--all",
|
|
37
|
+
"/app/senat-data",
|
|
38
|
+
"--fromSession",
|
|
39
|
+
"2022",
|
|
40
|
+
"--fetchDocuments",
|
|
41
|
+
"--parseDocuments",
|
|
42
|
+
"--only-recent",
|
|
43
|
+
"30",
|
|
44
|
+
"--commit",
|
|
45
|
+
"--keepDir",
|
|
46
|
+
];
|
|
47
|
+
const parsed = parseArgs(rawArgs);
|
|
48
|
+
expect(parsed.dataDir).toBe("/app/senat-data");
|
|
49
|
+
expect(parsed.rest).toEqual([
|
|
50
|
+
"--all",
|
|
51
|
+
"--fromSession",
|
|
52
|
+
"2022",
|
|
53
|
+
"--fetchDocuments",
|
|
54
|
+
"--parseDocuments",
|
|
55
|
+
"--only-recent",
|
|
56
|
+
"30",
|
|
57
|
+
"--commit",
|
|
58
|
+
"--keepDir",
|
|
59
|
+
]);
|
|
60
|
+
});
|
|
61
|
+
it("collects --categories values including repeated occurrences", () => {
|
|
62
|
+
const parsed = parseArgs(["/data", "--categories", "Ameli", "--categories", "Sens"]);
|
|
63
|
+
expect(parsed.categories).toEqual(["Ameli", "Sens"]);
|
|
64
|
+
expect(parsed.dataDir).toBe("/data");
|
|
65
|
+
expect(parsed.rest).toEqual([]);
|
|
66
|
+
});
|
|
67
|
+
it("supports the -k alias and comma-separated values", () => {
|
|
68
|
+
const parsed = parseArgs(["-k", "Sens,Ameli", "/data"]);
|
|
69
|
+
expect(parsed.categories).toEqual(["Sens", "Ameli"]);
|
|
70
|
+
});
|
|
71
|
+
it("supports the --categories= form", () => {
|
|
72
|
+
const parsed = parseArgs(["--categories=Sens", "/data"]);
|
|
73
|
+
expect(parsed.categories).toEqual(["Sens"]);
|
|
74
|
+
});
|
|
75
|
+
it("does not mistake valued options for the dataDir positional", () => {
|
|
76
|
+
const parsed = parseArgs(["--sudo", "postgres", "--remote", "origin", "true-data", "--verbose"]);
|
|
77
|
+
expect(parsed.dataDir).toBe("true-data");
|
|
78
|
+
expect(parsed.rest).toEqual(["--sudo", "postgres", "--remote", "origin", "--verbose"]);
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
describe("forwardArgs", () => {
|
|
82
|
+
it("appends the dataDir then the requested categories to the forwarded options", () => {
|
|
83
|
+
const parsed = parseArgs(["--all", "/app/senat-data", "--categories", "All", "--fromSession", "2022"]);
|
|
84
|
+
const forwarded = forwardArgs(parsed, ["Sens"]);
|
|
85
|
+
expect(forwarded).toBe("--all --fromSession 2022 /app/senat-data --categories Sens");
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
describe("dataset status file", () => {
|
|
89
|
+
it("increments consecutiveFailures and keeps firstFailureAt across consecutive failures", () => {
|
|
90
|
+
const statuses = {};
|
|
91
|
+
recordFailure(statuses, "sens", new Date("2026-09-01T00:00:00Z"), "dump empty");
|
|
92
|
+
recordFailure(statuses, "sens", new Date("2026-09-02T00:00:00Z"), "dump empty again");
|
|
93
|
+
expect(statuses["sens"]).toMatchObject({
|
|
94
|
+
status: "failed",
|
|
95
|
+
consecutiveFailures: 2,
|
|
96
|
+
firstFailureAt: "2026-09-01T00:00:00.000Z",
|
|
97
|
+
lastFailureAt: "2026-09-02T00:00:00.000Z",
|
|
98
|
+
lastError: "dump empty again",
|
|
99
|
+
lastSuccessAt: undefined,
|
|
100
|
+
});
|
|
101
|
+
});
|
|
102
|
+
it("resets the failure streak on success while updating lastSuccessAt", () => {
|
|
103
|
+
const statuses = {};
|
|
104
|
+
recordFailure(statuses, "sens", new Date("2026-09-01T00:00:00Z"), "dump empty");
|
|
105
|
+
recordSuccess(statuses, "sens", new Date("2026-09-02T00:00:00Z"));
|
|
106
|
+
expect(statuses["sens"]).toEqual({
|
|
107
|
+
status: "ok",
|
|
108
|
+
consecutiveFailures: 0,
|
|
109
|
+
lastRunAt: "2026-09-02T00:00:00.000Z",
|
|
110
|
+
lastSuccessAt: "2026-09-02T00:00:00.000Z",
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
it("returns an empty state for a missing or corrupted status file", () => {
|
|
114
|
+
const dir = createTempDataDir();
|
|
115
|
+
expect(readStatusFile(path.join(dir, STATUS_FILENAME))).toEqual({});
|
|
116
|
+
const corruptedPath = path.join(dir, "corrupted.json");
|
|
117
|
+
writeFileSync(corruptedPath, "{not json", { encoding: "utf-8" });
|
|
118
|
+
expect(readStatusFile(corruptedPath)).toEqual({});
|
|
119
|
+
});
|
|
120
|
+
it("round-trips statuses through the status file", () => {
|
|
121
|
+
const dir = createTempDataDir();
|
|
122
|
+
const filePath = path.join(dir, STATUS_FILENAME);
|
|
123
|
+
const statuses = {};
|
|
124
|
+
recordFailure(statuses, "sens", fixedDate, "dump empty");
|
|
125
|
+
writeStatusFile(filePath, statuses);
|
|
126
|
+
expect(readStatusFile(filePath)).toEqual(statuses);
|
|
127
|
+
});
|
|
128
|
+
});
|
|
129
|
+
describe("runDataDownload", () => {
|
|
130
|
+
it("continues after a dataset failure, converts only successful categories and exits 0", () => {
|
|
131
|
+
const dir = createTempDataDir();
|
|
132
|
+
const { commands, exec } = makeExec((command) => command.includes("--categories Sens")
|
|
133
|
+
? { status: 1, error: new Error("Staging schema sens_staging is empty") }
|
|
134
|
+
: undefined);
|
|
135
|
+
const exitCode = runDataDownload({
|
|
136
|
+
rawArgs: [dir, "--fromSession", "2022"],
|
|
137
|
+
exec,
|
|
138
|
+
now: () => fixedDate,
|
|
139
|
+
});
|
|
140
|
+
expect(exitCode).toBe(0);
|
|
141
|
+
const retrieveCommands = commands.filter((command) => command.includes("retrieve_open_data"));
|
|
142
|
+
expect(retrieveCommands).toHaveLength(5);
|
|
143
|
+
for (const command of retrieveCommands) {
|
|
144
|
+
expect(command).toContain("retrieve_open_data.ts --all ");
|
|
145
|
+
}
|
|
146
|
+
expect(retrieveCommands[0]).toContain("--categories Ameli");
|
|
147
|
+
expect(retrieveCommands[4]).toContain("--categories Sens");
|
|
148
|
+
const convertCommand = commands.find((command) => command.includes("convert_data"));
|
|
149
|
+
expect(convertCommand).toContain("--categories Ameli");
|
|
150
|
+
expect(convertCommand).toContain("--categories Debats");
|
|
151
|
+
expect(convertCommand).toContain("--categories DosLeg");
|
|
152
|
+
expect(convertCommand).toContain("--categories Questions");
|
|
153
|
+
expect(convertCommand).not.toContain("--categories Sens");
|
|
154
|
+
const agendaCommand = commands.find((command) => command.includes("retrieve_agenda"));
|
|
155
|
+
expect(agendaCommand).toBe(`cross-env TZ='Etc/UTC' tsx src/scripts/retrieve_agenda.ts ${dir} --fromSession 2022 --parseAgenda --silent`);
|
|
156
|
+
const statuses = JSON.parse(readFileSync(path.join(dir, STATUS_FILENAME), { encoding: "utf-8" }));
|
|
157
|
+
expect(statuses.sens).toMatchObject({ status: "failed", consecutiveFailures: 1 });
|
|
158
|
+
expect(statuses.ameli).toMatchObject({ status: "ok", consecutiveFailures: 0 });
|
|
159
|
+
});
|
|
160
|
+
it("exits 1 without conversion when all datasets fail", () => {
|
|
161
|
+
const dir = createTempDataDir();
|
|
162
|
+
const { commands, exec } = makeExec(() => ({ status: 1, error: new Error("boom") }));
|
|
163
|
+
const exitCode = runDataDownload({
|
|
164
|
+
rawArgs: [dir],
|
|
165
|
+
exec,
|
|
166
|
+
now: () => fixedDate,
|
|
167
|
+
});
|
|
168
|
+
expect(exitCode).toBe(1);
|
|
169
|
+
expect(commands.filter((command) => command.includes("retrieve_open_data"))).toHaveLength(5);
|
|
170
|
+
expect(commands.find((command) => command.includes("convert_data"))).toBeUndefined();
|
|
171
|
+
expect(commands.find((command) => command.includes("retrieve_agenda"))).toBeUndefined();
|
|
172
|
+
const statuses = JSON.parse(readFileSync(path.join(dir, STATUS_FILENAME), { encoding: "utf-8" }));
|
|
173
|
+
for (const dataset of ["ameli", "debats", "dosleg", "questions", "sens"]) {
|
|
174
|
+
expect(statuses[dataset]).toMatchObject({ status: "failed", consecutiveFailures: 1 });
|
|
175
|
+
}
|
|
176
|
+
});
|
|
177
|
+
it("runs every step when all datasets succeed", () => {
|
|
178
|
+
const dir = createTempDataDir();
|
|
179
|
+
const { commands, exec } = makeExec(() => undefined);
|
|
180
|
+
const exitCode = runDataDownload({
|
|
181
|
+
rawArgs: [dir, "--categories", "All"],
|
|
182
|
+
exec,
|
|
183
|
+
now: () => fixedDate,
|
|
184
|
+
});
|
|
185
|
+
expect(exitCode).toBe(0);
|
|
186
|
+
const convertCommand = commands.find((command) => command.includes("convert_data"));
|
|
187
|
+
for (const category of ["Ameli", "Debats", "DosLeg", "Questions", "Sens"]) {
|
|
188
|
+
expect(convertCommand).toContain(`--categories ${category}`);
|
|
189
|
+
}
|
|
190
|
+
expect(commands.find((command) => command.includes("retrieve_agenda"))).toBeDefined();
|
|
191
|
+
expect(commands.find((command) => command.includes("retrieve_cr_seance"))).toBeDefined();
|
|
192
|
+
expect(commands.find((command) => command.includes("retrieve_cr_commission"))).toBeDefined();
|
|
193
|
+
expect(commands.find((command) => command.includes("retrieve_videos"))).toBeDefined();
|
|
194
|
+
expect(commands.find((command) => command.includes("retrieve_collaborateurs"))).toBeDefined();
|
|
195
|
+
});
|
|
196
|
+
it("stops after convert_data when it fails and propagates its exit status", () => {
|
|
197
|
+
const dir = createTempDataDir();
|
|
198
|
+
const { commands, exec } = makeExec((command) => command.includes("convert_data") ? { status: 3, error: new Error("parse failure") } : undefined);
|
|
199
|
+
const exitCode = runDataDownload({
|
|
200
|
+
rawArgs: [dir],
|
|
201
|
+
exec,
|
|
202
|
+
now: () => fixedDate,
|
|
203
|
+
});
|
|
204
|
+
expect(exitCode).toBe(3);
|
|
205
|
+
expect(commands.find((command) => command.includes("retrieve_agenda"))).toBeUndefined();
|
|
206
|
+
});
|
|
207
|
+
it("exits 1 when no requested category matches a known dataset", () => {
|
|
208
|
+
const { commands, exec } = makeExec(() => undefined);
|
|
209
|
+
const exitCode = runDataDownload({
|
|
210
|
+
rawArgs: ["/data", "--categories", "Unknown"],
|
|
211
|
+
exec,
|
|
212
|
+
now: () => fixedDate,
|
|
213
|
+
});
|
|
214
|
+
expect(exitCode).toBe(1);
|
|
215
|
+
expect(commands).toHaveLength(0);
|
|
216
|
+
});
|
|
217
|
+
});
|