@tricoteuses/assemblee 3.8.4 → 3.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/lib/{amendements-BYKlEcWb.js → amendements-BtCdrXW8.js} +36 -33
- package/lib/{amendements-CvlD2dI4.js → amendements-DfZse75q.js} +924 -672
- package/lib/amendements-xiii.d.ts +56 -0
- package/lib/amendements-xiii.test.d.ts +1 -0
- package/lib/browser-Dl6SgoZX.js +8164 -0
- package/lib/cleaners/amendements-xiv.test.d.ts +1 -0
- package/lib/cleaners/amendements.d.ts +18 -1
- package/lib/cleaners.js +3 -3
- package/lib/index.d.ts +5 -2
- package/lib/index.js +596 -99
- package/lib/inserters.d.ts +2 -2
- package/lib/{loaders-BNXs79lC.js → loaders-Dd8i-E7o.js} +11 -2
- package/lib/loaders.js +1 -1
- package/lib/open_data/artifacts.d.ts +12 -0
- package/lib/parsers.js +359 -8521
- package/lib/schemas/clean/amendements-xiii.d.ts +579 -0
- package/lib/schemas/clean/amendements.d.ts +502 -11
- package/lib/schemas/enriched/amendements.d.ts +6 -6
- package/lib/schemas/raw/amendements-xiii.d.ts +754 -0
- package/lib/schemas/raw/amendements-xiv.test.d.ts +1 -0
- package/lib/schemas/raw/amendements.d.ts +1769 -2
- package/lib/schemas/raw/amendements.test.d.ts +1 -0
- package/lib/scripts/clean_reorganized_data.test.d.ts +1 -0
- package/lib/scripts/data-download.d.ts +132 -0
- package/lib/scripts/retrieve_amendements_xiii.d.ts +51 -0
- package/lib/scripts/retrieve_open_data.d.ts +39 -24
- package/lib/scripts/shared/amendements_xiv_stream.d.ts +29 -0
- package/lib/scripts/shared/amendements_xiv_stream.test.d.ts +1 -0
- package/lib/scripts/shared/cli_helpers.d.ts +20 -0
- package/lib/{scrutins-BUVxV-QV.js → scrutins-CLWnMYLE.js} +266 -266
- package/package.json +7 -7
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -1,4 +1,136 @@
|
|
|
1
1
|
import { ConversionScope } from '../open_data/artifacts.js';
|
|
2
|
+
/**
|
|
3
|
+
* Full option surface accepted by data-download: everything it forwards to
|
|
4
|
+
* retrieve/reorganize/clean, plus its own `--drop-intermediates`.
|
|
5
|
+
* Sub-scripts don't declare the drop flag (they must not silently ignore
|
|
6
|
+
* it), so it is stripped from the forwarded argument strings below.
|
|
7
|
+
*/
|
|
8
|
+
export declare const dataDownloadParser: import('zod-opts').Parser<{
|
|
9
|
+
dataset: {
|
|
10
|
+
description: string;
|
|
11
|
+
type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodString>>;
|
|
12
|
+
};
|
|
13
|
+
fetch: {
|
|
14
|
+
alias: string;
|
|
15
|
+
description: string;
|
|
16
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
17
|
+
};
|
|
18
|
+
"manifest-dir": {
|
|
19
|
+
description: string;
|
|
20
|
+
type: import('zod').ZodOptional<import('zod').ZodString>;
|
|
21
|
+
};
|
|
22
|
+
"only-if-new": {
|
|
23
|
+
description: string;
|
|
24
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
25
|
+
};
|
|
26
|
+
"probe-count": {
|
|
27
|
+
description: string;
|
|
28
|
+
type: import('zod').ZodDefault<import('zod').ZodNumber>;
|
|
29
|
+
};
|
|
30
|
+
"drop-intermediates": {
|
|
31
|
+
description: string;
|
|
32
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
33
|
+
};
|
|
34
|
+
categories: {
|
|
35
|
+
alias: string;
|
|
36
|
+
description: string;
|
|
37
|
+
type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodString>>;
|
|
38
|
+
};
|
|
39
|
+
legislature: {
|
|
40
|
+
alias: string;
|
|
41
|
+
description: string;
|
|
42
|
+
type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodNumber>>;
|
|
43
|
+
};
|
|
44
|
+
silent: {
|
|
45
|
+
alias: string;
|
|
46
|
+
description: string;
|
|
47
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
48
|
+
};
|
|
49
|
+
verbose: {
|
|
50
|
+
alias: string;
|
|
51
|
+
description: string;
|
|
52
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
53
|
+
};
|
|
54
|
+
"only-recent": {
|
|
55
|
+
description: string;
|
|
56
|
+
type: import('zod').ZodOptional<import('zod').ZodNumber>;
|
|
57
|
+
};
|
|
58
|
+
skipReindent: {
|
|
59
|
+
description: string;
|
|
60
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
61
|
+
};
|
|
62
|
+
keepDir: {
|
|
63
|
+
description: string;
|
|
64
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
65
|
+
};
|
|
66
|
+
clone: {
|
|
67
|
+
alias: string;
|
|
68
|
+
description: string;
|
|
69
|
+
type: import('zod').ZodOptional<import('zod').ZodString>;
|
|
70
|
+
};
|
|
71
|
+
commit: {
|
|
72
|
+
alias: string;
|
|
73
|
+
description: string;
|
|
74
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
75
|
+
};
|
|
76
|
+
remote: {
|
|
77
|
+
alias: string;
|
|
78
|
+
description: string;
|
|
79
|
+
type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodString>>;
|
|
80
|
+
};
|
|
81
|
+
pull: {
|
|
82
|
+
alias: string;
|
|
83
|
+
description: string;
|
|
84
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
85
|
+
};
|
|
86
|
+
fetchDocuments: {
|
|
87
|
+
description: string;
|
|
88
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
89
|
+
};
|
|
90
|
+
parseDocuments: {
|
|
91
|
+
description: string;
|
|
92
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
93
|
+
};
|
|
94
|
+
format: {
|
|
95
|
+
alias: string;
|
|
96
|
+
description: string;
|
|
97
|
+
type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodString>>;
|
|
98
|
+
};
|
|
99
|
+
fetchVideos: {
|
|
100
|
+
description: string;
|
|
101
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
102
|
+
};
|
|
103
|
+
fetchCrCommissions: {
|
|
104
|
+
description: string;
|
|
105
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
106
|
+
};
|
|
107
|
+
}, [{
|
|
108
|
+
description: string;
|
|
109
|
+
type: import('zod').ZodString;
|
|
110
|
+
name: "dataDir";
|
|
111
|
+
}]>;
|
|
112
|
+
export type DataDownloadOptions = ReturnType<typeof dataDownloadParser.parse>;
|
|
113
|
+
/**
|
|
114
|
+
* Remove the locally-consumed drop flag so sub-scripts (which don't declare
|
|
115
|
+
* it) never see it. Same routing rationale as RETRIEVE_ONLY_OPTIONS above.
|
|
116
|
+
*/
|
|
117
|
+
export declare function stripDropFlag(argv: string[]): string[];
|
|
118
|
+
export interface DropResult {
|
|
119
|
+
datasetName: string;
|
|
120
|
+
removed: string[];
|
|
121
|
+
skippedReason?: string;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Delete per-dataset intermediates (published ZIP + resume leftovers +
|
|
125
|
+
* extracted original + reorganized directory), keeping only the final
|
|
126
|
+
* `*_nettoye/` output (+ its git history), manifests and conversion scope.
|
|
127
|
+
*
|
|
128
|
+
* A dataset is only pruned when its `_nettoye/` directory exists and is
|
|
129
|
+
* non-empty — that is the downstream contract (`api-parlement` reads only
|
|
130
|
+
* `*_nettoye`). Unknown dataset names are skipped, never guessed. An empty
|
|
131
|
+
* name list fails open (nothing is deleted).
|
|
132
|
+
*/
|
|
133
|
+
export declare function dropIntermediates(dataDir: string, datasetNames: string[]): Promise<DropResult[]>;
|
|
2
134
|
export declare function dataDirArg(argv: string[]): string;
|
|
3
135
|
/**
|
|
4
136
|
* Returns the datasets to process. `undefined` means the scope is unknown, so
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { AmendementXiiiRaw } from '../schemas/raw/amendements-xiii.js';
|
|
2
|
+
import { AmendementXiiiClean } from '../schemas/clean/amendements-xiii.js';
|
|
3
|
+
/**
|
|
4
|
+
* List the amendment URLs of one text by paging through the archive search.
|
|
5
|
+
*
|
|
6
|
+
* The listing is the only enumeration endpoint: `/13/amendements/` is not
|
|
7
|
+
* browsable and slugs are not predictable. `offset` paginates by `nbres`.
|
|
8
|
+
*/
|
|
9
|
+
export declare function listAmendementUrls(texte: string, { limit, retries, nbres, }?: {
|
|
10
|
+
limit?: number;
|
|
11
|
+
retries?: number;
|
|
12
|
+
nbres?: number;
|
|
13
|
+
}): Promise<string[]>;
|
|
14
|
+
/**
|
|
15
|
+
* List every amendment URL of the legislature by paging through the global
|
|
16
|
+
* archive search (no `numInit` filter), which doubles as the enumeration of
|
|
17
|
+
* the texts.
|
|
18
|
+
*
|
|
19
|
+
* The search announces 49 140 rows for `leg=13`; the endpoint pages by
|
|
20
|
+
* `nbres` and repeats the tail on the last pages, hence the `added === 0`
|
|
21
|
+
* stop.
|
|
22
|
+
*/
|
|
23
|
+
export declare function listAllAmendementUrls({ logProgress, nbres, retries, }?: {
|
|
24
|
+
logProgress?: boolean;
|
|
25
|
+
nbres?: number;
|
|
26
|
+
retries?: number;
|
|
27
|
+
}): Promise<string[]>;
|
|
28
|
+
type CollectResult = {
|
|
29
|
+
clean: AmendementXiiiClean[];
|
|
30
|
+
errors: Array<{
|
|
31
|
+
url: string;
|
|
32
|
+
message: string;
|
|
33
|
+
}>;
|
|
34
|
+
raw: AmendementXiiiRaw[];
|
|
35
|
+
titre: string;
|
|
36
|
+
};
|
|
37
|
+
/**
|
|
38
|
+
* Fetch and parse every amendment of one text.
|
|
39
|
+
*
|
|
40
|
+
* A page that fails to parse is reported, not fatal: a run over a whole
|
|
41
|
+
* legislature must not lose 49 000 pages because of one notice page.
|
|
42
|
+
*/
|
|
43
|
+
export declare function collectTexte(texte: string, { concurrency, limit, retries, verbose, urls, }?: {
|
|
44
|
+
concurrency?: number;
|
|
45
|
+
limit?: number;
|
|
46
|
+
retries?: number;
|
|
47
|
+
verbose?: boolean;
|
|
48
|
+
/** URLs already listed (full-legislature mode); skips the listing. */
|
|
49
|
+
urls?: string[];
|
|
50
|
+
}): Promise<CollectResult>;
|
|
51
|
+
export {};
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { z } from 'zod';
|
|
2
1
|
import { Dataset } from '../datasets.js';
|
|
3
2
|
type ProcessDatasetArguments = {
|
|
4
3
|
dataset: Dataset;
|
|
@@ -8,106 +7,117 @@ type ProcessDatasetArguments = {
|
|
|
8
7
|
declare const retrieveOpenDataParser: import('zod-opts').Parser<{
|
|
9
8
|
dataset: {
|
|
10
9
|
description: string;
|
|
11
|
-
type:
|
|
10
|
+
type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodString>>;
|
|
12
11
|
};
|
|
13
12
|
fetch: {
|
|
14
13
|
alias: string;
|
|
15
14
|
description: string;
|
|
16
|
-
type:
|
|
15
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
17
16
|
};
|
|
18
17
|
"manifest-dir": {
|
|
19
18
|
description: string;
|
|
20
|
-
type:
|
|
19
|
+
type: import('zod').ZodOptional<import('zod').ZodString>;
|
|
21
20
|
};
|
|
22
21
|
"only-if-new": {
|
|
23
22
|
description: string;
|
|
24
|
-
type:
|
|
23
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
25
24
|
};
|
|
26
25
|
"probe-count": {
|
|
27
26
|
description: string;
|
|
28
|
-
type:
|
|
27
|
+
type: import('zod').ZodDefault<import('zod').ZodNumber>;
|
|
29
28
|
};
|
|
30
29
|
categories: {
|
|
31
30
|
alias: string;
|
|
32
31
|
description: string;
|
|
33
|
-
type:
|
|
32
|
+
type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodString>>;
|
|
34
33
|
};
|
|
35
34
|
legislature: {
|
|
36
35
|
alias: string;
|
|
37
36
|
description: string;
|
|
38
|
-
type:
|
|
37
|
+
type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodNumber>>;
|
|
39
38
|
};
|
|
40
39
|
silent: {
|
|
41
40
|
alias: string;
|
|
42
41
|
description: string;
|
|
43
|
-
type:
|
|
42
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
44
43
|
};
|
|
45
44
|
verbose: {
|
|
46
45
|
alias: string;
|
|
47
46
|
description: string;
|
|
48
|
-
type:
|
|
47
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
49
48
|
};
|
|
50
49
|
"only-recent": {
|
|
51
50
|
description: string;
|
|
52
|
-
type:
|
|
51
|
+
type: import('zod').ZodOptional<import('zod').ZodNumber>;
|
|
53
52
|
};
|
|
54
53
|
skipReindent: {
|
|
55
54
|
description: string;
|
|
56
|
-
type:
|
|
55
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
57
56
|
};
|
|
58
57
|
keepDir: {
|
|
59
58
|
description: string;
|
|
60
|
-
type:
|
|
59
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
61
60
|
};
|
|
62
61
|
clone: {
|
|
63
62
|
alias: string;
|
|
64
63
|
description: string;
|
|
65
|
-
type:
|
|
64
|
+
type: import('zod').ZodOptional<import('zod').ZodString>;
|
|
66
65
|
};
|
|
67
66
|
commit: {
|
|
68
67
|
alias: string;
|
|
69
68
|
description: string;
|
|
70
|
-
type:
|
|
69
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
71
70
|
};
|
|
72
71
|
remote: {
|
|
73
72
|
alias: string;
|
|
74
73
|
description: string;
|
|
75
|
-
type:
|
|
74
|
+
type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodString>>;
|
|
76
75
|
};
|
|
77
76
|
pull: {
|
|
78
77
|
alias: string;
|
|
79
78
|
description: string;
|
|
80
|
-
type:
|
|
79
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
81
80
|
};
|
|
82
81
|
fetchDocuments: {
|
|
83
82
|
description: string;
|
|
84
|
-
type:
|
|
83
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
85
84
|
};
|
|
86
85
|
parseDocuments: {
|
|
87
86
|
description: string;
|
|
88
|
-
type:
|
|
87
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
89
88
|
};
|
|
90
89
|
format: {
|
|
91
90
|
alias: string;
|
|
92
91
|
description: string;
|
|
93
|
-
type:
|
|
92
|
+
type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodString>>;
|
|
94
93
|
};
|
|
95
94
|
fetchVideos: {
|
|
96
95
|
description: string;
|
|
97
|
-
type:
|
|
96
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
98
97
|
};
|
|
99
98
|
fetchCrCommissions: {
|
|
100
99
|
description: string;
|
|
101
|
-
type:
|
|
100
|
+
type: import('zod').ZodDefault<import('zod').ZodBoolean>;
|
|
102
101
|
};
|
|
103
102
|
}, [{
|
|
104
103
|
description: string;
|
|
105
|
-
type:
|
|
104
|
+
type: import('zod').ZodString;
|
|
106
105
|
name: "dataDir";
|
|
107
106
|
}]>;
|
|
108
107
|
type Options = ReturnType<typeof retrieveOpenDataParser.parse>;
|
|
109
108
|
export declare function formatParisDateTime(value: string): string;
|
|
110
|
-
|
|
109
|
+
/**
|
|
110
|
+
* Extracts a ZIP archive into `outputDir`.
|
|
111
|
+
*
|
|
112
|
+
* Uses the system `unzip` (Info-ZIP) when available: on archives made of many
|
|
113
|
+
* small files (AMO50: 36k files, Amendements_XVII: 148k files) it is ~10×
|
|
114
|
+
* faster and uses ~10× less CPU than node-stream-zip, with a constant ~4 MB
|
|
115
|
+
* memory footprint. Falls back to node-stream-zip when `unzip` is not
|
|
116
|
+
* installed, so library consumers without it keep working.
|
|
117
|
+
*/
|
|
118
|
+
export declare function extractZip(zipFilePath: string, outputDir: string, { unzipCommand }?: {
|
|
119
|
+
unzipCommand?: string | null;
|
|
120
|
+
}): Promise<void>;
|
|
111
121
|
export declare function publishDataset(stagedDataPath: string, dataPath: string, partZipPath: string | undefined, zipPath: string): Promise<void>;
|
|
112
122
|
export declare function recoverPublication(dataPath: string): Promise<void>;
|
|
113
123
|
/**
|
|
@@ -119,6 +129,11 @@ export declare function recoverPublication(dataPath: string): Promise<void>;
|
|
|
119
129
|
* @returns {Promise<void>} A promise that resolves when the dataset processing is complete.
|
|
120
130
|
*/
|
|
121
131
|
export declare function processDataset({ dataset, dataDir, options, }: ProcessDatasetArguments): Promise<void>;
|
|
132
|
+
export declare const reindentMaxBytes: number;
|
|
133
|
+
export declare function reindentJsonFiles(dataDirOrFilePath: string, { silent, maxBytes, }?: {
|
|
134
|
+
silent?: boolean;
|
|
135
|
+
maxBytes?: number;
|
|
136
|
+
}): Promise<void>;
|
|
122
137
|
export type DatasetStatusEntry = {
|
|
123
138
|
status: "ok" | "failed";
|
|
124
139
|
lastRunAt: string;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Streaming reader for the XIV amendements single-file export.
|
|
3
|
+
*
|
|
4
|
+
* `Amendements_XIV.json` is a single 677MB JSON document. Node.js cannot
|
|
5
|
+
* create strings longer than 0x1fffffe8 (~537M) characters
|
|
6
|
+
* (ERR_STRING_TOO_LONG), so the file cannot be loaded whole with
|
|
7
|
+
* `fs.readFileSync` + `JSON.parse`. This scanner reads the document chunk
|
|
8
|
+
* by chunk and emits each element of the `textesEtAmendements.texteleg`
|
|
9
|
+
* array as a raw JSON string, keeping memory usage constant (each
|
|
10
|
+
* texteLegislatif element is well under 1MB).
|
|
11
|
+
*
|
|
12
|
+
* The document skeleton is validated on the fly: the root must be an
|
|
13
|
+
* object with a `textesEtAmendements` object whose `texteleg` member is an
|
|
14
|
+
* array (or null); unknown members of both objects are skipped, mirroring
|
|
15
|
+
* the non-strict behavior of the corresponding Zod object schemas.
|
|
16
|
+
* Per-element schema validation stays the responsibility of the caller.
|
|
17
|
+
*/
|
|
18
|
+
export declare class AmendementsXivStreamError extends Error {
|
|
19
|
+
byteOffset: number;
|
|
20
|
+
constructor(message: string, byteOffset: number);
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Streams the elements of the `textesEtAmendements.texteleg` array of a
|
|
24
|
+
* XIV amendements export. Calls `onTexteLegislatifJson` with the raw JSON
|
|
25
|
+
* string of every `texteleg` element, in document order.
|
|
26
|
+
*/
|
|
27
|
+
export declare function streamAmendementsXivFile(filePath: string, onTexteLegislatifJson: (json: string) => void, { highWaterMark }?: {
|
|
28
|
+
highWaterMark?: number;
|
|
29
|
+
}): Promise<void>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -91,6 +91,26 @@ export declare const fetchCrCommissionsOption: {
|
|
|
91
91
|
description: string;
|
|
92
92
|
type: z.ZodDefault<z.ZodBoolean>;
|
|
93
93
|
};
|
|
94
|
+
export declare const retrieveDatasetOption: {
|
|
95
|
+
description: string;
|
|
96
|
+
type: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
97
|
+
};
|
|
98
|
+
export declare const manifestDirOption: {
|
|
99
|
+
description: string;
|
|
100
|
+
type: z.ZodOptional<z.ZodString>;
|
|
101
|
+
};
|
|
102
|
+
export declare const onlyIfNewOption: {
|
|
103
|
+
description: string;
|
|
104
|
+
type: z.ZodDefault<z.ZodBoolean>;
|
|
105
|
+
};
|
|
106
|
+
export declare const probeCountOption: {
|
|
107
|
+
description: string;
|
|
108
|
+
type: z.ZodDefault<z.ZodNumber>;
|
|
109
|
+
};
|
|
110
|
+
export declare const dropIntermediatesOption: {
|
|
111
|
+
description: string;
|
|
112
|
+
type: z.ZodDefault<z.ZodBoolean>;
|
|
113
|
+
};
|
|
94
114
|
export declare const commonOptions: {
|
|
95
115
|
readonly categories: {
|
|
96
116
|
alias: string;
|