@tricoteuses/assemblee 3.8.3 → 3.8.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/README.md +2 -2
  2. package/lib/{amendements-BYKlEcWb.js → amendements-BtCdrXW8.js} +36 -33
  3. package/lib/{amendements-CvlD2dI4.js → amendements-DfZse75q.js} +924 -672
  4. package/lib/amendements-xiii.d.ts +56 -0
  5. package/lib/amendements-xiii.test.d.ts +1 -0
  6. package/lib/browser-Dl6SgoZX.js +8164 -0
  7. package/lib/cleaners/amendements-xiv.test.d.ts +1 -0
  8. package/lib/cleaners/amendements.d.ts +18 -1
  9. package/lib/cleaners.js +3 -3
  10. package/lib/index.d.ts +5 -2
  11. package/lib/index.js +596 -99
  12. package/lib/inserters.d.ts +2 -2
  13. package/lib/{loaders-BNXs79lC.js → loaders-Dd8i-E7o.js} +11 -2
  14. package/lib/loaders.js +1 -1
  15. package/lib/open_data/artifacts.d.ts +12 -0
  16. package/lib/parsers.js +359 -8521
  17. package/lib/schemas/clean/amendements-xiii.d.ts +579 -0
  18. package/lib/schemas/clean/amendements.d.ts +502 -11
  19. package/lib/schemas/enriched/amendements.d.ts +6 -6
  20. package/lib/schemas/raw/amendements-xiii.d.ts +754 -0
  21. package/lib/schemas/raw/amendements-xiv.test.d.ts +1 -0
  22. package/lib/schemas/raw/amendements.d.ts +1769 -2
  23. package/lib/schemas/raw/amendements.test.d.ts +1 -0
  24. package/lib/scripts/clean_reorganized_data.test.d.ts +1 -0
  25. package/lib/scripts/data-download.d.ts +132 -0
  26. package/lib/scripts/retrieve_amendements_xiii.d.ts +51 -0
  27. package/lib/scripts/retrieve_open_data.d.ts +39 -24
  28. package/lib/scripts/shared/amendements_xiv_stream.d.ts +29 -0
  29. package/lib/scripts/shared/amendements_xiv_stream.test.d.ts +1 -0
  30. package/lib/scripts/shared/cli_helpers.d.ts +20 -0
  31. package/lib/{scrutins-BUVxV-QV.js → scrutins-CLWnMYLE.js} +266 -266
  32. package/package.json +7 -7
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1 @@
1
+ export {};
@@ -1,4 +1,136 @@
1
1
  import { ConversionScope } from '../open_data/artifacts.js';
2
+ /**
3
+ * Full option surface accepted by data-download: everything it forwards to
4
+ * retrieve/reorganize/clean, plus its own `--drop-intermediates`.
5
+ * Sub-scripts don't declare the drop flag (they must not silently ignore
6
+ * it), so it is stripped from the forwarded argument strings below.
7
+ */
8
+ export declare const dataDownloadParser: import('zod-opts').Parser<{
9
+ dataset: {
10
+ description: string;
11
+ type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodString>>;
12
+ };
13
+ fetch: {
14
+ alias: string;
15
+ description: string;
16
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
17
+ };
18
+ "manifest-dir": {
19
+ description: string;
20
+ type: import('zod').ZodOptional<import('zod').ZodString>;
21
+ };
22
+ "only-if-new": {
23
+ description: string;
24
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
25
+ };
26
+ "probe-count": {
27
+ description: string;
28
+ type: import('zod').ZodDefault<import('zod').ZodNumber>;
29
+ };
30
+ "drop-intermediates": {
31
+ description: string;
32
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
33
+ };
34
+ categories: {
35
+ alias: string;
36
+ description: string;
37
+ type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodString>>;
38
+ };
39
+ legislature: {
40
+ alias: string;
41
+ description: string;
42
+ type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodNumber>>;
43
+ };
44
+ silent: {
45
+ alias: string;
46
+ description: string;
47
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
48
+ };
49
+ verbose: {
50
+ alias: string;
51
+ description: string;
52
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
53
+ };
54
+ "only-recent": {
55
+ description: string;
56
+ type: import('zod').ZodOptional<import('zod').ZodNumber>;
57
+ };
58
+ skipReindent: {
59
+ description: string;
60
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
61
+ };
62
+ keepDir: {
63
+ description: string;
64
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
65
+ };
66
+ clone: {
67
+ alias: string;
68
+ description: string;
69
+ type: import('zod').ZodOptional<import('zod').ZodString>;
70
+ };
71
+ commit: {
72
+ alias: string;
73
+ description: string;
74
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
75
+ };
76
+ remote: {
77
+ alias: string;
78
+ description: string;
79
+ type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodString>>;
80
+ };
81
+ pull: {
82
+ alias: string;
83
+ description: string;
84
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
85
+ };
86
+ fetchDocuments: {
87
+ description: string;
88
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
89
+ };
90
+ parseDocuments: {
91
+ description: string;
92
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
93
+ };
94
+ format: {
95
+ alias: string;
96
+ description: string;
97
+ type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodString>>;
98
+ };
99
+ fetchVideos: {
100
+ description: string;
101
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
102
+ };
103
+ fetchCrCommissions: {
104
+ description: string;
105
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
106
+ };
107
+ }, [{
108
+ description: string;
109
+ type: import('zod').ZodString;
110
+ name: "dataDir";
111
+ }]>;
112
+ export type DataDownloadOptions = ReturnType<typeof dataDownloadParser.parse>;
113
+ /**
114
+ * Remove the locally-consumed drop flag so sub-scripts (which don't declare
115
+ * it) never see it. Same routing rationale as RETRIEVE_ONLY_OPTIONS above.
116
+ */
117
+ export declare function stripDropFlag(argv: string[]): string[];
118
+ export interface DropResult {
119
+ datasetName: string;
120
+ removed: string[];
121
+ skippedReason?: string;
122
+ }
123
+ /**
124
+ * Delete per-dataset intermediates (published ZIP + resume leftovers +
125
+ * extracted original + reorganized directory), keeping only the final
126
+ * `*_nettoye/` output (+ its git history), manifests and conversion scope.
127
+ *
128
+ * A dataset is only pruned when its `_nettoye/` directory exists and is
129
+ * non-empty — that is the downstream contract (`api-parlement` reads only
130
+ * `*_nettoye`). Unknown dataset names are skipped, never guessed. An empty
131
+ * name list fails open (nothing is deleted).
132
+ */
133
+ export declare function dropIntermediates(dataDir: string, datasetNames: string[]): Promise<DropResult[]>;
2
134
  export declare function dataDirArg(argv: string[]): string;
3
135
  /**
4
136
  * Returns the datasets to process. `undefined` means the scope is unknown, so
@@ -0,0 +1,51 @@
1
+ import { AmendementXiiiRaw } from '../schemas/raw/amendements-xiii.js';
2
+ import { AmendementXiiiClean } from '../schemas/clean/amendements-xiii.js';
3
+ /**
4
+ * List the amendment URLs of one text by paging through the archive search.
5
+ *
6
+ * The listing is the only enumeration endpoint: `/13/amendements/` is not
7
+ * browsable and slugs are not predictable. `offset` paginates by `nbres`.
8
+ */
9
+ export declare function listAmendementUrls(texte: string, { limit, retries, nbres, }?: {
10
+ limit?: number;
11
+ retries?: number;
12
+ nbres?: number;
13
+ }): Promise<string[]>;
14
+ /**
15
+ * List every amendment URL of the legislature by paging through the global
16
+ * archive search (no `numInit` filter), which doubles as the enumeration of
17
+ * the texts.
18
+ *
19
+ * The search announces 49 140 rows for `leg=13`; the endpoint pages by
20
+ * `nbres` and repeats the tail on the last pages, hence the `added === 0`
21
+ * stop.
22
+ */
23
+ export declare function listAllAmendementUrls({ logProgress, nbres, retries, }?: {
24
+ logProgress?: boolean;
25
+ nbres?: number;
26
+ retries?: number;
27
+ }): Promise<string[]>;
28
+ type CollectResult = {
29
+ clean: AmendementXiiiClean[];
30
+ errors: Array<{
31
+ url: string;
32
+ message: string;
33
+ }>;
34
+ raw: AmendementXiiiRaw[];
35
+ titre: string;
36
+ };
37
+ /**
38
+ * Fetch and parse every amendment of one text.
39
+ *
40
+ * A page that fails to parse is reported, not fatal: a run over a whole
41
+ * legislature must not lose 49 000 pages because of one notice page.
42
+ */
43
+ export declare function collectTexte(texte: string, { concurrency, limit, retries, verbose, urls, }?: {
44
+ concurrency?: number;
45
+ limit?: number;
46
+ retries?: number;
47
+ verbose?: boolean;
48
+ /** URLs already listed (full-legislature mode); skips the listing. */
49
+ urls?: string[];
50
+ }): Promise<CollectResult>;
51
+ export {};
@@ -1,4 +1,3 @@
1
- import { z } from 'zod';
2
1
  import { Dataset } from '../datasets.js';
3
2
  type ProcessDatasetArguments = {
4
3
  dataset: Dataset;
@@ -8,106 +7,117 @@ type ProcessDatasetArguments = {
8
7
  declare const retrieveOpenDataParser: import('zod-opts').Parser<{
9
8
  dataset: {
10
9
  description: string;
11
- type: z.ZodDefault<z.ZodArray<z.ZodString>>;
10
+ type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodString>>;
12
11
  };
13
12
  fetch: {
14
13
  alias: string;
15
14
  description: string;
16
- type: z.ZodDefault<z.ZodBoolean>;
15
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
17
16
  };
18
17
  "manifest-dir": {
19
18
  description: string;
20
- type: z.ZodOptional<z.ZodString>;
19
+ type: import('zod').ZodOptional<import('zod').ZodString>;
21
20
  };
22
21
  "only-if-new": {
23
22
  description: string;
24
- type: z.ZodDefault<z.ZodBoolean>;
23
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
25
24
  };
26
25
  "probe-count": {
27
26
  description: string;
28
- type: z.ZodDefault<z.ZodNumber>;
27
+ type: import('zod').ZodDefault<import('zod').ZodNumber>;
29
28
  };
30
29
  categories: {
31
30
  alias: string;
32
31
  description: string;
33
- type: z.ZodDefault<z.ZodArray<z.ZodString>>;
32
+ type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodString>>;
34
33
  };
35
34
  legislature: {
36
35
  alias: string;
37
36
  description: string;
38
- type: z.ZodDefault<z.ZodArray<z.ZodNumber>>;
37
+ type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodNumber>>;
39
38
  };
40
39
  silent: {
41
40
  alias: string;
42
41
  description: string;
43
- type: z.ZodDefault<z.ZodBoolean>;
42
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
44
43
  };
45
44
  verbose: {
46
45
  alias: string;
47
46
  description: string;
48
- type: z.ZodDefault<z.ZodBoolean>;
47
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
49
48
  };
50
49
  "only-recent": {
51
50
  description: string;
52
- type: z.ZodOptional<z.ZodNumber>;
51
+ type: import('zod').ZodOptional<import('zod').ZodNumber>;
53
52
  };
54
53
  skipReindent: {
55
54
  description: string;
56
- type: z.ZodDefault<z.ZodBoolean>;
55
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
57
56
  };
58
57
  keepDir: {
59
58
  description: string;
60
- type: z.ZodDefault<z.ZodBoolean>;
59
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
61
60
  };
62
61
  clone: {
63
62
  alias: string;
64
63
  description: string;
65
- type: z.ZodOptional<z.ZodString>;
64
+ type: import('zod').ZodOptional<import('zod').ZodString>;
66
65
  };
67
66
  commit: {
68
67
  alias: string;
69
68
  description: string;
70
- type: z.ZodDefault<z.ZodBoolean>;
69
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
71
70
  };
72
71
  remote: {
73
72
  alias: string;
74
73
  description: string;
75
- type: z.ZodDefault<z.ZodArray<z.ZodString>>;
74
+ type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodString>>;
76
75
  };
77
76
  pull: {
78
77
  alias: string;
79
78
  description: string;
80
- type: z.ZodDefault<z.ZodBoolean>;
79
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
81
80
  };
82
81
  fetchDocuments: {
83
82
  description: string;
84
- type: z.ZodDefault<z.ZodBoolean>;
83
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
85
84
  };
86
85
  parseDocuments: {
87
86
  description: string;
88
- type: z.ZodDefault<z.ZodBoolean>;
87
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
89
88
  };
90
89
  format: {
91
90
  alias: string;
92
91
  description: string;
93
- type: z.ZodDefault<z.ZodArray<z.ZodString>>;
92
+ type: import('zod').ZodDefault<import('zod').ZodArray<import('zod').ZodString>>;
94
93
  };
95
94
  fetchVideos: {
96
95
  description: string;
97
- type: z.ZodDefault<z.ZodBoolean>;
96
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
98
97
  };
99
98
  fetchCrCommissions: {
100
99
  description: string;
101
- type: z.ZodDefault<z.ZodBoolean>;
100
+ type: import('zod').ZodDefault<import('zod').ZodBoolean>;
102
101
  };
103
102
  }, [{
104
103
  description: string;
105
- type: z.ZodString;
104
+ type: import('zod').ZodString;
106
105
  name: "dataDir";
107
106
  }]>;
108
107
  type Options = ReturnType<typeof retrieveOpenDataParser.parse>;
109
108
  export declare function formatParisDateTime(value: string): string;
110
- export declare function extractZip(zipFilePath: string, outputDir: string): Promise<void>;
109
+ /**
110
+ * Extracts a ZIP archive into `outputDir`.
111
+ *
112
+ * Uses the system `unzip` (Info-ZIP) when available: on archives made of many
113
+ * small files (AMO50: 36k files, Amendements_XVII: 148k files) it is ~10×
114
+ * faster and uses ~10× less CPU than node-stream-zip, with a constant ~4 MB
115
+ * memory footprint. Falls back to node-stream-zip when `unzip` is not
116
+ * installed, so library consumers without it keep working.
117
+ */
118
+ export declare function extractZip(zipFilePath: string, outputDir: string, { unzipCommand }?: {
119
+ unzipCommand?: string | null;
120
+ }): Promise<void>;
111
121
  export declare function publishDataset(stagedDataPath: string, dataPath: string, partZipPath: string | undefined, zipPath: string): Promise<void>;
112
122
  export declare function recoverPublication(dataPath: string): Promise<void>;
113
123
  /**
@@ -119,6 +129,11 @@ export declare function recoverPublication(dataPath: string): Promise<void>;
119
129
  * @returns {Promise<void>} A promise that resolves when the dataset processing is complete.
120
130
  */
121
131
  export declare function processDataset({ dataset, dataDir, options, }: ProcessDatasetArguments): Promise<void>;
132
+ export declare const reindentMaxBytes: number;
133
+ export declare function reindentJsonFiles(dataDirOrFilePath: string, { silent, maxBytes, }?: {
134
+ silent?: boolean;
135
+ maxBytes?: number;
136
+ }): Promise<void>;
122
137
  export type DatasetStatusEntry = {
123
138
  status: "ok" | "failed";
124
139
  lastRunAt: string;
@@ -0,0 +1,29 @@
1
+ /**
2
+ * Streaming reader for the XIV amendements single-file export.
3
+ *
4
+ * `Amendements_XIV.json` is a single 677MB JSON document. Node.js cannot
5
+ * create strings longer than 0x1fffffe8 (~537M) characters
6
+ * (ERR_STRING_TOO_LONG), so the file cannot be loaded whole with
7
+ * `fs.readFileSync` + `JSON.parse`. This scanner reads the document chunk
8
+ * by chunk and emits each element of the `textesEtAmendements.texteleg`
9
+ * array as a raw JSON string, keeping memory usage constant (each
10
+ * texteLegislatif element is well under 1MB).
11
+ *
12
+ * The document skeleton is validated on the fly: the root must be an
13
+ * object with a `textesEtAmendements` object whose `texteleg` member is an
14
+ * array (or null); unknown members of both objects are skipped, mirroring
15
+ * the non-strict behavior of the corresponding Zod object schemas.
16
+ * Per-element schema validation stays the responsibility of the caller.
17
+ */
18
+ export declare class AmendementsXivStreamError extends Error {
19
+ byteOffset: number;
20
+ constructor(message: string, byteOffset: number);
21
+ }
22
+ /**
23
+ * Streams the elements of the `textesEtAmendements.texteleg` array of a
24
+ * XIV amendements export. Calls `onTexteLegislatifJson` with the raw JSON
25
+ * string of every `texteleg` element, in document order.
26
+ */
27
+ export declare function streamAmendementsXivFile(filePath: string, onTexteLegislatifJson: (json: string) => void, { highWaterMark }?: {
28
+ highWaterMark?: number;
29
+ }): Promise<void>;
@@ -91,6 +91,26 @@ export declare const fetchCrCommissionsOption: {
91
91
  description: string;
92
92
  type: z.ZodDefault<z.ZodBoolean>;
93
93
  };
94
+ export declare const retrieveDatasetOption: {
95
+ description: string;
96
+ type: z.ZodDefault<z.ZodArray<z.ZodString>>;
97
+ };
98
+ export declare const manifestDirOption: {
99
+ description: string;
100
+ type: z.ZodOptional<z.ZodString>;
101
+ };
102
+ export declare const onlyIfNewOption: {
103
+ description: string;
104
+ type: z.ZodDefault<z.ZodBoolean>;
105
+ };
106
+ export declare const probeCountOption: {
107
+ description: string;
108
+ type: z.ZodDefault<z.ZodNumber>;
109
+ };
110
+ export declare const dropIntermediatesOption: {
111
+ description: string;
112
+ type: z.ZodDefault<z.ZodBoolean>;
113
+ };
94
114
  export declare const commonOptions: {
95
115
  readonly categories: {
96
116
  alias: string;