@tricoteuses/senat 2.22.4 → 2.22.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. package/lib/aggregates.d.ts +52 -0
  2. package/lib/aggregates.js +930 -0
  3. package/lib/aggregates.mjs +713 -0
  4. package/lib/aggregates.ts +833 -0
  5. package/lib/config.d.ts +10 -0
  6. package/lib/config.js +16 -0
  7. package/lib/config.mjs +16 -0
  8. package/lib/config.ts +26 -0
  9. package/lib/databases.d.ts +2 -0
  10. package/lib/databases.js +26 -0
  11. package/lib/databases.mjs +57 -0
  12. package/lib/databases.ts +71 -0
  13. package/lib/datasets.d.ts +34 -0
  14. package/lib/datasets.js +233 -0
  15. package/lib/datasets.mjs +78 -0
  16. package/lib/datasets.ts +118 -0
  17. package/lib/fields.d.ts +10 -0
  18. package/lib/fields.js +68 -0
  19. package/lib/fields.mjs +22 -0
  20. package/lib/fields.ts +29 -0
  21. package/lib/git.d.ts +26 -0
  22. package/lib/git.js +167 -0
  23. package/lib/index.d.ts +13 -0
  24. package/lib/index.js +1 -0
  25. package/lib/index.mjs +7 -0
  26. package/lib/index.ts +64 -0
  27. package/lib/inserters.d.ts +98 -0
  28. package/lib/inserters.js +500 -0
  29. package/lib/inserters.mjs +360 -0
  30. package/lib/inserters.ts +521 -0
  31. package/lib/legislatures.json +38 -0
  32. package/lib/loaders.d.ts +58 -0
  33. package/lib/loaders.js +286 -0
  34. package/lib/loaders.mjs +158 -0
  35. package/lib/loaders.ts +271 -0
  36. package/lib/model/agenda.d.ts +6 -0
  37. package/lib/model/agenda.js +148 -0
  38. package/lib/model/ameli.d.ts +51 -0
  39. package/lib/model/ameli.js +149 -0
  40. package/lib/model/ameli.mjs +84 -0
  41. package/lib/model/ameli.ts +100 -0
  42. package/lib/model/commission.d.ts +18 -0
  43. package/lib/model/commission.js +269 -0
  44. package/lib/model/debats.d.ts +67 -0
  45. package/lib/model/debats.js +95 -0
  46. package/lib/model/debats.mjs +43 -0
  47. package/lib/model/debats.ts +68 -0
  48. package/lib/model/documents.d.ts +12 -0
  49. package/lib/model/documents.js +151 -0
  50. package/lib/model/dosleg.d.ts +7 -0
  51. package/lib/model/dosleg.js +326 -0
  52. package/lib/model/dosleg.mjs +196 -0
  53. package/lib/model/dosleg.ts +240 -0
  54. package/lib/model/index.d.ts +7 -0
  55. package/lib/model/index.js +7 -0
  56. package/lib/model/index.mjs +5 -0
  57. package/lib/model/index.ts +15 -0
  58. package/lib/model/questions.d.ts +45 -0
  59. package/lib/model/questions.js +89 -0
  60. package/lib/model/questions.mjs +71 -0
  61. package/lib/model/questions.ts +93 -0
  62. package/lib/model/scrutins.d.ts +13 -0
  63. package/lib/model/scrutins.js +114 -0
  64. package/lib/model/seance.d.ts +3 -0
  65. package/lib/model/seance.js +267 -0
  66. package/lib/model/sens.d.ts +146 -0
  67. package/lib/model/sens.js +454 -0
  68. package/lib/model/sens.mjs +415 -0
  69. package/lib/model/sens.ts +516 -0
  70. package/lib/model/texte.d.ts +7 -0
  71. package/lib/model/texte.js +256 -0
  72. package/lib/model/texte.mjs +208 -0
  73. package/lib/model/texte.ts +229 -0
  74. package/lib/model/util.d.ts +9 -0
  75. package/lib/model/util.js +38 -0
  76. package/lib/model/util.mjs +19 -0
  77. package/lib/model/util.ts +32 -0
  78. package/lib/parsers/texte.d.ts +7 -0
  79. package/lib/parsers/texte.js +228 -0
  80. package/lib/raw_types/ameli.d.ts +914 -0
  81. package/lib/raw_types/ameli.js +5 -0
  82. package/lib/raw_types/ameli.mjs +163 -0
  83. package/lib/raw_types/debats.d.ts +207 -0
  84. package/lib/raw_types/debats.js +5 -0
  85. package/lib/raw_types/debats.mjs +58 -0
  86. package/lib/raw_types/dosleg.d.ts +1619 -0
  87. package/lib/raw_types/dosleg.js +5 -0
  88. package/lib/raw_types/dosleg.mjs +438 -0
  89. package/lib/raw_types/questions.d.ts +419 -0
  90. package/lib/raw_types/questions.js +5 -0
  91. package/lib/raw_types/questions.mjs +11 -0
  92. package/lib/raw_types/senat.d.ts +11368 -0
  93. package/lib/raw_types/senat.js +5 -0
  94. package/lib/raw_types/sens.d.ts +8248 -0
  95. package/lib/raw_types/sens.js +5 -0
  96. package/lib/raw_types/sens.mjs +508 -0
  97. package/lib/raw_types_kysely/ameli.d.ts +915 -0
  98. package/lib/raw_types_kysely/ameli.js +7 -0
  99. package/lib/raw_types_kysely/ameli.mjs +5 -0
  100. package/lib/raw_types_kysely/ameli.ts +951 -0
  101. package/lib/raw_types_kysely/debats.d.ts +207 -0
  102. package/lib/raw_types_kysely/debats.js +7 -0
  103. package/lib/raw_types_kysely/debats.mjs +5 -0
  104. package/lib/raw_types_kysely/debats.ts +222 -0
  105. package/lib/raw_types_kysely/dosleg.d.ts +3532 -0
  106. package/lib/raw_types_kysely/dosleg.js +7 -0
  107. package/lib/raw_types_kysely/dosleg.mjs +5 -0
  108. package/lib/raw_types_kysely/dosleg.ts +3621 -0
  109. package/lib/raw_types_kysely/questions.d.ts +414 -0
  110. package/lib/raw_types_kysely/questions.js +7 -0
  111. package/lib/raw_types_kysely/questions.mjs +5 -0
  112. package/lib/raw_types_kysely/questions.ts +426 -0
  113. package/lib/raw_types_kysely/sens.d.ts +4394 -0
  114. package/lib/raw_types_kysely/sens.js +7 -0
  115. package/lib/raw_types_kysely/sens.mjs +5 -0
  116. package/lib/raw_types_kysely/sens.ts +4499 -0
  117. package/lib/raw_types_schemats/ameli.d.ts +539 -0
  118. package/lib/raw_types_schemats/ameli.js +2 -0
  119. package/lib/raw_types_schemats/ameli.mjs +2 -0
  120. package/lib/raw_types_schemats/ameli.ts +601 -0
  121. package/lib/raw_types_schemats/debats.d.ts +127 -0
  122. package/lib/raw_types_schemats/debats.js +2 -0
  123. package/lib/raw_types_schemats/debats.mjs +2 -0
  124. package/lib/raw_types_schemats/debats.ts +145 -0
  125. package/lib/raw_types_schemats/dosleg.d.ts +977 -0
  126. package/lib/raw_types_schemats/dosleg.js +2 -0
  127. package/lib/raw_types_schemats/dosleg.mjs +2 -0
  128. package/lib/raw_types_schemats/dosleg.ts +2193 -0
  129. package/lib/raw_types_schemats/questions.d.ts +235 -0
  130. package/lib/raw_types_schemats/questions.js +2 -0
  131. package/lib/raw_types_schemats/questions.mjs +2 -0
  132. package/lib/raw_types_schemats/questions.ts +249 -0
  133. package/lib/raw_types_schemats/sens.d.ts +6915 -0
  134. package/lib/raw_types_schemats/sens.js +2 -0
  135. package/lib/raw_types_schemats/sens.mjs +2 -0
  136. package/lib/raw_types_schemats/sens.ts +2907 -0
  137. package/lib/scripts/convert_data.d.ts +1 -0
  138. package/lib/scripts/convert_data.js +354 -0
  139. package/lib/scripts/convert_data.mjs +181 -0
  140. package/lib/scripts/convert_data.ts +243 -0
  141. package/lib/scripts/data-download.d.ts +1 -0
  142. package/lib/scripts/data-download.js +12 -0
  143. package/lib/scripts/datautil.d.ts +8 -0
  144. package/lib/scripts/datautil.js +34 -0
  145. package/lib/scripts/datautil.mjs +16 -0
  146. package/lib/scripts/datautil.ts +19 -0
  147. package/lib/scripts/images/transparent_150x192.jpg +0 -0
  148. package/lib/scripts/images/transparent_155x225.jpg +0 -0
  149. package/lib/scripts/parse_textes.d.ts +1 -0
  150. package/lib/scripts/parse_textes.js +44 -0
  151. package/lib/scripts/parse_textes.mjs +46 -0
  152. package/lib/scripts/parse_textes.ts +65 -0
  153. package/lib/scripts/retrieve_agenda.d.ts +1 -0
  154. package/lib/scripts/retrieve_agenda.js +132 -0
  155. package/lib/scripts/retrieve_cr_commission.d.ts +1 -0
  156. package/lib/scripts/retrieve_cr_commission.js +364 -0
  157. package/lib/scripts/retrieve_cr_seance.d.ts +6 -0
  158. package/lib/scripts/retrieve_cr_seance.js +347 -0
  159. package/lib/scripts/retrieve_documents.d.ts +3 -0
  160. package/lib/scripts/retrieve_documents.js +219 -0
  161. package/lib/scripts/retrieve_documents.mjs +249 -0
  162. package/lib/scripts/retrieve_documents.ts +298 -0
  163. package/lib/scripts/retrieve_open_data.d.ts +1 -0
  164. package/lib/scripts/retrieve_open_data.js +315 -0
  165. package/lib/scripts/retrieve_open_data.mjs +217 -0
  166. package/lib/scripts/retrieve_open_data.ts +268 -0
  167. package/lib/scripts/retrieve_senateurs_photos.d.ts +1 -0
  168. package/lib/scripts/retrieve_senateurs_photos.js +147 -0
  169. package/lib/scripts/retrieve_senateurs_photos.mjs +147 -0
  170. package/lib/scripts/retrieve_senateurs_photos.ts +177 -0
  171. package/lib/scripts/retrieve_videos.d.ts +1 -0
  172. package/lib/scripts/retrieve_videos.js +461 -0
  173. package/lib/scripts/shared/cli_helpers.d.ts +95 -0
  174. package/lib/scripts/shared/cli_helpers.js +91 -0
  175. package/lib/scripts/shared/cli_helpers.ts +36 -0
  176. package/lib/scripts/shared/util.d.ts +4 -0
  177. package/lib/scripts/shared/util.js +35 -0
  178. package/lib/scripts/shared/util.ts +33 -0
  179. package/lib/scripts/test_iter_load.d.ts +1 -0
  180. package/lib/scripts/test_iter_load.js +12 -0
  181. package/lib/src/conversion_textes.js +10 -1
  182. package/lib/src/index.d.ts +1 -1
  183. package/lib/src/index.js +1 -1
  184. package/lib/src/loaders.d.ts +1 -0
  185. package/lib/src/loaders.js +1 -0
  186. package/lib/src/parsers/texte.js +2 -2
  187. package/lib/src/scripts/convert_data.js +11 -10
  188. package/lib/src/scripts/retrieve_documents.d.ts +1 -1
  189. package/lib/src/scripts/retrieve_documents.js +25 -10
  190. package/lib/src/scripts/retrieve_open_data.js +1 -1
  191. package/lib/src/types/sessions.d.ts +5 -4
  192. package/lib/src/types/sessions.js +10 -75
  193. package/lib/src/types/texte.d.ts +2 -0
  194. package/lib/strings.d.ts +1 -0
  195. package/lib/strings.js +18 -0
  196. package/lib/strings.mjs +18 -0
  197. package/lib/strings.ts +26 -0
  198. package/lib/types/agenda.d.ts +44 -0
  199. package/lib/types/agenda.js +1 -0
  200. package/lib/types/ameli.d.ts +5 -0
  201. package/lib/types/ameli.js +1 -0
  202. package/lib/types/ameli.mjs +13 -0
  203. package/lib/types/ameli.ts +21 -0
  204. package/lib/types/compte_rendu.d.ts +83 -0
  205. package/lib/types/compte_rendu.js +1 -0
  206. package/lib/types/debats.d.ts +2 -0
  207. package/lib/types/debats.js +1 -0
  208. package/lib/types/debats.mjs +2 -0
  209. package/lib/types/debats.ts +6 -0
  210. package/lib/types/dosleg.d.ts +70 -0
  211. package/lib/types/dosleg.js +1 -0
  212. package/lib/types/dosleg.mjs +151 -0
  213. package/lib/types/dosleg.ts +284 -0
  214. package/lib/types/questions.d.ts +2 -0
  215. package/lib/types/questions.js +1 -0
  216. package/lib/types/questions.mjs +1 -0
  217. package/lib/types/questions.ts +3 -0
  218. package/lib/types/sens.d.ts +10 -0
  219. package/lib/types/sens.js +1 -0
  220. package/lib/types/sens.mjs +1 -0
  221. package/lib/types/sens.ts +12 -0
  222. package/lib/types/sessions.d.ts +5 -0
  223. package/lib/types/sessions.js +84 -0
  224. package/lib/types/sessions.mjs +43 -0
  225. package/lib/types/sessions.ts +42 -0
  226. package/lib/types/texte.d.ts +74 -0
  227. package/lib/types/texte.js +16 -0
  228. package/lib/types/texte.mjs +16 -0
  229. package/lib/types/texte.ts +76 -0
  230. package/lib/typings/windows-1252.d.js +2 -0
  231. package/lib/typings/windows-1252.d.mjs +2 -0
  232. package/lib/typings/windows-1252.d.ts +11 -0
  233. package/lib/utils/cr_spliting.d.ts +28 -0
  234. package/lib/utils/cr_spliting.js +265 -0
  235. package/lib/utils/date.d.ts +10 -0
  236. package/lib/utils/date.js +100 -0
  237. package/lib/utils/nvs-timecode.d.ts +7 -0
  238. package/lib/utils/nvs-timecode.js +79 -0
  239. package/lib/utils/reunion_grouping.d.ts +9 -0
  240. package/lib/utils/reunion_grouping.js +361 -0
  241. package/lib/utils/reunion_odj_building.d.ts +5 -0
  242. package/lib/utils/reunion_odj_building.js +154 -0
  243. package/lib/utils/reunion_parsing.d.ts +23 -0
  244. package/lib/utils/reunion_parsing.js +209 -0
  245. package/lib/utils/scoring.d.ts +14 -0
  246. package/lib/utils/scoring.js +147 -0
  247. package/lib/utils/string_cleaning.d.ts +7 -0
  248. package/lib/utils/string_cleaning.js +57 -0
  249. package/lib/validators/config.d.ts +9 -0
  250. package/lib/validators/config.js +10 -0
  251. package/lib/validators/config.mjs +54 -0
  252. package/lib/validators/config.ts +79 -0
  253. package/lib/validators/senat.d.ts +0 -0
  254. package/lib/validators/senat.js +28 -0
  255. package/lib/validators/senat.mjs +24 -0
  256. package/lib/validators/senat.ts +26 -0
  257. package/package.json +5 -5
@@ -0,0 +1,95 @@
1
+ export declare const categoriesOption: {
2
+ alias: string;
3
+ defaultValue: string[];
4
+ help: string;
5
+ multiple: boolean;
6
+ name: string;
7
+ type: StringConstructor;
8
+ };
9
+ export declare const dataDirDefaultOption: {
10
+ defaultOption: boolean;
11
+ help: string;
12
+ name: string;
13
+ type: StringConstructor;
14
+ };
15
+ export declare const fromSessionOption: {
16
+ defaultValue: number;
17
+ help: string;
18
+ name: string;
19
+ type: NumberConstructor;
20
+ };
21
+ export declare const silentOption: {
22
+ alias: string;
23
+ help: string;
24
+ name: string;
25
+ type: BooleanConstructor;
26
+ };
27
+ export declare const verboseOption: {
28
+ alias: string;
29
+ help: string;
30
+ name: string;
31
+ type: BooleanConstructor;
32
+ };
33
+ export declare const onlyRecentOption: {
34
+ help: string;
35
+ name: string;
36
+ type: NumberConstructor;
37
+ };
38
+ export declare const keepDirOption: {
39
+ help: string;
40
+ name: string;
41
+ type: BooleanConstructor;
42
+ };
43
+ export declare const cloneOption: {
44
+ alias: string;
45
+ help: string;
46
+ name: string;
47
+ type: StringConstructor;
48
+ };
49
+ export declare const commitOption: {
50
+ help: string;
51
+ name: string;
52
+ type: BooleanConstructor;
53
+ };
54
+ export declare const remoteOption: {
55
+ alias: string;
56
+ help: string;
57
+ multiple: boolean;
58
+ name: string;
59
+ type: StringConstructor;
60
+ };
61
+ export declare const pullOption: {
62
+ alias: string;
63
+ help: string;
64
+ name: string;
65
+ type: BooleanConstructor;
66
+ };
67
+ export declare const fetchDocumentsOption: {
68
+ help: string;
69
+ name: string;
70
+ type: BooleanConstructor;
71
+ };
72
+ export declare const parseDocumentsOption: {
73
+ help: string;
74
+ name: string;
75
+ type: BooleanConstructor;
76
+ };
77
+ export declare const commonOptions: ({
78
+ defaultOption: boolean;
79
+ help: string;
80
+ name: string;
81
+ type: StringConstructor;
82
+ } | {
83
+ help: string;
84
+ name: string;
85
+ type: NumberConstructor;
86
+ } | {
87
+ help: string;
88
+ name: string;
89
+ type: BooleanConstructor;
90
+ } | {
91
+ alias: string;
92
+ help: string;
93
+ name: string;
94
+ type: StringConstructor;
95
+ })[];
@@ -0,0 +1,91 @@
1
+ export const categoriesOption = {
2
+ alias: "k",
3
+ defaultValue: ["All"],
4
+ help: "categories of datasets to reorganize; default All",
5
+ multiple: true,
6
+ name: "categories",
7
+ type: String,
8
+ };
9
+ export const dataDirDefaultOption = {
10
+ defaultOption: true,
11
+ help: "directory containing Sénat open data files",
12
+ name: "dataDir",
13
+ type: String,
14
+ };
15
+ export const fromSessionOption = {
16
+ defaultValue: 2022,
17
+ help: "session year to retrieve data from; default 2022",
18
+ name: "fromSession",
19
+ type: Number,
20
+ };
21
+ export const silentOption = {
22
+ alias: "s",
23
+ help: "don't log anything",
24
+ name: "silent",
25
+ type: Boolean,
26
+ };
27
+ export const verboseOption = {
28
+ alias: "v",
29
+ help: "verbose logs",
30
+ name: "verbose",
31
+ type: Boolean,
32
+ };
33
+ export const onlyRecentOption = {
34
+ help: "retrieve only documents created within the last N days (that are not already downloaded)",
35
+ name: "only-recent",
36
+ type: Number,
37
+ };
38
+ export const keepDirOption = {
39
+ help: "keep directories when cleaning data",
40
+ name: "keepDir",
41
+ type: Boolean,
42
+ };
43
+ export const cloneOption = {
44
+ alias: "C",
45
+ help: "clone repositories from given group (or organization) git URL",
46
+ name: "clone",
47
+ type: String,
48
+ };
49
+ export const commitOption = {
50
+ help: "commit clean files",
51
+ name: "commit",
52
+ type: Boolean,
53
+ };
54
+ export const remoteOption = {
55
+ alias: "r",
56
+ help: "push commit to given remote",
57
+ multiple: true,
58
+ name: "remote",
59
+ type: String,
60
+ };
61
+ export const pullOption = {
62
+ alias: "p",
63
+ help: "pull repositories before proceeding",
64
+ name: "pull",
65
+ type: Boolean,
66
+ };
67
+ export const fetchDocumentsOption = {
68
+ help: "download documents",
69
+ name: "fetchDocuments",
70
+ type: Boolean,
71
+ };
72
+ export const parseDocumentsOption = {
73
+ help: "parse documents",
74
+ name: "parseDocuments",
75
+ type: Boolean,
76
+ };
77
+ export const commonOptions = [
78
+ categoriesOption,
79
+ dataDirDefaultOption,
80
+ fromSessionOption,
81
+ silentOption,
82
+ verboseOption,
83
+ onlyRecentOption,
84
+ keepDirOption,
85
+ cloneOption,
86
+ commitOption,
87
+ remoteOption,
88
+ pullOption,
89
+ fetchDocumentsOption,
90
+ parseDocumentsOption,
91
+ ];
@@ -0,0 +1,36 @@
1
+ export const categoriesOption = {
2
+ alias: "k",
3
+ defaultValue: ["All"],
4
+ help: "categories of datasets to reorganize",
5
+ multiple: true,
6
+ name: "categories",
7
+ type: String,
8
+ }
9
+
10
+ export const dataDirDefaultOption = {
11
+ defaultOption: true,
12
+ help: "directory containing Sénat open data files",
13
+ name: "dataDir",
14
+ type: String,
15
+ }
16
+
17
+ export const silentOption = {
18
+ alias: "s",
19
+ help: "don't log anything",
20
+ name: "silent",
21
+ type: Boolean,
22
+ }
23
+
24
+ export const verboseOption = {
25
+ alias: "v",
26
+ help: "verbose logs",
27
+ name: "verbose",
28
+ type: Boolean,
29
+ }
30
+
31
+ export const commonOptions = [
32
+ categoriesOption,
33
+ dataDirDefaultOption,
34
+ silentOption,
35
+ verboseOption,
36
+ ]
@@ -0,0 +1,4 @@
1
+ export declare function isOptionEmptyOrHasValue(option: string, value: string): boolean;
2
+ export declare function ensureAndClearDirSync(dir: string): void;
3
+ export declare function ensureAndClearDir(path: string): void;
4
+ export declare function fetchWithRetry(url: string, retries?: number, backoff?: number): Promise<Response>;
@@ -0,0 +1,35 @@
1
+ import fs from "fs-extra";
2
+ import path from "path";
3
+ export function isOptionEmptyOrHasValue(option, value) {
4
+ return !option || option.length === 0 || option.includes(value);
5
+ }
6
+ export function ensureAndClearDirSync(dir) {
7
+ fs.ensureDirSync(dir);
8
+ for (const name of fs.readdirSync(dir)) {
9
+ fs.rmSync(path.join(dir, name), { recursive: true, force: true });
10
+ }
11
+ }
12
+ export function ensureAndClearDir(path) {
13
+ if (!fs.existsSync(path)) {
14
+ fs.mkdirSync(path, { recursive: true });
15
+ }
16
+ else {
17
+ fs.emptyDirSync(path);
18
+ }
19
+ }
20
+ export async function fetchWithRetry(url, retries = 3, backoff = 300) {
21
+ for (let attempt = 0; attempt < retries; attempt++) {
22
+ try {
23
+ return await fetch(url);
24
+ }
25
+ catch (error) {
26
+ if (attempt === retries - 1) {
27
+ throw error;
28
+ }
29
+ console.warn(`Fetch attempt ${attempt + 1} for ${url} failed. Retrying in ${backoff}ms…`);
30
+ await new Promise((resolve) => setTimeout(resolve, backoff));
31
+ backoff *= 2;
32
+ }
33
+ }
34
+ throw new Error(`Failed to fetch ${url} after ${retries} attempts`);
35
+ }
@@ -0,0 +1,33 @@
1
+ import fs from "fs-extra"
2
+
3
+ export function isOptionEmptyOrHasValue (option: string, value: string) {
4
+ return !option || option.length === 0 || option.includes(value)
5
+ }
6
+
7
+ export function ensureAndClearDir (path: string) {
8
+ if (!fs.existsSync(path)) {
9
+ fs.mkdirSync(path, { recursive: true })
10
+ } else {
11
+ fs.emptyDirSync(path)
12
+ }
13
+ }
14
+
15
+ export async function fetchWithRetry (
16
+ url: string,
17
+ retries: number = 3,
18
+ backoff: number = 300,
19
+ ): Promise<Response> {
20
+ for (let attempt = 0; attempt < retries; attempt++) {
21
+ try {
22
+ return await fetch(url)
23
+ } catch (error) {
24
+ if (attempt === retries - 1) {
25
+ throw error
26
+ }
27
+ console.warn(`Fetch attempt ${attempt + 1} for ${url} failed. Retrying in ${backoff}ms…`)
28
+ await new Promise((resolve) => setTimeout(resolve, backoff))
29
+ backoff *= 2
30
+ }
31
+ }
32
+ throw new Error(`Failed to fetch ${url} after ${retries} attempts`)
33
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,12 @@
1
+ import { iterLoadSenatRapports } from "../loaders";
2
+ import commandLineArgs from "command-line-args";
3
+ import { dataDirDefaultOption } from "./shared/cli_helpers";
4
+ const optionsDefinitions = [dataDirDefaultOption];
5
+ const options = commandLineArgs(optionsDefinitions);
6
+ const session = 2024;
7
+ const sinceCommit = undefined;
8
+ for (const { item: rapport } of iterLoadSenatRapports(options["dataDir"], session, {
9
+ sinceCommit: sinceCommit,
10
+ })) {
11
+ console.log(rapport);
12
+ }
@@ -70,7 +70,16 @@ export function extractMetadata(xmlDoc) {
70
70
  return metadata;
71
71
  }
72
72
  export async function convertSenatXmlToHtml(texteXml, outputFilePath) {
73
- const { document: xmlDoc } = new JSDOM(texteXml, { contentType: "text/xml" }).window;
73
+ let xmlDoc;
74
+ try {
75
+ xmlDoc = new JSDOM(texteXml, { contentType: "text/xml" }).window.document;
76
+ }
77
+ catch (err) {
78
+ if (await fs.pathExists(outputFilePath)) {
79
+ await fs.remove(outputFilePath);
80
+ }
81
+ throw err;
82
+ }
74
83
  const metadata = extractMetadata(xmlDoc);
75
84
  const xmlBody = xmlDoc.querySelector("body");
76
85
  const style = `
@@ -9,5 +9,5 @@ export type { CompteRendu } from "./types/compte_rendu";
9
9
  export type { Debat, LecAssDeb } from "./types/debats";
10
10
  export type { Ass, Aud, Auteur, DateSeance, DecCoc, DenRap, DocAtt, Ecr, EtaLoi, LecAss, LecAssRap, Lecture, Loi, Org, OriTxt, Qua, Rap, RapOrg, Scr, Texte, TypAtt, TypLec, TypLoi, TypTxt, TypUrl, } from "./types/dosleg";
11
11
  export type { Photo, Sen } from "./types/sens";
12
- export { UNDEFINED_SESSION, type Session, getSessionsFromStart } from "./types/sessions";
12
+ export { sessionOptions, sessionOptionsOrAll, UNDEFINED_SESSION, type Session, type SessionOrAll, getSessionsFromStart, } from "./types/sessions";
13
13
  export type { Alinea, Article, Division, DivisionTag, FlatTexte } from "./types/texte";
package/lib/src/index.js CHANGED
@@ -1 +1 @@
1
- export { UNDEFINED_SESSION, getSessionsFromStart } from "./types/sessions";
1
+ export { sessionOptions, sessionOptionsOrAll, UNDEFINED_SESSION, getSessionsFromStart, } from "./types/sessions";
@@ -14,6 +14,7 @@ export declare const AGENDA_FOLDER = "agenda";
14
14
  export declare const COMPTES_RENDUS_FOLDER = "seances";
15
15
  export declare const COMMISSION_FOLDER = "commissions";
16
16
  export declare const DOSLEG_DOSSIERS_FOLDER = "dossiers";
17
+ export declare const ENRICHED_TEXTE_FOLDER = "leg_enrichi";
17
18
  export declare const SCRUTINS_FOLDER = "scrutins";
18
19
  export declare const SENS_CIRCONSCRIPTIONS_FOLDER = "circonscriptions";
19
20
  export declare const SENS_ORGANISMES_FOLDER = "organismes";
@@ -9,6 +9,7 @@ export const AGENDA_FOLDER = "agenda";
9
9
  export const COMPTES_RENDUS_FOLDER = "seances";
10
10
  export const COMMISSION_FOLDER = "commissions";
11
11
  export const DOSLEG_DOSSIERS_FOLDER = "dossiers";
12
+ export const ENRICHED_TEXTE_FOLDER = "leg_enrichi";
12
13
  export const SCRUTINS_FOLDER = "scrutins";
13
14
  export const SENS_CIRCONSCRIPTIONS_FOLDER = "circonscriptions";
14
15
  export const SENS_ORGANISMES_FOLDER = "organismes";
@@ -62,7 +62,7 @@ function buildEmptyArticle(index) {
62
62
  alineas: [],
63
63
  };
64
64
  }
65
- function flattenTexte(texteContentRoot) {
65
+ function splitTexte(texteContentRoot) {
66
66
  const divisions = [];
67
67
  let divisionIndex = 0;
68
68
  const iter = (node) => {
@@ -129,7 +129,7 @@ export function transformTexte(document) {
129
129
  date_depot: dateDepot ? new Date(dateDepot) : null,
130
130
  date_publication_xml: datePublicationXml ? new Date(datePublicationXml) : null,
131
131
  version: identificationParts?.["version"] ? identificationParts["version"] : null,
132
- divisions: bodyElement ? flattenTexte(bodyElement) : [],
132
+ divisions: bodyElement ? splitTexte(bodyElement) : [],
133
133
  };
134
134
  }
135
135
  export function transformExposeDesMotifs(document) {
@@ -5,7 +5,7 @@ import path from "path";
5
5
  import pLimit from "p-limit";
6
6
  import * as git from "../git";
7
7
  import { datasets, EnabledDatasets, getEnabledDatasets } from "../datasets";
8
- import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, SCRUTINS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER, } from "../loaders";
8
+ import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, SCRUTINS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER, ENRICHED_TEXTE_FOLDER, } from "../loaders";
9
9
  import { findAllAmendements, findAllCirconscriptions, findAllDebats, findAllDossiers, findAllScrutins, findAllOrganismes, findAllQuestions, findAllSens, findAllTextes, findAllRapports, } from "../model";
10
10
  import { processRapport, processTexte } from "./retrieve_documents";
11
11
  import { buildActesLegislatifs } from "../model/dosleg";
@@ -40,7 +40,7 @@ async function convertData() {
40
40
  commitAndPushGit(ameliDir, options);
41
41
  }
42
42
  catch (error) {
43
- console.error(`Error converting Ameli dataset:`, error);
43
+ console.error("Error converting Ameli dataset:", error);
44
44
  }
45
45
  }
46
46
  if (enabledDatasets & EnabledDatasets.Debats) {
@@ -50,7 +50,7 @@ async function convertData() {
50
50
  commitAndPushGit(debatsDir, options);
51
51
  }
52
52
  catch (error) {
53
- console.error(`Error converting Debats dataset:`, error);
53
+ console.error("Error converting Debats dataset:", error);
54
54
  }
55
55
  }
56
56
  if (enabledDatasets & EnabledDatasets.DosLeg) {
@@ -60,7 +60,7 @@ async function convertData() {
60
60
  commitAndPushGit(doslegDir, options);
61
61
  }
62
62
  catch (error) {
63
- console.error(`Error converting DosLeg dataset:`, error);
63
+ console.error("Error converting DosLeg dataset:", error);
64
64
  }
65
65
  try {
66
66
  await convertDatasetScrutins(dataDir, options);
@@ -68,7 +68,7 @@ async function convertData() {
68
68
  commitAndPushGit(scrutinsDir, options);
69
69
  }
70
70
  catch (error) {
71
- console.error(`Error converting Scrutins dataset:`, error);
71
+ console.error("Error converting Scrutins dataset:", error);
72
72
  }
73
73
  }
74
74
  if (enabledDatasets & EnabledDatasets.Questions) {
@@ -78,7 +78,7 @@ async function convertData() {
78
78
  commitAndPushGit(questionsDir, options);
79
79
  }
80
80
  catch (error) {
81
- console.error(`Error converting Questions dataset:`, error);
81
+ console.error("Error converting Questions dataset:", error);
82
82
  }
83
83
  }
84
84
  if (enabledDatasets & EnabledDatasets.Sens) {
@@ -88,7 +88,7 @@ async function convertData() {
88
88
  commitAndPushGit(sensDir, options);
89
89
  }
90
90
  catch (error) {
91
- console.error(`Error converting Sens dataset:`, error);
91
+ console.error("Error converting Sens dataset:", error);
92
92
  }
93
93
  }
94
94
  if (!options["silent"]) {
@@ -220,8 +220,9 @@ async function convertDatasetQuestions(dataDir, options) {
220
220
  async function convertTextes(dataDir, options) {
221
221
  const originalTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_ORIGINAL_FOLDER);
222
222
  const transformedTextesDir = path.join(dataDir, TEXTE_FOLDER, DATA_TRANSFORMED_FOLDER);
223
+ const enrichedTextesDir = path.join(dataDir, ENRICHED_TEXTE_FOLDER);
223
224
  if (!options["silent"]) {
224
- console.log(`Converting database textes data into files…`);
225
+ console.log("Converting database textes data into files…");
225
226
  }
226
227
  for await (const texte of findAllTextes()) {
227
228
  const session = texte["session"] ?? UNDEFINED_SESSION;
@@ -253,14 +254,14 @@ async function convertTextes(dataDir, options) {
253
254
  spaces: 2,
254
255
  });
255
256
  if (options.fetchDocuments) {
256
- await processTexte(metadata, originalTextesDir, transformedTextesDir, options);
257
+ await processTexte(metadata, originalTextesDir, transformedTextesDir, enrichedTextesDir, options);
257
258
  }
258
259
  }
259
260
  }
260
261
  async function convertRapports(dataDir, options) {
261
262
  const originalRapportsDir = path.join(dataDir, RAPPORT_FOLDER, DATA_ORIGINAL_FOLDER);
262
263
  if (!options["silent"]) {
263
- console.log(`Converting database rapports data into files…`);
264
+ console.log("Converting database rapports data into files…");
264
265
  }
265
266
  for await (const rapport of findAllRapports()) {
266
267
  const session = rapport["session"] ?? UNDEFINED_SESSION;
@@ -1,3 +1,3 @@
1
1
  import { DocumentMetadata } from "../types/texte";
2
- export declare function processTexte(texteMetadata: DocumentMetadata, originalTextesDir: string, transformedTextesDir: string, options: any): Promise<void>;
2
+ export declare function processTexte(texteMetadata: DocumentMetadata, originalTextesDir: string, transformedTextesDir: string, enrichedTextesDir: string, options: any): Promise<void>;
3
3
  export declare function processRapport(rapportMetadata: any, originalRapportsDir: string, options: any): Promise<void>;
@@ -3,8 +3,9 @@ import commandLineArgs from "command-line-args";
3
3
  import fs from "fs-extra";
4
4
  import { DateTime } from "luxon";
5
5
  import path from "path";
6
+ import { convertSenatXmlToHtml } from "../conversion_textes";
6
7
  import * as git from "../git";
7
- import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatRapportUrls, iterLoadSenatTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, } from "../loaders";
8
+ import { DATA_ORIGINAL_FOLDER, DATA_TRANSFORMED_FOLDER, ENRICHED_TEXTE_FOLDER, iterLoadSenatRapportUrls, iterLoadSenatTexteUrls, RAPPORT_FOLDER, TEXTE_FOLDER, } from "../loaders";
8
9
  import { parseExposeDesMotifs, parseTexte, parseTexteFromFile } from "../parsers/texte";
9
10
  import { getSessionsFromStart, UNDEFINED_SESSION } from "../types/sessions";
10
11
  import { commonOptions } from "./shared/cli_helpers";
@@ -98,7 +99,7 @@ async function processDocument(url, destPath, docDate, options) {
98
99
  await fs.outputFile(destPath, buffer);
99
100
  return { success: true, skipped: false, buffer };
100
101
  }
101
- export async function processTexte(texteMetadata, originalTextesDir, transformedTextesDir, options) {
102
+ export async function processTexte(texteMetadata, originalTextesDir, transformedTextesDir, enrichedTextesDir, options) {
102
103
  const texteDir = path.join(originalTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name);
103
104
  let exposeDesMotifsContent = null;
104
105
  if (texteMetadata.url_expose_des_motifs) {
@@ -121,19 +122,30 @@ export async function processTexte(texteMetadata, originalTextesDir, transformed
121
122
  for (const format of formats) {
122
123
  if (!isOptionEmptyOrHasValue(options.formats, format.type))
123
124
  continue;
124
- if (!format.url)
125
+ if (format.url === undefined || format.url.toString().includes("#"))
125
126
  continue;
126
127
  const destPath = path.join(texteDir, `${texteMetadata.name}.${format.type}`);
127
128
  const result = await processDocument(format.url.toString(), destPath, texteMetadata.date, options);
128
129
  // Specific logic: Parsing (Only applies to XML)
129
130
  if (format.isParseTarget && options.parseDocuments) {
130
- if (!result.buffer && !(await fs.pathExists(destPath))) {
131
- if (options.verbose) {
132
- console.warn(`Skipping parse for missing XML file: ${destPath}`);
131
+ if (result.buffer !== null || (await fs.pathExists(destPath))) {
132
+ await parseDocument(texteMetadata.session, transformedTextesDir, destPath, texteMetadata.name, result.buffer, exposeDesMotifsContent, options);
133
+ }
134
+ let texteXmlContent = null;
135
+ if (result.buffer === null && (await fs.pathExists(destPath))) {
136
+ texteXmlContent = await fs.readFile(destPath, "utf-8");
137
+ }
138
+ else if (result.buffer !== null) {
139
+ texteXmlContent = textDecoder.decode(result.buffer);
140
+ }
141
+ if (texteXmlContent !== null) {
142
+ try {
143
+ await convertSenatXmlToHtml(texteXmlContent, path.join(enrichedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name, `${texteMetadata.name}.html`));
144
+ }
145
+ catch (error) {
146
+ console.error(`Error converting ${texteMetadata.name} to HTML: ${error.message}`);
133
147
  }
134
- continue;
135
148
  }
136
- await parseDocument(texteMetadata.session, transformedTextesDir, destPath, texteMetadata.name, result.buffer, exposeDesMotifsContent, options);
137
149
  }
138
150
  }
139
151
  }
@@ -154,8 +166,10 @@ async function processTextes(dataDir, sessions) {
154
166
  const textesDir = path.join(dataDir, TEXTE_FOLDER);
155
167
  const originalTextesDir = path.join(textesDir, DATA_ORIGINAL_FOLDER);
156
168
  const transformedTextesDir = path.join(textesDir, DATA_TRANSFORMED_FOLDER);
157
- if (options["parseDocuments"]) {
169
+ const enrichedTextesDir = path.join(dataDir, ENRICHED_TEXTE_FOLDER);
170
+ if (options["parseDocuments"] && options["only-recent"] === undefined) {
158
171
  ensureAndClearDir(transformedTextesDir);
172
+ ensureAndClearDir(enrichedTextesDir);
159
173
  }
160
174
  const dlOptions = {
161
175
  force: options["force"],
@@ -167,10 +181,11 @@ async function processTextes(dataDir, sessions) {
167
181
  };
168
182
  for (const session of sessions) {
169
183
  for (const { item: texteMetadata } of iterLoadSenatTexteUrls(dataDir, session)) {
170
- await processTexte(texteMetadata, originalTextesDir, transformedTextesDir, dlOptions);
184
+ await processTexte(texteMetadata, originalTextesDir, transformedTextesDir, enrichedTextesDir, dlOptions);
171
185
  }
172
186
  }
173
187
  commitAndPushGit(textesDir);
188
+ commitAndPushGit(enrichedTextesDir);
174
189
  }
175
190
  async function processRapports(dataDir, sessions) {
176
191
  const rapportsDir = path.join(dataDir, RAPPORT_FOLDER);
@@ -297,7 +297,7 @@ async function retrieveOpenData() {
297
297
  if (options["schema"]) {
298
298
  const dbConnectionString = `postgres://${process.env["PGUSER"]}:${process.env["PGPASSWORD"]}@${process.env["PGHOST"]}:${process.env["PGPORT"]}/senat`;
299
299
  const definitionsDir = path.resolve("src", "raw_types");
300
- const definitionFilePath = path.join(definitionsDir, `senat.ts`);
300
+ const definitionFilePath = path.join(definitionsDir, "senat.ts");
301
301
  execSync(`npx kysely-codegen --url '${dbConnectionString}' --out-file ${definitionFilePath}`, {
302
302
  env: process.env,
303
303
  encoding: "utf-8",
@@ -1,5 +1,6 @@
1
1
  export declare const UNDEFINED_SESSION = 0;
2
- declare const sessions: readonly [0, 1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026];
3
- export type Session = (typeof sessions)[number];
4
- export declare function getSessionsFromStart(startSession: Session): (0 | 1958 | 1959 | 1960 | 1961 | 1962 | 1963 | 1964 | 1965 | 1966 | 1967 | 1968 | 1969 | 1970 | 1971 | 1972 | 1973 | 1974 | 1975 | 1976 | 1977 | 1978 | 1979 | 1980 | 1981 | 1982 | 1983 | 1984 | 1985 | 1986 | 1987 | 1988 | 1989 | 1990 | 1991 | 1992 | 1993 | 1994 | 1995 | 1996 | 1997 | 1998 | 1999 | 2000 | 2001 | 2002 | 2003 | 2004 | 2005 | 2006 | 2007 | 2008 | 2009 | 2010 | 2011 | 2012 | 2013 | 2014 | 2015 | 2016 | 2017 | 2018 | 2019 | 2020 | 2021 | 2022 | 2023 | 2024 | 2025 | 2026)[];
5
- export {};
2
+ export declare const sessionOptions: readonly [1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026];
3
+ export declare const sessionOptionsOrAll: readonly [0, 1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026];
4
+ export type Session = (typeof sessionOptions)[number];
5
+ export type SessionOrAll = (typeof sessionOptionsOrAll)[number];
6
+ export declare function getSessionsFromStart(startSession: SessionOrAll): (1958 | 1959 | 1960 | 1961 | 1962 | 1963 | 1964 | 1965 | 1966 | 1967 | 1968 | 1969 | 1970 | 1971 | 1972 | 1973 | 1974 | 1975 | 1976 | 1977 | 1978 | 1979 | 1980 | 1981 | 1982 | 1983 | 1984 | 1985 | 1986 | 1987 | 1988 | 1989 | 1990 | 1991 | 1992 | 1993 | 1994 | 1995 | 1996 | 1997 | 1998 | 1999 | 2000 | 2001 | 2002 | 2003 | 2004 | 2005 | 2006 | 2007 | 2008 | 2009 | 2010 | 2011 | 2012 | 2013 | 2014 | 2015 | 2016 | 2017 | 2018 | 2019 | 2020 | 2021 | 2022 | 2023 | 2024 | 2025 | 2026)[];