@tricoteuses/senat 2.22.3 → 2.22.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (258) hide show
  1. package/lib/aggregates.d.ts +52 -0
  2. package/lib/aggregates.js +930 -0
  3. package/lib/aggregates.mjs +713 -0
  4. package/lib/aggregates.ts +833 -0
  5. package/lib/config.d.ts +10 -0
  6. package/lib/config.js +16 -0
  7. package/lib/config.mjs +16 -0
  8. package/lib/config.ts +26 -0
  9. package/lib/databases.d.ts +2 -0
  10. package/lib/databases.js +26 -0
  11. package/lib/databases.mjs +57 -0
  12. package/lib/databases.ts +71 -0
  13. package/lib/datasets.d.ts +34 -0
  14. package/lib/datasets.js +233 -0
  15. package/lib/datasets.mjs +78 -0
  16. package/lib/datasets.ts +118 -0
  17. package/lib/fields.d.ts +10 -0
  18. package/lib/fields.js +68 -0
  19. package/lib/fields.mjs +22 -0
  20. package/lib/fields.ts +29 -0
  21. package/lib/git.d.ts +26 -0
  22. package/lib/git.js +167 -0
  23. package/lib/index.d.ts +13 -0
  24. package/lib/index.js +1 -0
  25. package/lib/index.mjs +7 -0
  26. package/lib/index.ts +64 -0
  27. package/lib/inserters.d.ts +98 -0
  28. package/lib/inserters.js +500 -0
  29. package/lib/inserters.mjs +360 -0
  30. package/lib/inserters.ts +521 -0
  31. package/lib/legislatures.json +38 -0
  32. package/lib/loaders.d.ts +58 -0
  33. package/lib/loaders.js +286 -0
  34. package/lib/loaders.mjs +158 -0
  35. package/lib/loaders.ts +271 -0
  36. package/lib/model/agenda.d.ts +6 -0
  37. package/lib/model/agenda.js +148 -0
  38. package/lib/model/ameli.d.ts +51 -0
  39. package/lib/model/ameli.js +149 -0
  40. package/lib/model/ameli.mjs +84 -0
  41. package/lib/model/ameli.ts +100 -0
  42. package/lib/model/commission.d.ts +18 -0
  43. package/lib/model/commission.js +269 -0
  44. package/lib/model/debats.d.ts +67 -0
  45. package/lib/model/debats.js +95 -0
  46. package/lib/model/debats.mjs +43 -0
  47. package/lib/model/debats.ts +68 -0
  48. package/lib/model/documents.d.ts +12 -0
  49. package/lib/model/documents.js +151 -0
  50. package/lib/model/dosleg.d.ts +7 -0
  51. package/lib/model/dosleg.js +326 -0
  52. package/lib/model/dosleg.mjs +196 -0
  53. package/lib/model/dosleg.ts +240 -0
  54. package/lib/model/index.d.ts +7 -0
  55. package/lib/model/index.js +7 -0
  56. package/lib/model/index.mjs +5 -0
  57. package/lib/model/index.ts +15 -0
  58. package/lib/model/questions.d.ts +45 -0
  59. package/lib/model/questions.js +89 -0
  60. package/lib/model/questions.mjs +71 -0
  61. package/lib/model/questions.ts +93 -0
  62. package/lib/model/scrutins.d.ts +13 -0
  63. package/lib/model/scrutins.js +114 -0
  64. package/lib/model/seance.d.ts +3 -0
  65. package/lib/model/seance.js +267 -0
  66. package/lib/model/sens.d.ts +146 -0
  67. package/lib/model/sens.js +454 -0
  68. package/lib/model/sens.mjs +415 -0
  69. package/lib/model/sens.ts +516 -0
  70. package/lib/model/texte.d.ts +7 -0
  71. package/lib/model/texte.js +256 -0
  72. package/lib/model/texte.mjs +208 -0
  73. package/lib/model/texte.ts +229 -0
  74. package/lib/model/util.d.ts +9 -0
  75. package/lib/model/util.js +38 -0
  76. package/lib/model/util.mjs +19 -0
  77. package/lib/model/util.ts +32 -0
  78. package/lib/parsers/texte.d.ts +7 -0
  79. package/lib/parsers/texte.js +228 -0
  80. package/lib/raw_types/ameli.d.ts +914 -0
  81. package/lib/raw_types/ameli.js +5 -0
  82. package/lib/raw_types/ameli.mjs +163 -0
  83. package/lib/raw_types/debats.d.ts +207 -0
  84. package/lib/raw_types/debats.js +5 -0
  85. package/lib/raw_types/debats.mjs +58 -0
  86. package/lib/raw_types/dosleg.d.ts +1619 -0
  87. package/lib/raw_types/dosleg.js +5 -0
  88. package/lib/raw_types/dosleg.mjs +438 -0
  89. package/lib/raw_types/questions.d.ts +419 -0
  90. package/lib/raw_types/questions.js +5 -0
  91. package/lib/raw_types/questions.mjs +11 -0
  92. package/lib/raw_types/senat.d.ts +11368 -0
  93. package/lib/raw_types/senat.js +5 -0
  94. package/lib/raw_types/sens.d.ts +8248 -0
  95. package/lib/raw_types/sens.js +5 -0
  96. package/lib/raw_types/sens.mjs +508 -0
  97. package/lib/raw_types_kysely/ameli.d.ts +915 -0
  98. package/lib/raw_types_kysely/ameli.js +7 -0
  99. package/lib/raw_types_kysely/ameli.mjs +5 -0
  100. package/lib/raw_types_kysely/ameli.ts +951 -0
  101. package/lib/raw_types_kysely/debats.d.ts +207 -0
  102. package/lib/raw_types_kysely/debats.js +7 -0
  103. package/lib/raw_types_kysely/debats.mjs +5 -0
  104. package/lib/raw_types_kysely/debats.ts +222 -0
  105. package/lib/raw_types_kysely/dosleg.d.ts +3532 -0
  106. package/lib/raw_types_kysely/dosleg.js +7 -0
  107. package/lib/raw_types_kysely/dosleg.mjs +5 -0
  108. package/lib/raw_types_kysely/dosleg.ts +3621 -0
  109. package/lib/raw_types_kysely/questions.d.ts +414 -0
  110. package/lib/raw_types_kysely/questions.js +7 -0
  111. package/lib/raw_types_kysely/questions.mjs +5 -0
  112. package/lib/raw_types_kysely/questions.ts +426 -0
  113. package/lib/raw_types_kysely/sens.d.ts +4394 -0
  114. package/lib/raw_types_kysely/sens.js +7 -0
  115. package/lib/raw_types_kysely/sens.mjs +5 -0
  116. package/lib/raw_types_kysely/sens.ts +4499 -0
  117. package/lib/raw_types_schemats/ameli.d.ts +539 -0
  118. package/lib/raw_types_schemats/ameli.js +2 -0
  119. package/lib/raw_types_schemats/ameli.mjs +2 -0
  120. package/lib/raw_types_schemats/ameli.ts +601 -0
  121. package/lib/raw_types_schemats/debats.d.ts +127 -0
  122. package/lib/raw_types_schemats/debats.js +2 -0
  123. package/lib/raw_types_schemats/debats.mjs +2 -0
  124. package/lib/raw_types_schemats/debats.ts +145 -0
  125. package/lib/raw_types_schemats/dosleg.d.ts +977 -0
  126. package/lib/raw_types_schemats/dosleg.js +2 -0
  127. package/lib/raw_types_schemats/dosleg.mjs +2 -0
  128. package/lib/raw_types_schemats/dosleg.ts +2193 -0
  129. package/lib/raw_types_schemats/questions.d.ts +235 -0
  130. package/lib/raw_types_schemats/questions.js +2 -0
  131. package/lib/raw_types_schemats/questions.mjs +2 -0
  132. package/lib/raw_types_schemats/questions.ts +249 -0
  133. package/lib/raw_types_schemats/sens.d.ts +6915 -0
  134. package/lib/raw_types_schemats/sens.js +2 -0
  135. package/lib/raw_types_schemats/sens.mjs +2 -0
  136. package/lib/raw_types_schemats/sens.ts +2907 -0
  137. package/lib/scripts/convert_data.d.ts +1 -0
  138. package/lib/scripts/convert_data.js +354 -0
  139. package/lib/scripts/convert_data.mjs +181 -0
  140. package/lib/scripts/convert_data.ts +243 -0
  141. package/lib/scripts/data-download.d.ts +1 -0
  142. package/lib/scripts/data-download.js +12 -0
  143. package/lib/scripts/datautil.d.ts +8 -0
  144. package/lib/scripts/datautil.js +34 -0
  145. package/lib/scripts/datautil.mjs +16 -0
  146. package/lib/scripts/datautil.ts +19 -0
  147. package/lib/scripts/images/transparent_150x192.jpg +0 -0
  148. package/lib/scripts/images/transparent_155x225.jpg +0 -0
  149. package/lib/scripts/parse_textes.d.ts +1 -0
  150. package/lib/scripts/parse_textes.js +44 -0
  151. package/lib/scripts/parse_textes.mjs +46 -0
  152. package/lib/scripts/parse_textes.ts +65 -0
  153. package/lib/scripts/retrieve_agenda.d.ts +1 -0
  154. package/lib/scripts/retrieve_agenda.js +132 -0
  155. package/lib/scripts/retrieve_cr_commission.d.ts +1 -0
  156. package/lib/scripts/retrieve_cr_commission.js +364 -0
  157. package/lib/scripts/retrieve_cr_seance.d.ts +6 -0
  158. package/lib/scripts/retrieve_cr_seance.js +347 -0
  159. package/lib/scripts/retrieve_documents.d.ts +3 -0
  160. package/lib/scripts/retrieve_documents.js +219 -0
  161. package/lib/scripts/retrieve_documents.mjs +249 -0
  162. package/lib/scripts/retrieve_documents.ts +298 -0
  163. package/lib/scripts/retrieve_open_data.d.ts +1 -0
  164. package/lib/scripts/retrieve_open_data.js +315 -0
  165. package/lib/scripts/retrieve_open_data.mjs +217 -0
  166. package/lib/scripts/retrieve_open_data.ts +268 -0
  167. package/lib/scripts/retrieve_senateurs_photos.d.ts +1 -0
  168. package/lib/scripts/retrieve_senateurs_photos.js +147 -0
  169. package/lib/scripts/retrieve_senateurs_photos.mjs +147 -0
  170. package/lib/scripts/retrieve_senateurs_photos.ts +177 -0
  171. package/lib/scripts/retrieve_videos.d.ts +1 -0
  172. package/lib/scripts/retrieve_videos.js +461 -0
  173. package/lib/scripts/shared/cli_helpers.d.ts +95 -0
  174. package/lib/scripts/shared/cli_helpers.js +91 -0
  175. package/lib/scripts/shared/cli_helpers.ts +36 -0
  176. package/lib/scripts/shared/util.d.ts +4 -0
  177. package/lib/scripts/shared/util.js +35 -0
  178. package/lib/scripts/shared/util.ts +33 -0
  179. package/lib/scripts/test_iter_load.d.ts +1 -0
  180. package/lib/scripts/test_iter_load.js +12 -0
  181. package/lib/src/conversion_textes.js +10 -1
  182. package/lib/src/index.d.ts +1 -1
  183. package/lib/src/index.js +1 -1
  184. package/lib/src/loaders.d.ts +1 -0
  185. package/lib/src/loaders.js +1 -0
  186. package/lib/src/model/ameli.js +1 -0
  187. package/lib/src/parsers/texte.js +2 -2
  188. package/lib/src/scripts/convert_data.js +11 -10
  189. package/lib/src/scripts/retrieve_documents.d.ts +1 -1
  190. package/lib/src/scripts/retrieve_documents.js +25 -10
  191. package/lib/src/scripts/retrieve_open_data.js +1 -1
  192. package/lib/src/types/sessions.d.ts +5 -4
  193. package/lib/src/types/sessions.js +10 -75
  194. package/lib/src/types/texte.d.ts +2 -0
  195. package/lib/strings.d.ts +1 -0
  196. package/lib/strings.js +18 -0
  197. package/lib/strings.mjs +18 -0
  198. package/lib/strings.ts +26 -0
  199. package/lib/types/agenda.d.ts +44 -0
  200. package/lib/types/agenda.js +1 -0
  201. package/lib/types/ameli.d.ts +5 -0
  202. package/lib/types/ameli.js +1 -0
  203. package/lib/types/ameli.mjs +13 -0
  204. package/lib/types/ameli.ts +21 -0
  205. package/lib/types/compte_rendu.d.ts +83 -0
  206. package/lib/types/compte_rendu.js +1 -0
  207. package/lib/types/debats.d.ts +2 -0
  208. package/lib/types/debats.js +1 -0
  209. package/lib/types/debats.mjs +2 -0
  210. package/lib/types/debats.ts +6 -0
  211. package/lib/types/dosleg.d.ts +70 -0
  212. package/lib/types/dosleg.js +1 -0
  213. package/lib/types/dosleg.mjs +151 -0
  214. package/lib/types/dosleg.ts +284 -0
  215. package/lib/types/questions.d.ts +2 -0
  216. package/lib/types/questions.js +1 -0
  217. package/lib/types/questions.mjs +1 -0
  218. package/lib/types/questions.ts +3 -0
  219. package/lib/types/sens.d.ts +10 -0
  220. package/lib/types/sens.js +1 -0
  221. package/lib/types/sens.mjs +1 -0
  222. package/lib/types/sens.ts +12 -0
  223. package/lib/types/sessions.d.ts +5 -0
  224. package/lib/types/sessions.js +84 -0
  225. package/lib/types/sessions.mjs +43 -0
  226. package/lib/types/sessions.ts +42 -0
  227. package/lib/types/texte.d.ts +74 -0
  228. package/lib/types/texte.js +16 -0
  229. package/lib/types/texte.mjs +16 -0
  230. package/lib/types/texte.ts +76 -0
  231. package/lib/typings/windows-1252.d.js +2 -0
  232. package/lib/typings/windows-1252.d.mjs +2 -0
  233. package/lib/typings/windows-1252.d.ts +11 -0
  234. package/lib/utils/cr_spliting.d.ts +28 -0
  235. package/lib/utils/cr_spliting.js +265 -0
  236. package/lib/utils/date.d.ts +10 -0
  237. package/lib/utils/date.js +100 -0
  238. package/lib/utils/nvs-timecode.d.ts +7 -0
  239. package/lib/utils/nvs-timecode.js +79 -0
  240. package/lib/utils/reunion_grouping.d.ts +9 -0
  241. package/lib/utils/reunion_grouping.js +361 -0
  242. package/lib/utils/reunion_odj_building.d.ts +5 -0
  243. package/lib/utils/reunion_odj_building.js +154 -0
  244. package/lib/utils/reunion_parsing.d.ts +23 -0
  245. package/lib/utils/reunion_parsing.js +209 -0
  246. package/lib/utils/scoring.d.ts +14 -0
  247. package/lib/utils/scoring.js +147 -0
  248. package/lib/utils/string_cleaning.d.ts +7 -0
  249. package/lib/utils/string_cleaning.js +57 -0
  250. package/lib/validators/config.d.ts +9 -0
  251. package/lib/validators/config.js +10 -0
  252. package/lib/validators/config.mjs +54 -0
  253. package/lib/validators/config.ts +79 -0
  254. package/lib/validators/senat.d.ts +0 -0
  255. package/lib/validators/senat.js +28 -0
  256. package/lib/validators/senat.mjs +24 -0
  257. package/lib/validators/senat.ts +26 -0
  258. package/package.json +5 -5
package/lib/loaders.ts ADDED
@@ -0,0 +1,271 @@
1
+ import fs from "fs"
2
+ import path from "path"
3
+
4
+ import legislatures from "./legislatures.json"
5
+
6
+ import { Dataset, datasets } from "./datasets"
7
+ import { AmendementResult } from "./model/ameli"
8
+ import { DossierLegislatifResult } from "./model/dosleg"
9
+ import { QuestionResult } from "./model/questions"
10
+ import { CirconscriptionResult, OrganismeResult, SenateurResult } from "./model/sens"
11
+ import { UNDEFINED_SESSION } from "./scripts/datautil"
12
+ import { FlatTexte } from "./types/texte"
13
+
14
+ export { EnabledDatasets } from "./datasets"
15
+
16
+ export const DOSLEG_DOSSIERS_FOLDER = "dossiers"
17
+ export const SENS_CIRCONSCRIPTIONS_FOLDER = "circonscriptions"
18
+ export const SENS_ORGANISMES_FOLDER = "organismes"
19
+ export const SENS_SENATEURS_FOLDER = "senateurs"
20
+ export const TEXTE_FOLDER = "leg"
21
+ export const TEXTE_ORIGINAL_FOLDER = "original"
22
+ export const TEXTE_TRANSFORMED_FOLDER = "transformed"
23
+ export const DOCUMENT_METADATA_FILE = "metadata.json"
24
+ export const RAPPORT_FOLDER = "rap"
25
+
26
+ type IterItem<T> = {
27
+ item: T
28
+ filePathFromDataset?: string,
29
+ legislature?: number
30
+ }
31
+
32
+ export interface TexteMetadata {
33
+ name: string,
34
+ session: string | null | undefined,
35
+ url_expose_des_motifs?: URL,
36
+ url_xml: URL,
37
+ url_html: URL,
38
+ url_pdf: URL,
39
+ }
40
+
41
+ export interface RapportMetadata {
42
+ name: string,
43
+ session: string | null | undefined,
44
+ url_html: URL,
45
+ url_pdf: URL,
46
+ }
47
+
48
+ export interface DossierLegislatifTexteResult {
49
+ signet_dossier: string,
50
+ url_dossier_senat: string,
51
+ url_dossier_assemblee_nationale: string | null,
52
+ type_lecture: string
53
+ libelle_lecture: string
54
+ libelle_organisme: string | null
55
+ numero: string | null
56
+ url: string
57
+ origine_texte: string
58
+ type_texte: string
59
+ date: string
60
+ session: string | null
61
+ auteurs: {
62
+ prenom: string | null
63
+ nom_usuel: string
64
+ matricule: string | null
65
+ }[]
66
+ }
67
+
68
+ export function * iterFilePaths (dirPath: string): Generator<string> {
69
+ if (dirPath && fs.existsSync(dirPath)) {
70
+ const files = fs.readdirSync(dirPath, { withFileTypes: true, recursive: true })
71
+ for (const file of files) {
72
+ if (file.isFile()) {
73
+ yield path.join(file.parentPath, file.name)
74
+ }
75
+ }
76
+ }
77
+ }
78
+
79
+ function * iterLoadSenatItems<T> (
80
+ dataDir: string,
81
+ dataset: Dataset,
82
+ legislatureOrSession?: number,
83
+ subDir?: string,
84
+ { log = false } = {},
85
+ ): Generator<IterItem<T>> {
86
+ let itemsDir = path.join(dataDir, dataset.database)
87
+ if (subDir) {
88
+ itemsDir = path.join(itemsDir, subDir)
89
+ }
90
+ if (legislatureOrSession) {
91
+ itemsDir = path.join(itemsDir, String(legislatureOrSession))
92
+ }
93
+ for (const filePath of iterFilePaths(itemsDir)) {
94
+ if (log) {
95
+ console.log(`Loading file: ${filePath}…`)
96
+ }
97
+
98
+ const itemJson = fs.readFileSync(filePath, { encoding: "utf8" })
99
+ const item: T = JSON.parse(itemJson)
100
+
101
+ const filePathFromDataset = filePath.substring(filePath.indexOf(dataset.database) + dataset.database.length)
102
+
103
+ yield {
104
+ item,
105
+ filePathFromDataset,
106
+ legislature: legislatureOrSession,
107
+ }
108
+ }
109
+ }
110
+
111
+ export function * iterLoadSenatAmendements (
112
+ dataDir: string,
113
+ session: number | undefined,
114
+ options = {},
115
+ ): Generator<IterItem<AmendementResult>> {
116
+ for (const amendementItem
117
+ of iterLoadSenatItems<AmendementResult>(dataDir, datasets.ameli, session, undefined, options)
118
+ ) {
119
+ yield amendementItem
120
+ }
121
+ }
122
+
123
+ export function * iterLoadSenatDossiersLegislatifs (
124
+ dataDir: string,
125
+ session: number | undefined,
126
+ options = {},
127
+ ): Generator<IterItem<DossierLegislatifResult>> {
128
+ for (const dossierLegislatifItem
129
+ of iterLoadSenatItems<DossierLegislatifResult>(dataDir, datasets.dosleg, session, DOSLEG_DOSSIERS_FOLDER, options)
130
+ ) {
131
+ yield dossierLegislatifItem
132
+ }
133
+ }
134
+
135
+ export function * iterLoadSenatDossiersLegislatifsTexteUrls (
136
+ dataDir: string,
137
+ session: number | undefined,
138
+ ): Generator<IterItem<TexteMetadata>> {
139
+ let itemsDir = path.join(dataDir, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER)
140
+ if (session) {
141
+ itemsDir = path.join(itemsDir, session.toString())
142
+ }
143
+ for (const filePath of iterFilePaths(itemsDir)) {
144
+ const parsedFilePath = path.parse(filePath)
145
+ if (parsedFilePath.base === DOCUMENT_METADATA_FILE) {
146
+ const itemJson = fs.readFileSync(filePath, { encoding: "utf8" })
147
+ const item = JSON.parse(itemJson)
148
+ yield {
149
+ item,
150
+ }
151
+ }
152
+ }
153
+ }
154
+
155
+ export function * iterLoadSenatDossiersLegislatifsRapportUrls (
156
+ dataDir: string,
157
+ session: number | undefined,
158
+ ): Generator<IterItem<RapportMetadata>> {
159
+ let itemsDir = path.join(dataDir, RAPPORT_FOLDER)
160
+ if (session) {
161
+ itemsDir = path.join(itemsDir, session.toString())
162
+ }
163
+ for (const filePath of iterFilePaths(itemsDir)) {
164
+ const parsedFilePath = path.parse(filePath)
165
+ if (parsedFilePath.base === DOCUMENT_METADATA_FILE) {
166
+ const itemJson = fs.readFileSync(filePath, { encoding: "utf8" })
167
+ const item = JSON.parse(itemJson)
168
+ yield {
169
+ item,
170
+ }
171
+ }
172
+ }
173
+ }
174
+
175
+ export function * iterLoadSenatDossiersLegislatifsTextes (
176
+ dataDir: string,
177
+ session: number | undefined,
178
+ options = {},
179
+ ): Generator<IterItem<DossierLegislatifTexteResult>> {
180
+ for (const { item: dossierLegislatif } of iterLoadSenatDossiersLegislatifs(dataDir, session, options)) {
181
+ for (const lecture of dossierLegislatif.lectures) {
182
+ const lecturesSenat = lecture.lectures_assemblee
183
+ .filter(lectureAssemblee => lectureAssemblee.assemblee === "Sénat")
184
+ for (const lectureSenat of lecturesSenat) {
185
+ for (const texte of lectureSenat.textes) {
186
+ const enrichedTexte = {
187
+ signet_dossier: dossierLegislatif.signet,
188
+ url_dossier_senat: dossierLegislatif.url,
189
+ url_dossier_assemblee_nationale: dossierLegislatif.url_dossier_assemblee_nationale,
190
+ type_lecture: lecture.type_lecture,
191
+ libelle_lecture: lecture.libelle,
192
+ libelle_organisme: lectureSenat.libelle_organisme,
193
+ ...texte,
194
+ }
195
+ const texteItem: IterItem<DossierLegislatifTexteResult> = {
196
+ item: enrichedTexte,
197
+ }
198
+ if (texte.url) {
199
+ const texteName = path.parse(texte.url).name
200
+ texteItem.filePathFromDataset
201
+ = path.join(`${texte.session ?? UNDEFINED_SESSION}`, texteName, `${texteName}.pdf`)
202
+ }
203
+ yield texteItem
204
+ }
205
+ }
206
+ }
207
+ }
208
+ }
209
+
210
+ export function loadSenatTexteContent (
211
+ dataDir: string,
212
+ textePathFromDataset: string,
213
+ ): IterItem<FlatTexte | null> {
214
+ const parsedTextePath = path.parse(textePathFromDataset)
215
+ const jsonTexteName = `${parsedTextePath.name}.json`
216
+ const fullTextePath = path.join(dataDir, TEXTE_FOLDER, TEXTE_TRANSFORMED_FOLDER, parsedTextePath.dir, jsonTexteName)
217
+ if (!fs.existsSync(fullTextePath)) {
218
+ return { item: null }
219
+ }
220
+ const texteJson = fs.readFileSync(fullTextePath, { encoding: "utf8" })
221
+ return { item: JSON.parse(texteJson) }
222
+ }
223
+
224
+ export function * iterLoadSenatOrganismes (
225
+ dataDir: string,
226
+ options = {},
227
+ ): Generator<IterItem<OrganismeResult>> {
228
+ for (const organismeItem of iterLoadSenatItems<OrganismeResult>(dataDir, datasets.sens, undefined, SENS_ORGANISMES_FOLDER, options)) {
229
+ yield organismeItem
230
+ }
231
+ }
232
+
233
+ export function * iterLoadSenatSenateurs (
234
+ dataDir: string,
235
+ legislature: number,
236
+ options = {},
237
+ ): Generator<IterItem<SenateurResult>> {
238
+ const dateDebutLegislatureStr = legislatures
239
+ .find(legislatureInfo => legislatureInfo.numero === legislature)
240
+ ?.date_debut
241
+ const dateDebutLegislature = new Date(dateDebutLegislatureStr!)
242
+ for (const senateurItem of iterLoadSenatItems<SenateurResult>(dataDir, datasets.sens, undefined, SENS_SENATEURS_FOLDER, options)) {
243
+ const dateFinMandatSenateur = senateurItem.item.mandats_senateur[0]?.date_fin ?
244
+ new Date(senateurItem.item.mandats_senateur[0]?.date_fin) : null
245
+ if (dateFinMandatSenateur && dateFinMandatSenateur < dateDebutLegislature) {
246
+ continue
247
+ }
248
+ yield senateurItem
249
+ }
250
+ }
251
+
252
+ export function * iterLoadSenatCirconscriptions (
253
+ dataDir: string,
254
+ options = {},
255
+ ): Generator<IterItem<CirconscriptionResult>> {
256
+ for (const circonscriptionItem of iterLoadSenatItems<CirconscriptionResult>(dataDir, datasets.sens, undefined, SENS_CIRCONSCRIPTIONS_FOLDER, options)) {
257
+ yield circonscriptionItem
258
+ }
259
+ }
260
+
261
+ export function * iterLoadSenatQuestions (
262
+ dataDir: string,
263
+ legislature: number,
264
+ options = {},
265
+ ): Generator<IterItem<QuestionResult>> {
266
+ for (const questionItem
267
+ of iterLoadSenatItems<QuestionResult>(dataDir, datasets.questions, legislature, undefined, options)
268
+ ) {
269
+ yield questionItem
270
+ }
271
+ }
@@ -0,0 +1,6 @@
1
+ import { AgendaEvent } from "../types/agenda";
2
+ export declare function getStartAndEndTimes(timeStr: string | null | undefined, dateISO: string): {
3
+ startTime: string | null;
4
+ endTime: string | null;
5
+ };
6
+ export declare function parseAgendaFromFile(htmlFilePath: string): Promise<AgendaEvent[] | null>;
@@ -0,0 +1,148 @@
1
+ import { JSDOM } from "jsdom";
2
+ import { DateTime } from "luxon";
3
+ import path from "path";
4
+ import { ID_DATE_FORMAT, STANDARD_DATE_FORMAT } from "../scripts/datautil";
5
+ const FR_TZ = "Europe/Paris";
6
+ function eventIsSeance(eventElement) {
7
+ return eventElement.classList.contains("evt-seance");
8
+ }
9
+ function getEventType(eventClasses) {
10
+ const typeClass = [...eventClasses].find((className) => className.startsWith("evt-")) || null;
11
+ switch (typeClass) {
12
+ case "evt-seance":
13
+ return "Séance publique";
14
+ case "evt-instanz":
15
+ return "Commissions";
16
+ case "evt-cemi":
17
+ return "Mission de contrôle";
18
+ case "evt-deleg":
19
+ return "Offices et délégations";
20
+ case "evt-bureau":
21
+ return "Instances décisionnelles";
22
+ }
23
+ return null;
24
+ }
25
+ function getUrlDossierSenat(lienElements) {
26
+ const urlElement = [...lienElements].find((lienElement) => lienElement.textContent?.includes("dossier législatif"));
27
+ return urlElement ? urlElement.getAttribute("href") : null;
28
+ }
29
+ function getQuantieme(eventElement, seancesElements) {
30
+ const seanceIndex = seancesElements.indexOf(eventElement);
31
+ if (seancesElements.length === 1 && seanceIndex === 0) {
32
+ return "Unique";
33
+ }
34
+ else {
35
+ switch (seanceIndex) {
36
+ case 0:
37
+ return "Première";
38
+ case 1:
39
+ return "Deuxième";
40
+ case 2:
41
+ return "Troisième";
42
+ case 3:
43
+ return "Quatrième";
44
+ case 4:
45
+ return "Cinquième";
46
+ }
47
+ }
48
+ return "Non défini";
49
+ }
50
+ /**
51
+ * Normalize time string to become a simple start time ("H'h'mm") or a duration ("'de 'H'h'mm' à 'H'h'mm").
52
+ */
53
+ function normalizeTime(timeStr) {
54
+ return timeStr
55
+ ?.replace(/^À l'issue de l'espace réservé .* et au plus tard\s/i, "") // Must be processed first
56
+ ?.replace(/^(?:le )?matin/i, "10h00") // We chose "matin" to mean 10h00
57
+ ?.replace(/^(?:l')?après-midi/i, "16h00") // We chose "après-midi" to mean 16h00
58
+ ?.replace(/^(?:le )?soir/i, "20h00") // We chose "soir" to mean 20h00
59
+ ?.replace(/^(?:la )?nuit/i, "22h00") // We chose "nuit" to mean 22h00
60
+ ?.replace(/^à\s/gi, "")
61
+ ?.replace(/heures/gi, "h00")
62
+ ?.replace(/\set.*/i, "")
63
+ ?.replace(/,.*/, "")
64
+ ?.replace(/\s\(hors hémicycle\)/i, "")
65
+ ?.replace(/\s*h\s*/gi, "h");
66
+ }
67
+ export function getStartAndEndTimes(timeStr, dateISO) {
68
+ const normalizedTime = normalizeTime(timeStr);
69
+ if (!normalizedTime) {
70
+ return { startTime: null, endTime: null };
71
+ }
72
+ const rangeMatch = normalizedTime.match(/^de (?<start>\d{1,2}h\d{2}) à (?<end>\d{1,2}h\d{2})$/i);
73
+ const toUtcTimeOnly = (value) => {
74
+ if (!value)
75
+ return null;
76
+ const time = DateTime.fromFormat(value, "H'h'mm", { zone: FR_TZ });
77
+ if (!time.isValid)
78
+ return null;
79
+ const local = DateTime.fromISO(dateISO, { zone: FR_TZ }).set({
80
+ hour: time.hour,
81
+ minute: time.minute,
82
+ second: 0,
83
+ millisecond: 0,
84
+ });
85
+ if (!local.isValid)
86
+ return null;
87
+ return local.toUTC().toFormat("HH:mm:ss.SSS'Z'");
88
+ };
89
+ if (rangeMatch?.groups) {
90
+ const { start, end } = rangeMatch.groups;
91
+ return {
92
+ startTime: toUtcTimeOnly(start),
93
+ endTime: toUtcTimeOnly(end),
94
+ };
95
+ }
96
+ return {
97
+ startTime: toUtcTimeOnly(normalizedTime),
98
+ endTime: null,
99
+ };
100
+ }
101
+ function transformAgenda(document, fileName) {
102
+ const agendaEvents = [];
103
+ const eventElements = document.querySelectorAll(".evt");
104
+ const seanceElements = Array.from(eventElements).filter((eventElement) => eventIsSeance(eventElement));
105
+ for (const eventElement of eventElements) {
106
+ const id = eventElement.previousElementSibling?.getAttribute("name") || null;
107
+ if (!id) {
108
+ continue;
109
+ }
110
+ const type = getEventType(eventElement.classList);
111
+ const date = DateTime.fromFormat(fileName, ID_DATE_FORMAT).toFormat(STANDARD_DATE_FORMAT);
112
+ const timeOriginal = eventElement.querySelector(".time")?.textContent || null;
113
+ const { startTime, endTime } = getStartAndEndTimes(timeOriginal, date);
114
+ const titre = eventElement.querySelector(".titre")?.textContent?.trim() || "";
115
+ const organe = eventElement.querySelector(".organe")?.textContent?.trim() || null;
116
+ const objet = eventElement.querySelector(".objet")?.textContent?.trim()?.replace(/^- /, "") || null;
117
+ const lieu = eventElement.querySelector(".lieu")?.textContent || null;
118
+ const videoElement = eventElement.querySelector(".video");
119
+ const urlDossierSenat = getUrlDossierSenat(eventElement.querySelectorAll(".lien a"));
120
+ agendaEvents.push({
121
+ id,
122
+ type,
123
+ date,
124
+ startTime,
125
+ endTime,
126
+ timeOriginal,
127
+ titre,
128
+ organe,
129
+ objet,
130
+ lieu,
131
+ captationVideo: videoElement !== null,
132
+ urlDossierSenat: urlDossierSenat,
133
+ quantieme: eventIsSeance(eventElement) ? getQuantieme(eventElement, seanceElements) : null,
134
+ });
135
+ }
136
+ return agendaEvents;
137
+ }
138
+ export async function parseAgendaFromFile(htmlFilePath) {
139
+ try {
140
+ const { document } = (await JSDOM.fromFile(htmlFilePath, { contentType: "text/html" })).window;
141
+ const fileName = path.parse(htmlFilePath).name;
142
+ return transformAgenda(document, fileName);
143
+ }
144
+ catch (error) {
145
+ console.error(`Could not parse texte with error ${error}`);
146
+ }
147
+ return null;
148
+ }
@@ -0,0 +1,51 @@
1
+ import { InferResult } from "kysely";
2
+ export type AmendementResult = InferResult<typeof findAllAmendementsQuery>[0];
3
+ declare const findAllAmendementsQuery: import("kysely").SelectQueryBuilder<{
4
+ [x: string]: any;
5
+ [x: number]: any;
6
+ [x: symbol]: any;
7
+ }, "ameli.amd" | "ameli.sub" | "ameli.typsub" | "ameli.typrect" | "ameli.txt_ameli" | "ameli.etatxt" | "ameli.ses" | "ameli.typses" | "ameli.nat" | "ameli.lec_ameli" | "dosleg.texte" | "dosleg.lecass" | "ameli.mot" | "ameli.avicom" | "ameli.avigvt" | "ameli.sor" | "ameli.irr" | "ameli.grppol_ameli" | "ameli.com_ameli" | "ameli.cab", {
8
+ [x: string]: any;
9
+ nature: string;
10
+ date_depot: string;
11
+ etat: string;
12
+ sort: any;
13
+ url: string;
14
+ auteur_est_gouvernement: boolean;
15
+ scrutin_num: any;
16
+ auteurs: {
17
+ prenom: any;
18
+ homonyme: any;
19
+ nom: any;
20
+ qualite: any;
21
+ rang: any;
22
+ matricule: any;
23
+ groupe_politique_id: any;
24
+ group_politique_code: any;
25
+ groupe_politique_libelle_court: any;
26
+ groupe_politique_libelle: any;
27
+ }[];
28
+ }>;
29
+ export declare function findAllAmendements(fromSession?: number): AsyncIterableIterator<{
30
+ [x: string]: any;
31
+ nature: string;
32
+ date_depot: string;
33
+ etat: string;
34
+ sort: any;
35
+ url: string;
36
+ auteur_est_gouvernement: boolean;
37
+ scrutin_num: any;
38
+ auteurs: {
39
+ prenom: any;
40
+ homonyme: any;
41
+ nom: any;
42
+ qualite: any;
43
+ rang: any;
44
+ matricule: any;
45
+ groupe_politique_id: any;
46
+ group_politique_code: any;
47
+ groupe_politique_libelle_court: any;
48
+ groupe_politique_libelle: any;
49
+ }[];
50
+ }>;
51
+ export {};
@@ -0,0 +1,149 @@
1
+ import { sql } from "kysely";
2
+ import { jsonArrayFrom } from "kysely/helpers/postgres";
3
+ import { dbSenat } from "../databases";
4
+ import { concat, toDateString } from "./util";
5
+ function auteurs(amendementId) {
6
+ return jsonArrayFrom(dbSenat
7
+ .selectFrom("ameli.amdsen")
8
+ .leftJoin("ameli.sen_ameli", "ameli.amdsen.senid", "ameli.sen_ameli.entid")
9
+ .leftJoin("ameli.grppol_ameli", "ameli.amdsen.grpid", "ameli.grppol_ameli.entid")
10
+ .where("ameli.amdsen.amdid", "=", amendementId)
11
+ .select([
12
+ "ameli.amdsen.prenomuse as prenom",
13
+ "ameli.amdsen.hom as homonyme",
14
+ "ameli.amdsen.nomuse as nom",
15
+ "ameli.amdsen.qua as qualite",
16
+ "ameli.amdsen.rng as rang",
17
+ "ameli.sen_ameli.mat as matricule",
18
+ "ameli.amdsen.grpid as groupe_politique_id",
19
+ "ameli.grppol_ameli.cod as group_politique_code",
20
+ "ameli.grppol_ameli.libcou as groupe_politique_libelle_court",
21
+ "ameli.grppol_ameli.lilcou as groupe_politique_libelle",
22
+ ])
23
+ .orderBy("ameli.amdsen.rng", "asc"));
24
+ }
25
+ function scrutin(amendementNum, sesann, lecassidt) {
26
+ return dbSenat
27
+ .selectFrom("dosleg.amescr")
28
+ .leftJoin("dosleg.scr", (join) => join
29
+ .onRef("dosleg.amescr.scrnum", "=", "dosleg.scr.scrnum")
30
+ .onRef("dosleg.amescr.sesann", "=", "dosleg.scr.sesann"))
31
+ .leftJoin("dosleg.date_seance", "dosleg.scr.code", "dosleg.date_seance.code")
32
+ .where("dosleg.amescr.amescrnum", "=", amendementNum)
33
+ .where("dosleg.amescr.sesann", "=", sesann)
34
+ .where("dosleg.date_seance.lecidt", "=", lecassidt)
35
+ .select(["dosleg.amescr.scrnum as scrutin_num"])
36
+ .limit(1)
37
+ .as("scrutin_num");
38
+ }
39
+ const findAllAmendementsQuery = dbSenat
40
+ .selectFrom("ameli.amd")
41
+ .leftJoin("ameli.sub", "ameli.amd.subid", "ameli.sub.id")
42
+ .leftJoin("ameli.typsub", "ameli.sub.typid", "ameli.typsub.id")
43
+ .leftJoin("ameli.typrect", "ameli.amd.typrectid", "ameli.typrect.id")
44
+ .leftJoin("ameli.txt_ameli", "ameli.amd.txtid", "ameli.txt_ameli.id")
45
+ .leftJoin("ameli.etatxt", "ameli.txt_ameli.txtetaid", "ameli.etatxt.id")
46
+ .leftJoin("ameli.ses", "ameli.txt_ameli.sesdepid", "ameli.ses.id")
47
+ .leftJoin("ameli.typses", "ameli.typses.id", "ameli.ses.typid")
48
+ .leftJoin("ameli.nat", "ameli.txt_ameli.natid", "ameli.nat.id")
49
+ .leftJoin("ameli.lec_ameli", "ameli.txt_ameli.lecid", "ameli.lec_ameli.id")
50
+ .leftJoin("dosleg.texte", (join) => join
51
+ .onRef("ameli.ses.ann", "=", "dosleg.texte.sesann")
52
+ .onRef("ameli.txt_ameli.numabs", "=", "dosleg.texte.texnum"))
53
+ .leftJoin("dosleg.lecass", "dosleg.texte.lecassidt", "dosleg.lecass.lecassidt")
54
+ .leftJoin("ameli.mot", "ameli.amd.motid", "ameli.mot.id")
55
+ .leftJoin("ameli.avicom", "ameli.amd.avcid", "ameli.avicom.id")
56
+ .leftJoin("ameli.avigvt", "ameli.amd.avgid", "ameli.avigvt.id")
57
+ .leftJoin("ameli.sor", "ameli.amd.sorid", "ameli.sor.id")
58
+ .leftJoin("ameli.irr", "ameli.amd.irrid", "ameli.irr.id")
59
+ .leftJoin("ameli.grppol_ameli", "ameli.amd.nomentid", "ameli.grppol_ameli.entid")
60
+ .leftJoin("ameli.com_ameli", "ameli.amd.nomentid", "ameli.com_ameli.entid")
61
+ .leftJoin("ameli.cab", "ameli.amd.nomentid", "ameli.cab.entid")
62
+ .select(({ eb, ref, val }) => [
63
+ "ameli.ses.ann as session",
64
+ "ameli.ses.lil as session_libelle",
65
+ "ameli.typses.lib as type_session",
66
+ "ameli.txt_ameli.doslegsignet as signet_dossier_legislatif",
67
+ "ameli.nat.libcourt as nature_texte",
68
+ "ameli.nat.lib as nature_texte_libelle",
69
+ "ameli.txt_ameli.numabs as numero_texte",
70
+ "ameli.txt_ameli.numado as numero_adoption_texte",
71
+ "ameli.txt_ameli.int as intitule_texte",
72
+ "ameli.etatxt.lic as etat_texte",
73
+ "ameli.etatxt.lib as etat_texte_libelle",
74
+ "ameli.etatxt.txttyp as type_texte",
75
+ "ameli.lec_ameli.lib as lecture",
76
+ eb
77
+ .case()
78
+ .when("ameli.amd.typ", "=", "A")
79
+ .then(val("Amendement"))
80
+ .when("ameli.amd.typ", "=", "M")
81
+ .then(val("Motion"))
82
+ .when("ameli.amd.typ", "=", "S")
83
+ .then(val("Sous-amendement"))
84
+ .else("")
85
+ .end()
86
+ .as("nature"),
87
+ "ameli.amd.id as id",
88
+ "ameli.amd.amdperid as parent_id",
89
+ "ameli.amd.ideid as identique_id",
90
+ "ameli.amd.discomid as discussion_commune_id",
91
+ "ameli.amd.num as numero",
92
+ "ameli.amd.numabs as numero_absolu",
93
+ "ameli.amd.ord as ordre",
94
+ "ameli.amd.accgou as accepte_gouvernement",
95
+ "ameli.amd.txtid as texte_id",
96
+ "ameli.sub.lib as subdivision_libelle",
97
+ "ameli.sub.lic as subdivision_libelle_court",
98
+ "ameli.sub.pos as subdivision_position_texte",
99
+ "ameli.sub.posder as subdivision_position_discussion",
100
+ "ameli.sub.merid as subdivision_mere_id",
101
+ "ameli.sub.sig as subdivision_signet",
102
+ "ameli.sub.comdelid as subdivision_commission_id",
103
+ "ameli.sub.dupl as subdivision_dupliquee",
104
+ "ameli.typsub.lib as subdivision_type",
105
+ "ameli.amd.alinea as alinea",
106
+ "ameli.amd.obs as observations",
107
+ "ameli.amd.mot as observations_additionnelles",
108
+ toDateString(ref("ameli.amd.datdep")).as("date_depot"),
109
+ "ameli.amd.dis as dispositif",
110
+ "ameli.amd.obj as objet",
111
+ "ameli.typrect.lib as type_rectification",
112
+ "ameli.mot.lib as motion_libelle",
113
+ eb
114
+ .case()
115
+ .when("ameli.amd.etaid", "=", 7)
116
+ .then(val("Diffusé"))
117
+ .when("ameli.amd.etaid", "=", 8)
118
+ .then(val("Retiré avant réunion ou séance"))
119
+ .when("ameli.amd.etaid", "=", 9)
120
+ .then(val("Examiné en commission ou séance"))
121
+ .when("ameli.amd.etaid", "=", 10)
122
+ .then(val("Irrecevable"))
123
+ .when("ameli.amd.etaid", "=", 11)
124
+ .then(val("Irrecevable"))
125
+ .else("")
126
+ .end()
127
+ .as("etat"),
128
+ "ameli.avicom.lib as avis_commission",
129
+ "ameli.avigvt.lib as avis_gouvernement",
130
+ eb.fn.coalesce("ameli.sor.lib", "ameli.irr.libirr").as("sort"),
131
+ "ameli.amd.rev as revision",
132
+ concat(eb
133
+ .case()
134
+ .when("ameli.amd.num", "like", "%COM%")
135
+ .then(val("https://www.senat.fr/amendements/commissions/"))
136
+ .else(val("https://www.senat.fr/amendements/"))
137
+ .end(), ref("ameli.ses.ann"), val("-"), sql `(ameli.ses.ann + 1)`, val("/"), ref("ameli.txt_ameli.numabs"), val("/Amdt_"), ref("ameli.amd.num"), val(".html")).as("url"),
138
+ "ameli.grppol_ameli.lilcou as au_nom_de_groupe_politique",
139
+ "ameli.com_ameli.lil as au_nom_de_commission",
140
+ eb.case().when("ameli.cab.entid", "is not", null).then(true).else(false).end().as("auteur_est_gouvernement"),
141
+ scrutin(ref("ameli.amd.num"), ref("ameli.ses.ann"), ref("dosleg.texte.lecassidt")),
142
+ auteurs(ref("ameli.amd.id")).as("auteurs"),
143
+ ]);
144
+ export function findAllAmendements(fromSession) {
145
+ if (fromSession !== undefined) {
146
+ return findAllAmendementsQuery.where("ameli.ses.ann", ">=", fromSession).stream();
147
+ }
148
+ return findAllAmendementsQuery.stream();
149
+ }