@tricoteuses/senat 3.1.22 → 3.1.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (388) hide show
  1. package/LICENSE.md +32 -32
  2. package/README.md +326 -326
  3. package/lib/src/other_types/collaborateurs.d.ts +5 -0
  4. package/lib/src/parsers/collaborateurs.d.ts +44 -0
  5. package/lib/src/parsers/collaborateurs.js +157 -0
  6. package/lib/src/scripts/retrieve_collaborateurs.js +166 -0
  7. package/lib/src/scripts/retrieve_cr_seance.js +34 -25
  8. package/lib/src/scripts/retrieve_videos.js +13 -8
  9. package/lib/src/scripts/shared/incremental_import_sql.js +866 -859
  10. package/lib/src/scripts/shared/schema_version.js +84 -84
  11. package/lib/src/scripts/shared/staging_metadata_sql.js +214 -214
  12. package/lib/src/scripts/validate_prefixed_tables.js +12 -12
  13. package/lib/src/server/ameli.js +14 -13
  14. package/lib/src/server/conversion_textes.js +106 -106
  15. package/lib/src/server/debats.js +10 -10
  16. package/lib/src/server/documents.js +22 -22
  17. package/lib/src/server/dosleg.js +33 -33
  18. package/lib/src/server/questions.js +10 -10
  19. package/lib/src/server/scrutins.js +3 -3
  20. package/lib/src/server/sens.js +19 -19
  21. package/lib/src/utils/reunion_odj_building.js +0 -4
  22. package/lib/src/videos/match.js +7 -5
  23. package/lib/src/videos/pipeline.js +26 -19
  24. package/lib/tests/collaborateurs.test.js +115 -0
  25. package/lib/tests/incrementalImportSql.test.js +4 -1
  26. package/package.json +119 -119
  27. package/lib/add-js-extensions-v2.js +0 -23
  28. package/lib/add-js-extensions.js +0 -17
  29. package/lib/aggregates.d.ts +0 -52
  30. package/lib/aggregates.js +0 -930
  31. package/lib/aggregates.mjs +0 -713
  32. package/lib/aggregates.ts +0 -833
  33. package/lib/config.d.ts +0 -10
  34. package/lib/config.js +0 -16
  35. package/lib/config.mjs +0 -16
  36. package/lib/config.ts +0 -26
  37. package/lib/databases.d.ts +0 -2
  38. package/lib/databases.js +0 -26
  39. package/lib/databases.mjs +0 -57
  40. package/lib/databases.ts +0 -71
  41. package/lib/datasets.d.ts +0 -34
  42. package/lib/datasets.js +0 -233
  43. package/lib/datasets.mjs +0 -78
  44. package/lib/datasets.ts +0 -118
  45. package/lib/fields.d.ts +0 -10
  46. package/lib/fields.js +0 -68
  47. package/lib/fields.mjs +0 -22
  48. package/lib/fields.ts +0 -29
  49. package/lib/git.d.ts +0 -26
  50. package/lib/git.js +0 -167
  51. package/lib/index.d.ts +0 -13
  52. package/lib/index.js +0 -1
  53. package/lib/index.mjs +0 -7
  54. package/lib/index.ts +0 -64
  55. package/lib/inserters.d.ts +0 -98
  56. package/lib/inserters.js +0 -500
  57. package/lib/inserters.mjs +0 -360
  58. package/lib/inserters.ts +0 -521
  59. package/lib/legislatures.json +0 -38
  60. package/lib/loaders.d.ts +0 -58
  61. package/lib/loaders.js +0 -286
  62. package/lib/loaders.mjs +0 -158
  63. package/lib/loaders.ts +0 -271
  64. package/lib/model/agenda.d.ts +0 -6
  65. package/lib/model/agenda.js +0 -148
  66. package/lib/model/ameli.d.ts +0 -51
  67. package/lib/model/ameli.js +0 -149
  68. package/lib/model/ameli.mjs +0 -84
  69. package/lib/model/ameli.ts +0 -100
  70. package/lib/model/commission.d.ts +0 -18
  71. package/lib/model/commission.js +0 -269
  72. package/lib/model/debats.d.ts +0 -67
  73. package/lib/model/debats.js +0 -95
  74. package/lib/model/debats.mjs +0 -43
  75. package/lib/model/debats.ts +0 -68
  76. package/lib/model/documents.d.ts +0 -12
  77. package/lib/model/documents.js +0 -151
  78. package/lib/model/dosleg.d.ts +0 -7
  79. package/lib/model/dosleg.js +0 -326
  80. package/lib/model/dosleg.mjs +0 -196
  81. package/lib/model/dosleg.ts +0 -240
  82. package/lib/model/index.d.ts +0 -7
  83. package/lib/model/index.js +0 -7
  84. package/lib/model/index.mjs +0 -5
  85. package/lib/model/index.ts +0 -15
  86. package/lib/model/questions.d.ts +0 -45
  87. package/lib/model/questions.js +0 -89
  88. package/lib/model/questions.mjs +0 -71
  89. package/lib/model/questions.ts +0 -93
  90. package/lib/model/scrutins.d.ts +0 -13
  91. package/lib/model/scrutins.js +0 -114
  92. package/lib/model/seance.d.ts +0 -3
  93. package/lib/model/seance.js +0 -267
  94. package/lib/model/sens.d.ts +0 -146
  95. package/lib/model/sens.js +0 -454
  96. package/lib/model/sens.mjs +0 -415
  97. package/lib/model/sens.ts +0 -516
  98. package/lib/model/texte.d.ts +0 -7
  99. package/lib/model/texte.js +0 -256
  100. package/lib/model/texte.mjs +0 -208
  101. package/lib/model/texte.ts +0 -229
  102. package/lib/model/util.d.ts +0 -9
  103. package/lib/model/util.js +0 -38
  104. package/lib/model/util.mjs +0 -19
  105. package/lib/model/util.ts +0 -32
  106. package/lib/parsers/texte.d.ts +0 -7
  107. package/lib/parsers/texte.js +0 -228
  108. package/lib/raw_types/ameli.d.ts +0 -914
  109. package/lib/raw_types/ameli.js +0 -5
  110. package/lib/raw_types/ameli.mjs +0 -163
  111. package/lib/raw_types/debats.d.ts +0 -207
  112. package/lib/raw_types/debats.js +0 -5
  113. package/lib/raw_types/debats.mjs +0 -58
  114. package/lib/raw_types/dosleg.d.ts +0 -1619
  115. package/lib/raw_types/dosleg.js +0 -5
  116. package/lib/raw_types/dosleg.mjs +0 -438
  117. package/lib/raw_types/questions.d.ts +0 -419
  118. package/lib/raw_types/questions.js +0 -5
  119. package/lib/raw_types/questions.mjs +0 -11
  120. package/lib/raw_types/senat.d.ts +0 -11368
  121. package/lib/raw_types/senat.js +0 -5
  122. package/lib/raw_types/sens.d.ts +0 -8248
  123. package/lib/raw_types/sens.js +0 -5
  124. package/lib/raw_types/sens.mjs +0 -508
  125. package/lib/raw_types_kysely/ameli.d.ts +0 -915
  126. package/lib/raw_types_kysely/ameli.js +0 -7
  127. package/lib/raw_types_kysely/ameli.mjs +0 -5
  128. package/lib/raw_types_kysely/ameli.ts +0 -951
  129. package/lib/raw_types_kysely/debats.d.ts +0 -207
  130. package/lib/raw_types_kysely/debats.js +0 -7
  131. package/lib/raw_types_kysely/debats.mjs +0 -5
  132. package/lib/raw_types_kysely/debats.ts +0 -222
  133. package/lib/raw_types_kysely/dosleg.d.ts +0 -3532
  134. package/lib/raw_types_kysely/dosleg.js +0 -7
  135. package/lib/raw_types_kysely/dosleg.mjs +0 -5
  136. package/lib/raw_types_kysely/dosleg.ts +0 -3621
  137. package/lib/raw_types_kysely/questions.d.ts +0 -414
  138. package/lib/raw_types_kysely/questions.js +0 -7
  139. package/lib/raw_types_kysely/questions.mjs +0 -5
  140. package/lib/raw_types_kysely/questions.ts +0 -426
  141. package/lib/raw_types_kysely/sens.d.ts +0 -4394
  142. package/lib/raw_types_kysely/sens.js +0 -7
  143. package/lib/raw_types_kysely/sens.mjs +0 -5
  144. package/lib/raw_types_kysely/sens.ts +0 -4499
  145. package/lib/raw_types_schemats/ameli.d.ts +0 -539
  146. package/lib/raw_types_schemats/ameli.js +0 -2
  147. package/lib/raw_types_schemats/ameli.mjs +0 -2
  148. package/lib/raw_types_schemats/ameli.ts +0 -601
  149. package/lib/raw_types_schemats/debats.d.ts +0 -127
  150. package/lib/raw_types_schemats/debats.js +0 -2
  151. package/lib/raw_types_schemats/debats.mjs +0 -2
  152. package/lib/raw_types_schemats/debats.ts +0 -145
  153. package/lib/raw_types_schemats/dosleg.d.ts +0 -977
  154. package/lib/raw_types_schemats/dosleg.js +0 -2
  155. package/lib/raw_types_schemats/dosleg.mjs +0 -2
  156. package/lib/raw_types_schemats/dosleg.ts +0 -2193
  157. package/lib/raw_types_schemats/questions.d.ts +0 -235
  158. package/lib/raw_types_schemats/questions.js +0 -2
  159. package/lib/raw_types_schemats/questions.mjs +0 -2
  160. package/lib/raw_types_schemats/questions.ts +0 -249
  161. package/lib/raw_types_schemats/sens.d.ts +0 -6915
  162. package/lib/raw_types_schemats/sens.js +0 -2
  163. package/lib/raw_types_schemats/sens.mjs +0 -2
  164. package/lib/raw_types_schemats/sens.ts +0 -2907
  165. package/lib/scripts/convert_data.js +0 -354
  166. package/lib/scripts/convert_data.mjs +0 -181
  167. package/lib/scripts/convert_data.ts +0 -243
  168. package/lib/scripts/data-download.d.ts +0 -1
  169. package/lib/scripts/data-download.js +0 -12
  170. package/lib/scripts/datautil.d.ts +0 -8
  171. package/lib/scripts/datautil.js +0 -34
  172. package/lib/scripts/datautil.mjs +0 -16
  173. package/lib/scripts/datautil.ts +0 -19
  174. package/lib/scripts/images/transparent_150x192.jpg +0 -0
  175. package/lib/scripts/images/transparent_155x225.jpg +0 -0
  176. package/lib/scripts/parse_textes.d.ts +0 -1
  177. package/lib/scripts/parse_textes.js +0 -44
  178. package/lib/scripts/parse_textes.mjs +0 -46
  179. package/lib/scripts/parse_textes.ts +0 -65
  180. package/lib/scripts/retrieve_agenda.d.ts +0 -1
  181. package/lib/scripts/retrieve_agenda.js +0 -132
  182. package/lib/scripts/retrieve_cr_commission.d.ts +0 -1
  183. package/lib/scripts/retrieve_cr_commission.js +0 -364
  184. package/lib/scripts/retrieve_cr_seance.d.ts +0 -6
  185. package/lib/scripts/retrieve_cr_seance.js +0 -347
  186. package/lib/scripts/retrieve_documents.d.ts +0 -3
  187. package/lib/scripts/retrieve_documents.js +0 -219
  188. package/lib/scripts/retrieve_documents.mjs +0 -249
  189. package/lib/scripts/retrieve_documents.ts +0 -298
  190. package/lib/scripts/retrieve_open_data.d.ts +0 -1
  191. package/lib/scripts/retrieve_open_data.js +0 -315
  192. package/lib/scripts/retrieve_open_data.mjs +0 -217
  193. package/lib/scripts/retrieve_open_data.ts +0 -268
  194. package/lib/scripts/retrieve_senateurs_photos.d.ts +0 -1
  195. package/lib/scripts/retrieve_senateurs_photos.js +0 -147
  196. package/lib/scripts/retrieve_senateurs_photos.mjs +0 -147
  197. package/lib/scripts/retrieve_senateurs_photos.ts +0 -177
  198. package/lib/scripts/retrieve_videos.d.ts +0 -1
  199. package/lib/scripts/retrieve_videos.js +0 -461
  200. package/lib/scripts/shared/cli_helpers.d.ts +0 -95
  201. package/lib/scripts/shared/cli_helpers.js +0 -91
  202. package/lib/scripts/shared/cli_helpers.ts +0 -36
  203. package/lib/scripts/shared/util.d.ts +0 -4
  204. package/lib/scripts/shared/util.js +0 -35
  205. package/lib/scripts/shared/util.ts +0 -33
  206. package/lib/scripts/test_iter_load.d.ts +0 -1
  207. package/lib/scripts/test_iter_load.js +0 -12
  208. package/lib/src/ameli.d.ts +0 -66
  209. package/lib/src/ameli.js +0 -1
  210. package/lib/src/config.d.ts +0 -43
  211. package/lib/src/config.js +0 -37
  212. package/lib/src/conversion_textes.d.ts +0 -11
  213. package/lib/src/conversion_textes.js +0 -320
  214. package/lib/src/databases.d.ts +0 -3
  215. package/lib/src/databases.js +0 -26
  216. package/lib/src/databases_postgres.d.ts +0 -4
  217. package/lib/src/databases_postgres.js +0 -23
  218. package/lib/src/datasets.d.ts +0 -38
  219. package/lib/src/datasets.js +0 -247
  220. package/lib/src/db_types/ameli.d.ts +0 -1762
  221. package/lib/src/db_types/ameli.js +0 -1074
  222. package/lib/src/db_types/debats.d.ts +0 -380
  223. package/lib/src/db_types/debats.js +0 -266
  224. package/lib/src/db_types/dosleg.d.ts +0 -2954
  225. package/lib/src/db_types/dosleg.js +0 -2005
  226. package/lib/src/db_types/questions.d.ts +0 -699
  227. package/lib/src/db_types/questions.js +0 -493
  228. package/lib/src/db_types/sens.d.ts +0 -7843
  229. package/lib/src/db_types/sens.js +0 -4691
  230. package/lib/src/debats.d.ts +0 -38
  231. package/lib/src/debats.js +0 -1
  232. package/lib/src/dosleg.d.ts +0 -142
  233. package/lib/src/dosleg.js +0 -193
  234. package/lib/src/git.d.ts +0 -27
  235. package/lib/src/git.js +0 -251
  236. package/lib/src/loaders.d.ts +0 -52
  237. package/lib/src/loaders.js +0 -260
  238. package/lib/src/model/agenda.d.ts +0 -6
  239. package/lib/src/model/agenda.js +0 -148
  240. package/lib/src/model/ameli.d.ts +0 -67
  241. package/lib/src/model/ameli.js +0 -150
  242. package/lib/src/model/ameli_postgres.d.ts +0 -67
  243. package/lib/src/model/ameli_postgres.js +0 -150
  244. package/lib/src/model/commission.d.ts +0 -19
  245. package/lib/src/model/commission.js +0 -269
  246. package/lib/src/model/debats.d.ts +0 -39
  247. package/lib/src/model/debats.js +0 -112
  248. package/lib/src/model/documents.d.ts +0 -32
  249. package/lib/src/model/documents.js +0 -182
  250. package/lib/src/model/dosleg.d.ts +0 -144
  251. package/lib/src/model/dosleg.js +0 -468
  252. package/lib/src/model/index.d.ts +0 -7
  253. package/lib/src/model/index.js +0 -7
  254. package/lib/src/model/questions.d.ts +0 -54
  255. package/lib/src/model/questions.js +0 -91
  256. package/lib/src/model/scrutins.d.ts +0 -48
  257. package/lib/src/model/scrutins.js +0 -121
  258. package/lib/src/model/seance.d.ts +0 -3
  259. package/lib/src/model/seance.js +0 -267
  260. package/lib/src/model/sens.d.ts +0 -112
  261. package/lib/src/model/sens.js +0 -385
  262. package/lib/src/model/util.d.ts +0 -1
  263. package/lib/src/model/util.js +0 -15
  264. package/lib/src/other_types/questions.d.ts +0 -2
  265. package/lib/src/other_types/questions.js +0 -1
  266. package/lib/src/questions.d.ts +0 -53
  267. package/lib/src/questions.js +0 -1
  268. package/lib/src/raw_types/ameli.d.ts +0 -1762
  269. package/lib/src/raw_types/ameli.js +0 -1074
  270. package/lib/src/raw_types/debats.d.ts +0 -380
  271. package/lib/src/raw_types/debats.js +0 -266
  272. package/lib/src/raw_types/dosleg.d.ts +0 -2954
  273. package/lib/src/raw_types/dosleg.js +0 -2005
  274. package/lib/src/raw_types/questions.d.ts +0 -699
  275. package/lib/src/raw_types/questions.js +0 -493
  276. package/lib/src/raw_types/senat.d.ts +0 -11372
  277. package/lib/src/raw_types/senat.js +0 -5
  278. package/lib/src/raw_types/sens.d.ts +0 -7843
  279. package/lib/src/raw_types/sens.js +0 -4691
  280. package/lib/src/raw_types_schemats/ameli.d.ts +0 -541
  281. package/lib/src/raw_types_schemats/ameli.js +0 -2
  282. package/lib/src/raw_types_schemats/debats.d.ts +0 -127
  283. package/lib/src/raw_types_schemats/debats.js +0 -2
  284. package/lib/src/raw_types_schemats/dosleg.d.ts +0 -977
  285. package/lib/src/raw_types_schemats/dosleg.js +0 -2
  286. package/lib/src/raw_types_schemats/questions.d.ts +0 -237
  287. package/lib/src/raw_types_schemats/questions.js +0 -2
  288. package/lib/src/raw_types_schemats/sens.d.ts +0 -2709
  289. package/lib/src/raw_types_schemats/sens.js +0 -2
  290. package/lib/src/rich_types/agenda.d.ts +0 -45
  291. package/lib/src/rich_types/agenda.js +0 -1
  292. package/lib/src/rich_types/compte_rendu.d.ts +0 -83
  293. package/lib/src/rich_types/compte_rendu.js +0 -1
  294. package/lib/src/rich_types/sessions.d.ts +0 -6
  295. package/lib/src/rich_types/sessions.js +0 -19
  296. package/lib/src/rich_types/texte.d.ts +0 -72
  297. package/lib/src/rich_types/texte.js +0 -15
  298. package/lib/src/scripts/test_iter_load.d.ts +0 -1
  299. package/lib/src/scripts/test_iter_load.js +0 -12
  300. package/lib/src/sens.d.ts +0 -104
  301. package/lib/src/sens.js +0 -1
  302. package/lib/src/types/agenda.d.ts +0 -45
  303. package/lib/src/types/agenda.js +0 -1
  304. package/lib/src/types/ameli.d.ts +0 -1762
  305. package/lib/src/types/ameli.js +0 -1074
  306. package/lib/src/types/compte_rendu.d.ts +0 -83
  307. package/lib/src/types/compte_rendu.js +0 -1
  308. package/lib/src/types/debats.d.ts +0 -380
  309. package/lib/src/types/debats.js +0 -266
  310. package/lib/src/types/dosleg.d.ts +0 -2954
  311. package/lib/src/types/dosleg.js +0 -2005
  312. package/lib/src/types/questions.d.ts +0 -699
  313. package/lib/src/types/questions.js +0 -493
  314. package/lib/src/types/sens.d.ts +0 -7843
  315. package/lib/src/types/sens.js +0 -4691
  316. package/lib/src/types/sessions.d.ts +0 -6
  317. package/lib/src/types/sessions.js +0 -19
  318. package/lib/src/types/texte.d.ts +0 -72
  319. package/lib/src/types/texte.js +0 -15
  320. package/lib/src/validators/config.d.ts +0 -9
  321. package/lib/src/validators/config.js +0 -10
  322. package/lib/strings.d.ts +0 -1
  323. package/lib/strings.js +0 -18
  324. package/lib/strings.mjs +0 -18
  325. package/lib/strings.ts +0 -26
  326. package/lib/tsconfig.tsbuildinfo +0 -1
  327. package/lib/types/agenda.d.ts +0 -44
  328. package/lib/types/agenda.js +0 -1
  329. package/lib/types/ameli.d.ts +0 -5
  330. package/lib/types/ameli.js +0 -1
  331. package/lib/types/ameli.mjs +0 -13
  332. package/lib/types/ameli.ts +0 -21
  333. package/lib/types/compte_rendu.d.ts +0 -83
  334. package/lib/types/compte_rendu.js +0 -1
  335. package/lib/types/debats.d.ts +0 -2
  336. package/lib/types/debats.js +0 -1
  337. package/lib/types/debats.mjs +0 -2
  338. package/lib/types/debats.ts +0 -6
  339. package/lib/types/dosleg.d.ts +0 -70
  340. package/lib/types/dosleg.js +0 -1
  341. package/lib/types/dosleg.mjs +0 -151
  342. package/lib/types/dosleg.ts +0 -284
  343. package/lib/types/questions.d.ts +0 -2
  344. package/lib/types/questions.js +0 -1
  345. package/lib/types/questions.mjs +0 -1
  346. package/lib/types/questions.ts +0 -3
  347. package/lib/types/sens.d.ts +0 -10
  348. package/lib/types/sens.js +0 -1
  349. package/lib/types/sens.mjs +0 -1
  350. package/lib/types/sens.ts +0 -12
  351. package/lib/types/sessions.d.ts +0 -5
  352. package/lib/types/sessions.js +0 -84
  353. package/lib/types/sessions.mjs +0 -43
  354. package/lib/types/sessions.ts +0 -42
  355. package/lib/types/texte.d.ts +0 -74
  356. package/lib/types/texte.js +0 -16
  357. package/lib/types/texte.mjs +0 -16
  358. package/lib/types/texte.ts +0 -76
  359. package/lib/typings/windows-1252.d.js +0 -2
  360. package/lib/typings/windows-1252.d.mjs +0 -2
  361. package/lib/typings/windows-1252.d.ts +0 -11
  362. package/lib/utils/cr_spliting.d.ts +0 -28
  363. package/lib/utils/cr_spliting.js +0 -265
  364. package/lib/utils/date.d.ts +0 -10
  365. package/lib/utils/date.js +0 -100
  366. package/lib/utils/nvs-timecode.d.ts +0 -7
  367. package/lib/utils/nvs-timecode.js +0 -79
  368. package/lib/utils/reunion_grouping.d.ts +0 -9
  369. package/lib/utils/reunion_grouping.js +0 -361
  370. package/lib/utils/reunion_odj_building.d.ts +0 -5
  371. package/lib/utils/reunion_odj_building.js +0 -154
  372. package/lib/utils/reunion_parsing.d.ts +0 -23
  373. package/lib/utils/reunion_parsing.js +0 -209
  374. package/lib/utils/scoring.d.ts +0 -14
  375. package/lib/utils/scoring.js +0 -147
  376. package/lib/utils/string_cleaning.d.ts +0 -7
  377. package/lib/utils/string_cleaning.js +0 -57
  378. package/lib/validators/config.d.ts +0 -9
  379. package/lib/validators/config.js +0 -10
  380. package/lib/validators/config.mjs +0 -54
  381. package/lib/validators/config.ts +0 -79
  382. package/lib/validators/senat.d.ts +0 -0
  383. package/lib/validators/senat.js +0 -28
  384. package/lib/validators/senat.mjs +0 -24
  385. package/lib/validators/senat.ts +0 -26
  386. /package/lib/{add-js-extensions-v2.d.ts → src/other_types/collaborateurs.js} +0 -0
  387. /package/lib/{add-js-extensions.d.ts → src/scripts/retrieve_collaborateurs.d.ts} +0 -0
  388. /package/lib/{scripts/convert_data.d.ts → tests/collaborateurs.test.d.ts} +0 -0
@@ -1,177 +0,0 @@
1
- import assert from "assert"
2
- import { execSync } from "child_process"
3
- import commandLineArgs from "command-line-args"
4
- import fs from "fs-extra"
5
- // import fetch from "node-fetch"
6
- import path from "path"
7
- // import stream from "stream"
8
- // import util from "util"
9
-
10
- import { findActif as findActifSenateurs } from "../model/sens"
11
- import { slugify } from "../strings"
12
- import { Photo } from "../types/sens"
13
- import { commonOptions } from "./shared/cli_helpers"
14
-
15
- const optionsDefinitions = [
16
- ...commonOptions,
17
- {
18
- alias: "f",
19
- help: "fetch sénateurs' pictures instead of retrieving them from files",
20
- name: "fetch",
21
- type: Boolean,
22
- },
23
- ]
24
- const options = commandLineArgs(optionsDefinitions)
25
- // const pipeline = util.promisify(stream.pipeline)
26
-
27
- async function retrievePhotosSenateurs(): Promise<void> {
28
- const dataDir = options.dataDir
29
- assert(dataDir, "Missing argument: data directory")
30
-
31
- const photosDir = path.join(dataDir, "photos_senateurs")
32
- const missingPhotoFilePath = path.resolve(__dirname, "images", "transparent_155x225.jpg")
33
-
34
- const sens = await Array.fromAsync(findActifSenateurs())
35
-
36
- // Download photos.
37
- fs.ensureDirSync(photosDir)
38
- if (options.fetch) {
39
- for (const sen of sens) {
40
- const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(
41
- sen.senprenomuse,
42
- "_",
43
- )}${slugify(sen.senmat, "_")}`
44
- const photoFilename = photoStem + ".jpg"
45
- const photoFilePath = path.join(photosDir, photoFilename)
46
- const photoTempFilename = photoStem + "_temp.jpg"
47
- const photoTempFilePath = path.join(photosDir, photoTempFilename)
48
- const urlPhoto = `https://www.senat.fr/senimg/${photoFilename}`
49
- if (!options.silent) {
50
- console.log(
51
- `Loading photo ${urlPhoto} for ${sen.senprenomuse} ${sen.sennomuse}…`,
52
- )
53
- }
54
- // Fetch fails with OpenSSL error: dh key too small.
55
- // (so does "curl").
56
- // for (let retries = 0; retries < 3; retries++) {
57
- // const response = await fetch(urlPhoto)
58
- // if (response.ok) {
59
- // await pipeline(response.body, fs.createWriteStream(photoTempFilePath))
60
- // fs.renameSync(photoTempFilePath, photoFilePath)
61
- // break
62
- // }
63
- // if (retries >= 2) {
64
- // console.warn(`Fetch failed: ${urlPhoto} (${sen.senprenomuse} ${sen.sennomuse})`)
65
- // console.warn(response.status, response.statusText)
66
- // console.warn(await response.text())
67
- // if (fs.existsSync(photoFilePath)) {
68
- // console.warn(" => Reusing existing image")
69
- // } else {
70
- // console.warn(" => Using blank image")
71
- // fs.copyFileSync(missingPhotoFilePath, photoFilePath)
72
- // }
73
- // break
74
- // }
75
- // }
76
- try {
77
- execSync(`wget --quiet -O ${photoTempFilename} ${urlPhoto}`, {
78
- cwd: photosDir,
79
- env: process.env,
80
- encoding: "utf-8",
81
- // stdio: ["ignore", "ignore", "pipe"],
82
- })
83
- fs.renameSync(photoTempFilePath, photoFilePath)
84
- } catch (error) {
85
- if (typeof error === "object" && error && "status" in error && error.status === 8) {
86
- console.error(`Unable to load photo for ${sen.senprenomuse} ${sen.sennomuse}`)
87
- continue
88
- }
89
- throw error
90
- }
91
- }
92
- }
93
-
94
- // Resize photos to 155x225, because some haven't exactly this size.
95
- for (const sen of sens) {
96
- const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(
97
- sen.senprenomuse,
98
- "_",
99
- )}${slugify(sen.senmat, "_")}`
100
- const photoFilename = photoStem + ".jpg"
101
- const photoFilePath = path.join(photosDir, photoFilename)
102
- if (fs.existsSync(photoFilePath)) {
103
- if (!options.silent) {
104
- console.log(
105
- `Resizing photo ${photoStem} for ${sen.senprenomuse} ${sen.sennomuse}…`,
106
- )
107
- }
108
-
109
- execSync(
110
- `gm convert -resize 155x225! ${photoStem}.jpg ${photoStem}_155x225.jpg`,
111
- {
112
- cwd: photosDir,
113
- },
114
- )
115
- } else {
116
- if (!options.silent) {
117
- console.warn(`Missing photo for ${sen.senprenomuse} ${sen.sennomuse}: using blank image`)
118
- }
119
- fs.copyFileSync(missingPhotoFilePath, path.join(photosDir, `${photoStem}_155x225.jpg`))
120
- }
121
- }
122
-
123
- // Create a mosaic of photos.
124
- if (!options.silent) {
125
- console.log("Creating mosaic of photos…")
126
- }
127
- const photoBySenmat: { [senmat: string]: Photo } = {}
128
- const rowsFilenames: string[] = []
129
- for (
130
- let senIndex = 0, rowIndex = 0;
131
- senIndex < sens.length;
132
- senIndex += 25, rowIndex++
133
- ) {
134
- const row = sens.slice(senIndex, senIndex + 25)
135
- const photosFilenames: string[] = []
136
- for (const [columnIndex, sen] of row.entries()) {
137
- const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(
138
- sen.senprenomuse,
139
- "_",
140
- )}${slugify(sen.senmat, "_")}`
141
- const photoFilename = `${photoStem}_155x225.jpg`
142
- photosFilenames.push(photoFilename)
143
- photoBySenmat[sen.senmat] = {
144
- chemin: `photos_senateurs/${photoFilename}`,
145
- cheminMosaique: "photos_senateurs/senateurs.jpg",
146
- hauteur: 225,
147
- largeur: 155,
148
- xMosaique: columnIndex * 155,
149
- yMosaique: rowIndex * 225,
150
- }
151
- }
152
- const rowFilename = `row-${rowIndex}.jpg`
153
- execSync(`gm convert ${photosFilenames.join(" ")} +append ${rowFilename}`, {
154
- cwd: photosDir,
155
- })
156
- rowsFilenames.push(rowFilename)
157
- }
158
- execSync(`gm convert ${rowsFilenames.join(" ")} -append senateurs.jpg`, {
159
- cwd: photosDir,
160
- })
161
- for (const rowFilename of rowsFilenames) {
162
- fs.unlinkSync(path.join(photosDir, rowFilename))
163
- }
164
-
165
- if (!options.silent) {
166
- console.log("Creating JSON file containing informations on all pictures…")
167
- }
168
- const jsonFilePath = path.join(photosDir, "senateurs.json")
169
- fs.writeFileSync(jsonFilePath, JSON.stringify(photoBySenmat, null, 2))
170
- }
171
-
172
- retrievePhotosSenateurs()
173
- .then(() => process.exit(0))
174
- .catch((error) => {
175
- console.log(error)
176
- process.exit(1)
177
- })
@@ -1 +0,0 @@
1
- export {};
@@ -1,461 +0,0 @@
1
- // scripts/retrieve_senat_videos_from_agendas.ts
2
- import assert from "assert";
3
- import commandLineArgs from "command-line-args";
4
- import fs from "fs-extra";
5
- import fsp from "fs/promises";
6
- import path from "path";
7
- import * as cheerio from "cheerio";
8
- import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatAgendas } from "../loaders";
9
- import { getSessionsFromStart } from "../types/sessions";
10
- import { commonOptions } from "./shared/cli_helpers";
11
- import { getAgendaSegmentTimecodes } from "../utils/nvs-timecode";
12
- import { decodeHtmlEntities } from "../utils/string_cleaning";
13
- import { dice, normalize, scoreVideo } from "../utils/scoring";
14
- import { epochToParisDateTime, toFRDate, toTargetEpoch } from "../utils/date";
15
- // ===================== Constants =====================
16
- const MATCH_THRESHOLD = 0.5;
17
- const MAX_CANDIDATES = 15;
18
- const STATS = { total: 0, accepted: 0 };
19
- const VIDEOS_ROOT_FOLDER = "videos";
20
- const SENAT_VIDEOS_SEARCH_AJAX = "https://videos.senat.fr/senat_videos_search.php";
21
- const SENAT_DATAS_ROOT = "https://videos.senat.fr/Datas/senat";
22
- // ===================== CLI =====================
23
- const optionsDefinitions = [...commonOptions];
24
- const options = commandLineArgs(optionsDefinitions);
25
- // ===================== Utils =====================
26
- async function fetchText(url) {
27
- const res = await fetch(url);
28
- if (!res.ok)
29
- return null;
30
- return await res.text();
31
- }
32
- async function fetchBuffer(url) {
33
- const res = await fetch(url);
34
- if (!res.ok)
35
- return null;
36
- const ab = await res.arrayBuffer();
37
- return Buffer.from(ab);
38
- }
39
- async function writeIfChanged(p, content) {
40
- const exists = await fs.pathExists(p);
41
- if (exists) {
42
- const old = await fsp.readFile(p, "utf-8");
43
- if (old === content)
44
- return;
45
- }
46
- await fsp.writeFile(p, content, "utf-8");
47
- }
48
- function queryString(obj) {
49
- return Object.entries(obj)
50
- .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
51
- .join("&");
52
- }
53
- function extractCandidatesFromSearchHtml(html) {
54
- const $ = cheerio.load(html);
55
- const out = [];
56
- const re = /video\.(\d+)_([a-z0-9]+)/i;
57
- $('h3.card-title a.stretched-link[href*="video."]').each((_, a) => {
58
- const href = $(a).attr("href") || "";
59
- const m = href.match(re);
60
- if (!m)
61
- return;
62
- const id = m[1];
63
- const hash = m[2];
64
- const pageUrl = `https://videos.senat.fr/video.${id}_${hash}.html`;
65
- const title = ($(a).attr("title") || $(a).text() || "").replace(/\s+/g, " ").trim() || undefined;
66
- const isSeancePublique = title?.toLowerCase().includes("séance publique") ?? false;
67
- out.push({ id, hash, pageUrl, title, isSeancePublique });
68
- });
69
- // dedupe
70
- const seen = new Set();
71
- return out.filter((c) => {
72
- const k = `${c.id}_${c.hash}`;
73
- if (seen.has(k))
74
- return false;
75
- seen.add(k);
76
- return true;
77
- });
78
- }
79
- function parseDataNvs(nvs) {
80
- const epochStr = nvs.match(/<metadata\s+name="date"\s+value="(\d+)"/i)?.[1];
81
- const epoch = epochStr ? Number(epochStr) : undefined;
82
- // There can be multiple organes for one video in meta
83
- const organes = [];
84
- const organesRegex = /<metadata\b[^>]*\bname="organes"[^>]*>/gi;
85
- let m;
86
- while ((m = organesRegex.exec(nvs)) !== null) {
87
- const tag = m[0];
88
- const label = tag.match(/\blabel="([^"]+)"/i)?.[1];
89
- if (label) {
90
- const decoded = decodeHtmlEntities(label).trim();
91
- if (decoded)
92
- organes.push(decoded);
93
- }
94
- }
95
- if (organes.length === 0) {
96
- organes.push("Séance publique");
97
- }
98
- const firstChapterLabelMatch = nvs.match(/<chapter\b[^>]*\blabel="([^"]+)"/i);
99
- const firstChapterLabel = firstChapterLabelMatch ? decodeHtmlEntities(firstChapterLabelMatch[1]).trim() : undefined;
100
- return { epoch, organes, firstChapterLabel };
101
- }
102
- function buildSenatVodMasterM3u8FromNvs(nvsText) {
103
- // serverfiles://senat/2025/10/encoder10_20251022084451_2.mp4
104
- const m = nvsText.match(/serverfiles:\/\/senat\/(\d{4})\/(\d{2})\/(encoder\d+)_([0-9]{14})/i);
105
- if (!m)
106
- return null;
107
- const [, yyyy, mm, encoder, stamp] = m;
108
- const base = `https://vodsenat.akamaized.net/senat/${yyyy}/${mm}/${encoder}_${stamp}`;
109
- return `${base}.smil/master.m3u8`;
110
- }
111
- function isAmbiguousTimeOriginal(timeOriginal) {
112
- if (!timeOriginal)
113
- return false;
114
- const s = timeOriginal.toLowerCase();
115
- // Catches "14h", "14 h", "14h30", "14 h 30", "14 heures", "14 heure"
116
- const timeRe = /\b([01]?\d|2[0-3])\s*(?:h|heures?|heure)\s*(?:([0-5]\d))?\b/g;
117
- const times = new Set();
118
- let m;
119
- while ((m = timeRe.exec(s))) {
120
- const hh = String(m[1]).padStart(2, "0");
121
- const mm = m[2] ? String(m[2]).padStart(2, "0") : "00";
122
- times.add(`${hh}:${mm}`);
123
- }
124
- // "midi" / "minuit"
125
- if (/\bmidi\b/.test(s))
126
- times.add("12:00");
127
- if (/\bminuit\b/.test(s))
128
- times.add("00:00");
129
- if (times.size >= 2)
130
- return true;
131
- const hasDayPeriod = /\b(matin|après-?midi|soir|nuit|journée|toute la journée)\b/.test(s);
132
- const hasLinking = /,|\bet\b|\bou\b|\bpuis\b/.test(s);
133
- if (times.size === 1 && hasDayPeriod && hasLinking)
134
- return true;
135
- return false;
136
- }
137
- function getAgendaType(agenda) {
138
- const o = agenda.organe || "";
139
- if (/séance publique/i.test(o))
140
- return "Séance publique";
141
- return "Commission";
142
- }
143
- async function fetchAllSearchPages(args, maxPages = 3) {
144
- const pages = [];
145
- for (let p = 1; p <= maxPages; p++) {
146
- const url = `${SENAT_VIDEOS_SEARCH_AJAX}?${queryString({ ...args, page: String(p) })}`;
147
- const html = await fetchText(url);
148
- if (!html)
149
- break;
150
- pages.push(html);
151
- if (!/href="\/?video\.\d+_[a-z0-9]+\./i.test(html))
152
- break;
153
- }
154
- return pages;
155
- }
156
- function getOrgKey(norm) {
157
- if (!norm)
158
- return "autre";
159
- if (norm.includes("seance publique"))
160
- return "seance_publique";
161
- if (norm.includes("culture"))
162
- return "culture";
163
- if (norm.includes("finances"))
164
- return "finances";
165
- if (norm.includes("sociales"))
166
- return "affaires_sociales";
167
- if (norm.includes("economiques"))
168
- return "affaires_economiques";
169
- if (norm.includes("europeennes"))
170
- return "affaires_europeennes";
171
- if (norm.includes("etrangeres") || norm.includes("forces armees") || norm.includes("defense")) {
172
- return "affaires_etrangeres_defense";
173
- }
174
- if (norm.includes("territoire") || norm.includes("durable")) {
175
- return "amenagement_territoire_dd";
176
- }
177
- if (norm.includes("commission des lois"))
178
- return "lois";
179
- if (norm.includes("delegation aux collectivites territoriales") || norm.includes("delegation a la decentralisation"))
180
- return "delegation_collectivites";
181
- if (norm.includes("delegation aux droits des femmes") ||
182
- norm.includes("egalite des chances entre les hommes et les femmes"))
183
- return "delegation_droits_femmes";
184
- if (norm.includes("delegation aux entreprises"))
185
- return "delegation_entreprises";
186
- if (norm.includes("delegation senatoriale aux outre mer") || norm.includes("delegation aux outre mer"))
187
- return "delegation_outre_mer";
188
- if (norm.includes("delegation a la prospective"))
189
- return "delegation_prospective";
190
- if (norm.includes("office parlementaire d evaluation des choix scientifiques et technologiques") ||
191
- norm.includes("opecst"))
192
- return "opecst";
193
- return "autre";
194
- }
195
- async function processGroupedReunion(agenda, session, dataDir) {
196
- // 1) GuardRails
197
- if (!agenda.captationVideo) {
198
- // if (!options["silent"]) console.log(`[skip] ${agenda.uid} captationVideo=false`)
199
- return;
200
- }
201
- if (!agenda.date || !agenda.startTime) {
202
- // if (!options["silent"]) console.log(`[skip] ${agenda.uid} date/hour missing`)
203
- return;
204
- }
205
- const agendaTs = toTargetEpoch(agenda.startTime, agenda.date);
206
- const now = Date.now();
207
- if (agendaTs && agendaTs * 1000 > now) {
208
- return;
209
- }
210
- const reunionUid = agenda.uid;
211
- const baseDir = path.join(dataDir, VIDEOS_ROOT_FOLDER, String(session), reunionUid);
212
- await fs.ensureDir(baseDir);
213
- let skipDownload = false;
214
- if (options["only-recent"]) {
215
- const now = Date.now();
216
- const cutoff = now - options["only-recent"] * 24 * 3600 * 1000;
217
- const reunionTs = Date.parse(agenda.date);
218
- if (reunionTs < cutoff) {
219
- // Check if files already exist
220
- const dataNvsPath = path.join(baseDir, "data.nvs");
221
- const finalplayerNvsPath = path.join(baseDir, "finalplayer.nvs");
222
- if (fs.existsSync(dataNvsPath) && fs.existsSync(finalplayerNvsPath)) {
223
- skipDownload = true;
224
- }
225
- }
226
- }
227
- let master = null;
228
- let dataTxt = null;
229
- let finalTxt = null;
230
- let accepted = false;
231
- if (!skipDownload) {
232
- STATS.total++;
233
- const searchParams = {
234
- search: "true",
235
- videotype: getAgendaType(agenda),
236
- };
237
- if (agenda.date) {
238
- const fr = toFRDate(agenda.date);
239
- searchParams.period = "custom";
240
- searchParams.begin = fr;
241
- searchParams.end = fr;
242
- }
243
- if (agenda.organe) {
244
- searchParams.organe = agenda.organe;
245
- }
246
- const pages = await fetchAllSearchPages(searchParams);
247
- if (!pages.length) {
248
- if (!options["silent"]) {
249
- console.log(`[miss] ${agenda.uid} no candidates (videotype=${searchParams.videotype}, organe=${searchParams.organe || "-"}, date=${searchParams.begin || "-"})`);
250
- }
251
- return;
252
- }
253
- const combinedHtml = pages.join("\n<!-- PAGE SPLIT -->\n");
254
- const candidates = extractCandidatesFromSearchHtml(combinedHtml).slice(0, MAX_CANDIDATES);
255
- if (!candidates.length) {
256
- if (!options["silent"]) {
257
- console.log(`[miss] ${agenda.uid} no candidates after parse (videotype=${searchParams.videotype}, organe=${searchParams.organe || "-"}, date=${searchParams.begin || "-"})`);
258
- }
259
- return;
260
- }
261
- // ==== 2) Enrich via data.nvs + scoring; pick best ====
262
- let best = null;
263
- const timeAmbigious = isAmbiguousTimeOriginal(agenda.events[0].timeOriginal);
264
- if (timeAmbigious) {
265
- console.log(`[match] ${agenda.uid} timeOriginal ambiguous => ignoring time scoring: "${agenda.events[0].timeOriginal}"`);
266
- }
267
- for (const c of candidates) {
268
- const dataUrl = `${SENAT_DATAS_ROOT}/${c.id}_${c.hash}/content/data.nvs`;
269
- const finalUrl = `${SENAT_DATAS_ROOT}/${c.id}_${c.hash}/content/finalplayer.nvs`;
270
- const dataBuf = await fetchBuffer(dataUrl);
271
- if (!dataBuf)
272
- continue;
273
- const meta = parseDataNvs(dataBuf.toString("utf-8"));
274
- let sameOrg = false;
275
- // If organes are too different, go to next candidates
276
- if (agenda.organe && meta.organes?.length) {
277
- const agendaOrgNorm = normalize(agenda.organe);
278
- const agendaKey = getOrgKey(agendaOrgNorm);
279
- let bestDice = 0;
280
- let hasSameKey = false;
281
- for (const vo of meta.organes) {
282
- const videoOrgNorm = normalize(vo);
283
- const videoKey = getOrgKey(videoOrgNorm);
284
- const d = dice(agendaOrgNorm, videoOrgNorm);
285
- if (videoKey === agendaKey && videoKey !== "autre") {
286
- hasSameKey = true;
287
- }
288
- if (d > bestDice)
289
- bestDice = d;
290
- }
291
- if (hasSameKey) {
292
- sameOrg = true; // we are sure this is the same org
293
- }
294
- else if (bestDice < 0.8) {
295
- // if diff org and dice too low we skip
296
- continue;
297
- }
298
- }
299
- let videoTitle = c.title;
300
- if (c.isSeancePublique && meta.firstChapterLabel) {
301
- videoTitle = meta.firstChapterLabel;
302
- }
303
- const s = scoreVideo(agenda, agendaTs, sameOrg, videoTitle, meta.epoch, meta.organes, timeAmbigious);
304
- if (!best || s > best.score) {
305
- best = {
306
- id: c.id,
307
- hash: c.hash,
308
- pageUrl: c.pageUrl,
309
- epoch: meta.epoch,
310
- vtitle: videoTitle,
311
- score: s,
312
- vorgane: meta.organes[0],
313
- };
314
- }
315
- }
316
- if (!best) {
317
- if (!options["silent"])
318
- console.log(`[miss] ${agenda.uid} No candidate found for this reunion`);
319
- return;
320
- }
321
- accepted = best.score >= MATCH_THRESHOLD;
322
- if (accepted)
323
- STATS.accepted++;
324
- if (!options["silent"]) {
325
- console.log(`[pick] ${agenda.uid} score=${best.score.toFixed(2)}
326
- agenda title="${agenda.titre ?? ""}" agenda organe="${agenda.organe ?? ""}" agenda heure=${agenda.startTime}
327
- best title="${best.vtitle ?? ""}" best organe="${best.vorgane ?? ""}"
328
- accepted=${accepted}`);
329
- }
330
- // ==== 3) Write metadata + NVS of the best candidate (always) ====
331
- const bestDt = best?.epoch ? epochToParisDateTime(best.epoch) : null;
332
- const metadata = {
333
- reunionUid,
334
- session,
335
- accepted,
336
- threshold: MATCH_THRESHOLD,
337
- agenda: {
338
- date: agenda.date,
339
- startTime: agenda.startTime,
340
- titre: agenda.titre,
341
- organe: agenda.organe ?? undefined,
342
- uid: agenda.uid,
343
- },
344
- best: {
345
- id: best.id,
346
- hash: best.hash,
347
- pageUrl: best.pageUrl,
348
- epoch: best.epoch ?? null,
349
- date: bestDt?.date ?? null,
350
- startTime: bestDt?.startTime ?? null,
351
- title: best.vtitle ?? null,
352
- score: best.score,
353
- },
354
- };
355
- await writeIfChanged(path.join(baseDir, "metadata.json"), JSON.stringify(metadata, null, 2));
356
- const dataUrl = `${SENAT_DATAS_ROOT}/${best.id}_${best.hash}/content/data.nvs`;
357
- const finalUrl = `${SENAT_DATAS_ROOT}/${best.id}_${best.hash}/content/finalplayer.nvs`;
358
- dataTxt = await fetchText(dataUrl);
359
- finalTxt = await fetchText(finalUrl);
360
- if (dataTxt)
361
- await fsp.writeFile(path.join(baseDir, "data.nvs"), dataTxt, "utf-8");
362
- if (finalTxt)
363
- await fsp.writeFile(path.join(baseDir, "finalplayer.nvs"), finalTxt, "utf-8");
364
- if (dataTxt) {
365
- master = buildSenatVodMasterM3u8FromNvs(dataTxt);
366
- }
367
- else {
368
- console.log("Cannot download data nvs");
369
- }
370
- }
371
- else {
372
- // Skipped download, but need to read data.nvs for urlVideo
373
- try {
374
- dataTxt = await fsp.readFile(path.join(baseDir, "data.nvs"), "utf-8");
375
- finalTxt = await fsp.readFile(path.join(baseDir, "finalplayer.nvs"), "utf-8");
376
- master = buildSenatVodMasterM3u8FromNvs(dataTxt);
377
- }
378
- catch (e) {
379
- console.warn(e);
380
- }
381
- }
382
- // ==== 4) Update agenda file (only if accepted + m3u8) ====
383
- if ((accepted || skipDownload) && master) {
384
- const agendaJsonPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session), `${agenda.uid}.json`);
385
- let timecodeDebutVideo = null;
386
- let timecodeFinVideo = null;
387
- if (dataTxt && finalTxt) {
388
- const agendaKey = agenda.titre || agenda.objet || "";
389
- const seg = getAgendaSegmentTimecodes(dataTxt, finalTxt, agendaKey);
390
- if (!seg) {
391
- console.warn(`[warn] Cannot retrieve agenda segment timecodes from reunion ${reunionUid}`);
392
- }
393
- else {
394
- timecodeDebutVideo = seg.start;
395
- timecodeFinVideo = seg.end;
396
- }
397
- }
398
- if (await fs.pathExists(agendaJsonPath)) {
399
- const raw = await fsp.readFile(agendaJsonPath, "utf-8");
400
- let obj;
401
- try {
402
- obj = JSON.parse(raw);
403
- }
404
- catch (e) {
405
- console.warn(`[warn] invalid JSON in ${agendaJsonPath}:`, e?.message);
406
- obj = null;
407
- }
408
- if (obj && typeof obj === "object" && !Array.isArray(obj)) {
409
- const next = { ...obj, urlVideo: master };
410
- if (timecodeDebutVideo != null) {
411
- next.timecodeDebutVideo = timecodeDebutVideo;
412
- next.timecodeFinVideo = timecodeFinVideo;
413
- }
414
- await writeIfChanged(agendaJsonPath, JSON.stringify(next, null, 2));
415
- if (!options["silent"]) {
416
- console.log(`[write] ${agenda.uid} urlVideo ← ${master}` +
417
- (timecodeDebutVideo != null ? ` (timecodeDebutVideo ← ${timecodeDebutVideo}s)` : ""));
418
- }
419
- }
420
- else {
421
- console.warn(`[warn] expected an object in ${agendaJsonPath}, got ${Array.isArray(obj) ? "array" : typeof obj}`);
422
- }
423
- }
424
- else {
425
- console.warn(`[warn] agenda file not found for update: ${agendaJsonPath}`);
426
- }
427
- }
428
- else {
429
- console.warn(`[warn] The video url could not be built for reunion `, reunionUid);
430
- }
431
- }
432
- async function processAll(dataDir, sessions) {
433
- console.log("Process all Agendas and fetch video's url");
434
- for (const session of sessions) {
435
- for (const { item: agenda } of iterLoadSenatAgendas(dataDir, session)) {
436
- try {
437
- await processGroupedReunion(agenda, session, dataDir);
438
- }
439
- catch (e) {
440
- console.error(`[error] ${agenda?.uid ?? "unknown-uid"}:`, e?.message || e);
441
- }
442
- }
443
- }
444
- }
445
- async function main() {
446
- const dataDir = options["dataDir"];
447
- assert(dataDir, "Missing argument: data directory");
448
- const sessions = getSessionsFromStart(options["fromSession"]);
449
- console.time("senat-agendas→videos start processing time");
450
- await processAll(dataDir, sessions);
451
- console.timeEnd("senat-agendas→videos processing time");
452
- const { total, accepted } = STATS;
453
- const ratio = total ? ((accepted / total) * 100).toFixed(1) : "0.0";
454
- console.log(`[summary] accepted=${accepted} / total=${total} (${ratio}%)`);
455
- }
456
- main()
457
- .then(() => process.exit(0))
458
- .catch((err) => {
459
- console.error(err);
460
- process.exit(1);
461
- });