@tricoteuses/senat 3.1.0 → 3.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (306) hide show
  1. package/lib/src/loaders.d.ts +3 -3
  2. package/lib/src/loaders.js +1 -1
  3. package/lib/src/model/agenda.d.ts +1 -1
  4. package/lib/src/model/commission.d.ts +2 -2
  5. package/lib/src/model/seance.d.ts +1 -1
  6. package/lib/src/types/ameli.d.ts +4 -1761
  7. package/lib/src/types/ameli.js +1 -1074
  8. package/lib/src/types/debats.d.ts +2 -380
  9. package/lib/src/types/debats.js +1 -266
  10. package/lib/src/types/dosleg.d.ts +69 -2953
  11. package/lib/src/types/dosleg.js +1 -2005
  12. package/lib/src/types/questions.d.ts +2 -699
  13. package/lib/src/types/questions.js +1 -493
  14. package/lib/src/types/sens.d.ts +7 -7842
  15. package/lib/src/types/sens.js +1 -4691
  16. package/lib/src/utils/nvs-parsing.d.ts +1 -1
  17. package/lib/src/utils/nvs-parsing.js +9 -1
  18. package/lib/src/videos/pipeline.d.ts +3 -3
  19. package/lib/src/videos/pipeline.js +2 -2
  20. package/package.json +1 -1
  21. package/lib/add-js-extensions-v2.d.ts +0 -1
  22. package/lib/add-js-extensions-v2.js +0 -23
  23. package/lib/add-js-extensions.d.ts +0 -1
  24. package/lib/add-js-extensions.js +0 -17
  25. package/lib/aggregates.d.ts +0 -52
  26. package/lib/aggregates.js +0 -930
  27. package/lib/aggregates.mjs +0 -713
  28. package/lib/aggregates.ts +0 -833
  29. package/lib/config.d.ts +0 -10
  30. package/lib/config.js +0 -16
  31. package/lib/config.mjs +0 -16
  32. package/lib/config.ts +0 -26
  33. package/lib/databases.d.ts +0 -2
  34. package/lib/databases.js +0 -26
  35. package/lib/databases.mjs +0 -57
  36. package/lib/databases.ts +0 -71
  37. package/lib/datasets.d.ts +0 -34
  38. package/lib/datasets.js +0 -233
  39. package/lib/datasets.mjs +0 -78
  40. package/lib/datasets.ts +0 -118
  41. package/lib/fields.d.ts +0 -10
  42. package/lib/fields.js +0 -68
  43. package/lib/fields.mjs +0 -22
  44. package/lib/fields.ts +0 -29
  45. package/lib/git.d.ts +0 -26
  46. package/lib/git.js +0 -167
  47. package/lib/index.d.ts +0 -13
  48. package/lib/index.js +0 -1
  49. package/lib/index.mjs +0 -7
  50. package/lib/index.ts +0 -64
  51. package/lib/inserters.d.ts +0 -98
  52. package/lib/inserters.js +0 -500
  53. package/lib/inserters.mjs +0 -360
  54. package/lib/inserters.ts +0 -521
  55. package/lib/legislatures.json +0 -38
  56. package/lib/loaders.d.ts +0 -58
  57. package/lib/loaders.js +0 -286
  58. package/lib/loaders.mjs +0 -158
  59. package/lib/loaders.ts +0 -271
  60. package/lib/model/agenda.d.ts +0 -6
  61. package/lib/model/agenda.js +0 -148
  62. package/lib/model/ameli.d.ts +0 -51
  63. package/lib/model/ameli.js +0 -149
  64. package/lib/model/ameli.mjs +0 -84
  65. package/lib/model/ameli.ts +0 -100
  66. package/lib/model/commission.d.ts +0 -18
  67. package/lib/model/commission.js +0 -269
  68. package/lib/model/debats.d.ts +0 -67
  69. package/lib/model/debats.js +0 -95
  70. package/lib/model/debats.mjs +0 -43
  71. package/lib/model/debats.ts +0 -68
  72. package/lib/model/documents.d.ts +0 -12
  73. package/lib/model/documents.js +0 -151
  74. package/lib/model/dosleg.d.ts +0 -7
  75. package/lib/model/dosleg.js +0 -326
  76. package/lib/model/dosleg.mjs +0 -196
  77. package/lib/model/dosleg.ts +0 -240
  78. package/lib/model/index.d.ts +0 -7
  79. package/lib/model/index.js +0 -7
  80. package/lib/model/index.mjs +0 -5
  81. package/lib/model/index.ts +0 -15
  82. package/lib/model/questions.d.ts +0 -45
  83. package/lib/model/questions.js +0 -89
  84. package/lib/model/questions.mjs +0 -71
  85. package/lib/model/questions.ts +0 -93
  86. package/lib/model/scrutins.d.ts +0 -13
  87. package/lib/model/scrutins.js +0 -114
  88. package/lib/model/seance.d.ts +0 -3
  89. package/lib/model/seance.js +0 -267
  90. package/lib/model/sens.d.ts +0 -146
  91. package/lib/model/sens.js +0 -454
  92. package/lib/model/sens.mjs +0 -415
  93. package/lib/model/sens.ts +0 -516
  94. package/lib/model/texte.d.ts +0 -7
  95. package/lib/model/texte.js +0 -256
  96. package/lib/model/texte.mjs +0 -208
  97. package/lib/model/texte.ts +0 -229
  98. package/lib/model/util.d.ts +0 -9
  99. package/lib/model/util.js +0 -38
  100. package/lib/model/util.mjs +0 -19
  101. package/lib/model/util.ts +0 -32
  102. package/lib/parsers/texte.d.ts +0 -7
  103. package/lib/parsers/texte.js +0 -228
  104. package/lib/raw_types/ameli.d.ts +0 -914
  105. package/lib/raw_types/ameli.js +0 -5
  106. package/lib/raw_types/ameli.mjs +0 -163
  107. package/lib/raw_types/debats.d.ts +0 -207
  108. package/lib/raw_types/debats.js +0 -5
  109. package/lib/raw_types/debats.mjs +0 -58
  110. package/lib/raw_types/dosleg.d.ts +0 -1619
  111. package/lib/raw_types/dosleg.js +0 -5
  112. package/lib/raw_types/dosleg.mjs +0 -438
  113. package/lib/raw_types/questions.d.ts +0 -419
  114. package/lib/raw_types/questions.js +0 -5
  115. package/lib/raw_types/questions.mjs +0 -11
  116. package/lib/raw_types/senat.d.ts +0 -11368
  117. package/lib/raw_types/senat.js +0 -5
  118. package/lib/raw_types/sens.d.ts +0 -8248
  119. package/lib/raw_types/sens.js +0 -5
  120. package/lib/raw_types/sens.mjs +0 -508
  121. package/lib/raw_types_kysely/ameli.d.ts +0 -915
  122. package/lib/raw_types_kysely/ameli.js +0 -7
  123. package/lib/raw_types_kysely/ameli.mjs +0 -5
  124. package/lib/raw_types_kysely/ameli.ts +0 -951
  125. package/lib/raw_types_kysely/debats.d.ts +0 -207
  126. package/lib/raw_types_kysely/debats.js +0 -7
  127. package/lib/raw_types_kysely/debats.mjs +0 -5
  128. package/lib/raw_types_kysely/debats.ts +0 -222
  129. package/lib/raw_types_kysely/dosleg.d.ts +0 -3532
  130. package/lib/raw_types_kysely/dosleg.js +0 -7
  131. package/lib/raw_types_kysely/dosleg.mjs +0 -5
  132. package/lib/raw_types_kysely/dosleg.ts +0 -3621
  133. package/lib/raw_types_kysely/questions.d.ts +0 -414
  134. package/lib/raw_types_kysely/questions.js +0 -7
  135. package/lib/raw_types_kysely/questions.mjs +0 -5
  136. package/lib/raw_types_kysely/questions.ts +0 -426
  137. package/lib/raw_types_kysely/sens.d.ts +0 -4394
  138. package/lib/raw_types_kysely/sens.js +0 -7
  139. package/lib/raw_types_kysely/sens.mjs +0 -5
  140. package/lib/raw_types_kysely/sens.ts +0 -4499
  141. package/lib/raw_types_schemats/ameli.d.ts +0 -539
  142. package/lib/raw_types_schemats/ameli.js +0 -2
  143. package/lib/raw_types_schemats/ameli.mjs +0 -2
  144. package/lib/raw_types_schemats/ameli.ts +0 -601
  145. package/lib/raw_types_schemats/debats.d.ts +0 -127
  146. package/lib/raw_types_schemats/debats.js +0 -2
  147. package/lib/raw_types_schemats/debats.mjs +0 -2
  148. package/lib/raw_types_schemats/debats.ts +0 -145
  149. package/lib/raw_types_schemats/dosleg.d.ts +0 -977
  150. package/lib/raw_types_schemats/dosleg.js +0 -2
  151. package/lib/raw_types_schemats/dosleg.mjs +0 -2
  152. package/lib/raw_types_schemats/dosleg.ts +0 -2193
  153. package/lib/raw_types_schemats/questions.d.ts +0 -235
  154. package/lib/raw_types_schemats/questions.js +0 -2
  155. package/lib/raw_types_schemats/questions.mjs +0 -2
  156. package/lib/raw_types_schemats/questions.ts +0 -249
  157. package/lib/raw_types_schemats/sens.d.ts +0 -6915
  158. package/lib/raw_types_schemats/sens.js +0 -2
  159. package/lib/raw_types_schemats/sens.mjs +0 -2
  160. package/lib/raw_types_schemats/sens.ts +0 -2907
  161. package/lib/scripts/convert_data.d.ts +0 -1
  162. package/lib/scripts/convert_data.js +0 -354
  163. package/lib/scripts/convert_data.mjs +0 -181
  164. package/lib/scripts/convert_data.ts +0 -243
  165. package/lib/scripts/data-download.d.ts +0 -1
  166. package/lib/scripts/data-download.js +0 -12
  167. package/lib/scripts/datautil.d.ts +0 -8
  168. package/lib/scripts/datautil.js +0 -34
  169. package/lib/scripts/datautil.mjs +0 -16
  170. package/lib/scripts/datautil.ts +0 -19
  171. package/lib/scripts/images/transparent_150x192.jpg +0 -0
  172. package/lib/scripts/images/transparent_155x225.jpg +0 -0
  173. package/lib/scripts/parse_textes.d.ts +0 -1
  174. package/lib/scripts/parse_textes.js +0 -44
  175. package/lib/scripts/parse_textes.mjs +0 -46
  176. package/lib/scripts/parse_textes.ts +0 -65
  177. package/lib/scripts/retrieve_agenda.d.ts +0 -1
  178. package/lib/scripts/retrieve_agenda.js +0 -132
  179. package/lib/scripts/retrieve_cr_commission.d.ts +0 -1
  180. package/lib/scripts/retrieve_cr_commission.js +0 -364
  181. package/lib/scripts/retrieve_cr_seance.d.ts +0 -6
  182. package/lib/scripts/retrieve_cr_seance.js +0 -347
  183. package/lib/scripts/retrieve_documents.d.ts +0 -3
  184. package/lib/scripts/retrieve_documents.js +0 -219
  185. package/lib/scripts/retrieve_documents.mjs +0 -249
  186. package/lib/scripts/retrieve_documents.ts +0 -298
  187. package/lib/scripts/retrieve_open_data.d.ts +0 -1
  188. package/lib/scripts/retrieve_open_data.js +0 -315
  189. package/lib/scripts/retrieve_open_data.mjs +0 -217
  190. package/lib/scripts/retrieve_open_data.ts +0 -268
  191. package/lib/scripts/retrieve_senateurs_photos.d.ts +0 -1
  192. package/lib/scripts/retrieve_senateurs_photos.js +0 -147
  193. package/lib/scripts/retrieve_senateurs_photos.mjs +0 -147
  194. package/lib/scripts/retrieve_senateurs_photos.ts +0 -177
  195. package/lib/scripts/retrieve_videos.d.ts +0 -1
  196. package/lib/scripts/retrieve_videos.js +0 -461
  197. package/lib/scripts/shared/cli_helpers.d.ts +0 -95
  198. package/lib/scripts/shared/cli_helpers.js +0 -91
  199. package/lib/scripts/shared/cli_helpers.ts +0 -36
  200. package/lib/scripts/shared/util.d.ts +0 -4
  201. package/lib/scripts/shared/util.js +0 -35
  202. package/lib/scripts/shared/util.ts +0 -33
  203. package/lib/scripts/test_iter_load.d.ts +0 -1
  204. package/lib/scripts/test_iter_load.js +0 -12
  205. package/lib/src/ameli.d.ts +0 -66
  206. package/lib/src/ameli.js +0 -1
  207. package/lib/src/databases.d.ts +0 -3
  208. package/lib/src/databases.js +0 -26
  209. package/lib/src/db_types/ameli.d.ts +0 -1762
  210. package/lib/src/db_types/ameli.js +0 -1074
  211. package/lib/src/db_types/debats.d.ts +0 -380
  212. package/lib/src/db_types/debats.js +0 -266
  213. package/lib/src/db_types/dosleg.d.ts +0 -2954
  214. package/lib/src/db_types/dosleg.js +0 -2005
  215. package/lib/src/db_types/questions.d.ts +0 -699
  216. package/lib/src/db_types/questions.js +0 -493
  217. package/lib/src/db_types/sens.d.ts +0 -7843
  218. package/lib/src/db_types/sens.js +0 -4691
  219. package/lib/src/debats.d.ts +0 -38
  220. package/lib/src/debats.js +0 -1
  221. package/lib/src/dosleg.d.ts +0 -142
  222. package/lib/src/dosleg.js +0 -193
  223. package/lib/src/model/ameli_postgres.d.ts +0 -67
  224. package/lib/src/model/ameli_postgres.js +0 -150
  225. package/lib/src/other_types/questions.d.ts +0 -2
  226. package/lib/src/other_types/questions.js +0 -1
  227. package/lib/src/questions.d.ts +0 -53
  228. package/lib/src/questions.js +0 -1
  229. package/lib/src/raw_types/senat.d.ts +0 -11372
  230. package/lib/src/raw_types/senat.js +0 -5
  231. package/lib/src/rich_types/agenda.d.ts +0 -45
  232. package/lib/src/rich_types/agenda.js +0 -1
  233. package/lib/src/rich_types/compte_rendu.d.ts +0 -83
  234. package/lib/src/rich_types/compte_rendu.js +0 -1
  235. package/lib/src/rich_types/sessions.d.ts +0 -6
  236. package/lib/src/rich_types/sessions.js +0 -19
  237. package/lib/src/rich_types/texte.d.ts +0 -72
  238. package/lib/src/rich_types/texte.js +0 -15
  239. package/lib/src/scripts/test_iter_load.d.ts +0 -1
  240. package/lib/src/scripts/test_iter_load.js +0 -12
  241. package/lib/src/sens.d.ts +0 -104
  242. package/lib/src/sens.js +0 -1
  243. package/lib/strings.d.ts +0 -1
  244. package/lib/strings.js +0 -18
  245. package/lib/strings.mjs +0 -18
  246. package/lib/strings.ts +0 -26
  247. package/lib/tsconfig.tsbuildinfo +0 -1
  248. package/lib/types/agenda.d.ts +0 -44
  249. package/lib/types/agenda.js +0 -1
  250. package/lib/types/ameli.d.ts +0 -5
  251. package/lib/types/ameli.js +0 -1
  252. package/lib/types/ameli.mjs +0 -13
  253. package/lib/types/ameli.ts +0 -21
  254. package/lib/types/compte_rendu.d.ts +0 -83
  255. package/lib/types/compte_rendu.js +0 -1
  256. package/lib/types/debats.d.ts +0 -2
  257. package/lib/types/debats.js +0 -1
  258. package/lib/types/debats.mjs +0 -2
  259. package/lib/types/debats.ts +0 -6
  260. package/lib/types/dosleg.d.ts +0 -70
  261. package/lib/types/dosleg.js +0 -1
  262. package/lib/types/dosleg.mjs +0 -151
  263. package/lib/types/dosleg.ts +0 -284
  264. package/lib/types/questions.d.ts +0 -2
  265. package/lib/types/questions.js +0 -1
  266. package/lib/types/questions.mjs +0 -1
  267. package/lib/types/questions.ts +0 -3
  268. package/lib/types/sens.d.ts +0 -10
  269. package/lib/types/sens.js +0 -1
  270. package/lib/types/sens.mjs +0 -1
  271. package/lib/types/sens.ts +0 -12
  272. package/lib/types/sessions.d.ts +0 -5
  273. package/lib/types/sessions.js +0 -84
  274. package/lib/types/sessions.mjs +0 -43
  275. package/lib/types/sessions.ts +0 -42
  276. package/lib/types/texte.d.ts +0 -74
  277. package/lib/types/texte.js +0 -16
  278. package/lib/types/texte.mjs +0 -16
  279. package/lib/types/texte.ts +0 -76
  280. package/lib/typings/windows-1252.d.js +0 -2
  281. package/lib/typings/windows-1252.d.mjs +0 -2
  282. package/lib/typings/windows-1252.d.ts +0 -11
  283. package/lib/utils/cr_spliting.d.ts +0 -28
  284. package/lib/utils/cr_spliting.js +0 -265
  285. package/lib/utils/date.d.ts +0 -10
  286. package/lib/utils/date.js +0 -100
  287. package/lib/utils/nvs-timecode.d.ts +0 -7
  288. package/lib/utils/nvs-timecode.js +0 -79
  289. package/lib/utils/reunion_grouping.d.ts +0 -9
  290. package/lib/utils/reunion_grouping.js +0 -361
  291. package/lib/utils/reunion_odj_building.d.ts +0 -5
  292. package/lib/utils/reunion_odj_building.js +0 -154
  293. package/lib/utils/reunion_parsing.d.ts +0 -23
  294. package/lib/utils/reunion_parsing.js +0 -209
  295. package/lib/utils/scoring.d.ts +0 -14
  296. package/lib/utils/scoring.js +0 -147
  297. package/lib/utils/string_cleaning.d.ts +0 -7
  298. package/lib/utils/string_cleaning.js +0 -57
  299. package/lib/validators/config.d.ts +0 -9
  300. package/lib/validators/config.js +0 -10
  301. package/lib/validators/config.mjs +0 -54
  302. package/lib/validators/config.ts +0 -79
  303. package/lib/validators/senat.d.ts +0 -0
  304. package/lib/validators/senat.js +0 -28
  305. package/lib/validators/senat.mjs +0 -24
  306. package/lib/validators/senat.ts +0 -26
@@ -1,177 +0,0 @@
1
- import assert from "assert"
2
- import { execSync } from "child_process"
3
- import commandLineArgs from "command-line-args"
4
- import fs from "fs-extra"
5
- // import fetch from "node-fetch"
6
- import path from "path"
7
- // import stream from "stream"
8
- // import util from "util"
9
-
10
- import { findActif as findActifSenateurs } from "../model/sens"
11
- import { slugify } from "../strings"
12
- import { Photo } from "../types/sens"
13
- import { commonOptions } from "./shared/cli_helpers"
14
-
15
- const optionsDefinitions = [
16
- ...commonOptions,
17
- {
18
- alias: "f",
19
- help: "fetch sénateurs' pictures instead of retrieving them from files",
20
- name: "fetch",
21
- type: Boolean,
22
- },
23
- ]
24
- const options = commandLineArgs(optionsDefinitions)
25
- // const pipeline = util.promisify(stream.pipeline)
26
-
27
- async function retrievePhotosSenateurs(): Promise<void> {
28
- const dataDir = options.dataDir
29
- assert(dataDir, "Missing argument: data directory")
30
-
31
- const photosDir = path.join(dataDir, "photos_senateurs")
32
- const missingPhotoFilePath = path.resolve(__dirname, "images", "transparent_155x225.jpg")
33
-
34
- const sens = await Array.fromAsync(findActifSenateurs())
35
-
36
- // Download photos.
37
- fs.ensureDirSync(photosDir)
38
- if (options.fetch) {
39
- for (const sen of sens) {
40
- const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(
41
- sen.senprenomuse,
42
- "_",
43
- )}${slugify(sen.senmat, "_")}`
44
- const photoFilename = photoStem + ".jpg"
45
- const photoFilePath = path.join(photosDir, photoFilename)
46
- const photoTempFilename = photoStem + "_temp.jpg"
47
- const photoTempFilePath = path.join(photosDir, photoTempFilename)
48
- const urlPhoto = `https://www.senat.fr/senimg/${photoFilename}`
49
- if (!options.silent) {
50
- console.log(
51
- `Loading photo ${urlPhoto} for ${sen.senprenomuse} ${sen.sennomuse}…`,
52
- )
53
- }
54
- // Fetch fails with OpenSSL error: dh key too small.
55
- // (so does "curl").
56
- // for (let retries = 0; retries < 3; retries++) {
57
- // const response = await fetch(urlPhoto)
58
- // if (response.ok) {
59
- // await pipeline(response.body, fs.createWriteStream(photoTempFilePath))
60
- // fs.renameSync(photoTempFilePath, photoFilePath)
61
- // break
62
- // }
63
- // if (retries >= 2) {
64
- // console.warn(`Fetch failed: ${urlPhoto} (${sen.senprenomuse} ${sen.sennomuse})`)
65
- // console.warn(response.status, response.statusText)
66
- // console.warn(await response.text())
67
- // if (fs.existsSync(photoFilePath)) {
68
- // console.warn(" => Reusing existing image")
69
- // } else {
70
- // console.warn(" => Using blank image")
71
- // fs.copyFileSync(missingPhotoFilePath, photoFilePath)
72
- // }
73
- // break
74
- // }
75
- // }
76
- try {
77
- execSync(`wget --quiet -O ${photoTempFilename} ${urlPhoto}`, {
78
- cwd: photosDir,
79
- env: process.env,
80
- encoding: "utf-8",
81
- // stdio: ["ignore", "ignore", "pipe"],
82
- })
83
- fs.renameSync(photoTempFilePath, photoFilePath)
84
- } catch (error) {
85
- if (typeof error === "object" && error && "status" in error && error.status === 8) {
86
- console.error(`Unable to load photo for ${sen.senprenomuse} ${sen.sennomuse}`)
87
- continue
88
- }
89
- throw error
90
- }
91
- }
92
- }
93
-
94
- // Resize photos to 155x225, because some haven't exactly this size.
95
- for (const sen of sens) {
96
- const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(
97
- sen.senprenomuse,
98
- "_",
99
- )}${slugify(sen.senmat, "_")}`
100
- const photoFilename = photoStem + ".jpg"
101
- const photoFilePath = path.join(photosDir, photoFilename)
102
- if (fs.existsSync(photoFilePath)) {
103
- if (!options.silent) {
104
- console.log(
105
- `Resizing photo ${photoStem} for ${sen.senprenomuse} ${sen.sennomuse}…`,
106
- )
107
- }
108
-
109
- execSync(
110
- `gm convert -resize 155x225! ${photoStem}.jpg ${photoStem}_155x225.jpg`,
111
- {
112
- cwd: photosDir,
113
- },
114
- )
115
- } else {
116
- if (!options.silent) {
117
- console.warn(`Missing photo for ${sen.senprenomuse} ${sen.sennomuse}: using blank image`)
118
- }
119
- fs.copyFileSync(missingPhotoFilePath, path.join(photosDir, `${photoStem}_155x225.jpg`))
120
- }
121
- }
122
-
123
- // Create a mosaic of photos.
124
- if (!options.silent) {
125
- console.log("Creating mosaic of photos…")
126
- }
127
- const photoBySenmat: { [senmat: string]: Photo } = {}
128
- const rowsFilenames: string[] = []
129
- for (
130
- let senIndex = 0, rowIndex = 0;
131
- senIndex < sens.length;
132
- senIndex += 25, rowIndex++
133
- ) {
134
- const row = sens.slice(senIndex, senIndex + 25)
135
- const photosFilenames: string[] = []
136
- for (const [columnIndex, sen] of row.entries()) {
137
- const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(
138
- sen.senprenomuse,
139
- "_",
140
- )}${slugify(sen.senmat, "_")}`
141
- const photoFilename = `${photoStem}_155x225.jpg`
142
- photosFilenames.push(photoFilename)
143
- photoBySenmat[sen.senmat] = {
144
- chemin: `photos_senateurs/${photoFilename}`,
145
- cheminMosaique: "photos_senateurs/senateurs.jpg",
146
- hauteur: 225,
147
- largeur: 155,
148
- xMosaique: columnIndex * 155,
149
- yMosaique: rowIndex * 225,
150
- }
151
- }
152
- const rowFilename = `row-${rowIndex}.jpg`
153
- execSync(`gm convert ${photosFilenames.join(" ")} +append ${rowFilename}`, {
154
- cwd: photosDir,
155
- })
156
- rowsFilenames.push(rowFilename)
157
- }
158
- execSync(`gm convert ${rowsFilenames.join(" ")} -append senateurs.jpg`, {
159
- cwd: photosDir,
160
- })
161
- for (const rowFilename of rowsFilenames) {
162
- fs.unlinkSync(path.join(photosDir, rowFilename))
163
- }
164
-
165
- if (!options.silent) {
166
- console.log("Creating JSON file containing informations on all pictures…")
167
- }
168
- const jsonFilePath = path.join(photosDir, "senateurs.json")
169
- fs.writeFileSync(jsonFilePath, JSON.stringify(photoBySenmat, null, 2))
170
- }
171
-
172
- retrievePhotosSenateurs()
173
- .then(() => process.exit(0))
174
- .catch((error) => {
175
- console.log(error)
176
- process.exit(1)
177
- })
@@ -1 +0,0 @@
1
- export {};
@@ -1,461 +0,0 @@
1
- // scripts/retrieve_senat_videos_from_agendas.ts
2
- import assert from "assert";
3
- import commandLineArgs from "command-line-args";
4
- import fs from "fs-extra";
5
- import fsp from "fs/promises";
6
- import path from "path";
7
- import * as cheerio from "cheerio";
8
- import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatAgendas } from "../loaders";
9
- import { getSessionsFromStart } from "../types/sessions";
10
- import { commonOptions } from "./shared/cli_helpers";
11
- import { getAgendaSegmentTimecodes } from "../utils/nvs-timecode";
12
- import { decodeHtmlEntities } from "../utils/string_cleaning";
13
- import { dice, normalize, scoreVideo } from "../utils/scoring";
14
- import { epochToParisDateTime, toFRDate, toTargetEpoch } from "../utils/date";
15
- // ===================== Constants =====================
16
- const MATCH_THRESHOLD = 0.5;
17
- const MAX_CANDIDATES = 15;
18
- const STATS = { total: 0, accepted: 0 };
19
- const VIDEOS_ROOT_FOLDER = "videos";
20
- const SENAT_VIDEOS_SEARCH_AJAX = "https://videos.senat.fr/senat_videos_search.php";
21
- const SENAT_DATAS_ROOT = "https://videos.senat.fr/Datas/senat";
22
- // ===================== CLI =====================
23
- const optionsDefinitions = [...commonOptions];
24
- const options = commandLineArgs(optionsDefinitions);
25
- // ===================== Utils =====================
26
- async function fetchText(url) {
27
- const res = await fetch(url);
28
- if (!res.ok)
29
- return null;
30
- return await res.text();
31
- }
32
- async function fetchBuffer(url) {
33
- const res = await fetch(url);
34
- if (!res.ok)
35
- return null;
36
- const ab = await res.arrayBuffer();
37
- return Buffer.from(ab);
38
- }
39
- async function writeIfChanged(p, content) {
40
- const exists = await fs.pathExists(p);
41
- if (exists) {
42
- const old = await fsp.readFile(p, "utf-8");
43
- if (old === content)
44
- return;
45
- }
46
- await fsp.writeFile(p, content, "utf-8");
47
- }
48
- function queryString(obj) {
49
- return Object.entries(obj)
50
- .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
51
- .join("&");
52
- }
53
- function extractCandidatesFromSearchHtml(html) {
54
- const $ = cheerio.load(html);
55
- const out = [];
56
- const re = /video\.(\d+)_([a-z0-9]+)/i;
57
- $('h3.card-title a.stretched-link[href*="video."]').each((_, a) => {
58
- const href = $(a).attr("href") || "";
59
- const m = href.match(re);
60
- if (!m)
61
- return;
62
- const id = m[1];
63
- const hash = m[2];
64
- const pageUrl = `https://videos.senat.fr/video.${id}_${hash}.html`;
65
- const title = ($(a).attr("title") || $(a).text() || "").replace(/\s+/g, " ").trim() || undefined;
66
- const isSeancePublique = title?.toLowerCase().includes("séance publique") ?? false;
67
- out.push({ id, hash, pageUrl, title, isSeancePublique });
68
- });
69
- // dedupe
70
- const seen = new Set();
71
- return out.filter((c) => {
72
- const k = `${c.id}_${c.hash}`;
73
- if (seen.has(k))
74
- return false;
75
- seen.add(k);
76
- return true;
77
- });
78
- }
79
- function parseDataNvs(nvs) {
80
- const epochStr = nvs.match(/<metadata\s+name="date"\s+value="(\d+)"/i)?.[1];
81
- const epoch = epochStr ? Number(epochStr) : undefined;
82
- // There can be multiple organes for one video in meta
83
- const organes = [];
84
- const organesRegex = /<metadata\b[^>]*\bname="organes"[^>]*>/gi;
85
- let m;
86
- while ((m = organesRegex.exec(nvs)) !== null) {
87
- const tag = m[0];
88
- const label = tag.match(/\blabel="([^"]+)"/i)?.[1];
89
- if (label) {
90
- const decoded = decodeHtmlEntities(label).trim();
91
- if (decoded)
92
- organes.push(decoded);
93
- }
94
- }
95
- if (organes.length === 0) {
96
- organes.push("Séance publique");
97
- }
98
- const firstChapterLabelMatch = nvs.match(/<chapter\b[^>]*\blabel="([^"]+)"/i);
99
- const firstChapterLabel = firstChapterLabelMatch ? decodeHtmlEntities(firstChapterLabelMatch[1]).trim() : undefined;
100
- return { epoch, organes, firstChapterLabel };
101
- }
102
- function buildSenatVodMasterM3u8FromNvs(nvsText) {
103
- // serverfiles://senat/2025/10/encoder10_20251022084451_2.mp4
104
- const m = nvsText.match(/serverfiles:\/\/senat\/(\d{4})\/(\d{2})\/(encoder\d+)_([0-9]{14})/i);
105
- if (!m)
106
- return null;
107
- const [, yyyy, mm, encoder, stamp] = m;
108
- const base = `https://vodsenat.akamaized.net/senat/${yyyy}/${mm}/${encoder}_${stamp}`;
109
- return `${base}.smil/master.m3u8`;
110
- }
111
- function isAmbiguousTimeOriginal(timeOriginal) {
112
- if (!timeOriginal)
113
- return false;
114
- const s = timeOriginal.toLowerCase();
115
- // Catches "14h", "14 h", "14h30", "14 h 30", "14 heures", "14 heure"
116
- const timeRe = /\b([01]?\d|2[0-3])\s*(?:h|heures?|heure)\s*(?:([0-5]\d))?\b/g;
117
- const times = new Set();
118
- let m;
119
- while ((m = timeRe.exec(s))) {
120
- const hh = String(m[1]).padStart(2, "0");
121
- const mm = m[2] ? String(m[2]).padStart(2, "0") : "00";
122
- times.add(`${hh}:${mm}`);
123
- }
124
- // "midi" / "minuit"
125
- if (/\bmidi\b/.test(s))
126
- times.add("12:00");
127
- if (/\bminuit\b/.test(s))
128
- times.add("00:00");
129
- if (times.size >= 2)
130
- return true;
131
- const hasDayPeriod = /\b(matin|après-?midi|soir|nuit|journée|toute la journée)\b/.test(s);
132
- const hasLinking = /,|\bet\b|\bou\b|\bpuis\b/.test(s);
133
- if (times.size === 1 && hasDayPeriod && hasLinking)
134
- return true;
135
- return false;
136
- }
137
- function getAgendaType(agenda) {
138
- const o = agenda.organe || "";
139
- if (/séance publique/i.test(o))
140
- return "Séance publique";
141
- return "Commission";
142
- }
143
- async function fetchAllSearchPages(args, maxPages = 3) {
144
- const pages = [];
145
- for (let p = 1; p <= maxPages; p++) {
146
- const url = `${SENAT_VIDEOS_SEARCH_AJAX}?${queryString({ ...args, page: String(p) })}`;
147
- const html = await fetchText(url);
148
- if (!html)
149
- break;
150
- pages.push(html);
151
- if (!/href="\/?video\.\d+_[a-z0-9]+\./i.test(html))
152
- break;
153
- }
154
- return pages;
155
- }
156
- function getOrgKey(norm) {
157
- if (!norm)
158
- return "autre";
159
- if (norm.includes("seance publique"))
160
- return "seance_publique";
161
- if (norm.includes("culture"))
162
- return "culture";
163
- if (norm.includes("finances"))
164
- return "finances";
165
- if (norm.includes("sociales"))
166
- return "affaires_sociales";
167
- if (norm.includes("economiques"))
168
- return "affaires_economiques";
169
- if (norm.includes("europeennes"))
170
- return "affaires_europeennes";
171
- if (norm.includes("etrangeres") || norm.includes("forces armees") || norm.includes("defense")) {
172
- return "affaires_etrangeres_defense";
173
- }
174
- if (norm.includes("territoire") || norm.includes("durable")) {
175
- return "amenagement_territoire_dd";
176
- }
177
- if (norm.includes("commission des lois"))
178
- return "lois";
179
- if (norm.includes("delegation aux collectivites territoriales") || norm.includes("delegation a la decentralisation"))
180
- return "delegation_collectivites";
181
- if (norm.includes("delegation aux droits des femmes") ||
182
- norm.includes("egalite des chances entre les hommes et les femmes"))
183
- return "delegation_droits_femmes";
184
- if (norm.includes("delegation aux entreprises"))
185
- return "delegation_entreprises";
186
- if (norm.includes("delegation senatoriale aux outre mer") || norm.includes("delegation aux outre mer"))
187
- return "delegation_outre_mer";
188
- if (norm.includes("delegation a la prospective"))
189
- return "delegation_prospective";
190
- if (norm.includes("office parlementaire d evaluation des choix scientifiques et technologiques") ||
191
- norm.includes("opecst"))
192
- return "opecst";
193
- return "autre";
194
- }
195
- async function processGroupedReunion(agenda, session, dataDir) {
196
- // 1) GuardRails
197
- if (!agenda.captationVideo) {
198
- // if (!options["silent"]) console.log(`[skip] ${agenda.uid} captationVideo=false`)
199
- return;
200
- }
201
- if (!agenda.date || !agenda.startTime) {
202
- // if (!options["silent"]) console.log(`[skip] ${agenda.uid} date/hour missing`)
203
- return;
204
- }
205
- const agendaTs = toTargetEpoch(agenda.startTime, agenda.date);
206
- const now = Date.now();
207
- if (agendaTs && agendaTs * 1000 > now) {
208
- return;
209
- }
210
- const reunionUid = agenda.uid;
211
- const baseDir = path.join(dataDir, VIDEOS_ROOT_FOLDER, String(session), reunionUid);
212
- await fs.ensureDir(baseDir);
213
- let skipDownload = false;
214
- if (options["only-recent"]) {
215
- const now = Date.now();
216
- const cutoff = now - options["only-recent"] * 24 * 3600 * 1000;
217
- const reunionTs = Date.parse(agenda.date);
218
- if (reunionTs < cutoff) {
219
- // Check if files already exist
220
- const dataNvsPath = path.join(baseDir, "data.nvs");
221
- const finalplayerNvsPath = path.join(baseDir, "finalplayer.nvs");
222
- if (fs.existsSync(dataNvsPath) && fs.existsSync(finalplayerNvsPath)) {
223
- skipDownload = true;
224
- }
225
- }
226
- }
227
- let master = null;
228
- let dataTxt = null;
229
- let finalTxt = null;
230
- let accepted = false;
231
- if (!skipDownload) {
232
- STATS.total++;
233
- const searchParams = {
234
- search: "true",
235
- videotype: getAgendaType(agenda),
236
- };
237
- if (agenda.date) {
238
- const fr = toFRDate(agenda.date);
239
- searchParams.period = "custom";
240
- searchParams.begin = fr;
241
- searchParams.end = fr;
242
- }
243
- if (agenda.organe) {
244
- searchParams.organe = agenda.organe;
245
- }
246
- const pages = await fetchAllSearchPages(searchParams);
247
- if (!pages.length) {
248
- if (!options["silent"]) {
249
- console.log(`[miss] ${agenda.uid} no candidates (videotype=${searchParams.videotype}, organe=${searchParams.organe || "-"}, date=${searchParams.begin || "-"})`);
250
- }
251
- return;
252
- }
253
- const combinedHtml = pages.join("\n<!-- PAGE SPLIT -->\n");
254
- const candidates = extractCandidatesFromSearchHtml(combinedHtml).slice(0, MAX_CANDIDATES);
255
- if (!candidates.length) {
256
- if (!options["silent"]) {
257
- console.log(`[miss] ${agenda.uid} no candidates after parse (videotype=${searchParams.videotype}, organe=${searchParams.organe || "-"}, date=${searchParams.begin || "-"})`);
258
- }
259
- return;
260
- }
261
- // ==== 2) Enrich via data.nvs + scoring; pick best ====
262
- let best = null;
263
- const timeAmbigious = isAmbiguousTimeOriginal(agenda.events[0].timeOriginal);
264
- if (timeAmbigious) {
265
- console.log(`[match] ${agenda.uid} timeOriginal ambiguous => ignoring time scoring: "${agenda.events[0].timeOriginal}"`);
266
- }
267
- for (const c of candidates) {
268
- const dataUrl = `${SENAT_DATAS_ROOT}/${c.id}_${c.hash}/content/data.nvs`;
269
- const finalUrl = `${SENAT_DATAS_ROOT}/${c.id}_${c.hash}/content/finalplayer.nvs`;
270
- const dataBuf = await fetchBuffer(dataUrl);
271
- if (!dataBuf)
272
- continue;
273
- const meta = parseDataNvs(dataBuf.toString("utf-8"));
274
- let sameOrg = false;
275
- // If organes are too different, go to next candidates
276
- if (agenda.organe && meta.organes?.length) {
277
- const agendaOrgNorm = normalize(agenda.organe);
278
- const agendaKey = getOrgKey(agendaOrgNorm);
279
- let bestDice = 0;
280
- let hasSameKey = false;
281
- for (const vo of meta.organes) {
282
- const videoOrgNorm = normalize(vo);
283
- const videoKey = getOrgKey(videoOrgNorm);
284
- const d = dice(agendaOrgNorm, videoOrgNorm);
285
- if (videoKey === agendaKey && videoKey !== "autre") {
286
- hasSameKey = true;
287
- }
288
- if (d > bestDice)
289
- bestDice = d;
290
- }
291
- if (hasSameKey) {
292
- sameOrg = true; // we are sure this is the same org
293
- }
294
- else if (bestDice < 0.8) {
295
- // if diff org and dice too low we skip
296
- continue;
297
- }
298
- }
299
- let videoTitle = c.title;
300
- if (c.isSeancePublique && meta.firstChapterLabel) {
301
- videoTitle = meta.firstChapterLabel;
302
- }
303
- const s = scoreVideo(agenda, agendaTs, sameOrg, videoTitle, meta.epoch, meta.organes, timeAmbigious);
304
- if (!best || s > best.score) {
305
- best = {
306
- id: c.id,
307
- hash: c.hash,
308
- pageUrl: c.pageUrl,
309
- epoch: meta.epoch,
310
- vtitle: videoTitle,
311
- score: s,
312
- vorgane: meta.organes[0],
313
- };
314
- }
315
- }
316
- if (!best) {
317
- if (!options["silent"])
318
- console.log(`[miss] ${agenda.uid} No candidate found for this reunion`);
319
- return;
320
- }
321
- accepted = best.score >= MATCH_THRESHOLD;
322
- if (accepted)
323
- STATS.accepted++;
324
- if (!options["silent"]) {
325
- console.log(`[pick] ${agenda.uid} score=${best.score.toFixed(2)}
326
- agenda title="${agenda.titre ?? ""}" agenda organe="${agenda.organe ?? ""}" agenda heure=${agenda.startTime}
327
- best title="${best.vtitle ?? ""}" best organe="${best.vorgane ?? ""}"
328
- accepted=${accepted}`);
329
- }
330
- // ==== 3) Write metadata + NVS of the best candidate (always) ====
331
- const bestDt = best?.epoch ? epochToParisDateTime(best.epoch) : null;
332
- const metadata = {
333
- reunionUid,
334
- session,
335
- accepted,
336
- threshold: MATCH_THRESHOLD,
337
- agenda: {
338
- date: agenda.date,
339
- startTime: agenda.startTime,
340
- titre: agenda.titre,
341
- organe: agenda.organe ?? undefined,
342
- uid: agenda.uid,
343
- },
344
- best: {
345
- id: best.id,
346
- hash: best.hash,
347
- pageUrl: best.pageUrl,
348
- epoch: best.epoch ?? null,
349
- date: bestDt?.date ?? null,
350
- startTime: bestDt?.startTime ?? null,
351
- title: best.vtitle ?? null,
352
- score: best.score,
353
- },
354
- };
355
- await writeIfChanged(path.join(baseDir, "metadata.json"), JSON.stringify(metadata, null, 2));
356
- const dataUrl = `${SENAT_DATAS_ROOT}/${best.id}_${best.hash}/content/data.nvs`;
357
- const finalUrl = `${SENAT_DATAS_ROOT}/${best.id}_${best.hash}/content/finalplayer.nvs`;
358
- dataTxt = await fetchText(dataUrl);
359
- finalTxt = await fetchText(finalUrl);
360
- if (dataTxt)
361
- await fsp.writeFile(path.join(baseDir, "data.nvs"), dataTxt, "utf-8");
362
- if (finalTxt)
363
- await fsp.writeFile(path.join(baseDir, "finalplayer.nvs"), finalTxt, "utf-8");
364
- if (dataTxt) {
365
- master = buildSenatVodMasterM3u8FromNvs(dataTxt);
366
- }
367
- else {
368
- console.log("Cannot download data nvs");
369
- }
370
- }
371
- else {
372
- // Skipped download, but need to read data.nvs for urlVideo
373
- try {
374
- dataTxt = await fsp.readFile(path.join(baseDir, "data.nvs"), "utf-8");
375
- finalTxt = await fsp.readFile(path.join(baseDir, "finalplayer.nvs"), "utf-8");
376
- master = buildSenatVodMasterM3u8FromNvs(dataTxt);
377
- }
378
- catch (e) {
379
- console.warn(e);
380
- }
381
- }
382
- // ==== 4) Update agenda file (only if accepted + m3u8) ====
383
- if ((accepted || skipDownload) && master) {
384
- const agendaJsonPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session), `${agenda.uid}.json`);
385
- let timecodeDebutVideo = null;
386
- let timecodeFinVideo = null;
387
- if (dataTxt && finalTxt) {
388
- const agendaKey = agenda.titre || agenda.objet || "";
389
- const seg = getAgendaSegmentTimecodes(dataTxt, finalTxt, agendaKey);
390
- if (!seg) {
391
- console.warn(`[warn] Cannot retrieve agenda segment timecodes from reunion ${reunionUid}`);
392
- }
393
- else {
394
- timecodeDebutVideo = seg.start;
395
- timecodeFinVideo = seg.end;
396
- }
397
- }
398
- if (await fs.pathExists(agendaJsonPath)) {
399
- const raw = await fsp.readFile(agendaJsonPath, "utf-8");
400
- let obj;
401
- try {
402
- obj = JSON.parse(raw);
403
- }
404
- catch (e) {
405
- console.warn(`[warn] invalid JSON in ${agendaJsonPath}:`, e?.message);
406
- obj = null;
407
- }
408
- if (obj && typeof obj === "object" && !Array.isArray(obj)) {
409
- const next = { ...obj, urlVideo: master };
410
- if (timecodeDebutVideo != null) {
411
- next.timecodeDebutVideo = timecodeDebutVideo;
412
- next.timecodeFinVideo = timecodeFinVideo;
413
- }
414
- await writeIfChanged(agendaJsonPath, JSON.stringify(next, null, 2));
415
- if (!options["silent"]) {
416
- console.log(`[write] ${agenda.uid} urlVideo ← ${master}` +
417
- (timecodeDebutVideo != null ? ` (timecodeDebutVideo ← ${timecodeDebutVideo}s)` : ""));
418
- }
419
- }
420
- else {
421
- console.warn(`[warn] expected an object in ${agendaJsonPath}, got ${Array.isArray(obj) ? "array" : typeof obj}`);
422
- }
423
- }
424
- else {
425
- console.warn(`[warn] agenda file not found for update: ${agendaJsonPath}`);
426
- }
427
- }
428
- else {
429
- console.warn(`[warn] The video url could not be built for reunion `, reunionUid);
430
- }
431
- }
432
- async function processAll(dataDir, sessions) {
433
- console.log("Process all Agendas and fetch video's url");
434
- for (const session of sessions) {
435
- for (const { item: agenda } of iterLoadSenatAgendas(dataDir, session)) {
436
- try {
437
- await processGroupedReunion(agenda, session, dataDir);
438
- }
439
- catch (e) {
440
- console.error(`[error] ${agenda?.uid ?? "unknown-uid"}:`, e?.message || e);
441
- }
442
- }
443
- }
444
- }
445
- async function main() {
446
- const dataDir = options["dataDir"];
447
- assert(dataDir, "Missing argument: data directory");
448
- const sessions = getSessionsFromStart(options["fromSession"]);
449
- console.time("senat-agendas→videos start processing time");
450
- await processAll(dataDir, sessions);
451
- console.timeEnd("senat-agendas→videos processing time");
452
- const { total, accepted } = STATS;
453
- const ratio = total ? ((accepted / total) * 100).toFixed(1) : "0.0";
454
- console.log(`[summary] accepted=${accepted} / total=${total} (${ratio}%)`);
455
- }
456
- main()
457
- .then(() => process.exit(0))
458
- .catch((err) => {
459
- console.error(err);
460
- process.exit(1);
461
- });