@tricoteuses/senat 3.1.0 → 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/src/loaders.d.ts +3 -3
- package/lib/src/loaders.js +1 -1
- package/lib/src/model/agenda.d.ts +1 -1
- package/lib/src/model/commission.d.ts +2 -2
- package/lib/src/model/seance.d.ts +1 -1
- package/lib/src/types/ameli.d.ts +4 -1761
- package/lib/src/types/ameli.js +1 -1074
- package/lib/src/types/debats.d.ts +2 -380
- package/lib/src/types/debats.js +1 -266
- package/lib/src/types/dosleg.d.ts +69 -2953
- package/lib/src/types/dosleg.js +1 -2005
- package/lib/src/types/questions.d.ts +2 -699
- package/lib/src/types/questions.js +1 -493
- package/lib/src/types/sens.d.ts +7 -7842
- package/lib/src/types/sens.js +1 -4691
- package/lib/src/utils/nvs-parsing.d.ts +1 -1
- package/lib/src/utils/nvs-parsing.js +9 -1
- package/lib/src/videos/pipeline.d.ts +3 -3
- package/lib/src/videos/pipeline.js +2 -2
- package/package.json +1 -1
- package/lib/add-js-extensions-v2.d.ts +0 -1
- package/lib/add-js-extensions-v2.js +0 -23
- package/lib/add-js-extensions.d.ts +0 -1
- package/lib/add-js-extensions.js +0 -17
- package/lib/aggregates.d.ts +0 -52
- package/lib/aggregates.js +0 -930
- package/lib/aggregates.mjs +0 -713
- package/lib/aggregates.ts +0 -833
- package/lib/config.d.ts +0 -10
- package/lib/config.js +0 -16
- package/lib/config.mjs +0 -16
- package/lib/config.ts +0 -26
- package/lib/databases.d.ts +0 -2
- package/lib/databases.js +0 -26
- package/lib/databases.mjs +0 -57
- package/lib/databases.ts +0 -71
- package/lib/datasets.d.ts +0 -34
- package/lib/datasets.js +0 -233
- package/lib/datasets.mjs +0 -78
- package/lib/datasets.ts +0 -118
- package/lib/fields.d.ts +0 -10
- package/lib/fields.js +0 -68
- package/lib/fields.mjs +0 -22
- package/lib/fields.ts +0 -29
- package/lib/git.d.ts +0 -26
- package/lib/git.js +0 -167
- package/lib/index.d.ts +0 -13
- package/lib/index.js +0 -1
- package/lib/index.mjs +0 -7
- package/lib/index.ts +0 -64
- package/lib/inserters.d.ts +0 -98
- package/lib/inserters.js +0 -500
- package/lib/inserters.mjs +0 -360
- package/lib/inserters.ts +0 -521
- package/lib/legislatures.json +0 -38
- package/lib/loaders.d.ts +0 -58
- package/lib/loaders.js +0 -286
- package/lib/loaders.mjs +0 -158
- package/lib/loaders.ts +0 -271
- package/lib/model/agenda.d.ts +0 -6
- package/lib/model/agenda.js +0 -148
- package/lib/model/ameli.d.ts +0 -51
- package/lib/model/ameli.js +0 -149
- package/lib/model/ameli.mjs +0 -84
- package/lib/model/ameli.ts +0 -100
- package/lib/model/commission.d.ts +0 -18
- package/lib/model/commission.js +0 -269
- package/lib/model/debats.d.ts +0 -67
- package/lib/model/debats.js +0 -95
- package/lib/model/debats.mjs +0 -43
- package/lib/model/debats.ts +0 -68
- package/lib/model/documents.d.ts +0 -12
- package/lib/model/documents.js +0 -151
- package/lib/model/dosleg.d.ts +0 -7
- package/lib/model/dosleg.js +0 -326
- package/lib/model/dosleg.mjs +0 -196
- package/lib/model/dosleg.ts +0 -240
- package/lib/model/index.d.ts +0 -7
- package/lib/model/index.js +0 -7
- package/lib/model/index.mjs +0 -5
- package/lib/model/index.ts +0 -15
- package/lib/model/questions.d.ts +0 -45
- package/lib/model/questions.js +0 -89
- package/lib/model/questions.mjs +0 -71
- package/lib/model/questions.ts +0 -93
- package/lib/model/scrutins.d.ts +0 -13
- package/lib/model/scrutins.js +0 -114
- package/lib/model/seance.d.ts +0 -3
- package/lib/model/seance.js +0 -267
- package/lib/model/sens.d.ts +0 -146
- package/lib/model/sens.js +0 -454
- package/lib/model/sens.mjs +0 -415
- package/lib/model/sens.ts +0 -516
- package/lib/model/texte.d.ts +0 -7
- package/lib/model/texte.js +0 -256
- package/lib/model/texte.mjs +0 -208
- package/lib/model/texte.ts +0 -229
- package/lib/model/util.d.ts +0 -9
- package/lib/model/util.js +0 -38
- package/lib/model/util.mjs +0 -19
- package/lib/model/util.ts +0 -32
- package/lib/parsers/texte.d.ts +0 -7
- package/lib/parsers/texte.js +0 -228
- package/lib/raw_types/ameli.d.ts +0 -914
- package/lib/raw_types/ameli.js +0 -5
- package/lib/raw_types/ameli.mjs +0 -163
- package/lib/raw_types/debats.d.ts +0 -207
- package/lib/raw_types/debats.js +0 -5
- package/lib/raw_types/debats.mjs +0 -58
- package/lib/raw_types/dosleg.d.ts +0 -1619
- package/lib/raw_types/dosleg.js +0 -5
- package/lib/raw_types/dosleg.mjs +0 -438
- package/lib/raw_types/questions.d.ts +0 -419
- package/lib/raw_types/questions.js +0 -5
- package/lib/raw_types/questions.mjs +0 -11
- package/lib/raw_types/senat.d.ts +0 -11368
- package/lib/raw_types/senat.js +0 -5
- package/lib/raw_types/sens.d.ts +0 -8248
- package/lib/raw_types/sens.js +0 -5
- package/lib/raw_types/sens.mjs +0 -508
- package/lib/raw_types_kysely/ameli.d.ts +0 -915
- package/lib/raw_types_kysely/ameli.js +0 -7
- package/lib/raw_types_kysely/ameli.mjs +0 -5
- package/lib/raw_types_kysely/ameli.ts +0 -951
- package/lib/raw_types_kysely/debats.d.ts +0 -207
- package/lib/raw_types_kysely/debats.js +0 -7
- package/lib/raw_types_kysely/debats.mjs +0 -5
- package/lib/raw_types_kysely/debats.ts +0 -222
- package/lib/raw_types_kysely/dosleg.d.ts +0 -3532
- package/lib/raw_types_kysely/dosleg.js +0 -7
- package/lib/raw_types_kysely/dosleg.mjs +0 -5
- package/lib/raw_types_kysely/dosleg.ts +0 -3621
- package/lib/raw_types_kysely/questions.d.ts +0 -414
- package/lib/raw_types_kysely/questions.js +0 -7
- package/lib/raw_types_kysely/questions.mjs +0 -5
- package/lib/raw_types_kysely/questions.ts +0 -426
- package/lib/raw_types_kysely/sens.d.ts +0 -4394
- package/lib/raw_types_kysely/sens.js +0 -7
- package/lib/raw_types_kysely/sens.mjs +0 -5
- package/lib/raw_types_kysely/sens.ts +0 -4499
- package/lib/raw_types_schemats/ameli.d.ts +0 -539
- package/lib/raw_types_schemats/ameli.js +0 -2
- package/lib/raw_types_schemats/ameli.mjs +0 -2
- package/lib/raw_types_schemats/ameli.ts +0 -601
- package/lib/raw_types_schemats/debats.d.ts +0 -127
- package/lib/raw_types_schemats/debats.js +0 -2
- package/lib/raw_types_schemats/debats.mjs +0 -2
- package/lib/raw_types_schemats/debats.ts +0 -145
- package/lib/raw_types_schemats/dosleg.d.ts +0 -977
- package/lib/raw_types_schemats/dosleg.js +0 -2
- package/lib/raw_types_schemats/dosleg.mjs +0 -2
- package/lib/raw_types_schemats/dosleg.ts +0 -2193
- package/lib/raw_types_schemats/questions.d.ts +0 -235
- package/lib/raw_types_schemats/questions.js +0 -2
- package/lib/raw_types_schemats/questions.mjs +0 -2
- package/lib/raw_types_schemats/questions.ts +0 -249
- package/lib/raw_types_schemats/sens.d.ts +0 -6915
- package/lib/raw_types_schemats/sens.js +0 -2
- package/lib/raw_types_schemats/sens.mjs +0 -2
- package/lib/raw_types_schemats/sens.ts +0 -2907
- package/lib/scripts/convert_data.d.ts +0 -1
- package/lib/scripts/convert_data.js +0 -354
- package/lib/scripts/convert_data.mjs +0 -181
- package/lib/scripts/convert_data.ts +0 -243
- package/lib/scripts/data-download.d.ts +0 -1
- package/lib/scripts/data-download.js +0 -12
- package/lib/scripts/datautil.d.ts +0 -8
- package/lib/scripts/datautil.js +0 -34
- package/lib/scripts/datautil.mjs +0 -16
- package/lib/scripts/datautil.ts +0 -19
- package/lib/scripts/images/transparent_150x192.jpg +0 -0
- package/lib/scripts/images/transparent_155x225.jpg +0 -0
- package/lib/scripts/parse_textes.d.ts +0 -1
- package/lib/scripts/parse_textes.js +0 -44
- package/lib/scripts/parse_textes.mjs +0 -46
- package/lib/scripts/parse_textes.ts +0 -65
- package/lib/scripts/retrieve_agenda.d.ts +0 -1
- package/lib/scripts/retrieve_agenda.js +0 -132
- package/lib/scripts/retrieve_cr_commission.d.ts +0 -1
- package/lib/scripts/retrieve_cr_commission.js +0 -364
- package/lib/scripts/retrieve_cr_seance.d.ts +0 -6
- package/lib/scripts/retrieve_cr_seance.js +0 -347
- package/lib/scripts/retrieve_documents.d.ts +0 -3
- package/lib/scripts/retrieve_documents.js +0 -219
- package/lib/scripts/retrieve_documents.mjs +0 -249
- package/lib/scripts/retrieve_documents.ts +0 -298
- package/lib/scripts/retrieve_open_data.d.ts +0 -1
- package/lib/scripts/retrieve_open_data.js +0 -315
- package/lib/scripts/retrieve_open_data.mjs +0 -217
- package/lib/scripts/retrieve_open_data.ts +0 -268
- package/lib/scripts/retrieve_senateurs_photos.d.ts +0 -1
- package/lib/scripts/retrieve_senateurs_photos.js +0 -147
- package/lib/scripts/retrieve_senateurs_photos.mjs +0 -147
- package/lib/scripts/retrieve_senateurs_photos.ts +0 -177
- package/lib/scripts/retrieve_videos.d.ts +0 -1
- package/lib/scripts/retrieve_videos.js +0 -461
- package/lib/scripts/shared/cli_helpers.d.ts +0 -95
- package/lib/scripts/shared/cli_helpers.js +0 -91
- package/lib/scripts/shared/cli_helpers.ts +0 -36
- package/lib/scripts/shared/util.d.ts +0 -4
- package/lib/scripts/shared/util.js +0 -35
- package/lib/scripts/shared/util.ts +0 -33
- package/lib/scripts/test_iter_load.d.ts +0 -1
- package/lib/scripts/test_iter_load.js +0 -12
- package/lib/src/ameli.d.ts +0 -66
- package/lib/src/ameli.js +0 -1
- package/lib/src/databases.d.ts +0 -3
- package/lib/src/databases.js +0 -26
- package/lib/src/db_types/ameli.d.ts +0 -1762
- package/lib/src/db_types/ameli.js +0 -1074
- package/lib/src/db_types/debats.d.ts +0 -380
- package/lib/src/db_types/debats.js +0 -266
- package/lib/src/db_types/dosleg.d.ts +0 -2954
- package/lib/src/db_types/dosleg.js +0 -2005
- package/lib/src/db_types/questions.d.ts +0 -699
- package/lib/src/db_types/questions.js +0 -493
- package/lib/src/db_types/sens.d.ts +0 -7843
- package/lib/src/db_types/sens.js +0 -4691
- package/lib/src/debats.d.ts +0 -38
- package/lib/src/debats.js +0 -1
- package/lib/src/dosleg.d.ts +0 -142
- package/lib/src/dosleg.js +0 -193
- package/lib/src/model/ameli_postgres.d.ts +0 -67
- package/lib/src/model/ameli_postgres.js +0 -150
- package/lib/src/other_types/questions.d.ts +0 -2
- package/lib/src/other_types/questions.js +0 -1
- package/lib/src/questions.d.ts +0 -53
- package/lib/src/questions.js +0 -1
- package/lib/src/raw_types/senat.d.ts +0 -11372
- package/lib/src/raw_types/senat.js +0 -5
- package/lib/src/rich_types/agenda.d.ts +0 -45
- package/lib/src/rich_types/agenda.js +0 -1
- package/lib/src/rich_types/compte_rendu.d.ts +0 -83
- package/lib/src/rich_types/compte_rendu.js +0 -1
- package/lib/src/rich_types/sessions.d.ts +0 -6
- package/lib/src/rich_types/sessions.js +0 -19
- package/lib/src/rich_types/texte.d.ts +0 -72
- package/lib/src/rich_types/texte.js +0 -15
- package/lib/src/scripts/test_iter_load.d.ts +0 -1
- package/lib/src/scripts/test_iter_load.js +0 -12
- package/lib/src/sens.d.ts +0 -104
- package/lib/src/sens.js +0 -1
- package/lib/strings.d.ts +0 -1
- package/lib/strings.js +0 -18
- package/lib/strings.mjs +0 -18
- package/lib/strings.ts +0 -26
- package/lib/tsconfig.tsbuildinfo +0 -1
- package/lib/types/agenda.d.ts +0 -44
- package/lib/types/agenda.js +0 -1
- package/lib/types/ameli.d.ts +0 -5
- package/lib/types/ameli.js +0 -1
- package/lib/types/ameli.mjs +0 -13
- package/lib/types/ameli.ts +0 -21
- package/lib/types/compte_rendu.d.ts +0 -83
- package/lib/types/compte_rendu.js +0 -1
- package/lib/types/debats.d.ts +0 -2
- package/lib/types/debats.js +0 -1
- package/lib/types/debats.mjs +0 -2
- package/lib/types/debats.ts +0 -6
- package/lib/types/dosleg.d.ts +0 -70
- package/lib/types/dosleg.js +0 -1
- package/lib/types/dosleg.mjs +0 -151
- package/lib/types/dosleg.ts +0 -284
- package/lib/types/questions.d.ts +0 -2
- package/lib/types/questions.js +0 -1
- package/lib/types/questions.mjs +0 -1
- package/lib/types/questions.ts +0 -3
- package/lib/types/sens.d.ts +0 -10
- package/lib/types/sens.js +0 -1
- package/lib/types/sens.mjs +0 -1
- package/lib/types/sens.ts +0 -12
- package/lib/types/sessions.d.ts +0 -5
- package/lib/types/sessions.js +0 -84
- package/lib/types/sessions.mjs +0 -43
- package/lib/types/sessions.ts +0 -42
- package/lib/types/texte.d.ts +0 -74
- package/lib/types/texte.js +0 -16
- package/lib/types/texte.mjs +0 -16
- package/lib/types/texte.ts +0 -76
- package/lib/typings/windows-1252.d.js +0 -2
- package/lib/typings/windows-1252.d.mjs +0 -2
- package/lib/typings/windows-1252.d.ts +0 -11
- package/lib/utils/cr_spliting.d.ts +0 -28
- package/lib/utils/cr_spliting.js +0 -265
- package/lib/utils/date.d.ts +0 -10
- package/lib/utils/date.js +0 -100
- package/lib/utils/nvs-timecode.d.ts +0 -7
- package/lib/utils/nvs-timecode.js +0 -79
- package/lib/utils/reunion_grouping.d.ts +0 -9
- package/lib/utils/reunion_grouping.js +0 -361
- package/lib/utils/reunion_odj_building.d.ts +0 -5
- package/lib/utils/reunion_odj_building.js +0 -154
- package/lib/utils/reunion_parsing.d.ts +0 -23
- package/lib/utils/reunion_parsing.js +0 -209
- package/lib/utils/scoring.d.ts +0 -14
- package/lib/utils/scoring.js +0 -147
- package/lib/utils/string_cleaning.d.ts +0 -7
- package/lib/utils/string_cleaning.js +0 -57
- package/lib/validators/config.d.ts +0 -9
- package/lib/validators/config.js +0 -10
- package/lib/validators/config.mjs +0 -54
- package/lib/validators/config.ts +0 -79
- package/lib/validators/senat.d.ts +0 -0
- package/lib/validators/senat.js +0 -28
- package/lib/validators/senat.mjs +0 -24
- package/lib/validators/senat.ts +0 -26
|
@@ -1,177 +0,0 @@
|
|
|
1
|
-
import assert from "assert"
|
|
2
|
-
import { execSync } from "child_process"
|
|
3
|
-
import commandLineArgs from "command-line-args"
|
|
4
|
-
import fs from "fs-extra"
|
|
5
|
-
// import fetch from "node-fetch"
|
|
6
|
-
import path from "path"
|
|
7
|
-
// import stream from "stream"
|
|
8
|
-
// import util from "util"
|
|
9
|
-
|
|
10
|
-
import { findActif as findActifSenateurs } from "../model/sens"
|
|
11
|
-
import { slugify } from "../strings"
|
|
12
|
-
import { Photo } from "../types/sens"
|
|
13
|
-
import { commonOptions } from "./shared/cli_helpers"
|
|
14
|
-
|
|
15
|
-
const optionsDefinitions = [
|
|
16
|
-
...commonOptions,
|
|
17
|
-
{
|
|
18
|
-
alias: "f",
|
|
19
|
-
help: "fetch sénateurs' pictures instead of retrieving them from files",
|
|
20
|
-
name: "fetch",
|
|
21
|
-
type: Boolean,
|
|
22
|
-
},
|
|
23
|
-
]
|
|
24
|
-
const options = commandLineArgs(optionsDefinitions)
|
|
25
|
-
// const pipeline = util.promisify(stream.pipeline)
|
|
26
|
-
|
|
27
|
-
async function retrievePhotosSenateurs(): Promise<void> {
|
|
28
|
-
const dataDir = options.dataDir
|
|
29
|
-
assert(dataDir, "Missing argument: data directory")
|
|
30
|
-
|
|
31
|
-
const photosDir = path.join(dataDir, "photos_senateurs")
|
|
32
|
-
const missingPhotoFilePath = path.resolve(__dirname, "images", "transparent_155x225.jpg")
|
|
33
|
-
|
|
34
|
-
const sens = await Array.fromAsync(findActifSenateurs())
|
|
35
|
-
|
|
36
|
-
// Download photos.
|
|
37
|
-
fs.ensureDirSync(photosDir)
|
|
38
|
-
if (options.fetch) {
|
|
39
|
-
for (const sen of sens) {
|
|
40
|
-
const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(
|
|
41
|
-
sen.senprenomuse,
|
|
42
|
-
"_",
|
|
43
|
-
)}${slugify(sen.senmat, "_")}`
|
|
44
|
-
const photoFilename = photoStem + ".jpg"
|
|
45
|
-
const photoFilePath = path.join(photosDir, photoFilename)
|
|
46
|
-
const photoTempFilename = photoStem + "_temp.jpg"
|
|
47
|
-
const photoTempFilePath = path.join(photosDir, photoTempFilename)
|
|
48
|
-
const urlPhoto = `https://www.senat.fr/senimg/${photoFilename}`
|
|
49
|
-
if (!options.silent) {
|
|
50
|
-
console.log(
|
|
51
|
-
`Loading photo ${urlPhoto} for ${sen.senprenomuse} ${sen.sennomuse}…`,
|
|
52
|
-
)
|
|
53
|
-
}
|
|
54
|
-
// Fetch fails with OpenSSL error: dh key too small.
|
|
55
|
-
// (so does "curl").
|
|
56
|
-
// for (let retries = 0; retries < 3; retries++) {
|
|
57
|
-
// const response = await fetch(urlPhoto)
|
|
58
|
-
// if (response.ok) {
|
|
59
|
-
// await pipeline(response.body, fs.createWriteStream(photoTempFilePath))
|
|
60
|
-
// fs.renameSync(photoTempFilePath, photoFilePath)
|
|
61
|
-
// break
|
|
62
|
-
// }
|
|
63
|
-
// if (retries >= 2) {
|
|
64
|
-
// console.warn(`Fetch failed: ${urlPhoto} (${sen.senprenomuse} ${sen.sennomuse})`)
|
|
65
|
-
// console.warn(response.status, response.statusText)
|
|
66
|
-
// console.warn(await response.text())
|
|
67
|
-
// if (fs.existsSync(photoFilePath)) {
|
|
68
|
-
// console.warn(" => Reusing existing image")
|
|
69
|
-
// } else {
|
|
70
|
-
// console.warn(" => Using blank image")
|
|
71
|
-
// fs.copyFileSync(missingPhotoFilePath, photoFilePath)
|
|
72
|
-
// }
|
|
73
|
-
// break
|
|
74
|
-
// }
|
|
75
|
-
// }
|
|
76
|
-
try {
|
|
77
|
-
execSync(`wget --quiet -O ${photoTempFilename} ${urlPhoto}`, {
|
|
78
|
-
cwd: photosDir,
|
|
79
|
-
env: process.env,
|
|
80
|
-
encoding: "utf-8",
|
|
81
|
-
// stdio: ["ignore", "ignore", "pipe"],
|
|
82
|
-
})
|
|
83
|
-
fs.renameSync(photoTempFilePath, photoFilePath)
|
|
84
|
-
} catch (error) {
|
|
85
|
-
if (typeof error === "object" && error && "status" in error && error.status === 8) {
|
|
86
|
-
console.error(`Unable to load photo for ${sen.senprenomuse} ${sen.sennomuse}`)
|
|
87
|
-
continue
|
|
88
|
-
}
|
|
89
|
-
throw error
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
// Resize photos to 155x225, because some haven't exactly this size.
|
|
95
|
-
for (const sen of sens) {
|
|
96
|
-
const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(
|
|
97
|
-
sen.senprenomuse,
|
|
98
|
-
"_",
|
|
99
|
-
)}${slugify(sen.senmat, "_")}`
|
|
100
|
-
const photoFilename = photoStem + ".jpg"
|
|
101
|
-
const photoFilePath = path.join(photosDir, photoFilename)
|
|
102
|
-
if (fs.existsSync(photoFilePath)) {
|
|
103
|
-
if (!options.silent) {
|
|
104
|
-
console.log(
|
|
105
|
-
`Resizing photo ${photoStem} for ${sen.senprenomuse} ${sen.sennomuse}…`,
|
|
106
|
-
)
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
execSync(
|
|
110
|
-
`gm convert -resize 155x225! ${photoStem}.jpg ${photoStem}_155x225.jpg`,
|
|
111
|
-
{
|
|
112
|
-
cwd: photosDir,
|
|
113
|
-
},
|
|
114
|
-
)
|
|
115
|
-
} else {
|
|
116
|
-
if (!options.silent) {
|
|
117
|
-
console.warn(`Missing photo for ${sen.senprenomuse} ${sen.sennomuse}: using blank image`)
|
|
118
|
-
}
|
|
119
|
-
fs.copyFileSync(missingPhotoFilePath, path.join(photosDir, `${photoStem}_155x225.jpg`))
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
// Create a mosaic of photos.
|
|
124
|
-
if (!options.silent) {
|
|
125
|
-
console.log("Creating mosaic of photos…")
|
|
126
|
-
}
|
|
127
|
-
const photoBySenmat: { [senmat: string]: Photo } = {}
|
|
128
|
-
const rowsFilenames: string[] = []
|
|
129
|
-
for (
|
|
130
|
-
let senIndex = 0, rowIndex = 0;
|
|
131
|
-
senIndex < sens.length;
|
|
132
|
-
senIndex += 25, rowIndex++
|
|
133
|
-
) {
|
|
134
|
-
const row = sens.slice(senIndex, senIndex + 25)
|
|
135
|
-
const photosFilenames: string[] = []
|
|
136
|
-
for (const [columnIndex, sen] of row.entries()) {
|
|
137
|
-
const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(
|
|
138
|
-
sen.senprenomuse,
|
|
139
|
-
"_",
|
|
140
|
-
)}${slugify(sen.senmat, "_")}`
|
|
141
|
-
const photoFilename = `${photoStem}_155x225.jpg`
|
|
142
|
-
photosFilenames.push(photoFilename)
|
|
143
|
-
photoBySenmat[sen.senmat] = {
|
|
144
|
-
chemin: `photos_senateurs/${photoFilename}`,
|
|
145
|
-
cheminMosaique: "photos_senateurs/senateurs.jpg",
|
|
146
|
-
hauteur: 225,
|
|
147
|
-
largeur: 155,
|
|
148
|
-
xMosaique: columnIndex * 155,
|
|
149
|
-
yMosaique: rowIndex * 225,
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
const rowFilename = `row-${rowIndex}.jpg`
|
|
153
|
-
execSync(`gm convert ${photosFilenames.join(" ")} +append ${rowFilename}`, {
|
|
154
|
-
cwd: photosDir,
|
|
155
|
-
})
|
|
156
|
-
rowsFilenames.push(rowFilename)
|
|
157
|
-
}
|
|
158
|
-
execSync(`gm convert ${rowsFilenames.join(" ")} -append senateurs.jpg`, {
|
|
159
|
-
cwd: photosDir,
|
|
160
|
-
})
|
|
161
|
-
for (const rowFilename of rowsFilenames) {
|
|
162
|
-
fs.unlinkSync(path.join(photosDir, rowFilename))
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
if (!options.silent) {
|
|
166
|
-
console.log("Creating JSON file containing informations on all pictures…")
|
|
167
|
-
}
|
|
168
|
-
const jsonFilePath = path.join(photosDir, "senateurs.json")
|
|
169
|
-
fs.writeFileSync(jsonFilePath, JSON.stringify(photoBySenmat, null, 2))
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
retrievePhotosSenateurs()
|
|
173
|
-
.then(() => process.exit(0))
|
|
174
|
-
.catch((error) => {
|
|
175
|
-
console.log(error)
|
|
176
|
-
process.exit(1)
|
|
177
|
-
})
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,461 +0,0 @@
|
|
|
1
|
-
// scripts/retrieve_senat_videos_from_agendas.ts
|
|
2
|
-
import assert from "assert";
|
|
3
|
-
import commandLineArgs from "command-line-args";
|
|
4
|
-
import fs from "fs-extra";
|
|
5
|
-
import fsp from "fs/promises";
|
|
6
|
-
import path from "path";
|
|
7
|
-
import * as cheerio from "cheerio";
|
|
8
|
-
import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatAgendas } from "../loaders";
|
|
9
|
-
import { getSessionsFromStart } from "../types/sessions";
|
|
10
|
-
import { commonOptions } from "./shared/cli_helpers";
|
|
11
|
-
import { getAgendaSegmentTimecodes } from "../utils/nvs-timecode";
|
|
12
|
-
import { decodeHtmlEntities } from "../utils/string_cleaning";
|
|
13
|
-
import { dice, normalize, scoreVideo } from "../utils/scoring";
|
|
14
|
-
import { epochToParisDateTime, toFRDate, toTargetEpoch } from "../utils/date";
|
|
15
|
-
// ===================== Constants =====================
|
|
16
|
-
const MATCH_THRESHOLD = 0.5;
|
|
17
|
-
const MAX_CANDIDATES = 15;
|
|
18
|
-
const STATS = { total: 0, accepted: 0 };
|
|
19
|
-
const VIDEOS_ROOT_FOLDER = "videos";
|
|
20
|
-
const SENAT_VIDEOS_SEARCH_AJAX = "https://videos.senat.fr/senat_videos_search.php";
|
|
21
|
-
const SENAT_DATAS_ROOT = "https://videos.senat.fr/Datas/senat";
|
|
22
|
-
// ===================== CLI =====================
|
|
23
|
-
const optionsDefinitions = [...commonOptions];
|
|
24
|
-
const options = commandLineArgs(optionsDefinitions);
|
|
25
|
-
// ===================== Utils =====================
|
|
26
|
-
async function fetchText(url) {
|
|
27
|
-
const res = await fetch(url);
|
|
28
|
-
if (!res.ok)
|
|
29
|
-
return null;
|
|
30
|
-
return await res.text();
|
|
31
|
-
}
|
|
32
|
-
async function fetchBuffer(url) {
|
|
33
|
-
const res = await fetch(url);
|
|
34
|
-
if (!res.ok)
|
|
35
|
-
return null;
|
|
36
|
-
const ab = await res.arrayBuffer();
|
|
37
|
-
return Buffer.from(ab);
|
|
38
|
-
}
|
|
39
|
-
async function writeIfChanged(p, content) {
|
|
40
|
-
const exists = await fs.pathExists(p);
|
|
41
|
-
if (exists) {
|
|
42
|
-
const old = await fsp.readFile(p, "utf-8");
|
|
43
|
-
if (old === content)
|
|
44
|
-
return;
|
|
45
|
-
}
|
|
46
|
-
await fsp.writeFile(p, content, "utf-8");
|
|
47
|
-
}
|
|
48
|
-
function queryString(obj) {
|
|
49
|
-
return Object.entries(obj)
|
|
50
|
-
.map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
|
|
51
|
-
.join("&");
|
|
52
|
-
}
|
|
53
|
-
function extractCandidatesFromSearchHtml(html) {
|
|
54
|
-
const $ = cheerio.load(html);
|
|
55
|
-
const out = [];
|
|
56
|
-
const re = /video\.(\d+)_([a-z0-9]+)/i;
|
|
57
|
-
$('h3.card-title a.stretched-link[href*="video."]').each((_, a) => {
|
|
58
|
-
const href = $(a).attr("href") || "";
|
|
59
|
-
const m = href.match(re);
|
|
60
|
-
if (!m)
|
|
61
|
-
return;
|
|
62
|
-
const id = m[1];
|
|
63
|
-
const hash = m[2];
|
|
64
|
-
const pageUrl = `https://videos.senat.fr/video.${id}_${hash}.html`;
|
|
65
|
-
const title = ($(a).attr("title") || $(a).text() || "").replace(/\s+/g, " ").trim() || undefined;
|
|
66
|
-
const isSeancePublique = title?.toLowerCase().includes("séance publique") ?? false;
|
|
67
|
-
out.push({ id, hash, pageUrl, title, isSeancePublique });
|
|
68
|
-
});
|
|
69
|
-
// dedupe
|
|
70
|
-
const seen = new Set();
|
|
71
|
-
return out.filter((c) => {
|
|
72
|
-
const k = `${c.id}_${c.hash}`;
|
|
73
|
-
if (seen.has(k))
|
|
74
|
-
return false;
|
|
75
|
-
seen.add(k);
|
|
76
|
-
return true;
|
|
77
|
-
});
|
|
78
|
-
}
|
|
79
|
-
function parseDataNvs(nvs) {
|
|
80
|
-
const epochStr = nvs.match(/<metadata\s+name="date"\s+value="(\d+)"/i)?.[1];
|
|
81
|
-
const epoch = epochStr ? Number(epochStr) : undefined;
|
|
82
|
-
// There can be multiple organes for one video in meta
|
|
83
|
-
const organes = [];
|
|
84
|
-
const organesRegex = /<metadata\b[^>]*\bname="organes"[^>]*>/gi;
|
|
85
|
-
let m;
|
|
86
|
-
while ((m = organesRegex.exec(nvs)) !== null) {
|
|
87
|
-
const tag = m[0];
|
|
88
|
-
const label = tag.match(/\blabel="([^"]+)"/i)?.[1];
|
|
89
|
-
if (label) {
|
|
90
|
-
const decoded = decodeHtmlEntities(label).trim();
|
|
91
|
-
if (decoded)
|
|
92
|
-
organes.push(decoded);
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
if (organes.length === 0) {
|
|
96
|
-
organes.push("Séance publique");
|
|
97
|
-
}
|
|
98
|
-
const firstChapterLabelMatch = nvs.match(/<chapter\b[^>]*\blabel="([^"]+)"/i);
|
|
99
|
-
const firstChapterLabel = firstChapterLabelMatch ? decodeHtmlEntities(firstChapterLabelMatch[1]).trim() : undefined;
|
|
100
|
-
return { epoch, organes, firstChapterLabel };
|
|
101
|
-
}
|
|
102
|
-
function buildSenatVodMasterM3u8FromNvs(nvsText) {
|
|
103
|
-
// serverfiles://senat/2025/10/encoder10_20251022084451_2.mp4
|
|
104
|
-
const m = nvsText.match(/serverfiles:\/\/senat\/(\d{4})\/(\d{2})\/(encoder\d+)_([0-9]{14})/i);
|
|
105
|
-
if (!m)
|
|
106
|
-
return null;
|
|
107
|
-
const [, yyyy, mm, encoder, stamp] = m;
|
|
108
|
-
const base = `https://vodsenat.akamaized.net/senat/${yyyy}/${mm}/${encoder}_${stamp}`;
|
|
109
|
-
return `${base}.smil/master.m3u8`;
|
|
110
|
-
}
|
|
111
|
-
function isAmbiguousTimeOriginal(timeOriginal) {
|
|
112
|
-
if (!timeOriginal)
|
|
113
|
-
return false;
|
|
114
|
-
const s = timeOriginal.toLowerCase();
|
|
115
|
-
// Catches "14h", "14 h", "14h30", "14 h 30", "14 heures", "14 heure"
|
|
116
|
-
const timeRe = /\b([01]?\d|2[0-3])\s*(?:h|heures?|heure)\s*(?:([0-5]\d))?\b/g;
|
|
117
|
-
const times = new Set();
|
|
118
|
-
let m;
|
|
119
|
-
while ((m = timeRe.exec(s))) {
|
|
120
|
-
const hh = String(m[1]).padStart(2, "0");
|
|
121
|
-
const mm = m[2] ? String(m[2]).padStart(2, "0") : "00";
|
|
122
|
-
times.add(`${hh}:${mm}`);
|
|
123
|
-
}
|
|
124
|
-
// "midi" / "minuit"
|
|
125
|
-
if (/\bmidi\b/.test(s))
|
|
126
|
-
times.add("12:00");
|
|
127
|
-
if (/\bminuit\b/.test(s))
|
|
128
|
-
times.add("00:00");
|
|
129
|
-
if (times.size >= 2)
|
|
130
|
-
return true;
|
|
131
|
-
const hasDayPeriod = /\b(matin|après-?midi|soir|nuit|journée|toute la journée)\b/.test(s);
|
|
132
|
-
const hasLinking = /,|\bet\b|\bou\b|\bpuis\b/.test(s);
|
|
133
|
-
if (times.size === 1 && hasDayPeriod && hasLinking)
|
|
134
|
-
return true;
|
|
135
|
-
return false;
|
|
136
|
-
}
|
|
137
|
-
function getAgendaType(agenda) {
|
|
138
|
-
const o = agenda.organe || "";
|
|
139
|
-
if (/séance publique/i.test(o))
|
|
140
|
-
return "Séance publique";
|
|
141
|
-
return "Commission";
|
|
142
|
-
}
|
|
143
|
-
async function fetchAllSearchPages(args, maxPages = 3) {
|
|
144
|
-
const pages = [];
|
|
145
|
-
for (let p = 1; p <= maxPages; p++) {
|
|
146
|
-
const url = `${SENAT_VIDEOS_SEARCH_AJAX}?${queryString({ ...args, page: String(p) })}`;
|
|
147
|
-
const html = await fetchText(url);
|
|
148
|
-
if (!html)
|
|
149
|
-
break;
|
|
150
|
-
pages.push(html);
|
|
151
|
-
if (!/href="\/?video\.\d+_[a-z0-9]+\./i.test(html))
|
|
152
|
-
break;
|
|
153
|
-
}
|
|
154
|
-
return pages;
|
|
155
|
-
}
|
|
156
|
-
function getOrgKey(norm) {
|
|
157
|
-
if (!norm)
|
|
158
|
-
return "autre";
|
|
159
|
-
if (norm.includes("seance publique"))
|
|
160
|
-
return "seance_publique";
|
|
161
|
-
if (norm.includes("culture"))
|
|
162
|
-
return "culture";
|
|
163
|
-
if (norm.includes("finances"))
|
|
164
|
-
return "finances";
|
|
165
|
-
if (norm.includes("sociales"))
|
|
166
|
-
return "affaires_sociales";
|
|
167
|
-
if (norm.includes("economiques"))
|
|
168
|
-
return "affaires_economiques";
|
|
169
|
-
if (norm.includes("europeennes"))
|
|
170
|
-
return "affaires_europeennes";
|
|
171
|
-
if (norm.includes("etrangeres") || norm.includes("forces armees") || norm.includes("defense")) {
|
|
172
|
-
return "affaires_etrangeres_defense";
|
|
173
|
-
}
|
|
174
|
-
if (norm.includes("territoire") || norm.includes("durable")) {
|
|
175
|
-
return "amenagement_territoire_dd";
|
|
176
|
-
}
|
|
177
|
-
if (norm.includes("commission des lois"))
|
|
178
|
-
return "lois";
|
|
179
|
-
if (norm.includes("delegation aux collectivites territoriales") || norm.includes("delegation a la decentralisation"))
|
|
180
|
-
return "delegation_collectivites";
|
|
181
|
-
if (norm.includes("delegation aux droits des femmes") ||
|
|
182
|
-
norm.includes("egalite des chances entre les hommes et les femmes"))
|
|
183
|
-
return "delegation_droits_femmes";
|
|
184
|
-
if (norm.includes("delegation aux entreprises"))
|
|
185
|
-
return "delegation_entreprises";
|
|
186
|
-
if (norm.includes("delegation senatoriale aux outre mer") || norm.includes("delegation aux outre mer"))
|
|
187
|
-
return "delegation_outre_mer";
|
|
188
|
-
if (norm.includes("delegation a la prospective"))
|
|
189
|
-
return "delegation_prospective";
|
|
190
|
-
if (norm.includes("office parlementaire d evaluation des choix scientifiques et technologiques") ||
|
|
191
|
-
norm.includes("opecst"))
|
|
192
|
-
return "opecst";
|
|
193
|
-
return "autre";
|
|
194
|
-
}
|
|
195
|
-
async function processGroupedReunion(agenda, session, dataDir) {
|
|
196
|
-
// 1) GuardRails
|
|
197
|
-
if (!agenda.captationVideo) {
|
|
198
|
-
// if (!options["silent"]) console.log(`[skip] ${agenda.uid} captationVideo=false`)
|
|
199
|
-
return;
|
|
200
|
-
}
|
|
201
|
-
if (!agenda.date || !agenda.startTime) {
|
|
202
|
-
// if (!options["silent"]) console.log(`[skip] ${agenda.uid} date/hour missing`)
|
|
203
|
-
return;
|
|
204
|
-
}
|
|
205
|
-
const agendaTs = toTargetEpoch(agenda.startTime, agenda.date);
|
|
206
|
-
const now = Date.now();
|
|
207
|
-
if (agendaTs && agendaTs * 1000 > now) {
|
|
208
|
-
return;
|
|
209
|
-
}
|
|
210
|
-
const reunionUid = agenda.uid;
|
|
211
|
-
const baseDir = path.join(dataDir, VIDEOS_ROOT_FOLDER, String(session), reunionUid);
|
|
212
|
-
await fs.ensureDir(baseDir);
|
|
213
|
-
let skipDownload = false;
|
|
214
|
-
if (options["only-recent"]) {
|
|
215
|
-
const now = Date.now();
|
|
216
|
-
const cutoff = now - options["only-recent"] * 24 * 3600 * 1000;
|
|
217
|
-
const reunionTs = Date.parse(agenda.date);
|
|
218
|
-
if (reunionTs < cutoff) {
|
|
219
|
-
// Check if files already exist
|
|
220
|
-
const dataNvsPath = path.join(baseDir, "data.nvs");
|
|
221
|
-
const finalplayerNvsPath = path.join(baseDir, "finalplayer.nvs");
|
|
222
|
-
if (fs.existsSync(dataNvsPath) && fs.existsSync(finalplayerNvsPath)) {
|
|
223
|
-
skipDownload = true;
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
}
|
|
227
|
-
let master = null;
|
|
228
|
-
let dataTxt = null;
|
|
229
|
-
let finalTxt = null;
|
|
230
|
-
let accepted = false;
|
|
231
|
-
if (!skipDownload) {
|
|
232
|
-
STATS.total++;
|
|
233
|
-
const searchParams = {
|
|
234
|
-
search: "true",
|
|
235
|
-
videotype: getAgendaType(agenda),
|
|
236
|
-
};
|
|
237
|
-
if (agenda.date) {
|
|
238
|
-
const fr = toFRDate(agenda.date);
|
|
239
|
-
searchParams.period = "custom";
|
|
240
|
-
searchParams.begin = fr;
|
|
241
|
-
searchParams.end = fr;
|
|
242
|
-
}
|
|
243
|
-
if (agenda.organe) {
|
|
244
|
-
searchParams.organe = agenda.organe;
|
|
245
|
-
}
|
|
246
|
-
const pages = await fetchAllSearchPages(searchParams);
|
|
247
|
-
if (!pages.length) {
|
|
248
|
-
if (!options["silent"]) {
|
|
249
|
-
console.log(`[miss] ${agenda.uid} no candidates (videotype=${searchParams.videotype}, organe=${searchParams.organe || "-"}, date=${searchParams.begin || "-"})`);
|
|
250
|
-
}
|
|
251
|
-
return;
|
|
252
|
-
}
|
|
253
|
-
const combinedHtml = pages.join("\n<!-- PAGE SPLIT -->\n");
|
|
254
|
-
const candidates = extractCandidatesFromSearchHtml(combinedHtml).slice(0, MAX_CANDIDATES);
|
|
255
|
-
if (!candidates.length) {
|
|
256
|
-
if (!options["silent"]) {
|
|
257
|
-
console.log(`[miss] ${agenda.uid} no candidates after parse (videotype=${searchParams.videotype}, organe=${searchParams.organe || "-"}, date=${searchParams.begin || "-"})`);
|
|
258
|
-
}
|
|
259
|
-
return;
|
|
260
|
-
}
|
|
261
|
-
// ==== 2) Enrich via data.nvs + scoring; pick best ====
|
|
262
|
-
let best = null;
|
|
263
|
-
const timeAmbigious = isAmbiguousTimeOriginal(agenda.events[0].timeOriginal);
|
|
264
|
-
if (timeAmbigious) {
|
|
265
|
-
console.log(`[match] ${agenda.uid} timeOriginal ambiguous => ignoring time scoring: "${agenda.events[0].timeOriginal}"`);
|
|
266
|
-
}
|
|
267
|
-
for (const c of candidates) {
|
|
268
|
-
const dataUrl = `${SENAT_DATAS_ROOT}/${c.id}_${c.hash}/content/data.nvs`;
|
|
269
|
-
const finalUrl = `${SENAT_DATAS_ROOT}/${c.id}_${c.hash}/content/finalplayer.nvs`;
|
|
270
|
-
const dataBuf = await fetchBuffer(dataUrl);
|
|
271
|
-
if (!dataBuf)
|
|
272
|
-
continue;
|
|
273
|
-
const meta = parseDataNvs(dataBuf.toString("utf-8"));
|
|
274
|
-
let sameOrg = false;
|
|
275
|
-
// If organes are too different, go to next candidates
|
|
276
|
-
if (agenda.organe && meta.organes?.length) {
|
|
277
|
-
const agendaOrgNorm = normalize(agenda.organe);
|
|
278
|
-
const agendaKey = getOrgKey(agendaOrgNorm);
|
|
279
|
-
let bestDice = 0;
|
|
280
|
-
let hasSameKey = false;
|
|
281
|
-
for (const vo of meta.organes) {
|
|
282
|
-
const videoOrgNorm = normalize(vo);
|
|
283
|
-
const videoKey = getOrgKey(videoOrgNorm);
|
|
284
|
-
const d = dice(agendaOrgNorm, videoOrgNorm);
|
|
285
|
-
if (videoKey === agendaKey && videoKey !== "autre") {
|
|
286
|
-
hasSameKey = true;
|
|
287
|
-
}
|
|
288
|
-
if (d > bestDice)
|
|
289
|
-
bestDice = d;
|
|
290
|
-
}
|
|
291
|
-
if (hasSameKey) {
|
|
292
|
-
sameOrg = true; // we are sure this is the same org
|
|
293
|
-
}
|
|
294
|
-
else if (bestDice < 0.8) {
|
|
295
|
-
// if diff org and dice too low we skip
|
|
296
|
-
continue;
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
let videoTitle = c.title;
|
|
300
|
-
if (c.isSeancePublique && meta.firstChapterLabel) {
|
|
301
|
-
videoTitle = meta.firstChapterLabel;
|
|
302
|
-
}
|
|
303
|
-
const s = scoreVideo(agenda, agendaTs, sameOrg, videoTitle, meta.epoch, meta.organes, timeAmbigious);
|
|
304
|
-
if (!best || s > best.score) {
|
|
305
|
-
best = {
|
|
306
|
-
id: c.id,
|
|
307
|
-
hash: c.hash,
|
|
308
|
-
pageUrl: c.pageUrl,
|
|
309
|
-
epoch: meta.epoch,
|
|
310
|
-
vtitle: videoTitle,
|
|
311
|
-
score: s,
|
|
312
|
-
vorgane: meta.organes[0],
|
|
313
|
-
};
|
|
314
|
-
}
|
|
315
|
-
}
|
|
316
|
-
if (!best) {
|
|
317
|
-
if (!options["silent"])
|
|
318
|
-
console.log(`[miss] ${agenda.uid} No candidate found for this reunion`);
|
|
319
|
-
return;
|
|
320
|
-
}
|
|
321
|
-
accepted = best.score >= MATCH_THRESHOLD;
|
|
322
|
-
if (accepted)
|
|
323
|
-
STATS.accepted++;
|
|
324
|
-
if (!options["silent"]) {
|
|
325
|
-
console.log(`[pick] ${agenda.uid} score=${best.score.toFixed(2)}
|
|
326
|
-
agenda title="${agenda.titre ?? ""}" agenda organe="${agenda.organe ?? ""}" agenda heure=${agenda.startTime}
|
|
327
|
-
best title="${best.vtitle ?? ""}" best organe="${best.vorgane ?? ""}"
|
|
328
|
-
accepted=${accepted}`);
|
|
329
|
-
}
|
|
330
|
-
// ==== 3) Write metadata + NVS of the best candidate (always) ====
|
|
331
|
-
const bestDt = best?.epoch ? epochToParisDateTime(best.epoch) : null;
|
|
332
|
-
const metadata = {
|
|
333
|
-
reunionUid,
|
|
334
|
-
session,
|
|
335
|
-
accepted,
|
|
336
|
-
threshold: MATCH_THRESHOLD,
|
|
337
|
-
agenda: {
|
|
338
|
-
date: agenda.date,
|
|
339
|
-
startTime: agenda.startTime,
|
|
340
|
-
titre: agenda.titre,
|
|
341
|
-
organe: agenda.organe ?? undefined,
|
|
342
|
-
uid: agenda.uid,
|
|
343
|
-
},
|
|
344
|
-
best: {
|
|
345
|
-
id: best.id,
|
|
346
|
-
hash: best.hash,
|
|
347
|
-
pageUrl: best.pageUrl,
|
|
348
|
-
epoch: best.epoch ?? null,
|
|
349
|
-
date: bestDt?.date ?? null,
|
|
350
|
-
startTime: bestDt?.startTime ?? null,
|
|
351
|
-
title: best.vtitle ?? null,
|
|
352
|
-
score: best.score,
|
|
353
|
-
},
|
|
354
|
-
};
|
|
355
|
-
await writeIfChanged(path.join(baseDir, "metadata.json"), JSON.stringify(metadata, null, 2));
|
|
356
|
-
const dataUrl = `${SENAT_DATAS_ROOT}/${best.id}_${best.hash}/content/data.nvs`;
|
|
357
|
-
const finalUrl = `${SENAT_DATAS_ROOT}/${best.id}_${best.hash}/content/finalplayer.nvs`;
|
|
358
|
-
dataTxt = await fetchText(dataUrl);
|
|
359
|
-
finalTxt = await fetchText(finalUrl);
|
|
360
|
-
if (dataTxt)
|
|
361
|
-
await fsp.writeFile(path.join(baseDir, "data.nvs"), dataTxt, "utf-8");
|
|
362
|
-
if (finalTxt)
|
|
363
|
-
await fsp.writeFile(path.join(baseDir, "finalplayer.nvs"), finalTxt, "utf-8");
|
|
364
|
-
if (dataTxt) {
|
|
365
|
-
master = buildSenatVodMasterM3u8FromNvs(dataTxt);
|
|
366
|
-
}
|
|
367
|
-
else {
|
|
368
|
-
console.log("Cannot download data nvs");
|
|
369
|
-
}
|
|
370
|
-
}
|
|
371
|
-
else {
|
|
372
|
-
// Skipped download, but need to read data.nvs for urlVideo
|
|
373
|
-
try {
|
|
374
|
-
dataTxt = await fsp.readFile(path.join(baseDir, "data.nvs"), "utf-8");
|
|
375
|
-
finalTxt = await fsp.readFile(path.join(baseDir, "finalplayer.nvs"), "utf-8");
|
|
376
|
-
master = buildSenatVodMasterM3u8FromNvs(dataTxt);
|
|
377
|
-
}
|
|
378
|
-
catch (e) {
|
|
379
|
-
console.warn(e);
|
|
380
|
-
}
|
|
381
|
-
}
|
|
382
|
-
// ==== 4) Update agenda file (only if accepted + m3u8) ====
|
|
383
|
-
if ((accepted || skipDownload) && master) {
|
|
384
|
-
const agendaJsonPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session), `${agenda.uid}.json`);
|
|
385
|
-
let timecodeDebutVideo = null;
|
|
386
|
-
let timecodeFinVideo = null;
|
|
387
|
-
if (dataTxt && finalTxt) {
|
|
388
|
-
const agendaKey = agenda.titre || agenda.objet || "";
|
|
389
|
-
const seg = getAgendaSegmentTimecodes(dataTxt, finalTxt, agendaKey);
|
|
390
|
-
if (!seg) {
|
|
391
|
-
console.warn(`[warn] Cannot retrieve agenda segment timecodes from reunion ${reunionUid}`);
|
|
392
|
-
}
|
|
393
|
-
else {
|
|
394
|
-
timecodeDebutVideo = seg.start;
|
|
395
|
-
timecodeFinVideo = seg.end;
|
|
396
|
-
}
|
|
397
|
-
}
|
|
398
|
-
if (await fs.pathExists(agendaJsonPath)) {
|
|
399
|
-
const raw = await fsp.readFile(agendaJsonPath, "utf-8");
|
|
400
|
-
let obj;
|
|
401
|
-
try {
|
|
402
|
-
obj = JSON.parse(raw);
|
|
403
|
-
}
|
|
404
|
-
catch (e) {
|
|
405
|
-
console.warn(`[warn] invalid JSON in ${agendaJsonPath}:`, e?.message);
|
|
406
|
-
obj = null;
|
|
407
|
-
}
|
|
408
|
-
if (obj && typeof obj === "object" && !Array.isArray(obj)) {
|
|
409
|
-
const next = { ...obj, urlVideo: master };
|
|
410
|
-
if (timecodeDebutVideo != null) {
|
|
411
|
-
next.timecodeDebutVideo = timecodeDebutVideo;
|
|
412
|
-
next.timecodeFinVideo = timecodeFinVideo;
|
|
413
|
-
}
|
|
414
|
-
await writeIfChanged(agendaJsonPath, JSON.stringify(next, null, 2));
|
|
415
|
-
if (!options["silent"]) {
|
|
416
|
-
console.log(`[write] ${agenda.uid} urlVideo ← ${master}` +
|
|
417
|
-
(timecodeDebutVideo != null ? ` (timecodeDebutVideo ← ${timecodeDebutVideo}s)` : ""));
|
|
418
|
-
}
|
|
419
|
-
}
|
|
420
|
-
else {
|
|
421
|
-
console.warn(`[warn] expected an object in ${agendaJsonPath}, got ${Array.isArray(obj) ? "array" : typeof obj}`);
|
|
422
|
-
}
|
|
423
|
-
}
|
|
424
|
-
else {
|
|
425
|
-
console.warn(`[warn] agenda file not found for update: ${agendaJsonPath}`);
|
|
426
|
-
}
|
|
427
|
-
}
|
|
428
|
-
else {
|
|
429
|
-
console.warn(`[warn] The video url could not be built for reunion `, reunionUid);
|
|
430
|
-
}
|
|
431
|
-
}
|
|
432
|
-
async function processAll(dataDir, sessions) {
|
|
433
|
-
console.log("Process all Agendas and fetch video's url");
|
|
434
|
-
for (const session of sessions) {
|
|
435
|
-
for (const { item: agenda } of iterLoadSenatAgendas(dataDir, session)) {
|
|
436
|
-
try {
|
|
437
|
-
await processGroupedReunion(agenda, session, dataDir);
|
|
438
|
-
}
|
|
439
|
-
catch (e) {
|
|
440
|
-
console.error(`[error] ${agenda?.uid ?? "unknown-uid"}:`, e?.message || e);
|
|
441
|
-
}
|
|
442
|
-
}
|
|
443
|
-
}
|
|
444
|
-
}
|
|
445
|
-
async function main() {
|
|
446
|
-
const dataDir = options["dataDir"];
|
|
447
|
-
assert(dataDir, "Missing argument: data directory");
|
|
448
|
-
const sessions = getSessionsFromStart(options["fromSession"]);
|
|
449
|
-
console.time("senat-agendas→videos start processing time");
|
|
450
|
-
await processAll(dataDir, sessions);
|
|
451
|
-
console.timeEnd("senat-agendas→videos processing time");
|
|
452
|
-
const { total, accepted } = STATS;
|
|
453
|
-
const ratio = total ? ((accepted / total) * 100).toFixed(1) : "0.0";
|
|
454
|
-
console.log(`[summary] accepted=${accepted} / total=${total} (${ratio}%)`);
|
|
455
|
-
}
|
|
456
|
-
main()
|
|
457
|
-
.then(() => process.exit(0))
|
|
458
|
-
.catch((err) => {
|
|
459
|
-
console.error(err);
|
|
460
|
-
process.exit(1);
|
|
461
|
-
});
|