@tricoteuses/senat 2.19.1 → 2.19.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +22 -22
- package/README.md +123 -123
- package/lib/model/agenda.js +5 -5
- package/lib/scripts/retrieve_videos.d.ts +1 -1
- package/lib/scripts/retrieve_videos.js +21 -37
- package/package.json +101 -101
package/LICENSE.md
CHANGED
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
# Tricoteuses-Senat
|
|
2
|
-
|
|
3
|
-
## _Handle French Sénat's open data_
|
|
4
|
-
|
|
5
|
-
By: Emmanuel Raviart <mailto:emmanuel@raviart.com>
|
|
6
|
-
|
|
7
|
-
Copyright (C) 2019, 2020, 2021 Emmanuel Raviart
|
|
8
|
-
|
|
9
|
-
https://git.tricoteuses.fr/logiciels/tricoteuses-senat
|
|
10
|
-
|
|
11
|
-
> Tricoteuses-Senat is free software; you can redistribute it and/or modify
|
|
12
|
-
> it under the terms of the GNU Affero General Public License as
|
|
13
|
-
> published by the Free Software Foundation, either version 3 of the
|
|
14
|
-
> License, or (at your option) any later version.
|
|
15
|
-
>
|
|
16
|
-
> Tricoteuses-Senat is distributed in the hope that it will be useful,
|
|
17
|
-
> but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
18
|
-
> MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
19
|
-
> GNU Affero General Public License for more details.
|
|
20
|
-
>
|
|
21
|
-
> You should have received a copy of the GNU Affero General Public License
|
|
22
|
-
> along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
1
|
+
# Tricoteuses-Senat
|
|
2
|
+
|
|
3
|
+
## _Handle French Sénat's open data_
|
|
4
|
+
|
|
5
|
+
By: Emmanuel Raviart <mailto:emmanuel@raviart.com>
|
|
6
|
+
|
|
7
|
+
Copyright (C) 2019, 2020, 2021 Emmanuel Raviart
|
|
8
|
+
|
|
9
|
+
https://git.tricoteuses.fr/logiciels/tricoteuses-senat
|
|
10
|
+
|
|
11
|
+
> Tricoteuses-Senat is free software; you can redistribute it and/or modify
|
|
12
|
+
> it under the terms of the GNU Affero General Public License as
|
|
13
|
+
> published by the Free Software Foundation, either version 3 of the
|
|
14
|
+
> License, or (at your option) any later version.
|
|
15
|
+
>
|
|
16
|
+
> Tricoteuses-Senat is distributed in the hope that it will be useful,
|
|
17
|
+
> but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
18
|
+
> MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
19
|
+
> GNU Affero General Public License for more details.
|
|
20
|
+
>
|
|
21
|
+
> You should have received a copy of the GNU Affero General Public License
|
|
22
|
+
> along with this program. If not, see <http://www.gnu.org/licenses/>.
|
package/README.md
CHANGED
|
@@ -1,123 +1,123 @@
|
|
|
1
|
-
# Tricoteuses-Senat
|
|
2
|
-
|
|
3
|
-
## _Retrieve, clean up & handle French Sénat's open data_
|
|
4
|
-
|
|
5
|
-
## Requirements
|
|
6
|
-
|
|
7
|
-
- Node >= 22
|
|
8
|
-
|
|
9
|
-
## Installation
|
|
10
|
-
|
|
11
|
-
```bash
|
|
12
|
-
git clone https://git.tricoteuses.fr/logiciels/tricoteuses-senat
|
|
13
|
-
cd tricoteuses-senat/
|
|
14
|
-
```
|
|
15
|
-
|
|
16
|
-
Create a `.env` file to set PostgreSQL database informations and other configuration variables (you can use `example.env` as a template). Then
|
|
17
|
-
|
|
18
|
-
```bash
|
|
19
|
-
npm install
|
|
20
|
-
```
|
|
21
|
-
|
|
22
|
-
### Database creation (not needed if downloading with Docker image)
|
|
23
|
-
|
|
24
|
-
#### Using Docker
|
|
25
|
-
|
|
26
|
-
```bash
|
|
27
|
-
docker run --name local-postgres -d -p 5432:5432 -e POSTGRES_PASSWORD=$YOUR_CUSTOM_DB_PASSWORD postgres
|
|
28
|
-
# Default Postgres user is postgres
|
|
29
|
-
# But scripts require an "opendata" role
|
|
30
|
-
docker exec -it local-postgres psql -U postgres -c "CREATE ROLE opendata;"
|
|
31
|
-
```
|
|
32
|
-
|
|
33
|
-
## Download data
|
|
34
|
-
|
|
35
|
-
Create a folder where the data will be downloaded and run the following command to download the data and convert it into JSON files.
|
|
36
|
-
|
|
37
|
-
```bash
|
|
38
|
-
mkdir ../senat-data/
|
|
39
|
-
|
|
40
|
-
# Available options for optional `categories` parameter : All, Ameli, Debats, DosLeg, Questions, Sens
|
|
41
|
-
npm run data:download ../senat-data -- [--categories All]
|
|
42
|
-
```
|
|
43
|
-
|
|
44
|
-
Data from other sources is also available :
|
|
45
|
-
|
|
46
|
-
```bash
|
|
47
|
-
# Retrieval of textes and rapports from Sénat's website
|
|
48
|
-
# Available options for optional `formats` parameter : xml, html, pdf
|
|
49
|
-
# Available options for optional `types` parameter : textes, rapports
|
|
50
|
-
npm run data:retrieve_documents ../senat-data -- --fromSession 2022 [--formats xml pdf] [--types textes]
|
|
51
|
-
|
|
52
|
-
# Retrieval & parsing (textes in xml format only for now)
|
|
53
|
-
npm run data:retrieve_documents ../senat-data -- --fromSession 2022 --parseDocuments
|
|
54
|
-
|
|
55
|
-
# Parsing only
|
|
56
|
-
npm run data:parse_textes_lois ../senat-data
|
|
57
|
-
|
|
58
|
-
# Retrieval (& parsing) of agenda from Sénat's website
|
|
59
|
-
npm run data:retrieve_agenda ../senat-data -- --fromSession 2022 [--parseAgenda]
|
|
60
|
-
|
|
61
|
-
# Retrieval (& parsing) of comptes-rendus de séance from Sénat's data
|
|
62
|
-
npm run data:retrieve_cr_seance ../senat-data -- [--parseDebats]
|
|
63
|
-
|
|
64
|
-
# Retrieval (& parsing) of comptes-rendus de commissions from Sénat's website
|
|
65
|
-
npm run data:retrieve_cr_commission ../senat-data -- [--parseDebats]
|
|
66
|
-
|
|
67
|
-
# Retrieval of sénateurs' pictures from Sénat's website
|
|
68
|
-
npm run data:retrieve_senateurs_photos ../senat-data
|
|
69
|
-
```
|
|
70
|
-
|
|
71
|
-
## Data download using Docker
|
|
72
|
-
|
|
73
|
-
A Docker image that downloads and converts the data all at once is available. Build it locally or run it from the container registry.
|
|
74
|
-
Use the environment variables `FROM_SESSION` and `CATEGORIES` if needed.
|
|
75
|
-
|
|
76
|
-
```bash
|
|
77
|
-
docker run --pull always --name tricoteuses-senat -v ../senat-data:/app/senat-data -d git.tricoteuses.fr/logiciels/tricoteuses-senat:latest
|
|
78
|
-
```
|
|
79
|
-
|
|
80
|
-
Use the environment variable `CATEGORIES` and `FROM_SESSION` if needed.
|
|
81
|
-
|
|
82
|
-
## Using the data
|
|
83
|
-
|
|
84
|
-
Once the data is downloaded, you can use loaders to retrieve it.
|
|
85
|
-
To use loaders in your project, you can install the _@tricoteuses/senat_ package, and import the iterator functions that you need.
|
|
86
|
-
|
|
87
|
-
```bash
|
|
88
|
-
npm install @tricoteuses/senat
|
|
89
|
-
```
|
|
90
|
-
|
|
91
|
-
```js
|
|
92
|
-
import { iterLoadSenatQuestions } from "@tricoteuses/senat/loaders"
|
|
93
|
-
|
|
94
|
-
// Pass data directory and legislature as arguments
|
|
95
|
-
for (const { item: question } of iterLoadSenatQuestions("../senat-data", 17)) {
|
|
96
|
-
console.log(question.id)
|
|
97
|
-
}
|
|
98
|
-
```
|
|
99
|
-
|
|
100
|
-
## Generation of raw types from SQL schema (for contributors only)
|
|
101
|
-
|
|
102
|
-
```bash
|
|
103
|
-
npm run data:generate_schemas ../senat-data
|
|
104
|
-
```
|
|
105
|
-
|
|
106
|
-
## Publishing
|
|
107
|
-
|
|
108
|
-
To publish a new version of this package onto npm, bump the package version and publish.
|
|
109
|
-
|
|
110
|
-
```bash
|
|
111
|
-
# Increment version and create a new Git tag automatically
|
|
112
|
-
npm version patch # +0.0.1 → small fixes
|
|
113
|
-
npm version minor # +0.1.0 → new features
|
|
114
|
-
npm version major # +1.0.0 → breaking changes
|
|
115
|
-
npx tsc
|
|
116
|
-
npm publish
|
|
117
|
-
```
|
|
118
|
-
|
|
119
|
-
The Docker image will be automatically built during a CI Workflow if you push the tag to the remote repository.
|
|
120
|
-
|
|
121
|
-
```bash
|
|
122
|
-
git push --tags
|
|
123
|
-
```
|
|
1
|
+
# Tricoteuses-Senat
|
|
2
|
+
|
|
3
|
+
## _Retrieve, clean up & handle French Sénat's open data_
|
|
4
|
+
|
|
5
|
+
## Requirements
|
|
6
|
+
|
|
7
|
+
- Node >= 22
|
|
8
|
+
|
|
9
|
+
## Installation
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
git clone https://git.tricoteuses.fr/logiciels/tricoteuses-senat
|
|
13
|
+
cd tricoteuses-senat/
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
Create a `.env` file to set PostgreSQL database informations and other configuration variables (you can use `example.env` as a template). Then
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
npm install
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
### Database creation (not needed if downloading with Docker image)
|
|
23
|
+
|
|
24
|
+
#### Using Docker
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
docker run --name local-postgres -d -p 5432:5432 -e POSTGRES_PASSWORD=$YOUR_CUSTOM_DB_PASSWORD postgres
|
|
28
|
+
# Default Postgres user is postgres
|
|
29
|
+
# But scripts require an "opendata" role
|
|
30
|
+
docker exec -it local-postgres psql -U postgres -c "CREATE ROLE opendata;"
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Download data
|
|
34
|
+
|
|
35
|
+
Create a folder where the data will be downloaded and run the following command to download the data and convert it into JSON files.
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
mkdir ../senat-data/
|
|
39
|
+
|
|
40
|
+
# Available options for optional `categories` parameter : All, Ameli, Debats, DosLeg, Questions, Sens
|
|
41
|
+
npm run data:download ../senat-data -- [--categories All]
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Data from other sources is also available :
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
# Retrieval of textes and rapports from Sénat's website
|
|
48
|
+
# Available options for optional `formats` parameter : xml, html, pdf
|
|
49
|
+
# Available options for optional `types` parameter : textes, rapports
|
|
50
|
+
npm run data:retrieve_documents ../senat-data -- --fromSession 2022 [--formats xml pdf] [--types textes]
|
|
51
|
+
|
|
52
|
+
# Retrieval & parsing (textes in xml format only for now)
|
|
53
|
+
npm run data:retrieve_documents ../senat-data -- --fromSession 2022 --parseDocuments
|
|
54
|
+
|
|
55
|
+
# Parsing only
|
|
56
|
+
npm run data:parse_textes_lois ../senat-data
|
|
57
|
+
|
|
58
|
+
# Retrieval (& parsing) of agenda from Sénat's website
|
|
59
|
+
npm run data:retrieve_agenda ../senat-data -- --fromSession 2022 [--parseAgenda]
|
|
60
|
+
|
|
61
|
+
# Retrieval (& parsing) of comptes-rendus de séance from Sénat's data
|
|
62
|
+
npm run data:retrieve_cr_seance ../senat-data -- [--parseDebats]
|
|
63
|
+
|
|
64
|
+
# Retrieval (& parsing) of comptes-rendus de commissions from Sénat's website
|
|
65
|
+
npm run data:retrieve_cr_commission ../senat-data -- [--parseDebats]
|
|
66
|
+
|
|
67
|
+
# Retrieval of sénateurs' pictures from Sénat's website
|
|
68
|
+
npm run data:retrieve_senateurs_photos ../senat-data
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Data download using Docker
|
|
72
|
+
|
|
73
|
+
A Docker image that downloads and converts the data all at once is available. Build it locally or run it from the container registry.
|
|
74
|
+
Use the environment variables `FROM_SESSION` and `CATEGORIES` if needed.
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
docker run --pull always --name tricoteuses-senat -v ../senat-data:/app/senat-data -d git.tricoteuses.fr/logiciels/tricoteuses-senat:latest
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Use the environment variable `CATEGORIES` and `FROM_SESSION` if needed.
|
|
81
|
+
|
|
82
|
+
## Using the data
|
|
83
|
+
|
|
84
|
+
Once the data is downloaded, you can use loaders to retrieve it.
|
|
85
|
+
To use loaders in your project, you can install the _@tricoteuses/senat_ package, and import the iterator functions that you need.
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
npm install @tricoteuses/senat
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
```js
|
|
92
|
+
import { iterLoadSenatQuestions } from "@tricoteuses/senat/loaders"
|
|
93
|
+
|
|
94
|
+
// Pass data directory and legislature as arguments
|
|
95
|
+
for (const { item: question } of iterLoadSenatQuestions("../senat-data", 17)) {
|
|
96
|
+
console.log(question.id)
|
|
97
|
+
}
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## Generation of raw types from SQL schema (for contributors only)
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
npm run data:generate_schemas ../senat-data
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
## Publishing
|
|
107
|
+
|
|
108
|
+
To publish a new version of this package onto npm, bump the package version and publish.
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
# Increment version and create a new Git tag automatically
|
|
112
|
+
npm version patch # +0.0.1 → small fixes
|
|
113
|
+
npm version minor # +0.1.0 → new features
|
|
114
|
+
npm version major # +1.0.0 → breaking changes
|
|
115
|
+
npx tsc
|
|
116
|
+
npm publish
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
The Docker image will be automatically built during a CI Workflow if you push the tag to the remote repository.
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
git push --tags
|
|
123
|
+
```
|
package/lib/model/agenda.js
CHANGED
|
@@ -70,7 +70,7 @@ export function getStartAndEndTimes(timeStr, dateISO) {
|
|
|
70
70
|
return { startTime: null, endTime: null };
|
|
71
71
|
}
|
|
72
72
|
const rangeMatch = normalizedTime.match(/^de (?<start>\d{1,2}h\d{2}) à (?<end>\d{1,2}h\d{2})$/i);
|
|
73
|
-
const
|
|
73
|
+
const toUtcTimeOnly = (value) => {
|
|
74
74
|
if (!value)
|
|
75
75
|
return null;
|
|
76
76
|
const time = DateTime.fromFormat(value, "H'h'mm", { zone: FR_TZ });
|
|
@@ -84,17 +84,17 @@ export function getStartAndEndTimes(timeStr, dateISO) {
|
|
|
84
84
|
});
|
|
85
85
|
if (!local.isValid)
|
|
86
86
|
return null;
|
|
87
|
-
return local.toUTC().
|
|
87
|
+
return local.toUTC().toFormat("HH:mm:ss.SSS'Z'");
|
|
88
88
|
};
|
|
89
89
|
if (rangeMatch?.groups) {
|
|
90
90
|
const { start, end } = rangeMatch.groups;
|
|
91
91
|
return {
|
|
92
|
-
startTime:
|
|
93
|
-
endTime:
|
|
92
|
+
startTime: toUtcTimeOnly(start),
|
|
93
|
+
endTime: toUtcTimeOnly(end),
|
|
94
94
|
};
|
|
95
95
|
}
|
|
96
96
|
return {
|
|
97
|
-
startTime:
|
|
97
|
+
startTime: toUtcTimeOnly(normalizedTime),
|
|
98
98
|
endTime: null,
|
|
99
99
|
};
|
|
100
100
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export
|
|
1
|
+
export declare function buildSenatVodMasterM3u8FromNvs(nvsText: string): string | null;
|
|
@@ -158,38 +158,13 @@ function parseDataNvs(nvs) {
|
|
|
158
158
|
const firstChapterLabel = firstChapterLabelMatch ? decodeHtmlEntities(firstChapterLabelMatch[1]).trim() : undefined;
|
|
159
159
|
return { epoch, organes, firstChapterLabel };
|
|
160
160
|
}
|
|
161
|
-
function buildSenatVodMasterM3u8FromNvs(nvsText
|
|
162
|
-
//
|
|
163
|
-
const
|
|
164
|
-
if (!
|
|
161
|
+
export function buildSenatVodMasterM3u8FromNvs(nvsText) {
|
|
162
|
+
// serverfiles://senat/2025/10/encoder10_20251022084451_2.mp4
|
|
163
|
+
const m = nvsText.match(/serverfiles:\/\/senat\/(\d{4})\/(\d{2})\/(encoder\d+)_([0-9]{14})/i);
|
|
164
|
+
if (!m)
|
|
165
165
|
return null;
|
|
166
|
-
const [, yyyy, mm, encoder, stamp] =
|
|
166
|
+
const [, yyyy, mm, encoder, stamp] = m;
|
|
167
167
|
const base = `https://vodsenat.akamaized.net/senat/${yyyy}/${mm}/${encoder}_${stamp}`;
|
|
168
|
-
// 2) start/end from finalplayer.nvs
|
|
169
|
-
let start = null, end = null;
|
|
170
|
-
const playerAttr = finalText.match(/player[^>]*\bstarttime="(\d+)"[^>]*\bendtime="(\d+)"/i);
|
|
171
|
-
if (playerAttr) {
|
|
172
|
-
start = parseInt(playerAttr[1], 10);
|
|
173
|
-
end = parseInt(playerAttr[2], 10);
|
|
174
|
-
}
|
|
175
|
-
else {
|
|
176
|
-
// fallback: take smallest timecode of <synchro timecode="...">
|
|
177
|
-
const tc = Array.from(finalText.matchAll(/timecode="(\d+)"/g)).map((m) => parseInt(m[1], 10));
|
|
178
|
-
if (tc.length)
|
|
179
|
-
start = Math.min(...tc);
|
|
180
|
-
}
|
|
181
|
-
if (end == null) {
|
|
182
|
-
const durMeta = nvsText.match(/<metadata[^>]*\bname="duree"[^>]*\bvalue="(\d+)"[^>]*>/i);
|
|
183
|
-
if (durMeta && start != null) {
|
|
184
|
-
const durMs = parseInt(durMeta[1], 10) * 1000; // sec → ms
|
|
185
|
-
end = start + durMs;
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
// 4) Construct URL
|
|
189
|
-
if (start != null && end != null && end > start) {
|
|
190
|
-
const pd = end - start;
|
|
191
|
-
return `${base}_ps${start}_pd${pd}.smil/master.m3u8`;
|
|
192
|
-
}
|
|
193
168
|
return `${base}.smil/master.m3u8`;
|
|
194
169
|
}
|
|
195
170
|
function score(agenda, agendaTs, sameOrg, videoTitle, videoEpoch, videoOrganes) {
|
|
@@ -390,9 +365,9 @@ async function processGroupedReunion(agenda, session, dataDir) {
|
|
|
390
365
|
if (accepted)
|
|
391
366
|
STATS.accepted++;
|
|
392
367
|
if (!options["silent"]) {
|
|
393
|
-
console.log(`[pick] ${agenda.uid} score=${best.score.toFixed(2)}
|
|
394
|
-
agenda title="${agenda.titre ?? ""}" agenda organe="${agenda.organe ?? ""}" agenda heure=${agenda.startTime}
|
|
395
|
-
best title="${best.vtitle ?? ""}" best organe="${best.vorgane ?? ""}"
|
|
368
|
+
console.log(`[pick] ${agenda.uid} score=${best.score.toFixed(2)}
|
|
369
|
+
agenda title="${agenda.titre ?? ""}" agenda organe="${agenda.organe ?? ""}" agenda heure=${agenda.startTime}
|
|
370
|
+
best title="${best.vtitle ?? ""}" best organe="${best.vorgane ?? ""}"
|
|
396
371
|
accepted=${accepted}`);
|
|
397
372
|
}
|
|
398
373
|
// ==== 3) Write metadata + NVS of the best candidate (always) ====
|
|
@@ -429,17 +404,23 @@ async function processGroupedReunion(agenda, session, dataDir) {
|
|
|
429
404
|
await fsp.writeFile(path.join(baseDir, "data.nvs"), dataTxt, "utf-8");
|
|
430
405
|
if (finalTxt)
|
|
431
406
|
await fsp.writeFile(path.join(baseDir, "finalplayer.nvs"), finalTxt, "utf-8");
|
|
432
|
-
if (dataTxt
|
|
433
|
-
master = buildSenatVodMasterM3u8FromNvs(dataTxt
|
|
407
|
+
if (dataTxt) {
|
|
408
|
+
master = buildSenatVodMasterM3u8FromNvs(dataTxt);
|
|
409
|
+
}
|
|
410
|
+
else {
|
|
411
|
+
console.log("Cannot download data nvs");
|
|
412
|
+
}
|
|
434
413
|
}
|
|
435
414
|
else {
|
|
436
415
|
// Skipped download, but need to read data.nvs for urlVideo
|
|
437
416
|
try {
|
|
438
417
|
const dataTxt = await fsp.readFile(path.join(baseDir, "data.nvs"), "utf-8");
|
|
439
418
|
const finalTxt = await fsp.readFile(path.join(baseDir, "finalplayer.nvs"), "utf-8");
|
|
440
|
-
master = buildSenatVodMasterM3u8FromNvs(dataTxt
|
|
419
|
+
master = buildSenatVodMasterM3u8FromNvs(dataTxt);
|
|
420
|
+
}
|
|
421
|
+
catch (e) {
|
|
422
|
+
console.warn(e);
|
|
441
423
|
}
|
|
442
|
-
catch { }
|
|
443
424
|
}
|
|
444
425
|
// ==== 4) Update agenda file (only if accepted + m3u8) ====
|
|
445
426
|
if ((accepted || skipDownload) && master) {
|
|
@@ -469,6 +450,9 @@ async function processGroupedReunion(agenda, session, dataDir) {
|
|
|
469
450
|
console.warn(`[warn] agenda file not found for update: ${agendaJsonPath}`);
|
|
470
451
|
}
|
|
471
452
|
}
|
|
453
|
+
else {
|
|
454
|
+
console.warn(`[warn] The video url could not be built`);
|
|
455
|
+
}
|
|
472
456
|
}
|
|
473
457
|
async function processAll(dataDir, sessions) {
|
|
474
458
|
console.log("Process all Agendas and fetch video's url");
|
package/package.json
CHANGED
|
@@ -1,101 +1,101 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "@tricoteuses/senat",
|
|
3
|
-
"version": "2.19.
|
|
4
|
-
"description": "Handle French Sénat's open data",
|
|
5
|
-
"keywords": [
|
|
6
|
-
"France",
|
|
7
|
-
"open data",
|
|
8
|
-
"Parliament",
|
|
9
|
-
"Sénat"
|
|
10
|
-
],
|
|
11
|
-
"author": "Emmanuel Raviart <emmanuel@raviart.com>",
|
|
12
|
-
"bugs": {
|
|
13
|
-
"url": "https://git.tricoteuses.fr/logiciels/tricoteuses-senat/issues"
|
|
14
|
-
},
|
|
15
|
-
"homepage": "https://tricoteuses.fr/",
|
|
16
|
-
"license": "AGPL-3.0-or-later",
|
|
17
|
-
"repository": {
|
|
18
|
-
"type": "git",
|
|
19
|
-
"url": "https://git.tricoteuses.fr/logiciels/tricoteuses-senat.git"
|
|
20
|
-
},
|
|
21
|
-
"type": "module",
|
|
22
|
-
"engines": {
|
|
23
|
-
"node": ">=22"
|
|
24
|
-
},
|
|
25
|
-
"files": [
|
|
26
|
-
"lib"
|
|
27
|
-
],
|
|
28
|
-
"exports": {
|
|
29
|
-
".": {
|
|
30
|
-
"import": "./lib/index.js",
|
|
31
|
-
"types": "./lib/index.d.ts"
|
|
32
|
-
},
|
|
33
|
-
"./loaders": {
|
|
34
|
-
"import": "./lib/loaders.js",
|
|
35
|
-
"types": "./lib/loaders.d.ts"
|
|
36
|
-
},
|
|
37
|
-
"./package.json": "./package.json"
|
|
38
|
-
},
|
|
39
|
-
"publishConfig": {
|
|
40
|
-
"access": "public"
|
|
41
|
-
},
|
|
42
|
-
"scripts": {
|
|
43
|
-
"build": "tsc",
|
|
44
|
-
"build:types": "tsc --emitDeclarationOnly",
|
|
45
|
-
"data:convert_data": "tsx src/scripts/convert_data.ts",
|
|
46
|
-
"data:download": "tsx src/scripts/data-download.ts",
|
|
47
|
-
"data:generate_schemas": "tsx src/scripts/retrieve_open_data.ts --schema",
|
|
48
|
-
"data:retrieve_agenda": "cross-env TZ='Etc/UTC' tsx src/scripts/retrieve_agenda.ts",
|
|
49
|
-
"data:retrieve_cr_seance": "tsx src/scripts/retrieve_cr_seance.ts",
|
|
50
|
-
"data:retrieve_cr_commission": "tsx src/scripts/retrieve_cr_commission.ts",
|
|
51
|
-
"data:retrieve_documents": "tsx src/scripts/retrieve_documents.ts",
|
|
52
|
-
"data:retrieve_open_data": "tsx src/scripts/retrieve_open_data.ts --all",
|
|
53
|
-
"data:retrieve_senateurs_photos": "tsx src/scripts/retrieve_senateurs_photos.ts --fetch",
|
|
54
|
-
"data:retrieve_videos": "tsx src/scripts/retrieve_videos.ts",
|
|
55
|
-
"data:parse_textes_lois": "tsx src/scripts/parse_textes.ts",
|
|
56
|
-
"prepare": "npm run build",
|
|
57
|
-
"prepublishOnly": "npm run build",
|
|
58
|
-
"prettier": "prettier --write 'src/**/*.ts' 'tests/**/*.test.ts'",
|
|
59
|
-
"test:iter_load": "tsx src/scripts/test_iter_load.ts",
|
|
60
|
-
"type-check": "tsc --noEmit",
|
|
61
|
-
"type-check:watch": "npm run type-check -- --watch"
|
|
62
|
-
},
|
|
63
|
-
"dependencies": {
|
|
64
|
-
"@biryani/core": "^0.2.1",
|
|
65
|
-
"cheerio": "^1.1.2",
|
|
66
|
-
"command-line-args": "^5.1.1",
|
|
67
|
-
"dotenv": "^8.2.0",
|
|
68
|
-
"fs-extra": "^9.1.0",
|
|
69
|
-
"jsdom": "^26.0.0",
|
|
70
|
-
"kysely": "^0.27.4",
|
|
71
|
-
"luxon": "^3.7.2",
|
|
72
|
-
"node-stream-zip": "^1.8.2",
|
|
73
|
-
"pg": "^8.13.1",
|
|
74
|
-
"pg-cursor": "^2.12.1",
|
|
75
|
-
"p-limit": "^7.2.0",
|
|
76
|
-
"slug": "^11.0.0",
|
|
77
|
-
"tsx": "^4.20.6",
|
|
78
|
-
"windows-1252": "^1.0.0"
|
|
79
|
-
},
|
|
80
|
-
"devDependencies": {
|
|
81
|
-
"@typed-code/schemats": "^5.0.1",
|
|
82
|
-
"@types/cheerio": "^1.0.0",
|
|
83
|
-
"@types/command-line-args": "^5.0.0",
|
|
84
|
-
"@types/fs-extra": "^9.0.7",
|
|
85
|
-
"@types/jsdom": "^21.1.7",
|
|
86
|
-
"@types/luxon": "^3.7.1",
|
|
87
|
-
"@types/node": "^20.17.6",
|
|
88
|
-
"@types/pg": "^8.15.5",
|
|
89
|
-
"@types/pg-cursor": "^2.7.2",
|
|
90
|
-
"@types/slug": "^5.0.9",
|
|
91
|
-
"@typescript-eslint/eslint-plugin": "^8.46.0",
|
|
92
|
-
"@typescript-eslint/parser": "^8.46.0",
|
|
93
|
-
"cross-env": "^10.1.0",
|
|
94
|
-
"eslint": "^8.57.1",
|
|
95
|
-
"iconv-lite": "^0.7.0",
|
|
96
|
-
"kysely-codegen": "^0.19.0",
|
|
97
|
-
"prettier": "^3.5.3",
|
|
98
|
-
"tslib": "^2.1.0",
|
|
99
|
-
"typescript": "^5.9.3"
|
|
100
|
-
}
|
|
101
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"name": "@tricoteuses/senat",
|
|
3
|
+
"version": "2.19.3",
|
|
4
|
+
"description": "Handle French Sénat's open data",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"France",
|
|
7
|
+
"open data",
|
|
8
|
+
"Parliament",
|
|
9
|
+
"Sénat"
|
|
10
|
+
],
|
|
11
|
+
"author": "Emmanuel Raviart <emmanuel@raviart.com>",
|
|
12
|
+
"bugs": {
|
|
13
|
+
"url": "https://git.tricoteuses.fr/logiciels/tricoteuses-senat/issues"
|
|
14
|
+
},
|
|
15
|
+
"homepage": "https://tricoteuses.fr/",
|
|
16
|
+
"license": "AGPL-3.0-or-later",
|
|
17
|
+
"repository": {
|
|
18
|
+
"type": "git",
|
|
19
|
+
"url": "https://git.tricoteuses.fr/logiciels/tricoteuses-senat.git"
|
|
20
|
+
},
|
|
21
|
+
"type": "module",
|
|
22
|
+
"engines": {
|
|
23
|
+
"node": ">=22"
|
|
24
|
+
},
|
|
25
|
+
"files": [
|
|
26
|
+
"lib"
|
|
27
|
+
],
|
|
28
|
+
"exports": {
|
|
29
|
+
".": {
|
|
30
|
+
"import": "./lib/index.js",
|
|
31
|
+
"types": "./lib/index.d.ts"
|
|
32
|
+
},
|
|
33
|
+
"./loaders": {
|
|
34
|
+
"import": "./lib/loaders.js",
|
|
35
|
+
"types": "./lib/loaders.d.ts"
|
|
36
|
+
},
|
|
37
|
+
"./package.json": "./package.json"
|
|
38
|
+
},
|
|
39
|
+
"publishConfig": {
|
|
40
|
+
"access": "public"
|
|
41
|
+
},
|
|
42
|
+
"scripts": {
|
|
43
|
+
"build": "tsc",
|
|
44
|
+
"build:types": "tsc --emitDeclarationOnly",
|
|
45
|
+
"data:convert_data": "tsx src/scripts/convert_data.ts",
|
|
46
|
+
"data:download": "tsx src/scripts/data-download.ts",
|
|
47
|
+
"data:generate_schemas": "tsx src/scripts/retrieve_open_data.ts --schema",
|
|
48
|
+
"data:retrieve_agenda": "cross-env TZ='Etc/UTC' tsx src/scripts/retrieve_agenda.ts",
|
|
49
|
+
"data:retrieve_cr_seance": "tsx src/scripts/retrieve_cr_seance.ts",
|
|
50
|
+
"data:retrieve_cr_commission": "tsx src/scripts/retrieve_cr_commission.ts",
|
|
51
|
+
"data:retrieve_documents": "tsx src/scripts/retrieve_documents.ts",
|
|
52
|
+
"data:retrieve_open_data": "tsx src/scripts/retrieve_open_data.ts --all",
|
|
53
|
+
"data:retrieve_senateurs_photos": "tsx src/scripts/retrieve_senateurs_photos.ts --fetch",
|
|
54
|
+
"data:retrieve_videos": "tsx src/scripts/retrieve_videos.ts",
|
|
55
|
+
"data:parse_textes_lois": "tsx src/scripts/parse_textes.ts",
|
|
56
|
+
"prepare": "npm run build",
|
|
57
|
+
"prepublishOnly": "npm run build",
|
|
58
|
+
"prettier": "prettier --write 'src/**/*.ts' 'tests/**/*.test.ts'",
|
|
59
|
+
"test:iter_load": "tsx src/scripts/test_iter_load.ts",
|
|
60
|
+
"type-check": "tsc --noEmit",
|
|
61
|
+
"type-check:watch": "npm run type-check -- --watch"
|
|
62
|
+
},
|
|
63
|
+
"dependencies": {
|
|
64
|
+
"@biryani/core": "^0.2.1",
|
|
65
|
+
"cheerio": "^1.1.2",
|
|
66
|
+
"command-line-args": "^5.1.1",
|
|
67
|
+
"dotenv": "^8.2.0",
|
|
68
|
+
"fs-extra": "^9.1.0",
|
|
69
|
+
"jsdom": "^26.0.0",
|
|
70
|
+
"kysely": "^0.27.4",
|
|
71
|
+
"luxon": "^3.7.2",
|
|
72
|
+
"node-stream-zip": "^1.8.2",
|
|
73
|
+
"pg": "^8.13.1",
|
|
74
|
+
"pg-cursor": "^2.12.1",
|
|
75
|
+
"p-limit": "^7.2.0",
|
|
76
|
+
"slug": "^11.0.0",
|
|
77
|
+
"tsx": "^4.20.6",
|
|
78
|
+
"windows-1252": "^1.0.0"
|
|
79
|
+
},
|
|
80
|
+
"devDependencies": {
|
|
81
|
+
"@typed-code/schemats": "^5.0.1",
|
|
82
|
+
"@types/cheerio": "^1.0.0",
|
|
83
|
+
"@types/command-line-args": "^5.0.0",
|
|
84
|
+
"@types/fs-extra": "^9.0.7",
|
|
85
|
+
"@types/jsdom": "^21.1.7",
|
|
86
|
+
"@types/luxon": "^3.7.1",
|
|
87
|
+
"@types/node": "^20.17.6",
|
|
88
|
+
"@types/pg": "^8.15.5",
|
|
89
|
+
"@types/pg-cursor": "^2.7.2",
|
|
90
|
+
"@types/slug": "^5.0.9",
|
|
91
|
+
"@typescript-eslint/eslint-plugin": "^8.46.0",
|
|
92
|
+
"@typescript-eslint/parser": "^8.46.0",
|
|
93
|
+
"cross-env": "^10.1.0",
|
|
94
|
+
"eslint": "^8.57.1",
|
|
95
|
+
"iconv-lite": "^0.7.0",
|
|
96
|
+
"kysely-codegen": "^0.19.0",
|
|
97
|
+
"prettier": "^3.5.3",
|
|
98
|
+
"tslib": "^2.1.0",
|
|
99
|
+
"typescript": "^5.9.3"
|
|
100
|
+
}
|
|
101
|
+
}
|