@tricoteuses/assemblee 3.2.15 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +38 -38
- package/README.md +153 -153
- package/lib/cleaners.js +44 -36
- package/lib/index.d.ts +1 -0
- package/lib/index.js +20 -20
- package/lib/{loaders-Bqy18Qs1.js → loaders-DbLMTnJA.js} +152 -152
- package/lib/loaders.js +1 -1
- package/lib/open_data/artifacts.d.ts +38 -0
- package/lib/parsers/index.d.ts +2 -0
- package/lib/parsers/plf/credits.d.ts +2 -0
- package/lib/parsers/plf/credits.test.d.ts +1 -0
- package/lib/parsers/plf/depot.d.ts +2 -0
- package/lib/parsers/plf/depot.test.d.ts +1 -0
- package/lib/parsers/plf/index.d.ts +3 -0
- package/lib/parsers/plf/integration.test.d.ts +1 -0
- package/lib/parsers/plf/later_stage.d.ts +2 -0
- package/lib/parsers/plf/later_stage.test.d.ts +1 -0
- package/lib/parsers/plf/parser_utils.d.ts +11 -0
- package/lib/parsers/plf/test_utils.d.ts +18 -0
- package/lib/parsers/plf/variant_detection.d.ts +2 -0
- package/lib/parsers/plf/variant_detection.test.d.ts +1 -0
- package/lib/parsers/utils.d.ts +7 -0
- package/lib/parsers.js +3726 -3326
- package/lib/schemas/clean/dossiers_legislatifs.d.ts +97 -0
- package/lib/schemas/enriched/dossiers_legislatifs.d.ts +6 -0
- package/lib/scripts/monitor_open_data_updates.d.ts +2 -0
- package/lib/scripts/retrieve_open_data.d.ts +1 -0
- package/lib/{scrutins-D1qWLlRT.js → scrutins-CcBKz4TE.js} +156 -121
- package/package.json +135 -135
package/LICENSE.md
CHANGED
|
@@ -1,38 +1,38 @@
|
|
|
1
|
-
# Tricoteuses-Assemblee
|
|
2
|
-
|
|
3
|
-
## _Retrieve, clean up & handle French Assemblée nationale's open data_
|
|
4
|
-
|
|
5
|
-
By:
|
|
6
|
-
|
|
7
|
-
- Henry Boisgibault <mailto:henry.boisgibault@proton.me>
|
|
8
|
-
- Jérémie Corpinot <mailto:jerem.freelance@codingbox.fr>
|
|
9
|
-
- Mathilde Couvreur <mailto:mathilde.couvreur@gmail.com>
|
|
10
|
-
- Loïc Dachary <mailto:loic@dachary.org>
|
|
11
|
-
- Pierre Drege <mailto:dregop@proton.me>
|
|
12
|
-
- Nelson Herbin <mailto:nelson@herbin.info>
|
|
13
|
-
- implicitdef
|
|
14
|
-
- Hélène Jonin <mailto:helene.jonin@gmail.com>
|
|
15
|
-
- Pauline Léon <mailto:paulineleon@tutanota.com>
|
|
16
|
-
- Paul-Henry Ngounou <mailto:ngpaulhenry@gmail.com>
|
|
17
|
-
- Emmanuel Raviart <mailto:emmanuel@raviart.com>
|
|
18
|
-
|
|
19
|
-
Copyright:
|
|
20
|
-
|
|
21
|
-
- © 2019, 2020, 2021 Loïc Dachary & Emmanuel Raviart
|
|
22
|
-
- © 2022 Emmanuel Raviart
|
|
23
|
-
- © 2023, 2024, 2025, 2026 Logora & Emmanuel Raviart
|
|
24
|
-
|
|
25
|
-
https://git.tricoteuses.fr/logiciels/tricoteuses-assemblee
|
|
26
|
-
|
|
27
|
-
> Tricoteuses-Assemblee is free software; you can redistribute it and/or modify
|
|
28
|
-
> it under the terms of the GNU Affero General Public License as
|
|
29
|
-
> published by the Free Software Foundation, either version 3 of the
|
|
30
|
-
> License, or (at your option) any later version.
|
|
31
|
-
>
|
|
32
|
-
> Tricoteuses-Assemblee is distributed in the hope that it will be useful,
|
|
33
|
-
> but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
34
|
-
> MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
35
|
-
> GNU Affero General Public License for more details.
|
|
36
|
-
>
|
|
37
|
-
> You should have received a copy of the GNU Affero General Public License
|
|
38
|
-
> along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
1
|
+
# Tricoteuses-Assemblee
|
|
2
|
+
|
|
3
|
+
## _Retrieve, clean up & handle French Assemblée nationale's open data_
|
|
4
|
+
|
|
5
|
+
By:
|
|
6
|
+
|
|
7
|
+
- Henry Boisgibault <mailto:henry.boisgibault@proton.me>
|
|
8
|
+
- Jérémie Corpinot <mailto:jerem.freelance@codingbox.fr>
|
|
9
|
+
- Mathilde Couvreur <mailto:mathilde.couvreur@gmail.com>
|
|
10
|
+
- Loïc Dachary <mailto:loic@dachary.org>
|
|
11
|
+
- Pierre Drege <mailto:dregop@proton.me>
|
|
12
|
+
- Nelson Herbin <mailto:nelson@herbin.info>
|
|
13
|
+
- implicitdef
|
|
14
|
+
- Hélène Jonin <mailto:helene.jonin@gmail.com>
|
|
15
|
+
- Pauline Léon <mailto:paulineleon@tutanota.com>
|
|
16
|
+
- Paul-Henry Ngounou <mailto:ngpaulhenry@gmail.com>
|
|
17
|
+
- Emmanuel Raviart <mailto:emmanuel@raviart.com>
|
|
18
|
+
|
|
19
|
+
Copyright:
|
|
20
|
+
|
|
21
|
+
- © 2019, 2020, 2021 Loïc Dachary & Emmanuel Raviart
|
|
22
|
+
- © 2022 Emmanuel Raviart
|
|
23
|
+
- © 2023, 2024, 2025, 2026 Logora & Emmanuel Raviart
|
|
24
|
+
|
|
25
|
+
https://git.tricoteuses.fr/logiciels/tricoteuses-assemblee
|
|
26
|
+
|
|
27
|
+
> Tricoteuses-Assemblee is free software; you can redistribute it and/or modify
|
|
28
|
+
> it under the terms of the GNU Affero General Public License as
|
|
29
|
+
> published by the Free Software Foundation, either version 3 of the
|
|
30
|
+
> License, or (at your option) any later version.
|
|
31
|
+
>
|
|
32
|
+
> Tricoteuses-Assemblee is distributed in the hope that it will be useful,
|
|
33
|
+
> but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
34
|
+
> MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
35
|
+
> GNU Affero General Public License for more details.
|
|
36
|
+
>
|
|
37
|
+
> You should have received a copy of the GNU Affero General Public License
|
|
38
|
+
> along with this program. If not, see <http://www.gnu.org/licenses/>.
|
package/README.md
CHANGED
|
@@ -1,153 +1,153 @@
|
|
|
1
|
-
# Tricoteuses-Assemblee
|
|
2
|
-
|
|
3
|
-
## _Retrieve, clean up & handle French Assemblée nationale's open data_
|
|
4
|
-
|
|
5
|
-
_Tricoteuses Légifrance_ is free and open source software.
|
|
6
|
-
|
|
7
|
-
- [software repository](https://git.tricoteuses.fr/logiciels/tricoteuses-assemblee)
|
|
8
|
-
- [GNU Affero General Public License version 3 or greater](https://git.tricoteuses.fr/logiciels/tricoteuses-assemblee/-/tree/master/LICENSE.md)
|
|
9
|
-
|
|
10
|
-
## documentation
|
|
11
|
-
|
|
12
|
-
- [Architecture](doc/architecture.md)
|
|
13
|
-
- [Browser Usage](doc/BROWSER_USAGE.md) - Using this package in browser/Vite projects
|
|
14
|
-
|
|
15
|
-
## Installation
|
|
16
|
-
|
|
17
|
-
```bash
|
|
18
|
-
git clone https://git.tricoteuses.fr/logiciels/tricoteuses-assemblee
|
|
19
|
-
cd tricoteuses-assemblee/
|
|
20
|
-
```
|
|
21
|
-
|
|
22
|
-
```bash
|
|
23
|
-
npm install
|
|
24
|
-
```
|
|
25
|
-
|
|
26
|
-
## Download and clean data
|
|
27
|
-
|
|
28
|
-
### Basic usage
|
|
29
|
-
|
|
30
|
-
Create a directory to store the data, then run the following command to download, reorganize and clean the data.
|
|
31
|
-
|
|
32
|
-
```bash
|
|
33
|
-
mkdir ../assemblee-data/
|
|
34
|
-
npm run data:download ../assemblee-data
|
|
35
|
-
```
|
|
36
|
-
|
|
37
|
-
### Available Commands
|
|
38
|
-
|
|
39
|
-
- `npm run data:download <dir>`: Download, reorganize, and clean data
|
|
40
|
-
- `npm run data:retrieve_open_data <dir>`: Download raw data files.
|
|
41
|
-
- `npm run data:reorganize_data <dir>`: Reorganize raw files by entity.
|
|
42
|
-
- `npm run data:clean_data <dir>`: Clean and validate reorganized files.
|
|
43
|
-
- `npm run data:retrieve_deputes_photos <dir>`: Retrieval of députés' pictures from Assemblée nationale's website
|
|
44
|
-
- `npm run data:retrieve_senateurs_photos <dir>`: Retrieval of sénateurs' pictures from Assemblée nationale's website
|
|
45
|
-
- `npm run data:retrieve_documents <dir>`: Retrieval of legislative documents from Assemblée nationale's website
|
|
46
|
-
- `npm run data:retrieve_pending_amendements <dir>`: Retrieval of pending amendments from Assemblée nationale's website (waiting to be processed by Assemblée services)
|
|
47
|
-
|
|
48
|
-
_Notes_:
|
|
49
|
-
|
|
50
|
-
- Reorganized files (generated by the _data:reorganize_data_ command) are also available in [Tricoteuses / Data / Données brutes de l'Assemblée](https://git.en-root.org/tricoteuses/data/assemblee-brut). They are updated on a regular basis.
|
|
51
|
-
- Split & cleaned files (generated by the _data:clean_data_ command) are also available in [Tricoteuses / Data / Données nettoyées de l'Assemblée](https://git.en-root.org/tricoteuses/data/assemblee-nettoye) with the `_nettoye` suffix. They are updated on a regular basis.
|
|
52
|
-
|
|
53
|
-
### Filtering Options
|
|
54
|
-
|
|
55
|
-
Downloading and cleaning all the data is long and takes up a lot of disk space. It is possible to choose the type of data that you want to retrieve to reduce the load.
|
|
56
|
-
|
|
57
|
-
Examples:
|
|
58
|
-
|
|
59
|
-
```bash
|
|
60
|
-
# Only download amendments
|
|
61
|
-
npm run data:download ../assemblee-data -- -k Amendements
|
|
62
|
-
|
|
63
|
-
# Only process 16th and 17th legislatures
|
|
64
|
-
npm run data:download ../assemblee-data -- -l 16 -l 17
|
|
65
|
-
|
|
66
|
-
# Retrieve comptes rendus de seance and commissions for one legislature
|
|
67
|
-
npm run data:retrieve_open_data ../assemblee-data -- --categories ComptesRendus --legislature 17 --fetchCrCommissions
|
|
68
|
-
```
|
|
69
|
-
|
|
70
|
-
### Common Options
|
|
71
|
-
|
|
72
|
-
- `--categories` or `-k <name>`: Filter by dataset categories (Available options : `ActeursEtOrganes`, `Agendas`, `Amendements`, `DossiersLegislatifs`, `Photos`, `Scrutins`, `Questions`, `ComptesRendus`)
|
|
73
|
-
|
|
74
|
-
- `--legislature` or `-l <number>`: Specify one or more legislatures to process (e.g., `-l 15 -l 16`)
|
|
75
|
-
- `--dataDir <path>` (Mandatory): Path to the working directory where all data is stored (required)
|
|
76
|
-
- `--silent` or `-s`: Disable logging
|
|
77
|
-
- `--verbose` or `-v`: Enable verbose logging
|
|
78
|
-
- `--fetch` or `-f`: Force re-download of data even if already present
|
|
79
|
-
- `--commit` or `-c`: Automatically commit cleaned data
|
|
80
|
-
- `--pull` or `-p`: Pull repositories before starting
|
|
81
|
-
- `--clone` or `-C <url>`: Clone Git repositories from a remote group or organization
|
|
82
|
-
- `--remote` or `-r <name>`: Push commits to specified Git remote(s)
|
|
83
|
-
- `--keepDir`: Keep Dir (Implement before cleaning data)
|
|
84
|
-
- `--only-recent` (number): If files are already present, skip files that are above the specified number of days and skip old legislatures (e.g. `-only-recent 30`)
|
|
85
|
-
|
|
86
|
-
If you use such options, use them in all subsequent commands too (_data:regorganize_data_ and _data:clean_data_).
|
|
87
|
-
|
|
88
|
-
### Options for Data Retrieval
|
|
89
|
-
|
|
90
|
-
With `data:retrieve_open_data`,
|
|
91
|
-
|
|
92
|
-
- use `--categories=ComptesRendus` to retrieve comptes rendus de séance for the selected legislature(s).
|
|
93
|
-
|
|
94
|
-
Note:
|
|
95
|
-
|
|
96
|
-
- Comptes-rendus & videos of commissions are retrieved when cleaning agendas data.
|
|
97
|
-
|
|
98
|
-
### Options for Cleaning Data
|
|
99
|
-
|
|
100
|
-
- `--dataset` or `-d <name>`: Clean a specific dataset only
|
|
101
|
-
- `--fetchCrCommissions`: Retrieve and parse CR commissions
|
|
102
|
-
- `--fetchVideos`: Retrieve videos
|
|
103
|
-
- `--fetchDocuments` : Specify to retrieve documents
|
|
104
|
-
- `--no-reset-after-commit`: Skip Git reset after committing (useful to preserve local changes)
|
|
105
|
-
- `--no-validate` or `-V`: Skip schema validation during cleaning
|
|
106
|
-
- `--parseDocuments`: Specify to parse documents into cleaned json
|
|
107
|
-
|
|
108
|
-
Note:
|
|
109
|
-
|
|
110
|
-
- use `--categories=Agendas` together with `--fetchCrCommissions` to retrieve comptes rendus de commission.
|
|
111
|
-
- use `--categories=Agendas` together with `--fetchVideos` to retrieve videos of commissions.
|
|
112
|
-
|
|
113
|
-
### Options for Retrieving Documents
|
|
114
|
-
|
|
115
|
-
- `--full` or `-f`: Retrieve all documents, even those already downloaded
|
|
116
|
-
- `--document-type` or `-T <type>`: Restrict to specific document types (e.g., `PION`)
|
|
117
|
-
|
|
118
|
-
## Download using Docker
|
|
119
|
-
|
|
120
|
-
A Docker image that downloads and cleans the data all at once is available. Build it locally or run it from the container registry.
|
|
121
|
-
Use the environment variables `LEGISLATURE` and `CATEGORIES` if needed.
|
|
122
|
-
|
|
123
|
-
```bash
|
|
124
|
-
docker run --pull always --name tricoteuses-assemblee -v ../assemblee-data:/app/assemblee-data -e LEGISLATURE=17 -d git.tricoteuses.fr/logiciels/tricoteuses-assemblee:latest
|
|
125
|
-
```
|
|
126
|
-
|
|
127
|
-
## Using the data
|
|
128
|
-
|
|
129
|
-
Once the data is downloaded and cleaned, you can use loaders to retrieve it.
|
|
130
|
-
To use loaders in your project, you can install the _@tricoteuses/assemblee_ package, and import the iterator functions that you need.
|
|
131
|
-
|
|
132
|
-
```bash
|
|
133
|
-
npm install @tricoteuses/assemblee
|
|
134
|
-
```
|
|
135
|
-
|
|
136
|
-
```js
|
|
137
|
-
import {
|
|
138
|
-
iterLoadAssembleeActeurs,
|
|
139
|
-
iterLoadAssembleeOrganes,
|
|
140
|
-
iterLoadAssembleeReunions,
|
|
141
|
-
iterLoadAssembleeScrutins,
|
|
142
|
-
iterLoadAssembleeDocuments,
|
|
143
|
-
iterLoadAssembleeDossiersParlementaires,
|
|
144
|
-
iterLoadAssembleeAmendements,
|
|
145
|
-
iterLoadAssembleeQuestions,
|
|
146
|
-
iterLoadAssembleeComptesRendus,
|
|
147
|
-
} from "@tricoteuses/assemblee/loaders"
|
|
148
|
-
|
|
149
|
-
// Pass data directory and legislature as arguments
|
|
150
|
-
for (const { acteur } of iterLoadAssembleeActeurs("../assemblee-data", 17)) {
|
|
151
|
-
console.log(acteur.uid)
|
|
152
|
-
}
|
|
153
|
-
```
|
|
1
|
+
# Tricoteuses-Assemblee
|
|
2
|
+
|
|
3
|
+
## _Retrieve, clean up & handle French Assemblée nationale's open data_
|
|
4
|
+
|
|
5
|
+
_Tricoteuses Légifrance_ is free and open source software.
|
|
6
|
+
|
|
7
|
+
- [software repository](https://git.tricoteuses.fr/logiciels/tricoteuses-assemblee)
|
|
8
|
+
- [GNU Affero General Public License version 3 or greater](https://git.tricoteuses.fr/logiciels/tricoteuses-assemblee/-/tree/master/LICENSE.md)
|
|
9
|
+
|
|
10
|
+
## documentation
|
|
11
|
+
|
|
12
|
+
- [Architecture](doc/architecture.md)
|
|
13
|
+
- [Browser Usage](doc/BROWSER_USAGE.md) - Using this package in browser/Vite projects
|
|
14
|
+
|
|
15
|
+
## Installation
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
git clone https://git.tricoteuses.fr/logiciels/tricoteuses-assemblee
|
|
19
|
+
cd tricoteuses-assemblee/
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
npm install
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Download and clean data
|
|
27
|
+
|
|
28
|
+
### Basic usage
|
|
29
|
+
|
|
30
|
+
Create a directory to store the data, then run the following command to download, reorganize and clean the data.
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
mkdir ../assemblee-data/
|
|
34
|
+
npm run data:download ../assemblee-data
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
### Available Commands
|
|
38
|
+
|
|
39
|
+
- `npm run data:download <dir>`: Download, reorganize, and clean data
|
|
40
|
+
- `npm run data:retrieve_open_data <dir>`: Download raw data files.
|
|
41
|
+
- `npm run data:reorganize_data <dir>`: Reorganize raw files by entity.
|
|
42
|
+
- `npm run data:clean_data <dir>`: Clean and validate reorganized files.
|
|
43
|
+
- `npm run data:retrieve_deputes_photos <dir>`: Retrieval of députés' pictures from Assemblée nationale's website
|
|
44
|
+
- `npm run data:retrieve_senateurs_photos <dir>`: Retrieval of sénateurs' pictures from Assemblée nationale's website
|
|
45
|
+
- `npm run data:retrieve_documents <dir>`: Retrieval of legislative documents from Assemblée nationale's website
|
|
46
|
+
- `npm run data:retrieve_pending_amendements <dir>`: Retrieval of pending amendments from Assemblée nationale's website (waiting to be processed by Assemblée services)
|
|
47
|
+
|
|
48
|
+
_Notes_:
|
|
49
|
+
|
|
50
|
+
- Reorganized files (generated by the _data:reorganize_data_ command) are also available in [Tricoteuses / Data / Données brutes de l'Assemblée](https://git.en-root.org/tricoteuses/data/assemblee-brut). They are updated on a regular basis.
|
|
51
|
+
- Split & cleaned files (generated by the _data:clean_data_ command) are also available in [Tricoteuses / Data / Données nettoyées de l'Assemblée](https://git.en-root.org/tricoteuses/data/assemblee-nettoye) with the `_nettoye` suffix. They are updated on a regular basis.
|
|
52
|
+
|
|
53
|
+
### Filtering Options
|
|
54
|
+
|
|
55
|
+
Downloading and cleaning all the data is long and takes up a lot of disk space. It is possible to choose the type of data that you want to retrieve to reduce the load.
|
|
56
|
+
|
|
57
|
+
Examples:
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
# Only download amendments
|
|
61
|
+
npm run data:download ../assemblee-data -- -k Amendements
|
|
62
|
+
|
|
63
|
+
# Only process 16th and 17th legislatures
|
|
64
|
+
npm run data:download ../assemblee-data -- -l 16 -l 17
|
|
65
|
+
|
|
66
|
+
# Retrieve comptes rendus de seance and commissions for one legislature
|
|
67
|
+
npm run data:retrieve_open_data ../assemblee-data -- --categories ComptesRendus --legislature 17 --fetchCrCommissions
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### Common Options
|
|
71
|
+
|
|
72
|
+
- `--categories` or `-k <name>`: Filter by dataset categories (Available options : `ActeursEtOrganes`, `Agendas`, `Amendements`, `DossiersLegislatifs`, `Photos`, `Scrutins`, `Questions`, `ComptesRendus`)
|
|
73
|
+
|
|
74
|
+
- `--legislature` or `-l <number>`: Specify one or more legislatures to process (e.g., `-l 15 -l 16`)
|
|
75
|
+
- `--dataDir <path>` (Mandatory): Path to the working directory where all data is stored (required)
|
|
76
|
+
- `--silent` or `-s`: Disable logging
|
|
77
|
+
- `--verbose` or `-v`: Enable verbose logging
|
|
78
|
+
- `--fetch` or `-f`: Force re-download of data even if already present
|
|
79
|
+
- `--commit` or `-c`: Automatically commit cleaned data
|
|
80
|
+
- `--pull` or `-p`: Pull repositories before starting
|
|
81
|
+
- `--clone` or `-C <url>`: Clone Git repositories from a remote group or organization
|
|
82
|
+
- `--remote` or `-r <name>`: Push commits to specified Git remote(s)
|
|
83
|
+
- `--keepDir`: Keep Dir (Implement before cleaning data)
|
|
84
|
+
- `--only-recent` (number): If files are already present, skip files that are above the specified number of days and skip old legislatures (e.g. `-only-recent 30`)
|
|
85
|
+
|
|
86
|
+
If you use such options, use them in all subsequent commands too (_data:regorganize_data_ and _data:clean_data_).
|
|
87
|
+
|
|
88
|
+
### Options for Data Retrieval
|
|
89
|
+
|
|
90
|
+
With `data:retrieve_open_data`,
|
|
91
|
+
|
|
92
|
+
- use `--categories=ComptesRendus` to retrieve comptes rendus de séance for the selected legislature(s).
|
|
93
|
+
|
|
94
|
+
Note:
|
|
95
|
+
|
|
96
|
+
- Comptes-rendus & videos of commissions are retrieved when cleaning agendas data.
|
|
97
|
+
|
|
98
|
+
### Options for Cleaning Data
|
|
99
|
+
|
|
100
|
+
- `--dataset` or `-d <name>`: Clean a specific dataset only
|
|
101
|
+
- `--fetchCrCommissions`: Retrieve and parse CR commissions
|
|
102
|
+
- `--fetchVideos`: Retrieve videos
|
|
103
|
+
- `--fetchDocuments` : Specify to retrieve documents
|
|
104
|
+
- `--no-reset-after-commit`: Skip Git reset after committing (useful to preserve local changes)
|
|
105
|
+
- `--no-validate` or `-V`: Skip schema validation during cleaning
|
|
106
|
+
- `--parseDocuments`: Specify to parse documents into cleaned json
|
|
107
|
+
|
|
108
|
+
Note:
|
|
109
|
+
|
|
110
|
+
- use `--categories=Agendas` together with `--fetchCrCommissions` to retrieve comptes rendus de commission.
|
|
111
|
+
- use `--categories=Agendas` together with `--fetchVideos` to retrieve videos of commissions.
|
|
112
|
+
|
|
113
|
+
### Options for Retrieving Documents
|
|
114
|
+
|
|
115
|
+
- `--full` or `-f`: Retrieve all documents, even those already downloaded
|
|
116
|
+
- `--document-type` or `-T <type>`: Restrict to specific document types (e.g., `PION`)
|
|
117
|
+
|
|
118
|
+
## Download using Docker
|
|
119
|
+
|
|
120
|
+
A Docker image that downloads and cleans the data all at once is available. Build it locally or run it from the container registry.
|
|
121
|
+
Use the environment variables `LEGISLATURE` and `CATEGORIES` if needed.
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
docker run --pull always --name tricoteuses-assemblee -v ../assemblee-data:/app/assemblee-data -e LEGISLATURE=17 -d git.tricoteuses.fr/logiciels/tricoteuses-assemblee:latest
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
## Using the data
|
|
128
|
+
|
|
129
|
+
Once the data is downloaded and cleaned, you can use loaders to retrieve it.
|
|
130
|
+
To use loaders in your project, you can install the _@tricoteuses/assemblee_ package, and import the iterator functions that you need.
|
|
131
|
+
|
|
132
|
+
```bash
|
|
133
|
+
npm install @tricoteuses/assemblee
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
```js
|
|
137
|
+
import {
|
|
138
|
+
iterLoadAssembleeActeurs,
|
|
139
|
+
iterLoadAssembleeOrganes,
|
|
140
|
+
iterLoadAssembleeReunions,
|
|
141
|
+
iterLoadAssembleeScrutins,
|
|
142
|
+
iterLoadAssembleeDocuments,
|
|
143
|
+
iterLoadAssembleeDossiersParlementaires,
|
|
144
|
+
iterLoadAssembleeAmendements,
|
|
145
|
+
iterLoadAssembleeQuestions,
|
|
146
|
+
iterLoadAssembleeComptesRendus,
|
|
147
|
+
} from "@tricoteuses/assemblee/loaders"
|
|
148
|
+
|
|
149
|
+
// Pass data directory and legislature as arguments
|
|
150
|
+
for (const { acteur } of iterLoadAssembleeActeurs("../assemblee-data", 17)) {
|
|
151
|
+
console.log(acteur.uid)
|
|
152
|
+
}
|
|
153
|
+
```
|
package/lib/cleaners.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { At as e, cr as t, tr as n } from "./scrutins-CcBKz4TE.js";
|
|
2
2
|
import { i as r, n as i, r as a, t as o } from "./amendements-C3RgTrF-.js";
|
|
3
|
-
import "./loaders-
|
|
3
|
+
import "./loaders-DbLMTnJA.js";
|
|
4
4
|
import s from "node:assert";
|
|
5
5
|
import "fs-extra";
|
|
6
6
|
import "node:path";
|
|
@@ -46,40 +46,48 @@ function c(e) {
|
|
|
46
46
|
if (l !== void 0) {
|
|
47
47
|
let e = [], n = Array.isArray(l) ? l : l.mandat;
|
|
48
48
|
Array.isArray(n) ? e = t.mandats = n : typeof n == "object" && (e = t.mandats = [n]);
|
|
49
|
-
for (let
|
|
50
|
-
r(t);
|
|
51
|
-
let e = t["@xsi:type"];
|
|
52
|
-
e !== void 0 && (t.xsiType = e, delete t["@xsi:type"]);
|
|
53
|
-
let n = t.infosQualite;
|
|
49
|
+
for (let n of e) {
|
|
54
50
|
r(n);
|
|
55
|
-
let
|
|
56
|
-
|
|
57
|
-
let
|
|
58
|
-
|
|
59
|
-
let
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
51
|
+
let e = n["@xsi:type"];
|
|
52
|
+
e !== void 0 && (n.xsiType = e, delete n["@xsi:type"]);
|
|
53
|
+
let i = n.infosQualite;
|
|
54
|
+
r(i);
|
|
55
|
+
let a = n.organes;
|
|
56
|
+
s(a), delete n.organes;
|
|
57
|
+
let o = a.organeRef;
|
|
58
|
+
s(o), Array.isArray(o) || (o = [o]), n.organesRefs = o;
|
|
59
|
+
let c = n.suppleants;
|
|
60
|
+
if (delete n.suppleants, c) {
|
|
61
|
+
let e = c.suppleant;
|
|
62
|
+
s(e), n.suppleant = e, r(e);
|
|
63
63
|
}
|
|
64
|
-
let
|
|
65
|
-
if (
|
|
66
|
-
r(
|
|
67
|
-
let e =
|
|
64
|
+
let l = n.election;
|
|
65
|
+
if (l !== void 0) {
|
|
66
|
+
r(l);
|
|
67
|
+
let e = l.lieu;
|
|
68
68
|
r(e);
|
|
69
69
|
}
|
|
70
|
-
let
|
|
71
|
-
|
|
72
|
-
let
|
|
73
|
-
if (
|
|
74
|
-
let e = Array.isArray(
|
|
75
|
-
e.map(({ collaborateur: e }) => {
|
|
76
|
-
if (Array.isArray(e)) for (let t of e) t !== null &&
|
|
77
|
-
else e &&
|
|
78
|
-
}),
|
|
79
|
-
|
|
70
|
+
let u = n.mandature;
|
|
71
|
+
u !== void 0 && r(u), s.strictEqual(n.chambre, void 0);
|
|
72
|
+
let d = n.collaborateurs;
|
|
73
|
+
if (d != null) {
|
|
74
|
+
let e = Array.isArray(d) ? d.filter((e) => e !== null) : [d], i = [];
|
|
75
|
+
if (e.map(({ collaborateur: e }) => {
|
|
76
|
+
if (Array.isArray(e)) for (let t of e) t !== null && i.push(t);
|
|
77
|
+
else e && i.push(e);
|
|
78
|
+
}), i.length === 0) delete n.collaborateurs;
|
|
79
|
+
else {
|
|
80
|
+
let e = /* @__PURE__ */ new Map(), a = [], o = 0;
|
|
81
|
+
for (let r of i) {
|
|
82
|
+
let i = r.nom, s = r.prenom, c = r.qualite, l = `${i || ""}_${s || ""}`;
|
|
83
|
+
e.has(l) ? (o++, console.warn(`[Dublicate COLLABORATEUR] Acteur="${t.uid}" | Mandat="${n.uid}" | Nom="${i}" Prénom="${s}" | Qualité="${c || ""}" | → Skipping (${o} in this mandat)`)) : (e.set(l, r), a.push(r));
|
|
84
|
+
}
|
|
85
|
+
n.collaborateurs = a;
|
|
86
|
+
for (let e of a) r(e), s.strictEqual(e.dateDebut, void 0), s.strictEqual(e.dateFin, void 0);
|
|
87
|
+
}
|
|
80
88
|
}
|
|
81
|
-
let
|
|
82
|
-
|
|
89
|
+
let f = n.InfosHorsSIAN;
|
|
90
|
+
f !== void 0 && (r(f), s.strictEqual(f.HATVP_URI, void 0), s.strictEqual(Object.keys(f).length, 0), delete n.InfosHorsSIAN);
|
|
83
91
|
}
|
|
84
92
|
e.sort((e, t) => e.uid.length === t.uid.length ? e.uid.localeCompare(t.uid) : e.uid.length - t.uid.length);
|
|
85
93
|
}
|
|
@@ -600,18 +608,18 @@ function D(e) {
|
|
|
600
608
|
//#endregion
|
|
601
609
|
//#region src/cleaners/questions.ts
|
|
602
610
|
function O(e) {
|
|
603
|
-
let
|
|
604
|
-
return r(
|
|
611
|
+
let t = n.parse(e);
|
|
612
|
+
return r(t, { recursive: !0 }), t;
|
|
605
613
|
}
|
|
606
614
|
//#endregion
|
|
607
615
|
//#region src/cleaners/debats.ts
|
|
608
|
-
function k(
|
|
609
|
-
let i =
|
|
616
|
+
function k(n) {
|
|
617
|
+
let i = t.safeParse(n);
|
|
610
618
|
if (i.success) {
|
|
611
619
|
let e = i.data;
|
|
612
620
|
return r(e, { recursive: !0 }), e;
|
|
613
621
|
}
|
|
614
|
-
return e.parse(
|
|
622
|
+
return e.parse(n);
|
|
615
623
|
}
|
|
616
624
|
//#endregion
|
|
617
625
|
export { c as cleanActeur, o as cleanAmendement, k as cleanCompteRendu, d as cleanDocumentOrDivision, _ as cleanDossierParlementaire, y as cleanOrgane, O as cleanQuestion, x as cleanReunion, E as cleanScrutin };
|
package/lib/index.d.ts
CHANGED
|
@@ -20,5 +20,6 @@ export { avantAApresOptions, etatAmendementCodeOptions, etatAmendementLibelleOpt
|
|
|
20
20
|
export { type CompteRenduClean as CompteRendu, CompteRenduCleanSchema as CompteRenduSchema, type ContenuClean as Contenu, ContenuCleanSchema as ContenuSchema, type FinSeanceClean as FinSeance, FinSeanceCleanSchema as FinSeanceSchema, type MetadonneesClean as Metadonnees, MetadonneesCleanSchema as MetadonneesSchema, type OrateurElementClean as OrateurElement, OrateurElementCleanSchema as OrateurElementSchema, type OrateursClean as Orateurs, OrateursCleanSchema as OrateursSchema, type PointClean as Point, PointCleanSchema as PointSchema, type QuantiemesClean as Quantiemes, QuantiemesCleanSchema as QuantiemesSchema, type SommaireClean as Sommaire, SommaireCleanSchema as SommaireSchema, type SommaireElementClean as SommaireElement, SommaireElementCleanSchema as SommaireElementSchema, type TexteClean as Texte, TexteCleanSchema as TexteSchema, type TitreStructClean as TitreStruct, TitreStructCleanSchema as TitreStructSchema, } from './schemas/clean/debats.js';
|
|
21
21
|
export { acteLegislatifXsiTypeOptions, documentXsiTypeOptions, dossierParlementaireXsiTypeOptions, type ActeLegislatifClean as ActeLegislatif, ActeLegislatifCleanSchema as ActeLegislatifSchema, type ActeLegislatifXsiType as TypeActeLegislatif, type AmendementsCommissionElementClean as AmendementsCommission, AmendementsCommissionElementCleanSchema as AmendementsCommissionSchema, type AmendementsSeanceClean as AmendementsSeance, AmendementsSeanceCleanSchema as AmendementsSeanceSchema, type AuteurActeurClean as AuteurActeur, AuteurActeurCleanSchema as AuteurActeurSchema, type AuteurClean as Auteur, AuteurCleanSchema as AuteurSchema, type Cause, type ChronoClean as ChronoDocument, ChronoCleanSchema as ChronoDocumentSchema, type ClassificationClean as Classification, ClassificationCleanSchema as ClassificationSchema, type ContributionInternauteClean as ContributionInternaute, ContributionInternauteCleanSchema as ContributionInternauteSchema, type CorrectionClean as Correction, CorrectionCleanSchema as CorrectionSchema, type CoSignataireElementClean as CoSignataireElement, CoSignataireElementCleanSchema as CoSignataireElementSchema, type CoSignataireOrganeClean as CoSignataireOrgane, CoSignataireOrganeCleanSchema as CoSignataireOrganeSchema, type CycleDeVieClean as CycleDeVieDocument, CycleDeVieCleanSchema as CycleDeVieDocumentSchema, type DepotAmendementsClean as DepotAmendements, DepotAmendementsCleanSchema as DepotAmendementsSchema, type DivisionClean as Division, DivisionCleanSchema as DivisionSchema, type DocumentClean as Document, DocumentCleanSchema as DocumentSchema, type DocumentDenominationStructurelle, type DocumentXsiType as TypeDocument, type DossierParlementaireClean as DossierParlementaire, DossierParlementaireCleanSchema as DossierParlementaireSchema, type DossierParlementaireXsiType, type EtudePlfClean as EtudePlf, EtudePlfCleanSchema as EtudePlfSchema, type FamCodeEtLibelleClean as FamCodeEtLibelle, FamCodeEtLibelleCleanSchema as FamCodeEtLibelleSchema, type FamilleClean as Famille, FamilleCleanSchema as FamilleSchema, type FusionDossierClean as FusionDossier, FusionDossierCleanSchema as FusionDossierSchema, type ImprimerieClean as Imprimerie, ImprimerieCleanSchema as ImprimerieSchema, type IndexationClean as Indexation, IndexationCleanSchema as IndexationSchema, type InfoJoceClean as InfoJoce, InfoJoceCleanSchema as InfoJoceSchema, type InfoJoElementClean as InfoJo, InfoJoElementCleanSchema as InfoJoSchema, type InitiateurActeurElementClean as InitiateurActeur, InitiateurActeurElementCleanSchema as InitiateurActeurSchema, type InitiateurClean as Initiateur, InitiateurCleanSchema as InitiateurSchema, type LibelleActeClean as LibelleActe, LibelleActeCleanSchema as LibelleActeSchema, type MissionMinefiElementClean as MissionMinefi, MissionMinefiElementCleanSchema as MissionMinefiSchema, type Motif, type NoticeClean as Notice, NoticeCleanSchema as NoticeSchema, type ProcedureParlementaireClean as ProcedureParlementaire, ProcedureParlementaireCleanSchema as ProcedureParlementaireSchema, type Provenance, type Qualite, type RapporteurClean as Rapporteur, RapporteurCleanSchema as RapporteurSchema, type SousTypeClean as SousType, SousTypeCleanSchema as SousTypeSchema, type StatutAdoption, type TexteAssocieElementClean as TexteAssocie, TexteAssocieElementCleanSchema as TexteAssocieSchema, type TexteEuropeenClean as TexteEuropeen, TexteEuropeenCleanSchema as TexteEuropeenSchema, type TexteExterneAssocieClean as TexteExterneAssocie, TexteExterneAssocieCleanSchema as TexteExterneAssocieSchema, type TexteExterneAssocieTypeDoc, type ThemeClean as Theme, ThemeCleanSchema as ThemeSchema, type ThemesClean as Themes, ThemesCleanSchema as ThemesSchema, type TitreDossierClean as TitreDossier, TitreDossierCleanSchema as TitreDossierSchema, type TitresClean as Titres, TitresCleanSchema as TitresSchema, type TypeBudget, type TypeCorrection, type TypeJo, type TypeMission, type TypeTexte, } from './schemas/clean/dossiers_legislatifs.js';
|
|
22
22
|
export { type ContenuSubdivisionEnriched as ContenuSubdivision, type SubdivisionEnriched as Subdivision, } from './schemas/enriched/dossiers_legislatifs.js';
|
|
23
|
+
export { PlfBudgetTypeSchema, PlfLigneSchema, PlfProgrammeSchema, PlfMissionSchema, PlfCreditsSectionSchema, type PlfBudgetType, type PlfLigne, type PlfProgramme, type PlfMission, type PlfCreditsSection, } from './schemas/clean/dossiers_legislatifs.js';
|
|
23
24
|
export { typeJoQuestionOptions, type AnalyseClean as Analyse, AnalyseCleanSchema as AnalyseSchema, type AuteurQuestionClean as AuteurQuestion, AuteurQuestionCleanSchema as AuteurQuestionSchema, type ClotureClean as Cloture, ClotureCleanSchema as ClotureSchema, type GroupeClean as Groupe, GroupeCleanSchema as GroupeSchema, type IdentifiantClean as Identifiant, IdentifiantCleanSchema as IdentifiantSchema, type IdentiteAuteurQuestionClean as IdentiteAuteurQuestion, IdentiteAuteurQuestionCleanSchema as IdentiteAuteurQuestionSchema, type IndexationAnClean as IndexationAn, IndexationAnCleanSchema as IndexationAnSchema, type InfoJoQuestionClean as InfoJoQuestion, InfoJoQuestionCleanSchema as InfoJoQuestionSchema, type MinAttribElementClean as MinAttribElement, MinAttribElementCleanSchema as MinAttribElementSchema, type MinAttribsClean as MinAttribs, MinAttribsCleanSchema as MinAttribsSchema, type QuestionClean as Question, QuestionCleanSchema as QuestionSchema, type RenouvellementClean as Renouvellement, RenouvellementCleanSchema as RenouvellementSchema, type RenouvellementCleans as Renouvellements, RenouvellementCleansSchema as RenouvellementsSchema, type SignalementClean as Signalement, SignalementCleanSchema as SignalementSchema, type TexteQuestionClean as TexteQuestion, TexteQuestionCleanSchema as TexteQuestionSchema, type TextesQuestionClean as TextesQuestion, TextesQuestionCleanSchema as TextesQuestionSchema, type TextesReponseClean as TextesReponse, TextesReponseCleanSchema as TextesReponseSchema, type TypeJoQuestion, } from './schemas/clean/questions.js';
|
|
24
25
|
export { type DossierLegislatifObjetClean as DossierLegislatifObjet, DossierLegislatifObjetCleanSchema as DossierLegislatifObjetSchema, type DecompteNominatifClean as DecompteNominatif, DecompteNominatifCleanSchema as DecompteNominatifSchema, type DecompteVoixClean as DecompteVoix, DecompteVoixCleanSchema as DecompteVoixSchema, type DemandeurClean as Demandeur, DemandeurCleanSchema as DemandeurSchema, type DysfonctionnementClean as Dysfonctionnement, DysfonctionnementCleanSchema as DysfonctionnementSchema, type MiseAuPointClean as MiseAuPoint, MiseAuPointCleanSchema as MiseAuPointSchema, type ObjetVoteClean as ObjetVote, ObjetVoteCleanSchema as ObjetVoteSchema, type ScrutinClean as Scrutin, ScrutinCleanSchema as ScrutinSchema, type SortScrutinClean as SortScrutin, SortScrutinCleanSchema as SortScrutinSchema, type SyntheseVoteClean as SyntheseVote, SyntheseVoteCleanSchema as SyntheseVoteSchema, type TypeVoteClean as TypeVote, TypeVoteCleanSchema as TypeVoteSchema, type VentilationVotesClean as VentilationVotes, VentilationVotesCleanSchema as VentilationVotesSchema, type VentilationVotesGroupeClean as VentilationVotesGroupe, VentilationVotesGroupeCleanSchema as VentilationVotesGroupeSchema, type VotantClean as Votant, VotantCleanSchema as VotantSchema, type VoteClean as Vote, VoteCleanSchema as VoteSchema, } from './schemas/clean/scrutins.js';
|