@tricoteuses/senat 1.3.2 → 1.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/config.js +4 -6
- package/lib/databases.js +34 -75
- package/lib/datasets.js +20 -28
- package/lib/index.d.ts +5 -0
- package/lib/index.js +4 -43
- package/lib/loaders.js +56 -74
- package/lib/model/ameli.js +8 -11
- package/lib/model/dosleg.js +47 -52
- package/lib/model/index.js +4 -13
- package/lib/model/questions.js +15 -18
- package/lib/model/sens.d.ts +1 -1
- package/lib/model/sens.js +65 -71
- package/lib/model/texte.js +17 -25
- package/lib/model/util.js +13 -21
- package/lib/raw_types/ameli.js +1 -2
- package/lib/raw_types/debats.js +1 -2
- package/lib/raw_types/dosleg.js +1 -2
- package/lib/raw_types/questions.js +1 -2
- package/lib/raw_types/sens.js +1 -2
- package/lib/raw_types_schemats/ameli.js +1 -2
- package/lib/raw_types_schemats/debats.js +1 -2
- package/lib/raw_types_schemats/dosleg.js +1 -2
- package/lib/raw_types_schemats/questions.js +1 -2
- package/lib/raw_types_schemats/sens.js +1 -2
- package/lib/scripts/convert_data.js +78 -83
- package/lib/scripts/datautil.js +9 -13
- package/lib/scripts/parse_textes.js +23 -28
- package/lib/scripts/retrieve_documents.js +56 -61
- package/lib/scripts/retrieve_open_data.js +44 -49
- package/lib/scripts/retrieve_senateurs_photos.js +31 -36
- package/lib/scripts/shared/cli_helpers.js +9 -12
- package/lib/scripts/shared/util.js +7 -15
- package/lib/strings.js +4 -10
- package/lib/types/ameli.js +5 -8
- package/lib/types/debats.js +2 -5
- package/lib/types/dosleg.js +28 -31
- package/lib/types/questions.js +1 -2
- package/lib/types/sens.js +1 -2
- package/lib/types/sessions.js +2 -5
- package/lib/types/texte.js +2 -5
- package/lib/validators/config.js +4 -7
- package/package.json +4 -4
|
@@ -1,25 +1,20 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const assert_1 = __importDefault(require("assert"));
|
|
7
|
-
const child_process_1 = require("child_process");
|
|
8
|
-
const command_line_args_1 = __importDefault(require("command-line-args"));
|
|
9
|
-
const fs_extra_1 = __importDefault(require("fs-extra"));
|
|
1
|
+
import assert from "assert";
|
|
2
|
+
import { execSync } from "child_process";
|
|
3
|
+
import commandLineArgs from "command-line-args";
|
|
4
|
+
import fs from "fs-extra";
|
|
10
5
|
// import fetch from "node-fetch"
|
|
11
|
-
|
|
6
|
+
import path from "path";
|
|
12
7
|
// import stream from "stream"
|
|
13
|
-
|
|
14
|
-
|
|
8
|
+
import StreamZip from "node-stream-zip";
|
|
9
|
+
import readline from "readline";
|
|
15
10
|
// import util from "util"
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
11
|
+
import windows1252 from "windows-1252";
|
|
12
|
+
import config from "../config";
|
|
13
|
+
import { datasets, getChosenFromEnabledDatasets } from "../datasets";
|
|
14
|
+
import { commonOptions } from "./shared/cli_helpers";
|
|
20
15
|
const badWindows1252CharacterRegex = /[\u0080-\u009f]/g;
|
|
21
16
|
const optionsDefinitions = [
|
|
22
|
-
...
|
|
17
|
+
...commonOptions,
|
|
23
18
|
{
|
|
24
19
|
alias: "a",
|
|
25
20
|
help: "all options: fetch, unzip, repair-encoding, import",
|
|
@@ -63,11 +58,11 @@ const optionsDefinitions = [
|
|
|
63
58
|
type: Boolean,
|
|
64
59
|
},
|
|
65
60
|
];
|
|
66
|
-
const options = (
|
|
61
|
+
const options = commandLineArgs(optionsDefinitions);
|
|
67
62
|
// const pipeline = util.promisify(stream.pipeline)
|
|
68
63
|
async function retrieveDataset(dataDir, dataset) {
|
|
69
64
|
const zipFilename = dataset.url.substring(dataset.url.lastIndexOf("/") + 1);
|
|
70
|
-
const zipFilePath =
|
|
65
|
+
const zipFilePath = path.join(dataDir, zipFilename);
|
|
71
66
|
if (options["all"] || options["fetch"]) {
|
|
72
67
|
// Fetch & save ZIP file.
|
|
73
68
|
if (!options["silent"]) {
|
|
@@ -82,8 +77,8 @@ async function retrieveDataset(dataDir, dataset) {
|
|
|
82
77
|
// throw new Error(`Fetch failed: ${dataset.url}`)
|
|
83
78
|
// }
|
|
84
79
|
// await pipeline(response.body!, fs.createWriteStream(zipFilePath))
|
|
85
|
-
|
|
86
|
-
|
|
80
|
+
fs.removeSync(zipFilePath);
|
|
81
|
+
execSync(`wget --quiet ${dataset.url}`, {
|
|
87
82
|
cwd: dataDir,
|
|
88
83
|
env: process.env,
|
|
89
84
|
encoding: "utf-8",
|
|
@@ -91,13 +86,13 @@ async function retrieveDataset(dataDir, dataset) {
|
|
|
91
86
|
});
|
|
92
87
|
}
|
|
93
88
|
const sqlFilename = `${dataset.database}.sql`;
|
|
94
|
-
const sqlFilePath =
|
|
89
|
+
const sqlFilePath = path.join(dataDir, sqlFilename);
|
|
95
90
|
if (options["all"] || options["unzip"]) {
|
|
96
91
|
if (!options["silent"]) {
|
|
97
92
|
console.log(`Unzipping ${dataset.title}: ${zipFilename}…`);
|
|
98
93
|
}
|
|
99
|
-
|
|
100
|
-
const zip = new
|
|
94
|
+
fs.removeSync(sqlFilePath);
|
|
95
|
+
const zip = new StreamZip({
|
|
101
96
|
file: zipFilePath,
|
|
102
97
|
storeEntries: true,
|
|
103
98
|
});
|
|
@@ -126,24 +121,24 @@ async function retrieveDataset(dataDir, dataset) {
|
|
|
126
121
|
console.log(`Repairing Windows CP1252 encoding of ${dataset.title}: ${sqlFilename}…`);
|
|
127
122
|
}
|
|
128
123
|
const repairedSqlFilePath = sqlFilePath + ".repaired";
|
|
129
|
-
const repairedSqlWriter =
|
|
124
|
+
const repairedSqlWriter = fs.createWriteStream(repairedSqlFilePath, {
|
|
130
125
|
encoding: "utf8",
|
|
131
126
|
});
|
|
132
|
-
const lineReader =
|
|
133
|
-
input:
|
|
127
|
+
const lineReader = readline.createInterface({
|
|
128
|
+
input: fs.createReadStream(sqlFilePath, { encoding: "utf8" }),
|
|
134
129
|
crlfDelay: Infinity,
|
|
135
130
|
});
|
|
136
131
|
for await (const line of lineReader) {
|
|
137
|
-
repairedSqlWriter.write(line.replace(badWindows1252CharacterRegex, (match) =>
|
|
132
|
+
repairedSqlWriter.write(line.replace(badWindows1252CharacterRegex, (match) => windows1252.decode(match, { mode: "fatal" })) + "\n");
|
|
138
133
|
}
|
|
139
134
|
repairedSqlWriter.end();
|
|
140
|
-
await
|
|
135
|
+
await fs.move(repairedSqlFilePath, sqlFilePath, { overwrite: true });
|
|
141
136
|
}
|
|
142
137
|
if (options["all"] || options["import"] || options["schema"]) {
|
|
143
138
|
if (!options["silent"]) {
|
|
144
139
|
console.log(`Importing ${dataset.title}: ${sqlFilename}…`);
|
|
145
140
|
}
|
|
146
|
-
|
|
141
|
+
execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -d ${dataset.database} -f ${sqlFilename}`, {
|
|
147
142
|
cwd: dataDir,
|
|
148
143
|
env: process.env,
|
|
149
144
|
encoding: "utf-8",
|
|
@@ -151,27 +146,27 @@ async function retrieveDataset(dataDir, dataset) {
|
|
|
151
146
|
});
|
|
152
147
|
}
|
|
153
148
|
if (options["schema"]) {
|
|
154
|
-
let definitionsDir =
|
|
155
|
-
(
|
|
149
|
+
let definitionsDir = path.resolve("src", "raw_types_schemats");
|
|
150
|
+
assert(fs.statSync(definitionsDir).isDirectory());
|
|
156
151
|
if (!options["silent"]) {
|
|
157
152
|
console.log(`Creating TypeScript definitions from schema of database ${dataset.database}…`);
|
|
158
153
|
}
|
|
159
154
|
const dbConnectionString = `postgres://${process.env["PGUSER"]}:${process.env["PGPASSWORD"]}@${process.env["PGHOST"]}:${process.env["PGPORT"]}/${dataset.database}`;
|
|
160
|
-
let definitionFilePath =
|
|
161
|
-
|
|
155
|
+
let definitionFilePath = path.join(definitionsDir, `${dataset.database}.ts`);
|
|
156
|
+
execSync(`npx schemats generate -c ${dbConnectionString} -s ${dataset.schema} -o ${definitionFilePath}`, {
|
|
162
157
|
// cwd: dataDir,
|
|
163
158
|
env: process.env,
|
|
164
159
|
encoding: "utf-8",
|
|
165
160
|
// stdio: ["ignore", "ignore", "pipe"],
|
|
166
161
|
});
|
|
167
|
-
const definition =
|
|
162
|
+
const definition = fs.readFileSync(definitionFilePath, { encoding: "utf8" });
|
|
168
163
|
const definitionRepaired = definition
|
|
169
164
|
.replace(/\r\n/g, "\n")
|
|
170
165
|
.replace(/AUTO-GENERATED FILE @ \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/, "AUTO-GENERATED FILE");
|
|
171
|
-
|
|
172
|
-
definitionsDir =
|
|
173
|
-
definitionFilePath =
|
|
174
|
-
|
|
166
|
+
fs.writeFileSync(definitionFilePath, definitionRepaired);
|
|
167
|
+
definitionsDir = path.resolve("src", "raw_types");
|
|
168
|
+
definitionFilePath = path.join(definitionsDir, `${dataset.database}.ts`);
|
|
169
|
+
execSync(`kysely-codegen --url ${dbConnectionString} --default-schema=${dataset.schema} --out-file=${definitionFilePath}`, {
|
|
175
170
|
// cwd: dataDir,
|
|
176
171
|
env: process.env,
|
|
177
172
|
encoding: "utf-8",
|
|
@@ -181,32 +176,32 @@ async function retrieveDataset(dataDir, dataset) {
|
|
|
181
176
|
}
|
|
182
177
|
async function retrieveOpenData() {
|
|
183
178
|
const dataDir = options["dataDir"];
|
|
184
|
-
(
|
|
179
|
+
assert(dataDir, "Missing argument: data directory");
|
|
185
180
|
process.env = {
|
|
186
181
|
...process.env,
|
|
187
|
-
PGHOST: process.env["PGHOST"] ||
|
|
188
|
-
PGPORT: process.env["PGPORT"] ||
|
|
189
|
-
PGUSER: process.env["PGUSER"] ||
|
|
190
|
-
PGPASSWORD: process.env["PGPASSWORD"] ||
|
|
182
|
+
PGHOST: process.env["PGHOST"] || config.db.host,
|
|
183
|
+
PGPORT: process.env["PGPORT"] || config.db.port,
|
|
184
|
+
PGUSER: process.env["PGUSER"] || config.db.user,
|
|
185
|
+
PGPASSWORD: process.env["PGPASSWORD"] || config.db.password,
|
|
191
186
|
};
|
|
192
|
-
(
|
|
187
|
+
assert(process.env["PGHOST"] &&
|
|
193
188
|
process.env["PGPORT"] &&
|
|
194
189
|
process.env["PGUSER"] &&
|
|
195
190
|
process.env["PGPASSWORD"], "Missing database configuration: environment variables PGHOST, PGPORT, PGUSER and PGPASSWORD or TRICOTEUSES_SENAT_DB_* in .env file");
|
|
196
191
|
console.time("data extraction time");
|
|
197
|
-
for (const [, dataset] of Object.entries(
|
|
198
|
-
|
|
192
|
+
for (const [, dataset] of Object.entries(datasets)) {
|
|
193
|
+
execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -c "DROP DATABASE IF EXISTS ${dataset.database}"`, {
|
|
199
194
|
cwd: dataDir,
|
|
200
195
|
env: process.env,
|
|
201
196
|
encoding: "utf-8",
|
|
202
197
|
});
|
|
203
|
-
|
|
198
|
+
execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -c "CREATE DATABASE ${dataset.database} WITH OWNER opendata"`, {
|
|
204
199
|
cwd: dataDir,
|
|
205
200
|
env: process.env,
|
|
206
201
|
encoding: "utf-8",
|
|
207
202
|
});
|
|
208
203
|
}
|
|
209
|
-
const choosenDatasets =
|
|
204
|
+
const choosenDatasets = getChosenFromEnabledDatasets(options["categories"]);
|
|
210
205
|
for (const dataset of choosenDatasets) {
|
|
211
206
|
await retrieveDataset(dataDir, dataset);
|
|
212
207
|
}
|
|
@@ -1,21 +1,16 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const assert_1 = __importDefault(require("assert"));
|
|
7
|
-
const child_process_1 = require("child_process");
|
|
8
|
-
const command_line_args_1 = __importDefault(require("command-line-args"));
|
|
9
|
-
const fs_extra_1 = __importDefault(require("fs-extra"));
|
|
1
|
+
import assert from "assert";
|
|
2
|
+
import { execSync } from "child_process";
|
|
3
|
+
import commandLineArgs from "command-line-args";
|
|
4
|
+
import fs from "fs-extra";
|
|
10
5
|
// import fetch from "node-fetch"
|
|
11
|
-
|
|
6
|
+
import path from "path";
|
|
12
7
|
// import stream from "stream"
|
|
13
8
|
// import util from "util"
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
9
|
+
import { findActif as findActifSenateurs } from "../model/sens";
|
|
10
|
+
import { slugify } from "../strings";
|
|
11
|
+
import { commonOptions } from "./shared/cli_helpers";
|
|
17
12
|
const optionsDefinitions = [
|
|
18
|
-
...
|
|
13
|
+
...commonOptions,
|
|
19
14
|
{
|
|
20
15
|
alias: "f",
|
|
21
16
|
help: "fetch sénateurs' pictures instead of retrieving them from files",
|
|
@@ -23,23 +18,23 @@ const optionsDefinitions = [
|
|
|
23
18
|
type: Boolean,
|
|
24
19
|
},
|
|
25
20
|
];
|
|
26
|
-
const options = (
|
|
21
|
+
const options = commandLineArgs(optionsDefinitions);
|
|
27
22
|
// const pipeline = util.promisify(stream.pipeline)
|
|
28
23
|
async function retrievePhotosSenateurs() {
|
|
29
24
|
const dataDir = options["dataDir"];
|
|
30
|
-
(
|
|
31
|
-
const photosDir =
|
|
32
|
-
const missingPhotoFilePath =
|
|
33
|
-
const sens = await Array.fromAsync((
|
|
25
|
+
assert(dataDir, "Missing argument: data directory");
|
|
26
|
+
const photosDir = path.join(dataDir, "photos_senateurs");
|
|
27
|
+
const missingPhotoFilePath = path.resolve(__dirname, "images", "transparent_155x225.jpg");
|
|
28
|
+
const sens = await Array.fromAsync(findActifSenateurs());
|
|
34
29
|
// Download photos.
|
|
35
|
-
|
|
30
|
+
fs.ensureDirSync(photosDir);
|
|
36
31
|
if (options["fetch"]) {
|
|
37
32
|
for (const sen of sens) {
|
|
38
|
-
const photoStem = `${
|
|
33
|
+
const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(sen.senprenomuse, "_")}${slugify(sen.senmat, "_")}`;
|
|
39
34
|
const photoFilename = photoStem + ".jpg";
|
|
40
|
-
const photoFilePath =
|
|
35
|
+
const photoFilePath = path.join(photosDir, photoFilename);
|
|
41
36
|
const photoTempFilename = photoStem + "_temp.jpg";
|
|
42
|
-
const photoTempFilePath =
|
|
37
|
+
const photoTempFilePath = path.join(photosDir, photoTempFilename);
|
|
43
38
|
const urlPhoto = `https://www.senat.fr/senimg/${photoFilename}`;
|
|
44
39
|
if (!options["silent"]) {
|
|
45
40
|
console.log(`Loading photo ${urlPhoto} for ${sen.senprenomuse} ${sen.sennomuse}…`);
|
|
@@ -67,13 +62,13 @@ async function retrievePhotosSenateurs() {
|
|
|
67
62
|
// }
|
|
68
63
|
// }
|
|
69
64
|
try {
|
|
70
|
-
|
|
65
|
+
execSync(`wget --quiet -O ${photoTempFilename} ${urlPhoto}`, {
|
|
71
66
|
cwd: photosDir,
|
|
72
67
|
env: process.env,
|
|
73
68
|
encoding: "utf-8",
|
|
74
69
|
// stdio: ["ignore", "ignore", "pipe"],
|
|
75
70
|
});
|
|
76
|
-
|
|
71
|
+
fs.renameSync(photoTempFilePath, photoFilePath);
|
|
77
72
|
}
|
|
78
73
|
catch (error) {
|
|
79
74
|
if (typeof error === "object" &&
|
|
@@ -89,14 +84,14 @@ async function retrievePhotosSenateurs() {
|
|
|
89
84
|
}
|
|
90
85
|
// Resize photos to 155x225, because some haven't exactly this size.
|
|
91
86
|
for (const sen of sens) {
|
|
92
|
-
const photoStem = `${
|
|
87
|
+
const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(sen.senprenomuse, "_")}${slugify(sen.senmat, "_")}`;
|
|
93
88
|
const photoFilename = photoStem + ".jpg";
|
|
94
|
-
const photoFilePath =
|
|
95
|
-
if (
|
|
89
|
+
const photoFilePath = path.join(photosDir, photoFilename);
|
|
90
|
+
if (fs.existsSync(photoFilePath)) {
|
|
96
91
|
if (!options["silent"]) {
|
|
97
92
|
console.log(`Resizing photo ${photoStem} for ${sen.senprenomuse} ${sen.sennomuse}…`);
|
|
98
93
|
}
|
|
99
|
-
|
|
94
|
+
execSync(`gm convert -resize 155x225! ${photoStem}.jpg ${photoStem}_155x225.jpg`, {
|
|
100
95
|
cwd: photosDir,
|
|
101
96
|
});
|
|
102
97
|
}
|
|
@@ -104,7 +99,7 @@ async function retrievePhotosSenateurs() {
|
|
|
104
99
|
if (!options["silent"]) {
|
|
105
100
|
console.warn(`Missing photo for ${sen.senprenomuse} ${sen.sennomuse}: using blank image`);
|
|
106
101
|
}
|
|
107
|
-
|
|
102
|
+
fs.copyFileSync(missingPhotoFilePath, path.join(photosDir, `${photoStem}_155x225.jpg`));
|
|
108
103
|
}
|
|
109
104
|
}
|
|
110
105
|
// Create a mosaic of photos.
|
|
@@ -117,7 +112,7 @@ async function retrievePhotosSenateurs() {
|
|
|
117
112
|
const row = sens.slice(senIndex, senIndex + 25);
|
|
118
113
|
const photosFilenames = [];
|
|
119
114
|
for (const [columnIndex, sen] of row.entries()) {
|
|
120
|
-
const photoStem = `${
|
|
115
|
+
const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(sen.senprenomuse, "_")}${slugify(sen.senmat, "_")}`;
|
|
121
116
|
const photoFilename = `${photoStem}_155x225.jpg`;
|
|
122
117
|
photosFilenames.push(photoFilename);
|
|
123
118
|
photoBySenmat[sen.senmat] = {
|
|
@@ -130,22 +125,22 @@ async function retrievePhotosSenateurs() {
|
|
|
130
125
|
};
|
|
131
126
|
}
|
|
132
127
|
const rowFilename = `row-${rowIndex}.jpg`;
|
|
133
|
-
|
|
128
|
+
execSync(`gm convert ${photosFilenames.join(" ")} +append ${rowFilename}`, {
|
|
134
129
|
cwd: photosDir,
|
|
135
130
|
});
|
|
136
131
|
rowsFilenames.push(rowFilename);
|
|
137
132
|
}
|
|
138
|
-
|
|
133
|
+
execSync(`gm convert ${rowsFilenames.join(" ")} -append senateurs.jpg`, {
|
|
139
134
|
cwd: photosDir,
|
|
140
135
|
});
|
|
141
136
|
for (const rowFilename of rowsFilenames) {
|
|
142
|
-
|
|
137
|
+
fs.unlinkSync(path.join(photosDir, rowFilename));
|
|
143
138
|
}
|
|
144
139
|
if (!options["silent"]) {
|
|
145
140
|
console.log("Creating JSON file containing informations on all pictures…");
|
|
146
141
|
}
|
|
147
|
-
const jsonFilePath =
|
|
148
|
-
|
|
142
|
+
const jsonFilePath = path.join(photosDir, "senateurs.json");
|
|
143
|
+
fs.writeFileSync(jsonFilePath, JSON.stringify(photoBySenmat, null, 2));
|
|
149
144
|
}
|
|
150
145
|
retrievePhotosSenateurs()
|
|
151
146
|
.then(() => process.exit(0))
|
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.commonOptions = exports.verboseOption = exports.silentOption = exports.dataDirDefaultOption = exports.categoriesOption = void 0;
|
|
4
|
-
exports.categoriesOption = {
|
|
1
|
+
export const categoriesOption = {
|
|
5
2
|
alias: "k",
|
|
6
3
|
defaultValue: ["All"],
|
|
7
4
|
help: "categories of datasets to reorganize",
|
|
@@ -9,27 +6,27 @@ exports.categoriesOption = {
|
|
|
9
6
|
name: "categories",
|
|
10
7
|
type: String,
|
|
11
8
|
};
|
|
12
|
-
|
|
9
|
+
export const dataDirDefaultOption = {
|
|
13
10
|
defaultOption: true,
|
|
14
11
|
help: "directory containing Sénat open data files",
|
|
15
12
|
name: "dataDir",
|
|
16
13
|
type: String,
|
|
17
14
|
};
|
|
18
|
-
|
|
15
|
+
export const silentOption = {
|
|
19
16
|
alias: "s",
|
|
20
17
|
help: "don't log anything",
|
|
21
18
|
name: "silent",
|
|
22
19
|
type: Boolean,
|
|
23
20
|
};
|
|
24
|
-
|
|
21
|
+
export const verboseOption = {
|
|
25
22
|
alias: "v",
|
|
26
23
|
help: "verbose logs",
|
|
27
24
|
name: "verbose",
|
|
28
25
|
type: Boolean,
|
|
29
26
|
};
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
27
|
+
export const commonOptions = [
|
|
28
|
+
categoriesOption,
|
|
29
|
+
dataDirDefaultOption,
|
|
30
|
+
silentOption,
|
|
31
|
+
verboseOption,
|
|
35
32
|
];
|
|
@@ -1,24 +1,16 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.isOptionEmptyOrHasValue = isOptionEmptyOrHasValue;
|
|
7
|
-
exports.ensureAndClearDir = ensureAndClearDir;
|
|
8
|
-
exports.fetchWithRetry = fetchWithRetry;
|
|
9
|
-
const fs_extra_1 = __importDefault(require("fs-extra"));
|
|
10
|
-
function isOptionEmptyOrHasValue(option, value) {
|
|
1
|
+
import fs from "fs-extra";
|
|
2
|
+
export function isOptionEmptyOrHasValue(option, value) {
|
|
11
3
|
return !option || option.length === 0 || option.includes(value);
|
|
12
4
|
}
|
|
13
|
-
function ensureAndClearDir(path) {
|
|
14
|
-
if (!
|
|
15
|
-
|
|
5
|
+
export function ensureAndClearDir(path) {
|
|
6
|
+
if (!fs.existsSync(path)) {
|
|
7
|
+
fs.mkdirSync(path, { recursive: true });
|
|
16
8
|
}
|
|
17
9
|
else {
|
|
18
|
-
|
|
10
|
+
fs.emptyDirSync(path);
|
|
19
11
|
}
|
|
20
12
|
}
|
|
21
|
-
async function fetchWithRetry(url, retries = 3, backoff = 300) {
|
|
13
|
+
export async function fetchWithRetry(url, retries = 3, backoff = 300) {
|
|
22
14
|
for (let attempt = 0; attempt < retries; attempt++) {
|
|
23
15
|
try {
|
|
24
16
|
return await fetch(url);
|
package/lib/strings.js
CHANGED
|
@@ -1,18 +1,12 @@
|
|
|
1
|
-
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.slugify = slugify;
|
|
7
|
-
const slug_1 = __importDefault(require("slug"));
|
|
1
|
+
import originalSlugify from "slug";
|
|
8
2
|
const slugifyCharmap = {
|
|
9
|
-
...
|
|
3
|
+
...originalSlugify.defaults.charmap,
|
|
10
4
|
"'": " ",
|
|
11
5
|
"@": " ",
|
|
12
6
|
".": " ",
|
|
13
7
|
"-": "_",
|
|
14
8
|
};
|
|
15
|
-
function slugify(string, replacement) {
|
|
9
|
+
export function slugify(string, replacement) {
|
|
16
10
|
const options = {
|
|
17
11
|
charmap: slugifyCharmap,
|
|
18
12
|
mode: "rfc3986",
|
|
@@ -20,5 +14,5 @@ function slugify(string, replacement) {
|
|
|
20
14
|
if (replacement) {
|
|
21
15
|
options.replacement = replacement;
|
|
22
16
|
}
|
|
23
|
-
return (
|
|
17
|
+
return originalSlugify(string, options);
|
|
24
18
|
}
|
package/lib/types/ameli.js
CHANGED
|
@@ -1,11 +1,8 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
exports.subFieldsToParseInt = ["pos", "posder", "prires"];
|
|
7
|
-
exports.subFieldsToTrim = ["lic", "lib", "sig", "style"];
|
|
8
|
-
exports.txtAmeliFieldsToTrim = [
|
|
1
|
+
export const sesFieldsToParseInt = ["ann"];
|
|
2
|
+
export const sesFieldsToTrim = ["lil"];
|
|
3
|
+
export const subFieldsToParseInt = ["pos", "posder", "prires"];
|
|
4
|
+
export const subFieldsToTrim = ["lic", "lib", "sig", "style"];
|
|
5
|
+
export const txtAmeliFieldsToTrim = [
|
|
9
6
|
"num",
|
|
10
7
|
"int",
|
|
11
8
|
"inl",
|
package/lib/types/debats.js
CHANGED
|
@@ -1,5 +1,2 @@
|
|
|
1
|
-
"
|
|
2
|
-
|
|
3
|
-
exports.lecassdebFieldsToTrim = exports.debatsFieldsToTrim = void 0;
|
|
4
|
-
exports.debatsFieldsToTrim = ["deburl", "libspec"];
|
|
5
|
-
exports.lecassdebFieldsToTrim = ["lecassidt"];
|
|
1
|
+
export const debatsFieldsToTrim = ["deburl", "libspec"];
|
|
2
|
+
export const lecassdebFieldsToTrim = ["lecassidt"];
|
package/lib/types/dosleg.js
CHANGED
|
@@ -1,9 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
exports.assFieldsToTrim = ["libass"];
|
|
5
|
-
exports.audFieldsToTrim = ["lecassidt", "audtit", "audurl", "orgcod"];
|
|
6
|
-
exports.auteurFieldsToTrim = [
|
|
1
|
+
export const assFieldsToTrim = ["libass"];
|
|
2
|
+
export const audFieldsToTrim = ["lecassidt", "audtit", "audurl", "orgcod"];
|
|
3
|
+
export const auteurFieldsToTrim = [
|
|
7
4
|
"autcod",
|
|
8
5
|
"quacod",
|
|
9
6
|
"typautcod",
|
|
@@ -13,9 +10,9 @@ exports.auteurFieldsToTrim = [
|
|
|
13
10
|
"autmat",
|
|
14
11
|
"autfct",
|
|
15
12
|
];
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
13
|
+
export const dateSeanceFieldsToTrim = ["lecidt", "statut"];
|
|
14
|
+
export const deccocFieldsToTrim = ["deccoccod", "deccoclib"];
|
|
15
|
+
export const denrapFieldsToTrim = [
|
|
19
16
|
"coddenrap",
|
|
20
17
|
"typraprap",
|
|
21
18
|
"libdenrap",
|
|
@@ -24,11 +21,11 @@ exports.denrapFieldsToTrim = [
|
|
|
24
21
|
"denrapstymin",
|
|
25
22
|
"solnatrapcod",
|
|
26
23
|
];
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
24
|
+
export const docattFieldsToParseInt = ["docattcle", "rapcod"];
|
|
25
|
+
export const docattFieldsToTrim = ["docatturl"];
|
|
26
|
+
export const ecrFieldsToTrim = ["autcod", "ecrqua"];
|
|
27
|
+
export const etaloiFieldsToTrim = ["etaloilib"];
|
|
28
|
+
export const lecassFieldsToTrim = [
|
|
32
29
|
"lecassidt",
|
|
33
30
|
"lecidt",
|
|
34
31
|
"ptlurl",
|
|
@@ -50,9 +47,9 @@ exports.lecassFieldsToTrim = [
|
|
|
50
47
|
"aliasppr",
|
|
51
48
|
"lecassamecom",
|
|
52
49
|
];
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
50
|
+
export const lecassrapFieldsToTrim = ["lecassidt"];
|
|
51
|
+
export const lectureFieldsToTrim = ["lecidt", "loicod", "typleccod", "leccom"];
|
|
52
|
+
export const loiFieldsToTrim = [
|
|
56
53
|
"loicod",
|
|
57
54
|
"typloicod",
|
|
58
55
|
"deccoccod",
|
|
@@ -82,7 +79,7 @@ exports.loiFieldsToTrim = [
|
|
|
82
79
|
"doscocurl",
|
|
83
80
|
"loiintori",
|
|
84
81
|
];
|
|
85
|
-
|
|
82
|
+
export const orgFieldsToTrim = [
|
|
86
83
|
"orgcod",
|
|
87
84
|
"typorgcod",
|
|
88
85
|
"orgnom",
|
|
@@ -97,10 +94,10 @@ exports.orgFieldsToTrim = [
|
|
|
97
94
|
"senorgcod",
|
|
98
95
|
"html_color",
|
|
99
96
|
];
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
97
|
+
export const oritxtFieldsToTrim = ["oritxtcod", "oritxtlib", "oritxtlibfem"];
|
|
98
|
+
export const quaFieldsToTrim = ["quacod", "qualic", "quaabr", "quaabrplu"];
|
|
99
|
+
export const rapFieldsToParseInt = ["rapcod", "sesann", "rapnum", "rapnuman"];
|
|
100
|
+
export const rapFieldsToTrim = [
|
|
104
101
|
"coddenrap",
|
|
105
102
|
"blecod",
|
|
106
103
|
"raptitcou",
|
|
@@ -118,10 +115,10 @@ exports.rapFieldsToTrim = [
|
|
|
118
115
|
"rapres",
|
|
119
116
|
"forpubcod",
|
|
120
117
|
];
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
118
|
+
export const raporgFieldsToTrim = ["orgcod"];
|
|
119
|
+
export const scrFieldsToTrim = ["scrint", "soslib"];
|
|
120
|
+
export const texteFieldsToParseInt = ["texcod", "sesann", "texnum"];
|
|
121
|
+
export const texteFieldsToTrim = [
|
|
125
122
|
"oritxtcod",
|
|
126
123
|
"typtxtcod",
|
|
127
124
|
"lecassidt",
|
|
@@ -137,9 +134,9 @@ exports.texteFieldsToTrim = [
|
|
|
137
134
|
"numerobis",
|
|
138
135
|
"reserve_comspe",
|
|
139
136
|
];
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
137
|
+
export const typattFieldsToTrim = ["typattlib"];
|
|
138
|
+
export const typlecFieldsToTrim = ["typleccod", "typleclib"];
|
|
139
|
+
export const typloiFieldsToTrim = [
|
|
143
140
|
"typloicod",
|
|
144
141
|
"typloilib",
|
|
145
142
|
"groupe",
|
|
@@ -150,5 +147,5 @@ exports.typloiFieldsToTrim = [
|
|
|
150
147
|
"typloide",
|
|
151
148
|
"typloiabr",
|
|
152
149
|
];
|
|
153
|
-
|
|
154
|
-
|
|
150
|
+
export const typtxtFieldsToTrim = ["typtxtcod", "typtxtlib"];
|
|
151
|
+
export const typurlFieldsToTrim = ["libtypurl"];
|
package/lib/types/questions.js
CHANGED
|
@@ -1,2 +1 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1
|
+
export {};
|
package/lib/types/sens.js
CHANGED
|
@@ -1,2 +1 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1
|
+
export {};
|
package/lib/types/sessions.js
CHANGED
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Session = void 0;
|
|
4
|
-
var Session;
|
|
1
|
+
export var Session;
|
|
5
2
|
(function (Session) {
|
|
6
3
|
Session[Session["2012-2013"] = 2012] = "2012-2013";
|
|
7
4
|
Session[Session["2013-2014"] = 2013] = "2013-2014";
|
|
@@ -43,4 +40,4 @@ var Session;
|
|
|
43
40
|
Session[Session["2049-2050"] = 2049] = "2049-2050";
|
|
44
41
|
Session[Session["2050-2051"] = 2050] = "2050-2051";
|
|
45
42
|
Session[Session["All"] = 0] = "All";
|
|
46
|
-
})(Session || (
|
|
43
|
+
})(Session || (Session = {}));
|
package/lib/types/texte.js
CHANGED
|
@@ -1,8 +1,5 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
/* eslint no-use-before-define: 0 */
|
|
3
|
-
|
|
4
|
-
exports.DivisionType = void 0;
|
|
5
|
-
var DivisionType;
|
|
2
|
+
export var DivisionType;
|
|
6
3
|
(function (DivisionType) {
|
|
7
4
|
DivisionType[DivisionType["tome"] = 1] = "tome";
|
|
8
5
|
DivisionType[DivisionType["part"] = 2] = "part";
|
|
@@ -16,4 +13,4 @@ var DivisionType;
|
|
|
16
13
|
DivisionType[DivisionType["article"] = 10] = "article";
|
|
17
14
|
DivisionType[DivisionType["alinea"] = 11] = "alinea";
|
|
18
15
|
DivisionType[DivisionType["division"] = 12] = "division";
|
|
19
|
-
})(DivisionType || (
|
|
16
|
+
})(DivisionType || (DivisionType = {}));
|