@tricoteuses/senat 2.22.10 → 2.22.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/src/model/sens.d.ts +36 -0
- package/lib/src/model/sens.js +35 -4
- package/lib/src/scripts/retrieve_documents.js +24 -15
- package/package.json +1 -1
- package/lib/config.d.ts +0 -21
- package/lib/config.js +0 -27
- package/lib/databases.d.ts +0 -2
- package/lib/databases.js +0 -26
- package/lib/datasets.d.ts +0 -34
- package/lib/datasets.js +0 -233
- package/lib/git.d.ts +0 -26
- package/lib/git.js +0 -167
- package/lib/index.d.ts +0 -13
- package/lib/index.js +0 -1
- package/lib/loaders.d.ts +0 -58
- package/lib/loaders.js +0 -286
- package/lib/model/agenda.d.ts +0 -6
- package/lib/model/agenda.js +0 -148
- package/lib/model/ameli.d.ts +0 -51
- package/lib/model/ameli.js +0 -147
- package/lib/model/commission.d.ts +0 -18
- package/lib/model/commission.js +0 -269
- package/lib/model/debats.d.ts +0 -67
- package/lib/model/debats.js +0 -95
- package/lib/model/documents.d.ts +0 -12
- package/lib/model/documents.js +0 -138
- package/lib/model/dosleg.d.ts +0 -7
- package/lib/model/dosleg.js +0 -326
- package/lib/model/index.d.ts +0 -7
- package/lib/model/index.js +0 -7
- package/lib/model/questions.d.ts +0 -45
- package/lib/model/questions.js +0 -89
- package/lib/model/scrutins.d.ts +0 -13
- package/lib/model/scrutins.js +0 -114
- package/lib/model/seance.d.ts +0 -3
- package/lib/model/seance.js +0 -267
- package/lib/model/sens.d.ts +0 -146
- package/lib/model/sens.js +0 -454
- package/lib/model/texte.d.ts +0 -7
- package/lib/model/texte.js +0 -228
- package/lib/model/util.d.ts +0 -9
- package/lib/model/util.js +0 -38
- package/lib/parsers/texte.d.ts +0 -7
- package/lib/parsers/texte.js +0 -228
- package/lib/raw_types/ameli.d.ts +0 -914
- package/lib/raw_types/ameli.js +0 -5
- package/lib/raw_types/debats.d.ts +0 -207
- package/lib/raw_types/debats.js +0 -5
- package/lib/raw_types/dosleg.d.ts +0 -1619
- package/lib/raw_types/dosleg.js +0 -5
- package/lib/raw_types/questions.d.ts +0 -423
- package/lib/raw_types/questions.js +0 -5
- package/lib/raw_types/senat.d.ts +0 -11372
- package/lib/raw_types/senat.js +0 -5
- package/lib/raw_types/sens.d.ts +0 -8248
- package/lib/raw_types/sens.js +0 -5
- package/lib/raw_types_schemats/ameli.d.ts +0 -539
- package/lib/raw_types_schemats/ameli.js +0 -2
- package/lib/raw_types_schemats/debats.d.ts +0 -127
- package/lib/raw_types_schemats/debats.js +0 -2
- package/lib/raw_types_schemats/dosleg.d.ts +0 -977
- package/lib/raw_types_schemats/dosleg.js +0 -2
- package/lib/raw_types_schemats/questions.d.ts +0 -237
- package/lib/raw_types_schemats/questions.js +0 -2
- package/lib/raw_types_schemats/sens.d.ts +0 -6915
- package/lib/raw_types_schemats/sens.js +0 -2
- package/lib/scripts/convert_data.d.ts +0 -1
- package/lib/scripts/convert_data.js +0 -354
- package/lib/scripts/data-download.d.ts +0 -1
- package/lib/scripts/data-download.js +0 -12
- package/lib/scripts/datautil.d.ts +0 -8
- package/lib/scripts/datautil.js +0 -34
- package/lib/scripts/parse_textes.d.ts +0 -1
- package/lib/scripts/parse_textes.js +0 -44
- package/lib/scripts/retrieve_agenda.d.ts +0 -1
- package/lib/scripts/retrieve_agenda.js +0 -132
- package/lib/scripts/retrieve_cr_commission.d.ts +0 -1
- package/lib/scripts/retrieve_cr_commission.js +0 -364
- package/lib/scripts/retrieve_cr_seance.d.ts +0 -6
- package/lib/scripts/retrieve_cr_seance.js +0 -347
- package/lib/scripts/retrieve_documents.d.ts +0 -3
- package/lib/scripts/retrieve_documents.js +0 -219
- package/lib/scripts/retrieve_open_data.d.ts +0 -1
- package/lib/scripts/retrieve_open_data.js +0 -316
- package/lib/scripts/retrieve_senateurs_photos.d.ts +0 -1
- package/lib/scripts/retrieve_senateurs_photos.js +0 -147
- package/lib/scripts/retrieve_videos.d.ts +0 -1
- package/lib/scripts/retrieve_videos.js +0 -461
- package/lib/scripts/shared/cli_helpers.d.ts +0 -95
- package/lib/scripts/shared/cli_helpers.js +0 -91
- package/lib/scripts/shared/util.d.ts +0 -4
- package/lib/scripts/shared/util.js +0 -35
- package/lib/scripts/test_iter_load.d.ts +0 -1
- package/lib/scripts/test_iter_load.js +0 -12
- package/lib/src/utils/nvs-timecode.d.ts +0 -17
- package/lib/src/utils/nvs-timecode.js +0 -79
- package/lib/src/utils/weights_scoring_config.d.ts +0 -2
- package/lib/src/utils/weights_scoring_config.js +0 -15
- package/lib/strings.d.ts +0 -1
- package/lib/strings.js +0 -18
- package/lib/types/agenda.d.ts +0 -44
- package/lib/types/agenda.js +0 -1
- package/lib/types/ameli.d.ts +0 -5
- package/lib/types/ameli.js +0 -1
- package/lib/types/compte_rendu.d.ts +0 -83
- package/lib/types/compte_rendu.js +0 -1
- package/lib/types/debats.d.ts +0 -2
- package/lib/types/debats.js +0 -1
- package/lib/types/dosleg.d.ts +0 -70
- package/lib/types/dosleg.js +0 -1
- package/lib/types/questions.d.ts +0 -2
- package/lib/types/questions.js +0 -1
- package/lib/types/sens.d.ts +0 -10
- package/lib/types/sens.js +0 -1
- package/lib/types/sessions.d.ts +0 -5
- package/lib/types/sessions.js +0 -84
- package/lib/types/texte.d.ts +0 -74
- package/lib/types/texte.js +0 -16
- package/lib/utils/cr_spliting.d.ts +0 -28
- package/lib/utils/cr_spliting.js +0 -265
- package/lib/utils/date.d.ts +0 -10
- package/lib/utils/date.js +0 -100
- package/lib/utils/nvs-timecode.d.ts +0 -7
- package/lib/utils/nvs-timecode.js +0 -79
- package/lib/utils/reunion_grouping.d.ts +0 -11
- package/lib/utils/reunion_grouping.js +0 -337
- package/lib/utils/reunion_odj_building.d.ts +0 -5
- package/lib/utils/reunion_odj_building.js +0 -154
- package/lib/utils/reunion_parsing.d.ts +0 -23
- package/lib/utils/reunion_parsing.js +0 -209
- package/lib/utils/scoring.d.ts +0 -14
- package/lib/utils/scoring.js +0 -147
- package/lib/utils/string_cleaning.d.ts +0 -7
- package/lib/utils/string_cleaning.js +0 -57
- package/lib/validators/config.d.ts +0 -9
- package/lib/validators/config.js +0 -10
package/lib/src/model/sens.d.ts
CHANGED
|
@@ -54,6 +54,24 @@ declare const findAllQuery: import("kysely").SelectQueryBuilder<{
|
|
|
54
54
|
date_fin: string;
|
|
55
55
|
libelle: unknown;
|
|
56
56
|
}[];
|
|
57
|
+
points_contact: {
|
|
58
|
+
[x: string]: any;
|
|
59
|
+
adresses: {
|
|
60
|
+
numero_voie: any;
|
|
61
|
+
nom_voie: any;
|
|
62
|
+
complement: any;
|
|
63
|
+
complement2: any;
|
|
64
|
+
code_postal: any;
|
|
65
|
+
commune: any;
|
|
66
|
+
code_cedex: any;
|
|
67
|
+
libelle_cedex: any;
|
|
68
|
+
bureau_distributeur: any;
|
|
69
|
+
}[];
|
|
70
|
+
telephones: {
|
|
71
|
+
type: any;
|
|
72
|
+
numero: any;
|
|
73
|
+
}[];
|
|
74
|
+
}[];
|
|
57
75
|
}>;
|
|
58
76
|
declare const findAllCirconscriptionsQuery: import("kysely").SelectQueryBuilder<{
|
|
59
77
|
[x: string]: any;
|
|
@@ -124,6 +142,24 @@ export declare function findAll(): AsyncIterableIterator<{
|
|
|
124
142
|
date_fin: string;
|
|
125
143
|
libelle: unknown;
|
|
126
144
|
}[];
|
|
145
|
+
points_contact: {
|
|
146
|
+
[x: string]: any;
|
|
147
|
+
adresses: {
|
|
148
|
+
numero_voie: any;
|
|
149
|
+
nom_voie: any;
|
|
150
|
+
complement: any;
|
|
151
|
+
complement2: any;
|
|
152
|
+
code_postal: any;
|
|
153
|
+
commune: any;
|
|
154
|
+
code_cedex: any;
|
|
155
|
+
libelle_cedex: any;
|
|
156
|
+
bureau_distributeur: any;
|
|
157
|
+
}[];
|
|
158
|
+
telephones: {
|
|
159
|
+
type: any;
|
|
160
|
+
numero: any;
|
|
161
|
+
}[];
|
|
162
|
+
}[];
|
|
127
163
|
}>;
|
|
128
164
|
export declare function findAllCirconscriptions(): AsyncIterableIterator<{
|
|
129
165
|
[x: string]: any;
|
package/lib/src/model/sens.js
CHANGED
|
@@ -290,14 +290,44 @@ function mandatsMinistre(senMat) {
|
|
|
290
290
|
.where("minind.senmat", "=", senMat)
|
|
291
291
|
.orderBy("minind.mindatdeb", "desc"));
|
|
292
292
|
}
|
|
293
|
-
function
|
|
293
|
+
function adressesPoicon(poiconId) {
|
|
294
|
+
return jsonArrayFrom(dbSenat
|
|
295
|
+
.withSchema("sens")
|
|
296
|
+
.selectFrom("adresse")
|
|
297
|
+
.where("adresse.poiconid", "=", poiconId)
|
|
298
|
+
.select([
|
|
299
|
+
"adresse.adrnumvoi as numero_voie",
|
|
300
|
+
"adresse.adrnomvoi as nom_voie",
|
|
301
|
+
"adresse.adrcmp as complement",
|
|
302
|
+
"adresse.adrcmp2 as complement2",
|
|
303
|
+
"adresse.adrcodpos as code_postal",
|
|
304
|
+
"adresse.adrcom as commune",
|
|
305
|
+
"adresse.adrcdxcod as code_cedex",
|
|
306
|
+
"adresse.adrcdxlib as libelle_cedex",
|
|
307
|
+
"adresse.adrburdis as bureau_distributeur",
|
|
308
|
+
])
|
|
309
|
+
.orderBy("adresse.adrnumtri", "asc"));
|
|
310
|
+
}
|
|
311
|
+
function telephonesPoicon(poiconId) {
|
|
312
|
+
return jsonArrayFrom(dbSenat
|
|
313
|
+
.withSchema("sens")
|
|
314
|
+
.selectFrom("telephone")
|
|
315
|
+
.where("telephone.poiconid", "=", poiconId)
|
|
316
|
+
.select(["telephone.typtelcod as type", "telephone.telnum as numero"])
|
|
317
|
+
.orderBy("telephone.telnumtri", "asc"));
|
|
318
|
+
}
|
|
319
|
+
function pointsContact(senMat) {
|
|
294
320
|
return jsonArrayFrom(dbSenat
|
|
295
321
|
.withSchema("sens")
|
|
296
322
|
.selectFrom("poicon")
|
|
297
|
-
.leftJoin("adresse", "poicon.poiconid", "adresse.poiconid")
|
|
298
|
-
.leftJoin("mel", "poicon.poiconid", "mel.poiconid")
|
|
299
|
-
.leftJoin("telephone", "poicon.poiconid", "telephone.poiconid")
|
|
300
323
|
.where("poicon.senmat", "=", senMat)
|
|
324
|
+
.select(({ ref }) => [
|
|
325
|
+
"poicon.poiconid as id",
|
|
326
|
+
"poicon.typpoiconcod as type",
|
|
327
|
+
"poicon.poiconlib as libelle",
|
|
328
|
+
adressesPoicon(ref("poicon.poiconid")).as("adresses"),
|
|
329
|
+
telephonesPoicon(ref("poicon.poiconid")).as("telephones"),
|
|
330
|
+
])
|
|
301
331
|
.orderBy("poicon.poiconnumtri", "asc"));
|
|
302
332
|
}
|
|
303
333
|
function urls(senMat) {
|
|
@@ -347,6 +377,7 @@ const findAllQuery = dbSenat
|
|
|
347
377
|
mandatsMembreDelegation(ref("sen.senmat")).as("delegations"),
|
|
348
378
|
mandatsMembreGroupePolitique(ref("sen.senmat")).as("groupes"),
|
|
349
379
|
fonctionsBureau(ref("sen.senmat")).as("fonctions_bureau"),
|
|
380
|
+
pointsContact(ref("sen.senmat")).as("points_contact"),
|
|
350
381
|
]);
|
|
351
382
|
const findAllCirconscriptionsQuery = dbSenat
|
|
352
383
|
.withSchema("sens")
|
|
@@ -101,6 +101,9 @@ async function processDocument(url, destPath, docDate, options) {
|
|
|
101
101
|
}
|
|
102
102
|
export async function processTexte(texteMetadata, originalTextesDir, transformedTextesDir, enrichedTextesDir, options) {
|
|
103
103
|
const texteDir = path.join(originalTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name);
|
|
104
|
+
// Pre-compute whether the parsed JSON output already exists, to avoid re-parsing unchanged files
|
|
105
|
+
const parsedJsonPath = path.join(transformedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name, `${texteMetadata.name}.json`);
|
|
106
|
+
const parsedOutputExists = options.parseDocuments ? await fs.pathExists(parsedJsonPath) : false;
|
|
104
107
|
let exposeDesMotifsContent = null;
|
|
105
108
|
if (texteMetadata.url_expose_des_motifs) {
|
|
106
109
|
const exposePath = path.join(texteDir, `${texteMetadata.name}-expose.html`);
|
|
@@ -108,7 +111,8 @@ export async function processTexte(texteMetadata, originalTextesDir, transformed
|
|
|
108
111
|
if (res.buffer) {
|
|
109
112
|
exposeDesMotifsContent = res.buffer;
|
|
110
113
|
}
|
|
111
|
-
else if (res.skipped && options.parseDocuments) {
|
|
114
|
+
else if (res.skipped && options.parseDocuments && !parsedOutputExists) {
|
|
115
|
+
// Only load expose from disk if we'll actually need to re-parse
|
|
112
116
|
if (await fs.pathExists(exposePath)) {
|
|
113
117
|
exposeDesMotifsContent = await fs.readFile(exposePath);
|
|
114
118
|
}
|
|
@@ -128,23 +132,28 @@ export async function processTexte(texteMetadata, originalTextesDir, transformed
|
|
|
128
132
|
const result = await processDocument(format.url.toString(), destPath, texteMetadata.date, options);
|
|
129
133
|
// Specific logic: Parsing (Only applies to XML)
|
|
130
134
|
if (format.isParseTarget && options.parseDocuments) {
|
|
131
|
-
if
|
|
135
|
+
// Skip re-parsing if the XML was not newly downloaded AND the parsed output already exists
|
|
136
|
+
const needsParsing = !result.skipped || !parsedOutputExists;
|
|
137
|
+
if (needsParsing && (result.buffer !== null || (await fs.pathExists(destPath)))) {
|
|
132
138
|
await parseDocument(texteMetadata.session, transformedTextesDir, destPath, texteMetadata.name, result.buffer, exposeDesMotifsContent, options);
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
texteXmlContent = await fs.readFile(destPath, "utf-8");
|
|
137
|
-
}
|
|
138
|
-
else if (result.buffer !== null) {
|
|
139
|
-
texteXmlContent = textDecoder.decode(result.buffer);
|
|
140
|
-
}
|
|
141
|
-
if (texteXmlContent !== null) {
|
|
142
|
-
try {
|
|
143
|
-
await convertSenatXmlToHtml(texteXmlContent, path.join(enrichedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name, `${texteMetadata.name}.html`));
|
|
139
|
+
let texteXmlContent = null;
|
|
140
|
+
if (result.buffer === null && (await fs.pathExists(destPath))) {
|
|
141
|
+
texteXmlContent = await fs.readFile(destPath, "utf-8");
|
|
144
142
|
}
|
|
145
|
-
|
|
146
|
-
|
|
143
|
+
else if (result.buffer !== null) {
|
|
144
|
+
texteXmlContent = textDecoder.decode(result.buffer);
|
|
147
145
|
}
|
|
146
|
+
if (texteXmlContent !== null) {
|
|
147
|
+
try {
|
|
148
|
+
await convertSenatXmlToHtml(texteXmlContent, path.join(enrichedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name, `${texteMetadata.name}.html`));
|
|
149
|
+
}
|
|
150
|
+
catch (error) {
|
|
151
|
+
console.error(`Error converting ${texteMetadata.name} to HTML: ${error.message}`);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
else if (options.verbose) {
|
|
156
|
+
console.info(`Skipping parse for already processed texte ${texteMetadata.name}…`);
|
|
148
157
|
}
|
|
149
158
|
}
|
|
150
159
|
}
|
package/package.json
CHANGED
package/lib/config.d.ts
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import "dotenv/config";
|
|
2
|
-
import { z } from "zod";
|
|
3
|
-
export declare const configSchema: z.ZodObject<{
|
|
4
|
-
db: z.ZodObject<{
|
|
5
|
-
host: z.ZodString;
|
|
6
|
-
name: z.ZodString;
|
|
7
|
-
password: z.ZodString;
|
|
8
|
-
port: z.ZodCoercedNumber<unknown>;
|
|
9
|
-
user: z.ZodString;
|
|
10
|
-
}, z.core.$strip>;
|
|
11
|
-
}, z.core.$strip>;
|
|
12
|
-
declare const _default: {
|
|
13
|
-
db: {
|
|
14
|
-
host: string;
|
|
15
|
-
name: string;
|
|
16
|
-
password: string;
|
|
17
|
-
port: number;
|
|
18
|
-
user: string;
|
|
19
|
-
};
|
|
20
|
-
};
|
|
21
|
-
export default _default;
|
package/lib/config.js
DELETED
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
import "dotenv/config";
|
|
2
|
-
import { z } from "zod";
|
|
3
|
-
const dbSchema = z.object({
|
|
4
|
-
host: z.string().trim().min(1, "Must not be empty"),
|
|
5
|
-
name: z.string().trim().min(1, "Must not be empty"),
|
|
6
|
-
password: z.string().trim().min(1, "Must not be empty"),
|
|
7
|
-
port: z.coerce.number().int().min(0).max(65535),
|
|
8
|
-
user: z.string().trim().min(1, "Must not be empty"),
|
|
9
|
-
});
|
|
10
|
-
export const configSchema = z.object({
|
|
11
|
-
db: dbSchema,
|
|
12
|
-
});
|
|
13
|
-
const config = {
|
|
14
|
-
db: {
|
|
15
|
-
host: process.env["TRICOTEUSES_SENAT_DB_HOST"] || "localhost",
|
|
16
|
-
name: process.env["TRICOTEUSES_SENAT_DB_NAME"] || "postgres",
|
|
17
|
-
password: process.env["TRICOTEUSES_SENAT_DB_PASSWORD"] || "opendata",
|
|
18
|
-
port: process.env["TRICOTEUSES_SENAT_DB_PORT"] || 5432,
|
|
19
|
-
user: process.env["TRICOTEUSES_SENAT_DB_USER"] || "opendata",
|
|
20
|
-
},
|
|
21
|
-
};
|
|
22
|
-
const result = configSchema.safeParse(config);
|
|
23
|
-
if (!result.success) {
|
|
24
|
-
console.error(`Error in configuration:\n${JSON.stringify(config, null, 2)}\nError:\n${JSON.stringify(result.error.issues, null, 2)}`);
|
|
25
|
-
process.exit(-1);
|
|
26
|
-
}
|
|
27
|
-
export default result.data;
|
package/lib/databases.d.ts
DELETED
package/lib/databases.js
DELETED
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
import { Kysely, PostgresDialect } from "kysely";
|
|
2
|
-
import * as pg from "pg";
|
|
3
|
-
import { types } from "pg";
|
|
4
|
-
import Cursor from "pg-cursor";
|
|
5
|
-
import config from "./config";
|
|
6
|
-
// Map int8 to number instead of string
|
|
7
|
-
// https://github.com/brianc/node-pg-types#use
|
|
8
|
-
pg.types.setTypeParser(types.builtins.INT8, (val) => {
|
|
9
|
-
return parseInt(val, 10);
|
|
10
|
-
});
|
|
11
|
-
const senatPool = new pg.Pool({
|
|
12
|
-
database: "senat",
|
|
13
|
-
host: config.db.host,
|
|
14
|
-
user: config.db.user,
|
|
15
|
-
password: config.db.password,
|
|
16
|
-
port: config.db.port,
|
|
17
|
-
max: 10,
|
|
18
|
-
});
|
|
19
|
-
const senatDialect = new PostgresDialect({
|
|
20
|
-
pool: senatPool,
|
|
21
|
-
cursor: Cursor,
|
|
22
|
-
});
|
|
23
|
-
export const dbSenat = new Kysely({
|
|
24
|
-
log: ["error"],
|
|
25
|
-
dialect: senatDialect,
|
|
26
|
-
});
|
package/lib/datasets.d.ts
DELETED
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
export interface Dataset {
|
|
2
|
-
database: string;
|
|
3
|
-
repairEncoding: boolean;
|
|
4
|
-
repairZip?: (dataset: Dataset, dataDir: string) => void;
|
|
5
|
-
title: string;
|
|
6
|
-
url: string;
|
|
7
|
-
indexes?: {
|
|
8
|
-
[table: string]: Array<{
|
|
9
|
-
name: string;
|
|
10
|
-
columns: string[];
|
|
11
|
-
}>;
|
|
12
|
-
};
|
|
13
|
-
}
|
|
14
|
-
export interface Datasets {
|
|
15
|
-
ameli: Dataset;
|
|
16
|
-
debats: Dataset;
|
|
17
|
-
dosleg: Dataset;
|
|
18
|
-
questions: Dataset;
|
|
19
|
-
sens: Dataset;
|
|
20
|
-
}
|
|
21
|
-
export declare enum EnabledDatasets {
|
|
22
|
-
None = 0,
|
|
23
|
-
Ameli = 1,
|
|
24
|
-
Debats = 2,
|
|
25
|
-
DosLeg = 4,
|
|
26
|
-
Questions = 8,
|
|
27
|
-
Sens = 16,
|
|
28
|
-
PhotosSenateurs = 32,
|
|
29
|
-
Agenda = 64,
|
|
30
|
-
All = 127
|
|
31
|
-
}
|
|
32
|
-
export declare const datasets: Datasets;
|
|
33
|
-
export declare function getEnabledDatasets(categories: string[]): EnabledDatasets;
|
|
34
|
-
export declare function getChosenDatasets(enabledDatasets: EnabledDatasets): Dataset[];
|
package/lib/datasets.js
DELETED
|
@@ -1,233 +0,0 @@
|
|
|
1
|
-
import fs from "fs-extra";
|
|
2
|
-
import path from "path";
|
|
3
|
-
import assert from "assert";
|
|
4
|
-
export var EnabledDatasets;
|
|
5
|
-
(function (EnabledDatasets) {
|
|
6
|
-
EnabledDatasets[EnabledDatasets["None"] = 0] = "None";
|
|
7
|
-
EnabledDatasets[EnabledDatasets["Ameli"] = 1] = "Ameli";
|
|
8
|
-
EnabledDatasets[EnabledDatasets["Debats"] = 2] = "Debats";
|
|
9
|
-
EnabledDatasets[EnabledDatasets["DosLeg"] = 4] = "DosLeg";
|
|
10
|
-
EnabledDatasets[EnabledDatasets["Questions"] = 8] = "Questions";
|
|
11
|
-
EnabledDatasets[EnabledDatasets["Sens"] = 16] = "Sens";
|
|
12
|
-
EnabledDatasets[EnabledDatasets["PhotosSenateurs"] = 32] = "PhotosSenateurs";
|
|
13
|
-
EnabledDatasets[EnabledDatasets["Agenda"] = 64] = "Agenda";
|
|
14
|
-
EnabledDatasets[EnabledDatasets["All"] = 127] = "All";
|
|
15
|
-
})(EnabledDatasets || (EnabledDatasets = {}));
|
|
16
|
-
export const datasets = {
|
|
17
|
-
ameli: {
|
|
18
|
-
database: "ameli",
|
|
19
|
-
repairEncoding: true,
|
|
20
|
-
repairZip: (dataset, dataDir) => {
|
|
21
|
-
const sqlFilename = `${dataset.database}.sql`;
|
|
22
|
-
const sqlFilePath = path.join(dataDir, sqlFilename);
|
|
23
|
-
fs.removeSync(sqlFilePath);
|
|
24
|
-
fs.moveSync(path.join(dataDir, "var", "opt", "opendata", sqlFilename), sqlFilePath);
|
|
25
|
-
},
|
|
26
|
-
title: "Amendements",
|
|
27
|
-
url: "https://data.senat.fr/data/ameli/ameli.zip",
|
|
28
|
-
indexes: {
|
|
29
|
-
amdsen: [
|
|
30
|
-
{ name: "idx_amdid", columns: ["amdid"] },
|
|
31
|
-
{ name: "idx_senid", columns: ["senid"] },
|
|
32
|
-
{ name: "idx_grpid", columns: ["grpid"] },
|
|
33
|
-
],
|
|
34
|
-
sen_ameli: [{ name: "idx_entid", columns: ["entid"] }],
|
|
35
|
-
grppol_ameli: [{ name: "idx_entid", columns: ["entid"] }],
|
|
36
|
-
sub: [
|
|
37
|
-
{ name: "idx_id", columns: ["id"] },
|
|
38
|
-
{ name: "idx_typid", columns: ["typid"] },
|
|
39
|
-
{ name: "idx_merid", columns: ["merid"] },
|
|
40
|
-
{ name: "idx_comdelid", columns: ["comdelid"] },
|
|
41
|
-
],
|
|
42
|
-
typsub: [{ name: "idx_id", columns: ["id"] }],
|
|
43
|
-
typrect: [{ name: "idx_id", columns: ["id"] }],
|
|
44
|
-
txt_ameli: [
|
|
45
|
-
{ name: "idx_id", columns: ["id"] },
|
|
46
|
-
{ name: "idx_txtetaid", columns: ["txtetaid"] },
|
|
47
|
-
{ name: "idx_sesdepid", columns: ["sesdepid"] },
|
|
48
|
-
{ name: "idx_natid", columns: ["natid"] },
|
|
49
|
-
{ name: "idx_lecid", columns: ["lecid"] },
|
|
50
|
-
],
|
|
51
|
-
etatxt: [{ name: "idx_id", columns: ["id"] }],
|
|
52
|
-
ses: [
|
|
53
|
-
{ name: "idx_id", columns: ["id"] },
|
|
54
|
-
{ name: "idx_typid", columns: ["typid"] },
|
|
55
|
-
{ name: "idx_ann", columns: ["ann"] },
|
|
56
|
-
],
|
|
57
|
-
typses: [{ name: "idx_id", columns: ["id"] }],
|
|
58
|
-
nat: [{ name: "idx_id", columns: ["id"] }],
|
|
59
|
-
lec_ameli: [{ name: "idx_id", columns: ["id"] }],
|
|
60
|
-
mot: [{ name: "idx_id", columns: ["id"] }],
|
|
61
|
-
avicom: [{ name: "idx_id", columns: ["id"] }],
|
|
62
|
-
avigvt: [{ name: "idx_id", columns: ["id"] }],
|
|
63
|
-
sor: [{ name: "idx_id", columns: ["id"] }],
|
|
64
|
-
irr: [{ name: "idx_id", columns: ["id"] }],
|
|
65
|
-
com_ameli: [{ name: "idx_entid", columns: ["entid"] }],
|
|
66
|
-
cab: [{ name: "idx_entid", columns: ["entid"] }],
|
|
67
|
-
amd: [
|
|
68
|
-
{ name: "idx_subid", columns: ["subid"] },
|
|
69
|
-
{ name: "idx_typrectid", columns: ["typrectid"] },
|
|
70
|
-
{ name: "idx_txtid", columns: ["txtid"] },
|
|
71
|
-
{ name: "idx_motid", columns: ["motid"] },
|
|
72
|
-
{ name: "idx_avcid", columns: ["avcid"] },
|
|
73
|
-
{ name: "idx_avgid", columns: ["avgid"] },
|
|
74
|
-
{ name: "idx_sorid", columns: ["sorid"] },
|
|
75
|
-
{ name: "idx_irrid", columns: ["irrid"] },
|
|
76
|
-
{ name: "idx_nomentid", columns: ["nomentid"] },
|
|
77
|
-
],
|
|
78
|
-
},
|
|
79
|
-
},
|
|
80
|
-
debats: {
|
|
81
|
-
database: "debats",
|
|
82
|
-
repairEncoding: false,
|
|
83
|
-
title: "Informations relatives aux comptes rendus intégraux de la séance publique",
|
|
84
|
-
url: "https://data.senat.fr/data/debats/debats.zip",
|
|
85
|
-
indexes: {
|
|
86
|
-
secdis: [
|
|
87
|
-
{ name: "idx_datsea", columns: ["datsea"] },
|
|
88
|
-
{ name: "idx_typseccod", columns: ["typseccod"] },
|
|
89
|
-
],
|
|
90
|
-
intpjl: [{ name: "idx_secdiscle", columns: ["secdiscle"] }],
|
|
91
|
-
secdivers: [
|
|
92
|
-
{ name: "idx_datsea", columns: ["datsea"] },
|
|
93
|
-
{ name: "idx_typseccod", columns: ["typseccod"] },
|
|
94
|
-
],
|
|
95
|
-
intdivers: [{ name: "idx_intdiverscle", columns: ["intdiverscle"] }],
|
|
96
|
-
lecassdeb: [{ name: "idx_datsea", columns: ["datsea"] }],
|
|
97
|
-
},
|
|
98
|
-
},
|
|
99
|
-
dosleg: {
|
|
100
|
-
database: "dosleg",
|
|
101
|
-
repairEncoding: false,
|
|
102
|
-
title: "Dossiers législatifs",
|
|
103
|
-
url: "https://data.senat.fr/data/dosleg/dosleg.zip",
|
|
104
|
-
indexes: {
|
|
105
|
-
amescr: [{ name: "idx_scrnum", columns: ["scrnum"] }],
|
|
106
|
-
auteur: [{ name: "idx_autcod", columns: ["autcod"] }],
|
|
107
|
-
corscr: [{ name: "idx_scrnum", columns: ["scrnum"] }],
|
|
108
|
-
date_seance: [
|
|
109
|
-
{ name: "idx_code", columns: ["code"] },
|
|
110
|
-
{ name: "idx_lecidt", columns: ["lecidt"] },
|
|
111
|
-
],
|
|
112
|
-
deccoc: [{ name: "idx_deccoccod", columns: ["deccoccod"] }],
|
|
113
|
-
etaloi: [{ name: "idx_etaloicod", columns: ["etaloicod"] }],
|
|
114
|
-
lecass: [
|
|
115
|
-
{ name: "idx_lecidt", columns: ["lecidt"] },
|
|
116
|
-
{ name: "idx_codass", columns: ["codass"] },
|
|
117
|
-
{ name: "idx_orgcod", columns: ["orgcod"] },
|
|
118
|
-
{ name: "idx_orippr", columns: ["orippr"] },
|
|
119
|
-
],
|
|
120
|
-
lecassrap: [
|
|
121
|
-
{ name: "idx_rapcod", columns: ["rapcod"] },
|
|
122
|
-
{ name: "idx_lecassidt", columns: ["lecassidt"] },
|
|
123
|
-
],
|
|
124
|
-
lecture: [
|
|
125
|
-
{ name: "idx_loicod", columns: ["loicod"] },
|
|
126
|
-
{ name: "idx_typleccod", columns: ["typleccod"] },
|
|
127
|
-
{ name: "idx_lecidt", columns: ["lecidt"] },
|
|
128
|
-
],
|
|
129
|
-
loi: [
|
|
130
|
-
{ name: "idx_typloicod", columns: ["typloicod"] },
|
|
131
|
-
{ name: "idx_etaloicod", columns: ["etaloicod"] },
|
|
132
|
-
{ name: "idx_deccoccod", columns: ["deccoccod"] },
|
|
133
|
-
{ name: "idx_loicod", columns: ["loicod"] },
|
|
134
|
-
],
|
|
135
|
-
loithe: [
|
|
136
|
-
{ name: "idx_thecle", columns: ["thecle"] },
|
|
137
|
-
{ name: "idx_loicod", columns: ["loicod"] },
|
|
138
|
-
],
|
|
139
|
-
posvot: [{ name: "idx_posvotcod", columns: ["posvotcod"] }],
|
|
140
|
-
rap: [
|
|
141
|
-
{ name: "idx_rapcod", columns: ["rapcod"] },
|
|
142
|
-
{ name: "idx_coddenrap", columns: ["coddenrap"] },
|
|
143
|
-
],
|
|
144
|
-
rolsig: [{ name: "idx_signataire", columns: ["signataire"] }],
|
|
145
|
-
scr: [
|
|
146
|
-
{ name: "idx_code", columns: ["code"] },
|
|
147
|
-
{ name: "idx_scrnum", columns: ["scrnum"] },
|
|
148
|
-
{ name: "idx_sesann", columns: ["sesann"] },
|
|
149
|
-
],
|
|
150
|
-
stavot: [{ name: "idx_stavotidt", columns: ["stavotidt"] }],
|
|
151
|
-
texte: [
|
|
152
|
-
{ name: "idx_lecassidt", columns: ["lecassidt"] },
|
|
153
|
-
{ name: "idx_oritxtcod", columns: ["oritxtcod"] },
|
|
154
|
-
{ name: "idx_typtxtcod", columns: ["typtxtcod"] },
|
|
155
|
-
{ name: "idx_texcod", columns: ["texcod"] },
|
|
156
|
-
],
|
|
157
|
-
the: [{ name: "idx_thecle", columns: ["thecle"] }],
|
|
158
|
-
titsen: [{ name: "idx_titsencod", columns: ["titsencod"] }],
|
|
159
|
-
typloi: [{ name: "idx_typloicod", columns: ["typloicod"] }],
|
|
160
|
-
typtxt: [{ name: "idx_typtxtcod", columns: ["typtxtcod"] }],
|
|
161
|
-
votsen: [
|
|
162
|
-
{ name: "idx_scrnum", columns: ["scrnum"] },
|
|
163
|
-
{ name: "idx_sesann", columns: ["sesann"] },
|
|
164
|
-
{ name: "idx_titsencod", columns: ["titsencod"] },
|
|
165
|
-
{ name: "idx_stavotidt", columns: ["stavotidt"] },
|
|
166
|
-
{ name: "idx_posvotcod", columns: ["posvotcod"] },
|
|
167
|
-
{ name: "idx_senmat", columns: ["senmat"] },
|
|
168
|
-
],
|
|
169
|
-
},
|
|
170
|
-
},
|
|
171
|
-
questions: {
|
|
172
|
-
database: "questions",
|
|
173
|
-
repairEncoding: false,
|
|
174
|
-
title: "Questions écrites et orales posées par les sénateurs au Gouvernement",
|
|
175
|
-
url: "https://data.senat.fr/data/questions/questions.zip",
|
|
176
|
-
indexes: {
|
|
177
|
-
tam_reponses: [{ name: "idx_idque", columns: ["idque"] }],
|
|
178
|
-
tam_questions: [
|
|
179
|
-
{ name: "idx_sorquecod", columns: ["sorquecod"] },
|
|
180
|
-
{ name: "idx_etaquecod", columns: ["etaquecod"] },
|
|
181
|
-
],
|
|
182
|
-
the: [{ name: "idx_thenouidt", columns: ["thenouidt"] }],
|
|
183
|
-
sortquestion: [{ name: "idx_sorquecod", columns: ["sorquecod"] }],
|
|
184
|
-
etatquestion: [{ name: "idx_etaquecod", columns: ["etaquecod"] }],
|
|
185
|
-
},
|
|
186
|
-
},
|
|
187
|
-
sens: {
|
|
188
|
-
database: "sens",
|
|
189
|
-
repairEncoding: false,
|
|
190
|
-
repairZip: (dataset, dataDir) => {
|
|
191
|
-
const sqlFilename = `${dataset.database}.sql`;
|
|
192
|
-
const sqlFilePath = path.join(dataDir, sqlFilename);
|
|
193
|
-
fs.removeSync(sqlFilePath);
|
|
194
|
-
fs.moveSync(path.join(dataDir, "export_sens.sql"), sqlFilePath);
|
|
195
|
-
},
|
|
196
|
-
title: "Sénateurs (y compris organes et présence)",
|
|
197
|
-
url: "https://data.senat.fr/data/senateurs/export_sens.zip",
|
|
198
|
-
indexes: {
|
|
199
|
-
dpt: [
|
|
200
|
-
{ name: "idx_dptnum", columns: ["dptnum"] },
|
|
201
|
-
{ name: "idx_regcod", columns: ["regcod"] },
|
|
202
|
-
],
|
|
203
|
-
activite_loi: [{ name: "idx_actid", columns: ["actid"] }],
|
|
204
|
-
activite_delegation: [{ name: "idx_actid", columns: ["actid"] }],
|
|
205
|
-
activite_participant: [{ name: "idx_senmat", columns: ["senmat"] }],
|
|
206
|
-
activite_senateur: [{ name: "idx_senmat", columns: ["senmat"] }],
|
|
207
|
-
senbur: [
|
|
208
|
-
{ name: "idx_senmat", columns: ["senmat"] },
|
|
209
|
-
{ name: "idx_burcod", columns: ["burcod"] },
|
|
210
|
-
],
|
|
211
|
-
elucan: [
|
|
212
|
-
{ name: "idx_senmat", columns: ["senmat"] },
|
|
213
|
-
{ name: "idx_typmancod", columns: ["typmancod"] },
|
|
214
|
-
],
|
|
215
|
-
},
|
|
216
|
-
},
|
|
217
|
-
};
|
|
218
|
-
export function getEnabledDatasets(categories) {
|
|
219
|
-
categories.map((datasetName) => assert.notStrictEqual(EnabledDatasets[datasetName], undefined, `Unknown name of dataset: ${datasetName}`));
|
|
220
|
-
return categories.reduce((enabledDatasets, datasetName) => {
|
|
221
|
-
const enabledDataset = EnabledDatasets[datasetName];
|
|
222
|
-
return enabledDatasets | (enabledDataset || EnabledDatasets.None);
|
|
223
|
-
}, EnabledDatasets.None);
|
|
224
|
-
}
|
|
225
|
-
export function getChosenDatasets(enabledDatasets) {
|
|
226
|
-
return [
|
|
227
|
-
enabledDatasets & EnabledDatasets.Ameli ? datasets.ameli : null,
|
|
228
|
-
enabledDatasets & EnabledDatasets.Debats ? datasets.debats : null,
|
|
229
|
-
enabledDatasets & EnabledDatasets.DosLeg ? datasets.dosleg : null,
|
|
230
|
-
enabledDatasets & EnabledDatasets.Questions ? datasets.questions : null,
|
|
231
|
-
enabledDatasets & EnabledDatasets.Sens ? datasets.sens : null,
|
|
232
|
-
].filter((dataset) => dataset !== null);
|
|
233
|
-
}
|
package/lib/git.d.ts
DELETED
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
export declare function initRepo(repositoryDir: string): void;
|
|
2
|
-
export declare function commit(repositoryDir: string, message: string): boolean;
|
|
3
|
-
export declare function commitAndPush(repositoryDir: string, message: string, remotes?: string[]): number;
|
|
4
|
-
export declare function resetAndPull(gitDir: string): boolean;
|
|
5
|
-
export declare function clone(gitGroupUrl: string | undefined, gitName: string, workingDir: string): void;
|
|
6
|
-
export declare function run(repositoryDir: string, args: string, verbose?: boolean): string;
|
|
7
|
-
export declare function test(repositoryDir: string, args: string, verbose?: boolean): boolean;
|
|
8
|
-
/**
|
|
9
|
-
* Information about a changed file in git
|
|
10
|
-
*/
|
|
11
|
-
export interface GitChangedFile {
|
|
12
|
-
path: string;
|
|
13
|
-
status: "A" | "M" | "D" | "R" | "C" | "T" | "U";
|
|
14
|
-
}
|
|
15
|
-
/**
|
|
16
|
-
* Get the list of files that have changed since a specific commit in a git repository.
|
|
17
|
-
* @param repositoryDir The directory of the git repository
|
|
18
|
-
* @param sinceCommit The commit hash to compare against (e.g., "HEAD~1", "abc123", etc.)
|
|
19
|
-
* @param options Options for filtering
|
|
20
|
-
* @param options.diffFilter Git diff-filter string (default: "AMR").
|
|
21
|
-
* A=Added, M=Modified, D=Deleted, R=Renamed, C=Copied, T=Type changed, U=Unmerged
|
|
22
|
-
* @returns A Map of file paths to their git status
|
|
23
|
-
*/
|
|
24
|
-
export declare function getChangedFilesSinceCommit(repositoryDir: string, sinceCommit: string, options?: {
|
|
25
|
-
diffFilter?: string;
|
|
26
|
-
}): Map<string, GitChangedFile["status"]>;
|