@tricoteuses/senat 2.22.13 → 2.22.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/src/loaders.d.ts +2 -8
- package/lib/src/loaders.js +7 -25
- package/lib/src/scripts/retrieve_cr_commission.js +1 -1
- package/lib/src/scripts/retrieve_cr_seance.js +1 -1
- package/lib/src/scripts/retrieve_videos.js +2 -2
- package/lib/tests/test_iter_load.test.js +17 -0
- package/package.json +2 -2
- package/lib/config.d.ts +0 -21
- package/lib/config.js +0 -27
- package/lib/databases.d.ts +0 -2
- package/lib/databases.js +0 -26
- package/lib/datasets.d.ts +0 -34
- package/lib/datasets.js +0 -233
- package/lib/git.d.ts +0 -26
- package/lib/git.js +0 -167
- package/lib/index.d.ts +0 -13
- package/lib/index.js +0 -1
- package/lib/loaders.d.ts +0 -58
- package/lib/loaders.js +0 -286
- package/lib/model/agenda.d.ts +0 -6
- package/lib/model/agenda.js +0 -148
- package/lib/model/ameli.d.ts +0 -51
- package/lib/model/ameli.js +0 -147
- package/lib/model/commission.d.ts +0 -18
- package/lib/model/commission.js +0 -269
- package/lib/model/debats.d.ts +0 -67
- package/lib/model/debats.js +0 -95
- package/lib/model/documents.d.ts +0 -12
- package/lib/model/documents.js +0 -138
- package/lib/model/dosleg.d.ts +0 -7
- package/lib/model/dosleg.js +0 -326
- package/lib/model/index.d.ts +0 -7
- package/lib/model/index.js +0 -7
- package/lib/model/questions.d.ts +0 -45
- package/lib/model/questions.js +0 -89
- package/lib/model/scrutins.d.ts +0 -13
- package/lib/model/scrutins.js +0 -114
- package/lib/model/seance.d.ts +0 -3
- package/lib/model/seance.js +0 -267
- package/lib/model/sens.d.ts +0 -146
- package/lib/model/sens.js +0 -454
- package/lib/model/texte.d.ts +0 -7
- package/lib/model/texte.js +0 -228
- package/lib/model/util.d.ts +0 -9
- package/lib/model/util.js +0 -38
- package/lib/parsers/texte.d.ts +0 -7
- package/lib/parsers/texte.js +0 -228
- package/lib/raw_types/ameli.d.ts +0 -914
- package/lib/raw_types/ameli.js +0 -5
- package/lib/raw_types/debats.d.ts +0 -207
- package/lib/raw_types/debats.js +0 -5
- package/lib/raw_types/dosleg.d.ts +0 -1619
- package/lib/raw_types/dosleg.js +0 -5
- package/lib/raw_types/questions.d.ts +0 -423
- package/lib/raw_types/questions.js +0 -5
- package/lib/raw_types/senat.d.ts +0 -11372
- package/lib/raw_types/senat.js +0 -5
- package/lib/raw_types/sens.d.ts +0 -8248
- package/lib/raw_types/sens.js +0 -5
- package/lib/raw_types_schemats/ameli.d.ts +0 -539
- package/lib/raw_types_schemats/ameli.js +0 -2
- package/lib/raw_types_schemats/debats.d.ts +0 -127
- package/lib/raw_types_schemats/debats.js +0 -2
- package/lib/raw_types_schemats/dosleg.d.ts +0 -977
- package/lib/raw_types_schemats/dosleg.js +0 -2
- package/lib/raw_types_schemats/questions.d.ts +0 -237
- package/lib/raw_types_schemats/questions.js +0 -2
- package/lib/raw_types_schemats/sens.d.ts +0 -6915
- package/lib/raw_types_schemats/sens.js +0 -2
- package/lib/scripts/convert_data.js +0 -354
- package/lib/scripts/data-download.d.ts +0 -1
- package/lib/scripts/data-download.js +0 -12
- package/lib/scripts/datautil.d.ts +0 -8
- package/lib/scripts/datautil.js +0 -34
- package/lib/scripts/parse_textes.d.ts +0 -1
- package/lib/scripts/parse_textes.js +0 -44
- package/lib/scripts/retrieve_agenda.d.ts +0 -1
- package/lib/scripts/retrieve_agenda.js +0 -132
- package/lib/scripts/retrieve_cr_commission.d.ts +0 -1
- package/lib/scripts/retrieve_cr_commission.js +0 -364
- package/lib/scripts/retrieve_cr_seance.d.ts +0 -6
- package/lib/scripts/retrieve_cr_seance.js +0 -347
- package/lib/scripts/retrieve_documents.d.ts +0 -3
- package/lib/scripts/retrieve_documents.js +0 -219
- package/lib/scripts/retrieve_open_data.d.ts +0 -1
- package/lib/scripts/retrieve_open_data.js +0 -316
- package/lib/scripts/retrieve_senateurs_photos.d.ts +0 -1
- package/lib/scripts/retrieve_senateurs_photos.js +0 -147
- package/lib/scripts/retrieve_videos.d.ts +0 -1
- package/lib/scripts/retrieve_videos.js +0 -461
- package/lib/scripts/shared/cli_helpers.d.ts +0 -95
- package/lib/scripts/shared/cli_helpers.js +0 -91
- package/lib/scripts/shared/util.d.ts +0 -4
- package/lib/scripts/shared/util.js +0 -35
- package/lib/scripts/test_iter_load.d.ts +0 -1
- package/lib/scripts/test_iter_load.js +0 -12
- package/lib/src/utils/nvs-timecode.d.ts +0 -17
- package/lib/src/utils/nvs-timecode.js +0 -79
- package/lib/src/utils/weights_scoring_config.d.ts +0 -2
- package/lib/src/utils/weights_scoring_config.js +0 -15
- package/lib/strings.d.ts +0 -1
- package/lib/strings.js +0 -18
- package/lib/types/agenda.d.ts +0 -44
- package/lib/types/agenda.js +0 -1
- package/lib/types/ameli.d.ts +0 -5
- package/lib/types/ameli.js +0 -1
- package/lib/types/compte_rendu.d.ts +0 -83
- package/lib/types/compte_rendu.js +0 -1
- package/lib/types/debats.d.ts +0 -2
- package/lib/types/debats.js +0 -1
- package/lib/types/dosleg.d.ts +0 -70
- package/lib/types/dosleg.js +0 -1
- package/lib/types/questions.d.ts +0 -2
- package/lib/types/questions.js +0 -1
- package/lib/types/sens.d.ts +0 -10
- package/lib/types/sens.js +0 -1
- package/lib/types/sessions.d.ts +0 -5
- package/lib/types/sessions.js +0 -84
- package/lib/types/texte.d.ts +0 -74
- package/lib/types/texte.js +0 -16
- package/lib/utils/cr_spliting.d.ts +0 -28
- package/lib/utils/cr_spliting.js +0 -265
- package/lib/utils/date.d.ts +0 -10
- package/lib/utils/date.js +0 -100
- package/lib/utils/nvs-timecode.d.ts +0 -7
- package/lib/utils/nvs-timecode.js +0 -79
- package/lib/utils/reunion_grouping.d.ts +0 -11
- package/lib/utils/reunion_grouping.js +0 -337
- package/lib/utils/reunion_odj_building.d.ts +0 -5
- package/lib/utils/reunion_odj_building.js +0 -154
- package/lib/utils/reunion_parsing.d.ts +0 -23
- package/lib/utils/reunion_parsing.js +0 -209
- package/lib/utils/scoring.d.ts +0 -14
- package/lib/utils/scoring.js +0 -147
- package/lib/utils/string_cleaning.d.ts +0 -7
- package/lib/utils/string_cleaning.js +0 -57
- package/lib/validators/config.d.ts +0 -9
- package/lib/validators/config.js +0 -10
- /package/lib/{scripts/convert_data.d.ts → tests/test_iter_load.test.d.ts} +0 -0
package/lib/src/loaders.d.ts
CHANGED
|
@@ -33,14 +33,8 @@ export type IterItem<T> = {
|
|
|
33
33
|
export declare function iterFilePaths(dirPath: string): Generator<string>;
|
|
34
34
|
export declare function iterLoadSenatAmendements(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<AmendementResult>>;
|
|
35
35
|
export declare function iterLoadSenatDebats(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DebatResult>>;
|
|
36
|
-
export declare function iterLoadSenatComptesRendusSeances(dataDir: string, session: number): Generator<
|
|
37
|
-
|
|
38
|
-
session: number;
|
|
39
|
-
}>;
|
|
40
|
-
export declare function iterLoadSenatComptesRendusCommissions(dataDir: string, session: number): Generator<{
|
|
41
|
-
compteRendu: CompteRendu;
|
|
42
|
-
session: number;
|
|
43
|
-
}>;
|
|
36
|
+
export declare function iterLoadSenatComptesRendusSeances(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<CompteRendu>>;
|
|
37
|
+
export declare function iterLoadSenatComptesRendusCommissions(dataDir: string, session: number, options?: {}): Generator<IterItem<CompteRendu>>;
|
|
44
38
|
export declare function iterLoadSenatDossiersLegislatifs(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DossierLegislatifResult>>;
|
|
45
39
|
export declare function iterLoadSenatRapportUrls(dataDir: string, session: number | undefined): Generator<IterItem<DocumentMetadata>>;
|
|
46
40
|
export declare function iterLoadSenatTexteUrls(dataDir: string, session: number | undefined): Generator<IterItem<DocumentMetadata>>;
|
package/lib/src/loaders.js
CHANGED
|
@@ -118,33 +118,15 @@ export function* iterLoadSenatDebats(dataDir, session, options = {}) {
|
|
|
118
118
|
yield debatItem;
|
|
119
119
|
}
|
|
120
120
|
}
|
|
121
|
-
function*
|
|
122
|
-
const
|
|
123
|
-
|
|
124
|
-
return;
|
|
125
|
-
}
|
|
126
|
-
const files = (fs.readdirSync(basePath) || []).filter((f) => f.endsWith(".json")).sort();
|
|
127
|
-
for (const fileName of files) {
|
|
128
|
-
const filePath = path.join(basePath, fileName);
|
|
129
|
-
try {
|
|
130
|
-
const fileContent = fs.readFileSync(filePath, "utf-8");
|
|
131
|
-
const compteRendu = JSON.parse(fileContent);
|
|
132
|
-
if (!compteRendu?.uid) {
|
|
133
|
-
console.warn(`[SN] CR without uid → ${fileName}`);
|
|
134
|
-
continue;
|
|
135
|
-
}
|
|
136
|
-
yield { compteRendu, session };
|
|
137
|
-
}
|
|
138
|
-
catch (err) {
|
|
139
|
-
console.warn(`[SN] error reading CR → ${fileName}`, err);
|
|
140
|
-
}
|
|
121
|
+
export function* iterLoadSenatComptesRendusSeances(dataDir, session, options = {}) {
|
|
122
|
+
for (const compteRenduItem of iterLoadSenatItems(dataDir, COMPTES_RENDUS_FOLDER, session, DATA_TRANSFORMED_FOLDER, options)) {
|
|
123
|
+
yield compteRenduItem;
|
|
141
124
|
}
|
|
142
125
|
}
|
|
143
|
-
export function*
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
yield* iterLoadSenatComptesRendusGeneric(dataDir, session, COMMISSION_FOLDER);
|
|
126
|
+
export function* iterLoadSenatComptesRendusCommissions(dataDir, session, options = {}) {
|
|
127
|
+
for (const compteRenduItem of iterLoadSenatItems(dataDir, COMMISSION_FOLDER, session, DATA_TRANSFORMED_FOLDER, options)) {
|
|
128
|
+
yield compteRenduItem;
|
|
129
|
+
}
|
|
148
130
|
}
|
|
149
131
|
export function* iterLoadSenatDossiersLegislatifs(dataDir, session, options = {}) {
|
|
150
132
|
for (const dossierLegislatifItem of iterLoadSenatItems(dataDir, datasets.dosleg.database, session, DOSLEG_DOSSIERS_FOLDER, options)) {
|
|
@@ -358,7 +358,7 @@ async function retrieveCommissionCRs(options = {}) {
|
|
|
358
358
|
}
|
|
359
359
|
}
|
|
360
360
|
}
|
|
361
|
-
const debatsDir = path.join(dataDir, COMMISSION_FOLDER
|
|
361
|
+
const debatsDir = path.join(dataDir, COMMISSION_FOLDER);
|
|
362
362
|
commitAndPushGit(debatsDir, options);
|
|
363
363
|
}
|
|
364
364
|
async function main() {
|
|
@@ -205,7 +205,7 @@ export async function retrieveCriXmlDump(dataDir, options = {}) {
|
|
|
205
205
|
}
|
|
206
206
|
}
|
|
207
207
|
}
|
|
208
|
-
const debatsDir = path.join(dataDir, COMPTES_RENDUS_FOLDER
|
|
208
|
+
const debatsDir = path.join(dataDir, COMPTES_RENDUS_FOLDER);
|
|
209
209
|
commitAndPushGit(debatsDir, options);
|
|
210
210
|
}
|
|
211
211
|
function commitAndPushGit(datasetDir, options) {
|
|
@@ -5,7 +5,7 @@ import fs from "fs-extra";
|
|
|
5
5
|
import fsp from "fs/promises";
|
|
6
6
|
import path from "path";
|
|
7
7
|
import * as git from "../git.js";
|
|
8
|
-
import { AGENDA_FOLDER,
|
|
8
|
+
import { AGENDA_FOLDER, iterLoadSenatAgendas } from "../loaders";
|
|
9
9
|
import { getSessionsFromStart } from "../types/sessions";
|
|
10
10
|
import { commonOptions } from "./shared/cli_helpers";
|
|
11
11
|
import { getAgendaSegmentTimecodes, buildSenatVodMasterM3u8FromNvs } from "../utils/nvs-parsing";
|
|
@@ -169,7 +169,7 @@ async function processAll(dataDir, sessions) {
|
|
|
169
169
|
}
|
|
170
170
|
}
|
|
171
171
|
}
|
|
172
|
-
const debatsDir = path.join(dataDir, AGENDA_FOLDER
|
|
172
|
+
const debatsDir = path.join(dataDir, AGENDA_FOLDER);
|
|
173
173
|
commitAndPushGit(debatsDir, options);
|
|
174
174
|
}
|
|
175
175
|
function commitAndPushGit(datasetDir, options) {
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import os from "os";
|
|
3
|
+
import path from "path";
|
|
4
|
+
import { describe, it, expect } from "vitest";
|
|
5
|
+
import { COMPTES_RENDUS_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatComptesRendusSeances } from "../src/loaders";
|
|
6
|
+
describe("iterLoadSenatComptesRendusSeances", () => {
|
|
7
|
+
it("loads comptes rendus from transformed/session folder", () => {
|
|
8
|
+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "senat-cr-"));
|
|
9
|
+
const session = 2025;
|
|
10
|
+
const targetDir = path.join(tmpDir, COMPTES_RENDUS_FOLDER, DATA_TRANSFORMED_FOLDER, String(session));
|
|
11
|
+
fs.mkdirSync(targetDir, { recursive: true });
|
|
12
|
+
fs.writeFileSync(path.join(targetDir, "cr-1.json"), JSON.stringify({ uid: "cr-1", titre: "CR 1" }), "utf8");
|
|
13
|
+
const results = Array.from(iterLoadSenatComptesRendusSeances(tmpDir, session));
|
|
14
|
+
expect(results).toHaveLength(1);
|
|
15
|
+
expect(results[0].item.uid).toBe("cr-1");
|
|
16
|
+
});
|
|
17
|
+
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tricoteuses/senat",
|
|
3
|
-
"version": "2.22.
|
|
3
|
+
"version": "2.22.15",
|
|
4
4
|
"description": "Handle French Sénat's open data",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"France",
|
|
@@ -63,7 +63,7 @@
|
|
|
63
63
|
"prepare": "npm run build",
|
|
64
64
|
"prepublishOnly": "npm run build",
|
|
65
65
|
"prettier": "prettier --write 'src/**/*.ts'",
|
|
66
|
-
"test:iter_load": "
|
|
66
|
+
"test:iter_load": "vitest tests/test_iter_load.test.ts",
|
|
67
67
|
"type-check": "tsc --noEmit",
|
|
68
68
|
"type-check:watch": "npm run type-check -- --watch",
|
|
69
69
|
"test:video-matching": "vitest tests/videoMatching.test.ts"
|
package/lib/config.d.ts
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import "dotenv/config";
|
|
2
|
-
import { z } from "zod";
|
|
3
|
-
export declare const configSchema: z.ZodObject<{
|
|
4
|
-
db: z.ZodObject<{
|
|
5
|
-
host: z.ZodString;
|
|
6
|
-
name: z.ZodString;
|
|
7
|
-
password: z.ZodString;
|
|
8
|
-
port: z.ZodCoercedNumber<unknown>;
|
|
9
|
-
user: z.ZodString;
|
|
10
|
-
}, z.core.$strip>;
|
|
11
|
-
}, z.core.$strip>;
|
|
12
|
-
declare const _default: {
|
|
13
|
-
db: {
|
|
14
|
-
host: string;
|
|
15
|
-
name: string;
|
|
16
|
-
password: string;
|
|
17
|
-
port: number;
|
|
18
|
-
user: string;
|
|
19
|
-
};
|
|
20
|
-
};
|
|
21
|
-
export default _default;
|
package/lib/config.js
DELETED
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
import "dotenv/config";
|
|
2
|
-
import { z } from "zod";
|
|
3
|
-
const dbSchema = z.object({
|
|
4
|
-
host: z.string().trim().min(1, "Must not be empty"),
|
|
5
|
-
name: z.string().trim().min(1, "Must not be empty"),
|
|
6
|
-
password: z.string().trim().min(1, "Must not be empty"),
|
|
7
|
-
port: z.coerce.number().int().min(0).max(65535),
|
|
8
|
-
user: z.string().trim().min(1, "Must not be empty"),
|
|
9
|
-
});
|
|
10
|
-
export const configSchema = z.object({
|
|
11
|
-
db: dbSchema,
|
|
12
|
-
});
|
|
13
|
-
const config = {
|
|
14
|
-
db: {
|
|
15
|
-
host: process.env["TRICOTEUSES_SENAT_DB_HOST"] || "localhost",
|
|
16
|
-
name: process.env["TRICOTEUSES_SENAT_DB_NAME"] || "postgres",
|
|
17
|
-
password: process.env["TRICOTEUSES_SENAT_DB_PASSWORD"] || "opendata",
|
|
18
|
-
port: process.env["TRICOTEUSES_SENAT_DB_PORT"] || 5432,
|
|
19
|
-
user: process.env["TRICOTEUSES_SENAT_DB_USER"] || "opendata",
|
|
20
|
-
},
|
|
21
|
-
};
|
|
22
|
-
const result = configSchema.safeParse(config);
|
|
23
|
-
if (!result.success) {
|
|
24
|
-
console.error(`Error in configuration:\n${JSON.stringify(config, null, 2)}\nError:\n${JSON.stringify(result.error.issues, null, 2)}`);
|
|
25
|
-
process.exit(-1);
|
|
26
|
-
}
|
|
27
|
-
export default result.data;
|
package/lib/databases.d.ts
DELETED
package/lib/databases.js
DELETED
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
import { Kysely, PostgresDialect } from "kysely";
|
|
2
|
-
import * as pg from "pg";
|
|
3
|
-
import { types } from "pg";
|
|
4
|
-
import Cursor from "pg-cursor";
|
|
5
|
-
import config from "./config";
|
|
6
|
-
// Map int8 to number instead of string
|
|
7
|
-
// https://github.com/brianc/node-pg-types#use
|
|
8
|
-
pg.types.setTypeParser(types.builtins.INT8, (val) => {
|
|
9
|
-
return parseInt(val, 10);
|
|
10
|
-
});
|
|
11
|
-
const senatPool = new pg.Pool({
|
|
12
|
-
database: "senat",
|
|
13
|
-
host: config.db.host,
|
|
14
|
-
user: config.db.user,
|
|
15
|
-
password: config.db.password,
|
|
16
|
-
port: config.db.port,
|
|
17
|
-
max: 10,
|
|
18
|
-
});
|
|
19
|
-
const senatDialect = new PostgresDialect({
|
|
20
|
-
pool: senatPool,
|
|
21
|
-
cursor: Cursor,
|
|
22
|
-
});
|
|
23
|
-
export const dbSenat = new Kysely({
|
|
24
|
-
log: ["error"],
|
|
25
|
-
dialect: senatDialect,
|
|
26
|
-
});
|
package/lib/datasets.d.ts
DELETED
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
export interface Dataset {
|
|
2
|
-
database: string;
|
|
3
|
-
repairEncoding: boolean;
|
|
4
|
-
repairZip?: (dataset: Dataset, dataDir: string) => void;
|
|
5
|
-
title: string;
|
|
6
|
-
url: string;
|
|
7
|
-
indexes?: {
|
|
8
|
-
[table: string]: Array<{
|
|
9
|
-
name: string;
|
|
10
|
-
columns: string[];
|
|
11
|
-
}>;
|
|
12
|
-
};
|
|
13
|
-
}
|
|
14
|
-
export interface Datasets {
|
|
15
|
-
ameli: Dataset;
|
|
16
|
-
debats: Dataset;
|
|
17
|
-
dosleg: Dataset;
|
|
18
|
-
questions: Dataset;
|
|
19
|
-
sens: Dataset;
|
|
20
|
-
}
|
|
21
|
-
export declare enum EnabledDatasets {
|
|
22
|
-
None = 0,
|
|
23
|
-
Ameli = 1,
|
|
24
|
-
Debats = 2,
|
|
25
|
-
DosLeg = 4,
|
|
26
|
-
Questions = 8,
|
|
27
|
-
Sens = 16,
|
|
28
|
-
PhotosSenateurs = 32,
|
|
29
|
-
Agenda = 64,
|
|
30
|
-
All = 127
|
|
31
|
-
}
|
|
32
|
-
export declare const datasets: Datasets;
|
|
33
|
-
export declare function getEnabledDatasets(categories: string[]): EnabledDatasets;
|
|
34
|
-
export declare function getChosenDatasets(enabledDatasets: EnabledDatasets): Dataset[];
|
package/lib/datasets.js
DELETED
|
@@ -1,233 +0,0 @@
|
|
|
1
|
-
import fs from "fs-extra";
|
|
2
|
-
import path from "path";
|
|
3
|
-
import assert from "assert";
|
|
4
|
-
export var EnabledDatasets;
|
|
5
|
-
(function (EnabledDatasets) {
|
|
6
|
-
EnabledDatasets[EnabledDatasets["None"] = 0] = "None";
|
|
7
|
-
EnabledDatasets[EnabledDatasets["Ameli"] = 1] = "Ameli";
|
|
8
|
-
EnabledDatasets[EnabledDatasets["Debats"] = 2] = "Debats";
|
|
9
|
-
EnabledDatasets[EnabledDatasets["DosLeg"] = 4] = "DosLeg";
|
|
10
|
-
EnabledDatasets[EnabledDatasets["Questions"] = 8] = "Questions";
|
|
11
|
-
EnabledDatasets[EnabledDatasets["Sens"] = 16] = "Sens";
|
|
12
|
-
EnabledDatasets[EnabledDatasets["PhotosSenateurs"] = 32] = "PhotosSenateurs";
|
|
13
|
-
EnabledDatasets[EnabledDatasets["Agenda"] = 64] = "Agenda";
|
|
14
|
-
EnabledDatasets[EnabledDatasets["All"] = 127] = "All";
|
|
15
|
-
})(EnabledDatasets || (EnabledDatasets = {}));
|
|
16
|
-
export const datasets = {
|
|
17
|
-
ameli: {
|
|
18
|
-
database: "ameli",
|
|
19
|
-
repairEncoding: true,
|
|
20
|
-
repairZip: (dataset, dataDir) => {
|
|
21
|
-
const sqlFilename = `${dataset.database}.sql`;
|
|
22
|
-
const sqlFilePath = path.join(dataDir, sqlFilename);
|
|
23
|
-
fs.removeSync(sqlFilePath);
|
|
24
|
-
fs.moveSync(path.join(dataDir, "var", "opt", "opendata", sqlFilename), sqlFilePath);
|
|
25
|
-
},
|
|
26
|
-
title: "Amendements",
|
|
27
|
-
url: "https://data.senat.fr/data/ameli/ameli.zip",
|
|
28
|
-
indexes: {
|
|
29
|
-
amdsen: [
|
|
30
|
-
{ name: "idx_amdid", columns: ["amdid"] },
|
|
31
|
-
{ name: "idx_senid", columns: ["senid"] },
|
|
32
|
-
{ name: "idx_grpid", columns: ["grpid"] },
|
|
33
|
-
],
|
|
34
|
-
sen_ameli: [{ name: "idx_entid", columns: ["entid"] }],
|
|
35
|
-
grppol_ameli: [{ name: "idx_entid", columns: ["entid"] }],
|
|
36
|
-
sub: [
|
|
37
|
-
{ name: "idx_id", columns: ["id"] },
|
|
38
|
-
{ name: "idx_typid", columns: ["typid"] },
|
|
39
|
-
{ name: "idx_merid", columns: ["merid"] },
|
|
40
|
-
{ name: "idx_comdelid", columns: ["comdelid"] },
|
|
41
|
-
],
|
|
42
|
-
typsub: [{ name: "idx_id", columns: ["id"] }],
|
|
43
|
-
typrect: [{ name: "idx_id", columns: ["id"] }],
|
|
44
|
-
txt_ameli: [
|
|
45
|
-
{ name: "idx_id", columns: ["id"] },
|
|
46
|
-
{ name: "idx_txtetaid", columns: ["txtetaid"] },
|
|
47
|
-
{ name: "idx_sesdepid", columns: ["sesdepid"] },
|
|
48
|
-
{ name: "idx_natid", columns: ["natid"] },
|
|
49
|
-
{ name: "idx_lecid", columns: ["lecid"] },
|
|
50
|
-
],
|
|
51
|
-
etatxt: [{ name: "idx_id", columns: ["id"] }],
|
|
52
|
-
ses: [
|
|
53
|
-
{ name: "idx_id", columns: ["id"] },
|
|
54
|
-
{ name: "idx_typid", columns: ["typid"] },
|
|
55
|
-
{ name: "idx_ann", columns: ["ann"] },
|
|
56
|
-
],
|
|
57
|
-
typses: [{ name: "idx_id", columns: ["id"] }],
|
|
58
|
-
nat: [{ name: "idx_id", columns: ["id"] }],
|
|
59
|
-
lec_ameli: [{ name: "idx_id", columns: ["id"] }],
|
|
60
|
-
mot: [{ name: "idx_id", columns: ["id"] }],
|
|
61
|
-
avicom: [{ name: "idx_id", columns: ["id"] }],
|
|
62
|
-
avigvt: [{ name: "idx_id", columns: ["id"] }],
|
|
63
|
-
sor: [{ name: "idx_id", columns: ["id"] }],
|
|
64
|
-
irr: [{ name: "idx_id", columns: ["id"] }],
|
|
65
|
-
com_ameli: [{ name: "idx_entid", columns: ["entid"] }],
|
|
66
|
-
cab: [{ name: "idx_entid", columns: ["entid"] }],
|
|
67
|
-
amd: [
|
|
68
|
-
{ name: "idx_subid", columns: ["subid"] },
|
|
69
|
-
{ name: "idx_typrectid", columns: ["typrectid"] },
|
|
70
|
-
{ name: "idx_txtid", columns: ["txtid"] },
|
|
71
|
-
{ name: "idx_motid", columns: ["motid"] },
|
|
72
|
-
{ name: "idx_avcid", columns: ["avcid"] },
|
|
73
|
-
{ name: "idx_avgid", columns: ["avgid"] },
|
|
74
|
-
{ name: "idx_sorid", columns: ["sorid"] },
|
|
75
|
-
{ name: "idx_irrid", columns: ["irrid"] },
|
|
76
|
-
{ name: "idx_nomentid", columns: ["nomentid"] },
|
|
77
|
-
],
|
|
78
|
-
},
|
|
79
|
-
},
|
|
80
|
-
debats: {
|
|
81
|
-
database: "debats",
|
|
82
|
-
repairEncoding: false,
|
|
83
|
-
title: "Informations relatives aux comptes rendus intégraux de la séance publique",
|
|
84
|
-
url: "https://data.senat.fr/data/debats/debats.zip",
|
|
85
|
-
indexes: {
|
|
86
|
-
secdis: [
|
|
87
|
-
{ name: "idx_datsea", columns: ["datsea"] },
|
|
88
|
-
{ name: "idx_typseccod", columns: ["typseccod"] },
|
|
89
|
-
],
|
|
90
|
-
intpjl: [{ name: "idx_secdiscle", columns: ["secdiscle"] }],
|
|
91
|
-
secdivers: [
|
|
92
|
-
{ name: "idx_datsea", columns: ["datsea"] },
|
|
93
|
-
{ name: "idx_typseccod", columns: ["typseccod"] },
|
|
94
|
-
],
|
|
95
|
-
intdivers: [{ name: "idx_intdiverscle", columns: ["intdiverscle"] }],
|
|
96
|
-
lecassdeb: [{ name: "idx_datsea", columns: ["datsea"] }],
|
|
97
|
-
},
|
|
98
|
-
},
|
|
99
|
-
dosleg: {
|
|
100
|
-
database: "dosleg",
|
|
101
|
-
repairEncoding: false,
|
|
102
|
-
title: "Dossiers législatifs",
|
|
103
|
-
url: "https://data.senat.fr/data/dosleg/dosleg.zip",
|
|
104
|
-
indexes: {
|
|
105
|
-
amescr: [{ name: "idx_scrnum", columns: ["scrnum"] }],
|
|
106
|
-
auteur: [{ name: "idx_autcod", columns: ["autcod"] }],
|
|
107
|
-
corscr: [{ name: "idx_scrnum", columns: ["scrnum"] }],
|
|
108
|
-
date_seance: [
|
|
109
|
-
{ name: "idx_code", columns: ["code"] },
|
|
110
|
-
{ name: "idx_lecidt", columns: ["lecidt"] },
|
|
111
|
-
],
|
|
112
|
-
deccoc: [{ name: "idx_deccoccod", columns: ["deccoccod"] }],
|
|
113
|
-
etaloi: [{ name: "idx_etaloicod", columns: ["etaloicod"] }],
|
|
114
|
-
lecass: [
|
|
115
|
-
{ name: "idx_lecidt", columns: ["lecidt"] },
|
|
116
|
-
{ name: "idx_codass", columns: ["codass"] },
|
|
117
|
-
{ name: "idx_orgcod", columns: ["orgcod"] },
|
|
118
|
-
{ name: "idx_orippr", columns: ["orippr"] },
|
|
119
|
-
],
|
|
120
|
-
lecassrap: [
|
|
121
|
-
{ name: "idx_rapcod", columns: ["rapcod"] },
|
|
122
|
-
{ name: "idx_lecassidt", columns: ["lecassidt"] },
|
|
123
|
-
],
|
|
124
|
-
lecture: [
|
|
125
|
-
{ name: "idx_loicod", columns: ["loicod"] },
|
|
126
|
-
{ name: "idx_typleccod", columns: ["typleccod"] },
|
|
127
|
-
{ name: "idx_lecidt", columns: ["lecidt"] },
|
|
128
|
-
],
|
|
129
|
-
loi: [
|
|
130
|
-
{ name: "idx_typloicod", columns: ["typloicod"] },
|
|
131
|
-
{ name: "idx_etaloicod", columns: ["etaloicod"] },
|
|
132
|
-
{ name: "idx_deccoccod", columns: ["deccoccod"] },
|
|
133
|
-
{ name: "idx_loicod", columns: ["loicod"] },
|
|
134
|
-
],
|
|
135
|
-
loithe: [
|
|
136
|
-
{ name: "idx_thecle", columns: ["thecle"] },
|
|
137
|
-
{ name: "idx_loicod", columns: ["loicod"] },
|
|
138
|
-
],
|
|
139
|
-
posvot: [{ name: "idx_posvotcod", columns: ["posvotcod"] }],
|
|
140
|
-
rap: [
|
|
141
|
-
{ name: "idx_rapcod", columns: ["rapcod"] },
|
|
142
|
-
{ name: "idx_coddenrap", columns: ["coddenrap"] },
|
|
143
|
-
],
|
|
144
|
-
rolsig: [{ name: "idx_signataire", columns: ["signataire"] }],
|
|
145
|
-
scr: [
|
|
146
|
-
{ name: "idx_code", columns: ["code"] },
|
|
147
|
-
{ name: "idx_scrnum", columns: ["scrnum"] },
|
|
148
|
-
{ name: "idx_sesann", columns: ["sesann"] },
|
|
149
|
-
],
|
|
150
|
-
stavot: [{ name: "idx_stavotidt", columns: ["stavotidt"] }],
|
|
151
|
-
texte: [
|
|
152
|
-
{ name: "idx_lecassidt", columns: ["lecassidt"] },
|
|
153
|
-
{ name: "idx_oritxtcod", columns: ["oritxtcod"] },
|
|
154
|
-
{ name: "idx_typtxtcod", columns: ["typtxtcod"] },
|
|
155
|
-
{ name: "idx_texcod", columns: ["texcod"] },
|
|
156
|
-
],
|
|
157
|
-
the: [{ name: "idx_thecle", columns: ["thecle"] }],
|
|
158
|
-
titsen: [{ name: "idx_titsencod", columns: ["titsencod"] }],
|
|
159
|
-
typloi: [{ name: "idx_typloicod", columns: ["typloicod"] }],
|
|
160
|
-
typtxt: [{ name: "idx_typtxtcod", columns: ["typtxtcod"] }],
|
|
161
|
-
votsen: [
|
|
162
|
-
{ name: "idx_scrnum", columns: ["scrnum"] },
|
|
163
|
-
{ name: "idx_sesann", columns: ["sesann"] },
|
|
164
|
-
{ name: "idx_titsencod", columns: ["titsencod"] },
|
|
165
|
-
{ name: "idx_stavotidt", columns: ["stavotidt"] },
|
|
166
|
-
{ name: "idx_posvotcod", columns: ["posvotcod"] },
|
|
167
|
-
{ name: "idx_senmat", columns: ["senmat"] },
|
|
168
|
-
],
|
|
169
|
-
},
|
|
170
|
-
},
|
|
171
|
-
questions: {
|
|
172
|
-
database: "questions",
|
|
173
|
-
repairEncoding: false,
|
|
174
|
-
title: "Questions écrites et orales posées par les sénateurs au Gouvernement",
|
|
175
|
-
url: "https://data.senat.fr/data/questions/questions.zip",
|
|
176
|
-
indexes: {
|
|
177
|
-
tam_reponses: [{ name: "idx_idque", columns: ["idque"] }],
|
|
178
|
-
tam_questions: [
|
|
179
|
-
{ name: "idx_sorquecod", columns: ["sorquecod"] },
|
|
180
|
-
{ name: "idx_etaquecod", columns: ["etaquecod"] },
|
|
181
|
-
],
|
|
182
|
-
the: [{ name: "idx_thenouidt", columns: ["thenouidt"] }],
|
|
183
|
-
sortquestion: [{ name: "idx_sorquecod", columns: ["sorquecod"] }],
|
|
184
|
-
etatquestion: [{ name: "idx_etaquecod", columns: ["etaquecod"] }],
|
|
185
|
-
},
|
|
186
|
-
},
|
|
187
|
-
sens: {
|
|
188
|
-
database: "sens",
|
|
189
|
-
repairEncoding: false,
|
|
190
|
-
repairZip: (dataset, dataDir) => {
|
|
191
|
-
const sqlFilename = `${dataset.database}.sql`;
|
|
192
|
-
const sqlFilePath = path.join(dataDir, sqlFilename);
|
|
193
|
-
fs.removeSync(sqlFilePath);
|
|
194
|
-
fs.moveSync(path.join(dataDir, "export_sens.sql"), sqlFilePath);
|
|
195
|
-
},
|
|
196
|
-
title: "Sénateurs (y compris organes et présence)",
|
|
197
|
-
url: "https://data.senat.fr/data/senateurs/export_sens.zip",
|
|
198
|
-
indexes: {
|
|
199
|
-
dpt: [
|
|
200
|
-
{ name: "idx_dptnum", columns: ["dptnum"] },
|
|
201
|
-
{ name: "idx_regcod", columns: ["regcod"] },
|
|
202
|
-
],
|
|
203
|
-
activite_loi: [{ name: "idx_actid", columns: ["actid"] }],
|
|
204
|
-
activite_delegation: [{ name: "idx_actid", columns: ["actid"] }],
|
|
205
|
-
activite_participant: [{ name: "idx_senmat", columns: ["senmat"] }],
|
|
206
|
-
activite_senateur: [{ name: "idx_senmat", columns: ["senmat"] }],
|
|
207
|
-
senbur: [
|
|
208
|
-
{ name: "idx_senmat", columns: ["senmat"] },
|
|
209
|
-
{ name: "idx_burcod", columns: ["burcod"] },
|
|
210
|
-
],
|
|
211
|
-
elucan: [
|
|
212
|
-
{ name: "idx_senmat", columns: ["senmat"] },
|
|
213
|
-
{ name: "idx_typmancod", columns: ["typmancod"] },
|
|
214
|
-
],
|
|
215
|
-
},
|
|
216
|
-
},
|
|
217
|
-
};
|
|
218
|
-
export function getEnabledDatasets(categories) {
|
|
219
|
-
categories.map((datasetName) => assert.notStrictEqual(EnabledDatasets[datasetName], undefined, `Unknown name of dataset: ${datasetName}`));
|
|
220
|
-
return categories.reduce((enabledDatasets, datasetName) => {
|
|
221
|
-
const enabledDataset = EnabledDatasets[datasetName];
|
|
222
|
-
return enabledDatasets | (enabledDataset || EnabledDatasets.None);
|
|
223
|
-
}, EnabledDatasets.None);
|
|
224
|
-
}
|
|
225
|
-
export function getChosenDatasets(enabledDatasets) {
|
|
226
|
-
return [
|
|
227
|
-
enabledDatasets & EnabledDatasets.Ameli ? datasets.ameli : null,
|
|
228
|
-
enabledDatasets & EnabledDatasets.Debats ? datasets.debats : null,
|
|
229
|
-
enabledDatasets & EnabledDatasets.DosLeg ? datasets.dosleg : null,
|
|
230
|
-
enabledDatasets & EnabledDatasets.Questions ? datasets.questions : null,
|
|
231
|
-
enabledDatasets & EnabledDatasets.Sens ? datasets.sens : null,
|
|
232
|
-
].filter((dataset) => dataset !== null);
|
|
233
|
-
}
|
package/lib/git.d.ts
DELETED
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
export declare function initRepo(repositoryDir: string): void;
|
|
2
|
-
export declare function commit(repositoryDir: string, message: string): boolean;
|
|
3
|
-
export declare function commitAndPush(repositoryDir: string, message: string, remotes?: string[]): number;
|
|
4
|
-
export declare function resetAndPull(gitDir: string): boolean;
|
|
5
|
-
export declare function clone(gitGroupUrl: string | undefined, gitName: string, workingDir: string): void;
|
|
6
|
-
export declare function run(repositoryDir: string, args: string, verbose?: boolean): string;
|
|
7
|
-
export declare function test(repositoryDir: string, args: string, verbose?: boolean): boolean;
|
|
8
|
-
/**
|
|
9
|
-
* Information about a changed file in git
|
|
10
|
-
*/
|
|
11
|
-
export interface GitChangedFile {
|
|
12
|
-
path: string;
|
|
13
|
-
status: "A" | "M" | "D" | "R" | "C" | "T" | "U";
|
|
14
|
-
}
|
|
15
|
-
/**
|
|
16
|
-
* Get the list of files that have changed since a specific commit in a git repository.
|
|
17
|
-
* @param repositoryDir The directory of the git repository
|
|
18
|
-
* @param sinceCommit The commit hash to compare against (e.g., "HEAD~1", "abc123", etc.)
|
|
19
|
-
* @param options Options for filtering
|
|
20
|
-
* @param options.diffFilter Git diff-filter string (default: "AMR").
|
|
21
|
-
* A=Added, M=Modified, D=Deleted, R=Renamed, C=Copied, T=Type changed, U=Unmerged
|
|
22
|
-
* @returns A Map of file paths to their git status
|
|
23
|
-
*/
|
|
24
|
-
export declare function getChangedFilesSinceCommit(repositoryDir: string, sinceCommit: string, options?: {
|
|
25
|
-
diffFilter?: string;
|
|
26
|
-
}): Map<string, GitChangedFile["status"]>;
|