@tricoteuses/senat 1.3.2 → 1.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/config.js +4 -6
- package/lib/databases.js +34 -75
- package/lib/datasets.js +20 -28
- package/lib/index.d.ts +5 -0
- package/lib/index.js +4 -43
- package/lib/loaders.js +56 -74
- package/lib/model/ameli.js +8 -11
- package/lib/model/dosleg.js +47 -52
- package/lib/model/index.js +4 -13
- package/lib/model/questions.js +15 -18
- package/lib/model/sens.d.ts +1 -1
- package/lib/model/sens.js +65 -71
- package/lib/model/texte.js +17 -25
- package/lib/model/util.js +13 -21
- package/lib/raw_types/ameli.js +1 -2
- package/lib/raw_types/debats.js +1 -2
- package/lib/raw_types/dosleg.js +1 -2
- package/lib/raw_types/questions.js +1 -2
- package/lib/raw_types/sens.js +1 -2
- package/lib/raw_types_schemats/ameli.js +1 -2
- package/lib/raw_types_schemats/debats.js +1 -2
- package/lib/raw_types_schemats/dosleg.js +1 -2
- package/lib/raw_types_schemats/questions.js +1 -2
- package/lib/raw_types_schemats/sens.js +1 -2
- package/lib/scripts/convert_data.js +78 -83
- package/lib/scripts/datautil.js +9 -13
- package/lib/scripts/parse_textes.js +23 -28
- package/lib/scripts/retrieve_documents.js +56 -61
- package/lib/scripts/retrieve_open_data.js +44 -49
- package/lib/scripts/retrieve_senateurs_photos.js +31 -36
- package/lib/scripts/shared/cli_helpers.js +9 -12
- package/lib/scripts/shared/util.js +7 -15
- package/lib/strings.js +4 -10
- package/lib/types/ameli.js +5 -8
- package/lib/types/debats.js +2 -5
- package/lib/types/dosleg.js +28 -31
- package/lib/types/questions.js +1 -2
- package/lib/types/sens.js +1 -2
- package/lib/types/sessions.js +2 -5
- package/lib/types/texte.js +2 -5
- package/lib/validators/config.js +4 -7
- package/package.json +4 -4
package/lib/config.js
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
require("dotenv/config");
|
|
4
|
-
const config_1 = require("./validators/config");
|
|
1
|
+
import "dotenv/config";
|
|
2
|
+
import { validateConfig } from "./validators/config";
|
|
5
3
|
const config = {
|
|
6
4
|
db: {
|
|
7
5
|
host: process.env["TRICOTEUSES_SENAT_DB_HOST"] || "localhost",
|
|
@@ -10,9 +8,9 @@ const config = {
|
|
|
10
8
|
user: process.env["TRICOTEUSES_SENAT_DB_USER"] || "opendata",
|
|
11
9
|
},
|
|
12
10
|
};
|
|
13
|
-
const [validConfig, error] =
|
|
11
|
+
const [validConfig, error] = validateConfig(config);
|
|
14
12
|
if (error !== null) {
|
|
15
13
|
console.error(`Error in configuration:\n${JSON.stringify(validConfig, null, 2)}\nError:\n${JSON.stringify(error, null, 2)}`);
|
|
16
14
|
process.exit(-1);
|
|
17
15
|
}
|
|
18
|
-
|
|
16
|
+
export default validConfig;
|
package/lib/databases.js
CHANGED
|
@@ -1,98 +1,57 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
|
-
};
|
|
38
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
-
exports.dbSharedConnectionObjectByName = exports.dbSens = exports.dbQuestions = exports.dbDosleg = exports.dbAmeli = exports.dbByName = exports.dbNames = exports.pgPromise = void 0;
|
|
40
|
-
exports.checkDatabase = checkDatabase;
|
|
41
|
-
exports.checkDatabases = checkDatabases;
|
|
42
|
-
const kysely_1 = require("kysely");
|
|
43
|
-
const pg = __importStar(require("pg"));
|
|
44
|
-
const pg_1 = require("pg");
|
|
45
|
-
const pg_cursor_1 = __importDefault(require("pg-cursor"));
|
|
46
|
-
const pg_promise_1 = __importDefault(require("pg-promise"));
|
|
47
|
-
const config_1 = __importDefault(require("./config"));
|
|
48
|
-
const datasets_1 = require("./datasets");
|
|
1
|
+
import { Kysely, PostgresDialect } from "kysely";
|
|
2
|
+
import * as pg from "pg";
|
|
3
|
+
import { types } from "pg";
|
|
4
|
+
import Cursor from "pg-cursor";
|
|
5
|
+
import PgPromise from "pg-promise";
|
|
6
|
+
import config from "./config";
|
|
7
|
+
import { datasets } from "./datasets";
|
|
49
8
|
// Map int8 to number instead of string
|
|
50
9
|
// https://github.com/brianc/node-pg-types#use
|
|
51
|
-
pg.types.setTypeParser(
|
|
10
|
+
pg.types.setTypeParser(types.builtins.INT8, (val) => {
|
|
52
11
|
return parseInt(val, 10);
|
|
53
12
|
});
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
for (const name of
|
|
58
|
-
|
|
13
|
+
export const pgPromise = PgPromise();
|
|
14
|
+
export const dbNames = ["ameli", "debats", "dosleg"];
|
|
15
|
+
export const dbByName = {};
|
|
16
|
+
for (const name of dbNames) {
|
|
17
|
+
dbByName[name] = pgPromise({
|
|
59
18
|
database: name,
|
|
60
|
-
host:
|
|
61
|
-
password:
|
|
62
|
-
port:
|
|
63
|
-
user:
|
|
19
|
+
host: config.db.host,
|
|
20
|
+
password: config.db.password,
|
|
21
|
+
port: config.db.port,
|
|
22
|
+
user: config.db.user,
|
|
64
23
|
});
|
|
65
24
|
}
|
|
66
25
|
function createDb(database, schema) {
|
|
67
|
-
const dialect = new
|
|
26
|
+
const dialect = new PostgresDialect({
|
|
68
27
|
pool: new pg.Pool({
|
|
69
28
|
database: database,
|
|
70
|
-
host:
|
|
71
|
-
user:
|
|
72
|
-
password:
|
|
73
|
-
port:
|
|
29
|
+
host: config.db.host,
|
|
30
|
+
user: config.db.user,
|
|
31
|
+
password: config.db.password,
|
|
32
|
+
port: config.db.port,
|
|
74
33
|
max: 10,
|
|
75
34
|
}),
|
|
76
|
-
cursor:
|
|
35
|
+
cursor: Cursor,
|
|
77
36
|
});
|
|
78
|
-
return new
|
|
37
|
+
return new Kysely({
|
|
79
38
|
log: ["error"],
|
|
80
39
|
dialect: dialect,
|
|
81
40
|
}).withSchema(schema);
|
|
82
41
|
}
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
async function checkDatabase(name) {
|
|
42
|
+
export const dbAmeli = createDb(datasets.ameli.database, datasets.ameli.schema);
|
|
43
|
+
export const dbDosleg = createDb(datasets.dosleg.database, datasets.dosleg.schema);
|
|
44
|
+
export const dbQuestions = createDb(datasets.questions.database, datasets.questions.schema);
|
|
45
|
+
export const dbSens = createDb(datasets.sens.database, datasets.sens.schema);
|
|
46
|
+
export let dbSharedConnectionObjectByName = {};
|
|
47
|
+
export async function checkDatabase(name) {
|
|
89
48
|
// Check that database exists.
|
|
90
|
-
const db =
|
|
91
|
-
|
|
49
|
+
const db = dbByName[name];
|
|
50
|
+
dbSharedConnectionObjectByName[name] = await db.connect();
|
|
92
51
|
return db;
|
|
93
52
|
}
|
|
94
|
-
async function checkDatabases() {
|
|
95
|
-
for (const name of
|
|
53
|
+
export async function checkDatabases() {
|
|
54
|
+
for (const name of dbNames) {
|
|
96
55
|
await checkDatabase(name);
|
|
97
56
|
}
|
|
98
57
|
}
|
package/lib/datasets.js
CHANGED
|
@@ -1,15 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.datasets = exports.EnabledDatasets = void 0;
|
|
7
|
-
exports.getEnabledDatasets = getEnabledDatasets;
|
|
8
|
-
exports.getChosenFromEnabledDatasets = getChosenFromEnabledDatasets;
|
|
9
|
-
const fs_extra_1 = __importDefault(require("fs-extra"));
|
|
10
|
-
const path_1 = __importDefault(require("path"));
|
|
11
|
-
const assert_1 = __importDefault(require("assert"));
|
|
12
|
-
var EnabledDatasets;
|
|
1
|
+
import fs from "fs-extra";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import assert from "assert";
|
|
4
|
+
export var EnabledDatasets;
|
|
13
5
|
(function (EnabledDatasets) {
|
|
14
6
|
EnabledDatasets[EnabledDatasets["None"] = 0] = "None";
|
|
15
7
|
EnabledDatasets[EnabledDatasets["Ameli"] = 1] = "Ameli";
|
|
@@ -19,16 +11,16 @@ var EnabledDatasets;
|
|
|
19
11
|
EnabledDatasets[EnabledDatasets["Sens"] = 16] = "Sens";
|
|
20
12
|
EnabledDatasets[EnabledDatasets["PhotosSenateurs"] = 32] = "PhotosSenateurs";
|
|
21
13
|
EnabledDatasets[EnabledDatasets["All"] = 63] = "All";
|
|
22
|
-
})(EnabledDatasets || (
|
|
23
|
-
|
|
14
|
+
})(EnabledDatasets || (EnabledDatasets = {}));
|
|
15
|
+
export const datasets = {
|
|
24
16
|
ameli: {
|
|
25
17
|
database: "ameli",
|
|
26
18
|
repairEncoding: false,
|
|
27
19
|
repairZip: (dataset, dataDir) => {
|
|
28
20
|
const sqlFilename = `${dataset.database}.sql`;
|
|
29
|
-
const sqlFilePath =
|
|
30
|
-
|
|
31
|
-
|
|
21
|
+
const sqlFilePath = path.join(dataDir, sqlFilename);
|
|
22
|
+
fs.removeSync(sqlFilePath);
|
|
23
|
+
fs.moveSync(path.join(dataDir, "var", "opt", "opendata", sqlFilename), sqlFilePath);
|
|
32
24
|
},
|
|
33
25
|
schema: "public",
|
|
34
26
|
title: "Amendements",
|
|
@@ -60,29 +52,29 @@ exports.datasets = {
|
|
|
60
52
|
repairEncoding: true,
|
|
61
53
|
repairZip: (dataset, dataDir) => {
|
|
62
54
|
const sqlFilename = `${dataset.database}.sql`;
|
|
63
|
-
const sqlFilePath =
|
|
64
|
-
|
|
65
|
-
|
|
55
|
+
const sqlFilePath = path.join(dataDir, sqlFilename);
|
|
56
|
+
fs.removeSync(sqlFilePath);
|
|
57
|
+
fs.moveSync(path.join(dataDir, "export_sens.sql"), sqlFilePath);
|
|
66
58
|
},
|
|
67
59
|
schema: "public",
|
|
68
60
|
title: "Sénateurs (y compris organes et présence)",
|
|
69
61
|
url: "https://data.senat.fr/data/senateurs/export_sens.zip",
|
|
70
62
|
},
|
|
71
63
|
};
|
|
72
|
-
function getEnabledDatasets(categories) {
|
|
73
|
-
categories.map((datasetName) =>
|
|
64
|
+
export function getEnabledDatasets(categories) {
|
|
65
|
+
categories.map((datasetName) => assert.notStrictEqual(EnabledDatasets[datasetName], undefined, `Unknown name of dataset: ${datasetName}`));
|
|
74
66
|
return categories.reduce((enabledDatasets, datasetName) => {
|
|
75
67
|
const enabledDataset = EnabledDatasets[datasetName];
|
|
76
68
|
return enabledDatasets | (enabledDataset || EnabledDatasets.None);
|
|
77
69
|
}, EnabledDatasets.None);
|
|
78
70
|
}
|
|
79
|
-
function getChosenFromEnabledDatasets(categories) {
|
|
71
|
+
export function getChosenFromEnabledDatasets(categories) {
|
|
80
72
|
const enabledDatasets = getEnabledDatasets(categories);
|
|
81
73
|
return [
|
|
82
|
-
enabledDatasets & EnabledDatasets.Ameli ?
|
|
83
|
-
enabledDatasets & EnabledDatasets.Debats ?
|
|
84
|
-
enabledDatasets & EnabledDatasets.DosLeg ?
|
|
85
|
-
enabledDatasets & EnabledDatasets.Questions ?
|
|
86
|
-
enabledDatasets & EnabledDatasets.Sens ?
|
|
74
|
+
enabledDatasets & EnabledDatasets.Ameli ? datasets.ameli : null,
|
|
75
|
+
enabledDatasets & EnabledDatasets.Debats ? datasets.debats : null,
|
|
76
|
+
enabledDatasets & EnabledDatasets.DosLeg ? datasets.dosleg : null,
|
|
77
|
+
enabledDatasets & EnabledDatasets.Questions ? datasets.questions : null,
|
|
78
|
+
enabledDatasets & EnabledDatasets.Sens ? datasets.sens : null,
|
|
87
79
|
].filter((dataset) => dataset !== null);
|
|
88
80
|
}
|
package/lib/index.d.ts
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
export type { AmendementResult, } from "./model/ameli";
|
|
2
|
+
export type { DossierLegislatifResult, } from "./model/dosleg";
|
|
3
|
+
export type { QuestionResult, } from "./model/questions";
|
|
4
|
+
export type { CirconscriptionResult, OrganismeResult, SenateurResult, } from "./model/sens";
|
|
1
5
|
export { sesFieldsToParseInt, sesFieldsToTrim, subFieldsToParseInt, subFieldsToTrim, txtAmeliFieldsToTrim, } from "./types/ameli";
|
|
2
6
|
export type { Ses, Sub, TxtAmeli } from "./types/ameli";
|
|
3
7
|
export { debatsFieldsToTrim, lecassdebFieldsToTrim } from "./types/debats";
|
|
@@ -6,3 +10,4 @@ export { assFieldsToTrim, audFieldsToTrim, auteurFieldsToTrim, dateSeanceFieldsT
|
|
|
6
10
|
export type { Ass, Aud, Auteur, DateSeance, DecCoc, DenRap, DocAtt, Ecr, EtaLoi, LecAss, LecAssRap, Lecture, Loi, Org, OriTxt, Qua, Rap, RapOrg, Scr, Texte, TypAtt, TypLec, TypLoi, TypTxt, TypUrl, } from "./types/dosleg";
|
|
7
11
|
export type { Photo, Sen } from "./types/sens";
|
|
8
12
|
export { Session } from "./types/sessions";
|
|
13
|
+
export type { Alinea, Article, Division, DivisionTag, FlatTexte } from "./types/texte";
|
package/lib/index.js
CHANGED
|
@@ -1,43 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
Object.defineProperty(exports, "sesFieldsToParseInt", { enumerable: true, get: function () { return ameli_1.sesFieldsToParseInt; } });
|
|
6
|
-
Object.defineProperty(exports, "sesFieldsToTrim", { enumerable: true, get: function () { return ameli_1.sesFieldsToTrim; } });
|
|
7
|
-
Object.defineProperty(exports, "subFieldsToParseInt", { enumerable: true, get: function () { return ameli_1.subFieldsToParseInt; } });
|
|
8
|
-
Object.defineProperty(exports, "subFieldsToTrim", { enumerable: true, get: function () { return ameli_1.subFieldsToTrim; } });
|
|
9
|
-
Object.defineProperty(exports, "txtAmeliFieldsToTrim", { enumerable: true, get: function () { return ameli_1.txtAmeliFieldsToTrim; } });
|
|
10
|
-
var debats_1 = require("./types/debats");
|
|
11
|
-
Object.defineProperty(exports, "debatsFieldsToTrim", { enumerable: true, get: function () { return debats_1.debatsFieldsToTrim; } });
|
|
12
|
-
Object.defineProperty(exports, "lecassdebFieldsToTrim", { enumerable: true, get: function () { return debats_1.lecassdebFieldsToTrim; } });
|
|
13
|
-
var dosleg_1 = require("./types/dosleg");
|
|
14
|
-
Object.defineProperty(exports, "assFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.assFieldsToTrim; } });
|
|
15
|
-
Object.defineProperty(exports, "audFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.audFieldsToTrim; } });
|
|
16
|
-
Object.defineProperty(exports, "auteurFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.auteurFieldsToTrim; } });
|
|
17
|
-
Object.defineProperty(exports, "dateSeanceFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.dateSeanceFieldsToTrim; } });
|
|
18
|
-
Object.defineProperty(exports, "deccocFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.deccocFieldsToTrim; } });
|
|
19
|
-
Object.defineProperty(exports, "denrapFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.denrapFieldsToTrim; } });
|
|
20
|
-
Object.defineProperty(exports, "docattFieldsToParseInt", { enumerable: true, get: function () { return dosleg_1.docattFieldsToParseInt; } });
|
|
21
|
-
Object.defineProperty(exports, "docattFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.docattFieldsToTrim; } });
|
|
22
|
-
Object.defineProperty(exports, "ecrFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.ecrFieldsToTrim; } });
|
|
23
|
-
Object.defineProperty(exports, "etaloiFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.etaloiFieldsToTrim; } });
|
|
24
|
-
Object.defineProperty(exports, "lecassFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.lecassFieldsToTrim; } });
|
|
25
|
-
Object.defineProperty(exports, "lecassrapFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.lecassrapFieldsToTrim; } });
|
|
26
|
-
Object.defineProperty(exports, "lectureFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.lectureFieldsToTrim; } });
|
|
27
|
-
Object.defineProperty(exports, "loiFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.loiFieldsToTrim; } });
|
|
28
|
-
Object.defineProperty(exports, "orgFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.orgFieldsToTrim; } });
|
|
29
|
-
Object.defineProperty(exports, "oritxtFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.oritxtFieldsToTrim; } });
|
|
30
|
-
Object.defineProperty(exports, "quaFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.quaFieldsToTrim; } });
|
|
31
|
-
Object.defineProperty(exports, "rapFieldsToParseInt", { enumerable: true, get: function () { return dosleg_1.rapFieldsToParseInt; } });
|
|
32
|
-
Object.defineProperty(exports, "rapFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.rapFieldsToTrim; } });
|
|
33
|
-
Object.defineProperty(exports, "raporgFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.raporgFieldsToTrim; } });
|
|
34
|
-
Object.defineProperty(exports, "scrFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.scrFieldsToTrim; } });
|
|
35
|
-
Object.defineProperty(exports, "texteFieldsToParseInt", { enumerable: true, get: function () { return dosleg_1.texteFieldsToParseInt; } });
|
|
36
|
-
Object.defineProperty(exports, "texteFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.texteFieldsToTrim; } });
|
|
37
|
-
Object.defineProperty(exports, "typattFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.typattFieldsToTrim; } });
|
|
38
|
-
Object.defineProperty(exports, "typlecFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.typlecFieldsToTrim; } });
|
|
39
|
-
Object.defineProperty(exports, "typloiFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.typloiFieldsToTrim; } });
|
|
40
|
-
Object.defineProperty(exports, "typtxtFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.typtxtFieldsToTrim; } });
|
|
41
|
-
Object.defineProperty(exports, "typurlFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.typurlFieldsToTrim; } });
|
|
42
|
-
var sessions_1 = require("./types/sessions");
|
|
43
|
-
Object.defineProperty(exports, "Session", { enumerable: true, get: function () { return sessions_1.Session; } });
|
|
1
|
+
export { sesFieldsToParseInt, sesFieldsToTrim, subFieldsToParseInt, subFieldsToTrim, txtAmeliFieldsToTrim, } from "./types/ameli";
|
|
2
|
+
export { debatsFieldsToTrim, lecassdebFieldsToTrim } from "./types/debats";
|
|
3
|
+
export { assFieldsToTrim, audFieldsToTrim, auteurFieldsToTrim, dateSeanceFieldsToTrim, deccocFieldsToTrim, denrapFieldsToTrim, docattFieldsToParseInt, docattFieldsToTrim, ecrFieldsToTrim, etaloiFieldsToTrim, lecassFieldsToTrim, lecassrapFieldsToTrim, lectureFieldsToTrim, loiFieldsToTrim, orgFieldsToTrim, oritxtFieldsToTrim, quaFieldsToTrim, rapFieldsToParseInt, rapFieldsToTrim, raporgFieldsToTrim, scrFieldsToTrim, texteFieldsToParseInt, texteFieldsToTrim, typattFieldsToTrim, typlecFieldsToTrim, typloiFieldsToTrim, typtxtFieldsToTrim, typurlFieldsToTrim, } from "./types/dosleg";
|
|
4
|
+
export { Session } from "./types/sessions";
|
package/lib/loaders.js
CHANGED
|
@@ -1,62 +1,44 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
};
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
const
|
|
19
|
-
const path_1 = __importDefault(require("path"));
|
|
20
|
-
const legislatures_json_1 = __importDefault(require("./legislatures.json"));
|
|
21
|
-
const datasets_1 = require("./datasets");
|
|
22
|
-
const datautil_1 = require("./scripts/datautil");
|
|
23
|
-
var datasets_2 = require("./datasets");
|
|
24
|
-
Object.defineProperty(exports, "EnabledDatasets", { enumerable: true, get: function () { return datasets_2.EnabledDatasets; } });
|
|
25
|
-
exports.DOSLEG_DOSSIERS_FOLDER = "dossiers";
|
|
26
|
-
exports.SENS_CIRCONSCRIPTIONS_FOLDER = "circonscriptions";
|
|
27
|
-
exports.SENS_ORGANISMES_FOLDER = "organismes";
|
|
28
|
-
exports.SENS_SENATEURS_FOLDER = "senateurs";
|
|
29
|
-
exports.TEXTE_FOLDER = "leg";
|
|
30
|
-
exports.TEXTE_ORIGINAL_FOLDER = "original";
|
|
31
|
-
exports.TEXTE_TRANSFORMED_FOLDER = "transformed";
|
|
32
|
-
exports.DOCUMENT_METADATA_FILE = "metadata.json";
|
|
33
|
-
exports.RAPPORT_FOLDER = "rap";
|
|
34
|
-
function* iterFilePaths(dirPath) {
|
|
35
|
-
if (dirPath && fs_1.default.existsSync(dirPath)) {
|
|
36
|
-
const files = fs_1.default.readdirSync(dirPath, {
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import legislatures from "./legislatures.json";
|
|
4
|
+
import { datasets } from "./datasets";
|
|
5
|
+
import { UNDEFINED_SESSION } from "./scripts/datautil";
|
|
6
|
+
export { EnabledDatasets } from "./datasets";
|
|
7
|
+
export const DOSLEG_DOSSIERS_FOLDER = "dossiers";
|
|
8
|
+
export const SENS_CIRCONSCRIPTIONS_FOLDER = "circonscriptions";
|
|
9
|
+
export const SENS_ORGANISMES_FOLDER = "organismes";
|
|
10
|
+
export const SENS_SENATEURS_FOLDER = "senateurs";
|
|
11
|
+
export const TEXTE_FOLDER = "leg";
|
|
12
|
+
export const TEXTE_ORIGINAL_FOLDER = "original";
|
|
13
|
+
export const TEXTE_TRANSFORMED_FOLDER = "transformed";
|
|
14
|
+
export const DOCUMENT_METADATA_FILE = "metadata.json";
|
|
15
|
+
export const RAPPORT_FOLDER = "rap";
|
|
16
|
+
export function* iterFilePaths(dirPath) {
|
|
17
|
+
if (dirPath && fs.existsSync(dirPath)) {
|
|
18
|
+
const files = fs.readdirSync(dirPath, {
|
|
37
19
|
withFileTypes: true,
|
|
38
20
|
recursive: true,
|
|
39
21
|
});
|
|
40
22
|
for (const file of files) {
|
|
41
23
|
if (file.isFile()) {
|
|
42
|
-
yield
|
|
24
|
+
yield path.join(file.parentPath, file.name);
|
|
43
25
|
}
|
|
44
26
|
}
|
|
45
27
|
}
|
|
46
28
|
}
|
|
47
29
|
function* iterLoadSenatItems(dataDir, dataset, legislatureOrSession, subDir, { log = false } = {}) {
|
|
48
|
-
let itemsDir =
|
|
30
|
+
let itemsDir = path.join(dataDir, dataset.database);
|
|
49
31
|
if (subDir) {
|
|
50
|
-
itemsDir =
|
|
32
|
+
itemsDir = path.join(itemsDir, subDir);
|
|
51
33
|
}
|
|
52
34
|
if (legislatureOrSession) {
|
|
53
|
-
itemsDir =
|
|
35
|
+
itemsDir = path.join(itemsDir, String(legislatureOrSession));
|
|
54
36
|
}
|
|
55
37
|
for (const filePath of iterFilePaths(itemsDir)) {
|
|
56
38
|
if (log) {
|
|
57
39
|
console.log(`Loading file: ${filePath}…`);
|
|
58
40
|
}
|
|
59
|
-
const itemJson =
|
|
41
|
+
const itemJson = fs.readFileSync(filePath, { encoding: "utf8" });
|
|
60
42
|
const item = JSON.parse(itemJson);
|
|
61
43
|
const filePathFromDataset = filePath.substring(filePath.indexOf(dataset.database) + dataset.database.length);
|
|
62
44
|
yield {
|
|
@@ -66,25 +48,25 @@ function* iterLoadSenatItems(dataDir, dataset, legislatureOrSession, subDir, { l
|
|
|
66
48
|
};
|
|
67
49
|
}
|
|
68
50
|
}
|
|
69
|
-
function* iterLoadSenatAmendements(dataDir, session, options = {}) {
|
|
70
|
-
for (const amendementItem of iterLoadSenatItems(dataDir,
|
|
51
|
+
export function* iterLoadSenatAmendements(dataDir, session, options = {}) {
|
|
52
|
+
for (const amendementItem of iterLoadSenatItems(dataDir, datasets.ameli, session, undefined, options)) {
|
|
71
53
|
yield amendementItem;
|
|
72
54
|
}
|
|
73
55
|
}
|
|
74
|
-
function* iterLoadSenatDossiersLegislatifs(dataDir, session, options = {}) {
|
|
75
|
-
for (const dossierLegislatifItem of iterLoadSenatItems(dataDir,
|
|
56
|
+
export function* iterLoadSenatDossiersLegislatifs(dataDir, session, options = {}) {
|
|
57
|
+
for (const dossierLegislatifItem of iterLoadSenatItems(dataDir, datasets.dosleg, session, DOSLEG_DOSSIERS_FOLDER, options)) {
|
|
76
58
|
yield dossierLegislatifItem;
|
|
77
59
|
}
|
|
78
60
|
}
|
|
79
|
-
function* iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session) {
|
|
80
|
-
let itemsDir =
|
|
61
|
+
export function* iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session) {
|
|
62
|
+
let itemsDir = path.join(dataDir, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER);
|
|
81
63
|
if (session) {
|
|
82
|
-
itemsDir =
|
|
64
|
+
itemsDir = path.join(itemsDir, session.toString());
|
|
83
65
|
}
|
|
84
66
|
for (const filePath of iterFilePaths(itemsDir)) {
|
|
85
|
-
const parsedFilePath =
|
|
86
|
-
if (parsedFilePath.base ===
|
|
87
|
-
const itemJson =
|
|
67
|
+
const parsedFilePath = path.parse(filePath);
|
|
68
|
+
if (parsedFilePath.base === DOCUMENT_METADATA_FILE) {
|
|
69
|
+
const itemJson = fs.readFileSync(filePath, { encoding: "utf8" });
|
|
88
70
|
const item = JSON.parse(itemJson);
|
|
89
71
|
yield {
|
|
90
72
|
item,
|
|
@@ -92,15 +74,15 @@ function* iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session) {
|
|
|
92
74
|
}
|
|
93
75
|
}
|
|
94
76
|
}
|
|
95
|
-
function* iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session) {
|
|
96
|
-
let itemsDir =
|
|
77
|
+
export function* iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session) {
|
|
78
|
+
let itemsDir = path.join(dataDir, RAPPORT_FOLDER);
|
|
97
79
|
if (session) {
|
|
98
|
-
itemsDir =
|
|
80
|
+
itemsDir = path.join(itemsDir, session.toString());
|
|
99
81
|
}
|
|
100
82
|
for (const filePath of iterFilePaths(itemsDir)) {
|
|
101
|
-
const parsedFilePath =
|
|
102
|
-
if (parsedFilePath.base ===
|
|
103
|
-
const itemJson =
|
|
83
|
+
const parsedFilePath = path.parse(filePath);
|
|
84
|
+
if (parsedFilePath.base === DOCUMENT_METADATA_FILE) {
|
|
85
|
+
const itemJson = fs.readFileSync(filePath, { encoding: "utf8" });
|
|
104
86
|
const item = JSON.parse(itemJson);
|
|
105
87
|
yield {
|
|
106
88
|
item,
|
|
@@ -108,7 +90,7 @@ function* iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session) {
|
|
|
108
90
|
}
|
|
109
91
|
}
|
|
110
92
|
}
|
|
111
|
-
function* iterLoadSenatDossiersLegislatifsTextes(dataDir, session, options = {}) {
|
|
93
|
+
export function* iterLoadSenatDossiersLegislatifsTextes(dataDir, session, options = {}) {
|
|
112
94
|
for (const { item: dossierLegislatif } of iterLoadSenatDossiersLegislatifs(dataDir, session, options)) {
|
|
113
95
|
for (const lecture of dossierLegislatif.lectures) {
|
|
114
96
|
const lecturesSenat = lecture.lectures_assemblee.filter((lectureAssemblee) => lectureAssemblee.assemblee === "Sénat");
|
|
@@ -127,8 +109,8 @@ function* iterLoadSenatDossiersLegislatifsTextes(dataDir, session, options = {})
|
|
|
127
109
|
item: enrichedTexte,
|
|
128
110
|
};
|
|
129
111
|
if (texte.url) {
|
|
130
|
-
const texteName =
|
|
131
|
-
texteItem.filePathFromDataset =
|
|
112
|
+
const texteName = path.parse(texte.url).name;
|
|
113
|
+
texteItem.filePathFromDataset = path.join(`${texte.session ?? UNDEFINED_SESSION}`, texteName, `${texteName}.pdf`);
|
|
132
114
|
}
|
|
133
115
|
yield texteItem;
|
|
134
116
|
}
|
|
@@ -136,25 +118,25 @@ function* iterLoadSenatDossiersLegislatifsTextes(dataDir, session, options = {})
|
|
|
136
118
|
}
|
|
137
119
|
}
|
|
138
120
|
}
|
|
139
|
-
function loadSenatTexteContent(dataDir, textePathFromDataset) {
|
|
140
|
-
const parsedTextePath =
|
|
121
|
+
export function loadSenatTexteContent(dataDir, textePathFromDataset) {
|
|
122
|
+
const parsedTextePath = path.parse(textePathFromDataset);
|
|
141
123
|
const jsonTexteName = `${parsedTextePath.name}.json`;
|
|
142
|
-
const fullTextePath =
|
|
143
|
-
if (!
|
|
124
|
+
const fullTextePath = path.join(dataDir, TEXTE_FOLDER, TEXTE_TRANSFORMED_FOLDER, parsedTextePath.dir, jsonTexteName);
|
|
125
|
+
if (!fs.existsSync(fullTextePath)) {
|
|
144
126
|
return { item: null };
|
|
145
127
|
}
|
|
146
|
-
const texteJson =
|
|
128
|
+
const texteJson = fs.readFileSync(fullTextePath, { encoding: "utf8" });
|
|
147
129
|
return { item: JSON.parse(texteJson) };
|
|
148
130
|
}
|
|
149
|
-
function* iterLoadSenatOrganismes(dataDir, options = {}) {
|
|
150
|
-
for (const organismeItem of iterLoadSenatItems(dataDir,
|
|
131
|
+
export function* iterLoadSenatOrganismes(dataDir, options = {}) {
|
|
132
|
+
for (const organismeItem of iterLoadSenatItems(dataDir, datasets.sens, undefined, SENS_ORGANISMES_FOLDER, options)) {
|
|
151
133
|
yield organismeItem;
|
|
152
134
|
}
|
|
153
135
|
}
|
|
154
|
-
function* iterLoadSenatSenateurs(dataDir, legislature, options = {}) {
|
|
155
|
-
const dateDebutLegislatureStr =
|
|
136
|
+
export function* iterLoadSenatSenateurs(dataDir, legislature, options = {}) {
|
|
137
|
+
const dateDebutLegislatureStr = legislatures.find((legislatureInfo) => legislatureInfo.numero === legislature)?.date_debut;
|
|
156
138
|
const dateDebutLegislature = new Date(dateDebutLegislatureStr);
|
|
157
|
-
for (const senateurItem of iterLoadSenatItems(dataDir,
|
|
139
|
+
for (const senateurItem of iterLoadSenatItems(dataDir, datasets.sens, undefined, SENS_SENATEURS_FOLDER, options)) {
|
|
158
140
|
const dateFinMandatSenateur = senateurItem.item.mandats_senateur[0]
|
|
159
141
|
?.date_fin
|
|
160
142
|
? new Date(senateurItem.item.mandats_senateur[0]?.date_fin)
|
|
@@ -165,13 +147,13 @@ function* iterLoadSenatSenateurs(dataDir, legislature, options = {}) {
|
|
|
165
147
|
yield senateurItem;
|
|
166
148
|
}
|
|
167
149
|
}
|
|
168
|
-
function* iterLoadSenatCirconscriptions(dataDir, options = {}) {
|
|
169
|
-
for (const circonscriptionItem of iterLoadSenatItems(dataDir,
|
|
150
|
+
export function* iterLoadSenatCirconscriptions(dataDir, options = {}) {
|
|
151
|
+
for (const circonscriptionItem of iterLoadSenatItems(dataDir, datasets.sens, undefined, SENS_CIRCONSCRIPTIONS_FOLDER, options)) {
|
|
170
152
|
yield circonscriptionItem;
|
|
171
153
|
}
|
|
172
154
|
}
|
|
173
|
-
function* iterLoadSenatQuestions(dataDir, legislature, options = {}) {
|
|
174
|
-
for (const questionItem of iterLoadSenatItems(dataDir,
|
|
155
|
+
export function* iterLoadSenatQuestions(dataDir, legislature, options = {}) {
|
|
156
|
+
for (const questionItem of iterLoadSenatItems(dataDir, datasets.questions, legislature, undefined, options)) {
|
|
175
157
|
yield questionItem;
|
|
176
158
|
}
|
|
177
159
|
}
|
package/lib/model/ameli.js
CHANGED
|
@@ -1,11 +1,8 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const postgres_1 = require("kysely/helpers/postgres");
|
|
5
|
-
const databases_1 = require("../databases");
|
|
6
|
-
const util_1 = require("./util");
|
|
1
|
+
import { jsonArrayFrom } from "kysely/helpers/postgres";
|
|
2
|
+
import { dbAmeli } from "../databases";
|
|
3
|
+
import { concat, toDateString } from "./util";
|
|
7
4
|
function auteurs(amendementId) {
|
|
8
|
-
return
|
|
5
|
+
return jsonArrayFrom(dbAmeli
|
|
9
6
|
.selectFrom("amdsen")
|
|
10
7
|
.leftJoin("sen_ameli", "amdsen.senid", "sen_ameli.entid")
|
|
11
8
|
.where("amdsen.amdid", "=", amendementId)
|
|
@@ -16,7 +13,7 @@ function auteurs(amendementId) {
|
|
|
16
13
|
])
|
|
17
14
|
.orderBy("amdsen.rng asc"));
|
|
18
15
|
}
|
|
19
|
-
const findAllQuery =
|
|
16
|
+
const findAllQuery = dbAmeli
|
|
20
17
|
.selectFrom("amd")
|
|
21
18
|
.leftJoin("sub", "amd.subid", "sub.id")
|
|
22
19
|
.leftJoin("typsub", "sub.typid", "typsub.id")
|
|
@@ -58,7 +55,7 @@ const findAllQuery = databases_1.dbAmeli
|
|
|
58
55
|
"sub.lic as subdivision_libelle_court",
|
|
59
56
|
"typsub.lib as subdivision_type",
|
|
60
57
|
"amd.alinea as alinea",
|
|
61
|
-
|
|
58
|
+
toDateString(ref("amd.datdep")).as("date_depot"),
|
|
62
59
|
eb
|
|
63
60
|
.case()
|
|
64
61
|
.when("amd.etaid", "=", 8)
|
|
@@ -72,7 +69,7 @@ const findAllQuery = databases_1.dbAmeli
|
|
|
72
69
|
"avigvt.lib as avis_gouvernement",
|
|
73
70
|
eb.fn.coalesce("sor.lib", "irr.libirr").as("sort"),
|
|
74
71
|
"amd.rev as revision",
|
|
75
|
-
|
|
72
|
+
concat(val("https://www.senat.fr/amendements/"), ref("ses.lil"), val("/"), ref("txt_ameli.numabs"), val("/Amdt_"), ref("amd.numabs"), val(".html")).as("url"),
|
|
76
73
|
"grppol_ameli.lilcou as au_nom_de_groupe_politique",
|
|
77
74
|
"com_ameli.lil as au_nom_de_commission",
|
|
78
75
|
eb
|
|
@@ -84,6 +81,6 @@ const findAllQuery = databases_1.dbAmeli
|
|
|
84
81
|
.as("auteur_est_gouvernement"),
|
|
85
82
|
])
|
|
86
83
|
.select(({ ref }) => [auteurs(ref("amd.id")).as("auteurs")]);
|
|
87
|
-
function findAll() {
|
|
84
|
+
export function findAll() {
|
|
88
85
|
return findAllQuery.stream();
|
|
89
86
|
}
|