@tricoteuses/senat 1.3.2 → 1.3.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/lib/config.js +4 -6
  2. package/lib/databases.js +34 -75
  3. package/lib/datasets.js +20 -28
  4. package/lib/index.d.ts +5 -0
  5. package/lib/index.js +4 -43
  6. package/lib/loaders.js +56 -74
  7. package/lib/model/ameli.js +8 -11
  8. package/lib/model/dosleg.js +47 -52
  9. package/lib/model/index.js +4 -13
  10. package/lib/model/questions.js +15 -18
  11. package/lib/model/sens.d.ts +1 -1
  12. package/lib/model/sens.js +65 -71
  13. package/lib/model/texte.js +17 -25
  14. package/lib/model/util.js +13 -21
  15. package/lib/raw_types/ameli.js +1 -2
  16. package/lib/raw_types/debats.js +1 -2
  17. package/lib/raw_types/dosleg.js +1 -2
  18. package/lib/raw_types/questions.js +1 -2
  19. package/lib/raw_types/sens.js +1 -2
  20. package/lib/raw_types_schemats/ameli.js +1 -2
  21. package/lib/raw_types_schemats/debats.js +1 -2
  22. package/lib/raw_types_schemats/dosleg.js +1 -2
  23. package/lib/raw_types_schemats/questions.js +1 -2
  24. package/lib/raw_types_schemats/sens.js +1 -2
  25. package/lib/scripts/convert_data.js +78 -83
  26. package/lib/scripts/datautil.js +9 -13
  27. package/lib/scripts/parse_textes.js +23 -28
  28. package/lib/scripts/retrieve_documents.js +56 -61
  29. package/lib/scripts/retrieve_open_data.js +44 -49
  30. package/lib/scripts/retrieve_senateurs_photos.js +31 -36
  31. package/lib/scripts/shared/cli_helpers.js +9 -12
  32. package/lib/scripts/shared/util.js +7 -15
  33. package/lib/strings.js +4 -10
  34. package/lib/types/ameli.js +5 -8
  35. package/lib/types/debats.js +2 -5
  36. package/lib/types/dosleg.js +28 -31
  37. package/lib/types/questions.js +1 -2
  38. package/lib/types/sens.js +1 -2
  39. package/lib/types/sessions.js +2 -5
  40. package/lib/types/texte.js +2 -5
  41. package/lib/validators/config.js +4 -7
  42. package/package.json +4 -4
package/lib/config.js CHANGED
@@ -1,7 +1,5 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- require("dotenv/config");
4
- const config_1 = require("./validators/config");
1
+ import "dotenv/config";
2
+ import { validateConfig } from "./validators/config";
5
3
  const config = {
6
4
  db: {
7
5
  host: process.env["TRICOTEUSES_SENAT_DB_HOST"] || "localhost",
@@ -10,9 +8,9 @@ const config = {
10
8
  user: process.env["TRICOTEUSES_SENAT_DB_USER"] || "opendata",
11
9
  },
12
10
  };
13
- const [validConfig, error] = (0, config_1.validateConfig)(config);
11
+ const [validConfig, error] = validateConfig(config);
14
12
  if (error !== null) {
15
13
  console.error(`Error in configuration:\n${JSON.stringify(validConfig, null, 2)}\nError:\n${JSON.stringify(error, null, 2)}`);
16
14
  process.exit(-1);
17
15
  }
18
- exports.default = validConfig;
16
+ export default validConfig;
package/lib/databases.js CHANGED
@@ -1,98 +1,57 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || (function () {
19
- var ownKeys = function(o) {
20
- ownKeys = Object.getOwnPropertyNames || function (o) {
21
- var ar = [];
22
- for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
- return ar;
24
- };
25
- return ownKeys(o);
26
- };
27
- return function (mod) {
28
- if (mod && mod.__esModule) return mod;
29
- var result = {};
30
- if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
- __setModuleDefault(result, mod);
32
- return result;
33
- };
34
- })();
35
- var __importDefault = (this && this.__importDefault) || function (mod) {
36
- return (mod && mod.__esModule) ? mod : { "default": mod };
37
- };
38
- Object.defineProperty(exports, "__esModule", { value: true });
39
- exports.dbSharedConnectionObjectByName = exports.dbSens = exports.dbQuestions = exports.dbDosleg = exports.dbAmeli = exports.dbByName = exports.dbNames = exports.pgPromise = void 0;
40
- exports.checkDatabase = checkDatabase;
41
- exports.checkDatabases = checkDatabases;
42
- const kysely_1 = require("kysely");
43
- const pg = __importStar(require("pg"));
44
- const pg_1 = require("pg");
45
- const pg_cursor_1 = __importDefault(require("pg-cursor"));
46
- const pg_promise_1 = __importDefault(require("pg-promise"));
47
- const config_1 = __importDefault(require("./config"));
48
- const datasets_1 = require("./datasets");
1
+ import { Kysely, PostgresDialect } from "kysely";
2
+ import * as pg from "pg";
3
+ import { types } from "pg";
4
+ import Cursor from "pg-cursor";
5
+ import PgPromise from "pg-promise";
6
+ import config from "./config";
7
+ import { datasets } from "./datasets";
49
8
  // Map int8 to number instead of string
50
9
  // https://github.com/brianc/node-pg-types#use
51
- pg.types.setTypeParser(pg_1.types.builtins.INT8, (val) => {
10
+ pg.types.setTypeParser(types.builtins.INT8, (val) => {
52
11
  return parseInt(val, 10);
53
12
  });
54
- exports.pgPromise = (0, pg_promise_1.default)();
55
- exports.dbNames = ["ameli", "debats", "dosleg"];
56
- exports.dbByName = {};
57
- for (const name of exports.dbNames) {
58
- exports.dbByName[name] = (0, exports.pgPromise)({
13
+ export const pgPromise = PgPromise();
14
+ export const dbNames = ["ameli", "debats", "dosleg"];
15
+ export const dbByName = {};
16
+ for (const name of dbNames) {
17
+ dbByName[name] = pgPromise({
59
18
  database: name,
60
- host: config_1.default.db.host,
61
- password: config_1.default.db.password,
62
- port: config_1.default.db.port,
63
- user: config_1.default.db.user,
19
+ host: config.db.host,
20
+ password: config.db.password,
21
+ port: config.db.port,
22
+ user: config.db.user,
64
23
  });
65
24
  }
66
25
  function createDb(database, schema) {
67
- const dialect = new kysely_1.PostgresDialect({
26
+ const dialect = new PostgresDialect({
68
27
  pool: new pg.Pool({
69
28
  database: database,
70
- host: config_1.default.db.host,
71
- user: config_1.default.db.user,
72
- password: config_1.default.db.password,
73
- port: config_1.default.db.port,
29
+ host: config.db.host,
30
+ user: config.db.user,
31
+ password: config.db.password,
32
+ port: config.db.port,
74
33
  max: 10,
75
34
  }),
76
- cursor: pg_cursor_1.default,
35
+ cursor: Cursor,
77
36
  });
78
- return new kysely_1.Kysely({
37
+ return new Kysely({
79
38
  log: ["error"],
80
39
  dialect: dialect,
81
40
  }).withSchema(schema);
82
41
  }
83
- exports.dbAmeli = createDb(datasets_1.datasets.ameli.database, datasets_1.datasets.ameli.schema);
84
- exports.dbDosleg = createDb(datasets_1.datasets.dosleg.database, datasets_1.datasets.dosleg.schema);
85
- exports.dbQuestions = createDb(datasets_1.datasets.questions.database, datasets_1.datasets.questions.schema);
86
- exports.dbSens = createDb(datasets_1.datasets.sens.database, datasets_1.datasets.sens.schema);
87
- exports.dbSharedConnectionObjectByName = {};
88
- async function checkDatabase(name) {
42
+ export const dbAmeli = createDb(datasets.ameli.database, datasets.ameli.schema);
43
+ export const dbDosleg = createDb(datasets.dosleg.database, datasets.dosleg.schema);
44
+ export const dbQuestions = createDb(datasets.questions.database, datasets.questions.schema);
45
+ export const dbSens = createDb(datasets.sens.database, datasets.sens.schema);
46
+ export let dbSharedConnectionObjectByName = {};
47
+ export async function checkDatabase(name) {
89
48
  // Check that database exists.
90
- const db = exports.dbByName[name];
91
- exports.dbSharedConnectionObjectByName[name] = await db.connect();
49
+ const db = dbByName[name];
50
+ dbSharedConnectionObjectByName[name] = await db.connect();
92
51
  return db;
93
52
  }
94
- async function checkDatabases() {
95
- for (const name of exports.dbNames) {
53
+ export async function checkDatabases() {
54
+ for (const name of dbNames) {
96
55
  await checkDatabase(name);
97
56
  }
98
57
  }
package/lib/datasets.js CHANGED
@@ -1,15 +1,7 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.datasets = exports.EnabledDatasets = void 0;
7
- exports.getEnabledDatasets = getEnabledDatasets;
8
- exports.getChosenFromEnabledDatasets = getChosenFromEnabledDatasets;
9
- const fs_extra_1 = __importDefault(require("fs-extra"));
10
- const path_1 = __importDefault(require("path"));
11
- const assert_1 = __importDefault(require("assert"));
12
- var EnabledDatasets;
1
+ import fs from "fs-extra";
2
+ import path from "path";
3
+ import assert from "assert";
4
+ export var EnabledDatasets;
13
5
  (function (EnabledDatasets) {
14
6
  EnabledDatasets[EnabledDatasets["None"] = 0] = "None";
15
7
  EnabledDatasets[EnabledDatasets["Ameli"] = 1] = "Ameli";
@@ -19,16 +11,16 @@ var EnabledDatasets;
19
11
  EnabledDatasets[EnabledDatasets["Sens"] = 16] = "Sens";
20
12
  EnabledDatasets[EnabledDatasets["PhotosSenateurs"] = 32] = "PhotosSenateurs";
21
13
  EnabledDatasets[EnabledDatasets["All"] = 63] = "All";
22
- })(EnabledDatasets || (exports.EnabledDatasets = EnabledDatasets = {}));
23
- exports.datasets = {
14
+ })(EnabledDatasets || (EnabledDatasets = {}));
15
+ export const datasets = {
24
16
  ameli: {
25
17
  database: "ameli",
26
18
  repairEncoding: false,
27
19
  repairZip: (dataset, dataDir) => {
28
20
  const sqlFilename = `${dataset.database}.sql`;
29
- const sqlFilePath = path_1.default.join(dataDir, sqlFilename);
30
- fs_extra_1.default.removeSync(sqlFilePath);
31
- fs_extra_1.default.moveSync(path_1.default.join(dataDir, "var", "opt", "opendata", sqlFilename), sqlFilePath);
21
+ const sqlFilePath = path.join(dataDir, sqlFilename);
22
+ fs.removeSync(sqlFilePath);
23
+ fs.moveSync(path.join(dataDir, "var", "opt", "opendata", sqlFilename), sqlFilePath);
32
24
  },
33
25
  schema: "public",
34
26
  title: "Amendements",
@@ -60,29 +52,29 @@ exports.datasets = {
60
52
  repairEncoding: true,
61
53
  repairZip: (dataset, dataDir) => {
62
54
  const sqlFilename = `${dataset.database}.sql`;
63
- const sqlFilePath = path_1.default.join(dataDir, sqlFilename);
64
- fs_extra_1.default.removeSync(sqlFilePath);
65
- fs_extra_1.default.moveSync(path_1.default.join(dataDir, "export_sens.sql"), sqlFilePath);
55
+ const sqlFilePath = path.join(dataDir, sqlFilename);
56
+ fs.removeSync(sqlFilePath);
57
+ fs.moveSync(path.join(dataDir, "export_sens.sql"), sqlFilePath);
66
58
  },
67
59
  schema: "public",
68
60
  title: "Sénateurs (y compris organes et présence)",
69
61
  url: "https://data.senat.fr/data/senateurs/export_sens.zip",
70
62
  },
71
63
  };
72
- function getEnabledDatasets(categories) {
73
- categories.map((datasetName) => assert_1.default.notStrictEqual(EnabledDatasets[datasetName], undefined, `Unknown name of dataset: ${datasetName}`));
64
+ export function getEnabledDatasets(categories) {
65
+ categories.map((datasetName) => assert.notStrictEqual(EnabledDatasets[datasetName], undefined, `Unknown name of dataset: ${datasetName}`));
74
66
  return categories.reduce((enabledDatasets, datasetName) => {
75
67
  const enabledDataset = EnabledDatasets[datasetName];
76
68
  return enabledDatasets | (enabledDataset || EnabledDatasets.None);
77
69
  }, EnabledDatasets.None);
78
70
  }
79
- function getChosenFromEnabledDatasets(categories) {
71
+ export function getChosenFromEnabledDatasets(categories) {
80
72
  const enabledDatasets = getEnabledDatasets(categories);
81
73
  return [
82
- enabledDatasets & EnabledDatasets.Ameli ? exports.datasets.ameli : null,
83
- enabledDatasets & EnabledDatasets.Debats ? exports.datasets.debats : null,
84
- enabledDatasets & EnabledDatasets.DosLeg ? exports.datasets.dosleg : null,
85
- enabledDatasets & EnabledDatasets.Questions ? exports.datasets.questions : null,
86
- enabledDatasets & EnabledDatasets.Sens ? exports.datasets.sens : null,
74
+ enabledDatasets & EnabledDatasets.Ameli ? datasets.ameli : null,
75
+ enabledDatasets & EnabledDatasets.Debats ? datasets.debats : null,
76
+ enabledDatasets & EnabledDatasets.DosLeg ? datasets.dosleg : null,
77
+ enabledDatasets & EnabledDatasets.Questions ? datasets.questions : null,
78
+ enabledDatasets & EnabledDatasets.Sens ? datasets.sens : null,
87
79
  ].filter((dataset) => dataset !== null);
88
80
  }
package/lib/index.d.ts CHANGED
@@ -1,3 +1,7 @@
1
+ export type { AmendementResult, } from "./model/ameli";
2
+ export type { DossierLegislatifResult, } from "./model/dosleg";
3
+ export type { QuestionResult, } from "./model/questions";
4
+ export type { CirconscriptionResult, OrganismeResult, SenateurResult, } from "./model/sens";
1
5
  export { sesFieldsToParseInt, sesFieldsToTrim, subFieldsToParseInt, subFieldsToTrim, txtAmeliFieldsToTrim, } from "./types/ameli";
2
6
  export type { Ses, Sub, TxtAmeli } from "./types/ameli";
3
7
  export { debatsFieldsToTrim, lecassdebFieldsToTrim } from "./types/debats";
@@ -6,3 +10,4 @@ export { assFieldsToTrim, audFieldsToTrim, auteurFieldsToTrim, dateSeanceFieldsT
6
10
  export type { Ass, Aud, Auteur, DateSeance, DecCoc, DenRap, DocAtt, Ecr, EtaLoi, LecAss, LecAssRap, Lecture, Loi, Org, OriTxt, Qua, Rap, RapOrg, Scr, Texte, TypAtt, TypLec, TypLoi, TypTxt, TypUrl, } from "./types/dosleg";
7
11
  export type { Photo, Sen } from "./types/sens";
8
12
  export { Session } from "./types/sessions";
13
+ export type { Alinea, Article, Division, DivisionTag, FlatTexte } from "./types/texte";
package/lib/index.js CHANGED
@@ -1,43 +1,4 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.Session = exports.typurlFieldsToTrim = exports.typtxtFieldsToTrim = exports.typloiFieldsToTrim = exports.typlecFieldsToTrim = exports.typattFieldsToTrim = exports.texteFieldsToTrim = exports.texteFieldsToParseInt = exports.scrFieldsToTrim = exports.raporgFieldsToTrim = exports.rapFieldsToTrim = exports.rapFieldsToParseInt = exports.quaFieldsToTrim = exports.oritxtFieldsToTrim = exports.orgFieldsToTrim = exports.loiFieldsToTrim = exports.lectureFieldsToTrim = exports.lecassrapFieldsToTrim = exports.lecassFieldsToTrim = exports.etaloiFieldsToTrim = exports.ecrFieldsToTrim = exports.docattFieldsToTrim = exports.docattFieldsToParseInt = exports.denrapFieldsToTrim = exports.deccocFieldsToTrim = exports.dateSeanceFieldsToTrim = exports.auteurFieldsToTrim = exports.audFieldsToTrim = exports.assFieldsToTrim = exports.lecassdebFieldsToTrim = exports.debatsFieldsToTrim = exports.txtAmeliFieldsToTrim = exports.subFieldsToTrim = exports.subFieldsToParseInt = exports.sesFieldsToTrim = exports.sesFieldsToParseInt = void 0;
4
- var ameli_1 = require("./types/ameli");
5
- Object.defineProperty(exports, "sesFieldsToParseInt", { enumerable: true, get: function () { return ameli_1.sesFieldsToParseInt; } });
6
- Object.defineProperty(exports, "sesFieldsToTrim", { enumerable: true, get: function () { return ameli_1.sesFieldsToTrim; } });
7
- Object.defineProperty(exports, "subFieldsToParseInt", { enumerable: true, get: function () { return ameli_1.subFieldsToParseInt; } });
8
- Object.defineProperty(exports, "subFieldsToTrim", { enumerable: true, get: function () { return ameli_1.subFieldsToTrim; } });
9
- Object.defineProperty(exports, "txtAmeliFieldsToTrim", { enumerable: true, get: function () { return ameli_1.txtAmeliFieldsToTrim; } });
10
- var debats_1 = require("./types/debats");
11
- Object.defineProperty(exports, "debatsFieldsToTrim", { enumerable: true, get: function () { return debats_1.debatsFieldsToTrim; } });
12
- Object.defineProperty(exports, "lecassdebFieldsToTrim", { enumerable: true, get: function () { return debats_1.lecassdebFieldsToTrim; } });
13
- var dosleg_1 = require("./types/dosleg");
14
- Object.defineProperty(exports, "assFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.assFieldsToTrim; } });
15
- Object.defineProperty(exports, "audFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.audFieldsToTrim; } });
16
- Object.defineProperty(exports, "auteurFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.auteurFieldsToTrim; } });
17
- Object.defineProperty(exports, "dateSeanceFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.dateSeanceFieldsToTrim; } });
18
- Object.defineProperty(exports, "deccocFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.deccocFieldsToTrim; } });
19
- Object.defineProperty(exports, "denrapFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.denrapFieldsToTrim; } });
20
- Object.defineProperty(exports, "docattFieldsToParseInt", { enumerable: true, get: function () { return dosleg_1.docattFieldsToParseInt; } });
21
- Object.defineProperty(exports, "docattFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.docattFieldsToTrim; } });
22
- Object.defineProperty(exports, "ecrFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.ecrFieldsToTrim; } });
23
- Object.defineProperty(exports, "etaloiFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.etaloiFieldsToTrim; } });
24
- Object.defineProperty(exports, "lecassFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.lecassFieldsToTrim; } });
25
- Object.defineProperty(exports, "lecassrapFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.lecassrapFieldsToTrim; } });
26
- Object.defineProperty(exports, "lectureFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.lectureFieldsToTrim; } });
27
- Object.defineProperty(exports, "loiFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.loiFieldsToTrim; } });
28
- Object.defineProperty(exports, "orgFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.orgFieldsToTrim; } });
29
- Object.defineProperty(exports, "oritxtFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.oritxtFieldsToTrim; } });
30
- Object.defineProperty(exports, "quaFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.quaFieldsToTrim; } });
31
- Object.defineProperty(exports, "rapFieldsToParseInt", { enumerable: true, get: function () { return dosleg_1.rapFieldsToParseInt; } });
32
- Object.defineProperty(exports, "rapFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.rapFieldsToTrim; } });
33
- Object.defineProperty(exports, "raporgFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.raporgFieldsToTrim; } });
34
- Object.defineProperty(exports, "scrFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.scrFieldsToTrim; } });
35
- Object.defineProperty(exports, "texteFieldsToParseInt", { enumerable: true, get: function () { return dosleg_1.texteFieldsToParseInt; } });
36
- Object.defineProperty(exports, "texteFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.texteFieldsToTrim; } });
37
- Object.defineProperty(exports, "typattFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.typattFieldsToTrim; } });
38
- Object.defineProperty(exports, "typlecFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.typlecFieldsToTrim; } });
39
- Object.defineProperty(exports, "typloiFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.typloiFieldsToTrim; } });
40
- Object.defineProperty(exports, "typtxtFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.typtxtFieldsToTrim; } });
41
- Object.defineProperty(exports, "typurlFieldsToTrim", { enumerable: true, get: function () { return dosleg_1.typurlFieldsToTrim; } });
42
- var sessions_1 = require("./types/sessions");
43
- Object.defineProperty(exports, "Session", { enumerable: true, get: function () { return sessions_1.Session; } });
1
+ export { sesFieldsToParseInt, sesFieldsToTrim, subFieldsToParseInt, subFieldsToTrim, txtAmeliFieldsToTrim, } from "./types/ameli";
2
+ export { debatsFieldsToTrim, lecassdebFieldsToTrim } from "./types/debats";
3
+ export { assFieldsToTrim, audFieldsToTrim, auteurFieldsToTrim, dateSeanceFieldsToTrim, deccocFieldsToTrim, denrapFieldsToTrim, docattFieldsToParseInt, docattFieldsToTrim, ecrFieldsToTrim, etaloiFieldsToTrim, lecassFieldsToTrim, lecassrapFieldsToTrim, lectureFieldsToTrim, loiFieldsToTrim, orgFieldsToTrim, oritxtFieldsToTrim, quaFieldsToTrim, rapFieldsToParseInt, rapFieldsToTrim, raporgFieldsToTrim, scrFieldsToTrim, texteFieldsToParseInt, texteFieldsToTrim, typattFieldsToTrim, typlecFieldsToTrim, typloiFieldsToTrim, typtxtFieldsToTrim, typurlFieldsToTrim, } from "./types/dosleg";
4
+ export { Session } from "./types/sessions";
package/lib/loaders.js CHANGED
@@ -1,62 +1,44 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.RAPPORT_FOLDER = exports.DOCUMENT_METADATA_FILE = exports.TEXTE_TRANSFORMED_FOLDER = exports.TEXTE_ORIGINAL_FOLDER = exports.TEXTE_FOLDER = exports.SENS_SENATEURS_FOLDER = exports.SENS_ORGANISMES_FOLDER = exports.SENS_CIRCONSCRIPTIONS_FOLDER = exports.DOSLEG_DOSSIERS_FOLDER = exports.EnabledDatasets = void 0;
7
- exports.iterFilePaths = iterFilePaths;
8
- exports.iterLoadSenatAmendements = iterLoadSenatAmendements;
9
- exports.iterLoadSenatDossiersLegislatifs = iterLoadSenatDossiersLegislatifs;
10
- exports.iterLoadSenatDossiersLegislatifsTexteUrls = iterLoadSenatDossiersLegislatifsTexteUrls;
11
- exports.iterLoadSenatDossiersLegislatifsRapportUrls = iterLoadSenatDossiersLegislatifsRapportUrls;
12
- exports.iterLoadSenatDossiersLegislatifsTextes = iterLoadSenatDossiersLegislatifsTextes;
13
- exports.loadSenatTexteContent = loadSenatTexteContent;
14
- exports.iterLoadSenatOrganismes = iterLoadSenatOrganismes;
15
- exports.iterLoadSenatSenateurs = iterLoadSenatSenateurs;
16
- exports.iterLoadSenatCirconscriptions = iterLoadSenatCirconscriptions;
17
- exports.iterLoadSenatQuestions = iterLoadSenatQuestions;
18
- const fs_1 = __importDefault(require("fs"));
19
- const path_1 = __importDefault(require("path"));
20
- const legislatures_json_1 = __importDefault(require("./legislatures.json"));
21
- const datasets_1 = require("./datasets");
22
- const datautil_1 = require("./scripts/datautil");
23
- var datasets_2 = require("./datasets");
24
- Object.defineProperty(exports, "EnabledDatasets", { enumerable: true, get: function () { return datasets_2.EnabledDatasets; } });
25
- exports.DOSLEG_DOSSIERS_FOLDER = "dossiers";
26
- exports.SENS_CIRCONSCRIPTIONS_FOLDER = "circonscriptions";
27
- exports.SENS_ORGANISMES_FOLDER = "organismes";
28
- exports.SENS_SENATEURS_FOLDER = "senateurs";
29
- exports.TEXTE_FOLDER = "leg";
30
- exports.TEXTE_ORIGINAL_FOLDER = "original";
31
- exports.TEXTE_TRANSFORMED_FOLDER = "transformed";
32
- exports.DOCUMENT_METADATA_FILE = "metadata.json";
33
- exports.RAPPORT_FOLDER = "rap";
34
- function* iterFilePaths(dirPath) {
35
- if (dirPath && fs_1.default.existsSync(dirPath)) {
36
- const files = fs_1.default.readdirSync(dirPath, {
1
+ import fs from "fs";
2
+ import path from "path";
3
+ import legislatures from "./legislatures.json";
4
+ import { datasets } from "./datasets";
5
+ import { UNDEFINED_SESSION } from "./scripts/datautil";
6
+ export { EnabledDatasets } from "./datasets";
7
+ export const DOSLEG_DOSSIERS_FOLDER = "dossiers";
8
+ export const SENS_CIRCONSCRIPTIONS_FOLDER = "circonscriptions";
9
+ export const SENS_ORGANISMES_FOLDER = "organismes";
10
+ export const SENS_SENATEURS_FOLDER = "senateurs";
11
+ export const TEXTE_FOLDER = "leg";
12
+ export const TEXTE_ORIGINAL_FOLDER = "original";
13
+ export const TEXTE_TRANSFORMED_FOLDER = "transformed";
14
+ export const DOCUMENT_METADATA_FILE = "metadata.json";
15
+ export const RAPPORT_FOLDER = "rap";
16
+ export function* iterFilePaths(dirPath) {
17
+ if (dirPath && fs.existsSync(dirPath)) {
18
+ const files = fs.readdirSync(dirPath, {
37
19
  withFileTypes: true,
38
20
  recursive: true,
39
21
  });
40
22
  for (const file of files) {
41
23
  if (file.isFile()) {
42
- yield path_1.default.join(file.parentPath, file.name);
24
+ yield path.join(file.parentPath, file.name);
43
25
  }
44
26
  }
45
27
  }
46
28
  }
47
29
  function* iterLoadSenatItems(dataDir, dataset, legislatureOrSession, subDir, { log = false } = {}) {
48
- let itemsDir = path_1.default.join(dataDir, dataset.database);
30
+ let itemsDir = path.join(dataDir, dataset.database);
49
31
  if (subDir) {
50
- itemsDir = path_1.default.join(itemsDir, subDir);
32
+ itemsDir = path.join(itemsDir, subDir);
51
33
  }
52
34
  if (legislatureOrSession) {
53
- itemsDir = path_1.default.join(itemsDir, String(legislatureOrSession));
35
+ itemsDir = path.join(itemsDir, String(legislatureOrSession));
54
36
  }
55
37
  for (const filePath of iterFilePaths(itemsDir)) {
56
38
  if (log) {
57
39
  console.log(`Loading file: ${filePath}…`);
58
40
  }
59
- const itemJson = fs_1.default.readFileSync(filePath, { encoding: "utf8" });
41
+ const itemJson = fs.readFileSync(filePath, { encoding: "utf8" });
60
42
  const item = JSON.parse(itemJson);
61
43
  const filePathFromDataset = filePath.substring(filePath.indexOf(dataset.database) + dataset.database.length);
62
44
  yield {
@@ -66,25 +48,25 @@ function* iterLoadSenatItems(dataDir, dataset, legislatureOrSession, subDir, { l
66
48
  };
67
49
  }
68
50
  }
69
- function* iterLoadSenatAmendements(dataDir, session, options = {}) {
70
- for (const amendementItem of iterLoadSenatItems(dataDir, datasets_1.datasets.ameli, session, undefined, options)) {
51
+ export function* iterLoadSenatAmendements(dataDir, session, options = {}) {
52
+ for (const amendementItem of iterLoadSenatItems(dataDir, datasets.ameli, session, undefined, options)) {
71
53
  yield amendementItem;
72
54
  }
73
55
  }
74
- function* iterLoadSenatDossiersLegislatifs(dataDir, session, options = {}) {
75
- for (const dossierLegislatifItem of iterLoadSenatItems(dataDir, datasets_1.datasets.dosleg, session, exports.DOSLEG_DOSSIERS_FOLDER, options)) {
56
+ export function* iterLoadSenatDossiersLegislatifs(dataDir, session, options = {}) {
57
+ for (const dossierLegislatifItem of iterLoadSenatItems(dataDir, datasets.dosleg, session, DOSLEG_DOSSIERS_FOLDER, options)) {
76
58
  yield dossierLegislatifItem;
77
59
  }
78
60
  }
79
- function* iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session) {
80
- let itemsDir = path_1.default.join(dataDir, exports.TEXTE_FOLDER, exports.TEXTE_ORIGINAL_FOLDER);
61
+ export function* iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session) {
62
+ let itemsDir = path.join(dataDir, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER);
81
63
  if (session) {
82
- itemsDir = path_1.default.join(itemsDir, session.toString());
64
+ itemsDir = path.join(itemsDir, session.toString());
83
65
  }
84
66
  for (const filePath of iterFilePaths(itemsDir)) {
85
- const parsedFilePath = path_1.default.parse(filePath);
86
- if (parsedFilePath.base === exports.DOCUMENT_METADATA_FILE) {
87
- const itemJson = fs_1.default.readFileSync(filePath, { encoding: "utf8" });
67
+ const parsedFilePath = path.parse(filePath);
68
+ if (parsedFilePath.base === DOCUMENT_METADATA_FILE) {
69
+ const itemJson = fs.readFileSync(filePath, { encoding: "utf8" });
88
70
  const item = JSON.parse(itemJson);
89
71
  yield {
90
72
  item,
@@ -92,15 +74,15 @@ function* iterLoadSenatDossiersLegislatifsTexteUrls(dataDir, session) {
92
74
  }
93
75
  }
94
76
  }
95
- function* iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session) {
96
- let itemsDir = path_1.default.join(dataDir, exports.RAPPORT_FOLDER);
77
+ export function* iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session) {
78
+ let itemsDir = path.join(dataDir, RAPPORT_FOLDER);
97
79
  if (session) {
98
- itemsDir = path_1.default.join(itemsDir, session.toString());
80
+ itemsDir = path.join(itemsDir, session.toString());
99
81
  }
100
82
  for (const filePath of iterFilePaths(itemsDir)) {
101
- const parsedFilePath = path_1.default.parse(filePath);
102
- if (parsedFilePath.base === exports.DOCUMENT_METADATA_FILE) {
103
- const itemJson = fs_1.default.readFileSync(filePath, { encoding: "utf8" });
83
+ const parsedFilePath = path.parse(filePath);
84
+ if (parsedFilePath.base === DOCUMENT_METADATA_FILE) {
85
+ const itemJson = fs.readFileSync(filePath, { encoding: "utf8" });
104
86
  const item = JSON.parse(itemJson);
105
87
  yield {
106
88
  item,
@@ -108,7 +90,7 @@ function* iterLoadSenatDossiersLegislatifsRapportUrls(dataDir, session) {
108
90
  }
109
91
  }
110
92
  }
111
- function* iterLoadSenatDossiersLegislatifsTextes(dataDir, session, options = {}) {
93
+ export function* iterLoadSenatDossiersLegislatifsTextes(dataDir, session, options = {}) {
112
94
  for (const { item: dossierLegislatif } of iterLoadSenatDossiersLegislatifs(dataDir, session, options)) {
113
95
  for (const lecture of dossierLegislatif.lectures) {
114
96
  const lecturesSenat = lecture.lectures_assemblee.filter((lectureAssemblee) => lectureAssemblee.assemblee === "Sénat");
@@ -127,8 +109,8 @@ function* iterLoadSenatDossiersLegislatifsTextes(dataDir, session, options = {})
127
109
  item: enrichedTexte,
128
110
  };
129
111
  if (texte.url) {
130
- const texteName = path_1.default.parse(texte.url).name;
131
- texteItem.filePathFromDataset = path_1.default.join(`${texte.session ?? datautil_1.UNDEFINED_SESSION}`, texteName, `${texteName}.pdf`);
112
+ const texteName = path.parse(texte.url).name;
113
+ texteItem.filePathFromDataset = path.join(`${texte.session ?? UNDEFINED_SESSION}`, texteName, `${texteName}.pdf`);
132
114
  }
133
115
  yield texteItem;
134
116
  }
@@ -136,25 +118,25 @@ function* iterLoadSenatDossiersLegislatifsTextes(dataDir, session, options = {})
136
118
  }
137
119
  }
138
120
  }
139
- function loadSenatTexteContent(dataDir, textePathFromDataset) {
140
- const parsedTextePath = path_1.default.parse(textePathFromDataset);
121
+ export function loadSenatTexteContent(dataDir, textePathFromDataset) {
122
+ const parsedTextePath = path.parse(textePathFromDataset);
141
123
  const jsonTexteName = `${parsedTextePath.name}.json`;
142
- const fullTextePath = path_1.default.join(dataDir, exports.TEXTE_FOLDER, exports.TEXTE_TRANSFORMED_FOLDER, parsedTextePath.dir, jsonTexteName);
143
- if (!fs_1.default.existsSync(fullTextePath)) {
124
+ const fullTextePath = path.join(dataDir, TEXTE_FOLDER, TEXTE_TRANSFORMED_FOLDER, parsedTextePath.dir, jsonTexteName);
125
+ if (!fs.existsSync(fullTextePath)) {
144
126
  return { item: null };
145
127
  }
146
- const texteJson = fs_1.default.readFileSync(fullTextePath, { encoding: "utf8" });
128
+ const texteJson = fs.readFileSync(fullTextePath, { encoding: "utf8" });
147
129
  return { item: JSON.parse(texteJson) };
148
130
  }
149
- function* iterLoadSenatOrganismes(dataDir, options = {}) {
150
- for (const organismeItem of iterLoadSenatItems(dataDir, datasets_1.datasets.sens, undefined, exports.SENS_ORGANISMES_FOLDER, options)) {
131
+ export function* iterLoadSenatOrganismes(dataDir, options = {}) {
132
+ for (const organismeItem of iterLoadSenatItems(dataDir, datasets.sens, undefined, SENS_ORGANISMES_FOLDER, options)) {
151
133
  yield organismeItem;
152
134
  }
153
135
  }
154
- function* iterLoadSenatSenateurs(dataDir, legislature, options = {}) {
155
- const dateDebutLegislatureStr = legislatures_json_1.default.find((legislatureInfo) => legislatureInfo.numero === legislature)?.date_debut;
136
+ export function* iterLoadSenatSenateurs(dataDir, legislature, options = {}) {
137
+ const dateDebutLegislatureStr = legislatures.find((legislatureInfo) => legislatureInfo.numero === legislature)?.date_debut;
156
138
  const dateDebutLegislature = new Date(dateDebutLegislatureStr);
157
- for (const senateurItem of iterLoadSenatItems(dataDir, datasets_1.datasets.sens, undefined, exports.SENS_SENATEURS_FOLDER, options)) {
139
+ for (const senateurItem of iterLoadSenatItems(dataDir, datasets.sens, undefined, SENS_SENATEURS_FOLDER, options)) {
158
140
  const dateFinMandatSenateur = senateurItem.item.mandats_senateur[0]
159
141
  ?.date_fin
160
142
  ? new Date(senateurItem.item.mandats_senateur[0]?.date_fin)
@@ -165,13 +147,13 @@ function* iterLoadSenatSenateurs(dataDir, legislature, options = {}) {
165
147
  yield senateurItem;
166
148
  }
167
149
  }
168
- function* iterLoadSenatCirconscriptions(dataDir, options = {}) {
169
- for (const circonscriptionItem of iterLoadSenatItems(dataDir, datasets_1.datasets.sens, undefined, exports.SENS_CIRCONSCRIPTIONS_FOLDER, options)) {
150
+ export function* iterLoadSenatCirconscriptions(dataDir, options = {}) {
151
+ for (const circonscriptionItem of iterLoadSenatItems(dataDir, datasets.sens, undefined, SENS_CIRCONSCRIPTIONS_FOLDER, options)) {
170
152
  yield circonscriptionItem;
171
153
  }
172
154
  }
173
- function* iterLoadSenatQuestions(dataDir, legislature, options = {}) {
174
- for (const questionItem of iterLoadSenatItems(dataDir, datasets_1.datasets.questions, legislature, undefined, options)) {
155
+ export function* iterLoadSenatQuestions(dataDir, legislature, options = {}) {
156
+ for (const questionItem of iterLoadSenatItems(dataDir, datasets.questions, legislature, undefined, options)) {
175
157
  yield questionItem;
176
158
  }
177
159
  }
@@ -1,11 +1,8 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.findAll = findAll;
4
- const postgres_1 = require("kysely/helpers/postgres");
5
- const databases_1 = require("../databases");
6
- const util_1 = require("./util");
1
+ import { jsonArrayFrom } from "kysely/helpers/postgres";
2
+ import { dbAmeli } from "../databases";
3
+ import { concat, toDateString } from "./util";
7
4
  function auteurs(amendementId) {
8
- return (0, postgres_1.jsonArrayFrom)(databases_1.dbAmeli
5
+ return jsonArrayFrom(dbAmeli
9
6
  .selectFrom("amdsen")
10
7
  .leftJoin("sen_ameli", "amdsen.senid", "sen_ameli.entid")
11
8
  .where("amdsen.amdid", "=", amendementId)
@@ -16,7 +13,7 @@ function auteurs(amendementId) {
16
13
  ])
17
14
  .orderBy("amdsen.rng asc"));
18
15
  }
19
- const findAllQuery = databases_1.dbAmeli
16
+ const findAllQuery = dbAmeli
20
17
  .selectFrom("amd")
21
18
  .leftJoin("sub", "amd.subid", "sub.id")
22
19
  .leftJoin("typsub", "sub.typid", "typsub.id")
@@ -58,7 +55,7 @@ const findAllQuery = databases_1.dbAmeli
58
55
  "sub.lic as subdivision_libelle_court",
59
56
  "typsub.lib as subdivision_type",
60
57
  "amd.alinea as alinea",
61
- (0, util_1.toDateString)(ref("amd.datdep")).as("date_depot"),
58
+ toDateString(ref("amd.datdep")).as("date_depot"),
62
59
  eb
63
60
  .case()
64
61
  .when("amd.etaid", "=", 8)
@@ -72,7 +69,7 @@ const findAllQuery = databases_1.dbAmeli
72
69
  "avigvt.lib as avis_gouvernement",
73
70
  eb.fn.coalesce("sor.lib", "irr.libirr").as("sort"),
74
71
  "amd.rev as revision",
75
- (0, util_1.concat)(val("https://www.senat.fr/amendements/"), ref("ses.lil"), val("/"), ref("txt_ameli.numabs"), val("/Amdt_"), ref("amd.numabs"), val(".html")).as("url"),
72
+ concat(val("https://www.senat.fr/amendements/"), ref("ses.lil"), val("/"), ref("txt_ameli.numabs"), val("/Amdt_"), ref("amd.numabs"), val(".html")).as("url"),
76
73
  "grppol_ameli.lilcou as au_nom_de_groupe_politique",
77
74
  "com_ameli.lil as au_nom_de_commission",
78
75
  eb
@@ -84,6 +81,6 @@ const findAllQuery = databases_1.dbAmeli
84
81
  .as("auteur_est_gouvernement"),
85
82
  ])
86
83
  .select(({ ref }) => [auteurs(ref("amd.id")).as("auteurs")]);
87
- function findAll() {
84
+ export function findAll() {
88
85
  return findAllQuery.stream();
89
86
  }