@tricoteuses/senat 2.22.10 → 2.22.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. package/lib/src/scripts/retrieve_documents.js +24 -15
  2. package/package.json +1 -1
  3. package/lib/config.d.ts +0 -21
  4. package/lib/config.js +0 -27
  5. package/lib/databases.d.ts +0 -2
  6. package/lib/databases.js +0 -26
  7. package/lib/datasets.d.ts +0 -34
  8. package/lib/datasets.js +0 -233
  9. package/lib/git.d.ts +0 -26
  10. package/lib/git.js +0 -167
  11. package/lib/index.d.ts +0 -13
  12. package/lib/index.js +0 -1
  13. package/lib/loaders.d.ts +0 -58
  14. package/lib/loaders.js +0 -286
  15. package/lib/model/agenda.d.ts +0 -6
  16. package/lib/model/agenda.js +0 -148
  17. package/lib/model/ameli.d.ts +0 -51
  18. package/lib/model/ameli.js +0 -147
  19. package/lib/model/commission.d.ts +0 -18
  20. package/lib/model/commission.js +0 -269
  21. package/lib/model/debats.d.ts +0 -67
  22. package/lib/model/debats.js +0 -95
  23. package/lib/model/documents.d.ts +0 -12
  24. package/lib/model/documents.js +0 -138
  25. package/lib/model/dosleg.d.ts +0 -7
  26. package/lib/model/dosleg.js +0 -326
  27. package/lib/model/index.d.ts +0 -7
  28. package/lib/model/index.js +0 -7
  29. package/lib/model/questions.d.ts +0 -45
  30. package/lib/model/questions.js +0 -89
  31. package/lib/model/scrutins.d.ts +0 -13
  32. package/lib/model/scrutins.js +0 -114
  33. package/lib/model/seance.d.ts +0 -3
  34. package/lib/model/seance.js +0 -267
  35. package/lib/model/sens.d.ts +0 -146
  36. package/lib/model/sens.js +0 -454
  37. package/lib/model/texte.d.ts +0 -7
  38. package/lib/model/texte.js +0 -228
  39. package/lib/model/util.d.ts +0 -9
  40. package/lib/model/util.js +0 -38
  41. package/lib/parsers/texte.d.ts +0 -7
  42. package/lib/parsers/texte.js +0 -228
  43. package/lib/raw_types/ameli.d.ts +0 -914
  44. package/lib/raw_types/ameli.js +0 -5
  45. package/lib/raw_types/debats.d.ts +0 -207
  46. package/lib/raw_types/debats.js +0 -5
  47. package/lib/raw_types/dosleg.d.ts +0 -1619
  48. package/lib/raw_types/dosleg.js +0 -5
  49. package/lib/raw_types/questions.d.ts +0 -423
  50. package/lib/raw_types/questions.js +0 -5
  51. package/lib/raw_types/senat.d.ts +0 -11372
  52. package/lib/raw_types/senat.js +0 -5
  53. package/lib/raw_types/sens.d.ts +0 -8248
  54. package/lib/raw_types/sens.js +0 -5
  55. package/lib/raw_types_schemats/ameli.d.ts +0 -539
  56. package/lib/raw_types_schemats/ameli.js +0 -2
  57. package/lib/raw_types_schemats/debats.d.ts +0 -127
  58. package/lib/raw_types_schemats/debats.js +0 -2
  59. package/lib/raw_types_schemats/dosleg.d.ts +0 -977
  60. package/lib/raw_types_schemats/dosleg.js +0 -2
  61. package/lib/raw_types_schemats/questions.d.ts +0 -237
  62. package/lib/raw_types_schemats/questions.js +0 -2
  63. package/lib/raw_types_schemats/sens.d.ts +0 -6915
  64. package/lib/raw_types_schemats/sens.js +0 -2
  65. package/lib/scripts/convert_data.d.ts +0 -1
  66. package/lib/scripts/convert_data.js +0 -354
  67. package/lib/scripts/data-download.d.ts +0 -1
  68. package/lib/scripts/data-download.js +0 -12
  69. package/lib/scripts/datautil.d.ts +0 -8
  70. package/lib/scripts/datautil.js +0 -34
  71. package/lib/scripts/parse_textes.d.ts +0 -1
  72. package/lib/scripts/parse_textes.js +0 -44
  73. package/lib/scripts/retrieve_agenda.d.ts +0 -1
  74. package/lib/scripts/retrieve_agenda.js +0 -132
  75. package/lib/scripts/retrieve_cr_commission.d.ts +0 -1
  76. package/lib/scripts/retrieve_cr_commission.js +0 -364
  77. package/lib/scripts/retrieve_cr_seance.d.ts +0 -6
  78. package/lib/scripts/retrieve_cr_seance.js +0 -347
  79. package/lib/scripts/retrieve_documents.d.ts +0 -3
  80. package/lib/scripts/retrieve_documents.js +0 -219
  81. package/lib/scripts/retrieve_open_data.d.ts +0 -1
  82. package/lib/scripts/retrieve_open_data.js +0 -316
  83. package/lib/scripts/retrieve_senateurs_photos.d.ts +0 -1
  84. package/lib/scripts/retrieve_senateurs_photos.js +0 -147
  85. package/lib/scripts/retrieve_videos.d.ts +0 -1
  86. package/lib/scripts/retrieve_videos.js +0 -461
  87. package/lib/scripts/shared/cli_helpers.d.ts +0 -95
  88. package/lib/scripts/shared/cli_helpers.js +0 -91
  89. package/lib/scripts/shared/util.d.ts +0 -4
  90. package/lib/scripts/shared/util.js +0 -35
  91. package/lib/scripts/test_iter_load.d.ts +0 -1
  92. package/lib/scripts/test_iter_load.js +0 -12
  93. package/lib/src/utils/nvs-timecode.d.ts +0 -17
  94. package/lib/src/utils/nvs-timecode.js +0 -79
  95. package/lib/src/utils/weights_scoring_config.d.ts +0 -2
  96. package/lib/src/utils/weights_scoring_config.js +0 -15
  97. package/lib/strings.d.ts +0 -1
  98. package/lib/strings.js +0 -18
  99. package/lib/types/agenda.d.ts +0 -44
  100. package/lib/types/agenda.js +0 -1
  101. package/lib/types/ameli.d.ts +0 -5
  102. package/lib/types/ameli.js +0 -1
  103. package/lib/types/compte_rendu.d.ts +0 -83
  104. package/lib/types/compte_rendu.js +0 -1
  105. package/lib/types/debats.d.ts +0 -2
  106. package/lib/types/debats.js +0 -1
  107. package/lib/types/dosleg.d.ts +0 -70
  108. package/lib/types/dosleg.js +0 -1
  109. package/lib/types/questions.d.ts +0 -2
  110. package/lib/types/questions.js +0 -1
  111. package/lib/types/sens.d.ts +0 -10
  112. package/lib/types/sens.js +0 -1
  113. package/lib/types/sessions.d.ts +0 -5
  114. package/lib/types/sessions.js +0 -84
  115. package/lib/types/texte.d.ts +0 -74
  116. package/lib/types/texte.js +0 -16
  117. package/lib/utils/cr_spliting.d.ts +0 -28
  118. package/lib/utils/cr_spliting.js +0 -265
  119. package/lib/utils/date.d.ts +0 -10
  120. package/lib/utils/date.js +0 -100
  121. package/lib/utils/nvs-timecode.d.ts +0 -7
  122. package/lib/utils/nvs-timecode.js +0 -79
  123. package/lib/utils/reunion_grouping.d.ts +0 -11
  124. package/lib/utils/reunion_grouping.js +0 -337
  125. package/lib/utils/reunion_odj_building.d.ts +0 -5
  126. package/lib/utils/reunion_odj_building.js +0 -154
  127. package/lib/utils/reunion_parsing.d.ts +0 -23
  128. package/lib/utils/reunion_parsing.js +0 -209
  129. package/lib/utils/scoring.d.ts +0 -14
  130. package/lib/utils/scoring.js +0 -147
  131. package/lib/utils/string_cleaning.d.ts +0 -7
  132. package/lib/utils/string_cleaning.js +0 -57
  133. package/lib/validators/config.d.ts +0 -9
  134. package/lib/validators/config.js +0 -10
@@ -101,6 +101,9 @@ async function processDocument(url, destPath, docDate, options) {
101
101
  }
102
102
  export async function processTexte(texteMetadata, originalTextesDir, transformedTextesDir, enrichedTextesDir, options) {
103
103
  const texteDir = path.join(originalTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name);
104
+ // Pre-compute whether the parsed JSON output already exists, to avoid re-parsing unchanged files
105
+ const parsedJsonPath = path.join(transformedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name, `${texteMetadata.name}.json`);
106
+ const parsedOutputExists = options.parseDocuments ? await fs.pathExists(parsedJsonPath) : false;
104
107
  let exposeDesMotifsContent = null;
105
108
  if (texteMetadata.url_expose_des_motifs) {
106
109
  const exposePath = path.join(texteDir, `${texteMetadata.name}-expose.html`);
@@ -108,7 +111,8 @@ export async function processTexte(texteMetadata, originalTextesDir, transformed
108
111
  if (res.buffer) {
109
112
  exposeDesMotifsContent = res.buffer;
110
113
  }
111
- else if (res.skipped && options.parseDocuments) {
114
+ else if (res.skipped && options.parseDocuments && !parsedOutputExists) {
115
+ // Only load expose from disk if we'll actually need to re-parse
112
116
  if (await fs.pathExists(exposePath)) {
113
117
  exposeDesMotifsContent = await fs.readFile(exposePath);
114
118
  }
@@ -128,23 +132,28 @@ export async function processTexte(texteMetadata, originalTextesDir, transformed
128
132
  const result = await processDocument(format.url.toString(), destPath, texteMetadata.date, options);
129
133
  // Specific logic: Parsing (Only applies to XML)
130
134
  if (format.isParseTarget && options.parseDocuments) {
131
- if (result.buffer !== null || (await fs.pathExists(destPath))) {
135
+ // Skip re-parsing if the XML was not newly downloaded AND the parsed output already exists
136
+ const needsParsing = !result.skipped || !parsedOutputExists;
137
+ if (needsParsing && (result.buffer !== null || (await fs.pathExists(destPath)))) {
132
138
  await parseDocument(texteMetadata.session, transformedTextesDir, destPath, texteMetadata.name, result.buffer, exposeDesMotifsContent, options);
133
- }
134
- let texteXmlContent = null;
135
- if (result.buffer === null && (await fs.pathExists(destPath))) {
136
- texteXmlContent = await fs.readFile(destPath, "utf-8");
137
- }
138
- else if (result.buffer !== null) {
139
- texteXmlContent = textDecoder.decode(result.buffer);
140
- }
141
- if (texteXmlContent !== null) {
142
- try {
143
- await convertSenatXmlToHtml(texteXmlContent, path.join(enrichedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name, `${texteMetadata.name}.html`));
139
+ let texteXmlContent = null;
140
+ if (result.buffer === null && (await fs.pathExists(destPath))) {
141
+ texteXmlContent = await fs.readFile(destPath, "utf-8");
144
142
  }
145
- catch (error) {
146
- console.error(`Error converting ${texteMetadata.name} to HTML: ${error.message}`);
143
+ else if (result.buffer !== null) {
144
+ texteXmlContent = textDecoder.decode(result.buffer);
147
145
  }
146
+ if (texteXmlContent !== null) {
147
+ try {
148
+ await convertSenatXmlToHtml(texteXmlContent, path.join(enrichedTextesDir, `${texteMetadata.session ?? UNDEFINED_SESSION}`, texteMetadata.name, `${texteMetadata.name}.html`));
149
+ }
150
+ catch (error) {
151
+ console.error(`Error converting ${texteMetadata.name} to HTML: ${error.message}`);
152
+ }
153
+ }
154
+ }
155
+ else if (options.verbose) {
156
+ console.info(`Skipping parse for already processed texte ${texteMetadata.name}…`);
148
157
  }
149
158
  }
150
159
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@tricoteuses/senat",
3
- "version": "2.22.10",
3
+ "version": "2.22.11",
4
4
  "description": "Handle French Sénat's open data",
5
5
  "keywords": [
6
6
  "France",
package/lib/config.d.ts DELETED
@@ -1,21 +0,0 @@
1
- import "dotenv/config";
2
- import { z } from "zod";
3
- export declare const configSchema: z.ZodObject<{
4
- db: z.ZodObject<{
5
- host: z.ZodString;
6
- name: z.ZodString;
7
- password: z.ZodString;
8
- port: z.ZodCoercedNumber<unknown>;
9
- user: z.ZodString;
10
- }, z.core.$strip>;
11
- }, z.core.$strip>;
12
- declare const _default: {
13
- db: {
14
- host: string;
15
- name: string;
16
- password: string;
17
- port: number;
18
- user: string;
19
- };
20
- };
21
- export default _default;
package/lib/config.js DELETED
@@ -1,27 +0,0 @@
1
- import "dotenv/config";
2
- import { z } from "zod";
3
- const dbSchema = z.object({
4
- host: z.string().trim().min(1, "Must not be empty"),
5
- name: z.string().trim().min(1, "Must not be empty"),
6
- password: z.string().trim().min(1, "Must not be empty"),
7
- port: z.coerce.number().int().min(0).max(65535),
8
- user: z.string().trim().min(1, "Must not be empty"),
9
- });
10
- export const configSchema = z.object({
11
- db: dbSchema,
12
- });
13
- const config = {
14
- db: {
15
- host: process.env["TRICOTEUSES_SENAT_DB_HOST"] || "localhost",
16
- name: process.env["TRICOTEUSES_SENAT_DB_NAME"] || "postgres",
17
- password: process.env["TRICOTEUSES_SENAT_DB_PASSWORD"] || "opendata",
18
- port: process.env["TRICOTEUSES_SENAT_DB_PORT"] || 5432,
19
- user: process.env["TRICOTEUSES_SENAT_DB_USER"] || "opendata",
20
- },
21
- };
22
- const result = configSchema.safeParse(config);
23
- if (!result.success) {
24
- console.error(`Error in configuration:\n${JSON.stringify(config, null, 2)}\nError:\n${JSON.stringify(result.error.issues, null, 2)}`);
25
- process.exit(-1);
26
- }
27
- export default result.data;
@@ -1,2 +0,0 @@
1
- import { Kysely } from "kysely";
2
- export declare const dbSenat: Kysely<any>;
package/lib/databases.js DELETED
@@ -1,26 +0,0 @@
1
- import { Kysely, PostgresDialect } from "kysely";
2
- import * as pg from "pg";
3
- import { types } from "pg";
4
- import Cursor from "pg-cursor";
5
- import config from "./config";
6
- // Map int8 to number instead of string
7
- // https://github.com/brianc/node-pg-types#use
8
- pg.types.setTypeParser(types.builtins.INT8, (val) => {
9
- return parseInt(val, 10);
10
- });
11
- const senatPool = new pg.Pool({
12
- database: "senat",
13
- host: config.db.host,
14
- user: config.db.user,
15
- password: config.db.password,
16
- port: config.db.port,
17
- max: 10,
18
- });
19
- const senatDialect = new PostgresDialect({
20
- pool: senatPool,
21
- cursor: Cursor,
22
- });
23
- export const dbSenat = new Kysely({
24
- log: ["error"],
25
- dialect: senatDialect,
26
- });
package/lib/datasets.d.ts DELETED
@@ -1,34 +0,0 @@
1
- export interface Dataset {
2
- database: string;
3
- repairEncoding: boolean;
4
- repairZip?: (dataset: Dataset, dataDir: string) => void;
5
- title: string;
6
- url: string;
7
- indexes?: {
8
- [table: string]: Array<{
9
- name: string;
10
- columns: string[];
11
- }>;
12
- };
13
- }
14
- export interface Datasets {
15
- ameli: Dataset;
16
- debats: Dataset;
17
- dosleg: Dataset;
18
- questions: Dataset;
19
- sens: Dataset;
20
- }
21
- export declare enum EnabledDatasets {
22
- None = 0,
23
- Ameli = 1,
24
- Debats = 2,
25
- DosLeg = 4,
26
- Questions = 8,
27
- Sens = 16,
28
- PhotosSenateurs = 32,
29
- Agenda = 64,
30
- All = 127
31
- }
32
- export declare const datasets: Datasets;
33
- export declare function getEnabledDatasets(categories: string[]): EnabledDatasets;
34
- export declare function getChosenDatasets(enabledDatasets: EnabledDatasets): Dataset[];
package/lib/datasets.js DELETED
@@ -1,233 +0,0 @@
1
- import fs from "fs-extra";
2
- import path from "path";
3
- import assert from "assert";
4
- export var EnabledDatasets;
5
- (function (EnabledDatasets) {
6
- EnabledDatasets[EnabledDatasets["None"] = 0] = "None";
7
- EnabledDatasets[EnabledDatasets["Ameli"] = 1] = "Ameli";
8
- EnabledDatasets[EnabledDatasets["Debats"] = 2] = "Debats";
9
- EnabledDatasets[EnabledDatasets["DosLeg"] = 4] = "DosLeg";
10
- EnabledDatasets[EnabledDatasets["Questions"] = 8] = "Questions";
11
- EnabledDatasets[EnabledDatasets["Sens"] = 16] = "Sens";
12
- EnabledDatasets[EnabledDatasets["PhotosSenateurs"] = 32] = "PhotosSenateurs";
13
- EnabledDatasets[EnabledDatasets["Agenda"] = 64] = "Agenda";
14
- EnabledDatasets[EnabledDatasets["All"] = 127] = "All";
15
- })(EnabledDatasets || (EnabledDatasets = {}));
16
- export const datasets = {
17
- ameli: {
18
- database: "ameli",
19
- repairEncoding: true,
20
- repairZip: (dataset, dataDir) => {
21
- const sqlFilename = `${dataset.database}.sql`;
22
- const sqlFilePath = path.join(dataDir, sqlFilename);
23
- fs.removeSync(sqlFilePath);
24
- fs.moveSync(path.join(dataDir, "var", "opt", "opendata", sqlFilename), sqlFilePath);
25
- },
26
- title: "Amendements",
27
- url: "https://data.senat.fr/data/ameli/ameli.zip",
28
- indexes: {
29
- amdsen: [
30
- { name: "idx_amdid", columns: ["amdid"] },
31
- { name: "idx_senid", columns: ["senid"] },
32
- { name: "idx_grpid", columns: ["grpid"] },
33
- ],
34
- sen_ameli: [{ name: "idx_entid", columns: ["entid"] }],
35
- grppol_ameli: [{ name: "idx_entid", columns: ["entid"] }],
36
- sub: [
37
- { name: "idx_id", columns: ["id"] },
38
- { name: "idx_typid", columns: ["typid"] },
39
- { name: "idx_merid", columns: ["merid"] },
40
- { name: "idx_comdelid", columns: ["comdelid"] },
41
- ],
42
- typsub: [{ name: "idx_id", columns: ["id"] }],
43
- typrect: [{ name: "idx_id", columns: ["id"] }],
44
- txt_ameli: [
45
- { name: "idx_id", columns: ["id"] },
46
- { name: "idx_txtetaid", columns: ["txtetaid"] },
47
- { name: "idx_sesdepid", columns: ["sesdepid"] },
48
- { name: "idx_natid", columns: ["natid"] },
49
- { name: "idx_lecid", columns: ["lecid"] },
50
- ],
51
- etatxt: [{ name: "idx_id", columns: ["id"] }],
52
- ses: [
53
- { name: "idx_id", columns: ["id"] },
54
- { name: "idx_typid", columns: ["typid"] },
55
- { name: "idx_ann", columns: ["ann"] },
56
- ],
57
- typses: [{ name: "idx_id", columns: ["id"] }],
58
- nat: [{ name: "idx_id", columns: ["id"] }],
59
- lec_ameli: [{ name: "idx_id", columns: ["id"] }],
60
- mot: [{ name: "idx_id", columns: ["id"] }],
61
- avicom: [{ name: "idx_id", columns: ["id"] }],
62
- avigvt: [{ name: "idx_id", columns: ["id"] }],
63
- sor: [{ name: "idx_id", columns: ["id"] }],
64
- irr: [{ name: "idx_id", columns: ["id"] }],
65
- com_ameli: [{ name: "idx_entid", columns: ["entid"] }],
66
- cab: [{ name: "idx_entid", columns: ["entid"] }],
67
- amd: [
68
- { name: "idx_subid", columns: ["subid"] },
69
- { name: "idx_typrectid", columns: ["typrectid"] },
70
- { name: "idx_txtid", columns: ["txtid"] },
71
- { name: "idx_motid", columns: ["motid"] },
72
- { name: "idx_avcid", columns: ["avcid"] },
73
- { name: "idx_avgid", columns: ["avgid"] },
74
- { name: "idx_sorid", columns: ["sorid"] },
75
- { name: "idx_irrid", columns: ["irrid"] },
76
- { name: "idx_nomentid", columns: ["nomentid"] },
77
- ],
78
- },
79
- },
80
- debats: {
81
- database: "debats",
82
- repairEncoding: false,
83
- title: "Informations relatives aux comptes rendus intégraux de la séance publique",
84
- url: "https://data.senat.fr/data/debats/debats.zip",
85
- indexes: {
86
- secdis: [
87
- { name: "idx_datsea", columns: ["datsea"] },
88
- { name: "idx_typseccod", columns: ["typseccod"] },
89
- ],
90
- intpjl: [{ name: "idx_secdiscle", columns: ["secdiscle"] }],
91
- secdivers: [
92
- { name: "idx_datsea", columns: ["datsea"] },
93
- { name: "idx_typseccod", columns: ["typseccod"] },
94
- ],
95
- intdivers: [{ name: "idx_intdiverscle", columns: ["intdiverscle"] }],
96
- lecassdeb: [{ name: "idx_datsea", columns: ["datsea"] }],
97
- },
98
- },
99
- dosleg: {
100
- database: "dosleg",
101
- repairEncoding: false,
102
- title: "Dossiers législatifs",
103
- url: "https://data.senat.fr/data/dosleg/dosleg.zip",
104
- indexes: {
105
- amescr: [{ name: "idx_scrnum", columns: ["scrnum"] }],
106
- auteur: [{ name: "idx_autcod", columns: ["autcod"] }],
107
- corscr: [{ name: "idx_scrnum", columns: ["scrnum"] }],
108
- date_seance: [
109
- { name: "idx_code", columns: ["code"] },
110
- { name: "idx_lecidt", columns: ["lecidt"] },
111
- ],
112
- deccoc: [{ name: "idx_deccoccod", columns: ["deccoccod"] }],
113
- etaloi: [{ name: "idx_etaloicod", columns: ["etaloicod"] }],
114
- lecass: [
115
- { name: "idx_lecidt", columns: ["lecidt"] },
116
- { name: "idx_codass", columns: ["codass"] },
117
- { name: "idx_orgcod", columns: ["orgcod"] },
118
- { name: "idx_orippr", columns: ["orippr"] },
119
- ],
120
- lecassrap: [
121
- { name: "idx_rapcod", columns: ["rapcod"] },
122
- { name: "idx_lecassidt", columns: ["lecassidt"] },
123
- ],
124
- lecture: [
125
- { name: "idx_loicod", columns: ["loicod"] },
126
- { name: "idx_typleccod", columns: ["typleccod"] },
127
- { name: "idx_lecidt", columns: ["lecidt"] },
128
- ],
129
- loi: [
130
- { name: "idx_typloicod", columns: ["typloicod"] },
131
- { name: "idx_etaloicod", columns: ["etaloicod"] },
132
- { name: "idx_deccoccod", columns: ["deccoccod"] },
133
- { name: "idx_loicod", columns: ["loicod"] },
134
- ],
135
- loithe: [
136
- { name: "idx_thecle", columns: ["thecle"] },
137
- { name: "idx_loicod", columns: ["loicod"] },
138
- ],
139
- posvot: [{ name: "idx_posvotcod", columns: ["posvotcod"] }],
140
- rap: [
141
- { name: "idx_rapcod", columns: ["rapcod"] },
142
- { name: "idx_coddenrap", columns: ["coddenrap"] },
143
- ],
144
- rolsig: [{ name: "idx_signataire", columns: ["signataire"] }],
145
- scr: [
146
- { name: "idx_code", columns: ["code"] },
147
- { name: "idx_scrnum", columns: ["scrnum"] },
148
- { name: "idx_sesann", columns: ["sesann"] },
149
- ],
150
- stavot: [{ name: "idx_stavotidt", columns: ["stavotidt"] }],
151
- texte: [
152
- { name: "idx_lecassidt", columns: ["lecassidt"] },
153
- { name: "idx_oritxtcod", columns: ["oritxtcod"] },
154
- { name: "idx_typtxtcod", columns: ["typtxtcod"] },
155
- { name: "idx_texcod", columns: ["texcod"] },
156
- ],
157
- the: [{ name: "idx_thecle", columns: ["thecle"] }],
158
- titsen: [{ name: "idx_titsencod", columns: ["titsencod"] }],
159
- typloi: [{ name: "idx_typloicod", columns: ["typloicod"] }],
160
- typtxt: [{ name: "idx_typtxtcod", columns: ["typtxtcod"] }],
161
- votsen: [
162
- { name: "idx_scrnum", columns: ["scrnum"] },
163
- { name: "idx_sesann", columns: ["sesann"] },
164
- { name: "idx_titsencod", columns: ["titsencod"] },
165
- { name: "idx_stavotidt", columns: ["stavotidt"] },
166
- { name: "idx_posvotcod", columns: ["posvotcod"] },
167
- { name: "idx_senmat", columns: ["senmat"] },
168
- ],
169
- },
170
- },
171
- questions: {
172
- database: "questions",
173
- repairEncoding: false,
174
- title: "Questions écrites et orales posées par les sénateurs au Gouvernement",
175
- url: "https://data.senat.fr/data/questions/questions.zip",
176
- indexes: {
177
- tam_reponses: [{ name: "idx_idque", columns: ["idque"] }],
178
- tam_questions: [
179
- { name: "idx_sorquecod", columns: ["sorquecod"] },
180
- { name: "idx_etaquecod", columns: ["etaquecod"] },
181
- ],
182
- the: [{ name: "idx_thenouidt", columns: ["thenouidt"] }],
183
- sortquestion: [{ name: "idx_sorquecod", columns: ["sorquecod"] }],
184
- etatquestion: [{ name: "idx_etaquecod", columns: ["etaquecod"] }],
185
- },
186
- },
187
- sens: {
188
- database: "sens",
189
- repairEncoding: false,
190
- repairZip: (dataset, dataDir) => {
191
- const sqlFilename = `${dataset.database}.sql`;
192
- const sqlFilePath = path.join(dataDir, sqlFilename);
193
- fs.removeSync(sqlFilePath);
194
- fs.moveSync(path.join(dataDir, "export_sens.sql"), sqlFilePath);
195
- },
196
- title: "Sénateurs (y compris organes et présence)",
197
- url: "https://data.senat.fr/data/senateurs/export_sens.zip",
198
- indexes: {
199
- dpt: [
200
- { name: "idx_dptnum", columns: ["dptnum"] },
201
- { name: "idx_regcod", columns: ["regcod"] },
202
- ],
203
- activite_loi: [{ name: "idx_actid", columns: ["actid"] }],
204
- activite_delegation: [{ name: "idx_actid", columns: ["actid"] }],
205
- activite_participant: [{ name: "idx_senmat", columns: ["senmat"] }],
206
- activite_senateur: [{ name: "idx_senmat", columns: ["senmat"] }],
207
- senbur: [
208
- { name: "idx_senmat", columns: ["senmat"] },
209
- { name: "idx_burcod", columns: ["burcod"] },
210
- ],
211
- elucan: [
212
- { name: "idx_senmat", columns: ["senmat"] },
213
- { name: "idx_typmancod", columns: ["typmancod"] },
214
- ],
215
- },
216
- },
217
- };
218
- export function getEnabledDatasets(categories) {
219
- categories.map((datasetName) => assert.notStrictEqual(EnabledDatasets[datasetName], undefined, `Unknown name of dataset: ${datasetName}`));
220
- return categories.reduce((enabledDatasets, datasetName) => {
221
- const enabledDataset = EnabledDatasets[datasetName];
222
- return enabledDatasets | (enabledDataset || EnabledDatasets.None);
223
- }, EnabledDatasets.None);
224
- }
225
- export function getChosenDatasets(enabledDatasets) {
226
- return [
227
- enabledDatasets & EnabledDatasets.Ameli ? datasets.ameli : null,
228
- enabledDatasets & EnabledDatasets.Debats ? datasets.debats : null,
229
- enabledDatasets & EnabledDatasets.DosLeg ? datasets.dosleg : null,
230
- enabledDatasets & EnabledDatasets.Questions ? datasets.questions : null,
231
- enabledDatasets & EnabledDatasets.Sens ? datasets.sens : null,
232
- ].filter((dataset) => dataset !== null);
233
- }
package/lib/git.d.ts DELETED
@@ -1,26 +0,0 @@
1
- export declare function initRepo(repositoryDir: string): void;
2
- export declare function commit(repositoryDir: string, message: string): boolean;
3
- export declare function commitAndPush(repositoryDir: string, message: string, remotes?: string[]): number;
4
- export declare function resetAndPull(gitDir: string): boolean;
5
- export declare function clone(gitGroupUrl: string | undefined, gitName: string, workingDir: string): void;
6
- export declare function run(repositoryDir: string, args: string, verbose?: boolean): string;
7
- export declare function test(repositoryDir: string, args: string, verbose?: boolean): boolean;
8
- /**
9
- * Information about a changed file in git
10
- */
11
- export interface GitChangedFile {
12
- path: string;
13
- status: "A" | "M" | "D" | "R" | "C" | "T" | "U";
14
- }
15
- /**
16
- * Get the list of files that have changed since a specific commit in a git repository.
17
- * @param repositoryDir The directory of the git repository
18
- * @param sinceCommit The commit hash to compare against (e.g., "HEAD~1", "abc123", etc.)
19
- * @param options Options for filtering
20
- * @param options.diffFilter Git diff-filter string (default: "AMR").
21
- * A=Added, M=Modified, D=Deleted, R=Renamed, C=Copied, T=Type changed, U=Unmerged
22
- * @returns A Map of file paths to their git status
23
- */
24
- export declare function getChangedFilesSinceCommit(repositoryDir: string, sinceCommit: string, options?: {
25
- diffFilter?: string;
26
- }): Map<string, GitChangedFile["status"]>;
package/lib/git.js DELETED
@@ -1,167 +0,0 @@
1
- import { execSync } from "node:child_process";
2
- import fs from "fs-extra";
3
- import path from "node:path";
4
- const MAXBUFFER = 50 * 1024 * 1024;
5
- export function initRepo(repositoryDir) {
6
- if (!fs.existsSync(path.join(repositoryDir, ".git"))) {
7
- fs.ensureDirSync(repositoryDir);
8
- execSync("git init", {
9
- cwd: repositoryDir,
10
- env: process.env,
11
- encoding: "utf-8",
12
- stdio: ["ignore", "ignore", "pipe"],
13
- });
14
- }
15
- }
16
- export function commit(repositoryDir, message) {
17
- initRepo(repositoryDir);
18
- execSync("git add .", {
19
- cwd: repositoryDir,
20
- env: process.env,
21
- encoding: "utf-8",
22
- stdio: ["ignore", "ignore", "pipe"],
23
- maxBuffer: MAXBUFFER,
24
- });
25
- try {
26
- execSync(`git commit -m "${message}" --quiet`, {
27
- cwd: repositoryDir,
28
- env: process.env,
29
- encoding: "utf-8",
30
- stdio: ["ignore", "pipe", "pipe"],
31
- });
32
- return true;
33
- }
34
- catch (childProcess) {
35
- if (childProcess.stdout === null ||
36
- !/nothing to commit|rien à valider/.test(childProcess.stdout)) {
37
- console.error(childProcess.output);
38
- throw childProcess;
39
- }
40
- return false;
41
- }
42
- }
43
- export function commitAndPush(repositoryDir, message, remotes) {
44
- let exitCode = 0;
45
- if (commit(repositoryDir, message)) {
46
- for (const remote of remotes || []) {
47
- try {
48
- execSync(`git push ${remote} master`, {
49
- cwd: repositoryDir,
50
- env: process.env,
51
- encoding: "utf-8",
52
- stdio: ["ignore", "ignore", "pipe"],
53
- });
54
- }
55
- catch (childProcess) {
56
- // Don't stop when push fails.
57
- console.error(childProcess.output);
58
- exitCode = childProcess.status;
59
- }
60
- }
61
- }
62
- else {
63
- // There was nothing to commit.
64
- exitCode = 10;
65
- }
66
- return exitCode;
67
- }
68
- export function resetAndPull(gitDir) {
69
- execSync("git reset --hard origin/master", {
70
- cwd: gitDir,
71
- env: process.env,
72
- encoding: "utf-8",
73
- stdio: ["ignore", "ignore", "pipe"],
74
- });
75
- execSync("git pull --rebase", {
76
- cwd: gitDir,
77
- env: process.env,
78
- encoding: "utf-8",
79
- stdio: ["ignore", "ignore", "pipe"],
80
- });
81
- return true;
82
- }
83
- export function clone(gitGroupUrl, gitName, workingDir) {
84
- if (gitGroupUrl !== undefined) {
85
- execSync(`git clone ${gitGroupUrl}/${gitName}.git`, {
86
- cwd: workingDir,
87
- env: process.env,
88
- encoding: "utf-8",
89
- stdio: ["ignore", "ignore", "pipe"],
90
- });
91
- }
92
- }
93
- export function run(repositoryDir, args, verbose) {
94
- try {
95
- if (verbose)
96
- console.log(`git -C ${repositoryDir} ${args}`);
97
- const output = execSync(`git ${args}`, {
98
- cwd: repositoryDir,
99
- maxBuffer: MAXBUFFER,
100
- })
101
- .toString()
102
- .trim();
103
- if (verbose)
104
- console.log(output);
105
- return output;
106
- }
107
- catch (childProcess) {
108
- for (const output of ["stdout", "stderr"])
109
- console.error(`${output}: ${childProcess[output]}`);
110
- throw childProcess;
111
- }
112
- }
113
- export function test(repositoryDir, args, verbose) {
114
- try {
115
- if (verbose)
116
- console.log(`git -C ${repositoryDir} ${args}`);
117
- const output = execSync(`git ${args}`, {
118
- cwd: repositoryDir,
119
- stdio: ["ignore", "pipe", "pipe"],
120
- maxBuffer: MAXBUFFER,
121
- })
122
- .toString()
123
- .trim();
124
- if (verbose)
125
- console.log(output);
126
- return true;
127
- }
128
- catch (childProcess) {
129
- if (childProcess.status != 0)
130
- return false;
131
- throw childProcess;
132
- }
133
- }
134
- /**
135
- * Get the list of files that have changed since a specific commit in a git repository.
136
- * @param repositoryDir The directory of the git repository
137
- * @param sinceCommit The commit hash to compare against (e.g., "HEAD~1", "abc123", etc.)
138
- * @param options Options for filtering
139
- * @param options.diffFilter Git diff-filter string (default: "AMR").
140
- * A=Added, M=Modified, D=Deleted, R=Renamed, C=Copied, T=Type changed, U=Unmerged
141
- * @returns A Map of file paths to their git status
142
- */
143
- export function getChangedFilesSinceCommit(repositoryDir, sinceCommit, options = {}) {
144
- const { diffFilter } = options;
145
- try {
146
- // Using diff-filter: A = Added, M = Modified, R = Renamed, D = Deleted, etc.
147
- // Default to AMR (excludes deleted files to prevent loading errors)
148
- const filter = diffFilter ?? "AMR";
149
- const output = run(repositoryDir, `diff --name-status --diff-filter=${filter} ${sinceCommit}`, false);
150
- const changedFiles = new Map();
151
- for (const line of output.split("\n")) {
152
- if (line.trim().length === 0)
153
- continue;
154
- const parts = line.split("\t");
155
- if (parts.length >= 2) {
156
- const status = parts[0].charAt(0);
157
- const path = parts[1];
158
- changedFiles.set(path, status);
159
- }
160
- }
161
- return changedFiles;
162
- }
163
- catch (error) {
164
- console.error(`Error getting changed files since commit ${sinceCommit}:`, error);
165
- return new Map();
166
- }
167
- }
package/lib/index.d.ts DELETED
@@ -1,13 +0,0 @@
1
- export type { AmendementResult } from "./model/ameli";
2
- export type { DossierLegislatifResult } from "./model/dosleg";
3
- export type { ScrutinResult } from "./model/scrutins";
4
- export type { QuestionResult } from "./model/questions";
5
- export type { CirconscriptionResult, OrganismeResult, SenateurResult } from "./model/sens";
6
- export type { AgendaEvent, Reunion, ReunionOdjPoint } from "./types/agenda";
7
- export type { Ses, Sub, TxtAmeli } from "./types/ameli";
8
- export type { CompteRendu } from "./types/compte_rendu";
9
- export type { Debat, LecAssDeb } from "./types/debats";
10
- export type { Ass, Aud, Auteur, DateSeance, DecCoc, DenRap, DocAtt, Ecr, EtaLoi, LecAss, LecAssRap, Lecture, Loi, Org, OriTxt, Qua, Rap, RapOrg, Scr, Texte, TypAtt, TypLec, TypLoi, TypTxt, TypUrl, } from "./types/dosleg";
11
- export type { Photo, Sen } from "./types/sens";
12
- export { UNDEFINED_SESSION, type Session, getSessionsFromStart } from "./types/sessions";
13
- export type { Alinea, Article, Division, DivisionTag, FlatTexte } from "./types/texte";
package/lib/index.js DELETED
@@ -1 +0,0 @@
1
- export { UNDEFINED_SESSION, getSessionsFromStart } from "./types/sessions";