@tricoteuses/senat 2.22.13 → 2.22.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. package/lib/src/loaders.d.ts +2 -8
  2. package/lib/src/loaders.js +7 -25
  3. package/lib/tests/test_iter_load.test.js +17 -0
  4. package/package.json +2 -2
  5. package/lib/config.d.ts +0 -21
  6. package/lib/config.js +0 -27
  7. package/lib/databases.d.ts +0 -2
  8. package/lib/databases.js +0 -26
  9. package/lib/datasets.d.ts +0 -34
  10. package/lib/datasets.js +0 -233
  11. package/lib/git.d.ts +0 -26
  12. package/lib/git.js +0 -167
  13. package/lib/index.d.ts +0 -13
  14. package/lib/index.js +0 -1
  15. package/lib/loaders.d.ts +0 -58
  16. package/lib/loaders.js +0 -286
  17. package/lib/model/agenda.d.ts +0 -6
  18. package/lib/model/agenda.js +0 -148
  19. package/lib/model/ameli.d.ts +0 -51
  20. package/lib/model/ameli.js +0 -147
  21. package/lib/model/commission.d.ts +0 -18
  22. package/lib/model/commission.js +0 -269
  23. package/lib/model/debats.d.ts +0 -67
  24. package/lib/model/debats.js +0 -95
  25. package/lib/model/documents.d.ts +0 -12
  26. package/lib/model/documents.js +0 -138
  27. package/lib/model/dosleg.d.ts +0 -7
  28. package/lib/model/dosleg.js +0 -326
  29. package/lib/model/index.d.ts +0 -7
  30. package/lib/model/index.js +0 -7
  31. package/lib/model/questions.d.ts +0 -45
  32. package/lib/model/questions.js +0 -89
  33. package/lib/model/scrutins.d.ts +0 -13
  34. package/lib/model/scrutins.js +0 -114
  35. package/lib/model/seance.d.ts +0 -3
  36. package/lib/model/seance.js +0 -267
  37. package/lib/model/sens.d.ts +0 -146
  38. package/lib/model/sens.js +0 -454
  39. package/lib/model/texte.d.ts +0 -7
  40. package/lib/model/texte.js +0 -228
  41. package/lib/model/util.d.ts +0 -9
  42. package/lib/model/util.js +0 -38
  43. package/lib/parsers/texte.d.ts +0 -7
  44. package/lib/parsers/texte.js +0 -228
  45. package/lib/raw_types/ameli.d.ts +0 -914
  46. package/lib/raw_types/ameli.js +0 -5
  47. package/lib/raw_types/debats.d.ts +0 -207
  48. package/lib/raw_types/debats.js +0 -5
  49. package/lib/raw_types/dosleg.d.ts +0 -1619
  50. package/lib/raw_types/dosleg.js +0 -5
  51. package/lib/raw_types/questions.d.ts +0 -423
  52. package/lib/raw_types/questions.js +0 -5
  53. package/lib/raw_types/senat.d.ts +0 -11372
  54. package/lib/raw_types/senat.js +0 -5
  55. package/lib/raw_types/sens.d.ts +0 -8248
  56. package/lib/raw_types/sens.js +0 -5
  57. package/lib/raw_types_schemats/ameli.d.ts +0 -539
  58. package/lib/raw_types_schemats/ameli.js +0 -2
  59. package/lib/raw_types_schemats/debats.d.ts +0 -127
  60. package/lib/raw_types_schemats/debats.js +0 -2
  61. package/lib/raw_types_schemats/dosleg.d.ts +0 -977
  62. package/lib/raw_types_schemats/dosleg.js +0 -2
  63. package/lib/raw_types_schemats/questions.d.ts +0 -237
  64. package/lib/raw_types_schemats/questions.js +0 -2
  65. package/lib/raw_types_schemats/sens.d.ts +0 -6915
  66. package/lib/raw_types_schemats/sens.js +0 -2
  67. package/lib/scripts/convert_data.js +0 -354
  68. package/lib/scripts/data-download.d.ts +0 -1
  69. package/lib/scripts/data-download.js +0 -12
  70. package/lib/scripts/datautil.d.ts +0 -8
  71. package/lib/scripts/datautil.js +0 -34
  72. package/lib/scripts/parse_textes.d.ts +0 -1
  73. package/lib/scripts/parse_textes.js +0 -44
  74. package/lib/scripts/retrieve_agenda.d.ts +0 -1
  75. package/lib/scripts/retrieve_agenda.js +0 -132
  76. package/lib/scripts/retrieve_cr_commission.d.ts +0 -1
  77. package/lib/scripts/retrieve_cr_commission.js +0 -364
  78. package/lib/scripts/retrieve_cr_seance.d.ts +0 -6
  79. package/lib/scripts/retrieve_cr_seance.js +0 -347
  80. package/lib/scripts/retrieve_documents.d.ts +0 -3
  81. package/lib/scripts/retrieve_documents.js +0 -219
  82. package/lib/scripts/retrieve_open_data.d.ts +0 -1
  83. package/lib/scripts/retrieve_open_data.js +0 -316
  84. package/lib/scripts/retrieve_senateurs_photos.d.ts +0 -1
  85. package/lib/scripts/retrieve_senateurs_photos.js +0 -147
  86. package/lib/scripts/retrieve_videos.d.ts +0 -1
  87. package/lib/scripts/retrieve_videos.js +0 -461
  88. package/lib/scripts/shared/cli_helpers.d.ts +0 -95
  89. package/lib/scripts/shared/cli_helpers.js +0 -91
  90. package/lib/scripts/shared/util.d.ts +0 -4
  91. package/lib/scripts/shared/util.js +0 -35
  92. package/lib/scripts/test_iter_load.d.ts +0 -1
  93. package/lib/scripts/test_iter_load.js +0 -12
  94. package/lib/src/utils/nvs-timecode.d.ts +0 -17
  95. package/lib/src/utils/nvs-timecode.js +0 -79
  96. package/lib/src/utils/weights_scoring_config.d.ts +0 -2
  97. package/lib/src/utils/weights_scoring_config.js +0 -15
  98. package/lib/strings.d.ts +0 -1
  99. package/lib/strings.js +0 -18
  100. package/lib/types/agenda.d.ts +0 -44
  101. package/lib/types/agenda.js +0 -1
  102. package/lib/types/ameli.d.ts +0 -5
  103. package/lib/types/ameli.js +0 -1
  104. package/lib/types/compte_rendu.d.ts +0 -83
  105. package/lib/types/compte_rendu.js +0 -1
  106. package/lib/types/debats.d.ts +0 -2
  107. package/lib/types/debats.js +0 -1
  108. package/lib/types/dosleg.d.ts +0 -70
  109. package/lib/types/dosleg.js +0 -1
  110. package/lib/types/questions.d.ts +0 -2
  111. package/lib/types/questions.js +0 -1
  112. package/lib/types/sens.d.ts +0 -10
  113. package/lib/types/sens.js +0 -1
  114. package/lib/types/sessions.d.ts +0 -5
  115. package/lib/types/sessions.js +0 -84
  116. package/lib/types/texte.d.ts +0 -74
  117. package/lib/types/texte.js +0 -16
  118. package/lib/utils/cr_spliting.d.ts +0 -28
  119. package/lib/utils/cr_spliting.js +0 -265
  120. package/lib/utils/date.d.ts +0 -10
  121. package/lib/utils/date.js +0 -100
  122. package/lib/utils/nvs-timecode.d.ts +0 -7
  123. package/lib/utils/nvs-timecode.js +0 -79
  124. package/lib/utils/reunion_grouping.d.ts +0 -11
  125. package/lib/utils/reunion_grouping.js +0 -337
  126. package/lib/utils/reunion_odj_building.d.ts +0 -5
  127. package/lib/utils/reunion_odj_building.js +0 -154
  128. package/lib/utils/reunion_parsing.d.ts +0 -23
  129. package/lib/utils/reunion_parsing.js +0 -209
  130. package/lib/utils/scoring.d.ts +0 -14
  131. package/lib/utils/scoring.js +0 -147
  132. package/lib/utils/string_cleaning.d.ts +0 -7
  133. package/lib/utils/string_cleaning.js +0 -57
  134. package/lib/validators/config.d.ts +0 -9
  135. package/lib/validators/config.js +0 -10
  136. /package/lib/{scripts/convert_data.d.ts → tests/test_iter_load.test.d.ts} +0 -0
@@ -33,14 +33,8 @@ export type IterItem<T> = {
33
33
  export declare function iterFilePaths(dirPath: string): Generator<string>;
34
34
  export declare function iterLoadSenatAmendements(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<AmendementResult>>;
35
35
  export declare function iterLoadSenatDebats(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DebatResult>>;
36
- export declare function iterLoadSenatComptesRendusSeances(dataDir: string, session: number): Generator<{
37
- compteRendu: CompteRendu;
38
- session: number;
39
- }>;
40
- export declare function iterLoadSenatComptesRendusCommissions(dataDir: string, session: number): Generator<{
41
- compteRendu: CompteRendu;
42
- session: number;
43
- }>;
36
+ export declare function iterLoadSenatComptesRendusSeances(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<CompteRendu>>;
37
+ export declare function iterLoadSenatComptesRendusCommissions(dataDir: string, session: number, options?: {}): Generator<IterItem<CompteRendu>>;
44
38
  export declare function iterLoadSenatDossiersLegislatifs(dataDir: string, session: number | undefined, options?: {}): Generator<IterItem<DossierLegislatifResult>>;
45
39
  export declare function iterLoadSenatRapportUrls(dataDir: string, session: number | undefined): Generator<IterItem<DocumentMetadata>>;
46
40
  export declare function iterLoadSenatTexteUrls(dataDir: string, session: number | undefined): Generator<IterItem<DocumentMetadata>>;
@@ -118,33 +118,15 @@ export function* iterLoadSenatDebats(dataDir, session, options = {}) {
118
118
  yield debatItem;
119
119
  }
120
120
  }
121
- function* iterLoadSenatComptesRendusGeneric(dataDir, session, subFolder) {
122
- const basePath = path.join(dataDir, subFolder, DATA_TRANSFORMED_FOLDER, String(session));
123
- if (!fs.existsSync(basePath)) {
124
- return;
125
- }
126
- const files = (fs.readdirSync(basePath) || []).filter((f) => f.endsWith(".json")).sort();
127
- for (const fileName of files) {
128
- const filePath = path.join(basePath, fileName);
129
- try {
130
- const fileContent = fs.readFileSync(filePath, "utf-8");
131
- const compteRendu = JSON.parse(fileContent);
132
- if (!compteRendu?.uid) {
133
- console.warn(`[SN] CR without uid → ${fileName}`);
134
- continue;
135
- }
136
- yield { compteRendu, session };
137
- }
138
- catch (err) {
139
- console.warn(`[SN] error reading CR → ${fileName}`, err);
140
- }
121
+ export function* iterLoadSenatComptesRendusSeances(dataDir, session, options = {}) {
122
+ for (const compteRenduItem of iterLoadSenatItems(dataDir, COMPTES_RENDUS_FOLDER, session, DATA_TRANSFORMED_FOLDER, options)) {
123
+ yield compteRenduItem;
141
124
  }
142
125
  }
143
- export function* iterLoadSenatComptesRendusSeances(dataDir, session) {
144
- yield* iterLoadSenatComptesRendusGeneric(dataDir, session, COMPTES_RENDUS_FOLDER);
145
- }
146
- export function* iterLoadSenatComptesRendusCommissions(dataDir, session) {
147
- yield* iterLoadSenatComptesRendusGeneric(dataDir, session, COMMISSION_FOLDER);
126
+ export function* iterLoadSenatComptesRendusCommissions(dataDir, session, options = {}) {
127
+ for (const compteRenduItem of iterLoadSenatItems(dataDir, COMMISSION_FOLDER, session, DATA_TRANSFORMED_FOLDER, options)) {
128
+ yield compteRenduItem;
129
+ }
148
130
  }
149
131
  export function* iterLoadSenatDossiersLegislatifs(dataDir, session, options = {}) {
150
132
  for (const dossierLegislatifItem of iterLoadSenatItems(dataDir, datasets.dosleg.database, session, DOSLEG_DOSSIERS_FOLDER, options)) {
@@ -0,0 +1,17 @@
1
+ import fs from "fs";
2
+ import os from "os";
3
+ import path from "path";
4
+ import { describe, it, expect } from "vitest";
5
+ import { COMPTES_RENDUS_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatComptesRendusSeances } from "../src/loaders";
6
+ describe("iterLoadSenatComptesRendusSeances", () => {
7
+ it("loads comptes rendus from transformed/session folder", () => {
8
+ const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "senat-cr-"));
9
+ const session = 2025;
10
+ const targetDir = path.join(tmpDir, COMPTES_RENDUS_FOLDER, DATA_TRANSFORMED_FOLDER, String(session));
11
+ fs.mkdirSync(targetDir, { recursive: true });
12
+ fs.writeFileSync(path.join(targetDir, "cr-1.json"), JSON.stringify({ uid: "cr-1", titre: "CR 1" }), "utf8");
13
+ const results = Array.from(iterLoadSenatComptesRendusSeances(tmpDir, session));
14
+ expect(results).toHaveLength(1);
15
+ expect(results[0].item.uid).toBe("cr-1");
16
+ });
17
+ });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@tricoteuses/senat",
3
- "version": "2.22.13",
3
+ "version": "2.22.14",
4
4
  "description": "Handle French Sénat's open data",
5
5
  "keywords": [
6
6
  "France",
@@ -63,7 +63,7 @@
63
63
  "prepare": "npm run build",
64
64
  "prepublishOnly": "npm run build",
65
65
  "prettier": "prettier --write 'src/**/*.ts'",
66
- "test:iter_load": "tsx src/scripts/test_iter_load.ts",
66
+ "test:iter_load": "vitest tests/test_iter_load.test.ts",
67
67
  "type-check": "tsc --noEmit",
68
68
  "type-check:watch": "npm run type-check -- --watch",
69
69
  "test:video-matching": "vitest tests/videoMatching.test.ts"
package/lib/config.d.ts DELETED
@@ -1,21 +0,0 @@
1
- import "dotenv/config";
2
- import { z } from "zod";
3
- export declare const configSchema: z.ZodObject<{
4
- db: z.ZodObject<{
5
- host: z.ZodString;
6
- name: z.ZodString;
7
- password: z.ZodString;
8
- port: z.ZodCoercedNumber<unknown>;
9
- user: z.ZodString;
10
- }, z.core.$strip>;
11
- }, z.core.$strip>;
12
- declare const _default: {
13
- db: {
14
- host: string;
15
- name: string;
16
- password: string;
17
- port: number;
18
- user: string;
19
- };
20
- };
21
- export default _default;
package/lib/config.js DELETED
@@ -1,27 +0,0 @@
1
- import "dotenv/config";
2
- import { z } from "zod";
3
- const dbSchema = z.object({
4
- host: z.string().trim().min(1, "Must not be empty"),
5
- name: z.string().trim().min(1, "Must not be empty"),
6
- password: z.string().trim().min(1, "Must not be empty"),
7
- port: z.coerce.number().int().min(0).max(65535),
8
- user: z.string().trim().min(1, "Must not be empty"),
9
- });
10
- export const configSchema = z.object({
11
- db: dbSchema,
12
- });
13
- const config = {
14
- db: {
15
- host: process.env["TRICOTEUSES_SENAT_DB_HOST"] || "localhost",
16
- name: process.env["TRICOTEUSES_SENAT_DB_NAME"] || "postgres",
17
- password: process.env["TRICOTEUSES_SENAT_DB_PASSWORD"] || "opendata",
18
- port: process.env["TRICOTEUSES_SENAT_DB_PORT"] || 5432,
19
- user: process.env["TRICOTEUSES_SENAT_DB_USER"] || "opendata",
20
- },
21
- };
22
- const result = configSchema.safeParse(config);
23
- if (!result.success) {
24
- console.error(`Error in configuration:\n${JSON.stringify(config, null, 2)}\nError:\n${JSON.stringify(result.error.issues, null, 2)}`);
25
- process.exit(-1);
26
- }
27
- export default result.data;
@@ -1,2 +0,0 @@
1
- import { Kysely } from "kysely";
2
- export declare const dbSenat: Kysely<any>;
package/lib/databases.js DELETED
@@ -1,26 +0,0 @@
1
- import { Kysely, PostgresDialect } from "kysely";
2
- import * as pg from "pg";
3
- import { types } from "pg";
4
- import Cursor from "pg-cursor";
5
- import config from "./config";
6
- // Map int8 to number instead of string
7
- // https://github.com/brianc/node-pg-types#use
8
- pg.types.setTypeParser(types.builtins.INT8, (val) => {
9
- return parseInt(val, 10);
10
- });
11
- const senatPool = new pg.Pool({
12
- database: "senat",
13
- host: config.db.host,
14
- user: config.db.user,
15
- password: config.db.password,
16
- port: config.db.port,
17
- max: 10,
18
- });
19
- const senatDialect = new PostgresDialect({
20
- pool: senatPool,
21
- cursor: Cursor,
22
- });
23
- export const dbSenat = new Kysely({
24
- log: ["error"],
25
- dialect: senatDialect,
26
- });
package/lib/datasets.d.ts DELETED
@@ -1,34 +0,0 @@
1
- export interface Dataset {
2
- database: string;
3
- repairEncoding: boolean;
4
- repairZip?: (dataset: Dataset, dataDir: string) => void;
5
- title: string;
6
- url: string;
7
- indexes?: {
8
- [table: string]: Array<{
9
- name: string;
10
- columns: string[];
11
- }>;
12
- };
13
- }
14
- export interface Datasets {
15
- ameli: Dataset;
16
- debats: Dataset;
17
- dosleg: Dataset;
18
- questions: Dataset;
19
- sens: Dataset;
20
- }
21
- export declare enum EnabledDatasets {
22
- None = 0,
23
- Ameli = 1,
24
- Debats = 2,
25
- DosLeg = 4,
26
- Questions = 8,
27
- Sens = 16,
28
- PhotosSenateurs = 32,
29
- Agenda = 64,
30
- All = 127
31
- }
32
- export declare const datasets: Datasets;
33
- export declare function getEnabledDatasets(categories: string[]): EnabledDatasets;
34
- export declare function getChosenDatasets(enabledDatasets: EnabledDatasets): Dataset[];
package/lib/datasets.js DELETED
@@ -1,233 +0,0 @@
1
- import fs from "fs-extra";
2
- import path from "path";
3
- import assert from "assert";
4
- export var EnabledDatasets;
5
- (function (EnabledDatasets) {
6
- EnabledDatasets[EnabledDatasets["None"] = 0] = "None";
7
- EnabledDatasets[EnabledDatasets["Ameli"] = 1] = "Ameli";
8
- EnabledDatasets[EnabledDatasets["Debats"] = 2] = "Debats";
9
- EnabledDatasets[EnabledDatasets["DosLeg"] = 4] = "DosLeg";
10
- EnabledDatasets[EnabledDatasets["Questions"] = 8] = "Questions";
11
- EnabledDatasets[EnabledDatasets["Sens"] = 16] = "Sens";
12
- EnabledDatasets[EnabledDatasets["PhotosSenateurs"] = 32] = "PhotosSenateurs";
13
- EnabledDatasets[EnabledDatasets["Agenda"] = 64] = "Agenda";
14
- EnabledDatasets[EnabledDatasets["All"] = 127] = "All";
15
- })(EnabledDatasets || (EnabledDatasets = {}));
16
- export const datasets = {
17
- ameli: {
18
- database: "ameli",
19
- repairEncoding: true,
20
- repairZip: (dataset, dataDir) => {
21
- const sqlFilename = `${dataset.database}.sql`;
22
- const sqlFilePath = path.join(dataDir, sqlFilename);
23
- fs.removeSync(sqlFilePath);
24
- fs.moveSync(path.join(dataDir, "var", "opt", "opendata", sqlFilename), sqlFilePath);
25
- },
26
- title: "Amendements",
27
- url: "https://data.senat.fr/data/ameli/ameli.zip",
28
- indexes: {
29
- amdsen: [
30
- { name: "idx_amdid", columns: ["amdid"] },
31
- { name: "idx_senid", columns: ["senid"] },
32
- { name: "idx_grpid", columns: ["grpid"] },
33
- ],
34
- sen_ameli: [{ name: "idx_entid", columns: ["entid"] }],
35
- grppol_ameli: [{ name: "idx_entid", columns: ["entid"] }],
36
- sub: [
37
- { name: "idx_id", columns: ["id"] },
38
- { name: "idx_typid", columns: ["typid"] },
39
- { name: "idx_merid", columns: ["merid"] },
40
- { name: "idx_comdelid", columns: ["comdelid"] },
41
- ],
42
- typsub: [{ name: "idx_id", columns: ["id"] }],
43
- typrect: [{ name: "idx_id", columns: ["id"] }],
44
- txt_ameli: [
45
- { name: "idx_id", columns: ["id"] },
46
- { name: "idx_txtetaid", columns: ["txtetaid"] },
47
- { name: "idx_sesdepid", columns: ["sesdepid"] },
48
- { name: "idx_natid", columns: ["natid"] },
49
- { name: "idx_lecid", columns: ["lecid"] },
50
- ],
51
- etatxt: [{ name: "idx_id", columns: ["id"] }],
52
- ses: [
53
- { name: "idx_id", columns: ["id"] },
54
- { name: "idx_typid", columns: ["typid"] },
55
- { name: "idx_ann", columns: ["ann"] },
56
- ],
57
- typses: [{ name: "idx_id", columns: ["id"] }],
58
- nat: [{ name: "idx_id", columns: ["id"] }],
59
- lec_ameli: [{ name: "idx_id", columns: ["id"] }],
60
- mot: [{ name: "idx_id", columns: ["id"] }],
61
- avicom: [{ name: "idx_id", columns: ["id"] }],
62
- avigvt: [{ name: "idx_id", columns: ["id"] }],
63
- sor: [{ name: "idx_id", columns: ["id"] }],
64
- irr: [{ name: "idx_id", columns: ["id"] }],
65
- com_ameli: [{ name: "idx_entid", columns: ["entid"] }],
66
- cab: [{ name: "idx_entid", columns: ["entid"] }],
67
- amd: [
68
- { name: "idx_subid", columns: ["subid"] },
69
- { name: "idx_typrectid", columns: ["typrectid"] },
70
- { name: "idx_txtid", columns: ["txtid"] },
71
- { name: "idx_motid", columns: ["motid"] },
72
- { name: "idx_avcid", columns: ["avcid"] },
73
- { name: "idx_avgid", columns: ["avgid"] },
74
- { name: "idx_sorid", columns: ["sorid"] },
75
- { name: "idx_irrid", columns: ["irrid"] },
76
- { name: "idx_nomentid", columns: ["nomentid"] },
77
- ],
78
- },
79
- },
80
- debats: {
81
- database: "debats",
82
- repairEncoding: false,
83
- title: "Informations relatives aux comptes rendus intégraux de la séance publique",
84
- url: "https://data.senat.fr/data/debats/debats.zip",
85
- indexes: {
86
- secdis: [
87
- { name: "idx_datsea", columns: ["datsea"] },
88
- { name: "idx_typseccod", columns: ["typseccod"] },
89
- ],
90
- intpjl: [{ name: "idx_secdiscle", columns: ["secdiscle"] }],
91
- secdivers: [
92
- { name: "idx_datsea", columns: ["datsea"] },
93
- { name: "idx_typseccod", columns: ["typseccod"] },
94
- ],
95
- intdivers: [{ name: "idx_intdiverscle", columns: ["intdiverscle"] }],
96
- lecassdeb: [{ name: "idx_datsea", columns: ["datsea"] }],
97
- },
98
- },
99
- dosleg: {
100
- database: "dosleg",
101
- repairEncoding: false,
102
- title: "Dossiers législatifs",
103
- url: "https://data.senat.fr/data/dosleg/dosleg.zip",
104
- indexes: {
105
- amescr: [{ name: "idx_scrnum", columns: ["scrnum"] }],
106
- auteur: [{ name: "idx_autcod", columns: ["autcod"] }],
107
- corscr: [{ name: "idx_scrnum", columns: ["scrnum"] }],
108
- date_seance: [
109
- { name: "idx_code", columns: ["code"] },
110
- { name: "idx_lecidt", columns: ["lecidt"] },
111
- ],
112
- deccoc: [{ name: "idx_deccoccod", columns: ["deccoccod"] }],
113
- etaloi: [{ name: "idx_etaloicod", columns: ["etaloicod"] }],
114
- lecass: [
115
- { name: "idx_lecidt", columns: ["lecidt"] },
116
- { name: "idx_codass", columns: ["codass"] },
117
- { name: "idx_orgcod", columns: ["orgcod"] },
118
- { name: "idx_orippr", columns: ["orippr"] },
119
- ],
120
- lecassrap: [
121
- { name: "idx_rapcod", columns: ["rapcod"] },
122
- { name: "idx_lecassidt", columns: ["lecassidt"] },
123
- ],
124
- lecture: [
125
- { name: "idx_loicod", columns: ["loicod"] },
126
- { name: "idx_typleccod", columns: ["typleccod"] },
127
- { name: "idx_lecidt", columns: ["lecidt"] },
128
- ],
129
- loi: [
130
- { name: "idx_typloicod", columns: ["typloicod"] },
131
- { name: "idx_etaloicod", columns: ["etaloicod"] },
132
- { name: "idx_deccoccod", columns: ["deccoccod"] },
133
- { name: "idx_loicod", columns: ["loicod"] },
134
- ],
135
- loithe: [
136
- { name: "idx_thecle", columns: ["thecle"] },
137
- { name: "idx_loicod", columns: ["loicod"] },
138
- ],
139
- posvot: [{ name: "idx_posvotcod", columns: ["posvotcod"] }],
140
- rap: [
141
- { name: "idx_rapcod", columns: ["rapcod"] },
142
- { name: "idx_coddenrap", columns: ["coddenrap"] },
143
- ],
144
- rolsig: [{ name: "idx_signataire", columns: ["signataire"] }],
145
- scr: [
146
- { name: "idx_code", columns: ["code"] },
147
- { name: "idx_scrnum", columns: ["scrnum"] },
148
- { name: "idx_sesann", columns: ["sesann"] },
149
- ],
150
- stavot: [{ name: "idx_stavotidt", columns: ["stavotidt"] }],
151
- texte: [
152
- { name: "idx_lecassidt", columns: ["lecassidt"] },
153
- { name: "idx_oritxtcod", columns: ["oritxtcod"] },
154
- { name: "idx_typtxtcod", columns: ["typtxtcod"] },
155
- { name: "idx_texcod", columns: ["texcod"] },
156
- ],
157
- the: [{ name: "idx_thecle", columns: ["thecle"] }],
158
- titsen: [{ name: "idx_titsencod", columns: ["titsencod"] }],
159
- typloi: [{ name: "idx_typloicod", columns: ["typloicod"] }],
160
- typtxt: [{ name: "idx_typtxtcod", columns: ["typtxtcod"] }],
161
- votsen: [
162
- { name: "idx_scrnum", columns: ["scrnum"] },
163
- { name: "idx_sesann", columns: ["sesann"] },
164
- { name: "idx_titsencod", columns: ["titsencod"] },
165
- { name: "idx_stavotidt", columns: ["stavotidt"] },
166
- { name: "idx_posvotcod", columns: ["posvotcod"] },
167
- { name: "idx_senmat", columns: ["senmat"] },
168
- ],
169
- },
170
- },
171
- questions: {
172
- database: "questions",
173
- repairEncoding: false,
174
- title: "Questions écrites et orales posées par les sénateurs au Gouvernement",
175
- url: "https://data.senat.fr/data/questions/questions.zip",
176
- indexes: {
177
- tam_reponses: [{ name: "idx_idque", columns: ["idque"] }],
178
- tam_questions: [
179
- { name: "idx_sorquecod", columns: ["sorquecod"] },
180
- { name: "idx_etaquecod", columns: ["etaquecod"] },
181
- ],
182
- the: [{ name: "idx_thenouidt", columns: ["thenouidt"] }],
183
- sortquestion: [{ name: "idx_sorquecod", columns: ["sorquecod"] }],
184
- etatquestion: [{ name: "idx_etaquecod", columns: ["etaquecod"] }],
185
- },
186
- },
187
- sens: {
188
- database: "sens",
189
- repairEncoding: false,
190
- repairZip: (dataset, dataDir) => {
191
- const sqlFilename = `${dataset.database}.sql`;
192
- const sqlFilePath = path.join(dataDir, sqlFilename);
193
- fs.removeSync(sqlFilePath);
194
- fs.moveSync(path.join(dataDir, "export_sens.sql"), sqlFilePath);
195
- },
196
- title: "Sénateurs (y compris organes et présence)",
197
- url: "https://data.senat.fr/data/senateurs/export_sens.zip",
198
- indexes: {
199
- dpt: [
200
- { name: "idx_dptnum", columns: ["dptnum"] },
201
- { name: "idx_regcod", columns: ["regcod"] },
202
- ],
203
- activite_loi: [{ name: "idx_actid", columns: ["actid"] }],
204
- activite_delegation: [{ name: "idx_actid", columns: ["actid"] }],
205
- activite_participant: [{ name: "idx_senmat", columns: ["senmat"] }],
206
- activite_senateur: [{ name: "idx_senmat", columns: ["senmat"] }],
207
- senbur: [
208
- { name: "idx_senmat", columns: ["senmat"] },
209
- { name: "idx_burcod", columns: ["burcod"] },
210
- ],
211
- elucan: [
212
- { name: "idx_senmat", columns: ["senmat"] },
213
- { name: "idx_typmancod", columns: ["typmancod"] },
214
- ],
215
- },
216
- },
217
- };
218
- export function getEnabledDatasets(categories) {
219
- categories.map((datasetName) => assert.notStrictEqual(EnabledDatasets[datasetName], undefined, `Unknown name of dataset: ${datasetName}`));
220
- return categories.reduce((enabledDatasets, datasetName) => {
221
- const enabledDataset = EnabledDatasets[datasetName];
222
- return enabledDatasets | (enabledDataset || EnabledDatasets.None);
223
- }, EnabledDatasets.None);
224
- }
225
- export function getChosenDatasets(enabledDatasets) {
226
- return [
227
- enabledDatasets & EnabledDatasets.Ameli ? datasets.ameli : null,
228
- enabledDatasets & EnabledDatasets.Debats ? datasets.debats : null,
229
- enabledDatasets & EnabledDatasets.DosLeg ? datasets.dosleg : null,
230
- enabledDatasets & EnabledDatasets.Questions ? datasets.questions : null,
231
- enabledDatasets & EnabledDatasets.Sens ? datasets.sens : null,
232
- ].filter((dataset) => dataset !== null);
233
- }
package/lib/git.d.ts DELETED
@@ -1,26 +0,0 @@
1
- export declare function initRepo(repositoryDir: string): void;
2
- export declare function commit(repositoryDir: string, message: string): boolean;
3
- export declare function commitAndPush(repositoryDir: string, message: string, remotes?: string[]): number;
4
- export declare function resetAndPull(gitDir: string): boolean;
5
- export declare function clone(gitGroupUrl: string | undefined, gitName: string, workingDir: string): void;
6
- export declare function run(repositoryDir: string, args: string, verbose?: boolean): string;
7
- export declare function test(repositoryDir: string, args: string, verbose?: boolean): boolean;
8
- /**
9
- * Information about a changed file in git
10
- */
11
- export interface GitChangedFile {
12
- path: string;
13
- status: "A" | "M" | "D" | "R" | "C" | "T" | "U";
14
- }
15
- /**
16
- * Get the list of files that have changed since a specific commit in a git repository.
17
- * @param repositoryDir The directory of the git repository
18
- * @param sinceCommit The commit hash to compare against (e.g., "HEAD~1", "abc123", etc.)
19
- * @param options Options for filtering
20
- * @param options.diffFilter Git diff-filter string (default: "AMR").
21
- * A=Added, M=Modified, D=Deleted, R=Renamed, C=Copied, T=Type changed, U=Unmerged
22
- * @returns A Map of file paths to their git status
23
- */
24
- export declare function getChangedFilesSinceCommit(repositoryDir: string, sinceCommit: string, options?: {
25
- diffFilter?: string;
26
- }): Map<string, GitChangedFile["status"]>;
package/lib/git.js DELETED
@@ -1,167 +0,0 @@
1
- import { execSync } from "node:child_process";
2
- import fs from "fs-extra";
3
- import path from "node:path";
4
- const MAXBUFFER = 50 * 1024 * 1024;
5
- export function initRepo(repositoryDir) {
6
- if (!fs.existsSync(path.join(repositoryDir, ".git"))) {
7
- fs.ensureDirSync(repositoryDir);
8
- execSync("git init", {
9
- cwd: repositoryDir,
10
- env: process.env,
11
- encoding: "utf-8",
12
- stdio: ["ignore", "ignore", "pipe"],
13
- });
14
- }
15
- }
16
- export function commit(repositoryDir, message) {
17
- initRepo(repositoryDir);
18
- execSync("git add .", {
19
- cwd: repositoryDir,
20
- env: process.env,
21
- encoding: "utf-8",
22
- stdio: ["ignore", "ignore", "pipe"],
23
- maxBuffer: MAXBUFFER,
24
- });
25
- try {
26
- execSync(`git commit -m "${message}" --quiet`, {
27
- cwd: repositoryDir,
28
- env: process.env,
29
- encoding: "utf-8",
30
- stdio: ["ignore", "pipe", "pipe"],
31
- });
32
- return true;
33
- }
34
- catch (childProcess) {
35
- if (childProcess.stdout === null ||
36
- !/nothing to commit|rien à valider/.test(childProcess.stdout)) {
37
- console.error(childProcess.output);
38
- throw childProcess;
39
- }
40
- return false;
41
- }
42
- }
43
- export function commitAndPush(repositoryDir, message, remotes) {
44
- let exitCode = 0;
45
- if (commit(repositoryDir, message)) {
46
- for (const remote of remotes || []) {
47
- try {
48
- execSync(`git push ${remote} master`, {
49
- cwd: repositoryDir,
50
- env: process.env,
51
- encoding: "utf-8",
52
- stdio: ["ignore", "ignore", "pipe"],
53
- });
54
- }
55
- catch (childProcess) {
56
- // Don't stop when push fails.
57
- console.error(childProcess.output);
58
- exitCode = childProcess.status;
59
- }
60
- }
61
- }
62
- else {
63
- // There was nothing to commit.
64
- exitCode = 10;
65
- }
66
- return exitCode;
67
- }
68
- export function resetAndPull(gitDir) {
69
- execSync("git reset --hard origin/master", {
70
- cwd: gitDir,
71
- env: process.env,
72
- encoding: "utf-8",
73
- stdio: ["ignore", "ignore", "pipe"],
74
- });
75
- execSync("git pull --rebase", {
76
- cwd: gitDir,
77
- env: process.env,
78
- encoding: "utf-8",
79
- stdio: ["ignore", "ignore", "pipe"],
80
- });
81
- return true;
82
- }
83
- export function clone(gitGroupUrl, gitName, workingDir) {
84
- if (gitGroupUrl !== undefined) {
85
- execSync(`git clone ${gitGroupUrl}/${gitName}.git`, {
86
- cwd: workingDir,
87
- env: process.env,
88
- encoding: "utf-8",
89
- stdio: ["ignore", "ignore", "pipe"],
90
- });
91
- }
92
- }
93
- export function run(repositoryDir, args, verbose) {
94
- try {
95
- if (verbose)
96
- console.log(`git -C ${repositoryDir} ${args}`);
97
- const output = execSync(`git ${args}`, {
98
- cwd: repositoryDir,
99
- maxBuffer: MAXBUFFER,
100
- })
101
- .toString()
102
- .trim();
103
- if (verbose)
104
- console.log(output);
105
- return output;
106
- }
107
- catch (childProcess) {
108
- for (const output of ["stdout", "stderr"])
109
- console.error(`${output}: ${childProcess[output]}`);
110
- throw childProcess;
111
- }
112
- }
113
- export function test(repositoryDir, args, verbose) {
114
- try {
115
- if (verbose)
116
- console.log(`git -C ${repositoryDir} ${args}`);
117
- const output = execSync(`git ${args}`, {
118
- cwd: repositoryDir,
119
- stdio: ["ignore", "pipe", "pipe"],
120
- maxBuffer: MAXBUFFER,
121
- })
122
- .toString()
123
- .trim();
124
- if (verbose)
125
- console.log(output);
126
- return true;
127
- }
128
- catch (childProcess) {
129
- if (childProcess.status != 0)
130
- return false;
131
- throw childProcess;
132
- }
133
- }
134
- /**
135
- * Get the list of files that have changed since a specific commit in a git repository.
136
- * @param repositoryDir The directory of the git repository
137
- * @param sinceCommit The commit hash to compare against (e.g., "HEAD~1", "abc123", etc.)
138
- * @param options Options for filtering
139
- * @param options.diffFilter Git diff-filter string (default: "AMR").
140
- * A=Added, M=Modified, D=Deleted, R=Renamed, C=Copied, T=Type changed, U=Unmerged
141
- * @returns A Map of file paths to their git status
142
- */
143
- export function getChangedFilesSinceCommit(repositoryDir, sinceCommit, options = {}) {
144
- const { diffFilter } = options;
145
- try {
146
- // Using diff-filter: A = Added, M = Modified, R = Renamed, D = Deleted, etc.
147
- // Default to AMR (excludes deleted files to prevent loading errors)
148
- const filter = diffFilter ?? "AMR";
149
- const output = run(repositoryDir, `diff --name-status --diff-filter=${filter} ${sinceCommit}`, false);
150
- const changedFiles = new Map();
151
- for (const line of output.split("\n")) {
152
- if (line.trim().length === 0)
153
- continue;
154
- const parts = line.split("\t");
155
- if (parts.length >= 2) {
156
- const status = parts[0].charAt(0);
157
- const path = parts[1];
158
- changedFiles.set(path, status);
159
- }
160
- }
161
- return changedFiles;
162
- }
163
- catch (error) {
164
- console.error(`Error getting changed files since commit ${sinceCommit}:`, error);
165
- return new Map();
166
- }
167
- }
package/lib/index.d.ts DELETED
@@ -1,13 +0,0 @@
1
- export type { AmendementResult } from "./model/ameli";
2
- export type { DossierLegislatifResult } from "./model/dosleg";
3
- export type { ScrutinResult } from "./model/scrutins";
4
- export type { QuestionResult } from "./model/questions";
5
- export type { CirconscriptionResult, OrganismeResult, SenateurResult } from "./model/sens";
6
- export type { AgendaEvent, Reunion, ReunionOdjPoint } from "./types/agenda";
7
- export type { Ses, Sub, TxtAmeli } from "./types/ameli";
8
- export type { CompteRendu } from "./types/compte_rendu";
9
- export type { Debat, LecAssDeb } from "./types/debats";
10
- export type { Ass, Aud, Auteur, DateSeance, DecCoc, DenRap, DocAtt, Ecr, EtaLoi, LecAss, LecAssRap, Lecture, Loi, Org, OriTxt, Qua, Rap, RapOrg, Scr, Texte, TypAtt, TypLec, TypLoi, TypTxt, TypUrl, } from "./types/dosleg";
11
- export type { Photo, Sen } from "./types/sens";
12
- export { UNDEFINED_SESSION, type Session, getSessionsFromStart } from "./types/sessions";
13
- export type { Alinea, Article, Division, DivisionTag, FlatTexte } from "./types/texte";
package/lib/index.js DELETED
@@ -1 +0,0 @@
1
- export { UNDEFINED_SESSION, getSessionsFromStart } from "./types/sessions";