@tricoteuses/senat 2.22.16 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (345) hide show
  1. package/README.md +168 -0
  2. package/lib/aggregates.d.ts +52 -0
  3. package/lib/aggregates.js +930 -0
  4. package/lib/aggregates.mjs +713 -0
  5. package/lib/aggregates.ts +833 -0
  6. package/lib/config.d.ts +10 -0
  7. package/lib/config.js +16 -0
  8. package/lib/config.mjs +16 -0
  9. package/lib/config.ts +26 -0
  10. package/lib/databases.d.ts +2 -0
  11. package/lib/databases.js +26 -0
  12. package/lib/databases.mjs +57 -0
  13. package/lib/databases.ts +71 -0
  14. package/lib/datasets.d.ts +34 -0
  15. package/lib/datasets.js +233 -0
  16. package/lib/datasets.mjs +78 -0
  17. package/lib/datasets.ts +118 -0
  18. package/lib/fields.d.ts +10 -0
  19. package/lib/fields.js +68 -0
  20. package/lib/fields.mjs +22 -0
  21. package/lib/fields.ts +29 -0
  22. package/lib/git.d.ts +26 -0
  23. package/lib/git.js +167 -0
  24. package/lib/index.d.ts +13 -0
  25. package/lib/index.js +1 -0
  26. package/lib/index.mjs +7 -0
  27. package/lib/index.ts +64 -0
  28. package/lib/inserters.d.ts +98 -0
  29. package/lib/inserters.js +500 -0
  30. package/lib/inserters.mjs +360 -0
  31. package/lib/inserters.ts +521 -0
  32. package/lib/legislatures.json +38 -0
  33. package/lib/loaders.d.ts +58 -0
  34. package/lib/loaders.js +286 -0
  35. package/lib/loaders.mjs +158 -0
  36. package/lib/loaders.ts +271 -0
  37. package/lib/model/agenda.d.ts +6 -0
  38. package/lib/model/agenda.js +148 -0
  39. package/lib/model/ameli.d.ts +51 -0
  40. package/lib/model/ameli.js +149 -0
  41. package/lib/model/ameli.mjs +84 -0
  42. package/lib/model/ameli.ts +100 -0
  43. package/lib/model/commission.d.ts +18 -0
  44. package/lib/model/commission.js +269 -0
  45. package/lib/model/debats.d.ts +67 -0
  46. package/lib/model/debats.js +95 -0
  47. package/lib/model/debats.mjs +43 -0
  48. package/lib/model/debats.ts +68 -0
  49. package/lib/model/documents.d.ts +12 -0
  50. package/lib/model/documents.js +151 -0
  51. package/lib/model/dosleg.d.ts +7 -0
  52. package/lib/model/dosleg.js +326 -0
  53. package/lib/model/dosleg.mjs +196 -0
  54. package/lib/model/dosleg.ts +240 -0
  55. package/lib/model/index.d.ts +7 -0
  56. package/lib/model/index.js +7 -0
  57. package/lib/model/index.mjs +5 -0
  58. package/lib/model/index.ts +15 -0
  59. package/lib/model/questions.d.ts +45 -0
  60. package/lib/model/questions.js +89 -0
  61. package/lib/model/questions.mjs +71 -0
  62. package/lib/model/questions.ts +93 -0
  63. package/lib/model/scrutins.d.ts +13 -0
  64. package/lib/model/scrutins.js +114 -0
  65. package/lib/model/seance.d.ts +3 -0
  66. package/lib/model/seance.js +267 -0
  67. package/lib/model/sens.d.ts +146 -0
  68. package/lib/model/sens.js +454 -0
  69. package/lib/model/sens.mjs +415 -0
  70. package/lib/model/sens.ts +516 -0
  71. package/lib/model/texte.d.ts +7 -0
  72. package/lib/model/texte.js +256 -0
  73. package/lib/model/texte.mjs +208 -0
  74. package/lib/model/texte.ts +229 -0
  75. package/lib/model/util.d.ts +9 -0
  76. package/lib/model/util.js +38 -0
  77. package/lib/model/util.mjs +19 -0
  78. package/lib/model/util.ts +32 -0
  79. package/lib/parsers/texte.d.ts +7 -0
  80. package/lib/parsers/texte.js +228 -0
  81. package/lib/raw_types/ameli.d.ts +914 -0
  82. package/lib/raw_types/ameli.js +5 -0
  83. package/lib/raw_types/ameli.mjs +163 -0
  84. package/lib/raw_types/debats.d.ts +207 -0
  85. package/lib/raw_types/debats.js +5 -0
  86. package/lib/raw_types/debats.mjs +58 -0
  87. package/lib/raw_types/dosleg.d.ts +1619 -0
  88. package/lib/raw_types/dosleg.js +5 -0
  89. package/lib/raw_types/dosleg.mjs +438 -0
  90. package/lib/raw_types/questions.d.ts +419 -0
  91. package/lib/raw_types/questions.js +5 -0
  92. package/lib/raw_types/questions.mjs +11 -0
  93. package/lib/raw_types/senat.d.ts +11368 -0
  94. package/lib/raw_types/senat.js +5 -0
  95. package/lib/raw_types/sens.d.ts +8248 -0
  96. package/lib/raw_types/sens.js +5 -0
  97. package/lib/raw_types/sens.mjs +508 -0
  98. package/lib/raw_types_kysely/ameli.d.ts +915 -0
  99. package/lib/raw_types_kysely/ameli.js +7 -0
  100. package/lib/raw_types_kysely/ameli.mjs +5 -0
  101. package/lib/raw_types_kysely/ameli.ts +951 -0
  102. package/lib/raw_types_kysely/debats.d.ts +207 -0
  103. package/lib/raw_types_kysely/debats.js +7 -0
  104. package/lib/raw_types_kysely/debats.mjs +5 -0
  105. package/lib/raw_types_kysely/debats.ts +222 -0
  106. package/lib/raw_types_kysely/dosleg.d.ts +3532 -0
  107. package/lib/raw_types_kysely/dosleg.js +7 -0
  108. package/lib/raw_types_kysely/dosleg.mjs +5 -0
  109. package/lib/raw_types_kysely/dosleg.ts +3621 -0
  110. package/lib/raw_types_kysely/questions.d.ts +414 -0
  111. package/lib/raw_types_kysely/questions.js +7 -0
  112. package/lib/raw_types_kysely/questions.mjs +5 -0
  113. package/lib/raw_types_kysely/questions.ts +426 -0
  114. package/lib/raw_types_kysely/sens.d.ts +4394 -0
  115. package/lib/raw_types_kysely/sens.js +7 -0
  116. package/lib/raw_types_kysely/sens.mjs +5 -0
  117. package/lib/raw_types_kysely/sens.ts +4499 -0
  118. package/lib/raw_types_schemats/ameli.d.ts +539 -0
  119. package/lib/raw_types_schemats/ameli.js +2 -0
  120. package/lib/raw_types_schemats/ameli.mjs +2 -0
  121. package/lib/raw_types_schemats/ameli.ts +601 -0
  122. package/lib/raw_types_schemats/debats.d.ts +127 -0
  123. package/lib/raw_types_schemats/debats.js +2 -0
  124. package/lib/raw_types_schemats/debats.mjs +2 -0
  125. package/lib/raw_types_schemats/debats.ts +145 -0
  126. package/lib/raw_types_schemats/dosleg.d.ts +977 -0
  127. package/lib/raw_types_schemats/dosleg.js +2 -0
  128. package/lib/raw_types_schemats/dosleg.mjs +2 -0
  129. package/lib/raw_types_schemats/dosleg.ts +2193 -0
  130. package/lib/raw_types_schemats/questions.d.ts +235 -0
  131. package/lib/raw_types_schemats/questions.js +2 -0
  132. package/lib/raw_types_schemats/questions.mjs +2 -0
  133. package/lib/raw_types_schemats/questions.ts +249 -0
  134. package/lib/raw_types_schemats/sens.d.ts +6915 -0
  135. package/lib/raw_types_schemats/sens.js +2 -0
  136. package/lib/raw_types_schemats/sens.mjs +2 -0
  137. package/lib/raw_types_schemats/sens.ts +2907 -0
  138. package/lib/scripts/convert_data.d.ts +1 -0
  139. package/lib/scripts/convert_data.js +354 -0
  140. package/lib/scripts/convert_data.mjs +181 -0
  141. package/lib/scripts/convert_data.ts +243 -0
  142. package/lib/scripts/data-download.d.ts +1 -0
  143. package/lib/scripts/data-download.js +12 -0
  144. package/lib/scripts/datautil.d.ts +8 -0
  145. package/lib/scripts/datautil.js +34 -0
  146. package/lib/scripts/datautil.mjs +16 -0
  147. package/lib/scripts/datautil.ts +19 -0
  148. package/lib/scripts/images/transparent_150x192.jpg +0 -0
  149. package/lib/scripts/images/transparent_155x225.jpg +0 -0
  150. package/lib/scripts/parse_textes.d.ts +1 -0
  151. package/lib/scripts/parse_textes.js +44 -0
  152. package/lib/scripts/parse_textes.mjs +46 -0
  153. package/lib/scripts/parse_textes.ts +65 -0
  154. package/lib/scripts/retrieve_agenda.d.ts +1 -0
  155. package/lib/scripts/retrieve_agenda.js +132 -0
  156. package/lib/scripts/retrieve_cr_commission.d.ts +1 -0
  157. package/lib/scripts/retrieve_cr_commission.js +364 -0
  158. package/lib/scripts/retrieve_cr_seance.d.ts +6 -0
  159. package/lib/scripts/retrieve_cr_seance.js +347 -0
  160. package/lib/scripts/retrieve_documents.d.ts +3 -0
  161. package/lib/scripts/retrieve_documents.js +219 -0
  162. package/lib/scripts/retrieve_documents.mjs +249 -0
  163. package/lib/scripts/retrieve_documents.ts +298 -0
  164. package/lib/scripts/retrieve_open_data.d.ts +1 -0
  165. package/lib/scripts/retrieve_open_data.js +315 -0
  166. package/lib/scripts/retrieve_open_data.mjs +217 -0
  167. package/lib/scripts/retrieve_open_data.ts +268 -0
  168. package/lib/scripts/retrieve_senateurs_photos.d.ts +1 -0
  169. package/lib/scripts/retrieve_senateurs_photos.js +147 -0
  170. package/lib/scripts/retrieve_senateurs_photos.mjs +147 -0
  171. package/lib/scripts/retrieve_senateurs_photos.ts +177 -0
  172. package/lib/scripts/retrieve_videos.d.ts +1 -0
  173. package/lib/scripts/retrieve_videos.js +461 -0
  174. package/lib/scripts/shared/cli_helpers.d.ts +95 -0
  175. package/lib/scripts/shared/cli_helpers.js +91 -0
  176. package/lib/scripts/shared/cli_helpers.ts +36 -0
  177. package/lib/scripts/shared/util.d.ts +4 -0
  178. package/lib/scripts/shared/util.js +35 -0
  179. package/lib/scripts/shared/util.ts +33 -0
  180. package/lib/scripts/test_iter_load.d.ts +1 -0
  181. package/lib/scripts/test_iter_load.js +12 -0
  182. package/lib/src/config.d.ts +22 -0
  183. package/lib/src/config.js +17 -7
  184. package/lib/src/conversion_textes.js +5 -1
  185. package/lib/src/databases.d.ts +2 -1
  186. package/lib/src/databases_postgres.d.ts +4 -0
  187. package/lib/src/databases_postgres.js +23 -0
  188. package/lib/src/datasets.d.ts +4 -0
  189. package/lib/src/datasets.js +16 -2
  190. package/lib/src/git.d.ts +1 -0
  191. package/lib/src/git.js +45 -11
  192. package/lib/src/index.d.ts +19 -8
  193. package/lib/src/index.js +6 -1
  194. package/lib/src/loaders.js +10 -4
  195. package/lib/src/model/agenda.js +2 -2
  196. package/lib/src/model/ameli.d.ts +64 -52
  197. package/lib/src/model/ameli.js +147 -145
  198. package/lib/src/model/ameli_postgres.d.ts +67 -0
  199. package/lib/src/model/ameli_postgres.js +150 -0
  200. package/lib/src/model/commission.d.ts +3 -2
  201. package/lib/src/model/commission.js +2 -2
  202. package/lib/src/model/debats.d.ts +38 -66
  203. package/lib/src/model/debats.js +110 -93
  204. package/lib/src/model/documents.d.ts +32 -12
  205. package/lib/src/model/documents.js +171 -130
  206. package/lib/src/model/dosleg.d.ts +142 -5
  207. package/lib/src/model/dosleg.js +298 -156
  208. package/lib/src/model/questions.d.ts +54 -45
  209. package/lib/src/model/questions.js +89 -87
  210. package/lib/src/model/scrutins.d.ts +48 -13
  211. package/lib/src/model/scrutins.js +118 -111
  212. package/lib/src/model/seance.js +3 -3
  213. package/lib/src/model/sens.d.ts +109 -179
  214. package/lib/src/model/sens.js +384 -484
  215. package/lib/src/model/util.d.ts +0 -8
  216. package/lib/src/model/util.js +0 -23
  217. package/lib/src/parsers/texte.js +7 -7
  218. package/lib/src/raw_types/ameli.d.ts +1651 -803
  219. package/lib/src/raw_types/ameli.js +1816 -5
  220. package/lib/src/raw_types/debats.d.ts +353 -180
  221. package/lib/src/raw_types/debats.js +517 -5
  222. package/lib/src/raw_types/dosleg.d.ts +2862 -1527
  223. package/lib/src/raw_types/dosleg.js +4354 -5
  224. package/lib/src/raw_types/questions.d.ts +671 -395
  225. package/lib/src/raw_types/questions.js +1303 -5
  226. package/lib/src/raw_types/sens.d.ts +7743 -8148
  227. package/lib/src/raw_types/sens.js +10429 -5
  228. package/lib/src/raw_types_schemats/ameli.d.ts +4 -2
  229. package/lib/src/raw_types_schemats/debats.d.ts +2 -2
  230. package/lib/src/raw_types_schemats/dosleg.d.ts +2 -2
  231. package/lib/src/raw_types_schemats/questions.d.ts +2 -2
  232. package/lib/src/raw_types_schemats/sens.d.ts +10 -4216
  233. package/lib/src/scripts/convert_data.js +7 -6
  234. package/lib/src/scripts/convert_xml_to_html.js +2 -2
  235. package/lib/src/scripts/data-download.js +3 -2
  236. package/lib/src/scripts/retrieve_agenda.js +21 -9
  237. package/lib/src/scripts/retrieve_cr_commission.js +17 -17
  238. package/lib/src/scripts/retrieve_cr_seance.d.ts +14 -1
  239. package/lib/src/scripts/retrieve_cr_seance.js +10 -11
  240. package/lib/src/scripts/retrieve_documents.d.ts +11 -2
  241. package/lib/src/scripts/retrieve_documents.js +25 -14
  242. package/lib/src/scripts/retrieve_open_data.js +514 -153
  243. package/lib/src/scripts/retrieve_senateurs_photos.js +25 -11
  244. package/lib/src/scripts/retrieve_videos.js +12 -11
  245. package/lib/src/scripts/shared/cli_helpers.d.ts +1 -6
  246. package/lib/src/scripts/shared/cli_helpers.js +9 -8
  247. package/lib/src/scripts/shared/incremental_import_sql.d.ts +2 -0
  248. package/lib/src/scripts/shared/incremental_import_sql.js +894 -0
  249. package/lib/src/scripts/shared/prefixed_tables.d.ts +10 -0
  250. package/lib/src/scripts/shared/prefixed_tables.js +36 -0
  251. package/lib/src/scripts/shared/schema_version.d.ts +3 -0
  252. package/lib/src/scripts/shared/schema_version.js +97 -0
  253. package/lib/src/scripts/shared/staging_import.d.ts +3 -0
  254. package/lib/src/scripts/shared/staging_import.js +80 -0
  255. package/lib/src/scripts/shared/staging_metadata_sql.d.ts +1 -0
  256. package/lib/src/scripts/shared/staging_metadata_sql.js +221 -0
  257. package/lib/src/scripts/validate_prefixed_tables.d.ts +1 -0
  258. package/lib/src/scripts/validate_prefixed_tables.js +101 -0
  259. package/lib/src/types/ameli.d.ts +4 -4
  260. package/lib/src/types/debats.d.ts +2 -2
  261. package/lib/src/types/dosleg.d.ts +39 -39
  262. package/lib/src/types/questions.d.ts +2 -2
  263. package/lib/src/types/sens.d.ts +0 -2
  264. package/lib/src/types/texte.d.ts +1 -1
  265. package/lib/src/utils/cr_spliting.d.ts +9 -6
  266. package/lib/src/utils/cr_spliting.js +6 -101
  267. package/lib/src/utils/reunion_odj_building.d.ts +7 -3
  268. package/lib/src/utils/reunion_parsing.d.ts +2 -1
  269. package/lib/src/utils/reunion_parsing.js +2 -2
  270. package/lib/src/videos/match.js +8 -5
  271. package/lib/src/videos/pipeline.d.ts +6 -2
  272. package/lib/src/videos/pipeline.js +21 -8
  273. package/lib/src/videos/search.js +6 -2
  274. package/lib/strings.d.ts +1 -0
  275. package/lib/strings.js +18 -0
  276. package/lib/strings.mjs +18 -0
  277. package/lib/strings.ts +26 -0
  278. package/lib/tests/incrementalImportSql.test.d.ts +1 -0
  279. package/lib/tests/incrementalImportSql.test.js +155 -0
  280. package/lib/tests/prefixedTables.test.d.ts +1 -0
  281. package/lib/tests/prefixedTables.test.js +22 -0
  282. package/lib/tests/schemaVersion.test.d.ts +1 -0
  283. package/lib/tests/schemaVersion.test.js +23 -0
  284. package/lib/tests/validatePrefixedTables.test.d.ts +1 -0
  285. package/lib/tests/validatePrefixedTables.test.js +14 -0
  286. package/lib/types/agenda.d.ts +44 -0
  287. package/lib/types/agenda.js +1 -0
  288. package/lib/types/ameli.d.ts +5 -0
  289. package/lib/types/ameli.js +1 -0
  290. package/lib/types/ameli.mjs +13 -0
  291. package/lib/types/ameli.ts +21 -0
  292. package/lib/types/compte_rendu.d.ts +83 -0
  293. package/lib/types/compte_rendu.js +1 -0
  294. package/lib/types/debats.d.ts +2 -0
  295. package/lib/types/debats.js +1 -0
  296. package/lib/types/debats.mjs +2 -0
  297. package/lib/types/debats.ts +6 -0
  298. package/lib/types/dosleg.d.ts +70 -0
  299. package/lib/types/dosleg.js +1 -0
  300. package/lib/types/dosleg.mjs +151 -0
  301. package/lib/types/dosleg.ts +284 -0
  302. package/lib/types/questions.d.ts +2 -0
  303. package/lib/types/questions.js +1 -0
  304. package/lib/types/questions.mjs +1 -0
  305. package/lib/types/questions.ts +3 -0
  306. package/lib/types/sens.d.ts +10 -0
  307. package/lib/types/sens.js +1 -0
  308. package/lib/types/sens.mjs +1 -0
  309. package/lib/types/sens.ts +12 -0
  310. package/lib/types/sessions.d.ts +5 -0
  311. package/lib/types/sessions.js +84 -0
  312. package/lib/types/sessions.mjs +43 -0
  313. package/lib/types/sessions.ts +42 -0
  314. package/lib/types/texte.d.ts +74 -0
  315. package/lib/types/texte.js +16 -0
  316. package/lib/types/texte.mjs +16 -0
  317. package/lib/types/texte.ts +76 -0
  318. package/lib/typings/windows-1252.d.js +2 -0
  319. package/lib/typings/windows-1252.d.mjs +2 -0
  320. package/lib/typings/windows-1252.d.ts +11 -0
  321. package/lib/utils/cr_spliting.d.ts +28 -0
  322. package/lib/utils/cr_spliting.js +265 -0
  323. package/lib/utils/date.d.ts +10 -0
  324. package/lib/utils/date.js +100 -0
  325. package/lib/utils/nvs-timecode.d.ts +7 -0
  326. package/lib/utils/nvs-timecode.js +79 -0
  327. package/lib/utils/reunion_grouping.d.ts +9 -0
  328. package/lib/utils/reunion_grouping.js +361 -0
  329. package/lib/utils/reunion_odj_building.d.ts +5 -0
  330. package/lib/utils/reunion_odj_building.js +154 -0
  331. package/lib/utils/reunion_parsing.d.ts +23 -0
  332. package/lib/utils/reunion_parsing.js +209 -0
  333. package/lib/utils/scoring.d.ts +14 -0
  334. package/lib/utils/scoring.js +147 -0
  335. package/lib/utils/string_cleaning.d.ts +7 -0
  336. package/lib/utils/string_cleaning.js +57 -0
  337. package/lib/validators/config.d.ts +9 -0
  338. package/lib/validators/config.js +10 -0
  339. package/lib/validators/config.mjs +54 -0
  340. package/lib/validators/config.ts +79 -0
  341. package/lib/validators/senat.d.ts +0 -0
  342. package/lib/validators/senat.js +28 -0
  343. package/lib/validators/senat.mjs +24 -0
  344. package/lib/validators/senat.ts +26 -0
  345. package/package.json +11 -11
@@ -0,0 +1,10 @@
1
+ export declare const senatSchemaName = "senat";
2
+ export declare const rawTypesDir: string;
3
+ export declare function prefixedName(datasetName: string, relationName: string): string;
4
+ export declare function stagingSchemaName(datasetName: string): string;
5
+ export declare function escapeRegExp(text: string): string;
6
+ export declare function stripDatasetPrefix(relationName: string, datasetName: string): string;
7
+ export declare function getGeneratedDefinitionPath(datasetName: string): string;
8
+ export declare function getGeneratedTableManifestPath(datasetName: string): string;
9
+ export declare function buildGeneratedTableManifest(datasetName: string, prefixedTables: string[]): string;
10
+ export declare function extractPrefixedTableNamesFromGeneratedManifest(manifest: string, datasetName: string): string[];
@@ -0,0 +1,36 @@
1
+ import path from "path";
2
+ export const senatSchemaName = "senat";
3
+ export const rawTypesDir = path.resolve("src", "raw_types");
4
+ export function prefixedName(datasetName, relationName) {
5
+ return `${datasetName}_${relationName}`;
6
+ }
7
+ export function stagingSchemaName(datasetName) {
8
+ return `${datasetName}_staging`;
9
+ }
10
+ export function escapeRegExp(text) {
11
+ return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
12
+ }
13
+ export function stripDatasetPrefix(relationName, datasetName) {
14
+ const datasetPrefix = `${datasetName}_`;
15
+ return relationName.startsWith(datasetPrefix) ? relationName.slice(datasetPrefix.length) : relationName;
16
+ }
17
+ export function getGeneratedDefinitionPath(datasetName) {
18
+ return path.join(rawTypesDir, `${datasetName}.ts`);
19
+ }
20
+ export function getGeneratedTableManifestPath(datasetName) {
21
+ return path.join(rawTypesDir, `${datasetName}.tables.json`);
22
+ }
23
+ export function buildGeneratedTableManifest(datasetName, prefixedTables) {
24
+ const manifest = {
25
+ dataset: datasetName,
26
+ prefixedTables: [...new Set(prefixedTables)],
27
+ };
28
+ return JSON.stringify(manifest, null, 2) + "\n";
29
+ }
30
+ export function extractPrefixedTableNamesFromGeneratedManifest(manifest, datasetName) {
31
+ const parsed = JSON.parse(manifest);
32
+ if (parsed.dataset !== datasetName || !Array.isArray(parsed.prefixedTables)) {
33
+ return [];
34
+ }
35
+ return [...new Set(parsed.prefixedTables.filter((tableName) => typeof tableName === "string"))];
36
+ }
@@ -0,0 +1,3 @@
1
+ export declare function buildEnsureSchemaVersionTableSql(schemaName: string): string;
2
+ export declare function buildSchemaStructureFingerprintQuery(schemaName: string): string;
3
+ export declare function buildIncrementSchemaVersionSql(schemaName: string): string;
@@ -0,0 +1,97 @@
1
+ function escapeSqlLiteral(value) {
2
+ return value.replace(/'/g, "''");
3
+ }
4
+ export function buildEnsureSchemaVersionTableSql(schemaName) {
5
+ return `
6
+ CREATE TABLE IF NOT EXISTS ${schemaName}.version(
7
+ number integer PRIMARY KEY
8
+ );
9
+ COMMENT ON TABLE ${schemaName}.version IS 'version of database';
10
+ COMMENT ON COLUMN ${schemaName}.version.number IS 'version number of database schema';
11
+ INSERT INTO ${schemaName}.version(number)
12
+ SELECT 0
13
+ WHERE NOT EXISTS (SELECT 1 FROM ${schemaName}.version);
14
+ `;
15
+ }
16
+ export function buildSchemaStructureFingerprintQuery(schemaName) {
17
+ const escapedSchemaName = escapeSqlLiteral(schemaName);
18
+ return `
19
+ WITH objects AS (
20
+ SELECT format('table:%s', table_class.relname) AS entry
21
+ FROM pg_class table_class
22
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
23
+ WHERE table_namespace.nspname = '${escapedSchemaName}'
24
+ AND table_class.relkind IN ('r', 'p')
25
+ AND table_class.relname <> 'version'
26
+
27
+ UNION ALL
28
+
29
+ SELECT format(
30
+ 'column:%s:%s:%s:%s:%s:%s:%s:%s',
31
+ columns.table_name,
32
+ columns.ordinal_position,
33
+ columns.column_name,
34
+ columns.udt_schema,
35
+ columns.udt_name,
36
+ columns.is_nullable,
37
+ coalesce(columns.column_default, ''),
38
+ columns.is_identity
39
+ ) AS entry
40
+ FROM information_schema.columns columns
41
+ WHERE columns.table_schema = '${escapedSchemaName}'
42
+ AND columns.table_name <> 'version'
43
+
44
+ UNION ALL
45
+
46
+ SELECT format(
47
+ 'constraint:%s:%s:%s:%s',
48
+ table_class.relname,
49
+ constraint_data.contype,
50
+ constraint_data.conname,
51
+ pg_get_constraintdef(constraint_data.oid, true)
52
+ ) AS entry
53
+ FROM pg_constraint constraint_data
54
+ JOIN pg_class table_class ON table_class.oid = constraint_data.conrelid
55
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
56
+ WHERE table_namespace.nspname = '${escapedSchemaName}'
57
+ AND table_class.relname <> 'version'
58
+ AND constraint_data.contype <> 'n'
59
+
60
+ UNION ALL
61
+
62
+ SELECT format(
63
+ 'index:%s:%s',
64
+ table_class.relname,
65
+ pg_get_indexdef(idx.indexrelid)
66
+ ) AS entry
67
+ FROM pg_index idx
68
+ JOIN pg_class table_class ON table_class.oid = idx.indrelid
69
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
70
+ LEFT JOIN pg_constraint constraint_data ON constraint_data.conindid = idx.indexrelid
71
+ WHERE table_namespace.nspname = '${escapedSchemaName}'
72
+ AND table_class.relname <> 'version'
73
+ AND constraint_data.oid IS NULL
74
+
75
+ UNION ALL
76
+
77
+ SELECT format(
78
+ 'sequence:%s:%s:%s:%s:%s:%s:%s:%s',
79
+ sequences.sequencename,
80
+ sequences.data_type,
81
+ sequences.start_value,
82
+ sequences.min_value,
83
+ sequences.max_value,
84
+ sequences.increment_by,
85
+ sequences.cycle,
86
+ sequences.cache_size
87
+ ) AS entry
88
+ FROM pg_sequences sequences
89
+ WHERE sequences.schemaname = '${escapedSchemaName}'
90
+ )
91
+ SELECT coalesce(md5(string_agg(entry, E'\\n' ORDER BY entry)), '')
92
+ FROM objects
93
+ `;
94
+ }
95
+ export function buildIncrementSchemaVersionSql(schemaName) {
96
+ return `UPDATE ${schemaName}.version SET number = number + 1`;
97
+ }
@@ -0,0 +1,3 @@
1
+ import type { Dataset } from "../../datasets";
2
+ export declare function isCopyFromStdinLine(line: string): boolean;
3
+ export declare function rewriteLineForStagingImport(line: string, dataset: Dataset, stagingSchema: string, inCopyData?: boolean): string;
@@ -0,0 +1,80 @@
1
+ function sourceSchemaNames(dataset) {
2
+ if (dataset.database === "sens") {
3
+ return ["sens", "senat"];
4
+ }
5
+ return [dataset.database];
6
+ }
7
+ function sourceSchemaAlternation(dataset) {
8
+ return sourceSchemaNames(dataset).join("|");
9
+ }
10
+ export function isCopyFromStdinLine(line) {
11
+ return /^COPY .* FROM stdin;$/i.test(line);
12
+ }
13
+ export function rewriteLineForStagingImport(line, dataset, stagingSchema, inCopyData = false) {
14
+ if (inCopyData) {
15
+ return line;
16
+ }
17
+ const sourceSchemas = sourceSchemaNames(dataset);
18
+ const sourceSchemaPattern = sourceSchemaAlternation(dataset);
19
+ const searchPathRegex = new RegExp(`^SET search_path = (public|${sourceSchemaPattern}), pg_catalog;$`, "i");
20
+ if (searchPathRegex.test(line)) {
21
+ return `SET search_path = ${stagingSchema}, pg_catalog;`;
22
+ }
23
+ const alterOwnerRegex = /^ALTER (SCHEMA|TABLE|SEQUENCE|INDEX|VIEW) .* OWNER TO /i;
24
+ if (alterOwnerRegex.test(line)) {
25
+ return "-- skipped OWNER TO during staging import";
26
+ }
27
+ if (/^(GRANT|REVOKE) /i.test(line)) {
28
+ return "-- skipped GRANT/REVOKE during staging import";
29
+ }
30
+ if (/^ALTER TABLE ONLY .* DROP CONSTRAINT /i.test(line)) {
31
+ return "-- skipped redundant DROP CONSTRAINT during staging import";
32
+ }
33
+ if (/^DROP INDEX /i.test(line)) {
34
+ return "-- skipped redundant DROP INDEX during staging import";
35
+ }
36
+ if (/^DROP TABLE /i.test(line)) {
37
+ return "-- skipped redundant DROP TABLE during staging import";
38
+ }
39
+ if (/^DROP SCHEMA public;$/i.test(line)) {
40
+ return "-- skipped redundant DROP SCHEMA public during staging import";
41
+ }
42
+ if (/^CREATE SCHEMA public;$/i.test(line)) {
43
+ return "-- skipped redundant CREATE SCHEMA public during staging import";
44
+ }
45
+ if (/^COMMENT ON SCHEMA public IS /i.test(line)) {
46
+ return "-- skipped redundant COMMENT ON SCHEMA public during staging import";
47
+ }
48
+ if (/^CREATE TABLE /i.test(line)) {
49
+ line = line.replace(/^CREATE TABLE /i, "CREATE UNLOGGED TABLE ");
50
+ }
51
+ if (/^CREATE SEQUENCE /i.test(line)) {
52
+ line = line.replace(/^CREATE SEQUENCE /i, "CREATE UNLOGGED SEQUENCE ");
53
+ }
54
+ const createSchemaRegex = new RegExp(`^CREATE SCHEMA (${sourceSchemaPattern});$`, "i");
55
+ if (createSchemaRegex.test(line)) {
56
+ return `CREATE SCHEMA IF NOT EXISTS ${stagingSchema};`;
57
+ }
58
+ // In dumps like debats, "COMMENT ON COLUMN debats.datsea" means table.column,
59
+ // not schema.table.column. Keep these statements untouched and let search_path resolve them.
60
+ const commentOnColumnRegex = /^COMMENT ON COLUMN ([A-Za-z_][A-Za-z0-9_]*)\.([A-Za-z_][A-Za-z0-9_]*) IS /i;
61
+ const commentOnColumnMatch = line.match(commentOnColumnRegex);
62
+ if (commentOnColumnMatch && sourceSchemas.includes(commentOnColumnMatch[1])) {
63
+ return line;
64
+ }
65
+ const parts = line.split(/(')/);
66
+ let inString = false;
67
+ for (let i = 0; i < parts.length; i++) {
68
+ if (parts[i] === "'") {
69
+ inString = !inString;
70
+ }
71
+ else if (!inString) {
72
+ let rewrittenPart = parts[i].replace(/\bpublic\./g, `${stagingSchema}.`);
73
+ for (const sourceSchema of sourceSchemas) {
74
+ rewrittenPart = rewrittenPart.replace(new RegExp(`\\b${sourceSchema}\\.`, "g"), `${stagingSchema}.`);
75
+ }
76
+ parts[i] = rewrittenPart;
77
+ }
78
+ }
79
+ return parts.join("");
80
+ }
@@ -0,0 +1 @@
1
+ export declare function buildExportStagingMetadataStatementsQuery(stagingSchema: string, targetSchema: string): string;
@@ -0,0 +1,221 @@
1
+ function escapeSqlLiteral(value) {
2
+ return value.replace(/'/g, "''");
3
+ }
4
+ export function buildExportStagingMetadataStatementsQuery(stagingSchema, targetSchema) {
5
+ const escapedStagingSchema = escapeSqlLiteral(stagingSchema);
6
+ return `
7
+ WITH ordered_statements AS (
8
+ SELECT
9
+ 10 AS group_no,
10
+ 0 AS order_no,
11
+ sequences.sequencename AS object_name,
12
+ format('CREATE SEQUENCE IF NOT EXISTS %I.%I', '${targetSchema}', sequences.sequencename) AS statement
13
+ FROM pg_sequences sequences
14
+ WHERE sequences.schemaname = '${escapedStagingSchema}'
15
+
16
+ UNION ALL
17
+
18
+ SELECT
19
+ 20 AS group_no,
20
+ 0 AS order_no,
21
+ sequences.sequencename AS object_name,
22
+ format(
23
+ 'ALTER SEQUENCE %I.%I AS %s INCREMENT BY %s MINVALUE %s MAXVALUE %s START WITH %s CACHE %s %s',
24
+ '${targetSchema}',
25
+ sequences.sequencename,
26
+ sequences.data_type,
27
+ sequences.increment_by,
28
+ sequences.min_value,
29
+ sequences.max_value,
30
+ sequences.start_value,
31
+ sequences.cache_size,
32
+ CASE WHEN sequences.cycle THEN 'CYCLE' ELSE 'NO CYCLE' END
33
+ ) AS statement
34
+ FROM pg_sequences sequences
35
+ WHERE sequences.schemaname = '${escapedStagingSchema}'
36
+
37
+ UNION ALL
38
+
39
+ SELECT
40
+ 30 AS group_no,
41
+ 0 AS order_no,
42
+ sequence_class.relname AS object_name,
43
+ CASE
44
+ WHEN dependent_table.relname IS NULL THEN format(
45
+ 'ALTER SEQUENCE %I.%I OWNED BY NONE',
46
+ '${targetSchema}',
47
+ sequence_class.relname
48
+ )
49
+ ELSE format(
50
+ 'ALTER SEQUENCE %I.%I OWNED BY %I.%I.%I',
51
+ '${targetSchema}',
52
+ sequence_class.relname,
53
+ '${targetSchema}',
54
+ dependent_table.relname,
55
+ dependent_column.attname
56
+ )
57
+ END AS statement
58
+ FROM pg_class sequence_class
59
+ JOIN pg_namespace sequence_namespace ON sequence_namespace.oid = sequence_class.relnamespace
60
+ LEFT JOIN pg_depend dependency
61
+ ON dependency.objid = sequence_class.oid
62
+ AND dependency.deptype = 'a'
63
+ LEFT JOIN pg_class dependent_table ON dependent_table.oid = dependency.refobjid
64
+ LEFT JOIN pg_namespace dependent_namespace ON dependent_namespace.oid = dependent_table.relnamespace
65
+ LEFT JOIN pg_attribute dependent_column
66
+ ON dependent_column.attrelid = dependent_table.oid
67
+ AND dependent_column.attnum = dependency.refobjsubid
68
+ WHERE sequence_namespace.nspname = '${escapedStagingSchema}'
69
+ AND sequence_class.relkind = 'S'
70
+ AND (
71
+ dependent_namespace.nspname IS NULL
72
+ OR dependent_namespace.nspname = '${escapedStagingSchema}'
73
+ )
74
+
75
+ UNION ALL
76
+
77
+ SELECT
78
+ 40 AS group_no,
79
+ 0 AS order_no,
80
+ format('%s.%s', table_class.relname, attribute.attname) AS object_name,
81
+ CASE
82
+ WHEN default_expr.expression IS NULL THEN format(
83
+ 'ALTER TABLE %I.%I ALTER COLUMN %I DROP DEFAULT',
84
+ '${targetSchema}',
85
+ table_class.relname,
86
+ attribute.attname
87
+ )
88
+ ELSE format(
89
+ 'ALTER TABLE %I.%I ALTER COLUMN %I SET DEFAULT %s',
90
+ '${targetSchema}',
91
+ table_class.relname,
92
+ attribute.attname,
93
+ replace(default_expr.expression, '${stagingSchema}.', '${targetSchema}.')
94
+ )
95
+ END AS statement
96
+ FROM pg_class table_class
97
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
98
+ JOIN pg_attribute attribute ON attribute.attrelid = table_class.oid
99
+ LEFT JOIN LATERAL (
100
+ SELECT pg_get_expr(attr_def.adbin, attr_def.adrelid) AS expression
101
+ FROM pg_attrdef attr_def
102
+ WHERE attr_def.adrelid = attribute.attrelid
103
+ AND attr_def.adnum = attribute.attnum
104
+ ) default_expr ON true
105
+ WHERE table_namespace.nspname = '${escapedStagingSchema}'
106
+ AND table_class.relkind IN ('r', 'p')
107
+ AND attribute.attnum > 0
108
+ AND NOT attribute.attisdropped
109
+ AND attribute.attgenerated = ''
110
+ AND attribute.attidentity = ''
111
+
112
+ UNION ALL
113
+
114
+ SELECT
115
+ 50 AS group_no,
116
+ 0 AS order_no,
117
+ table_class.relname AS object_name,
118
+ format(
119
+ 'COMMENT ON TABLE %I.%I IS %L',
120
+ '${targetSchema}',
121
+ table_class.relname,
122
+ obj_description(table_class.oid, 'pg_class')
123
+ ) AS statement
124
+ FROM pg_class table_class
125
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
126
+ WHERE table_namespace.nspname = '${escapedStagingSchema}'
127
+ AND table_class.relkind IN ('r', 'p')
128
+
129
+ UNION ALL
130
+
131
+ SELECT
132
+ 60 AS group_no,
133
+ 0 AS order_no,
134
+ format('%s.%s', table_class.relname, attribute.attname) AS object_name,
135
+ format(
136
+ 'COMMENT ON COLUMN %I.%I.%I IS %L',
137
+ '${targetSchema}',
138
+ table_class.relname,
139
+ attribute.attname,
140
+ col_description(table_class.oid, attribute.attnum)
141
+ ) AS statement
142
+ FROM pg_class table_class
143
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
144
+ JOIN pg_attribute attribute ON attribute.attrelid = table_class.oid
145
+ WHERE table_namespace.nspname = '${escapedStagingSchema}'
146
+ AND table_class.relkind IN ('r', 'p')
147
+ AND attribute.attnum > 0
148
+ AND NOT attribute.attisdropped
149
+
150
+ UNION ALL
151
+
152
+ SELECT
153
+ 70 AS group_no,
154
+ CASE constraint_data.contype
155
+ WHEN 'p' THEN 1
156
+ WHEN 'u' THEN 2
157
+ WHEN 'c' THEN 3
158
+ WHEN 'f' THEN 4
159
+ ELSE 5
160
+ END AS order_no,
161
+ format('%s.%s', table_class.relname, constraint_data.conname) AS object_name,
162
+ format(
163
+ 'ALTER TABLE %I.%I ADD CONSTRAINT %I %s',
164
+ '${targetSchema}',
165
+ table_class.relname,
166
+ constraint_data.conname,
167
+ replace(
168
+ replace(
169
+ pg_get_constraintdef(constraint_data.oid, true),
170
+ ' REFERENCES ' || '${stagingSchema}' || '.',
171
+ ' REFERENCES ${targetSchema}.'
172
+ ),
173
+ ' REFERENCES ONLY ' || '${stagingSchema}' || '.',
174
+ ' REFERENCES ONLY ${targetSchema}.'
175
+ )
176
+ ) AS statement
177
+ FROM pg_constraint constraint_data
178
+ JOIN pg_class table_class ON table_class.oid = constraint_data.conrelid
179
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
180
+ WHERE table_namespace.nspname = '${escapedStagingSchema}'
181
+ AND constraint_data.contype <> 'n'
182
+
183
+ UNION ALL
184
+
185
+ SELECT
186
+ 80 AS group_no,
187
+ 0 AS order_no,
188
+ format('%s.%s', table_class.relname, index_class.relname) AS object_name,
189
+ replace(
190
+ pg_get_indexdef(index_data.indexrelid),
191
+ ' ON ${stagingSchema}.',
192
+ ' ON ${targetSchema}.'
193
+ ) AS statement
194
+ FROM pg_index index_data
195
+ JOIN pg_class table_class ON table_class.oid = index_data.indrelid
196
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
197
+ JOIN pg_class index_class ON index_class.oid = index_data.indexrelid
198
+ LEFT JOIN pg_constraint constraint_data ON constraint_data.conindid = index_data.indexrelid
199
+ WHERE table_namespace.nspname = '${escapedStagingSchema}'
200
+ AND constraint_data.oid IS NULL
201
+
202
+ UNION ALL
203
+
204
+ SELECT
205
+ 90 AS group_no,
206
+ 0 AS order_no,
207
+ sequences.sequencename AS object_name,
208
+ format(
209
+ 'SELECT setval(%L, %s, %s)',
210
+ '${targetSchema}.' || sequences.sequencename,
211
+ coalesce(sequences.last_value, sequences.start_value),
212
+ CASE WHEN sequences.last_value IS NULL THEN 'false' ELSE 'true' END
213
+ ) AS statement
214
+ FROM pg_sequences sequences
215
+ WHERE sequences.schemaname = '${escapedStagingSchema}'
216
+ )
217
+ SELECT encode(convert_to(statement, 'UTF8'), 'hex')
218
+ FROM ordered_statements
219
+ ORDER BY group_no, order_no, object_name;
220
+ `;
221
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,101 @@
1
+ import commandLineArgs from "command-line-args";
2
+ import fs from "fs-extra";
3
+ import { sql } from "../databases_postgres";
4
+ import { getChosenDatasets, getEnabledDatasets } from "../datasets";
5
+ import { categoriesOption, silentOption } from "./shared/cli_helpers";
6
+ import { extractPrefixedTableNamesFromGeneratedManifest, getGeneratedTableManifestPath, senatSchemaName, } from "./shared/prefixed_tables";
7
+ const optionsDefinitions = [categoriesOption, silentOption];
8
+ const options = commandLineArgs(optionsDefinitions);
9
+ async function listPrefixedSenatTables(dataset) {
10
+ const prefix = `${dataset.database}_`;
11
+ const rows = await sql `
12
+ select tablename as relation_name
13
+ from pg_tables
14
+ where schemaname = ${senatSchemaName}
15
+ and tablename like ${prefix + "%"}
16
+ order by tablename
17
+ `;
18
+ return rows.map(({ relation_name }) => relation_name);
19
+ }
20
+ function getGeneratedDefinitionPath(dataset) {
21
+ return getGeneratedTableManifestPath(dataset.database);
22
+ }
23
+ function listExpectedPrefixedSenatTables(dataset) {
24
+ const manifestFilePath = getGeneratedDefinitionPath(dataset);
25
+ const manifest = fs.readFileSync(manifestFilePath, { encoding: "utf8" });
26
+ return extractPrefixedTableNamesFromGeneratedManifest(manifest, dataset.database);
27
+ }
28
+ async function listColumns(schema, relationName) {
29
+ const rows = await sql `
30
+ select column_name
31
+ from information_schema.columns
32
+ where table_schema = ${schema}
33
+ and table_name = ${relationName}
34
+ order by ordinal_position
35
+ `;
36
+ return rows.map(({ column_name }) => column_name);
37
+ }
38
+ async function validateDataset(dataset) {
39
+ const issues = [];
40
+ const expectedTables = listExpectedPrefixedSenatTables(dataset);
41
+ const prefixedTables = await listPrefixedSenatTables(dataset);
42
+ const expectedTableSet = new Set(expectedTables);
43
+ const senatTableSet = new Set(prefixedTables);
44
+ for (const tableName of expectedTables) {
45
+ if (!senatTableSet.has(tableName)) {
46
+ issues.push(`${senatSchemaName}.${tableName}: missing prefixed senat table`);
47
+ }
48
+ }
49
+ for (const tableName of prefixedTables) {
50
+ if (!expectedTableSet.has(tableName)) {
51
+ issues.push(`${senatSchemaName}.${tableName}: unexpected prefixed senat table`);
52
+ }
53
+ }
54
+ for (const tableName of expectedTables) {
55
+ if (!senatTableSet.has(tableName)) {
56
+ continue;
57
+ }
58
+ const senatColumns = await listColumns(senatSchemaName, tableName);
59
+ if (senatColumns.length === 0) {
60
+ issues.push(`${senatSchemaName}.${tableName}: no columns found for prefixed senat table`);
61
+ }
62
+ }
63
+ if (expectedTables.length === 0) {
64
+ issues.push(`${dataset.database}: no expected prefixed tables found in generated definitions`);
65
+ }
66
+ if (prefixedTables.length === 0) {
67
+ issues.push(`${dataset.database}: no prefixed senat tables found`);
68
+ }
69
+ return issues;
70
+ }
71
+ async function validatePrefixedTables() {
72
+ const enabledDatasets = getEnabledDatasets(options.categories);
73
+ const chosenDatasets = getChosenDatasets(enabledDatasets);
74
+ let hasErrors = false;
75
+ for (const dataset of chosenDatasets) {
76
+ const issues = await validateDataset(dataset);
77
+ if (issues.length === 0) {
78
+ if (!options.silent) {
79
+ console.log(`OK ${dataset.database}`);
80
+ }
81
+ continue;
82
+ }
83
+ hasErrors = true;
84
+ console.error(`Validation failed for ${dataset.database}:`);
85
+ for (const issue of issues) {
86
+ console.error(`- ${issue}`);
87
+ }
88
+ }
89
+ await sql.end();
90
+ if (hasErrors) {
91
+ process.exit(1);
92
+ }
93
+ if (!options.silent) {
94
+ console.log("All selected datasets passed prefixed-table validation.");
95
+ }
96
+ }
97
+ validatePrefixedTables().catch(async (error) => {
98
+ await sql.end();
99
+ console.error(error);
100
+ process.exit(1);
101
+ });
@@ -1,5 +1,5 @@
1
- import { ses as Ses, sub as Sub, subFields, txt_ameli } from "../raw_types_schemats/ameli";
2
- export type { Ses, Sub };
3
- export interface TxtAmeli extends txt_ameli {
4
- subids?: subFields.id[];
1
+ import { TxtAmeli } from "../raw_types/ameli";
2
+ import { Sub } from "../raw_types/ameli";
3
+ export interface TxtAmeliCustom extends TxtAmeli {
4
+ subids?: Sub["id"][];
5
5
  }
@@ -1,2 +1,2 @@
1
- import { debats as Debat, lecassdeb as LecAssDeb } from "../raw_types_schemats/debats";
2
- export type { Debat, LecAssDeb };
1
+ import { Debats, Lecassdeb } from "../raw_types/debats";
2
+ export type { Debats as Debat, Lecassdeb as LecAssDeb };