@tricoteuses/senat 2.22.16 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +168 -0
- package/lib/aggregates.d.ts +52 -0
- package/lib/aggregates.js +930 -0
- package/lib/aggregates.mjs +713 -0
- package/lib/aggregates.ts +833 -0
- package/lib/config.d.ts +10 -0
- package/lib/config.js +16 -0
- package/lib/config.mjs +16 -0
- package/lib/config.ts +26 -0
- package/lib/databases.d.ts +2 -0
- package/lib/databases.js +26 -0
- package/lib/databases.mjs +57 -0
- package/lib/databases.ts +71 -0
- package/lib/datasets.d.ts +34 -0
- package/lib/datasets.js +233 -0
- package/lib/datasets.mjs +78 -0
- package/lib/datasets.ts +118 -0
- package/lib/fields.d.ts +10 -0
- package/lib/fields.js +68 -0
- package/lib/fields.mjs +22 -0
- package/lib/fields.ts +29 -0
- package/lib/git.d.ts +26 -0
- package/lib/git.js +167 -0
- package/lib/index.d.ts +13 -0
- package/lib/index.js +1 -0
- package/lib/index.mjs +7 -0
- package/lib/index.ts +64 -0
- package/lib/inserters.d.ts +98 -0
- package/lib/inserters.js +500 -0
- package/lib/inserters.mjs +360 -0
- package/lib/inserters.ts +521 -0
- package/lib/legislatures.json +38 -0
- package/lib/loaders.d.ts +58 -0
- package/lib/loaders.js +286 -0
- package/lib/loaders.mjs +158 -0
- package/lib/loaders.ts +271 -0
- package/lib/model/agenda.d.ts +6 -0
- package/lib/model/agenda.js +148 -0
- package/lib/model/ameli.d.ts +51 -0
- package/lib/model/ameli.js +149 -0
- package/lib/model/ameli.mjs +84 -0
- package/lib/model/ameli.ts +100 -0
- package/lib/model/commission.d.ts +18 -0
- package/lib/model/commission.js +269 -0
- package/lib/model/debats.d.ts +67 -0
- package/lib/model/debats.js +95 -0
- package/lib/model/debats.mjs +43 -0
- package/lib/model/debats.ts +68 -0
- package/lib/model/documents.d.ts +12 -0
- package/lib/model/documents.js +151 -0
- package/lib/model/dosleg.d.ts +7 -0
- package/lib/model/dosleg.js +326 -0
- package/lib/model/dosleg.mjs +196 -0
- package/lib/model/dosleg.ts +240 -0
- package/lib/model/index.d.ts +7 -0
- package/lib/model/index.js +7 -0
- package/lib/model/index.mjs +5 -0
- package/lib/model/index.ts +15 -0
- package/lib/model/questions.d.ts +45 -0
- package/lib/model/questions.js +89 -0
- package/lib/model/questions.mjs +71 -0
- package/lib/model/questions.ts +93 -0
- package/lib/model/scrutins.d.ts +13 -0
- package/lib/model/scrutins.js +114 -0
- package/lib/model/seance.d.ts +3 -0
- package/lib/model/seance.js +267 -0
- package/lib/model/sens.d.ts +146 -0
- package/lib/model/sens.js +454 -0
- package/lib/model/sens.mjs +415 -0
- package/lib/model/sens.ts +516 -0
- package/lib/model/texte.d.ts +7 -0
- package/lib/model/texte.js +256 -0
- package/lib/model/texte.mjs +208 -0
- package/lib/model/texte.ts +229 -0
- package/lib/model/util.d.ts +9 -0
- package/lib/model/util.js +38 -0
- package/lib/model/util.mjs +19 -0
- package/lib/model/util.ts +32 -0
- package/lib/parsers/texte.d.ts +7 -0
- package/lib/parsers/texte.js +228 -0
- package/lib/raw_types/ameli.d.ts +914 -0
- package/lib/raw_types/ameli.js +5 -0
- package/lib/raw_types/ameli.mjs +163 -0
- package/lib/raw_types/debats.d.ts +207 -0
- package/lib/raw_types/debats.js +5 -0
- package/lib/raw_types/debats.mjs +58 -0
- package/lib/raw_types/dosleg.d.ts +1619 -0
- package/lib/raw_types/dosleg.js +5 -0
- package/lib/raw_types/dosleg.mjs +438 -0
- package/lib/raw_types/questions.d.ts +419 -0
- package/lib/raw_types/questions.js +5 -0
- package/lib/raw_types/questions.mjs +11 -0
- package/lib/raw_types/senat.d.ts +11368 -0
- package/lib/raw_types/senat.js +5 -0
- package/lib/raw_types/sens.d.ts +8248 -0
- package/lib/raw_types/sens.js +5 -0
- package/lib/raw_types/sens.mjs +508 -0
- package/lib/raw_types_kysely/ameli.d.ts +915 -0
- package/lib/raw_types_kysely/ameli.js +7 -0
- package/lib/raw_types_kysely/ameli.mjs +5 -0
- package/lib/raw_types_kysely/ameli.ts +951 -0
- package/lib/raw_types_kysely/debats.d.ts +207 -0
- package/lib/raw_types_kysely/debats.js +7 -0
- package/lib/raw_types_kysely/debats.mjs +5 -0
- package/lib/raw_types_kysely/debats.ts +222 -0
- package/lib/raw_types_kysely/dosleg.d.ts +3532 -0
- package/lib/raw_types_kysely/dosleg.js +7 -0
- package/lib/raw_types_kysely/dosleg.mjs +5 -0
- package/lib/raw_types_kysely/dosleg.ts +3621 -0
- package/lib/raw_types_kysely/questions.d.ts +414 -0
- package/lib/raw_types_kysely/questions.js +7 -0
- package/lib/raw_types_kysely/questions.mjs +5 -0
- package/lib/raw_types_kysely/questions.ts +426 -0
- package/lib/raw_types_kysely/sens.d.ts +4394 -0
- package/lib/raw_types_kysely/sens.js +7 -0
- package/lib/raw_types_kysely/sens.mjs +5 -0
- package/lib/raw_types_kysely/sens.ts +4499 -0
- package/lib/raw_types_schemats/ameli.d.ts +539 -0
- package/lib/raw_types_schemats/ameli.js +2 -0
- package/lib/raw_types_schemats/ameli.mjs +2 -0
- package/lib/raw_types_schemats/ameli.ts +601 -0
- package/lib/raw_types_schemats/debats.d.ts +127 -0
- package/lib/raw_types_schemats/debats.js +2 -0
- package/lib/raw_types_schemats/debats.mjs +2 -0
- package/lib/raw_types_schemats/debats.ts +145 -0
- package/lib/raw_types_schemats/dosleg.d.ts +977 -0
- package/lib/raw_types_schemats/dosleg.js +2 -0
- package/lib/raw_types_schemats/dosleg.mjs +2 -0
- package/lib/raw_types_schemats/dosleg.ts +2193 -0
- package/lib/raw_types_schemats/questions.d.ts +235 -0
- package/lib/raw_types_schemats/questions.js +2 -0
- package/lib/raw_types_schemats/questions.mjs +2 -0
- package/lib/raw_types_schemats/questions.ts +249 -0
- package/lib/raw_types_schemats/sens.d.ts +6915 -0
- package/lib/raw_types_schemats/sens.js +2 -0
- package/lib/raw_types_schemats/sens.mjs +2 -0
- package/lib/raw_types_schemats/sens.ts +2907 -0
- package/lib/scripts/convert_data.d.ts +1 -0
- package/lib/scripts/convert_data.js +354 -0
- package/lib/scripts/convert_data.mjs +181 -0
- package/lib/scripts/convert_data.ts +243 -0
- package/lib/scripts/data-download.d.ts +1 -0
- package/lib/scripts/data-download.js +12 -0
- package/lib/scripts/datautil.d.ts +8 -0
- package/lib/scripts/datautil.js +34 -0
- package/lib/scripts/datautil.mjs +16 -0
- package/lib/scripts/datautil.ts +19 -0
- package/lib/scripts/images/transparent_150x192.jpg +0 -0
- package/lib/scripts/images/transparent_155x225.jpg +0 -0
- package/lib/scripts/parse_textes.d.ts +1 -0
- package/lib/scripts/parse_textes.js +44 -0
- package/lib/scripts/parse_textes.mjs +46 -0
- package/lib/scripts/parse_textes.ts +65 -0
- package/lib/scripts/retrieve_agenda.d.ts +1 -0
- package/lib/scripts/retrieve_agenda.js +132 -0
- package/lib/scripts/retrieve_cr_commission.d.ts +1 -0
- package/lib/scripts/retrieve_cr_commission.js +364 -0
- package/lib/scripts/retrieve_cr_seance.d.ts +6 -0
- package/lib/scripts/retrieve_cr_seance.js +347 -0
- package/lib/scripts/retrieve_documents.d.ts +3 -0
- package/lib/scripts/retrieve_documents.js +219 -0
- package/lib/scripts/retrieve_documents.mjs +249 -0
- package/lib/scripts/retrieve_documents.ts +298 -0
- package/lib/scripts/retrieve_open_data.d.ts +1 -0
- package/lib/scripts/retrieve_open_data.js +315 -0
- package/lib/scripts/retrieve_open_data.mjs +217 -0
- package/lib/scripts/retrieve_open_data.ts +268 -0
- package/lib/scripts/retrieve_senateurs_photos.d.ts +1 -0
- package/lib/scripts/retrieve_senateurs_photos.js +147 -0
- package/lib/scripts/retrieve_senateurs_photos.mjs +147 -0
- package/lib/scripts/retrieve_senateurs_photos.ts +177 -0
- package/lib/scripts/retrieve_videos.d.ts +1 -0
- package/lib/scripts/retrieve_videos.js +461 -0
- package/lib/scripts/shared/cli_helpers.d.ts +95 -0
- package/lib/scripts/shared/cli_helpers.js +91 -0
- package/lib/scripts/shared/cli_helpers.ts +36 -0
- package/lib/scripts/shared/util.d.ts +4 -0
- package/lib/scripts/shared/util.js +35 -0
- package/lib/scripts/shared/util.ts +33 -0
- package/lib/scripts/test_iter_load.d.ts +1 -0
- package/lib/scripts/test_iter_load.js +12 -0
- package/lib/src/config.d.ts +22 -0
- package/lib/src/config.js +17 -7
- package/lib/src/conversion_textes.js +5 -1
- package/lib/src/databases.d.ts +2 -1
- package/lib/src/databases_postgres.d.ts +4 -0
- package/lib/src/databases_postgres.js +23 -0
- package/lib/src/datasets.d.ts +4 -0
- package/lib/src/datasets.js +16 -2
- package/lib/src/git.d.ts +1 -0
- package/lib/src/git.js +45 -11
- package/lib/src/index.d.ts +19 -8
- package/lib/src/index.js +6 -1
- package/lib/src/loaders.js +10 -4
- package/lib/src/model/agenda.js +2 -2
- package/lib/src/model/ameli.d.ts +64 -52
- package/lib/src/model/ameli.js +147 -145
- package/lib/src/model/ameli_postgres.d.ts +67 -0
- package/lib/src/model/ameli_postgres.js +150 -0
- package/lib/src/model/commission.d.ts +3 -2
- package/lib/src/model/commission.js +2 -2
- package/lib/src/model/debats.d.ts +38 -66
- package/lib/src/model/debats.js +110 -93
- package/lib/src/model/documents.d.ts +32 -12
- package/lib/src/model/documents.js +171 -130
- package/lib/src/model/dosleg.d.ts +142 -5
- package/lib/src/model/dosleg.js +298 -156
- package/lib/src/model/questions.d.ts +54 -45
- package/lib/src/model/questions.js +89 -87
- package/lib/src/model/scrutins.d.ts +48 -13
- package/lib/src/model/scrutins.js +118 -111
- package/lib/src/model/seance.js +3 -3
- package/lib/src/model/sens.d.ts +109 -179
- package/lib/src/model/sens.js +384 -484
- package/lib/src/model/util.d.ts +0 -8
- package/lib/src/model/util.js +0 -23
- package/lib/src/parsers/texte.js +7 -7
- package/lib/src/raw_types/ameli.d.ts +1651 -803
- package/lib/src/raw_types/ameli.js +1816 -5
- package/lib/src/raw_types/debats.d.ts +353 -180
- package/lib/src/raw_types/debats.js +517 -5
- package/lib/src/raw_types/dosleg.d.ts +2862 -1527
- package/lib/src/raw_types/dosleg.js +4354 -5
- package/lib/src/raw_types/questions.d.ts +671 -395
- package/lib/src/raw_types/questions.js +1303 -5
- package/lib/src/raw_types/sens.d.ts +7743 -8148
- package/lib/src/raw_types/sens.js +10429 -5
- package/lib/src/raw_types_schemats/ameli.d.ts +4 -2
- package/lib/src/raw_types_schemats/debats.d.ts +2 -2
- package/lib/src/raw_types_schemats/dosleg.d.ts +2 -2
- package/lib/src/raw_types_schemats/questions.d.ts +2 -2
- package/lib/src/raw_types_schemats/sens.d.ts +10 -4216
- package/lib/src/scripts/convert_data.js +7 -6
- package/lib/src/scripts/convert_xml_to_html.js +2 -2
- package/lib/src/scripts/data-download.js +3 -2
- package/lib/src/scripts/retrieve_agenda.js +21 -9
- package/lib/src/scripts/retrieve_cr_commission.js +17 -17
- package/lib/src/scripts/retrieve_cr_seance.d.ts +14 -1
- package/lib/src/scripts/retrieve_cr_seance.js +10 -11
- package/lib/src/scripts/retrieve_documents.d.ts +11 -2
- package/lib/src/scripts/retrieve_documents.js +25 -14
- package/lib/src/scripts/retrieve_open_data.js +514 -153
- package/lib/src/scripts/retrieve_senateurs_photos.js +25 -11
- package/lib/src/scripts/retrieve_videos.js +12 -11
- package/lib/src/scripts/shared/cli_helpers.d.ts +1 -6
- package/lib/src/scripts/shared/cli_helpers.js +9 -8
- package/lib/src/scripts/shared/incremental_import_sql.d.ts +2 -0
- package/lib/src/scripts/shared/incremental_import_sql.js +894 -0
- package/lib/src/scripts/shared/prefixed_tables.d.ts +10 -0
- package/lib/src/scripts/shared/prefixed_tables.js +36 -0
- package/lib/src/scripts/shared/schema_version.d.ts +3 -0
- package/lib/src/scripts/shared/schema_version.js +97 -0
- package/lib/src/scripts/shared/staging_import.d.ts +3 -0
- package/lib/src/scripts/shared/staging_import.js +80 -0
- package/lib/src/scripts/shared/staging_metadata_sql.d.ts +1 -0
- package/lib/src/scripts/shared/staging_metadata_sql.js +221 -0
- package/lib/src/scripts/validate_prefixed_tables.d.ts +1 -0
- package/lib/src/scripts/validate_prefixed_tables.js +101 -0
- package/lib/src/types/ameli.d.ts +4 -4
- package/lib/src/types/debats.d.ts +2 -2
- package/lib/src/types/dosleg.d.ts +39 -39
- package/lib/src/types/questions.d.ts +2 -2
- package/lib/src/types/sens.d.ts +0 -2
- package/lib/src/types/texte.d.ts +1 -1
- package/lib/src/utils/cr_spliting.d.ts +9 -6
- package/lib/src/utils/cr_spliting.js +6 -101
- package/lib/src/utils/reunion_odj_building.d.ts +7 -3
- package/lib/src/utils/reunion_parsing.d.ts +2 -1
- package/lib/src/utils/reunion_parsing.js +2 -2
- package/lib/src/videos/match.js +8 -5
- package/lib/src/videos/pipeline.d.ts +6 -2
- package/lib/src/videos/pipeline.js +21 -8
- package/lib/src/videos/search.js +6 -2
- package/lib/strings.d.ts +1 -0
- package/lib/strings.js +18 -0
- package/lib/strings.mjs +18 -0
- package/lib/strings.ts +26 -0
- package/lib/tests/incrementalImportSql.test.d.ts +1 -0
- package/lib/tests/incrementalImportSql.test.js +155 -0
- package/lib/tests/prefixedTables.test.d.ts +1 -0
- package/lib/tests/prefixedTables.test.js +22 -0
- package/lib/tests/schemaVersion.test.d.ts +1 -0
- package/lib/tests/schemaVersion.test.js +23 -0
- package/lib/tests/validatePrefixedTables.test.d.ts +1 -0
- package/lib/tests/validatePrefixedTables.test.js +14 -0
- package/lib/types/agenda.d.ts +44 -0
- package/lib/types/agenda.js +1 -0
- package/lib/types/ameli.d.ts +5 -0
- package/lib/types/ameli.js +1 -0
- package/lib/types/ameli.mjs +13 -0
- package/lib/types/ameli.ts +21 -0
- package/lib/types/compte_rendu.d.ts +83 -0
- package/lib/types/compte_rendu.js +1 -0
- package/lib/types/debats.d.ts +2 -0
- package/lib/types/debats.js +1 -0
- package/lib/types/debats.mjs +2 -0
- package/lib/types/debats.ts +6 -0
- package/lib/types/dosleg.d.ts +70 -0
- package/lib/types/dosleg.js +1 -0
- package/lib/types/dosleg.mjs +151 -0
- package/lib/types/dosleg.ts +284 -0
- package/lib/types/questions.d.ts +2 -0
- package/lib/types/questions.js +1 -0
- package/lib/types/questions.mjs +1 -0
- package/lib/types/questions.ts +3 -0
- package/lib/types/sens.d.ts +10 -0
- package/lib/types/sens.js +1 -0
- package/lib/types/sens.mjs +1 -0
- package/lib/types/sens.ts +12 -0
- package/lib/types/sessions.d.ts +5 -0
- package/lib/types/sessions.js +84 -0
- package/lib/types/sessions.mjs +43 -0
- package/lib/types/sessions.ts +42 -0
- package/lib/types/texte.d.ts +74 -0
- package/lib/types/texte.js +16 -0
- package/lib/types/texte.mjs +16 -0
- package/lib/types/texte.ts +76 -0
- package/lib/typings/windows-1252.d.js +2 -0
- package/lib/typings/windows-1252.d.mjs +2 -0
- package/lib/typings/windows-1252.d.ts +11 -0
- package/lib/utils/cr_spliting.d.ts +28 -0
- package/lib/utils/cr_spliting.js +265 -0
- package/lib/utils/date.d.ts +10 -0
- package/lib/utils/date.js +100 -0
- package/lib/utils/nvs-timecode.d.ts +7 -0
- package/lib/utils/nvs-timecode.js +79 -0
- package/lib/utils/reunion_grouping.d.ts +9 -0
- package/lib/utils/reunion_grouping.js +361 -0
- package/lib/utils/reunion_odj_building.d.ts +5 -0
- package/lib/utils/reunion_odj_building.js +154 -0
- package/lib/utils/reunion_parsing.d.ts +23 -0
- package/lib/utils/reunion_parsing.js +209 -0
- package/lib/utils/scoring.d.ts +14 -0
- package/lib/utils/scoring.js +147 -0
- package/lib/utils/string_cleaning.d.ts +7 -0
- package/lib/utils/string_cleaning.js +57 -0
- package/lib/validators/config.d.ts +9 -0
- package/lib/validators/config.js +10 -0
- package/lib/validators/config.mjs +54 -0
- package/lib/validators/config.ts +79 -0
- package/lib/validators/senat.d.ts +0 -0
- package/lib/validators/senat.js +28 -0
- package/lib/validators/senat.mjs +24 -0
- package/lib/validators/senat.ts +26 -0
- package/package.json +11 -11
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export declare const senatSchemaName = "senat";
|
|
2
|
+
export declare const rawTypesDir: string;
|
|
3
|
+
export declare function prefixedName(datasetName: string, relationName: string): string;
|
|
4
|
+
export declare function stagingSchemaName(datasetName: string): string;
|
|
5
|
+
export declare function escapeRegExp(text: string): string;
|
|
6
|
+
export declare function stripDatasetPrefix(relationName: string, datasetName: string): string;
|
|
7
|
+
export declare function getGeneratedDefinitionPath(datasetName: string): string;
|
|
8
|
+
export declare function getGeneratedTableManifestPath(datasetName: string): string;
|
|
9
|
+
export declare function buildGeneratedTableManifest(datasetName: string, prefixedTables: string[]): string;
|
|
10
|
+
export declare function extractPrefixedTableNamesFromGeneratedManifest(manifest: string, datasetName: string): string[];
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import path from "path";
|
|
2
|
+
export const senatSchemaName = "senat";
|
|
3
|
+
export const rawTypesDir = path.resolve("src", "raw_types");
|
|
4
|
+
export function prefixedName(datasetName, relationName) {
|
|
5
|
+
return `${datasetName}_${relationName}`;
|
|
6
|
+
}
|
|
7
|
+
export function stagingSchemaName(datasetName) {
|
|
8
|
+
return `${datasetName}_staging`;
|
|
9
|
+
}
|
|
10
|
+
export function escapeRegExp(text) {
|
|
11
|
+
return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
12
|
+
}
|
|
13
|
+
export function stripDatasetPrefix(relationName, datasetName) {
|
|
14
|
+
const datasetPrefix = `${datasetName}_`;
|
|
15
|
+
return relationName.startsWith(datasetPrefix) ? relationName.slice(datasetPrefix.length) : relationName;
|
|
16
|
+
}
|
|
17
|
+
export function getGeneratedDefinitionPath(datasetName) {
|
|
18
|
+
return path.join(rawTypesDir, `${datasetName}.ts`);
|
|
19
|
+
}
|
|
20
|
+
export function getGeneratedTableManifestPath(datasetName) {
|
|
21
|
+
return path.join(rawTypesDir, `${datasetName}.tables.json`);
|
|
22
|
+
}
|
|
23
|
+
export function buildGeneratedTableManifest(datasetName, prefixedTables) {
|
|
24
|
+
const manifest = {
|
|
25
|
+
dataset: datasetName,
|
|
26
|
+
prefixedTables: [...new Set(prefixedTables)],
|
|
27
|
+
};
|
|
28
|
+
return JSON.stringify(manifest, null, 2) + "\n";
|
|
29
|
+
}
|
|
30
|
+
export function extractPrefixedTableNamesFromGeneratedManifest(manifest, datasetName) {
|
|
31
|
+
const parsed = JSON.parse(manifest);
|
|
32
|
+
if (parsed.dataset !== datasetName || !Array.isArray(parsed.prefixedTables)) {
|
|
33
|
+
return [];
|
|
34
|
+
}
|
|
35
|
+
return [...new Set(parsed.prefixedTables.filter((tableName) => typeof tableName === "string"))];
|
|
36
|
+
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
function escapeSqlLiteral(value) {
|
|
2
|
+
return value.replace(/'/g, "''");
|
|
3
|
+
}
|
|
4
|
+
export function buildEnsureSchemaVersionTableSql(schemaName) {
|
|
5
|
+
return `
|
|
6
|
+
CREATE TABLE IF NOT EXISTS ${schemaName}.version(
|
|
7
|
+
number integer PRIMARY KEY
|
|
8
|
+
);
|
|
9
|
+
COMMENT ON TABLE ${schemaName}.version IS 'version of database';
|
|
10
|
+
COMMENT ON COLUMN ${schemaName}.version.number IS 'version number of database schema';
|
|
11
|
+
INSERT INTO ${schemaName}.version(number)
|
|
12
|
+
SELECT 0
|
|
13
|
+
WHERE NOT EXISTS (SELECT 1 FROM ${schemaName}.version);
|
|
14
|
+
`;
|
|
15
|
+
}
|
|
16
|
+
export function buildSchemaStructureFingerprintQuery(schemaName) {
|
|
17
|
+
const escapedSchemaName = escapeSqlLiteral(schemaName);
|
|
18
|
+
return `
|
|
19
|
+
WITH objects AS (
|
|
20
|
+
SELECT format('table:%s', table_class.relname) AS entry
|
|
21
|
+
FROM pg_class table_class
|
|
22
|
+
JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
|
|
23
|
+
WHERE table_namespace.nspname = '${escapedSchemaName}'
|
|
24
|
+
AND table_class.relkind IN ('r', 'p')
|
|
25
|
+
AND table_class.relname <> 'version'
|
|
26
|
+
|
|
27
|
+
UNION ALL
|
|
28
|
+
|
|
29
|
+
SELECT format(
|
|
30
|
+
'column:%s:%s:%s:%s:%s:%s:%s:%s',
|
|
31
|
+
columns.table_name,
|
|
32
|
+
columns.ordinal_position,
|
|
33
|
+
columns.column_name,
|
|
34
|
+
columns.udt_schema,
|
|
35
|
+
columns.udt_name,
|
|
36
|
+
columns.is_nullable,
|
|
37
|
+
coalesce(columns.column_default, ''),
|
|
38
|
+
columns.is_identity
|
|
39
|
+
) AS entry
|
|
40
|
+
FROM information_schema.columns columns
|
|
41
|
+
WHERE columns.table_schema = '${escapedSchemaName}'
|
|
42
|
+
AND columns.table_name <> 'version'
|
|
43
|
+
|
|
44
|
+
UNION ALL
|
|
45
|
+
|
|
46
|
+
SELECT format(
|
|
47
|
+
'constraint:%s:%s:%s:%s',
|
|
48
|
+
table_class.relname,
|
|
49
|
+
constraint_data.contype,
|
|
50
|
+
constraint_data.conname,
|
|
51
|
+
pg_get_constraintdef(constraint_data.oid, true)
|
|
52
|
+
) AS entry
|
|
53
|
+
FROM pg_constraint constraint_data
|
|
54
|
+
JOIN pg_class table_class ON table_class.oid = constraint_data.conrelid
|
|
55
|
+
JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
|
|
56
|
+
WHERE table_namespace.nspname = '${escapedSchemaName}'
|
|
57
|
+
AND table_class.relname <> 'version'
|
|
58
|
+
AND constraint_data.contype <> 'n'
|
|
59
|
+
|
|
60
|
+
UNION ALL
|
|
61
|
+
|
|
62
|
+
SELECT format(
|
|
63
|
+
'index:%s:%s',
|
|
64
|
+
table_class.relname,
|
|
65
|
+
pg_get_indexdef(idx.indexrelid)
|
|
66
|
+
) AS entry
|
|
67
|
+
FROM pg_index idx
|
|
68
|
+
JOIN pg_class table_class ON table_class.oid = idx.indrelid
|
|
69
|
+
JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
|
|
70
|
+
LEFT JOIN pg_constraint constraint_data ON constraint_data.conindid = idx.indexrelid
|
|
71
|
+
WHERE table_namespace.nspname = '${escapedSchemaName}'
|
|
72
|
+
AND table_class.relname <> 'version'
|
|
73
|
+
AND constraint_data.oid IS NULL
|
|
74
|
+
|
|
75
|
+
UNION ALL
|
|
76
|
+
|
|
77
|
+
SELECT format(
|
|
78
|
+
'sequence:%s:%s:%s:%s:%s:%s:%s:%s',
|
|
79
|
+
sequences.sequencename,
|
|
80
|
+
sequences.data_type,
|
|
81
|
+
sequences.start_value,
|
|
82
|
+
sequences.min_value,
|
|
83
|
+
sequences.max_value,
|
|
84
|
+
sequences.increment_by,
|
|
85
|
+
sequences.cycle,
|
|
86
|
+
sequences.cache_size
|
|
87
|
+
) AS entry
|
|
88
|
+
FROM pg_sequences sequences
|
|
89
|
+
WHERE sequences.schemaname = '${escapedSchemaName}'
|
|
90
|
+
)
|
|
91
|
+
SELECT coalesce(md5(string_agg(entry, E'\\n' ORDER BY entry)), '')
|
|
92
|
+
FROM objects
|
|
93
|
+
`;
|
|
94
|
+
}
|
|
95
|
+
export function buildIncrementSchemaVersionSql(schemaName) {
|
|
96
|
+
return `UPDATE ${schemaName}.version SET number = number + 1`;
|
|
97
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
function sourceSchemaNames(dataset) {
|
|
2
|
+
if (dataset.database === "sens") {
|
|
3
|
+
return ["sens", "senat"];
|
|
4
|
+
}
|
|
5
|
+
return [dataset.database];
|
|
6
|
+
}
|
|
7
|
+
function sourceSchemaAlternation(dataset) {
|
|
8
|
+
return sourceSchemaNames(dataset).join("|");
|
|
9
|
+
}
|
|
10
|
+
export function isCopyFromStdinLine(line) {
|
|
11
|
+
return /^COPY .* FROM stdin;$/i.test(line);
|
|
12
|
+
}
|
|
13
|
+
export function rewriteLineForStagingImport(line, dataset, stagingSchema, inCopyData = false) {
|
|
14
|
+
if (inCopyData) {
|
|
15
|
+
return line;
|
|
16
|
+
}
|
|
17
|
+
const sourceSchemas = sourceSchemaNames(dataset);
|
|
18
|
+
const sourceSchemaPattern = sourceSchemaAlternation(dataset);
|
|
19
|
+
const searchPathRegex = new RegExp(`^SET search_path = (public|${sourceSchemaPattern}), pg_catalog;$`, "i");
|
|
20
|
+
if (searchPathRegex.test(line)) {
|
|
21
|
+
return `SET search_path = ${stagingSchema}, pg_catalog;`;
|
|
22
|
+
}
|
|
23
|
+
const alterOwnerRegex = /^ALTER (SCHEMA|TABLE|SEQUENCE|INDEX|VIEW) .* OWNER TO /i;
|
|
24
|
+
if (alterOwnerRegex.test(line)) {
|
|
25
|
+
return "-- skipped OWNER TO during staging import";
|
|
26
|
+
}
|
|
27
|
+
if (/^(GRANT|REVOKE) /i.test(line)) {
|
|
28
|
+
return "-- skipped GRANT/REVOKE during staging import";
|
|
29
|
+
}
|
|
30
|
+
if (/^ALTER TABLE ONLY .* DROP CONSTRAINT /i.test(line)) {
|
|
31
|
+
return "-- skipped redundant DROP CONSTRAINT during staging import";
|
|
32
|
+
}
|
|
33
|
+
if (/^DROP INDEX /i.test(line)) {
|
|
34
|
+
return "-- skipped redundant DROP INDEX during staging import";
|
|
35
|
+
}
|
|
36
|
+
if (/^DROP TABLE /i.test(line)) {
|
|
37
|
+
return "-- skipped redundant DROP TABLE during staging import";
|
|
38
|
+
}
|
|
39
|
+
if (/^DROP SCHEMA public;$/i.test(line)) {
|
|
40
|
+
return "-- skipped redundant DROP SCHEMA public during staging import";
|
|
41
|
+
}
|
|
42
|
+
if (/^CREATE SCHEMA public;$/i.test(line)) {
|
|
43
|
+
return "-- skipped redundant CREATE SCHEMA public during staging import";
|
|
44
|
+
}
|
|
45
|
+
if (/^COMMENT ON SCHEMA public IS /i.test(line)) {
|
|
46
|
+
return "-- skipped redundant COMMENT ON SCHEMA public during staging import";
|
|
47
|
+
}
|
|
48
|
+
if (/^CREATE TABLE /i.test(line)) {
|
|
49
|
+
line = line.replace(/^CREATE TABLE /i, "CREATE UNLOGGED TABLE ");
|
|
50
|
+
}
|
|
51
|
+
if (/^CREATE SEQUENCE /i.test(line)) {
|
|
52
|
+
line = line.replace(/^CREATE SEQUENCE /i, "CREATE UNLOGGED SEQUENCE ");
|
|
53
|
+
}
|
|
54
|
+
const createSchemaRegex = new RegExp(`^CREATE SCHEMA (${sourceSchemaPattern});$`, "i");
|
|
55
|
+
if (createSchemaRegex.test(line)) {
|
|
56
|
+
return `CREATE SCHEMA IF NOT EXISTS ${stagingSchema};`;
|
|
57
|
+
}
|
|
58
|
+
// In dumps like debats, "COMMENT ON COLUMN debats.datsea" means table.column,
|
|
59
|
+
// not schema.table.column. Keep these statements untouched and let search_path resolve them.
|
|
60
|
+
const commentOnColumnRegex = /^COMMENT ON COLUMN ([A-Za-z_][A-Za-z0-9_]*)\.([A-Za-z_][A-Za-z0-9_]*) IS /i;
|
|
61
|
+
const commentOnColumnMatch = line.match(commentOnColumnRegex);
|
|
62
|
+
if (commentOnColumnMatch && sourceSchemas.includes(commentOnColumnMatch[1])) {
|
|
63
|
+
return line;
|
|
64
|
+
}
|
|
65
|
+
const parts = line.split(/(')/);
|
|
66
|
+
let inString = false;
|
|
67
|
+
for (let i = 0; i < parts.length; i++) {
|
|
68
|
+
if (parts[i] === "'") {
|
|
69
|
+
inString = !inString;
|
|
70
|
+
}
|
|
71
|
+
else if (!inString) {
|
|
72
|
+
let rewrittenPart = parts[i].replace(/\bpublic\./g, `${stagingSchema}.`);
|
|
73
|
+
for (const sourceSchema of sourceSchemas) {
|
|
74
|
+
rewrittenPart = rewrittenPart.replace(new RegExp(`\\b${sourceSchema}\\.`, "g"), `${stagingSchema}.`);
|
|
75
|
+
}
|
|
76
|
+
parts[i] = rewrittenPart;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return parts.join("");
|
|
80
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function buildExportStagingMetadataStatementsQuery(stagingSchema: string, targetSchema: string): string;
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
function escapeSqlLiteral(value) {
|
|
2
|
+
return value.replace(/'/g, "''");
|
|
3
|
+
}
|
|
4
|
+
export function buildExportStagingMetadataStatementsQuery(stagingSchema, targetSchema) {
|
|
5
|
+
const escapedStagingSchema = escapeSqlLiteral(stagingSchema);
|
|
6
|
+
return `
|
|
7
|
+
WITH ordered_statements AS (
|
|
8
|
+
SELECT
|
|
9
|
+
10 AS group_no,
|
|
10
|
+
0 AS order_no,
|
|
11
|
+
sequences.sequencename AS object_name,
|
|
12
|
+
format('CREATE SEQUENCE IF NOT EXISTS %I.%I', '${targetSchema}', sequences.sequencename) AS statement
|
|
13
|
+
FROM pg_sequences sequences
|
|
14
|
+
WHERE sequences.schemaname = '${escapedStagingSchema}'
|
|
15
|
+
|
|
16
|
+
UNION ALL
|
|
17
|
+
|
|
18
|
+
SELECT
|
|
19
|
+
20 AS group_no,
|
|
20
|
+
0 AS order_no,
|
|
21
|
+
sequences.sequencename AS object_name,
|
|
22
|
+
format(
|
|
23
|
+
'ALTER SEQUENCE %I.%I AS %s INCREMENT BY %s MINVALUE %s MAXVALUE %s START WITH %s CACHE %s %s',
|
|
24
|
+
'${targetSchema}',
|
|
25
|
+
sequences.sequencename,
|
|
26
|
+
sequences.data_type,
|
|
27
|
+
sequences.increment_by,
|
|
28
|
+
sequences.min_value,
|
|
29
|
+
sequences.max_value,
|
|
30
|
+
sequences.start_value,
|
|
31
|
+
sequences.cache_size,
|
|
32
|
+
CASE WHEN sequences.cycle THEN 'CYCLE' ELSE 'NO CYCLE' END
|
|
33
|
+
) AS statement
|
|
34
|
+
FROM pg_sequences sequences
|
|
35
|
+
WHERE sequences.schemaname = '${escapedStagingSchema}'
|
|
36
|
+
|
|
37
|
+
UNION ALL
|
|
38
|
+
|
|
39
|
+
SELECT
|
|
40
|
+
30 AS group_no,
|
|
41
|
+
0 AS order_no,
|
|
42
|
+
sequence_class.relname AS object_name,
|
|
43
|
+
CASE
|
|
44
|
+
WHEN dependent_table.relname IS NULL THEN format(
|
|
45
|
+
'ALTER SEQUENCE %I.%I OWNED BY NONE',
|
|
46
|
+
'${targetSchema}',
|
|
47
|
+
sequence_class.relname
|
|
48
|
+
)
|
|
49
|
+
ELSE format(
|
|
50
|
+
'ALTER SEQUENCE %I.%I OWNED BY %I.%I.%I',
|
|
51
|
+
'${targetSchema}',
|
|
52
|
+
sequence_class.relname,
|
|
53
|
+
'${targetSchema}',
|
|
54
|
+
dependent_table.relname,
|
|
55
|
+
dependent_column.attname
|
|
56
|
+
)
|
|
57
|
+
END AS statement
|
|
58
|
+
FROM pg_class sequence_class
|
|
59
|
+
JOIN pg_namespace sequence_namespace ON sequence_namespace.oid = sequence_class.relnamespace
|
|
60
|
+
LEFT JOIN pg_depend dependency
|
|
61
|
+
ON dependency.objid = sequence_class.oid
|
|
62
|
+
AND dependency.deptype = 'a'
|
|
63
|
+
LEFT JOIN pg_class dependent_table ON dependent_table.oid = dependency.refobjid
|
|
64
|
+
LEFT JOIN pg_namespace dependent_namespace ON dependent_namespace.oid = dependent_table.relnamespace
|
|
65
|
+
LEFT JOIN pg_attribute dependent_column
|
|
66
|
+
ON dependent_column.attrelid = dependent_table.oid
|
|
67
|
+
AND dependent_column.attnum = dependency.refobjsubid
|
|
68
|
+
WHERE sequence_namespace.nspname = '${escapedStagingSchema}'
|
|
69
|
+
AND sequence_class.relkind = 'S'
|
|
70
|
+
AND (
|
|
71
|
+
dependent_namespace.nspname IS NULL
|
|
72
|
+
OR dependent_namespace.nspname = '${escapedStagingSchema}'
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
UNION ALL
|
|
76
|
+
|
|
77
|
+
SELECT
|
|
78
|
+
40 AS group_no,
|
|
79
|
+
0 AS order_no,
|
|
80
|
+
format('%s.%s', table_class.relname, attribute.attname) AS object_name,
|
|
81
|
+
CASE
|
|
82
|
+
WHEN default_expr.expression IS NULL THEN format(
|
|
83
|
+
'ALTER TABLE %I.%I ALTER COLUMN %I DROP DEFAULT',
|
|
84
|
+
'${targetSchema}',
|
|
85
|
+
table_class.relname,
|
|
86
|
+
attribute.attname
|
|
87
|
+
)
|
|
88
|
+
ELSE format(
|
|
89
|
+
'ALTER TABLE %I.%I ALTER COLUMN %I SET DEFAULT %s',
|
|
90
|
+
'${targetSchema}',
|
|
91
|
+
table_class.relname,
|
|
92
|
+
attribute.attname,
|
|
93
|
+
replace(default_expr.expression, '${stagingSchema}.', '${targetSchema}.')
|
|
94
|
+
)
|
|
95
|
+
END AS statement
|
|
96
|
+
FROM pg_class table_class
|
|
97
|
+
JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
|
|
98
|
+
JOIN pg_attribute attribute ON attribute.attrelid = table_class.oid
|
|
99
|
+
LEFT JOIN LATERAL (
|
|
100
|
+
SELECT pg_get_expr(attr_def.adbin, attr_def.adrelid) AS expression
|
|
101
|
+
FROM pg_attrdef attr_def
|
|
102
|
+
WHERE attr_def.adrelid = attribute.attrelid
|
|
103
|
+
AND attr_def.adnum = attribute.attnum
|
|
104
|
+
) default_expr ON true
|
|
105
|
+
WHERE table_namespace.nspname = '${escapedStagingSchema}'
|
|
106
|
+
AND table_class.relkind IN ('r', 'p')
|
|
107
|
+
AND attribute.attnum > 0
|
|
108
|
+
AND NOT attribute.attisdropped
|
|
109
|
+
AND attribute.attgenerated = ''
|
|
110
|
+
AND attribute.attidentity = ''
|
|
111
|
+
|
|
112
|
+
UNION ALL
|
|
113
|
+
|
|
114
|
+
SELECT
|
|
115
|
+
50 AS group_no,
|
|
116
|
+
0 AS order_no,
|
|
117
|
+
table_class.relname AS object_name,
|
|
118
|
+
format(
|
|
119
|
+
'COMMENT ON TABLE %I.%I IS %L',
|
|
120
|
+
'${targetSchema}',
|
|
121
|
+
table_class.relname,
|
|
122
|
+
obj_description(table_class.oid, 'pg_class')
|
|
123
|
+
) AS statement
|
|
124
|
+
FROM pg_class table_class
|
|
125
|
+
JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
|
|
126
|
+
WHERE table_namespace.nspname = '${escapedStagingSchema}'
|
|
127
|
+
AND table_class.relkind IN ('r', 'p')
|
|
128
|
+
|
|
129
|
+
UNION ALL
|
|
130
|
+
|
|
131
|
+
SELECT
|
|
132
|
+
60 AS group_no,
|
|
133
|
+
0 AS order_no,
|
|
134
|
+
format('%s.%s', table_class.relname, attribute.attname) AS object_name,
|
|
135
|
+
format(
|
|
136
|
+
'COMMENT ON COLUMN %I.%I.%I IS %L',
|
|
137
|
+
'${targetSchema}',
|
|
138
|
+
table_class.relname,
|
|
139
|
+
attribute.attname,
|
|
140
|
+
col_description(table_class.oid, attribute.attnum)
|
|
141
|
+
) AS statement
|
|
142
|
+
FROM pg_class table_class
|
|
143
|
+
JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
|
|
144
|
+
JOIN pg_attribute attribute ON attribute.attrelid = table_class.oid
|
|
145
|
+
WHERE table_namespace.nspname = '${escapedStagingSchema}'
|
|
146
|
+
AND table_class.relkind IN ('r', 'p')
|
|
147
|
+
AND attribute.attnum > 0
|
|
148
|
+
AND NOT attribute.attisdropped
|
|
149
|
+
|
|
150
|
+
UNION ALL
|
|
151
|
+
|
|
152
|
+
SELECT
|
|
153
|
+
70 AS group_no,
|
|
154
|
+
CASE constraint_data.contype
|
|
155
|
+
WHEN 'p' THEN 1
|
|
156
|
+
WHEN 'u' THEN 2
|
|
157
|
+
WHEN 'c' THEN 3
|
|
158
|
+
WHEN 'f' THEN 4
|
|
159
|
+
ELSE 5
|
|
160
|
+
END AS order_no,
|
|
161
|
+
format('%s.%s', table_class.relname, constraint_data.conname) AS object_name,
|
|
162
|
+
format(
|
|
163
|
+
'ALTER TABLE %I.%I ADD CONSTRAINT %I %s',
|
|
164
|
+
'${targetSchema}',
|
|
165
|
+
table_class.relname,
|
|
166
|
+
constraint_data.conname,
|
|
167
|
+
replace(
|
|
168
|
+
replace(
|
|
169
|
+
pg_get_constraintdef(constraint_data.oid, true),
|
|
170
|
+
' REFERENCES ' || '${stagingSchema}' || '.',
|
|
171
|
+
' REFERENCES ${targetSchema}.'
|
|
172
|
+
),
|
|
173
|
+
' REFERENCES ONLY ' || '${stagingSchema}' || '.',
|
|
174
|
+
' REFERENCES ONLY ${targetSchema}.'
|
|
175
|
+
)
|
|
176
|
+
) AS statement
|
|
177
|
+
FROM pg_constraint constraint_data
|
|
178
|
+
JOIN pg_class table_class ON table_class.oid = constraint_data.conrelid
|
|
179
|
+
JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
|
|
180
|
+
WHERE table_namespace.nspname = '${escapedStagingSchema}'
|
|
181
|
+
AND constraint_data.contype <> 'n'
|
|
182
|
+
|
|
183
|
+
UNION ALL
|
|
184
|
+
|
|
185
|
+
SELECT
|
|
186
|
+
80 AS group_no,
|
|
187
|
+
0 AS order_no,
|
|
188
|
+
format('%s.%s', table_class.relname, index_class.relname) AS object_name,
|
|
189
|
+
replace(
|
|
190
|
+
pg_get_indexdef(index_data.indexrelid),
|
|
191
|
+
' ON ${stagingSchema}.',
|
|
192
|
+
' ON ${targetSchema}.'
|
|
193
|
+
) AS statement
|
|
194
|
+
FROM pg_index index_data
|
|
195
|
+
JOIN pg_class table_class ON table_class.oid = index_data.indrelid
|
|
196
|
+
JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
|
|
197
|
+
JOIN pg_class index_class ON index_class.oid = index_data.indexrelid
|
|
198
|
+
LEFT JOIN pg_constraint constraint_data ON constraint_data.conindid = index_data.indexrelid
|
|
199
|
+
WHERE table_namespace.nspname = '${escapedStagingSchema}'
|
|
200
|
+
AND constraint_data.oid IS NULL
|
|
201
|
+
|
|
202
|
+
UNION ALL
|
|
203
|
+
|
|
204
|
+
SELECT
|
|
205
|
+
90 AS group_no,
|
|
206
|
+
0 AS order_no,
|
|
207
|
+
sequences.sequencename AS object_name,
|
|
208
|
+
format(
|
|
209
|
+
'SELECT setval(%L, %s, %s)',
|
|
210
|
+
'${targetSchema}.' || sequences.sequencename,
|
|
211
|
+
coalesce(sequences.last_value, sequences.start_value),
|
|
212
|
+
CASE WHEN sequences.last_value IS NULL THEN 'false' ELSE 'true' END
|
|
213
|
+
) AS statement
|
|
214
|
+
FROM pg_sequences sequences
|
|
215
|
+
WHERE sequences.schemaname = '${escapedStagingSchema}'
|
|
216
|
+
)
|
|
217
|
+
SELECT encode(convert_to(statement, 'UTF8'), 'hex')
|
|
218
|
+
FROM ordered_statements
|
|
219
|
+
ORDER BY group_no, order_no, object_name;
|
|
220
|
+
`;
|
|
221
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import commandLineArgs from "command-line-args";
|
|
2
|
+
import fs from "fs-extra";
|
|
3
|
+
import { sql } from "../databases_postgres";
|
|
4
|
+
import { getChosenDatasets, getEnabledDatasets } from "../datasets";
|
|
5
|
+
import { categoriesOption, silentOption } from "./shared/cli_helpers";
|
|
6
|
+
import { extractPrefixedTableNamesFromGeneratedManifest, getGeneratedTableManifestPath, senatSchemaName, } from "./shared/prefixed_tables";
|
|
7
|
+
const optionsDefinitions = [categoriesOption, silentOption];
|
|
8
|
+
const options = commandLineArgs(optionsDefinitions);
|
|
9
|
+
async function listPrefixedSenatTables(dataset) {
|
|
10
|
+
const prefix = `${dataset.database}_`;
|
|
11
|
+
const rows = await sql `
|
|
12
|
+
select tablename as relation_name
|
|
13
|
+
from pg_tables
|
|
14
|
+
where schemaname = ${senatSchemaName}
|
|
15
|
+
and tablename like ${prefix + "%"}
|
|
16
|
+
order by tablename
|
|
17
|
+
`;
|
|
18
|
+
return rows.map(({ relation_name }) => relation_name);
|
|
19
|
+
}
|
|
20
|
+
function getGeneratedDefinitionPath(dataset) {
|
|
21
|
+
return getGeneratedTableManifestPath(dataset.database);
|
|
22
|
+
}
|
|
23
|
+
function listExpectedPrefixedSenatTables(dataset) {
|
|
24
|
+
const manifestFilePath = getGeneratedDefinitionPath(dataset);
|
|
25
|
+
const manifest = fs.readFileSync(manifestFilePath, { encoding: "utf8" });
|
|
26
|
+
return extractPrefixedTableNamesFromGeneratedManifest(manifest, dataset.database);
|
|
27
|
+
}
|
|
28
|
+
async function listColumns(schema, relationName) {
|
|
29
|
+
const rows = await sql `
|
|
30
|
+
select column_name
|
|
31
|
+
from information_schema.columns
|
|
32
|
+
where table_schema = ${schema}
|
|
33
|
+
and table_name = ${relationName}
|
|
34
|
+
order by ordinal_position
|
|
35
|
+
`;
|
|
36
|
+
return rows.map(({ column_name }) => column_name);
|
|
37
|
+
}
|
|
38
|
+
async function validateDataset(dataset) {
|
|
39
|
+
const issues = [];
|
|
40
|
+
const expectedTables = listExpectedPrefixedSenatTables(dataset);
|
|
41
|
+
const prefixedTables = await listPrefixedSenatTables(dataset);
|
|
42
|
+
const expectedTableSet = new Set(expectedTables);
|
|
43
|
+
const senatTableSet = new Set(prefixedTables);
|
|
44
|
+
for (const tableName of expectedTables) {
|
|
45
|
+
if (!senatTableSet.has(tableName)) {
|
|
46
|
+
issues.push(`${senatSchemaName}.${tableName}: missing prefixed senat table`);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
for (const tableName of prefixedTables) {
|
|
50
|
+
if (!expectedTableSet.has(tableName)) {
|
|
51
|
+
issues.push(`${senatSchemaName}.${tableName}: unexpected prefixed senat table`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
for (const tableName of expectedTables) {
|
|
55
|
+
if (!senatTableSet.has(tableName)) {
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
const senatColumns = await listColumns(senatSchemaName, tableName);
|
|
59
|
+
if (senatColumns.length === 0) {
|
|
60
|
+
issues.push(`${senatSchemaName}.${tableName}: no columns found for prefixed senat table`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (expectedTables.length === 0) {
|
|
64
|
+
issues.push(`${dataset.database}: no expected prefixed tables found in generated definitions`);
|
|
65
|
+
}
|
|
66
|
+
if (prefixedTables.length === 0) {
|
|
67
|
+
issues.push(`${dataset.database}: no prefixed senat tables found`);
|
|
68
|
+
}
|
|
69
|
+
return issues;
|
|
70
|
+
}
|
|
71
|
+
async function validatePrefixedTables() {
|
|
72
|
+
const enabledDatasets = getEnabledDatasets(options.categories);
|
|
73
|
+
const chosenDatasets = getChosenDatasets(enabledDatasets);
|
|
74
|
+
let hasErrors = false;
|
|
75
|
+
for (const dataset of chosenDatasets) {
|
|
76
|
+
const issues = await validateDataset(dataset);
|
|
77
|
+
if (issues.length === 0) {
|
|
78
|
+
if (!options.silent) {
|
|
79
|
+
console.log(`OK ${dataset.database}`);
|
|
80
|
+
}
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
hasErrors = true;
|
|
84
|
+
console.error(`Validation failed for ${dataset.database}:`);
|
|
85
|
+
for (const issue of issues) {
|
|
86
|
+
console.error(`- ${issue}`);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
await sql.end();
|
|
90
|
+
if (hasErrors) {
|
|
91
|
+
process.exit(1);
|
|
92
|
+
}
|
|
93
|
+
if (!options.silent) {
|
|
94
|
+
console.log("All selected datasets passed prefixed-table validation.");
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
validatePrefixedTables().catch(async (error) => {
|
|
98
|
+
await sql.end();
|
|
99
|
+
console.error(error);
|
|
100
|
+
process.exit(1);
|
|
101
|
+
});
|
package/lib/src/types/ameli.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
export interface
|
|
4
|
-
subids?:
|
|
1
|
+
import { TxtAmeli } from "../raw_types/ameli";
|
|
2
|
+
import { Sub } from "../raw_types/ameli";
|
|
3
|
+
export interface TxtAmeliCustom extends TxtAmeli {
|
|
4
|
+
subids?: Sub["id"][];
|
|
5
5
|
}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export type { Debat, LecAssDeb };
|
|
1
|
+
import { Debats, Lecassdeb } from "../raw_types/debats";
|
|
2
|
+
export type { Debats as Debat, Lecassdeb as LecAssDeb };
|