@tricoteuses/senat 3.2.0 → 3.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/LICENSE.md +32 -32
  2. package/README.md +326 -326
  3. package/lib/src/config.d.ts +43 -0
  4. package/lib/src/config.js +37 -0
  5. package/lib/src/conversion_textes.d.ts +11 -0
  6. package/lib/src/conversion_textes.js +320 -0
  7. package/lib/src/databases_postgres.d.ts +4 -0
  8. package/lib/src/databases_postgres.js +23 -0
  9. package/lib/src/datasets.d.ts +38 -0
  10. package/lib/src/datasets.js +247 -0
  11. package/lib/src/git.d.ts +27 -0
  12. package/lib/src/git.js +251 -0
  13. package/lib/src/index.d.ts +1 -1
  14. package/lib/src/loaders.d.ts +52 -0
  15. package/lib/src/loaders.js +260 -0
  16. package/lib/src/model/agenda.d.ts +6 -0
  17. package/lib/src/model/agenda.js +148 -0
  18. package/lib/src/model/ameli.d.ts +67 -0
  19. package/lib/src/model/ameli.js +150 -0
  20. package/lib/src/model/commission.d.ts +19 -0
  21. package/lib/src/model/commission.js +269 -0
  22. package/lib/src/model/debats.d.ts +39 -0
  23. package/lib/src/model/debats.js +112 -0
  24. package/lib/src/model/documents.d.ts +32 -0
  25. package/lib/src/model/documents.js +182 -0
  26. package/lib/src/model/dosleg.d.ts +144 -0
  27. package/lib/src/model/dosleg.js +468 -0
  28. package/lib/src/model/index.d.ts +7 -0
  29. package/lib/src/model/index.js +7 -0
  30. package/lib/src/model/questions.d.ts +54 -0
  31. package/lib/src/model/questions.js +91 -0
  32. package/lib/src/model/scrutins.d.ts +48 -0
  33. package/lib/src/model/scrutins.js +121 -0
  34. package/lib/src/model/seance.d.ts +3 -0
  35. package/lib/src/model/seance.js +267 -0
  36. package/lib/src/model/sens.d.ts +112 -0
  37. package/lib/src/model/sens.js +385 -0
  38. package/lib/src/model/util.d.ts +1 -0
  39. package/lib/src/model/util.js +15 -0
  40. package/lib/src/parsers/plf/amendement.d.ts +3 -0
  41. package/lib/src/parsers/plf/amendement.js +152 -0
  42. package/lib/src/raw_types/ameli.d.ts +1762 -0
  43. package/lib/src/raw_types/ameli.js +1074 -0
  44. package/lib/src/raw_types/debats.d.ts +380 -0
  45. package/lib/src/raw_types/debats.js +266 -0
  46. package/lib/src/raw_types/dosleg.d.ts +2954 -0
  47. package/lib/src/raw_types/dosleg.js +2005 -0
  48. package/lib/src/raw_types/questions.d.ts +699 -0
  49. package/lib/src/raw_types/questions.js +493 -0
  50. package/lib/src/raw_types/sens.d.ts +7843 -0
  51. package/lib/src/raw_types/sens.js +4691 -0
  52. package/lib/src/raw_types_schemats/ameli.d.ts +541 -0
  53. package/lib/src/raw_types_schemats/ameli.js +2 -0
  54. package/lib/src/raw_types_schemats/debats.d.ts +127 -0
  55. package/lib/src/raw_types_schemats/debats.js +2 -0
  56. package/lib/src/raw_types_schemats/dosleg.d.ts +977 -0
  57. package/lib/src/raw_types_schemats/dosleg.js +2 -0
  58. package/lib/src/raw_types_schemats/questions.d.ts +237 -0
  59. package/lib/src/raw_types_schemats/questions.js +2 -0
  60. package/lib/src/raw_types_schemats/sens.d.ts +2709 -0
  61. package/lib/src/raw_types_schemats/sens.js +2 -0
  62. package/lib/src/rich_types/ameli.d.ts +12 -0
  63. package/lib/src/scripts/debug_dosleg_query.d.ts +6 -0
  64. package/lib/src/scripts/debug_dosleg_query.js +50 -0
  65. package/lib/src/scripts/retrieve_collaborateurs.js +6 -3
  66. package/lib/src/scripts/retrieve_open_data.js +2 -1
  67. package/lib/src/scripts/shared/incremental_import_sql.js +866 -866
  68. package/lib/src/scripts/shared/schema_version.js +85 -84
  69. package/lib/src/scripts/shared/staging_metadata_sql.js +214 -214
  70. package/lib/src/scripts/validate_prefixed_tables.js +12 -12
  71. package/lib/src/server/ameli.js +18 -11
  72. package/lib/src/server/conversion_textes.js +106 -106
  73. package/lib/src/server/datasets.js +1 -0
  74. package/lib/src/server/debats.js +10 -10
  75. package/lib/src/server/documents.js +22 -22
  76. package/lib/src/server/dosleg.js +33 -33
  77. package/lib/src/server/questions.js +10 -10
  78. package/lib/src/server/scrutins.js +3 -3
  79. package/lib/src/server/sens.js +19 -19
  80. package/lib/src/types/agenda.d.ts +45 -0
  81. package/lib/src/types/ameli.d.ts +5 -0
  82. package/lib/src/types/ameli.js +1 -0
  83. package/lib/src/types/compte_rendu.d.ts +83 -0
  84. package/lib/src/types/compte_rendu.js +1 -0
  85. package/lib/src/types/debats.d.ts +2 -0
  86. package/lib/src/types/debats.js +1 -0
  87. package/lib/src/types/dosleg.d.ts +70 -0
  88. package/lib/src/types/dosleg.js +1 -0
  89. package/lib/src/types/questions.d.ts +2 -0
  90. package/lib/src/types/questions.js +1 -0
  91. package/lib/src/types/sens.d.ts +8 -0
  92. package/lib/src/types/sens.js +1 -0
  93. package/lib/src/types/sessions.d.ts +6 -0
  94. package/lib/src/types/sessions.js +19 -0
  95. package/lib/src/types/texte.d.ts +72 -0
  96. package/lib/src/types/texte.js +15 -0
  97. package/lib/src/validators/config.d.ts +9 -0
  98. package/lib/src/validators/config.js +10 -0
  99. package/lib/src/videos/match.js +4 -4
  100. package/lib/tests/parsers/plf/amendement.test.d.ts +1 -0
  101. package/lib/tests/parsers/plf/amendement.test.js +153 -0
  102. package/lib/tests/parsers/plf/credits.test.js +72 -72
  103. package/lib/tests/parsers/plf/test_utils.js +13 -13
  104. package/lib/tests/parsers/texte.test_utils.js +22 -22
  105. package/lib/tests/videoMatching.test.js +1 -1
  106. package/package.json +123 -123
  107. package/lib/src/scripts/extract_plf_fixtures.d.ts +0 -1
  108. package/lib/src/scripts/extract_plf_fixtures.js +0 -101
  109. package/lib/tests/plf/credits.test.js +0 -458
  110. package/lib/tests/plf/test_utils.d.ts +0 -20
  111. package/lib/tests/plf/test_utils.js +0 -85
  112. /package/lib/{tests/plf/credits.test.d.ts → src/types/agenda.js} +0 -0
@@ -29,60 +29,60 @@ export function buildNormalizeStagingSchemaSql(datasetName) {
29
29
  const prefix = `${datasetName}_`;
30
30
  const escapedPrefix = escapeSqlLiteral(prefix);
31
31
  const escapedStagingSchema = escapeSqlLiteral(stagingSchema);
32
- return `
33
- DO $$
34
- DECLARE
35
- relation record;
36
- table_row record;
37
- BEGIN
38
- FOR relation IN
39
- SELECT c.relname
40
- FROM pg_class c
41
- JOIN pg_namespace n ON n.oid = c.relnamespace
42
- WHERE n.nspname = '${escapedStagingSchema}'
43
- AND c.relkind = 'i'
44
- AND c.relname NOT LIKE '${escapedPrefix}' || '%'
45
- LOOP
46
- EXECUTE format(
47
- 'ALTER INDEX %I.%I RENAME TO %I',
48
- '${stagingSchema}',
49
- relation.relname,
50
- '${prefix}' || relation.relname
51
- );
52
- END LOOP;
53
-
54
- FOR relation IN
55
- SELECT c.relname
56
- FROM pg_class c
57
- JOIN pg_namespace n ON n.oid = c.relnamespace
58
- WHERE n.nspname = '${escapedStagingSchema}'
59
- AND c.relkind = 'S'
60
- AND c.relname NOT LIKE '${escapedPrefix}' || '%'
61
- LOOP
62
- EXECUTE format(
63
- 'ALTER SEQUENCE %I.%I RENAME TO %I',
64
- '${stagingSchema}',
65
- relation.relname,
66
- '${prefix}' || relation.relname
67
- );
68
- END LOOP;
69
-
70
- FOR table_row IN
71
- SELECT tablename
72
- FROM pg_tables
73
- WHERE schemaname = '${escapedStagingSchema}'
74
- ORDER BY tablename
75
- LOOP
76
- IF table_row.tablename NOT LIKE '${escapedPrefix}' || '%' THEN
77
- EXECUTE format(
78
- 'ALTER TABLE %I.%I RENAME TO %I',
79
- '${stagingSchema}',
80
- table_row.tablename,
81
- '${prefix}' || table_row.tablename
82
- );
83
- END IF;
84
- END LOOP;
85
- END $$;
32
+ return `
33
+ DO $$
34
+ DECLARE
35
+ relation record;
36
+ table_row record;
37
+ BEGIN
38
+ FOR relation IN
39
+ SELECT c.relname
40
+ FROM pg_class c
41
+ JOIN pg_namespace n ON n.oid = c.relnamespace
42
+ WHERE n.nspname = '${escapedStagingSchema}'
43
+ AND c.relkind = 'i'
44
+ AND c.relname NOT LIKE '${escapedPrefix}' || '%'
45
+ LOOP
46
+ EXECUTE format(
47
+ 'ALTER INDEX %I.%I RENAME TO %I',
48
+ '${stagingSchema}',
49
+ relation.relname,
50
+ '${prefix}' || relation.relname
51
+ );
52
+ END LOOP;
53
+
54
+ FOR relation IN
55
+ SELECT c.relname
56
+ FROM pg_class c
57
+ JOIN pg_namespace n ON n.oid = c.relnamespace
58
+ WHERE n.nspname = '${escapedStagingSchema}'
59
+ AND c.relkind = 'S'
60
+ AND c.relname NOT LIKE '${escapedPrefix}' || '%'
61
+ LOOP
62
+ EXECUTE format(
63
+ 'ALTER SEQUENCE %I.%I RENAME TO %I',
64
+ '${stagingSchema}',
65
+ relation.relname,
66
+ '${prefix}' || relation.relname
67
+ );
68
+ END LOOP;
69
+
70
+ FOR table_row IN
71
+ SELECT tablename
72
+ FROM pg_tables
73
+ WHERE schemaname = '${escapedStagingSchema}'
74
+ ORDER BY tablename
75
+ LOOP
76
+ IF table_row.tablename NOT LIKE '${escapedPrefix}' || '%' THEN
77
+ EXECUTE format(
78
+ 'ALTER TABLE %I.%I RENAME TO %I',
79
+ '${stagingSchema}',
80
+ table_row.tablename,
81
+ '${prefix}' || table_row.tablename
82
+ );
83
+ END IF;
84
+ END LOOP;
85
+ END $$;
86
86
  `;
87
87
  }
88
88
  export function buildIncrementalDatasetImportSql(datasetName, dbUser, mergeKeys, rowMultisetMergeTables) {
@@ -100,817 +100,817 @@ export function buildIncrementalDatasetImportSql(datasetName, dbUser, mergeKeys,
100
100
  " AND c.relkind IN ('r', 'p', 'f')",
101
101
  "ORDER BY c.relname",
102
102
  ].join("\n");
103
- return `
104
- DO $$
105
- DECLARE
106
- table_row record;
107
- column_row record;
108
- sequence_row record;
109
- constraint_row record;
110
- index_row record;
111
- sequence_owner record;
112
- sequence_state record;
113
- staging_default text;
114
- target_default text;
115
- column_definition text;
116
- column_list text;
117
- staging_column_list text;
118
- key_columns text[];
119
- key_schema_name text;
120
- configured_key_columns text[];
121
- matching_key_column_count integer;
122
- use_row_multiset_merge boolean;
123
- join_condition text;
124
- update_assignments text;
125
- change_condition text;
126
- target_row_signature_expression text;
127
- staging_row_signature_expression text;
128
- sync_strategy text;
129
- staging_table_comment text;
130
- target_table_comment text;
131
- BEGIN
132
- PERFORM set_config('search_path', '${senatSchemaName}, pg_catalog', true);
133
-
134
- FOR table_row IN
135
- ${stagingRelationsQuery}
136
- LOOP
137
- IF NOT EXISTS (
138
- SELECT 1
139
- FROM pg_tables
140
- WHERE schemaname = '${senatSchemaName}'
141
- AND tablename = table_row.tablename
142
- ) THEN
143
- EXECUTE format(
144
- 'CREATE TABLE ${senatSchemaName}.%I (LIKE %I.%I INCLUDING IDENTITY INCLUDING GENERATED INCLUDING DEFAULTS)',
145
- table_row.tablename,
146
- '${stagingSchema}',
147
- table_row.tablename
148
- );
149
- END IF;
150
- END LOOP;
151
-
152
- FOR sequence_row IN
153
- SELECT
154
- sequencename,
155
- data_type,
156
- start_value,
157
- min_value,
158
- max_value,
159
- increment_by,
160
- cycle,
161
- cache_size
162
- FROM pg_sequences
163
- WHERE schemaname = '${escapedStagingSchema}'
164
- ORDER BY sequencename
165
- LOOP
166
- IF NOT EXISTS (
167
- SELECT 1
168
- FROM pg_class c
169
- JOIN pg_namespace n ON n.oid = c.relnamespace
170
- WHERE n.nspname = '${senatSchemaName}'
171
- AND c.relkind = 'S'
172
- AND c.relname = sequence_row.sequencename
173
- ) THEN
174
- EXECUTE format('CREATE SEQUENCE ${senatSchemaName}.%I', sequence_row.sequencename);
175
- END IF;
176
-
177
- EXECUTE format(
178
- 'ALTER SEQUENCE ${senatSchemaName}.%I AS %s INCREMENT BY %s MINVALUE %s MAXVALUE %s START WITH %s CACHE %s %s',
179
- sequence_row.sequencename,
180
- sequence_row.data_type,
181
- sequence_row.increment_by,
182
- sequence_row.min_value,
183
- sequence_row.max_value,
184
- sequence_row.start_value,
185
- sequence_row.cache_size,
186
- CASE WHEN sequence_row.cycle THEN 'CYCLE' ELSE 'NO CYCLE' END
187
- );
188
- END LOOP;
189
-
190
- FOR table_row IN
191
- SELECT tablename
192
- FROM pg_tables
193
- WHERE schemaname = '${senatSchemaName}'
194
- AND tablename LIKE '${escapedPrefix}' || '%'
195
- ORDER BY tablename
196
- LOOP
197
- IF NOT EXISTS (
198
- SELECT 1
199
- FROM pg_class c
200
- JOIN pg_namespace n ON n.oid = c.relnamespace
201
- WHERE n.nspname = '${escapedStagingSchema}'
202
- AND c.relkind IN ('r', 'p', 'f')
203
- AND c.relname = table_row.tablename
204
- ) THEN
205
- EXECUTE format('DROP TABLE IF EXISTS ${senatSchemaName}.%I CASCADE', table_row.tablename);
206
- END IF;
207
- END LOOP;
208
-
209
- FOR table_row IN
210
- ${stagingRelationsQuery}
211
- LOOP
212
- key_columns := NULL;
213
- key_schema_name := NULL;
214
- configured_key_columns := NULL;
215
- matching_key_column_count := 0;
216
- use_row_multiset_merge := false;
217
- join_condition := NULL;
218
- update_assignments := NULL;
219
- change_condition := NULL;
220
- target_row_signature_expression := NULL;
221
- staging_row_signature_expression := NULL;
222
-
223
- SELECT configured.key_columns
224
- INTO configured_key_columns
225
- FROM (
226
- ${configuredMergeKeysValues}
227
- ) AS configured
228
- WHERE configured.tablename = table_row.tablename;
229
-
230
- IF configured_key_columns IS NOT NULL THEN
231
- key_columns := configured_key_columns;
232
- key_schema_name := 'configured';
233
-
234
- SELECT string_agg(format('t.%1$I IS NOT DISTINCT FROM s.%1$I', key_column), ' AND ' ORDER BY ordinality)
235
- INTO join_condition
236
- FROM unnest(configured_key_columns) WITH ORDINALITY AS configured_key(key_column, ordinality);
237
- ELSE
238
- SELECT candidate.key_columns, candidate.join_condition, candidate.source_schema
239
- INTO key_columns, join_condition, key_schema_name
240
- FROM (
241
- SELECT
242
- array_agg(att.attname ORDER BY key_column.ordinality) AS key_columns,
243
- string_agg(format('t.%1$I = s.%1$I', att.attname), ' AND ' ORDER BY key_column.ordinality) AS join_condition,
244
- table_namespace.nspname AS source_schema,
245
- CASE WHEN table_namespace.nspname = '${escapedStagingSchema}' THEN 0 ELSE 1 END AS schema_priority,
246
- CASE WHEN index_data.indisprimary THEN 0 ELSE 1 END AS priority,
247
- count(*) AS key_column_count,
248
- index_data.indexrelid
249
- FROM pg_index index_data
250
- JOIN pg_class table_class ON table_class.oid = index_data.indrelid
251
- JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
252
- JOIN LATERAL unnest(index_data.indkey::int2[]) WITH ORDINALITY AS key_column(attnum, ordinality) ON true
253
- JOIN pg_attribute att
254
- ON att.attrelid = table_class.oid
255
- AND att.attnum = key_column.attnum
256
- WHERE table_namespace.nspname IN ('${escapedStagingSchema}', '${senatSchemaName}')
257
- AND table_class.relname = table_row.tablename
258
- AND index_data.indisunique
259
- AND index_data.indpred IS NULL
260
- AND index_data.indexprs IS NULL
261
- AND key_column.ordinality <= index_data.indnkeyatts
262
- AND key_column.attnum > 0
263
- AND EXISTS (
264
- SELECT 1
265
- FROM information_schema.columns staging_columns
266
- WHERE staging_columns.table_schema = '${escapedStagingSchema}'
267
- AND staging_columns.table_name = table_row.tablename
268
- AND staging_columns.column_name = att.attname
269
- )
270
- GROUP BY index_data.indexrelid, index_data.indisprimary, table_namespace.nspname
271
- HAVING index_data.indisprimary OR bool_and(att.attnotnull)
272
- ORDER BY schema_priority, priority, key_column_count, index_data.indexrelid
273
- LIMIT 1
274
- ) AS candidate;
275
- END IF;
276
-
277
- SELECT EXISTS (
278
- SELECT 1
279
- FROM (
280
- ${configuredRowMultisetTablesValues}
281
- ) AS configured
282
- WHERE configured.tablename = table_row.tablename
283
- )
284
- INTO use_row_multiset_merge;
285
-
286
- -- The synchronization below temporarily drops primary/unique constraints before
287
- -- issuing DELETE/UPDATE statements. On tables that belong to a publication with
288
- -- deletes/updates enabled, PostgreSQL then refuses those statements unless the
289
- -- table has an explicit replica identity. FULL keeps logical replication valid
290
- -- while the constraints are being rebuilt.
291
- EXECUTE format('ALTER TABLE ${senatSchemaName}.%I REPLICA IDENTITY FULL', table_row.tablename);
292
-
293
- FOR constraint_row IN
294
- SELECT con.conname
295
- FROM pg_constraint con
296
- JOIN pg_class c ON c.oid = con.conrelid
297
- JOIN pg_namespace n ON n.oid = c.relnamespace
298
- WHERE n.nspname = '${senatSchemaName}'
299
- AND c.relname = table_row.tablename
300
- AND con.contype <> 'n'
301
- LOOP
302
- EXECUTE format(
303
- 'ALTER TABLE ${senatSchemaName}.%I DROP CONSTRAINT IF EXISTS %I CASCADE',
304
- table_row.tablename,
305
- constraint_row.conname
306
- );
307
- END LOOP;
308
-
309
- FOR index_row IN
310
- SELECT index_class.relname AS index_name
311
- FROM pg_index idx
312
- JOIN pg_class table_class ON table_class.oid = idx.indrelid
313
- JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
314
- JOIN pg_class index_class ON index_class.oid = idx.indexrelid
315
- LEFT JOIN pg_constraint con ON con.conindid = idx.indexrelid
316
- WHERE table_namespace.nspname = '${senatSchemaName}'
317
- AND table_class.relname = table_row.tablename
318
- AND con.oid IS NULL
319
- LOOP
320
- EXECUTE format('DROP INDEX IF EXISTS ${senatSchemaName}.%I', index_row.index_name);
321
- END LOOP;
322
-
323
- FOR column_row IN
324
- SELECT target_columns.column_name
325
- FROM information_schema.columns target_columns
326
- WHERE target_columns.table_schema = '${senatSchemaName}'
327
- AND target_columns.table_name = table_row.tablename
328
- AND NOT EXISTS (
329
- SELECT 1
330
- FROM information_schema.columns staging_columns
331
- WHERE staging_columns.table_schema = '${escapedStagingSchema}'
332
- AND staging_columns.table_name = table_row.tablename
333
- AND staging_columns.column_name = target_columns.column_name
334
- )
335
- ORDER BY target_columns.ordinal_position DESC
336
- LOOP
337
- EXECUTE format(
338
- 'ALTER TABLE ${senatSchemaName}.%I DROP COLUMN IF EXISTS %I CASCADE',
339
- table_row.tablename,
340
- column_row.column_name
341
- );
342
- END LOOP;
343
-
344
- FOR column_row IN
345
- SELECT
346
- staging_columns.column_name,
347
- format_type(a.atttypid, a.atttypmod) AS data_type,
348
- a.attnotnull AS not_null,
349
- a.attidentity AS identity_kind,
350
- a.attgenerated AS generated_kind,
351
- pg_get_expr(ad.adbin, ad.adrelid) AS default_expression,
352
- CASE
353
- WHEN coll.oid IS NULL THEN NULL
354
- ELSE format('%I.%I', coll_namespace.nspname, coll.collname)
355
- END AS collation_name
356
- FROM information_schema.columns staging_columns
357
- JOIN pg_class c ON c.relname = staging_columns.table_name
358
- JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = staging_columns.table_schema
359
- JOIN pg_attribute a ON a.attrelid = c.oid AND a.attname = staging_columns.column_name
360
- LEFT JOIN pg_attrdef ad ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum
361
- LEFT JOIN pg_type typ ON typ.oid = a.atttypid
362
- LEFT JOIN pg_collation coll ON coll.oid = a.attcollation AND coll.oid <> typ.typcollation
363
- LEFT JOIN pg_namespace coll_namespace ON coll_namespace.oid = coll.collnamespace
364
- WHERE staging_columns.table_schema = '${escapedStagingSchema}'
365
- AND staging_columns.table_name = table_row.tablename
366
- AND NOT EXISTS (
367
- SELECT 1
368
- FROM information_schema.columns target_columns
369
- WHERE target_columns.table_schema = '${senatSchemaName}'
370
- AND target_columns.table_name = table_row.tablename
371
- AND target_columns.column_name = staging_columns.column_name
372
- )
373
- ORDER BY staging_columns.ordinal_position
374
- LOOP
375
- column_definition := column_row.data_type;
376
- IF column_row.collation_name IS NOT NULL THEN
377
- column_definition := column_definition || ' COLLATE ' || column_row.collation_name;
378
- END IF;
379
- IF column_row.generated_kind = 's' THEN
380
- column_definition :=
381
- column_definition ||
382
- ' GENERATED ALWAYS AS (' ||
383
- replace(column_row.default_expression,
384
- '${stagingSchema}.', '${senatSchemaName}.') ||
385
- ') STORED';
386
- ELSIF column_row.identity_kind = 'a' THEN
387
- column_definition := column_definition || ' GENERATED ALWAYS AS IDENTITY';
388
- ELSIF column_row.identity_kind = 'd' THEN
389
- column_definition := column_definition || ' GENERATED BY DEFAULT AS IDENTITY';
390
- END IF;
391
-
392
- EXECUTE format(
393
- 'ALTER TABLE ${senatSchemaName}.%I ADD COLUMN %I %s',
394
- table_row.tablename,
395
- column_row.column_name,
396
- column_definition
397
- );
398
-
399
- IF column_row.generated_kind = ''
400
- AND column_row.identity_kind = ''
401
- AND column_row.default_expression IS NOT NULL THEN
402
- EXECUTE format(
403
- 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I SET DEFAULT %s',
404
- table_row.tablename,
405
- column_row.column_name,
406
- replace(column_row.default_expression, '${stagingSchema}.', '${senatSchemaName}.')
407
- );
408
- END IF;
409
-
410
- IF column_row.not_null THEN
411
- EXECUTE format(
412
- 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I SET NOT NULL',
413
- table_row.tablename,
414
- column_row.column_name
415
- );
416
- END IF;
417
- END LOOP;
418
-
419
- FOR column_row IN
420
- SELECT
421
- staging_columns.column_name,
422
- format_type(staging_attribute.atttypid, staging_attribute.atttypmod) AS staging_data_type,
423
- format_type(target_attribute.atttypid, target_attribute.atttypmod) AS target_data_type,
424
- staging_attribute.attnotnull AS staging_not_null,
425
- target_attribute.attnotnull AS target_not_null,
426
- staging_attribute.attidentity AS staging_identity_kind,
427
- target_attribute.attidentity AS target_identity_kind,
428
- staging_attribute.attgenerated AS staging_generated_kind,
429
- target_attribute.attgenerated AS target_generated_kind,
430
- pg_get_expr(staging_default_expr.adbin, staging_default_expr.adrelid) AS staging_default_expression,
431
- pg_get_expr(target_default_expr.adbin, target_default_expr.adrelid) AS target_default_expression,
432
- CASE
433
- WHEN staging_collation.oid IS NULL THEN NULL
434
- ELSE format('%I.%I', staging_collation_namespace.nspname, staging_collation.collname)
435
- END AS staging_collation_name,
436
- CASE
437
- WHEN target_collation.oid IS NULL THEN NULL
438
- ELSE format('%I.%I', target_collation_namespace.nspname, target_collation.collname)
439
- END AS target_collation_name
440
- FROM information_schema.columns staging_columns
441
- JOIN information_schema.columns target_columns
442
- ON target_columns.table_schema = '${senatSchemaName}'
443
- AND target_columns.table_name = staging_columns.table_name
444
- AND target_columns.column_name = staging_columns.column_name
445
- JOIN pg_class staging_class ON staging_class.relname = staging_columns.table_name
446
- JOIN pg_namespace staging_namespace
447
- ON staging_namespace.oid = staging_class.relnamespace
448
- AND staging_namespace.nspname = staging_columns.table_schema
449
- JOIN pg_attribute staging_attribute
450
- ON staging_attribute.attrelid = staging_class.oid
451
- AND staging_attribute.attname = staging_columns.column_name
452
- LEFT JOIN pg_attrdef staging_default_expr
453
- ON staging_default_expr.adrelid = staging_attribute.attrelid
454
- AND staging_default_expr.adnum = staging_attribute.attnum
455
- LEFT JOIN pg_type staging_type ON staging_type.oid = staging_attribute.atttypid
456
- LEFT JOIN pg_collation staging_collation
457
- ON staging_collation.oid = staging_attribute.attcollation
458
- AND staging_collation.oid <> staging_type.typcollation
459
- LEFT JOIN pg_namespace staging_collation_namespace
460
- ON staging_collation_namespace.oid = staging_collation.collnamespace
461
- JOIN pg_class target_class ON target_class.relname = target_columns.table_name
462
- JOIN pg_namespace target_namespace
463
- ON target_namespace.oid = target_class.relnamespace
464
- AND target_namespace.nspname = target_columns.table_schema
465
- JOIN pg_attribute target_attribute
466
- ON target_attribute.attrelid = target_class.oid
467
- AND target_attribute.attname = target_columns.column_name
468
- LEFT JOIN pg_attrdef target_default_expr
469
- ON target_default_expr.adrelid = target_attribute.attrelid
470
- AND target_default_expr.adnum = target_attribute.attnum
471
- LEFT JOIN pg_type target_type ON target_type.oid = target_attribute.atttypid
472
- LEFT JOIN pg_collation target_collation
473
- ON target_collation.oid = target_attribute.attcollation
474
- AND target_collation.oid <> target_type.typcollation
475
- LEFT JOIN pg_namespace target_collation_namespace
476
- ON target_collation_namespace.oid = target_collation.collnamespace
477
- WHERE staging_columns.table_schema = '${escapedStagingSchema}'
478
- AND staging_columns.table_name = table_row.tablename
479
- ORDER BY staging_columns.ordinal_position
480
- LOOP
481
- IF column_row.staging_identity_kind <> column_row.target_identity_kind
482
- OR column_row.staging_generated_kind <> column_row.target_generated_kind THEN
483
- RAISE EXCEPTION
484
- 'Unsupported identity/generated column change for %.%',
485
- table_row.tablename,
486
- column_row.column_name;
487
- END IF;
488
-
489
- IF column_row.staging_data_type <> column_row.target_data_type
490
- OR column_row.staging_collation_name IS DISTINCT FROM column_row.target_collation_name THEN
491
- EXECUTE format(
492
- 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I TYPE %s%s USING %I::%s',
493
- table_row.tablename,
494
- column_row.column_name,
495
- column_row.staging_data_type,
496
- CASE
497
- WHEN column_row.staging_collation_name IS NULL THEN ''
498
- ELSE ' COLLATE ' || column_row.staging_collation_name
499
- END,
500
- column_row.column_name,
501
- column_row.staging_data_type
502
- );
503
- END IF;
504
-
505
- staging_default := replace(
506
- coalesce(column_row.staging_default_expression, ''),
507
- '${stagingSchema}.',
508
- '${senatSchemaName}.'
509
- );
510
- target_default := replace(
511
- coalesce(column_row.target_default_expression, ''),
512
- '${stagingSchema}.',
513
- '${senatSchemaName}.'
514
- );
515
-
516
- IF staging_default <> target_default THEN
517
- IF column_row.staging_default_expression IS NULL THEN
518
- EXECUTE format(
519
- 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I DROP DEFAULT',
520
- table_row.tablename,
521
- column_row.column_name
522
- );
523
- ELSE
524
- EXECUTE format(
525
- 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I SET DEFAULT %s',
526
- table_row.tablename,
527
- column_row.column_name,
528
- staging_default
529
- );
530
- END IF;
531
- END IF;
532
-
533
- IF column_row.staging_not_null <> column_row.target_not_null THEN
534
- EXECUTE format(
535
- 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I %s NOT NULL',
536
- table_row.tablename,
537
- column_row.column_name,
538
- CASE WHEN column_row.staging_not_null THEN 'SET' ELSE 'DROP' END
539
- );
540
- END IF;
541
- END LOOP;
542
-
543
- SELECT obj_description(c.oid, 'pg_class')
544
- INTO staging_table_comment
545
- FROM pg_class c
546
- JOIN pg_namespace n ON n.oid = c.relnamespace
547
- WHERE n.nspname = '${escapedStagingSchema}'
548
- AND c.relkind IN ('r', 'p', 'f')
549
- AND c.relname = table_row.tablename;
550
-
551
- SELECT obj_description(c.oid, 'pg_class')
552
- INTO target_table_comment
553
- FROM pg_class c
554
- JOIN pg_namespace n ON n.oid = c.relnamespace
555
- WHERE n.nspname = '${senatSchemaName}'
556
- AND c.relkind IN ('r', 'p')
557
- AND c.relname = table_row.tablename;
558
-
559
- IF staging_table_comment IS DISTINCT FROM target_table_comment THEN
560
- EXECUTE format(
561
- 'COMMENT ON TABLE ${senatSchemaName}.%I IS %L',
562
- table_row.tablename,
563
- staging_table_comment
564
- );
565
- END IF;
566
-
567
- FOR column_row IN
568
- SELECT
569
- staging_columns.column_name,
570
- col_description(staging_class.oid, staging_attribute.attnum) AS staging_comment,
571
- col_description(target_class.oid, target_attribute.attnum) AS target_comment
572
- FROM information_schema.columns staging_columns
573
- JOIN pg_class staging_class ON staging_class.relname = staging_columns.table_name
574
- JOIN pg_namespace staging_namespace
575
- ON staging_namespace.oid = staging_class.relnamespace
576
- AND staging_namespace.nspname = staging_columns.table_schema
577
- JOIN pg_attribute staging_attribute
578
- ON staging_attribute.attrelid = staging_class.oid
579
- AND staging_attribute.attname = staging_columns.column_name
580
- JOIN information_schema.columns target_columns
581
- ON target_columns.table_schema = '${senatSchemaName}'
582
- AND target_columns.table_name = staging_columns.table_name
583
- AND target_columns.column_name = staging_columns.column_name
584
- JOIN pg_class target_class ON target_class.relname = target_columns.table_name
585
- JOIN pg_namespace target_namespace
586
- ON target_namespace.oid = target_class.relnamespace
587
- AND target_namespace.nspname = target_columns.table_schema
588
- JOIN pg_attribute target_attribute
589
- ON target_attribute.attrelid = target_class.oid
590
- AND target_attribute.attname = target_columns.column_name
591
- WHERE staging_columns.table_schema = '${escapedStagingSchema}'
592
- AND staging_columns.table_name = table_row.tablename
593
- ORDER BY staging_columns.ordinal_position
594
- LOOP
595
- IF column_row.staging_comment IS DISTINCT FROM column_row.target_comment THEN
596
- EXECUTE format(
597
- 'COMMENT ON COLUMN ${senatSchemaName}.%I.%I IS %L',
598
- table_row.tablename,
599
- column_row.column_name,
600
- column_row.staging_comment
601
- );
602
- END IF;
603
- END LOOP;
604
-
605
- SELECT string_agg(format('%I', columns.column_name), ', ' ORDER BY columns.ordinal_position)
606
- , string_agg(format('s.%I', columns.column_name), ', ' ORDER BY columns.ordinal_position)
607
- INTO column_list, staging_column_list
608
- FROM information_schema.columns columns
609
- WHERE columns.table_schema = '${escapedStagingSchema}'
610
- AND columns.table_name = table_row.tablename;
611
-
612
- SELECT
613
- string_agg(
614
- format('%1$I = s.%1$I', columns.column_name),
615
- ', ' ORDER BY columns.ordinal_position),
616
- string_agg(
617
- format('t.%1$I IS DISTINCT FROM s.%1$I', columns.column_name),
618
- ' OR ' ORDER BY columns.ordinal_position)
619
- INTO update_assignments, change_condition
620
- FROM information_schema.columns columns
621
- WHERE columns.table_schema = '${escapedStagingSchema}'
622
- AND columns.table_name = table_row.tablename
623
- AND NOT (columns.column_name = ANY(coalesce(key_columns, ARRAY[]::text[])));
624
-
625
- IF key_columns IS NOT NULL THEN
626
- SELECT count(*)
627
- INTO matching_key_column_count
628
- FROM unnest(key_columns) AS key_column(column_name)
629
- JOIN information_schema.columns staging_columns
630
- ON staging_columns.table_schema = '${escapedStagingSchema}'
631
- AND staging_columns.table_name = table_row.tablename
632
- AND staging_columns.column_name = key_column.column_name
633
- JOIN information_schema.columns target_columns
634
- ON target_columns.table_schema = '${senatSchemaName}'
635
- AND target_columns.table_name = table_row.tablename
636
- AND target_columns.column_name = key_column.column_name;
637
-
638
- IF matching_key_column_count <> cardinality(key_columns) THEN
639
- RAISE NOTICE
640
- 'Rejected sync key for %.% from % because only %/% columns are present in both staging and target: %',
641
- '${senatSchemaName}',
642
- table_row.tablename,
643
- coalesce(key_schema_name, 'unknown'),
644
- matching_key_column_count,
645
- cardinality(key_columns),
646
- array_to_string(key_columns, ', ');
647
- key_columns := NULL;
648
- key_schema_name := NULL;
649
- join_condition := NULL;
650
- END IF;
651
- END IF;
652
-
653
- SELECT
654
- 'jsonb_build_array(' ||
655
- string_agg(format('t.%I', columns.column_name),
656
- ', ' ORDER BY columns.ordinal_position) || ')',
657
- 'jsonb_build_array(' ||
658
- string_agg(format('s.%I', columns.column_name),
659
- ', ' ORDER BY columns.ordinal_position) || ')'
660
- INTO target_row_signature_expression, staging_row_signature_expression
661
- FROM information_schema.columns columns
662
- WHERE columns.table_schema = '${escapedStagingSchema}'
663
- AND columns.table_name = table_row.tablename;
664
-
665
- IF use_row_multiset_merge THEN
666
- sync_strategy := 'incremental_row_multiset';
667
- RAISE NOTICE
668
- 'Sync strategy for %.%: %',
669
- '${senatSchemaName}',
670
- table_row.tablename,
671
- sync_strategy;
672
-
673
- EXECUTE format(
674
- 'WITH staging_counts AS (' ||
675
- ' SELECT %2$s AS row_signature, count(*) AS row_count' ||
676
- ' FROM %3$I.%1$I s' ||
677
- ' GROUP BY 1' ||
678
- '), target_ranked AS (' ||
679
- ' SELECT ctid, %4$s AS row_signature, row_number() OVER (PARTITION BY %4$s) AS row_number' ||
680
- ' FROM ${senatSchemaName}.%1$I t' ||
681
- ')' ||
682
- ' DELETE FROM ${senatSchemaName}.%1$I t' ||
683
- ' USING (' ||
684
- ' SELECT candidate_row.ctid' ||
685
- ' FROM target_ranked candidate_row' ||
686
- ' LEFT JOIN staging_counts reference_row USING (row_signature)' ||
687
- ' WHERE candidate_row.row_number > coalesce(reference_row.row_count, 0)' ||
688
- ' ) rows_to_delete' ||
689
- ' WHERE t.ctid = rows_to_delete.ctid',
690
- table_row.tablename,
691
- staging_row_signature_expression,
692
- '${stagingSchema}',
693
- target_row_signature_expression
694
- );
695
-
696
- EXECUTE format(
697
- 'WITH target_counts AS (' ||
698
- ' SELECT %2$s AS row_signature, count(*) AS row_count' ||
699
- ' FROM ${senatSchemaName}.%1$I t' ||
700
- ' GROUP BY 1' ||
701
- '), staging_ranked AS (' ||
702
- ' SELECT %3$s, %4$s AS row_signature, row_number() OVER (PARTITION BY %4$s) AS row_number' ||
703
- ' FROM %5$I.%1$I s' ||
704
- ')' ||
705
- ' INSERT INTO ${senatSchemaName}.%1$I (%3$s)' ||
706
- ' SELECT %3$s' ||
707
- ' FROM staging_ranked candidate_row' ||
708
- ' LEFT JOIN target_counts reference_row USING (row_signature)' ||
709
- ' WHERE candidate_row.row_number > coalesce(reference_row.row_count, 0)',
710
- table_row.tablename,
711
- target_row_signature_expression,
712
- column_list,
713
- staging_row_signature_expression,
714
- '${stagingSchema}'
715
- );
716
- ELSIF key_columns IS NULL THEN
717
- sync_strategy := 'full_refresh_fallback';
718
- RAISE NOTICE
719
- 'Sync strategy for %.%: %',
720
- '${senatSchemaName}',
721
- table_row.tablename,
722
- sync_strategy;
723
- RAISE WARNING
724
- 'No primary key or all-NOT-NULL unique key found for %.%; falling back to full refresh',
725
- '${senatSchemaName}',
726
- table_row.tablename;
727
-
728
- EXECUTE format('TRUNCATE TABLE ${senatSchemaName}.%I', table_row.tablename);
729
-
730
- EXECUTE format(
731
- 'INSERT INTO ${senatSchemaName}.%I (%s) SELECT %s FROM %I.%I s',
732
- table_row.tablename,
733
- column_list,
734
- staging_column_list,
735
- '${stagingSchema}',
736
- table_row.tablename
737
- );
738
- ELSE
739
- sync_strategy := 'incremental';
740
- RAISE NOTICE
741
- 'Sync strategy for %.%: % (key from %: %)',
742
- '${senatSchemaName}',
743
- table_row.tablename,
744
- sync_strategy,
745
- key_schema_name,
746
- array_to_string(key_columns, ', ');
747
-
748
- EXECUTE format(
749
- 'DELETE FROM ${senatSchemaName}.%1$I t WHERE NOT EXISTS (SELECT 1 FROM %2$I.%1$I s WHERE %3$s)',
750
- table_row.tablename,
751
- '${stagingSchema}',
752
- join_condition
753
- );
754
-
755
- IF update_assignments IS NOT NULL THEN
756
- EXECUTE format(
757
- 'UPDATE ${senatSchemaName}.%1$I t SET %2$s FROM %3$I.%1$I s WHERE %4$s AND (%5$s)',
758
- table_row.tablename,
759
- update_assignments,
760
- '${stagingSchema}',
761
- join_condition,
762
- change_condition
763
- );
764
- END IF;
765
-
766
- EXECUTE format(
767
- 'INSERT INTO ${senatSchemaName}.%1$I (%2$s) ' ||
768
- 'SELECT %3$s FROM %4$I.%1$I s ' ||
769
- 'WHERE NOT EXISTS ' ||
770
- '(SELECT 1 FROM ${senatSchemaName}.%1$I t WHERE %5$s)',
771
- table_row.tablename,
772
- column_list,
773
- staging_column_list,
774
- '${stagingSchema}',
775
- join_condition
776
- );
777
- END IF;
778
- END LOOP;
779
-
780
- FOR sequence_row IN
781
- SELECT sequencename
782
- FROM pg_sequences
783
- WHERE schemaname = '${escapedStagingSchema}'
784
- ORDER BY sequencename
785
- LOOP
786
- SELECT
787
- dependent_table.relname AS table_name,
788
- dependent_column.attname AS column_name
789
- INTO sequence_owner
790
- FROM pg_class sequence_class
791
- JOIN pg_namespace sequence_namespace ON sequence_namespace.oid = sequence_class.relnamespace
792
- JOIN pg_depend dependency
793
- ON dependency.objid = sequence_class.oid
794
- AND dependency.deptype = 'a'
795
- JOIN pg_class dependent_table ON dependent_table.oid = dependency.refobjid
796
- JOIN pg_namespace dependent_namespace ON dependent_namespace.oid = dependent_table.relnamespace
797
- JOIN pg_attribute dependent_column
798
- ON dependent_column.attrelid = dependent_table.oid
799
- AND dependent_column.attnum = dependency.refobjsubid
800
- WHERE sequence_namespace.nspname = '${escapedStagingSchema}'
801
- AND dependent_namespace.nspname = '${escapedStagingSchema}'
802
- AND sequence_class.relname = sequence_row.sequencename;
803
-
804
- IF sequence_owner.table_name IS NULL THEN
805
- EXECUTE format('ALTER SEQUENCE ${senatSchemaName}.%I OWNED BY NONE', sequence_row.sequencename);
806
- ELSE
807
- EXECUTE format(
808
- 'ALTER SEQUENCE ${senatSchemaName}.%I OWNED BY ${senatSchemaName}.%I.%I',
809
- sequence_row.sequencename,
810
- sequence_owner.table_name,
811
- sequence_owner.column_name
812
- );
813
- END IF;
814
- END LOOP;
815
-
816
- FOR constraint_row IN
817
- SELECT
818
- table_class.relname AS table_name,
819
- con.conname,
820
- con.contype,
821
- pg_get_constraintdef(con.oid, true) AS constraint_definition
822
- FROM pg_constraint con
823
- JOIN pg_class table_class ON table_class.oid = con.conrelid
824
- JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
825
- WHERE table_namespace.nspname = '${escapedStagingSchema}'
826
- AND con.contype <> 'n'
827
- ORDER BY
828
- CASE con.contype
829
- WHEN 'p' THEN 1
830
- WHEN 'u' THEN 2
831
- WHEN 'c' THEN 3
832
- WHEN 'f' THEN 4
833
- ELSE 5
834
- END,
835
- table_class.relname,
836
- con.conname
837
- LOOP
838
- EXECUTE format(
839
- 'ALTER TABLE ${senatSchemaName}.%I ADD CONSTRAINT %I %s',
840
- constraint_row.table_name,
841
- constraint_row.conname,
842
- replace(
843
- replace(
844
- constraint_row.constraint_definition,
845
- ' REFERENCES ' || '${stagingSchema}' || '.',
846
- ' REFERENCES ${senatSchemaName}.'
847
- ),
848
- ' REFERENCES ONLY ' || '${stagingSchema}' || '.',
849
- ' REFERENCES ONLY ${senatSchemaName}.'
850
- )
851
- );
852
- END LOOP;
853
-
854
- FOR index_row IN
855
- SELECT
856
- table_class.relname AS table_name,
857
- pg_get_indexdef(index_data.indexrelid) AS index_definition
858
- FROM pg_index index_data
859
- JOIN pg_class table_class ON table_class.oid = index_data.indrelid
860
- JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
861
- LEFT JOIN pg_constraint con ON con.conindid = index_data.indexrelid
862
- WHERE table_namespace.nspname = '${escapedStagingSchema}'
863
- AND con.oid IS NULL
864
- ORDER BY table_class.relname, pg_get_indexdef(index_data.indexrelid)
865
- LOOP
866
- EXECUTE replace(
867
- index_row.index_definition,
868
- ' ON ${stagingSchema}.',
869
- ' ON ${senatSchemaName}.'
870
- );
871
- END LOOP;
872
-
873
- FOR sequence_row IN
874
- SELECT sequencename
875
- FROM pg_sequences
876
- WHERE schemaname = '${escapedStagingSchema}'
877
- ORDER BY sequencename
878
- LOOP
879
- EXECUTE format(
880
- 'SELECT last_value, is_called FROM %I.%I',
881
- '${stagingSchema}',
882
- sequence_row.sequencename
883
- )
884
- INTO sequence_state;
885
-
886
- EXECUTE format(
887
- 'SELECT setval(%L, %s, %L)',
888
- '${senatSchemaName}.' || sequence_row.sequencename,
889
- sequence_state.last_value,
890
- sequence_state.is_called
891
- );
892
- END LOOP;
893
-
894
- FOR sequence_row IN
895
- SELECT c.relname AS sequencename
896
- FROM pg_class c
897
- JOIN pg_namespace n ON n.oid = c.relnamespace
898
- WHERE n.nspname = '${senatSchemaName}'
899
- AND c.relkind = 'S'
900
- AND c.relname LIKE '${escapedPrefix}' || '%'
901
- ORDER BY c.relname
902
- LOOP
903
- IF NOT EXISTS (
904
- SELECT 1
905
- FROM pg_sequences
906
- WHERE schemaname = '${escapedStagingSchema}'
907
- AND sequencename = sequence_row.sequencename
908
- ) THEN
909
- EXECUTE format('DROP SEQUENCE IF EXISTS ${senatSchemaName}.%I CASCADE', sequence_row.sequencename);
910
- END IF;
911
- END LOOP;
912
- END $$;
913
-
914
- GRANT SELECT ON ALL TABLES IN SCHEMA ${senatSchemaName} TO ${dbUser};
103
+ return `
104
+ DO $$
105
+ DECLARE
106
+ table_row record;
107
+ column_row record;
108
+ sequence_row record;
109
+ constraint_row record;
110
+ index_row record;
111
+ sequence_owner record;
112
+ sequence_state record;
113
+ staging_default text;
114
+ target_default text;
115
+ column_definition text;
116
+ column_list text;
117
+ staging_column_list text;
118
+ key_columns text[];
119
+ key_schema_name text;
120
+ configured_key_columns text[];
121
+ matching_key_column_count integer;
122
+ use_row_multiset_merge boolean;
123
+ join_condition text;
124
+ update_assignments text;
125
+ change_condition text;
126
+ target_row_signature_expression text;
127
+ staging_row_signature_expression text;
128
+ sync_strategy text;
129
+ staging_table_comment text;
130
+ target_table_comment text;
131
+ BEGIN
132
+ PERFORM set_config('search_path', '${senatSchemaName}, pg_catalog', true);
133
+
134
+ FOR table_row IN
135
+ ${stagingRelationsQuery}
136
+ LOOP
137
+ IF NOT EXISTS (
138
+ SELECT 1
139
+ FROM pg_tables
140
+ WHERE schemaname = '${senatSchemaName}'
141
+ AND tablename = table_row.tablename
142
+ ) THEN
143
+ EXECUTE format(
144
+ 'CREATE TABLE ${senatSchemaName}.%I (LIKE %I.%I INCLUDING IDENTITY INCLUDING GENERATED INCLUDING DEFAULTS)',
145
+ table_row.tablename,
146
+ '${stagingSchema}',
147
+ table_row.tablename
148
+ );
149
+ END IF;
150
+ END LOOP;
151
+
152
+ FOR sequence_row IN
153
+ SELECT
154
+ sequencename,
155
+ data_type,
156
+ start_value,
157
+ min_value,
158
+ max_value,
159
+ increment_by,
160
+ cycle,
161
+ cache_size
162
+ FROM pg_sequences
163
+ WHERE schemaname = '${escapedStagingSchema}'
164
+ ORDER BY sequencename
165
+ LOOP
166
+ IF NOT EXISTS (
167
+ SELECT 1
168
+ FROM pg_class c
169
+ JOIN pg_namespace n ON n.oid = c.relnamespace
170
+ WHERE n.nspname = '${senatSchemaName}'
171
+ AND c.relkind = 'S'
172
+ AND c.relname = sequence_row.sequencename
173
+ ) THEN
174
+ EXECUTE format('CREATE SEQUENCE ${senatSchemaName}.%I', sequence_row.sequencename);
175
+ END IF;
176
+
177
+ EXECUTE format(
178
+ 'ALTER SEQUENCE ${senatSchemaName}.%I AS %s INCREMENT BY %s MINVALUE %s MAXVALUE %s START WITH %s CACHE %s %s',
179
+ sequence_row.sequencename,
180
+ sequence_row.data_type,
181
+ sequence_row.increment_by,
182
+ sequence_row.min_value,
183
+ sequence_row.max_value,
184
+ sequence_row.start_value,
185
+ sequence_row.cache_size,
186
+ CASE WHEN sequence_row.cycle THEN 'CYCLE' ELSE 'NO CYCLE' END
187
+ );
188
+ END LOOP;
189
+
190
+ FOR table_row IN
191
+ SELECT tablename
192
+ FROM pg_tables
193
+ WHERE schemaname = '${senatSchemaName}'
194
+ AND tablename LIKE '${escapedPrefix}' || '%'
195
+ ORDER BY tablename
196
+ LOOP
197
+ IF NOT EXISTS (
198
+ SELECT 1
199
+ FROM pg_class c
200
+ JOIN pg_namespace n ON n.oid = c.relnamespace
201
+ WHERE n.nspname = '${escapedStagingSchema}'
202
+ AND c.relkind IN ('r', 'p', 'f')
203
+ AND c.relname = table_row.tablename
204
+ ) THEN
205
+ EXECUTE format('DROP TABLE IF EXISTS ${senatSchemaName}.%I CASCADE', table_row.tablename);
206
+ END IF;
207
+ END LOOP;
208
+
209
+ FOR table_row IN
210
+ ${stagingRelationsQuery}
211
+ LOOP
212
+ key_columns := NULL;
213
+ key_schema_name := NULL;
214
+ configured_key_columns := NULL;
215
+ matching_key_column_count := 0;
216
+ use_row_multiset_merge := false;
217
+ join_condition := NULL;
218
+ update_assignments := NULL;
219
+ change_condition := NULL;
220
+ target_row_signature_expression := NULL;
221
+ staging_row_signature_expression := NULL;
222
+
223
+ SELECT configured.key_columns
224
+ INTO configured_key_columns
225
+ FROM (
226
+ ${configuredMergeKeysValues}
227
+ ) AS configured
228
+ WHERE configured.tablename = table_row.tablename;
229
+
230
+ IF configured_key_columns IS NOT NULL THEN
231
+ key_columns := configured_key_columns;
232
+ key_schema_name := 'configured';
233
+
234
+ SELECT string_agg(format('t.%1$I IS NOT DISTINCT FROM s.%1$I', key_column), ' AND ' ORDER BY ordinality)
235
+ INTO join_condition
236
+ FROM unnest(configured_key_columns) WITH ORDINALITY AS configured_key(key_column, ordinality);
237
+ ELSE
238
+ SELECT candidate.key_columns, candidate.join_condition, candidate.source_schema
239
+ INTO key_columns, join_condition, key_schema_name
240
+ FROM (
241
+ SELECT
242
+ array_agg(att.attname ORDER BY key_column.ordinality) AS key_columns,
243
+ string_agg(format('t.%1$I = s.%1$I', att.attname), ' AND ' ORDER BY key_column.ordinality) AS join_condition,
244
+ table_namespace.nspname AS source_schema,
245
+ CASE WHEN table_namespace.nspname = '${escapedStagingSchema}' THEN 0 ELSE 1 END AS schema_priority,
246
+ CASE WHEN index_data.indisprimary THEN 0 ELSE 1 END AS priority,
247
+ count(*) AS key_column_count,
248
+ index_data.indexrelid
249
+ FROM pg_index index_data
250
+ JOIN pg_class table_class ON table_class.oid = index_data.indrelid
251
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
252
+ JOIN LATERAL unnest(index_data.indkey::int2[]) WITH ORDINALITY AS key_column(attnum, ordinality) ON true
253
+ JOIN pg_attribute att
254
+ ON att.attrelid = table_class.oid
255
+ AND att.attnum = key_column.attnum
256
+ WHERE table_namespace.nspname IN ('${escapedStagingSchema}', '${senatSchemaName}')
257
+ AND table_class.relname = table_row.tablename
258
+ AND index_data.indisunique
259
+ AND index_data.indpred IS NULL
260
+ AND index_data.indexprs IS NULL
261
+ AND key_column.ordinality <= index_data.indnkeyatts
262
+ AND key_column.attnum > 0
263
+ AND EXISTS (
264
+ SELECT 1
265
+ FROM information_schema.columns staging_columns
266
+ WHERE staging_columns.table_schema = '${escapedStagingSchema}'
267
+ AND staging_columns.table_name = table_row.tablename
268
+ AND staging_columns.column_name = att.attname
269
+ )
270
+ GROUP BY index_data.indexrelid, index_data.indisprimary, table_namespace.nspname
271
+ HAVING index_data.indisprimary OR bool_and(att.attnotnull)
272
+ ORDER BY schema_priority, priority, key_column_count, index_data.indexrelid
273
+ LIMIT 1
274
+ ) AS candidate;
275
+ END IF;
276
+
277
+ SELECT EXISTS (
278
+ SELECT 1
279
+ FROM (
280
+ ${configuredRowMultisetTablesValues}
281
+ ) AS configured
282
+ WHERE configured.tablename = table_row.tablename
283
+ )
284
+ INTO use_row_multiset_merge;
285
+
286
+ -- The synchronization below temporarily drops primary/unique constraints before
287
+ -- issuing DELETE/UPDATE statements. On tables that belong to a publication with
288
+ -- deletes/updates enabled, PostgreSQL then refuses those statements unless the
289
+ -- table has an explicit replica identity. FULL keeps logical replication valid
290
+ -- while the constraints are being rebuilt.
291
+ EXECUTE format('ALTER TABLE ${senatSchemaName}.%I REPLICA IDENTITY FULL', table_row.tablename);
292
+
293
+ FOR constraint_row IN
294
+ SELECT con.conname
295
+ FROM pg_constraint con
296
+ JOIN pg_class c ON c.oid = con.conrelid
297
+ JOIN pg_namespace n ON n.oid = c.relnamespace
298
+ WHERE n.nspname = '${senatSchemaName}'
299
+ AND c.relname = table_row.tablename
300
+ AND con.contype <> 'n'
301
+ LOOP
302
+ EXECUTE format(
303
+ 'ALTER TABLE ${senatSchemaName}.%I DROP CONSTRAINT IF EXISTS %I CASCADE',
304
+ table_row.tablename,
305
+ constraint_row.conname
306
+ );
307
+ END LOOP;
308
+
309
+ FOR index_row IN
310
+ SELECT index_class.relname AS index_name
311
+ FROM pg_index idx
312
+ JOIN pg_class table_class ON table_class.oid = idx.indrelid
313
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
314
+ JOIN pg_class index_class ON index_class.oid = idx.indexrelid
315
+ LEFT JOIN pg_constraint con ON con.conindid = idx.indexrelid
316
+ WHERE table_namespace.nspname = '${senatSchemaName}'
317
+ AND table_class.relname = table_row.tablename
318
+ AND con.oid IS NULL
319
+ LOOP
320
+ EXECUTE format('DROP INDEX IF EXISTS ${senatSchemaName}.%I', index_row.index_name);
321
+ END LOOP;
322
+
323
+ FOR column_row IN
324
+ SELECT target_columns.column_name
325
+ FROM information_schema.columns target_columns
326
+ WHERE target_columns.table_schema = '${senatSchemaName}'
327
+ AND target_columns.table_name = table_row.tablename
328
+ AND NOT EXISTS (
329
+ SELECT 1
330
+ FROM information_schema.columns staging_columns
331
+ WHERE staging_columns.table_schema = '${escapedStagingSchema}'
332
+ AND staging_columns.table_name = table_row.tablename
333
+ AND staging_columns.column_name = target_columns.column_name
334
+ )
335
+ ORDER BY target_columns.ordinal_position DESC
336
+ LOOP
337
+ EXECUTE format(
338
+ 'ALTER TABLE ${senatSchemaName}.%I DROP COLUMN IF EXISTS %I CASCADE',
339
+ table_row.tablename,
340
+ column_row.column_name
341
+ );
342
+ END LOOP;
343
+
344
+ FOR column_row IN
345
+ SELECT
346
+ staging_columns.column_name,
347
+ format_type(a.atttypid, a.atttypmod) AS data_type,
348
+ a.attnotnull AS not_null,
349
+ a.attidentity AS identity_kind,
350
+ a.attgenerated AS generated_kind,
351
+ pg_get_expr(ad.adbin, ad.adrelid) AS default_expression,
352
+ CASE
353
+ WHEN coll.oid IS NULL THEN NULL
354
+ ELSE format('%I.%I', coll_namespace.nspname, coll.collname)
355
+ END AS collation_name
356
+ FROM information_schema.columns staging_columns
357
+ JOIN pg_class c ON c.relname = staging_columns.table_name
358
+ JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = staging_columns.table_schema
359
+ JOIN pg_attribute a ON a.attrelid = c.oid AND a.attname = staging_columns.column_name
360
+ LEFT JOIN pg_attrdef ad ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum
361
+ LEFT JOIN pg_type typ ON typ.oid = a.atttypid
362
+ LEFT JOIN pg_collation coll ON coll.oid = a.attcollation AND coll.oid <> typ.typcollation
363
+ LEFT JOIN pg_namespace coll_namespace ON coll_namespace.oid = coll.collnamespace
364
+ WHERE staging_columns.table_schema = '${escapedStagingSchema}'
365
+ AND staging_columns.table_name = table_row.tablename
366
+ AND NOT EXISTS (
367
+ SELECT 1
368
+ FROM information_schema.columns target_columns
369
+ WHERE target_columns.table_schema = '${senatSchemaName}'
370
+ AND target_columns.table_name = table_row.tablename
371
+ AND target_columns.column_name = staging_columns.column_name
372
+ )
373
+ ORDER BY staging_columns.ordinal_position
374
+ LOOP
375
+ column_definition := column_row.data_type;
376
+ IF column_row.collation_name IS NOT NULL THEN
377
+ column_definition := column_definition || ' COLLATE ' || column_row.collation_name;
378
+ END IF;
379
+ IF column_row.generated_kind = 's' THEN
380
+ column_definition :=
381
+ column_definition ||
382
+ ' GENERATED ALWAYS AS (' ||
383
+ replace(column_row.default_expression,
384
+ '${stagingSchema}.', '${senatSchemaName}.') ||
385
+ ') STORED';
386
+ ELSIF column_row.identity_kind = 'a' THEN
387
+ column_definition := column_definition || ' GENERATED ALWAYS AS IDENTITY';
388
+ ELSIF column_row.identity_kind = 'd' THEN
389
+ column_definition := column_definition || ' GENERATED BY DEFAULT AS IDENTITY';
390
+ END IF;
391
+
392
+ EXECUTE format(
393
+ 'ALTER TABLE ${senatSchemaName}.%I ADD COLUMN %I %s',
394
+ table_row.tablename,
395
+ column_row.column_name,
396
+ column_definition
397
+ );
398
+
399
+ IF column_row.generated_kind = ''
400
+ AND column_row.identity_kind = ''
401
+ AND column_row.default_expression IS NOT NULL THEN
402
+ EXECUTE format(
403
+ 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I SET DEFAULT %s',
404
+ table_row.tablename,
405
+ column_row.column_name,
406
+ replace(column_row.default_expression, '${stagingSchema}.', '${senatSchemaName}.')
407
+ );
408
+ END IF;
409
+
410
+ IF column_row.not_null THEN
411
+ EXECUTE format(
412
+ 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I SET NOT NULL',
413
+ table_row.tablename,
414
+ column_row.column_name
415
+ );
416
+ END IF;
417
+ END LOOP;
418
+
419
+ FOR column_row IN
420
+ SELECT
421
+ staging_columns.column_name,
422
+ format_type(staging_attribute.atttypid, staging_attribute.atttypmod) AS staging_data_type,
423
+ format_type(target_attribute.atttypid, target_attribute.atttypmod) AS target_data_type,
424
+ staging_attribute.attnotnull AS staging_not_null,
425
+ target_attribute.attnotnull AS target_not_null,
426
+ staging_attribute.attidentity AS staging_identity_kind,
427
+ target_attribute.attidentity AS target_identity_kind,
428
+ staging_attribute.attgenerated AS staging_generated_kind,
429
+ target_attribute.attgenerated AS target_generated_kind,
430
+ pg_get_expr(staging_default_expr.adbin, staging_default_expr.adrelid) AS staging_default_expression,
431
+ pg_get_expr(target_default_expr.adbin, target_default_expr.adrelid) AS target_default_expression,
432
+ CASE
433
+ WHEN staging_collation.oid IS NULL THEN NULL
434
+ ELSE format('%I.%I', staging_collation_namespace.nspname, staging_collation.collname)
435
+ END AS staging_collation_name,
436
+ CASE
437
+ WHEN target_collation.oid IS NULL THEN NULL
438
+ ELSE format('%I.%I', target_collation_namespace.nspname, target_collation.collname)
439
+ END AS target_collation_name
440
+ FROM information_schema.columns staging_columns
441
+ JOIN information_schema.columns target_columns
442
+ ON target_columns.table_schema = '${senatSchemaName}'
443
+ AND target_columns.table_name = staging_columns.table_name
444
+ AND target_columns.column_name = staging_columns.column_name
445
+ JOIN pg_class staging_class ON staging_class.relname = staging_columns.table_name
446
+ JOIN pg_namespace staging_namespace
447
+ ON staging_namespace.oid = staging_class.relnamespace
448
+ AND staging_namespace.nspname = staging_columns.table_schema
449
+ JOIN pg_attribute staging_attribute
450
+ ON staging_attribute.attrelid = staging_class.oid
451
+ AND staging_attribute.attname = staging_columns.column_name
452
+ LEFT JOIN pg_attrdef staging_default_expr
453
+ ON staging_default_expr.adrelid = staging_attribute.attrelid
454
+ AND staging_default_expr.adnum = staging_attribute.attnum
455
+ LEFT JOIN pg_type staging_type ON staging_type.oid = staging_attribute.atttypid
456
+ LEFT JOIN pg_collation staging_collation
457
+ ON staging_collation.oid = staging_attribute.attcollation
458
+ AND staging_collation.oid <> staging_type.typcollation
459
+ LEFT JOIN pg_namespace staging_collation_namespace
460
+ ON staging_collation_namespace.oid = staging_collation.collnamespace
461
+ JOIN pg_class target_class ON target_class.relname = target_columns.table_name
462
+ JOIN pg_namespace target_namespace
463
+ ON target_namespace.oid = target_class.relnamespace
464
+ AND target_namespace.nspname = target_columns.table_schema
465
+ JOIN pg_attribute target_attribute
466
+ ON target_attribute.attrelid = target_class.oid
467
+ AND target_attribute.attname = target_columns.column_name
468
+ LEFT JOIN pg_attrdef target_default_expr
469
+ ON target_default_expr.adrelid = target_attribute.attrelid
470
+ AND target_default_expr.adnum = target_attribute.attnum
471
+ LEFT JOIN pg_type target_type ON target_type.oid = target_attribute.atttypid
472
+ LEFT JOIN pg_collation target_collation
473
+ ON target_collation.oid = target_attribute.attcollation
474
+ AND target_collation.oid <> target_type.typcollation
475
+ LEFT JOIN pg_namespace target_collation_namespace
476
+ ON target_collation_namespace.oid = target_collation.collnamespace
477
+ WHERE staging_columns.table_schema = '${escapedStagingSchema}'
478
+ AND staging_columns.table_name = table_row.tablename
479
+ ORDER BY staging_columns.ordinal_position
480
+ LOOP
481
+ IF column_row.staging_identity_kind <> column_row.target_identity_kind
482
+ OR column_row.staging_generated_kind <> column_row.target_generated_kind THEN
483
+ RAISE EXCEPTION
484
+ 'Unsupported identity/generated column change for %.%',
485
+ table_row.tablename,
486
+ column_row.column_name;
487
+ END IF;
488
+
489
+ IF column_row.staging_data_type <> column_row.target_data_type
490
+ OR column_row.staging_collation_name IS DISTINCT FROM column_row.target_collation_name THEN
491
+ EXECUTE format(
492
+ 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I TYPE %s%s USING %I::%s',
493
+ table_row.tablename,
494
+ column_row.column_name,
495
+ column_row.staging_data_type,
496
+ CASE
497
+ WHEN column_row.staging_collation_name IS NULL THEN ''
498
+ ELSE ' COLLATE ' || column_row.staging_collation_name
499
+ END,
500
+ column_row.column_name,
501
+ column_row.staging_data_type
502
+ );
503
+ END IF;
504
+
505
+ staging_default := replace(
506
+ coalesce(column_row.staging_default_expression, ''),
507
+ '${stagingSchema}.',
508
+ '${senatSchemaName}.'
509
+ );
510
+ target_default := replace(
511
+ coalesce(column_row.target_default_expression, ''),
512
+ '${stagingSchema}.',
513
+ '${senatSchemaName}.'
514
+ );
515
+
516
+ IF staging_default <> target_default THEN
517
+ IF column_row.staging_default_expression IS NULL THEN
518
+ EXECUTE format(
519
+ 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I DROP DEFAULT',
520
+ table_row.tablename,
521
+ column_row.column_name
522
+ );
523
+ ELSE
524
+ EXECUTE format(
525
+ 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I SET DEFAULT %s',
526
+ table_row.tablename,
527
+ column_row.column_name,
528
+ staging_default
529
+ );
530
+ END IF;
531
+ END IF;
532
+
533
+ IF column_row.staging_not_null <> column_row.target_not_null THEN
534
+ EXECUTE format(
535
+ 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I %s NOT NULL',
536
+ table_row.tablename,
537
+ column_row.column_name,
538
+ CASE WHEN column_row.staging_not_null THEN 'SET' ELSE 'DROP' END
539
+ );
540
+ END IF;
541
+ END LOOP;
542
+
543
+ SELECT obj_description(c.oid, 'pg_class')
544
+ INTO staging_table_comment
545
+ FROM pg_class c
546
+ JOIN pg_namespace n ON n.oid = c.relnamespace
547
+ WHERE n.nspname = '${escapedStagingSchema}'
548
+ AND c.relkind IN ('r', 'p', 'f')
549
+ AND c.relname = table_row.tablename;
550
+
551
+ SELECT obj_description(c.oid, 'pg_class')
552
+ INTO target_table_comment
553
+ FROM pg_class c
554
+ JOIN pg_namespace n ON n.oid = c.relnamespace
555
+ WHERE n.nspname = '${senatSchemaName}'
556
+ AND c.relkind IN ('r', 'p')
557
+ AND c.relname = table_row.tablename;
558
+
559
+ IF staging_table_comment IS DISTINCT FROM target_table_comment THEN
560
+ EXECUTE format(
561
+ 'COMMENT ON TABLE ${senatSchemaName}.%I IS %L',
562
+ table_row.tablename,
563
+ staging_table_comment
564
+ );
565
+ END IF;
566
+
567
+ FOR column_row IN
568
+ SELECT
569
+ staging_columns.column_name,
570
+ col_description(staging_class.oid, staging_attribute.attnum) AS staging_comment,
571
+ col_description(target_class.oid, target_attribute.attnum) AS target_comment
572
+ FROM information_schema.columns staging_columns
573
+ JOIN pg_class staging_class ON staging_class.relname = staging_columns.table_name
574
+ JOIN pg_namespace staging_namespace
575
+ ON staging_namespace.oid = staging_class.relnamespace
576
+ AND staging_namespace.nspname = staging_columns.table_schema
577
+ JOIN pg_attribute staging_attribute
578
+ ON staging_attribute.attrelid = staging_class.oid
579
+ AND staging_attribute.attname = staging_columns.column_name
580
+ JOIN information_schema.columns target_columns
581
+ ON target_columns.table_schema = '${senatSchemaName}'
582
+ AND target_columns.table_name = staging_columns.table_name
583
+ AND target_columns.column_name = staging_columns.column_name
584
+ JOIN pg_class target_class ON target_class.relname = target_columns.table_name
585
+ JOIN pg_namespace target_namespace
586
+ ON target_namespace.oid = target_class.relnamespace
587
+ AND target_namespace.nspname = target_columns.table_schema
588
+ JOIN pg_attribute target_attribute
589
+ ON target_attribute.attrelid = target_class.oid
590
+ AND target_attribute.attname = target_columns.column_name
591
+ WHERE staging_columns.table_schema = '${escapedStagingSchema}'
592
+ AND staging_columns.table_name = table_row.tablename
593
+ ORDER BY staging_columns.ordinal_position
594
+ LOOP
595
+ IF column_row.staging_comment IS DISTINCT FROM column_row.target_comment THEN
596
+ EXECUTE format(
597
+ 'COMMENT ON COLUMN ${senatSchemaName}.%I.%I IS %L',
598
+ table_row.tablename,
599
+ column_row.column_name,
600
+ column_row.staging_comment
601
+ );
602
+ END IF;
603
+ END LOOP;
604
+
605
+ SELECT string_agg(format('%I', columns.column_name), ', ' ORDER BY columns.ordinal_position)
606
+ , string_agg(format('s.%I', columns.column_name), ', ' ORDER BY columns.ordinal_position)
607
+ INTO column_list, staging_column_list
608
+ FROM information_schema.columns columns
609
+ WHERE columns.table_schema = '${escapedStagingSchema}'
610
+ AND columns.table_name = table_row.tablename;
611
+
612
+ SELECT
613
+ string_agg(
614
+ format('%1$I = s.%1$I', columns.column_name),
615
+ ', ' ORDER BY columns.ordinal_position),
616
+ string_agg(
617
+ format('t.%1$I IS DISTINCT FROM s.%1$I', columns.column_name),
618
+ ' OR ' ORDER BY columns.ordinal_position)
619
+ INTO update_assignments, change_condition
620
+ FROM information_schema.columns columns
621
+ WHERE columns.table_schema = '${escapedStagingSchema}'
622
+ AND columns.table_name = table_row.tablename
623
+ AND NOT (columns.column_name = ANY(coalesce(key_columns, ARRAY[]::text[])));
624
+
625
+ IF key_columns IS NOT NULL THEN
626
+ SELECT count(*)
627
+ INTO matching_key_column_count
628
+ FROM unnest(key_columns) AS key_column(column_name)
629
+ JOIN information_schema.columns staging_columns
630
+ ON staging_columns.table_schema = '${escapedStagingSchema}'
631
+ AND staging_columns.table_name = table_row.tablename
632
+ AND staging_columns.column_name = key_column.column_name
633
+ JOIN information_schema.columns target_columns
634
+ ON target_columns.table_schema = '${senatSchemaName}'
635
+ AND target_columns.table_name = table_row.tablename
636
+ AND target_columns.column_name = key_column.column_name;
637
+
638
+ IF matching_key_column_count <> cardinality(key_columns) THEN
639
+ RAISE NOTICE
640
+ 'Rejected sync key for %.% from % because only %/% columns are present in both staging and target: %',
641
+ '${senatSchemaName}',
642
+ table_row.tablename,
643
+ coalesce(key_schema_name, 'unknown'),
644
+ matching_key_column_count,
645
+ cardinality(key_columns),
646
+ array_to_string(key_columns, ', ');
647
+ key_columns := NULL;
648
+ key_schema_name := NULL;
649
+ join_condition := NULL;
650
+ END IF;
651
+ END IF;
652
+
653
+ SELECT
654
+ 'jsonb_build_array(' ||
655
+ string_agg(format('t.%I', columns.column_name),
656
+ ', ' ORDER BY columns.ordinal_position) || ')',
657
+ 'jsonb_build_array(' ||
658
+ string_agg(format('s.%I', columns.column_name),
659
+ ', ' ORDER BY columns.ordinal_position) || ')'
660
+ INTO target_row_signature_expression, staging_row_signature_expression
661
+ FROM information_schema.columns columns
662
+ WHERE columns.table_schema = '${escapedStagingSchema}'
663
+ AND columns.table_name = table_row.tablename;
664
+
665
+ IF use_row_multiset_merge THEN
666
+ sync_strategy := 'incremental_row_multiset';
667
+ RAISE NOTICE
668
+ 'Sync strategy for %.%: %',
669
+ '${senatSchemaName}',
670
+ table_row.tablename,
671
+ sync_strategy;
672
+
673
+ EXECUTE format(
674
+ 'WITH staging_counts AS (' ||
675
+ ' SELECT %2$s AS row_signature, count(*) AS row_count' ||
676
+ ' FROM %3$I.%1$I s' ||
677
+ ' GROUP BY 1' ||
678
+ '), target_ranked AS (' ||
679
+ ' SELECT ctid, %4$s AS row_signature, row_number() OVER (PARTITION BY %4$s) AS row_number' ||
680
+ ' FROM ${senatSchemaName}.%1$I t' ||
681
+ ')' ||
682
+ ' DELETE FROM ${senatSchemaName}.%1$I t' ||
683
+ ' USING (' ||
684
+ ' SELECT candidate_row.ctid' ||
685
+ ' FROM target_ranked candidate_row' ||
686
+ ' LEFT JOIN staging_counts reference_row USING (row_signature)' ||
687
+ ' WHERE candidate_row.row_number > coalesce(reference_row.row_count, 0)' ||
688
+ ' ) rows_to_delete' ||
689
+ ' WHERE t.ctid = rows_to_delete.ctid',
690
+ table_row.tablename,
691
+ staging_row_signature_expression,
692
+ '${stagingSchema}',
693
+ target_row_signature_expression
694
+ );
695
+
696
+ EXECUTE format(
697
+ 'WITH target_counts AS (' ||
698
+ ' SELECT %2$s AS row_signature, count(*) AS row_count' ||
699
+ ' FROM ${senatSchemaName}.%1$I t' ||
700
+ ' GROUP BY 1' ||
701
+ '), staging_ranked AS (' ||
702
+ ' SELECT %3$s, %4$s AS row_signature, row_number() OVER (PARTITION BY %4$s) AS row_number' ||
703
+ ' FROM %5$I.%1$I s' ||
704
+ ')' ||
705
+ ' INSERT INTO ${senatSchemaName}.%1$I (%3$s)' ||
706
+ ' SELECT %3$s' ||
707
+ ' FROM staging_ranked candidate_row' ||
708
+ ' LEFT JOIN target_counts reference_row USING (row_signature)' ||
709
+ ' WHERE candidate_row.row_number > coalesce(reference_row.row_count, 0)',
710
+ table_row.tablename,
711
+ target_row_signature_expression,
712
+ column_list,
713
+ staging_row_signature_expression,
714
+ '${stagingSchema}'
715
+ );
716
+ ELSIF key_columns IS NULL THEN
717
+ sync_strategy := 'full_refresh_fallback';
718
+ RAISE NOTICE
719
+ 'Sync strategy for %.%: %',
720
+ '${senatSchemaName}',
721
+ table_row.tablename,
722
+ sync_strategy;
723
+ RAISE WARNING
724
+ 'No primary key or all-NOT-NULL unique key found for %.%; falling back to full refresh',
725
+ '${senatSchemaName}',
726
+ table_row.tablename;
727
+
728
+ EXECUTE format('TRUNCATE TABLE ${senatSchemaName}.%I', table_row.tablename);
729
+
730
+ EXECUTE format(
731
+ 'INSERT INTO ${senatSchemaName}.%I (%s) SELECT %s FROM %I.%I s',
732
+ table_row.tablename,
733
+ column_list,
734
+ staging_column_list,
735
+ '${stagingSchema}',
736
+ table_row.tablename
737
+ );
738
+ ELSE
739
+ sync_strategy := 'incremental';
740
+ RAISE NOTICE
741
+ 'Sync strategy for %.%: % (key from %: %)',
742
+ '${senatSchemaName}',
743
+ table_row.tablename,
744
+ sync_strategy,
745
+ key_schema_name,
746
+ array_to_string(key_columns, ', ');
747
+
748
+ EXECUTE format(
749
+ 'DELETE FROM ${senatSchemaName}.%1$I t WHERE NOT EXISTS (SELECT 1 FROM %2$I.%1$I s WHERE %3$s)',
750
+ table_row.tablename,
751
+ '${stagingSchema}',
752
+ join_condition
753
+ );
754
+
755
+ IF update_assignments IS NOT NULL THEN
756
+ EXECUTE format(
757
+ 'UPDATE ${senatSchemaName}.%1$I t SET %2$s FROM %3$I.%1$I s WHERE %4$s AND (%5$s)',
758
+ table_row.tablename,
759
+ update_assignments,
760
+ '${stagingSchema}',
761
+ join_condition,
762
+ change_condition
763
+ );
764
+ END IF;
765
+
766
+ EXECUTE format(
767
+ 'INSERT INTO ${senatSchemaName}.%1$I (%2$s) ' ||
768
+ 'SELECT %3$s FROM %4$I.%1$I s ' ||
769
+ 'WHERE NOT EXISTS ' ||
770
+ '(SELECT 1 FROM ${senatSchemaName}.%1$I t WHERE %5$s)',
771
+ table_row.tablename,
772
+ column_list,
773
+ staging_column_list,
774
+ '${stagingSchema}',
775
+ join_condition
776
+ );
777
+ END IF;
778
+ END LOOP;
779
+
780
+ FOR sequence_row IN
781
+ SELECT sequencename
782
+ FROM pg_sequences
783
+ WHERE schemaname = '${escapedStagingSchema}'
784
+ ORDER BY sequencename
785
+ LOOP
786
+ SELECT
787
+ dependent_table.relname AS table_name,
788
+ dependent_column.attname AS column_name
789
+ INTO sequence_owner
790
+ FROM pg_class sequence_class
791
+ JOIN pg_namespace sequence_namespace ON sequence_namespace.oid = sequence_class.relnamespace
792
+ JOIN pg_depend dependency
793
+ ON dependency.objid = sequence_class.oid
794
+ AND dependency.deptype = 'a'
795
+ JOIN pg_class dependent_table ON dependent_table.oid = dependency.refobjid
796
+ JOIN pg_namespace dependent_namespace ON dependent_namespace.oid = dependent_table.relnamespace
797
+ JOIN pg_attribute dependent_column
798
+ ON dependent_column.attrelid = dependent_table.oid
799
+ AND dependent_column.attnum = dependency.refobjsubid
800
+ WHERE sequence_namespace.nspname = '${escapedStagingSchema}'
801
+ AND dependent_namespace.nspname = '${escapedStagingSchema}'
802
+ AND sequence_class.relname = sequence_row.sequencename;
803
+
804
+ IF sequence_owner.table_name IS NULL THEN
805
+ EXECUTE format('ALTER SEQUENCE ${senatSchemaName}.%I OWNED BY NONE', sequence_row.sequencename);
806
+ ELSE
807
+ EXECUTE format(
808
+ 'ALTER SEQUENCE ${senatSchemaName}.%I OWNED BY ${senatSchemaName}.%I.%I',
809
+ sequence_row.sequencename,
810
+ sequence_owner.table_name,
811
+ sequence_owner.column_name
812
+ );
813
+ END IF;
814
+ END LOOP;
815
+
816
+ FOR constraint_row IN
817
+ SELECT
818
+ table_class.relname AS table_name,
819
+ con.conname,
820
+ con.contype,
821
+ pg_get_constraintdef(con.oid, true) AS constraint_definition
822
+ FROM pg_constraint con
823
+ JOIN pg_class table_class ON table_class.oid = con.conrelid
824
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
825
+ WHERE table_namespace.nspname = '${escapedStagingSchema}'
826
+ AND con.contype <> 'n'
827
+ ORDER BY
828
+ CASE con.contype
829
+ WHEN 'p' THEN 1
830
+ WHEN 'u' THEN 2
831
+ WHEN 'c' THEN 3
832
+ WHEN 'f' THEN 4
833
+ ELSE 5
834
+ END,
835
+ table_class.relname,
836
+ con.conname
837
+ LOOP
838
+ EXECUTE format(
839
+ 'ALTER TABLE ${senatSchemaName}.%I ADD CONSTRAINT %I %s',
840
+ constraint_row.table_name,
841
+ constraint_row.conname,
842
+ replace(
843
+ replace(
844
+ constraint_row.constraint_definition,
845
+ ' REFERENCES ' || '${stagingSchema}' || '.',
846
+ ' REFERENCES ${senatSchemaName}.'
847
+ ),
848
+ ' REFERENCES ONLY ' || '${stagingSchema}' || '.',
849
+ ' REFERENCES ONLY ${senatSchemaName}.'
850
+ )
851
+ );
852
+ END LOOP;
853
+
854
+ FOR index_row IN
855
+ SELECT
856
+ table_class.relname AS table_name,
857
+ pg_get_indexdef(index_data.indexrelid) AS index_definition
858
+ FROM pg_index index_data
859
+ JOIN pg_class table_class ON table_class.oid = index_data.indrelid
860
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
861
+ LEFT JOIN pg_constraint con ON con.conindid = index_data.indexrelid
862
+ WHERE table_namespace.nspname = '${escapedStagingSchema}'
863
+ AND con.oid IS NULL
864
+ ORDER BY table_class.relname, pg_get_indexdef(index_data.indexrelid)
865
+ LOOP
866
+ EXECUTE replace(
867
+ index_row.index_definition,
868
+ ' ON ${stagingSchema}.',
869
+ ' ON ${senatSchemaName}.'
870
+ );
871
+ END LOOP;
872
+
873
+ FOR sequence_row IN
874
+ SELECT sequencename
875
+ FROM pg_sequences
876
+ WHERE schemaname = '${escapedStagingSchema}'
877
+ ORDER BY sequencename
878
+ LOOP
879
+ EXECUTE format(
880
+ 'SELECT last_value, is_called FROM %I.%I',
881
+ '${stagingSchema}',
882
+ sequence_row.sequencename
883
+ )
884
+ INTO sequence_state;
885
+
886
+ EXECUTE format(
887
+ 'SELECT setval(%L, %s, %L)',
888
+ '${senatSchemaName}.' || sequence_row.sequencename,
889
+ sequence_state.last_value,
890
+ sequence_state.is_called
891
+ );
892
+ END LOOP;
893
+
894
+ FOR sequence_row IN
895
+ SELECT c.relname AS sequencename
896
+ FROM pg_class c
897
+ JOIN pg_namespace n ON n.oid = c.relnamespace
898
+ WHERE n.nspname = '${senatSchemaName}'
899
+ AND c.relkind = 'S'
900
+ AND c.relname LIKE '${escapedPrefix}' || '%'
901
+ ORDER BY c.relname
902
+ LOOP
903
+ IF NOT EXISTS (
904
+ SELECT 1
905
+ FROM pg_sequences
906
+ WHERE schemaname = '${escapedStagingSchema}'
907
+ AND sequencename = sequence_row.sequencename
908
+ ) THEN
909
+ EXECUTE format('DROP SEQUENCE IF EXISTS ${senatSchemaName}.%I CASCADE', sequence_row.sequencename);
910
+ END IF;
911
+ END LOOP;
912
+ END $$;
913
+
914
+ GRANT SELECT ON ALL TABLES IN SCHEMA ${senatSchemaName} TO ${dbUser};
915
915
  `;
916
916
  }