@tricoteuses/senat 3.1.20 → 3.1.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/LICENSE.md +32 -32
  2. package/README.md +326 -326
  3. package/lib/src/other_types/collaborateurs.d.ts +5 -0
  4. package/lib/src/other_types/manifest.d.ts +11 -0
  5. package/lib/src/parsers/collaborateurs.d.ts +44 -0
  6. package/lib/src/parsers/collaborateurs.js +157 -0
  7. package/lib/src/scripts/retrieve_agenda.js +38 -4
  8. package/lib/src/scripts/retrieve_collaborateurs.js +166 -0
  9. package/lib/src/scripts/shared/incremental_import_sql.js +859 -859
  10. package/lib/src/scripts/shared/schema_version.js +84 -84
  11. package/lib/src/scripts/shared/staging_metadata_sql.js +214 -214
  12. package/lib/src/scripts/validate_prefixed_tables.js +12 -12
  13. package/lib/src/server/ameli.js +11 -11
  14. package/lib/src/server/conversion_textes.js +106 -106
  15. package/lib/src/server/debats.js +10 -10
  16. package/lib/src/server/documents.js +22 -22
  17. package/lib/src/server/dosleg.js +33 -33
  18. package/lib/src/server/questions.js +10 -10
  19. package/lib/src/server/scrutins.js +3 -3
  20. package/lib/src/server/sens.js +19 -19
  21. package/lib/src/utils/manifest.d.ts +6 -0
  22. package/lib/src/utils/manifest.js +37 -0
  23. package/lib/src/utils/reunion_deletion.d.ts +2 -0
  24. package/lib/src/utils/reunion_deletion.js +27 -0
  25. package/lib/src/videos/match.js +4 -4
  26. package/lib/tests/collaborateurs.test.js +115 -0
  27. package/package.json +117 -117
  28. package/lib/src/config.d.ts +0 -21
  29. package/lib/src/config.js +0 -27
  30. package/lib/src/conversion_textes.d.ts +0 -11
  31. package/lib/src/conversion_textes.js +0 -316
  32. package/lib/src/databases.d.ts +0 -2
  33. package/lib/src/databases.js +0 -26
  34. package/lib/src/datasets.d.ts +0 -34
  35. package/lib/src/datasets.js +0 -233
  36. package/lib/src/git.d.ts +0 -26
  37. package/lib/src/git.js +0 -217
  38. package/lib/src/loaders.d.ts +0 -52
  39. package/lib/src/loaders.js +0 -254
  40. package/lib/src/model/agenda.d.ts +0 -6
  41. package/lib/src/model/agenda.js +0 -148
  42. package/lib/src/model/ameli.d.ts +0 -55
  43. package/lib/src/model/ameli.js +0 -148
  44. package/lib/src/model/commission.d.ts +0 -18
  45. package/lib/src/model/commission.js +0 -269
  46. package/lib/src/model/debats.d.ts +0 -67
  47. package/lib/src/model/debats.js +0 -95
  48. package/lib/src/model/documents.d.ts +0 -12
  49. package/lib/src/model/documents.js +0 -141
  50. package/lib/src/model/dosleg.d.ts +0 -7
  51. package/lib/src/model/dosleg.js +0 -326
  52. package/lib/src/model/index.d.ts +0 -7
  53. package/lib/src/model/index.js +0 -7
  54. package/lib/src/model/questions.d.ts +0 -45
  55. package/lib/src/model/questions.js +0 -89
  56. package/lib/src/model/scrutins.d.ts +0 -13
  57. package/lib/src/model/scrutins.js +0 -114
  58. package/lib/src/model/seance.d.ts +0 -3
  59. package/lib/src/model/seance.js +0 -267
  60. package/lib/src/model/sens.d.ts +0 -182
  61. package/lib/src/model/sens.js +0 -485
  62. package/lib/src/model/util.d.ts +0 -9
  63. package/lib/src/model/util.js +0 -38
  64. package/lib/src/raw_types/ameli.d.ts +0 -914
  65. package/lib/src/raw_types/ameli.js +0 -5
  66. package/lib/src/raw_types/debats.d.ts +0 -207
  67. package/lib/src/raw_types/debats.js +0 -5
  68. package/lib/src/raw_types/dosleg.d.ts +0 -1619
  69. package/lib/src/raw_types/dosleg.js +0 -5
  70. package/lib/src/raw_types/questions.d.ts +0 -423
  71. package/lib/src/raw_types/questions.js +0 -5
  72. package/lib/src/raw_types/senat.d.ts +0 -11372
  73. package/lib/src/raw_types/senat.js +0 -5
  74. package/lib/src/raw_types/sens.d.ts +0 -8248
  75. package/lib/src/raw_types/sens.js +0 -5
  76. package/lib/src/raw_types_schemats/ameli.d.ts +0 -539
  77. package/lib/src/raw_types_schemats/ameli.js +0 -2
  78. package/lib/src/raw_types_schemats/debats.d.ts +0 -127
  79. package/lib/src/raw_types_schemats/debats.js +0 -2
  80. package/lib/src/raw_types_schemats/dosleg.d.ts +0 -977
  81. package/lib/src/raw_types_schemats/dosleg.js +0 -2
  82. package/lib/src/raw_types_schemats/questions.d.ts +0 -237
  83. package/lib/src/raw_types_schemats/questions.js +0 -2
  84. package/lib/src/raw_types_schemats/sens.d.ts +0 -6915
  85. package/lib/src/raw_types_schemats/sens.js +0 -2
  86. package/lib/src/scripts/test_iter_load.js +0 -12
  87. package/lib/src/types/agenda.d.ts +0 -45
  88. package/lib/src/types/ameli.d.ts +0 -5
  89. package/lib/src/types/compte_rendu.d.ts +0 -83
  90. package/lib/src/types/debats.d.ts +0 -2
  91. package/lib/src/types/debats.js +0 -1
  92. package/lib/src/types/dosleg.d.ts +0 -70
  93. package/lib/src/types/dosleg.js +0 -1
  94. package/lib/src/types/questions.d.ts +0 -2
  95. package/lib/src/types/questions.js +0 -1
  96. package/lib/src/types/sens.d.ts +0 -10
  97. package/lib/src/types/sens.js +0 -1
  98. package/lib/src/types/sessions.d.ts +0 -6
  99. package/lib/src/types/sessions.js +0 -19
  100. package/lib/src/types/texte.d.ts +0 -72
  101. package/lib/src/types/texte.js +0 -15
  102. package/lib/src/validators/config.d.ts +0 -9
  103. package/lib/src/validators/config.js +0 -10
  104. /package/lib/src/{scripts/test_iter_load.d.ts → other_types/collaborateurs.js} +0 -0
  105. /package/lib/src/{types/agenda.js → other_types/manifest.js} +0 -0
  106. /package/lib/src/{types/ameli.js → scripts/retrieve_collaborateurs.d.ts} +0 -0
  107. /package/lib/{src/types/compte_rendu.js → tests/collaborateurs.test.d.ts} +0 -0
@@ -29,60 +29,60 @@ export function buildNormalizeStagingSchemaSql(datasetName) {
29
29
  const prefix = `${datasetName}_`;
30
30
  const escapedPrefix = escapeSqlLiteral(prefix);
31
31
  const escapedStagingSchema = escapeSqlLiteral(stagingSchema);
32
- return `
33
- DO $$
34
- DECLARE
35
- relation record;
36
- table_row record;
37
- BEGIN
38
- FOR relation IN
39
- SELECT c.relname
40
- FROM pg_class c
41
- JOIN pg_namespace n ON n.oid = c.relnamespace
42
- WHERE n.nspname = '${escapedStagingSchema}'
43
- AND c.relkind = 'i'
44
- AND c.relname NOT LIKE '${escapedPrefix}' || '%'
45
- LOOP
46
- EXECUTE format(
47
- 'ALTER INDEX %I.%I RENAME TO %I',
48
- '${stagingSchema}',
49
- relation.relname,
50
- '${prefix}' || relation.relname
51
- );
52
- END LOOP;
53
-
54
- FOR relation IN
55
- SELECT c.relname
56
- FROM pg_class c
57
- JOIN pg_namespace n ON n.oid = c.relnamespace
58
- WHERE n.nspname = '${escapedStagingSchema}'
59
- AND c.relkind = 'S'
60
- AND c.relname NOT LIKE '${escapedPrefix}' || '%'
61
- LOOP
62
- EXECUTE format(
63
- 'ALTER SEQUENCE %I.%I RENAME TO %I',
64
- '${stagingSchema}',
65
- relation.relname,
66
- '${prefix}' || relation.relname
67
- );
68
- END LOOP;
69
-
70
- FOR table_row IN
71
- SELECT tablename
72
- FROM pg_tables
73
- WHERE schemaname = '${escapedStagingSchema}'
74
- ORDER BY tablename
75
- LOOP
76
- IF table_row.tablename NOT LIKE '${escapedPrefix}' || '%' THEN
77
- EXECUTE format(
78
- 'ALTER TABLE %I.%I RENAME TO %I',
79
- '${stagingSchema}',
80
- table_row.tablename,
81
- '${prefix}' || table_row.tablename
82
- );
83
- END IF;
84
- END LOOP;
85
- END $$;
32
+ return `
33
+ DO $$
34
+ DECLARE
35
+ relation record;
36
+ table_row record;
37
+ BEGIN
38
+ FOR relation IN
39
+ SELECT c.relname
40
+ FROM pg_class c
41
+ JOIN pg_namespace n ON n.oid = c.relnamespace
42
+ WHERE n.nspname = '${escapedStagingSchema}'
43
+ AND c.relkind = 'i'
44
+ AND c.relname NOT LIKE '${escapedPrefix}' || '%'
45
+ LOOP
46
+ EXECUTE format(
47
+ 'ALTER INDEX %I.%I RENAME TO %I',
48
+ '${stagingSchema}',
49
+ relation.relname,
50
+ '${prefix}' || relation.relname
51
+ );
52
+ END LOOP;
53
+
54
+ FOR relation IN
55
+ SELECT c.relname
56
+ FROM pg_class c
57
+ JOIN pg_namespace n ON n.oid = c.relnamespace
58
+ WHERE n.nspname = '${escapedStagingSchema}'
59
+ AND c.relkind = 'S'
60
+ AND c.relname NOT LIKE '${escapedPrefix}' || '%'
61
+ LOOP
62
+ EXECUTE format(
63
+ 'ALTER SEQUENCE %I.%I RENAME TO %I',
64
+ '${stagingSchema}',
65
+ relation.relname,
66
+ '${prefix}' || relation.relname
67
+ );
68
+ END LOOP;
69
+
70
+ FOR table_row IN
71
+ SELECT tablename
72
+ FROM pg_tables
73
+ WHERE schemaname = '${escapedStagingSchema}'
74
+ ORDER BY tablename
75
+ LOOP
76
+ IF table_row.tablename NOT LIKE '${escapedPrefix}' || '%' THEN
77
+ EXECUTE format(
78
+ 'ALTER TABLE %I.%I RENAME TO %I',
79
+ '${stagingSchema}',
80
+ table_row.tablename,
81
+ '${prefix}' || table_row.tablename
82
+ );
83
+ END IF;
84
+ END LOOP;
85
+ END $$;
86
86
  `;
87
87
  }
88
88
  export function buildIncrementalDatasetImportSql(datasetName, dbUser, mergeKeys, rowMultisetMergeTables) {
@@ -100,810 +100,810 @@ export function buildIncrementalDatasetImportSql(datasetName, dbUser, mergeKeys,
100
100
  " AND c.relkind IN ('r', 'p', 'f')",
101
101
  "ORDER BY c.relname",
102
102
  ].join("\n");
103
- return `
104
- DO $$
105
- DECLARE
106
- table_row record;
107
- column_row record;
108
- sequence_row record;
109
- constraint_row record;
110
- index_row record;
111
- sequence_owner record;
112
- sequence_state record;
113
- staging_default text;
114
- target_default text;
115
- column_definition text;
116
- column_list text;
117
- staging_column_list text;
118
- key_columns text[];
119
- key_schema_name text;
120
- configured_key_columns text[];
121
- matching_key_column_count integer;
122
- use_row_multiset_merge boolean;
123
- join_condition text;
124
- update_assignments text;
125
- change_condition text;
126
- target_row_signature_expression text;
127
- staging_row_signature_expression text;
128
- sync_strategy text;
129
- staging_table_comment text;
130
- target_table_comment text;
131
- BEGIN
132
- PERFORM set_config('search_path', '${senatSchemaName}, pg_catalog', true);
133
-
134
- FOR table_row IN
135
- ${stagingRelationsQuery}
136
- LOOP
137
- IF NOT EXISTS (
138
- SELECT 1
139
- FROM pg_tables
140
- WHERE schemaname = '${senatSchemaName}'
141
- AND tablename = table_row.tablename
142
- ) THEN
143
- EXECUTE format(
144
- 'CREATE TABLE ${senatSchemaName}.%I (LIKE %I.%I INCLUDING IDENTITY INCLUDING GENERATED INCLUDING DEFAULTS)',
145
- table_row.tablename,
146
- '${stagingSchema}',
147
- table_row.tablename
148
- );
149
- END IF;
150
- END LOOP;
151
-
152
- FOR sequence_row IN
153
- SELECT
154
- sequencename,
155
- data_type,
156
- start_value,
157
- min_value,
158
- max_value,
159
- increment_by,
160
- cycle,
161
- cache_size
162
- FROM pg_sequences
163
- WHERE schemaname = '${escapedStagingSchema}'
164
- ORDER BY sequencename
165
- LOOP
166
- IF NOT EXISTS (
167
- SELECT 1
168
- FROM pg_class c
169
- JOIN pg_namespace n ON n.oid = c.relnamespace
170
- WHERE n.nspname = '${senatSchemaName}'
171
- AND c.relkind = 'S'
172
- AND c.relname = sequence_row.sequencename
173
- ) THEN
174
- EXECUTE format('CREATE SEQUENCE ${senatSchemaName}.%I', sequence_row.sequencename);
175
- END IF;
176
-
177
- EXECUTE format(
178
- 'ALTER SEQUENCE ${senatSchemaName}.%I AS %s INCREMENT BY %s MINVALUE %s MAXVALUE %s START WITH %s CACHE %s %s',
179
- sequence_row.sequencename,
180
- sequence_row.data_type,
181
- sequence_row.increment_by,
182
- sequence_row.min_value,
183
- sequence_row.max_value,
184
- sequence_row.start_value,
185
- sequence_row.cache_size,
186
- CASE WHEN sequence_row.cycle THEN 'CYCLE' ELSE 'NO CYCLE' END
187
- );
188
- END LOOP;
189
-
190
- FOR table_row IN
191
- SELECT tablename
192
- FROM pg_tables
193
- WHERE schemaname = '${senatSchemaName}'
194
- AND tablename LIKE '${escapedPrefix}' || '%'
195
- ORDER BY tablename
196
- LOOP
197
- IF NOT EXISTS (
198
- SELECT 1
199
- FROM pg_class c
200
- JOIN pg_namespace n ON n.oid = c.relnamespace
201
- WHERE n.nspname = '${escapedStagingSchema}'
202
- AND c.relkind IN ('r', 'p', 'f')
203
- AND c.relname = table_row.tablename
204
- ) THEN
205
- EXECUTE format('DROP TABLE IF EXISTS ${senatSchemaName}.%I CASCADE', table_row.tablename);
206
- END IF;
207
- END LOOP;
208
-
209
- FOR table_row IN
210
- ${stagingRelationsQuery}
211
- LOOP
212
- key_columns := NULL;
213
- key_schema_name := NULL;
214
- configured_key_columns := NULL;
215
- matching_key_column_count := 0;
216
- use_row_multiset_merge := false;
217
- join_condition := NULL;
218
- update_assignments := NULL;
219
- change_condition := NULL;
220
- target_row_signature_expression := NULL;
221
- staging_row_signature_expression := NULL;
222
-
223
- SELECT configured.key_columns
224
- INTO configured_key_columns
225
- FROM (
226
- ${configuredMergeKeysValues}
227
- ) AS configured
228
- WHERE configured.tablename = table_row.tablename;
229
-
230
- IF configured_key_columns IS NOT NULL THEN
231
- key_columns := configured_key_columns;
232
- key_schema_name := 'configured';
233
-
234
- SELECT string_agg(format('t.%1$I IS NOT DISTINCT FROM s.%1$I', key_column), ' AND ' ORDER BY ordinality)
235
- INTO join_condition
236
- FROM unnest(configured_key_columns) WITH ORDINALITY AS configured_key(key_column, ordinality);
237
- ELSE
238
- SELECT candidate.key_columns, candidate.join_condition, candidate.source_schema
239
- INTO key_columns, join_condition, key_schema_name
240
- FROM (
241
- SELECT
242
- array_agg(att.attname ORDER BY key_column.ordinality) AS key_columns,
243
- string_agg(format('t.%1$I = s.%1$I', att.attname), ' AND ' ORDER BY key_column.ordinality) AS join_condition,
244
- table_namespace.nspname AS source_schema,
245
- CASE WHEN table_namespace.nspname = '${escapedStagingSchema}' THEN 0 ELSE 1 END AS schema_priority,
246
- CASE WHEN index_data.indisprimary THEN 0 ELSE 1 END AS priority,
247
- count(*) AS key_column_count,
248
- index_data.indexrelid
249
- FROM pg_index index_data
250
- JOIN pg_class table_class ON table_class.oid = index_data.indrelid
251
- JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
252
- JOIN LATERAL unnest(index_data.indkey::int2[]) WITH ORDINALITY AS key_column(attnum, ordinality) ON true
253
- JOIN pg_attribute att
254
- ON att.attrelid = table_class.oid
255
- AND att.attnum = key_column.attnum
256
- WHERE table_namespace.nspname IN ('${escapedStagingSchema}', '${senatSchemaName}')
257
- AND table_class.relname = table_row.tablename
258
- AND index_data.indisunique
259
- AND index_data.indpred IS NULL
260
- AND index_data.indexprs IS NULL
261
- AND key_column.ordinality <= index_data.indnkeyatts
262
- AND key_column.attnum > 0
263
- AND EXISTS (
264
- SELECT 1
265
- FROM information_schema.columns staging_columns
266
- WHERE staging_columns.table_schema = '${escapedStagingSchema}'
267
- AND staging_columns.table_name = table_row.tablename
268
- AND staging_columns.column_name = att.attname
269
- )
270
- GROUP BY index_data.indexrelid, index_data.indisprimary, table_namespace.nspname
271
- HAVING index_data.indisprimary OR bool_and(att.attnotnull)
272
- ORDER BY schema_priority, priority, key_column_count, index_data.indexrelid
273
- LIMIT 1
274
- ) AS candidate;
275
- END IF;
276
-
277
- SELECT EXISTS (
278
- SELECT 1
279
- FROM (
280
- ${configuredRowMultisetTablesValues}
281
- ) AS configured
282
- WHERE configured.tablename = table_row.tablename
283
- )
284
- INTO use_row_multiset_merge;
285
-
286
- FOR constraint_row IN
287
- SELECT con.conname
288
- FROM pg_constraint con
289
- JOIN pg_class c ON c.oid = con.conrelid
290
- JOIN pg_namespace n ON n.oid = c.relnamespace
291
- WHERE n.nspname = '${senatSchemaName}'
292
- AND c.relname = table_row.tablename
293
- AND con.contype <> 'n'
294
- LOOP
295
- EXECUTE format(
296
- 'ALTER TABLE ${senatSchemaName}.%I DROP CONSTRAINT IF EXISTS %I CASCADE',
297
- table_row.tablename,
298
- constraint_row.conname
299
- );
300
- END LOOP;
301
-
302
- FOR index_row IN
303
- SELECT index_class.relname AS index_name
304
- FROM pg_index idx
305
- JOIN pg_class table_class ON table_class.oid = idx.indrelid
306
- JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
307
- JOIN pg_class index_class ON index_class.oid = idx.indexrelid
308
- LEFT JOIN pg_constraint con ON con.conindid = idx.indexrelid
309
- WHERE table_namespace.nspname = '${senatSchemaName}'
310
- AND table_class.relname = table_row.tablename
311
- AND con.oid IS NULL
312
- LOOP
313
- EXECUTE format('DROP INDEX IF EXISTS ${senatSchemaName}.%I', index_row.index_name);
314
- END LOOP;
315
-
316
- FOR column_row IN
317
- SELECT target_columns.column_name
318
- FROM information_schema.columns target_columns
319
- WHERE target_columns.table_schema = '${senatSchemaName}'
320
- AND target_columns.table_name = table_row.tablename
321
- AND NOT EXISTS (
322
- SELECT 1
323
- FROM information_schema.columns staging_columns
324
- WHERE staging_columns.table_schema = '${escapedStagingSchema}'
325
- AND staging_columns.table_name = table_row.tablename
326
- AND staging_columns.column_name = target_columns.column_name
327
- )
328
- ORDER BY target_columns.ordinal_position DESC
329
- LOOP
330
- EXECUTE format(
331
- 'ALTER TABLE ${senatSchemaName}.%I DROP COLUMN IF EXISTS %I CASCADE',
332
- table_row.tablename,
333
- column_row.column_name
334
- );
335
- END LOOP;
336
-
337
- FOR column_row IN
338
- SELECT
339
- staging_columns.column_name,
340
- format_type(a.atttypid, a.atttypmod) AS data_type,
341
- a.attnotnull AS not_null,
342
- a.attidentity AS identity_kind,
343
- a.attgenerated AS generated_kind,
344
- pg_get_expr(ad.adbin, ad.adrelid) AS default_expression,
345
- CASE
346
- WHEN coll.oid IS NULL THEN NULL
347
- ELSE format('%I.%I', coll_namespace.nspname, coll.collname)
348
- END AS collation_name
349
- FROM information_schema.columns staging_columns
350
- JOIN pg_class c ON c.relname = staging_columns.table_name
351
- JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = staging_columns.table_schema
352
- JOIN pg_attribute a ON a.attrelid = c.oid AND a.attname = staging_columns.column_name
353
- LEFT JOIN pg_attrdef ad ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum
354
- LEFT JOIN pg_type typ ON typ.oid = a.atttypid
355
- LEFT JOIN pg_collation coll ON coll.oid = a.attcollation AND coll.oid <> typ.typcollation
356
- LEFT JOIN pg_namespace coll_namespace ON coll_namespace.oid = coll.collnamespace
357
- WHERE staging_columns.table_schema = '${escapedStagingSchema}'
358
- AND staging_columns.table_name = table_row.tablename
359
- AND NOT EXISTS (
360
- SELECT 1
361
- FROM information_schema.columns target_columns
362
- WHERE target_columns.table_schema = '${senatSchemaName}'
363
- AND target_columns.table_name = table_row.tablename
364
- AND target_columns.column_name = staging_columns.column_name
365
- )
366
- ORDER BY staging_columns.ordinal_position
367
- LOOP
368
- column_definition := column_row.data_type;
369
- IF column_row.collation_name IS NOT NULL THEN
370
- column_definition := column_definition || ' COLLATE ' || column_row.collation_name;
371
- END IF;
372
- IF column_row.generated_kind = 's' THEN
373
- column_definition :=
374
- column_definition ||
375
- ' GENERATED ALWAYS AS (' ||
376
- replace(column_row.default_expression,
377
- '${stagingSchema}.', '${senatSchemaName}.') ||
378
- ') STORED';
379
- ELSIF column_row.identity_kind = 'a' THEN
380
- column_definition := column_definition || ' GENERATED ALWAYS AS IDENTITY';
381
- ELSIF column_row.identity_kind = 'd' THEN
382
- column_definition := column_definition || ' GENERATED BY DEFAULT AS IDENTITY';
383
- END IF;
384
-
385
- EXECUTE format(
386
- 'ALTER TABLE ${senatSchemaName}.%I ADD COLUMN %I %s',
387
- table_row.tablename,
388
- column_row.column_name,
389
- column_definition
390
- );
391
-
392
- IF column_row.generated_kind = ''
393
- AND column_row.identity_kind = ''
394
- AND column_row.default_expression IS NOT NULL THEN
395
- EXECUTE format(
396
- 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I SET DEFAULT %s',
397
- table_row.tablename,
398
- column_row.column_name,
399
- replace(column_row.default_expression, '${stagingSchema}.', '${senatSchemaName}.')
400
- );
401
- END IF;
402
-
403
- IF column_row.not_null THEN
404
- EXECUTE format(
405
- 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I SET NOT NULL',
406
- table_row.tablename,
407
- column_row.column_name
408
- );
409
- END IF;
410
- END LOOP;
411
-
412
- FOR column_row IN
413
- SELECT
414
- staging_columns.column_name,
415
- format_type(staging_attribute.atttypid, staging_attribute.atttypmod) AS staging_data_type,
416
- format_type(target_attribute.atttypid, target_attribute.atttypmod) AS target_data_type,
417
- staging_attribute.attnotnull AS staging_not_null,
418
- target_attribute.attnotnull AS target_not_null,
419
- staging_attribute.attidentity AS staging_identity_kind,
420
- target_attribute.attidentity AS target_identity_kind,
421
- staging_attribute.attgenerated AS staging_generated_kind,
422
- target_attribute.attgenerated AS target_generated_kind,
423
- pg_get_expr(staging_default_expr.adbin, staging_default_expr.adrelid) AS staging_default_expression,
424
- pg_get_expr(target_default_expr.adbin, target_default_expr.adrelid) AS target_default_expression,
425
- CASE
426
- WHEN staging_collation.oid IS NULL THEN NULL
427
- ELSE format('%I.%I', staging_collation_namespace.nspname, staging_collation.collname)
428
- END AS staging_collation_name,
429
- CASE
430
- WHEN target_collation.oid IS NULL THEN NULL
431
- ELSE format('%I.%I', target_collation_namespace.nspname, target_collation.collname)
432
- END AS target_collation_name
433
- FROM information_schema.columns staging_columns
434
- JOIN information_schema.columns target_columns
435
- ON target_columns.table_schema = '${senatSchemaName}'
436
- AND target_columns.table_name = staging_columns.table_name
437
- AND target_columns.column_name = staging_columns.column_name
438
- JOIN pg_class staging_class ON staging_class.relname = staging_columns.table_name
439
- JOIN pg_namespace staging_namespace
440
- ON staging_namespace.oid = staging_class.relnamespace
441
- AND staging_namespace.nspname = staging_columns.table_schema
442
- JOIN pg_attribute staging_attribute
443
- ON staging_attribute.attrelid = staging_class.oid
444
- AND staging_attribute.attname = staging_columns.column_name
445
- LEFT JOIN pg_attrdef staging_default_expr
446
- ON staging_default_expr.adrelid = staging_attribute.attrelid
447
- AND staging_default_expr.adnum = staging_attribute.attnum
448
- LEFT JOIN pg_type staging_type ON staging_type.oid = staging_attribute.atttypid
449
- LEFT JOIN pg_collation staging_collation
450
- ON staging_collation.oid = staging_attribute.attcollation
451
- AND staging_collation.oid <> staging_type.typcollation
452
- LEFT JOIN pg_namespace staging_collation_namespace
453
- ON staging_collation_namespace.oid = staging_collation.collnamespace
454
- JOIN pg_class target_class ON target_class.relname = target_columns.table_name
455
- JOIN pg_namespace target_namespace
456
- ON target_namespace.oid = target_class.relnamespace
457
- AND target_namespace.nspname = target_columns.table_schema
458
- JOIN pg_attribute target_attribute
459
- ON target_attribute.attrelid = target_class.oid
460
- AND target_attribute.attname = target_columns.column_name
461
- LEFT JOIN pg_attrdef target_default_expr
462
- ON target_default_expr.adrelid = target_attribute.attrelid
463
- AND target_default_expr.adnum = target_attribute.attnum
464
- LEFT JOIN pg_type target_type ON target_type.oid = target_attribute.atttypid
465
- LEFT JOIN pg_collation target_collation
466
- ON target_collation.oid = target_attribute.attcollation
467
- AND target_collation.oid <> target_type.typcollation
468
- LEFT JOIN pg_namespace target_collation_namespace
469
- ON target_collation_namespace.oid = target_collation.collnamespace
470
- WHERE staging_columns.table_schema = '${escapedStagingSchema}'
471
- AND staging_columns.table_name = table_row.tablename
472
- ORDER BY staging_columns.ordinal_position
473
- LOOP
474
- IF column_row.staging_identity_kind <> column_row.target_identity_kind
475
- OR column_row.staging_generated_kind <> column_row.target_generated_kind THEN
476
- RAISE EXCEPTION
477
- 'Unsupported identity/generated column change for %.%',
478
- table_row.tablename,
479
- column_row.column_name;
480
- END IF;
481
-
482
- IF column_row.staging_data_type <> column_row.target_data_type
483
- OR column_row.staging_collation_name IS DISTINCT FROM column_row.target_collation_name THEN
484
- EXECUTE format(
485
- 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I TYPE %s%s USING %I::%s',
486
- table_row.tablename,
487
- column_row.column_name,
488
- column_row.staging_data_type,
489
- CASE
490
- WHEN column_row.staging_collation_name IS NULL THEN ''
491
- ELSE ' COLLATE ' || column_row.staging_collation_name
492
- END,
493
- column_row.column_name,
494
- column_row.staging_data_type
495
- );
496
- END IF;
497
-
498
- staging_default := replace(
499
- coalesce(column_row.staging_default_expression, ''),
500
- '${stagingSchema}.',
501
- '${senatSchemaName}.'
502
- );
503
- target_default := replace(
504
- coalesce(column_row.target_default_expression, ''),
505
- '${stagingSchema}.',
506
- '${senatSchemaName}.'
507
- );
508
-
509
- IF staging_default <> target_default THEN
510
- IF column_row.staging_default_expression IS NULL THEN
511
- EXECUTE format(
512
- 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I DROP DEFAULT',
513
- table_row.tablename,
514
- column_row.column_name
515
- );
516
- ELSE
517
- EXECUTE format(
518
- 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I SET DEFAULT %s',
519
- table_row.tablename,
520
- column_row.column_name,
521
- staging_default
522
- );
523
- END IF;
524
- END IF;
525
-
526
- IF column_row.staging_not_null <> column_row.target_not_null THEN
527
- EXECUTE format(
528
- 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I %s NOT NULL',
529
- table_row.tablename,
530
- column_row.column_name,
531
- CASE WHEN column_row.staging_not_null THEN 'SET' ELSE 'DROP' END
532
- );
533
- END IF;
534
- END LOOP;
535
-
536
- SELECT obj_description(c.oid, 'pg_class')
537
- INTO staging_table_comment
538
- FROM pg_class c
539
- JOIN pg_namespace n ON n.oid = c.relnamespace
540
- WHERE n.nspname = '${escapedStagingSchema}'
541
- AND c.relkind IN ('r', 'p', 'f')
542
- AND c.relname = table_row.tablename;
543
-
544
- SELECT obj_description(c.oid, 'pg_class')
545
- INTO target_table_comment
546
- FROM pg_class c
547
- JOIN pg_namespace n ON n.oid = c.relnamespace
548
- WHERE n.nspname = '${senatSchemaName}'
549
- AND c.relkind IN ('r', 'p')
550
- AND c.relname = table_row.tablename;
551
-
552
- IF staging_table_comment IS DISTINCT FROM target_table_comment THEN
553
- EXECUTE format(
554
- 'COMMENT ON TABLE ${senatSchemaName}.%I IS %L',
555
- table_row.tablename,
556
- staging_table_comment
557
- );
558
- END IF;
559
-
560
- FOR column_row IN
561
- SELECT
562
- staging_columns.column_name,
563
- col_description(staging_class.oid, staging_attribute.attnum) AS staging_comment,
564
- col_description(target_class.oid, target_attribute.attnum) AS target_comment
565
- FROM information_schema.columns staging_columns
566
- JOIN pg_class staging_class ON staging_class.relname = staging_columns.table_name
567
- JOIN pg_namespace staging_namespace
568
- ON staging_namespace.oid = staging_class.relnamespace
569
- AND staging_namespace.nspname = staging_columns.table_schema
570
- JOIN pg_attribute staging_attribute
571
- ON staging_attribute.attrelid = staging_class.oid
572
- AND staging_attribute.attname = staging_columns.column_name
573
- JOIN information_schema.columns target_columns
574
- ON target_columns.table_schema = '${senatSchemaName}'
575
- AND target_columns.table_name = staging_columns.table_name
576
- AND target_columns.column_name = staging_columns.column_name
577
- JOIN pg_class target_class ON target_class.relname = target_columns.table_name
578
- JOIN pg_namespace target_namespace
579
- ON target_namespace.oid = target_class.relnamespace
580
- AND target_namespace.nspname = target_columns.table_schema
581
- JOIN pg_attribute target_attribute
582
- ON target_attribute.attrelid = target_class.oid
583
- AND target_attribute.attname = target_columns.column_name
584
- WHERE staging_columns.table_schema = '${escapedStagingSchema}'
585
- AND staging_columns.table_name = table_row.tablename
586
- ORDER BY staging_columns.ordinal_position
587
- LOOP
588
- IF column_row.staging_comment IS DISTINCT FROM column_row.target_comment THEN
589
- EXECUTE format(
590
- 'COMMENT ON COLUMN ${senatSchemaName}.%I.%I IS %L',
591
- table_row.tablename,
592
- column_row.column_name,
593
- column_row.staging_comment
594
- );
595
- END IF;
596
- END LOOP;
597
-
598
- SELECT string_agg(format('%I', columns.column_name), ', ' ORDER BY columns.ordinal_position)
599
- , string_agg(format('s.%I', columns.column_name), ', ' ORDER BY columns.ordinal_position)
600
- INTO column_list, staging_column_list
601
- FROM information_schema.columns columns
602
- WHERE columns.table_schema = '${escapedStagingSchema}'
603
- AND columns.table_name = table_row.tablename;
604
-
605
- SELECT
606
- string_agg(
607
- format('%1$I = s.%1$I', columns.column_name),
608
- ', ' ORDER BY columns.ordinal_position),
609
- string_agg(
610
- format('t.%1$I IS DISTINCT FROM s.%1$I', columns.column_name),
611
- ' OR ' ORDER BY columns.ordinal_position)
612
- INTO update_assignments, change_condition
613
- FROM information_schema.columns columns
614
- WHERE columns.table_schema = '${escapedStagingSchema}'
615
- AND columns.table_name = table_row.tablename
616
- AND NOT (columns.column_name = ANY(coalesce(key_columns, ARRAY[]::text[])));
617
-
618
- IF key_columns IS NOT NULL THEN
619
- SELECT count(*)
620
- INTO matching_key_column_count
621
- FROM unnest(key_columns) AS key_column(column_name)
622
- JOIN information_schema.columns staging_columns
623
- ON staging_columns.table_schema = '${escapedStagingSchema}'
624
- AND staging_columns.table_name = table_row.tablename
625
- AND staging_columns.column_name = key_column.column_name
626
- JOIN information_schema.columns target_columns
627
- ON target_columns.table_schema = '${senatSchemaName}'
628
- AND target_columns.table_name = table_row.tablename
629
- AND target_columns.column_name = key_column.column_name;
630
-
631
- IF matching_key_column_count <> cardinality(key_columns) THEN
632
- RAISE NOTICE
633
- 'Rejected sync key for %.% from % because only %/% columns are present in both staging and target: %',
634
- '${senatSchemaName}',
635
- table_row.tablename,
636
- coalesce(key_schema_name, 'unknown'),
637
- matching_key_column_count,
638
- cardinality(key_columns),
639
- array_to_string(key_columns, ', ');
640
- key_columns := NULL;
641
- key_schema_name := NULL;
642
- join_condition := NULL;
643
- END IF;
644
- END IF;
645
-
646
- SELECT
647
- 'jsonb_build_array(' ||
648
- string_agg(format('t.%I', columns.column_name),
649
- ', ' ORDER BY columns.ordinal_position) || ')',
650
- 'jsonb_build_array(' ||
651
- string_agg(format('s.%I', columns.column_name),
652
- ', ' ORDER BY columns.ordinal_position) || ')'
653
- INTO target_row_signature_expression, staging_row_signature_expression
654
- FROM information_schema.columns columns
655
- WHERE columns.table_schema = '${escapedStagingSchema}'
656
- AND columns.table_name = table_row.tablename;
657
-
658
- IF use_row_multiset_merge THEN
659
- sync_strategy := 'incremental_row_multiset';
660
- RAISE NOTICE
661
- 'Sync strategy for %.%: %',
662
- '${senatSchemaName}',
663
- table_row.tablename,
664
- sync_strategy;
665
-
666
- EXECUTE format(
667
- 'WITH staging_counts AS (' ||
668
- ' SELECT %2$s AS row_signature, count(*) AS row_count' ||
669
- ' FROM %3$I.%1$I s' ||
670
- ' GROUP BY 1' ||
671
- '), target_ranked AS (' ||
672
- ' SELECT ctid, %4$s AS row_signature, row_number() OVER (PARTITION BY %4$s) AS row_number' ||
673
- ' FROM ${senatSchemaName}.%1$I t' ||
674
- ')' ||
675
- ' DELETE FROM ${senatSchemaName}.%1$I t' ||
676
- ' USING (' ||
677
- ' SELECT candidate_row.ctid' ||
678
- ' FROM target_ranked candidate_row' ||
679
- ' LEFT JOIN staging_counts reference_row USING (row_signature)' ||
680
- ' WHERE candidate_row.row_number > coalesce(reference_row.row_count, 0)' ||
681
- ' ) rows_to_delete' ||
682
- ' WHERE t.ctid = rows_to_delete.ctid',
683
- table_row.tablename,
684
- staging_row_signature_expression,
685
- '${stagingSchema}',
686
- target_row_signature_expression
687
- );
688
-
689
- EXECUTE format(
690
- 'WITH target_counts AS (' ||
691
- ' SELECT %2$s AS row_signature, count(*) AS row_count' ||
692
- ' FROM ${senatSchemaName}.%1$I t' ||
693
- ' GROUP BY 1' ||
694
- '), staging_ranked AS (' ||
695
- ' SELECT %3$s, %4$s AS row_signature, row_number() OVER (PARTITION BY %4$s) AS row_number' ||
696
- ' FROM %5$I.%1$I s' ||
697
- ')' ||
698
- ' INSERT INTO ${senatSchemaName}.%1$I (%3$s)' ||
699
- ' SELECT %3$s' ||
700
- ' FROM staging_ranked candidate_row' ||
701
- ' LEFT JOIN target_counts reference_row USING (row_signature)' ||
702
- ' WHERE candidate_row.row_number > coalesce(reference_row.row_count, 0)',
703
- table_row.tablename,
704
- target_row_signature_expression,
705
- column_list,
706
- staging_row_signature_expression,
707
- '${stagingSchema}'
708
- );
709
- ELSIF key_columns IS NULL THEN
710
- sync_strategy := 'full_refresh_fallback';
711
- RAISE NOTICE
712
- 'Sync strategy for %.%: %',
713
- '${senatSchemaName}',
714
- table_row.tablename,
715
- sync_strategy;
716
- RAISE WARNING
717
- 'No primary key or all-NOT-NULL unique key found for %.%; falling back to full refresh',
718
- '${senatSchemaName}',
719
- table_row.tablename;
720
-
721
- EXECUTE format('TRUNCATE TABLE ${senatSchemaName}.%I', table_row.tablename);
722
-
723
- EXECUTE format(
724
- 'INSERT INTO ${senatSchemaName}.%I (%s) SELECT %s FROM %I.%I s',
725
- table_row.tablename,
726
- column_list,
727
- staging_column_list,
728
- '${stagingSchema}',
729
- table_row.tablename
730
- );
731
- ELSE
732
- sync_strategy := 'incremental';
733
- RAISE NOTICE
734
- 'Sync strategy for %.%: % (key from %: %)',
735
- '${senatSchemaName}',
736
- table_row.tablename,
737
- sync_strategy,
738
- key_schema_name,
739
- array_to_string(key_columns, ', ');
740
-
741
- EXECUTE format(
742
- 'DELETE FROM ${senatSchemaName}.%1$I t WHERE NOT EXISTS (SELECT 1 FROM %2$I.%1$I s WHERE %3$s)',
743
- table_row.tablename,
744
- '${stagingSchema}',
745
- join_condition
746
- );
747
-
748
- IF update_assignments IS NOT NULL THEN
749
- EXECUTE format(
750
- 'UPDATE ${senatSchemaName}.%1$I t SET %2$s FROM %3$I.%1$I s WHERE %4$s AND (%5$s)',
751
- table_row.tablename,
752
- update_assignments,
753
- '${stagingSchema}',
754
- join_condition,
755
- change_condition
756
- );
757
- END IF;
758
-
759
- EXECUTE format(
760
- 'INSERT INTO ${senatSchemaName}.%1$I (%2$s) ' ||
761
- 'SELECT %3$s FROM %4$I.%1$I s ' ||
762
- 'WHERE NOT EXISTS ' ||
763
- '(SELECT 1 FROM ${senatSchemaName}.%1$I t WHERE %5$s)',
764
- table_row.tablename,
765
- column_list,
766
- staging_column_list,
767
- '${stagingSchema}',
768
- join_condition
769
- );
770
- END IF;
771
- END LOOP;
772
-
773
- FOR sequence_row IN
774
- SELECT sequencename
775
- FROM pg_sequences
776
- WHERE schemaname = '${escapedStagingSchema}'
777
- ORDER BY sequencename
778
- LOOP
779
- SELECT
780
- dependent_table.relname AS table_name,
781
- dependent_column.attname AS column_name
782
- INTO sequence_owner
783
- FROM pg_class sequence_class
784
- JOIN pg_namespace sequence_namespace ON sequence_namespace.oid = sequence_class.relnamespace
785
- JOIN pg_depend dependency
786
- ON dependency.objid = sequence_class.oid
787
- AND dependency.deptype = 'a'
788
- JOIN pg_class dependent_table ON dependent_table.oid = dependency.refobjid
789
- JOIN pg_namespace dependent_namespace ON dependent_namespace.oid = dependent_table.relnamespace
790
- JOIN pg_attribute dependent_column
791
- ON dependent_column.attrelid = dependent_table.oid
792
- AND dependent_column.attnum = dependency.refobjsubid
793
- WHERE sequence_namespace.nspname = '${escapedStagingSchema}'
794
- AND dependent_namespace.nspname = '${escapedStagingSchema}'
795
- AND sequence_class.relname = sequence_row.sequencename;
796
-
797
- IF sequence_owner.table_name IS NULL THEN
798
- EXECUTE format('ALTER SEQUENCE ${senatSchemaName}.%I OWNED BY NONE', sequence_row.sequencename);
799
- ELSE
800
- EXECUTE format(
801
- 'ALTER SEQUENCE ${senatSchemaName}.%I OWNED BY ${senatSchemaName}.%I.%I',
802
- sequence_row.sequencename,
803
- sequence_owner.table_name,
804
- sequence_owner.column_name
805
- );
806
- END IF;
807
- END LOOP;
808
-
809
- FOR constraint_row IN
810
- SELECT
811
- table_class.relname AS table_name,
812
- con.conname,
813
- con.contype,
814
- pg_get_constraintdef(con.oid, true) AS constraint_definition
815
- FROM pg_constraint con
816
- JOIN pg_class table_class ON table_class.oid = con.conrelid
817
- JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
818
- WHERE table_namespace.nspname = '${escapedStagingSchema}'
819
- AND con.contype <> 'n'
820
- ORDER BY
821
- CASE con.contype
822
- WHEN 'p' THEN 1
823
- WHEN 'u' THEN 2
824
- WHEN 'c' THEN 3
825
- WHEN 'f' THEN 4
826
- ELSE 5
827
- END,
828
- table_class.relname,
829
- con.conname
830
- LOOP
831
- EXECUTE format(
832
- 'ALTER TABLE ${senatSchemaName}.%I ADD CONSTRAINT %I %s',
833
- constraint_row.table_name,
834
- constraint_row.conname,
835
- replace(
836
- replace(
837
- constraint_row.constraint_definition,
838
- ' REFERENCES ' || '${stagingSchema}' || '.',
839
- ' REFERENCES ${senatSchemaName}.'
840
- ),
841
- ' REFERENCES ONLY ' || '${stagingSchema}' || '.',
842
- ' REFERENCES ONLY ${senatSchemaName}.'
843
- )
844
- );
845
- END LOOP;
846
-
847
- FOR index_row IN
848
- SELECT
849
- table_class.relname AS table_name,
850
- pg_get_indexdef(index_data.indexrelid) AS index_definition
851
- FROM pg_index index_data
852
- JOIN pg_class table_class ON table_class.oid = index_data.indrelid
853
- JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
854
- LEFT JOIN pg_constraint con ON con.conindid = index_data.indexrelid
855
- WHERE table_namespace.nspname = '${escapedStagingSchema}'
856
- AND con.oid IS NULL
857
- ORDER BY table_class.relname, pg_get_indexdef(index_data.indexrelid)
858
- LOOP
859
- EXECUTE replace(
860
- index_row.index_definition,
861
- ' ON ${stagingSchema}.',
862
- ' ON ${senatSchemaName}.'
863
- );
864
- END LOOP;
865
-
866
- FOR sequence_row IN
867
- SELECT sequencename
868
- FROM pg_sequences
869
- WHERE schemaname = '${escapedStagingSchema}'
870
- ORDER BY sequencename
871
- LOOP
872
- EXECUTE format(
873
- 'SELECT last_value, is_called FROM %I.%I',
874
- '${stagingSchema}',
875
- sequence_row.sequencename
876
- )
877
- INTO sequence_state;
878
-
879
- EXECUTE format(
880
- 'SELECT setval(%L, %s, %L)',
881
- '${senatSchemaName}.' || sequence_row.sequencename,
882
- sequence_state.last_value,
883
- sequence_state.is_called
884
- );
885
- END LOOP;
886
-
887
- FOR sequence_row IN
888
- SELECT c.relname AS sequencename
889
- FROM pg_class c
890
- JOIN pg_namespace n ON n.oid = c.relnamespace
891
- WHERE n.nspname = '${senatSchemaName}'
892
- AND c.relkind = 'S'
893
- AND c.relname LIKE '${escapedPrefix}' || '%'
894
- ORDER BY c.relname
895
- LOOP
896
- IF NOT EXISTS (
897
- SELECT 1
898
- FROM pg_sequences
899
- WHERE schemaname = '${escapedStagingSchema}'
900
- AND sequencename = sequence_row.sequencename
901
- ) THEN
902
- EXECUTE format('DROP SEQUENCE IF EXISTS ${senatSchemaName}.%I CASCADE', sequence_row.sequencename);
903
- END IF;
904
- END LOOP;
905
- END $$;
906
-
907
- GRANT SELECT ON ALL TABLES IN SCHEMA ${senatSchemaName} TO ${dbUser};
103
+ return `
104
+ DO $$
105
+ DECLARE
106
+ table_row record;
107
+ column_row record;
108
+ sequence_row record;
109
+ constraint_row record;
110
+ index_row record;
111
+ sequence_owner record;
112
+ sequence_state record;
113
+ staging_default text;
114
+ target_default text;
115
+ column_definition text;
116
+ column_list text;
117
+ staging_column_list text;
118
+ key_columns text[];
119
+ key_schema_name text;
120
+ configured_key_columns text[];
121
+ matching_key_column_count integer;
122
+ use_row_multiset_merge boolean;
123
+ join_condition text;
124
+ update_assignments text;
125
+ change_condition text;
126
+ target_row_signature_expression text;
127
+ staging_row_signature_expression text;
128
+ sync_strategy text;
129
+ staging_table_comment text;
130
+ target_table_comment text;
131
+ BEGIN
132
+ PERFORM set_config('search_path', '${senatSchemaName}, pg_catalog', true);
133
+
134
+ FOR table_row IN
135
+ ${stagingRelationsQuery}
136
+ LOOP
137
+ IF NOT EXISTS (
138
+ SELECT 1
139
+ FROM pg_tables
140
+ WHERE schemaname = '${senatSchemaName}'
141
+ AND tablename = table_row.tablename
142
+ ) THEN
143
+ EXECUTE format(
144
+ 'CREATE TABLE ${senatSchemaName}.%I (LIKE %I.%I INCLUDING IDENTITY INCLUDING GENERATED INCLUDING DEFAULTS)',
145
+ table_row.tablename,
146
+ '${stagingSchema}',
147
+ table_row.tablename
148
+ );
149
+ END IF;
150
+ END LOOP;
151
+
152
+ FOR sequence_row IN
153
+ SELECT
154
+ sequencename,
155
+ data_type,
156
+ start_value,
157
+ min_value,
158
+ max_value,
159
+ increment_by,
160
+ cycle,
161
+ cache_size
162
+ FROM pg_sequences
163
+ WHERE schemaname = '${escapedStagingSchema}'
164
+ ORDER BY sequencename
165
+ LOOP
166
+ IF NOT EXISTS (
167
+ SELECT 1
168
+ FROM pg_class c
169
+ JOIN pg_namespace n ON n.oid = c.relnamespace
170
+ WHERE n.nspname = '${senatSchemaName}'
171
+ AND c.relkind = 'S'
172
+ AND c.relname = sequence_row.sequencename
173
+ ) THEN
174
+ EXECUTE format('CREATE SEQUENCE ${senatSchemaName}.%I', sequence_row.sequencename);
175
+ END IF;
176
+
177
+ EXECUTE format(
178
+ 'ALTER SEQUENCE ${senatSchemaName}.%I AS %s INCREMENT BY %s MINVALUE %s MAXVALUE %s START WITH %s CACHE %s %s',
179
+ sequence_row.sequencename,
180
+ sequence_row.data_type,
181
+ sequence_row.increment_by,
182
+ sequence_row.min_value,
183
+ sequence_row.max_value,
184
+ sequence_row.start_value,
185
+ sequence_row.cache_size,
186
+ CASE WHEN sequence_row.cycle THEN 'CYCLE' ELSE 'NO CYCLE' END
187
+ );
188
+ END LOOP;
189
+
190
+ FOR table_row IN
191
+ SELECT tablename
192
+ FROM pg_tables
193
+ WHERE schemaname = '${senatSchemaName}'
194
+ AND tablename LIKE '${escapedPrefix}' || '%'
195
+ ORDER BY tablename
196
+ LOOP
197
+ IF NOT EXISTS (
198
+ SELECT 1
199
+ FROM pg_class c
200
+ JOIN pg_namespace n ON n.oid = c.relnamespace
201
+ WHERE n.nspname = '${escapedStagingSchema}'
202
+ AND c.relkind IN ('r', 'p', 'f')
203
+ AND c.relname = table_row.tablename
204
+ ) THEN
205
+ EXECUTE format('DROP TABLE IF EXISTS ${senatSchemaName}.%I CASCADE', table_row.tablename);
206
+ END IF;
207
+ END LOOP;
208
+
209
+ FOR table_row IN
210
+ ${stagingRelationsQuery}
211
+ LOOP
212
+ key_columns := NULL;
213
+ key_schema_name := NULL;
214
+ configured_key_columns := NULL;
215
+ matching_key_column_count := 0;
216
+ use_row_multiset_merge := false;
217
+ join_condition := NULL;
218
+ update_assignments := NULL;
219
+ change_condition := NULL;
220
+ target_row_signature_expression := NULL;
221
+ staging_row_signature_expression := NULL;
222
+
223
+ SELECT configured.key_columns
224
+ INTO configured_key_columns
225
+ FROM (
226
+ ${configuredMergeKeysValues}
227
+ ) AS configured
228
+ WHERE configured.tablename = table_row.tablename;
229
+
230
+ IF configured_key_columns IS NOT NULL THEN
231
+ key_columns := configured_key_columns;
232
+ key_schema_name := 'configured';
233
+
234
+ SELECT string_agg(format('t.%1$I IS NOT DISTINCT FROM s.%1$I', key_column), ' AND ' ORDER BY ordinality)
235
+ INTO join_condition
236
+ FROM unnest(configured_key_columns) WITH ORDINALITY AS configured_key(key_column, ordinality);
237
+ ELSE
238
+ SELECT candidate.key_columns, candidate.join_condition, candidate.source_schema
239
+ INTO key_columns, join_condition, key_schema_name
240
+ FROM (
241
+ SELECT
242
+ array_agg(att.attname ORDER BY key_column.ordinality) AS key_columns,
243
+ string_agg(format('t.%1$I = s.%1$I', att.attname), ' AND ' ORDER BY key_column.ordinality) AS join_condition,
244
+ table_namespace.nspname AS source_schema,
245
+ CASE WHEN table_namespace.nspname = '${escapedStagingSchema}' THEN 0 ELSE 1 END AS schema_priority,
246
+ CASE WHEN index_data.indisprimary THEN 0 ELSE 1 END AS priority,
247
+ count(*) AS key_column_count,
248
+ index_data.indexrelid
249
+ FROM pg_index index_data
250
+ JOIN pg_class table_class ON table_class.oid = index_data.indrelid
251
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
252
+ JOIN LATERAL unnest(index_data.indkey::int2[]) WITH ORDINALITY AS key_column(attnum, ordinality) ON true
253
+ JOIN pg_attribute att
254
+ ON att.attrelid = table_class.oid
255
+ AND att.attnum = key_column.attnum
256
+ WHERE table_namespace.nspname IN ('${escapedStagingSchema}', '${senatSchemaName}')
257
+ AND table_class.relname = table_row.tablename
258
+ AND index_data.indisunique
259
+ AND index_data.indpred IS NULL
260
+ AND index_data.indexprs IS NULL
261
+ AND key_column.ordinality <= index_data.indnkeyatts
262
+ AND key_column.attnum > 0
263
+ AND EXISTS (
264
+ SELECT 1
265
+ FROM information_schema.columns staging_columns
266
+ WHERE staging_columns.table_schema = '${escapedStagingSchema}'
267
+ AND staging_columns.table_name = table_row.tablename
268
+ AND staging_columns.column_name = att.attname
269
+ )
270
+ GROUP BY index_data.indexrelid, index_data.indisprimary, table_namespace.nspname
271
+ HAVING index_data.indisprimary OR bool_and(att.attnotnull)
272
+ ORDER BY schema_priority, priority, key_column_count, index_data.indexrelid
273
+ LIMIT 1
274
+ ) AS candidate;
275
+ END IF;
276
+
277
+ SELECT EXISTS (
278
+ SELECT 1
279
+ FROM (
280
+ ${configuredRowMultisetTablesValues}
281
+ ) AS configured
282
+ WHERE configured.tablename = table_row.tablename
283
+ )
284
+ INTO use_row_multiset_merge;
285
+
286
+ FOR constraint_row IN
287
+ SELECT con.conname
288
+ FROM pg_constraint con
289
+ JOIN pg_class c ON c.oid = con.conrelid
290
+ JOIN pg_namespace n ON n.oid = c.relnamespace
291
+ WHERE n.nspname = '${senatSchemaName}'
292
+ AND c.relname = table_row.tablename
293
+ AND con.contype <> 'n'
294
+ LOOP
295
+ EXECUTE format(
296
+ 'ALTER TABLE ${senatSchemaName}.%I DROP CONSTRAINT IF EXISTS %I CASCADE',
297
+ table_row.tablename,
298
+ constraint_row.conname
299
+ );
300
+ END LOOP;
301
+
302
+ FOR index_row IN
303
+ SELECT index_class.relname AS index_name
304
+ FROM pg_index idx
305
+ JOIN pg_class table_class ON table_class.oid = idx.indrelid
306
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
307
+ JOIN pg_class index_class ON index_class.oid = idx.indexrelid
308
+ LEFT JOIN pg_constraint con ON con.conindid = idx.indexrelid
309
+ WHERE table_namespace.nspname = '${senatSchemaName}'
310
+ AND table_class.relname = table_row.tablename
311
+ AND con.oid IS NULL
312
+ LOOP
313
+ EXECUTE format('DROP INDEX IF EXISTS ${senatSchemaName}.%I', index_row.index_name);
314
+ END LOOP;
315
+
316
+ FOR column_row IN
317
+ SELECT target_columns.column_name
318
+ FROM information_schema.columns target_columns
319
+ WHERE target_columns.table_schema = '${senatSchemaName}'
320
+ AND target_columns.table_name = table_row.tablename
321
+ AND NOT EXISTS (
322
+ SELECT 1
323
+ FROM information_schema.columns staging_columns
324
+ WHERE staging_columns.table_schema = '${escapedStagingSchema}'
325
+ AND staging_columns.table_name = table_row.tablename
326
+ AND staging_columns.column_name = target_columns.column_name
327
+ )
328
+ ORDER BY target_columns.ordinal_position DESC
329
+ LOOP
330
+ EXECUTE format(
331
+ 'ALTER TABLE ${senatSchemaName}.%I DROP COLUMN IF EXISTS %I CASCADE',
332
+ table_row.tablename,
333
+ column_row.column_name
334
+ );
335
+ END LOOP;
336
+
337
+ FOR column_row IN
338
+ SELECT
339
+ staging_columns.column_name,
340
+ format_type(a.atttypid, a.atttypmod) AS data_type,
341
+ a.attnotnull AS not_null,
342
+ a.attidentity AS identity_kind,
343
+ a.attgenerated AS generated_kind,
344
+ pg_get_expr(ad.adbin, ad.adrelid) AS default_expression,
345
+ CASE
346
+ WHEN coll.oid IS NULL THEN NULL
347
+ ELSE format('%I.%I', coll_namespace.nspname, coll.collname)
348
+ END AS collation_name
349
+ FROM information_schema.columns staging_columns
350
+ JOIN pg_class c ON c.relname = staging_columns.table_name
351
+ JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = staging_columns.table_schema
352
+ JOIN pg_attribute a ON a.attrelid = c.oid AND a.attname = staging_columns.column_name
353
+ LEFT JOIN pg_attrdef ad ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum
354
+ LEFT JOIN pg_type typ ON typ.oid = a.atttypid
355
+ LEFT JOIN pg_collation coll ON coll.oid = a.attcollation AND coll.oid <> typ.typcollation
356
+ LEFT JOIN pg_namespace coll_namespace ON coll_namespace.oid = coll.collnamespace
357
+ WHERE staging_columns.table_schema = '${escapedStagingSchema}'
358
+ AND staging_columns.table_name = table_row.tablename
359
+ AND NOT EXISTS (
360
+ SELECT 1
361
+ FROM information_schema.columns target_columns
362
+ WHERE target_columns.table_schema = '${senatSchemaName}'
363
+ AND target_columns.table_name = table_row.tablename
364
+ AND target_columns.column_name = staging_columns.column_name
365
+ )
366
+ ORDER BY staging_columns.ordinal_position
367
+ LOOP
368
+ column_definition := column_row.data_type;
369
+ IF column_row.collation_name IS NOT NULL THEN
370
+ column_definition := column_definition || ' COLLATE ' || column_row.collation_name;
371
+ END IF;
372
+ IF column_row.generated_kind = 's' THEN
373
+ column_definition :=
374
+ column_definition ||
375
+ ' GENERATED ALWAYS AS (' ||
376
+ replace(column_row.default_expression,
377
+ '${stagingSchema}.', '${senatSchemaName}.') ||
378
+ ') STORED';
379
+ ELSIF column_row.identity_kind = 'a' THEN
380
+ column_definition := column_definition || ' GENERATED ALWAYS AS IDENTITY';
381
+ ELSIF column_row.identity_kind = 'd' THEN
382
+ column_definition := column_definition || ' GENERATED BY DEFAULT AS IDENTITY';
383
+ END IF;
384
+
385
+ EXECUTE format(
386
+ 'ALTER TABLE ${senatSchemaName}.%I ADD COLUMN %I %s',
387
+ table_row.tablename,
388
+ column_row.column_name,
389
+ column_definition
390
+ );
391
+
392
+ IF column_row.generated_kind = ''
393
+ AND column_row.identity_kind = ''
394
+ AND column_row.default_expression IS NOT NULL THEN
395
+ EXECUTE format(
396
+ 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I SET DEFAULT %s',
397
+ table_row.tablename,
398
+ column_row.column_name,
399
+ replace(column_row.default_expression, '${stagingSchema}.', '${senatSchemaName}.')
400
+ );
401
+ END IF;
402
+
403
+ IF column_row.not_null THEN
404
+ EXECUTE format(
405
+ 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I SET NOT NULL',
406
+ table_row.tablename,
407
+ column_row.column_name
408
+ );
409
+ END IF;
410
+ END LOOP;
411
+
412
+ FOR column_row IN
413
+ SELECT
414
+ staging_columns.column_name,
415
+ format_type(staging_attribute.atttypid, staging_attribute.atttypmod) AS staging_data_type,
416
+ format_type(target_attribute.atttypid, target_attribute.atttypmod) AS target_data_type,
417
+ staging_attribute.attnotnull AS staging_not_null,
418
+ target_attribute.attnotnull AS target_not_null,
419
+ staging_attribute.attidentity AS staging_identity_kind,
420
+ target_attribute.attidentity AS target_identity_kind,
421
+ staging_attribute.attgenerated AS staging_generated_kind,
422
+ target_attribute.attgenerated AS target_generated_kind,
423
+ pg_get_expr(staging_default_expr.adbin, staging_default_expr.adrelid) AS staging_default_expression,
424
+ pg_get_expr(target_default_expr.adbin, target_default_expr.adrelid) AS target_default_expression,
425
+ CASE
426
+ WHEN staging_collation.oid IS NULL THEN NULL
427
+ ELSE format('%I.%I', staging_collation_namespace.nspname, staging_collation.collname)
428
+ END AS staging_collation_name,
429
+ CASE
430
+ WHEN target_collation.oid IS NULL THEN NULL
431
+ ELSE format('%I.%I', target_collation_namespace.nspname, target_collation.collname)
432
+ END AS target_collation_name
433
+ FROM information_schema.columns staging_columns
434
+ JOIN information_schema.columns target_columns
435
+ ON target_columns.table_schema = '${senatSchemaName}'
436
+ AND target_columns.table_name = staging_columns.table_name
437
+ AND target_columns.column_name = staging_columns.column_name
438
+ JOIN pg_class staging_class ON staging_class.relname = staging_columns.table_name
439
+ JOIN pg_namespace staging_namespace
440
+ ON staging_namespace.oid = staging_class.relnamespace
441
+ AND staging_namespace.nspname = staging_columns.table_schema
442
+ JOIN pg_attribute staging_attribute
443
+ ON staging_attribute.attrelid = staging_class.oid
444
+ AND staging_attribute.attname = staging_columns.column_name
445
+ LEFT JOIN pg_attrdef staging_default_expr
446
+ ON staging_default_expr.adrelid = staging_attribute.attrelid
447
+ AND staging_default_expr.adnum = staging_attribute.attnum
448
+ LEFT JOIN pg_type staging_type ON staging_type.oid = staging_attribute.atttypid
449
+ LEFT JOIN pg_collation staging_collation
450
+ ON staging_collation.oid = staging_attribute.attcollation
451
+ AND staging_collation.oid <> staging_type.typcollation
452
+ LEFT JOIN pg_namespace staging_collation_namespace
453
+ ON staging_collation_namespace.oid = staging_collation.collnamespace
454
+ JOIN pg_class target_class ON target_class.relname = target_columns.table_name
455
+ JOIN pg_namespace target_namespace
456
+ ON target_namespace.oid = target_class.relnamespace
457
+ AND target_namespace.nspname = target_columns.table_schema
458
+ JOIN pg_attribute target_attribute
459
+ ON target_attribute.attrelid = target_class.oid
460
+ AND target_attribute.attname = target_columns.column_name
461
+ LEFT JOIN pg_attrdef target_default_expr
462
+ ON target_default_expr.adrelid = target_attribute.attrelid
463
+ AND target_default_expr.adnum = target_attribute.attnum
464
+ LEFT JOIN pg_type target_type ON target_type.oid = target_attribute.atttypid
465
+ LEFT JOIN pg_collation target_collation
466
+ ON target_collation.oid = target_attribute.attcollation
467
+ AND target_collation.oid <> target_type.typcollation
468
+ LEFT JOIN pg_namespace target_collation_namespace
469
+ ON target_collation_namespace.oid = target_collation.collnamespace
470
+ WHERE staging_columns.table_schema = '${escapedStagingSchema}'
471
+ AND staging_columns.table_name = table_row.tablename
472
+ ORDER BY staging_columns.ordinal_position
473
+ LOOP
474
+ IF column_row.staging_identity_kind <> column_row.target_identity_kind
475
+ OR column_row.staging_generated_kind <> column_row.target_generated_kind THEN
476
+ RAISE EXCEPTION
477
+ 'Unsupported identity/generated column change for %.%',
478
+ table_row.tablename,
479
+ column_row.column_name;
480
+ END IF;
481
+
482
+ IF column_row.staging_data_type <> column_row.target_data_type
483
+ OR column_row.staging_collation_name IS DISTINCT FROM column_row.target_collation_name THEN
484
+ EXECUTE format(
485
+ 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I TYPE %s%s USING %I::%s',
486
+ table_row.tablename,
487
+ column_row.column_name,
488
+ column_row.staging_data_type,
489
+ CASE
490
+ WHEN column_row.staging_collation_name IS NULL THEN ''
491
+ ELSE ' COLLATE ' || column_row.staging_collation_name
492
+ END,
493
+ column_row.column_name,
494
+ column_row.staging_data_type
495
+ );
496
+ END IF;
497
+
498
+ staging_default := replace(
499
+ coalesce(column_row.staging_default_expression, ''),
500
+ '${stagingSchema}.',
501
+ '${senatSchemaName}.'
502
+ );
503
+ target_default := replace(
504
+ coalesce(column_row.target_default_expression, ''),
505
+ '${stagingSchema}.',
506
+ '${senatSchemaName}.'
507
+ );
508
+
509
+ IF staging_default <> target_default THEN
510
+ IF column_row.staging_default_expression IS NULL THEN
511
+ EXECUTE format(
512
+ 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I DROP DEFAULT',
513
+ table_row.tablename,
514
+ column_row.column_name
515
+ );
516
+ ELSE
517
+ EXECUTE format(
518
+ 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I SET DEFAULT %s',
519
+ table_row.tablename,
520
+ column_row.column_name,
521
+ staging_default
522
+ );
523
+ END IF;
524
+ END IF;
525
+
526
+ IF column_row.staging_not_null <> column_row.target_not_null THEN
527
+ EXECUTE format(
528
+ 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I %s NOT NULL',
529
+ table_row.tablename,
530
+ column_row.column_name,
531
+ CASE WHEN column_row.staging_not_null THEN 'SET' ELSE 'DROP' END
532
+ );
533
+ END IF;
534
+ END LOOP;
535
+
536
+ SELECT obj_description(c.oid, 'pg_class')
537
+ INTO staging_table_comment
538
+ FROM pg_class c
539
+ JOIN pg_namespace n ON n.oid = c.relnamespace
540
+ WHERE n.nspname = '${escapedStagingSchema}'
541
+ AND c.relkind IN ('r', 'p', 'f')
542
+ AND c.relname = table_row.tablename;
543
+
544
+ SELECT obj_description(c.oid, 'pg_class')
545
+ INTO target_table_comment
546
+ FROM pg_class c
547
+ JOIN pg_namespace n ON n.oid = c.relnamespace
548
+ WHERE n.nspname = '${senatSchemaName}'
549
+ AND c.relkind IN ('r', 'p')
550
+ AND c.relname = table_row.tablename;
551
+
552
+ IF staging_table_comment IS DISTINCT FROM target_table_comment THEN
553
+ EXECUTE format(
554
+ 'COMMENT ON TABLE ${senatSchemaName}.%I IS %L',
555
+ table_row.tablename,
556
+ staging_table_comment
557
+ );
558
+ END IF;
559
+
560
+ FOR column_row IN
561
+ SELECT
562
+ staging_columns.column_name,
563
+ col_description(staging_class.oid, staging_attribute.attnum) AS staging_comment,
564
+ col_description(target_class.oid, target_attribute.attnum) AS target_comment
565
+ FROM information_schema.columns staging_columns
566
+ JOIN pg_class staging_class ON staging_class.relname = staging_columns.table_name
567
+ JOIN pg_namespace staging_namespace
568
+ ON staging_namespace.oid = staging_class.relnamespace
569
+ AND staging_namespace.nspname = staging_columns.table_schema
570
+ JOIN pg_attribute staging_attribute
571
+ ON staging_attribute.attrelid = staging_class.oid
572
+ AND staging_attribute.attname = staging_columns.column_name
573
+ JOIN information_schema.columns target_columns
574
+ ON target_columns.table_schema = '${senatSchemaName}'
575
+ AND target_columns.table_name = staging_columns.table_name
576
+ AND target_columns.column_name = staging_columns.column_name
577
+ JOIN pg_class target_class ON target_class.relname = target_columns.table_name
578
+ JOIN pg_namespace target_namespace
579
+ ON target_namespace.oid = target_class.relnamespace
580
+ AND target_namespace.nspname = target_columns.table_schema
581
+ JOIN pg_attribute target_attribute
582
+ ON target_attribute.attrelid = target_class.oid
583
+ AND target_attribute.attname = target_columns.column_name
584
+ WHERE staging_columns.table_schema = '${escapedStagingSchema}'
585
+ AND staging_columns.table_name = table_row.tablename
586
+ ORDER BY staging_columns.ordinal_position
587
+ LOOP
588
+ IF column_row.staging_comment IS DISTINCT FROM column_row.target_comment THEN
589
+ EXECUTE format(
590
+ 'COMMENT ON COLUMN ${senatSchemaName}.%I.%I IS %L',
591
+ table_row.tablename,
592
+ column_row.column_name,
593
+ column_row.staging_comment
594
+ );
595
+ END IF;
596
+ END LOOP;
597
+
598
+ SELECT string_agg(format('%I', columns.column_name), ', ' ORDER BY columns.ordinal_position)
599
+ , string_agg(format('s.%I', columns.column_name), ', ' ORDER BY columns.ordinal_position)
600
+ INTO column_list, staging_column_list
601
+ FROM information_schema.columns columns
602
+ WHERE columns.table_schema = '${escapedStagingSchema}'
603
+ AND columns.table_name = table_row.tablename;
604
+
605
+ SELECT
606
+ string_agg(
607
+ format('%1$I = s.%1$I', columns.column_name),
608
+ ', ' ORDER BY columns.ordinal_position),
609
+ string_agg(
610
+ format('t.%1$I IS DISTINCT FROM s.%1$I', columns.column_name),
611
+ ' OR ' ORDER BY columns.ordinal_position)
612
+ INTO update_assignments, change_condition
613
+ FROM information_schema.columns columns
614
+ WHERE columns.table_schema = '${escapedStagingSchema}'
615
+ AND columns.table_name = table_row.tablename
616
+ AND NOT (columns.column_name = ANY(coalesce(key_columns, ARRAY[]::text[])));
617
+
618
+ IF key_columns IS NOT NULL THEN
619
+ SELECT count(*)
620
+ INTO matching_key_column_count
621
+ FROM unnest(key_columns) AS key_column(column_name)
622
+ JOIN information_schema.columns staging_columns
623
+ ON staging_columns.table_schema = '${escapedStagingSchema}'
624
+ AND staging_columns.table_name = table_row.tablename
625
+ AND staging_columns.column_name = key_column.column_name
626
+ JOIN information_schema.columns target_columns
627
+ ON target_columns.table_schema = '${senatSchemaName}'
628
+ AND target_columns.table_name = table_row.tablename
629
+ AND target_columns.column_name = key_column.column_name;
630
+
631
+ IF matching_key_column_count <> cardinality(key_columns) THEN
632
+ RAISE NOTICE
633
+ 'Rejected sync key for %.% from % because only %/% columns are present in both staging and target: %',
634
+ '${senatSchemaName}',
635
+ table_row.tablename,
636
+ coalesce(key_schema_name, 'unknown'),
637
+ matching_key_column_count,
638
+ cardinality(key_columns),
639
+ array_to_string(key_columns, ', ');
640
+ key_columns := NULL;
641
+ key_schema_name := NULL;
642
+ join_condition := NULL;
643
+ END IF;
644
+ END IF;
645
+
646
+ SELECT
647
+ 'jsonb_build_array(' ||
648
+ string_agg(format('t.%I', columns.column_name),
649
+ ', ' ORDER BY columns.ordinal_position) || ')',
650
+ 'jsonb_build_array(' ||
651
+ string_agg(format('s.%I', columns.column_name),
652
+ ', ' ORDER BY columns.ordinal_position) || ')'
653
+ INTO target_row_signature_expression, staging_row_signature_expression
654
+ FROM information_schema.columns columns
655
+ WHERE columns.table_schema = '${escapedStagingSchema}'
656
+ AND columns.table_name = table_row.tablename;
657
+
658
+ IF use_row_multiset_merge THEN
659
+ sync_strategy := 'incremental_row_multiset';
660
+ RAISE NOTICE
661
+ 'Sync strategy for %.%: %',
662
+ '${senatSchemaName}',
663
+ table_row.tablename,
664
+ sync_strategy;
665
+
666
+ EXECUTE format(
667
+ 'WITH staging_counts AS (' ||
668
+ ' SELECT %2$s AS row_signature, count(*) AS row_count' ||
669
+ ' FROM %3$I.%1$I s' ||
670
+ ' GROUP BY 1' ||
671
+ '), target_ranked AS (' ||
672
+ ' SELECT ctid, %4$s AS row_signature, row_number() OVER (PARTITION BY %4$s) AS row_number' ||
673
+ ' FROM ${senatSchemaName}.%1$I t' ||
674
+ ')' ||
675
+ ' DELETE FROM ${senatSchemaName}.%1$I t' ||
676
+ ' USING (' ||
677
+ ' SELECT candidate_row.ctid' ||
678
+ ' FROM target_ranked candidate_row' ||
679
+ ' LEFT JOIN staging_counts reference_row USING (row_signature)' ||
680
+ ' WHERE candidate_row.row_number > coalesce(reference_row.row_count, 0)' ||
681
+ ' ) rows_to_delete' ||
682
+ ' WHERE t.ctid = rows_to_delete.ctid',
683
+ table_row.tablename,
684
+ staging_row_signature_expression,
685
+ '${stagingSchema}',
686
+ target_row_signature_expression
687
+ );
688
+
689
+ EXECUTE format(
690
+ 'WITH target_counts AS (' ||
691
+ ' SELECT %2$s AS row_signature, count(*) AS row_count' ||
692
+ ' FROM ${senatSchemaName}.%1$I t' ||
693
+ ' GROUP BY 1' ||
694
+ '), staging_ranked AS (' ||
695
+ ' SELECT %3$s, %4$s AS row_signature, row_number() OVER (PARTITION BY %4$s) AS row_number' ||
696
+ ' FROM %5$I.%1$I s' ||
697
+ ')' ||
698
+ ' INSERT INTO ${senatSchemaName}.%1$I (%3$s)' ||
699
+ ' SELECT %3$s' ||
700
+ ' FROM staging_ranked candidate_row' ||
701
+ ' LEFT JOIN target_counts reference_row USING (row_signature)' ||
702
+ ' WHERE candidate_row.row_number > coalesce(reference_row.row_count, 0)',
703
+ table_row.tablename,
704
+ target_row_signature_expression,
705
+ column_list,
706
+ staging_row_signature_expression,
707
+ '${stagingSchema}'
708
+ );
709
+ ELSIF key_columns IS NULL THEN
710
+ sync_strategy := 'full_refresh_fallback';
711
+ RAISE NOTICE
712
+ 'Sync strategy for %.%: %',
713
+ '${senatSchemaName}',
714
+ table_row.tablename,
715
+ sync_strategy;
716
+ RAISE WARNING
717
+ 'No primary key or all-NOT-NULL unique key found for %.%; falling back to full refresh',
718
+ '${senatSchemaName}',
719
+ table_row.tablename;
720
+
721
+ EXECUTE format('TRUNCATE TABLE ${senatSchemaName}.%I', table_row.tablename);
722
+
723
+ EXECUTE format(
724
+ 'INSERT INTO ${senatSchemaName}.%I (%s) SELECT %s FROM %I.%I s',
725
+ table_row.tablename,
726
+ column_list,
727
+ staging_column_list,
728
+ '${stagingSchema}',
729
+ table_row.tablename
730
+ );
731
+ ELSE
732
+ sync_strategy := 'incremental';
733
+ RAISE NOTICE
734
+ 'Sync strategy for %.%: % (key from %: %)',
735
+ '${senatSchemaName}',
736
+ table_row.tablename,
737
+ sync_strategy,
738
+ key_schema_name,
739
+ array_to_string(key_columns, ', ');
740
+
741
+ EXECUTE format(
742
+ 'DELETE FROM ${senatSchemaName}.%1$I t WHERE NOT EXISTS (SELECT 1 FROM %2$I.%1$I s WHERE %3$s)',
743
+ table_row.tablename,
744
+ '${stagingSchema}',
745
+ join_condition
746
+ );
747
+
748
+ IF update_assignments IS NOT NULL THEN
749
+ EXECUTE format(
750
+ 'UPDATE ${senatSchemaName}.%1$I t SET %2$s FROM %3$I.%1$I s WHERE %4$s AND (%5$s)',
751
+ table_row.tablename,
752
+ update_assignments,
753
+ '${stagingSchema}',
754
+ join_condition,
755
+ change_condition
756
+ );
757
+ END IF;
758
+
759
+ EXECUTE format(
760
+ 'INSERT INTO ${senatSchemaName}.%1$I (%2$s) ' ||
761
+ 'SELECT %3$s FROM %4$I.%1$I s ' ||
762
+ 'WHERE NOT EXISTS ' ||
763
+ '(SELECT 1 FROM ${senatSchemaName}.%1$I t WHERE %5$s)',
764
+ table_row.tablename,
765
+ column_list,
766
+ staging_column_list,
767
+ '${stagingSchema}',
768
+ join_condition
769
+ );
770
+ END IF;
771
+ END LOOP;
772
+
773
+ FOR sequence_row IN
774
+ SELECT sequencename
775
+ FROM pg_sequences
776
+ WHERE schemaname = '${escapedStagingSchema}'
777
+ ORDER BY sequencename
778
+ LOOP
779
+ SELECT
780
+ dependent_table.relname AS table_name,
781
+ dependent_column.attname AS column_name
782
+ INTO sequence_owner
783
+ FROM pg_class sequence_class
784
+ JOIN pg_namespace sequence_namespace ON sequence_namespace.oid = sequence_class.relnamespace
785
+ JOIN pg_depend dependency
786
+ ON dependency.objid = sequence_class.oid
787
+ AND dependency.deptype = 'a'
788
+ JOIN pg_class dependent_table ON dependent_table.oid = dependency.refobjid
789
+ JOIN pg_namespace dependent_namespace ON dependent_namespace.oid = dependent_table.relnamespace
790
+ JOIN pg_attribute dependent_column
791
+ ON dependent_column.attrelid = dependent_table.oid
792
+ AND dependent_column.attnum = dependency.refobjsubid
793
+ WHERE sequence_namespace.nspname = '${escapedStagingSchema}'
794
+ AND dependent_namespace.nspname = '${escapedStagingSchema}'
795
+ AND sequence_class.relname = sequence_row.sequencename;
796
+
797
+ IF sequence_owner.table_name IS NULL THEN
798
+ EXECUTE format('ALTER SEQUENCE ${senatSchemaName}.%I OWNED BY NONE', sequence_row.sequencename);
799
+ ELSE
800
+ EXECUTE format(
801
+ 'ALTER SEQUENCE ${senatSchemaName}.%I OWNED BY ${senatSchemaName}.%I.%I',
802
+ sequence_row.sequencename,
803
+ sequence_owner.table_name,
804
+ sequence_owner.column_name
805
+ );
806
+ END IF;
807
+ END LOOP;
808
+
809
+ FOR constraint_row IN
810
+ SELECT
811
+ table_class.relname AS table_name,
812
+ con.conname,
813
+ con.contype,
814
+ pg_get_constraintdef(con.oid, true) AS constraint_definition
815
+ FROM pg_constraint con
816
+ JOIN pg_class table_class ON table_class.oid = con.conrelid
817
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
818
+ WHERE table_namespace.nspname = '${escapedStagingSchema}'
819
+ AND con.contype <> 'n'
820
+ ORDER BY
821
+ CASE con.contype
822
+ WHEN 'p' THEN 1
823
+ WHEN 'u' THEN 2
824
+ WHEN 'c' THEN 3
825
+ WHEN 'f' THEN 4
826
+ ELSE 5
827
+ END,
828
+ table_class.relname,
829
+ con.conname
830
+ LOOP
831
+ EXECUTE format(
832
+ 'ALTER TABLE ${senatSchemaName}.%I ADD CONSTRAINT %I %s',
833
+ constraint_row.table_name,
834
+ constraint_row.conname,
835
+ replace(
836
+ replace(
837
+ constraint_row.constraint_definition,
838
+ ' REFERENCES ' || '${stagingSchema}' || '.',
839
+ ' REFERENCES ${senatSchemaName}.'
840
+ ),
841
+ ' REFERENCES ONLY ' || '${stagingSchema}' || '.',
842
+ ' REFERENCES ONLY ${senatSchemaName}.'
843
+ )
844
+ );
845
+ END LOOP;
846
+
847
+ FOR index_row IN
848
+ SELECT
849
+ table_class.relname AS table_name,
850
+ pg_get_indexdef(index_data.indexrelid) AS index_definition
851
+ FROM pg_index index_data
852
+ JOIN pg_class table_class ON table_class.oid = index_data.indrelid
853
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
854
+ LEFT JOIN pg_constraint con ON con.conindid = index_data.indexrelid
855
+ WHERE table_namespace.nspname = '${escapedStagingSchema}'
856
+ AND con.oid IS NULL
857
+ ORDER BY table_class.relname, pg_get_indexdef(index_data.indexrelid)
858
+ LOOP
859
+ EXECUTE replace(
860
+ index_row.index_definition,
861
+ ' ON ${stagingSchema}.',
862
+ ' ON ${senatSchemaName}.'
863
+ );
864
+ END LOOP;
865
+
866
+ FOR sequence_row IN
867
+ SELECT sequencename
868
+ FROM pg_sequences
869
+ WHERE schemaname = '${escapedStagingSchema}'
870
+ ORDER BY sequencename
871
+ LOOP
872
+ EXECUTE format(
873
+ 'SELECT last_value, is_called FROM %I.%I',
874
+ '${stagingSchema}',
875
+ sequence_row.sequencename
876
+ )
877
+ INTO sequence_state;
878
+
879
+ EXECUTE format(
880
+ 'SELECT setval(%L, %s, %L)',
881
+ '${senatSchemaName}.' || sequence_row.sequencename,
882
+ sequence_state.last_value,
883
+ sequence_state.is_called
884
+ );
885
+ END LOOP;
886
+
887
+ FOR sequence_row IN
888
+ SELECT c.relname AS sequencename
889
+ FROM pg_class c
890
+ JOIN pg_namespace n ON n.oid = c.relnamespace
891
+ WHERE n.nspname = '${senatSchemaName}'
892
+ AND c.relkind = 'S'
893
+ AND c.relname LIKE '${escapedPrefix}' || '%'
894
+ ORDER BY c.relname
895
+ LOOP
896
+ IF NOT EXISTS (
897
+ SELECT 1
898
+ FROM pg_sequences
899
+ WHERE schemaname = '${escapedStagingSchema}'
900
+ AND sequencename = sequence_row.sequencename
901
+ ) THEN
902
+ EXECUTE format('DROP SEQUENCE IF EXISTS ${senatSchemaName}.%I CASCADE', sequence_row.sequencename);
903
+ END IF;
904
+ END LOOP;
905
+ END $$;
906
+
907
+ GRANT SELECT ON ALL TABLES IN SCHEMA ${senatSchemaName} TO ${dbUser};
908
908
  `;
909
909
  }