@tricoteuses/senat 3.1.22 → 3.1.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (388) hide show
  1. package/LICENSE.md +32 -32
  2. package/README.md +326 -326
  3. package/lib/src/other_types/collaborateurs.d.ts +5 -0
  4. package/lib/src/parsers/collaborateurs.d.ts +44 -0
  5. package/lib/src/parsers/collaborateurs.js +157 -0
  6. package/lib/src/scripts/retrieve_collaborateurs.js +166 -0
  7. package/lib/src/scripts/retrieve_cr_seance.js +34 -25
  8. package/lib/src/scripts/retrieve_videos.js +13 -8
  9. package/lib/src/scripts/shared/incremental_import_sql.js +866 -859
  10. package/lib/src/scripts/shared/schema_version.js +84 -84
  11. package/lib/src/scripts/shared/staging_metadata_sql.js +214 -214
  12. package/lib/src/scripts/validate_prefixed_tables.js +12 -12
  13. package/lib/src/server/ameli.js +14 -13
  14. package/lib/src/server/conversion_textes.js +106 -106
  15. package/lib/src/server/debats.js +10 -10
  16. package/lib/src/server/documents.js +22 -22
  17. package/lib/src/server/dosleg.js +33 -33
  18. package/lib/src/server/questions.js +10 -10
  19. package/lib/src/server/scrutins.js +3 -3
  20. package/lib/src/server/sens.js +19 -19
  21. package/lib/src/utils/reunion_odj_building.js +0 -4
  22. package/lib/src/videos/match.js +7 -5
  23. package/lib/src/videos/pipeline.js +26 -19
  24. package/lib/tests/collaborateurs.test.js +115 -0
  25. package/lib/tests/incrementalImportSql.test.js +4 -1
  26. package/package.json +119 -119
  27. package/lib/add-js-extensions-v2.js +0 -23
  28. package/lib/add-js-extensions.js +0 -17
  29. package/lib/aggregates.d.ts +0 -52
  30. package/lib/aggregates.js +0 -930
  31. package/lib/aggregates.mjs +0 -713
  32. package/lib/aggregates.ts +0 -833
  33. package/lib/config.d.ts +0 -10
  34. package/lib/config.js +0 -16
  35. package/lib/config.mjs +0 -16
  36. package/lib/config.ts +0 -26
  37. package/lib/databases.d.ts +0 -2
  38. package/lib/databases.js +0 -26
  39. package/lib/databases.mjs +0 -57
  40. package/lib/databases.ts +0 -71
  41. package/lib/datasets.d.ts +0 -34
  42. package/lib/datasets.js +0 -233
  43. package/lib/datasets.mjs +0 -78
  44. package/lib/datasets.ts +0 -118
  45. package/lib/fields.d.ts +0 -10
  46. package/lib/fields.js +0 -68
  47. package/lib/fields.mjs +0 -22
  48. package/lib/fields.ts +0 -29
  49. package/lib/git.d.ts +0 -26
  50. package/lib/git.js +0 -167
  51. package/lib/index.d.ts +0 -13
  52. package/lib/index.js +0 -1
  53. package/lib/index.mjs +0 -7
  54. package/lib/index.ts +0 -64
  55. package/lib/inserters.d.ts +0 -98
  56. package/lib/inserters.js +0 -500
  57. package/lib/inserters.mjs +0 -360
  58. package/lib/inserters.ts +0 -521
  59. package/lib/legislatures.json +0 -38
  60. package/lib/loaders.d.ts +0 -58
  61. package/lib/loaders.js +0 -286
  62. package/lib/loaders.mjs +0 -158
  63. package/lib/loaders.ts +0 -271
  64. package/lib/model/agenda.d.ts +0 -6
  65. package/lib/model/agenda.js +0 -148
  66. package/lib/model/ameli.d.ts +0 -51
  67. package/lib/model/ameli.js +0 -149
  68. package/lib/model/ameli.mjs +0 -84
  69. package/lib/model/ameli.ts +0 -100
  70. package/lib/model/commission.d.ts +0 -18
  71. package/lib/model/commission.js +0 -269
  72. package/lib/model/debats.d.ts +0 -67
  73. package/lib/model/debats.js +0 -95
  74. package/lib/model/debats.mjs +0 -43
  75. package/lib/model/debats.ts +0 -68
  76. package/lib/model/documents.d.ts +0 -12
  77. package/lib/model/documents.js +0 -151
  78. package/lib/model/dosleg.d.ts +0 -7
  79. package/lib/model/dosleg.js +0 -326
  80. package/lib/model/dosleg.mjs +0 -196
  81. package/lib/model/dosleg.ts +0 -240
  82. package/lib/model/index.d.ts +0 -7
  83. package/lib/model/index.js +0 -7
  84. package/lib/model/index.mjs +0 -5
  85. package/lib/model/index.ts +0 -15
  86. package/lib/model/questions.d.ts +0 -45
  87. package/lib/model/questions.js +0 -89
  88. package/lib/model/questions.mjs +0 -71
  89. package/lib/model/questions.ts +0 -93
  90. package/lib/model/scrutins.d.ts +0 -13
  91. package/lib/model/scrutins.js +0 -114
  92. package/lib/model/seance.d.ts +0 -3
  93. package/lib/model/seance.js +0 -267
  94. package/lib/model/sens.d.ts +0 -146
  95. package/lib/model/sens.js +0 -454
  96. package/lib/model/sens.mjs +0 -415
  97. package/lib/model/sens.ts +0 -516
  98. package/lib/model/texte.d.ts +0 -7
  99. package/lib/model/texte.js +0 -256
  100. package/lib/model/texte.mjs +0 -208
  101. package/lib/model/texte.ts +0 -229
  102. package/lib/model/util.d.ts +0 -9
  103. package/lib/model/util.js +0 -38
  104. package/lib/model/util.mjs +0 -19
  105. package/lib/model/util.ts +0 -32
  106. package/lib/parsers/texte.d.ts +0 -7
  107. package/lib/parsers/texte.js +0 -228
  108. package/lib/raw_types/ameli.d.ts +0 -914
  109. package/lib/raw_types/ameli.js +0 -5
  110. package/lib/raw_types/ameli.mjs +0 -163
  111. package/lib/raw_types/debats.d.ts +0 -207
  112. package/lib/raw_types/debats.js +0 -5
  113. package/lib/raw_types/debats.mjs +0 -58
  114. package/lib/raw_types/dosleg.d.ts +0 -1619
  115. package/lib/raw_types/dosleg.js +0 -5
  116. package/lib/raw_types/dosleg.mjs +0 -438
  117. package/lib/raw_types/questions.d.ts +0 -419
  118. package/lib/raw_types/questions.js +0 -5
  119. package/lib/raw_types/questions.mjs +0 -11
  120. package/lib/raw_types/senat.d.ts +0 -11368
  121. package/lib/raw_types/senat.js +0 -5
  122. package/lib/raw_types/sens.d.ts +0 -8248
  123. package/lib/raw_types/sens.js +0 -5
  124. package/lib/raw_types/sens.mjs +0 -508
  125. package/lib/raw_types_kysely/ameli.d.ts +0 -915
  126. package/lib/raw_types_kysely/ameli.js +0 -7
  127. package/lib/raw_types_kysely/ameli.mjs +0 -5
  128. package/lib/raw_types_kysely/ameli.ts +0 -951
  129. package/lib/raw_types_kysely/debats.d.ts +0 -207
  130. package/lib/raw_types_kysely/debats.js +0 -7
  131. package/lib/raw_types_kysely/debats.mjs +0 -5
  132. package/lib/raw_types_kysely/debats.ts +0 -222
  133. package/lib/raw_types_kysely/dosleg.d.ts +0 -3532
  134. package/lib/raw_types_kysely/dosleg.js +0 -7
  135. package/lib/raw_types_kysely/dosleg.mjs +0 -5
  136. package/lib/raw_types_kysely/dosleg.ts +0 -3621
  137. package/lib/raw_types_kysely/questions.d.ts +0 -414
  138. package/lib/raw_types_kysely/questions.js +0 -7
  139. package/lib/raw_types_kysely/questions.mjs +0 -5
  140. package/lib/raw_types_kysely/questions.ts +0 -426
  141. package/lib/raw_types_kysely/sens.d.ts +0 -4394
  142. package/lib/raw_types_kysely/sens.js +0 -7
  143. package/lib/raw_types_kysely/sens.mjs +0 -5
  144. package/lib/raw_types_kysely/sens.ts +0 -4499
  145. package/lib/raw_types_schemats/ameli.d.ts +0 -539
  146. package/lib/raw_types_schemats/ameli.js +0 -2
  147. package/lib/raw_types_schemats/ameli.mjs +0 -2
  148. package/lib/raw_types_schemats/ameli.ts +0 -601
  149. package/lib/raw_types_schemats/debats.d.ts +0 -127
  150. package/lib/raw_types_schemats/debats.js +0 -2
  151. package/lib/raw_types_schemats/debats.mjs +0 -2
  152. package/lib/raw_types_schemats/debats.ts +0 -145
  153. package/lib/raw_types_schemats/dosleg.d.ts +0 -977
  154. package/lib/raw_types_schemats/dosleg.js +0 -2
  155. package/lib/raw_types_schemats/dosleg.mjs +0 -2
  156. package/lib/raw_types_schemats/dosleg.ts +0 -2193
  157. package/lib/raw_types_schemats/questions.d.ts +0 -235
  158. package/lib/raw_types_schemats/questions.js +0 -2
  159. package/lib/raw_types_schemats/questions.mjs +0 -2
  160. package/lib/raw_types_schemats/questions.ts +0 -249
  161. package/lib/raw_types_schemats/sens.d.ts +0 -6915
  162. package/lib/raw_types_schemats/sens.js +0 -2
  163. package/lib/raw_types_schemats/sens.mjs +0 -2
  164. package/lib/raw_types_schemats/sens.ts +0 -2907
  165. package/lib/scripts/convert_data.js +0 -354
  166. package/lib/scripts/convert_data.mjs +0 -181
  167. package/lib/scripts/convert_data.ts +0 -243
  168. package/lib/scripts/data-download.d.ts +0 -1
  169. package/lib/scripts/data-download.js +0 -12
  170. package/lib/scripts/datautil.d.ts +0 -8
  171. package/lib/scripts/datautil.js +0 -34
  172. package/lib/scripts/datautil.mjs +0 -16
  173. package/lib/scripts/datautil.ts +0 -19
  174. package/lib/scripts/images/transparent_150x192.jpg +0 -0
  175. package/lib/scripts/images/transparent_155x225.jpg +0 -0
  176. package/lib/scripts/parse_textes.d.ts +0 -1
  177. package/lib/scripts/parse_textes.js +0 -44
  178. package/lib/scripts/parse_textes.mjs +0 -46
  179. package/lib/scripts/parse_textes.ts +0 -65
  180. package/lib/scripts/retrieve_agenda.d.ts +0 -1
  181. package/lib/scripts/retrieve_agenda.js +0 -132
  182. package/lib/scripts/retrieve_cr_commission.d.ts +0 -1
  183. package/lib/scripts/retrieve_cr_commission.js +0 -364
  184. package/lib/scripts/retrieve_cr_seance.d.ts +0 -6
  185. package/lib/scripts/retrieve_cr_seance.js +0 -347
  186. package/lib/scripts/retrieve_documents.d.ts +0 -3
  187. package/lib/scripts/retrieve_documents.js +0 -219
  188. package/lib/scripts/retrieve_documents.mjs +0 -249
  189. package/lib/scripts/retrieve_documents.ts +0 -298
  190. package/lib/scripts/retrieve_open_data.d.ts +0 -1
  191. package/lib/scripts/retrieve_open_data.js +0 -315
  192. package/lib/scripts/retrieve_open_data.mjs +0 -217
  193. package/lib/scripts/retrieve_open_data.ts +0 -268
  194. package/lib/scripts/retrieve_senateurs_photos.d.ts +0 -1
  195. package/lib/scripts/retrieve_senateurs_photos.js +0 -147
  196. package/lib/scripts/retrieve_senateurs_photos.mjs +0 -147
  197. package/lib/scripts/retrieve_senateurs_photos.ts +0 -177
  198. package/lib/scripts/retrieve_videos.d.ts +0 -1
  199. package/lib/scripts/retrieve_videos.js +0 -461
  200. package/lib/scripts/shared/cli_helpers.d.ts +0 -95
  201. package/lib/scripts/shared/cli_helpers.js +0 -91
  202. package/lib/scripts/shared/cli_helpers.ts +0 -36
  203. package/lib/scripts/shared/util.d.ts +0 -4
  204. package/lib/scripts/shared/util.js +0 -35
  205. package/lib/scripts/shared/util.ts +0 -33
  206. package/lib/scripts/test_iter_load.d.ts +0 -1
  207. package/lib/scripts/test_iter_load.js +0 -12
  208. package/lib/src/ameli.d.ts +0 -66
  209. package/lib/src/ameli.js +0 -1
  210. package/lib/src/config.d.ts +0 -43
  211. package/lib/src/config.js +0 -37
  212. package/lib/src/conversion_textes.d.ts +0 -11
  213. package/lib/src/conversion_textes.js +0 -320
  214. package/lib/src/databases.d.ts +0 -3
  215. package/lib/src/databases.js +0 -26
  216. package/lib/src/databases_postgres.d.ts +0 -4
  217. package/lib/src/databases_postgres.js +0 -23
  218. package/lib/src/datasets.d.ts +0 -38
  219. package/lib/src/datasets.js +0 -247
  220. package/lib/src/db_types/ameli.d.ts +0 -1762
  221. package/lib/src/db_types/ameli.js +0 -1074
  222. package/lib/src/db_types/debats.d.ts +0 -380
  223. package/lib/src/db_types/debats.js +0 -266
  224. package/lib/src/db_types/dosleg.d.ts +0 -2954
  225. package/lib/src/db_types/dosleg.js +0 -2005
  226. package/lib/src/db_types/questions.d.ts +0 -699
  227. package/lib/src/db_types/questions.js +0 -493
  228. package/lib/src/db_types/sens.d.ts +0 -7843
  229. package/lib/src/db_types/sens.js +0 -4691
  230. package/lib/src/debats.d.ts +0 -38
  231. package/lib/src/debats.js +0 -1
  232. package/lib/src/dosleg.d.ts +0 -142
  233. package/lib/src/dosleg.js +0 -193
  234. package/lib/src/git.d.ts +0 -27
  235. package/lib/src/git.js +0 -251
  236. package/lib/src/loaders.d.ts +0 -52
  237. package/lib/src/loaders.js +0 -260
  238. package/lib/src/model/agenda.d.ts +0 -6
  239. package/lib/src/model/agenda.js +0 -148
  240. package/lib/src/model/ameli.d.ts +0 -67
  241. package/lib/src/model/ameli.js +0 -150
  242. package/lib/src/model/ameli_postgres.d.ts +0 -67
  243. package/lib/src/model/ameli_postgres.js +0 -150
  244. package/lib/src/model/commission.d.ts +0 -19
  245. package/lib/src/model/commission.js +0 -269
  246. package/lib/src/model/debats.d.ts +0 -39
  247. package/lib/src/model/debats.js +0 -112
  248. package/lib/src/model/documents.d.ts +0 -32
  249. package/lib/src/model/documents.js +0 -182
  250. package/lib/src/model/dosleg.d.ts +0 -144
  251. package/lib/src/model/dosleg.js +0 -468
  252. package/lib/src/model/index.d.ts +0 -7
  253. package/lib/src/model/index.js +0 -7
  254. package/lib/src/model/questions.d.ts +0 -54
  255. package/lib/src/model/questions.js +0 -91
  256. package/lib/src/model/scrutins.d.ts +0 -48
  257. package/lib/src/model/scrutins.js +0 -121
  258. package/lib/src/model/seance.d.ts +0 -3
  259. package/lib/src/model/seance.js +0 -267
  260. package/lib/src/model/sens.d.ts +0 -112
  261. package/lib/src/model/sens.js +0 -385
  262. package/lib/src/model/util.d.ts +0 -1
  263. package/lib/src/model/util.js +0 -15
  264. package/lib/src/other_types/questions.d.ts +0 -2
  265. package/lib/src/other_types/questions.js +0 -1
  266. package/lib/src/questions.d.ts +0 -53
  267. package/lib/src/questions.js +0 -1
  268. package/lib/src/raw_types/ameli.d.ts +0 -1762
  269. package/lib/src/raw_types/ameli.js +0 -1074
  270. package/lib/src/raw_types/debats.d.ts +0 -380
  271. package/lib/src/raw_types/debats.js +0 -266
  272. package/lib/src/raw_types/dosleg.d.ts +0 -2954
  273. package/lib/src/raw_types/dosleg.js +0 -2005
  274. package/lib/src/raw_types/questions.d.ts +0 -699
  275. package/lib/src/raw_types/questions.js +0 -493
  276. package/lib/src/raw_types/senat.d.ts +0 -11372
  277. package/lib/src/raw_types/senat.js +0 -5
  278. package/lib/src/raw_types/sens.d.ts +0 -7843
  279. package/lib/src/raw_types/sens.js +0 -4691
  280. package/lib/src/raw_types_schemats/ameli.d.ts +0 -541
  281. package/lib/src/raw_types_schemats/ameli.js +0 -2
  282. package/lib/src/raw_types_schemats/debats.d.ts +0 -127
  283. package/lib/src/raw_types_schemats/debats.js +0 -2
  284. package/lib/src/raw_types_schemats/dosleg.d.ts +0 -977
  285. package/lib/src/raw_types_schemats/dosleg.js +0 -2
  286. package/lib/src/raw_types_schemats/questions.d.ts +0 -237
  287. package/lib/src/raw_types_schemats/questions.js +0 -2
  288. package/lib/src/raw_types_schemats/sens.d.ts +0 -2709
  289. package/lib/src/raw_types_schemats/sens.js +0 -2
  290. package/lib/src/rich_types/agenda.d.ts +0 -45
  291. package/lib/src/rich_types/agenda.js +0 -1
  292. package/lib/src/rich_types/compte_rendu.d.ts +0 -83
  293. package/lib/src/rich_types/compte_rendu.js +0 -1
  294. package/lib/src/rich_types/sessions.d.ts +0 -6
  295. package/lib/src/rich_types/sessions.js +0 -19
  296. package/lib/src/rich_types/texte.d.ts +0 -72
  297. package/lib/src/rich_types/texte.js +0 -15
  298. package/lib/src/scripts/test_iter_load.d.ts +0 -1
  299. package/lib/src/scripts/test_iter_load.js +0 -12
  300. package/lib/src/sens.d.ts +0 -104
  301. package/lib/src/sens.js +0 -1
  302. package/lib/src/types/agenda.d.ts +0 -45
  303. package/lib/src/types/agenda.js +0 -1
  304. package/lib/src/types/ameli.d.ts +0 -1762
  305. package/lib/src/types/ameli.js +0 -1074
  306. package/lib/src/types/compte_rendu.d.ts +0 -83
  307. package/lib/src/types/compte_rendu.js +0 -1
  308. package/lib/src/types/debats.d.ts +0 -380
  309. package/lib/src/types/debats.js +0 -266
  310. package/lib/src/types/dosleg.d.ts +0 -2954
  311. package/lib/src/types/dosleg.js +0 -2005
  312. package/lib/src/types/questions.d.ts +0 -699
  313. package/lib/src/types/questions.js +0 -493
  314. package/lib/src/types/sens.d.ts +0 -7843
  315. package/lib/src/types/sens.js +0 -4691
  316. package/lib/src/types/sessions.d.ts +0 -6
  317. package/lib/src/types/sessions.js +0 -19
  318. package/lib/src/types/texte.d.ts +0 -72
  319. package/lib/src/types/texte.js +0 -15
  320. package/lib/src/validators/config.d.ts +0 -9
  321. package/lib/src/validators/config.js +0 -10
  322. package/lib/strings.d.ts +0 -1
  323. package/lib/strings.js +0 -18
  324. package/lib/strings.mjs +0 -18
  325. package/lib/strings.ts +0 -26
  326. package/lib/tsconfig.tsbuildinfo +0 -1
  327. package/lib/types/agenda.d.ts +0 -44
  328. package/lib/types/agenda.js +0 -1
  329. package/lib/types/ameli.d.ts +0 -5
  330. package/lib/types/ameli.js +0 -1
  331. package/lib/types/ameli.mjs +0 -13
  332. package/lib/types/ameli.ts +0 -21
  333. package/lib/types/compte_rendu.d.ts +0 -83
  334. package/lib/types/compte_rendu.js +0 -1
  335. package/lib/types/debats.d.ts +0 -2
  336. package/lib/types/debats.js +0 -1
  337. package/lib/types/debats.mjs +0 -2
  338. package/lib/types/debats.ts +0 -6
  339. package/lib/types/dosleg.d.ts +0 -70
  340. package/lib/types/dosleg.js +0 -1
  341. package/lib/types/dosleg.mjs +0 -151
  342. package/lib/types/dosleg.ts +0 -284
  343. package/lib/types/questions.d.ts +0 -2
  344. package/lib/types/questions.js +0 -1
  345. package/lib/types/questions.mjs +0 -1
  346. package/lib/types/questions.ts +0 -3
  347. package/lib/types/sens.d.ts +0 -10
  348. package/lib/types/sens.js +0 -1
  349. package/lib/types/sens.mjs +0 -1
  350. package/lib/types/sens.ts +0 -12
  351. package/lib/types/sessions.d.ts +0 -5
  352. package/lib/types/sessions.js +0 -84
  353. package/lib/types/sessions.mjs +0 -43
  354. package/lib/types/sessions.ts +0 -42
  355. package/lib/types/texte.d.ts +0 -74
  356. package/lib/types/texte.js +0 -16
  357. package/lib/types/texte.mjs +0 -16
  358. package/lib/types/texte.ts +0 -76
  359. package/lib/typings/windows-1252.d.js +0 -2
  360. package/lib/typings/windows-1252.d.mjs +0 -2
  361. package/lib/typings/windows-1252.d.ts +0 -11
  362. package/lib/utils/cr_spliting.d.ts +0 -28
  363. package/lib/utils/cr_spliting.js +0 -265
  364. package/lib/utils/date.d.ts +0 -10
  365. package/lib/utils/date.js +0 -100
  366. package/lib/utils/nvs-timecode.d.ts +0 -7
  367. package/lib/utils/nvs-timecode.js +0 -79
  368. package/lib/utils/reunion_grouping.d.ts +0 -9
  369. package/lib/utils/reunion_grouping.js +0 -361
  370. package/lib/utils/reunion_odj_building.d.ts +0 -5
  371. package/lib/utils/reunion_odj_building.js +0 -154
  372. package/lib/utils/reunion_parsing.d.ts +0 -23
  373. package/lib/utils/reunion_parsing.js +0 -209
  374. package/lib/utils/scoring.d.ts +0 -14
  375. package/lib/utils/scoring.js +0 -147
  376. package/lib/utils/string_cleaning.d.ts +0 -7
  377. package/lib/utils/string_cleaning.js +0 -57
  378. package/lib/validators/config.d.ts +0 -9
  379. package/lib/validators/config.js +0 -10
  380. package/lib/validators/config.mjs +0 -54
  381. package/lib/validators/config.ts +0 -79
  382. package/lib/validators/senat.d.ts +0 -0
  383. package/lib/validators/senat.js +0 -28
  384. package/lib/validators/senat.mjs +0 -24
  385. package/lib/validators/senat.ts +0 -26
  386. /package/lib/{add-js-extensions-v2.d.ts → src/other_types/collaborateurs.js} +0 -0
  387. /package/lib/{add-js-extensions.d.ts → src/scripts/retrieve_collaborateurs.d.ts} +0 -0
  388. /package/lib/{scripts/convert_data.d.ts → tests/collaborateurs.test.d.ts} +0 -0
@@ -29,60 +29,60 @@ export function buildNormalizeStagingSchemaSql(datasetName) {
29
29
  const prefix = `${datasetName}_`;
30
30
  const escapedPrefix = escapeSqlLiteral(prefix);
31
31
  const escapedStagingSchema = escapeSqlLiteral(stagingSchema);
32
- return `
33
- DO $$
34
- DECLARE
35
- relation record;
36
- table_row record;
37
- BEGIN
38
- FOR relation IN
39
- SELECT c.relname
40
- FROM pg_class c
41
- JOIN pg_namespace n ON n.oid = c.relnamespace
42
- WHERE n.nspname = '${escapedStagingSchema}'
43
- AND c.relkind = 'i'
44
- AND c.relname NOT LIKE '${escapedPrefix}' || '%'
45
- LOOP
46
- EXECUTE format(
47
- 'ALTER INDEX %I.%I RENAME TO %I',
48
- '${stagingSchema}',
49
- relation.relname,
50
- '${prefix}' || relation.relname
51
- );
52
- END LOOP;
53
-
54
- FOR relation IN
55
- SELECT c.relname
56
- FROM pg_class c
57
- JOIN pg_namespace n ON n.oid = c.relnamespace
58
- WHERE n.nspname = '${escapedStagingSchema}'
59
- AND c.relkind = 'S'
60
- AND c.relname NOT LIKE '${escapedPrefix}' || '%'
61
- LOOP
62
- EXECUTE format(
63
- 'ALTER SEQUENCE %I.%I RENAME TO %I',
64
- '${stagingSchema}',
65
- relation.relname,
66
- '${prefix}' || relation.relname
67
- );
68
- END LOOP;
69
-
70
- FOR table_row IN
71
- SELECT tablename
72
- FROM pg_tables
73
- WHERE schemaname = '${escapedStagingSchema}'
74
- ORDER BY tablename
75
- LOOP
76
- IF table_row.tablename NOT LIKE '${escapedPrefix}' || '%' THEN
77
- EXECUTE format(
78
- 'ALTER TABLE %I.%I RENAME TO %I',
79
- '${stagingSchema}',
80
- table_row.tablename,
81
- '${prefix}' || table_row.tablename
82
- );
83
- END IF;
84
- END LOOP;
85
- END $$;
32
+ return `
33
+ DO $$
34
+ DECLARE
35
+ relation record;
36
+ table_row record;
37
+ BEGIN
38
+ FOR relation IN
39
+ SELECT c.relname
40
+ FROM pg_class c
41
+ JOIN pg_namespace n ON n.oid = c.relnamespace
42
+ WHERE n.nspname = '${escapedStagingSchema}'
43
+ AND c.relkind = 'i'
44
+ AND c.relname NOT LIKE '${escapedPrefix}' || '%'
45
+ LOOP
46
+ EXECUTE format(
47
+ 'ALTER INDEX %I.%I RENAME TO %I',
48
+ '${stagingSchema}',
49
+ relation.relname,
50
+ '${prefix}' || relation.relname
51
+ );
52
+ END LOOP;
53
+
54
+ FOR relation IN
55
+ SELECT c.relname
56
+ FROM pg_class c
57
+ JOIN pg_namespace n ON n.oid = c.relnamespace
58
+ WHERE n.nspname = '${escapedStagingSchema}'
59
+ AND c.relkind = 'S'
60
+ AND c.relname NOT LIKE '${escapedPrefix}' || '%'
61
+ LOOP
62
+ EXECUTE format(
63
+ 'ALTER SEQUENCE %I.%I RENAME TO %I',
64
+ '${stagingSchema}',
65
+ relation.relname,
66
+ '${prefix}' || relation.relname
67
+ );
68
+ END LOOP;
69
+
70
+ FOR table_row IN
71
+ SELECT tablename
72
+ FROM pg_tables
73
+ WHERE schemaname = '${escapedStagingSchema}'
74
+ ORDER BY tablename
75
+ LOOP
76
+ IF table_row.tablename NOT LIKE '${escapedPrefix}' || '%' THEN
77
+ EXECUTE format(
78
+ 'ALTER TABLE %I.%I RENAME TO %I',
79
+ '${stagingSchema}',
80
+ table_row.tablename,
81
+ '${prefix}' || table_row.tablename
82
+ );
83
+ END IF;
84
+ END LOOP;
85
+ END $$;
86
86
  `;
87
87
  }
88
88
  export function buildIncrementalDatasetImportSql(datasetName, dbUser, mergeKeys, rowMultisetMergeTables) {
@@ -100,810 +100,817 @@ export function buildIncrementalDatasetImportSql(datasetName, dbUser, mergeKeys,
100
100
  " AND c.relkind IN ('r', 'p', 'f')",
101
101
  "ORDER BY c.relname",
102
102
  ].join("\n");
103
- return `
104
- DO $$
105
- DECLARE
106
- table_row record;
107
- column_row record;
108
- sequence_row record;
109
- constraint_row record;
110
- index_row record;
111
- sequence_owner record;
112
- sequence_state record;
113
- staging_default text;
114
- target_default text;
115
- column_definition text;
116
- column_list text;
117
- staging_column_list text;
118
- key_columns text[];
119
- key_schema_name text;
120
- configured_key_columns text[];
121
- matching_key_column_count integer;
122
- use_row_multiset_merge boolean;
123
- join_condition text;
124
- update_assignments text;
125
- change_condition text;
126
- target_row_signature_expression text;
127
- staging_row_signature_expression text;
128
- sync_strategy text;
129
- staging_table_comment text;
130
- target_table_comment text;
131
- BEGIN
132
- PERFORM set_config('search_path', '${senatSchemaName}, pg_catalog', true);
133
-
134
- FOR table_row IN
135
- ${stagingRelationsQuery}
136
- LOOP
137
- IF NOT EXISTS (
138
- SELECT 1
139
- FROM pg_tables
140
- WHERE schemaname = '${senatSchemaName}'
141
- AND tablename = table_row.tablename
142
- ) THEN
143
- EXECUTE format(
144
- 'CREATE TABLE ${senatSchemaName}.%I (LIKE %I.%I INCLUDING IDENTITY INCLUDING GENERATED INCLUDING DEFAULTS)',
145
- table_row.tablename,
146
- '${stagingSchema}',
147
- table_row.tablename
148
- );
149
- END IF;
150
- END LOOP;
151
-
152
- FOR sequence_row IN
153
- SELECT
154
- sequencename,
155
- data_type,
156
- start_value,
157
- min_value,
158
- max_value,
159
- increment_by,
160
- cycle,
161
- cache_size
162
- FROM pg_sequences
163
- WHERE schemaname = '${escapedStagingSchema}'
164
- ORDER BY sequencename
165
- LOOP
166
- IF NOT EXISTS (
167
- SELECT 1
168
- FROM pg_class c
169
- JOIN pg_namespace n ON n.oid = c.relnamespace
170
- WHERE n.nspname = '${senatSchemaName}'
171
- AND c.relkind = 'S'
172
- AND c.relname = sequence_row.sequencename
173
- ) THEN
174
- EXECUTE format('CREATE SEQUENCE ${senatSchemaName}.%I', sequence_row.sequencename);
175
- END IF;
176
-
177
- EXECUTE format(
178
- 'ALTER SEQUENCE ${senatSchemaName}.%I AS %s INCREMENT BY %s MINVALUE %s MAXVALUE %s START WITH %s CACHE %s %s',
179
- sequence_row.sequencename,
180
- sequence_row.data_type,
181
- sequence_row.increment_by,
182
- sequence_row.min_value,
183
- sequence_row.max_value,
184
- sequence_row.start_value,
185
- sequence_row.cache_size,
186
- CASE WHEN sequence_row.cycle THEN 'CYCLE' ELSE 'NO CYCLE' END
187
- );
188
- END LOOP;
189
-
190
- FOR table_row IN
191
- SELECT tablename
192
- FROM pg_tables
193
- WHERE schemaname = '${senatSchemaName}'
194
- AND tablename LIKE '${escapedPrefix}' || '%'
195
- ORDER BY tablename
196
- LOOP
197
- IF NOT EXISTS (
198
- SELECT 1
199
- FROM pg_class c
200
- JOIN pg_namespace n ON n.oid = c.relnamespace
201
- WHERE n.nspname = '${escapedStagingSchema}'
202
- AND c.relkind IN ('r', 'p', 'f')
203
- AND c.relname = table_row.tablename
204
- ) THEN
205
- EXECUTE format('DROP TABLE IF EXISTS ${senatSchemaName}.%I CASCADE', table_row.tablename);
206
- END IF;
207
- END LOOP;
208
-
209
- FOR table_row IN
210
- ${stagingRelationsQuery}
211
- LOOP
212
- key_columns := NULL;
213
- key_schema_name := NULL;
214
- configured_key_columns := NULL;
215
- matching_key_column_count := 0;
216
- use_row_multiset_merge := false;
217
- join_condition := NULL;
218
- update_assignments := NULL;
219
- change_condition := NULL;
220
- target_row_signature_expression := NULL;
221
- staging_row_signature_expression := NULL;
222
-
223
- SELECT configured.key_columns
224
- INTO configured_key_columns
225
- FROM (
226
- ${configuredMergeKeysValues}
227
- ) AS configured
228
- WHERE configured.tablename = table_row.tablename;
229
-
230
- IF configured_key_columns IS NOT NULL THEN
231
- key_columns := configured_key_columns;
232
- key_schema_name := 'configured';
233
-
234
- SELECT string_agg(format('t.%1$I IS NOT DISTINCT FROM s.%1$I', key_column), ' AND ' ORDER BY ordinality)
235
- INTO join_condition
236
- FROM unnest(configured_key_columns) WITH ORDINALITY AS configured_key(key_column, ordinality);
237
- ELSE
238
- SELECT candidate.key_columns, candidate.join_condition, candidate.source_schema
239
- INTO key_columns, join_condition, key_schema_name
240
- FROM (
241
- SELECT
242
- array_agg(att.attname ORDER BY key_column.ordinality) AS key_columns,
243
- string_agg(format('t.%1$I = s.%1$I', att.attname), ' AND ' ORDER BY key_column.ordinality) AS join_condition,
244
- table_namespace.nspname AS source_schema,
245
- CASE WHEN table_namespace.nspname = '${escapedStagingSchema}' THEN 0 ELSE 1 END AS schema_priority,
246
- CASE WHEN index_data.indisprimary THEN 0 ELSE 1 END AS priority,
247
- count(*) AS key_column_count,
248
- index_data.indexrelid
249
- FROM pg_index index_data
250
- JOIN pg_class table_class ON table_class.oid = index_data.indrelid
251
- JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
252
- JOIN LATERAL unnest(index_data.indkey::int2[]) WITH ORDINALITY AS key_column(attnum, ordinality) ON true
253
- JOIN pg_attribute att
254
- ON att.attrelid = table_class.oid
255
- AND att.attnum = key_column.attnum
256
- WHERE table_namespace.nspname IN ('${escapedStagingSchema}', '${senatSchemaName}')
257
- AND table_class.relname = table_row.tablename
258
- AND index_data.indisunique
259
- AND index_data.indpred IS NULL
260
- AND index_data.indexprs IS NULL
261
- AND key_column.ordinality <= index_data.indnkeyatts
262
- AND key_column.attnum > 0
263
- AND EXISTS (
264
- SELECT 1
265
- FROM information_schema.columns staging_columns
266
- WHERE staging_columns.table_schema = '${escapedStagingSchema}'
267
- AND staging_columns.table_name = table_row.tablename
268
- AND staging_columns.column_name = att.attname
269
- )
270
- GROUP BY index_data.indexrelid, index_data.indisprimary, table_namespace.nspname
271
- HAVING index_data.indisprimary OR bool_and(att.attnotnull)
272
- ORDER BY schema_priority, priority, key_column_count, index_data.indexrelid
273
- LIMIT 1
274
- ) AS candidate;
275
- END IF;
276
-
277
- SELECT EXISTS (
278
- SELECT 1
279
- FROM (
280
- ${configuredRowMultisetTablesValues}
281
- ) AS configured
282
- WHERE configured.tablename = table_row.tablename
283
- )
284
- INTO use_row_multiset_merge;
285
-
286
- FOR constraint_row IN
287
- SELECT con.conname
288
- FROM pg_constraint con
289
- JOIN pg_class c ON c.oid = con.conrelid
290
- JOIN pg_namespace n ON n.oid = c.relnamespace
291
- WHERE n.nspname = '${senatSchemaName}'
292
- AND c.relname = table_row.tablename
293
- AND con.contype <> 'n'
294
- LOOP
295
- EXECUTE format(
296
- 'ALTER TABLE ${senatSchemaName}.%I DROP CONSTRAINT IF EXISTS %I CASCADE',
297
- table_row.tablename,
298
- constraint_row.conname
299
- );
300
- END LOOP;
301
-
302
- FOR index_row IN
303
- SELECT index_class.relname AS index_name
304
- FROM pg_index idx
305
- JOIN pg_class table_class ON table_class.oid = idx.indrelid
306
- JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
307
- JOIN pg_class index_class ON index_class.oid = idx.indexrelid
308
- LEFT JOIN pg_constraint con ON con.conindid = idx.indexrelid
309
- WHERE table_namespace.nspname = '${senatSchemaName}'
310
- AND table_class.relname = table_row.tablename
311
- AND con.oid IS NULL
312
- LOOP
313
- EXECUTE format('DROP INDEX IF EXISTS ${senatSchemaName}.%I', index_row.index_name);
314
- END LOOP;
315
-
316
- FOR column_row IN
317
- SELECT target_columns.column_name
318
- FROM information_schema.columns target_columns
319
- WHERE target_columns.table_schema = '${senatSchemaName}'
320
- AND target_columns.table_name = table_row.tablename
321
- AND NOT EXISTS (
322
- SELECT 1
323
- FROM information_schema.columns staging_columns
324
- WHERE staging_columns.table_schema = '${escapedStagingSchema}'
325
- AND staging_columns.table_name = table_row.tablename
326
- AND staging_columns.column_name = target_columns.column_name
327
- )
328
- ORDER BY target_columns.ordinal_position DESC
329
- LOOP
330
- EXECUTE format(
331
- 'ALTER TABLE ${senatSchemaName}.%I DROP COLUMN IF EXISTS %I CASCADE',
332
- table_row.tablename,
333
- column_row.column_name
334
- );
335
- END LOOP;
336
-
337
- FOR column_row IN
338
- SELECT
339
- staging_columns.column_name,
340
- format_type(a.atttypid, a.atttypmod) AS data_type,
341
- a.attnotnull AS not_null,
342
- a.attidentity AS identity_kind,
343
- a.attgenerated AS generated_kind,
344
- pg_get_expr(ad.adbin, ad.adrelid) AS default_expression,
345
- CASE
346
- WHEN coll.oid IS NULL THEN NULL
347
- ELSE format('%I.%I', coll_namespace.nspname, coll.collname)
348
- END AS collation_name
349
- FROM information_schema.columns staging_columns
350
- JOIN pg_class c ON c.relname = staging_columns.table_name
351
- JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = staging_columns.table_schema
352
- JOIN pg_attribute a ON a.attrelid = c.oid AND a.attname = staging_columns.column_name
353
- LEFT JOIN pg_attrdef ad ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum
354
- LEFT JOIN pg_type typ ON typ.oid = a.atttypid
355
- LEFT JOIN pg_collation coll ON coll.oid = a.attcollation AND coll.oid <> typ.typcollation
356
- LEFT JOIN pg_namespace coll_namespace ON coll_namespace.oid = coll.collnamespace
357
- WHERE staging_columns.table_schema = '${escapedStagingSchema}'
358
- AND staging_columns.table_name = table_row.tablename
359
- AND NOT EXISTS (
360
- SELECT 1
361
- FROM information_schema.columns target_columns
362
- WHERE target_columns.table_schema = '${senatSchemaName}'
363
- AND target_columns.table_name = table_row.tablename
364
- AND target_columns.column_name = staging_columns.column_name
365
- )
366
- ORDER BY staging_columns.ordinal_position
367
- LOOP
368
- column_definition := column_row.data_type;
369
- IF column_row.collation_name IS NOT NULL THEN
370
- column_definition := column_definition || ' COLLATE ' || column_row.collation_name;
371
- END IF;
372
- IF column_row.generated_kind = 's' THEN
373
- column_definition :=
374
- column_definition ||
375
- ' GENERATED ALWAYS AS (' ||
376
- replace(column_row.default_expression,
377
- '${stagingSchema}.', '${senatSchemaName}.') ||
378
- ') STORED';
379
- ELSIF column_row.identity_kind = 'a' THEN
380
- column_definition := column_definition || ' GENERATED ALWAYS AS IDENTITY';
381
- ELSIF column_row.identity_kind = 'd' THEN
382
- column_definition := column_definition || ' GENERATED BY DEFAULT AS IDENTITY';
383
- END IF;
384
-
385
- EXECUTE format(
386
- 'ALTER TABLE ${senatSchemaName}.%I ADD COLUMN %I %s',
387
- table_row.tablename,
388
- column_row.column_name,
389
- column_definition
390
- );
391
-
392
- IF column_row.generated_kind = ''
393
- AND column_row.identity_kind = ''
394
- AND column_row.default_expression IS NOT NULL THEN
395
- EXECUTE format(
396
- 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I SET DEFAULT %s',
397
- table_row.tablename,
398
- column_row.column_name,
399
- replace(column_row.default_expression, '${stagingSchema}.', '${senatSchemaName}.')
400
- );
401
- END IF;
402
-
403
- IF column_row.not_null THEN
404
- EXECUTE format(
405
- 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I SET NOT NULL',
406
- table_row.tablename,
407
- column_row.column_name
408
- );
409
- END IF;
410
- END LOOP;
411
-
412
- FOR column_row IN
413
- SELECT
414
- staging_columns.column_name,
415
- format_type(staging_attribute.atttypid, staging_attribute.atttypmod) AS staging_data_type,
416
- format_type(target_attribute.atttypid, target_attribute.atttypmod) AS target_data_type,
417
- staging_attribute.attnotnull AS staging_not_null,
418
- target_attribute.attnotnull AS target_not_null,
419
- staging_attribute.attidentity AS staging_identity_kind,
420
- target_attribute.attidentity AS target_identity_kind,
421
- staging_attribute.attgenerated AS staging_generated_kind,
422
- target_attribute.attgenerated AS target_generated_kind,
423
- pg_get_expr(staging_default_expr.adbin, staging_default_expr.adrelid) AS staging_default_expression,
424
- pg_get_expr(target_default_expr.adbin, target_default_expr.adrelid) AS target_default_expression,
425
- CASE
426
- WHEN staging_collation.oid IS NULL THEN NULL
427
- ELSE format('%I.%I', staging_collation_namespace.nspname, staging_collation.collname)
428
- END AS staging_collation_name,
429
- CASE
430
- WHEN target_collation.oid IS NULL THEN NULL
431
- ELSE format('%I.%I', target_collation_namespace.nspname, target_collation.collname)
432
- END AS target_collation_name
433
- FROM information_schema.columns staging_columns
434
- JOIN information_schema.columns target_columns
435
- ON target_columns.table_schema = '${senatSchemaName}'
436
- AND target_columns.table_name = staging_columns.table_name
437
- AND target_columns.column_name = staging_columns.column_name
438
- JOIN pg_class staging_class ON staging_class.relname = staging_columns.table_name
439
- JOIN pg_namespace staging_namespace
440
- ON staging_namespace.oid = staging_class.relnamespace
441
- AND staging_namespace.nspname = staging_columns.table_schema
442
- JOIN pg_attribute staging_attribute
443
- ON staging_attribute.attrelid = staging_class.oid
444
- AND staging_attribute.attname = staging_columns.column_name
445
- LEFT JOIN pg_attrdef staging_default_expr
446
- ON staging_default_expr.adrelid = staging_attribute.attrelid
447
- AND staging_default_expr.adnum = staging_attribute.attnum
448
- LEFT JOIN pg_type staging_type ON staging_type.oid = staging_attribute.atttypid
449
- LEFT JOIN pg_collation staging_collation
450
- ON staging_collation.oid = staging_attribute.attcollation
451
- AND staging_collation.oid <> staging_type.typcollation
452
- LEFT JOIN pg_namespace staging_collation_namespace
453
- ON staging_collation_namespace.oid = staging_collation.collnamespace
454
- JOIN pg_class target_class ON target_class.relname = target_columns.table_name
455
- JOIN pg_namespace target_namespace
456
- ON target_namespace.oid = target_class.relnamespace
457
- AND target_namespace.nspname = target_columns.table_schema
458
- JOIN pg_attribute target_attribute
459
- ON target_attribute.attrelid = target_class.oid
460
- AND target_attribute.attname = target_columns.column_name
461
- LEFT JOIN pg_attrdef target_default_expr
462
- ON target_default_expr.adrelid = target_attribute.attrelid
463
- AND target_default_expr.adnum = target_attribute.attnum
464
- LEFT JOIN pg_type target_type ON target_type.oid = target_attribute.atttypid
465
- LEFT JOIN pg_collation target_collation
466
- ON target_collation.oid = target_attribute.attcollation
467
- AND target_collation.oid <> target_type.typcollation
468
- LEFT JOIN pg_namespace target_collation_namespace
469
- ON target_collation_namespace.oid = target_collation.collnamespace
470
- WHERE staging_columns.table_schema = '${escapedStagingSchema}'
471
- AND staging_columns.table_name = table_row.tablename
472
- ORDER BY staging_columns.ordinal_position
473
- LOOP
474
- IF column_row.staging_identity_kind <> column_row.target_identity_kind
475
- OR column_row.staging_generated_kind <> column_row.target_generated_kind THEN
476
- RAISE EXCEPTION
477
- 'Unsupported identity/generated column change for %.%',
478
- table_row.tablename,
479
- column_row.column_name;
480
- END IF;
481
-
482
- IF column_row.staging_data_type <> column_row.target_data_type
483
- OR column_row.staging_collation_name IS DISTINCT FROM column_row.target_collation_name THEN
484
- EXECUTE format(
485
- 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I TYPE %s%s USING %I::%s',
486
- table_row.tablename,
487
- column_row.column_name,
488
- column_row.staging_data_type,
489
- CASE
490
- WHEN column_row.staging_collation_name IS NULL THEN ''
491
- ELSE ' COLLATE ' || column_row.staging_collation_name
492
- END,
493
- column_row.column_name,
494
- column_row.staging_data_type
495
- );
496
- END IF;
497
-
498
- staging_default := replace(
499
- coalesce(column_row.staging_default_expression, ''),
500
- '${stagingSchema}.',
501
- '${senatSchemaName}.'
502
- );
503
- target_default := replace(
504
- coalesce(column_row.target_default_expression, ''),
505
- '${stagingSchema}.',
506
- '${senatSchemaName}.'
507
- );
508
-
509
- IF staging_default <> target_default THEN
510
- IF column_row.staging_default_expression IS NULL THEN
511
- EXECUTE format(
512
- 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I DROP DEFAULT',
513
- table_row.tablename,
514
- column_row.column_name
515
- );
516
- ELSE
517
- EXECUTE format(
518
- 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I SET DEFAULT %s',
519
- table_row.tablename,
520
- column_row.column_name,
521
- staging_default
522
- );
523
- END IF;
524
- END IF;
525
-
526
- IF column_row.staging_not_null <> column_row.target_not_null THEN
527
- EXECUTE format(
528
- 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I %s NOT NULL',
529
- table_row.tablename,
530
- column_row.column_name,
531
- CASE WHEN column_row.staging_not_null THEN 'SET' ELSE 'DROP' END
532
- );
533
- END IF;
534
- END LOOP;
535
-
536
- SELECT obj_description(c.oid, 'pg_class')
537
- INTO staging_table_comment
538
- FROM pg_class c
539
- JOIN pg_namespace n ON n.oid = c.relnamespace
540
- WHERE n.nspname = '${escapedStagingSchema}'
541
- AND c.relkind IN ('r', 'p', 'f')
542
- AND c.relname = table_row.tablename;
543
-
544
- SELECT obj_description(c.oid, 'pg_class')
545
- INTO target_table_comment
546
- FROM pg_class c
547
- JOIN pg_namespace n ON n.oid = c.relnamespace
548
- WHERE n.nspname = '${senatSchemaName}'
549
- AND c.relkind IN ('r', 'p')
550
- AND c.relname = table_row.tablename;
551
-
552
- IF staging_table_comment IS DISTINCT FROM target_table_comment THEN
553
- EXECUTE format(
554
- 'COMMENT ON TABLE ${senatSchemaName}.%I IS %L',
555
- table_row.tablename,
556
- staging_table_comment
557
- );
558
- END IF;
559
-
560
- FOR column_row IN
561
- SELECT
562
- staging_columns.column_name,
563
- col_description(staging_class.oid, staging_attribute.attnum) AS staging_comment,
564
- col_description(target_class.oid, target_attribute.attnum) AS target_comment
565
- FROM information_schema.columns staging_columns
566
- JOIN pg_class staging_class ON staging_class.relname = staging_columns.table_name
567
- JOIN pg_namespace staging_namespace
568
- ON staging_namespace.oid = staging_class.relnamespace
569
- AND staging_namespace.nspname = staging_columns.table_schema
570
- JOIN pg_attribute staging_attribute
571
- ON staging_attribute.attrelid = staging_class.oid
572
- AND staging_attribute.attname = staging_columns.column_name
573
- JOIN information_schema.columns target_columns
574
- ON target_columns.table_schema = '${senatSchemaName}'
575
- AND target_columns.table_name = staging_columns.table_name
576
- AND target_columns.column_name = staging_columns.column_name
577
- JOIN pg_class target_class ON target_class.relname = target_columns.table_name
578
- JOIN pg_namespace target_namespace
579
- ON target_namespace.oid = target_class.relnamespace
580
- AND target_namespace.nspname = target_columns.table_schema
581
- JOIN pg_attribute target_attribute
582
- ON target_attribute.attrelid = target_class.oid
583
- AND target_attribute.attname = target_columns.column_name
584
- WHERE staging_columns.table_schema = '${escapedStagingSchema}'
585
- AND staging_columns.table_name = table_row.tablename
586
- ORDER BY staging_columns.ordinal_position
587
- LOOP
588
- IF column_row.staging_comment IS DISTINCT FROM column_row.target_comment THEN
589
- EXECUTE format(
590
- 'COMMENT ON COLUMN ${senatSchemaName}.%I.%I IS %L',
591
- table_row.tablename,
592
- column_row.column_name,
593
- column_row.staging_comment
594
- );
595
- END IF;
596
- END LOOP;
597
-
598
- SELECT string_agg(format('%I', columns.column_name), ', ' ORDER BY columns.ordinal_position)
599
- , string_agg(format('s.%I', columns.column_name), ', ' ORDER BY columns.ordinal_position)
600
- INTO column_list, staging_column_list
601
- FROM information_schema.columns columns
602
- WHERE columns.table_schema = '${escapedStagingSchema}'
603
- AND columns.table_name = table_row.tablename;
604
-
605
- SELECT
606
- string_agg(
607
- format('%1$I = s.%1$I', columns.column_name),
608
- ', ' ORDER BY columns.ordinal_position),
609
- string_agg(
610
- format('t.%1$I IS DISTINCT FROM s.%1$I', columns.column_name),
611
- ' OR ' ORDER BY columns.ordinal_position)
612
- INTO update_assignments, change_condition
613
- FROM information_schema.columns columns
614
- WHERE columns.table_schema = '${escapedStagingSchema}'
615
- AND columns.table_name = table_row.tablename
616
- AND NOT (columns.column_name = ANY(coalesce(key_columns, ARRAY[]::text[])));
617
-
618
- IF key_columns IS NOT NULL THEN
619
- SELECT count(*)
620
- INTO matching_key_column_count
621
- FROM unnest(key_columns) AS key_column(column_name)
622
- JOIN information_schema.columns staging_columns
623
- ON staging_columns.table_schema = '${escapedStagingSchema}'
624
- AND staging_columns.table_name = table_row.tablename
625
- AND staging_columns.column_name = key_column.column_name
626
- JOIN information_schema.columns target_columns
627
- ON target_columns.table_schema = '${senatSchemaName}'
628
- AND target_columns.table_name = table_row.tablename
629
- AND target_columns.column_name = key_column.column_name;
630
-
631
- IF matching_key_column_count <> cardinality(key_columns) THEN
632
- RAISE NOTICE
633
- 'Rejected sync key for %.% from % because only %/% columns are present in both staging and target: %',
634
- '${senatSchemaName}',
635
- table_row.tablename,
636
- coalesce(key_schema_name, 'unknown'),
637
- matching_key_column_count,
638
- cardinality(key_columns),
639
- array_to_string(key_columns, ', ');
640
- key_columns := NULL;
641
- key_schema_name := NULL;
642
- join_condition := NULL;
643
- END IF;
644
- END IF;
645
-
646
- SELECT
647
- 'jsonb_build_array(' ||
648
- string_agg(format('t.%I', columns.column_name),
649
- ', ' ORDER BY columns.ordinal_position) || ')',
650
- 'jsonb_build_array(' ||
651
- string_agg(format('s.%I', columns.column_name),
652
- ', ' ORDER BY columns.ordinal_position) || ')'
653
- INTO target_row_signature_expression, staging_row_signature_expression
654
- FROM information_schema.columns columns
655
- WHERE columns.table_schema = '${escapedStagingSchema}'
656
- AND columns.table_name = table_row.tablename;
657
-
658
- IF use_row_multiset_merge THEN
659
- sync_strategy := 'incremental_row_multiset';
660
- RAISE NOTICE
661
- 'Sync strategy for %.%: %',
662
- '${senatSchemaName}',
663
- table_row.tablename,
664
- sync_strategy;
665
-
666
- EXECUTE format(
667
- 'WITH staging_counts AS (' ||
668
- ' SELECT %2$s AS row_signature, count(*) AS row_count' ||
669
- ' FROM %3$I.%1$I s' ||
670
- ' GROUP BY 1' ||
671
- '), target_ranked AS (' ||
672
- ' SELECT ctid, %4$s AS row_signature, row_number() OVER (PARTITION BY %4$s) AS row_number' ||
673
- ' FROM ${senatSchemaName}.%1$I t' ||
674
- ')' ||
675
- ' DELETE FROM ${senatSchemaName}.%1$I t' ||
676
- ' USING (' ||
677
- ' SELECT candidate_row.ctid' ||
678
- ' FROM target_ranked candidate_row' ||
679
- ' LEFT JOIN staging_counts reference_row USING (row_signature)' ||
680
- ' WHERE candidate_row.row_number > coalesce(reference_row.row_count, 0)' ||
681
- ' ) rows_to_delete' ||
682
- ' WHERE t.ctid = rows_to_delete.ctid',
683
- table_row.tablename,
684
- staging_row_signature_expression,
685
- '${stagingSchema}',
686
- target_row_signature_expression
687
- );
688
-
689
- EXECUTE format(
690
- 'WITH target_counts AS (' ||
691
- ' SELECT %2$s AS row_signature, count(*) AS row_count' ||
692
- ' FROM ${senatSchemaName}.%1$I t' ||
693
- ' GROUP BY 1' ||
694
- '), staging_ranked AS (' ||
695
- ' SELECT %3$s, %4$s AS row_signature, row_number() OVER (PARTITION BY %4$s) AS row_number' ||
696
- ' FROM %5$I.%1$I s' ||
697
- ')' ||
698
- ' INSERT INTO ${senatSchemaName}.%1$I (%3$s)' ||
699
- ' SELECT %3$s' ||
700
- ' FROM staging_ranked candidate_row' ||
701
- ' LEFT JOIN target_counts reference_row USING (row_signature)' ||
702
- ' WHERE candidate_row.row_number > coalesce(reference_row.row_count, 0)',
703
- table_row.tablename,
704
- target_row_signature_expression,
705
- column_list,
706
- staging_row_signature_expression,
707
- '${stagingSchema}'
708
- );
709
- ELSIF key_columns IS NULL THEN
710
- sync_strategy := 'full_refresh_fallback';
711
- RAISE NOTICE
712
- 'Sync strategy for %.%: %',
713
- '${senatSchemaName}',
714
- table_row.tablename,
715
- sync_strategy;
716
- RAISE WARNING
717
- 'No primary key or all-NOT-NULL unique key found for %.%; falling back to full refresh',
718
- '${senatSchemaName}',
719
- table_row.tablename;
720
-
721
- EXECUTE format('TRUNCATE TABLE ${senatSchemaName}.%I', table_row.tablename);
722
-
723
- EXECUTE format(
724
- 'INSERT INTO ${senatSchemaName}.%I (%s) SELECT %s FROM %I.%I s',
725
- table_row.tablename,
726
- column_list,
727
- staging_column_list,
728
- '${stagingSchema}',
729
- table_row.tablename
730
- );
731
- ELSE
732
- sync_strategy := 'incremental';
733
- RAISE NOTICE
734
- 'Sync strategy for %.%: % (key from %: %)',
735
- '${senatSchemaName}',
736
- table_row.tablename,
737
- sync_strategy,
738
- key_schema_name,
739
- array_to_string(key_columns, ', ');
740
-
741
- EXECUTE format(
742
- 'DELETE FROM ${senatSchemaName}.%1$I t WHERE NOT EXISTS (SELECT 1 FROM %2$I.%1$I s WHERE %3$s)',
743
- table_row.tablename,
744
- '${stagingSchema}',
745
- join_condition
746
- );
747
-
748
- IF update_assignments IS NOT NULL THEN
749
- EXECUTE format(
750
- 'UPDATE ${senatSchemaName}.%1$I t SET %2$s FROM %3$I.%1$I s WHERE %4$s AND (%5$s)',
751
- table_row.tablename,
752
- update_assignments,
753
- '${stagingSchema}',
754
- join_condition,
755
- change_condition
756
- );
757
- END IF;
758
-
759
- EXECUTE format(
760
- 'INSERT INTO ${senatSchemaName}.%1$I (%2$s) ' ||
761
- 'SELECT %3$s FROM %4$I.%1$I s ' ||
762
- 'WHERE NOT EXISTS ' ||
763
- '(SELECT 1 FROM ${senatSchemaName}.%1$I t WHERE %5$s)',
764
- table_row.tablename,
765
- column_list,
766
- staging_column_list,
767
- '${stagingSchema}',
768
- join_condition
769
- );
770
- END IF;
771
- END LOOP;
772
-
773
- FOR sequence_row IN
774
- SELECT sequencename
775
- FROM pg_sequences
776
- WHERE schemaname = '${escapedStagingSchema}'
777
- ORDER BY sequencename
778
- LOOP
779
- SELECT
780
- dependent_table.relname AS table_name,
781
- dependent_column.attname AS column_name
782
- INTO sequence_owner
783
- FROM pg_class sequence_class
784
- JOIN pg_namespace sequence_namespace ON sequence_namespace.oid = sequence_class.relnamespace
785
- JOIN pg_depend dependency
786
- ON dependency.objid = sequence_class.oid
787
- AND dependency.deptype = 'a'
788
- JOIN pg_class dependent_table ON dependent_table.oid = dependency.refobjid
789
- JOIN pg_namespace dependent_namespace ON dependent_namespace.oid = dependent_table.relnamespace
790
- JOIN pg_attribute dependent_column
791
- ON dependent_column.attrelid = dependent_table.oid
792
- AND dependent_column.attnum = dependency.refobjsubid
793
- WHERE sequence_namespace.nspname = '${escapedStagingSchema}'
794
- AND dependent_namespace.nspname = '${escapedStagingSchema}'
795
- AND sequence_class.relname = sequence_row.sequencename;
796
-
797
- IF sequence_owner.table_name IS NULL THEN
798
- EXECUTE format('ALTER SEQUENCE ${senatSchemaName}.%I OWNED BY NONE', sequence_row.sequencename);
799
- ELSE
800
- EXECUTE format(
801
- 'ALTER SEQUENCE ${senatSchemaName}.%I OWNED BY ${senatSchemaName}.%I.%I',
802
- sequence_row.sequencename,
803
- sequence_owner.table_name,
804
- sequence_owner.column_name
805
- );
806
- END IF;
807
- END LOOP;
808
-
809
- FOR constraint_row IN
810
- SELECT
811
- table_class.relname AS table_name,
812
- con.conname,
813
- con.contype,
814
- pg_get_constraintdef(con.oid, true) AS constraint_definition
815
- FROM pg_constraint con
816
- JOIN pg_class table_class ON table_class.oid = con.conrelid
817
- JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
818
- WHERE table_namespace.nspname = '${escapedStagingSchema}'
819
- AND con.contype <> 'n'
820
- ORDER BY
821
- CASE con.contype
822
- WHEN 'p' THEN 1
823
- WHEN 'u' THEN 2
824
- WHEN 'c' THEN 3
825
- WHEN 'f' THEN 4
826
- ELSE 5
827
- END,
828
- table_class.relname,
829
- con.conname
830
- LOOP
831
- EXECUTE format(
832
- 'ALTER TABLE ${senatSchemaName}.%I ADD CONSTRAINT %I %s',
833
- constraint_row.table_name,
834
- constraint_row.conname,
835
- replace(
836
- replace(
837
- constraint_row.constraint_definition,
838
- ' REFERENCES ' || '${stagingSchema}' || '.',
839
- ' REFERENCES ${senatSchemaName}.'
840
- ),
841
- ' REFERENCES ONLY ' || '${stagingSchema}' || '.',
842
- ' REFERENCES ONLY ${senatSchemaName}.'
843
- )
844
- );
845
- END LOOP;
846
-
847
- FOR index_row IN
848
- SELECT
849
- table_class.relname AS table_name,
850
- pg_get_indexdef(index_data.indexrelid) AS index_definition
851
- FROM pg_index index_data
852
- JOIN pg_class table_class ON table_class.oid = index_data.indrelid
853
- JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
854
- LEFT JOIN pg_constraint con ON con.conindid = index_data.indexrelid
855
- WHERE table_namespace.nspname = '${escapedStagingSchema}'
856
- AND con.oid IS NULL
857
- ORDER BY table_class.relname, pg_get_indexdef(index_data.indexrelid)
858
- LOOP
859
- EXECUTE replace(
860
- index_row.index_definition,
861
- ' ON ${stagingSchema}.',
862
- ' ON ${senatSchemaName}.'
863
- );
864
- END LOOP;
865
-
866
- FOR sequence_row IN
867
- SELECT sequencename
868
- FROM pg_sequences
869
- WHERE schemaname = '${escapedStagingSchema}'
870
- ORDER BY sequencename
871
- LOOP
872
- EXECUTE format(
873
- 'SELECT last_value, is_called FROM %I.%I',
874
- '${stagingSchema}',
875
- sequence_row.sequencename
876
- )
877
- INTO sequence_state;
878
-
879
- EXECUTE format(
880
- 'SELECT setval(%L, %s, %L)',
881
- '${senatSchemaName}.' || sequence_row.sequencename,
882
- sequence_state.last_value,
883
- sequence_state.is_called
884
- );
885
- END LOOP;
886
-
887
- FOR sequence_row IN
888
- SELECT c.relname AS sequencename
889
- FROM pg_class c
890
- JOIN pg_namespace n ON n.oid = c.relnamespace
891
- WHERE n.nspname = '${senatSchemaName}'
892
- AND c.relkind = 'S'
893
- AND c.relname LIKE '${escapedPrefix}' || '%'
894
- ORDER BY c.relname
895
- LOOP
896
- IF NOT EXISTS (
897
- SELECT 1
898
- FROM pg_sequences
899
- WHERE schemaname = '${escapedStagingSchema}'
900
- AND sequencename = sequence_row.sequencename
901
- ) THEN
902
- EXECUTE format('DROP SEQUENCE IF EXISTS ${senatSchemaName}.%I CASCADE', sequence_row.sequencename);
903
- END IF;
904
- END LOOP;
905
- END $$;
906
-
907
- GRANT SELECT ON ALL TABLES IN SCHEMA ${senatSchemaName} TO ${dbUser};
103
+ return `
104
+ DO $$
105
+ DECLARE
106
+ table_row record;
107
+ column_row record;
108
+ sequence_row record;
109
+ constraint_row record;
110
+ index_row record;
111
+ sequence_owner record;
112
+ sequence_state record;
113
+ staging_default text;
114
+ target_default text;
115
+ column_definition text;
116
+ column_list text;
117
+ staging_column_list text;
118
+ key_columns text[];
119
+ key_schema_name text;
120
+ configured_key_columns text[];
121
+ matching_key_column_count integer;
122
+ use_row_multiset_merge boolean;
123
+ join_condition text;
124
+ update_assignments text;
125
+ change_condition text;
126
+ target_row_signature_expression text;
127
+ staging_row_signature_expression text;
128
+ sync_strategy text;
129
+ staging_table_comment text;
130
+ target_table_comment text;
131
+ BEGIN
132
+ PERFORM set_config('search_path', '${senatSchemaName}, pg_catalog', true);
133
+
134
+ FOR table_row IN
135
+ ${stagingRelationsQuery}
136
+ LOOP
137
+ IF NOT EXISTS (
138
+ SELECT 1
139
+ FROM pg_tables
140
+ WHERE schemaname = '${senatSchemaName}'
141
+ AND tablename = table_row.tablename
142
+ ) THEN
143
+ EXECUTE format(
144
+ 'CREATE TABLE ${senatSchemaName}.%I (LIKE %I.%I INCLUDING IDENTITY INCLUDING GENERATED INCLUDING DEFAULTS)',
145
+ table_row.tablename,
146
+ '${stagingSchema}',
147
+ table_row.tablename
148
+ );
149
+ END IF;
150
+ END LOOP;
151
+
152
+ FOR sequence_row IN
153
+ SELECT
154
+ sequencename,
155
+ data_type,
156
+ start_value,
157
+ min_value,
158
+ max_value,
159
+ increment_by,
160
+ cycle,
161
+ cache_size
162
+ FROM pg_sequences
163
+ WHERE schemaname = '${escapedStagingSchema}'
164
+ ORDER BY sequencename
165
+ LOOP
166
+ IF NOT EXISTS (
167
+ SELECT 1
168
+ FROM pg_class c
169
+ JOIN pg_namespace n ON n.oid = c.relnamespace
170
+ WHERE n.nspname = '${senatSchemaName}'
171
+ AND c.relkind = 'S'
172
+ AND c.relname = sequence_row.sequencename
173
+ ) THEN
174
+ EXECUTE format('CREATE SEQUENCE ${senatSchemaName}.%I', sequence_row.sequencename);
175
+ END IF;
176
+
177
+ EXECUTE format(
178
+ 'ALTER SEQUENCE ${senatSchemaName}.%I AS %s INCREMENT BY %s MINVALUE %s MAXVALUE %s START WITH %s CACHE %s %s',
179
+ sequence_row.sequencename,
180
+ sequence_row.data_type,
181
+ sequence_row.increment_by,
182
+ sequence_row.min_value,
183
+ sequence_row.max_value,
184
+ sequence_row.start_value,
185
+ sequence_row.cache_size,
186
+ CASE WHEN sequence_row.cycle THEN 'CYCLE' ELSE 'NO CYCLE' END
187
+ );
188
+ END LOOP;
189
+
190
+ FOR table_row IN
191
+ SELECT tablename
192
+ FROM pg_tables
193
+ WHERE schemaname = '${senatSchemaName}'
194
+ AND tablename LIKE '${escapedPrefix}' || '%'
195
+ ORDER BY tablename
196
+ LOOP
197
+ IF NOT EXISTS (
198
+ SELECT 1
199
+ FROM pg_class c
200
+ JOIN pg_namespace n ON n.oid = c.relnamespace
201
+ WHERE n.nspname = '${escapedStagingSchema}'
202
+ AND c.relkind IN ('r', 'p', 'f')
203
+ AND c.relname = table_row.tablename
204
+ ) THEN
205
+ EXECUTE format('DROP TABLE IF EXISTS ${senatSchemaName}.%I CASCADE', table_row.tablename);
206
+ END IF;
207
+ END LOOP;
208
+
209
+ FOR table_row IN
210
+ ${stagingRelationsQuery}
211
+ LOOP
212
+ key_columns := NULL;
213
+ key_schema_name := NULL;
214
+ configured_key_columns := NULL;
215
+ matching_key_column_count := 0;
216
+ use_row_multiset_merge := false;
217
+ join_condition := NULL;
218
+ update_assignments := NULL;
219
+ change_condition := NULL;
220
+ target_row_signature_expression := NULL;
221
+ staging_row_signature_expression := NULL;
222
+
223
+ SELECT configured.key_columns
224
+ INTO configured_key_columns
225
+ FROM (
226
+ ${configuredMergeKeysValues}
227
+ ) AS configured
228
+ WHERE configured.tablename = table_row.tablename;
229
+
230
+ IF configured_key_columns IS NOT NULL THEN
231
+ key_columns := configured_key_columns;
232
+ key_schema_name := 'configured';
233
+
234
+ SELECT string_agg(format('t.%1$I IS NOT DISTINCT FROM s.%1$I', key_column), ' AND ' ORDER BY ordinality)
235
+ INTO join_condition
236
+ FROM unnest(configured_key_columns) WITH ORDINALITY AS configured_key(key_column, ordinality);
237
+ ELSE
238
+ SELECT candidate.key_columns, candidate.join_condition, candidate.source_schema
239
+ INTO key_columns, join_condition, key_schema_name
240
+ FROM (
241
+ SELECT
242
+ array_agg(att.attname ORDER BY key_column.ordinality) AS key_columns,
243
+ string_agg(format('t.%1$I = s.%1$I', att.attname), ' AND ' ORDER BY key_column.ordinality) AS join_condition,
244
+ table_namespace.nspname AS source_schema,
245
+ CASE WHEN table_namespace.nspname = '${escapedStagingSchema}' THEN 0 ELSE 1 END AS schema_priority,
246
+ CASE WHEN index_data.indisprimary THEN 0 ELSE 1 END AS priority,
247
+ count(*) AS key_column_count,
248
+ index_data.indexrelid
249
+ FROM pg_index index_data
250
+ JOIN pg_class table_class ON table_class.oid = index_data.indrelid
251
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
252
+ JOIN LATERAL unnest(index_data.indkey::int2[]) WITH ORDINALITY AS key_column(attnum, ordinality) ON true
253
+ JOIN pg_attribute att
254
+ ON att.attrelid = table_class.oid
255
+ AND att.attnum = key_column.attnum
256
+ WHERE table_namespace.nspname IN ('${escapedStagingSchema}', '${senatSchemaName}')
257
+ AND table_class.relname = table_row.tablename
258
+ AND index_data.indisunique
259
+ AND index_data.indpred IS NULL
260
+ AND index_data.indexprs IS NULL
261
+ AND key_column.ordinality <= index_data.indnkeyatts
262
+ AND key_column.attnum > 0
263
+ AND EXISTS (
264
+ SELECT 1
265
+ FROM information_schema.columns staging_columns
266
+ WHERE staging_columns.table_schema = '${escapedStagingSchema}'
267
+ AND staging_columns.table_name = table_row.tablename
268
+ AND staging_columns.column_name = att.attname
269
+ )
270
+ GROUP BY index_data.indexrelid, index_data.indisprimary, table_namespace.nspname
271
+ HAVING index_data.indisprimary OR bool_and(att.attnotnull)
272
+ ORDER BY schema_priority, priority, key_column_count, index_data.indexrelid
273
+ LIMIT 1
274
+ ) AS candidate;
275
+ END IF;
276
+
277
+ SELECT EXISTS (
278
+ SELECT 1
279
+ FROM (
280
+ ${configuredRowMultisetTablesValues}
281
+ ) AS configured
282
+ WHERE configured.tablename = table_row.tablename
283
+ )
284
+ INTO use_row_multiset_merge;
285
+
286
+ -- The synchronization below temporarily drops primary/unique constraints before
287
+ -- issuing DELETE/UPDATE statements. On tables that belong to a publication with
288
+ -- deletes/updates enabled, PostgreSQL then refuses those statements unless the
289
+ -- table has an explicit replica identity. FULL keeps logical replication valid
290
+ -- while the constraints are being rebuilt.
291
+ EXECUTE format('ALTER TABLE ${senatSchemaName}.%I REPLICA IDENTITY FULL', table_row.tablename);
292
+
293
+ FOR constraint_row IN
294
+ SELECT con.conname
295
+ FROM pg_constraint con
296
+ JOIN pg_class c ON c.oid = con.conrelid
297
+ JOIN pg_namespace n ON n.oid = c.relnamespace
298
+ WHERE n.nspname = '${senatSchemaName}'
299
+ AND c.relname = table_row.tablename
300
+ AND con.contype <> 'n'
301
+ LOOP
302
+ EXECUTE format(
303
+ 'ALTER TABLE ${senatSchemaName}.%I DROP CONSTRAINT IF EXISTS %I CASCADE',
304
+ table_row.tablename,
305
+ constraint_row.conname
306
+ );
307
+ END LOOP;
308
+
309
+ FOR index_row IN
310
+ SELECT index_class.relname AS index_name
311
+ FROM pg_index idx
312
+ JOIN pg_class table_class ON table_class.oid = idx.indrelid
313
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
314
+ JOIN pg_class index_class ON index_class.oid = idx.indexrelid
315
+ LEFT JOIN pg_constraint con ON con.conindid = idx.indexrelid
316
+ WHERE table_namespace.nspname = '${senatSchemaName}'
317
+ AND table_class.relname = table_row.tablename
318
+ AND con.oid IS NULL
319
+ LOOP
320
+ EXECUTE format('DROP INDEX IF EXISTS ${senatSchemaName}.%I', index_row.index_name);
321
+ END LOOP;
322
+
323
+ FOR column_row IN
324
+ SELECT target_columns.column_name
325
+ FROM information_schema.columns target_columns
326
+ WHERE target_columns.table_schema = '${senatSchemaName}'
327
+ AND target_columns.table_name = table_row.tablename
328
+ AND NOT EXISTS (
329
+ SELECT 1
330
+ FROM information_schema.columns staging_columns
331
+ WHERE staging_columns.table_schema = '${escapedStagingSchema}'
332
+ AND staging_columns.table_name = table_row.tablename
333
+ AND staging_columns.column_name = target_columns.column_name
334
+ )
335
+ ORDER BY target_columns.ordinal_position DESC
336
+ LOOP
337
+ EXECUTE format(
338
+ 'ALTER TABLE ${senatSchemaName}.%I DROP COLUMN IF EXISTS %I CASCADE',
339
+ table_row.tablename,
340
+ column_row.column_name
341
+ );
342
+ END LOOP;
343
+
344
+ FOR column_row IN
345
+ SELECT
346
+ staging_columns.column_name,
347
+ format_type(a.atttypid, a.atttypmod) AS data_type,
348
+ a.attnotnull AS not_null,
349
+ a.attidentity AS identity_kind,
350
+ a.attgenerated AS generated_kind,
351
+ pg_get_expr(ad.adbin, ad.adrelid) AS default_expression,
352
+ CASE
353
+ WHEN coll.oid IS NULL THEN NULL
354
+ ELSE format('%I.%I', coll_namespace.nspname, coll.collname)
355
+ END AS collation_name
356
+ FROM information_schema.columns staging_columns
357
+ JOIN pg_class c ON c.relname = staging_columns.table_name
358
+ JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = staging_columns.table_schema
359
+ JOIN pg_attribute a ON a.attrelid = c.oid AND a.attname = staging_columns.column_name
360
+ LEFT JOIN pg_attrdef ad ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum
361
+ LEFT JOIN pg_type typ ON typ.oid = a.atttypid
362
+ LEFT JOIN pg_collation coll ON coll.oid = a.attcollation AND coll.oid <> typ.typcollation
363
+ LEFT JOIN pg_namespace coll_namespace ON coll_namespace.oid = coll.collnamespace
364
+ WHERE staging_columns.table_schema = '${escapedStagingSchema}'
365
+ AND staging_columns.table_name = table_row.tablename
366
+ AND NOT EXISTS (
367
+ SELECT 1
368
+ FROM information_schema.columns target_columns
369
+ WHERE target_columns.table_schema = '${senatSchemaName}'
370
+ AND target_columns.table_name = table_row.tablename
371
+ AND target_columns.column_name = staging_columns.column_name
372
+ )
373
+ ORDER BY staging_columns.ordinal_position
374
+ LOOP
375
+ column_definition := column_row.data_type;
376
+ IF column_row.collation_name IS NOT NULL THEN
377
+ column_definition := column_definition || ' COLLATE ' || column_row.collation_name;
378
+ END IF;
379
+ IF column_row.generated_kind = 's' THEN
380
+ column_definition :=
381
+ column_definition ||
382
+ ' GENERATED ALWAYS AS (' ||
383
+ replace(column_row.default_expression,
384
+ '${stagingSchema}.', '${senatSchemaName}.') ||
385
+ ') STORED';
386
+ ELSIF column_row.identity_kind = 'a' THEN
387
+ column_definition := column_definition || ' GENERATED ALWAYS AS IDENTITY';
388
+ ELSIF column_row.identity_kind = 'd' THEN
389
+ column_definition := column_definition || ' GENERATED BY DEFAULT AS IDENTITY';
390
+ END IF;
391
+
392
+ EXECUTE format(
393
+ 'ALTER TABLE ${senatSchemaName}.%I ADD COLUMN %I %s',
394
+ table_row.tablename,
395
+ column_row.column_name,
396
+ column_definition
397
+ );
398
+
399
+ IF column_row.generated_kind = ''
400
+ AND column_row.identity_kind = ''
401
+ AND column_row.default_expression IS NOT NULL THEN
402
+ EXECUTE format(
403
+ 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I SET DEFAULT %s',
404
+ table_row.tablename,
405
+ column_row.column_name,
406
+ replace(column_row.default_expression, '${stagingSchema}.', '${senatSchemaName}.')
407
+ );
408
+ END IF;
409
+
410
+ IF column_row.not_null THEN
411
+ EXECUTE format(
412
+ 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I SET NOT NULL',
413
+ table_row.tablename,
414
+ column_row.column_name
415
+ );
416
+ END IF;
417
+ END LOOP;
418
+
419
+ FOR column_row IN
420
+ SELECT
421
+ staging_columns.column_name,
422
+ format_type(staging_attribute.atttypid, staging_attribute.atttypmod) AS staging_data_type,
423
+ format_type(target_attribute.atttypid, target_attribute.atttypmod) AS target_data_type,
424
+ staging_attribute.attnotnull AS staging_not_null,
425
+ target_attribute.attnotnull AS target_not_null,
426
+ staging_attribute.attidentity AS staging_identity_kind,
427
+ target_attribute.attidentity AS target_identity_kind,
428
+ staging_attribute.attgenerated AS staging_generated_kind,
429
+ target_attribute.attgenerated AS target_generated_kind,
430
+ pg_get_expr(staging_default_expr.adbin, staging_default_expr.adrelid) AS staging_default_expression,
431
+ pg_get_expr(target_default_expr.adbin, target_default_expr.adrelid) AS target_default_expression,
432
+ CASE
433
+ WHEN staging_collation.oid IS NULL THEN NULL
434
+ ELSE format('%I.%I', staging_collation_namespace.nspname, staging_collation.collname)
435
+ END AS staging_collation_name,
436
+ CASE
437
+ WHEN target_collation.oid IS NULL THEN NULL
438
+ ELSE format('%I.%I', target_collation_namespace.nspname, target_collation.collname)
439
+ END AS target_collation_name
440
+ FROM information_schema.columns staging_columns
441
+ JOIN information_schema.columns target_columns
442
+ ON target_columns.table_schema = '${senatSchemaName}'
443
+ AND target_columns.table_name = staging_columns.table_name
444
+ AND target_columns.column_name = staging_columns.column_name
445
+ JOIN pg_class staging_class ON staging_class.relname = staging_columns.table_name
446
+ JOIN pg_namespace staging_namespace
447
+ ON staging_namespace.oid = staging_class.relnamespace
448
+ AND staging_namespace.nspname = staging_columns.table_schema
449
+ JOIN pg_attribute staging_attribute
450
+ ON staging_attribute.attrelid = staging_class.oid
451
+ AND staging_attribute.attname = staging_columns.column_name
452
+ LEFT JOIN pg_attrdef staging_default_expr
453
+ ON staging_default_expr.adrelid = staging_attribute.attrelid
454
+ AND staging_default_expr.adnum = staging_attribute.attnum
455
+ LEFT JOIN pg_type staging_type ON staging_type.oid = staging_attribute.atttypid
456
+ LEFT JOIN pg_collation staging_collation
457
+ ON staging_collation.oid = staging_attribute.attcollation
458
+ AND staging_collation.oid <> staging_type.typcollation
459
+ LEFT JOIN pg_namespace staging_collation_namespace
460
+ ON staging_collation_namespace.oid = staging_collation.collnamespace
461
+ JOIN pg_class target_class ON target_class.relname = target_columns.table_name
462
+ JOIN pg_namespace target_namespace
463
+ ON target_namespace.oid = target_class.relnamespace
464
+ AND target_namespace.nspname = target_columns.table_schema
465
+ JOIN pg_attribute target_attribute
466
+ ON target_attribute.attrelid = target_class.oid
467
+ AND target_attribute.attname = target_columns.column_name
468
+ LEFT JOIN pg_attrdef target_default_expr
469
+ ON target_default_expr.adrelid = target_attribute.attrelid
470
+ AND target_default_expr.adnum = target_attribute.attnum
471
+ LEFT JOIN pg_type target_type ON target_type.oid = target_attribute.atttypid
472
+ LEFT JOIN pg_collation target_collation
473
+ ON target_collation.oid = target_attribute.attcollation
474
+ AND target_collation.oid <> target_type.typcollation
475
+ LEFT JOIN pg_namespace target_collation_namespace
476
+ ON target_collation_namespace.oid = target_collation.collnamespace
477
+ WHERE staging_columns.table_schema = '${escapedStagingSchema}'
478
+ AND staging_columns.table_name = table_row.tablename
479
+ ORDER BY staging_columns.ordinal_position
480
+ LOOP
481
+ IF column_row.staging_identity_kind <> column_row.target_identity_kind
482
+ OR column_row.staging_generated_kind <> column_row.target_generated_kind THEN
483
+ RAISE EXCEPTION
484
+ 'Unsupported identity/generated column change for %.%',
485
+ table_row.tablename,
486
+ column_row.column_name;
487
+ END IF;
488
+
489
+ IF column_row.staging_data_type <> column_row.target_data_type
490
+ OR column_row.staging_collation_name IS DISTINCT FROM column_row.target_collation_name THEN
491
+ EXECUTE format(
492
+ 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I TYPE %s%s USING %I::%s',
493
+ table_row.tablename,
494
+ column_row.column_name,
495
+ column_row.staging_data_type,
496
+ CASE
497
+ WHEN column_row.staging_collation_name IS NULL THEN ''
498
+ ELSE ' COLLATE ' || column_row.staging_collation_name
499
+ END,
500
+ column_row.column_name,
501
+ column_row.staging_data_type
502
+ );
503
+ END IF;
504
+
505
+ staging_default := replace(
506
+ coalesce(column_row.staging_default_expression, ''),
507
+ '${stagingSchema}.',
508
+ '${senatSchemaName}.'
509
+ );
510
+ target_default := replace(
511
+ coalesce(column_row.target_default_expression, ''),
512
+ '${stagingSchema}.',
513
+ '${senatSchemaName}.'
514
+ );
515
+
516
+ IF staging_default <> target_default THEN
517
+ IF column_row.staging_default_expression IS NULL THEN
518
+ EXECUTE format(
519
+ 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I DROP DEFAULT',
520
+ table_row.tablename,
521
+ column_row.column_name
522
+ );
523
+ ELSE
524
+ EXECUTE format(
525
+ 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I SET DEFAULT %s',
526
+ table_row.tablename,
527
+ column_row.column_name,
528
+ staging_default
529
+ );
530
+ END IF;
531
+ END IF;
532
+
533
+ IF column_row.staging_not_null <> column_row.target_not_null THEN
534
+ EXECUTE format(
535
+ 'ALTER TABLE ${senatSchemaName}.%I ALTER COLUMN %I %s NOT NULL',
536
+ table_row.tablename,
537
+ column_row.column_name,
538
+ CASE WHEN column_row.staging_not_null THEN 'SET' ELSE 'DROP' END
539
+ );
540
+ END IF;
541
+ END LOOP;
542
+
543
+ SELECT obj_description(c.oid, 'pg_class')
544
+ INTO staging_table_comment
545
+ FROM pg_class c
546
+ JOIN pg_namespace n ON n.oid = c.relnamespace
547
+ WHERE n.nspname = '${escapedStagingSchema}'
548
+ AND c.relkind IN ('r', 'p', 'f')
549
+ AND c.relname = table_row.tablename;
550
+
551
+ SELECT obj_description(c.oid, 'pg_class')
552
+ INTO target_table_comment
553
+ FROM pg_class c
554
+ JOIN pg_namespace n ON n.oid = c.relnamespace
555
+ WHERE n.nspname = '${senatSchemaName}'
556
+ AND c.relkind IN ('r', 'p')
557
+ AND c.relname = table_row.tablename;
558
+
559
+ IF staging_table_comment IS DISTINCT FROM target_table_comment THEN
560
+ EXECUTE format(
561
+ 'COMMENT ON TABLE ${senatSchemaName}.%I IS %L',
562
+ table_row.tablename,
563
+ staging_table_comment
564
+ );
565
+ END IF;
566
+
567
+ FOR column_row IN
568
+ SELECT
569
+ staging_columns.column_name,
570
+ col_description(staging_class.oid, staging_attribute.attnum) AS staging_comment,
571
+ col_description(target_class.oid, target_attribute.attnum) AS target_comment
572
+ FROM information_schema.columns staging_columns
573
+ JOIN pg_class staging_class ON staging_class.relname = staging_columns.table_name
574
+ JOIN pg_namespace staging_namespace
575
+ ON staging_namespace.oid = staging_class.relnamespace
576
+ AND staging_namespace.nspname = staging_columns.table_schema
577
+ JOIN pg_attribute staging_attribute
578
+ ON staging_attribute.attrelid = staging_class.oid
579
+ AND staging_attribute.attname = staging_columns.column_name
580
+ JOIN information_schema.columns target_columns
581
+ ON target_columns.table_schema = '${senatSchemaName}'
582
+ AND target_columns.table_name = staging_columns.table_name
583
+ AND target_columns.column_name = staging_columns.column_name
584
+ JOIN pg_class target_class ON target_class.relname = target_columns.table_name
585
+ JOIN pg_namespace target_namespace
586
+ ON target_namespace.oid = target_class.relnamespace
587
+ AND target_namespace.nspname = target_columns.table_schema
588
+ JOIN pg_attribute target_attribute
589
+ ON target_attribute.attrelid = target_class.oid
590
+ AND target_attribute.attname = target_columns.column_name
591
+ WHERE staging_columns.table_schema = '${escapedStagingSchema}'
592
+ AND staging_columns.table_name = table_row.tablename
593
+ ORDER BY staging_columns.ordinal_position
594
+ LOOP
595
+ IF column_row.staging_comment IS DISTINCT FROM column_row.target_comment THEN
596
+ EXECUTE format(
597
+ 'COMMENT ON COLUMN ${senatSchemaName}.%I.%I IS %L',
598
+ table_row.tablename,
599
+ column_row.column_name,
600
+ column_row.staging_comment
601
+ );
602
+ END IF;
603
+ END LOOP;
604
+
605
+ SELECT string_agg(format('%I', columns.column_name), ', ' ORDER BY columns.ordinal_position)
606
+ , string_agg(format('s.%I', columns.column_name), ', ' ORDER BY columns.ordinal_position)
607
+ INTO column_list, staging_column_list
608
+ FROM information_schema.columns columns
609
+ WHERE columns.table_schema = '${escapedStagingSchema}'
610
+ AND columns.table_name = table_row.tablename;
611
+
612
+ SELECT
613
+ string_agg(
614
+ format('%1$I = s.%1$I', columns.column_name),
615
+ ', ' ORDER BY columns.ordinal_position),
616
+ string_agg(
617
+ format('t.%1$I IS DISTINCT FROM s.%1$I', columns.column_name),
618
+ ' OR ' ORDER BY columns.ordinal_position)
619
+ INTO update_assignments, change_condition
620
+ FROM information_schema.columns columns
621
+ WHERE columns.table_schema = '${escapedStagingSchema}'
622
+ AND columns.table_name = table_row.tablename
623
+ AND NOT (columns.column_name = ANY(coalesce(key_columns, ARRAY[]::text[])));
624
+
625
+ IF key_columns IS NOT NULL THEN
626
+ SELECT count(*)
627
+ INTO matching_key_column_count
628
+ FROM unnest(key_columns) AS key_column(column_name)
629
+ JOIN information_schema.columns staging_columns
630
+ ON staging_columns.table_schema = '${escapedStagingSchema}'
631
+ AND staging_columns.table_name = table_row.tablename
632
+ AND staging_columns.column_name = key_column.column_name
633
+ JOIN information_schema.columns target_columns
634
+ ON target_columns.table_schema = '${senatSchemaName}'
635
+ AND target_columns.table_name = table_row.tablename
636
+ AND target_columns.column_name = key_column.column_name;
637
+
638
+ IF matching_key_column_count <> cardinality(key_columns) THEN
639
+ RAISE NOTICE
640
+ 'Rejected sync key for %.% from % because only %/% columns are present in both staging and target: %',
641
+ '${senatSchemaName}',
642
+ table_row.tablename,
643
+ coalesce(key_schema_name, 'unknown'),
644
+ matching_key_column_count,
645
+ cardinality(key_columns),
646
+ array_to_string(key_columns, ', ');
647
+ key_columns := NULL;
648
+ key_schema_name := NULL;
649
+ join_condition := NULL;
650
+ END IF;
651
+ END IF;
652
+
653
+ SELECT
654
+ 'jsonb_build_array(' ||
655
+ string_agg(format('t.%I', columns.column_name),
656
+ ', ' ORDER BY columns.ordinal_position) || ')',
657
+ 'jsonb_build_array(' ||
658
+ string_agg(format('s.%I', columns.column_name),
659
+ ', ' ORDER BY columns.ordinal_position) || ')'
660
+ INTO target_row_signature_expression, staging_row_signature_expression
661
+ FROM information_schema.columns columns
662
+ WHERE columns.table_schema = '${escapedStagingSchema}'
663
+ AND columns.table_name = table_row.tablename;
664
+
665
+ IF use_row_multiset_merge THEN
666
+ sync_strategy := 'incremental_row_multiset';
667
+ RAISE NOTICE
668
+ 'Sync strategy for %.%: %',
669
+ '${senatSchemaName}',
670
+ table_row.tablename,
671
+ sync_strategy;
672
+
673
+ EXECUTE format(
674
+ 'WITH staging_counts AS (' ||
675
+ ' SELECT %2$s AS row_signature, count(*) AS row_count' ||
676
+ ' FROM %3$I.%1$I s' ||
677
+ ' GROUP BY 1' ||
678
+ '), target_ranked AS (' ||
679
+ ' SELECT ctid, %4$s AS row_signature, row_number() OVER (PARTITION BY %4$s) AS row_number' ||
680
+ ' FROM ${senatSchemaName}.%1$I t' ||
681
+ ')' ||
682
+ ' DELETE FROM ${senatSchemaName}.%1$I t' ||
683
+ ' USING (' ||
684
+ ' SELECT candidate_row.ctid' ||
685
+ ' FROM target_ranked candidate_row' ||
686
+ ' LEFT JOIN staging_counts reference_row USING (row_signature)' ||
687
+ ' WHERE candidate_row.row_number > coalesce(reference_row.row_count, 0)' ||
688
+ ' ) rows_to_delete' ||
689
+ ' WHERE t.ctid = rows_to_delete.ctid',
690
+ table_row.tablename,
691
+ staging_row_signature_expression,
692
+ '${stagingSchema}',
693
+ target_row_signature_expression
694
+ );
695
+
696
+ EXECUTE format(
697
+ 'WITH target_counts AS (' ||
698
+ ' SELECT %2$s AS row_signature, count(*) AS row_count' ||
699
+ ' FROM ${senatSchemaName}.%1$I t' ||
700
+ ' GROUP BY 1' ||
701
+ '), staging_ranked AS (' ||
702
+ ' SELECT %3$s, %4$s AS row_signature, row_number() OVER (PARTITION BY %4$s) AS row_number' ||
703
+ ' FROM %5$I.%1$I s' ||
704
+ ')' ||
705
+ ' INSERT INTO ${senatSchemaName}.%1$I (%3$s)' ||
706
+ ' SELECT %3$s' ||
707
+ ' FROM staging_ranked candidate_row' ||
708
+ ' LEFT JOIN target_counts reference_row USING (row_signature)' ||
709
+ ' WHERE candidate_row.row_number > coalesce(reference_row.row_count, 0)',
710
+ table_row.tablename,
711
+ target_row_signature_expression,
712
+ column_list,
713
+ staging_row_signature_expression,
714
+ '${stagingSchema}'
715
+ );
716
+ ELSIF key_columns IS NULL THEN
717
+ sync_strategy := 'full_refresh_fallback';
718
+ RAISE NOTICE
719
+ 'Sync strategy for %.%: %',
720
+ '${senatSchemaName}',
721
+ table_row.tablename,
722
+ sync_strategy;
723
+ RAISE WARNING
724
+ 'No primary key or all-NOT-NULL unique key found for %.%; falling back to full refresh',
725
+ '${senatSchemaName}',
726
+ table_row.tablename;
727
+
728
+ EXECUTE format('TRUNCATE TABLE ${senatSchemaName}.%I', table_row.tablename);
729
+
730
+ EXECUTE format(
731
+ 'INSERT INTO ${senatSchemaName}.%I (%s) SELECT %s FROM %I.%I s',
732
+ table_row.tablename,
733
+ column_list,
734
+ staging_column_list,
735
+ '${stagingSchema}',
736
+ table_row.tablename
737
+ );
738
+ ELSE
739
+ sync_strategy := 'incremental';
740
+ RAISE NOTICE
741
+ 'Sync strategy for %.%: % (key from %: %)',
742
+ '${senatSchemaName}',
743
+ table_row.tablename,
744
+ sync_strategy,
745
+ key_schema_name,
746
+ array_to_string(key_columns, ', ');
747
+
748
+ EXECUTE format(
749
+ 'DELETE FROM ${senatSchemaName}.%1$I t WHERE NOT EXISTS (SELECT 1 FROM %2$I.%1$I s WHERE %3$s)',
750
+ table_row.tablename,
751
+ '${stagingSchema}',
752
+ join_condition
753
+ );
754
+
755
+ IF update_assignments IS NOT NULL THEN
756
+ EXECUTE format(
757
+ 'UPDATE ${senatSchemaName}.%1$I t SET %2$s FROM %3$I.%1$I s WHERE %4$s AND (%5$s)',
758
+ table_row.tablename,
759
+ update_assignments,
760
+ '${stagingSchema}',
761
+ join_condition,
762
+ change_condition
763
+ );
764
+ END IF;
765
+
766
+ EXECUTE format(
767
+ 'INSERT INTO ${senatSchemaName}.%1$I (%2$s) ' ||
768
+ 'SELECT %3$s FROM %4$I.%1$I s ' ||
769
+ 'WHERE NOT EXISTS ' ||
770
+ '(SELECT 1 FROM ${senatSchemaName}.%1$I t WHERE %5$s)',
771
+ table_row.tablename,
772
+ column_list,
773
+ staging_column_list,
774
+ '${stagingSchema}',
775
+ join_condition
776
+ );
777
+ END IF;
778
+ END LOOP;
779
+
780
+ FOR sequence_row IN
781
+ SELECT sequencename
782
+ FROM pg_sequences
783
+ WHERE schemaname = '${escapedStagingSchema}'
784
+ ORDER BY sequencename
785
+ LOOP
786
+ SELECT
787
+ dependent_table.relname AS table_name,
788
+ dependent_column.attname AS column_name
789
+ INTO sequence_owner
790
+ FROM pg_class sequence_class
791
+ JOIN pg_namespace sequence_namespace ON sequence_namespace.oid = sequence_class.relnamespace
792
+ JOIN pg_depend dependency
793
+ ON dependency.objid = sequence_class.oid
794
+ AND dependency.deptype = 'a'
795
+ JOIN pg_class dependent_table ON dependent_table.oid = dependency.refobjid
796
+ JOIN pg_namespace dependent_namespace ON dependent_namespace.oid = dependent_table.relnamespace
797
+ JOIN pg_attribute dependent_column
798
+ ON dependent_column.attrelid = dependent_table.oid
799
+ AND dependent_column.attnum = dependency.refobjsubid
800
+ WHERE sequence_namespace.nspname = '${escapedStagingSchema}'
801
+ AND dependent_namespace.nspname = '${escapedStagingSchema}'
802
+ AND sequence_class.relname = sequence_row.sequencename;
803
+
804
+ IF sequence_owner.table_name IS NULL THEN
805
+ EXECUTE format('ALTER SEQUENCE ${senatSchemaName}.%I OWNED BY NONE', sequence_row.sequencename);
806
+ ELSE
807
+ EXECUTE format(
808
+ 'ALTER SEQUENCE ${senatSchemaName}.%I OWNED BY ${senatSchemaName}.%I.%I',
809
+ sequence_row.sequencename,
810
+ sequence_owner.table_name,
811
+ sequence_owner.column_name
812
+ );
813
+ END IF;
814
+ END LOOP;
815
+
816
+ FOR constraint_row IN
817
+ SELECT
818
+ table_class.relname AS table_name,
819
+ con.conname,
820
+ con.contype,
821
+ pg_get_constraintdef(con.oid, true) AS constraint_definition
822
+ FROM pg_constraint con
823
+ JOIN pg_class table_class ON table_class.oid = con.conrelid
824
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
825
+ WHERE table_namespace.nspname = '${escapedStagingSchema}'
826
+ AND con.contype <> 'n'
827
+ ORDER BY
828
+ CASE con.contype
829
+ WHEN 'p' THEN 1
830
+ WHEN 'u' THEN 2
831
+ WHEN 'c' THEN 3
832
+ WHEN 'f' THEN 4
833
+ ELSE 5
834
+ END,
835
+ table_class.relname,
836
+ con.conname
837
+ LOOP
838
+ EXECUTE format(
839
+ 'ALTER TABLE ${senatSchemaName}.%I ADD CONSTRAINT %I %s',
840
+ constraint_row.table_name,
841
+ constraint_row.conname,
842
+ replace(
843
+ replace(
844
+ constraint_row.constraint_definition,
845
+ ' REFERENCES ' || '${stagingSchema}' || '.',
846
+ ' REFERENCES ${senatSchemaName}.'
847
+ ),
848
+ ' REFERENCES ONLY ' || '${stagingSchema}' || '.',
849
+ ' REFERENCES ONLY ${senatSchemaName}.'
850
+ )
851
+ );
852
+ END LOOP;
853
+
854
+ FOR index_row IN
855
+ SELECT
856
+ table_class.relname AS table_name,
857
+ pg_get_indexdef(index_data.indexrelid) AS index_definition
858
+ FROM pg_index index_data
859
+ JOIN pg_class table_class ON table_class.oid = index_data.indrelid
860
+ JOIN pg_namespace table_namespace ON table_namespace.oid = table_class.relnamespace
861
+ LEFT JOIN pg_constraint con ON con.conindid = index_data.indexrelid
862
+ WHERE table_namespace.nspname = '${escapedStagingSchema}'
863
+ AND con.oid IS NULL
864
+ ORDER BY table_class.relname, pg_get_indexdef(index_data.indexrelid)
865
+ LOOP
866
+ EXECUTE replace(
867
+ index_row.index_definition,
868
+ ' ON ${stagingSchema}.',
869
+ ' ON ${senatSchemaName}.'
870
+ );
871
+ END LOOP;
872
+
873
+ FOR sequence_row IN
874
+ SELECT sequencename
875
+ FROM pg_sequences
876
+ WHERE schemaname = '${escapedStagingSchema}'
877
+ ORDER BY sequencename
878
+ LOOP
879
+ EXECUTE format(
880
+ 'SELECT last_value, is_called FROM %I.%I',
881
+ '${stagingSchema}',
882
+ sequence_row.sequencename
883
+ )
884
+ INTO sequence_state;
885
+
886
+ EXECUTE format(
887
+ 'SELECT setval(%L, %s, %L)',
888
+ '${senatSchemaName}.' || sequence_row.sequencename,
889
+ sequence_state.last_value,
890
+ sequence_state.is_called
891
+ );
892
+ END LOOP;
893
+
894
+ FOR sequence_row IN
895
+ SELECT c.relname AS sequencename
896
+ FROM pg_class c
897
+ JOIN pg_namespace n ON n.oid = c.relnamespace
898
+ WHERE n.nspname = '${senatSchemaName}'
899
+ AND c.relkind = 'S'
900
+ AND c.relname LIKE '${escapedPrefix}' || '%'
901
+ ORDER BY c.relname
902
+ LOOP
903
+ IF NOT EXISTS (
904
+ SELECT 1
905
+ FROM pg_sequences
906
+ WHERE schemaname = '${escapedStagingSchema}'
907
+ AND sequencename = sequence_row.sequencename
908
+ ) THEN
909
+ EXECUTE format('DROP SEQUENCE IF EXISTS ${senatSchemaName}.%I CASCADE', sequence_row.sequencename);
910
+ END IF;
911
+ END LOOP;
912
+ END $$;
913
+
914
+ GRANT SELECT ON ALL TABLES IN SCHEMA ${senatSchemaName} TO ${dbUser};
908
915
  `;
909
916
  }