@rebasepro/server-postgres 0.17.3 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/LICENSE +0 -1
  2. package/README.md +31 -0
  3. package/dist/backup-argv.d.ts +19 -0
  4. package/dist/branch-argv.d.ts +28 -0
  5. package/dist/branch-prune.d.ts +77 -0
  6. package/dist/cli-collections-path.d.ts +31 -0
  7. package/dist/cli-errors.d.ts +123 -0
  8. package/dist/cli-flags.d.ts +101 -0
  9. package/dist/cli-helpers.d.ts +94 -10
  10. package/dist/collection-index-C7LopMyX.js +247 -0
  11. package/dist/collection-index-C7LopMyX.js.map +1 -0
  12. package/dist/databasePoolManager.d.ts +27 -0
  13. package/dist/{ensure-collection-policies-UCqgv_8c.js → ensure-collection-policies-CagOzEPS.js} +6 -6
  14. package/dist/{ensure-collection-policies-UCqgv_8c.js.map → ensure-collection-policies-CagOzEPS.js.map} +1 -1
  15. package/dist/{ensure-collection-tables-DgVixhX3.js → ensure-collection-tables-DbAhrjnG.js} +614 -52
  16. package/dist/ensure-collection-tables-DbAhrjnG.js.map +1 -0
  17. package/dist/index.d.ts +12 -0
  18. package/dist/index.es.js +19307 -3059
  19. package/dist/index.es.js.map +1 -1
  20. package/dist/{rls-enforcement-DvAbL9YJ.js → rls-enforcement-ClJoTboT.js} +53 -189
  21. package/dist/rls-enforcement-ClJoTboT.js.map +1 -0
  22. package/dist/schema/column-type-drift.d.ts +79 -0
  23. package/dist/schema/doctor.d.ts +42 -0
  24. package/dist/schema/ensure-collection-tables.d.ts +90 -0
  25. package/dist/schema/generate-next-step.d.ts +30 -0
  26. package/dist/schema/generate-postgres-ddl-logic.d.ts +28 -0
  27. package/dist/schema/generated-schema-staleness.d.ts +37 -0
  28. package/dist/schema/introspect-db-queries.d.ts +34 -0
  29. package/dist/security/rls-enforcement.d.ts +63 -2
  30. package/dist/services/BranchService.d.ts +86 -1
  31. package/dist/services/collection-helpers.d.ts +1 -1
  32. package/dist/services/realtimeService.d.ts +30 -2
  33. package/dist/services/row-pipeline.d.ts +12 -1
  34. package/dist/utils/pg-error-utils.d.ts +3 -1
  35. package/dist/utils/pg-numeric-number-patch.d.ts +31 -0
  36. package/package.json +32 -27
  37. package/dist/backup-service-DCk7KhhL.js +0 -9044
  38. package/dist/backup-service-DCk7KhhL.js.map +0 -1
  39. package/dist/collection-index-BRUg10H5.js +0 -5690
  40. package/dist/collection-index-BRUg10H5.js.map +0 -1
  41. package/dist/connection-GOKU3Hu5.js +0 -277
  42. package/dist/connection-GOKU3Hu5.js.map +0 -1
  43. package/dist/ensure-collection-tables-DgVixhX3.js.map +0 -1
  44. package/dist/rls-bootstrap-sql-B5C9LoJ6.js +0 -244
  45. package/dist/rls-bootstrap-sql-B5C9LoJ6.js.map +0 -1
  46. package/dist/rls-enforcement-DvAbL9YJ.js.map +0 -1
  47. package/dist/src-DiB5RP2Z.js +0 -541
  48. package/dist/src-DiB5RP2Z.js.map +0 -1
  49. package/dist/websocket-BZ4H5wUz.js +0 -8198
  50. package/dist/websocket-BZ4H5wUz.js.map +0 -1
  51. package/src/PostgresAdapter.ts +0 -91
  52. package/src/PostgresBackendDriver.ts +0 -1857
  53. package/src/PostgresBootstrapper.ts +0 -1157
  54. package/src/auth/ensure-tables.ts +0 -896
  55. package/src/auth/schema-version.ts +0 -260
  56. package/src/auth/services.ts +0 -1610
  57. package/src/backup/backup-cli.ts +0 -520
  58. package/src/backup/backup-cron.ts +0 -212
  59. package/src/backup/backup-logic.ts +0 -93
  60. package/src/backup/backup-service.ts +0 -485
  61. package/src/backup/index.ts +0 -12
  62. package/src/backup/pg-tools.ts +0 -403
  63. package/src/backup/retention.ts +0 -86
  64. package/src/cli-errors.ts +0 -265
  65. package/src/cli-helpers.ts +0 -569
  66. package/src/cli-output.ts +0 -43
  67. package/src/cli.ts +0 -1339
  68. package/src/collections/PostgresCollectionRegistry.ts +0 -108
  69. package/src/collections/buildRegistry.ts +0 -68
  70. package/src/collections/validate-relations.ts +0 -402
  71. package/src/connection.ts +0 -333
  72. package/src/data-transformer.ts +0 -845
  73. package/src/databasePoolManager.ts +0 -95
  74. package/src/history/HistoryService.ts +0 -254
  75. package/src/history/ensure-history-table.ts +0 -53
  76. package/src/index.ts +0 -19
  77. package/src/interfaces.ts +0 -60
  78. package/src/module-dir.ts +0 -7
  79. package/src/schema/atlas-argv.ts +0 -94
  80. package/src/schema/auth-schema.ts +0 -298
  81. package/src/schema/auth-users-columns.ts +0 -131
  82. package/src/schema/carved-out-migration.ts +0 -404
  83. package/src/schema/classify-change.ts +0 -436
  84. package/src/schema/collection-index.ts +0 -427
  85. package/src/schema/collections-schema-version.ts +0 -103
  86. package/src/schema/destructive-sql.ts +0 -94
  87. package/src/schema/doctor-cli.ts +0 -66
  88. package/src/schema/doctor-policy-checks.ts +0 -105
  89. package/src/schema/doctor.ts +0 -837
  90. package/src/schema/drizzle-ddl.ts +0 -33
  91. package/src/schema/dynamic-tables.test.ts +0 -302
  92. package/src/schema/dynamic-tables.ts +0 -293
  93. package/src/schema/ensure-collection-policies.ts +0 -198
  94. package/src/schema/ensure-collection-tables.test.ts +0 -518
  95. package/src/schema/ensure-collection-tables.ts +0 -1267
  96. package/src/schema/generate-drizzle-schema-logic.ts +0 -957
  97. package/src/schema/generate-drizzle-schema.ts +0 -143
  98. package/src/schema/generate-postgres-ddl-logic.ts +0 -1339
  99. package/src/schema/generate-postgres-ddl.ts +0 -180
  100. package/src/schema/generate-schema-commit.ts +0 -267
  101. package/src/schema/generated-schema-staleness.ts +0 -284
  102. package/src/schema/introspect-db-constraints.ts +0 -385
  103. package/src/schema/introspect-db-inference.ts +0 -255
  104. package/src/schema/introspect-db-logic.ts +0 -1387
  105. package/src/schema/introspect-db-naming.ts +0 -15
  106. package/src/schema/introspect-db-project.ts +0 -78
  107. package/src/schema/introspect-db-queries.ts +0 -326
  108. package/src/schema/introspect-db-structure.ts +0 -670
  109. package/src/schema/introspect-db-types.ts +0 -56
  110. package/src/schema/introspect-db.ts +0 -257
  111. package/src/schema/introspect-runtime.test.ts +0 -260
  112. package/src/schema/introspect-runtime.ts +0 -327
  113. package/src/schema/non-sql-collections.test.ts +0 -131
  114. package/src/schema/rls-bootstrap-sql.ts +0 -288
  115. package/src/schema/search-column.ts +0 -643
  116. package/src/schema/test-schema.ts +0 -11
  117. package/src/schema/vector-index.ts +0 -491
  118. package/src/security/anonymous-grants.test.ts +0 -73
  119. package/src/security/policy-drift.test.ts +0 -435
  120. package/src/security/policy-drift.ts +0 -473
  121. package/src/security/rls-enforcement.ts +0 -505
  122. package/src/services/BranchService.ts +0 -326
  123. package/src/services/FetchService.ts +0 -2115
  124. package/src/services/PersistService.ts +0 -557
  125. package/src/services/RelationService.ts +0 -978
  126. package/src/services/RelationWriteService.ts +0 -653
  127. package/src/services/cdc/CdcListener.ts +0 -103
  128. package/src/services/cdc/junction-tables.ts +0 -91
  129. package/src/services/cdc/trigger-cdc.ts +0 -173
  130. package/src/services/channel-bus/ChannelBus.ts +0 -44
  131. package/src/services/channel-bus/PostgresChannelBus.ts +0 -299
  132. package/src/services/channel-bus/index.ts +0 -114
  133. package/src/services/channel-history.ts +0 -424
  134. package/src/services/channel-presence.ts +0 -185
  135. package/src/services/collection-helpers.ts +0 -420
  136. package/src/services/dataService.ts +0 -256
  137. package/src/services/index.ts +0 -14
  138. package/src/services/junction-writes.ts +0 -295
  139. package/src/services/nested-path.ts +0 -145
  140. package/src/services/pg-notify-listener.ts +0 -151
  141. package/src/services/realtimeService.ts +0 -2545
  142. package/src/services/row-pipeline.ts +0 -240
  143. package/src/services/write-denial.ts +0 -55
  144. package/src/types.ts +0 -4
  145. package/src/utils/connection-string.ts +0 -58
  146. package/src/utils/drizzle-conditions.ts +0 -2743
  147. package/src/utils/pg-array-null-patch.ts +0 -42
  148. package/src/utils/pg-error-utils.ts +0 -391
  149. package/src/websocket.ts +0 -855
@@ -1,1267 +0,0 @@
1
- /**
2
- * Bringing a database up to date with a bundle's collections, additively.
3
- *
4
- * ## Why this exists
5
- *
6
- * A managed runtime boots someone else's compiled project against a database it
7
- * has never seen. Auth tables are ensured at boot already, but collection tables
8
- * were not created by anything: the platform ran the app and every `/api/data/*`
9
- * request answered 500 on a missing relation. `rebase db push` cannot help — it
10
- * is an Atlas-driven CLI command, and the runtime image ships no CLI.
11
- *
12
- * ## Why additive-only, forever
13
- *
14
- * This runs unattended, against a database with customers' data in it, with no
15
- * human reading a diff. So it may only ever do things that cannot lose data:
16
- * create a missing table, add a missing column, create a missing enum type.
17
- *
18
- * It will **never** drop a table or a column, narrow a type, or alter a
19
- * constraint. A removed field leaves its column behind; a renamed field looks
20
- * like an addition and the old column stays. That is the correct trade for an
21
- * automated path — the alternative is an unattended process that can silently
22
- * destroy a column, which is precisely the failure `db push` was hardened
23
- * against. Destructive changes stay a deliberate, human-reviewed migration.
24
- *
25
- * Because of that, this is safe to run on every boot, and re-running it is a
26
- * no-op.
27
- */
28
- import {
29
- declaredDatabaseExtensions,
30
- isPostgresCollectionConfig,
31
- type CollectionConfig,
32
- type Property
33
- } from "@rebasepro/types";
34
- import { getTableName, relationalCollections } from "@rebasepro/common";
35
- import { logger, isConcurrentDdlRace, isDuplicateObjectRace } from "@rebasepro/server";
36
- import {
37
- assertSearchIsPostgresOnly,
38
- buildSearchColumnSpec,
39
- searchExtensionStatements,
40
- searchHelperFunctions,
41
- searchIndexStatements,
42
- searchColumnStamps,
43
- SEARCH_STAMP_PREFIX,
44
- SEARCH_TEXT_FN,
45
- SEARCH_UNACCENT_FN,
46
- type SearchColumnSpec
47
- } from "./search-column";
48
- import {
49
- getSqlColumnType,
50
- resolveColumnName,
51
- isIdProperty,
52
- planRelationalColumns,
53
- planJunctionTables,
54
- quoteSqlLiteral
55
- } from "./generate-postgres-ddl-logic";
56
- import {
57
- buildVectorColumnSpecs,
58
- buildVectorIndexPlan,
59
- vectorExtensionDeclared,
60
- vectorExtensionHint,
61
- vectorExtensionStatement,
62
- vectorIndexStatement,
63
- type SkippedVectorIndex
64
- } from "./vector-index";
65
- import { buildCollectionIndexPlan, collectionIndexStatement } from "./collection-index";
66
- import {
67
- AUTH_USERS_COLUMNS,
68
- authUsersColumnDefinition,
69
- authUsersColumnSql,
70
- isAuthCollection
71
- } from "./auth-users-columns";
72
-
73
- /**
74
- * The subset of a database handle this needs: run a statement, get rows back.
75
- *
76
- * Deliberately parameterless. Everything here is DDL or catalogue reads keyed by
77
- * schema name, and schema names are identifiers — they cannot be bound as
78
- * parameters anyway. They are validated against {@link SAFE_IDENTIFIER} before
79
- * they reach a statement, so a config that somehow carried a quote is refused
80
- * rather than concatenated.
81
- */
82
- export interface Queryable {
83
- query<T = unknown>(sql: string): Promise<{ rows: T[] }>;
84
- }
85
-
86
- /** Postgres identifiers this module is willing to interpolate. */
87
- const SAFE_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_$]*$/;
88
-
89
- function assertSafeIdentifier(value: string, what: string): string {
90
- if (!SAFE_IDENTIFIER.test(value)) {
91
- throw new Error(`Refusing to build SQL with an unsafe ${what}: ${JSON.stringify(value)}`);
92
- }
93
- return value;
94
- }
95
-
96
- /** What the database currently has, as the planner needs it. */
97
- export interface ExistingSchema {
98
- /** `schema.table` → set of column names. */
99
- tables: Map<string, Set<string>>;
100
- /** `schema.typename` of every enum type that already exists. */
101
- enums: Set<string>;
102
- /**
103
- * `schema.table.constraint` of every constraint that already exists.
104
- *
105
- * Optional so a caller that only cares about tables can still build one by
106
- * hand; absent is read as "none known", which at worst re-attempts a
107
- * constraint that then fails harmlessly as a duplicate.
108
- */
109
- constraints?: Set<string>;
110
- /**
111
- * `schema.table.column` → that column's comment, for the columns that have
112
- * one. This is where a generated search column's fingerprint lives, so it
113
- * is the only evidence that a `search` block has changed since the column
114
- * was built. Absent is read as "no column is stamped", which plans a stamp
115
- * and reports nothing as drifted.
116
- */
117
- columnComments?: Map<string, string>;
118
- /**
119
- * `schema.typename` → the values that type currently holds, in order.
120
- *
121
- * Without this, an enum type that already exists is skipped whole and a
122
- * value added to it never reaches the database — the type is there, so
123
- * nothing plans anything, and the first row using the new value is rejected
124
- * by a constraint nobody changed. Absent is read as "the values are
125
- * unknown", which keeps the old skip-by-name behaviour rather than guessing.
126
- */
127
- enumValues?: Map<string, string[]>;
128
- /** `schema.table.column` for every column the database marks NOT NULL. */
129
- notNullColumns?: Set<string>;
130
- /**
131
- * Tables known to hold at least one row.
132
- *
133
- * The only thing that decides whether a NOT NULL can be added without
134
- * reading the data: on an empty table the constraint cannot fail, on a
135
- * populated one it is checked against every existing row. Absent is read as
136
- * "assume populated", which is the conservative direction — it withholds a
137
- * constraint rather than attempting one that aborts the boot.
138
- */
139
- populatedTables?: Set<string>;
140
- }
141
-
142
- /**
143
- * How far the planner may go in making the database's constraints match the
144
- * configuration.
145
- *
146
- * - `additive` — the boot default. Columns, tables, indexes and enum values are
147
- * created; no existing column's constraints are touched. Unattended boots run
148
- * against customer data with nobody reading a diff, and a database adopted by
149
- * introspection legitimately carries NOT NULL on columns the generated
150
- * collection leaves optional (`introspect-db-logic` withholds `required` from
151
- * a column with a default or a trigger behind it). Converging there would
152
- * strip real constraints on first boot.
153
- * - `converge` — the live schema editor. Every statement is planned, shown to
154
- * the person making the change, and applied only once they confirm it. That
155
- * is the context in which changing an existing column's constraints is a
156
- * reviewed act rather than a surprise.
157
- */
158
- export type ConstraintPolicy = "additive" | "converge";
159
-
160
- export interface EnsureOptions {
161
- /** Defaults to `additive`. See {@link ConstraintPolicy}. */
162
- constraints?: ConstraintPolicy;
163
- /**
164
- * Server extensions the project's databases gave Rebase leave to install —
165
- * `declaredDatabaseExtensions()`. Absent means none, which is a refusal and
166
- * is the right default for a planner given no configuration at all.
167
- *
168
- * Explicit rather than read from the resource registry in here, because
169
- * this function is pure and several callers plan against fixtures: reaching
170
- * for a process-wide registry would make the plan depend on whatever some
171
- * other module happened to import. `ensureCollectionTables` is the boundary
172
- * that reads the world.
173
- */
174
- databaseExtensions?: readonly string[];
175
- }
176
-
177
- export interface EnsureAction {
178
- kind: "create-enum" | "create-table" | "add-column" | "add-constraint" | "rename-column"
179
- | "create-extension" | "create-function" | "create-index" | "comment-column"
180
- | "add-enum-value" | "set-not-null" | "drop-not-null";
181
- /** Qualified target, for logging: `public.posts` or `public.posts.title`. */
182
- target: string;
183
- sql: string;
184
- }
185
-
186
- export interface EnsurePlan {
187
- actions: EnsureAction[];
188
- /** Every statement, in dependency order. Empty when the schema is current. */
189
- statements: string[];
190
- /**
191
- * Relation columns this plan is about to create where the table already
192
- * carries the same column under its pre-singularization name.
193
- *
194
- * The reason this is reported rather than silently handled: the ensure is
195
- * additive, so it would add `category_id` beside a populated
196
- * `categorie_id` and the relation would then read the new, empty one. No
197
- * statement fails, no table is missing, and the only symptom is relations
198
- * resolving to nothing — which is indistinguishable from having no data.
199
- */
200
- legacyForeignKeys: LegacyForeignKey[];
201
- /**
202
- * Generated search columns whose `search` block has changed since they were
203
- * built. Reported, never planned into `actions` — see
204
- * {@link SearchColumnDrift} for why applying it is not this path's call.
205
- */
206
- searchDrift: SearchColumnDrift[];
207
- /**
208
- * Generated search columns that exist but carry no fingerprint — created
209
- * before this check existed, or by `search.sql` on an older CLI. The plan
210
- * stamps them so the *next* change is detectable; whether they match the
211
- * current block cannot be known, which is what the caller reports.
212
- */
213
- searchAdopted: { table: string; column: string }[];
214
- /**
215
- * Vector columns this plan is deliberately leaving unindexed, because
216
- * pgvector cannot build an ANN index that wide.
217
- *
218
- * Reported rather than thrown: the column is valid, storable and
219
- * searchable, and refusing the boot over it would make a working
220
- * configuration unbootable. Reported rather than dropped: an unindexed
221
- * vector column and an indexed one differ only in latency, so nothing
222
- * about the running system says which one you got.
223
- */
224
- vectorIndexSkipped: SkippedVectorIndex[];
225
- /**
226
- * Constraints the configuration asks for that this plan is not applying,
227
- * and why.
228
- *
229
- * This is the half of the feature that matters most. Every one of these was
230
- * previously withheld in silence: a required property arrived nullable, and
231
- * the only evidence was a database that disagreed with its own
232
- * configuration. Reporting them is what lets boot warn, the live editor
233
- * refuse, and the doctor explain — three surfaces that until now had nothing
234
- * to read.
235
- */
236
- withheldConstraints: WithheldConstraint[];
237
- }
238
-
239
- /** A constraint the configuration asks for that the planner is not applying. */
240
- export interface WithheldConstraint {
241
- /** `schema.table.column`. */
242
- target: string;
243
- kind: "not-null";
244
- /**
245
- * Why, in a sentence that names the obstacle rather than the rule. The
246
- * reader is looking at a column that is nullable when they asked for
247
- * required, and needs to know what to do about it.
248
- */
249
- reason: string;
250
- /** What would make it applicable. */
251
- remedy: string;
252
- }
253
-
254
- /**
255
- * A generated search column built from a `search` block that has since changed.
256
- *
257
- * Reported instead of applied because the two ways to apply it are both worse
258
- * than stopping. `ALTER COLUMN … SET EXPRESSION` exists only on PG17+ and
259
- * rewrites the table either way; `DROP COLUMN` + `ADD COLUMN` rewrites it under
260
- * an ACCESS EXCLUSIVE lock and rebuilds the GIN index. This module runs
261
- * unattended against live customer data with nobody reading a diff — the same
262
- * reason it withholds `SET NOT NULL` from an adopted table — so a multi-minute
263
- * outage is not a decision it may take on its own.
264
- *
265
- * Not applying it silently is not an option either: that is the bug this
266
- * detection exists for. A collection that added a field, flipped `unaccent` or
267
- * raised a weight kept indexing the *old* set forever, and the only symptom was
268
- * searches returning nothing for content plainly in the row.
269
- */
270
- export interface SearchColumnDrift {
271
- /** `schema.table`. */
272
- table: string;
273
- column: string;
274
- /** The fingerprint recorded on the column. */
275
- found: string;
276
- /** The fingerprint the current `search` block computes. */
277
- expected: string;
278
- /** The statements that would rebuild the column, for the operator to run. */
279
- rebuild: string[];
280
- }
281
-
282
- /** A relation column whose old and new spellings both plausibly apply. */
283
- export interface LegacyForeignKey {
284
- /** `schema.table`. */
285
- table: string;
286
- /** The name the current rule derives, and what this plan would create. */
287
- expected: string;
288
- /** The name the old rule derived, which the table already has. */
289
- legacy: string;
290
- }
291
-
292
- export interface EnsureOutcome extends EnsurePlan {
293
- /**
294
- * Actions that could not be applied and are non-fatal by nature.
295
- *
296
- * Two kinds qualify. A foreign key can only fail on data that already
297
- * violates it, and the column it would police exists either way, so the
298
- * collection still serves; refusing to boot over one would turn a
299
- * pre-existing data problem into an outage. A column comment is the search
300
- * fingerprint, which needs table ownership — losing it costs drift
301
- * detection on the next boot, not the deployment. Both are reported loudly.
302
- */
303
- failures: { kind: EnsureAction["kind"]; target: string; error: string }[];
304
- }
305
-
306
- function schemaOf(collection: CollectionConfig): string {
307
- return isPostgresCollectionConfig(collection) && collection.schema ? collection.schema : "public";
308
- }
309
-
310
- function qualified(collection: CollectionConfig): string {
311
- return `${schemaOf(collection)}.${getTableName(collection)}`;
312
- }
313
-
314
- /**
315
- * Enum types a collection's properties require, as `schema.typename`.
316
- *
317
- * Named exactly as the DDL generator names them (`<table>_<column>`), because
318
- * a column added here has to reference the same type the generator would have
319
- * created — a second, differently-named type for the same field would be a
320
- * silent schema fork.
321
- */
322
- function requiredEnums(collection: CollectionConfig): { name: string; values: string[] }[] {
323
- const table = getTableName(collection);
324
- const schema = schemaOf(collection);
325
- const out: { name: string; values: string[] }[] = [];
326
- for (const [propName, prop] of Object.entries(collection.properties ?? {})) {
327
- const p = prop as Property;
328
- if (!("enum" in p) || !p.enum) continue;
329
- if (p.type !== "string" && p.type !== "number") continue;
330
- const values = (p.enum as unknown[])
331
- .map(entry =>
332
- entry && typeof entry === "object" && "id" in (entry as Record<string, unknown>)
333
- ? String((entry as Record<string, unknown>).id)
334
- : String(entry)
335
- )
336
- .filter(v => v.length > 0);
337
- if (values.length === 0) continue;
338
- out.push({ name: `${schema}.${table}_${resolveColumnName(propName, p)}`, values });
339
- }
340
- return out;
341
- }
342
-
343
- /**
344
- * Decide what to add. Pure — the caller supplies what exists and runs the result.
345
- *
346
- * Ordering matters and is deliberate: enum types before the tables and columns
347
- * that reference them, tables before the columns added to other tables (a new
348
- * table may be the target of a relation), and nothing is emitted twice.
349
- */
350
- export function planCollectionSchemaEnsure(
351
- allCollections: CollectionConfig[],
352
- existing: ExistingSchema,
353
- options: EnsureOptions = {}
354
- ): EnsurePlan {
355
- const constraintPolicy: ConstraintPolicy = options.constraints ?? "additive";
356
- const withheldConstraints: WithheldConstraint[] = [];
357
- // Boot receives every collection the bundle declares, including the ones
358
- // served by another engine entirely. Creating a Postgres table for a
359
- // Firestore collection is not a harmless extra: the app keeps reading
360
- // documents from Firestore while an empty table with the same name accretes
361
- // policies and shows up in every drift report.
362
- // Before the filter, deliberately: a `search` block on a collection this
363
- // engine does not store would otherwise be dropped here without a word.
364
- assertSearchIsPostgresOnly(allCollections);
365
-
366
- const collections = relationalCollections(allCollections);
367
- const actions: EnsureAction[] = [];
368
- const plannedEnums = new Set<string>();
369
-
370
- // 1. Enum types. `CREATE TYPE` has no IF NOT EXISTS, so an existing type is
371
- // skipped by name rather than guarded in SQL.
372
- for (const collection of collections) {
373
- for (const { name, values } of requiredEnums(collection)) {
374
- if (existing.enums.has(name) || plannedEnums.has(name)) {
375
- // The type is there, but that says nothing about its *values*.
376
- // Skipping the whole type by name is what made an added enum
377
- // value vanish: nothing was planned, the boot reported success,
378
- // and the first row using the value was rejected by a type that
379
- // had never heard of it. `ADD VALUE` is the one alteration
380
- // Postgres offers here, it is purely additive, and it is
381
- // idempotent with `IF NOT EXISTS`.
382
- //
383
- // `enumValues` absent means the caller built the schema by hand
384
- // and does not know the values; skip by name as before rather
385
- // than plan against a guess.
386
- const current = existing.enumValues?.get(name);
387
- if (!current || plannedEnums.has(name)) continue;
388
- const [schema, typeName] = name.split(".");
389
- for (const value of values) {
390
- if (current.includes(value)) continue;
391
- actions.push({
392
- kind: "add-enum-value",
393
- target: `${name}.${value}`,
394
- // Not inside a transaction with any use of the value:
395
- // Postgres refuses to read a value added by the
396
- // transaction still adding it. The applier runs these
397
- // one statement at a time, which is what makes it legal.
398
- sql: `ALTER TYPE "${schema}"."${typeName}" ADD VALUE IF NOT EXISTS ${quoteSqlLiteral(value)};`
399
- });
400
- }
401
- continue;
402
- }
403
- plannedEnums.add(name);
404
- const [schema, typeName] = name.split(".");
405
- actions.push({
406
- kind: "create-enum",
407
- target: name,
408
- sql: `CREATE TYPE "${schema}"."${typeName}" AS ENUM (${values.map(quoteSqlLiteral).join(", ")});`
409
- });
410
- }
411
- }
412
-
413
- // 1b. Search support, for collections that declared a `search` block.
414
- //
415
- // Before the tables, because a generated column's expression is
416
- // resolved when the column is created: a table whose search column
417
- // calls `rebase_search_text` cannot be added before that function
418
- // exists. Both forms are idempotent, so a boot against a database that
419
- // already has them plans nothing.
420
- const searchSpecs = collections
421
- .map(c => buildSearchColumnSpec(c))
422
- .filter((spec): spec is SearchColumnSpec => spec !== undefined);
423
-
424
- const plannedExtensions = new Set<string>();
425
- const planExtension = (statement: string): void => {
426
- if (plannedExtensions.has(statement)) return;
427
- plannedExtensions.add(statement);
428
- actions.push({ kind: "create-extension", target: statement.replace(/^CREATE EXTENSION IF NOT EXISTS |;$/g, ""), sql: statement });
429
- };
430
- for (const spec of searchSpecs) {
431
- for (const statement of searchExtensionStatements(spec)) planExtension(statement);
432
- }
433
-
434
- // pgvector, for collections that declared a `vector` property — and only
435
- // when a database gave leave to install it. Unlike the search extensions
436
- // above, which are contrib modules every Postgres carries, pgvector is a
437
- // separate build behind an image, a grant and a provider allow-list, so
438
- // whether Rebase may install it is not Rebase's to decide. See
439
- // `DatabaseOptions.extensions`.
440
- //
441
- // Boot has to make the same call `rebase db push` makes, or the two produce
442
- // different databases from one commit — which is the rule
443
- // `contracts/derived-names.txt` enforces.
444
- if (vectorExtensionDeclared(options.databaseExtensions)
445
- && collections.some(c => buildVectorColumnSpecs(c, resolveColumnName).length > 0)) {
446
- planExtension(vectorExtensionStatement());
447
- }
448
- const plannedFunctions = new Set<string>();
449
- for (const spec of searchSpecs) {
450
- for (const statement of searchHelperFunctions(spec)) {
451
- if (plannedFunctions.has(statement)) continue;
452
- plannedFunctions.add(statement);
453
- actions.push({ kind: "create-function", target: statement.includes("unaccent") ? SEARCH_UNACCENT_FN : SEARCH_TEXT_FN, sql: statement });
454
- }
455
- }
456
-
457
- // 2. Missing tables. Only the identity column is created here; every other
458
- // column is added by step 3, so a new table and an existing table that
459
- // gained a field travel the exact same code path. One way to build a
460
- // column means one way for it to be wrong.
461
- const created = new Set<string>();
462
- for (const collection of collections) {
463
- const key = qualified(collection);
464
- if (existing.tables.has(key) || created.has(key)) continue;
465
- created.add(key);
466
- const schema = schemaOf(collection);
467
- const table = getTableName(collection);
468
- const idEntry = Object.entries(collection.properties ?? {}).find(([n, p]) =>
469
- isIdProperty(n, p as Property, collection)
470
- );
471
- const idName = idEntry ? resolveColumnName(idEntry[0], idEntry[1] as Property) : "id";
472
- const idProp = idEntry?.[1] as Property | undefined;
473
- // Derived through the generator's own type mapping, not re-decided here.
474
- // This branch used to emit BIGSERIAL for a numeric id while `db push`
475
- // emitted INTEGER GENERATED BY DEFAULT AS IDENTITY, so the same project
476
- // got an int8 key when the runtime brought the schema up and an int4 key
477
- // when a human pushed it. Two consequences, both real: node-postgres
478
- // hands back int8 as a *string*, so a collection declaring
479
- // `type: "number"` served `"1"` instead of `1` on the managed path only;
480
- // and every foreign key and junction column pointing at it stayed
481
- // INTEGER on both paths, which is a truncation waiting for the sequence
482
- // to pass 2^31. The agreement test now pins this.
483
- const idType = idProp
484
- ? getSqlColumnType(idEntry![0], idProp, collection, collections)
485
- : "TEXT";
486
- let idDef = `"${idName}" ${idType} PRIMARY KEY`;
487
- if (idProp?.type === "string" && (idProp as { isId?: unknown }).isId === "uuid") {
488
- idDef += " DEFAULT gen_random_uuid()";
489
- }
490
- actions.push({
491
- kind: "create-table",
492
- target: key,
493
- sql: `CREATE TABLE IF NOT EXISTS "${schema}"."${table}" (${idDef});`
494
- });
495
- }
496
-
497
- // 2b. Junction tables behind many-to-many relations. No collection declares
498
- // them, so the walk above never sees them — and until they existed, an
499
- // m2m write had nowhere to land and the junction's derived RLS had
500
- // nothing to attach to.
501
- const junctions = planJunctionTables(collections);
502
- for (const junction of junctions) {
503
- const key = `${junction.schema}.${junction.table}`;
504
- if (existing.tables.has(key) || created.has(key)) continue;
505
- created.add(key);
506
- actions.push({ kind: "create-table", target: key, sql: junction.createTable });
507
- }
508
-
509
- // 3. Missing columns, on both brand-new and pre-existing tables.
510
- const legacyForeignKeys: LegacyForeignKey[] = [];
511
-
512
- /**
513
- * Move a relation column that is only missing because it was renamed.
514
- *
515
- * Returns true when it handled the column, so the caller skips the ordinary
516
- * ADD. Adding here would be the wrong move and a quiet one: the data is in
517
- * the old column, `ADD COLUMN` creates the new one empty beside it, every
518
- * statement succeeds, and the relation reads the empty one. A rename is
519
- * metadata-only in Postgres, keeps the values, and carries the column's
520
- * indexes and constraints with it.
521
- *
522
- * Only ever reached when the new name is absent and the old name is
523
- * present, so there is nothing to overwrite and nothing to choose between.
524
- */
525
- const renameLegacyColumn = (
526
- key: string,
527
- schema: string,
528
- table: string,
529
- column: string,
530
- legacyName: string | undefined
531
- ): boolean => {
532
- const present = existing.tables.get(key);
533
- if (!legacyName || !present) return false;
534
- if (present.has(column) || !present.has(legacyName)) return false;
535
-
536
- legacyForeignKeys.push({ table: key, expected: column, legacy: legacyName });
537
- actions.push({
538
- kind: "rename-column",
539
- target: `${key}.${column}`,
540
- sql: `ALTER TABLE "${schema}"."${table}" RENAME COLUMN "${legacyName}" TO "${column}";`
541
- });
542
- return true;
543
- };
544
-
545
- const addColumn = (
546
- key: string,
547
- schema: string,
548
- table: string,
549
- column: string,
550
- definition: string
551
- ): void => {
552
- const present = existing.tables.get(key);
553
- if (present?.has(column)) return;
554
- actions.push({
555
- kind: "add-column",
556
- target: `${key}.${column}`,
557
- sql: `ALTER TABLE "${schema}"."${table}" ADD COLUMN IF NOT EXISTS "${column}" ${definition};`
558
- });
559
- };
560
-
561
- for (const collection of collections) {
562
- const key = qualified(collection);
563
- const schema = schemaOf(collection);
564
- const table = getTableName(collection);
565
- // A table this run is creating has no rows yet, so the constraints
566
- // `db push` writes are free to apply. On a table that already exists
567
- // they are not: `SET NOT NULL` is checked against live rows and a UNIQUE
568
- // would fail on existing duplicates, and this module runs unattended
569
- // against customer data with nobody reading a diff. So the constraints
570
- // are emitted for the fresh case — which is the whole managed-runtime
571
- // path, and the one that diverged from `db push` — and withheld for the
572
- // adopted one. `rebase db push` remains how an existing table gets them.
573
- const fresh = created.has(key);
574
- const auth = isAuthCollection(collection);
575
- for (const [propName, prop] of Object.entries(collection.properties ?? {})) {
576
- const p = prop as Property;
577
- if (isIdProperty(propName, p, collection)) continue;
578
- // Relation and reference columns are planned from the shared
579
- // relational planner below, which derives the column name, type and
580
- // foreign key the same way `db push` does. Deriving them here as
581
- // plain columns is what once produced a column with no constraint.
582
- if (p.type === "reference" || p.type === "relation") continue;
583
-
584
- const column = resolveColumnName(propName, p);
585
- // On an auth collection, the columns auth itself reads and writes
586
- // have exactly one definition, wherever the table is created from —
587
- // see `auth-users-columns`. Anything else on that collection is an
588
- // ordinary user-declared field and is generated like any other.
589
- const authDefinition = auth ? authUsersColumnDefinition(column) : undefined;
590
- if (authDefinition) {
591
- addColumn(key, schema, table, column, authDefinition);
592
- continue;
593
- }
594
-
595
- // Assembled in the generator's order — type, UNIQUE, DEFAULT,
596
- // NOT NULL — so the two produce byte-identical column definitions
597
- // and the agreement test can compare them directly instead of
598
- // checking that a column merely exists, which is how BIGSERIAL-vs-
599
- // INTEGER and every missing constraint went unnoticed.
600
- let definition = getSqlColumnType(propName, p, collection, collections);
601
- if (fresh && p.validation?.unique) definition += " UNIQUE";
602
- // Not gated on `fresh`: a default binds future writes only, so it is
603
- // safe on a live table, and a column added without it would take the
604
- // value the application forgot to send rather than `now()`.
605
- const autoValue = (p as { autoValue?: string }).autoValue;
606
- const hasDefault = p.type === "date" && (autoValue === "on_create" || autoValue === "on_update");
607
- if (hasDefault) definition += " DEFAULT now()";
608
-
609
- const required = p.validation?.required === true;
610
- const columnKey = `${key}.${column}`;
611
- const columnExists = existing.tables.get(key)?.has(column) === true;
612
-
613
- // A NOT NULL is safe exactly when it cannot fail against rows that
614
- // are already there, and there are three ways to know that:
615
- //
616
- // - the table is being created by this plan (no rows yet);
617
- // - the table exists and is empty;
618
- // - the column arrives with a DEFAULT, which Postgres backfills
619
- // into every existing row as part of ADD COLUMN.
620
- //
621
- // Anything else is checked against live data and can abort the boot,
622
- // which is why it used to be withheld — correctly. What was wrong was
623
- // withholding it in *silence*: the config said required, the column
624
- // came out nullable, and nothing anywhere said so.
625
- // `populatedTables` absent means the caller does not know, and not
626
- // knowing has to read as "assume rows" — the other direction emits a
627
- // NOT NULL that is checked against live data and aborts the boot.
628
- // Written as an explicit `!== undefined` because the optional-chain
629
- // form (`!existing.populatedTables?.has(key)`) quietly says *empty*
630
- // when the fact is missing, which is the wrong way to be wrong.
631
- const tableIsEmpty = existing.populatedTables !== undefined
632
- && existing.tables.has(key)
633
- && !existing.populatedTables.has(key);
634
- const notNullIsSafe = fresh || tableIsEmpty || hasDefault;
635
-
636
- if (required && !columnExists) {
637
- if (notNullIsSafe) {
638
- definition += " NOT NULL";
639
- } else {
640
- withheldConstraints.push({
641
- target: columnKey,
642
- kind: "not-null",
643
- reason:
644
- `"${column}" is required, but "${key}" already holds rows and the column ` +
645
- "has no default to backfill them with, so NOT NULL would be checked " +
646
- "against data that does not have a value yet.",
647
- remedy:
648
- "Backfill the column, then add the constraint — or give the property a " +
649
- "default so every existing row gets one."
650
- });
651
- }
652
- }
653
- addColumn(key, schema, table, column, definition);
654
-
655
- // The column is already there and only its constraint differs. Two
656
- // directions, and they are not equally safe — see `ConstraintPolicy`
657
- // for why neither runs at an unattended boot.
658
- if (columnExists && constraintPolicy === "converge") {
659
- const isNotNull = existing.notNullColumns?.has(columnKey) === true;
660
- if (required && !isNotNull) {
661
- if (tableIsEmpty) {
662
- actions.push({
663
- kind: "set-not-null",
664
- target: columnKey,
665
- sql: `ALTER TABLE "${schema}"."${table}" ALTER COLUMN "${column}" SET NOT NULL;`
666
- });
667
- } else {
668
- withheldConstraints.push({
669
- target: columnKey,
670
- kind: "not-null",
671
- reason:
672
- `"${column}" became required, but "${key}" holds rows and any of them ` +
673
- "with no value would make SET NOT NULL fail.",
674
- remedy:
675
- "Backfill the column first — `UPDATE … SET \"" + column +
676
- "\" = … WHERE \"" + column + "\" IS NULL` — then apply this again."
677
- });
678
- }
679
- }
680
- if (!required && isNotNull) {
681
- // Loosening never fails and never loses data. It is here
682
- // rather than at boot because a database adopted by
683
- // introspection carries NOT NULL on columns the generated
684
- // collection deliberately leaves optional, and converging
685
- // those unasked would drop constraints nobody edited.
686
- actions.push({
687
- kind: "drop-not-null",
688
- target: columnKey,
689
- sql: `ALTER TABLE "${schema}"."${table}" ALTER COLUMN "${column}" DROP NOT NULL;`
690
- });
691
- }
692
- }
693
- }
694
-
695
- // The auth columns the collection never mentions. The scaffold's users
696
- // collection describes 12 of the 14 auth reads and writes, so planning
697
- // only from properties left `is_anonymous` and `tokens_valid_after` to
698
- // `ensureAuthTablesExist` — which does create them, but only because
699
- // that function happens to run later in the same boot. Planning them
700
- // here makes this path self-contained and identical to `db push`, so
701
- // neither depends on the other having run.
702
- if (auth) {
703
- const declared = new Set(
704
- Object.entries(collection.properties ?? {})
705
- .map(([name, prop]) => resolveColumnName(name, prop as Property))
706
- );
707
- for (const spec of AUTH_USERS_COLUMNS) {
708
- if (declared.has(spec.column)) continue;
709
- addColumn(key, schema, table, spec.column, authUsersColumnSql(spec));
710
- }
711
- }
712
- }
713
-
714
- // 3aa. The generated search columns.
715
- //
716
- // Adding a STORED generated column rewrites the table, which on a large
717
- // one is not free — but it is the same additive shape as every other
718
- // column here, and the alternative (leaving it out until someone runs a
719
- // migration) is a declared `search` block that silently does nothing.
720
- //
721
- // Changing one is not additive, and `ADD COLUMN IF NOT EXISTS` is a
722
- // no-op against a column that is already there — which is why a `search`
723
- // block that gained a field, flipped `unaccent` or moved a weight used
724
- // to be inert forever, on every path, with nothing logged. Each column
725
- // therefore carries a fingerprint of the expression it was built from
726
- // (in its comment), and a mismatch is reported rather than applied.
727
- const searchDrift: SearchColumnDrift[] = [];
728
- const searchAdopted: { table: string; column: string }[] = [];
729
- for (const spec of searchSpecs) {
730
- const key = `${spec.schema}.${spec.table}`;
731
- const definitions: Record<string, string> = {
732
- [spec.column]: `tsvector GENERATED ALWAYS AS (${spec.expression}) STORED`
733
- };
734
- if (spec.fuzzy) {
735
- definitions[spec.fuzzy.column] = `text GENERATED ALWAYS AS (${spec.fuzzy.expression}) STORED`;
736
- }
737
-
738
- for (const stamp of searchColumnStamps(spec)) {
739
- const definition = definitions[stamp.column];
740
- const exists = existing.tables.get(key)?.has(stamp.column) === true;
741
- const recorded = existing.columnComments?.get(`${key}.${stamp.column}`);
742
-
743
- if (exists && recorded?.startsWith(SEARCH_STAMP_PREFIX) && recorded !== stamp.fingerprint) {
744
- searchDrift.push({
745
- table: key,
746
- column: stamp.column,
747
- found: recorded,
748
- expected: stamp.fingerprint,
749
- rebuild: [
750
- `ALTER TABLE "${spec.schema}"."${spec.table}" DROP COLUMN "${stamp.column}";`,
751
- `ALTER TABLE "${spec.schema}"."${spec.table}" ADD COLUMN "${stamp.column}" ${definition};`,
752
- stamp.sql
753
- ]
754
- });
755
- // The old stamp is the only evidence of what the column holds;
756
- // overwriting it here would erase the drift instead of fixing it.
757
- continue;
758
- }
759
-
760
- addColumn(key, spec.schema, spec.table, stamp.column, definition);
761
- if (exists && recorded === undefined) {
762
- searchAdopted.push({ table: key, column: stamp.column });
763
- }
764
- if (recorded !== stamp.fingerprint) {
765
- actions.push({
766
- kind: "comment-column",
767
- target: `${key}.${stamp.column}`,
768
- sql: stamp.sql
769
- });
770
- }
771
- }
772
- }
773
-
774
- // 3b. A junction that already existed, but is short a column. One created
775
- // above already carries both — unlike a collection table, whose CREATE
776
- // declares only the identity column — so re-listing them would log two
777
- // no-op statements and inflate the count of changes applied.
778
- for (const junction of junctions) {
779
- const key = `${junction.schema}.${junction.table}`;
780
- if (created.has(key)) continue;
781
- for (const column of junction.columns) {
782
- if (renameLegacyColumn(key, junction.schema, junction.table, column.name, column.legacyName)) continue;
783
- addColumn(key, junction.schema, junction.table, column.name, column.type);
784
- }
785
- }
786
-
787
- // 3c. The columns relation and reference properties own.
788
- for (const relational of planRelationalColumns(collections)) {
789
- const relKey = `${relational.schema}.${relational.table}`;
790
- if (renameLegacyColumn(relKey, relational.schema, relational.table, relational.column, relational.legacyColumn)) continue;
791
- addColumn(
792
- relKey,
793
- relational.schema,
794
- relational.table,
795
- relational.column,
796
- relational.type
797
- );
798
- }
799
-
800
- // 4. Foreign keys, last: the tables and columns on both ends have to exist
801
- // first, and a constraint is the one thing here that can fail on data
802
- // rather than on schema, so nothing else depends on it.
803
- const knownConstraints = existing.constraints ?? new Set<string>();
804
- const plannedConstraints = new Set<string>();
805
- const foreignKeys = [
806
- ...planRelationalColumns(collections).map(r => r.foreignKey),
807
- ...junctions.flatMap(j => j.foreignKeys)
808
- ];
809
- for (const fk of foreignKeys) {
810
- if (!fk) continue;
811
- const name = `${fk.schema}.${fk.table}.${fk.constraintName}`;
812
- if (knownConstraints.has(name) || plannedConstraints.has(name)) continue;
813
- plannedConstraints.add(name);
814
- actions.push({
815
- kind: "add-constraint",
816
- target: `${fk.schema}.${fk.table}.${fk.constraintName}`,
817
- sql: fk.sql
818
- });
819
- }
820
-
821
- // 5. Search indexes, after everything — the column has to exist, and this is
822
- // the one step that runs against a populated table for real work.
823
- //
824
- // CONCURRENTLY: a plain CREATE INDEX takes a lock that blocks writes for
825
- // the duration of the build, which on a live table is an outage. Each
826
- // statement here is issued on its own, outside any transaction, which is
827
- // the condition CONCURRENTLY requires.
828
- for (const spec of searchSpecs) {
829
- for (const statement of searchIndexStatements(spec)) {
830
- actions.push({
831
- kind: "create-index",
832
- target: `${spec.schema}.${spec.table}`,
833
- sql: statement.replace("CREATE INDEX IF NOT EXISTS", "CREATE INDEX CONCURRENTLY IF NOT EXISTS")
834
- });
835
- }
836
- }
837
-
838
- // ANN indexes for vector columns, on the same terms: the column has to
839
- // exist, the build is real work against real rows, and CONCURRENTLY is
840
- // what keeps that from locking writes for its duration.
841
- //
842
- // A column too wide for pgvector to index is reported, not planned —
843
- // silence there would read as "indexed" to anyone watching the boot.
844
- const vectorIndexSkipped: SkippedVectorIndex[] = [];
845
- for (const collection of collections) {
846
- const plan = buildVectorIndexPlan(collection, resolveColumnName);
847
- for (const spec of plan.specs) {
848
- actions.push({
849
- kind: "create-index",
850
- target: `${spec.schema}.${spec.table}`,
851
- sql: vectorIndexStatement(spec).replace("CREATE INDEX IF NOT EXISTS", "CREATE INDEX CONCURRENTLY IF NOT EXISTS")
852
- });
853
- }
854
- vectorIndexSkipped.push(...plan.skipped);
855
- }
856
-
857
- // Declared indexes, on exactly the same terms as the ANN ones above.
858
- //
859
- // Boot has to emit these, not just `db push`: the managed runtime
860
- // provisions at boot and never runs a push, so a push-only index would
861
- // simply not exist there — and nothing would say so.
862
- // `contracts/derived-names.txt` states the rule ("Both, or it is a
863
- // bug") and the gate enforces it, which is what caught this.
864
- //
865
- // `concurrently` is a parameter here rather than a string replacement
866
- // on the rendered SQL. The `.replace("CREATE INDEX IF NOT EXISTS", …)`
867
- // just above silently does nothing for a UNIQUE index, whose text is
868
- // `CREATE UNIQUE INDEX …` and never matches the pattern.
869
- for (const spec of buildCollectionIndexPlan(collections, resolveColumnName)) {
870
- actions.push({
871
- kind: "create-index",
872
- target: `${spec.schema}.${spec.table}`,
873
- sql: collectionIndexStatement(spec, { concurrently: true, ifNotExists: true })
874
- });
875
- }
876
-
877
- return {
878
- actions,
879
- statements: actions.map(a => a.sql),
880
- legacyForeignKeys,
881
- searchDrift,
882
- searchAdopted,
883
- vectorIndexSkipped,
884
- withheldConstraints
885
- };
886
- }
887
-
888
- /** Read what the database has, for the schemas the collections live in. */
889
- export async function readExistingSchema(
890
- client: Queryable,
891
- schemas: string[]
892
- ): Promise<ExistingSchema> {
893
- const tables = new Map<string, Set<string>>();
894
- const enums = new Set<string>();
895
- if (schemas.length === 0) return { tables, enums };
896
-
897
- const inList = schemas
898
- .map(schema => `'${assertSafeIdentifier(schema, "schema name")}'`)
899
- .join(", ");
900
-
901
- const notNullColumns = new Set<string>();
902
- const { rows: columns } = await client.query<{
903
- table_schema: string;
904
- table_name: string;
905
- column_name: string;
906
- is_nullable: string;
907
- }>(
908
- `SELECT table_schema, table_name, column_name, is_nullable
909
- FROM information_schema.columns
910
- WHERE table_schema IN (${inList})`
911
- );
912
- for (const row of columns) {
913
- const key = `${row.table_schema}.${row.table_name}`;
914
- if (!tables.has(key)) tables.set(key, new Set());
915
- tables.get(key)!.add(row.column_name);
916
- if (row.is_nullable === "NO") notNullColumns.add(`${key}.${row.column_name}`);
917
- }
918
-
919
- // Which tables hold rows. This is the only fact that decides whether a
920
- // NOT NULL can be added without reading the data, so it is worth a query.
921
- //
922
- // `reltuples` would be cheaper and is wrong for this: it is a planner
923
- // estimate, it is -1 on a table that has never been analyzed, and a table
924
- // that was full an hour ago still reads as full after a DELETE. A wrong
925
- // "empty" here means a boot that aborts on a constraint violation, so the
926
- // estimate is not good enough. `EXISTS … LIMIT 1` stops at the first row,
927
- // which makes the true cost one page read per table.
928
- //
929
- // Restricted to ordinary and partitioned tables: `information_schema.columns`
930
- // also lists views and materialized views, and probing those runs whatever
931
- // query defines them.
932
- const populatedTables = new Set<string>();
933
- const { rows: realTables } = await client.query<{ schema: string; name: string }>(
934
- `SELECT n.nspname AS schema, c.relname AS name
935
- FROM pg_class c
936
- JOIN pg_namespace n ON c.relnamespace = n.oid
937
- WHERE c.relkind IN ('r', 'p') AND n.nspname IN (${inList})`
938
- );
939
- if (realTables.length > 0) {
940
- const probes = realTables.map(row => {
941
- const schema = assertSafeIdentifier(row.schema, "schema name");
942
- const table = assertSafeIdentifier(row.name, "table name");
943
- return `SELECT ${quoteSqlLiteral(`${schema}.${table}`)} AS key, ` +
944
- `EXISTS(SELECT 1 FROM "${schema}"."${table}" LIMIT 1) AS populated`;
945
- });
946
- const { rows: populationRows } = await client.query<{ key: string; populated: boolean }>(
947
- probes.join(" UNION ALL ")
948
- );
949
- for (const row of populationRows) {
950
- if (row.populated) populatedTables.add(row.key);
951
- }
952
- }
953
-
954
- const enumValues = new Map<string, string[]>();
955
- const { rows: enumValueRows } = await client.query<{
956
- schema: string;
957
- name: string;
958
- value: string;
959
- }>(
960
- // Ordered by `enumsortorder`, not by label: an enum's order is part of
961
- // its meaning (it is what `<` compares), and reading it back sorted
962
- // alphabetically would make a correct type look drifted.
963
- `SELECT n.nspname AS schema, t.typname AS name, e.enumlabel AS value
964
- FROM pg_enum e
965
- JOIN pg_type t ON e.enumtypid = t.oid
966
- JOIN pg_namespace n ON t.typnamespace = n.oid
967
- WHERE n.nspname IN (${inList})
968
- ORDER BY t.typname, e.enumsortorder`
969
- );
970
- for (const row of enumValueRows) {
971
- const key = `${row.schema}.${row.name}`;
972
- if (!enumValues.has(key)) enumValues.set(key, []);
973
- enumValues.get(key)!.push(row.value);
974
- }
975
-
976
- const { rows: enumRows } = await client.query<{ schema: string; name: string }>(
977
- `SELECT n.nspname AS schema, t.typname AS name
978
- FROM pg_type t
979
- JOIN pg_namespace n ON t.typnamespace = n.oid
980
- WHERE t.typtype = 'e' AND n.nspname IN (${inList})`
981
- );
982
- for (const row of enumRows) enums.add(`${row.schema}.${row.name}`);
983
-
984
- // `ADD CONSTRAINT` has no IF NOT EXISTS, so an existing foreign key is
985
- // skipped by name rather than guarded in SQL.
986
- const constraints = new Set<string>();
987
- const { rows: constraintRows } = await client.query<{
988
- schema: string;
989
- table: string;
990
- name: string;
991
- }>(
992
- `SELECT n.nspname AS schema, c.relname AS table, con.conname AS name
993
- FROM pg_constraint con
994
- JOIN pg_class c ON con.conrelid = c.oid
995
- JOIN pg_namespace n ON c.relnamespace = n.oid
996
- WHERE n.nspname IN (${inList})`
997
- );
998
- for (const row of constraintRows) constraints.add(`${row.schema}.${row.table}.${row.name}`);
999
-
1000
- // Column comments, which is where a generated search column records the
1001
- // expression it was built from. `objsubid > 0` is what makes a row a
1002
- // *column* comment rather than the table's own.
1003
- const columnComments = new Map<string, string>();
1004
- const { rows: commentRows } = await client.query<{
1005
- schema: string;
1006
- table: string;
1007
- column: string;
1008
- comment: string | null;
1009
- }>(
1010
- `SELECT n.nspname AS schema, c.relname AS table, a.attname AS column, d.description AS comment
1011
- FROM pg_description d
1012
- JOIN pg_class c ON d.objoid = c.oid
1013
- JOIN pg_namespace n ON c.relnamespace = n.oid
1014
- JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = d.objsubid
1015
- WHERE d.objsubid > 0 AND n.nspname IN (${inList})`
1016
- );
1017
- for (const row of commentRows) {
1018
- if (row.comment == null) continue;
1019
- columnComments.set(`${row.schema}.${row.table}.${row.column}`, row.comment);
1020
- }
1021
-
1022
- return { tables, enums, constraints, columnComments, enumValues, notNullColumns, populatedTables };
1023
- }
1024
-
1025
- /**
1026
- * What to tell an operator whose `search` block no longer matches its column.
1027
- *
1028
- * Every line here is doing work: naming the collection is not enough, because
1029
- * the symptom (a search that finds nothing) points at the data, not the schema;
1030
- * and the remediation has to be exact, because it is a table rewrite the
1031
- * operator is being asked to schedule rather than discover.
1032
- */
1033
- function searchDriftMessage(drift: SearchColumnDrift[]): string {
1034
- const blocks = drift.map(d =>
1035
- ` "${d.table}"."${d.column}" was generated from a different \`search\` block ` +
1036
- `(recorded ${d.found}, current ${d.expected}).\n` +
1037
- d.rebuild.map(s => ` ${s}`).join("\n")
1038
- );
1039
- return (
1040
- "The `search` block changed after its generated column was created, and Postgres cannot alter a " +
1041
- "generated expression in place.\n" +
1042
- "Rebase will not rebuild it for you: dropping and re-adding a STORED generated column rewrites the whole " +
1043
- "table under an ACCESS EXCLUSIVE lock and rebuilds its GIN index, which is an outage this unattended path " +
1044
- "may not schedule on your behalf.\n" +
1045
- "Until it is rebuilt the column keeps indexing the previous fields, weights and language — searches for " +
1046
- "anything added since return nothing, which reads from outside as \"no such row\".\n" +
1047
- "Run these (or revert the block to what the column was built from), then boot again:\n" +
1048
- blocks.join("\n") +
1049
- "\n The GIN index is dropped with the column and recreated concurrently on the next boot."
1050
- );
1051
- }
1052
-
1053
-
1054
- /**
1055
- * Read what the database looks like, for the schemas a set of collections
1056
- * lives in.
1057
- *
1058
- * The same read `ensureCollectionTables` does at boot, exposed on its own for
1059
- * the callers that want to *plan* against a real database without changing it —
1060
- * the live schema editor, which has to tell somebody what a change would do
1061
- * before they agree to it.
1062
- */
1063
- export async function readSchemaFactsFor(
1064
- client: Queryable,
1065
- collections: CollectionConfig[]
1066
- ): Promise<ExistingSchema> {
1067
- const relational = relationalCollections(collections);
1068
- const schemas = Array.from(new Set([
1069
- ...relational.map(schemaOf),
1070
- ...planJunctionTables(relational).map(junction => junction.schema)
1071
- ]));
1072
- return readExistingSchema(client, schemas);
1073
- }
1074
-
1075
- /**
1076
- * Bring the database up to date. Returns what it did.
1077
- *
1078
- * Each statement runs on its own rather than in one transaction: they are all
1079
- * independently safe and idempotent, and a single failure (an enum label that
1080
- * cannot be added, say) should not roll back the tables that were created fine.
1081
- * The error is surfaced with the statement that caused it.
1082
- */
1083
- export async function ensureCollectionTables(
1084
- client: Queryable,
1085
- collections: CollectionConfig[],
1086
- log?: (message: string) => void,
1087
- options: EnsureOptions = {}
1088
- ): Promise<EnsureOutcome> {
1089
- // Junctions live alongside the collections that declare them, so their
1090
- // schema has to be read too — otherwise an existing junction reads as
1091
- // missing and its constraints as unplanned.
1092
- const schemas = Array.from(new Set([
1093
- ...collections.map(schemaOf),
1094
- ...planJunctionTables(collections).map(j => j.schema)
1095
- ]));
1096
- for (const schema of schemas) {
1097
- assertSafeIdentifier(schema, "schema name");
1098
- if (schema !== "public") {
1099
- await client.query(`CREATE SCHEMA IF NOT EXISTS "${schema}";`);
1100
- }
1101
- }
1102
-
1103
- const existing = await readExistingSchema(client, schemas);
1104
- // This is the boundary, so this is where the world is read: the planner
1105
- // itself takes the permission as an argument and never reaches for the
1106
- // registry. By the time boot gets here `loadBundleResourceGraph` has
1107
- // evaluated the project's `resources.ts`, so the declarations are there —
1108
- // and a caller that already knows them can still say so.
1109
- const plan = planCollectionSchemaEnsure(collections, existing, {
1110
- ...options,
1111
- databaseExtensions: options.databaseExtensions ?? declaredDatabaseExtensions()
1112
- });
1113
- const failures: EnsureOutcome["failures"] = [];
1114
-
1115
- // Reported, not warned: this is a rename the ensure is about to perform, and
1116
- // the operator should be able to see in the log why a column changed name.
1117
- // The interesting case is the one that no longer happens — before this,
1118
- // ensure added the new column empty beside the populated old one, every
1119
- // statement succeeded, and the relation read the empty one.
1120
- for (const legacy of plan.legacyForeignKeys) {
1121
- const message =
1122
- `Renaming "${legacy.table}"."${legacy.legacy}" to "${legacy.expected}". The old name is ` +
1123
- "the one Rebase derived for this relation before it singularized properly; the column " +
1124
- "keeps its data, indexes and constraints. To keep the old name instead, set " +
1125
- `\`localKey: "${legacy.legacy}"\` on the relation and this will stop.`;
1126
- logger.info(`[schema] ${message}`);
1127
- log?.(message);
1128
- }
1129
-
1130
- // Before anything is applied: a `search` block that changed after its column
1131
- // was generated cannot be honoured by an additive plan, and serving the old
1132
- // index while the config describes a new one is the silent failure this
1133
- // check exists to end. Refusing is the loud half — boot is fatal on purpose
1134
- // (see `ensureCollectionSchema` in the server's boot) and the message
1135
- // carries the exact statements that resolve it.
1136
- if (plan.searchDrift.length > 0) {
1137
- throw new Error(searchDriftMessage(plan.searchDrift));
1138
- }
1139
-
1140
- // Said once per column, at the moment the stamp is applied: from here on a
1141
- // change is detected, but whether *this* column matches the block it is
1142
- // being stamped with is not knowable — it predates the stamp.
1143
- for (const adopted of plan.searchAdopted) {
1144
- const message =
1145
- `Adopting the existing generated column "${adopted.table}"."${adopted.column}" and recording what the ` +
1146
- "current `search` block would generate. Any later change to that block will be detected and refused; a " +
1147
- "change made *before* this version was deployed cannot be, so if search has been missing content, " +
1148
- `rebuild the column once: ALTER TABLE "${adopted.table.split(".").join('"."')}" DROP COLUMN "${adopted.column}"; and boot again.`;
1149
- logger.info(`[schema] ${message}`);
1150
- log?.(message);
1151
- }
1152
-
1153
- // Said once per column, every boot: an unindexed vector column and an
1154
- // indexed one behave identically apart from latency, so the only way anyone
1155
- // learns which one they have is if the boot says so.
1156
- for (const skip of plan.vectorIndexSkipped) {
1157
- const message = `No ANN index on "${skip.table}"."${skip.column}": ${skip.reason}`;
1158
- logger.warn(`[schema] ${message}`);
1159
- log?.(message);
1160
- }
1161
-
1162
- // Said once per column, every boot, because the alternative is what this
1163
- // whole feature exists to end: a column the configuration calls required,
1164
- // sitting there nullable, with every surface reporting success. The boot
1165
- // does not fail over it — the column is usable and the data is intact — but
1166
- // it stops being invisible.
1167
- for (const withheld of plan.withheldConstraints) {
1168
- const message =
1169
- `No NOT NULL on "${withheld.target}": ${withheld.reason} ${withheld.remedy}`;
1170
- logger.warn(`[schema] ${message}`);
1171
- log?.(message);
1172
- }
1173
-
1174
- if (plan.actions.length === 0) {
1175
- log?.("Schema is up to date; nothing to create.");
1176
- return { ...plan, failures };
1177
- }
1178
-
1179
- for (const action of plan.actions) {
1180
- try {
1181
- if (await applyAction(client, action)) {
1182
- log?.(`${action.kind}: ${action.target}`);
1183
- } else {
1184
- log?.(`${action.kind}: ${action.target} (already created by a peer)`);
1185
- }
1186
- } catch (err) {
1187
- const message = err instanceof Error ? err.message : String(err);
1188
- // A foreign key is the only action that can fail on the customer's
1189
- // data rather than on the schema. The column it polices is already
1190
- // there, so the collection serves either way — record it and carry
1191
- // on rather than crash-looping the deployment.
1192
- //
1193
- // A comment is metadata about a column that was just created
1194
- // successfully, and it can only fail on ownership (COMMENT requires
1195
- // owning the table, which an adopted table may not grant). Losing
1196
- // the stamp costs drift detection on the next boot; it must not cost
1197
- // the deployment.
1198
- if (action.kind === "add-constraint" || action.kind === "comment-column") {
1199
- failures.push({ kind: action.kind, target: action.target, error: message });
1200
- continue;
1201
- }
1202
- throw new Error(
1203
- `Failed to ${action.kind} ${action.target}: ${message}${vectorExtensionHint(message)}\n ${action.sql}`
1204
- );
1205
- }
1206
- }
1207
- return { ...plan, failures };
1208
- }
1209
-
1210
- /** Attempts per action, including the first. Matches the server's bootstraps. */
1211
- const DDL_ATTEMPTS = 4;
1212
-
1213
- /**
1214
- * Run one planned statement, surviving a simultaneous boot.
1215
- *
1216
- * Every statement in a plan is written to be idempotent, and that is not the
1217
- * same as being safe to run concurrently: `CREATE … IF NOT EXISTS` reads the
1218
- * catalog and then writes to it as two steps, so peers starting together both
1219
- * see "absent" and the loser gets a duplicate key on a *catalog* index. Measured
1220
- * against Postgres 18: five instances, 8 of 10 calls lost. `CREATE TYPE` is
1221
- * worse, because Postgres has no `IF NOT EXISTS` for it at all.
1222
- *
1223
- * What made that fatal here rather than merely noisy is the loop this sits in.
1224
- * A losing statement threw, and the throw abandoned **every remaining action in
1225
- * the plan** — so a replica that lost one race came up missing tables it never
1226
- * attempted, and the boot log blamed the one statement that failed.
1227
- *
1228
- * @returns `true` if this process applied the statement, `false` if a peer had
1229
- * already created the object. The distinction is only for the log; both mean
1230
- * the object is now there.
1231
- * @throws the original error for anything that is not a race — a syntax error, a
1232
- * permission failure, a unique constraint the customer's own rows violate.
1233
- */
1234
- async function applyAction(
1235
- client: Queryable,
1236
- action: EnsureAction
1237
- ): Promise<boolean> {
1238
- for (let attempt = 1; ; attempt++) {
1239
- try {
1240
- await client.query(action.sql);
1241
- return true;
1242
- } catch (err) {
1243
- // Already there. Not "retry" — the end state this statement wanted
1244
- // is the end state the database is in, so carry on to the next
1245
- // action rather than spending three more attempts proving it.
1246
- if (isDuplicateObjectRace(err)) {
1247
- logger.debug(
1248
- `[schema] ${action.kind} ${action.target}: already created by another instance`
1249
- );
1250
- return false;
1251
- }
1252
- // Retryable but not yet satisfied — a deadlock between two boots
1253
- // taking catalog locks in step. The statement did nothing; run it
1254
- // again after a jittered pause so peers that collided once do not
1255
- // collide again in lockstep.
1256
- if (isConcurrentDdlRace(err) && attempt < DDL_ATTEMPTS) {
1257
- logger.debug(
1258
- `[schema] ${action.kind} ${action.target}: lost a race with another instance ` +
1259
- `(attempt ${attempt}/${DDL_ATTEMPTS}) — retrying`
1260
- );
1261
- await new Promise(resolve => setTimeout(resolve, 40 * attempt * (1 + Math.random())));
1262
- continue;
1263
- }
1264
- throw err;
1265
- }
1266
- }
1267
- }