@workerstack/database 0.22.0 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/README.md +95 -30
  2. package/docs/meta-contract.md +1 -1
  3. package/package.json +3 -2
  4. package/src/cli/args.ts +4 -2
  5. package/src/cli/codegen.ts +2 -2
  6. package/src/cli/comb-config.ts +8 -3
  7. package/src/cli/file-path.ts +25 -0
  8. package/src/cli/migrate-apply.ts +5 -21
  9. package/src/cli/migrate-d1.ts +5 -1
  10. package/src/cli/migrate-json.ts +48 -0
  11. package/src/cli/migrate.ts +26 -4
  12. package/src/formats/bcp47.ts +12 -0
  13. package/src/formats/catalog.ts +27 -4
  14. package/src/index.ts +1 -0
  15. package/src/migrate/core/apply.ts +63 -22
  16. package/src/migrate/core/atlas-apply-runs.ts +15 -26
  17. package/src/migrate/core/cleanup.ts +22 -10
  18. package/src/migrate/core/d1-engine-child.mjs +27 -11
  19. package/src/migrate/core/d1-history.ts +23 -26
  20. package/src/migrate/core/d1-wrangler-error.ts +24 -1
  21. package/src/migrate/core/pg-replay-commit.ts +41 -38
  22. package/src/migrate/core/pg-replay-drops.ts +17 -32
  23. package/src/migrate/core/pg-replay-hidden.ts +75 -0
  24. package/src/migrate/core/pg-replay-roles.ts +2 -4
  25. package/src/migrate/core/pg-replay.ts +21 -19
  26. package/src/migrate/core/pg-server-snapshot.ts +4 -24
  27. package/src/migrate/core/proof-readers.ts +80 -0
  28. package/src/migrate/core/proof-run.ts +19 -5
  29. package/src/migrate/core/proof-scope.ts +27 -14
  30. package/src/migrate/core/sql-code.ts +36 -5
  31. package/src/migrate/core/sql-scan.ts +7 -1
  32. package/src/migrate/core/sqlite-keywords.ts +70 -0
  33. package/src/migrate/core/stopped.ts +70 -0
  34. package/src/migrate/core/trigger-write.d.mts +7 -0
  35. package/src/migrate/core/trigger-write.mjs +42 -0
  36. package/src/query/cursor-mint.ts +11 -18
  37. package/src/query/cursor-refusals.ts +22 -11
  38. package/src/query/descriptions.ts +4 -9
  39. package/src/query/filter-parse.ts +80 -22
  40. package/src/query/filter-validate.ts +10 -13
  41. package/src/query/filter.ts +40 -18
  42. package/src/query/list-extras.ts +7 -10
  43. package/src/query/list-schema.ts +0 -2
  44. package/src/query/merged.ts +3 -2
  45. package/src/query/pagination.ts +30 -6
  46. package/src/query/rounded-key.ts +5 -8
  47. package/src/query/schema-params.ts +12 -5
  48. package/src/query/select-validate.ts +7 -4
  49. package/src/query/sqlite/cursor-fit.ts +10 -2
  50. package/src/query/trusted.ts +7 -2
  51. package/types/cli/args.d.ts.map +1 -1
  52. package/types/cli/comb-config.d.ts +5 -2
  53. package/types/cli/comb-config.d.ts.map +1 -1
  54. package/types/cli/file-path.d.ts +11 -0
  55. package/types/cli/file-path.d.ts.map +1 -0
  56. package/types/cli/migrate-apply.d.ts +1 -0
  57. package/types/cli/migrate-apply.d.ts.map +1 -1
  58. package/types/cli/migrate-d1.d.ts.map +1 -1
  59. package/types/cli/migrate-json.d.ts +4 -0
  60. package/types/cli/migrate-json.d.ts.map +1 -0
  61. package/types/cli/migrate.d.ts.map +1 -1
  62. package/types/formats/bcp47.d.ts +9 -0
  63. package/types/formats/bcp47.d.ts.map +1 -0
  64. package/types/formats/catalog.d.ts.map +1 -1
  65. package/types/index.d.ts +1 -0
  66. package/types/index.d.ts.map +1 -1
  67. package/types/migrate/core/apply.d.ts +3 -2
  68. package/types/migrate/core/apply.d.ts.map +1 -1
  69. package/types/migrate/core/atlas-apply-runs.d.ts +6 -0
  70. package/types/migrate/core/atlas-apply-runs.d.ts.map +1 -1
  71. package/types/migrate/core/cleanup.d.ts +16 -4
  72. package/types/migrate/core/cleanup.d.ts.map +1 -1
  73. package/types/migrate/core/d1-history.d.ts.map +1 -1
  74. package/types/migrate/core/d1-wrangler-error.d.ts +11 -0
  75. package/types/migrate/core/d1-wrangler-error.d.ts.map +1 -1
  76. package/types/migrate/core/pg-replay-commit.d.ts +14 -9
  77. package/types/migrate/core/pg-replay-commit.d.ts.map +1 -1
  78. package/types/migrate/core/pg-replay-drops.d.ts +7 -12
  79. package/types/migrate/core/pg-replay-drops.d.ts.map +1 -1
  80. package/types/migrate/core/pg-replay-hidden.d.ts +36 -0
  81. package/types/migrate/core/pg-replay-hidden.d.ts.map +1 -0
  82. package/types/migrate/core/pg-replay-roles.d.ts +2 -4
  83. package/types/migrate/core/pg-replay-roles.d.ts.map +1 -1
  84. package/types/migrate/core/pg-replay.d.ts.map +1 -1
  85. package/types/migrate/core/pg-server-snapshot.d.ts +4 -11
  86. package/types/migrate/core/pg-server-snapshot.d.ts.map +1 -1
  87. package/types/migrate/core/proof-readers.d.ts +19 -0
  88. package/types/migrate/core/proof-readers.d.ts.map +1 -0
  89. package/types/migrate/core/proof-run.d.ts.map +1 -1
  90. package/types/migrate/core/proof-scope.d.ts +15 -7
  91. package/types/migrate/core/proof-scope.d.ts.map +1 -1
  92. package/types/migrate/core/sql-code.d.ts +7 -3
  93. package/types/migrate/core/sql-code.d.ts.map +1 -1
  94. package/types/migrate/core/sql-scan.d.ts.map +1 -1
  95. package/types/migrate/core/sqlite-keywords.d.ts +11 -0
  96. package/types/migrate/core/sqlite-keywords.d.ts.map +1 -0
  97. package/types/migrate/core/stopped.d.ts +23 -0
  98. package/types/migrate/core/stopped.d.ts.map +1 -0
  99. package/types/query/cursor-mint.d.ts.map +1 -1
  100. package/types/query/cursor-refusals.d.ts +17 -1
  101. package/types/query/cursor-refusals.d.ts.map +1 -1
  102. package/types/query/descriptions.d.ts +1 -1
  103. package/types/query/descriptions.d.ts.map +1 -1
  104. package/types/query/filter-parse.d.ts +15 -4
  105. package/types/query/filter-parse.d.ts.map +1 -1
  106. package/types/query/filter-validate.d.ts +1 -1
  107. package/types/query/filter-validate.d.ts.map +1 -1
  108. package/types/query/filter.d.ts +9 -1
  109. package/types/query/filter.d.ts.map +1 -1
  110. package/types/query/list-extras.d.ts.map +1 -1
  111. package/types/query/list-schema.d.ts.map +1 -1
  112. package/types/query/merged.d.ts.map +1 -1
  113. package/types/query/pagination.d.ts +13 -1
  114. package/types/query/pagination.d.ts.map +1 -1
  115. package/types/query/rounded-key.d.ts +2 -11
  116. package/types/query/rounded-key.d.ts.map +1 -1
  117. package/types/query/schema-params.d.ts.map +1 -1
  118. package/types/query/select-validate.d.ts.map +1 -1
  119. package/types/query/sqlite/cursor-fit.d.ts.map +1 -1
  120. package/types/query/trusted.d.ts +1 -1
  121. package/types/query/trusted.d.ts.map +1 -1
@@ -2,12 +2,14 @@
2
2
  * The replay's transactions and the reads around them (pg-replay.ts): a file's or comb's own
3
3
  * statement's transaction, refused before its COMMIT when this backend's counts show it wrote past
4
4
  * its database, and the server's shared catalogs read as committed around it and around
5
- * every bare statement, a change there the file's when it is to an object the file names;
5
+ * every bare statement, a change there the file's when it is to the replay's own database, or to an
6
+ * object the file names beside a statement that can run code comb cannot read;
6
7
  * a `txmode none` file's statement in a transaction of its own first, bare only when
7
8
  * Postgres runs it outside a block alone.
8
9
  */
9
10
  import { adopt, bareAgain, type Run } from "./pg-replay-roles.ts"
10
- import { added, type Snapshot, snapshot, snapshotChanges, subscriptionsIn } from "./pg-server-snapshot.ts"
11
+ import { type Hidden, runsHidden } from "./pg-replay-hidden.ts"
12
+ import { type Snapshot, snapshot, snapshotChanges, subscriptionsIn } from "./pg-server-snapshot.ts"
11
13
  import {
12
14
  type Change,
13
15
  catalogsOf,
@@ -31,12 +33,13 @@ async function begin(run: Run): Promise<void> {
31
33
  }
32
34
 
33
35
  /**
34
- * Where a replayed transaction or bare statement is: its file's name and the names it spells, the
35
- * statement, if bare, and what a refusal stops; told each role the transaction adds that the replay
36
- * adopts
36
+ * Where a replayed transaction or bare statement is: its file's name and the names it spells, what
37
+ * its statements so far can run that comb cannot read (`hidden`), the statement, if bare, and what a
38
+ * refusal stops; told each role the transaction adds that the replay adopts
37
39
  */
38
40
  type At = {
39
41
  adopted?: ((role: string) => void) | undefined
42
+ hidden: Hidden
40
43
  name: string
41
44
  names: ReadonlySet<string>
42
45
  sent: string | undefined
@@ -46,52 +49,52 @@ type At = {
46
49
  /* the words that name devUrl's role in a file, as namesIn folds them: CURRENT_USER and its kin */
47
50
  const SESSION_ROLE = ["current_role", "current_user", "session_user", "user"]
48
51
 
52
+ /** The objects the file `names`, devUrl's role among them where it names that by keyword */
53
+ function spelled(run: Run, names: ReadonlySet<string>): Set<string> {
54
+ const its = new Set(names)
55
+ if (SESSION_ROLE.some((word) => names.has(word))) its.add(run.role)
56
+ return its
57
+ }
58
+
49
59
  /**
50
- * The server's shared catalogs read as committed against the last read: a change since to an object
51
- * the file `name` names (`names`), while `sent` or the whole file ran, is refused, and a role or
52
- * database it added is the cleanup's; a change to any other is another session's. The
53
- * replay's own database is the file's whatever it spells, as only this replay runs in it, and is
54
- * the cleanup's too. `read` is that read when the caller took it
60
+ * The server's shared catalogs read as committed against the last read: a change since to the
61
+ * replay's own database, or to what it holds, is the file's, as only this replay runs in it, and the
62
+ * cleanup drops it. A change to an object the file names is the file's when its statements so far
63
+ * can run code comb cannot read (pg-replay-hidden.ts), the one way a change past the database gets by
64
+ * the counts, and is refused, but no cleanup drops it: Postgres keeps no creator for a role or a
65
+ * database, and another session may have made it. A change to any other object, or beside statements
66
+ * that run only what their parse says, is another session's. `read` is that read when the caller took it
55
67
  */
56
68
  async function window(run: Run, at: At, read?: Snapshot): Promise<void> {
57
- strayed(run, at, read ?? (await snapshot(run.admin)))
69
+ const now = read ?? (await snapshot(run.admin))
70
+ strayed(run, { ...at, hid: await hides(run, at, now) }, now)
58
71
  }
59
72
 
73
+ /* whether a change in `now` to an object the file names may be the file's, as its statements can run code comb cannot read */
74
+ const hides = (run: Run, { hidden, names }: At, now: Snapshot): boolean | Promise<boolean> =>
75
+ snapshotChanges(run.seen, now, spelled(run, names)).length > 0 && runsHidden(run.sql, hidden)
76
+
60
77
  /**
61
78
  * window's verdict on `now`, the server's shared catalogs as committed after the file or `sent`,
62
- * against the last read; kept apart from window's await, as v8 miscounts a branch after an await
79
+ * against the last read, the objects the file names counted when it `hid` code comb cannot read;
80
+ * kept apart from window's await, as v8 miscounts a branch after an await
63
81
  */
64
- function strayed(run: Run, { name, names, sent, stops }: At, now: Snapshot): void {
65
- const its = new Set([...names, run.database])
66
- if (SESSION_ROLE.some((word) => names.has(word))) its.add(run.role)
67
- const changes = snapshotChanges(run.seen, now, its)
68
- const { databases, roles, slots } = added(run.seen, now, its)
82
+ function strayed(run: Run, { hid, name, names, sent, stops }: At & { hid: boolean }, now: Snapshot): void {
83
+ const replays = new Set([run.database])
84
+ const changes = snapshotChanges(run.seen, now, hid ? new Set([...spelled(run, names), ...replays]) : replays)
85
+ const own = new Set(snapshotChanges(run.seen, now, replays))
69
86
  const within = subscriptionsIn(run.seen, now, run.database)
70
87
  run.seen = now
71
88
  if (changes.length === 0) return
72
- run.strays.databases.push(...databases)
73
- run.strays.roles.push(...roles)
74
- run.strays.slots.push(...slots)
75
- const own = `database ${run.database} `
76
- const strays = [
77
- ...roles.map((role) => `role ${role}`),
78
- ...databases.map((database) => `database ${database} unless a session is in it`),
79
- ...slots.map((slot) => `replication slot ${slot} unless a session holds it`),
80
- ]
81
- const replays =
82
- changes.some((change) => change.startsWith(own)) || within.length > 0
89
+ const dropped =
90
+ own.size > 0
83
91
  ? `; comb drops the database it replays in${within.length === 0 ? "" : `, and ${within.join(", ")} in it`}`
84
92
  : ""
85
- const dropped = `${strays.length === 0 ? "" : `; comb drops ${strays.join(", ")}, which the file added`}${replays}`
86
- /* the read is committed, so a change it shows to an object the file did not add stays */
87
- const dropping = new Set([
88
- ...roles.map((role) => `role ${role} added`),
89
- ...databases.map((d) => `database ${d} added`),
90
- ...slots.map((slot) => `replication slot ${slot} added`),
91
- ...within.map((subject) => `${subject} added`),
92
- ])
93
- const kept = changes.filter((change) => !dropping.has(change) && !change.startsWith(own))
94
- const left = kept.length === 0 ? "" : `; devUrl's server keeps ${kept.join(", ")}, which comb cannot undo`
93
+ const kept = changes.filter((change) => !own.has(change))
94
+ const left =
95
+ kept.length === 0
96
+ ? ""
97
+ : `; comb leaves ${kept.join(", ")} on devUrl's server, as another session may have made ${kept.length === 1 ? "that change" : "those changes"}: undo there each that is the file's`
95
98
  throw new ReplayFailed(
96
99
  `${name}: devUrl's server changed past the database the replay runs in while ${sent === undefined ? "the file" : JSON.stringify(sent)} ran: ${changes.join(", ")}. A change made there, from inside a DO block, a function, EXECUTE or CALL, or over a second connection such as dblink's, apply would make on the target's server, so comb refuses the file; write the change as a statement of its own, which the replay leaves out and apply runs${dropped}${left}; ${stops}`,
97
100
  )
@@ -4,8 +4,8 @@
4
4
  * which the next replay or apply sweeps. Postgres drops no database that holds a subscription or a
5
5
  * logical slot, so both go first, and only from the replay databases it drops: a subscription from
6
6
  * a session in its database, the only one that can name it. A drop that fails never decides the
7
- * verdict, and is said, never left unsaid: what it left, dev's error, and whether a later sweep takes it, as a
8
- * stray of a refused window carries no tag a sweep finds. The replay database is dropped WITH
7
+ * verdict, and is said, never left unsaid: what it left and dev's error, which a later sweep takes. The
8
+ * replay database is dropped WITH
9
9
  * (FORCE), so a backend that came in after the cleanup ended the others cannot hold it.
10
10
  */
11
11
  import { sqlIdentifier } from "../../codegen/utils/case.ts"
@@ -24,14 +24,11 @@ async function settle(step: () => Promise<unknown>): Promise<void> {
24
24
  }
25
25
  }
26
26
 
27
- /** What a drop that failed left on devUrl's server, a database, role or slot by name, and whether a later sweep takes it */
28
- type Left = { swept: boolean; what: string }
29
-
30
- /** How a drop that failed is said: what it `left`, with dev's error */
31
- type Leave = (left: Left, error: unknown) => void
27
+ /** How a drop that failed is said: what it left on devUrl's server, a database, role or slot by name, with dev's error */
28
+ type Leave = (left: string, error: unknown) => void
32
29
 
33
30
  /** `drop` run, a failure `leave`d: cleaning up never replaces the replay's verdict, and never leaves something unsaid */
34
- async function dropOr(leave: Leave, left: Left, drop: () => Promise<unknown>): Promise<void> {
31
+ async function dropOr(leave: Leave, left: string, drop: () => Promise<unknown>): Promise<void> {
35
32
  try {
36
33
  await drop()
37
34
  } catch (error) {
@@ -45,9 +42,9 @@ type Drops = { into: (database: string) => Sql; leave: Leave }
45
42
 
46
43
  /** A `leave` that prints, `failed` naming devUrl in dev's error */
47
44
  function leaving(print: (line: string) => void, failed: Parameters<typeof failureOf>[1]): Leave {
48
- return ({ swept, what }, error) =>
45
+ return (left, error) =>
49
46
  print(
50
- `comb could not drop ${what}, which a replay left on devUrl's server: ${failureOf(error, failed)}; ${swept ? "the next check or apply that reaches that server drops it" : "drop it there"}`,
47
+ `comb could not drop ${left}, which a replay left on devUrl's server: ${failureOf(error, failed)}; the next check or apply that reaches that server drops it`,
51
48
  )
52
49
  }
53
50
 
@@ -80,14 +77,13 @@ async function emptyDatabase(admin: Sql, database: string, { into, leave }: Drop
80
77
  if (subscriptions.length > 0) {
81
78
  const sql = into(database)
82
79
  try {
83
- for (const name of subscriptions)
84
- await dropOr(leave, { swept: true, what: `subscription ${name}` }, () => dropSubscription(sql, name))
80
+ for (const name of subscriptions) await dropOr(leave, `subscription ${name}`, () => dropSubscription(sql, name))
85
81
  } finally {
86
82
  await settle(() => sql.end({ timeout: 0 }))
87
83
  }
88
84
  }
89
85
  for (const slot of (holds?.["slots"] ?? []) as string[])
90
- await dropOr(leave, { swept: true, what: `replication slot ${slot}` }, () => dropSlot(admin, slot))
86
+ await dropOr(leave, `replication slot ${slot}`, () => dropSlot(admin, slot))
91
87
  }
92
88
 
93
89
  /**
@@ -132,11 +128,9 @@ async function sweep(admin: Sql, now: number, drops: Drops): Promise<void> {
132
128
  await settle(() => emptyDatabase(admin, name, drops))
133
129
  /* Postgres drops no template, which a committing DO can make the replay's database */
134
130
  if (template) await settle(() => admin.unsafe(`ALTER DATABASE ${sqlIdentifier(name)} IS_TEMPLATE false`, []))
135
- await dropOr(leave, { swept: true, what: `database ${name}` }, () =>
136
- admin.unsafe(`DROP DATABASE IF EXISTS ${sqlIdentifier(name)}`, []),
137
- )
131
+ await dropOr(leave, `database ${name}`, () => admin.unsafe(`DROP DATABASE IF EXISTS ${sqlIdentifier(name)}`, []))
138
132
  }
139
- for (const role of roles) await dropOr(leave, { swept: true, what: `role ${role}` }, () => dropRole(admin, role))
133
+ for (const role of roles) await dropOr(leave, `role ${role}`, () => dropRole(admin, role))
140
134
  }
141
135
 
142
136
  /* why comb loads postgres.js in an apply */
@@ -196,10 +190,10 @@ async function sweepDev(
196
190
 
197
191
  /**
198
192
  * `run`'s cleanup, each drop that fails `leave`d: its connection closed, a query in flight with it,
199
- * any backend still in its database ended, the slots a refused window added, which a drop waits on,
200
- * the database's subscriptions and slots, which keep it from its drop, the database made no
201
- * template and dropped, forcing out a backend that came in since, each role it made, and that
202
- * window's roles and idle databases, which no sweep takes
193
+ * any backend still in its database ended, the database's subscriptions and slots, which keep it
194
+ * from its drop, the database made no template and dropped, forcing out a backend that came in
195
+ * since, and each role it made or adopted. It drops nothing else a refused window saw, as Postgres
196
+ * keeps no creator for a role, a database or a slot, and another session may have made it
203
197
  */
204
198
  async function cleanUp(run: Run, drops: Drops): Promise<void> {
205
199
  const { leave } = drops
@@ -210,21 +204,12 @@ async function cleanUp(run: Run, drops: Drops): Promise<void> {
210
204
  [run.database],
211
205
  ),
212
206
  )
213
- for (const slot of run.strays.slots)
214
- await dropOr(leave, { swept: false, what: `replication slot ${slot}` }, () => dropSlot(run.admin, slot))
215
207
  await settle(() => emptyDatabase(run.admin, run.database, drops))
216
208
  await settle(() => run.admin.unsafe(`ALTER DATABASE ${run.database} IS_TEMPLATE false`, []))
217
- await dropOr(leave, { swept: true, what: `database ${run.database}` }, () =>
209
+ await dropOr(leave, `database ${run.database}`, () =>
218
210
  run.admin.unsafe(`DROP DATABASE IF EXISTS ${run.database} WITH (FORCE)`, []),
219
211
  )
220
- for (const role of run.created)
221
- await dropOr(leave, { swept: true, what: `role ${role}` }, () => dropMade(run.admin, role))
222
- for (const name of run.strays.databases)
223
- await dropOr(leave, { swept: false, what: `database ${name}` }, () =>
224
- run.admin.unsafe(`DROP DATABASE IF EXISTS ${sqlIdentifier(name)}`, []),
225
- )
226
- for (const role of run.strays.roles)
227
- await dropOr(leave, { swept: false, what: `role ${role}` }, () => dropRole(run.admin, role))
212
+ for (const role of run.created) await dropOr(leave, `role ${role}`, () => dropMade(run.admin, role))
228
213
  }
229
214
 
230
215
  export { cleanUp, type Drops, dropMade, dropOr, dropRole, dropSlot, type Leave, leaving, settle, sweep, sweepDev }
@@ -0,0 +1,75 @@
1
+ /**
2
+ * Whether the statements a replay's window covers can run code comb cannot read, the only way one
3
+ * makes a change past its database that this backend's counts miss (pg-replay-commit.ts): a DO
4
+ * block, a CALL, an EXECUTE, or a call of a function outside pg_catalog, dblink's among them. A
5
+ * statement whose tree holds none of those runs only what its parse says, and BEYOND and SERVER_CALLS
6
+ * (pg-outside-transaction.ts, pg-server-calls.ts) leave out every such statement that reaches past
7
+ * the database, so a change the server's read shows beside it is another session's. A function
8
+ * called by a bare name is pg_catalog's only where the replay's database holds no other of that name,
9
+ * as an overload in a schema on the search path can take the call, and the file makes none of it.
10
+ * Code Postgres runs for a statement on its own, a trigger, a default, an operator or a cast that
11
+ * calls a function an earlier file defines, is no call in the tree, so it is not read here; what it
12
+ * writes in the replay's transaction its counts still see.
13
+ */
14
+ import type { Sql } from "./pg-replay-database.ts"
15
+
16
+ /**
17
+ * What the statements so far can run that comb cannot read: `code` when one surely can, the
18
+ * functions they call by a bare name (`calls`), which may be pg_catalog's, and those the file makes
19
+ * (`made`). A file's every later window counts what one ran, as a change a DO block makes in the
20
+ * file's own block commits at its COMMIT
21
+ */
22
+ type Hidden = { calls: Set<string>; code: boolean; made: Set<string> }
23
+
24
+ /** Nothing run yet */
25
+ const nothingHidden = (): Hidden => ({ calls: new Set(), code: false, made: new Set() })
26
+
27
+ // lookup-table: the statements that run code the parse does not hold: a DO block's body, a procedure, a prepared statement
28
+ const RUNS_CODE: ReadonlySet<string> = /* @__PURE__ */ new Set(["CallStmt", "DoStmt", "ExecuteStmt"])
29
+
30
+ /** The name parts of a FuncCall's or a CREATE FUNCTION's `funcname`, each a String node, the name last */
31
+ const partsOf = (funcname: unknown): string[] =>
32
+ (funcname as Array<{ String: { sval: string } }>).map((part) => part.String.sval)
33
+
34
+ /** `tree`, a parse tree or a part of one, read `into` */
35
+ function readTree(tree: unknown, into: Hidden): void {
36
+ if (tree === null || typeof tree !== "object") return
37
+ for (const [key, value] of Object.entries(tree)) {
38
+ if (RUNS_CODE.has(key)) into.code = true
39
+ else if (key === "FuncCall" || key === "CreateFunctionStmt") {
40
+ const parts = partsOf((value as { funcname: unknown }).funcname)
41
+ /* a funcname holds its name at least */
42
+ const name = parts.at(-1) as string
43
+ if (key === "CreateFunctionStmt") into.made.add(name)
44
+ else if (parts.length === 1) into.calls.add(name)
45
+ else if (parts.length > 2 || parts[0] !== "pg_catalog") into.code = true
46
+ }
47
+ readTree(value, into)
48
+ }
49
+ }
50
+
51
+ /** A statement the replay runs, read `into`: one the parser cannot read may run anything */
52
+ function readStatement(tree: unknown, into: Hidden): void {
53
+ if (tree === undefined) into.code = true
54
+ else readTree(tree, into)
55
+ }
56
+
57
+ /* of the functions a JSON array names, whether one is no pg_catalog function, or also one outside it */
58
+ const NOT_CATALOG = `SELECT EXISTS (
59
+ SELECT FROM jsonb_array_elements_text($1::text::jsonb) f(name)
60
+ WHERE NOT EXISTS (SELECT FROM pg_proc p WHERE p.proname = f.name AND p.pronamespace = 'pg_catalog'::regnamespace)
61
+ OR EXISTS (SELECT FROM pg_proc p WHERE p.proname = f.name AND p.pronamespace <> 'pg_catalog'::regnamespace)
62
+ ) AS hidden`
63
+
64
+ /**
65
+ * Whether what `hidden` holds can run code comb cannot read, a bare name resolved on `sql`, the
66
+ * replay's connection, which sees what its open transaction made
67
+ */
68
+ async function runsHidden(sql: Sql, { calls, code, made }: Hidden): Promise<boolean> {
69
+ if (code || [...calls].some((name) => made.has(name))) return true
70
+ if (calls.size === 0) return false
71
+ const rows = await sql.unsafe(NOT_CATALOG, [JSON.stringify([...calls])])
72
+ return rows.some((row) => row["hidden"] === true)
73
+ }
74
+
75
+ export { type Hidden, nothingHidden, readStatement, runsHidden }
@@ -18,7 +18,7 @@
18
18
  import { sqlIdentifier } from "../../codegen/utils/case.ts"
19
19
  import { codeOf, literal, type Sql, TAG } from "./pg-replay-database.ts"
20
20
  import { addWrites, catalogsOf, counts, type Rows, subjectRows, type Writes, writesSince } from "./pg-server-writes.ts"
21
- import type { Snapshot, Strays } from "./pg-server-snapshot.ts"
21
+ import type { Snapshot } from "./pg-server-snapshot.ts"
22
22
  import { flag } from "./pg-nodes.ts"
23
23
 
24
24
  /**
@@ -26,8 +26,7 @@ import { flag } from "./pg-nodes.ts"
26
26
  * that owns it, its role (`role`, which CURRENT_USER and its kin name in a file), whether that role
27
27
  * is a superuser, which reads pg_authid, whether it has CREATEROLE (`createrole`), and the roles it made; the
28
28
  * open transaction's own writes, comb's CREATE ROLEs', with the rows they left; the shared catalogs
29
- * as last read committed (`seen`), and the roles, databases and replication slots a refused window
30
- * added (`strays`), which the cleanup drops
29
+ * as last read committed (`seen`)
31
30
  */
32
31
  type Run = {
33
32
  admin: Sql
@@ -39,7 +38,6 @@ type Run = {
39
38
  role: string
40
39
  seen: Snapshot
41
40
  sql: Sql
42
- strays: Strays
43
41
  /** the tablespaces it left out the CREATE of, which a statement then runs in pg_default instead */
44
42
  tablespaces: Set<string>
45
43
  superuser: boolean
@@ -3,11 +3,10 @@
3
3
  * catalog query: Postgres, not a model of it, decides what a cascade, a rename or a
4
4
  * move leaves, and the two sides compare as one read of two databases.
5
5
  *
6
- * The replay runs as Atlas applies: each file on a session of its own, as apply
7
- * gives each file an Atlas call, and in a transaction of its own, a `--
8
- * atlas:txmode none` file statement by statement, each in a transaction of its own first and bare,
9
- * as Atlas sends it, where Postgres runs it only so, in a database
10
- * made for it and dropped after (pg-replay-database.ts). One transaction for the whole history
6
+ * The replay runs as Atlas applies: each file on a session of its own, as apply gives each file an
7
+ * Atlas call, and in a transaction of its own, a `-- atlas:txmode none` file statement by statement,
8
+ * each in a transaction of its own first and bare, as Atlas sends it, where Postgres runs it only
9
+ * so, in a database made for it and dropped after (pg-replay-database.ts). One transaction for the whole history
11
10
  * filled Postgres's lock table and left a seed's deferred trigger events pending, and devUrl's own
12
11
  * database lent the history what it held. Committed, a statement whose scope is the server or
13
12
  * another database would outlive the replay on devUrl's server, the target's own when they share
@@ -29,19 +28,19 @@
29
28
  * Bare runs only a statement a block refuses (an outside-only class, a DO block or CALL that
30
29
  * commits), one only a block runs (a LOCK, a SAVEPOINT, a cursor not held) and the file's own
31
30
  * block, and a change one makes past its database has committed when the read sees it: the refusal
32
- * names what devUrl's server keeps. The read is the server's, another session's work in it too, so
33
- * a change there is the file's only when the role, database or other object it is to is one the
34
- * file names: a name it spells, unquoted and folded as Postgres folds it, quoted and exact, or
35
- * inside a string a DO body, dblink_exec or format() may run (namesIn in sql-code.ts). That change
36
- * is refused, naming it, and a role it added is dropped with the cleanup, as is a database it added
37
- * that no session is in; a change to any other object is another session's and passes. A name the
38
- * file builds at run time, by concatenation, from a query, or in a function another file defines,
39
- * is none it spells, so comb neither refuses nor drops that change. A role's password reads from
40
- * pg_authid alone, so a password a bare statement changes is seen by a superuser's devUrl only.
31
+ * names what devUrl's server keeps. The read is the server's, another session's work in it too, and
32
+ * Postgres keeps no creator for a role or a database, so a change there is the file's only beside a
33
+ * statement that can run code comb cannot read (pg-replay-hidden.ts), and only when it is to an
34
+ * object the file names: a name it spells, unquoted and folded as Postgres folds it, quoted and
35
+ * exact, or inside a string a DO body, dblink_exec or format() may run (namesIn in sql-code.ts).
36
+ * That change is refused, naming it, and left, as another session may have made it; any other is
37
+ * another session's and passes. A name the file builds at run time, by concatenation, from a query,
38
+ * or in a function another file defines, is none it spells, so comb does not refuse that change. A
39
+ * role's password reads from pg_authid alone, so a bare statement's is seen by a superuser's only.
41
40
  *
42
- * A SIGINT or a SIGTERM, a Ctrl-C or a cancelled CI job, skips a `finally`, so either runs the
43
- * replay's cleanup first, once, and then ends the process as the signal would; a run
44
- * killed past it is swept by the next replay or apply on the server.
41
+ * A SIGINT or a SIGTERM, a Ctrl-C or a cancelled CI job, skips a `finally`, so either runs the replay's
42
+ * cleanup first, once, and then ends the process as the signal would; a run killed past it is swept
43
+ * by the next replay or apply on the server.
45
44
  */
46
45
  import { sqlIdentifier } from "../../codegen/utils/case.ts"
47
46
  import { type PgTarget, refuseUrlProblems } from "./atlas-target.ts"
@@ -65,6 +64,7 @@ import { prepareSnapshot, snapshot } from "./pg-server-snapshot.ts"
65
64
  import { refuseUncounted } from "./pg-server-writes.ts"
66
65
  import { pgRead } from "./pg-split.ts"
67
66
  import { namesIn } from "./sql-code.ts"
67
+ import { nothingHidden, readStatement } from "./pg-replay-hidden.ts"
68
68
 
69
69
  /** A history file as baseline reads it */
70
70
  type HistoryFile = { name: string; sql: string }
@@ -92,12 +92,14 @@ async function replayFile(run: Run, { name, sql: text }: HistoryFile, replay: Re
92
92
  const inFile = !outsideTransaction(text)
93
93
  const file = {
94
94
  adopted: (role: string) => replay.adopted?.(name, role),
95
+ hidden: nothingHidden(),
95
96
  name,
96
97
  names: namesIn(text),
97
98
  stops: replay.stops,
98
99
  }
99
100
  const statements = pgRead(text).statements.map(({ sent, tree }) => ({
100
101
  sent,
102
+ tree,
101
103
  ...(tree === undefined ? { node: {}, type: "" } : (([type, node]) => ({ node, type }))(nodeOf(tree))),
102
104
  }))
103
105
  const locks = lockReader(
@@ -110,13 +112,14 @@ async function replayFile(run: Run, { name, sql: text }: HistoryFile, replay: Re
110
112
  await begin(run)
111
113
  await locks.began()
112
114
  }
113
- for (const [index, { node, sent, type }] of statements.entries()) {
115
+ for (const [index, { node, sent, tree, type }] of statements.entries()) {
114
116
  const beyond = (await runsOnMade(run, type, node)) ? undefined : beyondDatabase(type, node)
115
117
  if (beyond !== undefined) {
116
118
  leaveOutTablespace(run.tablespaces, type, node)
117
119
  replay.leftOut(name, sent, beyond)
118
120
  continue
119
121
  }
122
+ readStatement(tree, file.hidden)
120
123
  await locks.next(index, type, node)
121
124
  const send = () => runStatement(run, { name, node, sent, type }, replay)
122
125
  if (inFile) await send()
@@ -206,7 +209,6 @@ async function opening(
206
209
  owner,
207
210
  seen: new Map(),
208
211
  sql: open(),
209
- strays: { databases: [], roles: [], slots: [] },
210
212
  tablespaces: new Set(),
211
213
  ...(await devRoleOf(admin)),
212
214
  }
@@ -2,8 +2,9 @@
2
2
  * The server's shared catalogs read as committed, in one query: what a bare
3
3
  * statement changed, which no transaction counts, and what a second connection committed, which this
4
4
  * backend's counts never see (pg-server-writes.ts). The read is the server's, another session's work
5
- * in it too, so each row carries the names of the objects it is about, and a change is a file's only
6
- * when one is a name the file spells (namesIn in sql-code.ts). A replay's own rows are left out, but
5
+ * in it too, so each row carries the names of the objects it is about: a change is a file's when it
6
+ * is to the replay's own database, or to a name the file spells (namesIn in sql-code.ts) beside a
7
+ * statement that can run code comb cannot read (pg-replay-hidden.ts). A replay's own rows are left out, but
7
8
  * for the fields of its own database's row that keep the cleanup from dropping it, which only that
8
9
  * replay changes. Beside the catalogs, the server's replication slots, which no catalog
9
10
  * holds.
@@ -142,27 +143,6 @@ function snapshotChanges(before: Snapshot, after: Snapshot, names?: ReadonlySet<
142
143
  return changes
143
144
  }
144
145
 
145
- /** What the cleanup drops, by the noun its subject starts with */
146
- type Strays = { databases: string[]; roles: string[]; slots: string[] }
147
-
148
- // lookup-table: the nouns of what a file adds that the cleanup drops, by where it keeps them
149
- const DROPPED: ReadonlyArray<[keyof Strays, string]> = [
150
- ["databases", "database "],
151
- ["roles", "role "],
152
- ["slots", "replication slot "],
153
- ]
154
-
155
- /** The roles, databases and replication slots named in `names` that `after` holds and `before` does not */
156
- function added(before: Snapshot, after: Snapshot, names: ReadonlySet<string>): Strays {
157
- const out: Strays = { databases: [], roles: [], slots: [] }
158
- for (const subject of after.keys()) {
159
- if (before.has(subject) || !about(subject, { after, before }, names)) continue
160
- const [where, noun] = DROPPED.find(([, prefix]) => subject.startsWith(prefix)) ?? []
161
- if (where !== undefined) out[where].push(subject.slice(noun?.length))
162
- }
163
- return out
164
- }
165
-
166
146
  /** The subscriptions `after` holds in `database` and `before` does not, by subject: its drop takes them */
167
147
  const subscriptionsIn = (before: Snapshot, after: Snapshot, database: string): string[] =>
168
148
  [...after]
@@ -171,4 +151,4 @@ const subscriptionsIn = (before: Snapshot, after: Snapshot, database: string): s
171
151
  )
172
152
  .map(([subject]) => subject)
173
153
 
174
- export { added, prepareSnapshot, type Snapshot, snapshot, snapshotChanges, type Strays, subscriptionsIn }
154
+ export { prepareSnapshot, type Snapshot, snapshot, snapshotChanges, subscriptionsIn }
@@ -0,0 +1,80 @@
1
+ /**
2
+ * What a file leaves unable to run: SQLite's DROP TABLE and DROP VIEW check no view or trigger that names what they
3
+ * drop, and SQLite, D1 and libsql resolve those names only when a statement runs. So a file can apply and leave every
4
+ * read of a view, or every write a trigger fires on, failing. After a file applies to the proof's copy, each view and
5
+ * trigger that survives it and names an object the file dropped or changed is compiled: a view's read, a trigger's
6
+ * own write on its table or view (trigger-write.mjs). SQLite resolves a name only by name, and the copy holds every
7
+ * reader that names a table a file reaches (proof-scope.ts), so this finds every reader the file breaks.
8
+ */
9
+ import type { LocalDb } from "../drivers/local-db.ts"
10
+ import type { LiveObject } from "./proof-copy.ts"
11
+ import { referencesOf } from "./proof-scope.ts"
12
+ import { asciiFold } from "./sql-code.ts"
13
+ import { quote, triggerEvent, writeOf } from "./trigger-write.mjs"
14
+
15
+ /** The copy's schema, each object by type and folded name */
16
+ function schemaOf(db: LocalDb): Map<string, LiveObject> {
17
+ const rows = db.all("SELECT type, name, sql, tbl_name FROM main.sqlite_master WHERE sql IS NOT NULL")
18
+ return new Map(
19
+ rows.map((row) => {
20
+ const object = {
21
+ name: String(row["name"]),
22
+ sql: String(row["sql"]),
23
+ table: String(row["tbl_name"]),
24
+ type: String(row["type"]),
25
+ }
26
+ return [`${object.type} ${asciiFold(object.name)}`, object]
27
+ }),
28
+ )
29
+ }
30
+
31
+ /** SQLite's error compiling `sql`, or undefined */
32
+ function compileError(db: LocalDb, sql: string): string | undefined {
33
+ try {
34
+ db.all(`EXPLAIN ${sql}`)
35
+ return undefined
36
+ } catch (error) {
37
+ // reason: what SQLite refuses to compile is the finding
38
+ return (error as Error).message
39
+ }
40
+ }
41
+
42
+ /** Why `reader`, a view or a trigger, fails on `db`, or undefined */
43
+ function brokenRead(db: LocalDb, reader: LiveObject): string | undefined {
44
+ if (reader.type === "view") {
45
+ const error = compileError(db, `SELECT * FROM ${quote(reader.name)}`)
46
+ return error === undefined ? undefined : `view ${reader.name} cannot be read: ${error}`
47
+ }
48
+ const event = triggerEvent(reader.sql)
49
+ const columns = db
50
+ .all(`PRAGMA main.table_xinfo(${quote(reader.table)})`)
51
+ .filter((row) => row["hidden"] === 0)
52
+ .map((row) => String(row["name"]))
53
+ const error = compileError(db, writeOf(reader.table, event, columns))
54
+ return error === undefined
55
+ ? undefined
56
+ : `every ${event} on ${reader.table} fails, as a trigger it fires cannot compile: ${error}`
57
+ }
58
+
59
+ /**
60
+ * Throws when a file that turned the copy's schema `before` into `db`'s leaves a view or trigger unable to run,
61
+ * naming it and SQLite's error: no approval overrides, since every read or write through it would fail
62
+ */
63
+ function refuseBrokenReaders(db: LocalDb, before: ReadonlyMap<string, LiveObject>): void {
64
+ const after = schemaOf(db)
65
+ const touched = new Set(
66
+ [...before]
67
+ .filter(([key, object]) => after.get(key)?.sql !== object.sql)
68
+ .map(([, object]) => asciiFold(object.name)),
69
+ )
70
+ if (touched.size === 0) return
71
+ for (const reader of after.values()) {
72
+ if (reader.type !== "view" && reader.type !== "trigger") continue
73
+ if (![...referencesOf(reader)].some((name) => touched.has(name))) continue
74
+ const broken = brokenRead(db, reader)
75
+ if (broken !== undefined)
76
+ throw new Error(`after this file, ${broken}; drop or change it in this file, or keep what it uses`)
77
+ }
78
+ }
79
+
80
+ export { refuseBrokenReaders, schemaOf }
@@ -12,7 +12,8 @@
12
12
  * Seeded rows cannot meet every predicate a file holds, so what each statement writes is read from
13
13
  * SQLite itself as it runs (proof-engine.ts): an update of a column that was there before, a
14
14
  * delete, a trigger fired or a temp shadow is unsafe whichever rows it meets, and a backfill of the
15
- * file's own columns is cautious. A file the copy cannot run is unproven, and unproven is unsafe.
15
+ * file's own columns is cautious. A file the copy cannot run is unproven, and unproven is unsafe. A
16
+ * file that leaves a view or trigger unable to run is refused outright (proof-readers.ts).
16
17
  */
17
18
  import { isMirror, mirrorFindings, mirrorRows } from "./proof-mirror.ts"
18
19
  import { type LocalDb, openScratch, type ScratchEngine } from "../drivers/local-db.ts"
@@ -21,6 +22,7 @@ import { notEngineTables } from "./engine-tables.ts"
21
22
  import { compare, type Shape } from "./proof-compare.ts"
22
23
  import { type Copy, copyOf, buildCopy, creatable, jsonColumns, type LiveObject, liveObjects } from "./proof-copy.ts"
23
24
  import { applyToCopy, engineFindings } from "./proof-engine.ts"
25
+ import { refuseBrokenReaders, schemaOf } from "./proof-readers.ts"
24
26
  import { quote } from "./proof-seed.ts"
25
27
  import { observable } from "./proof-scope.ts"
26
28
  import { asciiFold } from "./sql-code.ts"
@@ -174,7 +176,10 @@ function snapshotAfter(db: Copy["db"]): Map<string, Shape> | string {
174
176
  }
175
177
  }
176
178
 
177
- /** The proof of a file that applied as `run`, from the copy's tables and mirrored rows before it and its tables after */
179
+ /**
180
+ * The proof of file `name`, which applied as `run`, from the copy's tables, mirrored rows and schema before it and its
181
+ * tables after; throws, naming the file, when it leaves a view or trigger unable to run, which no approval overrides
182
+ */
178
183
  function appliedProof(
179
184
  copy: Copy,
180
185
  run: ReturnType<typeof applyToCopy>,
@@ -182,8 +187,17 @@ function appliedProof(
182
187
  after,
183
188
  before,
184
189
  mirrors,
185
- }: { after: Map<string, Shape>; before: Map<string, Shape>; mirrors: ReturnType<typeof mirrorRows> },
190
+ name,
191
+ schema,
192
+ }: {
193
+ after: Map<string, Shape>
194
+ before: Map<string, Shape>
195
+ mirrors: ReturnType<typeof mirrorRows>
196
+ name: string
197
+ schema: ReturnType<typeof schemaOf>
198
+ },
186
199
  ): Proof {
200
+ named(name, () => refuseBrokenReaders(copy.db, schema))
187
201
  const found = compare(copy.db, new Map([...before].filter(([table]) => !copy.unproven.has(table))), after)
188
202
  const steps = engineFindings(run.steps, run.start)
189
203
  return {
@@ -228,7 +242,7 @@ async function provePending(
228
242
  proofs.set(name, failedProof(copy, failed))
229
243
  continue
230
244
  }
231
- let [before, mirrors] = [snapshot(copy.db), mirrorRows(copy.db)]
245
+ let [before, mirrors, schema] = [snapshot(copy.db), mirrorRows(copy.db), schemaOf(copy.db)]
232
246
  let run = named(name, () => applyToCopy(copy.db, sql))
233
247
  const retry = run.error === undefined ? undefined : await reseeded(objects, applied, making)
234
248
  if (retry !== undefined) {
@@ -263,7 +277,7 @@ async function provePending(
263
277
  failed = `an earlier pending file, ${name}, left a table SQLite cannot read, so this one is unproven`
264
278
  continue
265
279
  }
266
- proofs.set(name, appliedProof(copy, run, { after, before, mirrors }))
280
+ proofs.set(name, appliedProof(copy, run, { after, before, mirrors, name, schema }))
267
281
  }
268
282
  } finally {
269
283
  copy.db.close()