@crombolo/data 0.9.1 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/package.json +2 -2
  2. package/src/audit.ts +291 -0
  3. package/src/bundle-tables/builder.ts +264 -0
  4. package/src/bundle-tables/capability.ts +377 -0
  5. package/src/bundle-tables/chain.ts +105 -0
  6. package/src/bundle-tables/ddl.ts +124 -0
  7. package/src/bundle-tables/guard.ts +116 -0
  8. package/src/bundle-tables/index.ts +7 -0
  9. package/src/bundle-tables/names.ts +58 -0
  10. package/src/bundle-tables/schema.ts +485 -0
  11. package/src/cluster.ts +1 -1
  12. package/src/database.ts +39 -9
  13. package/src/dialect.ts +32 -0
  14. package/src/index.ts +9 -0
  15. package/src/migrations.ts +147 -31
  16. package/src/pending.ts +1 -1
  17. package/src/repositories/bundles.ts +36 -4
  18. package/src/repositories/content-types.ts +41 -4
  19. package/src/repositories/documents.ts +13 -3
  20. package/src/repositories/form-entries.ts +1 -1
  21. package/src/repositories/form-workflows.ts +1 -1
  22. package/src/repositories/members.ts +15 -1
  23. package/src/repositories/nodes.ts +32 -7
  24. package/src/repositories/transfer-runs.ts +20 -2
  25. package/src/schema/audit-log.ts +45 -0
  26. package/src/schema/bundle-components.ts +1 -1
  27. package/src/schema/bundle-package.ts +27 -0
  28. package/src/schema/bundles-rename.ts +1 -1
  29. package/src/schema/content.ts +2 -2
  30. package/src/schema/created-packages.ts +1 -1
  31. package/src/schema/form-permissions.ts +1 -1
  32. package/src/schema/form-workflows.ts +1 -1
  33. package/src/schema/forms.ts +1 -1
  34. package/src/schema/identity.ts +1 -1
  35. package/src/schema/plan.ts +10 -0
  36. package/src/schema/property-searchable.ts +29 -0
  37. package/src/schema/search-index.ts +92 -0
  38. package/src/schema/transfer-run-package.ts +27 -0
  39. package/src/schema/upgrade-report.ts +1 -1
  40. package/src/schema-state.ts +1 -1
  41. package/src/search-text.ts +133 -0
  42. package/src/search.ts +397 -0
package/src/database.ts CHANGED
@@ -1,5 +1,7 @@
1
1
  /** Driver-agnostic database handle. */
2
+
2
3
  import { Database as BunSqlite } from 'bun:sqlite'
4
+ import { AsyncLocalStorage } from 'node:async_hooks'
3
5
  import { type Dialect, type DialectName, dialectFor } from './dialect.ts'
4
6
  import { createLockManager, type LockManager } from './locks.ts'
5
7
 
@@ -22,16 +24,32 @@ export interface ConnectOptions {
22
24
  url?: string
23
25
  }
24
26
 
27
+ /** The connection the current async chain holds a transaction on. */
28
+ const transacting = new AsyncLocalStorage<BunSqlite>()
29
+
30
+ /**
31
+ * One connection has one transaction at a time, but async chains interleave:
32
+ * a background job's BEGIN between a request's BEGIN and COMMIT would fail, or
33
+ * worse, land its statements in the request's transaction. So top-level
34
+ * transactions queue here, and a chain already inside one joins it.
35
+ */
25
36
  class SqliteDb implements Db {
26
37
  readonly dialect = dialectFor('sqlite')
27
38
  readonly locks: LockManager
28
39
  #db: BunSqlite
29
40
  #inTransaction: boolean
30
-
31
- constructor(db: BunSqlite, locks?: LockManager, inTransaction = false) {
41
+ #gate: { tail: Promise<void> }
42
+
43
+ constructor(
44
+ db: BunSqlite,
45
+ locks?: LockManager,
46
+ inTransaction = false,
47
+ gate: { tail: Promise<void> } = { tail: Promise.resolve() },
48
+ ) {
32
49
  this.#db = db
33
50
  this.locks = locks ?? createLockManager(this)
34
51
  this.#inTransaction = inTransaction
52
+ this.#gate = gate
35
53
  }
36
54
 
37
55
  async query<T = Record<string, unknown>>(sql: string, params: unknown[] = []): Promise<T[]> {
@@ -44,14 +62,26 @@ class SqliteDb implements Db {
44
62
 
45
63
  async transaction<T>(fn: (tx: Db) => Promise<T>): Promise<T> {
46
64
  if (this.#inTransaction) return fn(this)
47
- this.#db.exec('BEGIN')
65
+ const tx = new SqliteDb(this.#db, this.locks, true, this.#gate)
66
+ if (transacting.getStore() === this.#db) return fn(tx)
67
+ const previous = this.#gate.tail
68
+ let release!: () => void
69
+ this.#gate.tail = new Promise<void>((resolve) => {
70
+ release = resolve
71
+ })
72
+ await previous
48
73
  try {
49
- const result = await fn(new SqliteDb(this.#db, this.locks, true))
50
- this.#db.exec('COMMIT')
51
- return result
52
- } catch (error) {
53
- this.#db.exec('ROLLBACK')
54
- throw error
74
+ this.#db.exec('BEGIN')
75
+ try {
76
+ const result = await transacting.run(this.#db, () => fn(tx))
77
+ this.#db.exec('COMMIT')
78
+ return result
79
+ } catch (error) {
80
+ this.#db.exec('ROLLBACK')
81
+ throw error
82
+ }
83
+ } finally {
84
+ release()
55
85
  }
56
86
  }
57
87
 
package/src/dialect.ts CHANGED
@@ -15,6 +15,10 @@ export interface ColumnTypes {
15
15
  textCi: string
16
16
  text: string
17
17
  varchar: (n: number) => string
18
+ /** A varchar that compares case-sensitively on both dialects, as a bundle's tables do. */
19
+ varcharExact: (n: number) => string
20
+ /** JSON, held as text and parsed by whoever reads it. */
21
+ json: string
18
22
  timestamp: string
19
23
  numeric: string
20
24
  blob: string
@@ -32,6 +36,16 @@ export interface Dialect {
32
36
  /** Appended to a SELECT that must block a concurrent state change; empty where the engine is single-writer. */
33
37
  readonly forShare: string
34
38
  readonly forUpdate: string
39
+ /** Appended to a SELECT that claims rows, passing over any another connection holds. */
40
+ readonly forUpdateSkipLocked: string
41
+ /**
42
+ * The full-text match over `search_document`: a subquery of `node_id` and
43
+ * `score`, higher being better, for each item holding every term as a word
44
+ * prefix. Its one parameter is `fullTextQuery(terms)`. docs/search.md.
45
+ */
46
+ readonly fullTextHits: string
47
+ /** Terms are already folded words: letters and digits only. */
48
+ fullTextQuery(terms: readonly string[]): string
35
49
  /**
36
50
  * SQL that moves a table's identity past rows inserted with explicit ids, or
37
51
  * undefined where the engine does that itself.
@@ -53,6 +67,8 @@ export const sqliteDialect: Dialect = {
53
67
  textCi: 'TEXT COLLATE NOCASE',
54
68
  text: 'TEXT',
55
69
  varchar: (n) => `VARCHAR(${n}) COLLATE NOCASE`,
70
+ varcharExact: (n) => `VARCHAR(${n})`,
71
+ json: 'TEXT',
56
72
  timestamp: 'TEXT',
57
73
  // REAL would be lossy for decimal property values.
58
74
  numeric: 'TEXT',
@@ -65,6 +81,15 @@ export const sqliteDialect: Dialect = {
65
81
  boolValue: (value) => (value ? 1 : 0),
66
82
  forShare: '',
67
83
  forUpdate: '',
84
+ forUpdateSkipLocked: '',
85
+ // Name hits weigh ten times body hits; bm25 is lower-is-better, so negated.
86
+ // bm25 cannot be called under an aggregate, and `LIMIT -1` (no limit) keeps
87
+ // SQLite from flattening the inner query into the outer one, which would.
88
+ fullTextHits: `SELECT sd.node_id AS node_id, MAX(f.score) AS score
89
+ FROM (SELECT rowid AS id, -bm25(search_document_fts, 10.0, 1.0) AS score
90
+ FROM search_document_fts WHERE search_document_fts MATCH ? LIMIT -1) f
91
+ JOIN search_document sd ON sd.id = f.id GROUP BY sd.node_id`,
92
+ fullTextQuery: (terms) => terms.map((term) => `"${term}"*`).join(' AND '),
68
93
  syncIdentity: () => undefined,
69
94
  }
70
95
 
@@ -80,6 +105,8 @@ export const postgresDialect: Dialect = {
80
105
  textCi: 'CITEXT',
81
106
  text: 'TEXT',
82
107
  varchar: (n) => `VARCHAR(${n})`,
108
+ varcharExact: (n) => `VARCHAR(${n})`,
109
+ json: 'TEXT',
83
110
  timestamp: 'TIMESTAMPTZ',
84
111
  numeric: 'NUMERIC',
85
112
  blob: 'BYTEA',
@@ -94,6 +121,11 @@ export const postgresDialect: Dialect = {
94
121
  boolValue: (value) => value,
95
122
  forShare: 'FOR SHARE',
96
123
  forUpdate: 'FOR UPDATE',
124
+ forUpdateSkipLocked: 'FOR UPDATE SKIP LOCKED',
125
+ fullTextHits: `SELECT sd.node_id AS node_id, MAX(ts_rank(sd.terms, q)) AS score
126
+ FROM search_document sd, to_tsquery('simple', ?) q
127
+ WHERE sd.terms @@ q GROUP BY sd.node_id`,
128
+ fullTextQuery: (terms) => terms.map((term) => `${term}:*`).join(' & '),
97
129
  syncIdentity: (table) =>
98
130
  `SELECT setval(pg_get_serial_sequence('${table}', 'id'), (SELECT COALESCE(MAX(id), 1) FROM ${table}))`,
99
131
  }
package/src/index.ts CHANGED
@@ -1,3 +1,5 @@
1
+ export * from './audit.ts'
2
+ export * from './bundle-tables/index.ts'
1
3
  export * from './change-report.ts'
2
4
  export * from './cluster.ts'
3
5
  export * from './database.ts'
@@ -30,9 +32,11 @@ export * from './repositories/transfer-runs.ts'
30
32
  export * from './repositories/user-data.ts'
31
33
  export * from './repositories/users.ts'
32
34
  export * from './schema/assistant.ts'
35
+ export * from './schema/audit-log.ts'
33
36
  export * from './schema/built-in-content-types.ts'
34
37
  export * from './schema/built-in-media-types.ts'
35
38
  export * from './schema/built-in-member-types.ts'
39
+ export * from './schema/bundle-package.ts'
36
40
  export * from './schema/bundles-rename.ts'
37
41
  export * from './schema/change-report.ts'
38
42
  export * from './schema/client-id-prefix.ts'
@@ -47,14 +51,19 @@ export * from './schema/log-viewer.ts'
47
51
  export * from './schema/member-property-type.ts'
48
52
  export * from './schema/members.ts'
49
53
  export * from './schema/plan.ts'
54
+ export * from './schema/property-searchable.ts'
50
55
  export * from './schema/public-access.ts'
51
56
  export * from './schema/recycle-bin.ts'
52
57
  export * from './schema/rich-text-ui.ts'
58
+ export * from './schema/search-index.ts'
53
59
  export * from './schema/seed.ts'
54
60
  export * from './schema/since-version.ts'
61
+ export * from './schema/transfer-run-package.ts'
55
62
  export * from './schema/transfer-runs.ts'
56
63
  export * from './schema/upgrade-report.ts'
57
64
  export * from './schema/users-localisation.ts'
58
65
  export * from './schema/values.ts'
59
66
  export * from './schema-ops.ts'
60
67
  export * from './schema-state.ts'
68
+ export * from './search.ts'
69
+ export * from './search-text.ts'
package/src/migrations.ts CHANGED
@@ -3,7 +3,7 @@
3
3
  * version number and not a list of applied migrations: the plan is a chain of
4
4
  * transitions walked forward to its final state. That design handles concurrent
5
5
  * development branches far better than sequence numbers, so we keep it — and
6
- * harden it (docs/10-packaging-and-upgrades.md): a ledger of every step that
6
+ * harden it (docs/packaging-and-upgrades.md): a ledger of every step that
7
7
  * ran, expand/contract declared and linted, a recorded plan, and an explicit
8
8
  * refusal when the database is newer than the code.
9
9
  */
@@ -29,7 +29,24 @@ export interface Migration {
29
29
  release?: string
30
30
  /** For a contract: the name of the expand migration it removes. */
31
31
  contracts?: string
32
+ /** Recorded in the ledger, and compared when the plan verifies what ran. Defaults to `name`. */
33
+ checksum?: string
32
34
  up(db: Db): Promise<void>
35
+ /**
36
+ * Writes data once `up` has run, in the same transaction. `planUpgrade` cannot
37
+ * predict what it writes, so a step that has one is reported as unplannable.
38
+ */
39
+ backfill?(db: Db): Promise<void>
40
+ }
41
+
42
+ /** Where a chain keeps its state, and what it calls itself in the ledger. */
43
+ export interface ChainOptions {
44
+ /** The `key_value` key holding the chain's state. */
45
+ stateKey?: string
46
+ /** The state before the first step. */
47
+ initialState?: string
48
+ /** A bundle's id, for a bundle's chain; absent for core's. */
49
+ bundle?: string
33
50
  }
34
51
 
35
52
  function compareRelease(a: string, b: string): number {
@@ -71,10 +88,16 @@ export function isMajorRelease(release: string): boolean {
71
88
 
72
89
  export class MigrationPlan {
73
90
  readonly migrations: readonly Migration[]
91
+ readonly stateKey: string
92
+ readonly initialState: string
93
+ readonly bundle: string | undefined
74
94
 
75
- constructor(migrations: readonly Migration[]) {
95
+ constructor(migrations: readonly Migration[], options: ChainOptions = {}) {
76
96
  this.migrations = migrations
77
- let expected = INITIAL_STATE
97
+ this.stateKey = options.stateKey ?? MIGRATION_STATE_KEY
98
+ this.initialState = options.initialState ?? INITIAL_STATE
99
+ this.bundle = options.bundle
100
+ let expected = this.initialState
78
101
  const seen = new Map<string, Migration>()
79
102
  for (const migration of migrations) {
80
103
  if (migration.from !== expected) {
@@ -111,7 +134,14 @@ export class MigrationPlan {
111
134
  }
112
135
 
113
136
  get finalState(): string {
114
- return this.migrations.at(-1)?.to ?? INITIAL_STATE
137
+ return this.migrations.at(-1)?.to ?? this.initialState
138
+ }
139
+
140
+ /** The steps that have run when the chain is at a state; undefined when the state is not in it. */
141
+ appliedAt(state: string): Migration[] | undefined {
142
+ if (state === this.initialState) return []
143
+ const index = this.migrations.findIndex((m) => m.to === state)
144
+ return index < 0 ? undefined : this.migrations.slice(0, index + 1)
115
145
  }
116
146
 
117
147
  /** The steps still to run from a state; empty when current. Undefined when the state is not in the plan. */
@@ -123,7 +153,11 @@ export class MigrationPlan {
123
153
  }
124
154
  }
125
155
 
156
+ /** Connections whose ledger tables are known to exist with every column. */
157
+ const ensured = new WeakSet<Db>()
158
+
126
159
  async function ensureTables(db: Db): Promise<void> {
160
+ if (ensured.has(db)) return
127
161
  const t = db.dialect.types
128
162
  await db.exec(
129
163
  `CREATE TABLE IF NOT EXISTS key_value (
@@ -143,36 +177,62 @@ async function ensureTables(db: Db): Promise<void> {
143
177
  duration_ms ${t.integer} NOT NULL,
144
178
  checksum ${t.varchar(64)},
145
179
  applied_by ${t.varchar(255)},
146
- note ${t.text}
180
+ note ${t.text},
181
+ bundle ${t.varchar(64)}
147
182
  )`,
148
183
  )
184
+ // A ledger from before bundles could own tables has no `bundle` column. It is
185
+ // added here, where the table is, rather than by a core migration: the ledger
186
+ // has to be able to record a bundle's step whatever state core is at.
187
+ if (!(await hasColumn(db, 'migration_history', 'bundle')))
188
+ await db.exec(`ALTER TABLE migration_history ADD COLUMN bundle ${t.varchar(64)}`)
189
+ ensured.add(db)
190
+ }
191
+
192
+ async function hasColumn(db: Db, table: string, column: string): Promise<boolean> {
193
+ const rows =
194
+ db.dialect.name === 'sqlite'
195
+ ? await db.query<{ name: string }>(`SELECT name FROM pragma_table_info('${table}')`)
196
+ : await db.query<{ name: string }>(
197
+ 'SELECT column_name AS name FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = ?',
198
+ [table],
199
+ )
200
+ return rows.some((row) => row.name === column)
149
201
  }
150
202
 
151
- export async function readState(db: Db): Promise<string> {
203
+ /** A chain's state: core's unless a plan says otherwise. */
204
+ export async function readState(
205
+ db: Db,
206
+ chain: Pick<MigrationPlan, 'stateKey' | 'initialState'> = {
207
+ stateKey: MIGRATION_STATE_KEY,
208
+ initialState: INITIAL_STATE,
209
+ },
210
+ ): Promise<string> {
152
211
  await ensureTables(db)
153
212
  const rows = await db.query<{ value: string }>('SELECT value FROM key_value WHERE key = ?', [
154
- MIGRATION_STATE_KEY,
213
+ chain.stateKey,
155
214
  ])
156
- return rows[0]?.value ?? INITIAL_STATE
215
+ return rows[0]?.value ?? chain.initialState
157
216
  }
158
217
 
159
- async function writeState(db: Db, state: string): Promise<void> {
218
+ async function writeState(db: Db, state: string, key = MIGRATION_STATE_KEY): Promise<void> {
160
219
  const now = db.dialect.now()
161
- await db.query(`UPDATE key_value SET value = ?, update_date = ${now} WHERE key = ?`, [
162
- state,
163
- MIGRATION_STATE_KEY,
164
- ])
165
- const rows = await db.query<{ value: string }>('SELECT value FROM key_value WHERE key = ?', [
166
- MIGRATION_STATE_KEY,
167
- ])
220
+ await db.query(`UPDATE key_value SET value = ?, update_date = ${now} WHERE key = ?`, [state, key])
221
+ const rows = await db.query<{ value: string }>('SELECT value FROM key_value WHERE key = ?', [key])
168
222
  if (rows.length === 0) {
169
223
  await db.exec(
170
224
  `INSERT INTO key_value (key, value, update_date) VALUES (${db.dialect.placeholder(1)}, ${db.dialect.placeholder(2)}, ${now})`,
171
- [MIGRATION_STATE_KEY, state],
225
+ [key, state],
172
226
  )
173
227
  }
174
228
  }
175
229
 
230
+ /** Forgets a setting entirely, as a purged bundle's state is forgotten. */
231
+ export async function deleteKeyValue(db: Db, key: string): Promise<void> {
232
+ await ensureTables(db)
233
+ await db.exec('DELETE FROM key_value WHERE key = ?', [key])
234
+ }
235
+
176
236
  export interface LedgerEntry {
177
237
  name: string
178
238
  /** 'expand' | 'contract' for framework steps; 'value' for a value migration; 'upgrade' for a cut-over. */
@@ -183,6 +243,8 @@ export interface LedgerEntry {
183
243
  checksum?: string | null
184
244
  appliedBy?: string | null
185
245
  note?: string | null
246
+ /** The bundle whose chain this step belongs to; null for core. */
247
+ bundle?: string | null
186
248
  }
187
249
 
188
250
  /**
@@ -226,8 +288,8 @@ export async function ensureKeyValue(db: Db, key: string, create: () => string):
226
288
  export async function recordInLedger(db: Db, entry: LedgerEntry): Promise<void> {
227
289
  await ensureTables(db)
228
290
  await db.exec(
229
- `INSERT INTO migration_history (name, kind, from_state, to_state, applied_at, duration_ms, checksum, applied_by, note)
230
- VALUES (?, ?, ?, ?, ${db.dialect.now()}, ?, ?, ?, ?)`,
291
+ `INSERT INTO migration_history (name, kind, from_state, to_state, applied_at, duration_ms, checksum, applied_by, note, bundle)
292
+ VALUES (?, ?, ?, ?, ${db.dialect.now()}, ?, ?, ?, ?, ?)`,
231
293
  [
232
294
  entry.name,
233
295
  entry.kind,
@@ -237,6 +299,7 @@ export async function recordInLedger(db: Db, entry: LedgerEntry): Promise<void>
237
299
  entry.checksum ?? null,
238
300
  entry.appliedBy ?? null,
239
301
  entry.note ?? null,
302
+ entry.bundle ?? null,
240
303
  ],
241
304
  )
242
305
  }
@@ -248,10 +311,14 @@ export async function latestLedgerId(db: Db): Promise<number> {
248
311
  return Number(rows[0]?.n ?? 0)
249
312
  }
250
313
 
251
- /** How many contract steps — framework contracts and site purges — were ledgered after a row id. */
314
+ /**
315
+ * How many contract steps — framework contracts and site purges — were ledgered
316
+ * after a row id. A bundle's contract is not counted: it removes only the
317
+ * bundle's own tables, which no read of core's depends on.
318
+ */
252
319
  export async function contractsLedgeredAfter(db: Db, afterId: number): Promise<number> {
253
320
  const rows = await db.query<{ n: number }>(
254
- "SELECT COUNT(*) AS n FROM migration_history WHERE id > ? AND kind = 'contract'",
321
+ "SELECT COUNT(*) AS n FROM migration_history WHERE id > ? AND kind = 'contract' AND bundle IS NULL",
255
322
  [afterId],
256
323
  )
257
324
  return Number(rows[0]?.n ?? 0)
@@ -265,7 +332,7 @@ export interface LedgerRow extends LedgerEntry {
265
332
  export async function readLedger(db: Db): Promise<LedgerRow[]> {
266
333
  await ensureTables(db)
267
334
  const rows = await db.query(
268
- 'SELECT id, name, kind, from_state, to_state, applied_at, duration_ms, checksum, applied_by, note FROM migration_history ORDER BY id',
335
+ 'SELECT id, name, kind, from_state, to_state, applied_at, duration_ms, checksum, applied_by, note, bundle FROM migration_history ORDER BY id',
269
336
  )
270
337
  return rows.map((row) => ({
271
338
  id: Number(row.id),
@@ -278,6 +345,7 @@ export async function readLedger(db: Db): Promise<LedgerRow[]> {
278
345
  checksum: (row.checksum as string | null) ?? null,
279
346
  appliedBy: (row.applied_by as string | null) ?? null,
280
347
  note: (row.note as string | null) ?? null,
348
+ bundle: (row.bundle as string | null) ?? null,
281
349
  }))
282
350
  }
283
351
 
@@ -286,22 +354,66 @@ export class UpgradeStateError extends Error {
286
354
  constructor(
287
355
  readonly databaseState: string,
288
356
  readonly codeState: string,
357
+ /** The bundle whose chain it is, when it is not core's. */
358
+ readonly bundle?: string,
359
+ ) {
360
+ super(
361
+ bundle
362
+ ? `The database holds the bundle "${bundle}"'s tables at state ${databaseState}, which this version of the bundle (its steps end at ${codeState}) does not know. ` +
363
+ 'A newer release of the bundle wrote them; install that release, or restore the backup taken before the upgrade.'
364
+ : `The database is at migration state ${databaseState}, which this version of crombolo (plan ends at ${codeState}) does not know. ` +
365
+ 'It was upgraded by a newer version; run that version, or restore the backup taken before the upgrade.',
366
+ )
367
+ }
368
+ }
369
+
370
+ /** A bundle step changed after it ran here: what is in the database is not what the code says made it. */
371
+ export class BundleStepEditedError extends Error {
372
+ constructor(
373
+ readonly bundle: string,
374
+ readonly step: string,
289
375
  ) {
290
376
  super(
291
- `The database is at migration state ${databaseState}, which this version of crombolo (plan ends at ${codeState}) does not know. ` +
292
- 'It was upgraded by a newer version; run that version, or restore the backup taken before the upgrade.',
377
+ `The bundle "${bundle}"'s step "${step}" is not the step that ran on this database: it was edited after it shipped. ` +
378
+ 'A shipped step is never changed; put the change in a new step.',
293
379
  )
294
380
  }
295
381
  }
296
382
 
297
383
  /** Steps pending on this database, in order. Throws when the database is ahead of the code. */
298
384
  export async function pendingMigrations(db: Db, plan: MigrationPlan): Promise<Migration[]> {
299
- const state = await readState(db)
385
+ const state = await readState(db, plan)
300
386
  const pending = plan.pendingFrom(state)
301
- if (!pending) throw new UpgradeStateError(state, plan.finalState)
387
+ if (!pending) throw new UpgradeStateError(state, plan.finalState, plan.bundle)
302
388
  return pending
303
389
  }
304
390
 
391
+ /**
392
+ * Refuses a bundle chain whose applied steps no longer match the code. Only the
393
+ * rows since the bundle was last purged count: a purge starts its chain again.
394
+ */
395
+ async function verifyApplied(db: Db, plan: MigrationPlan, state: string): Promise<void> {
396
+ if (!plan.bundle) return
397
+ const applied = plan.appliedAt(state) ?? []
398
+ if (applied.length === 0) return
399
+ const rows = await db.query<{ name: string; kind: string; checksum: string | null }>(
400
+ 'SELECT name, kind, checksum FROM migration_history WHERE bundle = ? ORDER BY id',
401
+ [plan.bundle],
402
+ )
403
+ const purged = rows.map((row) => row.kind).lastIndexOf('purge')
404
+ const ran = new Map(
405
+ rows
406
+ .slice(purged + 1)
407
+ .filter((row) => row.kind === 'expand' || row.kind === 'contract')
408
+ .map((row) => [row.name, row.checksum]),
409
+ )
410
+ for (const step of applied) {
411
+ const recorded = ran.get(step.name)
412
+ if (recorded !== undefined && recorded !== (step.checksum ?? step.name))
413
+ throw new BundleStepEditedError(plan.bundle, step.name)
414
+ }
415
+ }
416
+
305
417
  export interface MigrateResult {
306
418
  from: string
307
419
  to: string
@@ -320,27 +432,30 @@ export async function migrate(
320
432
  options: MigrateOptions = {},
321
433
  ): Promise<MigrateResult> {
322
434
  return db.locks.withLock(Locks.KeyValues, async () => {
323
- const from = await readState(db)
435
+ const from = await readState(db, plan)
324
436
  let state = from
325
437
  const applied: string[] = []
326
438
  const pending = plan.pendingFrom(state)
327
- if (!pending) throw new UpgradeStateError(state, plan.finalState)
439
+ if (!pending) throw new UpgradeStateError(state, plan.finalState, plan.bundle)
440
+ await verifyApplied(db, plan, state)
328
441
  for (const next of pending) {
329
442
  const started = performance.now()
330
443
  await db.transaction(async (tx) => {
331
444
  await next.up(tx)
445
+ await next.backfill?.(tx)
332
446
  })
333
447
  state = next.to
334
- await writeState(db, state)
448
+ await writeState(db, state, plan.stateKey)
335
449
  await recordInLedger(db, {
336
450
  name: next.name,
337
451
  kind: next.kind ?? 'expand',
338
452
  fromState: next.from,
339
453
  toState: next.to,
340
454
  durationMs: Math.round(performance.now() - started),
341
- checksum: next.name,
455
+ checksum: next.checksum ?? next.name,
342
456
  appliedBy: options.by ?? null,
343
457
  note: next.release ? `release ${next.release}` : null,
458
+ bundle: plan.bundle ?? null,
344
459
  })
345
460
  applied.push(next.name)
346
461
  }
@@ -351,7 +466,7 @@ export async function migrate(
351
466
  /** A fresh install stamps the final state so no migrations then run. */
352
467
  export async function stampFinalState(db: Db, plan: MigrationPlan): Promise<void> {
353
468
  await ensureTables(db)
354
- await writeState(db, plan.finalState)
469
+ await writeState(db, plan.finalState, plan.stateKey)
355
470
  }
356
471
 
357
472
  // ---------------------------------------------------------------- the plan
@@ -421,6 +536,7 @@ export async function planUpgrade(db: Db, plan: MigrationPlan): Promise<PlannedS
421
536
  try {
422
537
  await migration.up(recorder)
423
538
  step.statements = recorder.statements
539
+ if (migration.backfill) step.unplannable = 'backfills data, which only running it can show'
424
540
  } catch (error) {
425
541
  if (error instanceof UnplannableError) step.unplannable = error.message
426
542
  else throw error
package/src/pending.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * An element is pending on a node when the deployment that created it is newer
3
3
  * than the node, or its file says `since` a version the node has not reached.
4
- * The one place the rule lives. docs/09-schema-as-code.md, "since".
4
+ * The one place the rule lives. docs/schema-as-code.md, "since".
5
5
  */
6
6
  import { compareStates, compareVersions, type NodeSchemaState } from './schema-state.ts'
7
7
 
@@ -1,6 +1,6 @@
1
1
  /**
2
2
  * The bundle definitions: what a bundle would contain, not a built bundle.
3
- * `docs/17-bundles.md`.
3
+ * `docs/bundles.md`.
4
4
  *
5
5
  * The selection lists travel as JSON text in single columns. A malformed list
6
6
  * reads back as empty rather than throwing: a definition whose row was edited
@@ -26,12 +26,21 @@ export interface Bundle {
26
26
  scripts: string[]
27
27
  languages: string[]
28
28
  dictionaryItems: string[]
29
+ /**
30
+ * The npm package the bundle is built as. Not in Umbraco's definition, so the
31
+ * contract never writes these; null until someone sets them.
32
+ */
33
+ packageName: string | null
34
+ packageVersion: string | null
29
35
  createDate: Date
30
36
  updateDate: Date
31
37
  }
32
38
 
33
39
  /** A definition to store; `id` is assigned when it is new, the dates always. */
34
- export type BundleInput = Omit<Bundle, 'id' | 'createDate' | 'updateDate'>
40
+ export type BundleInput = Omit<
41
+ Bundle,
42
+ 'id' | 'createDate' | 'updateDate' | 'packageName' | 'packageVersion'
43
+ >
35
44
 
36
45
  interface Row {
37
46
  id: string
@@ -49,6 +58,8 @@ interface Row {
49
58
  scripts: string
50
59
  languages: string
51
60
  dictionary_items: string
61
+ package_name: string | null
62
+ package_version: string | null
52
63
  create_date: unknown
53
64
  update_date: unknown
54
65
  }
@@ -56,7 +67,8 @@ interface Row {
56
67
  const SELECT = `SELECT id, name, content_node_id, content_load_child_nodes,
57
68
  media_load_child_nodes, media_ids, element_ids, content_types,
58
69
  media_types, data_types, components, stylesheets,
59
- scripts, languages, dictionary_items, create_date, update_date
70
+ scripts, languages, dictionary_items, package_name, package_version,
71
+ create_date, update_date
60
72
  FROM bundle`
61
73
 
62
74
  const list = (value: string | null): string[] => {
@@ -87,6 +99,8 @@ const hydrate = (row: Row): Bundle => ({
87
99
  scripts: list(row.scripts),
88
100
  languages: list(row.languages),
89
101
  dictionaryItems: list(row.dictionary_items),
102
+ packageName: row.package_name ?? null,
103
+ packageVersion: row.package_version ?? null,
90
104
  createDate: DbDate.fromDb(row.create_date) ?? new Date(0),
91
105
  updateDate: DbDate.fromDb(row.update_date) ?? new Date(0),
92
106
  })
@@ -152,7 +166,14 @@ export class BundleRepository {
152
166
  DbDate.toDb(now),
153
167
  ],
154
168
  )
155
- return { ...input, id, createDate: now, updateDate: now }
169
+ return {
170
+ ...input,
171
+ id,
172
+ packageName: null,
173
+ packageVersion: null,
174
+ createDate: now,
175
+ updateDate: now,
176
+ }
156
177
  }
157
178
 
158
179
  async update(id: string, input: BundleInput): Promise<void> {
@@ -185,6 +206,17 @@ export class BundleRepository {
185
206
  )
186
207
  }
187
208
 
209
+ /** The npm package a definition is built as; null clears a value back to its default. */
210
+ async setPackage(
211
+ id: string,
212
+ pkg: { name: string | null; version: string | null },
213
+ ): Promise<void> {
214
+ await this.#db.exec(
215
+ 'UPDATE bundle SET package_name = ?, package_version = ?, update_date = ? WHERE id = ?',
216
+ [pkg.name, pkg.version, DbDate.toDb(new Date()), id],
217
+ )
218
+ }
219
+
188
220
  async delete(id: string): Promise<void> {
189
221
  await this.#db.exec('DELETE FROM bundle WHERE id = ?', [id])
190
222
  }