@crombolo/data 0.9.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/audit.ts +291 -0
- package/src/bundle-tables/builder.ts +264 -0
- package/src/bundle-tables/capability.ts +377 -0
- package/src/bundle-tables/chain.ts +105 -0
- package/src/bundle-tables/ddl.ts +124 -0
- package/src/bundle-tables/guard.ts +116 -0
- package/src/bundle-tables/index.ts +7 -0
- package/src/bundle-tables/names.ts +58 -0
- package/src/bundle-tables/schema.ts +485 -0
- package/src/cluster.ts +1 -1
- package/src/database.ts +39 -9
- package/src/dialect.ts +32 -0
- package/src/index.ts +9 -0
- package/src/migrations.ts +147 -31
- package/src/pending.ts +1 -1
- package/src/repositories/bundles.ts +36 -4
- package/src/repositories/content-types.ts +41 -4
- package/src/repositories/documents.ts +13 -3
- package/src/repositories/form-entries.ts +1 -1
- package/src/repositories/form-workflows.ts +1 -1
- package/src/repositories/members.ts +15 -1
- package/src/repositories/nodes.ts +32 -7
- package/src/repositories/transfer-runs.ts +20 -2
- package/src/schema/audit-log.ts +45 -0
- package/src/schema/bundle-components.ts +1 -1
- package/src/schema/bundle-package.ts +27 -0
- package/src/schema/bundles-rename.ts +1 -1
- package/src/schema/content.ts +2 -2
- package/src/schema/created-packages.ts +1 -1
- package/src/schema/form-permissions.ts +1 -1
- package/src/schema/form-workflows.ts +1 -1
- package/src/schema/forms.ts +1 -1
- package/src/schema/identity.ts +1 -1
- package/src/schema/plan.ts +10 -0
- package/src/schema/property-searchable.ts +29 -0
- package/src/schema/search-index.ts +92 -0
- package/src/schema/transfer-run-package.ts +27 -0
- package/src/schema/upgrade-report.ts +1 -1
- package/src/schema-state.ts +1 -1
- package/src/search-text.ts +133 -0
- package/src/search.ts +397 -0
package/src/database.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/** Driver-agnostic database handle. */
|
|
2
|
+
|
|
2
3
|
import { Database as BunSqlite } from 'bun:sqlite'
|
|
4
|
+
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
3
5
|
import { type Dialect, type DialectName, dialectFor } from './dialect.ts'
|
|
4
6
|
import { createLockManager, type LockManager } from './locks.ts'
|
|
5
7
|
|
|
@@ -22,16 +24,32 @@ export interface ConnectOptions {
|
|
|
22
24
|
url?: string
|
|
23
25
|
}
|
|
24
26
|
|
|
27
|
+
/** The connection the current async chain holds a transaction on. */
|
|
28
|
+
const transacting = new AsyncLocalStorage<BunSqlite>()
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* One connection has one transaction at a time, but async chains interleave:
|
|
32
|
+
* a background job's BEGIN between a request's BEGIN and COMMIT would fail, or
|
|
33
|
+
* worse, land its statements in the request's transaction. So top-level
|
|
34
|
+
* transactions queue here, and a chain already inside one joins it.
|
|
35
|
+
*/
|
|
25
36
|
class SqliteDb implements Db {
|
|
26
37
|
readonly dialect = dialectFor('sqlite')
|
|
27
38
|
readonly locks: LockManager
|
|
28
39
|
#db: BunSqlite
|
|
29
40
|
#inTransaction: boolean
|
|
30
|
-
|
|
31
|
-
|
|
41
|
+
#gate: { tail: Promise<void> }
|
|
42
|
+
|
|
43
|
+
constructor(
|
|
44
|
+
db: BunSqlite,
|
|
45
|
+
locks?: LockManager,
|
|
46
|
+
inTransaction = false,
|
|
47
|
+
gate: { tail: Promise<void> } = { tail: Promise.resolve() },
|
|
48
|
+
) {
|
|
32
49
|
this.#db = db
|
|
33
50
|
this.locks = locks ?? createLockManager(this)
|
|
34
51
|
this.#inTransaction = inTransaction
|
|
52
|
+
this.#gate = gate
|
|
35
53
|
}
|
|
36
54
|
|
|
37
55
|
async query<T = Record<string, unknown>>(sql: string, params: unknown[] = []): Promise<T[]> {
|
|
@@ -44,14 +62,26 @@ class SqliteDb implements Db {
|
|
|
44
62
|
|
|
45
63
|
async transaction<T>(fn: (tx: Db) => Promise<T>): Promise<T> {
|
|
46
64
|
if (this.#inTransaction) return fn(this)
|
|
47
|
-
this.#db.
|
|
65
|
+
const tx = new SqliteDb(this.#db, this.locks, true, this.#gate)
|
|
66
|
+
if (transacting.getStore() === this.#db) return fn(tx)
|
|
67
|
+
const previous = this.#gate.tail
|
|
68
|
+
let release!: () => void
|
|
69
|
+
this.#gate.tail = new Promise<void>((resolve) => {
|
|
70
|
+
release = resolve
|
|
71
|
+
})
|
|
72
|
+
await previous
|
|
48
73
|
try {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
74
|
+
this.#db.exec('BEGIN')
|
|
75
|
+
try {
|
|
76
|
+
const result = await transacting.run(this.#db, () => fn(tx))
|
|
77
|
+
this.#db.exec('COMMIT')
|
|
78
|
+
return result
|
|
79
|
+
} catch (error) {
|
|
80
|
+
this.#db.exec('ROLLBACK')
|
|
81
|
+
throw error
|
|
82
|
+
}
|
|
83
|
+
} finally {
|
|
84
|
+
release()
|
|
55
85
|
}
|
|
56
86
|
}
|
|
57
87
|
|
package/src/dialect.ts
CHANGED
|
@@ -15,6 +15,10 @@ export interface ColumnTypes {
|
|
|
15
15
|
textCi: string
|
|
16
16
|
text: string
|
|
17
17
|
varchar: (n: number) => string
|
|
18
|
+
/** A varchar that compares case-sensitively on both dialects, as a bundle's tables do. */
|
|
19
|
+
varcharExact: (n: number) => string
|
|
20
|
+
/** JSON, held as text and parsed by whoever reads it. */
|
|
21
|
+
json: string
|
|
18
22
|
timestamp: string
|
|
19
23
|
numeric: string
|
|
20
24
|
blob: string
|
|
@@ -32,6 +36,16 @@ export interface Dialect {
|
|
|
32
36
|
/** Appended to a SELECT that must block a concurrent state change; empty where the engine is single-writer. */
|
|
33
37
|
readonly forShare: string
|
|
34
38
|
readonly forUpdate: string
|
|
39
|
+
/** Appended to a SELECT that claims rows, passing over any another connection holds. */
|
|
40
|
+
readonly forUpdateSkipLocked: string
|
|
41
|
+
/**
|
|
42
|
+
* The full-text match over `search_document`: a subquery of `node_id` and
|
|
43
|
+
* `score`, higher being better, for each item holding every term as a word
|
|
44
|
+
* prefix. Its one parameter is `fullTextQuery(terms)`. docs/search.md.
|
|
45
|
+
*/
|
|
46
|
+
readonly fullTextHits: string
|
|
47
|
+
/** Terms are already folded words: letters and digits only. */
|
|
48
|
+
fullTextQuery(terms: readonly string[]): string
|
|
35
49
|
/**
|
|
36
50
|
* SQL that moves a table's identity past rows inserted with explicit ids, or
|
|
37
51
|
* undefined where the engine does that itself.
|
|
@@ -53,6 +67,8 @@ export const sqliteDialect: Dialect = {
|
|
|
53
67
|
textCi: 'TEXT COLLATE NOCASE',
|
|
54
68
|
text: 'TEXT',
|
|
55
69
|
varchar: (n) => `VARCHAR(${n}) COLLATE NOCASE`,
|
|
70
|
+
varcharExact: (n) => `VARCHAR(${n})`,
|
|
71
|
+
json: 'TEXT',
|
|
56
72
|
timestamp: 'TEXT',
|
|
57
73
|
// REAL would be lossy for decimal property values.
|
|
58
74
|
numeric: 'TEXT',
|
|
@@ -65,6 +81,15 @@ export const sqliteDialect: Dialect = {
|
|
|
65
81
|
boolValue: (value) => (value ? 1 : 0),
|
|
66
82
|
forShare: '',
|
|
67
83
|
forUpdate: '',
|
|
84
|
+
forUpdateSkipLocked: '',
|
|
85
|
+
// Name hits weigh ten times body hits; bm25 is lower-is-better, so negated.
|
|
86
|
+
// bm25 cannot be called under an aggregate, and `LIMIT -1` (no limit) keeps
|
|
87
|
+
// SQLite from flattening the inner query into the outer one, which would.
|
|
88
|
+
fullTextHits: `SELECT sd.node_id AS node_id, MAX(f.score) AS score
|
|
89
|
+
FROM (SELECT rowid AS id, -bm25(search_document_fts, 10.0, 1.0) AS score
|
|
90
|
+
FROM search_document_fts WHERE search_document_fts MATCH ? LIMIT -1) f
|
|
91
|
+
JOIN search_document sd ON sd.id = f.id GROUP BY sd.node_id`,
|
|
92
|
+
fullTextQuery: (terms) => terms.map((term) => `"${term}"*`).join(' AND '),
|
|
68
93
|
syncIdentity: () => undefined,
|
|
69
94
|
}
|
|
70
95
|
|
|
@@ -80,6 +105,8 @@ export const postgresDialect: Dialect = {
|
|
|
80
105
|
textCi: 'CITEXT',
|
|
81
106
|
text: 'TEXT',
|
|
82
107
|
varchar: (n) => `VARCHAR(${n})`,
|
|
108
|
+
varcharExact: (n) => `VARCHAR(${n})`,
|
|
109
|
+
json: 'TEXT',
|
|
83
110
|
timestamp: 'TIMESTAMPTZ',
|
|
84
111
|
numeric: 'NUMERIC',
|
|
85
112
|
blob: 'BYTEA',
|
|
@@ -94,6 +121,11 @@ export const postgresDialect: Dialect = {
|
|
|
94
121
|
boolValue: (value) => value,
|
|
95
122
|
forShare: 'FOR SHARE',
|
|
96
123
|
forUpdate: 'FOR UPDATE',
|
|
124
|
+
forUpdateSkipLocked: 'FOR UPDATE SKIP LOCKED',
|
|
125
|
+
fullTextHits: `SELECT sd.node_id AS node_id, MAX(ts_rank(sd.terms, q)) AS score
|
|
126
|
+
FROM search_document sd, to_tsquery('simple', ?) q
|
|
127
|
+
WHERE sd.terms @@ q GROUP BY sd.node_id`,
|
|
128
|
+
fullTextQuery: (terms) => terms.map((term) => `${term}:*`).join(' & '),
|
|
97
129
|
syncIdentity: (table) =>
|
|
98
130
|
`SELECT setval(pg_get_serial_sequence('${table}', 'id'), (SELECT COALESCE(MAX(id), 1) FROM ${table}))`,
|
|
99
131
|
}
|
package/src/index.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
export * from './audit.ts'
|
|
2
|
+
export * from './bundle-tables/index.ts'
|
|
1
3
|
export * from './change-report.ts'
|
|
2
4
|
export * from './cluster.ts'
|
|
3
5
|
export * from './database.ts'
|
|
@@ -30,9 +32,11 @@ export * from './repositories/transfer-runs.ts'
|
|
|
30
32
|
export * from './repositories/user-data.ts'
|
|
31
33
|
export * from './repositories/users.ts'
|
|
32
34
|
export * from './schema/assistant.ts'
|
|
35
|
+
export * from './schema/audit-log.ts'
|
|
33
36
|
export * from './schema/built-in-content-types.ts'
|
|
34
37
|
export * from './schema/built-in-media-types.ts'
|
|
35
38
|
export * from './schema/built-in-member-types.ts'
|
|
39
|
+
export * from './schema/bundle-package.ts'
|
|
36
40
|
export * from './schema/bundles-rename.ts'
|
|
37
41
|
export * from './schema/change-report.ts'
|
|
38
42
|
export * from './schema/client-id-prefix.ts'
|
|
@@ -47,14 +51,19 @@ export * from './schema/log-viewer.ts'
|
|
|
47
51
|
export * from './schema/member-property-type.ts'
|
|
48
52
|
export * from './schema/members.ts'
|
|
49
53
|
export * from './schema/plan.ts'
|
|
54
|
+
export * from './schema/property-searchable.ts'
|
|
50
55
|
export * from './schema/public-access.ts'
|
|
51
56
|
export * from './schema/recycle-bin.ts'
|
|
52
57
|
export * from './schema/rich-text-ui.ts'
|
|
58
|
+
export * from './schema/search-index.ts'
|
|
53
59
|
export * from './schema/seed.ts'
|
|
54
60
|
export * from './schema/since-version.ts'
|
|
61
|
+
export * from './schema/transfer-run-package.ts'
|
|
55
62
|
export * from './schema/transfer-runs.ts'
|
|
56
63
|
export * from './schema/upgrade-report.ts'
|
|
57
64
|
export * from './schema/users-localisation.ts'
|
|
58
65
|
export * from './schema/values.ts'
|
|
59
66
|
export * from './schema-ops.ts'
|
|
60
67
|
export * from './schema-state.ts'
|
|
68
|
+
export * from './search.ts'
|
|
69
|
+
export * from './search-text.ts'
|
package/src/migrations.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* version number and not a list of applied migrations: the plan is a chain of
|
|
4
4
|
* transitions walked forward to its final state. That design handles concurrent
|
|
5
5
|
* development branches far better than sequence numbers, so we keep it — and
|
|
6
|
-
* harden it (docs/
|
|
6
|
+
* harden it (docs/packaging-and-upgrades.md): a ledger of every step that
|
|
7
7
|
* ran, expand/contract declared and linted, a recorded plan, and an explicit
|
|
8
8
|
* refusal when the database is newer than the code.
|
|
9
9
|
*/
|
|
@@ -29,7 +29,24 @@ export interface Migration {
|
|
|
29
29
|
release?: string
|
|
30
30
|
/** For a contract: the name of the expand migration it removes. */
|
|
31
31
|
contracts?: string
|
|
32
|
+
/** Recorded in the ledger, and compared when the plan verifies what ran. Defaults to `name`. */
|
|
33
|
+
checksum?: string
|
|
32
34
|
up(db: Db): Promise<void>
|
|
35
|
+
/**
|
|
36
|
+
* Writes data once `up` has run, in the same transaction. `planUpgrade` cannot
|
|
37
|
+
* predict what it writes, so a step that has one is reported as unplannable.
|
|
38
|
+
*/
|
|
39
|
+
backfill?(db: Db): Promise<void>
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Where a chain keeps its state, and what it calls itself in the ledger. */
|
|
43
|
+
export interface ChainOptions {
|
|
44
|
+
/** The `key_value` key holding the chain's state. */
|
|
45
|
+
stateKey?: string
|
|
46
|
+
/** The state before the first step. */
|
|
47
|
+
initialState?: string
|
|
48
|
+
/** A bundle's id, for a bundle's chain; absent for core's. */
|
|
49
|
+
bundle?: string
|
|
33
50
|
}
|
|
34
51
|
|
|
35
52
|
function compareRelease(a: string, b: string): number {
|
|
@@ -71,10 +88,16 @@ export function isMajorRelease(release: string): boolean {
|
|
|
71
88
|
|
|
72
89
|
export class MigrationPlan {
|
|
73
90
|
readonly migrations: readonly Migration[]
|
|
91
|
+
readonly stateKey: string
|
|
92
|
+
readonly initialState: string
|
|
93
|
+
readonly bundle: string | undefined
|
|
74
94
|
|
|
75
|
-
constructor(migrations: readonly Migration[]) {
|
|
95
|
+
constructor(migrations: readonly Migration[], options: ChainOptions = {}) {
|
|
76
96
|
this.migrations = migrations
|
|
77
|
-
|
|
97
|
+
this.stateKey = options.stateKey ?? MIGRATION_STATE_KEY
|
|
98
|
+
this.initialState = options.initialState ?? INITIAL_STATE
|
|
99
|
+
this.bundle = options.bundle
|
|
100
|
+
let expected = this.initialState
|
|
78
101
|
const seen = new Map<string, Migration>()
|
|
79
102
|
for (const migration of migrations) {
|
|
80
103
|
if (migration.from !== expected) {
|
|
@@ -111,7 +134,14 @@ export class MigrationPlan {
|
|
|
111
134
|
}
|
|
112
135
|
|
|
113
136
|
get finalState(): string {
|
|
114
|
-
return this.migrations.at(-1)?.to ??
|
|
137
|
+
return this.migrations.at(-1)?.to ?? this.initialState
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** The steps that have run when the chain is at a state; undefined when the state is not in it. */
|
|
141
|
+
appliedAt(state: string): Migration[] | undefined {
|
|
142
|
+
if (state === this.initialState) return []
|
|
143
|
+
const index = this.migrations.findIndex((m) => m.to === state)
|
|
144
|
+
return index < 0 ? undefined : this.migrations.slice(0, index + 1)
|
|
115
145
|
}
|
|
116
146
|
|
|
117
147
|
/** The steps still to run from a state; empty when current. Undefined when the state is not in the plan. */
|
|
@@ -123,7 +153,11 @@ export class MigrationPlan {
|
|
|
123
153
|
}
|
|
124
154
|
}
|
|
125
155
|
|
|
156
|
+
/** Connections whose ledger tables are known to exist with every column. */
|
|
157
|
+
const ensured = new WeakSet<Db>()
|
|
158
|
+
|
|
126
159
|
async function ensureTables(db: Db): Promise<void> {
|
|
160
|
+
if (ensured.has(db)) return
|
|
127
161
|
const t = db.dialect.types
|
|
128
162
|
await db.exec(
|
|
129
163
|
`CREATE TABLE IF NOT EXISTS key_value (
|
|
@@ -143,36 +177,62 @@ async function ensureTables(db: Db): Promise<void> {
|
|
|
143
177
|
duration_ms ${t.integer} NOT NULL,
|
|
144
178
|
checksum ${t.varchar(64)},
|
|
145
179
|
applied_by ${t.varchar(255)},
|
|
146
|
-
note ${t.text}
|
|
180
|
+
note ${t.text},
|
|
181
|
+
bundle ${t.varchar(64)}
|
|
147
182
|
)`,
|
|
148
183
|
)
|
|
184
|
+
// A ledger from before bundles could own tables has no `bundle` column. It is
|
|
185
|
+
// added here, where the table is, rather than by a core migration: the ledger
|
|
186
|
+
// has to be able to record a bundle's step whatever state core is at.
|
|
187
|
+
if (!(await hasColumn(db, 'migration_history', 'bundle')))
|
|
188
|
+
await db.exec(`ALTER TABLE migration_history ADD COLUMN bundle ${t.varchar(64)}`)
|
|
189
|
+
ensured.add(db)
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
async function hasColumn(db: Db, table: string, column: string): Promise<boolean> {
|
|
193
|
+
const rows =
|
|
194
|
+
db.dialect.name === 'sqlite'
|
|
195
|
+
? await db.query<{ name: string }>(`SELECT name FROM pragma_table_info('${table}')`)
|
|
196
|
+
: await db.query<{ name: string }>(
|
|
197
|
+
'SELECT column_name AS name FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = ?',
|
|
198
|
+
[table],
|
|
199
|
+
)
|
|
200
|
+
return rows.some((row) => row.name === column)
|
|
149
201
|
}
|
|
150
202
|
|
|
151
|
-
|
|
203
|
+
/** A chain's state: core's unless a plan says otherwise. */
|
|
204
|
+
export async function readState(
|
|
205
|
+
db: Db,
|
|
206
|
+
chain: Pick<MigrationPlan, 'stateKey' | 'initialState'> = {
|
|
207
|
+
stateKey: MIGRATION_STATE_KEY,
|
|
208
|
+
initialState: INITIAL_STATE,
|
|
209
|
+
},
|
|
210
|
+
): Promise<string> {
|
|
152
211
|
await ensureTables(db)
|
|
153
212
|
const rows = await db.query<{ value: string }>('SELECT value FROM key_value WHERE key = ?', [
|
|
154
|
-
|
|
213
|
+
chain.stateKey,
|
|
155
214
|
])
|
|
156
|
-
return rows[0]?.value ??
|
|
215
|
+
return rows[0]?.value ?? chain.initialState
|
|
157
216
|
}
|
|
158
217
|
|
|
159
|
-
async function writeState(db: Db, state: string): Promise<void> {
|
|
218
|
+
async function writeState(db: Db, state: string, key = MIGRATION_STATE_KEY): Promise<void> {
|
|
160
219
|
const now = db.dialect.now()
|
|
161
|
-
await db.query(`UPDATE key_value SET value = ?, update_date = ${now} WHERE key = ?`, [
|
|
162
|
-
|
|
163
|
-
MIGRATION_STATE_KEY,
|
|
164
|
-
])
|
|
165
|
-
const rows = await db.query<{ value: string }>('SELECT value FROM key_value WHERE key = ?', [
|
|
166
|
-
MIGRATION_STATE_KEY,
|
|
167
|
-
])
|
|
220
|
+
await db.query(`UPDATE key_value SET value = ?, update_date = ${now} WHERE key = ?`, [state, key])
|
|
221
|
+
const rows = await db.query<{ value: string }>('SELECT value FROM key_value WHERE key = ?', [key])
|
|
168
222
|
if (rows.length === 0) {
|
|
169
223
|
await db.exec(
|
|
170
224
|
`INSERT INTO key_value (key, value, update_date) VALUES (${db.dialect.placeholder(1)}, ${db.dialect.placeholder(2)}, ${now})`,
|
|
171
|
-
[
|
|
225
|
+
[key, state],
|
|
172
226
|
)
|
|
173
227
|
}
|
|
174
228
|
}
|
|
175
229
|
|
|
230
|
+
/** Forgets a setting entirely, as a purged bundle's state is forgotten. */
|
|
231
|
+
export async function deleteKeyValue(db: Db, key: string): Promise<void> {
|
|
232
|
+
await ensureTables(db)
|
|
233
|
+
await db.exec('DELETE FROM key_value WHERE key = ?', [key])
|
|
234
|
+
}
|
|
235
|
+
|
|
176
236
|
export interface LedgerEntry {
|
|
177
237
|
name: string
|
|
178
238
|
/** 'expand' | 'contract' for framework steps; 'value' for a value migration; 'upgrade' for a cut-over. */
|
|
@@ -183,6 +243,8 @@ export interface LedgerEntry {
|
|
|
183
243
|
checksum?: string | null
|
|
184
244
|
appliedBy?: string | null
|
|
185
245
|
note?: string | null
|
|
246
|
+
/** The bundle whose chain this step belongs to; null for core. */
|
|
247
|
+
bundle?: string | null
|
|
186
248
|
}
|
|
187
249
|
|
|
188
250
|
/**
|
|
@@ -226,8 +288,8 @@ export async function ensureKeyValue(db: Db, key: string, create: () => string):
|
|
|
226
288
|
export async function recordInLedger(db: Db, entry: LedgerEntry): Promise<void> {
|
|
227
289
|
await ensureTables(db)
|
|
228
290
|
await db.exec(
|
|
229
|
-
`INSERT INTO migration_history (name, kind, from_state, to_state, applied_at, duration_ms, checksum, applied_by, note)
|
|
230
|
-
VALUES (?, ?, ?, ?, ${db.dialect.now()}, ?, ?, ?, ?)`,
|
|
291
|
+
`INSERT INTO migration_history (name, kind, from_state, to_state, applied_at, duration_ms, checksum, applied_by, note, bundle)
|
|
292
|
+
VALUES (?, ?, ?, ?, ${db.dialect.now()}, ?, ?, ?, ?, ?)`,
|
|
231
293
|
[
|
|
232
294
|
entry.name,
|
|
233
295
|
entry.kind,
|
|
@@ -237,6 +299,7 @@ export async function recordInLedger(db: Db, entry: LedgerEntry): Promise<void>
|
|
|
237
299
|
entry.checksum ?? null,
|
|
238
300
|
entry.appliedBy ?? null,
|
|
239
301
|
entry.note ?? null,
|
|
302
|
+
entry.bundle ?? null,
|
|
240
303
|
],
|
|
241
304
|
)
|
|
242
305
|
}
|
|
@@ -248,10 +311,14 @@ export async function latestLedgerId(db: Db): Promise<number> {
|
|
|
248
311
|
return Number(rows[0]?.n ?? 0)
|
|
249
312
|
}
|
|
250
313
|
|
|
251
|
-
/**
|
|
314
|
+
/**
|
|
315
|
+
* How many contract steps — framework contracts and site purges — were ledgered
|
|
316
|
+
* after a row id. A bundle's contract is not counted: it removes only the
|
|
317
|
+
* bundle's own tables, which no read of core's depends on.
|
|
318
|
+
*/
|
|
252
319
|
export async function contractsLedgeredAfter(db: Db, afterId: number): Promise<number> {
|
|
253
320
|
const rows = await db.query<{ n: number }>(
|
|
254
|
-
"SELECT COUNT(*) AS n FROM migration_history WHERE id > ? AND kind = 'contract'",
|
|
321
|
+
"SELECT COUNT(*) AS n FROM migration_history WHERE id > ? AND kind = 'contract' AND bundle IS NULL",
|
|
255
322
|
[afterId],
|
|
256
323
|
)
|
|
257
324
|
return Number(rows[0]?.n ?? 0)
|
|
@@ -265,7 +332,7 @@ export interface LedgerRow extends LedgerEntry {
|
|
|
265
332
|
export async function readLedger(db: Db): Promise<LedgerRow[]> {
|
|
266
333
|
await ensureTables(db)
|
|
267
334
|
const rows = await db.query(
|
|
268
|
-
'SELECT id, name, kind, from_state, to_state, applied_at, duration_ms, checksum, applied_by, note FROM migration_history ORDER BY id',
|
|
335
|
+
'SELECT id, name, kind, from_state, to_state, applied_at, duration_ms, checksum, applied_by, note, bundle FROM migration_history ORDER BY id',
|
|
269
336
|
)
|
|
270
337
|
return rows.map((row) => ({
|
|
271
338
|
id: Number(row.id),
|
|
@@ -278,6 +345,7 @@ export async function readLedger(db: Db): Promise<LedgerRow[]> {
|
|
|
278
345
|
checksum: (row.checksum as string | null) ?? null,
|
|
279
346
|
appliedBy: (row.applied_by as string | null) ?? null,
|
|
280
347
|
note: (row.note as string | null) ?? null,
|
|
348
|
+
bundle: (row.bundle as string | null) ?? null,
|
|
281
349
|
}))
|
|
282
350
|
}
|
|
283
351
|
|
|
@@ -286,22 +354,66 @@ export class UpgradeStateError extends Error {
|
|
|
286
354
|
constructor(
|
|
287
355
|
readonly databaseState: string,
|
|
288
356
|
readonly codeState: string,
|
|
357
|
+
/** The bundle whose chain it is, when it is not core's. */
|
|
358
|
+
readonly bundle?: string,
|
|
359
|
+
) {
|
|
360
|
+
super(
|
|
361
|
+
bundle
|
|
362
|
+
? `The database holds the bundle "${bundle}"'s tables at state ${databaseState}, which this version of the bundle (its steps end at ${codeState}) does not know. ` +
|
|
363
|
+
'A newer release of the bundle wrote them; install that release, or restore the backup taken before the upgrade.'
|
|
364
|
+
: `The database is at migration state ${databaseState}, which this version of crombolo (plan ends at ${codeState}) does not know. ` +
|
|
365
|
+
'It was upgraded by a newer version; run that version, or restore the backup taken before the upgrade.',
|
|
366
|
+
)
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/** A bundle step changed after it ran here: what is in the database is not what the code says made it. */
|
|
371
|
+
export class BundleStepEditedError extends Error {
|
|
372
|
+
constructor(
|
|
373
|
+
readonly bundle: string,
|
|
374
|
+
readonly step: string,
|
|
289
375
|
) {
|
|
290
376
|
super(
|
|
291
|
-
`The
|
|
292
|
-
'
|
|
377
|
+
`The bundle "${bundle}"'s step "${step}" is not the step that ran on this database: it was edited after it shipped. ` +
|
|
378
|
+
'A shipped step is never changed; put the change in a new step.',
|
|
293
379
|
)
|
|
294
380
|
}
|
|
295
381
|
}
|
|
296
382
|
|
|
297
383
|
/** Steps pending on this database, in order. Throws when the database is ahead of the code. */
|
|
298
384
|
export async function pendingMigrations(db: Db, plan: MigrationPlan): Promise<Migration[]> {
|
|
299
|
-
const state = await readState(db)
|
|
385
|
+
const state = await readState(db, plan)
|
|
300
386
|
const pending = plan.pendingFrom(state)
|
|
301
|
-
if (!pending) throw new UpgradeStateError(state, plan.finalState)
|
|
387
|
+
if (!pending) throw new UpgradeStateError(state, plan.finalState, plan.bundle)
|
|
302
388
|
return pending
|
|
303
389
|
}
|
|
304
390
|
|
|
391
|
+
/**
|
|
392
|
+
* Refuses a bundle chain whose applied steps no longer match the code. Only the
|
|
393
|
+
* rows since the bundle was last purged count: a purge starts its chain again.
|
|
394
|
+
*/
|
|
395
|
+
async function verifyApplied(db: Db, plan: MigrationPlan, state: string): Promise<void> {
|
|
396
|
+
if (!plan.bundle) return
|
|
397
|
+
const applied = plan.appliedAt(state) ?? []
|
|
398
|
+
if (applied.length === 0) return
|
|
399
|
+
const rows = await db.query<{ name: string; kind: string; checksum: string | null }>(
|
|
400
|
+
'SELECT name, kind, checksum FROM migration_history WHERE bundle = ? ORDER BY id',
|
|
401
|
+
[plan.bundle],
|
|
402
|
+
)
|
|
403
|
+
const purged = rows.map((row) => row.kind).lastIndexOf('purge')
|
|
404
|
+
const ran = new Map(
|
|
405
|
+
rows
|
|
406
|
+
.slice(purged + 1)
|
|
407
|
+
.filter((row) => row.kind === 'expand' || row.kind === 'contract')
|
|
408
|
+
.map((row) => [row.name, row.checksum]),
|
|
409
|
+
)
|
|
410
|
+
for (const step of applied) {
|
|
411
|
+
const recorded = ran.get(step.name)
|
|
412
|
+
if (recorded !== undefined && recorded !== (step.checksum ?? step.name))
|
|
413
|
+
throw new BundleStepEditedError(plan.bundle, step.name)
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
|
|
305
417
|
export interface MigrateResult {
|
|
306
418
|
from: string
|
|
307
419
|
to: string
|
|
@@ -320,27 +432,30 @@ export async function migrate(
|
|
|
320
432
|
options: MigrateOptions = {},
|
|
321
433
|
): Promise<MigrateResult> {
|
|
322
434
|
return db.locks.withLock(Locks.KeyValues, async () => {
|
|
323
|
-
const from = await readState(db)
|
|
435
|
+
const from = await readState(db, plan)
|
|
324
436
|
let state = from
|
|
325
437
|
const applied: string[] = []
|
|
326
438
|
const pending = plan.pendingFrom(state)
|
|
327
|
-
if (!pending) throw new UpgradeStateError(state, plan.finalState)
|
|
439
|
+
if (!pending) throw new UpgradeStateError(state, plan.finalState, plan.bundle)
|
|
440
|
+
await verifyApplied(db, plan, state)
|
|
328
441
|
for (const next of pending) {
|
|
329
442
|
const started = performance.now()
|
|
330
443
|
await db.transaction(async (tx) => {
|
|
331
444
|
await next.up(tx)
|
|
445
|
+
await next.backfill?.(tx)
|
|
332
446
|
})
|
|
333
447
|
state = next.to
|
|
334
|
-
await writeState(db, state)
|
|
448
|
+
await writeState(db, state, plan.stateKey)
|
|
335
449
|
await recordInLedger(db, {
|
|
336
450
|
name: next.name,
|
|
337
451
|
kind: next.kind ?? 'expand',
|
|
338
452
|
fromState: next.from,
|
|
339
453
|
toState: next.to,
|
|
340
454
|
durationMs: Math.round(performance.now() - started),
|
|
341
|
-
checksum: next.name,
|
|
455
|
+
checksum: next.checksum ?? next.name,
|
|
342
456
|
appliedBy: options.by ?? null,
|
|
343
457
|
note: next.release ? `release ${next.release}` : null,
|
|
458
|
+
bundle: plan.bundle ?? null,
|
|
344
459
|
})
|
|
345
460
|
applied.push(next.name)
|
|
346
461
|
}
|
|
@@ -351,7 +466,7 @@ export async function migrate(
|
|
|
351
466
|
/** A fresh install stamps the final state so no migrations then run. */
|
|
352
467
|
export async function stampFinalState(db: Db, plan: MigrationPlan): Promise<void> {
|
|
353
468
|
await ensureTables(db)
|
|
354
|
-
await writeState(db, plan.finalState)
|
|
469
|
+
await writeState(db, plan.finalState, plan.stateKey)
|
|
355
470
|
}
|
|
356
471
|
|
|
357
472
|
// ---------------------------------------------------------------- the plan
|
|
@@ -421,6 +536,7 @@ export async function planUpgrade(db: Db, plan: MigrationPlan): Promise<PlannedS
|
|
|
421
536
|
try {
|
|
422
537
|
await migration.up(recorder)
|
|
423
538
|
step.statements = recorder.statements
|
|
539
|
+
if (migration.backfill) step.unplannable = 'backfills data, which only running it can show'
|
|
424
540
|
} catch (error) {
|
|
425
541
|
if (error instanceof UnplannableError) step.unplannable = error.message
|
|
426
542
|
else throw error
|
package/src/pending.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* An element is pending on a node when the deployment that created it is newer
|
|
3
3
|
* than the node, or its file says `since` a version the node has not reached.
|
|
4
|
-
* The one place the rule lives. docs/
|
|
4
|
+
* The one place the rule lives. docs/schema-as-code.md, "since".
|
|
5
5
|
*/
|
|
6
6
|
import { compareStates, compareVersions, type NodeSchemaState } from './schema-state.ts'
|
|
7
7
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The bundle definitions: what a bundle would contain, not a built bundle.
|
|
3
|
-
* `docs/
|
|
3
|
+
* `docs/bundles.md`.
|
|
4
4
|
*
|
|
5
5
|
* The selection lists travel as JSON text in single columns. A malformed list
|
|
6
6
|
* reads back as empty rather than throwing: a definition whose row was edited
|
|
@@ -26,12 +26,21 @@ export interface Bundle {
|
|
|
26
26
|
scripts: string[]
|
|
27
27
|
languages: string[]
|
|
28
28
|
dictionaryItems: string[]
|
|
29
|
+
/**
|
|
30
|
+
* The npm package the bundle is built as. Not in Umbraco's definition, so the
|
|
31
|
+
* contract never writes these; null until someone sets them.
|
|
32
|
+
*/
|
|
33
|
+
packageName: string | null
|
|
34
|
+
packageVersion: string | null
|
|
29
35
|
createDate: Date
|
|
30
36
|
updateDate: Date
|
|
31
37
|
}
|
|
32
38
|
|
|
33
39
|
/** A definition to store; `id` is assigned when it is new, the dates always. */
|
|
34
|
-
export type BundleInput = Omit<
|
|
40
|
+
export type BundleInput = Omit<
|
|
41
|
+
Bundle,
|
|
42
|
+
'id' | 'createDate' | 'updateDate' | 'packageName' | 'packageVersion'
|
|
43
|
+
>
|
|
35
44
|
|
|
36
45
|
interface Row {
|
|
37
46
|
id: string
|
|
@@ -49,6 +58,8 @@ interface Row {
|
|
|
49
58
|
scripts: string
|
|
50
59
|
languages: string
|
|
51
60
|
dictionary_items: string
|
|
61
|
+
package_name: string | null
|
|
62
|
+
package_version: string | null
|
|
52
63
|
create_date: unknown
|
|
53
64
|
update_date: unknown
|
|
54
65
|
}
|
|
@@ -56,7 +67,8 @@ interface Row {
|
|
|
56
67
|
const SELECT = `SELECT id, name, content_node_id, content_load_child_nodes,
|
|
57
68
|
media_load_child_nodes, media_ids, element_ids, content_types,
|
|
58
69
|
media_types, data_types, components, stylesheets,
|
|
59
|
-
scripts, languages, dictionary_items,
|
|
70
|
+
scripts, languages, dictionary_items, package_name, package_version,
|
|
71
|
+
create_date, update_date
|
|
60
72
|
FROM bundle`
|
|
61
73
|
|
|
62
74
|
const list = (value: string | null): string[] => {
|
|
@@ -87,6 +99,8 @@ const hydrate = (row: Row): Bundle => ({
|
|
|
87
99
|
scripts: list(row.scripts),
|
|
88
100
|
languages: list(row.languages),
|
|
89
101
|
dictionaryItems: list(row.dictionary_items),
|
|
102
|
+
packageName: row.package_name ?? null,
|
|
103
|
+
packageVersion: row.package_version ?? null,
|
|
90
104
|
createDate: DbDate.fromDb(row.create_date) ?? new Date(0),
|
|
91
105
|
updateDate: DbDate.fromDb(row.update_date) ?? new Date(0),
|
|
92
106
|
})
|
|
@@ -152,7 +166,14 @@ export class BundleRepository {
|
|
|
152
166
|
DbDate.toDb(now),
|
|
153
167
|
],
|
|
154
168
|
)
|
|
155
|
-
return {
|
|
169
|
+
return {
|
|
170
|
+
...input,
|
|
171
|
+
id,
|
|
172
|
+
packageName: null,
|
|
173
|
+
packageVersion: null,
|
|
174
|
+
createDate: now,
|
|
175
|
+
updateDate: now,
|
|
176
|
+
}
|
|
156
177
|
}
|
|
157
178
|
|
|
158
179
|
async update(id: string, input: BundleInput): Promise<void> {
|
|
@@ -185,6 +206,17 @@ export class BundleRepository {
|
|
|
185
206
|
)
|
|
186
207
|
}
|
|
187
208
|
|
|
209
|
+
/** The npm package a definition is built as; null clears a value back to its default. */
|
|
210
|
+
async setPackage(
|
|
211
|
+
id: string,
|
|
212
|
+
pkg: { name: string | null; version: string | null },
|
|
213
|
+
): Promise<void> {
|
|
214
|
+
await this.#db.exec(
|
|
215
|
+
'UPDATE bundle SET package_name = ?, package_version = ?, update_date = ? WHERE id = ?',
|
|
216
|
+
[pkg.name, pkg.version, DbDate.toDb(new Date()), id],
|
|
217
|
+
)
|
|
218
|
+
}
|
|
219
|
+
|
|
188
220
|
async delete(id: string): Promise<void> {
|
|
189
221
|
await this.#db.exec('DELETE FROM bundle WHERE id = ?', [id])
|
|
190
222
|
}
|