@crombolo/data 0.10.0 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/ambient.ts +20 -0
- package/src/audit.ts +291 -0
- package/src/busy.ts +74 -0
- package/src/change-report.ts +6 -0
- package/src/cluster.ts +17 -5
- package/src/database.ts +80 -17
- package/src/dialect.ts +24 -0
- package/src/index.ts +20 -0
- package/src/locks.ts +94 -25
- package/src/migrations.ts +42 -16
- package/src/relations.ts +346 -0
- package/src/repositories/content-types.ts +41 -4
- package/src/repositories/data-types.ts +38 -0
- package/src/repositories/delivery-member-tokens.ts +110 -0
- package/src/repositories/dictionary.ts +18 -0
- package/src/repositories/documents.ts +443 -88
- package/src/repositories/dynamic-roots.ts +151 -0
- package/src/repositories/form-entries.ts +32 -11
- package/src/repositories/form-workflows.ts +1 -1
- package/src/repositories/members.ts +26 -1
- package/src/repositories/nodes.ts +43 -7
- package/src/repositories/public-access.ts +30 -0
- package/src/repositories/references.ts +15 -0
- package/src/repositories/transfer-answers.ts +151 -0
- package/src/repositories/webhooks.ts +299 -0
- package/src/schema/audit-log.ts +45 -0
- package/src/schema/delivery-member-tokens.ts +40 -0
- package/src/schema/diagnostics.ts +28 -0
- package/src/schema/plan.ts +14 -0
- package/src/schema/property-searchable.ts +29 -0
- package/src/schema/search-index.ts +92 -0
- package/src/schema/seed.ts +13 -2
- package/src/schema/transfer-answers.ts +50 -0
- package/src/schema/webhooks.ts +78 -0
- package/src/search-text.ts +133 -0
- package/src/search.ts +397 -0
- package/src/setup.ts +57 -0
- package/src/tree-invariants.ts +167 -0
- package/src/url-options.ts +61 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crombolo/data",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.12.1",
|
|
4
4
|
"description": "SQLite and Postgres storage for crombolo: schema, migrations and repositories.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Crombolo",
|
|
@@ -28,6 +28,6 @@
|
|
|
28
28
|
"access": "public"
|
|
29
29
|
},
|
|
30
30
|
"dependencies": {
|
|
31
|
-
"@crombolo/core": "0.
|
|
31
|
+
"@crombolo/core": "0.12.1"
|
|
32
32
|
}
|
|
33
33
|
}
|
package/src/ambient.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/** The transaction the current async chain is inside, shared by the drivers and the lock managers. */
|
|
2
|
+
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
3
|
+
|
|
4
|
+
/** Cleared when its transaction ends, so work left running afterwards falls back to the pool. */
|
|
5
|
+
export interface AmbientTransaction<T> {
|
|
6
|
+
owner: object
|
|
7
|
+
tx: T | undefined
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export const ambient = new AsyncLocalStorage<AmbientTransaction<unknown>>()
|
|
11
|
+
|
|
12
|
+
/** The open transaction this chain holds on `owner`'s connection, if any. */
|
|
13
|
+
export function ambientFor<T>(owner: object): T | undefined {
|
|
14
|
+
const store = ambient.getStore()
|
|
15
|
+
return store?.owner === owner ? (store.tx as T | undefined) : undefined
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function insideTransaction(): boolean {
|
|
19
|
+
return ambient.getStore()?.tx !== undefined
|
|
20
|
+
}
|
package/src/audit.ts
ADDED
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The audit log: who did what, when and from where. Entries are only ever
|
|
3
|
+
* added, read and aged out; nothing edits one. docs/audit-log.md.
|
|
4
|
+
*/
|
|
5
|
+
import { normaliseUuid } from '@crombolo/core'
|
|
6
|
+
import type { Db } from './database.ts'
|
|
7
|
+
import { DbDate } from './dialect.ts'
|
|
8
|
+
import { STATE_AUDIT_LOG } from './schema/audit-log.ts'
|
|
9
|
+
|
|
10
|
+
export type AuditCategory =
|
|
11
|
+
| 'content'
|
|
12
|
+
| 'media'
|
|
13
|
+
| 'member'
|
|
14
|
+
| 'schema'
|
|
15
|
+
| 'settings'
|
|
16
|
+
| 'user'
|
|
17
|
+
| 'permission'
|
|
18
|
+
| 'sign-in'
|
|
19
|
+
| 'system'
|
|
20
|
+
|
|
21
|
+
export const AUDIT_CATEGORIES: readonly AuditCategory[] = [
|
|
22
|
+
'content',
|
|
23
|
+
'media',
|
|
24
|
+
'member',
|
|
25
|
+
'schema',
|
|
26
|
+
'settings',
|
|
27
|
+
'user',
|
|
28
|
+
'permission',
|
|
29
|
+
'sign-in',
|
|
30
|
+
'system',
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
/** Where an entry came from: a request (with its address), the command line, or a background job. */
|
|
34
|
+
export type AuditSource = 'request' | 'cli' | 'job'
|
|
35
|
+
|
|
36
|
+
export interface AuditEntryInput {
|
|
37
|
+
category: AuditCategory
|
|
38
|
+
action: string
|
|
39
|
+
source: AuditSource
|
|
40
|
+
userKey?: string | null
|
|
41
|
+
/** The user's name when the entry was written; for a failed sign-in, the login tried. */
|
|
42
|
+
userName?: string | null
|
|
43
|
+
entityType?: string | null
|
|
44
|
+
entityKey?: string | null
|
|
45
|
+
entityName?: string | null
|
|
46
|
+
ipAddress?: string | null
|
|
47
|
+
forwardedFor?: string | null
|
|
48
|
+
detail?: string | null
|
|
49
|
+
occurredAt?: Date
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface AuditEntry {
|
|
53
|
+
id: number
|
|
54
|
+
occurredAt: Date
|
|
55
|
+
category: AuditCategory
|
|
56
|
+
action: string
|
|
57
|
+
source: AuditSource
|
|
58
|
+
userKey: string | null
|
|
59
|
+
userName: string | null
|
|
60
|
+
entityType: string | null
|
|
61
|
+
entityKey: string | null
|
|
62
|
+
entityName: string | null
|
|
63
|
+
ipAddress: string | null
|
|
64
|
+
forwardedFor: string | null
|
|
65
|
+
detail: string | null
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export interface AuditFilter {
|
|
69
|
+
userKey?: string
|
|
70
|
+
entityKey?: string
|
|
71
|
+
category?: AuditCategory
|
|
72
|
+
action?: string
|
|
73
|
+
since?: Date
|
|
74
|
+
until?: Date
|
|
75
|
+
/** Matched anywhere in the item's name or key, or the user's name. */
|
|
76
|
+
text?: string
|
|
77
|
+
ascending?: boolean
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** What a key names, for an entry to carry once the thing itself has gone. */
|
|
81
|
+
export interface AuditSubject {
|
|
82
|
+
entityType: string
|
|
83
|
+
entityKey: string
|
|
84
|
+
entityName: string | null
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const UUID = /^\{?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\}?$/i
|
|
88
|
+
|
|
89
|
+
const LIMITS = {
|
|
90
|
+
action: 64,
|
|
91
|
+
userName: 255,
|
|
92
|
+
entityType: 64,
|
|
93
|
+
entityKey: 255,
|
|
94
|
+
entityName: 255,
|
|
95
|
+
ipAddress: 64,
|
|
96
|
+
forwardedFor: 255,
|
|
97
|
+
detail: 4000,
|
|
98
|
+
} as const
|
|
99
|
+
|
|
100
|
+
function clip(value: string | null | undefined, limit: number): string | null {
|
|
101
|
+
if (value === null || value === undefined || value === '') return null
|
|
102
|
+
return value.length > limit ? value.slice(0, limit) : value
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function text(value: unknown): string | null {
|
|
106
|
+
return value === null || value === undefined ? null : String(value)
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function toEntry(row: Record<string, unknown>): AuditEntry {
|
|
110
|
+
return {
|
|
111
|
+
id: Number(row.id),
|
|
112
|
+
occurredAt: DbDate.fromDb(row.occurred_at) ?? new Date(0),
|
|
113
|
+
category: String(row.category) as AuditCategory,
|
|
114
|
+
action: String(row.action),
|
|
115
|
+
source: String(row.source) as AuditSource,
|
|
116
|
+
userKey: row.user_key ? normaliseUuid(String(row.user_key)) : null,
|
|
117
|
+
userName: text(row.user_name),
|
|
118
|
+
entityType: text(row.entity_type),
|
|
119
|
+
entityKey: text(row.entity_key),
|
|
120
|
+
entityName: text(row.entity_name),
|
|
121
|
+
ipAddress: text(row.ip_address),
|
|
122
|
+
forwardedFor: text(row.forwarded_for),
|
|
123
|
+
detail: text(row.detail),
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export class AuditLog {
|
|
128
|
+
#db: Db
|
|
129
|
+
|
|
130
|
+
constructor(db: Db) {
|
|
131
|
+
this.#db = db
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
async record(entries: AuditEntryInput | readonly AuditEntryInput[]): Promise<void> {
|
|
135
|
+
const list = Array.isArray(entries) ? entries : [entries as AuditEntryInput]
|
|
136
|
+
for (const entry of list) {
|
|
137
|
+
const userKey =
|
|
138
|
+
entry.userKey && UUID.test(entry.userKey) ? normaliseUuid(entry.userKey) : null
|
|
139
|
+
const entityKey =
|
|
140
|
+
entry.entityKey && UUID.test(entry.entityKey)
|
|
141
|
+
? normaliseUuid(entry.entityKey)
|
|
142
|
+
: entry.entityKey
|
|
143
|
+
await this.#db.exec(
|
|
144
|
+
`INSERT INTO audit_entry (occurred_at, category, action, user_key, user_name, entity_type,
|
|
145
|
+
entity_key, entity_name, source, ip_address, forwarded_for, detail)
|
|
146
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
147
|
+
[
|
|
148
|
+
DbDate.toDb(entry.occurredAt ?? new Date()),
|
|
149
|
+
entry.category,
|
|
150
|
+
clip(entry.action, LIMITS.action),
|
|
151
|
+
userKey,
|
|
152
|
+
clip(entry.userName, LIMITS.userName),
|
|
153
|
+
clip(entry.entityType, LIMITS.entityType),
|
|
154
|
+
clip(entityKey, LIMITS.entityKey),
|
|
155
|
+
clip(entry.entityName, LIMITS.entityName),
|
|
156
|
+
entry.source,
|
|
157
|
+
clip(entry.ipAddress, LIMITS.ipAddress),
|
|
158
|
+
clip(entry.forwardedFor, LIMITS.forwardedFor),
|
|
159
|
+
clip(entry.detail, LIMITS.detail),
|
|
160
|
+
],
|
|
161
|
+
)
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
async query(
|
|
166
|
+
filter: AuditFilter = {},
|
|
167
|
+
page: { skip: number; take: number } = { skip: 0, take: 100 },
|
|
168
|
+
): Promise<{ total: number; items: AuditEntry[] }> {
|
|
169
|
+
const where: string[] = []
|
|
170
|
+
const params: unknown[] = []
|
|
171
|
+
if (filter.userKey !== undefined) {
|
|
172
|
+
if (!UUID.test(filter.userKey)) return { total: 0, items: [] }
|
|
173
|
+
where.push('user_key = ?')
|
|
174
|
+
params.push(normaliseUuid(filter.userKey))
|
|
175
|
+
}
|
|
176
|
+
if (filter.entityKey !== undefined) {
|
|
177
|
+
where.push('entity_key = ?')
|
|
178
|
+
params.push(UUID.test(filter.entityKey) ? normaliseUuid(filter.entityKey) : filter.entityKey)
|
|
179
|
+
}
|
|
180
|
+
if (filter.category !== undefined) {
|
|
181
|
+
where.push('category = ?')
|
|
182
|
+
params.push(filter.category)
|
|
183
|
+
}
|
|
184
|
+
if (filter.action !== undefined) {
|
|
185
|
+
where.push('action = ?')
|
|
186
|
+
params.push(filter.action)
|
|
187
|
+
}
|
|
188
|
+
if (filter.since !== undefined) {
|
|
189
|
+
where.push('occurred_at >= ?')
|
|
190
|
+
params.push(DbDate.toDb(filter.since))
|
|
191
|
+
}
|
|
192
|
+
if (filter.until !== undefined) {
|
|
193
|
+
where.push('occurred_at < ?')
|
|
194
|
+
params.push(DbDate.toDb(filter.until))
|
|
195
|
+
}
|
|
196
|
+
const needle = filter.text?.trim().toLowerCase()
|
|
197
|
+
if (needle) {
|
|
198
|
+
const like = `%${needle.replace(/[\\%_]/g, (c) => `\\${c}`)}%`
|
|
199
|
+
where.push(
|
|
200
|
+
`(LOWER(entity_name) LIKE ? ESCAPE '\\' OR LOWER(entity_key) LIKE ? ESCAPE '\\' OR LOWER(user_name) LIKE ? ESCAPE '\\')`,
|
|
201
|
+
)
|
|
202
|
+
params.push(like, like, like)
|
|
203
|
+
}
|
|
204
|
+
const clause = where.length > 0 ? `WHERE ${where.join(' AND ')}` : ''
|
|
205
|
+
const counted = await this.#db.query<{ total: number }>(
|
|
206
|
+
`SELECT COUNT(*) AS total FROM audit_entry ${clause}`,
|
|
207
|
+
params,
|
|
208
|
+
)
|
|
209
|
+
const order = filter.ascending ? 'ASC' : 'DESC'
|
|
210
|
+
const rows = await this.#db.query(
|
|
211
|
+
`SELECT * FROM audit_entry ${clause} ORDER BY occurred_at ${order}, id ${order} LIMIT ? OFFSET ?`,
|
|
212
|
+
[...params, page.take, page.skip],
|
|
213
|
+
)
|
|
214
|
+
return { total: Number(counted[0]?.total ?? 0), items: rows.map(toEntry) }
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** Everyone the log names, by the name they had most recently. */
|
|
218
|
+
async users(): Promise<Array<{ key: string; name: string }>> {
|
|
219
|
+
const rows = await this.#db.query<{ user_key: string; user_name: string | null }>(
|
|
220
|
+
`SELECT e.user_key, e.user_name FROM audit_entry e
|
|
221
|
+
JOIN (SELECT user_key, MAX(id) AS id FROM audit_entry WHERE user_key IS NOT NULL GROUP BY user_key) latest
|
|
222
|
+
ON latest.id = e.id
|
|
223
|
+
ORDER BY e.user_name`,
|
|
224
|
+
)
|
|
225
|
+
return rows.map((row) => ({
|
|
226
|
+
key: normaliseUuid(String(row.user_key)),
|
|
227
|
+
name: row.user_name ? String(row.user_name) : '',
|
|
228
|
+
}))
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** Deletes entries older than `before`, and returns how many went. */
|
|
232
|
+
async prune(before: Date): Promise<number> {
|
|
233
|
+
const counted = await this.#db.query<{ total: number }>(
|
|
234
|
+
'SELECT COUNT(*) AS total FROM audit_entry WHERE occurred_at < ?',
|
|
235
|
+
[DbDate.toDb(before)],
|
|
236
|
+
)
|
|
237
|
+
const total = Number(counted[0]?.total ?? 0)
|
|
238
|
+
if (total > 0)
|
|
239
|
+
await this.#db.exec('DELETE FROM audit_entry WHERE occurred_at < ?', [DbDate.toDb(before)])
|
|
240
|
+
return total
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** When this database began keeping the log; history before it is in the content versions. */
|
|
244
|
+
async startedAt(): Promise<Date | undefined> {
|
|
245
|
+
const rows = await this.#db.query<{ applied_at: unknown }>(
|
|
246
|
+
'SELECT MIN(applied_at) AS applied_at FROM migration_history WHERE to_state = ?',
|
|
247
|
+
[STATE_AUDIT_LOG],
|
|
248
|
+
)
|
|
249
|
+
return DbDate.fromDb(rows[0]?.applied_at)
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* What a key names. A version answers for the item it belongs to, so a
|
|
254
|
+
* rollback is filed under the page it rolled back.
|
|
255
|
+
*/
|
|
256
|
+
async describe(entityType: string, key: string): Promise<AuditSubject> {
|
|
257
|
+
const named = (entityName: unknown, type = entityType, entityKey = key): AuditSubject => ({
|
|
258
|
+
entityType: type,
|
|
259
|
+
entityKey,
|
|
260
|
+
entityName: entityName === null || entityName === undefined ? null : String(entityName),
|
|
261
|
+
})
|
|
262
|
+
if (entityType.endsWith('Version') && /^\d+$/.test(key)) {
|
|
263
|
+
const rows = await this.#db.query<{ unique_id: string; text: string | null }>(
|
|
264
|
+
'SELECT n.unique_id, n.text FROM content_version cv JOIN node n ON n.id = cv.node_id WHERE cv.id = ?',
|
|
265
|
+
[Number(key)],
|
|
266
|
+
)
|
|
267
|
+
const row = rows[0]
|
|
268
|
+
const owner = entityType.slice(0, -'Version'.length)
|
|
269
|
+
return row ? named(row.text, owner, normaliseUuid(String(row.unique_id))) : named(null)
|
|
270
|
+
}
|
|
271
|
+
if (entityType === 'Language') {
|
|
272
|
+
const rows = await this.#db.query<{ culture_name: string }>(
|
|
273
|
+
'SELECT culture_name FROM language WHERE iso_code = ?',
|
|
274
|
+
[key],
|
|
275
|
+
)
|
|
276
|
+
return named(rows[0]?.culture_name ?? null)
|
|
277
|
+
}
|
|
278
|
+
if (!UUID.test(key)) return named(null)
|
|
279
|
+
const uuid = normaliseUuid(key)
|
|
280
|
+
const sql =
|
|
281
|
+
entityType === 'User'
|
|
282
|
+
? 'SELECT user_name AS name FROM user_account WHERE key = ?'
|
|
283
|
+
: entityType === 'UserGroup'
|
|
284
|
+
? 'SELECT name FROM user_group WHERE key = ?'
|
|
285
|
+
: entityType === 'DictionaryItem'
|
|
286
|
+
? 'SELECT item_key AS name FROM dictionary_item WHERE key = ?'
|
|
287
|
+
: 'SELECT text AS name FROM node WHERE unique_id = ?'
|
|
288
|
+
const rows = await this.#db.query<{ name: string | null }>(sql, [uuid])
|
|
289
|
+
return named(rows[0]?.name ?? null, entityType, uuid)
|
|
290
|
+
}
|
|
291
|
+
}
|
package/src/busy.ts
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/** Lock contention: recognising it in driver errors, retrying it, and the error an editor sees. */
|
|
2
|
+
|
|
3
|
+
export type Contention = 'deadlock' | 'lockTimeout' | 'busy'
|
|
4
|
+
|
|
5
|
+
/** A write gave up waiting for another to finish. Answered as 503 with Retry-After. */
|
|
6
|
+
export class ContentBusyError extends Error {
|
|
7
|
+
constructor(
|
|
8
|
+
readonly contention: Contention,
|
|
9
|
+
readonly retryAfterSeconds = 2,
|
|
10
|
+
) {
|
|
11
|
+
super('Someone else is changing this content right now. Try again in a moment.')
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const POSTGRES_CODES: Record<string, Contention> = {
|
|
16
|
+
'40P01': 'deadlock',
|
|
17
|
+
'40001': 'deadlock',
|
|
18
|
+
'55P03': 'lockTimeout',
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** Which kind of contention a driver error reports, if any. */
|
|
22
|
+
export function classifyContention(error: unknown): Contention | undefined {
|
|
23
|
+
if (error instanceof ContentBusyError) return error.contention
|
|
24
|
+
if (!error || typeof error !== 'object') return undefined
|
|
25
|
+
const { code, errno, message } = error as { code?: unknown; errno?: unknown; message?: unknown }
|
|
26
|
+
// Bun's Postgres driver reports the SQLSTATE as `errno`, other drivers as `code`.
|
|
27
|
+
for (const state of [errno, code])
|
|
28
|
+
if (typeof state === 'string' && POSTGRES_CODES[state]) return POSTGRES_CODES[state]
|
|
29
|
+
if (
|
|
30
|
+
typeof code === 'string' &&
|
|
31
|
+
(code.startsWith('SQLITE_BUSY') || code.startsWith('SQLITE_LOCKED'))
|
|
32
|
+
)
|
|
33
|
+
return 'busy'
|
|
34
|
+
if (errno === 5 || errno === 6) return 'busy'
|
|
35
|
+
if (typeof message === 'string' && /database (table )?is locked/i.test(message)) return 'busy'
|
|
36
|
+
return undefined
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface RetryPolicy {
|
|
40
|
+
/** Attempts in all, the first included. */
|
|
41
|
+
attempts: number
|
|
42
|
+
/** Waits between attempts; injected so a test does not sleep. */
|
|
43
|
+
sleep: (ms: number) => Promise<void>
|
|
44
|
+
backoffMs: number
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export const DEFAULT_RETRY: RetryPolicy = {
|
|
48
|
+
attempts: 2,
|
|
49
|
+
sleep: (ms) => Bun.sleep(ms),
|
|
50
|
+
backoffMs: 50,
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Runs `fn`, retrying a deadlock: the database rolled the transaction back, so
|
|
55
|
+
* running it again is safe. A lock timeout is not retried, since the wait has
|
|
56
|
+
* already been spent. Contention that outlasts the policy becomes ContentBusyError.
|
|
57
|
+
*/
|
|
58
|
+
export async function retryContention<T>(
|
|
59
|
+
fn: () => Promise<T>,
|
|
60
|
+
policy: RetryPolicy = DEFAULT_RETRY,
|
|
61
|
+
): Promise<T> {
|
|
62
|
+
for (let attempt = 1; ; attempt++) {
|
|
63
|
+
try {
|
|
64
|
+
return await fn()
|
|
65
|
+
} catch (error) {
|
|
66
|
+
const contention = classifyContention(error)
|
|
67
|
+
if (!contention) throw error
|
|
68
|
+
if (contention !== 'deadlock' || attempt >= policy.attempts) {
|
|
69
|
+
throw error instanceof ContentBusyError ? error : new ContentBusyError(contention)
|
|
70
|
+
}
|
|
71
|
+
await policy.sleep(policy.backoffMs * attempt)
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
package/src/change-report.ts
CHANGED
package/src/cluster.ts
CHANGED
|
@@ -20,6 +20,9 @@ export interface CacheInstruction {
|
|
|
20
20
|
createdBy: string | null
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
+
/** Where Postgres says an instruction was written, so nodes poll now rather than on their timer. */
|
|
24
|
+
export const CACHE_INSTRUCTION_CHANNEL = 'crombolo_cache_instruction'
|
|
25
|
+
|
|
23
26
|
export async function appendCacheInstruction(
|
|
24
27
|
db: Db,
|
|
25
28
|
instruction: { kind: CacheInstructionKind; payload?: Record<string, unknown>; by?: string },
|
|
@@ -33,6 +36,9 @@ export async function appendCacheInstruction(
|
|
|
33
36
|
instruction.by ?? null,
|
|
34
37
|
],
|
|
35
38
|
)
|
|
39
|
+
// Delivered at commit, so a write that rolls back wakes nobody.
|
|
40
|
+
if (db.dialect.name === 'postgres')
|
|
41
|
+
await db.exec('SELECT pg_notify(?, ?)', [CACHE_INSTRUCTION_CHANNEL, instruction.kind])
|
|
36
42
|
}
|
|
37
43
|
|
|
38
44
|
/** The highest instruction id, so a fresh node ignores history. */
|
|
@@ -60,25 +66,28 @@ export interface ServerIdentity {
|
|
|
60
66
|
revision: string
|
|
61
67
|
/** `all`, `api` or `web`: what this node serves, so the cluster can be read back by role. */
|
|
62
68
|
role?: string
|
|
69
|
+
/** The Crombolo release this node runs; `version` is the schema set's. */
|
|
70
|
+
cromboloVersion?: string
|
|
63
71
|
}
|
|
64
72
|
|
|
65
73
|
/** Records this node in `server`; called at boot and on each poll. */
|
|
66
74
|
export async function touchServer(db: Db, identity: ServerIdentity): Promise<void> {
|
|
67
75
|
const now = DbDate.toDb(new Date())
|
|
68
76
|
const role = identity.role ?? null
|
|
77
|
+
const crombolo = identity.cromboloVersion ?? null
|
|
69
78
|
const existing = await db.query<{ node_id: string }>(
|
|
70
79
|
'SELECT node_id FROM server WHERE node_id = ?',
|
|
71
80
|
[identity.nodeId],
|
|
72
81
|
)
|
|
73
82
|
if (existing[0]) {
|
|
74
83
|
await db.exec(
|
|
75
|
-
'UPDATE server SET version = ?, revision = ?, role = ?, last_seen = ? WHERE node_id = ?',
|
|
76
|
-
[identity.version, identity.revision, role, now, identity.nodeId],
|
|
84
|
+
'UPDATE server SET version = ?, revision = ?, role = ?, crombolo_version = ?, last_seen = ? WHERE node_id = ?',
|
|
85
|
+
[identity.version, identity.revision, role, crombolo, now, identity.nodeId],
|
|
77
86
|
)
|
|
78
87
|
} else {
|
|
79
88
|
await db.exec(
|
|
80
|
-
'INSERT INTO server (node_id, version, revision, role, last_seen) VALUES (?, ?, ?, ?, ?)',
|
|
81
|
-
[identity.nodeId, identity.version, identity.revision, role, now],
|
|
89
|
+
'INSERT INTO server (node_id, version, revision, role, crombolo_version, last_seen) VALUES (?, ?, ?, ?, ?, ?)',
|
|
90
|
+
[identity.nodeId, identity.version, identity.revision, role, crombolo, now],
|
|
82
91
|
)
|
|
83
92
|
}
|
|
84
93
|
}
|
|
@@ -89,13 +98,15 @@ export interface LiveNode {
|
|
|
89
98
|
role: string | null
|
|
90
99
|
version: string | null
|
|
91
100
|
revision: string | null
|
|
101
|
+
/** Null for a row written before node releases were recorded. */
|
|
102
|
+
cromboloVersion: string | null
|
|
92
103
|
lastSeen: Date
|
|
93
104
|
}
|
|
94
105
|
|
|
95
106
|
/** The nodes that have polled within `seenWithinMs`: the cluster as it stands. */
|
|
96
107
|
export async function liveNodes(db: Db, options: { seenWithinMs: number }): Promise<LiveNode[]> {
|
|
97
108
|
const rows = await db.query(
|
|
98
|
-
'SELECT node_id, role, version, revision, last_seen FROM server WHERE last_seen >= ? ORDER BY node_id',
|
|
109
|
+
'SELECT node_id, role, version, revision, crombolo_version, last_seen FROM server WHERE last_seen >= ? ORDER BY node_id',
|
|
99
110
|
[DbDate.toDb(new Date(Date.now() - options.seenWithinMs))],
|
|
100
111
|
)
|
|
101
112
|
return rows.map((row) => ({
|
|
@@ -103,6 +114,7 @@ export async function liveNodes(db: Db, options: { seenWithinMs: number }): Prom
|
|
|
103
114
|
role: (row.role as string | null) ?? null,
|
|
104
115
|
version: (row.version as string | null) ?? null,
|
|
105
116
|
revision: (row.revision as string | null) ?? null,
|
|
117
|
+
cromboloVersion: (row.crombolo_version as string | null) ?? null,
|
|
106
118
|
lastSeen: DbDate.fromDb(row.last_seen) as Date,
|
|
107
119
|
}))
|
|
108
120
|
}
|
package/src/database.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
/** Driver-agnostic database handle. */
|
|
2
|
+
|
|
2
3
|
import { Database as BunSqlite } from 'bun:sqlite'
|
|
4
|
+
import { ambient, ambientFor } from './ambient.ts'
|
|
3
5
|
import { type Dialect, type DialectName, dialectFor } from './dialect.ts'
|
|
4
|
-
import { createLockManager, type LockManager } from './locks.ts'
|
|
6
|
+
import { createLockManager, DEFAULT_LOCK_TIMEOUT_MS, type LockManager } from './locks.ts'
|
|
5
7
|
|
|
6
8
|
export interface Db {
|
|
7
9
|
readonly dialect: Dialect
|
|
@@ -10,6 +12,8 @@ export interface Db {
|
|
|
10
12
|
exec(sql: string, params?: unknown[]): Promise<void>
|
|
11
13
|
transaction<T>(fn: (tx: Db) => Promise<T>): Promise<T>
|
|
12
14
|
close(): Promise<void>
|
|
15
|
+
/** Postgres only: calls `onNotify` for each `NOTIFY` on `channel`; resolves to the way to stop. */
|
|
16
|
+
listen?(channel: string, onNotify: (payload: string) => void): Promise<() => Promise<void>>
|
|
13
17
|
}
|
|
14
18
|
|
|
15
19
|
export interface ConnectOptions {
|
|
@@ -20,18 +24,40 @@ export interface ConnectOptions {
|
|
|
20
24
|
file?: string
|
|
21
25
|
/** Postgres connection string. Ignored for SQLite. */
|
|
22
26
|
url?: string
|
|
27
|
+
/** How long a write waits for a content lock; defaults to `CROMBOLO_LOCK_TIMEOUT_SECONDS`, else 15s. */
|
|
28
|
+
lockTimeoutMs?: number
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function resolveLockTimeoutMs(options: ConnectOptions = {}): number {
|
|
32
|
+
if (options.lockTimeoutMs !== undefined) return options.lockTimeoutMs
|
|
33
|
+
const seconds = Number(Bun.env.CROMBOLO_LOCK_TIMEOUT_SECONDS)
|
|
34
|
+
return Number.isFinite(seconds) && seconds > 0 ? seconds * 1000 : DEFAULT_LOCK_TIMEOUT_MS
|
|
23
35
|
}
|
|
24
36
|
|
|
37
|
+
/**
|
|
38
|
+
* One connection has one transaction at a time, but async chains interleave:
|
|
39
|
+
* a background job's BEGIN between a request's BEGIN and COMMIT would fail, or
|
|
40
|
+
* worse, land its statements in the request's transaction. So top-level
|
|
41
|
+
* transactions queue here, and a chain already inside one joins it.
|
|
42
|
+
*/
|
|
25
43
|
class SqliteDb implements Db {
|
|
26
44
|
readonly dialect = dialectFor('sqlite')
|
|
27
45
|
readonly locks: LockManager
|
|
28
46
|
#db: BunSqlite
|
|
29
47
|
#inTransaction: boolean
|
|
30
|
-
|
|
31
|
-
|
|
48
|
+
#gate: { tail: Promise<void> }
|
|
49
|
+
|
|
50
|
+
constructor(
|
|
51
|
+
db: BunSqlite,
|
|
52
|
+
locks?: LockManager,
|
|
53
|
+
inTransaction = false,
|
|
54
|
+
gate: { tail: Promise<void> } = { tail: Promise.resolve() },
|
|
55
|
+
lockTimeoutMs?: number,
|
|
56
|
+
) {
|
|
32
57
|
this.#db = db
|
|
33
|
-
this.locks = locks ?? createLockManager(this)
|
|
58
|
+
this.locks = locks ?? createLockManager(this, { timeoutMs: lockTimeoutMs })
|
|
34
59
|
this.#inTransaction = inTransaction
|
|
60
|
+
this.#gate = gate
|
|
35
61
|
}
|
|
36
62
|
|
|
37
63
|
async query<T = Record<string, unknown>>(sql: string, params: unknown[] = []): Promise<T[]> {
|
|
@@ -44,14 +70,29 @@ class SqliteDb implements Db {
|
|
|
44
70
|
|
|
45
71
|
async transaction<T>(fn: (tx: Db) => Promise<T>): Promise<T> {
|
|
46
72
|
if (this.#inTransaction) return fn(this)
|
|
47
|
-
this.#db.
|
|
73
|
+
const tx = new SqliteDb(this.#db, this.locks, true, this.#gate)
|
|
74
|
+
if (ambientFor(this.#db)) return fn(tx)
|
|
75
|
+
const previous = this.#gate.tail
|
|
76
|
+
let release!: () => void
|
|
77
|
+
this.#gate.tail = new Promise<void>((resolve) => {
|
|
78
|
+
release = resolve
|
|
79
|
+
})
|
|
80
|
+
await previous
|
|
48
81
|
try {
|
|
49
|
-
|
|
50
|
-
this.#db
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
82
|
+
this.#db.exec('BEGIN')
|
|
83
|
+
const store = { owner: this.#db, tx: tx as unknown }
|
|
84
|
+
try {
|
|
85
|
+
const result = await ambient.run(store, () => fn(tx))
|
|
86
|
+
this.#db.exec('COMMIT')
|
|
87
|
+
return result
|
|
88
|
+
} catch (error) {
|
|
89
|
+
this.#db.exec('ROLLBACK')
|
|
90
|
+
throw error
|
|
91
|
+
} finally {
|
|
92
|
+
store.tx = undefined
|
|
93
|
+
}
|
|
94
|
+
} finally {
|
|
95
|
+
release()
|
|
55
96
|
}
|
|
56
97
|
}
|
|
57
98
|
|
|
@@ -64,6 +105,10 @@ interface SqlTag {
|
|
|
64
105
|
unsafe(sql: string, params?: unknown[]): Promise<unknown>
|
|
65
106
|
begin<T>(fn: (tx: SqlTag) => Promise<T>): Promise<T>
|
|
66
107
|
close(): Promise<void>
|
|
108
|
+
listen(
|
|
109
|
+
channel: string,
|
|
110
|
+
onNotify: (payload: string) => void,
|
|
111
|
+
): Promise<{ unlisten(): Promise<void> }>
|
|
67
112
|
}
|
|
68
113
|
|
|
69
114
|
class PostgresDb implements Db {
|
|
@@ -72,9 +117,9 @@ class PostgresDb implements Db {
|
|
|
72
117
|
#sql: SqlTag
|
|
73
118
|
#inTransaction: boolean
|
|
74
119
|
|
|
75
|
-
constructor(sql: SqlTag, locks?: LockManager, inTransaction = false) {
|
|
120
|
+
constructor(sql: SqlTag, locks?: LockManager, inTransaction = false, lockTimeoutMs?: number) {
|
|
76
121
|
this.#sql = sql
|
|
77
|
-
this.locks = locks ?? createLockManager(this)
|
|
122
|
+
this.locks = locks ?? createLockManager(this, { timeoutMs: lockTimeoutMs })
|
|
78
123
|
this.#inTransaction = inTransaction
|
|
79
124
|
}
|
|
80
125
|
|
|
@@ -86,15 +131,33 @@ class PostgresDb implements Db {
|
|
|
86
131
|
await this.#sql.unsafe(this.dialect.render(sql), params)
|
|
87
132
|
}
|
|
88
133
|
|
|
89
|
-
/**
|
|
134
|
+
/**
|
|
135
|
+
* Nested calls join the enclosing transaction, as they do on SQLite. The
|
|
136
|
+
* chain is marked as inside one, so a deadlock is not retried into a
|
|
137
|
+
* transaction Postgres has already aborted.
|
|
138
|
+
*/
|
|
90
139
|
async transaction<T>(fn: (tx: Db) => Promise<T>): Promise<T> {
|
|
91
140
|
if (this.#inTransaction) return fn(this)
|
|
92
|
-
return this.#sql.begin(async (
|
|
141
|
+
return this.#sql.begin(async (sql) => {
|
|
142
|
+
const tx = new PostgresDb(sql, this.locks, true)
|
|
143
|
+
const store = { owner: this, tx: tx as unknown }
|
|
144
|
+
try {
|
|
145
|
+
return await ambient.run(store, () => fn(tx))
|
|
146
|
+
} finally {
|
|
147
|
+
store.tx = undefined
|
|
148
|
+
}
|
|
149
|
+
})
|
|
93
150
|
}
|
|
94
151
|
|
|
95
152
|
async close(): Promise<void> {
|
|
96
153
|
await this.#sql.close()
|
|
97
154
|
}
|
|
155
|
+
|
|
156
|
+
async listen(channel: string, onNotify: (payload: string) => void): Promise<() => Promise<void>> {
|
|
157
|
+
if (this.#inTransaction) throw new Error('A transaction cannot listen.')
|
|
158
|
+
const subscription = await this.#sql.listen(channel, onNotify)
|
|
159
|
+
return () => subscription.unlisten()
|
|
160
|
+
}
|
|
98
161
|
}
|
|
99
162
|
|
|
100
163
|
export function resolveDialectName(options: ConnectOptions = {}): DialectName {
|
|
@@ -109,7 +172,7 @@ export async function connect(options: ConnectOptions = {}): Promise<Db> {
|
|
|
109
172
|
// A modest pool: many short-lived connections exhaust Postgres' client limit,
|
|
110
173
|
// and a single process does not need more.
|
|
111
174
|
const sql = new SQL({ url, max: options.poolSize ?? 5 }) as unknown as SqlTag
|
|
112
|
-
const db = new PostgresDb(sql)
|
|
175
|
+
const db = new PostgresDb(sql, undefined, false, resolveLockTimeoutMs(options))
|
|
113
176
|
await db.exec('CREATE EXTENSION IF NOT EXISTS citext')
|
|
114
177
|
return db
|
|
115
178
|
}
|
|
@@ -117,5 +180,5 @@ export async function connect(options: ConnectOptions = {}): Promise<Db> {
|
|
|
117
180
|
sqlite.exec('PRAGMA journal_mode = WAL')
|
|
118
181
|
sqlite.exec('PRAGMA foreign_keys = ON')
|
|
119
182
|
sqlite.exec('PRAGMA busy_timeout = 5000')
|
|
120
|
-
return new SqliteDb(sqlite)
|
|
183
|
+
return new SqliteDb(sqlite, undefined, false, undefined, resolveLockTimeoutMs(options))
|
|
121
184
|
}
|