@crombolo/data 0.10.0 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/ambient.ts +20 -0
- package/src/audit.ts +291 -0
- package/src/busy.ts +74 -0
- package/src/change-report.ts +6 -0
- package/src/cluster.ts +17 -5
- package/src/database.ts +80 -17
- package/src/dialect.ts +24 -0
- package/src/index.ts +20 -0
- package/src/locks.ts +94 -25
- package/src/migrations.ts +42 -16
- package/src/relations.ts +346 -0
- package/src/repositories/content-types.ts +41 -4
- package/src/repositories/data-types.ts +38 -0
- package/src/repositories/delivery-member-tokens.ts +110 -0
- package/src/repositories/dictionary.ts +18 -0
- package/src/repositories/documents.ts +443 -88
- package/src/repositories/dynamic-roots.ts +151 -0
- package/src/repositories/form-entries.ts +32 -11
- package/src/repositories/form-workflows.ts +1 -1
- package/src/repositories/members.ts +26 -1
- package/src/repositories/nodes.ts +43 -7
- package/src/repositories/public-access.ts +30 -0
- package/src/repositories/references.ts +15 -0
- package/src/repositories/transfer-answers.ts +151 -0
- package/src/repositories/webhooks.ts +299 -0
- package/src/schema/audit-log.ts +45 -0
- package/src/schema/delivery-member-tokens.ts +40 -0
- package/src/schema/diagnostics.ts +28 -0
- package/src/schema/plan.ts +14 -0
- package/src/schema/property-searchable.ts +29 -0
- package/src/schema/search-index.ts +92 -0
- package/src/schema/seed.ts +13 -2
- package/src/schema/transfer-answers.ts +50 -0
- package/src/schema/webhooks.ts +78 -0
- package/src/search-text.ts +133 -0
- package/src/search.ts +397 -0
- package/src/setup.ts +57 -0
- package/src/tree-invariants.ts +167 -0
- package/src/url-options.ts +61 -0
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Webhooks kept in this environment, and the delivery queue and log every
|
|
3
|
+
* webhook and bundle event handler shares. docs/webhooks.md.
|
|
4
|
+
*
|
|
5
|
+
* A delivery is claimed with `UPDATE … WHERE state = 'pending' … RETURNING`, as
|
|
6
|
+
* form workflows are, so on a cluster exactly one node delivers each.
|
|
7
|
+
*/
|
|
8
|
+
import { randomUUID } from 'node:crypto'
|
|
9
|
+
import type { Db } from '../database.ts'
|
|
10
|
+
import { DbDate, fromDbBool } from '../dialect.ts'
|
|
11
|
+
|
|
12
|
+
export interface WebhookRow {
|
|
13
|
+
key: string
|
|
14
|
+
name: string | null
|
|
15
|
+
description: string | null
|
|
16
|
+
url: string
|
|
17
|
+
enabled: boolean
|
|
18
|
+
events: string[]
|
|
19
|
+
contentTypeKeys: string[]
|
|
20
|
+
/** The headers as one sealed value; the server holds the key that opens it. */
|
|
21
|
+
sealedHeaders: string | null
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const parseList = (value: unknown): string[] => {
|
|
25
|
+
try {
|
|
26
|
+
const parsed = JSON.parse(String(value ?? '[]')) as unknown
|
|
27
|
+
return Array.isArray(parsed) ? parsed.filter((item) => typeof item === 'string') : []
|
|
28
|
+
} catch {
|
|
29
|
+
return []
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export class WebhookRepository {
|
|
34
|
+
#db: Db
|
|
35
|
+
constructor(db: Db) {
|
|
36
|
+
this.#db = db
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async all(): Promise<WebhookRow[]> {
|
|
40
|
+
const rows = await this.#db.query(
|
|
41
|
+
'SELECT key, name, description, url, enabled, events, content_type_keys, headers FROM webhook ORDER BY create_date, key',
|
|
42
|
+
)
|
|
43
|
+
return rows.map((row) => ({
|
|
44
|
+
key: String(row.key),
|
|
45
|
+
name: (row.name as string | null) ?? null,
|
|
46
|
+
description: (row.description as string | null) ?? null,
|
|
47
|
+
url: String(row.url),
|
|
48
|
+
enabled: fromDbBool(row.enabled),
|
|
49
|
+
events: parseList(row.events),
|
|
50
|
+
contentTypeKeys: parseList(row.content_type_keys),
|
|
51
|
+
sealedHeaders: (row.headers as string | null) ?? null,
|
|
52
|
+
}))
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async byKey(key: string): Promise<WebhookRow | undefined> {
|
|
56
|
+
return (await this.all()).find((row) => row.key === key.toLowerCase())
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async save(webhook: WebhookRow): Promise<void> {
|
|
60
|
+
const now = DbDate.toDb(new Date())
|
|
61
|
+
const values = [
|
|
62
|
+
webhook.name,
|
|
63
|
+
webhook.description,
|
|
64
|
+
webhook.url,
|
|
65
|
+
this.#db.dialect.boolValue(webhook.enabled),
|
|
66
|
+
JSON.stringify(webhook.events),
|
|
67
|
+
JSON.stringify(webhook.contentTypeKeys),
|
|
68
|
+
webhook.sealedHeaders,
|
|
69
|
+
now,
|
|
70
|
+
]
|
|
71
|
+
const updated = await this.#db.query(
|
|
72
|
+
`UPDATE webhook SET name = ?, description = ?, url = ?, enabled = ?, events = ?,
|
|
73
|
+
content_type_keys = ?, headers = ?, update_date = ?
|
|
74
|
+
WHERE key = ? RETURNING key`,
|
|
75
|
+
[...values, webhook.key.toLowerCase()],
|
|
76
|
+
)
|
|
77
|
+
if (updated.length > 0) return
|
|
78
|
+
await this.#db.exec(
|
|
79
|
+
`INSERT INTO webhook (name, description, url, enabled, events, content_type_keys, headers,
|
|
80
|
+
update_date, key, create_date)
|
|
81
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
82
|
+
[...values, webhook.key.toLowerCase(), now],
|
|
83
|
+
)
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async remove(key: string): Promise<boolean> {
|
|
87
|
+
const rows = await this.#db.query('DELETE FROM webhook WHERE key = ? RETURNING key', [
|
|
88
|
+
key.toLowerCase(),
|
|
89
|
+
])
|
|
90
|
+
return rows.length > 0
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export type WebhookDeliveryState = 'pending' | 'running' | 'done' | 'failed'
|
|
95
|
+
|
|
96
|
+
export interface WebhookDelivery {
|
|
97
|
+
id: string
|
|
98
|
+
/** A webhook's key, or `bundle:<id>` for a bundle's handler. */
|
|
99
|
+
subscriber: string
|
|
100
|
+
eventAlias: string
|
|
101
|
+
payload: unknown
|
|
102
|
+
state: WebhookDeliveryState
|
|
103
|
+
attempts: number
|
|
104
|
+
lastError: string | null
|
|
105
|
+
runAfter: Date
|
|
106
|
+
createDate: Date
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export interface WebhookLogEntry {
|
|
110
|
+
key: string
|
|
111
|
+
deliveryId: string
|
|
112
|
+
subscriber: string
|
|
113
|
+
eventAlias: string
|
|
114
|
+
url: string
|
|
115
|
+
statusCode: number | null
|
|
116
|
+
success: boolean
|
|
117
|
+
exception: boolean
|
|
118
|
+
retryCount: number
|
|
119
|
+
requestHeaders: string
|
|
120
|
+
requestBody: string
|
|
121
|
+
responseHeaders: string
|
|
122
|
+
responseBody: string
|
|
123
|
+
date: Date
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const DELIVERY_COLUMNS =
|
|
127
|
+
'id, subscriber, event_alias, payload, state, attempts, last_error, run_after, create_date'
|
|
128
|
+
|
|
129
|
+
const hydrateDelivery = (row: Record<string, unknown>): WebhookDelivery => {
|
|
130
|
+
let payload: unknown = null
|
|
131
|
+
try {
|
|
132
|
+
payload = JSON.parse(String(row.payload))
|
|
133
|
+
} catch {
|
|
134
|
+
payload = null
|
|
135
|
+
}
|
|
136
|
+
return {
|
|
137
|
+
id: String(row.id),
|
|
138
|
+
subscriber: String(row.subscriber),
|
|
139
|
+
eventAlias: String(row.event_alias),
|
|
140
|
+
payload,
|
|
141
|
+
state: String(row.state) as WebhookDeliveryState,
|
|
142
|
+
attempts: Number(row.attempts),
|
|
143
|
+
lastError: (row.last_error as string | null) ?? null,
|
|
144
|
+
runAfter: DbDate.fromDb(row.run_after) ?? new Date(0),
|
|
145
|
+
createDate: DbDate.fromDb(row.create_date) ?? new Date(0),
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** Bodies and headers are kept to what a person will read, not what a server sent. */
|
|
150
|
+
const clip = (value: string) => value.slice(0, 10_000)
|
|
151
|
+
|
|
152
|
+
export class WebhookDeliveryRepository {
|
|
153
|
+
#db: Db
|
|
154
|
+
constructor(db: Db) {
|
|
155
|
+
this.#db = db
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
async enqueue(
|
|
159
|
+
deliveries: ReadonlyArray<{ subscriber: string; eventAlias: string; payload: unknown }>,
|
|
160
|
+
now = new Date(),
|
|
161
|
+
): Promise<string[]> {
|
|
162
|
+
const ids: string[] = []
|
|
163
|
+
for (const delivery of deliveries) {
|
|
164
|
+
const id = randomUUID()
|
|
165
|
+
await this.#db.exec(
|
|
166
|
+
`INSERT INTO webhook_delivery (${DELIVERY_COLUMNS}, update_date)
|
|
167
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
168
|
+
[
|
|
169
|
+
id,
|
|
170
|
+
delivery.subscriber,
|
|
171
|
+
delivery.eventAlias,
|
|
172
|
+
JSON.stringify(delivery.payload),
|
|
173
|
+
'pending',
|
|
174
|
+
0,
|
|
175
|
+
null,
|
|
176
|
+
DbDate.toDb(now),
|
|
177
|
+
DbDate.toDb(now),
|
|
178
|
+
DbDate.toDb(now),
|
|
179
|
+
],
|
|
180
|
+
)
|
|
181
|
+
ids.push(id)
|
|
182
|
+
}
|
|
183
|
+
return ids
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
async due(now = new Date(), limit = 25): Promise<WebhookDelivery[]> {
|
|
187
|
+
const rows = await this.#db.query(
|
|
188
|
+
`SELECT ${DELIVERY_COLUMNS} FROM webhook_delivery WHERE state = ? AND run_after <= ?
|
|
189
|
+
ORDER BY run_after, create_date LIMIT ?`,
|
|
190
|
+
['pending', DbDate.toDb(now), limit],
|
|
191
|
+
)
|
|
192
|
+
return rows.map(hydrateDelivery)
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/** Takes a delivery, or finds another node already has. */
|
|
196
|
+
async claim(id: string): Promise<WebhookDelivery | undefined> {
|
|
197
|
+
const rows = await this.#db.query(
|
|
198
|
+
`UPDATE webhook_delivery SET state = ?, attempts = attempts + 1, update_date = ?
|
|
199
|
+
WHERE id = ? AND state = ? RETURNING ${DELIVERY_COLUMNS}`,
|
|
200
|
+
['running', DbDate.toDb(new Date()), id, 'pending'],
|
|
201
|
+
)
|
|
202
|
+
return rows[0] ? hydrateDelivery(rows[0]) : undefined
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
async succeed(id: string): Promise<void> {
|
|
206
|
+
await this.#db.exec(
|
|
207
|
+
'UPDATE webhook_delivery SET state = ?, last_error = NULL, update_date = ? WHERE id = ?',
|
|
208
|
+
['done', DbDate.toDb(new Date()), id],
|
|
209
|
+
)
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/** `retryAt` puts it back in the queue; without one it has failed for good. */
|
|
213
|
+
async fail(id: string, error: string, retryAt?: Date): Promise<void> {
|
|
214
|
+
await this.#db.exec(
|
|
215
|
+
'UPDATE webhook_delivery SET state = ?, last_error = ?, run_after = ?, update_date = ? WHERE id = ?',
|
|
216
|
+
[
|
|
217
|
+
retryAt ? 'pending' : 'failed',
|
|
218
|
+
error.slice(0, 2000),
|
|
219
|
+
DbDate.toDb(retryAt ?? new Date()),
|
|
220
|
+
DbDate.toDb(new Date()),
|
|
221
|
+
id,
|
|
222
|
+
],
|
|
223
|
+
)
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
async byId(id: string): Promise<WebhookDelivery | undefined> {
|
|
227
|
+
const rows = await this.#db.query(
|
|
228
|
+
`SELECT ${DELIVERY_COLUMNS} FROM webhook_delivery WHERE id = ?`,
|
|
229
|
+
[id],
|
|
230
|
+
)
|
|
231
|
+
return rows[0] ? hydrateDelivery(rows[0]) : undefined
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
async log(entry: Omit<WebhookLogEntry, 'key' | 'date'>, date = new Date()): Promise<void> {
|
|
235
|
+
await this.#db.exec(
|
|
236
|
+
`INSERT INTO webhook_log (key, delivery_id, subscriber, event_alias, url, status_code, success,
|
|
237
|
+
exception, retry_count, request_headers, request_body, response_headers, response_body, log_date)
|
|
238
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
239
|
+
[
|
|
240
|
+
randomUUID(),
|
|
241
|
+
entry.deliveryId,
|
|
242
|
+
entry.subscriber,
|
|
243
|
+
entry.eventAlias,
|
|
244
|
+
entry.url,
|
|
245
|
+
entry.statusCode,
|
|
246
|
+
this.#db.dialect.boolValue(entry.success),
|
|
247
|
+
this.#db.dialect.boolValue(entry.exception),
|
|
248
|
+
entry.retryCount,
|
|
249
|
+
clip(entry.requestHeaders),
|
|
250
|
+
clip(entry.requestBody),
|
|
251
|
+
clip(entry.responseHeaders),
|
|
252
|
+
clip(entry.responseBody),
|
|
253
|
+
DbDate.toDb(date),
|
|
254
|
+
],
|
|
255
|
+
)
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** The log, newest first; one subscriber's when named. */
|
|
259
|
+
async logs(options: {
|
|
260
|
+
subscriber?: string
|
|
261
|
+
skip: number
|
|
262
|
+
take: number
|
|
263
|
+
}): Promise<{ total: number; items: WebhookLogEntry[] }> {
|
|
264
|
+
const where = options.subscriber ? 'WHERE subscriber = ?' : ''
|
|
265
|
+
const params = options.subscriber ? [options.subscriber] : []
|
|
266
|
+
const total = await this.#db.query<{ n: number }>(
|
|
267
|
+
`SELECT COUNT(*) AS n FROM webhook_log ${where}`,
|
|
268
|
+
params,
|
|
269
|
+
)
|
|
270
|
+
const rows = await this.#db.query(
|
|
271
|
+
`SELECT key, delivery_id, subscriber, event_alias, url, status_code, success, exception,
|
|
272
|
+
retry_count, request_headers, request_body, response_headers, response_body, log_date
|
|
273
|
+
FROM webhook_log ${where} ORDER BY log_date DESC, retry_count DESC, key LIMIT ? OFFSET ?`,
|
|
274
|
+
[...params, options.take, options.skip],
|
|
275
|
+
)
|
|
276
|
+
return {
|
|
277
|
+
total: Number(total[0]?.n ?? 0),
|
|
278
|
+
items: rows.map((row) => ({
|
|
279
|
+
key: String(row.key),
|
|
280
|
+
deliveryId: String(row.delivery_id),
|
|
281
|
+
subscriber: String(row.subscriber),
|
|
282
|
+
eventAlias: String(row.event_alias),
|
|
283
|
+
url: String(row.url),
|
|
284
|
+
statusCode:
|
|
285
|
+
row.status_code === null || row.status_code === undefined
|
|
286
|
+
? null
|
|
287
|
+
: Number(row.status_code),
|
|
288
|
+
success: fromDbBool(row.success),
|
|
289
|
+
exception: fromDbBool(row.exception),
|
|
290
|
+
retryCount: Number(row.retry_count),
|
|
291
|
+
requestHeaders: String(row.request_headers),
|
|
292
|
+
requestBody: String(row.request_body),
|
|
293
|
+
responseHeaders: String(row.response_headers),
|
|
294
|
+
responseBody: String(row.response_body),
|
|
295
|
+
date: DbDate.fromDb(row.log_date) ?? new Date(0),
|
|
296
|
+
})),
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Migration 033 — the audit log. docs/audit-log.md.
|
|
3
|
+
*
|
|
4
|
+
* `audit_entry` records who did what, when and from where: every backoffice
|
|
5
|
+
* change, every sign-in and failed sign-in, and what the command line and the
|
|
6
|
+
* background jobs changed. Names are copied in when an entry is written, so an
|
|
7
|
+
* entry still reads after its user or item is gone. It starts empty: history from
|
|
8
|
+
* before this migration stays in the content versions it came from.
|
|
9
|
+
*/
|
|
10
|
+
import type { Db } from '../database.ts'
|
|
11
|
+
import type { Migration } from '../migrations.ts'
|
|
12
|
+
import { STATE_PROPERTY_SEARCHABLE } from './property-searchable.ts'
|
|
13
|
+
|
|
14
|
+
export const STATE_AUDIT_LOG = '{6f1d4a8e-0033-4c1a-9e7d-3a5b8c2d4e33}'
|
|
15
|
+
|
|
16
|
+
export const auditLogMigration: Migration = {
|
|
17
|
+
from: STATE_PROPERTY_SEARCHABLE,
|
|
18
|
+
to: STATE_AUDIT_LOG,
|
|
19
|
+
name: 'AddAuditLog',
|
|
20
|
+
kind: 'expand',
|
|
21
|
+
release: '0.10.0',
|
|
22
|
+
async up(db: Db) {
|
|
23
|
+
const t = db.dialect.types
|
|
24
|
+
await db.exec(
|
|
25
|
+
`CREATE TABLE audit_entry (
|
|
26
|
+
id ${t.identity},
|
|
27
|
+
occurred_at ${t.timestamp} NOT NULL,
|
|
28
|
+
category ${t.varchar(32)} NOT NULL,
|
|
29
|
+
action ${t.varchar(64)} NOT NULL,
|
|
30
|
+
user_key ${t.uuid},
|
|
31
|
+
user_name ${t.varchar(255)},
|
|
32
|
+
entity_type ${t.varchar(64)},
|
|
33
|
+
entity_key ${t.varchar(255)},
|
|
34
|
+
entity_name ${t.varchar(255)},
|
|
35
|
+
source ${t.varchar(16)} NOT NULL,
|
|
36
|
+
ip_address ${t.varchar(64)},
|
|
37
|
+
forwarded_for ${t.varchar(255)},
|
|
38
|
+
detail ${t.text}
|
|
39
|
+
)`,
|
|
40
|
+
)
|
|
41
|
+
await db.exec('CREATE INDEX ix_audit_entry_occurred ON audit_entry (occurred_at)')
|
|
42
|
+
await db.exec('CREATE INDEX ix_audit_entry_entity ON audit_entry (entity_key, occurred_at)')
|
|
43
|
+
await db.exec('CREATE INDEX ix_audit_entry_user ON audit_entry (user_key, occurred_at)')
|
|
44
|
+
},
|
|
45
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Migration 037 — the Delivery API's member tokens (story 050):
|
|
3
|
+
* authorization codes, access and refresh tokens, each stored as the SHA-256
|
|
4
|
+
* of what the client holds, never the token itself. docs/delivery-api.md,
|
|
5
|
+
* "Member sign-in".
|
|
6
|
+
*/
|
|
7
|
+
import type { Db } from '../database.ts'
|
|
8
|
+
import type { Migration } from '../migrations.ts'
|
|
9
|
+
import { STATE_TRANSFER_ANSWERS } from './transfer-answers.ts'
|
|
10
|
+
|
|
11
|
+
export const STATE_DELIVERY_MEMBER_TOKENS = '{7d3b9e2c-0037-4a51-b8f6-2c4e1a9d5b37}'
|
|
12
|
+
|
|
13
|
+
export const deliveryMemberTokensMigration: Migration = {
|
|
14
|
+
from: STATE_TRANSFER_ANSWERS,
|
|
15
|
+
to: STATE_DELIVERY_MEMBER_TOKENS,
|
|
16
|
+
name: 'AddDeliveryMemberTokens',
|
|
17
|
+
kind: 'expand',
|
|
18
|
+
release: '0.12.0',
|
|
19
|
+
async up(db: Db) {
|
|
20
|
+
const t = db.dialect.types
|
|
21
|
+
await db.exec(
|
|
22
|
+
`CREATE TABLE delivery_member_token (
|
|
23
|
+
hash ${t.varchar(64)} NOT NULL,
|
|
24
|
+
kind ${t.varchar(16)} NOT NULL,
|
|
25
|
+
member_key ${t.varchar(64)} NOT NULL,
|
|
26
|
+
client_id ${t.varchar(255)} NOT NULL,
|
|
27
|
+
scopes ${t.varchar(255)} NOT NULL,
|
|
28
|
+
redirect_uri ${t.text},
|
|
29
|
+
code_challenge ${t.varchar(128)},
|
|
30
|
+
expires_at ${t.timestamp} NOT NULL,
|
|
31
|
+
redeemed_at ${t.timestamp},
|
|
32
|
+
create_date ${t.timestamp} NOT NULL,
|
|
33
|
+
PRIMARY KEY (hash)
|
|
34
|
+
)`,
|
|
35
|
+
)
|
|
36
|
+
await db.exec(
|
|
37
|
+
'CREATE INDEX ix_delivery_member_token_member ON delivery_member_token (member_key)',
|
|
38
|
+
)
|
|
39
|
+
},
|
|
40
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Migration 034 — diagnostics and choice sources. docs/diagnostics.md, docs/forms.md.
|
|
3
|
+
*
|
|
4
|
+
* `server.crombolo_version` is the Crombolo release each node runs, beside the
|
|
5
|
+
* schema set version `server.version` already holds, so the cluster health check
|
|
6
|
+
* can say when nodes disagree. `form_entry_value.label` is the option label a
|
|
7
|
+
* visitor saw for a choice drawn from a source, so an entry still reads after the
|
|
8
|
+
* node or dictionary item behind it is renamed or removed.
|
|
9
|
+
*/
|
|
10
|
+
import type { Db } from '../database.ts'
|
|
11
|
+
import type { Migration } from '../migrations.ts'
|
|
12
|
+
import { schemaOps } from '../schema-ops.ts'
|
|
13
|
+
import { STATE_AUDIT_LOG } from './audit-log.ts'
|
|
14
|
+
|
|
15
|
+
export const STATE_DIAGNOSTICS = '{6f1d4a8e-0034-4c1a-9e7d-3a5b8c2d4e34}'
|
|
16
|
+
|
|
17
|
+
export const diagnosticsMigration: Migration = {
|
|
18
|
+
from: STATE_AUDIT_LOG,
|
|
19
|
+
to: STATE_DIAGNOSTICS,
|
|
20
|
+
name: 'AddNodeVersionAndChoiceLabel',
|
|
21
|
+
kind: 'expand',
|
|
22
|
+
release: '0.12.0',
|
|
23
|
+
async up(db: Db) {
|
|
24
|
+
const ops = schemaOps(db)
|
|
25
|
+
await ops.addColumn('server', 'crombolo_version', db.dialect.types.varchar(64))
|
|
26
|
+
await ops.addColumn('form_entry_value', 'label', db.dialect.types.text)
|
|
27
|
+
},
|
|
28
|
+
}
|
package/src/schema/plan.ts
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import { MigrationPlan } from '../migrations.ts'
|
|
15
15
|
import { assistantMigration } from './assistant.ts'
|
|
16
|
+
import { auditLogMigration } from './audit-log.ts'
|
|
16
17
|
import { bundleComponentsMigration } from './bundle-components.ts'
|
|
17
18
|
import { bundlePackageMigration } from './bundle-package.ts'
|
|
18
19
|
import { bundlesRenameMigration } from './bundles-rename.ts'
|
|
@@ -21,6 +22,8 @@ import { clientIdPrefixMigration } from './client-id-prefix.ts'
|
|
|
21
22
|
import { contentMigration } from './content.ts'
|
|
22
23
|
import { contentEditingMigration } from './content-editing.ts'
|
|
23
24
|
import { createdPackagesMigration } from './created-packages.ts'
|
|
25
|
+
import { deliveryMemberTokensMigration } from './delivery-member-tokens.ts'
|
|
26
|
+
import { diagnosticsMigration } from './diagnostics.ts'
|
|
24
27
|
import { elementsMigration } from './elements.ts'
|
|
25
28
|
import { formPermissionPrefixMigration } from './form-permission-prefix.ts'
|
|
26
29
|
import { formPermissionsMigration } from './form-permissions.ts'
|
|
@@ -31,17 +34,21 @@ import { identityMigration } from './identity.ts'
|
|
|
31
34
|
import { logViewerMigration } from './log-viewer.ts'
|
|
32
35
|
import { memberPropertyTypeMigration } from './member-property-type.ts'
|
|
33
36
|
import { membersMigration } from './members.ts'
|
|
37
|
+
import { propertySearchableMigration } from './property-searchable.ts'
|
|
34
38
|
import { publicAccessMigration } from './public-access.ts'
|
|
35
39
|
import { recycleBinMigration } from './recycle-bin.ts'
|
|
36
40
|
import { redirectsMigration } from './redirects.ts'
|
|
37
41
|
import { richTextUiMigration } from './rich-text-ui.ts'
|
|
42
|
+
import { searchIndexMigration } from './search-index.ts'
|
|
38
43
|
import { serverRoleMigration } from './server-role.ts'
|
|
39
44
|
import { sinceVersionMigration } from './since-version.ts'
|
|
45
|
+
import { transferAnswersMigration } from './transfer-answers.ts'
|
|
40
46
|
import { transferRunPackageMigration } from './transfer-run-package.ts'
|
|
41
47
|
import { transferRunsMigration } from './transfer-runs.ts'
|
|
42
48
|
import { upgradeReportMigration } from './upgrade-report.ts'
|
|
43
49
|
import { usersLocalisationMigration } from './users-localisation.ts'
|
|
44
50
|
import { valuesMigration } from './values.ts'
|
|
51
|
+
import { webhooksMigration } from './webhooks.ts'
|
|
45
52
|
|
|
46
53
|
export const cromboloPlan = new MigrationPlan([
|
|
47
54
|
identityMigration,
|
|
@@ -74,4 +81,11 @@ export const cromboloPlan = new MigrationPlan([
|
|
|
74
81
|
bundleComponentsMigration,
|
|
75
82
|
bundlePackageMigration,
|
|
76
83
|
transferRunPackageMigration,
|
|
84
|
+
searchIndexMigration,
|
|
85
|
+
propertySearchableMigration,
|
|
86
|
+
auditLogMigration,
|
|
87
|
+
diagnosticsMigration,
|
|
88
|
+
webhooksMigration,
|
|
89
|
+
transferAnswersMigration,
|
|
90
|
+
deliveryMemberTokensMigration,
|
|
77
91
|
])
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Migration 032 — a property can be kept out of backoffice search.
|
|
3
|
+
*
|
|
4
|
+
* `searchable = false` on a property in the schema files. Stored on the
|
|
5
|
+
* property type, because the backoffice writes those files back from the
|
|
6
|
+
* database and its save carries no such flag, so the column is what a save
|
|
7
|
+
* preserves. Every existing property stays searchable. docs/search.md.
|
|
8
|
+
*/
|
|
9
|
+
import type { Db } from '../database.ts'
|
|
10
|
+
import type { Migration } from '../migrations.ts'
|
|
11
|
+
import { schemaOps } from '../schema-ops.ts'
|
|
12
|
+
import { STATE_SEARCH_INDEX } from './search-index.ts'
|
|
13
|
+
|
|
14
|
+
export const STATE_PROPERTY_SEARCHABLE = '{6f1d4a8e-0032-4b7e-8a2c-5d9f1e3b6c32}'
|
|
15
|
+
|
|
16
|
+
export const propertySearchableMigration: Migration = {
|
|
17
|
+
from: STATE_SEARCH_INDEX,
|
|
18
|
+
to: STATE_PROPERTY_SEARCHABLE,
|
|
19
|
+
name: 'AddPropertySearchable',
|
|
20
|
+
kind: 'expand',
|
|
21
|
+
release: '0.10.0',
|
|
22
|
+
async up(db: Db) {
|
|
23
|
+
await schemaOps(db).addColumn(
|
|
24
|
+
'property_type',
|
|
25
|
+
'searchable',
|
|
26
|
+
`${db.dialect.types.boolean} NOT NULL DEFAULT ${db.dialect.name === 'postgres' ? 'TRUE' : '1'}`,
|
|
27
|
+
)
|
|
28
|
+
},
|
|
29
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Migration 031 — the backoffice search index. docs/search.md.
|
|
3
|
+
*
|
|
4
|
+
* `search_document` holds one row per item and culture: its name and the text of
|
|
5
|
+
* its current values, folded to lowercase without diacritics. SQLite matches it
|
|
6
|
+
* through an FTS5 table kept in step by triggers; Postgres through a generated
|
|
7
|
+
* `tsvector` column. Both use language-neutral word splitting, so the two
|
|
8
|
+
* dialects find the same things.
|
|
9
|
+
*
|
|
10
|
+
* `search_index_queue` is what a write leaves for the indexer: a node id and a
|
|
11
|
+
* generation, bumped each time the node is queued again, so a row re-queued
|
|
12
|
+
* while it was being indexed survives the indexer's delete. Every existing
|
|
13
|
+
* page, media item and member is queued here, which is the backfill.
|
|
14
|
+
*/
|
|
15
|
+
import { ObjectTypes } from '@crombolo/core'
|
|
16
|
+
import type { Db } from '../database.ts'
|
|
17
|
+
import type { Migration } from '../migrations.ts'
|
|
18
|
+
import { STATE_TRANSFER_RUN_PACKAGE } from './transfer-run-package.ts'
|
|
19
|
+
|
|
20
|
+
export const STATE_SEARCH_INDEX = '{6f1d4a8e-0031-4f3a-9c6d-2b8e5a1f7d31}'
|
|
21
|
+
|
|
22
|
+
export const searchIndexMigration: Migration = {
|
|
23
|
+
from: STATE_TRANSFER_RUN_PACKAGE,
|
|
24
|
+
to: STATE_SEARCH_INDEX,
|
|
25
|
+
name: 'AddSearchIndex',
|
|
26
|
+
kind: 'expand',
|
|
27
|
+
release: '0.10.0',
|
|
28
|
+
async up(db: Db) {
|
|
29
|
+
const t = db.dialect.types
|
|
30
|
+
const postgres = db.dialect.name === 'postgres'
|
|
31
|
+
await db.exec(
|
|
32
|
+
`CREATE TABLE search_document (
|
|
33
|
+
id ${t.identity},
|
|
34
|
+
node_id ${t.integer} NOT NULL REFERENCES node (id) ON DELETE CASCADE,
|
|
35
|
+
culture ${t.varchar(14)} NOT NULL,
|
|
36
|
+
name ${t.text} NOT NULL,
|
|
37
|
+
body ${t.text} NOT NULL,
|
|
38
|
+
fields ${t.json} NOT NULL,
|
|
39
|
+
field_count ${t.integer} NOT NULL,
|
|
40
|
+
indexed_at ${t.timestamp} NOT NULL${
|
|
41
|
+
postgres
|
|
42
|
+
? `,
|
|
43
|
+
terms tsvector GENERATED ALWAYS AS (
|
|
44
|
+
setweight(to_tsvector('simple', name), 'A') || setweight(to_tsvector('simple', body), 'B')
|
|
45
|
+
) STORED`
|
|
46
|
+
: ''
|
|
47
|
+
},
|
|
48
|
+
UNIQUE (node_id, culture)
|
|
49
|
+
)`,
|
|
50
|
+
)
|
|
51
|
+
if (postgres) {
|
|
52
|
+
await db.exec('CREATE INDEX ix_search_document_terms ON search_document USING GIN (terms)')
|
|
53
|
+
} else {
|
|
54
|
+
await db.exec(
|
|
55
|
+
`CREATE VIRTUAL TABLE search_document_fts USING fts5(
|
|
56
|
+
name, body, content='search_document', content_rowid='id',
|
|
57
|
+
tokenize='unicode61 remove_diacritics 2'
|
|
58
|
+
)`,
|
|
59
|
+
)
|
|
60
|
+
await db.exec(
|
|
61
|
+
`CREATE TRIGGER search_document_ai AFTER INSERT ON search_document BEGIN
|
|
62
|
+
INSERT INTO search_document_fts (rowid, name, body) VALUES (new.id, new.name, new.body);
|
|
63
|
+
END`,
|
|
64
|
+
)
|
|
65
|
+
await db.exec(
|
|
66
|
+
`CREATE TRIGGER search_document_ad AFTER DELETE ON search_document BEGIN
|
|
67
|
+
INSERT INTO search_document_fts (search_document_fts, rowid, name, body)
|
|
68
|
+
VALUES ('delete', old.id, old.name, old.body);
|
|
69
|
+
END`,
|
|
70
|
+
)
|
|
71
|
+
await db.exec(
|
|
72
|
+
`CREATE TRIGGER search_document_au AFTER UPDATE ON search_document BEGIN
|
|
73
|
+
INSERT INTO search_document_fts (search_document_fts, rowid, name, body)
|
|
74
|
+
VALUES ('delete', old.id, old.name, old.body);
|
|
75
|
+
INSERT INTO search_document_fts (rowid, name, body) VALUES (new.id, new.name, new.body);
|
|
76
|
+
END`,
|
|
77
|
+
)
|
|
78
|
+
}
|
|
79
|
+
await db.exec(
|
|
80
|
+
`CREATE TABLE search_index_queue (
|
|
81
|
+
node_id ${t.integer} PRIMARY KEY REFERENCES node (id) ON DELETE CASCADE,
|
|
82
|
+
generation ${t.integer} NOT NULL,
|
|
83
|
+
queued_at ${t.timestamp} NOT NULL
|
|
84
|
+
)`,
|
|
85
|
+
)
|
|
86
|
+
await db.exec(
|
|
87
|
+
`INSERT INTO search_index_queue (node_id, generation, queued_at)
|
|
88
|
+
SELECT id, 1, ${db.dialect.now()} FROM node WHERE node_object_type IN (?, ?, ?)`,
|
|
89
|
+
[ObjectTypes.Document, ObjectTypes.Media, ObjectTypes.Member],
|
|
90
|
+
)
|
|
91
|
+
},
|
|
92
|
+
}
|
package/src/schema/seed.ts
CHANGED
|
@@ -211,7 +211,8 @@ export interface SeedAdmin {
|
|
|
211
211
|
}
|
|
212
212
|
|
|
213
213
|
export interface SeedOptions {
|
|
214
|
-
|
|
214
|
+
/** Omitted, the site is left for its owner to set up from the browser. */
|
|
215
|
+
admin?: SeedAdmin
|
|
215
216
|
hashPassword: (password: string) => Promise<string>
|
|
216
217
|
passwordConfig: string
|
|
217
218
|
}
|
|
@@ -259,6 +260,17 @@ export async function seedIdentity(db: Db, options: SeedOptions): Promise<boolea
|
|
|
259
260
|
if (sync) await db.exec(sync)
|
|
260
261
|
await grantBuiltInPermissions(db)
|
|
261
262
|
|
|
263
|
+
if (options.admin) await insertSuperUser(db, { ...options, admin: options.admin })
|
|
264
|
+
return true
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** Creates Umbraco's super user, in Administrators and Sensitive data. */
|
|
268
|
+
export async function insertSuperUser(
|
|
269
|
+
db: Db,
|
|
270
|
+
options: SeedOptions & { admin: SeedAdmin },
|
|
271
|
+
): Promise<void> {
|
|
272
|
+
const now = DbDate.toDb(new Date())
|
|
273
|
+
const bool = (value: boolean) => db.dialect.boolValue(value)
|
|
262
274
|
await db.exec(
|
|
263
275
|
`INSERT INTO user_account
|
|
264
276
|
(key, user_name, login, email, password_hash, password_config, security_stamp, language,
|
|
@@ -294,5 +306,4 @@ export async function seedIdentity(db: Db, options: SeedOptions): Promise<boolea
|
|
|
294
306
|
groupId,
|
|
295
307
|
])
|
|
296
308
|
}
|
|
297
|
-
return true
|
|
298
309
|
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Migration 036 — a pending import and its answers, so a content transfer's
|
|
3
|
+
* conflicts can be answered in the backoffice. docs/content-transfer.md.
|
|
4
|
+
*
|
|
5
|
+
* `transfer_pending` is the last check of each bundle: where it was found and
|
|
6
|
+
* what the check was asked to do, so the dashboard can run the same import.
|
|
7
|
+
* `transfer_answer` holds the choices made there, one per node and property
|
|
8
|
+
* handle, as `resolutions.json` would.
|
|
9
|
+
*/
|
|
10
|
+
import type { Db } from '../database.ts'
|
|
11
|
+
import type { Migration } from '../migrations.ts'
|
|
12
|
+
import { STATE_WEBHOOKS } from './webhooks.ts'
|
|
13
|
+
|
|
14
|
+
export const STATE_TRANSFER_ANSWERS = '{2c8e5b1a-0036-4f3d-8a6e-9d1c4b7a2e36}'
|
|
15
|
+
|
|
16
|
+
export const transferAnswersMigration: Migration = {
|
|
17
|
+
from: STATE_WEBHOOKS,
|
|
18
|
+
to: STATE_TRANSFER_ANSWERS,
|
|
19
|
+
name: 'AddTransferAnswers',
|
|
20
|
+
kind: 'expand',
|
|
21
|
+
release: '0.12.0',
|
|
22
|
+
async up(db: Db) {
|
|
23
|
+
const t = db.dialect.types
|
|
24
|
+
await db.exec(
|
|
25
|
+
`CREATE TABLE transfer_pending (
|
|
26
|
+
bundle_id ${t.varchar(64)} NOT NULL,
|
|
27
|
+
label ${t.varchar(255)},
|
|
28
|
+
target ${t.text} NOT NULL,
|
|
29
|
+
target_kind ${t.varchar(16)} NOT NULL,
|
|
30
|
+
under_target ${t.text},
|
|
31
|
+
publish ${t.boolean} NOT NULL,
|
|
32
|
+
allow_missing_blobs ${t.boolean} NOT NULL,
|
|
33
|
+
checked_at ${t.timestamp} NOT NULL,
|
|
34
|
+
checked_by ${t.varchar(255)},
|
|
35
|
+
PRIMARY KEY (bundle_id)
|
|
36
|
+
)`,
|
|
37
|
+
)
|
|
38
|
+
await db.exec(
|
|
39
|
+
`CREATE TABLE transfer_answer (
|
|
40
|
+
bundle_id ${t.varchar(64)} NOT NULL,
|
|
41
|
+
node_key ${t.varchar(64)} NOT NULL,
|
|
42
|
+
handle ${t.varchar(255)} NOT NULL,
|
|
43
|
+
resolution ${t.varchar(16)} NOT NULL,
|
|
44
|
+
answered_by ${t.varchar(255)},
|
|
45
|
+
answered_at ${t.timestamp} NOT NULL,
|
|
46
|
+
PRIMARY KEY (bundle_id, node_key, handle)
|
|
47
|
+
)`,
|
|
48
|
+
)
|
|
49
|
+
},
|
|
50
|
+
}
|