@crombolo/data 0.10.0 → 0.12.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/package.json +2 -2
  2. package/src/ambient.ts +20 -0
  3. package/src/audit.ts +291 -0
  4. package/src/busy.ts +74 -0
  5. package/src/change-report.ts +6 -0
  6. package/src/cluster.ts +17 -5
  7. package/src/database.ts +80 -17
  8. package/src/dialect.ts +24 -0
  9. package/src/index.ts +20 -0
  10. package/src/locks.ts +94 -25
  11. package/src/migrations.ts +42 -16
  12. package/src/relations.ts +346 -0
  13. package/src/repositories/content-types.ts +41 -4
  14. package/src/repositories/data-types.ts +38 -0
  15. package/src/repositories/delivery-member-tokens.ts +110 -0
  16. package/src/repositories/dictionary.ts +18 -0
  17. package/src/repositories/documents.ts +443 -88
  18. package/src/repositories/dynamic-roots.ts +151 -0
  19. package/src/repositories/form-entries.ts +32 -11
  20. package/src/repositories/form-workflows.ts +1 -1
  21. package/src/repositories/members.ts +26 -1
  22. package/src/repositories/nodes.ts +43 -7
  23. package/src/repositories/public-access.ts +30 -0
  24. package/src/repositories/references.ts +15 -0
  25. package/src/repositories/transfer-answers.ts +151 -0
  26. package/src/repositories/webhooks.ts +299 -0
  27. package/src/schema/audit-log.ts +45 -0
  28. package/src/schema/delivery-member-tokens.ts +40 -0
  29. package/src/schema/diagnostics.ts +28 -0
  30. package/src/schema/plan.ts +14 -0
  31. package/src/schema/property-searchable.ts +29 -0
  32. package/src/schema/search-index.ts +92 -0
  33. package/src/schema/seed.ts +13 -2
  34. package/src/schema/transfer-answers.ts +50 -0
  35. package/src/schema/webhooks.ts +78 -0
  36. package/src/search-text.ts +133 -0
  37. package/src/search.ts +397 -0
  38. package/src/setup.ts +57 -0
  39. package/src/tree-invariants.ts +167 -0
  40. package/src/url-options.ts +61 -0
@@ -0,0 +1,299 @@
1
+ /**
2
+ * Webhooks kept in this environment, and the delivery queue and log every
3
+ * webhook and bundle event handler shares. docs/webhooks.md.
4
+ *
5
+ * A delivery is claimed with `UPDATE … WHERE state = 'pending' … RETURNING`, as
6
+ * form workflows are, so on a cluster exactly one node delivers each.
7
+ */
8
+ import { randomUUID } from 'node:crypto'
9
+ import type { Db } from '../database.ts'
10
+ import { DbDate, fromDbBool } from '../dialect.ts'
11
+
12
+ export interface WebhookRow {
13
+ key: string
14
+ name: string | null
15
+ description: string | null
16
+ url: string
17
+ enabled: boolean
18
+ events: string[]
19
+ contentTypeKeys: string[]
20
+ /** The headers as one sealed value; the server holds the key that opens it. */
21
+ sealedHeaders: string | null
22
+ }
23
+
24
+ const parseList = (value: unknown): string[] => {
25
+ try {
26
+ const parsed = JSON.parse(String(value ?? '[]')) as unknown
27
+ return Array.isArray(parsed) ? parsed.filter((item) => typeof item === 'string') : []
28
+ } catch {
29
+ return []
30
+ }
31
+ }
32
+
33
+ export class WebhookRepository {
34
+ #db: Db
35
+ constructor(db: Db) {
36
+ this.#db = db
37
+ }
38
+
39
+ async all(): Promise<WebhookRow[]> {
40
+ const rows = await this.#db.query(
41
+ 'SELECT key, name, description, url, enabled, events, content_type_keys, headers FROM webhook ORDER BY create_date, key',
42
+ )
43
+ return rows.map((row) => ({
44
+ key: String(row.key),
45
+ name: (row.name as string | null) ?? null,
46
+ description: (row.description as string | null) ?? null,
47
+ url: String(row.url),
48
+ enabled: fromDbBool(row.enabled),
49
+ events: parseList(row.events),
50
+ contentTypeKeys: parseList(row.content_type_keys),
51
+ sealedHeaders: (row.headers as string | null) ?? null,
52
+ }))
53
+ }
54
+
55
+ async byKey(key: string): Promise<WebhookRow | undefined> {
56
+ return (await this.all()).find((row) => row.key === key.toLowerCase())
57
+ }
58
+
59
+ async save(webhook: WebhookRow): Promise<void> {
60
+ const now = DbDate.toDb(new Date())
61
+ const values = [
62
+ webhook.name,
63
+ webhook.description,
64
+ webhook.url,
65
+ this.#db.dialect.boolValue(webhook.enabled),
66
+ JSON.stringify(webhook.events),
67
+ JSON.stringify(webhook.contentTypeKeys),
68
+ webhook.sealedHeaders,
69
+ now,
70
+ ]
71
+ const updated = await this.#db.query(
72
+ `UPDATE webhook SET name = ?, description = ?, url = ?, enabled = ?, events = ?,
73
+ content_type_keys = ?, headers = ?, update_date = ?
74
+ WHERE key = ? RETURNING key`,
75
+ [...values, webhook.key.toLowerCase()],
76
+ )
77
+ if (updated.length > 0) return
78
+ await this.#db.exec(
79
+ `INSERT INTO webhook (name, description, url, enabled, events, content_type_keys, headers,
80
+ update_date, key, create_date)
81
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
82
+ [...values, webhook.key.toLowerCase(), now],
83
+ )
84
+ }
85
+
86
+ async remove(key: string): Promise<boolean> {
87
+ const rows = await this.#db.query('DELETE FROM webhook WHERE key = ? RETURNING key', [
88
+ key.toLowerCase(),
89
+ ])
90
+ return rows.length > 0
91
+ }
92
+ }
93
+
94
+ export type WebhookDeliveryState = 'pending' | 'running' | 'done' | 'failed'
95
+
96
+ export interface WebhookDelivery {
97
+ id: string
98
+ /** A webhook's key, or `bundle:<id>` for a bundle's handler. */
99
+ subscriber: string
100
+ eventAlias: string
101
+ payload: unknown
102
+ state: WebhookDeliveryState
103
+ attempts: number
104
+ lastError: string | null
105
+ runAfter: Date
106
+ createDate: Date
107
+ }
108
+
109
+ export interface WebhookLogEntry {
110
+ key: string
111
+ deliveryId: string
112
+ subscriber: string
113
+ eventAlias: string
114
+ url: string
115
+ statusCode: number | null
116
+ success: boolean
117
+ exception: boolean
118
+ retryCount: number
119
+ requestHeaders: string
120
+ requestBody: string
121
+ responseHeaders: string
122
+ responseBody: string
123
+ date: Date
124
+ }
125
+
126
+ const DELIVERY_COLUMNS =
127
+ 'id, subscriber, event_alias, payload, state, attempts, last_error, run_after, create_date'
128
+
129
+ const hydrateDelivery = (row: Record<string, unknown>): WebhookDelivery => {
130
+ let payload: unknown = null
131
+ try {
132
+ payload = JSON.parse(String(row.payload))
133
+ } catch {
134
+ payload = null
135
+ }
136
+ return {
137
+ id: String(row.id),
138
+ subscriber: String(row.subscriber),
139
+ eventAlias: String(row.event_alias),
140
+ payload,
141
+ state: String(row.state) as WebhookDeliveryState,
142
+ attempts: Number(row.attempts),
143
+ lastError: (row.last_error as string | null) ?? null,
144
+ runAfter: DbDate.fromDb(row.run_after) ?? new Date(0),
145
+ createDate: DbDate.fromDb(row.create_date) ?? new Date(0),
146
+ }
147
+ }
148
+
149
+ /** Bodies and headers are kept to what a person will read, not what a server sent. */
150
+ const clip = (value: string) => value.slice(0, 10_000)
151
+
152
+ export class WebhookDeliveryRepository {
153
+ #db: Db
154
+ constructor(db: Db) {
155
+ this.#db = db
156
+ }
157
+
158
+ async enqueue(
159
+ deliveries: ReadonlyArray<{ subscriber: string; eventAlias: string; payload: unknown }>,
160
+ now = new Date(),
161
+ ): Promise<string[]> {
162
+ const ids: string[] = []
163
+ for (const delivery of deliveries) {
164
+ const id = randomUUID()
165
+ await this.#db.exec(
166
+ `INSERT INTO webhook_delivery (${DELIVERY_COLUMNS}, update_date)
167
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
168
+ [
169
+ id,
170
+ delivery.subscriber,
171
+ delivery.eventAlias,
172
+ JSON.stringify(delivery.payload),
173
+ 'pending',
174
+ 0,
175
+ null,
176
+ DbDate.toDb(now),
177
+ DbDate.toDb(now),
178
+ DbDate.toDb(now),
179
+ ],
180
+ )
181
+ ids.push(id)
182
+ }
183
+ return ids
184
+ }
185
+
186
+ async due(now = new Date(), limit = 25): Promise<WebhookDelivery[]> {
187
+ const rows = await this.#db.query(
188
+ `SELECT ${DELIVERY_COLUMNS} FROM webhook_delivery WHERE state = ? AND run_after <= ?
189
+ ORDER BY run_after, create_date LIMIT ?`,
190
+ ['pending', DbDate.toDb(now), limit],
191
+ )
192
+ return rows.map(hydrateDelivery)
193
+ }
194
+
195
+ /** Takes a delivery, or finds another node already has. */
196
+ async claim(id: string): Promise<WebhookDelivery | undefined> {
197
+ const rows = await this.#db.query(
198
+ `UPDATE webhook_delivery SET state = ?, attempts = attempts + 1, update_date = ?
199
+ WHERE id = ? AND state = ? RETURNING ${DELIVERY_COLUMNS}`,
200
+ ['running', DbDate.toDb(new Date()), id, 'pending'],
201
+ )
202
+ return rows[0] ? hydrateDelivery(rows[0]) : undefined
203
+ }
204
+
205
+ async succeed(id: string): Promise<void> {
206
+ await this.#db.exec(
207
+ 'UPDATE webhook_delivery SET state = ?, last_error = NULL, update_date = ? WHERE id = ?',
208
+ ['done', DbDate.toDb(new Date()), id],
209
+ )
210
+ }
211
+
212
+ /** `retryAt` puts it back in the queue; without one it has failed for good. */
213
+ async fail(id: string, error: string, retryAt?: Date): Promise<void> {
214
+ await this.#db.exec(
215
+ 'UPDATE webhook_delivery SET state = ?, last_error = ?, run_after = ?, update_date = ? WHERE id = ?',
216
+ [
217
+ retryAt ? 'pending' : 'failed',
218
+ error.slice(0, 2000),
219
+ DbDate.toDb(retryAt ?? new Date()),
220
+ DbDate.toDb(new Date()),
221
+ id,
222
+ ],
223
+ )
224
+ }
225
+
226
+ async byId(id: string): Promise<WebhookDelivery | undefined> {
227
+ const rows = await this.#db.query(
228
+ `SELECT ${DELIVERY_COLUMNS} FROM webhook_delivery WHERE id = ?`,
229
+ [id],
230
+ )
231
+ return rows[0] ? hydrateDelivery(rows[0]) : undefined
232
+ }
233
+
234
+ async log(entry: Omit<WebhookLogEntry, 'key' | 'date'>, date = new Date()): Promise<void> {
235
+ await this.#db.exec(
236
+ `INSERT INTO webhook_log (key, delivery_id, subscriber, event_alias, url, status_code, success,
237
+ exception, retry_count, request_headers, request_body, response_headers, response_body, log_date)
238
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
239
+ [
240
+ randomUUID(),
241
+ entry.deliveryId,
242
+ entry.subscriber,
243
+ entry.eventAlias,
244
+ entry.url,
245
+ entry.statusCode,
246
+ this.#db.dialect.boolValue(entry.success),
247
+ this.#db.dialect.boolValue(entry.exception),
248
+ entry.retryCount,
249
+ clip(entry.requestHeaders),
250
+ clip(entry.requestBody),
251
+ clip(entry.responseHeaders),
252
+ clip(entry.responseBody),
253
+ DbDate.toDb(date),
254
+ ],
255
+ )
256
+ }
257
+
258
+ /** The log, newest first; one subscriber's when named. */
259
+ async logs(options: {
260
+ subscriber?: string
261
+ skip: number
262
+ take: number
263
+ }): Promise<{ total: number; items: WebhookLogEntry[] }> {
264
+ const where = options.subscriber ? 'WHERE subscriber = ?' : ''
265
+ const params = options.subscriber ? [options.subscriber] : []
266
+ const total = await this.#db.query<{ n: number }>(
267
+ `SELECT COUNT(*) AS n FROM webhook_log ${where}`,
268
+ params,
269
+ )
270
+ const rows = await this.#db.query(
271
+ `SELECT key, delivery_id, subscriber, event_alias, url, status_code, success, exception,
272
+ retry_count, request_headers, request_body, response_headers, response_body, log_date
273
+ FROM webhook_log ${where} ORDER BY log_date DESC, retry_count DESC, key LIMIT ? OFFSET ?`,
274
+ [...params, options.take, options.skip],
275
+ )
276
+ return {
277
+ total: Number(total[0]?.n ?? 0),
278
+ items: rows.map((row) => ({
279
+ key: String(row.key),
280
+ deliveryId: String(row.delivery_id),
281
+ subscriber: String(row.subscriber),
282
+ eventAlias: String(row.event_alias),
283
+ url: String(row.url),
284
+ statusCode:
285
+ row.status_code === null || row.status_code === undefined
286
+ ? null
287
+ : Number(row.status_code),
288
+ success: fromDbBool(row.success),
289
+ exception: fromDbBool(row.exception),
290
+ retryCount: Number(row.retry_count),
291
+ requestHeaders: String(row.request_headers),
292
+ requestBody: String(row.request_body),
293
+ responseHeaders: String(row.response_headers),
294
+ responseBody: String(row.response_body),
295
+ date: DbDate.fromDb(row.log_date) ?? new Date(0),
296
+ })),
297
+ }
298
+ }
299
+ }
@@ -0,0 +1,45 @@
1
+ /**
2
+ * Migration 033 — the audit log. docs/audit-log.md.
3
+ *
4
+ * `audit_entry` records who did what, when and from where: every backoffice
5
+ * change, every sign-in and failed sign-in, and what the command line and the
6
+ * background jobs changed. Names are copied in when an entry is written, so an
7
+ * entry still reads after its user or item is gone. It starts empty: history from
8
+ * before this migration stays in the content versions it came from.
9
+ */
10
+ import type { Db } from '../database.ts'
11
+ import type { Migration } from '../migrations.ts'
12
+ import { STATE_PROPERTY_SEARCHABLE } from './property-searchable.ts'
13
+
14
+ export const STATE_AUDIT_LOG = '{6f1d4a8e-0033-4c1a-9e7d-3a5b8c2d4e33}'
15
+
16
+ export const auditLogMigration: Migration = {
17
+ from: STATE_PROPERTY_SEARCHABLE,
18
+ to: STATE_AUDIT_LOG,
19
+ name: 'AddAuditLog',
20
+ kind: 'expand',
21
+ release: '0.10.0',
22
+ async up(db: Db) {
23
+ const t = db.dialect.types
24
+ await db.exec(
25
+ `CREATE TABLE audit_entry (
26
+ id ${t.identity},
27
+ occurred_at ${t.timestamp} NOT NULL,
28
+ category ${t.varchar(32)} NOT NULL,
29
+ action ${t.varchar(64)} NOT NULL,
30
+ user_key ${t.uuid},
31
+ user_name ${t.varchar(255)},
32
+ entity_type ${t.varchar(64)},
33
+ entity_key ${t.varchar(255)},
34
+ entity_name ${t.varchar(255)},
35
+ source ${t.varchar(16)} NOT NULL,
36
+ ip_address ${t.varchar(64)},
37
+ forwarded_for ${t.varchar(255)},
38
+ detail ${t.text}
39
+ )`,
40
+ )
41
+ await db.exec('CREATE INDEX ix_audit_entry_occurred ON audit_entry (occurred_at)')
42
+ await db.exec('CREATE INDEX ix_audit_entry_entity ON audit_entry (entity_key, occurred_at)')
43
+ await db.exec('CREATE INDEX ix_audit_entry_user ON audit_entry (user_key, occurred_at)')
44
+ },
45
+ }
@@ -0,0 +1,40 @@
1
+ /**
2
+ * Migration 037 — the Delivery API's member tokens (story 050):
3
+ * authorization codes, access and refresh tokens, each stored as the SHA-256
4
+ * of what the client holds, never the token itself. docs/delivery-api.md,
5
+ * "Member sign-in".
6
+ */
7
+ import type { Db } from '../database.ts'
8
+ import type { Migration } from '../migrations.ts'
9
+ import { STATE_TRANSFER_ANSWERS } from './transfer-answers.ts'
10
+
11
+ export const STATE_DELIVERY_MEMBER_TOKENS = '{7d3b9e2c-0037-4a51-b8f6-2c4e1a9d5b37}'
12
+
13
+ export const deliveryMemberTokensMigration: Migration = {
14
+ from: STATE_TRANSFER_ANSWERS,
15
+ to: STATE_DELIVERY_MEMBER_TOKENS,
16
+ name: 'AddDeliveryMemberTokens',
17
+ kind: 'expand',
18
+ release: '0.12.0',
19
+ async up(db: Db) {
20
+ const t = db.dialect.types
21
+ await db.exec(
22
+ `CREATE TABLE delivery_member_token (
23
+ hash ${t.varchar(64)} NOT NULL,
24
+ kind ${t.varchar(16)} NOT NULL,
25
+ member_key ${t.varchar(64)} NOT NULL,
26
+ client_id ${t.varchar(255)} NOT NULL,
27
+ scopes ${t.varchar(255)} NOT NULL,
28
+ redirect_uri ${t.text},
29
+ code_challenge ${t.varchar(128)},
30
+ expires_at ${t.timestamp} NOT NULL,
31
+ redeemed_at ${t.timestamp},
32
+ create_date ${t.timestamp} NOT NULL,
33
+ PRIMARY KEY (hash)
34
+ )`,
35
+ )
36
+ await db.exec(
37
+ 'CREATE INDEX ix_delivery_member_token_member ON delivery_member_token (member_key)',
38
+ )
39
+ },
40
+ }
@@ -0,0 +1,28 @@
1
+ /**
2
+ * Migration 034 — diagnostics and choice sources. docs/diagnostics.md, docs/forms.md.
3
+ *
4
+ * `server.crombolo_version` is the Crombolo release each node runs, beside the
5
+ * schema set version `server.version` already holds, so the cluster health check
6
+ * can say when nodes disagree. `form_entry_value.label` is the option label a
7
+ * visitor saw for a choice drawn from a source, so an entry still reads after the
8
+ * node or dictionary item behind it is renamed or removed.
9
+ */
10
+ import type { Db } from '../database.ts'
11
+ import type { Migration } from '../migrations.ts'
12
+ import { schemaOps } from '../schema-ops.ts'
13
+ import { STATE_AUDIT_LOG } from './audit-log.ts'
14
+
15
+ export const STATE_DIAGNOSTICS = '{6f1d4a8e-0034-4c1a-9e7d-3a5b8c2d4e34}'
16
+
17
+ export const diagnosticsMigration: Migration = {
18
+ from: STATE_AUDIT_LOG,
19
+ to: STATE_DIAGNOSTICS,
20
+ name: 'AddNodeVersionAndChoiceLabel',
21
+ kind: 'expand',
22
+ release: '0.12.0',
23
+ async up(db: Db) {
24
+ const ops = schemaOps(db)
25
+ await ops.addColumn('server', 'crombolo_version', db.dialect.types.varchar(64))
26
+ await ops.addColumn('form_entry_value', 'label', db.dialect.types.text)
27
+ },
28
+ }
@@ -13,6 +13,7 @@
13
13
  */
14
14
  import { MigrationPlan } from '../migrations.ts'
15
15
  import { assistantMigration } from './assistant.ts'
16
+ import { auditLogMigration } from './audit-log.ts'
16
17
  import { bundleComponentsMigration } from './bundle-components.ts'
17
18
  import { bundlePackageMigration } from './bundle-package.ts'
18
19
  import { bundlesRenameMigration } from './bundles-rename.ts'
@@ -21,6 +22,8 @@ import { clientIdPrefixMigration } from './client-id-prefix.ts'
21
22
  import { contentMigration } from './content.ts'
22
23
  import { contentEditingMigration } from './content-editing.ts'
23
24
  import { createdPackagesMigration } from './created-packages.ts'
25
+ import { deliveryMemberTokensMigration } from './delivery-member-tokens.ts'
26
+ import { diagnosticsMigration } from './diagnostics.ts'
24
27
  import { elementsMigration } from './elements.ts'
25
28
  import { formPermissionPrefixMigration } from './form-permission-prefix.ts'
26
29
  import { formPermissionsMigration } from './form-permissions.ts'
@@ -31,17 +34,21 @@ import { identityMigration } from './identity.ts'
31
34
  import { logViewerMigration } from './log-viewer.ts'
32
35
  import { memberPropertyTypeMigration } from './member-property-type.ts'
33
36
  import { membersMigration } from './members.ts'
37
+ import { propertySearchableMigration } from './property-searchable.ts'
34
38
  import { publicAccessMigration } from './public-access.ts'
35
39
  import { recycleBinMigration } from './recycle-bin.ts'
36
40
  import { redirectsMigration } from './redirects.ts'
37
41
  import { richTextUiMigration } from './rich-text-ui.ts'
42
+ import { searchIndexMigration } from './search-index.ts'
38
43
  import { serverRoleMigration } from './server-role.ts'
39
44
  import { sinceVersionMigration } from './since-version.ts'
45
+ import { transferAnswersMigration } from './transfer-answers.ts'
40
46
  import { transferRunPackageMigration } from './transfer-run-package.ts'
41
47
  import { transferRunsMigration } from './transfer-runs.ts'
42
48
  import { upgradeReportMigration } from './upgrade-report.ts'
43
49
  import { usersLocalisationMigration } from './users-localisation.ts'
44
50
  import { valuesMigration } from './values.ts'
51
+ import { webhooksMigration } from './webhooks.ts'
45
52
 
46
53
  export const cromboloPlan = new MigrationPlan([
47
54
  identityMigration,
@@ -74,4 +81,11 @@ export const cromboloPlan = new MigrationPlan([
74
81
  bundleComponentsMigration,
75
82
  bundlePackageMigration,
76
83
  transferRunPackageMigration,
84
+ searchIndexMigration,
85
+ propertySearchableMigration,
86
+ auditLogMigration,
87
+ diagnosticsMigration,
88
+ webhooksMigration,
89
+ transferAnswersMigration,
90
+ deliveryMemberTokensMigration,
77
91
  ])
@@ -0,0 +1,29 @@
1
+ /**
2
+ * Migration 032 — a property can be kept out of backoffice search.
3
+ *
4
+ * `searchable = false` on a property in the schema files. Stored on the
5
+ * property type, because the backoffice writes those files back from the
6
+ * database and its save carries no such flag, so the column is what a save
7
+ * preserves. Every existing property stays searchable. docs/search.md.
8
+ */
9
+ import type { Db } from '../database.ts'
10
+ import type { Migration } from '../migrations.ts'
11
+ import { schemaOps } from '../schema-ops.ts'
12
+ import { STATE_SEARCH_INDEX } from './search-index.ts'
13
+
14
+ export const STATE_PROPERTY_SEARCHABLE = '{6f1d4a8e-0032-4b7e-8a2c-5d9f1e3b6c32}'
15
+
16
+ export const propertySearchableMigration: Migration = {
17
+ from: STATE_SEARCH_INDEX,
18
+ to: STATE_PROPERTY_SEARCHABLE,
19
+ name: 'AddPropertySearchable',
20
+ kind: 'expand',
21
+ release: '0.10.0',
22
+ async up(db: Db) {
23
+ await schemaOps(db).addColumn(
24
+ 'property_type',
25
+ 'searchable',
26
+ `${db.dialect.types.boolean} NOT NULL DEFAULT ${db.dialect.name === 'postgres' ? 'TRUE' : '1'}`,
27
+ )
28
+ },
29
+ }
@@ -0,0 +1,92 @@
1
+ /**
2
+ * Migration 031 — the backoffice search index. docs/search.md.
3
+ *
4
+ * `search_document` holds one row per item and culture: its name and the text of
5
+ * its current values, folded to lowercase without diacritics. SQLite matches it
6
+ * through an FTS5 table kept in step by triggers; Postgres through a generated
7
+ * `tsvector` column. Both use language-neutral word splitting, so the two
8
+ * dialects find the same things.
9
+ *
10
+ * `search_index_queue` is what a write leaves for the indexer: a node id and a
11
+ * generation, bumped each time the node is queued again, so a row re-queued
12
+ * while it was being indexed survives the indexer's delete. Every existing
13
+ * page, media item and member is queued here, which is the backfill.
14
+ */
15
+ import { ObjectTypes } from '@crombolo/core'
16
+ import type { Db } from '../database.ts'
17
+ import type { Migration } from '../migrations.ts'
18
+ import { STATE_TRANSFER_RUN_PACKAGE } from './transfer-run-package.ts'
19
+
20
+ export const STATE_SEARCH_INDEX = '{6f1d4a8e-0031-4f3a-9c6d-2b8e5a1f7d31}'
21
+
22
+ export const searchIndexMigration: Migration = {
23
+ from: STATE_TRANSFER_RUN_PACKAGE,
24
+ to: STATE_SEARCH_INDEX,
25
+ name: 'AddSearchIndex',
26
+ kind: 'expand',
27
+ release: '0.10.0',
28
+ async up(db: Db) {
29
+ const t = db.dialect.types
30
+ const postgres = db.dialect.name === 'postgres'
31
+ await db.exec(
32
+ `CREATE TABLE search_document (
33
+ id ${t.identity},
34
+ node_id ${t.integer} NOT NULL REFERENCES node (id) ON DELETE CASCADE,
35
+ culture ${t.varchar(14)} NOT NULL,
36
+ name ${t.text} NOT NULL,
37
+ body ${t.text} NOT NULL,
38
+ fields ${t.json} NOT NULL,
39
+ field_count ${t.integer} NOT NULL,
40
+ indexed_at ${t.timestamp} NOT NULL${
41
+ postgres
42
+ ? `,
43
+ terms tsvector GENERATED ALWAYS AS (
44
+ setweight(to_tsvector('simple', name), 'A') || setweight(to_tsvector('simple', body), 'B')
45
+ ) STORED`
46
+ : ''
47
+ },
48
+ UNIQUE (node_id, culture)
49
+ )`,
50
+ )
51
+ if (postgres) {
52
+ await db.exec('CREATE INDEX ix_search_document_terms ON search_document USING GIN (terms)')
53
+ } else {
54
+ await db.exec(
55
+ `CREATE VIRTUAL TABLE search_document_fts USING fts5(
56
+ name, body, content='search_document', content_rowid='id',
57
+ tokenize='unicode61 remove_diacritics 2'
58
+ )`,
59
+ )
60
+ await db.exec(
61
+ `CREATE TRIGGER search_document_ai AFTER INSERT ON search_document BEGIN
62
+ INSERT INTO search_document_fts (rowid, name, body) VALUES (new.id, new.name, new.body);
63
+ END`,
64
+ )
65
+ await db.exec(
66
+ `CREATE TRIGGER search_document_ad AFTER DELETE ON search_document BEGIN
67
+ INSERT INTO search_document_fts (search_document_fts, rowid, name, body)
68
+ VALUES ('delete', old.id, old.name, old.body);
69
+ END`,
70
+ )
71
+ await db.exec(
72
+ `CREATE TRIGGER search_document_au AFTER UPDATE ON search_document BEGIN
73
+ INSERT INTO search_document_fts (search_document_fts, rowid, name, body)
74
+ VALUES ('delete', old.id, old.name, old.body);
75
+ INSERT INTO search_document_fts (rowid, name, body) VALUES (new.id, new.name, new.body);
76
+ END`,
77
+ )
78
+ }
79
+ await db.exec(
80
+ `CREATE TABLE search_index_queue (
81
+ node_id ${t.integer} PRIMARY KEY REFERENCES node (id) ON DELETE CASCADE,
82
+ generation ${t.integer} NOT NULL,
83
+ queued_at ${t.timestamp} NOT NULL
84
+ )`,
85
+ )
86
+ await db.exec(
87
+ `INSERT INTO search_index_queue (node_id, generation, queued_at)
88
+ SELECT id, 1, ${db.dialect.now()} FROM node WHERE node_object_type IN (?, ?, ?)`,
89
+ [ObjectTypes.Document, ObjectTypes.Media, ObjectTypes.Member],
90
+ )
91
+ },
92
+ }
@@ -211,7 +211,8 @@ export interface SeedAdmin {
211
211
  }
212
212
 
213
213
  export interface SeedOptions {
214
- admin: SeedAdmin
214
+ /** Omitted, the site is left for its owner to set up from the browser. */
215
+ admin?: SeedAdmin
215
216
  hashPassword: (password: string) => Promise<string>
216
217
  passwordConfig: string
217
218
  }
@@ -259,6 +260,17 @@ export async function seedIdentity(db: Db, options: SeedOptions): Promise<boolea
259
260
  if (sync) await db.exec(sync)
260
261
  await grantBuiltInPermissions(db)
261
262
 
263
+ if (options.admin) await insertSuperUser(db, { ...options, admin: options.admin })
264
+ return true
265
+ }
266
+
267
+ /** Creates Umbraco's super user, in Administrators and Sensitive data. */
268
+ export async function insertSuperUser(
269
+ db: Db,
270
+ options: SeedOptions & { admin: SeedAdmin },
271
+ ): Promise<void> {
272
+ const now = DbDate.toDb(new Date())
273
+ const bool = (value: boolean) => db.dialect.boolValue(value)
262
274
  await db.exec(
263
275
  `INSERT INTO user_account
264
276
  (key, user_name, login, email, password_hash, password_config, security_stamp, language,
@@ -294,5 +306,4 @@ export async function seedIdentity(db: Db, options: SeedOptions): Promise<boolea
294
306
  groupId,
295
307
  ])
296
308
  }
297
- return true
298
309
  }
@@ -0,0 +1,50 @@
1
+ /**
2
+ * Migration 036 — a pending import and its answers, so a content transfer's
3
+ * conflicts can be answered in the backoffice. docs/content-transfer.md.
4
+ *
5
+ * `transfer_pending` is the last check of each bundle: where it was found and
6
+ * what the check was asked to do, so the dashboard can run the same import.
7
+ * `transfer_answer` holds the choices made there, one per node and property
8
+ * handle, as `resolutions.json` would.
9
+ */
10
+ import type { Db } from '../database.ts'
11
+ import type { Migration } from '../migrations.ts'
12
+ import { STATE_WEBHOOKS } from './webhooks.ts'
13
+
14
+ export const STATE_TRANSFER_ANSWERS = '{2c8e5b1a-0036-4f3d-8a6e-9d1c4b7a2e36}'
15
+
16
+ export const transferAnswersMigration: Migration = {
17
+ from: STATE_WEBHOOKS,
18
+ to: STATE_TRANSFER_ANSWERS,
19
+ name: 'AddTransferAnswers',
20
+ kind: 'expand',
21
+ release: '0.12.0',
22
+ async up(db: Db) {
23
+ const t = db.dialect.types
24
+ await db.exec(
25
+ `CREATE TABLE transfer_pending (
26
+ bundle_id ${t.varchar(64)} NOT NULL,
27
+ label ${t.varchar(255)},
28
+ target ${t.text} NOT NULL,
29
+ target_kind ${t.varchar(16)} NOT NULL,
30
+ under_target ${t.text},
31
+ publish ${t.boolean} NOT NULL,
32
+ allow_missing_blobs ${t.boolean} NOT NULL,
33
+ checked_at ${t.timestamp} NOT NULL,
34
+ checked_by ${t.varchar(255)},
35
+ PRIMARY KEY (bundle_id)
36
+ )`,
37
+ )
38
+ await db.exec(
39
+ `CREATE TABLE transfer_answer (
40
+ bundle_id ${t.varchar(64)} NOT NULL,
41
+ node_key ${t.varchar(64)} NOT NULL,
42
+ handle ${t.varchar(255)} NOT NULL,
43
+ resolution ${t.varchar(16)} NOT NULL,
44
+ answered_by ${t.varchar(255)},
45
+ answered_at ${t.timestamp} NOT NULL,
46
+ PRIMARY KEY (bundle_id, node_key, handle)
47
+ )`,
48
+ )
49
+ },
50
+ }