@crombolo/data 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/audit.ts +291 -0
- package/src/database.ts +39 -9
- package/src/dialect.ts +24 -0
- package/src/index.ts +6 -0
- package/src/repositories/content-types.ts +41 -4
- package/src/repositories/documents.ts +12 -2
- package/src/repositories/members.ts +15 -1
- package/src/repositories/nodes.ts +32 -7
- package/src/schema/audit-log.ts +45 -0
- package/src/schema/plan.ts +6 -0
- package/src/schema/property-searchable.ts +29 -0
- package/src/schema/search-index.ts +92 -0
- package/src/search-text.ts +133 -0
- package/src/search.ts +397 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crombolo/data",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.0",
|
|
4
4
|
"description": "SQLite and Postgres storage for crombolo: schema, migrations and repositories.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Crombolo",
|
|
@@ -28,6 +28,6 @@
|
|
|
28
28
|
"access": "public"
|
|
29
29
|
},
|
|
30
30
|
"dependencies": {
|
|
31
|
-
"@crombolo/core": "0.
|
|
31
|
+
"@crombolo/core": "0.11.0"
|
|
32
32
|
}
|
|
33
33
|
}
|
package/src/audit.ts
ADDED
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The audit log: who did what, when and from where. Entries are only ever
|
|
3
|
+
* added, read and aged out; nothing edits one. docs/audit-log.md.
|
|
4
|
+
*/
|
|
5
|
+
import { normaliseUuid } from '@crombolo/core'
|
|
6
|
+
import type { Db } from './database.ts'
|
|
7
|
+
import { DbDate } from './dialect.ts'
|
|
8
|
+
import { STATE_AUDIT_LOG } from './schema/audit-log.ts'
|
|
9
|
+
|
|
10
|
+
export type AuditCategory =
|
|
11
|
+
| 'content'
|
|
12
|
+
| 'media'
|
|
13
|
+
| 'member'
|
|
14
|
+
| 'schema'
|
|
15
|
+
| 'settings'
|
|
16
|
+
| 'user'
|
|
17
|
+
| 'permission'
|
|
18
|
+
| 'sign-in'
|
|
19
|
+
| 'system'
|
|
20
|
+
|
|
21
|
+
export const AUDIT_CATEGORIES: readonly AuditCategory[] = [
|
|
22
|
+
'content',
|
|
23
|
+
'media',
|
|
24
|
+
'member',
|
|
25
|
+
'schema',
|
|
26
|
+
'settings',
|
|
27
|
+
'user',
|
|
28
|
+
'permission',
|
|
29
|
+
'sign-in',
|
|
30
|
+
'system',
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
/** Where an entry came from: a request (with its address), the command line, or a background job. */
|
|
34
|
+
export type AuditSource = 'request' | 'cli' | 'job'
|
|
35
|
+
|
|
36
|
+
export interface AuditEntryInput {
|
|
37
|
+
category: AuditCategory
|
|
38
|
+
action: string
|
|
39
|
+
source: AuditSource
|
|
40
|
+
userKey?: string | null
|
|
41
|
+
/** The user's name when the entry was written; for a failed sign-in, the login tried. */
|
|
42
|
+
userName?: string | null
|
|
43
|
+
entityType?: string | null
|
|
44
|
+
entityKey?: string | null
|
|
45
|
+
entityName?: string | null
|
|
46
|
+
ipAddress?: string | null
|
|
47
|
+
forwardedFor?: string | null
|
|
48
|
+
detail?: string | null
|
|
49
|
+
occurredAt?: Date
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface AuditEntry {
|
|
53
|
+
id: number
|
|
54
|
+
occurredAt: Date
|
|
55
|
+
category: AuditCategory
|
|
56
|
+
action: string
|
|
57
|
+
source: AuditSource
|
|
58
|
+
userKey: string | null
|
|
59
|
+
userName: string | null
|
|
60
|
+
entityType: string | null
|
|
61
|
+
entityKey: string | null
|
|
62
|
+
entityName: string | null
|
|
63
|
+
ipAddress: string | null
|
|
64
|
+
forwardedFor: string | null
|
|
65
|
+
detail: string | null
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export interface AuditFilter {
|
|
69
|
+
userKey?: string
|
|
70
|
+
entityKey?: string
|
|
71
|
+
category?: AuditCategory
|
|
72
|
+
action?: string
|
|
73
|
+
since?: Date
|
|
74
|
+
until?: Date
|
|
75
|
+
/** Matched anywhere in the item's name or key, or the user's name. */
|
|
76
|
+
text?: string
|
|
77
|
+
ascending?: boolean
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** What a key names, for an entry to carry once the thing itself has gone. */
|
|
81
|
+
export interface AuditSubject {
|
|
82
|
+
entityType: string
|
|
83
|
+
entityKey: string
|
|
84
|
+
entityName: string | null
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const UUID = /^\{?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\}?$/i
|
|
88
|
+
|
|
89
|
+
const LIMITS = {
|
|
90
|
+
action: 64,
|
|
91
|
+
userName: 255,
|
|
92
|
+
entityType: 64,
|
|
93
|
+
entityKey: 255,
|
|
94
|
+
entityName: 255,
|
|
95
|
+
ipAddress: 64,
|
|
96
|
+
forwardedFor: 255,
|
|
97
|
+
detail: 4000,
|
|
98
|
+
} as const
|
|
99
|
+
|
|
100
|
+
function clip(value: string | null | undefined, limit: number): string | null {
|
|
101
|
+
if (value === null || value === undefined || value === '') return null
|
|
102
|
+
return value.length > limit ? value.slice(0, limit) : value
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function text(value: unknown): string | null {
|
|
106
|
+
return value === null || value === undefined ? null : String(value)
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function toEntry(row: Record<string, unknown>): AuditEntry {
|
|
110
|
+
return {
|
|
111
|
+
id: Number(row.id),
|
|
112
|
+
occurredAt: DbDate.fromDb(row.occurred_at) ?? new Date(0),
|
|
113
|
+
category: String(row.category) as AuditCategory,
|
|
114
|
+
action: String(row.action),
|
|
115
|
+
source: String(row.source) as AuditSource,
|
|
116
|
+
userKey: row.user_key ? normaliseUuid(String(row.user_key)) : null,
|
|
117
|
+
userName: text(row.user_name),
|
|
118
|
+
entityType: text(row.entity_type),
|
|
119
|
+
entityKey: text(row.entity_key),
|
|
120
|
+
entityName: text(row.entity_name),
|
|
121
|
+
ipAddress: text(row.ip_address),
|
|
122
|
+
forwardedFor: text(row.forwarded_for),
|
|
123
|
+
detail: text(row.detail),
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export class AuditLog {
|
|
128
|
+
#db: Db
|
|
129
|
+
|
|
130
|
+
constructor(db: Db) {
|
|
131
|
+
this.#db = db
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
async record(entries: AuditEntryInput | readonly AuditEntryInput[]): Promise<void> {
|
|
135
|
+
const list = Array.isArray(entries) ? entries : [entries as AuditEntryInput]
|
|
136
|
+
for (const entry of list) {
|
|
137
|
+
const userKey =
|
|
138
|
+
entry.userKey && UUID.test(entry.userKey) ? normaliseUuid(entry.userKey) : null
|
|
139
|
+
const entityKey =
|
|
140
|
+
entry.entityKey && UUID.test(entry.entityKey)
|
|
141
|
+
? normaliseUuid(entry.entityKey)
|
|
142
|
+
: entry.entityKey
|
|
143
|
+
await this.#db.exec(
|
|
144
|
+
`INSERT INTO audit_entry (occurred_at, category, action, user_key, user_name, entity_type,
|
|
145
|
+
entity_key, entity_name, source, ip_address, forwarded_for, detail)
|
|
146
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
147
|
+
[
|
|
148
|
+
DbDate.toDb(entry.occurredAt ?? new Date()),
|
|
149
|
+
entry.category,
|
|
150
|
+
clip(entry.action, LIMITS.action),
|
|
151
|
+
userKey,
|
|
152
|
+
clip(entry.userName, LIMITS.userName),
|
|
153
|
+
clip(entry.entityType, LIMITS.entityType),
|
|
154
|
+
clip(entityKey, LIMITS.entityKey),
|
|
155
|
+
clip(entry.entityName, LIMITS.entityName),
|
|
156
|
+
entry.source,
|
|
157
|
+
clip(entry.ipAddress, LIMITS.ipAddress),
|
|
158
|
+
clip(entry.forwardedFor, LIMITS.forwardedFor),
|
|
159
|
+
clip(entry.detail, LIMITS.detail),
|
|
160
|
+
],
|
|
161
|
+
)
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
async query(
|
|
166
|
+
filter: AuditFilter = {},
|
|
167
|
+
page: { skip: number; take: number } = { skip: 0, take: 100 },
|
|
168
|
+
): Promise<{ total: number; items: AuditEntry[] }> {
|
|
169
|
+
const where: string[] = []
|
|
170
|
+
const params: unknown[] = []
|
|
171
|
+
if (filter.userKey !== undefined) {
|
|
172
|
+
if (!UUID.test(filter.userKey)) return { total: 0, items: [] }
|
|
173
|
+
where.push('user_key = ?')
|
|
174
|
+
params.push(normaliseUuid(filter.userKey))
|
|
175
|
+
}
|
|
176
|
+
if (filter.entityKey !== undefined) {
|
|
177
|
+
where.push('entity_key = ?')
|
|
178
|
+
params.push(UUID.test(filter.entityKey) ? normaliseUuid(filter.entityKey) : filter.entityKey)
|
|
179
|
+
}
|
|
180
|
+
if (filter.category !== undefined) {
|
|
181
|
+
where.push('category = ?')
|
|
182
|
+
params.push(filter.category)
|
|
183
|
+
}
|
|
184
|
+
if (filter.action !== undefined) {
|
|
185
|
+
where.push('action = ?')
|
|
186
|
+
params.push(filter.action)
|
|
187
|
+
}
|
|
188
|
+
if (filter.since !== undefined) {
|
|
189
|
+
where.push('occurred_at >= ?')
|
|
190
|
+
params.push(DbDate.toDb(filter.since))
|
|
191
|
+
}
|
|
192
|
+
if (filter.until !== undefined) {
|
|
193
|
+
where.push('occurred_at < ?')
|
|
194
|
+
params.push(DbDate.toDb(filter.until))
|
|
195
|
+
}
|
|
196
|
+
const needle = filter.text?.trim().toLowerCase()
|
|
197
|
+
if (needle) {
|
|
198
|
+
const like = `%${needle.replace(/[\\%_]/g, (c) => `\\${c}`)}%`
|
|
199
|
+
where.push(
|
|
200
|
+
`(LOWER(entity_name) LIKE ? ESCAPE '\\' OR LOWER(entity_key) LIKE ? ESCAPE '\\' OR LOWER(user_name) LIKE ? ESCAPE '\\')`,
|
|
201
|
+
)
|
|
202
|
+
params.push(like, like, like)
|
|
203
|
+
}
|
|
204
|
+
const clause = where.length > 0 ? `WHERE ${where.join(' AND ')}` : ''
|
|
205
|
+
const counted = await this.#db.query<{ total: number }>(
|
|
206
|
+
`SELECT COUNT(*) AS total FROM audit_entry ${clause}`,
|
|
207
|
+
params,
|
|
208
|
+
)
|
|
209
|
+
const order = filter.ascending ? 'ASC' : 'DESC'
|
|
210
|
+
const rows = await this.#db.query(
|
|
211
|
+
`SELECT * FROM audit_entry ${clause} ORDER BY occurred_at ${order}, id ${order} LIMIT ? OFFSET ?`,
|
|
212
|
+
[...params, page.take, page.skip],
|
|
213
|
+
)
|
|
214
|
+
return { total: Number(counted[0]?.total ?? 0), items: rows.map(toEntry) }
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** Everyone the log names, by the name they had most recently. */
|
|
218
|
+
async users(): Promise<Array<{ key: string; name: string }>> {
|
|
219
|
+
const rows = await this.#db.query<{ user_key: string; user_name: string | null }>(
|
|
220
|
+
`SELECT e.user_key, e.user_name FROM audit_entry e
|
|
221
|
+
JOIN (SELECT user_key, MAX(id) AS id FROM audit_entry WHERE user_key IS NOT NULL GROUP BY user_key) latest
|
|
222
|
+
ON latest.id = e.id
|
|
223
|
+
ORDER BY e.user_name`,
|
|
224
|
+
)
|
|
225
|
+
return rows.map((row) => ({
|
|
226
|
+
key: normaliseUuid(String(row.user_key)),
|
|
227
|
+
name: row.user_name ? String(row.user_name) : '',
|
|
228
|
+
}))
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** Deletes entries older than `before`, and returns how many went. */
|
|
232
|
+
async prune(before: Date): Promise<number> {
|
|
233
|
+
const counted = await this.#db.query<{ total: number }>(
|
|
234
|
+
'SELECT COUNT(*) AS total FROM audit_entry WHERE occurred_at < ?',
|
|
235
|
+
[DbDate.toDb(before)],
|
|
236
|
+
)
|
|
237
|
+
const total = Number(counted[0]?.total ?? 0)
|
|
238
|
+
if (total > 0)
|
|
239
|
+
await this.#db.exec('DELETE FROM audit_entry WHERE occurred_at < ?', [DbDate.toDb(before)])
|
|
240
|
+
return total
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** When this database began keeping the log; history before it is in the content versions. */
|
|
244
|
+
async startedAt(): Promise<Date | undefined> {
|
|
245
|
+
const rows = await this.#db.query<{ applied_at: unknown }>(
|
|
246
|
+
'SELECT MIN(applied_at) AS applied_at FROM migration_history WHERE to_state = ?',
|
|
247
|
+
[STATE_AUDIT_LOG],
|
|
248
|
+
)
|
|
249
|
+
return DbDate.fromDb(rows[0]?.applied_at)
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* What a key names. A version answers for the item it belongs to, so a
|
|
254
|
+
* rollback is filed under the page it rolled back.
|
|
255
|
+
*/
|
|
256
|
+
async describe(entityType: string, key: string): Promise<AuditSubject> {
|
|
257
|
+
const named = (entityName: unknown, type = entityType, entityKey = key): AuditSubject => ({
|
|
258
|
+
entityType: type,
|
|
259
|
+
entityKey,
|
|
260
|
+
entityName: entityName === null || entityName === undefined ? null : String(entityName),
|
|
261
|
+
})
|
|
262
|
+
if (entityType.endsWith('Version') && /^\d+$/.test(key)) {
|
|
263
|
+
const rows = await this.#db.query<{ unique_id: string; text: string | null }>(
|
|
264
|
+
'SELECT n.unique_id, n.text FROM content_version cv JOIN node n ON n.id = cv.node_id WHERE cv.id = ?',
|
|
265
|
+
[Number(key)],
|
|
266
|
+
)
|
|
267
|
+
const row = rows[0]
|
|
268
|
+
const owner = entityType.slice(0, -'Version'.length)
|
|
269
|
+
return row ? named(row.text, owner, normaliseUuid(String(row.unique_id))) : named(null)
|
|
270
|
+
}
|
|
271
|
+
if (entityType === 'Language') {
|
|
272
|
+
const rows = await this.#db.query<{ culture_name: string }>(
|
|
273
|
+
'SELECT culture_name FROM language WHERE iso_code = ?',
|
|
274
|
+
[key],
|
|
275
|
+
)
|
|
276
|
+
return named(rows[0]?.culture_name ?? null)
|
|
277
|
+
}
|
|
278
|
+
if (!UUID.test(key)) return named(null)
|
|
279
|
+
const uuid = normaliseUuid(key)
|
|
280
|
+
const sql =
|
|
281
|
+
entityType === 'User'
|
|
282
|
+
? 'SELECT user_name AS name FROM user_account WHERE key = ?'
|
|
283
|
+
: entityType === 'UserGroup'
|
|
284
|
+
? 'SELECT name FROM user_group WHERE key = ?'
|
|
285
|
+
: entityType === 'DictionaryItem'
|
|
286
|
+
? 'SELECT item_key AS name FROM dictionary_item WHERE key = ?'
|
|
287
|
+
: 'SELECT text AS name FROM node WHERE unique_id = ?'
|
|
288
|
+
const rows = await this.#db.query<{ name: string | null }>(sql, [uuid])
|
|
289
|
+
return named(rows[0]?.name ?? null, entityType, uuid)
|
|
290
|
+
}
|
|
291
|
+
}
|
package/src/database.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/** Driver-agnostic database handle. */
|
|
2
|
+
|
|
2
3
|
import { Database as BunSqlite } from 'bun:sqlite'
|
|
4
|
+
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
3
5
|
import { type Dialect, type DialectName, dialectFor } from './dialect.ts'
|
|
4
6
|
import { createLockManager, type LockManager } from './locks.ts'
|
|
5
7
|
|
|
@@ -22,16 +24,32 @@ export interface ConnectOptions {
|
|
|
22
24
|
url?: string
|
|
23
25
|
}
|
|
24
26
|
|
|
27
|
+
/** The connection the current async chain holds a transaction on. */
|
|
28
|
+
const transacting = new AsyncLocalStorage<BunSqlite>()
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* One connection has one transaction at a time, but async chains interleave:
|
|
32
|
+
* a background job's BEGIN between a request's BEGIN and COMMIT would fail, or
|
|
33
|
+
* worse, land its statements in the request's transaction. So top-level
|
|
34
|
+
* transactions queue here, and a chain already inside one joins it.
|
|
35
|
+
*/
|
|
25
36
|
class SqliteDb implements Db {
|
|
26
37
|
readonly dialect = dialectFor('sqlite')
|
|
27
38
|
readonly locks: LockManager
|
|
28
39
|
#db: BunSqlite
|
|
29
40
|
#inTransaction: boolean
|
|
30
|
-
|
|
31
|
-
|
|
41
|
+
#gate: { tail: Promise<void> }
|
|
42
|
+
|
|
43
|
+
constructor(
|
|
44
|
+
db: BunSqlite,
|
|
45
|
+
locks?: LockManager,
|
|
46
|
+
inTransaction = false,
|
|
47
|
+
gate: { tail: Promise<void> } = { tail: Promise.resolve() },
|
|
48
|
+
) {
|
|
32
49
|
this.#db = db
|
|
33
50
|
this.locks = locks ?? createLockManager(this)
|
|
34
51
|
this.#inTransaction = inTransaction
|
|
52
|
+
this.#gate = gate
|
|
35
53
|
}
|
|
36
54
|
|
|
37
55
|
async query<T = Record<string, unknown>>(sql: string, params: unknown[] = []): Promise<T[]> {
|
|
@@ -44,14 +62,26 @@ class SqliteDb implements Db {
|
|
|
44
62
|
|
|
45
63
|
async transaction<T>(fn: (tx: Db) => Promise<T>): Promise<T> {
|
|
46
64
|
if (this.#inTransaction) return fn(this)
|
|
47
|
-
this.#db.
|
|
65
|
+
const tx = new SqliteDb(this.#db, this.locks, true, this.#gate)
|
|
66
|
+
if (transacting.getStore() === this.#db) return fn(tx)
|
|
67
|
+
const previous = this.#gate.tail
|
|
68
|
+
let release!: () => void
|
|
69
|
+
this.#gate.tail = new Promise<void>((resolve) => {
|
|
70
|
+
release = resolve
|
|
71
|
+
})
|
|
72
|
+
await previous
|
|
48
73
|
try {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
74
|
+
this.#db.exec('BEGIN')
|
|
75
|
+
try {
|
|
76
|
+
const result = await transacting.run(this.#db, () => fn(tx))
|
|
77
|
+
this.#db.exec('COMMIT')
|
|
78
|
+
return result
|
|
79
|
+
} catch (error) {
|
|
80
|
+
this.#db.exec('ROLLBACK')
|
|
81
|
+
throw error
|
|
82
|
+
}
|
|
83
|
+
} finally {
|
|
84
|
+
release()
|
|
55
85
|
}
|
|
56
86
|
}
|
|
57
87
|
|
package/src/dialect.ts
CHANGED
|
@@ -36,6 +36,16 @@ export interface Dialect {
|
|
|
36
36
|
/** Appended to a SELECT that must block a concurrent state change; empty where the engine is single-writer. */
|
|
37
37
|
readonly forShare: string
|
|
38
38
|
readonly forUpdate: string
|
|
39
|
+
/** Appended to a SELECT that claims rows, passing over any another connection holds. */
|
|
40
|
+
readonly forUpdateSkipLocked: string
|
|
41
|
+
/**
|
|
42
|
+
* The full-text match over `search_document`: a subquery of `node_id` and
|
|
43
|
+
* `score`, higher being better, for each item holding every term as a word
|
|
44
|
+
* prefix. Its one parameter is `fullTextQuery(terms)`. docs/search.md.
|
|
45
|
+
*/
|
|
46
|
+
readonly fullTextHits: string
|
|
47
|
+
/** Terms are already folded words: letters and digits only. */
|
|
48
|
+
fullTextQuery(terms: readonly string[]): string
|
|
39
49
|
/**
|
|
40
50
|
* SQL that moves a table's identity past rows inserted with explicit ids, or
|
|
41
51
|
* undefined where the engine does that itself.
|
|
@@ -71,6 +81,15 @@ export const sqliteDialect: Dialect = {
|
|
|
71
81
|
boolValue: (value) => (value ? 1 : 0),
|
|
72
82
|
forShare: '',
|
|
73
83
|
forUpdate: '',
|
|
84
|
+
forUpdateSkipLocked: '',
|
|
85
|
+
// Name hits weigh ten times body hits; bm25 is lower-is-better, so negated.
|
|
86
|
+
// bm25 cannot be called under an aggregate, and `LIMIT -1` (no limit) keeps
|
|
87
|
+
// SQLite from flattening the inner query into the outer one, which would.
|
|
88
|
+
fullTextHits: `SELECT sd.node_id AS node_id, MAX(f.score) AS score
|
|
89
|
+
FROM (SELECT rowid AS id, -bm25(search_document_fts, 10.0, 1.0) AS score
|
|
90
|
+
FROM search_document_fts WHERE search_document_fts MATCH ? LIMIT -1) f
|
|
91
|
+
JOIN search_document sd ON sd.id = f.id GROUP BY sd.node_id`,
|
|
92
|
+
fullTextQuery: (terms) => terms.map((term) => `"${term}"*`).join(' AND '),
|
|
74
93
|
syncIdentity: () => undefined,
|
|
75
94
|
}
|
|
76
95
|
|
|
@@ -102,6 +121,11 @@ export const postgresDialect: Dialect = {
|
|
|
102
121
|
boolValue: (value) => value,
|
|
103
122
|
forShare: 'FOR SHARE',
|
|
104
123
|
forUpdate: 'FOR UPDATE',
|
|
124
|
+
forUpdateSkipLocked: 'FOR UPDATE SKIP LOCKED',
|
|
125
|
+
fullTextHits: `SELECT sd.node_id AS node_id, MAX(ts_rank(sd.terms, q)) AS score
|
|
126
|
+
FROM search_document sd, to_tsquery('simple', ?) q
|
|
127
|
+
WHERE sd.terms @@ q GROUP BY sd.node_id`,
|
|
128
|
+
fullTextQuery: (terms) => terms.map((term) => `${term}:*`).join(' & '),
|
|
105
129
|
syncIdentity: (table) =>
|
|
106
130
|
`SELECT setval(pg_get_serial_sequence('${table}', 'id'), (SELECT COALESCE(MAX(id), 1) FROM ${table}))`,
|
|
107
131
|
}
|
package/src/index.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export * from './audit.ts'
|
|
1
2
|
export * from './bundle-tables/index.ts'
|
|
2
3
|
export * from './change-report.ts'
|
|
3
4
|
export * from './cluster.ts'
|
|
@@ -31,6 +32,7 @@ export * from './repositories/transfer-runs.ts'
|
|
|
31
32
|
export * from './repositories/user-data.ts'
|
|
32
33
|
export * from './repositories/users.ts'
|
|
33
34
|
export * from './schema/assistant.ts'
|
|
35
|
+
export * from './schema/audit-log.ts'
|
|
34
36
|
export * from './schema/built-in-content-types.ts'
|
|
35
37
|
export * from './schema/built-in-media-types.ts'
|
|
36
38
|
export * from './schema/built-in-member-types.ts'
|
|
@@ -49,9 +51,11 @@ export * from './schema/log-viewer.ts'
|
|
|
49
51
|
export * from './schema/member-property-type.ts'
|
|
50
52
|
export * from './schema/members.ts'
|
|
51
53
|
export * from './schema/plan.ts'
|
|
54
|
+
export * from './schema/property-searchable.ts'
|
|
52
55
|
export * from './schema/public-access.ts'
|
|
53
56
|
export * from './schema/recycle-bin.ts'
|
|
54
57
|
export * from './schema/rich-text-ui.ts'
|
|
58
|
+
export * from './schema/search-index.ts'
|
|
55
59
|
export * from './schema/seed.ts'
|
|
56
60
|
export * from './schema/since-version.ts'
|
|
57
61
|
export * from './schema/transfer-run-package.ts'
|
|
@@ -61,3 +65,5 @@ export * from './schema/users-localisation.ts'
|
|
|
61
65
|
export * from './schema/values.ts'
|
|
62
66
|
export * from './schema-ops.ts'
|
|
63
67
|
export * from './schema-state.ts'
|
|
68
|
+
export * from './search.ts'
|
|
69
|
+
export * from './search-text.ts'
|
|
@@ -27,10 +27,26 @@ import type { Db } from '../database.ts'
|
|
|
27
27
|
import { DbDate, fromDbBool } from '../dialect.ts'
|
|
28
28
|
import { pendingUntil } from '../pending.ts'
|
|
29
29
|
import type { NodeSchemaState } from '../schema-state.ts'
|
|
30
|
+
import { queueContentTypesForSearch, queueEverythingForSearch } from '../search.ts'
|
|
30
31
|
import { NodeRepository } from './nodes.ts'
|
|
31
32
|
|
|
32
33
|
const GROUP_TYPE_NAMES = ['Group', 'Tab'] as const
|
|
33
34
|
|
|
35
|
+
/** The live properties of a type, by alias, variation and storage: what its items are indexed by. */
|
|
36
|
+
async function propertySignature(db: Db, contentTypeNodeId: number): Promise<string> {
|
|
37
|
+
const rows = await db.query<Record<string, unknown>>(
|
|
38
|
+
`SELECT p.id, p.alias, p.variations, p.data_type_id, p.searchable FROM property_type p
|
|
39
|
+
WHERE p.content_type_id = ? AND p.retired_at IS NULL ORDER BY p.id`,
|
|
40
|
+
[contentTypeNodeId],
|
|
41
|
+
)
|
|
42
|
+
return rows
|
|
43
|
+
.map(
|
|
44
|
+
(row) =>
|
|
45
|
+
`${row.id}:${row.alias}:${row.variations}:${row.data_type_id}:${fromDbBool(row.searchable)}`,
|
|
46
|
+
)
|
|
47
|
+
.join('|')
|
|
48
|
+
}
|
|
49
|
+
|
|
34
50
|
export interface SaveContentTypeOptions {
|
|
35
51
|
userId?: number
|
|
36
52
|
/** The schema state that introduced anything created by this save. */
|
|
@@ -228,6 +244,9 @@ export class ContentTypeRepository {
|
|
|
228
244
|
? ((await repo.nodes.byKey(aggregate.parentKey))?.id ?? SystemNodes.Root)
|
|
229
245
|
: SystemNodes.Root
|
|
230
246
|
|
|
247
|
+
// What the search index holds depends on which properties are live and what
|
|
248
|
+
// they are called, so a change to either has the type's items re-indexed.
|
|
249
|
+
const propertiesBefore = existing ? await propertySignature(tx, existing.id) : ''
|
|
231
250
|
let nodeId: number
|
|
232
251
|
if (existing) {
|
|
233
252
|
nodeId = existing.id
|
|
@@ -361,13 +380,25 @@ export class ContentTypeRepository {
|
|
|
361
380
|
WHERE id = ?`,
|
|
362
381
|
[...columns, Number(found[0].id)],
|
|
363
382
|
)
|
|
383
|
+
// Only a schema sync says; a backoffice save leaves it as it was.
|
|
384
|
+
if (property.searchable !== undefined)
|
|
385
|
+
await tx.exec('UPDATE property_type SET searchable = ? WHERE id = ?', [
|
|
386
|
+
bool(property.searchable),
|
|
387
|
+
Number(found[0].id),
|
|
388
|
+
])
|
|
364
389
|
} else {
|
|
365
390
|
await tx.exec(
|
|
366
391
|
`INSERT INTO property_type
|
|
367
392
|
(unique_id, content_type_id, data_type_id, property_type_group_id, alias, name, description, sort_order,
|
|
368
|
-
mandatory, mandatory_message, validation_reg_exp, validation_reg_exp_message, label_on_top, variations, since_version, since_state_id)
|
|
369
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
370
|
-
[
|
|
393
|
+
mandatory, mandatory_message, validation_reg_exp, validation_reg_exp_message, label_on_top, variations, since_version, since_state_id, searchable)
|
|
394
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
395
|
+
[
|
|
396
|
+
key,
|
|
397
|
+
nodeId,
|
|
398
|
+
...columns,
|
|
399
|
+
options.sinceStateId ?? null,
|
|
400
|
+
bool(property.searchable !== false),
|
|
401
|
+
],
|
|
371
402
|
)
|
|
372
403
|
}
|
|
373
404
|
}
|
|
@@ -451,6 +482,11 @@ export class ContentTypeRepository {
|
|
|
451
482
|
)
|
|
452
483
|
}
|
|
453
484
|
|
|
485
|
+
if (existing && propertiesBefore !== (await propertySignature(tx, nodeId))) {
|
|
486
|
+
if (aggregate.isElement) await queueEverythingForSearch(tx)
|
|
487
|
+
else await queueContentTypesForSearch(tx, [nodeId])
|
|
488
|
+
}
|
|
489
|
+
|
|
454
490
|
const saved = await repo.byKey(aggregate.key)
|
|
455
491
|
if (!saved) throw new Error('Content type could not be read back after saving.')
|
|
456
492
|
return saved
|
|
@@ -739,7 +775,7 @@ export class ContentTypeRepository {
|
|
|
739
775
|
p.validation_reg_exp, p.validation_reg_exp_message, p.label_on_top, p.variations,
|
|
740
776
|
dtn.unique_id AS data_type_key, g.unique_id AS container_key,
|
|
741
777
|
p.since_version, ss.version AS since_v, ss.revision AS since_r,
|
|
742
|
-
m.member_can_edit, m.member_can_view, m.is_sensitive
|
|
778
|
+
m.member_can_edit, m.member_can_view, m.is_sensitive, p.searchable
|
|
743
779
|
FROM property_type p
|
|
744
780
|
JOIN node dtn ON dtn.id = p.data_type_id
|
|
745
781
|
LEFT JOIN property_type_group g ON g.id = p.property_type_group_id
|
|
@@ -766,6 +802,7 @@ export class ContentTypeRepository {
|
|
|
766
802
|
labelOnTop: fromDbBool(row.label_on_top),
|
|
767
803
|
sinceVersion: (row.since_version as string | null) ?? null,
|
|
768
804
|
pending: this.#pending(row),
|
|
805
|
+
...(fromDbBool(row.searchable) ? {} : { searchable: false }),
|
|
769
806
|
...(this.#kind === 'member'
|
|
770
807
|
? {
|
|
771
808
|
memberCanEdit: fromDbBool(row.member_can_edit),
|
|
@@ -36,6 +36,7 @@ import {
|
|
|
36
36
|
type SchemaStateRow,
|
|
37
37
|
visibleStateIdFor,
|
|
38
38
|
} from '../schema-state.ts'
|
|
39
|
+
import { queueForSearch } from '../search.ts'
|
|
39
40
|
import { NodeRepository, type NodeRow } from './nodes.ts'
|
|
40
41
|
|
|
41
42
|
export type EventKind = 'save' | 'publish' | 'rollback' | 'migrate'
|
|
@@ -334,7 +335,11 @@ export class DocumentRepository {
|
|
|
334
335
|
? (await this.#nodes.byKey(options.parentKey))?.id
|
|
335
336
|
: undefined
|
|
336
337
|
if (options.parentKey && parentId === undefined) return { total: 0, items: [] }
|
|
337
|
-
return this.#nodes.search(this.objectType, query, {
|
|
338
|
+
return this.#nodes.search(this.objectType, query, {
|
|
339
|
+
...options,
|
|
340
|
+
parentId,
|
|
341
|
+
fullText: this.kind === 'document' || this.kind === 'media',
|
|
342
|
+
})
|
|
338
343
|
}
|
|
339
344
|
|
|
340
345
|
async versions(key: string): Promise<DocumentVersionSummary[]> {
|
|
@@ -594,7 +599,10 @@ export class DocumentRepository {
|
|
|
594
599
|
)
|
|
595
600
|
written = true
|
|
596
601
|
}
|
|
597
|
-
if (written)
|
|
602
|
+
if (written) {
|
|
603
|
+
await repo.#recomputeEdited(node.id)
|
|
604
|
+
await queueForSearch(tx, [node.id])
|
|
605
|
+
}
|
|
598
606
|
return written
|
|
599
607
|
})
|
|
600
608
|
}
|
|
@@ -2053,6 +2061,8 @@ export class DocumentRepository {
|
|
|
2053
2061
|
'INSERT INTO document_version (id, template_id, published) VALUES (?, ?, ?)',
|
|
2054
2062
|
[eventId, templateId, this.#db.dialect.boolValue(published)],
|
|
2055
2063
|
)
|
|
2064
|
+
// Every change to what an item says is an event, so this is where the search index hears of it.
|
|
2065
|
+
await queueForSearch(this.#db, [nodeId])
|
|
2056
2066
|
return eventId
|
|
2057
2067
|
}
|
|
2058
2068
|
|
|
@@ -14,6 +14,8 @@
|
|
|
14
14
|
import { normaliseUuid, ObjectTypes } from '@crombolo/core'
|
|
15
15
|
import type { Db } from '../database.ts'
|
|
16
16
|
import { DbDate, fromDbBool } from '../dialect.ts'
|
|
17
|
+
import { queueForSearch } from '../search.ts'
|
|
18
|
+
import { searchTerms } from '../search-text.ts'
|
|
17
19
|
import {
|
|
18
20
|
DocumentRepository,
|
|
19
21
|
type DocumentRepositoryOptions,
|
|
@@ -242,6 +244,8 @@ export class MemberRepository {
|
|
|
242
244
|
],
|
|
243
245
|
)
|
|
244
246
|
await this.#writeGroups(node.id, identity.groupKeys ?? [])
|
|
247
|
+
// Again, now the e-mail and username it is found by are written.
|
|
248
|
+
await queueForSearch(this.#db, [node.id])
|
|
245
249
|
const member = await this.byKey(created.key)
|
|
246
250
|
if (!member) throw new Error(`the member ${created.key} could not be read back`)
|
|
247
251
|
return member
|
|
@@ -290,6 +294,7 @@ export class MemberRepository {
|
|
|
290
294
|
...values,
|
|
291
295
|
node.id,
|
|
292
296
|
])
|
|
297
|
+
await queueForSearch(this.#db, [node.id])
|
|
293
298
|
if (identity.groupKeys) await this.#writeGroups(node.id, identity.groupKeys)
|
|
294
299
|
return this.byKey(key)
|
|
295
300
|
}
|
|
@@ -365,9 +370,18 @@ export class MemberRepository {
|
|
|
365
370
|
}
|
|
366
371
|
const text = criteria.filter?.trim()
|
|
367
372
|
if (text) {
|
|
368
|
-
wheres.push('(LOWER(m.email) LIKE ? OR LOWER(m.login_name) LIKE ? OR LOWER(n.text) LIKE ?)')
|
|
369
373
|
const like = `%${text.toLowerCase()}%`
|
|
374
|
+
const terms = searchTerms(text)
|
|
375
|
+
// The index adds what a member's properties say to the name, username and e-mail.
|
|
376
|
+
const indexed =
|
|
377
|
+
terms.length > 0
|
|
378
|
+
? ` OR m.node_id IN (SELECT node_id FROM (${this.#db.dialect.fullTextHits}) h)`
|
|
379
|
+
: ''
|
|
380
|
+
wheres.push(
|
|
381
|
+
`(LOWER(m.email) LIKE ? OR LOWER(m.login_name) LIKE ? OR LOWER(n.text) LIKE ?${indexed})`,
|
|
382
|
+
)
|
|
370
383
|
values.push(like, like, like)
|
|
384
|
+
if (indexed) values.push(this.#db.dialect.fullTextQuery(terms))
|
|
371
385
|
}
|
|
372
386
|
const where = wheres.length > 0 ? `WHERE ${wheres.join(' AND ')}` : ''
|
|
373
387
|
const order = ORDER_COLUMNS[(criteria.orderBy ?? '').toLowerCase()] ?? ORDER_COLUMNS.username
|