@ultimat3/entity 30.9.0 → 31.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +68 -106
- package/README.md +180 -8
- package/package.json +6 -5
- package/src/adapter.ts +65 -0
- package/src/batch-read.ts +12 -0
- package/src/column.ts +12 -0
- package/src/columns.ts +20 -8
- package/src/cross-tenant.ts +11 -14
- package/src/cursor.ts +3 -0
- package/src/database.ts +2 -2
- package/src/declared-index.ts +72 -0
- package/src/define-driver.ts +156 -0
- package/src/describe.ts +5 -0
- package/src/driver-delete.ts +62 -0
- package/src/driver-errors.ts +80 -0
- package/src/driver-kit-types.ts +107 -0
- package/src/driver-registry.ts +26 -0
- package/src/driver-repo-reads.ts +102 -0
- package/src/driver-repo-writes.ts +236 -0
- package/src/driver-rows.ts +108 -0
- package/src/driver-scope.ts +77 -0
- package/src/driver-write-scope.ts +107 -0
- package/src/entity-error.ts +10 -0
- package/src/entity.ts +40 -54
- package/src/enum-column.ts +8 -0
- package/src/index.ts +43 -27
- package/src/memory-match.ts +35 -5
- package/src/memory-observed.ts +16 -11
- package/src/memory-repo.ts +46 -99
- package/src/memory-seek.ts +12 -25
- package/src/memory-unique.ts +7 -7
- package/src/memory-update.ts +7 -2
- package/src/memory-upsert.ts +102 -0
- package/src/partition.ts +140 -0
- package/src/personal-data-kind.ts +15 -0
- package/src/personal-data.ts +66 -0
- package/src/pg-driver.ts +83 -30
- package/src/pg-write-sql.ts +39 -4
- package/src/plan.ts +2 -0
- package/src/predicate-screen.ts +36 -0
- package/src/query.ts +13 -4
- package/src/registry.ts +3 -0
- package/src/row-key.ts +19 -0
- package/src/row-observer.ts +38 -90
- package/src/search-index-installed.ts +24 -0
- package/src/search-index-memory.ts +132 -0
- package/src/search-index-postgres.ts +107 -0
- package/src/search-index-sync.ts +96 -0
- package/src/search-index-version.ts +93 -0
- package/src/search-index.ts +168 -0
- package/src/search-mirror-batch.ts +133 -0
- package/src/search-mirrored-repo.ts +98 -0
- package/src/search-repo.ts +91 -0
- package/src/search-written-rows.ts +36 -0
- package/src/tenant-index.ts +113 -0
- package/src/types.ts +12 -0
- package/src/write-commit.ts +28 -0
- package/src/write-tag.ts +6 -2
package/src/row-observer.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
// Committed row changes, as the framework's own repositories saw them
|
|
2
|
-
//
|
|
1
|
+
// Committed row changes, as the framework's own repositories saw them, emitted on the bus as
|
|
2
|
+
// `entity.row` / `entity.bulk`. One emitter, above the driver, so it reports the same changes
|
|
3
|
+
// whether rows live in memory or in Postgres.
|
|
3
4
|
//
|
|
4
5
|
// It exists because a change feed needs a source and only one of the two has one. Production
|
|
5
6
|
// decodes the write-ahead log (`@ultimat3/realtime`'s `postgresChangeFeed()`); PGlite has no
|
|
@@ -13,80 +14,35 @@
|
|
|
13
14
|
// the replicator — `@ultimat3/realtime`'s `selectChangeFeed` still decides, and this is never in
|
|
14
15
|
// that decision.
|
|
15
16
|
|
|
16
|
-
import { currentWriteOrigin } from '@ultimat3/core';
|
|
17
|
-
import {
|
|
17
|
+
import { currentWriteOrigin, type EntityRowOp, instrument, subscribed } from '@ultimat3/core';
|
|
18
|
+
import { expectedQueryLoop } from '@ultimat3/db';
|
|
18
19
|
import type { EntityCore } from './entity';
|
|
19
|
-
import { memoryJournal } from './memory-journal';
|
|
20
20
|
import { MAX_PAGE_SIZE } from './plan';
|
|
21
21
|
import { readRecorder } from './read-capture';
|
|
22
22
|
import type { Repo, RepoOptions, UpsertArgs } from './repo';
|
|
23
23
|
import { sealedFields } from './sealed';
|
|
24
|
+
import { searchMirroredRepo } from './search-mirrored-repo';
|
|
24
25
|
import type { IdOf, RowPatch } from './types';
|
|
25
|
-
|
|
26
|
-
export type RowChangeOp = 'insert' | 'update' | 'delete';
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* One committed row change. `before`/`after` are whole rows, exactly as logical replication reports
|
|
30
|
-
* them with `REPLICA IDENTITY FULL` — a consumer diffs them itself rather than trusting a patch it
|
|
31
|
-
* did not compile.
|
|
32
|
-
*/
|
|
33
|
-
export interface RowChange {
|
|
34
|
-
/** Entity name as declared, never a table name — the vocabulary a matcher's dependency set uses. */
|
|
35
|
-
readonly entity: string;
|
|
36
|
-
readonly op: RowChangeOp;
|
|
37
|
-
readonly before: Readonly<Record<string, unknown>> | null;
|
|
38
|
-
readonly after: Readonly<Record<string, unknown>> | null;
|
|
39
|
-
/**
|
|
40
|
-
* The keyed write this change belongs to — `@ultimat3/core`'s `currentWriteOrigin()` at the
|
|
41
|
-
* moment the repository wrote, i.e. the digest of the idempotency key the request arrived with.
|
|
42
|
-
* The WAL decoder reads the same fact off the transaction's opening message; absent both ways
|
|
43
|
-
* for a write no keyed request made.
|
|
44
|
-
*/
|
|
45
|
-
readonly write?: string;
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
/**
|
|
49
|
-
* A write this seam cannot itemise: `deleteWhere` and `updateWhere` name a filter, not a row, and
|
|
50
|
-
* reading the matching rows first would turn one statement into two and change what the code under
|
|
51
|
-
* test issues. Reported as a bulk fact so a consumer can re-read rather than be told nothing —
|
|
52
|
-
* silence is the one answer that diverges silently, which is the failure `invalidate()` exists for.
|
|
53
|
-
*/
|
|
54
|
-
export interface RowBulkChange {
|
|
55
|
-
readonly entity: string;
|
|
56
|
-
readonly op: 'delete' | 'update';
|
|
57
|
-
readonly rows: number;
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
export interface RowObserver {
|
|
61
|
-
onChange(change: RowChange): void;
|
|
62
|
-
/** Optional: an observer that re-reads everything on any change has nothing to do here. */
|
|
63
|
-
onBulk?(change: RowBulkChange): void;
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
let installed: RowObserver | null = null;
|
|
26
|
+
import { afterCommit } from './write-commit';
|
|
67
27
|
|
|
68
28
|
/**
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
29
|
+
* A committed row change is `entity.row` on the bus (`EntityRowEvent`, core's shape): `before` and
|
|
30
|
+
* `after` are whole rows, exactly as logical replication reports them with `REPLICA IDENTITY FULL`
|
|
31
|
+
* — a subscriber diffs them itself rather than trusting a patch it did not compile. `write` is
|
|
32
|
+
* `currentWriteOrigin()` at the moment the repository wrote; the WAL decoder reads the same fact.
|
|
72
33
|
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
34
|
+
* `deleteWhere` and `updateWhere` name a filter, not a row, and reading the matching rows first
|
|
35
|
+
* would turn one statement into two — so they are `entity.bulk` (`EntityBulkEvent`), a count a
|
|
36
|
+
* subscriber re-reads on rather than silence, which is the one answer that diverges silently.
|
|
76
37
|
*/
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
installed = next;
|
|
80
|
-
return previous;
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
export const rowObserver = (): RowObserver | null => installed;
|
|
38
|
+
const ROW = 'entity.row';
|
|
39
|
+
const BULK = 'entity.bulk';
|
|
84
40
|
|
|
85
41
|
/**
|
|
86
42
|
* The reads a change needs and a plain write does not. `before` costs one `findById` per update and
|
|
87
|
-
* per delete — paid ONLY while
|
|
88
|
-
* every method below rather than a flag read once at wrap time. With
|
|
89
|
-
*
|
|
43
|
+
* per delete — paid ONLY while `entity.row` has a subscriber, which is why the guard is the first line
|
|
44
|
+
* of every method below rather than a flag read once at wrap time. With none this wrapper is one map
|
|
45
|
+
* read and a delegation, on a path that already awaits a database.
|
|
90
46
|
*/
|
|
91
47
|
const idOf = (value: unknown): string | undefined => {
|
|
92
48
|
const id = (value as { readonly id?: unknown } | null)?.id;
|
|
@@ -114,7 +70,7 @@ const readsById = (entity: EntityCore<unknown>): boolean =>
|
|
|
114
70
|
*
|
|
115
71
|
* The `catch` is a FLOOR, not a case: no write in this repo is known to succeed while its own
|
|
116
72
|
* `findById` refuses, because every guard a read applies — tenancy, soft delete — the write applies
|
|
117
|
-
* too. It stays because the cost of being wrong is asymmetric:
|
|
73
|
+
* too. It stays because the cost of being wrong is asymmetric: a subscriber is a diagnostic, and a
|
|
118
74
|
* diagnostic that turns a working write into a failing one is worse than the gap it was closing.
|
|
119
75
|
*/
|
|
120
76
|
const beforeOf = async <Row>(
|
|
@@ -129,7 +85,7 @@ const beforeOf = async <Row>(
|
|
|
129
85
|
// caller's loop, reported on the caller's writes. Without the scope the same loop was reported
|
|
130
86
|
// twice, the second verdict naming a `findById` no app code issued.
|
|
131
87
|
return await expectedQueryLoop(
|
|
132
|
-
'the row
|
|
88
|
+
'the entity.row event reads the row a point write is about to change — one read per write',
|
|
133
89
|
() => repo.findById(id),
|
|
134
90
|
);
|
|
135
91
|
} catch {
|
|
@@ -180,12 +136,15 @@ const beforeAllOf = async <Row>(
|
|
|
180
136
|
|
|
181
137
|
/**
|
|
182
138
|
* Wrap one repository so its writes are reported. Applied by `database()` to every table it builds,
|
|
183
|
-
* so an app opts in by
|
|
139
|
+
* so an app opts in by subscribing to `entity.row` and never by choosing a different repository — the
|
|
184
140
|
* rows under test are the rows the app reads, which is the whole reason `defaultDriver()` is
|
|
185
141
|
* exported at all.
|
|
186
142
|
*/
|
|
187
|
-
export function observedRepo<Row>(entity: EntityCore<Row>,
|
|
143
|
+
export function observedRepo<Row>(entity: EntityCore<Row>, driven: Repo<Row>): Repo<Row> {
|
|
188
144
|
const name = entity.$name;
|
|
145
|
+
// Beneath the observer, so a searchable entity's writes reach an external index whether or not
|
|
146
|
+
// anything hears `entity.row` (`search-mirrored-repo.ts`).
|
|
147
|
+
const repo = searchMirroredRepo(entity, driven);
|
|
189
148
|
const sealed = sealedFields(entity).map((field) => field.property);
|
|
190
149
|
|
|
191
150
|
/**
|
|
@@ -202,42 +161,31 @@ export function observedRepo<Row>(entity: EntityCore<Row>, repo: Repo<Row>): Rep
|
|
|
202
161
|
};
|
|
203
162
|
|
|
204
163
|
/**
|
|
205
|
-
* Reported at COMMIT.
|
|
206
|
-
*
|
|
207
|
-
*
|
|
208
|
-
* the write is already durable, so it is reported at once. The write origin is read NOW: it is
|
|
209
|
-
* the request's, and the commit callback may run outside that request's scope.
|
|
164
|
+
* Reported at COMMIT (`write-commit.ts`): a change reported before COMMIT put a row in a live
|
|
165
|
+
* query that a rollback then erased (plan 101, 06 m). The write origin is read NOW: it is the
|
|
166
|
+
* request's, and the commit callback may run outside that request's scope.
|
|
210
167
|
*/
|
|
211
|
-
const afterCommit = (options: RepoOptions | undefined, report: () => void): void => {
|
|
212
|
-
const tx: { onCommit?(effect: () => void): void } | undefined =
|
|
213
|
-
options?.tx ?? currentTx() ?? memoryJournal();
|
|
214
|
-
if (tx?.onCommit === undefined) report();
|
|
215
|
-
else tx.onCommit(report);
|
|
216
|
-
};
|
|
217
|
-
|
|
218
168
|
const emit = (
|
|
219
|
-
op:
|
|
169
|
+
op: EntityRowOp,
|
|
220
170
|
before: unknown,
|
|
221
171
|
after: unknown,
|
|
222
172
|
options: RepoOptions | undefined,
|
|
223
173
|
): void => {
|
|
224
|
-
|
|
225
|
-
if (observer === null) return;
|
|
174
|
+
if (!subscribed(ROW)) return;
|
|
226
175
|
const write = currentWriteOrigin();
|
|
227
|
-
const change
|
|
176
|
+
const change = {
|
|
228
177
|
entity: name,
|
|
229
178
|
op,
|
|
230
179
|
before: reported(before),
|
|
231
180
|
after: reported(after),
|
|
232
181
|
...(write === undefined ? {} : { write }),
|
|
233
182
|
};
|
|
234
|
-
afterCommit(options, () =>
|
|
183
|
+
afterCommit(options, () => instrument(ROW, change));
|
|
235
184
|
};
|
|
236
185
|
|
|
237
186
|
const bulk = (op: 'delete' | 'update', rows: number, options: RepoOptions | undefined): void => {
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
afterCommit(options, () => observer.onBulk?.({ entity: name, op, rows }));
|
|
187
|
+
if (rows > 0 && subscribed(BULK)) {
|
|
188
|
+
afterCommit(options, () => instrument(BULK, { entity: name, op, rows }));
|
|
241
189
|
}
|
|
242
190
|
};
|
|
243
191
|
|
|
@@ -282,7 +230,7 @@ export function observedRepo<Row>(entity: EntityCore<Row>, repo: Repo<Row>): Rep
|
|
|
282
230
|
* did not occur.
|
|
283
231
|
*/
|
|
284
232
|
upsertAll: async (rows: readonly Row[], args: UpsertArgs<Row>): Promise<readonly Row[]> => {
|
|
285
|
-
if (
|
|
233
|
+
if (!subscribed(ROW)) return await repo.upsertAll(rows, args);
|
|
286
234
|
const ids = [...new Set(rows.flatMap((row) => idOf(row) ?? []))];
|
|
287
235
|
const before = await beforeAllOf(entity, repo, ids);
|
|
288
236
|
const stored = await repo.upsertAll(rows, args);
|
|
@@ -295,7 +243,7 @@ export function observedRepo<Row>(entity: EntityCore<Row>, repo: Repo<Row>): Rep
|
|
|
295
243
|
},
|
|
296
244
|
|
|
297
245
|
update: async (id, patch: RowPatch<Row>, options?: RepoOptions): Promise<Row> => {
|
|
298
|
-
if (
|
|
246
|
+
if (!subscribed(ROW)) return await repo.update(id, patch, options);
|
|
299
247
|
const before = await beforeOf(entity, repo, id);
|
|
300
248
|
const after = await repo.update(id, patch, options);
|
|
301
249
|
emit('update', before, after, options);
|
|
@@ -303,7 +251,7 @@ export function observedRepo<Row>(entity: EntityCore<Row>, repo: Repo<Row>): Rep
|
|
|
303
251
|
},
|
|
304
252
|
|
|
305
253
|
delete: async (id, options?: RepoOptions): Promise<void> => {
|
|
306
|
-
if (
|
|
254
|
+
if (!subscribed(ROW)) return await repo.delete(id, options);
|
|
307
255
|
const before = await beforeOf(entity, repo, id);
|
|
308
256
|
await repo.delete(id, options);
|
|
309
257
|
emit('delete', before, null, options);
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// The installed `SearchIndex`: `runtime.ts`'s `searchIndex`, else `postgresSearchIndex()` over the
|
|
2
|
+
// ambient pool. One slot (`adapterSlot`), read per call, so a test's `installSearchIndex(memory)`
|
|
3
|
+
// and the boot's install both release only their own frame.
|
|
4
|
+
|
|
5
|
+
import { adapterSlot } from '@ultimat3/core/adapter';
|
|
6
|
+
import type { SearchIndex } from './search-index';
|
|
7
|
+
import { postgresSearchIndex } from './search-index-postgres';
|
|
8
|
+
|
|
9
|
+
let builtIn: SearchIndex | undefined;
|
|
10
|
+
|
|
11
|
+
const slot = /* @__PURE__ */ adapterSlot<SearchIndex>(() => {
|
|
12
|
+
builtIn ??= postgresSearchIndex();
|
|
13
|
+
return builtIn;
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* On `./adapter` only. Answers the release: it drops this install and no other. While an external
|
|
18
|
+
* index is the current one, every committed write of a searchable entity is mirrored to it
|
|
19
|
+
* (`search-index-sync.ts`, asked per write by `row-observer.ts`) — no subscription to hold.
|
|
20
|
+
*/
|
|
21
|
+
export const installSearchIndex = (index: SearchIndex): (() => void) => slot.install(index);
|
|
22
|
+
|
|
23
|
+
/** The index every search through the port asks — what `reindex` writes and a read queries. */
|
|
24
|
+
export const searchIndex = (): SearchIndex => slot.current();
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
// `memorySearchIndex()`: the reference `SearchIndex` an adapter author copies, and the one a unit
|
|
2
|
+
// test installs. Words, case-folded, every query word required — NOT Postgres: no stemming, no
|
|
3
|
+
// stop words, no `websearch` operators — why `memoryDriver()` still refuses `.search()`. Newest
|
|
4
|
+
// version per id wins; tombstones live as long as the index (a reference and a test index only).
|
|
5
|
+
|
|
6
|
+
import type {
|
|
7
|
+
SearchDocument,
|
|
8
|
+
SearchHit,
|
|
9
|
+
SearchIndex,
|
|
10
|
+
SearchListEntry,
|
|
11
|
+
SearchQuery,
|
|
12
|
+
SearchScope,
|
|
13
|
+
} from './search-index';
|
|
14
|
+
import { searchLimitOf, searchListLimitOf } from './search-index';
|
|
15
|
+
|
|
16
|
+
/** Letters and digits in any script; everything else separates words. */
|
|
17
|
+
const WORD = /[\p{L}\p{N}]+/gu;
|
|
18
|
+
|
|
19
|
+
/** The distinct lower-cased words of `text`. */
|
|
20
|
+
export const searchWords = (text: string): readonly string[] => [
|
|
21
|
+
...new Set(text.toLocaleLowerCase('und').match(WORD) ?? []),
|
|
22
|
+
];
|
|
23
|
+
|
|
24
|
+
/** One id's newest state: its words, or `null` — the tombstone a `remove` leaves. */
|
|
25
|
+
interface Held {
|
|
26
|
+
readonly id: string;
|
|
27
|
+
readonly version: number;
|
|
28
|
+
readonly words: ReadonlyMap<string, number> | null;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const countWords = (document: SearchDocument): ReadonlyMap<string, number> => {
|
|
32
|
+
const counts = new Map<string, number>();
|
|
33
|
+
for (const text of Object.values(document.fields)) {
|
|
34
|
+
for (const word of text.toLocaleLowerCase('und').match(WORD) ?? []) {
|
|
35
|
+
counts.set(word, (counts.get(word) ?? 0) + 1);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return counts;
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
/** One key per `(entity, tenant)`: JSON, so no entity name can be spelled into another scope. */
|
|
42
|
+
const scopeKey = (scope: SearchScope): string => JSON.stringify([scope.entity, scope.tenant]);
|
|
43
|
+
|
|
44
|
+
export const memorySearchIndex = (): SearchIndex => {
|
|
45
|
+
const scopes = new Map<string, Map<string, Held>>();
|
|
46
|
+
/** Which tenant each scope key is, so `list` can walk one entity's scopes. */
|
|
47
|
+
const scopeOwners = new Map<string, SearchScope>();
|
|
48
|
+
const scopeOf = (scope: SearchScope): Map<string, Held> => {
|
|
49
|
+
const key = scopeKey(scope);
|
|
50
|
+
const found = scopes.get(key);
|
|
51
|
+
if (found !== undefined) return found;
|
|
52
|
+
const created = new Map<string, Held>();
|
|
53
|
+
scopes.set(key, created);
|
|
54
|
+
scopeOwners.set(key, { entity: scope.entity, tenant: scope.tenant });
|
|
55
|
+
return created;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
return {
|
|
59
|
+
name: 'memory',
|
|
60
|
+
index: async (documents, options) => {
|
|
61
|
+
for (const document of documents) {
|
|
62
|
+
const scope = scopeOf(document);
|
|
63
|
+
const held = scope.get(document.id);
|
|
64
|
+
// A committed write's row exists: it replaces the tombstone of an earlier delete.
|
|
65
|
+
const replaces = options?.committed === true && held?.words === null;
|
|
66
|
+
// Keep only newer: a reindex batch that read the row before a write — or before its
|
|
67
|
+
// delete — arrives here after it, and must not put the old text back.
|
|
68
|
+
if (!replaces && (held?.version ?? Number.NEGATIVE_INFINITY) >= document.version) {
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
scope.set(document.id, {
|
|
72
|
+
id: document.id,
|
|
73
|
+
version: document.version,
|
|
74
|
+
words: countWords(document),
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
},
|
|
78
|
+
remove: async (scope, ids, version) => {
|
|
79
|
+
const held = scopeOf(scope);
|
|
80
|
+
for (const id of ids) {
|
|
81
|
+
// A remove older than what is held (a re-create after it) changes nothing.
|
|
82
|
+
if ((held.get(id)?.version ?? Number.NEGATIVE_INFINITY) > version) continue;
|
|
83
|
+
held.set(id, { id, version, words: null });
|
|
84
|
+
}
|
|
85
|
+
},
|
|
86
|
+
// Ordered by `[tenant, id]` as JSON, which is also the cursor: a total order, stable across calls.
|
|
87
|
+
list: async (query) => {
|
|
88
|
+
const limit = searchListLimitOf(query);
|
|
89
|
+
const entries: (SearchListEntry & { readonly at: string })[] = [];
|
|
90
|
+
for (const [key, owner] of scopeOwners) {
|
|
91
|
+
if (owner.entity !== query.entity) continue;
|
|
92
|
+
if (query.tenant !== undefined && owner.tenant !== query.tenant) continue;
|
|
93
|
+
for (const each of scopes.get(key)?.values() ?? []) {
|
|
94
|
+
if (each.words === null) continue;
|
|
95
|
+
const at = JSON.stringify([owner.tenant, each.id]);
|
|
96
|
+
if (query.after !== null && at <= query.after) continue;
|
|
97
|
+
entries.push({ tenant: owner.tenant, id: each.id, version: each.version, at });
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
entries.sort((a, b) => (a.at < b.at ? -1 : a.at > b.at ? 1 : 0));
|
|
101
|
+
const page = entries.slice(0, limit);
|
|
102
|
+
const last = page.at(-1);
|
|
103
|
+
return {
|
|
104
|
+
entries: page.map(({ tenant, id, version }) => ({ tenant, id, version })),
|
|
105
|
+
next: entries.length > limit && last !== undefined ? last.at : null,
|
|
106
|
+
};
|
|
107
|
+
},
|
|
108
|
+
query: async (query: SearchQuery): Promise<readonly SearchHit[]> => {
|
|
109
|
+
const limit = searchLimitOf(query);
|
|
110
|
+
const words = searchWords(query.term);
|
|
111
|
+
if (words.length === 0) return [];
|
|
112
|
+
const held = scopes.get(scopeKey(query)) ?? new Map<string, Held>();
|
|
113
|
+
const scored: { readonly id: string; readonly score: number }[] = [];
|
|
114
|
+
for (const each of held.values()) {
|
|
115
|
+
if (each.words === null) continue;
|
|
116
|
+
let score = 0;
|
|
117
|
+
for (const word of words) {
|
|
118
|
+
const count = each.words.get(word) ?? 0;
|
|
119
|
+
if (count === 0) {
|
|
120
|
+
score = -1;
|
|
121
|
+
break;
|
|
122
|
+
}
|
|
123
|
+
score += count;
|
|
124
|
+
}
|
|
125
|
+
if (score > 0) scored.push({ id: each.id, score });
|
|
126
|
+
}
|
|
127
|
+
// Best first; the id breaks a tie so the same index answers the same order every time.
|
|
128
|
+
scored.sort((a, b) => b.score - a.score || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
|
129
|
+
return scored.slice(0, limit).map(({ id }) => ({ id }));
|
|
130
|
+
},
|
|
131
|
+
};
|
|
132
|
+
};
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
// `postgresSearchIndex()`: the built-in `SearchIndex` — `.searchable()`'s generated `tsvector`,
|
|
2
|
+
// behind the port. Postgres keeps the vector itself on every write (`generated always as … stored`),
|
|
3
|
+
// so `index`/`remove` only check the entity and send nothing; `query` is the repository's own
|
|
4
|
+
// `matches` read, so tenancy, the actor check and soft delete are the ones `.search()` already has.
|
|
5
|
+
|
|
6
|
+
import type { DbClient } from '@ultimat3/db';
|
|
7
|
+
import type { EntityCore } from './entity';
|
|
8
|
+
import { EntityError } from './errors';
|
|
9
|
+
import { searchUndeclared } from './feature-errors';
|
|
10
|
+
import { postgresRepo } from './pg-driver';
|
|
11
|
+
import { getEntity } from './registry';
|
|
12
|
+
import { rowKey } from './row-key';
|
|
13
|
+
import { SEARCH_PROPERTY } from './search';
|
|
14
|
+
import type { SearchHit, SearchIndex, SearchListPage, SearchScope } from './search-index';
|
|
15
|
+
import { searchLimitOf, searchListLimitOf } from './search-index';
|
|
16
|
+
import { searchVersionOf } from './search-index-version';
|
|
17
|
+
import type { Predicate } from './tenancy';
|
|
18
|
+
|
|
19
|
+
export interface PostgresSearchIndexOptions {
|
|
20
|
+
/** Pin a client; left out, the ambient pool (or the open transaction) per statement. */
|
|
21
|
+
readonly client?: DbClient;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** The registered entity a scope names, with a search vector — refused otherwise. */
|
|
25
|
+
const searchableEntity = (name: string): EntityCore<unknown> => {
|
|
26
|
+
const core = getEntity(name)?.core;
|
|
27
|
+
if (core === undefined || core.$search === null) throw searchUndeclared(name);
|
|
28
|
+
return core;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
/** The tenant predicate a scope adds; a tenant named on an untenanted entity is refused. */
|
|
32
|
+
const tenantPredicate = (core: EntityCore<unknown>, scope: SearchScope): readonly Predicate[] => {
|
|
33
|
+
const column = core.$tenantColumn;
|
|
34
|
+
if (column === null) {
|
|
35
|
+
if (scope.tenant === null) return [];
|
|
36
|
+
throw new EntityError({
|
|
37
|
+
code: 'X_INVARIANT_VIOLATED',
|
|
38
|
+
cause: `search of ${core.$name} named a tenant, but ${core.$name} has no tenant column — the tenant would scope nothing`,
|
|
39
|
+
fix: 'searchIndex().query({ entity, tenant: null, term }) — an untenanted entity is searched whole',
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
// `null` on a tenanted entity adds no predicate: the repository then takes the acting actor's
|
|
43
|
+
// tenant, or refuses `X_TENANCY_UNSCOPED` — the rule every other read of it already follows.
|
|
44
|
+
return scope.tenant === null ? [] : [{ column, op: 'eq', value: scope.tenant }];
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* `index` and `remove` send nothing, so the version contract holds by construction: the table IS
|
|
49
|
+
* the newest state, and a stale document never reaches it.
|
|
50
|
+
*
|
|
51
|
+
* The indexes whose match is the statement's own `matches` predicate — a chain's `.search()` then
|
|
52
|
+
* sends today's single statement (`pg-sql.ts`'s `searchSql`), never a round trip for ids. Only
|
|
53
|
+
* this file adds to it: an app's index is always asked through the port.
|
|
54
|
+
*/
|
|
55
|
+
const inStatement = new WeakSet<SearchIndex>();
|
|
56
|
+
|
|
57
|
+
export const answersInStatement = (index: SearchIndex): boolean => inStatement.has(index);
|
|
58
|
+
|
|
59
|
+
export const postgresSearchIndex = (options: PostgresSearchIndexOptions = {}): SearchIndex => {
|
|
60
|
+
const repoOf = (core: EntityCore<unknown>) =>
|
|
61
|
+
postgresRepo(core, options.client === undefined ? {} : { client: options.client });
|
|
62
|
+
const index: SearchIndex = {
|
|
63
|
+
name: 'postgres',
|
|
64
|
+
index: async (documents) => {
|
|
65
|
+
for (const document of documents)
|
|
66
|
+
tenantPredicate(searchableEntity(document.entity), document);
|
|
67
|
+
},
|
|
68
|
+
remove: async (scope) => {
|
|
69
|
+
tenantPredicate(searchableEntity(scope.entity), scope);
|
|
70
|
+
},
|
|
71
|
+
// What it holds IS the table: the rows a `reindex` source reads, so its sweep removes nothing.
|
|
72
|
+
// Primary-key order, the repository's own cursor; a tenant left out is the ambient rule.
|
|
73
|
+
list: async (query): Promise<SearchListPage> => {
|
|
74
|
+
const limit = searchListLimitOf(query);
|
|
75
|
+
const core = searchableEntity(query.entity);
|
|
76
|
+
const where = tenantPredicate(core, { entity: query.entity, tenant: query.tenant ?? null });
|
|
77
|
+
const page = await repoOf(core).findMany({ where, limit, cursor: query.after });
|
|
78
|
+
const tenantColumn = core.$tenantColumn;
|
|
79
|
+
return {
|
|
80
|
+
entries: page.rows.map((row) => {
|
|
81
|
+
const tenant =
|
|
82
|
+
tenantColumn === null ? null : (row as Record<string, unknown>)[tenantColumn];
|
|
83
|
+
return {
|
|
84
|
+
tenant: typeof tenant === 'string' ? tenant : null,
|
|
85
|
+
id: rowKey(core, row),
|
|
86
|
+
version: searchVersionOf(core, row),
|
|
87
|
+
};
|
|
88
|
+
}),
|
|
89
|
+
next: page.nextCursor,
|
|
90
|
+
};
|
|
91
|
+
},
|
|
92
|
+
query: async (query): Promise<readonly SearchHit[]> => {
|
|
93
|
+
const limit = searchLimitOf(query);
|
|
94
|
+
const core = searchableEntity(query.entity);
|
|
95
|
+
const where: readonly Predicate[] = [
|
|
96
|
+
...tenantPredicate(core, query),
|
|
97
|
+
{ column: SEARCH_PROPERTY, op: 'matches', value: query.term },
|
|
98
|
+
];
|
|
99
|
+
// Primary-key order, as `.search()` with no `orderBy` answers: a rank is not a sort key the
|
|
100
|
+
// plan can seek on, so the built-in index returns matches, not a relevance order.
|
|
101
|
+
const page = await repoOf(core).findMany({ where, limit });
|
|
102
|
+
return page.rows.map((row) => ({ id: rowKey(core, row) }));
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
inStatement.add(index);
|
|
106
|
+
return index;
|
|
107
|
+
};
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
// The search index WRITE path: what a committed write of a searchable entity sends to an installed
|
|
2
|
+
// external index. A live row is indexed (`committed`: it replaces a tombstone); a deleted one — hard,
|
|
3
|
+
// or soft-stamped — removed at a tombstone no older than its last version. Ops are taken when the
|
|
4
|
+
// write runs and sent at COMMIT (`search-mirror-batch.ts`), so a rollback indexes nothing.
|
|
5
|
+
|
|
6
|
+
import { entityNow } from './clock';
|
|
7
|
+
import type { EntityCore } from './entity';
|
|
8
|
+
import { SOFT_DELETE_COLUMN } from './entity';
|
|
9
|
+
import type { RepoOptions } from './repo';
|
|
10
|
+
import { rowKey } from './row-key';
|
|
11
|
+
import { searchDocumentOf } from './search-index';
|
|
12
|
+
import { searchIndex } from './search-index-installed';
|
|
13
|
+
import { answersInStatement } from './search-index-postgres';
|
|
14
|
+
import { searchVersionOf, VERSION_STEP_MS } from './search-index-version';
|
|
15
|
+
import { mirrorBatch, mirrorFailed } from './search-mirror-batch';
|
|
16
|
+
import { commitScope } from './write-commit';
|
|
17
|
+
|
|
18
|
+
export { SEARCH_INDEX_WRITE_FAILED } from './search-mirror-batch';
|
|
19
|
+
|
|
20
|
+
type Record_ = Readonly<Record<string, unknown>>;
|
|
21
|
+
|
|
22
|
+
/** One write's mirror: bound to the index installed and the transaction open when the write ran. */
|
|
23
|
+
export interface SearchMirror {
|
|
24
|
+
/** Rows the write left in place: indexed, or removed when a soft-delete stamp is on them. */
|
|
25
|
+
changed(op: string, rows: readonly unknown[]): void;
|
|
26
|
+
/** Rows the write deleted. */
|
|
27
|
+
deleted(op: string, rows: readonly unknown[]): void;
|
|
28
|
+
/** The write touched `count` rows its driver did not name: said out loud, never silent. */
|
|
29
|
+
unnamed(op: string, count: number): void;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const isRecord = (row: unknown): row is Record_ => typeof row === 'object' && row !== null;
|
|
33
|
+
|
|
34
|
+
const tenantOf = (core: EntityCore<unknown>, row: Record_): string | null => {
|
|
35
|
+
const column = core.$tenantColumn;
|
|
36
|
+
const tenant = column === null ? null : row[column];
|
|
37
|
+
return typeof tenant === 'string' ? tenant : null;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
const keyOf = (entity: string, tenant: string | null, id: string): string =>
|
|
41
|
+
JSON.stringify([entity, tenant, id]);
|
|
42
|
+
|
|
43
|
+
export const mirrorFor = (
|
|
44
|
+
core: EntityCore<unknown>,
|
|
45
|
+
options: RepoOptions | undefined,
|
|
46
|
+
): SearchMirror | undefined => {
|
|
47
|
+
if (core.$search === null) return undefined;
|
|
48
|
+
const index = searchIndex();
|
|
49
|
+
if (answersInStatement(index)) return undefined;
|
|
50
|
+
const entity = core.$name;
|
|
51
|
+
const batch = mirrorBatch(index, commitScope(options));
|
|
52
|
+
// Read when the write runs, not at commit: a hard delete's tombstone is no older than its instant.
|
|
53
|
+
const deletedAt = entityNow().getTime();
|
|
54
|
+
/** The tombstone a row's removal leaves: past every version the row was indexed at. */
|
|
55
|
+
const removal = (op: string, row: Record_, at: number) => {
|
|
56
|
+
const tenant = tenantOf(core, row);
|
|
57
|
+
const id = rowKey(core, row);
|
|
58
|
+
const version = Math.max(at, searchVersionOf(core, row) + VERSION_STEP_MS);
|
|
59
|
+
batch.put(keyOf(entity, tenant, id), { op, entity, tenant, id, version });
|
|
60
|
+
};
|
|
61
|
+
const guarded = (op: string, work: () => void) => {
|
|
62
|
+
try {
|
|
63
|
+
work();
|
|
64
|
+
} catch (error) {
|
|
65
|
+
mirrorFailed(entity, op, error);
|
|
66
|
+
}
|
|
67
|
+
batch.done();
|
|
68
|
+
};
|
|
69
|
+
return {
|
|
70
|
+
changed: (op, rows) =>
|
|
71
|
+
guarded(op, () => {
|
|
72
|
+
for (const row of rows.filter(isRecord)) {
|
|
73
|
+
const stamp = core.$softDelete ? row[SOFT_DELETE_COLUMN] : null;
|
|
74
|
+
if (stamp instanceof Date) {
|
|
75
|
+
removal(op, row, stamp.getTime());
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
const document = searchDocumentOf(core, row);
|
|
79
|
+
batch.put(keyOf(entity, document.tenant, document.id), { op, document });
|
|
80
|
+
}
|
|
81
|
+
}),
|
|
82
|
+
deleted: (op, rows) =>
|
|
83
|
+
guarded(op, () => {
|
|
84
|
+
for (const row of rows.filter(isRecord)) removal(op, row, deletedAt);
|
|
85
|
+
}),
|
|
86
|
+
unnamed: (op, count) => guarded(op, () => batch.unnamed(entity, op, count)),
|
|
87
|
+
};
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Whether a write of `entity` owes the index anything: nothing for an entity with nothing
|
|
92
|
+
* searchable (one property read), nor under the built-in Postgres index, whose vector is the row.
|
|
93
|
+
* Asked per write, so a test's install over the boot's mirrors to the newest one only.
|
|
94
|
+
*/
|
|
95
|
+
export const mirrors = (entity: EntityCore<unknown>): boolean =>
|
|
96
|
+
entity.$search !== null && !answersInStatement(searchIndex());
|