@ultimat3/entity 30.8.0 → 31.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +68 -106
- package/README.md +180 -8
- package/package.json +6 -5
- package/src/adapter.ts +65 -0
- package/src/batch-read.ts +12 -0
- package/src/column.ts +12 -0
- package/src/columns.ts +20 -8
- package/src/cross-tenant.ts +11 -14
- package/src/cursor.ts +3 -0
- package/src/database.ts +2 -2
- package/src/declared-index.ts +72 -0
- package/src/define-driver.ts +156 -0
- package/src/describe.ts +5 -0
- package/src/driver-delete.ts +62 -0
- package/src/driver-errors.ts +80 -0
- package/src/driver-kit-types.ts +107 -0
- package/src/driver-registry.ts +26 -0
- package/src/driver-repo-reads.ts +102 -0
- package/src/driver-repo-writes.ts +236 -0
- package/src/driver-rows.ts +108 -0
- package/src/driver-scope.ts +77 -0
- package/src/driver-write-scope.ts +107 -0
- package/src/entity-error.ts +10 -0
- package/src/entity-parse.ts +3 -2
- package/src/entity-patch.ts +3 -1
- package/src/entity.ts +40 -54
- package/src/enum-column.ts +8 -0
- package/src/index.ts +43 -27
- package/src/json-null.ts +13 -0
- package/src/memory-match.ts +51 -17
- package/src/memory-observed.ts +16 -11
- package/src/memory-repo.ts +46 -99
- package/src/memory-seek.ts +12 -25
- package/src/memory-unique.ts +7 -7
- package/src/memory-update.ts +7 -2
- package/src/memory-upsert.ts +102 -0
- package/src/partition.ts +140 -0
- package/src/personal-data-kind.ts +15 -0
- package/src/personal-data.ts +66 -0
- package/src/pg-driver.ts +83 -30
- package/src/pg-row.ts +7 -1
- package/src/pg-write-sql.ts +39 -4
- package/src/plan.ts +2 -0
- package/src/predicate-screen.ts +36 -0
- package/src/query.ts +13 -4
- package/src/registry.ts +3 -0
- package/src/row-key.ts +19 -0
- package/src/row-observer.ts +38 -90
- package/src/search-index-installed.ts +24 -0
- package/src/search-index-memory.ts +132 -0
- package/src/search-index-postgres.ts +107 -0
- package/src/search-index-sync.ts +96 -0
- package/src/search-index-version.ts +93 -0
- package/src/search-index.ts +168 -0
- package/src/search-mirror-batch.ts +133 -0
- package/src/search-mirrored-repo.ts +98 -0
- package/src/search-repo.ts +91 -0
- package/src/search-written-rows.ts +36 -0
- package/src/tenant-index.ts +113 -0
- package/src/types.ts +12 -0
- package/src/view.ts +2 -1
- package/src/write-commit.ts +28 -0
- package/src/write-tag.ts +6 -2
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
// A search document's VERSION: the row's `onUpdateNow()` instant, epoch ms — STRICTLY increasing
|
|
2
|
+
// per row. `touch()` stamps it from `entityNow()`, and every driver writes it no earlier than the
|
|
3
|
+
// stored value + 1 ms (`advanceVersion`, `pg-write-sql.ts`'s `greatest`), so two writes inside one
|
|
4
|
+
// millisecond are two versions and an index can keep only newer without losing the second.
|
|
5
|
+
|
|
6
|
+
import type { EntityCore } from './entity';
|
|
7
|
+
import { EntityError } from './errors';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The property a searchable entity is versioned by: its first `onUpdateNow()` timestamp. Refused
|
|
11
|
+
* when there is none — without it no index can tell a `reindex` batch's stale read from a write.
|
|
12
|
+
*/
|
|
13
|
+
export const searchVersionColumn = (entity: EntityCore<unknown>): string => {
|
|
14
|
+
for (const [property, column] of Object.entries(entity.$columns)) {
|
|
15
|
+
if (column.$meta.kind === 'timestamptz' && column.$meta.onUpdate !== undefined) return property;
|
|
16
|
+
}
|
|
17
|
+
throw new EntityError({
|
|
18
|
+
code: 'X_INVARIANT_VIOLATED',
|
|
19
|
+
cause: `${entity.$name} is searchable but declares no onUpdateNow() column, so a search document has no version and an index cannot refuse a stale one`,
|
|
20
|
+
fix: `updatedAt: timestamp().defaultNow().onUpdateNow() # in entity('${entity.$name}'), then: x db gen "version ${entity.$name} for search"`,
|
|
21
|
+
});
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
/** How far a write moves a version past the stored one when the clock has not: one millisecond. */
|
|
25
|
+
export const VERSION_STEP_MS = 1;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* The property a write of `entity` must ADVANCE, or `undefined`: the version column of a
|
|
29
|
+
* searchable entity. Every other `onUpdateNow()` column is the clock's reading, unchanged.
|
|
30
|
+
*/
|
|
31
|
+
export const advancingVersionColumn = (entity: EntityCore<unknown>): string | undefined => {
|
|
32
|
+
if (entity.$search === null) return undefined;
|
|
33
|
+
for (const [property, column] of Object.entries(entity.$columns)) {
|
|
34
|
+
if (column.$meta.kind === 'timestamptz' && column.$meta.onUpdate !== undefined) return property;
|
|
35
|
+
}
|
|
36
|
+
return undefined;
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* A soft delete's stamp: `deletedAt`, plus the version column of a searchable entity — a deletion
|
|
41
|
+
* is a write, so it advances the version too (`advanceVersion`, `greatest(…)` in the statement), and
|
|
42
|
+
* the tombstone sits past it.
|
|
43
|
+
*/
|
|
44
|
+
export const deletionStamp = (
|
|
45
|
+
entity: EntityCore<unknown>,
|
|
46
|
+
softDeleteColumn: string,
|
|
47
|
+
now: Date,
|
|
48
|
+
): Readonly<Record<string, Date>> => {
|
|
49
|
+
const version = advancingVersionColumn(entity);
|
|
50
|
+
return version === undefined
|
|
51
|
+
? { [softDeleteColumn]: now }
|
|
52
|
+
: { [softDeleteColumn]: now, [version]: now };
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* `merged` with its version no earlier than `stored`'s + `VERSION_STEP_MS`, when the write named
|
|
57
|
+
* the version column (`named`) — the memory driver's and a custom store's half of the rule; the
|
|
58
|
+
* Postgres statement says it as `greatest(…)`.
|
|
59
|
+
*/
|
|
60
|
+
export const advanceVersion = <Row>(
|
|
61
|
+
entity: EntityCore<Row>,
|
|
62
|
+
stored: Row,
|
|
63
|
+
merged: Row,
|
|
64
|
+
named: (property: string) => boolean,
|
|
65
|
+
): Row => {
|
|
66
|
+
const property = advancingVersionColumn(entity as EntityCore<unknown>);
|
|
67
|
+
if (property === undefined || !named(property)) return merged;
|
|
68
|
+
const before = instantMs((stored as Readonly<Record<string, unknown>>)[property]);
|
|
69
|
+
const after = instantMs((merged as Readonly<Record<string, unknown>>)[property]);
|
|
70
|
+
if (before === undefined || after === undefined || after > before) return merged;
|
|
71
|
+
return Object.assign({}, merged, { [property]: new Date(before + VERSION_STEP_MS) }) as Row;
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
const instantMs = (value: unknown): number | undefined => {
|
|
75
|
+
if (value instanceof Date) return value.getTime();
|
|
76
|
+
if (typeof value === 'string') return Date.parse(value);
|
|
77
|
+
return undefined;
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
/** The version of one row of `entity`: its version column, epoch ms. */
|
|
81
|
+
export const searchVersionOf = <Row>(entity: EntityCore<Row>, row: Row): number => {
|
|
82
|
+
const core = entity as EntityCore<unknown>;
|
|
83
|
+
const property = searchVersionColumn(core);
|
|
84
|
+
const ms = instantMs((row as Readonly<Record<string, unknown>>)[property]);
|
|
85
|
+
if (ms === undefined || !Number.isFinite(ms)) {
|
|
86
|
+
throw new EntityError({
|
|
87
|
+
code: 'X_INVARIANT_VIOLATED',
|
|
88
|
+
cause: `a ${entity.$name} row reached the search index without its ${property} instant, so its document has no version`,
|
|
89
|
+
fix: `searchDocumentOf(${entity.$name}, await db.${entity.$name}.findById(id)) # the whole row: a select() that drops ${property} has no version`,
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
return ms;
|
|
93
|
+
};
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
// The `searchIndex` port: where a full-text search over an entity's `.searchable()` columns is
|
|
2
|
+
// answered. Postgres' generated `tsvector` is the built-in one (`search-index-postgres.ts`); an app
|
|
3
|
+
// writes Meilisearch or Typesense against this interface. Tenant-scoped on every call: a document
|
|
4
|
+
// is indexed under its tenant, and a query or a removal names the tenant it may reach.
|
|
5
|
+
|
|
6
|
+
import type { EntityCore } from './entity';
|
|
7
|
+
import { EntityError } from './errors';
|
|
8
|
+
import { MAX_PAGE_SIZE } from './plan';
|
|
9
|
+
import { rowKey } from './row-key';
|
|
10
|
+
import { searchVersionOf } from './search-index-version';
|
|
11
|
+
|
|
12
|
+
/** What a search answers: the entity's name and the rows' tenant, never one without the other. */
|
|
13
|
+
export interface SearchScope {
|
|
14
|
+
/** The entity NAME (`entity('<name>', …)`), not its table. */
|
|
15
|
+
readonly entity: string;
|
|
16
|
+
/** The tenant column's value; `null` for an untenanted entity, and only for one. */
|
|
17
|
+
readonly tenant: string | null;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** One row as the index sees it: its key and the text of every searchable column it holds. */
|
|
21
|
+
export interface SearchDocument extends SearchScope {
|
|
22
|
+
/** `rowKey(entity, row)` — the id a hit hands back to `findById`. */
|
|
23
|
+
readonly id: string;
|
|
24
|
+
/**
|
|
25
|
+
* The row's `onUpdateNow()` instant, epoch ms (`search-index-version.ts`). An index keeps the
|
|
26
|
+
* NEWEST version per id and refuses an older one — a `reindex` batch read before a write.
|
|
27
|
+
*/
|
|
28
|
+
readonly version: number;
|
|
29
|
+
/** Searchable property → its text. A NULL column is absent, never `''`. */
|
|
30
|
+
readonly fields: Readonly<Record<string, string>>;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface SearchQuery extends SearchScope {
|
|
34
|
+
/**
|
|
35
|
+
* What a search box sent. Every word must match (case-insensitively); punctuation is text and
|
|
36
|
+
* never an error; a term with no word in it answers no hits.
|
|
37
|
+
*/
|
|
38
|
+
readonly term: string;
|
|
39
|
+
/** At most this many hits. Default `DEFAULT_SEARCH_LIMIT`, at most `MAX_PAGE_SIZE`. */
|
|
40
|
+
readonly limit?: number;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** How a batch of documents reached `index`. */
|
|
44
|
+
export interface SearchIndexOptions {
|
|
45
|
+
/**
|
|
46
|
+
* `true` from the write path: each document is a row a COMMITTED write left, so it replaces a
|
|
47
|
+
* TOMBSTONE whatever the two versions say — a key deleted and written again is live, even inside
|
|
48
|
+
* one millisecond. Against a held document it still keeps only newer. Unset (`reindex`'s reads):
|
|
49
|
+
* a tombstone at the document's version or newer refuses it, so a stale read never brings a
|
|
50
|
+
* deleted row back.
|
|
51
|
+
*/
|
|
52
|
+
readonly committed?: boolean;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* One page of what an index holds for an entity — what `reindex`'s sweep compares with the table.
|
|
57
|
+
* `tenant` absent is EVERY tenant's documents: a maintenance read, under the same tenancy rule a
|
|
58
|
+
* `query` with `tenant: null` follows (the built-in index needs `crossTenant` for it).
|
|
59
|
+
*/
|
|
60
|
+
export interface SearchListQuery {
|
|
61
|
+
readonly entity: string;
|
|
62
|
+
readonly tenant?: string | null;
|
|
63
|
+
/** The previous page's `next`; `null` for the first page. Opaque to the caller. */
|
|
64
|
+
readonly after: string | null;
|
|
65
|
+
/** At most this many entries, from 1 to `MAX_PAGE_SIZE`. */
|
|
66
|
+
readonly limit: number;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** A document the index holds — never a tombstone. */
|
|
70
|
+
export interface SearchListEntry {
|
|
71
|
+
readonly tenant: string | null;
|
|
72
|
+
readonly id: string;
|
|
73
|
+
/** The version it holds for the id: a sweep's removal is tombstoned just above it. */
|
|
74
|
+
readonly version: number;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export interface SearchListPage {
|
|
78
|
+
readonly entries: readonly SearchListEntry[];
|
|
79
|
+
/** Where the next page starts; `null` once there is none. */
|
|
80
|
+
readonly next: string | null;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** A match: the row's id, best first in the index's own relevance order. */
|
|
84
|
+
export interface SearchHit {
|
|
85
|
+
readonly id: string;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* The port. `index` and `remove` take batches because `reindex` (a `backfill()` job) walks a table
|
|
90
|
+
* page by page. Every method is tenant-scoped: an index that answers tenant A a document indexed
|
|
91
|
+
* under tenant B is broken, and `searchIndexConformance` says so.
|
|
92
|
+
*
|
|
93
|
+
* Keep only newer: a document is applied only when its `version` is newer than what the index
|
|
94
|
+
* holds for that id — a document, or the TOMBSTONE a `remove` leaves, unless the write path sent it
|
|
95
|
+
* (`committed`), which replaces a tombstone. A version equal to the one held is a replay.
|
|
96
|
+
*/
|
|
97
|
+
export interface SearchIndex {
|
|
98
|
+
readonly name: string;
|
|
99
|
+
/**
|
|
100
|
+
* Insert or replace each document under its `(entity, tenant, id)`, when it is newer than the
|
|
101
|
+
* document held — or, `committed`, whatever tombstone is held (`SearchIndexOptions`).
|
|
102
|
+
*/
|
|
103
|
+
index(documents: readonly SearchDocument[], options?: SearchIndexOptions): Promise<void>;
|
|
104
|
+
/**
|
|
105
|
+
* Drop these ids of `scope` and leave a tombstone at `version` (the delete's instant, epoch ms),
|
|
106
|
+
* so a document no newer than the delete is refused. Another tenant's id is left where it is.
|
|
107
|
+
*/
|
|
108
|
+
remove(scope: SearchScope, ids: readonly string[], version: number): Promise<void>;
|
|
109
|
+
query(query: SearchQuery): Promise<readonly SearchHit[]>;
|
|
110
|
+
/**
|
|
111
|
+
* Every live document of an entity, paged in an order the index keeps stable across calls: each
|
|
112
|
+
* id once, none removed. `reindex` sweeps with it, removing what the table no longer holds.
|
|
113
|
+
*/
|
|
114
|
+
list(query: SearchListQuery): Promise<SearchListPage>;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export const DEFAULT_SEARCH_LIMIT = 20;
|
|
118
|
+
|
|
119
|
+
/** The limit a query asked for, checked once for every index. */
|
|
120
|
+
export const searchLimitOf = (query: SearchQuery): number => {
|
|
121
|
+
const limit = query.limit ?? DEFAULT_SEARCH_LIMIT;
|
|
122
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_PAGE_SIZE) {
|
|
123
|
+
throw new EntityError({
|
|
124
|
+
code: 'X_INVARIANT_VIOLATED',
|
|
125
|
+
cause: `search of ${query.entity}: limit must be a whole number from 1 to ${MAX_PAGE_SIZE}, got ${String(limit)}`,
|
|
126
|
+
fix: `searchIndex().query({ entity, tenant, term, limit: ${DEFAULT_SEARCH_LIMIT} })`,
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
return limit;
|
|
130
|
+
};
|
|
131
|
+
|
|
132
|
+
/** A `list` page size, checked once for every index. */
|
|
133
|
+
export const searchListLimitOf = (query: SearchListQuery): number => {
|
|
134
|
+
const { limit } = query;
|
|
135
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_PAGE_SIZE) {
|
|
136
|
+
throw new EntityError({
|
|
137
|
+
code: 'X_INVARIANT_VIOLATED',
|
|
138
|
+
cause: `list of ${query.entity}'s search documents: limit must be a whole number from 1 to ${MAX_PAGE_SIZE}, got ${String(limit)}`,
|
|
139
|
+
fix: `searchIndex().list({ entity, after: null, limit: ${DEFAULT_SEARCH_LIMIT} })`,
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
return limit;
|
|
143
|
+
};
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* The document a row of `entity` is indexed as — the ONE builder, so a write hook and `reindex`
|
|
147
|
+
* cannot index the same row two ways. Only `.searchable()` columns travel; nothing else of the row
|
|
148
|
+
* (a sealed column is never searchable, `sealed-column.ts`) leaves for an external index. Its
|
|
149
|
+
* version is the row's `onUpdateNow()` instant; an entity declaring none is refused.
|
|
150
|
+
*/
|
|
151
|
+
export const searchDocumentOf = <Row>(entity: EntityCore<Row>, row: Row): SearchDocument => {
|
|
152
|
+
const record = row as Readonly<Record<string, unknown>>;
|
|
153
|
+
const fields: Record<string, string> = {};
|
|
154
|
+
for (const [property, column] of Object.entries(entity.$columns)) {
|
|
155
|
+
if (column.$meta.searchable === undefined) continue;
|
|
156
|
+
const value = record[property];
|
|
157
|
+
if (typeof value === 'string') fields[property] = value;
|
|
158
|
+
}
|
|
159
|
+
const tenantColumn = entity.$tenantColumn;
|
|
160
|
+
const tenant = tenantColumn === null ? null : record[tenantColumn];
|
|
161
|
+
return {
|
|
162
|
+
entity: entity.$name,
|
|
163
|
+
tenant: typeof tenant === 'string' ? tenant : null,
|
|
164
|
+
id: rowKey(entity, row),
|
|
165
|
+
version: searchVersionOf(entity, row),
|
|
166
|
+
fields,
|
|
167
|
+
};
|
|
168
|
+
};
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
// One transaction's search-index ops, merged per `(entity, tenant, id)`: a key's last document, and
|
|
2
|
+
// its removal unless a later op wrote it back — sent once the transaction COMMITS, removals first.
|
|
3
|
+
// Every send to one index rides one chain: an index hears this process's writes in commit order.
|
|
4
|
+
|
|
5
|
+
import { isUltimateError, logger, renderThrowable } from '@ultimat3/core';
|
|
6
|
+
import type { SearchDocument, SearchIndex } from './search-index';
|
|
7
|
+
import type { CommitScope } from './write-commit';
|
|
8
|
+
|
|
9
|
+
/** The one log event a failed mirror writes — its fields name the entity, the op and the code. */
|
|
10
|
+
export const SEARCH_INDEX_WRITE_FAILED = 'entity.search-index.write-failed';
|
|
11
|
+
|
|
12
|
+
export const unmirrored = (entity: string, op: string, cause: string, code?: string): void => {
|
|
13
|
+
logger.warn(SEARCH_INDEX_WRITE_FAILED, {
|
|
14
|
+
entity,
|
|
15
|
+
op,
|
|
16
|
+
...(code === undefined ? {} : { code }),
|
|
17
|
+
cause,
|
|
18
|
+
fix: `reindex({ name: 'reindex-${entity}', entity, source, tenant: 'none' }) # rebuild the index, which missed this ${op}`,
|
|
19
|
+
});
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
export const mirrorFailed = (entity: string, op: string, error: unknown): void =>
|
|
23
|
+
unmirrored(entity, op, renderThrowable(error), isUltimateError(error) ? error.code : undefined);
|
|
24
|
+
|
|
25
|
+
/** One key's op: its document, or its removal at a tombstone version. */
|
|
26
|
+
export type MirrorOp = { readonly op: string; readonly document: SearchDocument } | Removal;
|
|
27
|
+
|
|
28
|
+
interface Removal {
|
|
29
|
+
readonly op: string;
|
|
30
|
+
readonly entity: string;
|
|
31
|
+
readonly tenant: string | null;
|
|
32
|
+
readonly id: string;
|
|
33
|
+
readonly version: number;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* What one key owes at commit. A removal followed by a document keeps BOTH, in that order: the
|
|
38
|
+
* document (`committed`) replaces the tombstone, and the removal is what retires the document the
|
|
39
|
+
* index held — of an older incarnation, at a version the new row may not pass. A document followed
|
|
40
|
+
* by a removal is the removal alone.
|
|
41
|
+
*/
|
|
42
|
+
interface KeyOps {
|
|
43
|
+
removal?: Removal;
|
|
44
|
+
document?: SearchDocument;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
interface Pending {
|
|
48
|
+
readonly index: SearchIndex;
|
|
49
|
+
readonly ops: Map<string, KeyOps>;
|
|
50
|
+
readonly unnamed: { readonly entity: string; readonly op: string; readonly count: number }[];
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Each index's tail: the next send waits for the last, whatever it answered. */
|
|
54
|
+
const tails = new WeakMap<SearchIndex, Promise<void>>();
|
|
55
|
+
|
|
56
|
+
const send = (index: SearchIndex, entity: string, op: string, call: () => Promise<void>) => {
|
|
57
|
+
const next = (tails.get(index) ?? Promise.resolve()).then(async () => {
|
|
58
|
+
try {
|
|
59
|
+
await call();
|
|
60
|
+
} catch (error) {
|
|
61
|
+
mirrorFailed(entity, op, error);
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
tails.set(index, next);
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
const flush = ({ index, ops, unnamed }: Pending): void => {
|
|
68
|
+
const documents: SearchDocument[] = [];
|
|
69
|
+
const removals = new Map<
|
|
70
|
+
string,
|
|
71
|
+
{ entity: string; tenant: string | null; version: number; op: string; ids: string[] }
|
|
72
|
+
>();
|
|
73
|
+
for (const { removal, document } of ops.values()) {
|
|
74
|
+
if (document !== undefined) documents.push(document);
|
|
75
|
+
if (removal === undefined) continue;
|
|
76
|
+
const at = JSON.stringify([removal.entity, removal.tenant, removal.version]);
|
|
77
|
+
const group = removals.get(at) ?? { ...removal, ids: [] };
|
|
78
|
+
group.ids.push(removal.id);
|
|
79
|
+
removals.set(at, group);
|
|
80
|
+
}
|
|
81
|
+
for (const { entity, tenant, version, op, ids } of removals.values()) {
|
|
82
|
+
send(index, entity, op, () => index.remove({ entity, tenant }, ids, version));
|
|
83
|
+
}
|
|
84
|
+
if (documents.length > 0) {
|
|
85
|
+
const [first] = documents;
|
|
86
|
+
send(index, first?.entity ?? '', 'index', () => index.index(documents, { committed: true }));
|
|
87
|
+
}
|
|
88
|
+
for (const { entity, op, count } of unnamed) {
|
|
89
|
+
unmirrored(entity, op, `${count} rows changed by a store that did not name them`);
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
/** The open batch per transaction: one per scope, flushed by its commit and dropped by rollback. */
|
|
94
|
+
const open = new WeakMap<CommitScope, Pending>();
|
|
95
|
+
|
|
96
|
+
export interface MirrorBatch {
|
|
97
|
+
put(key: string, op: MirrorOp): void;
|
|
98
|
+
unnamed(entity: string, op: string, count: number): void;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* The batch `index` takes this write's ops into: the transaction's, registered on its commit the
|
|
103
|
+
* first time — or, outside one, a batch sent as soon as it is filled (`done`).
|
|
104
|
+
*/
|
|
105
|
+
export const mirrorBatch = (
|
|
106
|
+
index: SearchIndex,
|
|
107
|
+
scope: CommitScope | undefined,
|
|
108
|
+
): MirrorBatch & { done(): void } => {
|
|
109
|
+
const existing = scope === undefined ? undefined : open.get(scope);
|
|
110
|
+
const pending = existing ?? { index, ops: new Map<string, KeyOps>(), unnamed: [] };
|
|
111
|
+
if (scope !== undefined && existing === undefined) {
|
|
112
|
+
open.set(scope, pending);
|
|
113
|
+
scope.onCommit?.(() => {
|
|
114
|
+
open.delete(scope);
|
|
115
|
+
flush(pending);
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
return {
|
|
119
|
+
put: (key, op) => {
|
|
120
|
+
if (!('document' in op)) {
|
|
121
|
+
pending.ops.set(key, { removal: op });
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
pending.ops.set(key, { ...pending.ops.get(key), document: op.document });
|
|
125
|
+
},
|
|
126
|
+
unnamed: (entity, op, count) => {
|
|
127
|
+
pending.unnamed.push({ entity, op, count });
|
|
128
|
+
},
|
|
129
|
+
done: () => {
|
|
130
|
+
if (scope === undefined) flush(pending);
|
|
131
|
+
},
|
|
132
|
+
};
|
|
133
|
+
};
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
// Every write of a searchable entity, mirrored to an installed external index at COMMIT. A delete
|
|
2
|
+
// and a filtered write run inside `collectWrittenRows`, so the driver answers the rows it touched
|
|
3
|
+
// (`returning *`) — for a composite or non-`id` key too. Not searchable, or the built-in index
|
|
4
|
+
// installed: the repository is handed back, or the write delegates with no extra statement.
|
|
5
|
+
|
|
6
|
+
import type { EntityCore } from './entity';
|
|
7
|
+
import type { Repo, RepoOptions } from './repo';
|
|
8
|
+
import type { SearchMirror } from './search-index-sync';
|
|
9
|
+
import { mirrorFor, mirrors } from './search-index-sync';
|
|
10
|
+
import { collectWrittenRows } from './search-written-rows';
|
|
11
|
+
|
|
12
|
+
type Told = (mirror: SearchMirror, rows: readonly unknown[]) => void;
|
|
13
|
+
|
|
14
|
+
/** A write that answers its rows: each one mirrored as changed. */
|
|
15
|
+
const stored = async <T extends readonly unknown[]>(
|
|
16
|
+
entity: EntityCore<unknown>,
|
|
17
|
+
op: string,
|
|
18
|
+
options: RepoOptions | undefined,
|
|
19
|
+
write: () => Promise<T>,
|
|
20
|
+
): Promise<T> => {
|
|
21
|
+
const rows = await write();
|
|
22
|
+
if (mirrors(entity)) mirrorFor(entity, options)?.changed(op, rows);
|
|
23
|
+
return rows;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* A write that answers a count, run so its driver hands the rows over. `rows` absent is a store
|
|
28
|
+
* that named none (a custom driver's, or a write past the bound): logged, never silent, never the
|
|
29
|
+
* write's failure.
|
|
30
|
+
*/
|
|
31
|
+
const collected = async <T>(
|
|
32
|
+
entity: EntityCore<unknown>,
|
|
33
|
+
op: string,
|
|
34
|
+
options: RepoOptions | undefined,
|
|
35
|
+
write: () => Promise<T>,
|
|
36
|
+
told: Told,
|
|
37
|
+
touched: (result: T) => number,
|
|
38
|
+
): Promise<T> => {
|
|
39
|
+
if (!mirrors(entity)) return await write();
|
|
40
|
+
const { result, rows } = await collectWrittenRows(write);
|
|
41
|
+
const mirror = mirrorFor(entity, options);
|
|
42
|
+
if (mirror === undefined) return result;
|
|
43
|
+
if (rows !== undefined) told(mirror, rows);
|
|
44
|
+
else if (touched(result) > 0) mirror.unnamed(op, touched(result));
|
|
45
|
+
return result;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
export function searchMirroredRepo<Row>(entity: EntityCore<Row>, repo: Repo<Row>): Repo<Row> {
|
|
49
|
+
if (entity.$search === null) return repo;
|
|
50
|
+
const core = entity as EntityCore<unknown>;
|
|
51
|
+
const single = async <T>(row: Promise<T>): Promise<readonly [T]> => [await row];
|
|
52
|
+
const changed =
|
|
53
|
+
(op: string): Told =>
|
|
54
|
+
(mirror, rows) =>
|
|
55
|
+
mirror.changed(op, rows);
|
|
56
|
+
const deleted =
|
|
57
|
+
(op: string): Told =>
|
|
58
|
+
(mirror, rows) =>
|
|
59
|
+
mirror.deleted(op, rows);
|
|
60
|
+
|
|
61
|
+
return {
|
|
62
|
+
...repo,
|
|
63
|
+
insert: async (values, options) =>
|
|
64
|
+
(await stored(core, 'insert', options, () => single(repo.insert(values, options))))[0],
|
|
65
|
+
insertAll: (rows, options) =>
|
|
66
|
+
stored(core, 'insertAll', options, () => repo.insertAll(rows, options)),
|
|
67
|
+
upsertAll: (rows, args) => stored(core, 'upsertAll', args, () => repo.upsertAll(rows, args)),
|
|
68
|
+
update: async (id, patch, options) =>
|
|
69
|
+
(await stored(core, 'update', options, () => single(repo.update(id, patch, options))))[0],
|
|
70
|
+
delete: (id, options) =>
|
|
71
|
+
collected(
|
|
72
|
+
core,
|
|
73
|
+
'delete',
|
|
74
|
+
options,
|
|
75
|
+
() => repo.delete(id, options),
|
|
76
|
+
deleted('delete'),
|
|
77
|
+
() => 1,
|
|
78
|
+
),
|
|
79
|
+
deleteWhere: (filter, options) =>
|
|
80
|
+
collected(
|
|
81
|
+
core,
|
|
82
|
+
'deleteWhere',
|
|
83
|
+
options,
|
|
84
|
+
() => repo.deleteWhere(filter, options),
|
|
85
|
+
deleted('deleteWhere'),
|
|
86
|
+
(count) => count,
|
|
87
|
+
),
|
|
88
|
+
updateWhere: (filter, patch, options) =>
|
|
89
|
+
collected(
|
|
90
|
+
core,
|
|
91
|
+
'updateWhere',
|
|
92
|
+
options,
|
|
93
|
+
() => repo.updateWhere(filter, patch, options),
|
|
94
|
+
changed('updateWhere'),
|
|
95
|
+
(count) => count,
|
|
96
|
+
),
|
|
97
|
+
};
|
|
98
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
// The ONE search path: a chain's `.search(term)` asks the INSTALLED `SearchIndex`. The built-in
|
|
2
|
+
// Postgres index answers inside the statement (its `matches` predicate, today's SQL, untouched);
|
|
3
|
+
// any other index is asked for ids first, and the chain then reads those rows with its own
|
|
4
|
+
// tenancy, soft delete, order and cursor — so a Meilisearch hit can never widen what the chain sees.
|
|
5
|
+
|
|
6
|
+
import { tryUseContext } from '@ultimat3/core';
|
|
7
|
+
import type { EntityCore } from './entity';
|
|
8
|
+
import { EntityError } from './errors';
|
|
9
|
+
import { MAX_PAGE_SIZE } from './plan';
|
|
10
|
+
import type { FindManyArgs, Repo } from './repo';
|
|
11
|
+
import { SEARCH_PROPERTY } from './search';
|
|
12
|
+
import { searchIndex } from './search-index-installed';
|
|
13
|
+
import { answersInStatement } from './search-index-postgres';
|
|
14
|
+
import type { Predicate } from './tenancy';
|
|
15
|
+
|
|
16
|
+
const isMatch = (predicate: Predicate): boolean =>
|
|
17
|
+
predicate.column === SEARCH_PROPERTY && predicate.op === 'matches';
|
|
18
|
+
|
|
19
|
+
/** The tenant the chain is scoped to: its own `eq` on the tenant column, else the actor's. */
|
|
20
|
+
const tenantOf = (entity: EntityCore<unknown>, where: readonly Predicate[]): string | null => {
|
|
21
|
+
const column = entity.$tenantColumn;
|
|
22
|
+
if (column === null) return null;
|
|
23
|
+
const named = where.find((each) => each.column === column && each.op === 'eq');
|
|
24
|
+
if (typeof named?.value === 'string') return named.value;
|
|
25
|
+
return tryUseContext()?.actor.orgId ?? null;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
/** The single key a hit's id is read back by; a composite key has no `in` an index id can name. */
|
|
29
|
+
const keyOf = (entity: EntityCore<unknown>): string => {
|
|
30
|
+
const [key, ...rest] = entity.$primaryKey;
|
|
31
|
+
if (key === undefined || rest.length > 0) {
|
|
32
|
+
throw new EntityError({
|
|
33
|
+
code: 'X_INVARIANT_VIOLATED',
|
|
34
|
+
cause: `search of ${entity.$name} through an external index needs a single-column primary key; it has ${entity.$primaryKey.length}`,
|
|
35
|
+
fix: 'installSearchIndex(postgresSearchIndex()) — the built-in index matches inside the statement, whatever the key',
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
return key;
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
/** `args` with every `matches` predicate replaced by the ids the installed index answers. */
|
|
42
|
+
const resolved = async (entity: EntityCore<unknown>, args: FindManyArgs): Promise<FindManyArgs> => {
|
|
43
|
+
const where = args.where ?? [];
|
|
44
|
+
const index = searchIndex();
|
|
45
|
+
if (answersInStatement(index)) return args;
|
|
46
|
+
const tenant = tenantOf(entity, where);
|
|
47
|
+
const swapped: Predicate[] = [];
|
|
48
|
+
for (const predicate of where) {
|
|
49
|
+
if (!isMatch(predicate)) {
|
|
50
|
+
swapped.push(predicate);
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
const hits = await index.query({
|
|
54
|
+
entity: entity.$name,
|
|
55
|
+
tenant,
|
|
56
|
+
term: String(predicate.value ?? ''),
|
|
57
|
+
limit: MAX_PAGE_SIZE,
|
|
58
|
+
});
|
|
59
|
+
swapped.push({ column: keyOf(entity), op: 'in', value: hits.map((hit) => hit.id) });
|
|
60
|
+
}
|
|
61
|
+
return { ...args, where: swapped };
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
const searching = (args: FindManyArgs | undefined): boolean => args?.where?.some(isMatch) === true;
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* `repo` with every read that takes a chain's arguments routed through `resolved` — only when the
|
|
68
|
+
* chain searched, so a read with no `.search()` reaches the repository exactly as before.
|
|
69
|
+
*/
|
|
70
|
+
export const searchedRepo = <Row>(entity: EntityCore<Row>, repo: Repo<Row>): Repo<Row> => {
|
|
71
|
+
const core = entity as EntityCore<unknown>;
|
|
72
|
+
const series = repo.series;
|
|
73
|
+
return {
|
|
74
|
+
...repo,
|
|
75
|
+
findMany: async (args) =>
|
|
76
|
+
repo.findMany(searching(args) ? await resolved(core, args ?? {}) : args),
|
|
77
|
+
count: async (args) => repo.count(searching(args) ? await resolved(core, args ?? {}) : args),
|
|
78
|
+
countBy: async (column, args) =>
|
|
79
|
+
repo.countBy(column, searching(args) ? await resolved(core, args ?? {}) : args),
|
|
80
|
+
aggregate: async (fn, column, args) =>
|
|
81
|
+
repo.aggregate(fn, column, searching(args) ? await resolved(core, args ?? {}) : args),
|
|
82
|
+
approximateCount: async (args) =>
|
|
83
|
+
repo.approximateCount(searching(args) ? await resolved(core, args ?? {}) : args),
|
|
84
|
+
...(series === undefined
|
|
85
|
+
? {}
|
|
86
|
+
: {
|
|
87
|
+
series: async (spec, args) =>
|
|
88
|
+
series.call(repo, spec, searching(args) ? await resolved(core, args ?? {}) : args),
|
|
89
|
+
}),
|
|
90
|
+
};
|
|
91
|
+
};
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// The rows a delete or a filtered write touched, handed up to the search mirror. A driver hands
|
|
2
|
+
// them over only while `collectWrittenRows` runs — the mirror opens it for a searchable entity
|
|
3
|
+
// under an external index, and nowhere else — so every other write sends no `returning *`.
|
|
4
|
+
|
|
5
|
+
import { asyncContext } from '@ultimat3/core';
|
|
6
|
+
|
|
7
|
+
/** What one write answered: its rows, or nothing — a store that cannot say which rows it touched. */
|
|
8
|
+
interface WrittenRows {
|
|
9
|
+
rows: unknown[] | undefined;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
const collecting = asyncContext<WrittenRows>('the search index write path');
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* The sink a driver hands a write's rows to, or `undefined` when nobody asked — the one check a
|
|
16
|
+
* driver pays per delete and filtered write. A driver that holds the rows anyway calls it too.
|
|
17
|
+
*/
|
|
18
|
+
export const writtenRowsSink = (): ((rows: readonly unknown[]) => void) | undefined => {
|
|
19
|
+
const open = collecting.get();
|
|
20
|
+
if (open === undefined) return undefined;
|
|
21
|
+
return (rows) => {
|
|
22
|
+
open.rows = [...(open.rows ?? []), ...rows];
|
|
23
|
+
};
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Run one write and answer what it returned plus the rows its driver handed over. `rows` is
|
|
28
|
+
* `undefined` when the driver handed none — a custom store's raw update, which reports a count.
|
|
29
|
+
*/
|
|
30
|
+
export const collectWrittenRows = async <T>(
|
|
31
|
+
write: () => Promise<T>,
|
|
32
|
+
): Promise<{ readonly result: T; readonly rows: readonly unknown[] | undefined }> => {
|
|
33
|
+
const open: WrittenRows = { rows: undefined };
|
|
34
|
+
const result = await collecting.run(open, write);
|
|
35
|
+
return { result, rows: open.rows };
|
|
36
|
+
};
|