@dokebi/search 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AI.md +134 -0
- package/LICENSE +21 -0
- package/README.md +8 -2
- package/dist/api.d.ts +78 -0
- package/dist/api.js +1 -0
- package/dist/cursor.d.ts +19 -0
- package/dist/cursor.js +68 -0
- package/dist/dependency.d.ts +4 -0
- package/dist/dependency.js +13 -0
- package/dist/errors.d.ts +6 -0
- package/dist/errors.js +58 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.js +7 -0
- package/dist/indexes.d.ts +27 -0
- package/dist/indexes.js +87 -0
- package/dist/install.d.ts +37 -0
- package/dist/install.js +56 -0
- package/dist/options.d.ts +23 -0
- package/dist/options.js +73 -0
- package/dist/plugin.d.ts +6 -0
- package/dist/plugin.js +671 -0
- package/dist/sql.d.ts +32 -0
- package/dist/sql.js +104 -0
- package/dist/store.d.ts +66 -0
- package/dist/store.js +137 -0
- package/dist/terms.d.ts +31 -0
- package/dist/terms.js +58 -0
- package/options.schema.json +95 -0
- package/package.json +86 -4
- package/templates/indexes.ts +10 -0
- package/templates/migrations/dokebi_search.ts +92 -0
package/dist/plugin.js
ADDED
|
@@ -0,0 +1,671 @@
|
|
|
1
|
+
import { definePlugin, definePoint, providerDependency, } from '@dokebi/core';
|
|
2
|
+
import { diagnostics } from '@dokebi/core/libraries';
|
|
3
|
+
import { Cursors, queryFingerprint } from './cursor.js';
|
|
4
|
+
import { SearchError } from './errors.js';
|
|
5
|
+
import { indexesOf } from './indexes.js';
|
|
6
|
+
import { DEFAULT_SECRETS_ENV, MIN_SECRET_LENGTH, options } from './options.js';
|
|
7
|
+
import { detectDriver, isMissingTable, isQueryable, serverDialect, session, } from './sql.js';
|
|
8
|
+
import { COUNTS_TABLE, INDEXES_TABLE, Store, TERMS_TABLE, } from './store.js';
|
|
9
|
+
import { MAX_QUERY_WORDS, queryTerms, terms } from './terms.js';
|
|
10
|
+
const db = providerDependency({
|
|
11
|
+
name: 'dokebi-db',
|
|
12
|
+
version: '^0.4.1',
|
|
13
|
+
packageName: '@dokebi/db',
|
|
14
|
+
load: () => import('@dokebi/db'),
|
|
15
|
+
});
|
|
16
|
+
/** The `core:describe` point (core 0.4.2), declared by name. */
|
|
17
|
+
export const describePoint = definePoint('core:describe');
|
|
18
|
+
/** How often a call tries to reach a database that was down (an optional connection, D-0127). */
|
|
19
|
+
const RETRY_MS = 1000;
|
|
20
|
+
const DEFAULT_LIMIT = 20;
|
|
21
|
+
/** Rows of the leading word one statement reads (a request reads up to examineLimit). */
|
|
22
|
+
const CHUNK = 500;
|
|
23
|
+
const MAX_ID = 128;
|
|
24
|
+
const MAX_SCOPE = 64;
|
|
25
|
+
const MAX_SCOPES = 32;
|
|
26
|
+
/** A field's text above this many characters is refused (the terms of 1 MB of text). */
|
|
27
|
+
const MAX_TEXT = 1_000_000;
|
|
28
|
+
const SEP = '\u0000';
|
|
29
|
+
/** Buffered words that start a flush before its interval. */
|
|
30
|
+
const FLUSH_WORDS = 2000;
|
|
31
|
+
/** How often repeated flush failures are logged (D-0047). */
|
|
32
|
+
const REPEAT_MS = 60_000;
|
|
33
|
+
function optionalConnection(database, name) {
|
|
34
|
+
if (typeof database.status !== 'function')
|
|
35
|
+
return false;
|
|
36
|
+
return database.status(name).startup === 'optional';
|
|
37
|
+
}
|
|
38
|
+
/** A failure that waiting for the database does not cure (a missing table, a wrong server). */
|
|
39
|
+
function permanent(error) {
|
|
40
|
+
const { plugin, code } = (error ?? {});
|
|
41
|
+
if (plugin === 'dokebi-search')
|
|
42
|
+
return code !== 'NOT_READY';
|
|
43
|
+
return plugin === 'dokebi-db' && (code === 'INVALID_OPTIONS' || code === 'INVALID_RESOURCE');
|
|
44
|
+
}
|
|
45
|
+
/** The signing secrets: the environment variable of secretsEnv wins over dokebi.yaml. */
|
|
46
|
+
function signingSecrets(settings) {
|
|
47
|
+
const variable = settings.secretsEnv ?? DEFAULT_SECRETS_ENV;
|
|
48
|
+
const value = process.env[variable];
|
|
49
|
+
if (value?.trim()) {
|
|
50
|
+
const secrets = value.split(',').map((part) => part.trim());
|
|
51
|
+
if (secrets.length > 8 ||
|
|
52
|
+
new Set(secrets).size !== secrets.length ||
|
|
53
|
+
secrets.some((secret) => secret.length < MIN_SECRET_LENGTH || secret.length > 512))
|
|
54
|
+
throw new SearchError('INVALID_OPTIONS', `Search secrets in ${variable} are invalid: one to eight different secrets of ${MIN_SECRET_LENGTH} to 512 characters, separated by commas`, { data: { variable } });
|
|
55
|
+
return { secrets, from: 'environment', variable };
|
|
56
|
+
}
|
|
57
|
+
if (settings.secrets?.length)
|
|
58
|
+
return { secrets: settings.secrets, from: 'options', variable };
|
|
59
|
+
throw new SearchError('SECRET_MISSING', `Search cursor secret missing: options.dokebi-search.secrets in dokebi.yaml is empty and ${variable} is not set`, { data: { variable } });
|
|
60
|
+
}
|
|
61
|
+
function plain(value) {
|
|
62
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
63
|
+
}
|
|
64
|
+
const byKey = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
|
|
65
|
+
class SearchRuntime {
|
|
66
|
+
#options;
|
|
67
|
+
#indexes;
|
|
68
|
+
#context;
|
|
69
|
+
#database;
|
|
70
|
+
#resource;
|
|
71
|
+
#live;
|
|
72
|
+
#bringing;
|
|
73
|
+
#lastTry = 0;
|
|
74
|
+
#lastError;
|
|
75
|
+
#derived;
|
|
76
|
+
#cursors;
|
|
77
|
+
#stale = new Set();
|
|
78
|
+
/**
|
|
79
|
+
* Word count changes not yet in the table: `index\0field\0term` → delta. The counts only
|
|
80
|
+
* choose a search's leading word (a hint), so they are added after the writes, outside the
|
|
81
|
+
* caller's transaction: a common word's count is never a row every writer waits on (db AI.md
|
|
82
|
+
* 6.4), and MariaDB's snapshot isolation never refuses the caller's commit over it (errno 1020).
|
|
83
|
+
* A rolled-back write leaves its delta: the counts may drift, never the results.
|
|
84
|
+
*/
|
|
85
|
+
#pending = new Map();
|
|
86
|
+
#flushing;
|
|
87
|
+
#repeats;
|
|
88
|
+
constructor(parsed, indexes) {
|
|
89
|
+
this.#options = parsed;
|
|
90
|
+
this.#indexes = indexes;
|
|
91
|
+
}
|
|
92
|
+
async setup(context) {
|
|
93
|
+
const started = performance.now();
|
|
94
|
+
const settings = this.#options;
|
|
95
|
+
if (settings.searchLimit < settings.examineLimit)
|
|
96
|
+
throw new SearchError('INVALID_OPTIONS', `searchLimit (${settings.searchLimit}) is below examineLimit (${settings.examineLimit})`, { data: { field: 'searchLimit' } });
|
|
97
|
+
const database = context.getDependency(db);
|
|
98
|
+
let resource;
|
|
99
|
+
try {
|
|
100
|
+
resource = database.resource(settings.connection);
|
|
101
|
+
}
|
|
102
|
+
catch (cause) {
|
|
103
|
+
throw new SearchError('UNKNOWN_CONNECTION', `The search connection ${settings.connection} is not a dokebi-db connection`, { cause, data: { connection: settings.connection } });
|
|
104
|
+
}
|
|
105
|
+
if (!isQueryable(resource))
|
|
106
|
+
throw new SearchError('UNSUPPORTED_RESOURCE', `Connection ${settings.connection} is not a pg, mysql2/promise or mariadb pool`, { data: { connection: settings.connection } });
|
|
107
|
+
this.#context = context;
|
|
108
|
+
this.#database = database;
|
|
109
|
+
this.#resource = resource;
|
|
110
|
+
if (context.runtime.describing === true)
|
|
111
|
+
return this.#describing(context);
|
|
112
|
+
const signing = signingSecrets(settings);
|
|
113
|
+
this.#cursors = new Cursors(signing.secrets);
|
|
114
|
+
// Names only, never the values (D-0047).
|
|
115
|
+
context.logger.log('debug', 'Search cursor secrets resolved.', {
|
|
116
|
+
code: 'SEARCH_SECRETS',
|
|
117
|
+
from: signing.from,
|
|
118
|
+
variable: signing.variable,
|
|
119
|
+
count: signing.secrets.length,
|
|
120
|
+
});
|
|
121
|
+
try {
|
|
122
|
+
this.#live = await this.#bringUp();
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
// D-0127: an optional connection that is down does not stop the start; calls answer
|
|
126
|
+
// NOT_READY (with the cause) and reach the database again once it answers.
|
|
127
|
+
if (!optionalConnection(database, settings.connection) || permanent(error))
|
|
128
|
+
throw error;
|
|
129
|
+
this.#unavailable(error);
|
|
130
|
+
}
|
|
131
|
+
this.#repeats = new diagnostics.Repeats({ logger: context.logger, intervalMs: REPEAT_MS });
|
|
132
|
+
const timer = setInterval(() => {
|
|
133
|
+
void this.#flush();
|
|
134
|
+
}, settings.counts.flushMs);
|
|
135
|
+
timer.unref();
|
|
136
|
+
context.defer(async () => {
|
|
137
|
+
clearInterval(timer);
|
|
138
|
+
await this.#flush();
|
|
139
|
+
this.#repeats?.flush();
|
|
140
|
+
});
|
|
141
|
+
context.logger.log('debug', 'Search set up.', {
|
|
142
|
+
code: 'SEARCH_SETUP',
|
|
143
|
+
connection: settings.connection,
|
|
144
|
+
indexes: this.#indexes.names.length,
|
|
145
|
+
ready: this.#live !== undefined,
|
|
146
|
+
ms: Math.round(performance.now() - started),
|
|
147
|
+
});
|
|
148
|
+
return this.#api();
|
|
149
|
+
}
|
|
150
|
+
async #bringUp() {
|
|
151
|
+
const settings = this.#options;
|
|
152
|
+
const context = this.#context;
|
|
153
|
+
const resource = this.#resource;
|
|
154
|
+
const driver = await detectDriver(resource);
|
|
155
|
+
if (!driver)
|
|
156
|
+
throw new SearchError('UNSUPPORTED_RESOURCE', `Connection ${settings.connection} is not a pg, mysql2/promise or mariadb pool`, { data: { connection: settings.connection } });
|
|
157
|
+
const server = await serverDialect(driver, session(driver, resource));
|
|
158
|
+
// The plugin's own pool of the same connection, so searches never take the request pool;
|
|
159
|
+
// pool.max 0 shares the connection's.
|
|
160
|
+
this.#derived ??=
|
|
161
|
+
settings.pool.max > 0
|
|
162
|
+
? await this.#database.derive(settings.connection, {
|
|
163
|
+
purpose: 'search',
|
|
164
|
+
overrides: { max: settings.pool.max },
|
|
165
|
+
})
|
|
166
|
+
: resource;
|
|
167
|
+
const pool = session(driver, this.#derived);
|
|
168
|
+
const store = new Store(server.dialect, settings.counts.slots);
|
|
169
|
+
let stored;
|
|
170
|
+
try {
|
|
171
|
+
await store.check(pool);
|
|
172
|
+
stored = await store.fingerprints(pool);
|
|
173
|
+
}
|
|
174
|
+
catch (error) {
|
|
175
|
+
if (isMissingTable(error))
|
|
176
|
+
throw new SearchError('TABLE_MISSING', 'The search tables are missing or older than this version', { cause: error, data: { connection: settings.connection } });
|
|
177
|
+
throw error;
|
|
178
|
+
}
|
|
179
|
+
// A new index is current (its rows are written from now on); a changed one is stale until
|
|
180
|
+
// the application indexed its rows again and called markCurrent.
|
|
181
|
+
for (const name of this.#indexes.names) {
|
|
182
|
+
const fingerprint = this.#indexes.fingerprint(name);
|
|
183
|
+
const before = stored.get(name);
|
|
184
|
+
if (before === undefined)
|
|
185
|
+
await store.saveFingerprint(pool, name, fingerprint, Date.now());
|
|
186
|
+
else if (before !== fingerprint) {
|
|
187
|
+
this.#stale.add(name);
|
|
188
|
+
context.logger.log('warn', 'A search index declaration changed: its rows need indexing again.', {
|
|
189
|
+
code: 'SEARCH_INDEX_CHANGED',
|
|
190
|
+
index: name,
|
|
191
|
+
fix: `Index the rows of ${name} again (search.indexMany in batches), then call search.markCurrent('${name}')`,
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
context.logger.log('debug', 'Search connected.', {
|
|
196
|
+
code: 'SEARCH_CONNECTED',
|
|
197
|
+
connection: settings.connection,
|
|
198
|
+
dialect: server.dialect,
|
|
199
|
+
version: server.version,
|
|
200
|
+
pool: settings.pool.max,
|
|
201
|
+
stale: this.#stale.size,
|
|
202
|
+
});
|
|
203
|
+
return { driver, store, pool };
|
|
204
|
+
}
|
|
205
|
+
#unavailable(error) {
|
|
206
|
+
this.#lastError = error;
|
|
207
|
+
this.#lastTry = Date.now();
|
|
208
|
+
this.#context?.logger.log('warn', 'Search database is not reachable; calls answer NOT_READY until it is.', {
|
|
209
|
+
code: 'SEARCH_DB_UNAVAILABLE',
|
|
210
|
+
connection: this.#options.connection,
|
|
211
|
+
error: diagnostics.describeError(error),
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
/** The live store, or one more try when the database was down (at most once per second). */
|
|
215
|
+
async #ready() {
|
|
216
|
+
if (this.#live)
|
|
217
|
+
return this.#live;
|
|
218
|
+
if (!this.#bringing && Date.now() - this.#lastTry >= RETRY_MS) {
|
|
219
|
+
this.#lastTry = Date.now();
|
|
220
|
+
this.#bringing = this.#bringUp().finally(() => {
|
|
221
|
+
this.#bringing = undefined;
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
if (this.#bringing) {
|
|
225
|
+
try {
|
|
226
|
+
this.#live = await this.#bringing;
|
|
227
|
+
this.#context?.logger.log('info', 'Search database reached.', {
|
|
228
|
+
code: 'SEARCH_DB_RECOVERED',
|
|
229
|
+
connection: this.#options.connection,
|
|
230
|
+
});
|
|
231
|
+
return this.#live;
|
|
232
|
+
}
|
|
233
|
+
catch (error) {
|
|
234
|
+
this.#lastError = error;
|
|
235
|
+
if (permanent(error))
|
|
236
|
+
throw error;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
throw new SearchError('NOT_READY', 'The search database is not reachable', {
|
|
240
|
+
cause: this.#lastError,
|
|
241
|
+
data: { connection: this.#options.connection, retryAfterMs: RETRY_MS },
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
/** A statement failure as a coded error that keeps the driver's cause. */
|
|
245
|
+
async #run(operation, work) {
|
|
246
|
+
const live = await this.#ready();
|
|
247
|
+
try {
|
|
248
|
+
return await work(live);
|
|
249
|
+
}
|
|
250
|
+
catch (error) {
|
|
251
|
+
if (error instanceof SearchError)
|
|
252
|
+
throw error;
|
|
253
|
+
if (this.#context?.logger.isEnabled('debug'))
|
|
254
|
+
this.#context.logger.log('debug', 'Search statement failed.', {
|
|
255
|
+
code: 'SEARCH_STORE_FAILED',
|
|
256
|
+
operation,
|
|
257
|
+
error: diagnostics.describeError(error),
|
|
258
|
+
});
|
|
259
|
+
throw new SearchError('STORE_FAILED', `The search could not ${operation}`, {
|
|
260
|
+
cause: error,
|
|
261
|
+
data: { operation },
|
|
262
|
+
});
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
#fields(index) {
|
|
266
|
+
const fields = typeof index === 'string' ? this.#indexes.fields(index) : undefined;
|
|
267
|
+
if (!fields)
|
|
268
|
+
throw new SearchError('UNKNOWN_INDEX', `Search index ${String(index)} is not declared`, {
|
|
269
|
+
data: { index: String(index).slice(0, 40) },
|
|
270
|
+
});
|
|
271
|
+
return fields;
|
|
272
|
+
}
|
|
273
|
+
#id(value, error = 'INVALID_ITEM') {
|
|
274
|
+
if (typeof value !== 'string' || !value.length || value.length > MAX_ID)
|
|
275
|
+
throw new SearchError(error, `id is a string of 1 to ${MAX_ID} characters`, {
|
|
276
|
+
data: { field: 'id' },
|
|
277
|
+
});
|
|
278
|
+
return value;
|
|
279
|
+
}
|
|
280
|
+
#scope(value, at, error) {
|
|
281
|
+
if (typeof value !== 'string' || value.length > MAX_SCOPE)
|
|
282
|
+
throw new SearchError(error, `${at} is a string of at most ${MAX_SCOPE} characters`, {
|
|
283
|
+
data: { field: at },
|
|
284
|
+
});
|
|
285
|
+
return value;
|
|
286
|
+
}
|
|
287
|
+
/** The rows and words of an item: one row per term, field and scope. */
|
|
288
|
+
#rowsOf(index, item) {
|
|
289
|
+
const fields = this.#fields(index);
|
|
290
|
+
if (!plain(item))
|
|
291
|
+
throw new SearchError('INVALID_ITEM', 'The item is { id, sortKey, scopes?, fields }');
|
|
292
|
+
this.#id(item.id);
|
|
293
|
+
if (!Number.isSafeInteger(item.sortKey))
|
|
294
|
+
throw new SearchError('INVALID_ITEM', 'sortKey is a safe integer', {
|
|
295
|
+
data: { field: 'sortKey' },
|
|
296
|
+
});
|
|
297
|
+
const scopes = item.scopes ?? [''];
|
|
298
|
+
if (!Array.isArray(scopes) || !scopes.length || scopes.length > MAX_SCOPES)
|
|
299
|
+
throw new SearchError('INVALID_ITEM', `scopes is a list of 1 to ${MAX_SCOPES} strings`, {
|
|
300
|
+
data: { field: 'scopes' },
|
|
301
|
+
});
|
|
302
|
+
for (const scope of scopes)
|
|
303
|
+
this.#scope(scope, 'scopes', 'INVALID_ITEM');
|
|
304
|
+
if (new Set(scopes).size !== scopes.length)
|
|
305
|
+
throw new SearchError('INVALID_ITEM', 'scopes are different strings', {
|
|
306
|
+
data: { field: 'scopes' },
|
|
307
|
+
});
|
|
308
|
+
if (!plain(item.fields))
|
|
309
|
+
throw new SearchError('INVALID_ITEM', 'fields is an object of texts', {
|
|
310
|
+
data: { field: 'fields' },
|
|
311
|
+
});
|
|
312
|
+
const rows = [];
|
|
313
|
+
const pairs = new Set();
|
|
314
|
+
for (const [field, text] of Object.entries(item.fields)) {
|
|
315
|
+
if (!fields.includes(field))
|
|
316
|
+
throw new SearchError('UNKNOWN_FIELD', `Field ${field} is not declared in ${index}`, {
|
|
317
|
+
data: { index, field: field.slice(0, 40) },
|
|
318
|
+
});
|
|
319
|
+
if (text === null || text === undefined)
|
|
320
|
+
continue;
|
|
321
|
+
if (typeof text !== 'string' || text.length > MAX_TEXT)
|
|
322
|
+
throw new SearchError('INVALID_ITEM', `fields.${field} is a string of at most ${MAX_TEXT} characters`, { data: { field } });
|
|
323
|
+
const rules = this.#indexes.rules(index, field);
|
|
324
|
+
if (!rules)
|
|
325
|
+
continue;
|
|
326
|
+
for (const term of terms(text, rules)) {
|
|
327
|
+
pairs.add(`${field}${SEP}${term}`);
|
|
328
|
+
for (const scope of scopes)
|
|
329
|
+
rows.push({ field, term, scope });
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
return { rows, pairs };
|
|
333
|
+
}
|
|
334
|
+
/** Writes one item: only the rows and counts that differ from what it had. */
|
|
335
|
+
async #write(live, tx, index, item) {
|
|
336
|
+
const { rows, pairs } = this.#rowsOf(index, item);
|
|
337
|
+
const old = await live.store.itemRows(tx, index, item.id);
|
|
338
|
+
const oldPairs = new Set(old.map((row) => `${row.field}${SEP}${row.term}`));
|
|
339
|
+
const key = (row) => `${row.field}${SEP}${row.term}${SEP}${row.scope}`;
|
|
340
|
+
if (old.some((row) => row.sortKey !== item.sortKey)) {
|
|
341
|
+
// The display order is part of every row's key: a new sortKey rewrites them all.
|
|
342
|
+
await live.store.deleteItem(tx, index, item.id);
|
|
343
|
+
await live.store.insertRows(tx, index, item.id, item.sortKey, rows);
|
|
344
|
+
}
|
|
345
|
+
else {
|
|
346
|
+
const now = new Set(rows.map(key));
|
|
347
|
+
const before = new Set(old.map(key));
|
|
348
|
+
const gone = old.filter((row) => !now.has(key(row)));
|
|
349
|
+
const added = rows.filter((row) => !before.has(key(row)));
|
|
350
|
+
if (gone.length)
|
|
351
|
+
await live.store.deleteRows(tx, index, item.id, gone);
|
|
352
|
+
if (added.length)
|
|
353
|
+
await live.store.insertRows(tx, index, item.id, item.sortKey, added);
|
|
354
|
+
}
|
|
355
|
+
this.#count(index, oldPairs, pairs);
|
|
356
|
+
}
|
|
357
|
+
/** Buffers the count changes of the words that came and went (added by #flush). */
|
|
358
|
+
#count(index, before, after) {
|
|
359
|
+
const add = (pair, delta) => {
|
|
360
|
+
const key = `${index}${SEP}${pair}`;
|
|
361
|
+
const value = (this.#pending.get(key) ?? 0) + delta;
|
|
362
|
+
if (value === 0)
|
|
363
|
+
this.#pending.delete(key);
|
|
364
|
+
else
|
|
365
|
+
this.#pending.set(key, value);
|
|
366
|
+
};
|
|
367
|
+
for (const pair of after)
|
|
368
|
+
if (!before.has(pair))
|
|
369
|
+
add(pair, 1);
|
|
370
|
+
for (const pair of before)
|
|
371
|
+
if (!after.has(pair))
|
|
372
|
+
add(pair, -1);
|
|
373
|
+
if (this.#pending.size >= FLUSH_WORDS)
|
|
374
|
+
void this.#flush();
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Adds the buffered count changes to the table, one autocommit statement per index on the
|
|
378
|
+
* plugin's pool, words in sorted order. A failure is logged and the changes are kept for the
|
|
379
|
+
* next flush. Never rejects.
|
|
380
|
+
*/
|
|
381
|
+
#flush() {
|
|
382
|
+
if (this.#flushing)
|
|
383
|
+
return this.#flushing;
|
|
384
|
+
if (!this.#pending.size || !this.#live)
|
|
385
|
+
return Promise.resolve();
|
|
386
|
+
const live = this.#live;
|
|
387
|
+
const taken = this.#pending;
|
|
388
|
+
this.#pending = new Map();
|
|
389
|
+
const work = async () => {
|
|
390
|
+
const byIndex = new Map();
|
|
391
|
+
for (const [key, delta] of taken) {
|
|
392
|
+
const first = key.indexOf(SEP);
|
|
393
|
+
const second = key.indexOf(SEP, first + 1);
|
|
394
|
+
const index = key.slice(0, first);
|
|
395
|
+
const list = byIndex.get(index) ?? [];
|
|
396
|
+
list.push({ field: key.slice(first + 1, second), term: key.slice(second + 1), delta });
|
|
397
|
+
byIndex.set(index, list);
|
|
398
|
+
}
|
|
399
|
+
for (const [index, deltas] of byIndex) {
|
|
400
|
+
deltas.sort((a, b) => byKey(a.field, b.field) || byKey(a.term, b.term));
|
|
401
|
+
try {
|
|
402
|
+
await live.store.addCounts(live.pool, index, deltas);
|
|
403
|
+
}
|
|
404
|
+
catch (error) {
|
|
405
|
+
// Kept for the next flush: the counts are late, never lost while the process runs.
|
|
406
|
+
for (const row of deltas) {
|
|
407
|
+
const key = `${index}${SEP}${row.field}${SEP}${row.term}`;
|
|
408
|
+
this.#pending.set(key, (this.#pending.get(key) ?? 0) + row.delta);
|
|
409
|
+
}
|
|
410
|
+
this.#repeats?.log('error', 'Search word counts could not be written; they are kept for the next flush.', 'SEARCH_COUNTS_FAILED', error);
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
};
|
|
414
|
+
this.#flushing = work().finally(() => {
|
|
415
|
+
this.#flushing = undefined;
|
|
416
|
+
});
|
|
417
|
+
return this.#flushing;
|
|
418
|
+
}
|
|
419
|
+
#query(index, query) {
|
|
420
|
+
const fields = this.#fields(index);
|
|
421
|
+
if (!plain(query))
|
|
422
|
+
throw new SearchError('INVALID_QUERY', 'The query is { field, q, scope?, limit?, cursor? }');
|
|
423
|
+
if (typeof query.field !== 'string' || !fields.includes(query.field))
|
|
424
|
+
throw new SearchError('UNKNOWN_FIELD', `Field ${String(query.field)} is not declared in ${index}`, { data: { index, field: String(query.field).slice(0, 40) } });
|
|
425
|
+
if (typeof query.q !== 'string' || query.q.length > 1000)
|
|
426
|
+
throw new SearchError('INVALID_QUERY', 'q is a string of at most 1000 characters', {
|
|
427
|
+
data: { field: 'q' },
|
|
428
|
+
});
|
|
429
|
+
const scope = this.#scope(query.scope ?? '', 'scope', 'INVALID_QUERY');
|
|
430
|
+
const limit = query.limit ?? DEFAULT_LIMIT;
|
|
431
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > this.#options.list.maxLimit)
|
|
432
|
+
throw new SearchError('INVALID_QUERY', `limit is an integer from 1 to ${this.#options.list.maxLimit}`, { data: { field: 'limit' } });
|
|
433
|
+
const rules = this.#indexes.rules(index, query.field);
|
|
434
|
+
const { terms: words, short } = queryTerms(query.q, rules);
|
|
435
|
+
if (!words.length)
|
|
436
|
+
throw new SearchError('INVALID_QUERY', 'The query has no word to search for', {
|
|
437
|
+
data: { field: 'q' },
|
|
438
|
+
});
|
|
439
|
+
if (words.length > MAX_QUERY_WORDS)
|
|
440
|
+
throw new SearchError('INVALID_QUERY', `The query has more than ${MAX_QUERY_WORDS} words`, {
|
|
441
|
+
data: { field: 'q', words: words.length },
|
|
442
|
+
});
|
|
443
|
+
if (short.length)
|
|
444
|
+
throw new SearchError('INVALID_QUERY', `Every word is at least ${rules.minLength} characters in ${index}.${query.field}`, { data: { field: 'q', minLength: rules.minLength } });
|
|
445
|
+
const fingerprint = queryFingerprint(index, query.field, scope, words);
|
|
446
|
+
const state = query.cursor === undefined
|
|
447
|
+
? undefined
|
|
448
|
+
: this.#cursors.open(query.cursor, fingerprint);
|
|
449
|
+
return { field: query.field, scope, limit, words, fingerprint, state };
|
|
450
|
+
}
|
|
451
|
+
async #find(live, index, query) {
|
|
452
|
+
const { field, scope, limit, words, fingerprint, state } = this.#query(index, query);
|
|
453
|
+
const cursors = this.#cursors;
|
|
454
|
+
const after = state && { sortKey: state.sortKey, id: state.id };
|
|
455
|
+
if (words.length === 1) {
|
|
456
|
+
// One word: one equality lookup in display order, whatever its frequency.
|
|
457
|
+
const rows = await live.store.page(live.pool, {
|
|
458
|
+
index,
|
|
459
|
+
field,
|
|
460
|
+
scope,
|
|
461
|
+
term: words[0],
|
|
462
|
+
after,
|
|
463
|
+
limit: limit + 1,
|
|
464
|
+
});
|
|
465
|
+
const items = rows.slice(0, limit);
|
|
466
|
+
const last = items.at(-1);
|
|
467
|
+
return Object.freeze({
|
|
468
|
+
items: Object.freeze(items.map((item) => Object.freeze(item))),
|
|
469
|
+
...(rows.length > limit && last
|
|
470
|
+
? {
|
|
471
|
+
next: cursors.sign({
|
|
472
|
+
query: fingerprint,
|
|
473
|
+
sortKey: last.sortKey,
|
|
474
|
+
id: last.id,
|
|
475
|
+
examined: 0,
|
|
476
|
+
}),
|
|
477
|
+
}
|
|
478
|
+
: {}),
|
|
479
|
+
partial: false,
|
|
480
|
+
examined: rows.length,
|
|
481
|
+
});
|
|
482
|
+
}
|
|
483
|
+
// Several words: the rarest leads (the same one on every page of the search).
|
|
484
|
+
let lead = state?.lead;
|
|
485
|
+
if (lead !== undefined && !words.includes(lead))
|
|
486
|
+
throw new SearchError('INVALID_CURSOR', 'The search cursor is not valid: lead', {
|
|
487
|
+
data: { reason: 'lead' },
|
|
488
|
+
});
|
|
489
|
+
if (lead === undefined) {
|
|
490
|
+
const counts = await live.store.counts(live.pool, index, field, words);
|
|
491
|
+
let best;
|
|
492
|
+
let fewest = Number.POSITIVE_INFINITY;
|
|
493
|
+
for (const word of words) {
|
|
494
|
+
// A hint only: a word counted 0 leads, and its scan ends at once when it has no rows.
|
|
495
|
+
const count = counts.get(word) ?? 0;
|
|
496
|
+
if (count < fewest ||
|
|
497
|
+
(count === fewest && best !== undefined && word.length > best.length)) {
|
|
498
|
+
best = word;
|
|
499
|
+
fewest = count;
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
lead = best;
|
|
503
|
+
}
|
|
504
|
+
const others = words.filter((word) => word !== lead);
|
|
505
|
+
const total = state?.examined ?? 0;
|
|
506
|
+
const budget = Math.min(this.#options.examineLimit, this.#options.searchLimit - total);
|
|
507
|
+
const items = [];
|
|
508
|
+
let examined = 0;
|
|
509
|
+
let position = after;
|
|
510
|
+
// Whether rows of the leading word may follow `position`.
|
|
511
|
+
let more = true;
|
|
512
|
+
while (items.length < limit && examined < budget) {
|
|
513
|
+
const chunk = Math.min(CHUNK, budget - examined);
|
|
514
|
+
const rows = await live.store.scan(live.pool, {
|
|
515
|
+
index,
|
|
516
|
+
field,
|
|
517
|
+
scope,
|
|
518
|
+
lead,
|
|
519
|
+
others,
|
|
520
|
+
after: position,
|
|
521
|
+
limit: chunk,
|
|
522
|
+
});
|
|
523
|
+
let read = 0;
|
|
524
|
+
for (const row of rows) {
|
|
525
|
+
read += 1;
|
|
526
|
+
position = { sortKey: row.sortKey, id: row.id };
|
|
527
|
+
if (row.hit) {
|
|
528
|
+
items.push(Object.freeze({ id: row.id, sortKey: row.sortKey }));
|
|
529
|
+
if (items.length === limit)
|
|
530
|
+
break;
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
examined += read;
|
|
534
|
+
// A short chunk is the end of the word's rows; the page may have filled before its end.
|
|
535
|
+
if (rows.length < chunk) {
|
|
536
|
+
if (read === rows.length)
|
|
537
|
+
more = false;
|
|
538
|
+
break;
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
const answer = { items: Object.freeze(items), examined };
|
|
542
|
+
if (!more || !position)
|
|
543
|
+
return Object.freeze({ ...answer, partial: false });
|
|
544
|
+
if (items.length < limit && total + examined >= this.#options.searchLimit) {
|
|
545
|
+
this.#context?.logger.log('debug', 'Search ended as too broad.', {
|
|
546
|
+
code: 'SEARCH_TOO_BROAD',
|
|
547
|
+
index,
|
|
548
|
+
field,
|
|
549
|
+
words: words.length,
|
|
550
|
+
examined: total + examined,
|
|
551
|
+
});
|
|
552
|
+
return Object.freeze({ ...answer, partial: false, stopped: 'SEARCH_TOO_BROAD' });
|
|
553
|
+
}
|
|
554
|
+
return Object.freeze({
|
|
555
|
+
...answer,
|
|
556
|
+
next: cursors.sign({
|
|
557
|
+
query: fingerprint,
|
|
558
|
+
lead,
|
|
559
|
+
sortKey: position.sortKey,
|
|
560
|
+
id: position.id,
|
|
561
|
+
examined: total + examined,
|
|
562
|
+
}),
|
|
563
|
+
partial: items.length < limit,
|
|
564
|
+
});
|
|
565
|
+
}
|
|
566
|
+
/** Drain: the buffered counts go to the table while the database is still open. */
|
|
567
|
+
async drain() {
|
|
568
|
+
await this.#flush();
|
|
569
|
+
}
|
|
570
|
+
#api() {
|
|
571
|
+
const indexes = this.#indexes;
|
|
572
|
+
return Object.freeze({
|
|
573
|
+
indexes,
|
|
574
|
+
terms: (index, field, text) => {
|
|
575
|
+
this.#fields(index);
|
|
576
|
+
const rules = indexes.rules(index, field);
|
|
577
|
+
if (!rules)
|
|
578
|
+
throw new SearchError('UNKNOWN_FIELD', `Field ${String(field)} is not declared in ${index}`);
|
|
579
|
+
if (typeof text !== 'string')
|
|
580
|
+
throw new SearchError('INVALID_ITEM', 'text is a string', { data: { field: 'text' } });
|
|
581
|
+
return Object.freeze([...terms(text, rules)].sort(byKey));
|
|
582
|
+
},
|
|
583
|
+
// The item is checked (and its terms made) once, inside #write: a SearchError passes #run.
|
|
584
|
+
index: async (tx, index, item) => {
|
|
585
|
+
await this.#run('index an item', async (live) => this.#write(live, session(live.driver, tx), index, item));
|
|
586
|
+
},
|
|
587
|
+
indexMany: async (tx, index, items) => {
|
|
588
|
+
if (!Array.isArray(items))
|
|
589
|
+
throw new SearchError('INVALID_ITEM', 'indexMany takes a list of items');
|
|
590
|
+
await this.#run('index items', async (live) => {
|
|
591
|
+
const client = session(live.driver, tx);
|
|
592
|
+
for (const item of items)
|
|
593
|
+
await this.#write(live, client, index, item);
|
|
594
|
+
});
|
|
595
|
+
},
|
|
596
|
+
remove: async (tx, index, id) => {
|
|
597
|
+
this.#fields(index);
|
|
598
|
+
const item = this.#id(id);
|
|
599
|
+
return this.#run('remove an item', async (live) => {
|
|
600
|
+
const client = session(live.driver, tx);
|
|
601
|
+
const old = await live.store.itemRows(client, index, item);
|
|
602
|
+
if (!old.length)
|
|
603
|
+
return Object.freeze({ removed: false });
|
|
604
|
+
await live.store.deleteItem(client, index, item);
|
|
605
|
+
this.#count(index, new Set(old.map((row) => `${row.field}${SEP}${row.term}`)), new Set());
|
|
606
|
+
return Object.freeze({ removed: true });
|
|
607
|
+
});
|
|
608
|
+
},
|
|
609
|
+
find: async (index, query) => {
|
|
610
|
+
return this.#run('search', (live) => this.#find(live, index, query));
|
|
611
|
+
},
|
|
612
|
+
flush: () => this.#flush(),
|
|
613
|
+
markCurrent: async (index) => {
|
|
614
|
+
this.#fields(index);
|
|
615
|
+
await this.#run('record an index as current', async (live) => {
|
|
616
|
+
await live.store.saveFingerprint(live.pool, index, indexes.fingerprint(index), Date.now());
|
|
617
|
+
this.#stale.delete(index);
|
|
618
|
+
});
|
|
619
|
+
},
|
|
620
|
+
status: () => Object.freeze(indexes.names.map((name) => Object.freeze({
|
|
621
|
+
name,
|
|
622
|
+
fingerprint: indexes.fingerprint(name),
|
|
623
|
+
stale: this.#stale.has(name),
|
|
624
|
+
}))),
|
|
625
|
+
});
|
|
626
|
+
}
|
|
627
|
+
/** `dokebi describe` opens no connection: the tables and indexes, every query refused. */
|
|
628
|
+
#describing(context) {
|
|
629
|
+
const settings = this.#options;
|
|
630
|
+
const indexes = this.#indexes;
|
|
631
|
+
context.contribute(describePoint, {
|
|
632
|
+
section: 'search',
|
|
633
|
+
describe: () => ({
|
|
634
|
+
connection: settings.connection,
|
|
635
|
+
tables: [TERMS_TABLE, COUNTS_TABLE, INDEXES_TABLE],
|
|
636
|
+
indexes: Object.fromEntries(indexes.names.map((name) => [name, indexes.fields(name)])),
|
|
637
|
+
examineLimit: settings.examineLimit,
|
|
638
|
+
searchLimit: settings.searchLimit,
|
|
639
|
+
}),
|
|
640
|
+
});
|
|
641
|
+
const refuse = () => Promise.reject(new SearchError('DESCRIBING', 'Search is being described: no query runs'));
|
|
642
|
+
return Object.freeze({
|
|
643
|
+
indexes,
|
|
644
|
+
terms: (index, field, text) => {
|
|
645
|
+
const rules = indexes.rules(index, field);
|
|
646
|
+
if (!rules)
|
|
647
|
+
throw new SearchError('UNKNOWN_FIELD', `Field ${field} is not declared in ${index}`);
|
|
648
|
+
return Object.freeze([...terms(String(text), rules)].sort(byKey));
|
|
649
|
+
},
|
|
650
|
+
index: refuse,
|
|
651
|
+
indexMany: refuse,
|
|
652
|
+
remove: refuse,
|
|
653
|
+
find: refuse,
|
|
654
|
+
flush: () => Promise.resolve(),
|
|
655
|
+
markCurrent: refuse,
|
|
656
|
+
status: () => Object.freeze([]),
|
|
657
|
+
});
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
/** Factory input: `{ app, options?, entry }`: `entry` is the module of src/infra/search/indexes.ts. */
|
|
661
|
+
export const createPlugin = definePlugin({
|
|
662
|
+
name: 'dokebi-search',
|
|
663
|
+
version: '0.1.0',
|
|
664
|
+
core: '^0.4.2',
|
|
665
|
+
options,
|
|
666
|
+
dependencies: () => [db],
|
|
667
|
+
contributes: [describePoint],
|
|
668
|
+
// The module of src/infra/search/indexes.ts (or the indexes themselves): INVALID_INDEXES otherwise.
|
|
669
|
+
entry: (input) => indexesOf(input),
|
|
670
|
+
instance: ({ options: parsed, entry }) => new SearchRuntime(parsed, entry),
|
|
671
|
+
});
|