@delali/narsil 0.3.1-next.130 → 0.3.1-next.131

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@delali/narsil",
3
- "version": "0.3.1-next.130",
3
+ "version": "0.3.1-next.131",
4
4
  "type": "module",
5
5
  "description": "Distributed search engine with full-text, vector, hybrid, and geosearch. One engine, embedded or on a server, with a cross-language .nrsl format.",
6
6
  "author": "Delali (https://sondelali.com)",
@@ -109,6 +109,10 @@
109
109
  "revisions:record": "tsx scripts/analysis-revisions.ts --record",
110
110
  "api": "tsx scripts/api-surface.ts",
111
111
  "api:write": "tsx scripts/api-surface.ts --write",
112
+ "openapi": "tsx scripts/openapi/generate.ts",
113
+ "openapi:write": "tsx scripts/openapi/generate.ts --write",
114
+ "openapi:lint": "spectral lint openapi.json --ruleset .spectral.yaml --fail-severity error",
115
+ "test:openapi-clients": "NARSIL_TEST_OPENAPI_CLIENTS=1 vitest run src/__tests__/server/openapi/generated-clients.test.ts",
112
116
  "bundle-size:write": "rm -rf dist && tsup && node scripts/check-browser-bundle.mjs --update-baseline",
113
117
  "lookalikes": "tsx scripts/lookalike-counts.ts",
114
118
  "lookalikes:collect": "tsx scripts/lookalike-counts.ts --collect",
@@ -216,10 +220,12 @@
216
220
  "./react": {
217
221
  "import": "./dist/react.mjs",
218
222
  "types": "./dist/react.d.ts"
219
- }
223
+ },
224
+ "./openapi.json": "./openapi.json"
220
225
  },
221
226
  "files": [
222
227
  "dist/",
228
+ "openapi.json",
223
229
  "LICENSE",
224
230
  "NOTICE"
225
231
  ],
@@ -231,10 +237,12 @@
231
237
  "@delali/narsil-embeddings-transformers": "workspace:*",
232
238
  "@grpc/grpc-js": "1.14.4",
233
239
  "@microsoft/api-extractor": "7.58.12",
240
+ "@stoplight/spectral-cli": "6.15.0",
234
241
  "@types/node": "22.15.32",
235
242
  "@types/react": "19.2.14",
236
243
  "@types/react-dom": "19.2.3",
237
244
  "@vitest/coverage-v8": "4.0.18",
245
+ "ajv": "8.17.1",
238
246
  "esbuild": "0.28.2",
239
247
  "etcd3": "1.1.2",
240
248
  "fflate": "0.8.3",
@@ -271,11 +279,11 @@
271
279
  }
272
280
  },
273
281
  "optionalDependencies": {
274
- "@delali/narsil-native-darwin-arm64": "0.3.1-next.130",
275
- "@delali/narsil-native-darwin-x64": "0.3.1-next.130",
276
- "@delali/narsil-native-linux-arm64": "0.3.1-next.130",
277
- "@delali/narsil-native-linux-x64": "0.3.1-next.130",
278
- "@delali/narsil-native-win32-arm64": "0.3.1-next.130",
279
- "@delali/narsil-native-win32-x64": "0.3.1-next.130"
282
+ "@delali/narsil-native-darwin-arm64": "0.3.1-next.131",
283
+ "@delali/narsil-native-darwin-x64": "0.3.1-next.131",
284
+ "@delali/narsil-native-linux-arm64": "0.3.1-next.131",
285
+ "@delali/narsil-native-linux-x64": "0.3.1-next.131",
286
+ "@delali/narsil-native-win32-arm64": "0.3.1-next.131",
287
+ "@delali/narsil-native-win32-x64": "0.3.1-next.131"
280
288
  }
281
289
  }
@@ -1,414 +0,0 @@
1
- import { P as PersistenceAdapter, I as InvalidationAdapter, E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
2
- import { Q as QueryParams, e as QueryResult, f as NarsilEventHandlers } from './events-DErop1LM.js';
3
- import { I as IndexConfig, A as AnyDocument } from './schema-CS-clLun.js';
4
-
5
- /**
6
- * Hooks that the engine calls alongside its own work, so that an application
7
- * can add validation, auditing, enrichment, or metrics without forking the
8
- * engine.
9
- *
10
- * Every hook is optional. A `before` hook fires first and rejects the
11
- * operation by throwing, while an `after` hook fires once the operation
12
- * succeeds. Where a `before` hook on a write throws a {@link NarsilError}, the
13
- * engine passes that error to the caller unchanged. It turns any other error
14
- * into `DOC_VALIDATION_FAILED` with the same message, for a single write and
15
- * for each entry of a batch alike. The engine awaits a hook that returns a
16
- * promise, so slow work in one hook slows every write or search that it
17
- * covers.
18
- *
19
- * @public
20
- */
21
- interface NarsilPlugin {
22
- /** This name identifies the plugin in errors and in event payloads. */
23
- name: string;
24
- /** Fires before the engine indexes a document. Throw to reject the write. */
25
- beforeInsert?(ctx: InsertContext): void | Promise<void>;
26
- /** Fires once the engine has indexed a document. */
27
- afterInsert?(ctx: InsertContext): void | Promise<void>;
28
- /** Fires before the engine removes a document. Throw to reject the removal. */
29
- beforeRemove?(ctx: RemoveContext): void | Promise<void>;
30
- /** Fires once the engine has removed a document. */
31
- afterRemove?(ctx: RemoveContext): void | Promise<void>;
32
- /** Fires before the engine replaces a document. Throw to reject the update. */
33
- beforeUpdate?(ctx: UpdateContext): void | Promise<void>;
34
- /** Fires once the engine has replaced a document. */
35
- afterUpdate?(ctx: UpdateContext): void | Promise<void>;
36
- /**
37
- * Fires before every search, and before every preflight. Throw to reject the
38
- * query. It fires before the engine embeds a `vector.text` query, so a hook
39
- * that counts or refuses work stops the call to the embedding provider, and
40
- * `ctx.params.vector` still holds the text that the caller passed.
41
- * {@link Narsil.suggest} scans the term dictionary and searches no document,
42
- * so it fires no hook.
43
- */
44
- beforeSearch?(ctx: SearchContext): void | Promise<void>;
45
- /**
46
- * Fires once a search returns. The context holds a copy of the results, so a
47
- * change that you make here leaves the caller's own result untouched. Where
48
- * this hook throws, the engine logs a warning and answers the query all the
49
- * same, because the search is already over.
50
- */
51
- afterSearch?(ctx: SearchContext): void | Promise<void>;
52
- /**
53
- * Fires once the engine creates an index, and a `restore` fires it too,
54
- * because a restore recreates the index that it replaces. Where this hook
55
- * throws, the engine logs a warning and keeps the index, because the hook
56
- * fires after the index exists.
57
- */
58
- onIndexCreate?(ctx: IndexContext): void | Promise<void>;
59
- /**
60
- * Fires once the engine drops an index, and a `restore` fires it too,
61
- * because a restore drops the index that it replaces. Where this hook
62
- * throws, the engine logs a warning, because the hook fires after the index
63
- * is gone.
64
- */
65
- onIndexDrop?(ctx: IndexContext): void | Promise<void>;
66
- /**
67
- * Fires once {@link Narsil.rebalance} moves an index onto a new partition
68
- * count, which is the only way that the count changes. An index opens with
69
- * the count that `partitions.maxPartitions` sets, because the engine adds no
70
- * partition on its own. Subscribe to the `partitionWatermark` event to learn
71
- * when an index fills its capacity.
72
- */
73
- onPartitionSplit?(ctx: PartitionContext): void | Promise<void>;
74
- /** Fires once the engine loads worker copies of its indexes. */
75
- onWorkerPromote?(ctx: WorkerContext): void | Promise<void>;
76
- }
77
- /**
78
- * What a plugin receives around an insert.
79
- *
80
- * @public
81
- */
82
- interface InsertContext {
83
- /** The engine writes the document to this index. */
84
- indexName: string;
85
- /** The document is stored under this id, which the engine resolves before the `before` hook runs. */
86
- docId: string;
87
- /** This is the document itself, and the engine indexes whatever a `before` hook leaves here. */
88
- document: AnyDocument;
89
- }
90
- /**
91
- * What a plugin receives around a removal.
92
- *
93
- * @public
94
- */
95
- interface RemoveContext {
96
- /** The engine removes the document from this index. */
97
- indexName: string;
98
- /** The engine removes this document. */
99
- docId: string;
100
- }
101
- /**
102
- * What a plugin receives around an update.
103
- *
104
- * @public
105
- */
106
- interface UpdateContext {
107
- /** The document belongs to this index. */
108
- indexName: string;
109
- /** The engine replaces this document. */
110
- docId: string;
111
- /** The index holds this document before the update. */
112
- oldDocument: AnyDocument;
113
- /** This document replaces it, and the engine indexes whatever a `before` hook leaves here. */
114
- newDocument: AnyDocument;
115
- }
116
- /**
117
- * What a plugin receives around a search.
118
- *
119
- * @public
120
- */
121
- interface SearchContext {
122
- /** The search runs against this index. */
123
- indexName: string;
124
- /** These are the parameters the query runs with, and a `before` hook that changes them rewrites the search. */
125
- params: QueryParams;
126
- /**
127
- * These are the results, which an `after` hook alone receives. A preflight
128
- * fires `beforeSearch` and stops there, because it answers with a count and
129
- * no hits. Read this field inside an `afterSearch` hook alone.
130
- */
131
- results?: QueryResult;
132
- }
133
- /**
134
- * What a plugin receives when the engine creates or drops an index.
135
- *
136
- * @public
137
- */
138
- interface IndexContext {
139
- /** The engine creates or drops this index. */
140
- indexName: string;
141
- /** The index holds this configuration. */
142
- config: IndexConfig;
143
- }
144
- /**
145
- * What a plugin receives when a rebalance changes an index's partition count.
146
- *
147
- * @public
148
- */
149
- interface PartitionContext {
150
- /** The rebalance moved this index. */
151
- indexName: string;
152
- /** The index held this many partitions before the rebalance. */
153
- oldPartitionCount: number;
154
- /** The index holds this many partitions now. */
155
- newPartitionCount: number;
156
- }
157
- /**
158
- * What a plugin receives when the engine moves onto worker threads.
159
- *
160
- * @public
161
- */
162
- interface WorkerContext {
163
- /** The pool holds this many workers. */
164
- workerCount: number;
165
- /** The engine promoted for this reason, such as the threshold that the index crossed. */
166
- reason: string;
167
- }
168
-
169
- /**
170
- * Everything {@link createNarsil} accepts when it builds an engine.
171
- *
172
- * Every field is optional, and an engine created without any of them keeps
173
- * its indexes in memory alone. Add persistence to survive a restart, add
174
- * invalidation to share a store with another process, and add durability to
175
- * recover the writes made since the last snapshot.
176
- *
177
- * @public
178
- */
179
- interface NarsilConfig {
180
- /** This adapter writes partitions somewhere they outlive the process. */
181
- persistence?: PersistenceAdapter;
182
- /** This channel carries partition changes to other instances sharing the store. */
183
- invalidation?: InvalidationAdapter;
184
- /** These hooks observe or reject writes, searches, and lifecycle changes. */
185
- plugins?: NarsilPlugin[];
186
- /** This supplies the id for a document inserted without one. The engine generates a UUID v7 by default. */
187
- idGenerator?: () => string;
188
- /** These settings control when an index gains worker copies, how many threads they share, and when an idle index gives them up. */
189
- workers?: WorkerConfig;
190
- /** Every index uses this adapter unless its own config names another. */
191
- embedding?: EmbeddingAdapter;
192
- /** Named adapters; names persist in index metadata so recovery can rebind. */
193
- embeddingAdapters?: Record<string, EmbeddingAdapter>;
194
- /**
195
- * These rerank adapters are registered under their keys, so that a query
196
- * can give a key in `rerank.adapter`. The engine throws
197
- * `RERANK_CONFIG_INVALID` for an empty key and for an entry without a
198
- * `rerank` function.
199
- */
200
- rerankAdapters?: Record<string, RerankAdapter>;
201
- /**
202
- * The engine waits this many milliseconds for a rerank adapter to score one
203
- * query, and 30,000 by default. It then cancels the call through the
204
- * adapter's abort signal and throws `RERANK_FAILED`. Set a whole number of
205
- * at least 1; {@link createNarsil} throws `CONFIG_INVALID` for any other
206
- * value.
207
- */
208
- rerankTimeout?: number;
209
- /**
210
- * A query may rerank at most this many of its top hits, which caps the
211
- * documents that one query sends to a model. Set a whole number from 1 to
212
- * 1,000, or leave it out for 100; {@link createNarsil} throws
213
- * `CONFIG_INVALID` for any other value.
214
- */
215
- maxRerankWindow?: number;
216
- /** These settings recover the writes made since the last snapshot. */
217
- durability?: DurabilityConfig;
218
- /** These settings control what the engine does when an index's stored analysis differs from its language module. */
219
- analysis?: AnalysisConfig;
220
- /** These limits close indexes while keeping their durable files ready to reopen. */
221
- lifecycle?: IndexLifecycleConfig;
222
- /**
223
- * These listeners receive events from the moment that the engine starts.
224
- * The engine registers each one before it recovers any index, so a listener
225
- * here receives the `layoutUpgrade` events of that recovery, which come
226
- * before {@link createNarsil} returns the engine. For a name outside
227
- * {@link NarsilEventMap}, or an entry that is not a function,
228
- * {@link createNarsil} throws `CONFIG_INVALID`.
229
- */
230
- events?: NarsilEventHandlers;
231
- /**
232
- * This is the most work that the engine may do on the pattern tests and
233
- * phrases of one search, counted over every partition that it searches. For
234
- * a phrase, the engine adds one unit for each posting of its starting word
235
- * that it visits, and one for each position that it compares. For a pattern
236
- * test, it adds one unit for each run of three code points that it looks up in a pattern
237
- * index, one for each entry that it reads from a pattern index, and one for
238
- * each step of its matcher. For a text whose case fold is shorter than three
239
- * code points, it counts each document that it checks as an entry that it
240
- * reads. It counts an `in` or `nin` test as one `eq` test for each distinct
241
- * value in its list. For an `eq` test with a `tolerance`, it also adds one
242
- * unit for each cell that it fills in the table of edit distances between
243
- * the text and a value. It also counts the preparation of that test once for
244
- * a search, however many worker copies prepare it. Once the count passes the
245
- * cap, the engine throws `SEARCH_WORK_CAP_EXCEEDED`. The cap
246
- * defaults to 25,000,000 units. For any value other than a whole number from
247
- * 1 to `Number.MAX_SAFE_INTEGER`, {@link createNarsil} throws
248
- * `CONFIG_INVALID`.
249
- */
250
- patternWorkCap?: number;
251
- }
252
- /**
253
- * Limits on the indexes one engine keeps open in memory.
254
- *
255
- * @public
256
- */
257
- interface IndexLifecycleConfig {
258
- /** An index closes after this many milliseconds without activity. */
259
- idleTimeoutMs?: number;
260
- /** The engine keeps at most this many indexes open at once. */
261
- maxOpenIndexes?: number;
262
- /** Open indexes may use at most this many estimated bytes in total. */
263
- maxOpenBytes?: number;
264
- /** At most this many callers may wait behind one index reopen. */
265
- maxReopenWaiters?: number;
266
- }
267
- /**
268
- * Describes an index whose stored terms came from an older analysis than the
269
- * one its language module carries today.
270
- *
271
- * The engine hands this to {@link AnalysisConfig.onStaleAnalysis} together
272
- * with the rebuild that fixes it, so you decide when the work runs.
273
- *
274
- * @public
275
- */
276
- interface StaleAnalysis {
277
- /** This index holds terms that are out of date. */
278
- indexName: string;
279
- /** The index was built with this language module. */
280
- language: string;
281
- /** The index recorded this revision when it was written, and `null` for an index written before revisions existed. */
282
- storedRevision: string | null;
283
- /** The language module carries this revision now. */
284
- currentRevision: string;
285
- /** The rebuild reindexes this many documents. */
286
- documentCount: number;
287
- }
288
- /**
289
- * What the engine does when an index's stored terms no longer match the
290
- * analysis its language module produces.
291
- *
292
- * A stale index answers text queries from terms an older analysis wrote, and
293
- * every such result reports `analysisStale`. Rebuilding fixes it, and this
294
- * setting controls whether the engine starts that work itself.
295
- *
296
- * @public
297
- */
298
- interface AnalysisConfig {
299
- /** `auto` rebuilds a stale index in the background, and `manual` leaves it to you. The engine rebuilds automatically by default. */
300
- rebuild?: 'auto' | 'manual';
301
- /**
302
- * Called once for each stale index the engine finds.
303
- *
304
- * @param index - The index that is stale, and how far behind it is.
305
- * @param rebuild - Starts the rebuild and resolves once every partition
306
- * carries current terms. Under `manual` nothing rebuilds until you call it.
307
- */
308
- onStaleAnalysis?(index: StaleAnalysis, rebuild: () => Promise<void>): void | Promise<void>;
309
- }
310
- /**
311
- * Write-ahead logging and checkpointing, which recover the writes made since
312
- * the last snapshot.
313
- *
314
- * The `wal` tier logs every mutation before applying it, so a crash costs
315
- * nothing once the log replays. The `snapshot` tier writes whole partitions on
316
- * an interval, which is cheaper to run and loses whatever arrived after the
317
- * last write.
318
- *
319
- * @public
320
- */
321
- interface DurabilityConfig {
322
- /** `wal` logs every mutation, and `snapshot` writes whole partitions on an interval. */
323
- tier?: 'wal' | 'snapshot';
324
- /** The engine writes the log and its checkpoints into this directory. */
325
- directory?: string;
326
- /** `sync` flushes before a write resolves, and `async` flushes on the interval below. */
327
- mode?: 'sync' | 'async';
328
- /** In `async` mode the engine flushes this many milliseconds apart. */
329
- flushIntervalMs?: number;
330
- /** The log rolls over to a new segment once it reaches this size. */
331
- segmentMaxBytes?: number;
332
- /** The engine checkpoints this many milliseconds apart, which shortens the replay a restart has to do. */
333
- checkpointIntervalMs?: number;
334
- /** This many mutations trigger a checkpoint before the interval elapses. */
335
- checkpointMutationThreshold?: number;
336
- /**
337
- * The engine merges a partition's checkpoint segments once that partition
338
- * holds more than this many of them, 12 by default. The value is a whole
339
- * number of segments.
340
- */
341
- compactionThreshold?: number;
342
- }
343
- /**
344
- * Which queries the main copy of an index answers once the index holds worker
345
- * copies. Under `lone`, the main copy answers a query the engine receives
346
- * while no copy holds one, on an index of at most 50,000 documents. Once
347
- * every copy is busy, it also answers one query per turn of the event loop.
348
- * Under `none`, every query on an index that holds copies goes to one of
349
- * them, while the main copy still answers a query on an index whose copies
350
- * are loading.
351
- *
352
- * @public
353
- */
354
- type MainCopyQueries = 'lone' | 'none';
355
- /**
356
- * How the engine holds worker copies of its indexes, and how many threads
357
- * they may use.
358
- *
359
- * A small index answers fastest on the main thread. Once an index holds as
360
- * many documents as the copy threshold, the engine loads a copy of it onto
361
- * every worker thread, and each query then runs whole on the copy with the
362
- * fewest queries in flight. The same switch governs the vector search pool.
363
- *
364
- * @public
365
- */
366
- interface WorkerConfig {
367
- /**
368
- * Whether the engine may hold worker copies and a vector search pool. It is
369
- * on everywhere but a browser page. Set it to false to hold the process to
370
- * one thread, which leaves both pools absent. Where a bundler folds
371
- * `@delali/narsil` into an application bundle, the engine finds no worker
372
- * entry: left unset, it answers on one thread and prints one warning, and
373
- * set to true, `createNarsil` throws `CONFIG_INVALID`.
374
- */
375
- enabled?: boolean;
376
- /**
377
- * The keyword copies and the vector search pool share this many threads
378
- * between them, half each, in an embedded engine, so a budget of 4 gives 2
379
- * keyword copies and 2 vector search workers. The HTTP server holds a copy
380
- * on every one of them and receives requests there. The engine takes the
381
- * host's cores minus one by default, between 2 and 8. It caps a value that
382
- * you set at 31, which is how many scratch slots the shared vector arena
383
- * reserves.
384
- */
385
- count?: number;
386
- /** An index gains worker copies once it holds this many documents, 1,000 by default. */
387
- promotionThreshold?: number;
388
- /**
389
- * The engine drops an index's copies after this many milliseconds without a
390
- * read or a write, five minutes by default, and loads them again on the
391
- * next request while the main copy answers it. It may not exceed
392
- * `lifecycle.idleTimeoutMs`, and where it is unset the copies take the
393
- * smaller of five minutes and that interval.
394
- */
395
- idleTimeoutMs?: number;
396
- /**
397
- * Which queries the main copy answers, `lone` by default. Set `none` where
398
- * the main thread also serves the host's own work, such as HTTP requests,
399
- * because a query answered on the main copy then delays every other request
400
- * waiting on that thread. A server created with `createServer` sets `none` on
401
- * an engine that leaves this unset.
402
- */
403
- mainCopyQueries?: MainCopyQueries;
404
- /**
405
- * Each worker imports this module on start-up, which is how a worker reaches
406
- * a custom tokeniser or language. Give an absolute path or a URL, such as
407
- * `new URL('./bootstrap.js', import.meta.url).href`, because a worker
408
- * resolves a relative specifier against Narsil's own directory. A relative
409
- * value raises `CONFIG_INVALID`.
410
- */
411
- bootstrapModule?: string;
412
- }
413
-
414
- export type { AnalysisConfig as A, DurabilityConfig as D, IndexContext as I, MainCopyQueries as M, NarsilConfig as N, PartitionContext as P, RemoveContext as R, SearchContext as S, UpdateContext as U, WorkerConfig as W, IndexLifecycleConfig as a, InsertContext as b, NarsilPlugin as c, StaleAnalysis as d, WorkerContext as e };