@delali/narsil 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +174 -4
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +46 -2
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-XODT2F3h.d.ts → adapters-Bo2JQg8b.d.ts} +8 -1
- package/dist/chunk-ALB35PRQ.mjs +98 -0
- package/dist/{chunk-TFWBALU4.mjs → chunk-FS7XLIUS.mjs} +3219 -2063
- package/dist/chunk-GQEHCZG7.mjs +6825 -0
- package/dist/chunk-KMMJMDUA.mjs +44 -0
- package/dist/chunk-RQLXC2YS.mjs +269 -0
- package/dist/chunk-S7RJJMHI.mjs +1086 -0
- package/dist/{chunk-WYC635GI.mjs → chunk-TW6BXTOL.mjs} +10 -7
- package/dist/chunk-WISGW24C.mjs +455 -0
- package/dist/config-cQLlYR3U.d.ts +470 -0
- package/dist/direct-executor-FX56DYD3.mjs +4 -0
- package/dist/distribution/coordinator/etcd.d.ts +16 -0
- package/dist/distribution/coordinator/etcd.mjs +2 -0
- package/dist/distribution/coordinator/in-memory.d.ts +7 -0
- package/dist/distribution/coordinator/in-memory.mjs +1 -0
- package/dist/distribution.d.ts +122 -0
- package/dist/distribution.mjs +7410 -0
- package/dist/embeddings/openai.d.ts +13 -0
- package/dist/embeddings/openai.mjs +248 -0
- package/dist/index.browser.mjs +12111 -5793
- package/dist/index.d.ts +10 -420
- package/dist/index.mjs +7 -2052
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{language-ClUGLuoC.d.ts → language-ONQ3iVDf.d.ts} +1 -0
- package/dist/languages/arabic.d.ts +1 -1
- package/dist/languages/armenian.d.ts +1 -1
- package/dist/languages/bulgarian.d.ts +1 -1
- package/dist/languages/chinese.d.ts +1 -1
- package/dist/languages/dagbani.d.ts +1 -1
- package/dist/languages/danish.d.ts +1 -1
- package/dist/languages/dutch.d.ts +1 -1
- package/dist/languages/english.d.ts +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/ewe.d.ts +1 -1
- package/dist/languages/finnish.d.ts +1 -1
- package/dist/languages/french.d.ts +1 -1
- package/dist/languages/ga.d.ts +1 -1
- package/dist/languages/german.d.ts +1 -1
- package/dist/languages/greek.d.ts +1 -1
- package/dist/languages/hausa.d.ts +1 -1
- package/dist/languages/hindi.d.ts +1 -1
- package/dist/languages/hungarian.d.ts +1 -1
- package/dist/languages/igbo.d.ts +1 -1
- package/dist/languages/indonesian.d.ts +1 -1
- package/dist/languages/irish.d.ts +1 -1
- package/dist/languages/italian.d.ts +1 -1
- package/dist/languages/japanese.d.ts +1 -1
- package/dist/languages/nepali.d.ts +1 -1
- package/dist/languages/norwegian.d.ts +1 -1
- package/dist/languages/portuguese.d.ts +1 -1
- package/dist/languages/romanian.d.ts +1 -1
- package/dist/languages/russian.d.ts +1 -1
- package/dist/languages/sanskrit.d.ts +1 -1
- package/dist/languages/serbian.d.ts +1 -1
- package/dist/languages/slovenian.d.ts +1 -1
- package/dist/languages/spanish.d.ts +1 -1
- package/dist/languages/swahili.d.ts +1 -1
- package/dist/languages/swedish.d.ts +1 -1
- package/dist/languages/tamil.d.ts +1 -1
- package/dist/languages/turkish.d.ts +1 -1
- package/dist/languages/twi.d.ts +1 -1
- package/dist/languages/ukrainian.d.ts +1 -1
- package/dist/languages/yoruba.d.ts +1 -1
- package/dist/languages/zulu.d.ts +1 -1
- package/dist/narsil-ChTdkwPu.d.ts +68 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +41 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +59 -0
- package/dist/schema-BVaw7nqz.d.ts +58 -0
- package/dist/serialization/crc32-worker.d.ts +20 -0
- package/dist/serialization/crc32-worker.mjs +61 -0
- package/dist/server.d.ts +218 -0
- package/dist/server.mjs +1185 -0
- package/dist/types-Bt7c4hDF.d.ts +73 -0
- package/dist/vector/hnsw-build-worker.d.ts +9 -2
- package/dist/vector/hnsw-build-worker.mjs +54 -10
- package/dist/workers/entry.mjs +1 -1
- package/package.json +46 -4
- package/dist/chunk-7GKPWGCN.mjs +0 -43
- package/dist/chunk-GVN2W4VA.mjs +0 -570
- package/dist/direct-executor-6TSG7GWH.mjs +0 -4
package/README.md
CHANGED
|
@@ -23,9 +23,9 @@ Requires Node.js >= 22.
|
|
|
23
23
|
## Quick start
|
|
24
24
|
|
|
25
25
|
```ts
|
|
26
|
-
import {
|
|
26
|
+
import { createNarsil } from '@delali/narsil'
|
|
27
27
|
|
|
28
|
-
const narsil =
|
|
28
|
+
const narsil = await createNarsil()
|
|
29
29
|
|
|
30
30
|
await narsil.createIndex('products', {
|
|
31
31
|
schema: {
|
|
@@ -59,6 +59,72 @@ const results = await narsil.query('products', {
|
|
|
59
59
|
})
|
|
60
60
|
```
|
|
61
61
|
|
|
62
|
+
## Configuration
|
|
63
|
+
|
|
64
|
+
`createNarsil` accepts an optional `NarsilConfig` object. All fields are optional.
|
|
65
|
+
|
|
66
|
+
```ts
|
|
67
|
+
import { createNarsil } from '@delali/narsil'
|
|
68
|
+
|
|
69
|
+
const narsil = await createNarsil({
|
|
70
|
+
persistence: myAdapter,
|
|
71
|
+
workers: { enabled: true, count: 4 },
|
|
72
|
+
flush: { interval: 5000, mutationThreshold: 100 },
|
|
73
|
+
})
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
### NarsilConfig
|
|
77
|
+
|
|
78
|
+
| Field | Type | Description |
|
|
79
|
+
| --- | --- | --- |
|
|
80
|
+
| `persistence` | `PersistenceAdapter` | Storage backend for durable indexes (filesystem, IndexedDB, or custom) |
|
|
81
|
+
| `invalidation` | `InvalidationAdapter` | Cross-instance cache coordination adapter for multi-process or multi-tab setups |
|
|
82
|
+
| `plugins` | `NarsilPlugin[]` | Lifecycle hooks for insert, remove, update, search, and index events |
|
|
83
|
+
| `idGenerator` | `() => string` | Custom function for generating document IDs (defaults to UUID v7) |
|
|
84
|
+
| `workers` | `WorkerConfig` | Worker thread configuration for parallel search |
|
|
85
|
+
| `flush` | `FlushConfig` | Controls when dirty partitions persist to storage |
|
|
86
|
+
| `eagerLoad` | `boolean` | When `true`, loads all persisted data into memory at creation time |
|
|
87
|
+
| `embedding` | `EmbeddingAdapter` | Default adapter for auto-embedding text fields into vectors (see [Embedding adapters](#embedding-adapters)) |
|
|
88
|
+
|
|
89
|
+
### WorkerConfig
|
|
90
|
+
|
|
91
|
+
| Field | Type | Default | Description |
|
|
92
|
+
| --- | --- | --- | --- |
|
|
93
|
+
| `enabled` | `boolean` | `false` | Enables worker thread pool for search |
|
|
94
|
+
| `count` | `number` | CPU count | Number of worker threads to spawn |
|
|
95
|
+
| `promotionThreshold` | `number` | - | Document count per index that triggers auto-promotion to workers |
|
|
96
|
+
| `totalPromotionThreshold` | `number` | - | Total document count across all indexes that triggers auto-promotion |
|
|
97
|
+
|
|
98
|
+
### FlushConfig
|
|
99
|
+
|
|
100
|
+
| Field | Type | Default | Description |
|
|
101
|
+
| --- | --- | --- | --- |
|
|
102
|
+
| `interval` | `number` | - | Milliseconds between persistence flushes |
|
|
103
|
+
| `mutationThreshold` | `number` | - | Number of mutations that triggers a flush |
|
|
104
|
+
|
|
105
|
+
### Tokenizer cache
|
|
106
|
+
|
|
107
|
+
The stemmer normalization cache auto-sizes based on the runtime environment. On Node.js it reads container memory limits (via `process.constrainedMemory()`), in browsers it checks `navigator.deviceMemory`, and falls back to a sensible default elsewhere.
|
|
108
|
+
|
|
109
|
+
This cache is process-global, shared across all Narsil instances in the same process. You can override the size by calling `configureNormalizationCache` once at startup, before creating any instances:
|
|
110
|
+
|
|
111
|
+
```ts
|
|
112
|
+
import { configureNormalizationCache, createNarsil } from '@delali/narsil'
|
|
113
|
+
|
|
114
|
+
configureNormalizationCache(500_000)
|
|
115
|
+
|
|
116
|
+
const narsil = await createNarsil()
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
The value clamps to a floor of 50,000 and a ceiling of 2,000,000 entries. Invalid values (NaN, Infinity, negative numbers, zero) throw a `NarsilError` with code `CONFIG_INVALID`.
|
|
120
|
+
|
|
121
|
+
Four utility functions are available for cache management:
|
|
122
|
+
|
|
123
|
+
- `configureNormalizationCache(maxSize)` - set the maximum cache size
|
|
124
|
+
- `clearNormalizationCache()` - drop all cached entries (reclaim memory after one-off indexing)
|
|
125
|
+
- `resetNormalizationCache()` - clear the cache and reset the size to the auto-detected default
|
|
126
|
+
- `getNormalizationCacheSize()` - return the current number of cached entries (useful for monitoring)
|
|
127
|
+
|
|
62
128
|
## Features
|
|
63
129
|
|
|
64
130
|
### Full-text search
|
|
@@ -69,10 +135,50 @@ BM25 scoring with field boosting, fuzzy matching via bounded Levenshtein distanc
|
|
|
69
135
|
|
|
70
136
|
Indexes split into shards when they grow beyond a configurable threshold. Partition assignment uses FNV-1a hashing for deterministic routing. Background rebalancing redistributes documents across new partitions without blocking queries, using a write-ahead queue for zero-downtime resharding.
|
|
71
137
|
|
|
138
|
+
**Single-process overhead.** Partitioning pays off when shards live on separate processes or hosts. Inside one Node.js process, going from 1 to 20 partitions costs about 14% of insert throughput, 28% of median search latency, and 27% at p95, with no scaling upside. Keep `partitions.maxPartitions` low (or unset) for single-process deployments and raise it once you fan partitions out across workers or distributed nodes.
|
|
139
|
+
|
|
140
|
+
**Rebalance latency spikes.** While a partition reshape is running, worst-tick p95 latency can climb to about 25ms compared with around 11ms in steady state. Schedule large bulk inserts that trigger rebalancing during low-traffic windows, or pre-size the index with `partitions.maxDocsPerPartition` to avoid mid-load reshapes.
|
|
141
|
+
|
|
72
142
|
### Worker isolation
|
|
73
143
|
|
|
74
144
|
Search operations move off the main thread through worker threads (Node.js, Bun) or Web Workers (browser, Deno). The engine starts in direct mode for small indexes and auto-promotes to workers once document counts cross a threshold. You keep using the same API throughout.
|
|
75
145
|
|
|
146
|
+
### Memory reporting
|
|
147
|
+
|
|
148
|
+
Narsil reports memory at three levels: per index, per partition, and per runtime. `getStats(indexName)` returns `estimatedMemoryBytes`, a formula-based estimate for the index's main-thread partitions and vector structures.
|
|
149
|
+
|
|
150
|
+
```ts
|
|
151
|
+
const indexStats = narsil.getStats('products')
|
|
152
|
+
|
|
153
|
+
console.log(indexStats.estimatedMemoryBytes)
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
The estimate comes from document counts, posting lists, field indexes, and vector indexes. It excludes V8 object headers, allocator overhead, and other host runtime costs, so use it for comparing indexes inside one process.
|
|
157
|
+
|
|
158
|
+
`getPartitionStats(indexName)` returns the same estimate for each partition. The partition values sum to `getStats(indexName).estimatedMemoryBytes`.
|
|
159
|
+
|
|
160
|
+
```ts
|
|
161
|
+
const partitions = narsil.getPartitionStats('products')
|
|
162
|
+
|
|
163
|
+
for (const partition of partitions) {
|
|
164
|
+
console.log(partition.partitionId, partition.estimatedMemoryBytes)
|
|
165
|
+
}
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
`getMemoryStats()` returns a runtime snapshot and worker reports. It is async because workers report their heap usage through the worker message channel.
|
|
169
|
+
|
|
170
|
+
```ts
|
|
171
|
+
const memory = await narsil.getMemoryStats()
|
|
172
|
+
|
|
173
|
+
console.log(memory.estimatedIndexBytes)
|
|
174
|
+
console.log(memory.process?.heapUsed)
|
|
175
|
+
console.log(memory.workers)
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
`memory.estimatedIndexBytes` sums `estimatedMemoryBytes` across every index in this Narsil instance. `memory.process` comes from `process.memoryUsage()` when the runtime exposes it. It measures the whole host process, so two Narsil instances in one Node.js process report the same process numbers. Browser runtimes return `null` for `memory.process`.
|
|
179
|
+
|
|
180
|
+
`memory.workers` lists `heapUsed`, `heapTotal`, and `external` for each active worker. The array is empty before worker promotion.
|
|
181
|
+
|
|
76
182
|
### Distributed BM25 scoring
|
|
77
183
|
|
|
78
184
|
Three scoring modes handle the partition-skew problem:
|
|
@@ -83,7 +189,7 @@ Three scoring modes handle the partition-skew problem:
|
|
|
83
189
|
|
|
84
190
|
### Vector search
|
|
85
191
|
|
|
86
|
-
Store and query high-dimensional embeddings with cosine similarity, dot product, or Euclidean distance. Small vector sets use brute-force
|
|
192
|
+
Store and query high-dimensional embeddings with cosine similarity, dot product, or Euclidean distance. Small vector sets use an exact brute-force scan. Once a field crosses 1,024 vectors, the engine builds an HNSW graph and switches the field to approximate search. The threshold is configurable per index via `vectorPromotion.threshold`.
|
|
87
193
|
|
|
88
194
|
### Hybrid search
|
|
89
195
|
|
|
@@ -146,6 +252,68 @@ const results = await narsil.query('products', {
|
|
|
146
252
|
// hit.highlights.title.snippet => '<mark>Mechanical</mark> Keyboard'
|
|
147
253
|
```
|
|
148
254
|
|
|
255
|
+
### Embedding adapters
|
|
256
|
+
|
|
257
|
+
Auto-embed text fields into vectors on insert and query. Pass an `EmbeddingAdapter` at the instance level (shared default) or per-index. When a document is inserted without a pre-computed vector, Narsil concatenates the configured source text fields and calls the adapter to generate the embedding. On query, pass `text` instead of `value` to auto-embed the search phrase.
|
|
258
|
+
|
|
259
|
+
```ts
|
|
260
|
+
import { createNarsil } from '@delali/narsil'
|
|
261
|
+
import { createOpenAIEmbedding } from '@delali/narsil/embeddings/openai'
|
|
262
|
+
|
|
263
|
+
const narsil = await createNarsil({
|
|
264
|
+
embedding: createOpenAIEmbedding({
|
|
265
|
+
apiKey: process.env.OPENAI_API_KEY,
|
|
266
|
+
model: 'text-embedding-3-small',
|
|
267
|
+
dimensions: 1536,
|
|
268
|
+
}),
|
|
269
|
+
})
|
|
270
|
+
|
|
271
|
+
await narsil.createIndex('articles', {
|
|
272
|
+
schema: {
|
|
273
|
+
title: 'string',
|
|
274
|
+
body: 'string',
|
|
275
|
+
embedding: 'vector[1536]',
|
|
276
|
+
},
|
|
277
|
+
embedding: {
|
|
278
|
+
fields: {
|
|
279
|
+
embedding: ['title', 'body'],
|
|
280
|
+
},
|
|
281
|
+
},
|
|
282
|
+
})
|
|
283
|
+
|
|
284
|
+
await narsil.insert('articles', {
|
|
285
|
+
title: 'Distributed search engines',
|
|
286
|
+
body: 'Partitioning data across shards improves throughput...',
|
|
287
|
+
})
|
|
288
|
+
|
|
289
|
+
const results = await narsil.query('articles', {
|
|
290
|
+
mode: 'vector',
|
|
291
|
+
vector: { field: 'embedding', text: 'how do search engines scale?' },
|
|
292
|
+
})
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
Two adapters ship with the project:
|
|
296
|
+
|
|
297
|
+
| Adapter | Package | Dependencies |
|
|
298
|
+
| --- | --- | --- |
|
|
299
|
+
| OpenAI | `@delali/narsil/embeddings/openai` | None (uses `fetch`) |
|
|
300
|
+
| Transformers.js | `@delali/narsil-embeddings-transformers` | `@huggingface/transformers` (peer) |
|
|
301
|
+
|
|
302
|
+
The OpenAI adapter includes retry logic with exponential backoff and jitter, batch chunking (up to 2,048 inputs per request), and configurable timeouts. The Transformers.js adapter runs models locally with lazy pipeline initialization, supports WebGPU/WASM/CPU backends, and handles asymmetric models through document/query prefixes.
|
|
303
|
+
|
|
304
|
+
Build a custom adapter by implementing the `EmbeddingAdapter` interface:
|
|
305
|
+
|
|
306
|
+
```ts
|
|
307
|
+
interface EmbeddingAdapter {
|
|
308
|
+
embed(input: string, purpose: 'document' | 'query', signal?: AbortSignal): Promise<Float32Array>
|
|
309
|
+
embedBatch?(inputs: string[], purpose: 'document' | 'query', signal?: AbortSignal): Promise<Float32Array[]>
|
|
310
|
+
readonly dimensions: number
|
|
311
|
+
shutdown?(): Promise<void>
|
|
312
|
+
}
|
|
313
|
+
```
|
|
314
|
+
|
|
315
|
+
If `embedBatch` is not provided, Narsil falls back to parallel `embed()` calls with a concurrency limit of 8.
|
|
316
|
+
|
|
149
317
|
### Term suggestions
|
|
150
318
|
|
|
151
319
|
Get autocomplete suggestions from the index's term dictionary. The API tokenizes the input, extracts the last word, and matches it against stored terms across all partitions. Results are ranked by document frequency.
|
|
@@ -228,7 +396,7 @@ Ranking quality is measured against the [Cranfield Collection](https://ir-datase
|
|
|
228
396
|
|
|
229
397
|
| Engine | nDCG@10 | P@10 | MAP | MRR |
|
|
230
398
|
| --- | ---: | ---: | ---: | ---: |
|
|
231
|
-
| **Narsil
|
|
399
|
+
| **Narsil** | **0.3739** | **0.2458** | **0.2614** | **0.5638** |
|
|
232
400
|
| Orama 3.1.18 | 0.2911 | 0.1836 | 0.1846 | 0.4821 |
|
|
233
401
|
| MiniSearch 7.2.0 | 0.0077 | 0.0067 | 0.0027 | 0.0139 |
|
|
234
402
|
|
|
@@ -243,6 +411,8 @@ These scores fall within the expected range for BM25 on standard IR benchmarks (
|
|
|
243
411
|
|
|
244
412
|
Reproduce: `pnpm -C packages/benchmarks bench --tiers quality`
|
|
245
413
|
|
|
414
|
+
Narsil also runs as a search server. On the BEIR information-retrieval datasets it is measured against Elasticsearch, OpenSearch, Qdrant, Weaviate, Typesense, and Meilisearch across keyword, vector, and hybrid retrieval. See [the full benchmarks](https://github.com/assetcorp/narsil/blob/main/BENCHMARKS.md) for those results.
|
|
415
|
+
|
|
246
416
|
## Runtime support (TypeScript)
|
|
247
417
|
|
|
248
418
|
| Runtime | Concurrency | Persistence | Invalidation |
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { NarsilError, ErrorCodes } from '../chunk-
|
|
1
|
+
import { NarsilError, ErrorCodes } from '../chunk-ALB35PRQ.mjs';
|
|
2
2
|
|
|
3
3
|
// src/persistence/filesystem.ts
|
|
4
4
|
var cachedFs = null;
|
|
@@ -30,6 +30,25 @@ async function resolveAndGuard(baseDir, key) {
|
|
|
30
30
|
}
|
|
31
31
|
return resolvedPath;
|
|
32
32
|
}
|
|
33
|
+
async function fsyncDirectory(dirPath, fs) {
|
|
34
|
+
let handle = null;
|
|
35
|
+
try {
|
|
36
|
+
handle = await fs.open(dirPath, "r");
|
|
37
|
+
await handle.sync();
|
|
38
|
+
} catch (err) {
|
|
39
|
+
const code = err.code;
|
|
40
|
+
if (code === "EISDIR" || code === "EPERM" || code === "EINVAL" || code === "ENOTSUP") {
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_FSYNC_FAILED, `fsync failed for directory "${dirPath}"`, {
|
|
44
|
+
cause: err instanceof Error ? err.message : String(err)
|
|
45
|
+
});
|
|
46
|
+
} finally {
|
|
47
|
+
if (handle !== null) {
|
|
48
|
+
await handle.close().catch(() => void 0);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
33
52
|
async function listDirectoryRecursive(dir, baseDir, pathMod, fs) {
|
|
34
53
|
const results = [];
|
|
35
54
|
let entries;
|
|
@@ -62,13 +81,38 @@ function createFilesystemPersistence(config) {
|
|
|
62
81
|
}
|
|
63
82
|
const baseDir = config.directory;
|
|
64
83
|
return {
|
|
84
|
+
directory: baseDir,
|
|
65
85
|
async save(key, data) {
|
|
66
86
|
const resolvedPath = await resolveAndGuard(baseDir, key);
|
|
67
87
|
const fs = await getFs();
|
|
68
88
|
const pathMod = await getPath();
|
|
69
89
|
const dir = pathMod.dirname(resolvedPath);
|
|
70
90
|
await fs.mkdir(dir, { recursive: true });
|
|
71
|
-
|
|
91
|
+
const tempName = `${pathMod.basename(resolvedPath)}.tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
|
92
|
+
const tempPath = pathMod.join(dir, tempName);
|
|
93
|
+
let handle = null;
|
|
94
|
+
try {
|
|
95
|
+
handle = await fs.open(tempPath, "w");
|
|
96
|
+
await handle.write(data);
|
|
97
|
+
try {
|
|
98
|
+
await handle.sync();
|
|
99
|
+
} catch (err) {
|
|
100
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_FSYNC_FAILED, `fsync failed for "${key}"`, {
|
|
101
|
+
key,
|
|
102
|
+
cause: err instanceof Error ? err.message : String(err)
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
await handle.close();
|
|
106
|
+
handle = null;
|
|
107
|
+
await fs.rename(tempPath, resolvedPath);
|
|
108
|
+
await fsyncDirectory(dir, fs);
|
|
109
|
+
} catch (err) {
|
|
110
|
+
if (handle !== null) {
|
|
111
|
+
await handle.close().catch(() => void 0);
|
|
112
|
+
}
|
|
113
|
+
await fs.unlink(tempPath).catch(() => void 0);
|
|
114
|
+
throw err;
|
|
115
|
+
}
|
|
72
116
|
},
|
|
73
117
|
async load(key) {
|
|
74
118
|
const resolvedPath = await resolveAndGuard(baseDir, key);
|
|
@@ -3,6 +3,7 @@ interface PersistenceAdapter {
|
|
|
3
3
|
load(key: string): Promise<Uint8Array | null>;
|
|
4
4
|
delete(key: string): Promise<void>;
|
|
5
5
|
list(prefix: string): Promise<string[]>;
|
|
6
|
+
readonly directory?: string;
|
|
6
7
|
}
|
|
7
8
|
interface InvalidationAdapter {
|
|
8
9
|
publish(event: InvalidationEvent): Promise<void>;
|
|
@@ -26,5 +27,11 @@ interface PartitionStatistics {
|
|
|
26
27
|
docFrequencies: Record<string, number>;
|
|
27
28
|
totalFieldLengths: Record<string, number>;
|
|
28
29
|
}
|
|
30
|
+
interface EmbeddingAdapter {
|
|
31
|
+
embed(input: string, purpose: 'document' | 'query', signal?: AbortSignal): Promise<Float32Array>;
|
|
32
|
+
embedBatch?(inputs: string[], purpose: 'document' | 'query', signal?: AbortSignal): Promise<Float32Array[]>;
|
|
33
|
+
readonly dimensions: number;
|
|
34
|
+
shutdown?(): Promise<void>;
|
|
35
|
+
}
|
|
29
36
|
|
|
30
|
-
export type { InvalidationAdapter as I,
|
|
37
|
+
export type { EmbeddingAdapter as E, InvalidationAdapter as I, PartitionStatistics as P, InvalidationEvent as a, PersistenceAdapter as b };
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
// src/errors.ts
|
|
2
|
+
var ErrorCodes = {
|
|
3
|
+
SCHEMA_INVALID_TYPE: "SCHEMA_INVALID_TYPE",
|
|
4
|
+
SCHEMA_MISSING_FIELD: "SCHEMA_MISSING_FIELD",
|
|
5
|
+
SCHEMA_DEPTH_EXCEEDED: "SCHEMA_DEPTH_EXCEEDED",
|
|
6
|
+
SCHEMA_INVALID_VECTOR_DIMENSION: "SCHEMA_INVALID_VECTOR_DIMENSION",
|
|
7
|
+
SCHEMA_INVALID_GEOPOINT: "SCHEMA_INVALID_GEOPOINT",
|
|
8
|
+
DOC_NOT_FOUND: "DOC_NOT_FOUND",
|
|
9
|
+
DOC_ALREADY_EXISTS: "DOC_ALREADY_EXISTS",
|
|
10
|
+
DOC_VALIDATION_FAILED: "DOC_VALIDATION_FAILED",
|
|
11
|
+
INDEX_NOT_FOUND: "INDEX_NOT_FOUND",
|
|
12
|
+
INDEX_ALREADY_EXISTS: "INDEX_ALREADY_EXISTS",
|
|
13
|
+
PARTITION_CORRUPTED: "PARTITION_CORRUPTED",
|
|
14
|
+
PARTITION_REBALANCING_BACKPRESSURE: "PARTITION_REBALANCING_BACKPRESSURE",
|
|
15
|
+
WORKER_CRASHED: "WORKER_CRASHED",
|
|
16
|
+
WORKER_BUSY: "WORKER_BUSY",
|
|
17
|
+
WORKER_TIMEOUT: "WORKER_TIMEOUT",
|
|
18
|
+
PERSISTENCE_SAVE_FAILED: "PERSISTENCE_SAVE_FAILED",
|
|
19
|
+
PERSISTENCE_LOAD_FAILED: "PERSISTENCE_LOAD_FAILED",
|
|
20
|
+
PERSISTENCE_DELETE_FAILED: "PERSISTENCE_DELETE_FAILED",
|
|
21
|
+
PERSISTENCE_CRC_MISMATCH: "PERSISTENCE_CRC_MISMATCH",
|
|
22
|
+
PERSISTENCE_WAL_CORRUPT: "PERSISTENCE_WAL_CORRUPT",
|
|
23
|
+
PERSISTENCE_FSYNC_FAILED: "PERSISTENCE_FSYNC_FAILED",
|
|
24
|
+
SEARCH_INVALID_FIELD: "SEARCH_INVALID_FIELD",
|
|
25
|
+
SEARCH_INVALID_VECTOR_SIZE: "SEARCH_INVALID_VECTOR_SIZE",
|
|
26
|
+
VECTOR_DIMENSION_MISMATCH: "VECTOR_DIMENSION_MISMATCH",
|
|
27
|
+
SEARCH_INVALID_FILTER: "SEARCH_INVALID_FILTER",
|
|
28
|
+
SEARCH_INVALID_MODE: "SEARCH_INVALID_MODE",
|
|
29
|
+
SEARCH_INVALID_CURSOR: "SEARCH_INVALID_CURSOR",
|
|
30
|
+
LANGUAGE_NOT_SUPPORTED: "LANGUAGE_NOT_SUPPORTED",
|
|
31
|
+
ENVELOPE_VERSION_MISMATCH: "ENVELOPE_VERSION_MISMATCH",
|
|
32
|
+
ENVELOPE_INVALID_MAGIC: "ENVELOPE_INVALID_MAGIC",
|
|
33
|
+
PARTITION_CAPACITY_EXCEEDED: "PARTITION_CAPACITY_EXCEEDED",
|
|
34
|
+
EMBEDDING_FAILED: "EMBEDDING_FAILED",
|
|
35
|
+
EMBEDDING_DIMENSION_MISMATCH: "EMBEDDING_DIMENSION_MISMATCH",
|
|
36
|
+
EMBEDDING_NO_SOURCE: "EMBEDDING_NO_SOURCE",
|
|
37
|
+
EMBEDDING_CONFIG_INVALID: "EMBEDDING_CONFIG_INVALID",
|
|
38
|
+
DOC_MISSING_REQUIRED_FIELD: "DOC_MISSING_REQUIRED_FIELD",
|
|
39
|
+
CONFIG_INVALID: "CONFIG_INVALID",
|
|
40
|
+
QUERY_ROUTING_FAILED: "QUERY_ROUTING_FAILED",
|
|
41
|
+
QUERY_PARTIAL_FAILURE: "QUERY_PARTIAL_FAILURE",
|
|
42
|
+
QUERY_NODE_TIMEOUT: "QUERY_NODE_TIMEOUT",
|
|
43
|
+
QUERY_NO_ACTIVE_REPLICA: "QUERY_NO_ACTIVE_REPLICA",
|
|
44
|
+
ALLOCATION_NO_DATA_NODES: "ALLOCATION_NO_DATA_NODES",
|
|
45
|
+
ALLOCATION_INVALID_CONFIG: "ALLOCATION_INVALID_CONFIG",
|
|
46
|
+
ALLOCATION_FAILED: "ALLOCATION_FAILED",
|
|
47
|
+
CONTROLLER_LEASE_LOST: "CONTROLLER_LEASE_LOST",
|
|
48
|
+
CONTROLLER_NOT_ACTIVE: "CONTROLLER_NOT_ACTIVE",
|
|
49
|
+
CONTROLLER_METADATA_INVALID: "CONTROLLER_METADATA_INVALID",
|
|
50
|
+
NODE_BOOTSTRAP_FAILED: "NODE_BOOTSTRAP_FAILED",
|
|
51
|
+
NODE_ALREADY_JOINED: "NODE_ALREADY_JOINED",
|
|
52
|
+
NODE_NOT_JOINED: "NODE_NOT_JOINED",
|
|
53
|
+
COORDINATOR_DEPENDENCY_MISSING: "COORDINATOR_DEPENDENCY_MISSING",
|
|
54
|
+
REPLICATION_ENTRY_INVALID: "REPLICATION_ENTRY_INVALID",
|
|
55
|
+
REPLICATION_INSYNC_REMOVAL_FAILED: "REPLICATION_INSYNC_REMOVAL_FAILED",
|
|
56
|
+
REPLICATION_ROLLBACK_FAILED: "REPLICATION_ROLLBACK_FAILED",
|
|
57
|
+
SNAPSHOT_SYNC_UNAUTHORIZED: "SNAPSHOT_SYNC_UNAUTHORIZED",
|
|
58
|
+
SNAPSHOT_SYNC_REQUEST_INVALID: "SNAPSHOT_SYNC_REQUEST_INVALID",
|
|
59
|
+
SNAPSHOT_SYNC_INDEX_NOT_FOUND: "SNAPSHOT_SYNC_INDEX_NOT_FOUND",
|
|
60
|
+
SNAPSHOT_SYNC_TOO_LARGE: "SNAPSHOT_SYNC_TOO_LARGE",
|
|
61
|
+
SNAPSHOT_SYNC_CAPACITY_EXHAUSTED: "SNAPSHOT_SYNC_CAPACITY_EXHAUSTED",
|
|
62
|
+
SNAPSHOT_SYNC_SNAPSHOT_FAILED: "SNAPSHOT_SYNC_SNAPSHOT_FAILED",
|
|
63
|
+
SNAPSHOT_SYNC_DECODE_FAILED: "SNAPSHOT_SYNC_DECODE_FAILED",
|
|
64
|
+
SNAPSHOT_SYNC_FRAME_INVALID: "SNAPSHOT_SYNC_FRAME_INVALID",
|
|
65
|
+
SNAPSHOT_SYNC_HEADER_INVALID: "SNAPSHOT_SYNC_HEADER_INVALID",
|
|
66
|
+
SNAPSHOT_SYNC_HEADER_MISMATCH: "SNAPSHOT_SYNC_HEADER_MISMATCH",
|
|
67
|
+
SNAPSHOT_SYNC_CHUNK_OUT_OF_ORDER: "SNAPSHOT_SYNC_CHUNK_OUT_OF_ORDER",
|
|
68
|
+
SNAPSHOT_SYNC_CHUNK_OVERFLOW: "SNAPSHOT_SYNC_CHUNK_OVERFLOW",
|
|
69
|
+
SNAPSHOT_SYNC_CHUNK_SIZE_EXCEEDED: "SNAPSHOT_SYNC_CHUNK_SIZE_EXCEEDED",
|
|
70
|
+
SNAPSHOT_SYNC_CHUNK_MISSING: "SNAPSHOT_SYNC_CHUNK_MISSING",
|
|
71
|
+
SNAPSHOT_SYNC_END_MISSING: "SNAPSHOT_SYNC_END_MISSING",
|
|
72
|
+
SNAPSHOT_SYNC_CHECKSUM_MISMATCH: "SNAPSHOT_SYNC_CHECKSUM_MISMATCH",
|
|
73
|
+
SNAPSHOT_SYNC_PRIMARY_ERROR: "SNAPSHOT_SYNC_PRIMARY_ERROR",
|
|
74
|
+
SNAPSHOT_SYNC_NO_TARGETS: "SNAPSHOT_SYNC_NO_TARGETS",
|
|
75
|
+
SNAPSHOT_SYNC_TRANSPORT_FAILED: "SNAPSHOT_SYNC_TRANSPORT_FAILED",
|
|
76
|
+
SNAPSHOT_SYNC_SCHEMA_UNAVAILABLE: "SNAPSHOT_SYNC_SCHEMA_UNAVAILABLE",
|
|
77
|
+
SNAPSHOT_SYNC_RESTORE_FAILED: "SNAPSHOT_SYNC_RESTORE_FAILED",
|
|
78
|
+
SNAPSHOT_SYNC_RESTORE_CLEANUP_FAILED: "SNAPSHOT_SYNC_RESTORE_CLEANUP_FAILED",
|
|
79
|
+
SNAPSHOT_SYNC_TIMEOUT: "SNAPSHOT_SYNC_TIMEOUT",
|
|
80
|
+
SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE: "SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE",
|
|
81
|
+
SNAPSHOT_SYNC_NOT_ASSIGNED: "SNAPSHOT_SYNC_NOT_ASSIGNED",
|
|
82
|
+
SNAPSHOT_SYNC_ABORTED: "SNAPSHOT_SYNC_ABORTED"
|
|
83
|
+
};
|
|
84
|
+
var NarsilError = class extends Error {
|
|
85
|
+
code;
|
|
86
|
+
details;
|
|
87
|
+
constructor(code, message, details) {
|
|
88
|
+
super(message);
|
|
89
|
+
this.name = "NarsilError";
|
|
90
|
+
this.code = code;
|
|
91
|
+
this.details = details ?? {};
|
|
92
|
+
}
|
|
93
|
+
};
|
|
94
|
+
function createNarsilError(code, message, details) {
|
|
95
|
+
return new NarsilError(code, message, details);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export { ErrorCodes, NarsilError, createNarsilError };
|