@delali/narsil 0.2.0 → 0.2.1-next.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.d.ts +23 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.d.ts +25 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +14 -1
- package/dist/adapters-CVqI5NBt.d.ts +149 -0
- package/dist/{chunk-S6IAXOBA.mjs → chunk-ASI6AOW2.mjs} +1 -1
- package/dist/{chunk-AUMKFYTG.mjs → chunk-FKK76OR7.mjs} +56 -55
- package/dist/{chunk-M2EDG55Y.mjs → chunk-UNJZ7ZDB.mjs} +10 -0
- package/dist/{chunk-XAKNSWIK.mjs → chunk-WLR5BFL3.mjs} +16 -14
- package/dist/config-CAjTHzkH.d.ts +256 -0
- package/dist/{direct-executor-2FV3MEDD.mjs → direct-executor-3W4QRA4E.mjs} +2 -2
- package/dist/distribution/coordinator/etcd.d.ts +44 -3
- package/dist/distribution/coordinator/etcd.mjs +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +14 -3
- package/dist/distribution.d.ts +312 -45
- package/dist/distribution.mjs +4 -4
- package/dist/embeddings/openai.d.ts +28 -1
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/engine-CCrlfLyq.d.ts +431 -0
- package/dist/index.browser.mjs +24 -12
- package/dist/index.d.ts +173 -8
- package/dist/index.mjs +3 -3
- package/dist/invalidation/broadcast-channel.d.ts +20 -1
- package/dist/invalidation/filesystem.d.ts +24 -1
- package/dist/invalidation/noop.d.ts +14 -1
- package/dist/language-DB8KG1z5.d.ts +53 -0
- package/dist/languages/albanian.d.ts +11 -1
- package/dist/languages/amharic.d.ts +11 -1
- package/dist/languages/arabic.d.ts +10 -1
- package/dist/languages/armenian.d.ts +10 -1
- package/dist/languages/azerbaijani.d.ts +11 -1
- package/dist/languages/bambara.d.ts +11 -1
- package/dist/languages/basque.d.ts +10 -1
- package/dist/languages/belarusian.d.ts +11 -1
- package/dist/languages/bengali.d.ts +11 -1
- package/dist/languages/bosnian.d.ts +11 -1
- package/dist/languages/breton.d.ts +11 -1
- package/dist/languages/bulgarian.d.ts +10 -1
- package/dist/languages/burmese.d.ts +11 -1
- package/dist/languages/catalan.d.ts +10 -1
- package/dist/languages/chinese.d.ts +11 -1
- package/dist/languages/croatian.d.ts +11 -1
- package/dist/languages/czech.d.ts +10 -1
- package/dist/languages/dagbani.d.ts +11 -1
- package/dist/languages/danish.d.ts +10 -1
- package/dist/languages/dutch.d.ts +10 -1
- package/dist/languages/english.d.ts +10 -1
- package/dist/languages/esperanto.d.ts +10 -1
- package/dist/languages/estonian.d.ts +10 -1
- package/dist/languages/ewe.d.ts +11 -1
- package/dist/languages/faroese.d.ts +11 -1
- package/dist/languages/fijian.d.ts +11 -1
- package/dist/languages/finnish.d.ts +10 -1
- package/dist/languages/french.d.ts +10 -1
- package/dist/languages/ga.d.ts +11 -1
- package/dist/languages/galician.d.ts +11 -1
- package/dist/languages/georgian.d.ts +11 -1
- package/dist/languages/german.d.ts +10 -1
- package/dist/languages/greek.d.ts +10 -1
- package/dist/languages/guarani.d.ts +11 -1
- package/dist/languages/gujarati.d.ts +11 -1
- package/dist/languages/haitian-creole.d.ts +11 -1
- package/dist/languages/hausa.d.ts +11 -1
- package/dist/languages/hawaiian.d.ts +11 -1
- package/dist/languages/hebrew.d.ts +11 -1
- package/dist/languages/hindi.d.ts +10 -1
- package/dist/languages/hungarian.d.ts +10 -1
- package/dist/languages/icelandic.d.ts +11 -1
- package/dist/languages/igbo.d.ts +11 -1
- package/dist/languages/indonesian.d.ts +10 -1
- package/dist/languages/irish.d.ts +10 -1
- package/dist/languages/italian.d.ts +10 -1
- package/dist/languages/japanese.d.ts +11 -1
- package/dist/languages/kannada.d.ts +11 -1
- package/dist/languages/kazakh.d.ts +11 -1
- package/dist/languages/khmer.d.ts +11 -1
- package/dist/languages/kinyarwanda.d.ts +11 -1
- package/dist/languages/kirundi.d.ts +11 -1
- package/dist/languages/korean.d.ts +11 -1
- package/dist/languages/kurmanji.d.ts +11 -1
- package/dist/languages/kyrgyz.d.ts +11 -1
- package/dist/languages/lao.d.ts +11 -1
- package/dist/languages/latin.d.ts +11 -1
- package/dist/languages/latvian.d.ts +11 -1
- package/dist/languages/lingala.d.ts +11 -1
- package/dist/languages/lithuanian.d.ts +10 -1
- package/dist/languages/luxembourgish.d.ts +11 -1
- package/dist/languages/macedonian.d.ts +11 -1
- package/dist/languages/malagasy.d.ts +11 -1
- package/dist/languages/malay.d.ts +11 -1
- package/dist/languages/malayalam.d.ts +11 -1
- package/dist/languages/maltese.d.ts +11 -1
- package/dist/languages/maori.d.ts +11 -1
- package/dist/languages/marathi.d.ts +11 -1
- package/dist/languages/nepali.d.ts +10 -1
- package/dist/languages/norwegian.d.ts +10 -1
- package/dist/languages/oromo.d.ts +11 -1
- package/dist/languages/persian.d.ts +10 -1
- package/dist/languages/polish.d.ts +10 -1
- package/dist/languages/portuguese.d.ts +10 -1
- package/dist/languages/punjabi.d.ts +11 -1
- package/dist/languages/romanian.d.ts +10 -1
- package/dist/languages/russian.d.ts +10 -1
- package/dist/languages/samoan.d.ts +11 -1
- package/dist/languages/sanskrit.d.ts +10 -1
- package/dist/languages/scottish-gaelic.d.ts +11 -1
- package/dist/languages/serbian.d.ts +10 -1
- package/dist/languages/shona.d.ts +11 -1
- package/dist/languages/sinhala.d.ts +11 -1
- package/dist/languages/slovak.d.ts +11 -1
- package/dist/languages/slovenian.d.ts +10 -1
- package/dist/languages/sorani.d.ts +11 -1
- package/dist/languages/spanish.d.ts +10 -1
- package/dist/languages/swahili.d.ts +10 -1
- package/dist/languages/swedish.d.ts +10 -1
- package/dist/languages/tagalog.d.ts +11 -1
- package/dist/languages/tamil.d.ts +10 -1
- package/dist/languages/tatar.d.ts +11 -1
- package/dist/languages/telugu.d.ts +11 -1
- package/dist/languages/thai.d.ts +11 -1
- package/dist/languages/tibetan.d.ts +11 -1
- package/dist/languages/tigrinya.d.ts +11 -1
- package/dist/languages/tongan.d.ts +11 -1
- package/dist/languages/turkish.d.ts +10 -1
- package/dist/languages/twi.d.ts +11 -1
- package/dist/languages/ukrainian.d.ts +10 -1
- package/dist/languages/urdu.d.ts +11 -1
- package/dist/languages/vietnamese.d.ts +11 -1
- package/dist/languages/welsh.d.ts +11 -1
- package/dist/languages/wolof.d.ts +11 -1
- package/dist/languages/xhosa.d.ts +11 -1
- package/dist/languages/yoruba.d.ts +11 -1
- package/dist/languages/zulu.d.ts +11 -1
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +3 -3
- package/dist/schema-CE6S_aAr.d.ts +208 -0
- package/dist/search-CgQQIciB.d.ts +786 -0
- package/dist/server.d.ts +213 -12
- package/dist/server.mjs +29 -1
- package/dist/types-BV3nAKTL.d.ts +302 -0
- package/dist/vector/hnsw-build-worker.d.ts +68 -2
- package/dist/vector/hnsw-build-worker.mjs +0 -2
- package/dist/workers/entry.mjs +1 -1
- package/package.json +5 -2
- package/dist/adapters-Bo2JQg8b.d.ts +0 -37
- package/dist/config-P27AtSn4.d.ts +0 -502
- package/dist/language-CmJBHf1j.d.ts +0 -17
- package/dist/narsil-CMZRPDlQ.d.ts +0 -159
- package/dist/schema-BnvLvjRp.d.ts +0 -67
- package/dist/types-BRTcBgAp.d.ts +0 -73
|
@@ -1,8 +1,30 @@
|
|
|
1
|
-
import { b as PersistenceAdapter } from '../adapters-
|
|
1
|
+
import { b as PersistenceAdapter } from '../adapters-CVqI5NBt.js';
|
|
2
2
|
|
|
3
|
+
/**
|
|
4
|
+
* Where {@link createFilesystemPersistence} writes its partitions.
|
|
5
|
+
*
|
|
6
|
+
* @public
|
|
7
|
+
*/
|
|
3
8
|
interface FilesystemPersistenceConfig {
|
|
9
|
+
/** The adapter writes every partition under this directory. It creates the directory, and it refuses any key that would escape it. */
|
|
4
10
|
directory: string;
|
|
5
11
|
}
|
|
12
|
+
/**
|
|
13
|
+
* Builds a persistence adapter that writes each partition to its own file, so
|
|
14
|
+
* an index comes back after a restart.
|
|
15
|
+
*
|
|
16
|
+
* The adapter writes each partition to a temporary file, flushes it to the
|
|
17
|
+
* disk, and renames it into place, which is what keeps a crash mid-write from
|
|
18
|
+
* leaving a half-written partition behind. It refuses a key that would escape
|
|
19
|
+
* the configured directory.
|
|
20
|
+
*
|
|
21
|
+
* @param config - The directory this adapter writes partitions under.
|
|
22
|
+
* @returns An adapter you pass as `persistence` when creating an engine.
|
|
23
|
+
* @throws A `NarsilError` with `PERSISTENCE_SAVE_FAILED` when the directory is
|
|
24
|
+
* empty or blank.
|
|
25
|
+
*
|
|
26
|
+
* @public
|
|
27
|
+
*/
|
|
6
28
|
declare function createFilesystemPersistence(config: FilesystemPersistenceConfig): PersistenceAdapter;
|
|
7
29
|
|
|
8
30
|
export { type FilesystemPersistenceConfig, createFilesystemPersistence };
|
|
@@ -1,9 +1,33 @@
|
|
|
1
|
-
import { b as PersistenceAdapter } from '../adapters-
|
|
1
|
+
import { b as PersistenceAdapter } from '../adapters-CVqI5NBt.js';
|
|
2
2
|
|
|
3
|
+
/**
|
|
4
|
+
* Which IndexedDB database and object store
|
|
5
|
+
* {@link createIndexedDBPersistence} writes to.
|
|
6
|
+
*
|
|
7
|
+
* @public
|
|
8
|
+
*/
|
|
3
9
|
interface IndexedDBPersistenceConfig {
|
|
10
|
+
/** The adapter opens this database, and opens `narsil` by default. */
|
|
4
11
|
dbName?: string;
|
|
12
|
+
/** The adapter writes partitions to this object store, and to `partitions` by default. */
|
|
5
13
|
storeName?: string;
|
|
6
14
|
}
|
|
15
|
+
/**
|
|
16
|
+
* Builds a persistence adapter that writes partitions to IndexedDB, so an
|
|
17
|
+
* index in a browser survives a reload.
|
|
18
|
+
*
|
|
19
|
+
* The adapter opens the database on its first call and keeps the handle, so
|
|
20
|
+
* creating it costs nothing until the engine writes.
|
|
21
|
+
*
|
|
22
|
+
* @param config - The database and object store to use. Omit it to accept the
|
|
23
|
+
* defaults.
|
|
24
|
+
* @returns An adapter you pass as `persistence` when creating an engine.
|
|
25
|
+
* @throws A `NarsilError` with `PERSISTENCE_SAVE_FAILED` in a runtime without
|
|
26
|
+
* IndexedDB, where {@link createMemoryPersistence} or
|
|
27
|
+
* {@link createFilesystemPersistence} applies instead.
|
|
28
|
+
*
|
|
29
|
+
* @public
|
|
30
|
+
*/
|
|
7
31
|
declare function createIndexedDBPersistence(config?: IndexedDBPersistenceConfig): PersistenceAdapter;
|
|
8
32
|
|
|
9
33
|
export { type IndexedDBPersistenceConfig, createIndexedDBPersistence };
|
|
@@ -1,5 +1,18 @@
|
|
|
1
|
-
import { b as PersistenceAdapter } from '../adapters-
|
|
1
|
+
import { b as PersistenceAdapter } from '../adapters-CVqI5NBt.js';
|
|
2
2
|
|
|
3
|
+
/**
|
|
4
|
+
* Builds a persistence adapter that keeps every partition in a map, which is
|
|
5
|
+
* what tests and short-lived processes want.
|
|
6
|
+
*
|
|
7
|
+
* Nothing survives the process, so reach for
|
|
8
|
+
* {@link createFilesystemPersistence} or
|
|
9
|
+
* {@link createIndexedDBPersistence} when an index has to come back after a
|
|
10
|
+
* restart.
|
|
11
|
+
*
|
|
12
|
+
* @returns An adapter you pass as `persistence` when creating an engine.
|
|
13
|
+
*
|
|
14
|
+
* @public
|
|
15
|
+
*/
|
|
3
16
|
declare function createMemoryPersistence(): PersistenceAdapter;
|
|
4
17
|
|
|
5
18
|
export { createMemoryPersistence };
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where the engine writes partitions so that they outlive the process.
|
|
3
|
+
*
|
|
4
|
+
* Every method addresses a partition by an opaque key, so any key-value store
|
|
5
|
+
* works: memory, the filesystem, IndexedDB, or a service you write yourself.
|
|
6
|
+
* The engine never interprets a key beyond the prefix it lists by.
|
|
7
|
+
*
|
|
8
|
+
* @public
|
|
9
|
+
*/
|
|
10
|
+
interface PersistenceAdapter {
|
|
11
|
+
/**
|
|
12
|
+
* Writes one partition, replacing whatever the key already held.
|
|
13
|
+
*
|
|
14
|
+
* @param key - The key the engine reads this partition back under.
|
|
15
|
+
* @param data - The serialised partition.
|
|
16
|
+
*/
|
|
17
|
+
save(key: string, data: Uint8Array): Promise<void>;
|
|
18
|
+
/**
|
|
19
|
+
* Reads one partition back.
|
|
20
|
+
*
|
|
21
|
+
* @param key - The key a matching {@link PersistenceAdapter.save} wrote.
|
|
22
|
+
* @returns The stored bytes, or `null` when the key holds nothing.
|
|
23
|
+
*/
|
|
24
|
+
load(key: string): Promise<Uint8Array | null>;
|
|
25
|
+
/**
|
|
26
|
+
* Removes one partition. A key that holds nothing is not an error.
|
|
27
|
+
*
|
|
28
|
+
* @param key - The key to clear.
|
|
29
|
+
*/
|
|
30
|
+
delete(key: string): Promise<void>;
|
|
31
|
+
/**
|
|
32
|
+
* Lists the keys the store holds under a prefix, which is how the engine
|
|
33
|
+
* finds an index's partitions on start-up.
|
|
34
|
+
*
|
|
35
|
+
* @param prefix - Matched against the start of each key.
|
|
36
|
+
* @returns Every matching key, in any order.
|
|
37
|
+
*/
|
|
38
|
+
list(prefix: string): Promise<string[]>;
|
|
39
|
+
/** This adapter writes to this directory, when a filesystem backs it. */
|
|
40
|
+
readonly directory?: string;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Carries partition changes between engine instances that share one store.
|
|
44
|
+
*
|
|
45
|
+
* Two processes reading the same persisted index each hold their own copy in
|
|
46
|
+
* memory, so a write in one has to reach the other. Publish an event on every
|
|
47
|
+
* write and each subscriber reloads the partitions that event names.
|
|
48
|
+
*
|
|
49
|
+
* @public
|
|
50
|
+
*/
|
|
51
|
+
interface InvalidationAdapter {
|
|
52
|
+
/**
|
|
53
|
+
* Broadcasts one change to every other subscriber.
|
|
54
|
+
*
|
|
55
|
+
* @param event - The partitions that changed, or the statistics that moved.
|
|
56
|
+
*/
|
|
57
|
+
publish(event: InvalidationEvent): Promise<void>;
|
|
58
|
+
/**
|
|
59
|
+
* Starts delivering the events other instances publish.
|
|
60
|
+
*
|
|
61
|
+
* @param handler - Called once per event.
|
|
62
|
+
*/
|
|
63
|
+
subscribe(handler: (event: InvalidationEvent) => void): Promise<void>;
|
|
64
|
+
/** Closes the channel and releases whatever it held open. */
|
|
65
|
+
shutdown(): Promise<void>;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* What one instance tells the others about a change it made.
|
|
69
|
+
*
|
|
70
|
+
* A `partition` event names the partitions a subscriber must reload. A
|
|
71
|
+
* `statistics` event carries the term counts that keep BM25 scores comparable
|
|
72
|
+
* across instances.
|
|
73
|
+
*
|
|
74
|
+
* @public
|
|
75
|
+
*/
|
|
76
|
+
type InvalidationEvent = {
|
|
77
|
+
/** This marks the event as a set of partitions to reload. */
|
|
78
|
+
type: 'partition';
|
|
79
|
+
/** The partitions belong to this index. */
|
|
80
|
+
indexName: string;
|
|
81
|
+
/** These partition ids changed. */
|
|
82
|
+
partitions: number[];
|
|
83
|
+
/** The publisher stamped the event at this many milliseconds since the epoch, which lets a subscriber drop a late one. */
|
|
84
|
+
timestamp: number;
|
|
85
|
+
/** This instance published the event, so a subscriber can skip its own. */
|
|
86
|
+
sourceInstanceId: string;
|
|
87
|
+
} | {
|
|
88
|
+
/** This marks the event as a statistics update. */
|
|
89
|
+
type: 'statistics';
|
|
90
|
+
/** The statistics describe this index. */
|
|
91
|
+
indexName: string;
|
|
92
|
+
/** The statistics were measured on this instance. */
|
|
93
|
+
instanceId: string;
|
|
94
|
+
/** That instance holds these term counts. */
|
|
95
|
+
stats: PartitionStatistics;
|
|
96
|
+
};
|
|
97
|
+
/**
|
|
98
|
+
* The term counts one instance holds, which the others fold into their own so
|
|
99
|
+
* that BM25 scores stay comparable across a shared index.
|
|
100
|
+
*
|
|
101
|
+
* @public
|
|
102
|
+
*/
|
|
103
|
+
interface PartitionStatistics {
|
|
104
|
+
/** This instance holds this many documents for the index. */
|
|
105
|
+
totalDocs: number;
|
|
106
|
+
/** Each term appears in this many documents, keyed by term. */
|
|
107
|
+
docFrequencies: Record<string, number>;
|
|
108
|
+
/** Each field holds this many tokens in total, which gives BM25 its average field length. */
|
|
109
|
+
totalFieldLengths: Record<string, number>;
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Turns text into vectors, so an index embeds documents as they arrive and
|
|
113
|
+
* embeds a query as it runs.
|
|
114
|
+
*
|
|
115
|
+
* Write one against any model you like. The engine calls it on every write to
|
|
116
|
+
* a field the index maps, and on every vector query that passes `text`.
|
|
117
|
+
*
|
|
118
|
+
* @public
|
|
119
|
+
*/
|
|
120
|
+
interface EmbeddingAdapter {
|
|
121
|
+
/**
|
|
122
|
+
* Embeds one string.
|
|
123
|
+
*
|
|
124
|
+
* @param input - The text to embed.
|
|
125
|
+
* @param purpose - `document` on a write and `query` on a search, which lets
|
|
126
|
+
* an asymmetric model pick the right prompt or prefix.
|
|
127
|
+
* @param signal - Aborts a call the engine no longer needs.
|
|
128
|
+
* @returns The vector, whose length must equal
|
|
129
|
+
* {@link EmbeddingAdapter.dimensions}.
|
|
130
|
+
*/
|
|
131
|
+
embed(input: string, purpose: 'document' | 'query', signal?: AbortSignal): Promise<Float32Array>;
|
|
132
|
+
/**
|
|
133
|
+
* Embeds many strings in one call, which a batch load uses when the adapter
|
|
134
|
+
* offers it. The engine falls back to {@link EmbeddingAdapter.embed} per
|
|
135
|
+
* item when it is absent.
|
|
136
|
+
*
|
|
137
|
+
* @param inputs - The texts to embed.
|
|
138
|
+
* @param purpose - `document` on a write and `query` on a search.
|
|
139
|
+
* @param signal - Aborts a call the engine no longer needs.
|
|
140
|
+
* @returns One vector per input, in the same order.
|
|
141
|
+
*/
|
|
142
|
+
embedBatch?(inputs: string[], purpose: 'document' | 'query', signal?: AbortSignal): Promise<Float32Array[]>;
|
|
143
|
+
/** Every vector this adapter returns is this long, which must match the schema's `vector[N]`. */
|
|
144
|
+
readonly dimensions: number;
|
|
145
|
+
/** Releases whatever the adapter holds open, such as a model or a connection pool. */
|
|
146
|
+
shutdown?(): Promise<void>;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export type { EmbeddingAdapter as E, InvalidationAdapter as I, PartitionStatistics as P, InvalidationEvent as a, PersistenceAdapter as b };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { english } from './chunk-TGP5M5R7.mjs';
|
|
2
2
|
import { createVectorStore, createArenaSimd, createHNSWIndex, magnitude, createBoundedMaxHeap, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-S7RJJMHI.mjs';
|
|
3
|
-
import { NarsilError, ErrorCodes, createNarsilError } from './chunk-
|
|
3
|
+
import { NarsilError, ErrorCodes, createNarsilError } from './chunk-UNJZ7ZDB.mjs';
|
|
4
4
|
import { encode, decode } from '@msgpack/msgpack';
|
|
5
5
|
|
|
6
6
|
// src/languages/registry.ts
|
|
@@ -1,6 +1,42 @@
|
|
|
1
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { NarsilError, ErrorCodes } from './chunk-UNJZ7ZDB.mjs';
|
|
2
2
|
import { decode, encode } from '@msgpack/msgpack';
|
|
3
3
|
|
|
4
|
+
// src/distribution/coordinator/etcd/types.ts
|
|
5
|
+
var DEFAULT_ETCD_CONFIG = {
|
|
6
|
+
endpoints: ["http://localhost:2379"],
|
|
7
|
+
keyPrefix: "_narsil",
|
|
8
|
+
nodeHeartbeatTtlSeconds: 30,
|
|
9
|
+
leaseTtlSeconds: 15
|
|
10
|
+
};
|
|
11
|
+
var ETCD_KEY_NODES = "nodes";
|
|
12
|
+
var ETCD_KEY_ALLOCATION = "allocation";
|
|
13
|
+
var ETCD_KEY_PARTITION = "partition";
|
|
14
|
+
var ETCD_KEY_SCHEMA = "schema";
|
|
15
|
+
function buildKey(prefix, ...segments) {
|
|
16
|
+
return [prefix, ...segments].join("/");
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
// src/distribution/coordinator/etcd/keys.ts
|
|
20
|
+
var ETCD_KEY_LEASE = "lease";
|
|
21
|
+
var ETCD_KEY_GENERIC = "kv";
|
|
22
|
+
function coordinatorKeys(config) {
|
|
23
|
+
const prefix = config.keyPrefix;
|
|
24
|
+
return {
|
|
25
|
+
node: (nodeId) => buildKey(prefix, ETCD_KEY_NODES, nodeId),
|
|
26
|
+
nodePrefix: () => buildKey(prefix, ETCD_KEY_NODES),
|
|
27
|
+
allocation: (indexName) => buildKey(prefix, ETCD_KEY_ALLOCATION, indexName),
|
|
28
|
+
allocationPrefix: () => buildKey(prefix, ETCD_KEY_ALLOCATION),
|
|
29
|
+
partition: (indexName, partitionId) => buildKey(prefix, ETCD_KEY_PARTITION, indexName, String(partitionId)),
|
|
30
|
+
schema: (indexName) => buildKey(prefix, ETCD_KEY_SCHEMA, indexName),
|
|
31
|
+
schemaPrefix: () => buildKey(prefix, ETCD_KEY_SCHEMA),
|
|
32
|
+
lease: (key) => buildKey(prefix, ETCD_KEY_LEASE, key),
|
|
33
|
+
generic: (key) => buildKey(prefix, ETCD_KEY_GENERIC, key)
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
function extractSuffix(fullKey, prefix) {
|
|
37
|
+
return fullKey.slice(prefix.length + 1);
|
|
38
|
+
}
|
|
39
|
+
|
|
4
40
|
// src/distribution/coordinator/etcd/leases.ts
|
|
5
41
|
var LeaseManager = class {
|
|
6
42
|
leases = /* @__PURE__ */ new Map();
|
|
@@ -99,21 +135,6 @@ function deserializeSchema(data) {
|
|
|
99
135
|
return decode(data);
|
|
100
136
|
}
|
|
101
137
|
|
|
102
|
-
// src/distribution/coordinator/etcd/types.ts
|
|
103
|
-
var DEFAULT_ETCD_CONFIG = {
|
|
104
|
-
endpoints: ["http://localhost:2379"],
|
|
105
|
-
keyPrefix: "_narsil",
|
|
106
|
-
nodeHeartbeatTtlSeconds: 30,
|
|
107
|
-
leaseTtlSeconds: 15
|
|
108
|
-
};
|
|
109
|
-
var ETCD_KEY_NODES = "nodes";
|
|
110
|
-
var ETCD_KEY_ALLOCATION = "allocation";
|
|
111
|
-
var ETCD_KEY_PARTITION = "partition";
|
|
112
|
-
var ETCD_KEY_SCHEMA = "schema";
|
|
113
|
-
function buildKey(prefix, ...segments) {
|
|
114
|
-
return [prefix, ...segments].join("/");
|
|
115
|
-
}
|
|
116
|
-
|
|
117
138
|
// src/distribution/coordinator/etcd/validation.ts
|
|
118
139
|
var MAX_NODE_ID_LENGTH = 255;
|
|
119
140
|
var VALID_PARTITION_STATES = /* @__PURE__ */ new Set(["UNASSIGNED", "INITIALISING", "ACTIVE", "MIGRATING", "DECOMMISSIONING"]);
|
|
@@ -164,27 +185,7 @@ async function createEtcdCoordinator(config) {
|
|
|
164
185
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Coordinator has been shut down");
|
|
165
186
|
}
|
|
166
187
|
}
|
|
167
|
-
|
|
168
|
-
return buildKey(resolvedConfig.keyPrefix, ETCD_KEY_NODES, nodeId);
|
|
169
|
-
}
|
|
170
|
-
function allocationKey(indexName) {
|
|
171
|
-
return buildKey(resolvedConfig.keyPrefix, ETCD_KEY_ALLOCATION, indexName);
|
|
172
|
-
}
|
|
173
|
-
function partitionKey(indexName, partitionId) {
|
|
174
|
-
return buildKey(resolvedConfig.keyPrefix, ETCD_KEY_PARTITION, indexName, String(partitionId));
|
|
175
|
-
}
|
|
176
|
-
function schemaKey(indexName) {
|
|
177
|
-
return buildKey(resolvedConfig.keyPrefix, ETCD_KEY_SCHEMA, indexName);
|
|
178
|
-
}
|
|
179
|
-
function leaseKey(key) {
|
|
180
|
-
return buildKey(resolvedConfig.keyPrefix, "lease", key);
|
|
181
|
-
}
|
|
182
|
-
function genericKey(key) {
|
|
183
|
-
return buildKey(resolvedConfig.keyPrefix, "kv", key);
|
|
184
|
-
}
|
|
185
|
-
function extractSuffix(fullKey, prefix) {
|
|
186
|
-
return fullKey.slice(prefix.length + 1);
|
|
187
|
-
}
|
|
188
|
+
const keys = coordinatorKeys(resolvedConfig);
|
|
188
189
|
function addWatcher(watcher) {
|
|
189
190
|
if (watchers.size >= MAX_WATCHERS) {
|
|
190
191
|
watcher.cancel().catch(() => {
|
|
@@ -207,7 +208,7 @@ async function createEtcdCoordinator(config) {
|
|
|
207
208
|
const ttl = resolvedConfig.nodeHeartbeatTtlSeconds;
|
|
208
209
|
const lease = client.lease(ttl);
|
|
209
210
|
await lease.grant();
|
|
210
|
-
const key =
|
|
211
|
+
const key = keys.node(registration.nodeId);
|
|
211
212
|
const data = Buffer.from(serializeNodeRegistration(registration));
|
|
212
213
|
await lease.put(key).value(data).exec();
|
|
213
214
|
const existing = leaseManager.get(key);
|
|
@@ -220,7 +221,7 @@ async function createEtcdCoordinator(config) {
|
|
|
220
221
|
async deregisterNode(nodeId) {
|
|
221
222
|
assertNotShutdown();
|
|
222
223
|
validateNodeId(nodeId);
|
|
223
|
-
const key =
|
|
224
|
+
const key = keys.node(nodeId);
|
|
224
225
|
const entry = leaseManager.remove(key);
|
|
225
226
|
if (entry !== void 0) {
|
|
226
227
|
await entry.lease.revoke().catch(() => {
|
|
@@ -230,7 +231,7 @@ async function createEtcdCoordinator(config) {
|
|
|
230
231
|
},
|
|
231
232
|
async listNodes() {
|
|
232
233
|
assertNotShutdown();
|
|
233
|
-
const prefix =
|
|
234
|
+
const prefix = keys.nodePrefix();
|
|
234
235
|
const result = await client.getAll().prefix(prefix).buffers();
|
|
235
236
|
const nodes = [];
|
|
236
237
|
for (const key of Object.keys(result)) {
|
|
@@ -243,7 +244,7 @@ async function createEtcdCoordinator(config) {
|
|
|
243
244
|
},
|
|
244
245
|
async watchNodes(handler) {
|
|
245
246
|
assertNotShutdown();
|
|
246
|
-
const prefix =
|
|
247
|
+
const prefix = keys.nodePrefix();
|
|
247
248
|
const watcher = await client.watch().prefix(prefix).create();
|
|
248
249
|
addWatcher(watcher);
|
|
249
250
|
watcher.on("put", (kv) => {
|
|
@@ -262,7 +263,7 @@ async function createEtcdCoordinator(config) {
|
|
|
262
263
|
},
|
|
263
264
|
async getAllocation(indexName) {
|
|
264
265
|
assertNotShutdown();
|
|
265
|
-
const key =
|
|
266
|
+
const key = keys.allocation(indexName);
|
|
266
267
|
const buf = await client.get(key).buffer();
|
|
267
268
|
if (buf === null) {
|
|
268
269
|
return null;
|
|
@@ -271,7 +272,7 @@ async function createEtcdCoordinator(config) {
|
|
|
271
272
|
},
|
|
272
273
|
async putAllocation(indexName, table, expectedVersion) {
|
|
273
274
|
assertNotShutdown();
|
|
274
|
-
const key =
|
|
275
|
+
const key = keys.allocation(indexName);
|
|
275
276
|
const data = Buffer.from(serializeAllocationTable(table));
|
|
276
277
|
if (expectedVersion === void 0) {
|
|
277
278
|
await client.put(key).value(data).exec();
|
|
@@ -295,7 +296,7 @@ async function createEtcdCoordinator(config) {
|
|
|
295
296
|
},
|
|
296
297
|
async watchAllocation(handler) {
|
|
297
298
|
assertNotShutdown();
|
|
298
|
-
const prefix =
|
|
299
|
+
const prefix = keys.allocationPrefix();
|
|
299
300
|
const watcher = await client.watch().prefix(prefix).create();
|
|
300
301
|
addWatcher(watcher);
|
|
301
302
|
watcher.on("put", (kv) => {
|
|
@@ -309,7 +310,7 @@ async function createEtcdCoordinator(config) {
|
|
|
309
310
|
},
|
|
310
311
|
async getPartitionState(indexName, partitionId) {
|
|
311
312
|
assertNotShutdown();
|
|
312
|
-
const key =
|
|
313
|
+
const key = keys.partition(indexName, partitionId);
|
|
313
314
|
const value = await client.get(key).string();
|
|
314
315
|
if (value === null) {
|
|
315
316
|
return "UNASSIGNED";
|
|
@@ -318,12 +319,12 @@ async function createEtcdCoordinator(config) {
|
|
|
318
319
|
},
|
|
319
320
|
async putPartitionState(indexName, partitionId, state) {
|
|
320
321
|
assertNotShutdown();
|
|
321
|
-
const key =
|
|
322
|
+
const key = keys.partition(indexName, partitionId);
|
|
322
323
|
await client.put(key).value(state).exec();
|
|
323
324
|
},
|
|
324
325
|
async acquireLease(key, nodeId, ttlMs) {
|
|
325
326
|
assertNotShutdown();
|
|
326
|
-
const etcdKey =
|
|
327
|
+
const etcdKey = keys.lease(key);
|
|
327
328
|
const ttlSeconds = Math.max(1, Math.ceil(ttlMs / 1e3));
|
|
328
329
|
const existing = leaseManager.get(etcdKey);
|
|
329
330
|
if (existing !== void 0 && existing.nodeId === nodeId) {
|
|
@@ -348,7 +349,7 @@ async function createEtcdCoordinator(config) {
|
|
|
348
349
|
},
|
|
349
350
|
async renewLease(key, nodeId, _ttlMs) {
|
|
350
351
|
assertNotShutdown();
|
|
351
|
-
const etcdKey =
|
|
352
|
+
const etcdKey = keys.lease(key);
|
|
352
353
|
const entry = leaseManager.getByNodeId(etcdKey, nodeId);
|
|
353
354
|
if (entry === void 0) {
|
|
354
355
|
return false;
|
|
@@ -363,7 +364,7 @@ async function createEtcdCoordinator(config) {
|
|
|
363
364
|
},
|
|
364
365
|
async releaseLease(key) {
|
|
365
366
|
assertNotShutdown();
|
|
366
|
-
const etcdKey =
|
|
367
|
+
const etcdKey = keys.lease(key);
|
|
367
368
|
const entry = leaseManager.remove(etcdKey);
|
|
368
369
|
if (entry !== void 0) {
|
|
369
370
|
await entry.lease.revoke().catch(() => {
|
|
@@ -373,7 +374,7 @@ async function createEtcdCoordinator(config) {
|
|
|
373
374
|
},
|
|
374
375
|
async get(key) {
|
|
375
376
|
assertNotShutdown();
|
|
376
|
-
const etcdKey =
|
|
377
|
+
const etcdKey = keys.generic(key);
|
|
377
378
|
const buf = await client.get(etcdKey).buffer();
|
|
378
379
|
if (buf === null) {
|
|
379
380
|
return null;
|
|
@@ -382,7 +383,7 @@ async function createEtcdCoordinator(config) {
|
|
|
382
383
|
},
|
|
383
384
|
async compareAndSet(key, expected, value) {
|
|
384
385
|
assertNotShutdown();
|
|
385
|
-
const etcdKey =
|
|
386
|
+
const etcdKey = keys.generic(key);
|
|
386
387
|
const newValue = Buffer.from(value);
|
|
387
388
|
if (expected === null) {
|
|
388
389
|
const txn2 = await client.if(etcdKey, "Create", "==", 0).then(client.put(etcdKey).value(newValue)).commit();
|
|
@@ -394,7 +395,7 @@ async function createEtcdCoordinator(config) {
|
|
|
394
395
|
},
|
|
395
396
|
async getSchema(indexName) {
|
|
396
397
|
assertNotShutdown();
|
|
397
|
-
const key =
|
|
398
|
+
const key = keys.schema(indexName);
|
|
398
399
|
const buf = await client.get(key).buffer();
|
|
399
400
|
if (buf === null) {
|
|
400
401
|
return null;
|
|
@@ -403,12 +404,12 @@ async function createEtcdCoordinator(config) {
|
|
|
403
404
|
},
|
|
404
405
|
async putSchema(indexName, schema) {
|
|
405
406
|
assertNotShutdown();
|
|
406
|
-
const key =
|
|
407
|
+
const key = keys.schema(indexName);
|
|
407
408
|
await client.put(key).value(serializeSchema(schema)).exec();
|
|
408
409
|
},
|
|
409
410
|
async watchSchemas(handler) {
|
|
410
411
|
assertNotShutdown();
|
|
411
|
-
const prefix =
|
|
412
|
+
const prefix = keys.schemaPrefix();
|
|
412
413
|
const watcher = await client.watch().prefix(prefix).create();
|
|
413
414
|
addWatcher(watcher);
|
|
414
415
|
watcher.on("put", (kv) => {
|
|
@@ -429,7 +430,7 @@ async function createEtcdCoordinator(config) {
|
|
|
429
430
|
},
|
|
430
431
|
async getLeaseHolder(key) {
|
|
431
432
|
assertNotShutdown();
|
|
432
|
-
const etcdKey =
|
|
433
|
+
const etcdKey = keys.lease(key);
|
|
433
434
|
const value = await client.get(etcdKey).string();
|
|
434
435
|
return value ?? null;
|
|
435
436
|
},
|
|
@@ -90,8 +90,18 @@ var ErrorCodes = {
|
|
|
90
90
|
SNAPSHOT_SYNC_ABORTED: "SNAPSHOT_SYNC_ABORTED"
|
|
91
91
|
};
|
|
92
92
|
var NarsilError = class extends Error {
|
|
93
|
+
/** This says which failure it is. */
|
|
93
94
|
code;
|
|
95
|
+
/** This carries the values behind the failure, such as the field, index, or limit involved. It is empty when the code says everything. */
|
|
94
96
|
details;
|
|
97
|
+
/**
|
|
98
|
+
* Builds an error carrying a code the caller can branch on.
|
|
99
|
+
*
|
|
100
|
+
* @param code - The failure this error reports.
|
|
101
|
+
* @param message - Plain description, for a log or a person.
|
|
102
|
+
* @param details - Values behind the failure, which reach
|
|
103
|
+
* {@link NarsilError.details}.
|
|
104
|
+
*/
|
|
95
105
|
constructor(code, message, details) {
|
|
96
106
|
super(message);
|
|
97
107
|
this.name = "NarsilError";
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { getNestedValue, getLanguage, extractVectorFieldsFromSchema, createDirectExecutor, createPartitionRouter, createPartitionManager, createVectorIndex, validateSchema, validateVectorPromotion, resolveIndexAnalysis, deserializePayloadV2, deserializePayloadV1, tokenize, fanOutQuery, validateRequiredFields, mergePartitionStats, createPartitionIndex, flattenSchema, pruneStatsToQueryTerms, fnv1a, detectRuntime, serializeMetadata, isNodeMainThread, spawnNodeWorker, deserializeMetadata, kWayMerge } from './chunk-
|
|
1
|
+
import { getNestedValue, getLanguage, extractVectorFieldsFromSchema, createDirectExecutor, createPartitionRouter, createPartitionManager, createVectorIndex, validateSchema, validateVectorPromotion, resolveIndexAnalysis, deserializePayloadV2, deserializePayloadV1, tokenize, fanOutQuery, validateRequiredFields, mergePartitionStats, createPartitionIndex, flattenSchema, pruneStatsToQueryTerms, fnv1a, detectRuntime, serializeMetadata, isNodeMainThread, spawnNodeWorker, deserializeMetadata, kWayMerge } from './chunk-ASI6AOW2.mjs';
|
|
2
2
|
import { crc32, crc32Init, crc32Update, crc32Final } from './chunk-KMMJMDUA.mjs';
|
|
3
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
3
|
+
import { ErrorCodes, NarsilError } from './chunk-UNJZ7ZDB.mjs';
|
|
4
4
|
import { generateId, createRequestId, isValidWorkerResponse } from './chunk-BKAI2WXI.mjs';
|
|
5
5
|
import { encode, decode } from '@msgpack/msgpack';
|
|
6
6
|
|
|
@@ -8023,18 +8023,7 @@ function readProcessMemory() {
|
|
|
8023
8023
|
}
|
|
8024
8024
|
}
|
|
8025
8025
|
|
|
8026
|
-
// src/narsil.ts
|
|
8027
|
-
async function createNarsil(config) {
|
|
8028
|
-
const core = createEngineCore(config);
|
|
8029
|
-
if (core.durability) {
|
|
8030
|
-
await core.durability.manager.recover();
|
|
8031
|
-
}
|
|
8032
|
-
if (core.invalidation) {
|
|
8033
|
-
await core.invalidation.start();
|
|
8034
|
-
}
|
|
8035
|
-
await core.analysisRebuild.reviewStaleIndexes();
|
|
8036
|
-
return createNarsilFromCore(core, config);
|
|
8037
|
-
}
|
|
8026
|
+
// src/narsil/operations.ts
|
|
8038
8027
|
function createNarsilFromCore(core, config) {
|
|
8039
8028
|
const {
|
|
8040
8029
|
executor,
|
|
@@ -8336,6 +8325,19 @@ function createNarsilFromCore(core, config) {
|
|
|
8336
8325
|
return narsil;
|
|
8337
8326
|
}
|
|
8338
8327
|
|
|
8328
|
+
// src/narsil/index.ts
|
|
8329
|
+
async function createNarsil(config) {
|
|
8330
|
+
const core = createEngineCore(config);
|
|
8331
|
+
if (core.durability) {
|
|
8332
|
+
await core.durability.manager.recover();
|
|
8333
|
+
}
|
|
8334
|
+
if (core.invalidation) {
|
|
8335
|
+
await core.invalidation.start();
|
|
8336
|
+
}
|
|
8337
|
+
await core.analysisRebuild.reviewStaleIndexes();
|
|
8338
|
+
return createNarsilFromCore(core, config);
|
|
8339
|
+
}
|
|
8340
|
+
|
|
8339
8341
|
// src/index.ts
|
|
8340
8342
|
var VERSION = "0.1.0";
|
|
8341
8343
|
|