@soulcraft/cortex 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +16 -0
- package/README.md +125 -0
- package/dist/graph/NativeGraphAdjacencyIndex.d.ts +92 -0
- package/dist/graph/NativeGraphAdjacencyIndex.js +671 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.js +23 -0
- package/dist/license.d.ts +18 -0
- package/dist/license.js +172 -0
- package/dist/native/NativeEmbeddingEngine.d.ts +79 -0
- package/dist/native/NativeEmbeddingEngine.js +302 -0
- package/dist/native/NativeRoaringBitmap32.d.ts +114 -0
- package/dist/native/NativeRoaringBitmap32.js +221 -0
- package/dist/native/ffi.d.ts +20 -0
- package/dist/native/ffi.js +48 -0
- package/dist/native/index.d.ts +30 -0
- package/dist/native/index.js +58 -0
- package/dist/native/napi.d.ts +21 -0
- package/dist/native/napi.js +88 -0
- package/dist/native/types.d.ts +710 -0
- package/dist/native/types.js +16 -0
- package/dist/plugin.d.ts +22 -0
- package/dist/plugin.js +115 -0
- package/dist/storage/mmapFileSystemStorage.d.ts +24 -0
- package/dist/storage/mmapFileSystemStorage.js +73 -0
- package/dist/utils/NativeMetadataIndex.d.ts +185 -0
- package/dist/utils/NativeMetadataIndex.js +1274 -0
- package/dist/utils/nativeEntityIdMapper.d.ts +84 -0
- package/dist/utils/nativeEntityIdMapper.js +134 -0
- package/native/brainy-native.darwin-arm64.node +0 -0
- package/native/brainy-native.darwin-x64.node +0 -0
- package/native/brainy-native.linux-arm64-gnu.node +0 -0
- package/native/brainy-native.linux-x64-gnu.node +0 -0
- package/native/brainy-native.win32-x64-msvc.node +0 -0
- package/native/index.d.ts +1068 -0
- package/package.json +66 -0
|
@@ -0,0 +1,1068 @@
|
|
|
1
|
+
/* auto-generated by NAPI-RS */
|
|
2
|
+
/* eslint-disable */
|
|
3
|
+
/** Native embedding engine (napi-rs class) */
|
|
4
|
+
export declare class NativeEmbeddingEngine {
|
|
5
|
+
/** Create a new embedding engine instance (not yet loaded). */
|
|
6
|
+
constructor()
|
|
7
|
+
/**
|
|
8
|
+
* Load the model and tokenizer from byte buffers.
|
|
9
|
+
*
|
|
10
|
+
* Arguments:
|
|
11
|
+
* - model_bytes: SafeTensors format model weights
|
|
12
|
+
* - tokenizer_bytes: tokenizer.json contents
|
|
13
|
+
* - config_bytes: config.json contents
|
|
14
|
+
*/
|
|
15
|
+
load(modelBytes: Buffer, tokenizerBytes: Buffer, configBytes: Buffer): void
|
|
16
|
+
/** Check if the engine has a loaded model and tokenizer. */
|
|
17
|
+
isReady(): boolean
|
|
18
|
+
/** Generate embedding for a single text. Returns Vec<f64> of 384 dimensions. */
|
|
19
|
+
embed(text: string): Array<number>
|
|
20
|
+
/**
|
|
21
|
+
* Generate embeddings for multiple texts.
|
|
22
|
+
* Returns Vec<Vec<f64>>.
|
|
23
|
+
*/
|
|
24
|
+
embedBatch(texts: Array<string>): Array<Array<number>>
|
|
25
|
+
/** Get the embedding dimension (384 for all-MiniLM-L6-v2). */
|
|
26
|
+
dimension(): number
|
|
27
|
+
/** Get the maximum sequence length. */
|
|
28
|
+
maxSequenceLength(): number
|
|
29
|
+
/** Get engine statistics. */
|
|
30
|
+
getStats(): EngineStats
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Bidirectional UUID ↔ integer mapper for roaring bitmap integration.
|
|
35
|
+
*
|
|
36
|
+
* All methods are synchronous (O(1) HashMap lookups).
|
|
37
|
+
* Persistence is handled by the TypeScript wrapper via load_from_json/save_to_json.
|
|
38
|
+
*/
|
|
39
|
+
export declare class NativeEntityIdMapper {
|
|
40
|
+
/** Create a new empty mapper */
|
|
41
|
+
constructor()
|
|
42
|
+
/**
|
|
43
|
+
* Load state from JSON string (storage format).
|
|
44
|
+
* Replaces all current state. Sets dirty = false.
|
|
45
|
+
*
|
|
46
|
+
* Expected format:
|
|
47
|
+
* { "noun": "EntityIdMapper", "nextId": N, "uuidToInt": {...}, "intToUuid": {...} }
|
|
48
|
+
*/
|
|
49
|
+
loadFromJson(json: string): void
|
|
50
|
+
/**
|
|
51
|
+
* Serialize state to JSON string for storage persistence.
|
|
52
|
+
* Format matches the TypeScript EntityIdMapper exactly.
|
|
53
|
+
*/
|
|
54
|
+
saveToJson(): string
|
|
55
|
+
/**
|
|
56
|
+
* Get integer ID for UUID, assigning a new one if not mapped.
|
|
57
|
+
* Returns the integer ID (existing or newly assigned).
|
|
58
|
+
*/
|
|
59
|
+
getOrAssign(uuid: string): number
|
|
60
|
+
/** Get UUID for integer ID. Returns null if not found. */
|
|
61
|
+
getUuid(intId: number): string | null
|
|
62
|
+
/** Get integer ID for UUID without assigning. Returns null if not found. */
|
|
63
|
+
getInt(uuid: string): number | null
|
|
64
|
+
/** Check if UUID has been assigned an integer ID. */
|
|
65
|
+
has(uuid: string): boolean
|
|
66
|
+
/** Remove mapping for UUID. Returns true if it existed. */
|
|
67
|
+
remove(uuid: string): boolean
|
|
68
|
+
/** Get total number of mappings. */
|
|
69
|
+
get size(): number
|
|
70
|
+
/** Convert array of UUIDs to array of integers (assigns new IDs as needed). */
|
|
71
|
+
uuidsToInts(uuids: Array<string>): Array<number>
|
|
72
|
+
/** Convert array of integers to array of UUIDs (skips unknown IDs). */
|
|
73
|
+
intsToUuids(ints: Array<number>): Array<string>
|
|
74
|
+
/** Whether there are unsaved changes. */
|
|
75
|
+
isDirty(): boolean
|
|
76
|
+
/** Clear all mappings and reset counter. */
|
|
77
|
+
clear(): void
|
|
78
|
+
/** Get statistics about the mapper. */
|
|
79
|
+
getStats(): NativeEntityIdMapperStats
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export declare class NativeGraphAdjacencyIndex {
|
|
83
|
+
constructor(config: GraphAdjacencyConfig)
|
|
84
|
+
/**
|
|
85
|
+
* Add a verb (relationship) to the graph.
|
|
86
|
+
* Updates all 4 LSM-trees and tracking state in a single call.
|
|
87
|
+
* Returns which trees need flushing.
|
|
88
|
+
*/
|
|
89
|
+
addVerb(verbId: string, sourceId: string, targetId: string, verbType: string): GraphAddVerbResult
|
|
90
|
+
/**
|
|
91
|
+
* Remove a verb from tracking (tombstone deletion).
|
|
92
|
+
* LSM-tree edges persist — removed verbs are filtered out in queries.
|
|
93
|
+
*/
|
|
94
|
+
removeVerb(verbId: string, verbType: string): void
|
|
95
|
+
/**
|
|
96
|
+
* Get neighbors for an entity.
|
|
97
|
+
* direction: "in", "out", or "both"
|
|
98
|
+
*/
|
|
99
|
+
getNeighbors(id: string, direction: string, limit?: number | undefined | null, offset?: number | undefined | null): Array<string>
|
|
100
|
+
/** Get verb IDs by source entity, filtered by verbIdSet (tombstone filtering). */
|
|
101
|
+
getVerbIdsBySource(sourceId: string, limit?: number | undefined | null, offset?: number | undefined | null): Array<string>
|
|
102
|
+
/** Get verb IDs by target entity, filtered by verbIdSet (tombstone filtering). */
|
|
103
|
+
getVerbIdsByTarget(targetId: string, limit?: number | undefined | null, offset?: number | undefined | null): Array<string>
|
|
104
|
+
/**
|
|
105
|
+
* Track a verb ID without adding to LSM-trees.
|
|
106
|
+
* Used during rebuild when LSM-trees already have the data.
|
|
107
|
+
*/
|
|
108
|
+
trackVerbId(verbId: string, verbType: string): void
|
|
109
|
+
/**
|
|
110
|
+
* Clear all verb tracking state (verbIdSet + relationship counts).
|
|
111
|
+
* Used before rebuild to ensure clean state.
|
|
112
|
+
*/
|
|
113
|
+
clearVerbTracking(): void
|
|
114
|
+
/**
|
|
115
|
+
* Flush a specific tree's MemTable.
|
|
116
|
+
* tree_name: "source", "target", "verbs-source", "verbs-target"
|
|
117
|
+
*/
|
|
118
|
+
flushTree(treeName: string): GraphFlushResult | null
|
|
119
|
+
/** Load an SSTable into a specific tree. */
|
|
120
|
+
loadSstable(treeName: string, id: string, level: number, data: Buffer): void
|
|
121
|
+
/** Check if a specific tree's level needs compaction. */
|
|
122
|
+
needsCompaction(treeName: string, level: number): boolean
|
|
123
|
+
/** Compact a specific tree's level. */
|
|
124
|
+
compactTree(treeName: string, level: number): GraphCompactResult | null
|
|
125
|
+
/**
|
|
126
|
+
* Flush a tree's MemTable to binary .sst format (no disk I/O).
|
|
127
|
+
* Returns binary data for the adapter to persist.
|
|
128
|
+
*/
|
|
129
|
+
flushTreeToBinary(treeName: string): NativeBinaryFlushResult | null
|
|
130
|
+
/**
|
|
131
|
+
* Compact a tree's level to binary .sst format (no disk I/O).
|
|
132
|
+
* Returns binary data + old IDs for the adapter to handle.
|
|
133
|
+
*/
|
|
134
|
+
compactTreeToBinary(treeName: string, level: number): NativeBinaryCompactResult | null
|
|
135
|
+
/** Mmap an adapter-written file, replacing InMemory variant with Mapped. */
|
|
136
|
+
registerMmapSstable(treeName: string, sstableId: string, level: number, filePath: string): void
|
|
137
|
+
/** Load an SSTable from binary .sst data (for non-mmap adapters). */
|
|
138
|
+
loadSstableFromBinary(treeName: string, sstableId: string, level: number, data: Buffer): void
|
|
139
|
+
/** Load an SSTable from a .sst file via mmap (adapter provides the path). */
|
|
140
|
+
loadSstableFromFile(treeName: string, sstableId: string, level: number, filePath: string): void
|
|
141
|
+
/** Get manifest JSON for a specific tree. */
|
|
142
|
+
getManifestJson(treeName: string): string
|
|
143
|
+
/** Load manifest JSON for a specific tree. */
|
|
144
|
+
loadManifestJson(treeName: string, json: string): void
|
|
145
|
+
/** Get SSTable IDs from a tree's manifest. */
|
|
146
|
+
getManifestSstableIds(treeName: string): Array<string>
|
|
147
|
+
/** Get the level for an SSTable ID in a tree's manifest. */
|
|
148
|
+
getManifestSstableLevel(treeName: string, id: string): number | null
|
|
149
|
+
/** Check if a specific tree's MemTable is empty. */
|
|
150
|
+
memTableIsEmpty(treeName: string): boolean
|
|
151
|
+
/** Total relationship count from the source tree */
|
|
152
|
+
size(): number
|
|
153
|
+
/** Number of tracked verb IDs */
|
|
154
|
+
verbIdCount(): number
|
|
155
|
+
/** Get relationship count for a specific type */
|
|
156
|
+
getRelationshipCountByType(verbType: string): number
|
|
157
|
+
/** Get all relationship counts as JSON */
|
|
158
|
+
getAllRelationshipCountsJson(): string
|
|
159
|
+
/** Get relationship statistics */
|
|
160
|
+
getRelationshipStatsJson(): string
|
|
161
|
+
/** Whether the index is healthy */
|
|
162
|
+
isHealthy(): boolean
|
|
163
|
+
/** Get comprehensive statistics */
|
|
164
|
+
getStats(): GraphAdjacencyStats
|
|
165
|
+
/** List all tree names (for iteration in TS wrapper) */
|
|
166
|
+
treeNames(): Array<string>
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export declare class NativeHNSWIndex {
|
|
170
|
+
constructor(config: HnswConfig)
|
|
171
|
+
/**
|
|
172
|
+
* Add a single item. Converts f64 -> f32 at boundary.
|
|
173
|
+
* Returns the item ID.
|
|
174
|
+
*/
|
|
175
|
+
addItem(id: string, vector: Array<number>): string
|
|
176
|
+
/** Add item at a specific level (for rebuild from storage). */
|
|
177
|
+
addItemWithLevel(id: string, vector: Array<number>, level: number): string
|
|
178
|
+
/**
|
|
179
|
+
* Add item and return all data needed for storage persistence in one FFI call.
|
|
180
|
+
* Eliminates N+3 FFI round trips (getNodeData for each neighbor + getSystemData).
|
|
181
|
+
*/
|
|
182
|
+
addItemFull(id: string, vector: Array<number>): AddItemResult
|
|
183
|
+
/**
|
|
184
|
+
* Search for k nearest neighbors.
|
|
185
|
+
* Returns sorted results (closest first).
|
|
186
|
+
*/
|
|
187
|
+
search(query: Array<number>, k: number, ef?: number | undefined | null): Array<SearchResult>
|
|
188
|
+
/**
|
|
189
|
+
* Search with candidate restriction (for metadata pre-filtering).
|
|
190
|
+
* Only considers nodes in the candidate set.
|
|
191
|
+
*/
|
|
192
|
+
searchWithCandidates(query: Array<number>, k: number, ef: number, candidateIds?: Array<string> | undefined | null): Array<SearchResult>
|
|
193
|
+
/**
|
|
194
|
+
* Remove an item from the index.
|
|
195
|
+
* Cleans all connections bidirectionally.
|
|
196
|
+
*/
|
|
197
|
+
removeItem(id: string): boolean
|
|
198
|
+
/**
|
|
199
|
+
* Fork: O(n) HashMap clone with Arc<HNSWNode> (each node is O(1) Arc::clone).
|
|
200
|
+
* Subsequent mutations trigger deep copy only for modified nodes.
|
|
201
|
+
*/
|
|
202
|
+
fork(): NativeHNSWIndex
|
|
203
|
+
/** Get node data for storage serialization. */
|
|
204
|
+
getNodeData(id: string): NodeData | null
|
|
205
|
+
/**
|
|
206
|
+
* Restore a node with pre-computed connections (no graph traversal).
|
|
207
|
+
* Used during rebuild from storage.
|
|
208
|
+
*/
|
|
209
|
+
restoreNode(id: string, vector: Array<number>, level: number, connections: Record<string, Array<string>>): void
|
|
210
|
+
/** Get system data (entry point, max level). */
|
|
211
|
+
getSystemData(): SystemData
|
|
212
|
+
/** Set system data (entry point, max level). */
|
|
213
|
+
setSystemData(entryPointId: string | undefined | null, maxLevel: number): void
|
|
214
|
+
/**
|
|
215
|
+
* Add multiple items. HNSW construction is inherently sequential
|
|
216
|
+
* (each new node navigates the existing graph), but internal
|
|
217
|
+
* distance calculations use SIMD.
|
|
218
|
+
*/
|
|
219
|
+
addItemsBatch(ids: Array<string>, vectors: Array<Array<number>>): Array<string>
|
|
220
|
+
/** Number of nodes in the index. */
|
|
221
|
+
size(): number
|
|
222
|
+
/** Clear the index. */
|
|
223
|
+
clear(): void
|
|
224
|
+
/** Get entry point ID. */
|
|
225
|
+
getEntryPointId(): string | null
|
|
226
|
+
/** Get maximum level. */
|
|
227
|
+
getMaxLevel(): number
|
|
228
|
+
/** Get vector dimension. */
|
|
229
|
+
getDimension(): number | null
|
|
230
|
+
/** Check if a node exists. */
|
|
231
|
+
hasNode(id: string): boolean
|
|
232
|
+
/** Get index health metrics. */
|
|
233
|
+
getIndexHealth(): IndexHealth
|
|
234
|
+
/** Get per-level statistics. */
|
|
235
|
+
getLevelStats(): Array<LevelStat>
|
|
236
|
+
/** Get node IDs at a given level. */
|
|
237
|
+
getNodesAtLevel(level: number): Array<string>
|
|
238
|
+
/** Get the vector for a node (returns f64 for JS compatibility). */
|
|
239
|
+
getVector(id: string): Array<number> | null
|
|
240
|
+
/** Get IDs of all nodes in the index. */
|
|
241
|
+
getAllIds(): Array<string>
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
export declare class NativeLSMTree {
|
|
245
|
+
constructor(config: NativeLsmConfig)
|
|
246
|
+
/**
|
|
247
|
+
* Add a source → target relationship to the MemTable.
|
|
248
|
+
* Returns true if the MemTable has reached the flush threshold.
|
|
249
|
+
*/
|
|
250
|
+
add(sourceId: string, targetId: string): boolean
|
|
251
|
+
/**
|
|
252
|
+
* Query targets for a sourceId.
|
|
253
|
+
* Searches MemTable first, then merges results from all SSTable levels.
|
|
254
|
+
* Returns null if sourceId has no targets anywhere.
|
|
255
|
+
*/
|
|
256
|
+
get(sourceId: string): Array<string> | null
|
|
257
|
+
/**
|
|
258
|
+
* Flush the MemTable to an L0 SSTable.
|
|
259
|
+
* Returns the serialized SSTable for the TS wrapper to persist,
|
|
260
|
+
* or null if the MemTable is empty.
|
|
261
|
+
*/
|
|
262
|
+
flushMemTable(): NativeFlushResult | null
|
|
263
|
+
/**
|
|
264
|
+
* Load a previously persisted SSTable into the tree.
|
|
265
|
+
* Called by the TS wrapper during initialization.
|
|
266
|
+
*/
|
|
267
|
+
loadSstable(id: string, level: number, data: Buffer): void
|
|
268
|
+
/** Check if a level needs compaction (has >= maxSSTablesPerLevel) */
|
|
269
|
+
needsCompaction(level: number): boolean
|
|
270
|
+
/**
|
|
271
|
+
* Compact a level by merging all SSTables at that level into one at the next level.
|
|
272
|
+
* Returns the new SSTable data and the list of old IDs to delete, or null if
|
|
273
|
+
* compaction is not needed.
|
|
274
|
+
*/
|
|
275
|
+
compact(level: number): NativeCompactResult | null
|
|
276
|
+
/** Get manifest as JSON for persistence */
|
|
277
|
+
getManifestJson(): string
|
|
278
|
+
/** Load manifest from JSON (called during initialization) */
|
|
279
|
+
loadManifestJson(json: string): void
|
|
280
|
+
/** Get the manifest SSTable ID → level mapping for SSTable loading */
|
|
281
|
+
getManifestSstableIds(): Array<string>
|
|
282
|
+
/** Get the level for a manifest SSTable ID */
|
|
283
|
+
getManifestSstableLevel(id: string): number | null
|
|
284
|
+
/** Number of relationships in the MemTable */
|
|
285
|
+
memTableSize(): number
|
|
286
|
+
/** Whether the MemTable is empty */
|
|
287
|
+
memTableIsEmpty(): boolean
|
|
288
|
+
/** Total relationship count (tracked across all adds) */
|
|
289
|
+
get totalRelationships(): number
|
|
290
|
+
/** Total relationship count (alias for total_relationships) */
|
|
291
|
+
size(): number
|
|
292
|
+
/** Whether the tree is healthy (not in an error state) */
|
|
293
|
+
isHealthy(): boolean
|
|
294
|
+
/** Get statistics about the LSM-tree */
|
|
295
|
+
getStats(): NativeLsmStats
|
|
296
|
+
/** Clear the MemTable (used when the TS wrapper handles its own flushing) */
|
|
297
|
+
clearMemTable(): void
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* NativeMetadataIndex — napi wrapper for the Rust metadata index.
|
|
302
|
+
*
|
|
303
|
+
* All query and mutation operations execute synchronously in Rust.
|
|
304
|
+
* TS wrapper handles async storage I/O (load/save chunks, sparse indices, etc.)
|
|
305
|
+
* and passes data into Rust via load_* methods.
|
|
306
|
+
*/
|
|
307
|
+
export declare class NativeMetadataIndex {
|
|
308
|
+
/** Create a new metadata index with optional configuration. */
|
|
309
|
+
constructor(configJson?: string | undefined | null)
|
|
310
|
+
/** Load entity ID mapper state from JSON string. */
|
|
311
|
+
loadEntityIdMapper(json: string): void
|
|
312
|
+
/** Serialize entity ID mapper state to JSON string. */
|
|
313
|
+
saveEntityIdMapper(): string
|
|
314
|
+
/** Whether the entity ID mapper has unsaved changes. */
|
|
315
|
+
isEntityIdMapperDirty(): boolean
|
|
316
|
+
/** Get entity ID mapper size. */
|
|
317
|
+
entityIdMapperSize(): number
|
|
318
|
+
/** Load a field's sparse index from JSON string. */
|
|
319
|
+
loadSparseIndex(field: string, json: string): void
|
|
320
|
+
/** Save a field's sparse index to JSON string. Returns None if field not loaded. */
|
|
321
|
+
saveSparseIndex(field: string): string | null
|
|
322
|
+
/** Get chunk IDs from a field's sparse index. */
|
|
323
|
+
getSparseIndexChunkIds(field: string): Array<number>
|
|
324
|
+
/** Check if a field's sparse index is loaded. */
|
|
325
|
+
isFieldLoaded(field: string): boolean
|
|
326
|
+
/** Get all loaded field names. */
|
|
327
|
+
getLoadedFields(): Array<string>
|
|
328
|
+
/** Load a chunk from JSON string. */
|
|
329
|
+
loadChunk(field: string, chunkId: number, json: string): void
|
|
330
|
+
/** Save a chunk to JSON string. Returns None if chunk not loaded. */
|
|
331
|
+
saveChunk(field: string, chunkId: number): string | null
|
|
332
|
+
/** Check if a chunk is loaded. */
|
|
333
|
+
isChunkLoaded(field: string, chunkId: number): boolean
|
|
334
|
+
/** Unload a chunk to free memory. */
|
|
335
|
+
unloadChunk(field: string, chunkId: number): void
|
|
336
|
+
/** Unload a field's sparse index and all its chunks to free memory. */
|
|
337
|
+
unloadField(field: string): void
|
|
338
|
+
/** Load field registry from JSON string. */
|
|
339
|
+
loadFieldRegistry(json: string): void
|
|
340
|
+
/** Save field registry to JSON string. */
|
|
341
|
+
saveFieldRegistry(): string
|
|
342
|
+
/** Load a field index from JSON string. */
|
|
343
|
+
loadFieldIndex(field: string, json: string): void
|
|
344
|
+
/** Save a field index to JSON string. */
|
|
345
|
+
saveFieldIndex(field: string): string | null
|
|
346
|
+
/** Get IDs for a single field-value pair. */
|
|
347
|
+
getIds(field: string, valueJson: string): Array<string>
|
|
348
|
+
/**
|
|
349
|
+
* Get IDs for a filter expression.
|
|
350
|
+
* all_ids_json: optional JSON array of all entity IDs (needed for ne/exists:false).
|
|
351
|
+
*/
|
|
352
|
+
getIdsForFilter(filterJson: string, allIdsJson?: string | undefined | null): Array<string>
|
|
353
|
+
/** Get IDs for multiple field-value pairs (AND intersection). */
|
|
354
|
+
getIdsForMultipleFields(pairsJson: string): Array<string>
|
|
355
|
+
/** Get IDs for a range query on a field. */
|
|
356
|
+
getIdsForRange(field: string, minJson: string | undefined | null, maxJson: string | undefined | null, includeMin: boolean, includeMax: boolean): Array<string>
|
|
357
|
+
/** Get IDs for a text query. Returns JSON array of {id, matchCount}. */
|
|
358
|
+
getIdsForTextQuery(queryStr: string): string
|
|
359
|
+
/** Get all unique values for a field (for filter discovery). */
|
|
360
|
+
getFilterValues(field: string): Array<string>
|
|
361
|
+
/** Get all indexed field names (for filter discovery). */
|
|
362
|
+
getFilterFields(): Array<string>
|
|
363
|
+
/** Extract field names from entity JSON (for lazy loading). */
|
|
364
|
+
extractFieldNames(entityJson: string): Array<string>
|
|
365
|
+
/** Add an entity to the index. Returns MutationResult JSON. */
|
|
366
|
+
addToIndex(id: string, entityJson: string): string
|
|
367
|
+
/** Remove an entity from the index. Returns MutationResult JSON. */
|
|
368
|
+
removeFromIndex(id: string, metadataJson?: string | undefined | null): string
|
|
369
|
+
/** Get entity count by type name. */
|
|
370
|
+
getEntityCountByType(typeName: string): number
|
|
371
|
+
/** Get total entity count. */
|
|
372
|
+
getTotalEntityCount(): number
|
|
373
|
+
/** Get top N noun types by count. Returns JSON. */
|
|
374
|
+
getTopNounTypes(n: number): string
|
|
375
|
+
/** Get top N verb types by count. Returns JSON. */
|
|
376
|
+
getTopVerbTypes(n: number): string
|
|
377
|
+
/** Get comprehensive statistics. Returns JSON. */
|
|
378
|
+
getStats(): string
|
|
379
|
+
/** Clear all in-memory state. */
|
|
380
|
+
clear(): void
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/**
|
|
384
|
+
* Memory-mapped vector store for zero-copy reads.
|
|
385
|
+
*
|
|
386
|
+
* Stores float32 vectors in a flat file with fixed-size records.
|
|
387
|
+
* OS page cache manages hot/cold vectors transparently.
|
|
388
|
+
*/
|
|
389
|
+
export declare class NativeMmapVectorStore {
|
|
390
|
+
/** Use factory methods: create(), open(), or openReadOnly(). */
|
|
391
|
+
constructor()
|
|
392
|
+
/** Create a new vector file with pre-allocated capacity. */
|
|
393
|
+
static create(path: string, dim: number, capacity: number): NativeMmapVectorStore
|
|
394
|
+
/** Open an existing vector file for reading and writing. */
|
|
395
|
+
static open(path: string): NativeMmapVectorStore
|
|
396
|
+
/** Open an existing vector file for read-only access. */
|
|
397
|
+
static openReadOnly(path: string): NativeMmapVectorStore
|
|
398
|
+
/**
|
|
399
|
+
* Write a single vector at the given index.
|
|
400
|
+
* Vector is provided as f64[] (JS number[]) and stored as f32.
|
|
401
|
+
*/
|
|
402
|
+
writeVector(index: number, vector: Array<number>): void
|
|
403
|
+
/**
|
|
404
|
+
* Write multiple vectors starting at start_index.
|
|
405
|
+
* vectors_flat is a flat f64 array of length n × dim.
|
|
406
|
+
* Returns the number of vectors written.
|
|
407
|
+
*/
|
|
408
|
+
writeVectorsBatch(startIndex: number, vectorsFlat: Array<number>): number
|
|
409
|
+
/** Read a single vector at the given index. Returns f64[]. */
|
|
410
|
+
readVector(index: number): Array<number>
|
|
411
|
+
/** Read multiple vectors by index. Returns flat f64 array of length n × dim. */
|
|
412
|
+
readVectorsBatch(indices: Array<number>): Array<number>
|
|
413
|
+
/**
|
|
414
|
+
* Prefetch vectors at given indices into OS page cache.
|
|
415
|
+
* Uses madvise(MADV_WILLNEED) on Linux/macOS for async readahead.
|
|
416
|
+
* No-op on unsupported platforms.
|
|
417
|
+
*/
|
|
418
|
+
prefetch(indices: Array<number>): void
|
|
419
|
+
/** Flush all pending writes to disk (msync). */
|
|
420
|
+
flush(): void
|
|
421
|
+
/**
|
|
422
|
+
* Grow the vector file to accommodate more vectors.
|
|
423
|
+
* Re-maps the file after growing.
|
|
424
|
+
*/
|
|
425
|
+
resize(newCapacity: number): void
|
|
426
|
+
/**
|
|
427
|
+
* Rewrite vectors in the specified order for improved disk locality.
|
|
428
|
+
* new_order[i] = old_index means new position i gets vector from old_index.
|
|
429
|
+
* Creates a temp file, writes in order, atomically replaces the original.
|
|
430
|
+
* Returns the new vector count.
|
|
431
|
+
*/
|
|
432
|
+
compact(newOrder: Array<number>): number
|
|
433
|
+
/** Number of vectors stored (highest written index + 1). */
|
|
434
|
+
get count(): number
|
|
435
|
+
/** Maximum vector capacity before resize is needed. */
|
|
436
|
+
get capacity(): number
|
|
437
|
+
/** Vector dimensionality. */
|
|
438
|
+
get dim(): number
|
|
439
|
+
/** File path. */
|
|
440
|
+
get path(): string
|
|
441
|
+
/** Size of a single vector record in bytes (dim × 4). */
|
|
442
|
+
get recordSizeBytes(): number
|
|
443
|
+
/** Total file size in bytes. */
|
|
444
|
+
get fileSizeBytes(): number
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
/**
|
|
448
|
+
* A trained PQ codebook.
|
|
449
|
+
*
|
|
450
|
+
* Contains M×ksub centroids, each of dimension dsub = dim/M.
|
|
451
|
+
* Stored as a flat f32 array: centroids[m][k][d] = data[m * ksub * dsub + k * dsub + d]
|
|
452
|
+
*/
|
|
453
|
+
export declare class NativePqCodebook {
|
|
454
|
+
/** Create a PQ codebook. Use `NativePQCodebook.train()` or `NativePQCodebook.deserialize()` instead. */
|
|
455
|
+
constructor()
|
|
456
|
+
/**
|
|
457
|
+
* Train a PQ codebook from a set of training vectors.
|
|
458
|
+
*
|
|
459
|
+
* Parameters:
|
|
460
|
+
* - vectors: training data (flat array, each vector is `dim` f64 values)
|
|
461
|
+
* - dim: vector dimension
|
|
462
|
+
* - m: number of subspaces (must divide dim evenly)
|
|
463
|
+
* - ksub: centroids per subspace (typically 256)
|
|
464
|
+
* - iterations: k-means iterations (typically 20-50)
|
|
465
|
+
*/
|
|
466
|
+
static train(vectors: Array<number>, dim: number, m: number, ksub: number, iterations: number): NativePqCodebook
|
|
467
|
+
/**
|
|
468
|
+
* Encode a single vector to a PQ code (M bytes).
|
|
469
|
+
*
|
|
470
|
+
* Each byte is the index of the nearest centroid in the corresponding subspace.
|
|
471
|
+
*/
|
|
472
|
+
encode(vector: Array<number>): Buffer
|
|
473
|
+
/**
|
|
474
|
+
* Encode multiple vectors to PQ codes.
|
|
475
|
+
* Returns flat buffer: n * m bytes.
|
|
476
|
+
*/
|
|
477
|
+
encodeBatch(vectors: Array<number>, dim: number): Buffer
|
|
478
|
+
/** Decode a PQ code back to an approximate vector. */
|
|
479
|
+
decode(code: Buffer): Array<number>
|
|
480
|
+
/**
|
|
481
|
+
* Precompute the ADC (Asymmetric Distance Computation) lookup table.
|
|
482
|
+
*
|
|
483
|
+
* Returns a flat f64 array of m * ksub distances.
|
|
484
|
+
* table[sub_idx * ksub + k] = squared distance from query subvector to centroid k.
|
|
485
|
+
*/
|
|
486
|
+
computeAdcTable(query: Array<number>): Array<number>
|
|
487
|
+
/**
|
|
488
|
+
* Compute approximate distance from a precomputed ADC table to a single PQ code.
|
|
489
|
+
*
|
|
490
|
+
* Returns squared Euclidean distance (sum of M table lookups).
|
|
491
|
+
*/
|
|
492
|
+
distanceAdc(table: Array<number>, code: Buffer): number
|
|
493
|
+
/**
|
|
494
|
+
* Batch compute ADC distances from a precomputed table to multiple PQ codes.
|
|
495
|
+
*
|
|
496
|
+
* codes: flat buffer of n * m bytes.
|
|
497
|
+
* Returns n distances.
|
|
498
|
+
*/
|
|
499
|
+
distanceAdcBatch(table: Array<number>, codes: Buffer, n: number): Array<number>
|
|
500
|
+
/**
|
|
501
|
+
* Serialize the codebook to a binary format for persistence.
|
|
502
|
+
*
|
|
503
|
+
* Format: [m:u32][ksub:u32][dsub:u32][dim:u32][centroids:f32×(m×ksub×dsub)]
|
|
504
|
+
*/
|
|
505
|
+
serialize(): Buffer
|
|
506
|
+
/** Deserialize a codebook from binary format. */
|
|
507
|
+
static deserialize(data: Buffer): NativePqCodebook
|
|
508
|
+
/** Get codebook parameters. */
|
|
509
|
+
get params(): PQParams
|
|
510
|
+
}
|
|
511
|
+
export type NativePQCodebook = NativePqCodebook
|
|
512
|
+
|
|
513
|
+
/** Native roaring bitmap (drop-in replacement for roaring-wasm RoaringBitmap32). */
|
|
514
|
+
export declare class NativeRoaringBitmap32 {
|
|
515
|
+
/** Create a new empty bitmap, or from an array of u32 values. */
|
|
516
|
+
constructor(values?: Array<number> | undefined | null)
|
|
517
|
+
/** Number of values in the bitmap. */
|
|
518
|
+
get size(): number
|
|
519
|
+
/** Whether the bitmap is empty. */
|
|
520
|
+
get isEmpty(): boolean
|
|
521
|
+
/** Add a value to the bitmap. */
|
|
522
|
+
add(value: number): void
|
|
523
|
+
/** Remove a value from the bitmap (called `delete` in roaring-wasm API). */
|
|
524
|
+
delete(value: number): void
|
|
525
|
+
/** Add a value, return true if it was newly added (false if already present). */
|
|
526
|
+
tryAdd(value: number): boolean
|
|
527
|
+
/** Check if the bitmap contains a value. */
|
|
528
|
+
has(value: number): boolean
|
|
529
|
+
/** Convert bitmap to an array of u32 values. */
|
|
530
|
+
toArray(): Array<number>
|
|
531
|
+
/** Serialize to portable format (binary-identical to roaring-wasm portable format). */
|
|
532
|
+
serialize(): Buffer
|
|
533
|
+
/** Get serialization size in bytes (portable format). */
|
|
534
|
+
serializationSizeInBytes(): number
|
|
535
|
+
/** Deserialize from portable format buffer. */
|
|
536
|
+
static deserialize(buffer: Buffer): NativeRoaringBitmap32
|
|
537
|
+
/** Minimum value in the bitmap. Returns undefined if empty. */
|
|
538
|
+
minimum(): number | null
|
|
539
|
+
/** Maximum value in the bitmap. Returns undefined if empty. */
|
|
540
|
+
maximum(): number | null
|
|
541
|
+
/** Compute intersection (AND) of two bitmaps. */
|
|
542
|
+
static and(a: NativeRoaringBitmap32, b: NativeRoaringBitmap32): NativeRoaringBitmap32
|
|
543
|
+
/** Compute union (OR) of two bitmaps. */
|
|
544
|
+
static or(a: NativeRoaringBitmap32, b: NativeRoaringBitmap32): NativeRoaringBitmap32
|
|
545
|
+
/** Compute union (OR) of multiple bitmaps. */
|
|
546
|
+
static orMany(bitmaps: Array<NativeRoaringBitmap32>): NativeRoaringBitmap32
|
|
547
|
+
/** Compute difference (AND NOT) of two bitmaps. */
|
|
548
|
+
static andNot(a: NativeRoaringBitmap32, b: NativeRoaringBitmap32): NativeRoaringBitmap32
|
|
549
|
+
/** Compute symmetric difference (XOR) of two bitmaps. */
|
|
550
|
+
static xor(a: NativeRoaringBitmap32, b: NativeRoaringBitmap32): NativeRoaringBitmap32
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
/**
|
|
554
|
+
* Native unified cache engine.
|
|
555
|
+
* Manages eviction decisions and metadata; TS wrapper holds actual JS data.
|
|
556
|
+
*/
|
|
557
|
+
export declare class NativeUnifiedCache {
|
|
558
|
+
/** Create a new cache with the given maximum size in bytes. */
|
|
559
|
+
constructor(maxSize: number)
|
|
560
|
+
/** Record a cache hit — increment access count for the key. */
|
|
561
|
+
recordAccess(key: string): void
|
|
562
|
+
/**
|
|
563
|
+
* Insert a cache entry. Returns keys that were evicted to make room.
|
|
564
|
+
* cache_type: 0=hnsw, 1=metadata, 2=embedding, 3=other
|
|
565
|
+
*/
|
|
566
|
+
insert(key: string, cacheType: number, size: number, rebuildCost: number): NativeEvictionResult
|
|
567
|
+
/** Remove a specific key. Returns freed bytes. */
|
|
568
|
+
remove(key: string): number
|
|
569
|
+
/** Remove all entries matching a key prefix. Returns list of removed keys. */
|
|
570
|
+
removeByPrefix(prefix: string): Array<string>
|
|
571
|
+
/**
|
|
572
|
+
* Clear all entries or entries of a specific type. Returns removed keys.
|
|
573
|
+
* Pass null/undefined to clear all.
|
|
574
|
+
*/
|
|
575
|
+
clear(cacheType?: number | undefined | null): Array<string>
|
|
576
|
+
/** Evict entries to free at least `bytes_needed` bytes. */
|
|
577
|
+
evictForSize(bytesNeeded: number): NativeEvictionResult
|
|
578
|
+
/**
|
|
579
|
+
* Fairness check: detect types using >70% cache with <15% access.
|
|
580
|
+
* Returns keys evicted during fairness correction.
|
|
581
|
+
*/
|
|
582
|
+
checkFairness(): NativeEvictionResult
|
|
583
|
+
/** Report native subsystem memory usage (called periodically by TS). */
|
|
584
|
+
reportNativeMemory(hnsw: number, metadata: number, graph: number): void
|
|
585
|
+
/** Get comprehensive statistics. */
|
|
586
|
+
getStats(): NativeCacheStats
|
|
587
|
+
/** Check if a key exists in the cache metadata. */
|
|
588
|
+
has(key: string): boolean
|
|
589
|
+
/** Number of entries in the cache. */
|
|
590
|
+
itemCount(): number
|
|
591
|
+
/** Save access patterns as JSON for warm start persistence. */
|
|
592
|
+
saveAccessPatterns(): string
|
|
593
|
+
/** Load access patterns from JSON for warm start. */
|
|
594
|
+
loadAccessPatterns(json: string): void
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
/**
|
|
598
|
+
* Full result from add_item including all persistence data.
|
|
599
|
+
* Returns everything TS needs to persist in a single FFI call,
|
|
600
|
+
* eliminating N+3 FFI round trips per add.
|
|
601
|
+
*/
|
|
602
|
+
export interface AddItemResult {
|
|
603
|
+
id: string
|
|
604
|
+
nodeData: NodeData
|
|
605
|
+
modifiedNeighbors: Array<NeighborPersistData>
|
|
606
|
+
systemData: SystemData
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
/**
|
|
610
|
+
* Batch read vectors from a .vec file without mmap.
|
|
611
|
+
*
|
|
612
|
+
* Returns a flat f64 array of length indices.len() × dim.
|
|
613
|
+
* Vectors are returned in the same order as the input indices.
|
|
614
|
+
* Indices are sorted internally for sequential disk access.
|
|
615
|
+
*/
|
|
616
|
+
export declare function batchReadVectorsDirect(path: string, indices: Array<number>): Array<number>
|
|
617
|
+
|
|
618
|
+
/**
|
|
619
|
+
* Compute BFS traversal order from an HNSW entry point.
|
|
620
|
+
*
|
|
621
|
+
* Returns an array where result[i] is the node index that should be placed
|
|
622
|
+
* at position i in the rewritten vector file. Nodes visited together during
|
|
623
|
+
* HNSW search will be adjacent on disk.
|
|
624
|
+
*
|
|
625
|
+
* Parameters:
|
|
626
|
+
* - entry_point: starting node index for BFS
|
|
627
|
+
* - adjacency_offsets: CSR offsets array (length = total_nodes + 1)
|
|
628
|
+
* - adjacency_neighbors: CSR flat neighbor array
|
|
629
|
+
*
|
|
630
|
+
* Any nodes not reachable from the entry point (disconnected components)
|
|
631
|
+
* are appended at the end in index order.
|
|
632
|
+
*/
|
|
633
|
+
export declare function computeBfsOrder(entryPoint: number, adjacencyOffsets: Array<number>, adjacencyNeighbors: Array<number>): Array<number>
|
|
634
|
+
|
|
635
|
+
/**
|
|
636
|
+
* Compute multi-level BFS order optimized for HNSW structure.
|
|
637
|
+
*
|
|
638
|
+
* Starts BFS from the entry point at the highest HNSW level, then continues
|
|
639
|
+
* from each discovered node at lower levels. This produces an ordering that
|
|
640
|
+
* follows the HNSW search traversal pattern more closely than a single-level BFS.
|
|
641
|
+
*
|
|
642
|
+
* Parameters:
|
|
643
|
+
* - entry_point: HNSW entry point node index
|
|
644
|
+
* - level_offsets: for each level, the CSR offsets array
|
|
645
|
+
* - level_neighbors: for each level, the CSR neighbors array
|
|
646
|
+
* - total_nodes: total number of nodes across all levels
|
|
647
|
+
*
|
|
648
|
+
* This is a simplified version — for most datasets, compute_bfs_order on
|
|
649
|
+
* the level-0 graph produces excellent results since level-0 has all nodes.
|
|
650
|
+
*/
|
|
651
|
+
export declare function computeHnswTraversalOrder(entryPoint: number, adjacencyOffsets: Array<number>, adjacencyNeighbors: Array<number>, neighborOfNeighborDepth?: number | undefined | null): Array<number>
|
|
652
|
+
|
|
653
|
+
/**
|
|
654
|
+
* Cosine distance between two vectors.
|
|
655
|
+
* Range: 0 (identical) to 2 (opposite).
|
|
656
|
+
* Lower values indicate higher similarity.
|
|
657
|
+
*/
|
|
658
|
+
export declare function cosineDistance(a: Array<number>, b: Array<number>): number
|
|
659
|
+
|
|
660
|
+
/**
|
|
661
|
+
* Batch cosine distance: compute distances from a query to multiple vectors.
|
|
662
|
+
* Uses Rayon parallelism for large batches.
|
|
663
|
+
* Returns array of distances.
|
|
664
|
+
*/
|
|
665
|
+
export declare function cosineDistanceBatch(query: Array<number>, vectors: Array<Array<number>>): Array<number>
|
|
666
|
+
|
|
667
|
+
/**
|
|
668
|
+
* Compute approximate cosine distance between two SQ4-quantized vectors.
|
|
669
|
+
*
|
|
670
|
+
* Unpacks nibbles to u8 values, then uses the same integer accumulation
|
|
671
|
+
* approach as SQ8 but with 4-bit precision.
|
|
672
|
+
*/
|
|
673
|
+
export declare function cosineDistanceSq4(a: Buffer, aMin: number, aMax: number, aDim: number, b: Buffer, bMin: number, bMax: number, bDim: number): number
|
|
674
|
+
|
|
675
|
+
/** Batch cosine distance from an SQ4 query to multiple SQ4 vectors. */
|
|
676
|
+
export declare function cosineDistanceSq4Batch(query: Buffer, queryMin: number, queryMax: number, queryDim: number, vectors: Array<Buffer>, mins: Array<number>, maxs: Array<number>, dims: Array<number>): Array<number>
|
|
677
|
+
|
|
678
|
+
/**
|
|
679
|
+
* Compute approximate cosine distance between two SQ8-quantized vectors.
|
|
680
|
+
*
|
|
681
|
+
* Operates directly on uint8 values using integer arithmetic in the inner loop.
|
|
682
|
+
* Avoids full dequantization by reconstructing the dot product and norms from
|
|
683
|
+
* integer sums and the per-vector scale factors.
|
|
684
|
+
*
|
|
685
|
+
* The accumulation loop auto-vectorizes with AVX2/NEON.
|
|
686
|
+
*/
|
|
687
|
+
export declare function cosineDistanceSq8(a: Buffer, aMin: number, aMax: number, b: Buffer, bMin: number, bMax: number): number
|
|
688
|
+
|
|
689
|
+
/**
|
|
690
|
+
* Batch cosine distance from an SQ8 query to multiple SQ8 vectors.
|
|
691
|
+
* Uses Rayon parallelism for large batches.
|
|
692
|
+
*/
|
|
693
|
+
export declare function cosineDistanceSq8Batch(query: Buffer, queryMin: number, queryMax: number, vectors: Array<Buffer>, mins: Array<number>, maxs: Array<number>): Array<number>
|
|
694
|
+
|
|
695
|
+
/** Calculate cosine similarity between two embedding vectors. */
|
|
696
|
+
export declare function cosineSimilarity(a: Array<number>, b: Array<number>): number
|
|
697
|
+
|
|
698
|
+
/** Decode delta-varint compressed bytes back to connection list. */
|
|
699
|
+
export declare function decodeConnections(data: Buffer): Array<number>
|
|
700
|
+
|
|
701
|
+
/** Dequantize SQ4 packed bytes back to float64 vector. */
|
|
702
|
+
export declare function dequantizeSq4(quantized: Buffer, min: number, max: number, dim: number): Array<number>
|
|
703
|
+
|
|
704
|
+
/** Dequantize SQ8 bytes back to float64 vector. */
|
|
705
|
+
export declare function dequantizeSq8(quantized: Buffer, min: number, max: number): Array<number>
|
|
706
|
+
|
|
707
|
+
/** Deserialize SQ8 data from compact binary format. */
|
|
708
|
+
export declare function deserializeSq8(data: Buffer): Sq8QuantizeResult
|
|
709
|
+
|
|
710
|
+
/** Device information returned to JavaScript */
|
|
711
|
+
export interface DeviceInfo {
|
|
712
|
+
/** Active device name: "cpu", "cuda", or "metal" */
|
|
713
|
+
device: string
|
|
714
|
+
/** Whether CUDA is available (compiled with cuda feature and GPU detected) */
|
|
715
|
+
cudaAvailable: boolean
|
|
716
|
+
/** Whether Metal is available (compiled with metal feature and GPU detected) */
|
|
717
|
+
metalAvailable: boolean
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
/**
|
|
721
|
+
* Dot product distance between two vectors.
|
|
722
|
+
* Converted to distance metric (lower is better): returns -dot_product.
|
|
723
|
+
*/
|
|
724
|
+
export declare function dotProductDistance(a: Array<number>, b: Array<number>): number
|
|
725
|
+
|
|
726
|
+
/**
|
|
727
|
+
* Encode a connection list (neighbor IDs) as delta-varint compressed bytes.
|
|
728
|
+
*
|
|
729
|
+
* 1. Sort the IDs
|
|
730
|
+
* 2. Compute deltas between consecutive values
|
|
731
|
+
* 3. Encode each delta as a variable-length integer (LEB128)
|
|
732
|
+
*
|
|
733
|
+
* Achieves 2-3x compression for typical HNSW neighbor lists.
|
|
734
|
+
*/
|
|
735
|
+
export declare function encodeConnections(ids: Array<number>): Buffer
|
|
736
|
+
|
|
737
|
+
/** Engine statistics returned to JavaScript. */
|
|
738
|
+
export interface EngineStats {
|
|
739
|
+
initialized: boolean
|
|
740
|
+
embedCount: number
|
|
741
|
+
totalProcessingTimeMs: number
|
|
742
|
+
avgProcessingTimeMs: number
|
|
743
|
+
modelName: string
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
/**
|
|
747
|
+
* Euclidean distance between two vectors.
|
|
748
|
+
* Lower values indicate higher similarity.
|
|
749
|
+
*/
|
|
750
|
+
export declare function euclideanDistance(a: Array<number>, b: Array<number>): number
|
|
751
|
+
|
|
752
|
+
/** Batch euclidean distance: compute distances from a query to multiple vectors. */
|
|
753
|
+
export declare function euclideanDistanceBatch(query: Array<number>, vectors: Array<Array<number>>): Array<number>
|
|
754
|
+
|
|
755
|
+
/** Get information about the active compute device. */
|
|
756
|
+
export declare function getDeviceInfo(): DeviceInfo
|
|
757
|
+
|
|
758
|
+
/** Result from addVerb — indicates which trees need flushing */
|
|
759
|
+
export interface GraphAddVerbResult {
|
|
760
|
+
/** Whether the source tree MemTable needs flush */
|
|
761
|
+
needsFlushSource: boolean
|
|
762
|
+
/** Whether the target tree MemTable needs flush */
|
|
763
|
+
needsFlushTarget: boolean
|
|
764
|
+
/** Whether the verbs-source tree MemTable needs flush */
|
|
765
|
+
needsFlushVerbsSource: boolean
|
|
766
|
+
/** Whether the verbs-target tree MemTable needs flush */
|
|
767
|
+
needsFlushVerbsTarget: boolean
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
/** Configuration for NativeGraphAdjacencyIndex */
|
|
771
|
+
export interface GraphAdjacencyConfig {
|
|
772
|
+
/** MemTable flush threshold per tree. Default: 100_000 */
|
|
773
|
+
memTableThreshold?: number
|
|
774
|
+
/** Max SSTables per level per tree. Default: 10 */
|
|
775
|
+
maxSstablesPerLevel?: number
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
/** Statistics for the graph adjacency index */
|
|
779
|
+
export interface GraphAdjacencyStats {
|
|
780
|
+
/** Total relationships in source tree */
|
|
781
|
+
totalRelationships: number
|
|
782
|
+
/** Total verb IDs tracked */
|
|
783
|
+
verbIdCount: number
|
|
784
|
+
/** Number of relationship types */
|
|
785
|
+
relationshipTypeCount: number
|
|
786
|
+
/** Source tree MemTable size */
|
|
787
|
+
sourceMemTableSize: number
|
|
788
|
+
/** Target tree MemTable size */
|
|
789
|
+
targetMemTableSize: number
|
|
790
|
+
/** Source tree SSTable count */
|
|
791
|
+
sourceSstableCount: number
|
|
792
|
+
/** Target tree SSTable count */
|
|
793
|
+
targetSstableCount: number
|
|
794
|
+
/** Source tree MemTable memory estimate */
|
|
795
|
+
sourceMemTableMemory: number
|
|
796
|
+
/** Target tree MemTable memory estimate */
|
|
797
|
+
targetMemTableMemory: number
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
/** Result from compacting a specific tree */
|
|
801
|
+
export interface GraphCompactResult {
|
|
802
|
+
/** Which tree was compacted */
|
|
803
|
+
treeName: string
|
|
804
|
+
/** New merged SSTable ID */
|
|
805
|
+
newSstableId: string
|
|
806
|
+
/** Serialized merged SSTable data */
|
|
807
|
+
newData: Buffer
|
|
808
|
+
/** Target level */
|
|
809
|
+
newLevel: number
|
|
810
|
+
/** Old SSTable IDs to delete */
|
|
811
|
+
oldIds: Array<string>
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
/** Result from flushing a specific tree's MemTable */
|
|
815
|
+
export interface GraphFlushResult {
|
|
816
|
+
/** Which tree was flushed */
|
|
817
|
+
treeName: string
|
|
818
|
+
/** SSTable ID */
|
|
819
|
+
sstableId: string
|
|
820
|
+
/** Serialized SSTable data (msgpack binary) */
|
|
821
|
+
data: Buffer
|
|
822
|
+
/** Compaction level (always 0 for flush) */
|
|
823
|
+
level: number
|
|
824
|
+
/** Number of unique entries */
|
|
825
|
+
entryCount: number
|
|
826
|
+
/** Total relationship count */
|
|
827
|
+
relationshipCount: number
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
/** HNSW configuration (mirrors TS HNSWConfig) */
|
|
831
|
+
export interface HnswConfig {
|
|
832
|
+
/** Max connections per node (default: 16) */
|
|
833
|
+
m: number
|
|
834
|
+
/** Dynamic candidate list during construction (default: 200) */
|
|
835
|
+
efConstruction: number
|
|
836
|
+
/** Dynamic candidate list during search (default: 50) */
|
|
837
|
+
efSearch: number
|
|
838
|
+
/** Max level (default: 16) */
|
|
839
|
+
ml: number
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
/** Index health metrics */
|
|
843
|
+
export interface IndexHealth {
|
|
844
|
+
averageConnections: number
|
|
845
|
+
layerDistribution: Array<number>
|
|
846
|
+
maxLayer: number
|
|
847
|
+
totalNodes: number
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
/** Level statistics */
|
|
851
|
+
export interface LevelStat {
|
|
852
|
+
level: number
|
|
853
|
+
nodeCount: number
|
|
854
|
+
avgConnections: number
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
/**
|
|
858
|
+
* Manhattan (L1) distance between two vectors.
|
|
859
|
+
* Lower values indicate higher similarity.
|
|
860
|
+
*/
|
|
861
|
+
export declare function manhattanDistance(a: Array<number>, b: Array<number>): number
|
|
862
|
+
|
|
863
|
+
/** Decode msgpack bytes to a JSON value. */
|
|
864
|
+
export declare function msgpackDecode(buffer: Buffer): any
|
|
865
|
+
|
|
866
|
+
/**
|
|
867
|
+
* Batch decode multiple msgpack buffers to JSON values.
|
|
868
|
+
* Uses Rayon parallelism for large batches.
|
|
869
|
+
*/
|
|
870
|
+
export declare function msgpackDecodeBatch(buffers: Array<Buffer>): Array<any>
|
|
871
|
+
|
|
872
|
+
/** Encode a JSON value to msgpack bytes. */
|
|
873
|
+
export declare function msgpackEncode(value: any): Buffer
|
|
874
|
+
|
|
875
|
+
/**
|
|
876
|
+
* Batch encode multiple JSON values to msgpack.
|
|
877
|
+
* Uses Rayon parallelism for large batches.
|
|
878
|
+
*/
|
|
879
|
+
export declare function msgpackEncodeBatch(values: Array<any>): Array<Buffer>
|
|
880
|
+
|
|
881
|
+
/** Result from compacting a tree to binary .sst format (adapter-controlled) */
|
|
882
|
+
export interface NativeBinaryCompactResult {
|
|
883
|
+
/** Which tree was compacted */
|
|
884
|
+
treeName: string
|
|
885
|
+
/** New merged SSTable ID */
|
|
886
|
+
newSstableId: string
|
|
887
|
+
/** Binary .sst data for the adapter to persist */
|
|
888
|
+
data: Buffer
|
|
889
|
+
/** Target level */
|
|
890
|
+
newLevel: number
|
|
891
|
+
/** Old SSTable IDs to remove from manifest/storage */
|
|
892
|
+
oldIds: Array<string>
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
/** Result from flushing a tree's MemTable to binary .sst format (adapter-controlled) */
|
|
896
|
+
export interface NativeBinaryFlushResult {
|
|
897
|
+
/** Which tree was flushed */
|
|
898
|
+
treeName: string
|
|
899
|
+
/** SSTable ID */
|
|
900
|
+
sstableId: string
|
|
901
|
+
/** Binary .sst data for the adapter to persist */
|
|
902
|
+
data: Buffer
|
|
903
|
+
/** Compaction level (always 0 for flush) */
|
|
904
|
+
level: number
|
|
905
|
+
/** Number of unique entries */
|
|
906
|
+
entryCount: number
|
|
907
|
+
/** Total relationship count */
|
|
908
|
+
relationshipCount: number
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
/** Cache statistics including native memory tracking. */
|
|
912
|
+
export interface NativeCacheStats {
|
|
913
|
+
totalSize: number
|
|
914
|
+
maxSize: number
|
|
915
|
+
itemCount: number
|
|
916
|
+
typeSizes: Array<number>
|
|
917
|
+
typeCounts: Array<number>
|
|
918
|
+
typeAccessCounts: Array<number>
|
|
919
|
+
totalAccessCount: number
|
|
920
|
+
nativeMemory: Array<number>
|
|
921
|
+
totalManagedMemory: number
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
/** Result from compacting a level */
|
|
925
|
+
export interface NativeCompactResult {
|
|
926
|
+
/** ID of the new merged SSTable */
|
|
927
|
+
newSstableId: string
|
|
928
|
+
/** Serialized merged SSTable data (msgpack binary) */
|
|
929
|
+
newData: Buffer
|
|
930
|
+
/** Target level for the merged SSTable */
|
|
931
|
+
newLevel: number
|
|
932
|
+
/** IDs of old SSTables that were merged (should be deleted from storage) */
|
|
933
|
+
oldIds: Array<string>
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
/** Statistics about the mapper state */
|
|
937
|
+
export interface NativeEntityIdMapperStats {
|
|
938
|
+
/** Number of active UUID ↔ integer mappings */
|
|
939
|
+
mappings: number
|
|
940
|
+
/** Next integer ID to assign */
|
|
941
|
+
nextId: number
|
|
942
|
+
/** Whether there are unsaved changes */
|
|
943
|
+
dirty: boolean
|
|
944
|
+
/** Estimated memory usage in bytes */
|
|
945
|
+
memoryEstimate: number
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
/** Result from eviction or clear operations. */
|
|
949
|
+
export interface NativeEvictionResult {
|
|
950
|
+
evictedKeys: Array<string>
|
|
951
|
+
bytesFreed: number
|
|
952
|
+
}
|
|
953
|
+
|
|
954
|
+
/** Result from flushing the MemTable to an SSTable */
|
|
955
|
+
export interface NativeFlushResult {
|
|
956
|
+
/** Unique ID assigned to the new SSTable */
|
|
957
|
+
sstableId: string
|
|
958
|
+
/** Serialized SSTable data (msgpack binary) */
|
|
959
|
+
data: Buffer
|
|
960
|
+
/** Compaction level (always 0 for fresh flush) */
|
|
961
|
+
level: number
|
|
962
|
+
/** Number of unique sourceIds in the SSTable */
|
|
963
|
+
entryCount: number
|
|
964
|
+
/** Total relationship count across all entries */
|
|
965
|
+
relationshipCount: number
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
/** Configuration for NativeLSMTree */
|
|
969
|
+
export interface NativeLsmConfig {
|
|
970
|
+
/** MemTable flush threshold (relationship count). Default: 100_000 */
|
|
971
|
+
memTableThreshold?: number
|
|
972
|
+
/** Max SSTables per level before compaction triggers. Default: 10 */
|
|
973
|
+
maxSstablesPerLevel?: number
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
/** LSM-tree statistics */
|
|
977
|
+
export interface NativeLsmStats {
|
|
978
|
+
/** Number of relationships in the MemTable */
|
|
979
|
+
memTableSize: number
|
|
980
|
+
/** Estimated MemTable memory usage in bytes */
|
|
981
|
+
memTableMemory: number
|
|
982
|
+
/** Total number of loaded SSTables across all levels */
|
|
983
|
+
sstableCount: number
|
|
984
|
+
/** JSON-encoded map of level → SSTable count */
|
|
985
|
+
sstablesByLevelJson: string
|
|
986
|
+
/** Total relationship count (MemTable + all SSTables) */
|
|
987
|
+
totalRelationships: number
|
|
988
|
+
/** Timestamp of last compaction (ms since epoch) */
|
|
989
|
+
lastCompaction: number
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
/** Neighbor persistence data (id + node data in one struct) */
|
|
993
|
+
export interface NeighborPersistData {
|
|
994
|
+
id: string
|
|
995
|
+
level: number
|
|
996
|
+
connections: Record<string, Array<string>>
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
/** Node data for storage persistence */
|
|
1000
|
+
export interface NodeData {
|
|
1001
|
+
level: number
|
|
1002
|
+
connections: Record<string, Array<string>>
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
/** PQ codebook parameters. */
|
|
1006
|
+
export interface PqParams {
|
|
1007
|
+
m: number
|
|
1008
|
+
ksub: number
|
|
1009
|
+
dsub: number
|
|
1010
|
+
dim: number
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
/**
|
|
1014
|
+
* Quantize a float64 vector to SQ4 (4-bit, two values per byte).
|
|
1015
|
+
*
|
|
1016
|
+
* Each dimension is mapped to [0, 15]. Pairs of values are packed into
|
|
1017
|
+
* single bytes (high nibble first). 8x compression vs float32.
|
|
1018
|
+
*/
|
|
1019
|
+
export declare function quantizeSq4(vector: Array<number>): Sq4QuantizeResult
|
|
1020
|
+
|
|
1021
|
+
/**
|
|
1022
|
+
* Quantize a float64 vector to SQ8 (uint8 with per-vector min/max).
|
|
1023
|
+
*
|
|
1024
|
+
* Each dimension is mapped to [0, 255] using the vector's value range.
|
|
1025
|
+
* Returns the quantized bytes plus min/max for reconstruction.
|
|
1026
|
+
*/
|
|
1027
|
+
export declare function quantizeSq8(vector: Array<number>): Sq8QuantizeResult
|
|
1028
|
+
|
|
1029
|
+
/**
|
|
1030
|
+
* Get the dimensionality of a .vec file without reading vectors.
|
|
1031
|
+
* Useful for determining dim before calling batch_read_vectors_direct.
|
|
1032
|
+
*/
|
|
1033
|
+
export declare function readVectorFileInfo(path: string): any
|
|
1034
|
+
|
|
1035
|
+
/** Search result returned to TypeScript */
|
|
1036
|
+
export interface SearchResult {
|
|
1037
|
+
id: string
|
|
1038
|
+
distance: number
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
/**
|
|
1042
|
+
* Serialize SQ8 data to compact binary format: [min:f32][max:f32][quantized bytes].
|
|
1043
|
+
* Total size: 8 + dim bytes.
|
|
1044
|
+
*/
|
|
1045
|
+
export declare function serializeSq8(quantized: Buffer, min: number, max: number): Buffer
|
|
1046
|
+
|
|
1047
|
+
/** Result of SQ4 quantization. */
|
|
1048
|
+
export interface Sq4QuantizeResult {
|
|
1049
|
+
/** Packed nibbles: two 4-bit values per byte. Length = ceil(dim / 2). */
|
|
1050
|
+
quantized: Buffer
|
|
1051
|
+
min: number
|
|
1052
|
+
max: number
|
|
1053
|
+
/** Original vector dimension (needed because packed length rounds up). */
|
|
1054
|
+
dim: number
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
/** Result of SQ8 quantization. */
|
|
1058
|
+
export interface Sq8QuantizeResult {
|
|
1059
|
+
quantized: Buffer
|
|
1060
|
+
min: number
|
|
1061
|
+
max: number
|
|
1062
|
+
}
|
|
1063
|
+
|
|
1064
|
+
/** System data for persistence */
|
|
1065
|
+
export interface SystemData {
|
|
1066
|
+
entryPointId?: string
|
|
1067
|
+
maxLevel: number
|
|
1068
|
+
}
|