@soulcraft/cortex 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +16 -0
- package/README.md +125 -0
- package/dist/graph/NativeGraphAdjacencyIndex.d.ts +92 -0
- package/dist/graph/NativeGraphAdjacencyIndex.js +671 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.js +23 -0
- package/dist/license.d.ts +18 -0
- package/dist/license.js +172 -0
- package/dist/native/NativeEmbeddingEngine.d.ts +79 -0
- package/dist/native/NativeEmbeddingEngine.js +302 -0
- package/dist/native/NativeRoaringBitmap32.d.ts +114 -0
- package/dist/native/NativeRoaringBitmap32.js +221 -0
- package/dist/native/ffi.d.ts +20 -0
- package/dist/native/ffi.js +48 -0
- package/dist/native/index.d.ts +30 -0
- package/dist/native/index.js +58 -0
- package/dist/native/napi.d.ts +21 -0
- package/dist/native/napi.js +88 -0
- package/dist/native/types.d.ts +710 -0
- package/dist/native/types.js +16 -0
- package/dist/plugin.d.ts +22 -0
- package/dist/plugin.js +115 -0
- package/dist/storage/mmapFileSystemStorage.d.ts +24 -0
- package/dist/storage/mmapFileSystemStorage.js +73 -0
- package/dist/utils/NativeMetadataIndex.d.ts +185 -0
- package/dist/utils/NativeMetadataIndex.js +1274 -0
- package/dist/utils/nativeEntityIdMapper.d.ts +84 -0
- package/dist/utils/nativeEntityIdMapper.js +134 -0
- package/native/brainy-native.darwin-arm64.node +0 -0
- package/native/brainy-native.darwin-x64.node +0 -0
- package/native/brainy-native.linux-arm64-gnu.node +0 -0
- package/native/brainy-native.linux-x64-gnu.node +0 -0
- package/native/brainy-native.win32-x64-msvc.node +0 -0
- package/native/index.d.ts +1068 -0
- package/package.json +66 -0
|
@@ -0,0 +1,710 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared types for the native module integration.
|
|
3
|
+
*
|
|
4
|
+
* These types are used by both the napi-rs and bun:ffi loaders.
|
|
5
|
+
* Ported from src/embeddings/wasm/types.ts — keeps the same API surface.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Embedding result with metadata
|
|
9
|
+
*/
|
|
10
|
+
export interface EmbeddingResult {
|
|
11
|
+
/** 384-dimensional embedding vector */
|
|
12
|
+
embedding: number[];
|
|
13
|
+
/** Number of tokens processed (0 when using native Candle — handled internally) */
|
|
14
|
+
tokenCount: number;
|
|
15
|
+
/** Processing time in milliseconds */
|
|
16
|
+
processingTimeMs: number;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Engine statistics
|
|
20
|
+
*/
|
|
21
|
+
export interface EngineStats {
|
|
22
|
+
/** Whether the engine is initialized */
|
|
23
|
+
initialized: boolean;
|
|
24
|
+
/** Total number of embeddings generated */
|
|
25
|
+
embedCount: number;
|
|
26
|
+
/** Total processing time in milliseconds */
|
|
27
|
+
totalProcessingTimeMs: number;
|
|
28
|
+
/** Average processing time per embedding */
|
|
29
|
+
avgProcessingTimeMs: number;
|
|
30
|
+
/** Model name */
|
|
31
|
+
modelName: string;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Model constants for all-MiniLM-L6-v2
|
|
35
|
+
*/
|
|
36
|
+
export declare const MODEL_CONSTANTS: {
|
|
37
|
+
readonly HIDDEN_SIZE: 384;
|
|
38
|
+
readonly MAX_SEQUENCE_LENGTH: 256;
|
|
39
|
+
readonly VOCAB_SIZE: 30522;
|
|
40
|
+
readonly MODEL_NAME: "all-MiniLM-L6-v2";
|
|
41
|
+
};
|
|
42
|
+
/**
|
|
43
|
+
* Device information from the native module
|
|
44
|
+
*/
|
|
45
|
+
export interface DeviceInfo {
|
|
46
|
+
/** Active device: "cpu", "cuda", or "metal" */
|
|
47
|
+
device: string;
|
|
48
|
+
/** Whether CUDA is available */
|
|
49
|
+
cudaAvailable: boolean;
|
|
50
|
+
/** Whether Metal is available */
|
|
51
|
+
metalAvailable: boolean;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Native module bindings interface.
|
|
55
|
+
* Both napi-rs and bun:ffi loaders implement this interface.
|
|
56
|
+
*/
|
|
57
|
+
export interface NativeBindings {
|
|
58
|
+
NativeEmbeddingEngine: {
|
|
59
|
+
new (): NativeEmbeddingEngineInstance;
|
|
60
|
+
};
|
|
61
|
+
NativeRoaringBitmap32: {
|
|
62
|
+
new (values?: number[]): NativeRoaringBitmap32Instance;
|
|
63
|
+
and(a: NativeRoaringBitmap32Instance, b: NativeRoaringBitmap32Instance): NativeRoaringBitmap32Instance;
|
|
64
|
+
or(a: NativeRoaringBitmap32Instance, b: NativeRoaringBitmap32Instance): NativeRoaringBitmap32Instance;
|
|
65
|
+
orMany(bitmaps: NativeRoaringBitmap32Instance[]): NativeRoaringBitmap32Instance;
|
|
66
|
+
andNot(a: NativeRoaringBitmap32Instance, b: NativeRoaringBitmap32Instance): NativeRoaringBitmap32Instance;
|
|
67
|
+
xor(a: NativeRoaringBitmap32Instance, b: NativeRoaringBitmap32Instance): NativeRoaringBitmap32Instance;
|
|
68
|
+
deserialize(buffer: Buffer): NativeRoaringBitmap32Instance;
|
|
69
|
+
};
|
|
70
|
+
NativeEntityIdMapper: {
|
|
71
|
+
new (): NativeEntityIdMapperInstance;
|
|
72
|
+
};
|
|
73
|
+
NativeLSMTree: {
|
|
74
|
+
new (config: NativeLSMConfig): NativeLSMTreeInstance;
|
|
75
|
+
};
|
|
76
|
+
NativeGraphAdjacencyIndex: {
|
|
77
|
+
new (config: NativeGraphAdjacencyConfig): NativeGraphAdjacencyInstance;
|
|
78
|
+
};
|
|
79
|
+
NativeHNSWIndex: {
|
|
80
|
+
new (config: NativeHNSWConfig): NativeHNSWIndexInstance;
|
|
81
|
+
};
|
|
82
|
+
cosineDistance(a: number[], b: number[]): number;
|
|
83
|
+
euclideanDistance(a: number[], b: number[]): number;
|
|
84
|
+
manhattanDistance(a: number[], b: number[]): number;
|
|
85
|
+
dotProductDistance(a: number[], b: number[]): number;
|
|
86
|
+
cosineDistanceBatch(query: number[], vectors: number[][]): number[];
|
|
87
|
+
euclideanDistanceBatch(query: number[], vectors: number[][]): number[];
|
|
88
|
+
quantizeSq8(vector: number[]): {
|
|
89
|
+
quantized: Buffer;
|
|
90
|
+
min: number;
|
|
91
|
+
max: number;
|
|
92
|
+
};
|
|
93
|
+
dequantizeSq8(quantized: Buffer, min: number, max: number): number[];
|
|
94
|
+
serializeSq8(quantized: Buffer, min: number, max: number): Buffer;
|
|
95
|
+
deserializeSq8(data: Buffer): {
|
|
96
|
+
quantized: Buffer;
|
|
97
|
+
min: number;
|
|
98
|
+
max: number;
|
|
99
|
+
};
|
|
100
|
+
cosineDistanceSq8(a: Buffer, aMin: number, aMax: number, b: Buffer, bMin: number, bMax: number): number;
|
|
101
|
+
cosineDistanceSq8Batch(query: Buffer, queryMin: number, queryMax: number, vectors: Buffer[], mins: number[], maxs: number[]): number[];
|
|
102
|
+
quantizeSq4(vector: number[]): {
|
|
103
|
+
quantized: Buffer;
|
|
104
|
+
min: number;
|
|
105
|
+
max: number;
|
|
106
|
+
dim: number;
|
|
107
|
+
};
|
|
108
|
+
dequantizeSq4(quantized: Buffer, min: number, max: number, dim: number): number[];
|
|
109
|
+
cosineDistanceSq4(a: Buffer, aMin: number, aMax: number, aDim: number, b: Buffer, bMin: number, bMax: number, bDim: number): number;
|
|
110
|
+
cosineDistanceSq4Batch(query: Buffer, queryMin: number, queryMax: number, queryDim: number, vectors: Buffer[], mins: number[], maxs: number[], dims: number[]): number[];
|
|
111
|
+
encodeConnections(ids: number[]): Buffer;
|
|
112
|
+
decodeConnections(data: Buffer): number[];
|
|
113
|
+
NativePqCodebook: {
|
|
114
|
+
train(vectors: number[], dim: number, m: number, ksub: number, iterations: number): NativePQCodebookInstance;
|
|
115
|
+
deserialize(data: Buffer): NativePQCodebookInstance;
|
|
116
|
+
};
|
|
117
|
+
msgpackEncode(value: unknown): Buffer;
|
|
118
|
+
msgpackDecode(buffer: Buffer): unknown;
|
|
119
|
+
msgpackEncodeBatch(values: unknown[]): Buffer[];
|
|
120
|
+
msgpackDecodeBatch(buffers: Buffer[]): unknown[];
|
|
121
|
+
NativeMetadataIndex: {
|
|
122
|
+
new (configJson?: string | null): NativeMetadataIndexInstance;
|
|
123
|
+
};
|
|
124
|
+
NativeUnifiedCache: {
|
|
125
|
+
new (maxSize: number): NativeUnifiedCacheInstance;
|
|
126
|
+
};
|
|
127
|
+
NativeMmapVectorStore: {
|
|
128
|
+
create(path: string, dim: number, capacity: number): NativeMmapVectorStoreInstance;
|
|
129
|
+
open(path: string): NativeMmapVectorStoreInstance;
|
|
130
|
+
openReadOnly(path: string): NativeMmapVectorStoreInstance;
|
|
131
|
+
};
|
|
132
|
+
batchReadVectorsDirect(path: string, indices: number[]): number[];
|
|
133
|
+
readVectorFileInfo(path: string): VectorFileInfo;
|
|
134
|
+
computeBfsOrder(entryPoint: number, adjacencyOffsets: number[], adjacencyNeighbors: number[]): number[];
|
|
135
|
+
computeHnswTraversalOrder(entryPoint: number, adjacencyOffsets: number[], adjacencyNeighbors: number[], neighborOfNeighborDepth?: number | null): number[];
|
|
136
|
+
getDeviceInfo(): DeviceInfo;
|
|
137
|
+
cosineSimilarity(a: number[], b: number[]): number;
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Native embedding engine instance
|
|
141
|
+
*/
|
|
142
|
+
export interface NativeEmbeddingEngineInstance {
|
|
143
|
+
load(modelBytes: Buffer, tokenizerBytes: Buffer, configBytes: Buffer): void;
|
|
144
|
+
isReady(): boolean;
|
|
145
|
+
embed(text: string): number[];
|
|
146
|
+
embedBatch(texts: string[]): number[][];
|
|
147
|
+
dimension(): number;
|
|
148
|
+
maxSequenceLength(): number;
|
|
149
|
+
getStats(): EngineStats;
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Native roaring bitmap instance
|
|
153
|
+
*/
|
|
154
|
+
export interface NativeRoaringBitmap32Instance {
|
|
155
|
+
readonly size: number;
|
|
156
|
+
readonly isEmpty: boolean;
|
|
157
|
+
add(value: number): void;
|
|
158
|
+
delete(value: number): void;
|
|
159
|
+
tryAdd(value: number): boolean;
|
|
160
|
+
has(value: number): boolean;
|
|
161
|
+
toArray(): number[];
|
|
162
|
+
serialize(): Buffer;
|
|
163
|
+
serializationSizeInBytes(): number;
|
|
164
|
+
minimum(): number | undefined;
|
|
165
|
+
maximum(): number | undefined;
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Statistics from native EntityIdMapper
|
|
169
|
+
*/
|
|
170
|
+
export interface NativeEntityIdMapperStats {
|
|
171
|
+
/** Number of active UUID ↔ integer mappings */
|
|
172
|
+
mappings: number;
|
|
173
|
+
/** Next integer ID to assign */
|
|
174
|
+
nextId: number;
|
|
175
|
+
/** Whether there are unsaved changes */
|
|
176
|
+
dirty: boolean;
|
|
177
|
+
/** Estimated memory usage in bytes */
|
|
178
|
+
memoryEstimate: number;
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Native EntityIdMapper instance.
|
|
182
|
+
* Bidirectional UUID ↔ integer mapping for roaring bitmap integration.
|
|
183
|
+
*/
|
|
184
|
+
export interface NativeEntityIdMapperInstance {
|
|
185
|
+
/** Load state from JSON string (storage format) */
|
|
186
|
+
loadFromJson(json: string): void;
|
|
187
|
+
/** Serialize state to JSON string for storage persistence */
|
|
188
|
+
saveToJson(): string;
|
|
189
|
+
/** Get integer ID for UUID, assigning a new one if not mapped */
|
|
190
|
+
getOrAssign(uuid: string): number;
|
|
191
|
+
/** Get UUID for integer ID. Returns null if not found */
|
|
192
|
+
getUuid(intId: number): string | null;
|
|
193
|
+
/** Get integer ID for UUID without assigning. Returns null if not found */
|
|
194
|
+
getInt(uuid: string): number | null;
|
|
195
|
+
/** Check if UUID has been assigned an integer ID */
|
|
196
|
+
has(uuid: string): boolean;
|
|
197
|
+
/** Remove mapping for UUID. Returns true if it existed */
|
|
198
|
+
remove(uuid: string): boolean;
|
|
199
|
+
/** Get total number of mappings */
|
|
200
|
+
readonly size: number;
|
|
201
|
+
/** Convert array of UUIDs to array of integers (assigns new IDs as needed) */
|
|
202
|
+
uuidsToInts(uuids: string[]): number[];
|
|
203
|
+
/** Convert array of integers to array of UUIDs (skips unknown IDs) */
|
|
204
|
+
intsToUuids(ints: number[]): string[];
|
|
205
|
+
/** Whether there are unsaved changes */
|
|
206
|
+
isDirty(): boolean;
|
|
207
|
+
/** Clear all mappings and reset counter */
|
|
208
|
+
clear(): void;
|
|
209
|
+
/** Get statistics about the mapper */
|
|
210
|
+
getStats(): NativeEntityIdMapperStats;
|
|
211
|
+
}
|
|
212
|
+
/**
|
|
213
|
+
* HNSW configuration passed to the native engine.
|
|
214
|
+
* Mirrors the TypeScript HNSWConfig interface.
|
|
215
|
+
*/
|
|
216
|
+
export interface NativeHNSWConfig {
|
|
217
|
+
/** Max connections per node (default: 16) */
|
|
218
|
+
m: number;
|
|
219
|
+
/** Dynamic candidate list during construction (default: 200) */
|
|
220
|
+
efConstruction: number;
|
|
221
|
+
/** Dynamic candidate list during search (default: 50) */
|
|
222
|
+
efSearch: number;
|
|
223
|
+
/** Max level (default: 16) */
|
|
224
|
+
ml: number;
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* Search result from native HNSW engine
|
|
228
|
+
*/
|
|
229
|
+
export interface NativeSearchResult {
|
|
230
|
+
id: string;
|
|
231
|
+
distance: number;
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Node persistence data from native HNSW engine
|
|
235
|
+
*/
|
|
236
|
+
export interface NativeNodeData {
|
|
237
|
+
level: number;
|
|
238
|
+
connections: Record<string, string[]>;
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* System persistence data from native HNSW engine
|
|
242
|
+
*/
|
|
243
|
+
export interface NativeSystemData {
|
|
244
|
+
entryPointId: string | null | undefined;
|
|
245
|
+
maxLevel: number;
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* Index health metrics from native HNSW engine
|
|
249
|
+
*/
|
|
250
|
+
export interface NativeIndexHealth {
|
|
251
|
+
averageConnections: number;
|
|
252
|
+
layerDistribution: number[];
|
|
253
|
+
maxLayer: number;
|
|
254
|
+
totalNodes: number;
|
|
255
|
+
}
|
|
256
|
+
/**
|
|
257
|
+
* Per-level statistics from native HNSW engine
|
|
258
|
+
*/
|
|
259
|
+
export interface NativeLevelStat {
|
|
260
|
+
level: number;
|
|
261
|
+
nodeCount: number;
|
|
262
|
+
avgConnections: number;
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* Neighbor persistence data returned by addItemFull.
|
|
266
|
+
*/
|
|
267
|
+
export interface NativeNeighborPersistData {
|
|
268
|
+
id: string;
|
|
269
|
+
level: number;
|
|
270
|
+
connections: Record<string, string[]>;
|
|
271
|
+
}
|
|
272
|
+
/**
|
|
273
|
+
* Full result from addItemFull including all persistence data.
|
|
274
|
+
* Returns everything TS needs to persist in a single FFI call,
|
|
275
|
+
* eliminating N+3 FFI round trips per add.
|
|
276
|
+
*/
|
|
277
|
+
export interface NativeAddItemResult {
|
|
278
|
+
id: string;
|
|
279
|
+
nodeData: NativeNodeData;
|
|
280
|
+
modifiedNeighbors: NativeNeighborPersistData[];
|
|
281
|
+
systemData: NativeSystemData;
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* Native HNSW index instance.
|
|
285
|
+
* Full graph engine running in Rust with SIMD distance calculations,
|
|
286
|
+
* BinaryHeap priority queues, and Arc-based COW.
|
|
287
|
+
*/
|
|
288
|
+
export interface NativeHNSWIndexInstance {
|
|
289
|
+
addItem(id: string, vector: number[]): string;
|
|
290
|
+
addItemFull(id: string, vector: number[]): NativeAddItemResult;
|
|
291
|
+
addItemWithLevel(id: string, vector: number[], level: number): string;
|
|
292
|
+
addItemsBatch(ids: string[], vectors: number[][]): string[];
|
|
293
|
+
search(query: number[], k: number, ef?: number | null | undefined): NativeSearchResult[];
|
|
294
|
+
searchWithCandidates(query: number[], k: number, ef: number, candidateIds?: string[] | null | undefined): NativeSearchResult[];
|
|
295
|
+
removeItem(id: string): boolean;
|
|
296
|
+
fork(): NativeHNSWIndexInstance;
|
|
297
|
+
getNodeData(id: string): NativeNodeData | null;
|
|
298
|
+
restoreNode(id: string, vector: number[], level: number, connections: Record<string, string[]>): void;
|
|
299
|
+
getSystemData(): NativeSystemData;
|
|
300
|
+
setSystemData(entryPointId: string | null | undefined, maxLevel: number): void;
|
|
301
|
+
size(): number;
|
|
302
|
+
clear(): void;
|
|
303
|
+
getEntryPointId(): string | undefined;
|
|
304
|
+
getMaxLevel(): number;
|
|
305
|
+
getDimension(): number | undefined;
|
|
306
|
+
hasNode(id: string): boolean;
|
|
307
|
+
getIndexHealth(): NativeIndexHealth;
|
|
308
|
+
getLevelStats(): NativeLevelStat[];
|
|
309
|
+
getNodesAtLevel(level: number): string[];
|
|
310
|
+
getVector(id: string): number[] | null;
|
|
311
|
+
getAllIds(): string[];
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Configuration for NativeLSMTree
|
|
315
|
+
*/
|
|
316
|
+
export interface NativeLSMConfig {
|
|
317
|
+
/** MemTable flush threshold (relationship count). Default: 100_000 */
|
|
318
|
+
memTableThreshold?: number;
|
|
319
|
+
/** Max SSTables per level before compaction. Default: 10 */
|
|
320
|
+
maxSstablesPerLevel?: number;
|
|
321
|
+
}
|
|
322
|
+
/**
|
|
323
|
+
* Result from flushing the MemTable to an SSTable
|
|
324
|
+
*/
|
|
325
|
+
export interface NativeFlushResult {
|
|
326
|
+
/** Unique ID of the new SSTable */
|
|
327
|
+
sstableId: string;
|
|
328
|
+
/** Serialized SSTable data (msgpack binary) */
|
|
329
|
+
data: Buffer;
|
|
330
|
+
/** Compaction level (always 0 for fresh flush) */
|
|
331
|
+
level: number;
|
|
332
|
+
/** Number of unique sourceIds */
|
|
333
|
+
entryCount: number;
|
|
334
|
+
/** Total relationship count */
|
|
335
|
+
relationshipCount: number;
|
|
336
|
+
}
|
|
337
|
+
/**
|
|
338
|
+
* Result from compacting a level
|
|
339
|
+
*/
|
|
340
|
+
export interface NativeCompactResult {
|
|
341
|
+
/** ID of the new merged SSTable */
|
|
342
|
+
newSstableId: string;
|
|
343
|
+
/** Serialized merged SSTable data (msgpack binary) */
|
|
344
|
+
newData: Buffer;
|
|
345
|
+
/** Target level for the merged SSTable */
|
|
346
|
+
newLevel: number;
|
|
347
|
+
/** IDs of old SSTables that were merged (delete from storage) */
|
|
348
|
+
oldIds: string[];
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* LSM-tree statistics
|
|
352
|
+
*/
|
|
353
|
+
export interface NativeLSMStats {
|
|
354
|
+
/** Relationships currently in the MemTable */
|
|
355
|
+
memTableSize: number;
|
|
356
|
+
/** Estimated MemTable memory in bytes */
|
|
357
|
+
memTableMemory: number;
|
|
358
|
+
/** Total loaded SSTables */
|
|
359
|
+
sstableCount: number;
|
|
360
|
+
/** JSON map of level → SSTable count */
|
|
361
|
+
sstablesByLevelJson: string;
|
|
362
|
+
/** Total relationships tracked */
|
|
363
|
+
totalRelationships: number;
|
|
364
|
+
/** Timestamp of last compaction (ms) */
|
|
365
|
+
lastCompaction: number;
|
|
366
|
+
}
|
|
367
|
+
/**
|
|
368
|
+
* Native LSM-Tree instance.
|
|
369
|
+
* Log-Structured Merge Tree for graph relationship storage.
|
|
370
|
+
* All data operations execute synchronously in Rust.
|
|
371
|
+
*/
|
|
372
|
+
export interface NativeLSMTreeInstance {
|
|
373
|
+
/** Add source → target relationship. Returns true if MemTable needs flush */
|
|
374
|
+
add(sourceId: string, targetId: string): boolean;
|
|
375
|
+
/** Query targets for sourceId. Returns null if not found */
|
|
376
|
+
get(sourceId: string): string[] | null;
|
|
377
|
+
/** Flush MemTable to L0 SSTable. Returns serialized data or null if empty */
|
|
378
|
+
flushMemTable(): NativeFlushResult | null;
|
|
379
|
+
/** Load a persisted SSTable into the tree */
|
|
380
|
+
loadSstable(id: string, level: number, data: Buffer): void;
|
|
381
|
+
/** Check if a level needs compaction */
|
|
382
|
+
needsCompaction(level: number): boolean;
|
|
383
|
+
/** Compact a level. Returns merged SSTable data or null if not needed */
|
|
384
|
+
compact(level: number): NativeCompactResult | null;
|
|
385
|
+
/** Get manifest as JSON string */
|
|
386
|
+
getManifestJson(): string;
|
|
387
|
+
/** Load manifest from JSON string */
|
|
388
|
+
loadManifestJson(json: string): void;
|
|
389
|
+
/** Get SSTable IDs from manifest (for loading from storage) */
|
|
390
|
+
getManifestSstableIds(): string[];
|
|
391
|
+
/** Get the level for a manifest SSTable ID */
|
|
392
|
+
getManifestSstableLevel(id: string): number | null;
|
|
393
|
+
/** Number of relationships in MemTable */
|
|
394
|
+
memTableSize(): number;
|
|
395
|
+
/** Whether MemTable is empty */
|
|
396
|
+
memTableIsEmpty(): boolean;
|
|
397
|
+
/** Total relationship count */
|
|
398
|
+
readonly totalRelationships: number;
|
|
399
|
+
/** Total relationship count (alias) */
|
|
400
|
+
size(): number;
|
|
401
|
+
/** Whether tree is healthy */
|
|
402
|
+
isHealthy(): boolean;
|
|
403
|
+
/** Get statistics */
|
|
404
|
+
getStats(): NativeLSMStats;
|
|
405
|
+
/** Clear MemTable without flushing */
|
|
406
|
+
clearMemTable(): void;
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* Configuration for NativeGraphAdjacencyIndex
|
|
410
|
+
*/
|
|
411
|
+
export interface NativeGraphAdjacencyConfig {
|
|
412
|
+
/** MemTable flush threshold per tree. Default: 100_000 */
|
|
413
|
+
memTableThreshold?: number;
|
|
414
|
+
/** Max SSTables per level per tree. Default: 10 */
|
|
415
|
+
maxSstablesPerLevel?: number;
|
|
416
|
+
}
|
|
417
|
+
/**
|
|
418
|
+
* Result from addVerb — indicates which trees need flushing
|
|
419
|
+
*/
|
|
420
|
+
export interface GraphAddVerbResult {
|
|
421
|
+
needsFlushSource: boolean;
|
|
422
|
+
needsFlushTarget: boolean;
|
|
423
|
+
needsFlushVerbsSource: boolean;
|
|
424
|
+
needsFlushVerbsTarget: boolean;
|
|
425
|
+
}
|
|
426
|
+
/**
|
|
427
|
+
* Result from flushing a specific tree's MemTable
|
|
428
|
+
*/
|
|
429
|
+
export interface GraphFlushResult {
|
|
430
|
+
treeName: string;
|
|
431
|
+
sstableId: string;
|
|
432
|
+
data: Buffer;
|
|
433
|
+
level: number;
|
|
434
|
+
entryCount: number;
|
|
435
|
+
relationshipCount: number;
|
|
436
|
+
}
|
|
437
|
+
/**
|
|
438
|
+
* Result from compacting a specific tree
|
|
439
|
+
*/
|
|
440
|
+
export interface GraphCompactResult {
|
|
441
|
+
treeName: string;
|
|
442
|
+
newSstableId: string;
|
|
443
|
+
newData: Buffer;
|
|
444
|
+
newLevel: number;
|
|
445
|
+
oldIds: string[];
|
|
446
|
+
}
|
|
447
|
+
/**
|
|
448
|
+
* Statistics for the graph adjacency index
|
|
449
|
+
*/
|
|
450
|
+
export interface GraphAdjacencyStats {
|
|
451
|
+
totalRelationships: number;
|
|
452
|
+
verbIdCount: number;
|
|
453
|
+
relationshipTypeCount: number;
|
|
454
|
+
sourceMemTableSize: number;
|
|
455
|
+
targetMemTableSize: number;
|
|
456
|
+
sourceSstableCount: number;
|
|
457
|
+
targetSstableCount: number;
|
|
458
|
+
sourceMemTableMemory: number;
|
|
459
|
+
targetMemTableMemory: number;
|
|
460
|
+
}
|
|
461
|
+
/**
|
|
462
|
+
* Result from flushing a tree's MemTable to binary .sst format (adapter-controlled)
|
|
463
|
+
*/
|
|
464
|
+
export interface NativeBinaryFlushResult {
|
|
465
|
+
treeName: string;
|
|
466
|
+
sstableId: string;
|
|
467
|
+
data: Buffer;
|
|
468
|
+
level: number;
|
|
469
|
+
entryCount: number;
|
|
470
|
+
relationshipCount: number;
|
|
471
|
+
}
|
|
472
|
+
/**
|
|
473
|
+
* Result from compacting a tree's level to binary .sst format (adapter-controlled)
|
|
474
|
+
*/
|
|
475
|
+
export interface NativeBinaryCompactResult {
|
|
476
|
+
treeName: string;
|
|
477
|
+
newSstableId: string;
|
|
478
|
+
data: Buffer;
|
|
479
|
+
newLevel: number;
|
|
480
|
+
oldIds: string[];
|
|
481
|
+
}
|
|
482
|
+
/**
|
|
483
|
+
* Native graph adjacency index instance.
|
|
484
|
+
* Wraps 4 LSM-trees (source, target, verbs-source, verbs-target)
|
|
485
|
+
* plus verb tracking and relationship counts.
|
|
486
|
+
*/
|
|
487
|
+
export interface NativeGraphAdjacencyInstance {
|
|
488
|
+
/** Add a verb to the graph. Returns which trees need flushing */
|
|
489
|
+
addVerb(verbId: string, sourceId: string, targetId: string, verbType: string): GraphAddVerbResult;
|
|
490
|
+
/** Remove a verb (tombstone deletion) */
|
|
491
|
+
removeVerb(verbId: string, verbType: string): void;
|
|
492
|
+
/** Get neighbors with optional pagination */
|
|
493
|
+
getNeighbors(id: string, direction: string, limit?: number | null, offset?: number | null): string[];
|
|
494
|
+
/** Get verb IDs by source with tombstone filtering and pagination */
|
|
495
|
+
getVerbIdsBySource(sourceId: string, limit?: number | null, offset?: number | null): string[];
|
|
496
|
+
/** Get verb IDs by target with tombstone filtering and pagination */
|
|
497
|
+
getVerbIdsByTarget(targetId: string, limit?: number | null, offset?: number | null): string[];
|
|
498
|
+
/** Track a verb ID without adding to LSM-trees (for rebuild) */
|
|
499
|
+
trackVerbId(verbId: string, verbType: string): void;
|
|
500
|
+
/** Clear all verb tracking state */
|
|
501
|
+
clearVerbTracking(): void;
|
|
502
|
+
/** Flush a tree's MemTable to binary .sst format (no disk I/O) */
|
|
503
|
+
flushTreeToBinary(treeName: string): NativeBinaryFlushResult | null;
|
|
504
|
+
/** Compact a tree's level to binary .sst format (no disk I/O) */
|
|
505
|
+
compactTreeToBinary(treeName: string, level: number): NativeBinaryCompactResult | null;
|
|
506
|
+
/** Mmap an adapter-written file, replacing InMemory variant with Mapped */
|
|
507
|
+
registerMmapSstable(treeName: string, sstableId: string, level: number, filePath: string): void;
|
|
508
|
+
/** Load an SSTable from binary .sst data (for non-mmap adapters) */
|
|
509
|
+
loadSstableFromBinary(treeName: string, sstableId: string, level: number, data: Buffer): void;
|
|
510
|
+
/** Load an SSTable from a .sst file via mmap (adapter provides the path) */
|
|
511
|
+
loadSstableFromFile(treeName: string, sstableId: string, level: number, filePath: string): void;
|
|
512
|
+
/** Flush a specific tree's MemTable */
|
|
513
|
+
flushTree(treeName: string): GraphFlushResult | null;
|
|
514
|
+
/** Load an SSTable into a specific tree */
|
|
515
|
+
loadSstable(treeName: string, id: string, level: number, data: Buffer): void;
|
|
516
|
+
/** Check if a tree's level needs compaction */
|
|
517
|
+
needsCompaction(treeName: string, level: number): boolean;
|
|
518
|
+
/** Compact a specific tree's level */
|
|
519
|
+
compactTree(treeName: string, level: number): GraphCompactResult | null;
|
|
520
|
+
/** Get manifest JSON for a tree */
|
|
521
|
+
getManifestJson(treeName: string): string;
|
|
522
|
+
/** Load manifest JSON for a tree */
|
|
523
|
+
loadManifestJson(treeName: string, json: string): void;
|
|
524
|
+
/** Get SSTable IDs from a tree's manifest */
|
|
525
|
+
getManifestSstableIds(treeName: string): string[];
|
|
526
|
+
/** Get level for an SSTable ID in a tree's manifest */
|
|
527
|
+
getManifestSstableLevel(treeName: string, id: string): number | null;
|
|
528
|
+
/** Check if a tree's MemTable is empty */
|
|
529
|
+
memTableIsEmpty(treeName: string): boolean;
|
|
530
|
+
/** Total relationship count from source tree */
|
|
531
|
+
size(): number;
|
|
532
|
+
/** Number of tracked verb IDs */
|
|
533
|
+
verbIdCount(): number;
|
|
534
|
+
/** Get relationship count for a type */
|
|
535
|
+
getRelationshipCountByType(verbType: string): number;
|
|
536
|
+
/** Get all relationship counts as JSON */
|
|
537
|
+
getAllRelationshipCountsJson(): string;
|
|
538
|
+
/** Get relationship statistics as JSON */
|
|
539
|
+
getRelationshipStatsJson(): string;
|
|
540
|
+
/** Whether the index is healthy */
|
|
541
|
+
isHealthy(): boolean;
|
|
542
|
+
/** Get comprehensive statistics */
|
|
543
|
+
getStats(): GraphAdjacencyStats;
|
|
544
|
+
/** List all tree names */
|
|
545
|
+
treeNames(): string[];
|
|
546
|
+
}
|
|
547
|
+
/**
|
|
548
|
+
* Mutation result from add/remove operations.
|
|
549
|
+
* Tells the TS wrapper which data needs to be persisted.
|
|
550
|
+
*/
|
|
551
|
+
export interface NativeMetadataMutationResult {
|
|
552
|
+
dirtyChunks: Array<{
|
|
553
|
+
field: string;
|
|
554
|
+
chunkId: number;
|
|
555
|
+
}>;
|
|
556
|
+
dirtySparseIndices: string[];
|
|
557
|
+
dirtyFieldIndexes: string[];
|
|
558
|
+
newFields: string[];
|
|
559
|
+
dirtyFieldRegistry: boolean;
|
|
560
|
+
dirtyEntityIdMapper: boolean;
|
|
561
|
+
}
|
|
562
|
+
/**
|
|
563
|
+
* Native metadata index instance.
|
|
564
|
+
* All query and mutation operations execute synchronously in Rust.
|
|
565
|
+
* TS wrapper handles async storage I/O.
|
|
566
|
+
*/
|
|
567
|
+
export interface NativeMetadataIndexInstance {
|
|
568
|
+
loadEntityIdMapper(json: string): void;
|
|
569
|
+
saveEntityIdMapper(): string;
|
|
570
|
+
isEntityIdMapperDirty(): boolean;
|
|
571
|
+
entityIdMapperSize(): number;
|
|
572
|
+
loadSparseIndex(field: string, json: string): void;
|
|
573
|
+
saveSparseIndex(field: string): string | null;
|
|
574
|
+
getSparseIndexChunkIds(field: string): number[];
|
|
575
|
+
isFieldLoaded(field: string): boolean;
|
|
576
|
+
getLoadedFields(): string[];
|
|
577
|
+
loadChunk(field: string, chunkId: number, json: string): void;
|
|
578
|
+
saveChunk(field: string, chunkId: number): string | null;
|
|
579
|
+
isChunkLoaded(field: string, chunkId: number): boolean;
|
|
580
|
+
unloadChunk(field: string, chunkId: number): void;
|
|
581
|
+
unloadField(field: string): void;
|
|
582
|
+
loadFieldRegistry(json: string): void;
|
|
583
|
+
saveFieldRegistry(): string;
|
|
584
|
+
loadFieldIndex(field: string, json: string): void;
|
|
585
|
+
saveFieldIndex(field: string): string | null;
|
|
586
|
+
getIds(field: string, valueJson: string): string[];
|
|
587
|
+
getIdsForFilter(filterJson: string, allIdsJson?: string | null): string[];
|
|
588
|
+
getIdsForMultipleFields(pairsJson: string): string[];
|
|
589
|
+
getIdsForRange(field: string, minJson: string | null, maxJson: string | null, includeMin: boolean, includeMax: boolean): string[];
|
|
590
|
+
getIdsForTextQuery(queryStr: string): string;
|
|
591
|
+
getFilterValues(field: string): string[];
|
|
592
|
+
getFilterFields(): string[];
|
|
593
|
+
extractFieldNames(entityJson: string): string[];
|
|
594
|
+
addToIndex(id: string, entityJson: string): string;
|
|
595
|
+
removeFromIndex(id: string, metadataJson?: string | null): string;
|
|
596
|
+
getEntityCountByType(typeName: string): number;
|
|
597
|
+
getTotalEntityCount(): number;
|
|
598
|
+
getTopNounTypes(n: number): string;
|
|
599
|
+
getTopVerbTypes(n: number): string;
|
|
600
|
+
getStats(): string;
|
|
601
|
+
clear(): void;
|
|
602
|
+
}
|
|
603
|
+
/**
|
|
604
|
+
* Result from eviction or clear operations.
|
|
605
|
+
*/
|
|
606
|
+
export interface NativeEvictionResult {
|
|
607
|
+
evictedKeys: string[];
|
|
608
|
+
bytesFreed: number;
|
|
609
|
+
}
|
|
610
|
+
/**
|
|
611
|
+
* Cache statistics including native memory tracking.
|
|
612
|
+
*/
|
|
613
|
+
export interface NativeCacheStats {
|
|
614
|
+
totalSize: number;
|
|
615
|
+
maxSize: number;
|
|
616
|
+
itemCount: number;
|
|
617
|
+
typeSizes: number[];
|
|
618
|
+
typeCounts: number[];
|
|
619
|
+
typeAccessCounts: number[];
|
|
620
|
+
totalAccessCount: number;
|
|
621
|
+
nativeMemory: number[];
|
|
622
|
+
totalManagedMemory: number;
|
|
623
|
+
}
|
|
624
|
+
/**
|
|
625
|
+
* Native unified cache instance.
|
|
626
|
+
* Manages eviction decisions and cache entry metadata.
|
|
627
|
+
* TS wrapper holds actual JS data in a Map.
|
|
628
|
+
*/
|
|
629
|
+
export interface NativeUnifiedCacheInstance {
|
|
630
|
+
recordAccess(key: string): void;
|
|
631
|
+
insert(key: string, cacheType: number, size: number, rebuildCost: number): NativeEvictionResult;
|
|
632
|
+
remove(key: string): number;
|
|
633
|
+
removeByPrefix(prefix: string): string[];
|
|
634
|
+
clear(cacheType?: number | null): string[];
|
|
635
|
+
checkFairness(): NativeEvictionResult;
|
|
636
|
+
evictForSize(bytesNeeded: number): NativeEvictionResult;
|
|
637
|
+
reportNativeMemory(hnsw: number, metadata: number, graph: number): void;
|
|
638
|
+
getStats(): NativeCacheStats;
|
|
639
|
+
has(key: string): boolean;
|
|
640
|
+
itemCount(): number;
|
|
641
|
+
saveAccessPatterns(): string;
|
|
642
|
+
loadAccessPatterns(json: string): void;
|
|
643
|
+
}
|
|
644
|
+
/**
|
|
645
|
+
* PQ codebook parameters.
|
|
646
|
+
*/
|
|
647
|
+
export interface PQParams {
|
|
648
|
+
m: number;
|
|
649
|
+
ksub: number;
|
|
650
|
+
dsub: number;
|
|
651
|
+
dim: number;
|
|
652
|
+
}
|
|
653
|
+
/**
|
|
654
|
+
* Native Product Quantization codebook instance.
|
|
655
|
+
*/
|
|
656
|
+
export interface NativePQCodebookInstance {
|
|
657
|
+
readonly params: PQParams;
|
|
658
|
+
encode(vector: number[]): Buffer;
|
|
659
|
+
encodeBatch(vectors: number[], dim: number): Buffer;
|
|
660
|
+
decode(code: Buffer): number[];
|
|
661
|
+
computeAdcTable(query: number[]): number[];
|
|
662
|
+
distanceAdc(table: number[], code: Buffer): number;
|
|
663
|
+
distanceAdcBatch(table: number[], codes: Buffer, n: number): number[];
|
|
664
|
+
serialize(): Buffer;
|
|
665
|
+
}
|
|
666
|
+
/**
|
|
667
|
+
* Vector file info returned by readVectorFileInfo().
|
|
668
|
+
*/
|
|
669
|
+
export interface VectorFileInfo {
|
|
670
|
+
dim: number;
|
|
671
|
+
capacity: number;
|
|
672
|
+
count: number;
|
|
673
|
+
recordSizeBytes: number;
|
|
674
|
+
fileSizeBytes: number;
|
|
675
|
+
}
|
|
676
|
+
/**
|
|
677
|
+
* Native mmap vector store instance.
|
|
678
|
+
* Zero-copy vector reads via memory-mapped files.
|
|
679
|
+
*/
|
|
680
|
+
export interface NativeMmapVectorStoreInstance {
|
|
681
|
+
/** Write a single vector (f64[] → stored as f32) */
|
|
682
|
+
writeVector(index: number, vector: number[]): void;
|
|
683
|
+
/** Write batch of vectors from flat array. Returns count written. */
|
|
684
|
+
writeVectorsBatch(startIndex: number, vectorsFlat: number[]): number;
|
|
685
|
+
/** Read a single vector. Returns f64[]. */
|
|
686
|
+
readVector(index: number): number[];
|
|
687
|
+
/** Read multiple vectors by index. Returns flat f64[] of length n × dim. */
|
|
688
|
+
readVectorsBatch(indices: number[]): number[];
|
|
689
|
+
/** Prefetch vectors into OS page cache (madvise WILLNEED) */
|
|
690
|
+
prefetch(indices: number[]): void;
|
|
691
|
+
/** Flush pending writes to disk (msync) */
|
|
692
|
+
flush(): void;
|
|
693
|
+
/** Grow the file to new capacity */
|
|
694
|
+
resize(newCapacity: number): void;
|
|
695
|
+
/** Rewrite vectors in specified order for disk locality. Returns new count. */
|
|
696
|
+
compact(newOrder: number[]): number;
|
|
697
|
+
/** Number of vectors stored */
|
|
698
|
+
readonly count: number;
|
|
699
|
+
/** Maximum capacity */
|
|
700
|
+
readonly capacity: number;
|
|
701
|
+
/** Vector dimensionality */
|
|
702
|
+
readonly dim: number;
|
|
703
|
+
/** File path */
|
|
704
|
+
readonly path: string;
|
|
705
|
+
/** Size of one vector record in bytes */
|
|
706
|
+
readonly recordSizeBytes: number;
|
|
707
|
+
/** Total file size in bytes */
|
|
708
|
+
readonly fileSizeBytes: number;
|
|
709
|
+
}
|
|
710
|
+
//# sourceMappingURL=types.d.ts.map
|