@soulcraft/cor 3.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +16 -0
- package/README.md +328 -0
- package/dist/aggregation/NativeAggregationEngine.d.ts +118 -0
- package/dist/aggregation/NativeAggregationEngine.js +186 -0
- package/dist/cli.d.ts +12 -0
- package/dist/cli.js +236 -0
- package/dist/graph/GraphAccelerationAdapter.d.ts +153 -0
- package/dist/graph/GraphAccelerationAdapter.js +326 -0
- package/dist/graph/NativeGraphAdjacencyIndex.d.ts +286 -0
- package/dist/graph/NativeGraphAdjacencyIndex.js +944 -0
- package/dist/hnsw/AdaptiveDiskAnnModeSelector.d.ts +187 -0
- package/dist/hnsw/AdaptiveDiskAnnModeSelector.js +313 -0
- package/dist/hnsw/NativeDiskAnnWrapper.d.ts +489 -0
- package/dist/hnsw/NativeDiskAnnWrapper.js +1456 -0
- package/dist/index.d.ts +28 -0
- package/dist/index.js +29 -0
- package/dist/legacyLayoutGuard.d.ts +93 -0
- package/dist/legacyLayoutGuard.js +237 -0
- package/dist/license/constants.d.ts +30 -0
- package/dist/license/constants.js +30 -0
- package/dist/license/onlineValidator.d.ts +43 -0
- package/dist/license/onlineValidator.js +225 -0
- package/dist/license/types.d.ts +39 -0
- package/dist/license/types.js +5 -0
- package/dist/license.d.ts +64 -0
- package/dist/license.js +165 -0
- package/dist/migration/MigrationCoordinator.d.ts +171 -0
- package/dist/migration/MigrationCoordinator.js +249 -0
- package/dist/migration/indexEpochGuard.d.ts +163 -0
- package/dist/migration/indexEpochGuard.js +202 -0
- package/dist/native/NativeEmbeddingEngine.d.ts +79 -0
- package/dist/native/NativeEmbeddingEngine.js +318 -0
- package/dist/native/NativeRoaringBitmap32.d.ts +114 -0
- package/dist/native/NativeRoaringBitmap32.js +221 -0
- package/dist/native/ffi.d.ts +20 -0
- package/dist/native/ffi.js +48 -0
- package/dist/native/idMapperTestSupport.d.ts +67 -0
- package/dist/native/idMapperTestSupport.js +112 -0
- package/dist/native/index.d.ts +49 -0
- package/dist/native/index.js +87 -0
- package/dist/native/napi.d.ts +21 -0
- package/dist/native/napi.js +88 -0
- package/dist/native/types.d.ts +1298 -0
- package/dist/native/types.js +16 -0
- package/dist/plugin.d.ts +50 -0
- package/dist/plugin.js +388 -0
- package/dist/providerContracts.d.ts +277 -0
- package/dist/providerContracts.js +38 -0
- package/dist/resource/OsMemoryProbe.d.ts +175 -0
- package/dist/resource/OsMemoryProbe.js +206 -0
- package/dist/resource/ResourceManager.d.ts +491 -0
- package/dist/resource/ResourceManager.js +960 -0
- package/dist/utils/ColumnManifest.d.ts +97 -0
- package/dist/utils/ColumnManifest.js +129 -0
- package/dist/utils/NativeColumnStore.d.ts +284 -0
- package/dist/utils/NativeColumnStore.js +685 -0
- package/dist/utils/NativeMetadataIndex.d.ts +882 -0
- package/dist/utils/NativeMetadataIndex.js +2631 -0
- package/dist/utils/NativeUnifiedCache.d.ts +87 -0
- package/dist/utils/NativeUnifiedCache.js +273 -0
- package/dist/utils/binaryIdMapperFactory.d.ts +59 -0
- package/dist/utils/binaryIdMapperFactory.js +94 -0
- package/dist/utils/collation.d.ts +30 -0
- package/dist/utils/collation.js +40 -0
- package/dist/utils/columnStoreTypes.d.ts +161 -0
- package/dist/utils/columnStoreTypes.js +29 -0
- package/dist/utils/nativeBinaryEntityIdMapper.d.ts +211 -0
- package/dist/utils/nativeBinaryEntityIdMapper.js +381 -0
- package/dist/utils/nativeEntityIdMapper.d.ts +111 -0
- package/dist/utils/nativeEntityIdMapper.js +170 -0
- package/docs/ADR-002-diskann-100-percent-rust.md +294 -0
- package/docs/ADR-003-semantic-time-travel.md +100 -0
- package/docs/aggregation.md +251 -0
- package/docs/comparison.md +162 -0
- package/docs/deployment-limits.md +87 -0
- package/docs/diskann.md +184 -0
- package/docs/migration-3.0.md +396 -0
- package/docs/performance-budget.md +274 -0
- package/docs/performance.md +117 -0
- package/docs/scaling.md +439 -0
- package/docs/snapshot-safety.md +246 -0
- package/docs/u64-id-space.md +214 -0
- package/docs/verification-report.md +670 -0
- package/native/brainy-native.node +0 -0
- package/native/index.d.ts +3916 -0
- package/package.json +92 -0
- package/scripts/migrate-cortex-2x-to-3x.mjs +970 -0
|
@@ -0,0 +1,1298 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared types for the native module integration.
|
|
3
|
+
*
|
|
4
|
+
* These types are used by both the napi-rs and bun:ffi loaders.
|
|
5
|
+
* Ported from src/embeddings/wasm/types.ts — keeps the same API surface.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Embedding result with metadata
|
|
9
|
+
*/
|
|
10
|
+
export interface EmbeddingResult {
|
|
11
|
+
/** 384-dimensional embedding vector */
|
|
12
|
+
embedding: number[];
|
|
13
|
+
/** Number of tokens processed (0 when using native Candle — handled internally) */
|
|
14
|
+
tokenCount: number;
|
|
15
|
+
/** Processing time in milliseconds */
|
|
16
|
+
processingTimeMs: number;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Engine statistics
|
|
20
|
+
*/
|
|
21
|
+
export interface EngineStats {
|
|
22
|
+
/** Whether the engine is initialized */
|
|
23
|
+
initialized: boolean;
|
|
24
|
+
/** Total number of embeddings generated */
|
|
25
|
+
embedCount: number;
|
|
26
|
+
/** Total processing time in milliseconds */
|
|
27
|
+
totalProcessingTimeMs: number;
|
|
28
|
+
/** Average processing time per embedding */
|
|
29
|
+
avgProcessingTimeMs: number;
|
|
30
|
+
/** Model name */
|
|
31
|
+
modelName: string;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Model constants for all-MiniLM-L6-v2
|
|
35
|
+
*/
|
|
36
|
+
export declare const MODEL_CONSTANTS: {
|
|
37
|
+
readonly HIDDEN_SIZE: 384;
|
|
38
|
+
readonly MAX_SEQUENCE_LENGTH: 256;
|
|
39
|
+
readonly VOCAB_SIZE: 30522;
|
|
40
|
+
readonly MODEL_NAME: "all-MiniLM-L6-v2";
|
|
41
|
+
};
|
|
42
|
+
/**
|
|
43
|
+
* Device information from the native module
|
|
44
|
+
*/
|
|
45
|
+
export interface DeviceInfo {
|
|
46
|
+
/** Active device: "cpu", "cuda", or "metal" */
|
|
47
|
+
device: string;
|
|
48
|
+
/** Whether CUDA is available */
|
|
49
|
+
cudaAvailable: boolean;
|
|
50
|
+
/** Whether Metal is available */
|
|
51
|
+
metalAvailable: boolean;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* License validation result from Rust Ed25519 JWT verification.
|
|
55
|
+
*/
|
|
56
|
+
export interface NativeLicenseResult {
|
|
57
|
+
valid: boolean;
|
|
58
|
+
tier: string;
|
|
59
|
+
email: string;
|
|
60
|
+
expiresAt: number;
|
|
61
|
+
error?: string;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Native module bindings interface.
|
|
65
|
+
* Both napi-rs and bun:ffi loaders implement this interface.
|
|
66
|
+
*/
|
|
67
|
+
export interface NativeBindings {
|
|
68
|
+
validateLicenseKey(token: string): NativeLicenseResult;
|
|
69
|
+
/**
|
|
70
|
+
* Raise the process's soft `RLIMIT_NOFILE` toward its hard cap (zero-
|
|
71
|
+
* config fd headroom for the multi-tenant brain pool). Optional — the
|
|
72
|
+
* bun:ffi loader may not expose it; callers feature-detect. See
|
|
73
|
+
* `native/src/resource_limits.rs` + ADR-004.
|
|
74
|
+
*/
|
|
75
|
+
raiseOpenFileLimit?(): {
|
|
76
|
+
softBefore: number;
|
|
77
|
+
softAfter: number;
|
|
78
|
+
hard: number;
|
|
79
|
+
raised: boolean;
|
|
80
|
+
vmMaxMapCount: number;
|
|
81
|
+
};
|
|
82
|
+
NativeEmbeddingEngine: {
|
|
83
|
+
new (): NativeEmbeddingEngineInstance;
|
|
84
|
+
};
|
|
85
|
+
NativeRoaringBitmap32: {
|
|
86
|
+
new (values?: number[]): NativeRoaringBitmap32Instance;
|
|
87
|
+
and(a: NativeRoaringBitmap32Instance, b: NativeRoaringBitmap32Instance): NativeRoaringBitmap32Instance;
|
|
88
|
+
or(a: NativeRoaringBitmap32Instance, b: NativeRoaringBitmap32Instance): NativeRoaringBitmap32Instance;
|
|
89
|
+
orMany(bitmaps: NativeRoaringBitmap32Instance[]): NativeRoaringBitmap32Instance;
|
|
90
|
+
andNot(a: NativeRoaringBitmap32Instance, b: NativeRoaringBitmap32Instance): NativeRoaringBitmap32Instance;
|
|
91
|
+
xor(a: NativeRoaringBitmap32Instance, b: NativeRoaringBitmap32Instance): NativeRoaringBitmap32Instance;
|
|
92
|
+
deserialize(buffer: Buffer): NativeRoaringBitmap32Instance;
|
|
93
|
+
};
|
|
94
|
+
NativeEntityIdMapper: {
|
|
95
|
+
new (): NativeEntityIdMapperInstance;
|
|
96
|
+
};
|
|
97
|
+
NativeGraphAdjacencyIndex: {
|
|
98
|
+
new (config: NativeGraphAdjacencyConfig): NativeGraphAdjacencyInstance;
|
|
99
|
+
};
|
|
100
|
+
NativeAggregationEngine: {
|
|
101
|
+
new (): NativeAggregationEngineInstance;
|
|
102
|
+
};
|
|
103
|
+
cosineDistance(a: number[], b: number[]): number;
|
|
104
|
+
euclideanDistance(a: number[], b: number[]): number;
|
|
105
|
+
manhattanDistance(a: number[], b: number[]): number;
|
|
106
|
+
dotProductDistance(a: number[], b: number[]): number;
|
|
107
|
+
cosineDistanceBatch(query: number[], vectors: number[][]): number[];
|
|
108
|
+
euclideanDistanceBatch(query: number[], vectors: number[][]): number[];
|
|
109
|
+
quantizeSq8(vector: number[]): {
|
|
110
|
+
quantized: Buffer;
|
|
111
|
+
min: number;
|
|
112
|
+
max: number;
|
|
113
|
+
};
|
|
114
|
+
dequantizeSq8(quantized: Buffer, min: number, max: number): number[];
|
|
115
|
+
serializeSq8(quantized: Buffer, min: number, max: number): Buffer;
|
|
116
|
+
deserializeSq8(data: Buffer): {
|
|
117
|
+
quantized: Buffer;
|
|
118
|
+
min: number;
|
|
119
|
+
max: number;
|
|
120
|
+
};
|
|
121
|
+
cosineDistanceSq8(a: Buffer, aMin: number, aMax: number, b: Buffer, bMin: number, bMax: number): number;
|
|
122
|
+
cosineDistanceSq8Batch(query: Buffer, queryMin: number, queryMax: number, vectors: Buffer[], mins: number[], maxs: number[]): number[];
|
|
123
|
+
quantizeSq4(vector: number[]): {
|
|
124
|
+
quantized: Buffer;
|
|
125
|
+
min: number;
|
|
126
|
+
max: number;
|
|
127
|
+
dim: number;
|
|
128
|
+
};
|
|
129
|
+
dequantizeSq4(quantized: Buffer, min: number, max: number, dim: number): number[];
|
|
130
|
+
cosineDistanceSq4(a: Buffer, aMin: number, aMax: number, aDim: number, b: Buffer, bMin: number, bMax: number, bDim: number): number;
|
|
131
|
+
cosineDistanceSq4Batch(query: Buffer, queryMin: number, queryMax: number, queryDim: number, vectors: Buffer[], mins: number[], maxs: number[], dims: number[]): number[];
|
|
132
|
+
encodeConnections(ids: number[]): Buffer;
|
|
133
|
+
decodeConnections(data: Buffer): number[];
|
|
134
|
+
NativePqCodebook: {
|
|
135
|
+
train(vectors: number[], dim: number, m: number, ksub: number, iterations: number): NativePQCodebookInstance;
|
|
136
|
+
deserialize(data: Buffer): NativePQCodebookInstance;
|
|
137
|
+
};
|
|
138
|
+
msgpackEncode(value: unknown): Buffer;
|
|
139
|
+
msgpackDecode(buffer: Buffer): unknown;
|
|
140
|
+
msgpackEncodeBatch(values: unknown[]): Buffer[];
|
|
141
|
+
msgpackDecodeBatch(buffers: Buffer[]): unknown[];
|
|
142
|
+
NativeMetadataIndex: {
|
|
143
|
+
new (configJson?: string | null, idSpace?: 'u32' | 'u64' | null): NativeMetadataIndexInstance;
|
|
144
|
+
};
|
|
145
|
+
NativeLsmEngine: {
|
|
146
|
+
new (options?: NativeLsmEngineOptions | null): NativeLsmEngineInstance;
|
|
147
|
+
openWithManifest(manifestJson: string, options?: NativeLsmEngineOptions | null): NativeLsmEngineInstance;
|
|
148
|
+
/**
|
|
149
|
+
* **Piece 7 / LSM durability.** Construct an engine whose memtable is
|
|
150
|
+
* backed by mmap'd per-shard append logs under `<dir>/memtable/` (so a
|
|
151
|
+
* crash between inserts and the next `flush()` recovers transparently on
|
|
152
|
+
* reopen). Optionally seeds the epoch counter from a manifest so new
|
|
153
|
+
* mutations outrank every persisted SSTable epoch; when omitted, the
|
|
154
|
+
* post-replay epoch is `max(log_max_epoch) + 1`. The TS wrapper owns the
|
|
155
|
+
* SSTable + manifest disk I/O and registers each persisted SSTable via
|
|
156
|
+
* {@link NativeLsmEngineInstance.registerSstablePath} after open.
|
|
157
|
+
*/
|
|
158
|
+
openWithDir(dir: string, manifestJson?: string | null, options?: NativeLsmEngineOptions | null): NativeLsmEngineInstance;
|
|
159
|
+
};
|
|
160
|
+
NativeColumnStore: {
|
|
161
|
+
new (): NativeColumnStoreInstance;
|
|
162
|
+
};
|
|
163
|
+
NativeUnifiedCache: {
|
|
164
|
+
new (maxSize: number): NativeUnifiedCacheInstance;
|
|
165
|
+
};
|
|
166
|
+
NativeBinaryIdMapper: {
|
|
167
|
+
create(config: NativeBinaryIdMapperConfig): NativeBinaryIdMapperInstance;
|
|
168
|
+
openExisting(config: NativeBinaryIdMapperConfig): NativeBinaryIdMapperInstance;
|
|
169
|
+
};
|
|
170
|
+
sortTopKScores(scores: number[], k: number, descending: boolean): number[];
|
|
171
|
+
getDeviceInfo(): DeviceInfo;
|
|
172
|
+
cosineSimilarity(a: number[], b: number[]): number;
|
|
173
|
+
/** Tokenize text exactly as brainy's `MetadataIndexManager.tokenize` (ASCII `\w`, 2–50 length, dedupe). */
|
|
174
|
+
parityTokenize(text: string): string[];
|
|
175
|
+
/** FNV-1a word hash, identical to brainy's `hashWord` (signed int32, UTF-16 code units). */
|
|
176
|
+
parityHashWord(word: string): number;
|
|
177
|
+
/** Long-value hash, identical to brainy's `hashValue` (`__HASH_<base36>`, UTF-16 code units). */
|
|
178
|
+
parityHashValue(value: string): string;
|
|
179
|
+
/** Normalize a JSON-encoded value exactly as brainy's `normalizeValue` (default config, empty field stats). */
|
|
180
|
+
parityNormalizeValue(valueJson: string): string;
|
|
181
|
+
/** Compare two strings by UTF-8 byte / code-point order (brainy's `compareCodePoints`). Returns -1/0/1. */
|
|
182
|
+
parityCompareCodePoints(a: string, b: string): number;
|
|
183
|
+
/** Sort strings by the native column-store order (UTF-8 byte / code-point order). */
|
|
184
|
+
paritySortStrings(values: string[]): string[];
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Configuration for `NativeBinaryIdMapper.create` / `openExisting`.
|
|
188
|
+
* Mirror of the napi-exposed `BinaryIdMapperConfig` in
|
|
189
|
+
* `native/src/binary_id_mapper_napi.rs`.
|
|
190
|
+
*/
|
|
191
|
+
export interface NativeBinaryIdMapperConfig {
|
|
192
|
+
/** Path for `uuid_to_int.mkv` (extendible-hash KV file). */
|
|
193
|
+
uuidToIntPath: string;
|
|
194
|
+
/** Path for `int_to_uuid.bin` (direct-indexed sparse file). */
|
|
195
|
+
intToUuidPath: string;
|
|
196
|
+
/** Sparse file size for `int_to_uuid` in bytes. Default 32 GB. */
|
|
197
|
+
intToUuidSize?: bigint;
|
|
198
|
+
/** Sparse file size for `uuid_to_int` in bytes. Default 32 GB. */
|
|
199
|
+
uuidToIntSize?: bigint;
|
|
200
|
+
/** MmapKv bucket capacity (entries per bucket). Default 16. */
|
|
201
|
+
bucketCapacity?: number;
|
|
202
|
+
/** MmapKv max_global_depth (directory depth cap). Default 28. */
|
|
203
|
+
maxGlobalDepth?: number;
|
|
204
|
+
/**
|
|
205
|
+
* Entity-int wire-width. Default `"u32"` (cortex 2.x compatible —
|
|
206
|
+
* caps at 4.29 B entries). Set to `"u64"` to opt into the Piece 10
|
|
207
|
+
* U64 IdSpace (persists 8-byte ints + bumps `int_to_uuid.bin` to a
|
|
208
|
+
* v2 header). U64 mode is required when targeting >4.29 B entities.
|
|
209
|
+
*
|
|
210
|
+
* Ignored at `openExisting` time: the on-disk format dictates the
|
|
211
|
+
* IdSpace and any user-provided value is cross-checked against the
|
|
212
|
+
* file header (mismatch throws).
|
|
213
|
+
*/
|
|
214
|
+
idSpace?: 'u32' | 'u64';
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* `NativeBinaryIdMapper` instance (cor 3.0). Bidirectional
|
|
218
|
+
* UUID ↔ int mapping backed by two mmap'd files.
|
|
219
|
+
*
|
|
220
|
+
* ## IdSpace surface
|
|
221
|
+
*
|
|
222
|
+
* The mapper has two parallel method surfaces selected by the
|
|
223
|
+
* `idSpace` mode (configured at create time, persisted in the
|
|
224
|
+
* `int_to_uuid.bin` header):
|
|
225
|
+
*
|
|
226
|
+
* - **U32 mode** (default, cortex 2.x compatible): `getOrAssign` /
|
|
227
|
+
* `getInt` / `getUuid` / `size` / `nextInt` return `number`. The
|
|
228
|
+
* BigInt siblings still work — they widen to `bigint` at the JS
|
|
229
|
+
* boundary — but the `number` surface is the primary API.
|
|
230
|
+
* - **U64 mode** (Piece 10 — opt-in via `config.idSpace = 'u64'`):
|
|
231
|
+
* the `number`-returning methods throw with `"requires u32 mode"`.
|
|
232
|
+
* Callers must use the BigInt siblings (`getOrAssignBig`,
|
|
233
|
+
* `getIntBig`, `getUuidBig`, `sizeBig`, `nextIntBig`).
|
|
234
|
+
*
|
|
235
|
+
* `idSpace()` reports the actual mode of the open file, which is
|
|
236
|
+
* authoritative over any user-supplied config at `openExisting` time.
|
|
237
|
+
*/
|
|
238
|
+
export interface NativeBinaryIdMapperInstance {
|
|
239
|
+
/** Retrieve the int for `uuid`, allocating one if absent. */
|
|
240
|
+
getOrAssign(uuid: Buffer): number;
|
|
241
|
+
/** Lookup the int for `uuid`. Null if absent or tombstoned. */
|
|
242
|
+
getInt(uuid: Buffer): number | null;
|
|
243
|
+
/** Lookup the UUID for `int`. Null if absent or tombstoned. */
|
|
244
|
+
getUuid(int: number): Buffer | null;
|
|
245
|
+
/** Live (non-tombstone) entry count. */
|
|
246
|
+
size(): number;
|
|
247
|
+
/** Largest int ever assigned + 1. */
|
|
248
|
+
nextInt(): number;
|
|
249
|
+
/** All live int ids. At billion-scale, prefer streaming hooks (TBD). */
|
|
250
|
+
getAllIntIds(): number[];
|
|
251
|
+
/**
|
|
252
|
+
* Retrieve the int for `uuid`, allocating one if absent. Returns a
|
|
253
|
+
* `bigint` so the full u64 range is representable.
|
|
254
|
+
*/
|
|
255
|
+
getOrAssignBig(uuid: Buffer): bigint;
|
|
256
|
+
/** Lookup the int for `uuid`. `null` if absent or tombstoned. */
|
|
257
|
+
getIntBig(uuid: Buffer): bigint | null;
|
|
258
|
+
/** Lookup the UUID for `int`. `null` if absent or tombstoned. */
|
|
259
|
+
getUuidBig(int: bigint): Buffer | null;
|
|
260
|
+
/**
|
|
261
|
+
* Batch sibling of {@link getUuidBig} (ADR-006 Q5). Resolve many ints
|
|
262
|
+
* to their UUID bytes in one native call; `null` per unmapped int,
|
|
263
|
+
* index-aligned with the input.
|
|
264
|
+
*/
|
|
265
|
+
intsToUuidsBig(ints: bigint[]): (Buffer | null)[];
|
|
266
|
+
/** Live (non-tombstone) entry count as a `bigint`. */
|
|
267
|
+
sizeBig(): bigint;
|
|
268
|
+
/** Largest int ever assigned + 1, as a `bigint`. */
|
|
269
|
+
nextIntBig(): bigint;
|
|
270
|
+
/** Tombstone an entry. Returns true if it was present. */
|
|
271
|
+
remove(uuid: Buffer): boolean;
|
|
272
|
+
/** Flush dirty mmap pages to disk. */
|
|
273
|
+
flush(): void;
|
|
274
|
+
/** Filesystem paths to the underlying files (`[uuid_to_int, int_to_uuid]`). */
|
|
275
|
+
paths(): string[];
|
|
276
|
+
/** Diagnostic snapshot for tooling. */
|
|
277
|
+
diagnostics(): NativeBinaryIdMapperDiagnostics;
|
|
278
|
+
/**
|
|
279
|
+
* Report the mapper's actual IdSpace mode. Authoritative over the
|
|
280
|
+
* config's `idSpace` field — the on-disk header wins.
|
|
281
|
+
*/
|
|
282
|
+
idSpace(): 'u32' | 'u64';
|
|
283
|
+
}
|
|
284
|
+
/** Diagnostic snapshot returned by `NativeBinaryIdMapper.diagnostics`. */
|
|
285
|
+
export interface NativeBinaryIdMapperDiagnostics {
|
|
286
|
+
uuidSize: number;
|
|
287
|
+
intSize: number;
|
|
288
|
+
intToUuidEntrySize: number;
|
|
289
|
+
assignShards: number;
|
|
290
|
+
size: number;
|
|
291
|
+
nextInt: number;
|
|
292
|
+
}
|
|
293
|
+
/**
|
|
294
|
+
* Native embedding engine instance
|
|
295
|
+
*/
|
|
296
|
+
export interface NativeEmbeddingEngineInstance {
|
|
297
|
+
load(modelBytes: Buffer, tokenizerBytes: Buffer, configBytes: Buffer): void;
|
|
298
|
+
isReady(): boolean;
|
|
299
|
+
embed(text: string): number[];
|
|
300
|
+
embedBatch(texts: string[]): number[][];
|
|
301
|
+
dimension(): number;
|
|
302
|
+
maxSequenceLength(): number;
|
|
303
|
+
getStats(): EngineStats;
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* Native roaring bitmap instance
|
|
307
|
+
*/
|
|
308
|
+
export interface NativeRoaringBitmap32Instance {
|
|
309
|
+
readonly size: number;
|
|
310
|
+
readonly isEmpty: boolean;
|
|
311
|
+
add(value: number): void;
|
|
312
|
+
delete(value: number): void;
|
|
313
|
+
tryAdd(value: number): boolean;
|
|
314
|
+
has(value: number): boolean;
|
|
315
|
+
toArray(): number[];
|
|
316
|
+
serialize(): Buffer;
|
|
317
|
+
serializationSizeInBytes(): number;
|
|
318
|
+
minimum(): number | null;
|
|
319
|
+
maximum(): number | null;
|
|
320
|
+
}
|
|
321
|
+
/**
|
|
322
|
+
* Statistics from native EntityIdMapper
|
|
323
|
+
*/
|
|
324
|
+
export interface NativeEntityIdMapperStats {
|
|
325
|
+
/** Number of active UUID ↔ integer mappings */
|
|
326
|
+
mappings: number;
|
|
327
|
+
/** Next integer ID to assign */
|
|
328
|
+
nextId: number;
|
|
329
|
+
/** Whether there are unsaved changes */
|
|
330
|
+
dirty: boolean;
|
|
331
|
+
/** Estimated memory usage in bytes */
|
|
332
|
+
memoryEstimate: number;
|
|
333
|
+
}
|
|
334
|
+
/**
|
|
335
|
+
* Native EntityIdMapper instance.
|
|
336
|
+
* Bidirectional UUID ↔ integer mapping for roaring bitmap integration.
|
|
337
|
+
*/
|
|
338
|
+
export interface NativeEntityIdMapperInstance {
|
|
339
|
+
/** Load state from JSON string (storage format) */
|
|
340
|
+
loadFromJson(json: string): void;
|
|
341
|
+
/** Serialize state to JSON string for storage persistence */
|
|
342
|
+
saveToJson(): string;
|
|
343
|
+
/** Get integer ID for UUID, assigning a new one if not mapped */
|
|
344
|
+
getOrAssign(uuid: string): number;
|
|
345
|
+
/** Get UUID for integer ID. Returns null if not found */
|
|
346
|
+
getUuid(intId: number): string | null;
|
|
347
|
+
/** Get integer ID for UUID without assigning. Returns null if not found */
|
|
348
|
+
getInt(uuid: string): number | null;
|
|
349
|
+
/** Check if UUID has been assigned an integer ID */
|
|
350
|
+
has(uuid: string): boolean;
|
|
351
|
+
/** Remove mapping for UUID. Returns true if it existed */
|
|
352
|
+
remove(uuid: string): boolean;
|
|
353
|
+
/** Get total number of mappings */
|
|
354
|
+
readonly size: number;
|
|
355
|
+
/** Convert array of UUIDs to array of integers (assigns new IDs as needed) */
|
|
356
|
+
uuidsToInts(uuids: string[]): number[];
|
|
357
|
+
/** Convert array of integers to array of UUIDs (skips unknown IDs) */
|
|
358
|
+
intsToUuids(ints: number[]): string[];
|
|
359
|
+
/**
|
|
360
|
+
* BigInt sibling of {@link intsToUuids} (ADR-006 Q5). Resolve many
|
|
361
|
+
* u64 ints to UUID strings in one native call, index-aligned with the
|
|
362
|
+
* input — the `entityIntsToUuids` provider path for `Subgraph.nodes`.
|
|
363
|
+
*/
|
|
364
|
+
intsToUuidsBig(ints: bigint[]): string[];
|
|
365
|
+
/** Whether there are unsaved changes */
|
|
366
|
+
isDirty(): boolean;
|
|
367
|
+
/** Clear all mappings and reset counter */
|
|
368
|
+
clear(): void;
|
|
369
|
+
/** Get statistics about the mapper */
|
|
370
|
+
getStats(): NativeEntityIdMapperStats;
|
|
371
|
+
}
|
|
372
|
+
/**
|
|
373
|
+
* Configuration for NativeGraphAdjacencyIndex
|
|
374
|
+
*/
|
|
375
|
+
export interface NativeGraphAdjacencyConfig {
|
|
376
|
+
/** MemTable flush threshold per tree. Default: 100_000 */
|
|
377
|
+
memTableThreshold?: number;
|
|
378
|
+
/** Max SSTables per level per tree. Default: 10 */
|
|
379
|
+
maxSstablesPerLevel?: number;
|
|
380
|
+
/**
|
|
381
|
+
* **Piece 12.** Optional directory for the verb-ID interning
|
|
382
|
+
* namespace's mmap-backed files. When set, the namespace writes
|
|
383
|
+
* `verb-uuid-to-int.mkv` + `verb-int-to-uuid.bin` under this
|
|
384
|
+
* directory so verb-ID assignments survive process restart. When
|
|
385
|
+
* omitted, the namespace falls back to an in-memory interner —
|
|
386
|
+
* adequate for tests + transient brains but loses assignments on
|
|
387
|
+
* exit.
|
|
388
|
+
*/
|
|
389
|
+
persistenceDir?: string;
|
|
390
|
+
/**
|
|
391
|
+
* **Piece B (cor 3.0).** Wire width of the verb-ID interning
|
|
392
|
+
* namespace + the live-verb / per-(type, subtype) membership
|
|
393
|
+
* bitmaps. `"U32"` (raw napi default, brainy 7.x-compatible) caps
|
|
394
|
+
* at ~4.29 B unique verb-IDs. `"U64"` (cor 3.0 wrapper default
|
|
395
|
+
* via `GraphAdjacencyIndex`) lifts the cap to `u64::MAX - 1`. The
|
|
396
|
+
* choice is independent of the brain's *entity* IdSpace.
|
|
397
|
+
* Case-sensitive.
|
|
398
|
+
*/
|
|
399
|
+
verbIdSpace?: 'U32' | 'U64';
|
|
400
|
+
}
|
|
401
|
+
/**
|
|
402
|
+
* Result from addVerbWithEndpoints — indicates which trees need
|
|
403
|
+
* flushing and (cor 3.0 brainy-8.0 lockstep) returns the verb's
|
|
404
|
+
* interned `u64` integer ID so brainy 8.0's
|
|
405
|
+
* `verb_int ↔ verb_id_string` sidecar can be populated without a
|
|
406
|
+
* second FFI hop.
|
|
407
|
+
*/
|
|
408
|
+
export interface GraphAddVerbResult {
|
|
409
|
+
needsFlushSource: boolean;
|
|
410
|
+
needsFlushTarget: boolean;
|
|
411
|
+
needsFlushVerbsSource: boolean;
|
|
412
|
+
needsFlushVerbsTarget: boolean;
|
|
413
|
+
/**
|
|
414
|
+
* **Brainy 8.0 lockstep** (PLATFORM-HANDOFF
|
|
415
|
+
* BRAINY-8.0-RENAME-COORDINATION A.3): the verb's interned `u64`
|
|
416
|
+
* integer ID. Brainy 8.0 reads this directly to populate its
|
|
417
|
+
* sidecar map — one FFI round-trip per add saved at
|
|
418
|
+
* billion-scale ingest.
|
|
419
|
+
*/
|
|
420
|
+
verbInt: bigint;
|
|
421
|
+
}
|
|
422
|
+
/**
|
|
423
|
+
* Result from {@link NativeGraphAdjacencyInstance.getVerbEndpoints}.
|
|
424
|
+
* Carries the verb's (source, target) entity ints as u64 BigInts so
|
|
425
|
+
* brainy 8.0 callers can round-trip them via the brain's canonical
|
|
426
|
+
* `BinaryIdMapper` without precision loss at 10B+ scale.
|
|
427
|
+
*/
|
|
428
|
+
export interface VerbEndpointsResult {
|
|
429
|
+
sourceInt: bigint;
|
|
430
|
+
targetInt: bigint;
|
|
431
|
+
/**
|
|
432
|
+
* The verb's `VerbType` index (generation-stable; ADR-006 Phase A). Carried on
|
|
433
|
+
* the endpoint record so an as-of-`g` read can type an edge whose verb was
|
|
434
|
+
* removed after `g`. Present on the Rust struct + generated `index.d.ts`; this
|
|
435
|
+
* hand-maintained mirror was missing it (latent type-drift). (#79 cluster C.)
|
|
436
|
+
*/
|
|
437
|
+
verbType: number;
|
|
438
|
+
}
|
|
439
|
+
/**
|
|
440
|
+
* Statistics for the graph adjacency index.
|
|
441
|
+
*
|
|
442
|
+
* The legacy `number`-typed counters (`totalRelationships`,
|
|
443
|
+
* `verbIdCount`, `relationshipTypeCount`) cap at `0xFFFFFFFF` —
|
|
444
|
+
* preserved for cortex 2.x-compatible callers. The BigInt sibling
|
|
445
|
+
* fields (`*Big`) carry the same values losslessly at billion scale
|
|
446
|
+
* and should be preferred by any new consumer.
|
|
447
|
+
*/
|
|
448
|
+
export interface GraphAdjacencyStats {
|
|
449
|
+
totalRelationships: number;
|
|
450
|
+
verbIdCount: number;
|
|
451
|
+
relationshipTypeCount: number;
|
|
452
|
+
sourceMemTableSize: number;
|
|
453
|
+
targetMemTableSize: number;
|
|
454
|
+
sourceSstableCount: number;
|
|
455
|
+
targetSstableCount: number;
|
|
456
|
+
sourceMemTableMemory: number;
|
|
457
|
+
targetMemTableMemory: number;
|
|
458
|
+
/** BigInt total relationship count (no u32 ceiling). */
|
|
459
|
+
totalRelationshipsBig: bigint;
|
|
460
|
+
/** BigInt verb id count. */
|
|
461
|
+
verbIdCountBig: bigint;
|
|
462
|
+
/** BigInt relationship type count. */
|
|
463
|
+
relationshipTypeCountBig: bigint;
|
|
464
|
+
}
|
|
465
|
+
/**
|
|
466
|
+
* Result from flushing a tree's MemTable to binary .sst format
|
|
467
|
+
* (adapter-controlled).
|
|
468
|
+
*
|
|
469
|
+
* **Cor 3.0 Piece D**: `entryCount` and `valueCount` (renamed from
|
|
470
|
+
* pre-D `relationshipCount`) are u32 narrowings of the underlying u64
|
|
471
|
+
* counters. Per-SSTable totals never realistically approach the cap.
|
|
472
|
+
*/
|
|
473
|
+
export interface NativeBinaryFlushResult {
|
|
474
|
+
treeName: string;
|
|
475
|
+
sstableId: string;
|
|
476
|
+
data: Buffer;
|
|
477
|
+
level: number;
|
|
478
|
+
entryCount: number;
|
|
479
|
+
/** Sum of values across all entries in the flushed SSTable. */
|
|
480
|
+
valueCount: number;
|
|
481
|
+
}
|
|
482
|
+
/**
|
|
483
|
+
* Result from compacting a tree's level to binary .sst format
|
|
484
|
+
* (adapter-controlled).
|
|
485
|
+
*/
|
|
486
|
+
export interface NativeBinaryCompactResult {
|
|
487
|
+
treeName: string;
|
|
488
|
+
newSstableId: string;
|
|
489
|
+
data: Buffer;
|
|
490
|
+
newLevel: number;
|
|
491
|
+
oldIds: string[];
|
|
492
|
+
}
|
|
493
|
+
/**
|
|
494
|
+
* Native graph adjacency index instance.
|
|
495
|
+
* Wraps 4 u64-keyed LSM-trees (source, target, verbs-source,
|
|
496
|
+
* verbs-target) plus verb tracking and relationship counts.
|
|
497
|
+
*
|
|
498
|
+
* **Cor 3.0 Piece D**: entity ids cross as `bigint` (the canonical
|
|
499
|
+
* u64 from brainy's `BinaryIdMapper`). Verb ints (interned via cor's
|
|
500
|
+
* `verb_id_namespace`) also cross as `bigint`. Verb-id strings only
|
|
501
|
+
* appear at add-time (`addVerbWithEndpoints*`, `trackVerbId`) and
|
|
502
|
+
* endpoint lookups (`getVerbEndpoints`).
|
|
503
|
+
*
|
|
504
|
+
* Legacy `addVerb` / `addVerbByIndex` / `addVerb*WithSubtype` /
|
|
505
|
+
* `removeVerbByIndex` / `removeVerb*WithSubtype` / `trackVerbIdByIndex`
|
|
506
|
+
* / `flushTree` / `loadSstable` / `compactTree` are removed; brainy 8.0
|
|
507
|
+
* uses `addVerbWithEndpoints*` exclusively. See PLATFORM-HANDOFF row
|
|
508
|
+
* `CTX-30-U64-CONSISTENCY` for the brainy-side migration contract.
|
|
509
|
+
*/
|
|
510
|
+
/**
|
|
511
|
+
* Knobs for {@link NativeGraphAdjacencyInstance.traverse} (ADR-006 Phase A).
|
|
512
|
+
* Field names cross the napi boundary as camelCase.
|
|
513
|
+
*/
|
|
514
|
+
export interface TraverseOptions {
|
|
515
|
+
/** `"out"`, `"in"`, or `"both"` (any other value → `"both"`). */
|
|
516
|
+
direction: string;
|
|
517
|
+
/** Max hop count from the seed set (`0` = seeds only, no edges). */
|
|
518
|
+
depth: number;
|
|
519
|
+
/** Restrict to these `VerbType` indices (empty/omitted = all types). */
|
|
520
|
+
verbTypes?: Array<number>;
|
|
521
|
+
/** Cap on distinct nodes (sets `truncated` when hit). */
|
|
522
|
+
maxNodes?: number;
|
|
523
|
+
/** Cap on edges (sets `truncated` when hit). */
|
|
524
|
+
maxEdges?: number;
|
|
525
|
+
/** Capture edge columns (default `true`); `false` = nodes-only. */
|
|
526
|
+
includeEdges?: boolean;
|
|
527
|
+
/** Populate `nodeDepth` (default `true`). */
|
|
528
|
+
includeDepth?: boolean;
|
|
529
|
+
}
|
|
530
|
+
/**
|
|
531
|
+
* Columnar subgraph — parallel little-endian {@link Buffer} columns
|
|
532
|
+
* (ADR-006 wire format). View each as a typed array (`BigInt64Array` /
|
|
533
|
+
* `Uint16Array` / `Uint8Array`) with zero element copies; brainy resolves
|
|
534
|
+
* the u64 ints to UUIDs lazily. Shared by `traverse` + cursor chunks.
|
|
535
|
+
*/
|
|
536
|
+
export interface NativeSubgraph {
|
|
537
|
+
/** Distinct node ints, BFS-discovery order, `u64` LE. */
|
|
538
|
+
nodes: Buffer;
|
|
539
|
+
/** Per-node BFS depth (`u8`, parallel to `nodes`); empty when not requested. */
|
|
540
|
+
nodeDepth: Buffer;
|
|
541
|
+
/** Per-edge source node int (`u64` LE). */
|
|
542
|
+
edgeSources: Buffer;
|
|
543
|
+
/** Per-edge target node int (`u64` LE). */
|
|
544
|
+
edgeTargets: Buffer;
|
|
545
|
+
/** Per-edge interned verb int (`u64` LE); empty under `"light"` projection. */
|
|
546
|
+
edgeVerbInts: Buffer;
|
|
547
|
+
/** Per-edge verb-type index (`u16` LE). */
|
|
548
|
+
edgeTypes: Buffer;
|
|
549
|
+
/** `true` if a node/edge cap truncated a `traverse` (cursors page instead). */
|
|
550
|
+
truncated: boolean;
|
|
551
|
+
}
|
|
552
|
+
/**
|
|
553
|
+
* Options for {@link NativeGraphAdjacencyInstance.graphCursorOpen}
|
|
554
|
+
* (ADR-006 Phase B). Mirrors brainy's `GraphCursorOptions` + the trailing
|
|
555
|
+
* `generation`, flattened so seeds arrive as ints OR cor's opaque id-set.
|
|
556
|
+
*/
|
|
557
|
+
export interface GraphCursorOpenOptions {
|
|
558
|
+
/** `"out"`/`"in"`/`"both"` (default). Honored for seeded cursors only. */
|
|
559
|
+
direction?: string;
|
|
560
|
+
/** `"light"` (default; omits `edgeVerbInts`) or `"full"`. */
|
|
561
|
+
projection?: string;
|
|
562
|
+
/** Seed ints. Omitted (with `seedsOpaque`) = whole-graph walk. */
|
|
563
|
+
seeds?: Array<bigint>;
|
|
564
|
+
/** Seed set as cor's OpaqueIdSet envelope (the `find()`→export fusion). */
|
|
565
|
+
seedsOpaque?: Buffer;
|
|
566
|
+
/** Resume token from a prior chunk (reopen after `SnapshotExpired`). */
|
|
567
|
+
cursor?: string;
|
|
568
|
+
/** Explicit as-of generation to pin (omitted = current). */
|
|
569
|
+
generation?: bigint;
|
|
570
|
+
}
|
|
571
|
+
/**
|
|
572
|
+
* One streamed chunk from
|
|
573
|
+
* {@link NativeGraphAdjacencyInstance.graphCursorNext} (ADR-006 Phase B).
|
|
574
|
+
*/
|
|
575
|
+
export interface NativeGraphCursorChunk {
|
|
576
|
+
/** This chunk's nodes + edges, columnar (`nodeDepth` always empty here). */
|
|
577
|
+
subgraph: NativeSubgraph;
|
|
578
|
+
/** Resume token; `undefined` once the walk is exhausted. */
|
|
579
|
+
cursor?: string;
|
|
580
|
+
/** `true` once the walk is exhausted. */
|
|
581
|
+
done: boolean;
|
|
582
|
+
}
|
|
583
|
+
/** Per-node scalar scores (`pageRank` / `topByDegree`), descending — decoded
|
|
584
|
+
* to brainy's `GraphScores` (BigInt64Array + Float64Array). */
|
|
585
|
+
export interface NativeGraphScores {
|
|
586
|
+
/** Entity ints, `u64` LE. */
|
|
587
|
+
nodeInts: Buffer;
|
|
588
|
+
/** Per-node score, `f64` LE, parallel to `nodeInts`. */
|
|
589
|
+
scores: Buffer;
|
|
590
|
+
}
|
|
591
|
+
/** Connected-component labelling (`connectedComponents`) — decoded to brainy's
|
|
592
|
+
* `GraphComponents`. */
|
|
593
|
+
export interface NativeGraphComponents {
|
|
594
|
+
/** Entity ints, `u64` LE. */
|
|
595
|
+
nodeInts: Buffer;
|
|
596
|
+
/** Component label per node, `u32` LE, parallel to `nodeInts`. */
|
|
597
|
+
componentIds: Buffer;
|
|
598
|
+
/** Distinct component total (separate field, not `max(componentIds)`). */
|
|
599
|
+
componentCount: number;
|
|
600
|
+
}
|
|
601
|
+
/** A shortest path (`shortestPath`, `null` when unreachable) — decoded to
|
|
602
|
+
* brainy's `GraphPath`. */
|
|
603
|
+
export interface NativeGraphPath {
|
|
604
|
+
/** Path node sequence, `u64` LE. */
|
|
605
|
+
nodeInts: Buffer;
|
|
606
|
+
/** Edge verb ints, `u64` LE; `length === nodeInts.length/8 - 1` (gaps). */
|
|
607
|
+
edgeVerbInts: Buffer;
|
|
608
|
+
/** Hop count (cor stores no edge weights → hops-only). */
|
|
609
|
+
cost: number;
|
|
610
|
+
}
|
|
611
|
+
/** Options for {@link NativeGraphAdjacencyInstance.pageRank}. */
|
|
612
|
+
export interface NativePageRankOptions {
|
|
613
|
+
damping?: number;
|
|
614
|
+
maxIterations?: number;
|
|
615
|
+
tolerance?: number;
|
|
616
|
+
undirected?: boolean;
|
|
617
|
+
topK?: number;
|
|
618
|
+
/** Contextual scoping: rank only the seed-bounded induced subgraph (O(scope)). */
|
|
619
|
+
seeds?: Array<bigint>;
|
|
620
|
+
seedsOpaque?: Buffer;
|
|
621
|
+
scopeDepth?: number;
|
|
622
|
+
}
|
|
623
|
+
/** Options for {@link NativeGraphAdjacencyInstance.connectedComponents}. */
|
|
624
|
+
export interface NativeComponentsOptions {
|
|
625
|
+
/** `"weak"` (default) or `"strong"`. */
|
|
626
|
+
mode?: string;
|
|
627
|
+
/** Contextual scoping: label only the seed-bounded induced subgraph. */
|
|
628
|
+
seeds?: Array<bigint>;
|
|
629
|
+
seedsOpaque?: Buffer;
|
|
630
|
+
scopeDepth?: number;
|
|
631
|
+
}
|
|
632
|
+
/** Options for {@link NativeGraphAdjacencyInstance.shortestPath} (hops-only —
|
|
633
|
+
* no `weight` field; cor stores no edge weights). */
|
|
634
|
+
export interface NativeShortestPathOptions {
|
|
635
|
+
direction?: string;
|
|
636
|
+
verbTypes?: Array<number>;
|
|
637
|
+
maxDepth?: number;
|
|
638
|
+
}
|
|
639
|
+
/** Options for {@link NativeGraphAdjacencyInstance.neighborhoodSample}. */
|
|
640
|
+
export interface NativeNeighborhoodSampleOptions {
|
|
641
|
+
direction?: string;
|
|
642
|
+
depth: number;
|
|
643
|
+
fanout: number;
|
|
644
|
+
/** RNG seed (a JS number; narrowed to u64, NaN/negative → 0). */
|
|
645
|
+
seed?: number;
|
|
646
|
+
maxNodes?: number;
|
|
647
|
+
includeDepth?: boolean;
|
|
648
|
+
}
|
|
649
|
+
/** Options for {@link NativeGraphAdjacencyInstance.topByDegree}. */
|
|
650
|
+
export interface NativeTopByDegreeOptions {
|
|
651
|
+
/** `"out"` (default), `"in"`, or `"both"`. */
|
|
652
|
+
direction?: string;
|
|
653
|
+
topK: number;
|
|
654
|
+
/** Contextual scoping: rank degrees only within the seed-bounded subgraph. */
|
|
655
|
+
seeds?: Array<bigint>;
|
|
656
|
+
seedsOpaque?: Buffer;
|
|
657
|
+
scopeDepth?: number;
|
|
658
|
+
}
|
|
659
|
+
export interface NativeGraphAdjacencyInstance {
|
|
660
|
+
/**
|
|
661
|
+
* Add a verb with its `(sourceInt, targetInt)` endpoint pair.
|
|
662
|
+
* Updates all 4 LSM-trees, the membership bitmap, the per-type count
|
|
663
|
+
* slot, and (when `subtype` is set) the per-(type, subtype) bitmap +
|
|
664
|
+
* count slot. Returns which trees need flushing.
|
|
665
|
+
*
|
|
666
|
+
* The `verbType` string is resolved to its canonical `VerbTypeEnum`
|
|
667
|
+
* index — unknown strings coerce to `VerbType.RelatedTo` (3).
|
|
668
|
+
*/
|
|
669
|
+
addVerbWithEndpoints(verbId: string, sourceInt: bigint, targetInt: bigint, verbType: string, subtype: string,
|
|
670
|
+
/** brainy's commit generation for this write — recorded in the
|
|
671
|
+
* endpoint version chain so `getVerbEndpointsAtGeneration` (and
|
|
672
|
+
* thus `db.asOf(g)` graph hops) resolve historical edges. */
|
|
673
|
+
generation: bigint): GraphAddVerbResult;
|
|
674
|
+
/**
|
|
675
|
+
* Strict variant of {@link addVerbWithEndpoints} taking the canonical
|
|
676
|
+
* verb-type index directly. Brainy 8.0 happy path — resolves the
|
|
677
|
+
* index once via `TypeUtils.getVerbIndex` and hands it in.
|
|
678
|
+
* Out-of-range indices coerce to `VerbType.RelatedTo` (3).
|
|
679
|
+
*/
|
|
680
|
+
addVerbWithEndpointsByIndex(verbId: string, sourceInt: bigint, targetInt: bigint, verbTypeIndex: number, subtype: string,
|
|
681
|
+
/** brainy's commit generation — see {@link addVerbWithEndpoints}. */
|
|
682
|
+
generation: bigint): GraphAddVerbResult;
|
|
683
|
+
/**
|
|
684
|
+
* Remove a verb (tombstone deletion). LSM-tree edges persist —
|
|
685
|
+
* removed verbs are filtered out by the membership bitmap on query.
|
|
686
|
+
* When `subtype` is set, the per-(type, subtype) bitmap + count slot
|
|
687
|
+
* are also updated.
|
|
688
|
+
*/
|
|
689
|
+
removeVerb(verbId: string, verbTypeIndex: number, subtype: string,
|
|
690
|
+
/** brainy's commit generation — records the tombstone in the
|
|
691
|
+
* endpoint version chain at this generation for `db.asOf(g)`. */
|
|
692
|
+
generation: bigint): void;
|
|
693
|
+
/**
|
|
694
|
+
* Look up a verb's `(sourceInt, targetInt)` endpoint pair by verb-ID.
|
|
695
|
+
* Returns `null` when the verb was never added or has been tombstoned.
|
|
696
|
+
*/
|
|
697
|
+
getVerbEndpoints(verbId: string): VerbEndpointsResult | null;
|
|
698
|
+
/**
|
|
699
|
+
* Number of verb-IDs currently carrying endpoint info. Useful for
|
|
700
|
+
* tests + ops dashboards.
|
|
701
|
+
*/
|
|
702
|
+
verbEndpointsCount(): bigint;
|
|
703
|
+
/**
|
|
704
|
+
* Get neighbor entity ints for an entity.
|
|
705
|
+
* `direction`: `"in"`, `"out"`, or `"both"`. Returns deduped neighbor
|
|
706
|
+
* ints with optional pagination.
|
|
707
|
+
*/
|
|
708
|
+
getNeighbors(id: bigint, direction: string, limit?: number | null, offset?: number | null): bigint[];
|
|
709
|
+
/**
|
|
710
|
+
* Get verb ints by source entity, filtered by the live-membership
|
|
711
|
+
* bitmap (tombstone filtering). Returns verb ints (BigInt); brainy
|
|
712
|
+
* 8.0 owns the verb_int ↔ verb_id_string mapping on its side.
|
|
713
|
+
*/
|
|
714
|
+
getVerbIdsBySource(sourceInt: bigint, limit?: number | null, offset?: number | null): bigint[];
|
|
715
|
+
/**
|
|
716
|
+
* Get verb ints by target entity, filtered by the live-membership
|
|
717
|
+
* bitmap. See {@link getVerbIdsBySource}.
|
|
718
|
+
*/
|
|
719
|
+
getVerbIdsByTarget(targetInt: bigint, limit?: number | null, offset?: number | null): bigint[];
|
|
720
|
+
/**
|
|
721
|
+
* Get verb ints by source entity, filtered by both the live-membership
|
|
722
|
+
* bitmap AND the per-(verbTypeIndex, subtype) bitmap. Both filter
|
|
723
|
+
* dimensions are optional.
|
|
724
|
+
*/
|
|
725
|
+
getVerbIdsBySourceWithSubtype(sourceInt: bigint, verbTypeIndex?: number | null, subtype?: string | null, limit?: number | null, offset?: number | null): bigint[];
|
|
726
|
+
/**
|
|
727
|
+
* Get verb ints by target entity, filtered by both the live-membership
|
|
728
|
+
* bitmap AND the per-(verbTypeIndex, subtype) bitmap.
|
|
729
|
+
*/
|
|
730
|
+
getVerbIdsByTargetWithSubtype(targetInt: bigint, verbTypeIndex?: number | null, subtype?: string | null, limit?: number | null, offset?: number | null): bigint[];
|
|
731
|
+
/** Per-(verbTypeIndex, subtype) relationship count. */
|
|
732
|
+
getRelationshipSubtypeCount(verbTypeIndex: number, subtype: string): bigint;
|
|
733
|
+
/** Distinct subtype values observed for the given verb-type. */
|
|
734
|
+
listVerbSubtypes(verbTypeIndex: number): string[];
|
|
735
|
+
/**
|
|
736
|
+
* Track a verb ID without adding to LSM-trees. Used during rebuild
|
|
737
|
+
* when LSM-trees are already loaded from persisted SSTables.
|
|
738
|
+
*/
|
|
739
|
+
trackVerbId(verbId: string, verbType: string): void;
|
|
740
|
+
/** Clear membership bitmap + relationship counts (rebuild prelude). */
|
|
741
|
+
clearVerbTracking(): void;
|
|
742
|
+
/**
|
|
743
|
+
* Generation counter — bumped on every read-visible mutation. TS
|
|
744
|
+
* readers poll this to invalidate caches when state changes.
|
|
745
|
+
*/
|
|
746
|
+
currentGeneration(): bigint;
|
|
747
|
+
/**
|
|
748
|
+
* Bounded multi-hop BFS from explicit `seeds`, returning the reachable
|
|
749
|
+
* subgraph (nodes + edges + per-node depth) in one boundary crossing.
|
|
750
|
+
* `generation` (omitted = now) selects an as-of-`g` time-travel read.
|
|
751
|
+
*/
|
|
752
|
+
traverse(seeds: Array<bigint>, opts: TraverseOptions, generation?: bigint | null): NativeSubgraph;
|
|
753
|
+
/**
|
|
754
|
+
* {@link traverse} seeded from a `find()`-result OpaqueIdSet envelope
|
|
755
|
+
* (the query→expand fusion — no id materialization in TS). Fails loud
|
|
756
|
+
* on a stale/foreign buffer.
|
|
757
|
+
*/
|
|
758
|
+
traverseFromOpaqueIdSet(seed: Buffer, opts: TraverseOptions, generation?: bigint | null): NativeSubgraph;
|
|
759
|
+
/**
|
|
760
|
+
* Open a snapshot-consistent streaming walk of the whole graph (or a
|
|
761
|
+
* seeded region), pinned to a generation. Returns a TTL-bounded handle
|
|
762
|
+
* for {@link graphCursorNext} / {@link graphCursorClose}.
|
|
763
|
+
*/
|
|
764
|
+
graphCursorOpen(options: GraphCursorOpenOptions): string;
|
|
765
|
+
/**
|
|
766
|
+
* Pull the next chunk from an open cursor (up to `chunkSize` edges).
|
|
767
|
+
* `done: true` once exhausted; a handle idle past the TTL throws a
|
|
768
|
+
* `SnapshotExpired`-prefixed error.
|
|
769
|
+
*/
|
|
770
|
+
graphCursorNext(handle: string, chunkSize: number): NativeGraphCursorChunk;
|
|
771
|
+
/** Release a cursor's pinned generation + server-side state (idempotent). */
|
|
772
|
+
graphCursorClose(handle: string): void;
|
|
773
|
+
/** PageRank over the as-of-g graph (power iteration; descending; topK). */
|
|
774
|
+
pageRank(options: NativePageRankOptions, generation?: bigint | null): NativeGraphScores;
|
|
775
|
+
/** Weak (union-find) or strong (iterative Tarjan) connected components. */
|
|
776
|
+
connectedComponents(options: NativeComponentsOptions, generation?: bigint | null): NativeGraphComponents;
|
|
777
|
+
/** Shortest path (BFS hops; `null` when unreachable). */
|
|
778
|
+
shortestPath(fromInt: bigint, toInt: bigint, options: NativeShortestPathOptions, generation?: bigint | null): NativeGraphPath | null;
|
|
779
|
+
/** Deterministic seeded bounded neighborhood sample (columnar Subgraph). */
|
|
780
|
+
neighborhoodSample(seeds: Array<bigint>, seedsOpaque: Buffer | null, options: NativeNeighborhoodSampleOptions, generation?: bigint | null): NativeSubgraph;
|
|
781
|
+
/** Top-K nodes by degree (out/in/both), descending. */
|
|
782
|
+
topByDegree(options: NativeTopByDegreeOptions, generation?: bigint | null): NativeGraphScores;
|
|
783
|
+
/** Flush a tree's MemTable to binary .sst format (no disk I/O). */
|
|
784
|
+
flushTreeToBinary(treeName: string): NativeBinaryFlushResult | null;
|
|
785
|
+
/** Compact a tree's level to binary .sst format (no disk I/O). */
|
|
786
|
+
compactTreeToBinary(treeName: string, level: number): NativeBinaryCompactResult | null;
|
|
787
|
+
/** Mmap an adapter-written file, upgrading InMemory variant to Mapped. */
|
|
788
|
+
registerMmapSstable(treeName: string, sstableId: string, level: number, filePath: string): void;
|
|
789
|
+
/** Load an SSTable from binary .sst data. */
|
|
790
|
+
loadSstableFromBinary(treeName: string, sstableId: string, level: number, data: Buffer): void;
|
|
791
|
+
/** Load an SSTable from a .sst file via mmap. */
|
|
792
|
+
loadSstableFromFile(treeName: string, sstableId: string, level: number, filePath: string): void;
|
|
793
|
+
/** Check if a tree's level needs compaction. */
|
|
794
|
+
needsCompaction(treeName: string, level: number): boolean;
|
|
795
|
+
/** Get manifest JSON for a tree. */
|
|
796
|
+
getManifestJson(treeName: string): string;
|
|
797
|
+
/** Load manifest JSON for a tree. */
|
|
798
|
+
loadManifestJson(treeName: string, json: string): void;
|
|
799
|
+
/** Get SSTable IDs from a tree's manifest. */
|
|
800
|
+
getManifestSstableIds(treeName: string): string[];
|
|
801
|
+
/** Get level for an SSTable ID in a tree's manifest. */
|
|
802
|
+
getManifestSstableLevel(treeName: string, id: string): number | null;
|
|
803
|
+
/** Check if a tree's MemTable is empty. */
|
|
804
|
+
memTableIsEmpty(treeName: string): boolean;
|
|
805
|
+
/** Total relationship count (caps at 2^32-1). */
|
|
806
|
+
size(): number;
|
|
807
|
+
/** BigInt total relationship count. */
|
|
808
|
+
sizeBig(): bigint;
|
|
809
|
+
/** Number of tracked verb IDs (caps at 2^32-1). */
|
|
810
|
+
verbIdCount(): number;
|
|
811
|
+
/** BigInt verb id count. */
|
|
812
|
+
verbIdCountBig(): bigint;
|
|
813
|
+
/**
|
|
814
|
+
* Batch verb-int reverse resolver (brainy 8.0 lockstep,
|
|
815
|
+
* `GraphIndexProvider.verbIntsToIds` contract). Given a batch of
|
|
816
|
+
* verb-int BigInts, returns the canonical UUID string for each —
|
|
817
|
+
* `null`/`undefined` for any int with no entry in the persistent
|
|
818
|
+
* reverse map. O(1) per lookup via `verb-int-to-uuid.bin`.
|
|
819
|
+
*/
|
|
820
|
+
verbIntsToUuids(verbInts: Array<bigint>): Array<string | null | undefined>;
|
|
821
|
+
/** Per-type relationship count (saturates at 2^32-1). */
|
|
822
|
+
getRelationshipCountByType(verbType: string): number;
|
|
823
|
+
/** BigInt per-type relationship count (overflow-safe). */
|
|
824
|
+
getRelationshipCountByTypeBig(verbType: string): bigint;
|
|
825
|
+
/** Strict per-type count by canonical index. */
|
|
826
|
+
getRelationshipCountByIndex(verbTypeIndex: number): bigint;
|
|
827
|
+
/** All relationship counts as JSON `{type: count}` map. */
|
|
828
|
+
getAllRelationshipCountsJson(): string;
|
|
829
|
+
/** Relationship statistics as JSON. */
|
|
830
|
+
getRelationshipStatsJson(): string;
|
|
831
|
+
/** Whether the index is healthy. */
|
|
832
|
+
isHealthy(): boolean;
|
|
833
|
+
/**
|
|
834
|
+
* Whether the graph's edges are loaded and queryable WITHOUT a rebuild —
|
|
835
|
+
* brainy 8.0's rebuild gate (#36). `true` once an SSTable has cold-loaded
|
|
836
|
+
* (binary/file), a manifest named ≥1 SSTable, or a memtable was flushed,
|
|
837
|
+
* or while any tree's memtable is non-empty. Edges-loaded semantics only,
|
|
838
|
+
* never gated on verb-id membership.
|
|
839
|
+
*/
|
|
840
|
+
isReady(): boolean;
|
|
841
|
+
/** Comprehensive statistics. */
|
|
842
|
+
getStats(): GraphAdjacencyStats;
|
|
843
|
+
/** List all tree names: `source`, `target`, `verbs-source`, `verbs-target`. */
|
|
844
|
+
treeNames(): string[];
|
|
845
|
+
/** Verb-ID interning namespace wire width: `"U32"` or `"U64"`. */
|
|
846
|
+
verbIdSpace(): string;
|
|
847
|
+
}
|
|
848
|
+
/**
|
|
849
|
+
* Mutation result from add/remove operations.
|
|
850
|
+
* Tells the TS wrapper which data needs to be persisted. Posting lists are
|
|
851
|
+
* owned + made durable by the LSM engine, so the only TS-side persistence is
|
|
852
|
+
* the value-enumeration field indexes and the field registry.
|
|
853
|
+
*/
|
|
854
|
+
export interface NativeMetadataMutationResult {
|
|
855
|
+
dirtyFieldIndexes: string[];
|
|
856
|
+
newFields: string[];
|
|
857
|
+
dirtyFieldRegistry: boolean;
|
|
858
|
+
}
|
|
859
|
+
/**
|
|
860
|
+
* Native metadata index instance.
|
|
861
|
+
* All query and mutation operations execute synchronously in Rust.
|
|
862
|
+
* TS wrapper handles async storage I/O.
|
|
863
|
+
*/
|
|
864
|
+
export interface NativeMetadataIndexInstance {
|
|
865
|
+
/** IdSpace this metadata index was constructed with — `"u32"` or `"u64"`. */
|
|
866
|
+
idSpace(): string;
|
|
867
|
+
/**
|
|
868
|
+
* Replace this index's LSM engine with the durable, mmap-backed engine the
|
|
869
|
+
* TS wrapper opened for the brain's storage path. The index is constructed
|
|
870
|
+
* with a non-persistent in-RAM engine (so it is usable immediately);
|
|
871
|
+
* production swaps in the persistent engine here during `init()`, before
|
|
872
|
+
* any mutation. The engine's IdSpace MUST match the metadata index's —
|
|
873
|
+
* mismatches throw at attach time.
|
|
874
|
+
*/
|
|
875
|
+
setLsmEngine(engine: NativeLsmEngineInstance): void;
|
|
876
|
+
/**
|
|
877
|
+
* Whether mutations + queries route through the LSM engine. Always `true`
|
|
878
|
+
* in cor 3.0 — the LSM engine is the sole metadata engine and is attached
|
|
879
|
+
* at construction. Retained as a diagnostic surface for TS-side assertions.
|
|
880
|
+
*/
|
|
881
|
+
isLsmEnabled(): boolean;
|
|
882
|
+
/**
|
|
883
|
+
* **Piece 6 generation counter.** Bumped by every read-visible
|
|
884
|
+
* mutation (`addToIndex` / `removeFromIndex`). Compared by readers to
|
|
885
|
+
* invalidate cached lazy-load state.
|
|
886
|
+
*/
|
|
887
|
+
currentGeneration(): bigint;
|
|
888
|
+
/**
|
|
889
|
+
* **Piece 15 per-field generation counter.** Bumped by every
|
|
890
|
+
* `addToIndex` / `removeFromIndex` operation that touches the
|
|
891
|
+
* given field. Brainy 8.0+ wrappers use this to ask "did field X
|
|
892
|
+
* change since I last looked?" without dumping every loaded
|
|
893
|
+
* field's lazy-load state on every mutation. Returns `0n` for
|
|
894
|
+
* fields that have never been registered (cold-boot sentinel —
|
|
895
|
+
* matches `currentGeneration()` starting at `0n`).
|
|
896
|
+
*/
|
|
897
|
+
fieldGeneration(field: string): bigint;
|
|
898
|
+
/**
|
|
899
|
+
* **#72 Phase C.** Inject the shared mmap `BinaryIdMapper` as this
|
|
900
|
+
* metadata index's canonical UUID ↔ entity-int allocator. The native
|
|
901
|
+
* core is constructed with no storage path; the TS wrapper derives the
|
|
902
|
+
* `_id_mapper/*` paths, builds the mapper, and injects it here before
|
|
903
|
+
* any mutation. The mapper's IdSpace MUST match the index's (both
|
|
904
|
+
* `"u64"` in production) — mismatches throw. Replaces the legacy
|
|
905
|
+
* `loadEntityIdMapper` / `saveEntityIdMapper` JSON path.
|
|
906
|
+
*/
|
|
907
|
+
setIdMapper(mapper: NativeBinaryIdMapperInstance): void;
|
|
908
|
+
entityIdMapperSize(): number;
|
|
909
|
+
uuidToInt(uuid: string): number | null;
|
|
910
|
+
intToUuid(intId: number): string | null;
|
|
911
|
+
intsToUuids(ints: number[]): string[];
|
|
912
|
+
loadFieldRegistry(json: string): void;
|
|
913
|
+
saveFieldRegistry(): string;
|
|
914
|
+
loadFieldIndex(field: string, json: string): void;
|
|
915
|
+
saveFieldIndex(field: string): string | null;
|
|
916
|
+
getIds(field: string, valueJson: string): string[];
|
|
917
|
+
/**
|
|
918
|
+
* Count IDs for a single field-value pair WITHOUT materializing the UUID
|
|
919
|
+
* strings. Equals `getIds(field, valueJson).length` exactly (counts only
|
|
920
|
+
* ints that resolve in the entity-id mapper), but allocates no Strings —
|
|
921
|
+
* the count-only fast path for type/criteria/VFS counts at 10B scale.
|
|
922
|
+
*/
|
|
923
|
+
getIdsCount(field: string, valueJson: string): number;
|
|
924
|
+
/**
|
|
925
|
+
* Resolve a filter to its matching ids. `limit`/`offset` are the optional
|
|
926
|
+
* page bound (#75): when `limit` is supplied the native producer returns
|
|
927
|
+
* only the `[offset, offset+limit)` window, short-circuiting the posting-list
|
|
928
|
+
* → UUID conversion at `offset+limit` instead of materializing then slicing.
|
|
929
|
+
* Omitting `limit` returns the full match set (back-compat).
|
|
930
|
+
*/
|
|
931
|
+
getIdsForFilter(filterJson: string, allIdsJson?: string | null, limit?: number | null, offset?: number | null): string[];
|
|
932
|
+
/**
|
|
933
|
+
* Cumulative count of id-mapper resolutions performed by `getIdsForFilter`
|
|
934
|
+
* since construction (#75 observability). The page-bounded path keeps this
|
|
935
|
+
* at `offset+limit` per call instead of the full match cardinality.
|
|
936
|
+
*/
|
|
937
|
+
filterIdsResolved(): number;
|
|
938
|
+
getIdsForMultipleFields(pairsJson: string): string[];
|
|
939
|
+
getIdsForRange(field: string, minJson: string | null, maxJson: string | null, includeMin: boolean, includeMax: boolean): string[];
|
|
940
|
+
getIdsForTextQuery(queryStr: string): string;
|
|
941
|
+
getFilterValues(field: string): string[];
|
|
942
|
+
getFilterFields(): string[];
|
|
943
|
+
extractFieldNames(entityJson: string): string[];
|
|
944
|
+
addToIndex(id: string, entityJson: string): string;
|
|
945
|
+
removeFromIndex(id: string, metadataJson?: string | null): string;
|
|
946
|
+
getEntityCountByType(typeName: string): number;
|
|
947
|
+
getTotalEntityCount(): number;
|
|
948
|
+
getTopNounTypes(n: number): string;
|
|
949
|
+
getTopVerbTypes(n: number): string;
|
|
950
|
+
getStats(): string;
|
|
951
|
+
/**
|
|
952
|
+
* O(1)-ish internal consistency probe. Samples a bounded set of fields/ints
|
|
953
|
+
* and returns `false` if a sampled int sits in >1 value bucket of an
|
|
954
|
+
* otherwise-single-valued field (a cross-bucket phantom), `true` otherwise.
|
|
955
|
+
* Never throws; empty index → `true`. Backs the wrapper's `probeConsistency`.
|
|
956
|
+
*/
|
|
957
|
+
probeConsistency(): boolean;
|
|
958
|
+
/**
|
|
959
|
+
* Scan for the cross-bucket phantom signature (a query-visible int in >1
|
|
960
|
+
* value bucket of a field) and return up to `maxCandidates` (0 = unlimited)
|
|
961
|
+
* as a JSON array of `{ uuid, field }`. The `__words__` text index is
|
|
962
|
+
* skipped. Drives the state-level repair from `detectAndRepairCorruption`.
|
|
963
|
+
*/
|
|
964
|
+
detectCrossBucketPhantoms(maxCandidates: number): string;
|
|
965
|
+
/**
|
|
966
|
+
* Tombstone `uuid`'s int out of every value bucket of `field` except those
|
|
967
|
+
* matching its current stored value(s) (`currentValuesJson`: a JSON array of
|
|
968
|
+
* raw values). Empty array → evict from every bucket (orphan purge). Returns
|
|
969
|
+
* `true` if anything was tombstoned.
|
|
970
|
+
*/
|
|
971
|
+
repairEntityFieldPhantom(uuid: string, field: string, currentValuesJson: string): boolean;
|
|
972
|
+
clear(): void;
|
|
973
|
+
}
|
|
974
|
+
/**
|
|
975
|
+
* Construction options for {@link NativeLsmEngineInstance}.
|
|
976
|
+
*
|
|
977
|
+
* Every field is optional; defaults match production usage
|
|
978
|
+
* (`idSpace:"u32"`, bloom emit on at 10 bits/element, admission ratio
|
|
979
|
+
* 0.85). Test drivers may override `admissionThresholdRatio` to drive
|
|
980
|
+
* the admission gate under controlled budgets.
|
|
981
|
+
*/
|
|
982
|
+
export interface NativeLsmEngineOptions {
|
|
983
|
+
/** `"u32"` (default) or `"u64"`. */
|
|
984
|
+
idSpace?: string;
|
|
985
|
+
/** Emit bloom filters in flushed SSTables. Default `true`. */
|
|
986
|
+
emitBloom?: boolean;
|
|
987
|
+
/** Bits per element for the bloom filter. Default `10` (~1% FPR). */
|
|
988
|
+
bloomBitsPerElement?: number;
|
|
989
|
+
/**
|
|
990
|
+
* Admission threshold ratio (0.0..1.0). Default `0.85`. Hardcoded for
|
|
991
|
+
* production per the zero-config mandate — exposed here for test
|
|
992
|
+
* drivers only.
|
|
993
|
+
*/
|
|
994
|
+
admissionThresholdRatio?: number;
|
|
995
|
+
}
|
|
996
|
+
/**
|
|
997
|
+
* Native LSM metadata-index engine.
|
|
998
|
+
*
|
|
999
|
+
* Owns the sharded memtable, the global epoch counter, the
|
|
1000
|
+
* content-keyed reader cache, and the write admission-control gate.
|
|
1001
|
+
* The sole metadata storage engine; the wrapper swaps the durable
|
|
1002
|
+
* instance into a {@link NativeMetadataIndexInstance} via
|
|
1003
|
+
* {@link NativeMetadataIndexInstance.setLsmEngine} during init().
|
|
1004
|
+
*
|
|
1005
|
+
* The `*Big` siblings of `insert`/`delete` accept `bigint` ids; the
|
|
1006
|
+
* non-suffixed methods accept JS `number` and throw on U64 mode if the
|
|
1007
|
+
* value exceeds `Number.MAX_SAFE_INTEGER`.
|
|
1008
|
+
*/
|
|
1009
|
+
export interface NativeLsmEngineInstance {
|
|
1010
|
+
/** `"u32"` or `"u64"` — fixed at construction. */
|
|
1011
|
+
idSpace(): string;
|
|
1012
|
+
/** Add an entity int to `(field, value)`. */
|
|
1013
|
+
insert(field: string, value: string, intId: number): void;
|
|
1014
|
+
insertBig(field: string, value: string, intId: bigint): void;
|
|
1015
|
+
/** Tombstone an entity int at `(field, value)`. */
|
|
1016
|
+
delete(field: string, value: string, intId: number): void;
|
|
1017
|
+
deleteBig(field: string, value: string, intId: bigint): void;
|
|
1018
|
+
/**
|
|
1019
|
+
* Drain the memtable and produce SSTable bytes + manifest entry JSON
|
|
1020
|
+
* ready for `storage.saveBinaryBlob`. Returns `null` when there is
|
|
1021
|
+
* nothing to flush.
|
|
1022
|
+
*/
|
|
1023
|
+
flush(): NativeLsmFlushResult | null;
|
|
1024
|
+
/**
|
|
1025
|
+
* **Piece 7 / LSM durability.** Advance the flush watermark on every
|
|
1026
|
+
* shard's memtable log so records with `epoch <= throughEpoch` are dropped
|
|
1027
|
+
* from future replay. The wrapper calls this ONLY after the SSTable bytes
|
|
1028
|
+
* returned by the preceding {@link flush} AND the manifest referencing them
|
|
1029
|
+
* are durably persisted (and the SSTable registered) — at that point the
|
|
1030
|
+
* data lives in both the log and the persisted SSTable, so the log copy is
|
|
1031
|
+
* safe to retire. No-op for in-memory engines (constructed without
|
|
1032
|
+
* `openWithDir`).
|
|
1033
|
+
*/
|
|
1034
|
+
commitFlushThrough(throughEpoch: bigint): void;
|
|
1035
|
+
/**
|
|
1036
|
+
* Update the budget ceiling. Driven by the wrapper's
|
|
1037
|
+
* `ResourceManager` subscriber callback. Pass `0xffffffffffffffffn`
|
|
1038
|
+
* (`u64::MAX`) to disable admission control.
|
|
1039
|
+
*/
|
|
1040
|
+
setAllowedBytes(bytes: bigint): void;
|
|
1041
|
+
/** Current budget ceiling. */
|
|
1042
|
+
allowedBytes(): bigint;
|
|
1043
|
+
/** Estimated memtable resident bytes. */
|
|
1044
|
+
memtableBytes(): bigint;
|
|
1045
|
+
/** Outstanding admission-reserved bytes (diagnostic). */
|
|
1046
|
+
reservedBytes(): bigint;
|
|
1047
|
+
/** Distinct `(field, value)` keys buffered in the memtable. */
|
|
1048
|
+
memtableEntryCount(): bigint;
|
|
1049
|
+
/** Next epoch the engine will allocate. */
|
|
1050
|
+
currentEpoch(): bigint;
|
|
1051
|
+
/** Whether the next insert would block on admission control. */
|
|
1052
|
+
isOverAdmissionThreshold(): boolean;
|
|
1053
|
+
/**
|
|
1054
|
+
* Combined native memory footprint (memtable + reader cache). Reported
|
|
1055
|
+
* to `ResourceManager.reportNativeMemory` to keep the per-category
|
|
1056
|
+
* accounting honest.
|
|
1057
|
+
*/
|
|
1058
|
+
nativeMemoryBytes(): bigint;
|
|
1059
|
+
/**
|
|
1060
|
+
* Take a point-in-time snapshot for reading. Holds Arc clones of the
|
|
1061
|
+
* memtable shards + the supplied SSTable readers; reads through this
|
|
1062
|
+
* handle are valid for up to 30 seconds.
|
|
1063
|
+
*/
|
|
1064
|
+
takeSnapshot(sstables: Buffer[]): NativeLsmSnapshotInstance;
|
|
1065
|
+
/**
|
|
1066
|
+
* Read a posting list for `(field, value)` across the live memtable +
|
|
1067
|
+
* the supplied SSTable bytes. Returns croaring portable bytes; an
|
|
1068
|
+
* empty Buffer means "no matches".
|
|
1069
|
+
*/
|
|
1070
|
+
readPostingList(field: string, value: string, sstables: Buffer[]): Buffer;
|
|
1071
|
+
/** Zero-copy mmap variant of {@link readPostingList}. */
|
|
1072
|
+
readPostingListFromPaths(field: string, value: string, paths: string[]): Buffer;
|
|
1073
|
+
/** Open an SSTable from disk into the reader cache (mmap). */
|
|
1074
|
+
openSstablePath(path: string): void;
|
|
1075
|
+
/**
|
|
1076
|
+
* Open an SSTable from disk (mmap + SHA-256 verify) and register it
|
|
1077
|
+
* into the engine's live-SSTable set so every subsequent read
|
|
1078
|
+
* snapshot includes it. Unlike {@link openSstablePath} (which only
|
|
1079
|
+
* warms the evictable cache), this grows the engine-owned read view.
|
|
1080
|
+
*/
|
|
1081
|
+
registerSstablePath(path: string): void;
|
|
1082
|
+
/** Number of SSTable readers currently in the engine's live set. */
|
|
1083
|
+
liveSstableCount(): number;
|
|
1084
|
+
/** Phase 5 hook — drop the cache entry for a path. */
|
|
1085
|
+
evictReaderCacheByPath(path: string): void;
|
|
1086
|
+
/** Walk every directory entry in an SSTable in sorted order. */
|
|
1087
|
+
scanSstable(sstable: Buffer): NativeLsmScanEntry[];
|
|
1088
|
+
}
|
|
1089
|
+
/**
|
|
1090
|
+
* Result of {@link NativeLsmEngineInstance.flush}.
|
|
1091
|
+
*/
|
|
1092
|
+
export interface NativeLsmFlushResult {
|
|
1093
|
+
/** Raw SSTable bytes — hand to `storage.saveBinaryBlob`. */
|
|
1094
|
+
sstable: Buffer;
|
|
1095
|
+
/** `LsmSstableManifestEntry` JSON for the wrapper to splice in. */
|
|
1096
|
+
manifestEntryJson: string;
|
|
1097
|
+
/** Number of `(field, value)` keys in the flushed SSTable. */
|
|
1098
|
+
entryCount: bigint;
|
|
1099
|
+
/** New `last_durable_epoch` value to write into the manifest. */
|
|
1100
|
+
lastDurableEpoch: bigint;
|
|
1101
|
+
}
|
|
1102
|
+
/**
|
|
1103
|
+
* One entry materialized from {@link NativeLsmEngineInstance.scanSstable}.
|
|
1104
|
+
*/
|
|
1105
|
+
export interface NativeLsmScanEntry {
|
|
1106
|
+
field: string;
|
|
1107
|
+
value: string;
|
|
1108
|
+
additionsBytes: Buffer;
|
|
1109
|
+
deletionsBytes: Buffer;
|
|
1110
|
+
additionCount: bigint;
|
|
1111
|
+
deletionCount: bigint;
|
|
1112
|
+
epoch: bigint;
|
|
1113
|
+
isTombstone: boolean;
|
|
1114
|
+
}
|
|
1115
|
+
/**
|
|
1116
|
+
* Point-in-time read handle returned by
|
|
1117
|
+
* {@link NativeLsmEngineInstance.takeSnapshot}. Reads return
|
|
1118
|
+
* `SnapshotExpired` after 30 seconds.
|
|
1119
|
+
*/
|
|
1120
|
+
export interface NativeLsmSnapshotInstance {
|
|
1121
|
+
/** Read the posting list for `(field, value)` at this snapshot's epoch. */
|
|
1122
|
+
readPostingList(field: string, value: string): Buffer;
|
|
1123
|
+
/** Whether the snapshot has aged past its 30-second lifetime. */
|
|
1124
|
+
isExpired(): boolean;
|
|
1125
|
+
/** Number of SSTables held by this snapshot (diagnostic). */
|
|
1126
|
+
sstableCount(): number;
|
|
1127
|
+
}
|
|
1128
|
+
/**
|
|
1129
|
+
* Native column store instance.
|
|
1130
|
+
*
|
|
1131
|
+
* Owns per-field tail buffers + loaded `.cidx` segments and performs the
|
|
1132
|
+
* sort/filter/range value operations in Rust. The TS wrapper
|
|
1133
|
+
* (`utils/NativeColumnStore.ts`) bridges async storage I/O: it persists flushed
|
|
1134
|
+
* segments as binary blobs and loads them back (zero-copy via mmap when a local
|
|
1135
|
+
* path exists). All four brainy value types are native — `Number`/`Boolean`
|
|
1136
|
+
* share the i64 path, `Float` is f64, `String` is length-prefixed UTF-8.
|
|
1137
|
+
*/
|
|
1138
|
+
export interface NativeColumnStoreInstance {
|
|
1139
|
+
/** Load a segment from a buffer (remote storage, no local path). */
|
|
1140
|
+
loadSegmentFromBuffer(field: string, segmentId: string, data: Buffer): void;
|
|
1141
|
+
/** Load a segment by memory-mapping a file path (zero-copy reads). */
|
|
1142
|
+
loadSegmentFromPath(field: string, segmentId: string, path: string): void;
|
|
1143
|
+
/**
|
|
1144
|
+
* Add an entry to the in-memory tail buffer for a numeric (i64) field.
|
|
1145
|
+
*
|
|
1146
|
+
* **Cor 3.0 Piece E**: `entityIntId` is now `bigint` (full u64 range).
|
|
1147
|
+
* The pre-E `addNumericBig` alias was removed; this method accepts the
|
|
1148
|
+
* full u64 range directly.
|
|
1149
|
+
*/
|
|
1150
|
+
addNumeric(field: string, value: number, entityIntId: bigint): void;
|
|
1151
|
+
/** Add an entry to the in-memory tail buffer for a float (f64) field. */
|
|
1152
|
+
addFloat(field: string, value: number, entityIntId: bigint): void;
|
|
1153
|
+
/** Add an entry to the in-memory tail buffer for a string field. */
|
|
1154
|
+
addString(field: string, value: string, entityIntId: bigint): void;
|
|
1155
|
+
/** Mark an entity as deleted for a specific field. */
|
|
1156
|
+
deleteEntity(field: string, entityIntId: bigint): void;
|
|
1157
|
+
/**
|
|
1158
|
+
* Sort top-K entity ints by a field (auto-routed by field type).
|
|
1159
|
+
*
|
|
1160
|
+
* **Cor 3.0 Piece E**: returns `bigint[]` for full u64 fidelity. The
|
|
1161
|
+
* pre-E `sortTopKBig` alias was removed.
|
|
1162
|
+
*/
|
|
1163
|
+
sortTopK(field: string, order: 'asc' | 'desc', k: number): bigint[];
|
|
1164
|
+
/**
|
|
1165
|
+
* Point filter (numeric) → serialized portable Roaring64 bitmap.
|
|
1166
|
+
*
|
|
1167
|
+
* **Cor 3.0 Piece E**: result is now a Roaring64 (Treemap) portable
|
|
1168
|
+
* bitmap, not the pre-E Roaring32 (Bitmap). Brainy 8.0 TS-side decoder
|
|
1169
|
+
* must use the Roaring64 portable parser.
|
|
1170
|
+
*/
|
|
1171
|
+
filterNumeric(field: string, value: number): Buffer;
|
|
1172
|
+
/** Range filter (numeric, inclusive) → serialized portable Roaring64 bitmap. */
|
|
1173
|
+
rangeQueryNumeric(field: string, lo: number, hi: number): Buffer;
|
|
1174
|
+
/** Point filter (float) → serialized portable Roaring64 bitmap. */
|
|
1175
|
+
filterFloat(field: string, value: number): Buffer;
|
|
1176
|
+
/** Range filter (float, inclusive) → serialized portable Roaring64 bitmap. */
|
|
1177
|
+
rangeQueryFloat(field: string, lo: number, hi: number): Buffer;
|
|
1178
|
+
/** Point filter (string) → serialized portable Roaring64 bitmap. */
|
|
1179
|
+
filterString(field: string, value: string): Buffer;
|
|
1180
|
+
/** Range filter (string, inclusive) → serialized portable Roaring64 bitmap. */
|
|
1181
|
+
rangeQueryString(field: string, lo: string, hi: string): Buffer;
|
|
1182
|
+
/** Drain a field's tail buffer to a sorted `.cidx` segment buffer, or null if empty. */
|
|
1183
|
+
flushTailBuffer(field: string, level: number, segmentId: number): Buffer | null;
|
|
1184
|
+
/** Tail buffer entry count for a field. */
|
|
1185
|
+
tailBufferSize(field: string): number;
|
|
1186
|
+
/** Whether any segments or tail data exist for a field. */
|
|
1187
|
+
hasField(field: string): boolean;
|
|
1188
|
+
/** All distinct values for a field, as ascending strings. */
|
|
1189
|
+
distinctValues(field: string): string[];
|
|
1190
|
+
}
|
|
1191
|
+
/**
|
|
1192
|
+
* Result from eviction or clear operations.
|
|
1193
|
+
*/
|
|
1194
|
+
export interface NativeEvictionResult {
|
|
1195
|
+
evictedKeys: string[];
|
|
1196
|
+
bytesFreed: number;
|
|
1197
|
+
}
|
|
1198
|
+
/**
|
|
1199
|
+
* Cache statistics including native memory tracking.
|
|
1200
|
+
*/
|
|
1201
|
+
export interface NativeCacheStats {
|
|
1202
|
+
totalSize: number;
|
|
1203
|
+
maxSize: number;
|
|
1204
|
+
itemCount: number;
|
|
1205
|
+
typeSizes: number[];
|
|
1206
|
+
typeCounts: number[];
|
|
1207
|
+
typeAccessCounts: number[];
|
|
1208
|
+
totalAccessCount: number;
|
|
1209
|
+
nativeMemory: number[];
|
|
1210
|
+
totalManagedMemory: number;
|
|
1211
|
+
}
|
|
1212
|
+
/**
|
|
1213
|
+
* Native unified cache instance.
|
|
1214
|
+
* Manages eviction decisions and cache entry metadata.
|
|
1215
|
+
* TS wrapper holds actual JS data in a Map.
|
|
1216
|
+
*/
|
|
1217
|
+
export interface NativeUnifiedCacheInstance {
|
|
1218
|
+
recordAccess(key: string): void;
|
|
1219
|
+
insert(key: string, cacheType: number, size: number, rebuildCost: number): NativeEvictionResult;
|
|
1220
|
+
remove(key: string): number;
|
|
1221
|
+
removeByPrefix(prefix: string): string[];
|
|
1222
|
+
clear(cacheType?: number | null): string[];
|
|
1223
|
+
checkFairness(): NativeEvictionResult;
|
|
1224
|
+
evictForSize(bytesNeeded: number): NativeEvictionResult;
|
|
1225
|
+
reportNativeMemory(hnsw: number, metadata: number, graph: number): void;
|
|
1226
|
+
getStats(): NativeCacheStats;
|
|
1227
|
+
has(key: string): boolean;
|
|
1228
|
+
itemCount(): number;
|
|
1229
|
+
saveAccessPatterns(): string;
|
|
1230
|
+
loadAccessPatterns(json: string): void;
|
|
1231
|
+
}
|
|
1232
|
+
/**
|
|
1233
|
+
* PQ codebook parameters.
|
|
1234
|
+
*/
|
|
1235
|
+
export interface PQParams {
|
|
1236
|
+
m: number;
|
|
1237
|
+
ksub: number;
|
|
1238
|
+
dsub: number;
|
|
1239
|
+
dim: number;
|
|
1240
|
+
}
|
|
1241
|
+
/**
|
|
1242
|
+
* Native Product Quantization codebook instance.
|
|
1243
|
+
*/
|
|
1244
|
+
export interface NativePQCodebookInstance {
|
|
1245
|
+
readonly params: PQParams;
|
|
1246
|
+
encode(vector: number[]): Buffer;
|
|
1247
|
+
encodeBatch(vectors: number[], dim: number): Buffer;
|
|
1248
|
+
decode(code: Buffer): number[];
|
|
1249
|
+
computeAdcTable(query: number[]): number[];
|
|
1250
|
+
distanceAdc(table: number[], code: Buffer): number;
|
|
1251
|
+
distanceAdcBatch(table: number[], codes: Buffer, n: number): number[];
|
|
1252
|
+
serialize(): Buffer;
|
|
1253
|
+
}
|
|
1254
|
+
/**
|
|
1255
|
+
* Native aggregation engine instance.
|
|
1256
|
+
* Provides Rust-accelerated incremental aggregation with:
|
|
1257
|
+
* - BTreeMap precise MIN/MAX (never stale after deletes)
|
|
1258
|
+
* - Rayon parallel rebuild (100x on multi-core)
|
|
1259
|
+
* - Welford's online stddev/variance
|
|
1260
|
+
* - Integer timestamp bucketing (no Date allocation)
|
|
1261
|
+
* - Compiled source filters (rebuild path)
|
|
1262
|
+
*/
|
|
1263
|
+
export interface NativeAggregationEngineInstance {
|
|
1264
|
+
/** Register an aggregate definition (compiles and caches for hot path) */
|
|
1265
|
+
defineAggregate(definitionJson: string): void;
|
|
1266
|
+
/** Remove a registered aggregate definition and its state */
|
|
1267
|
+
removeAggregate(name: string): void;
|
|
1268
|
+
/**
|
|
1269
|
+
* Incrementally update aggregation state when an entity changes.
|
|
1270
|
+
* Entity is passed as serde_json::Value (NAPI converts JS objects directly).
|
|
1271
|
+
* Returns JSON array of modified AggregateGroupState objects.
|
|
1272
|
+
*/
|
|
1273
|
+
incrementalUpdate(name: string, entity: unknown, op: string, prev?: unknown): string;
|
|
1274
|
+
/**
|
|
1275
|
+
* Compute the group key for an entity given groupBy dimensions.
|
|
1276
|
+
* Returns JSON string of the group key object.
|
|
1277
|
+
*/
|
|
1278
|
+
computeGroupKey(entity: unknown, groupByJson: string): string;
|
|
1279
|
+
/**
|
|
1280
|
+
* Rebuild an entire aggregate from scratch using Rayon parallelism.
|
|
1281
|
+
* Takes definition and entities as JSON strings.
|
|
1282
|
+
* Returns JSON string of the state (array of AggregateGroupState).
|
|
1283
|
+
*/
|
|
1284
|
+
rebuildAggregate(definitionJson: string, entitiesJson: string): string;
|
|
1285
|
+
/**
|
|
1286
|
+
* Query aggregate state with filtering/sorting/pagination.
|
|
1287
|
+
* Takes state and params as JSON strings.
|
|
1288
|
+
* Returns JSON array of AggregateResult objects.
|
|
1289
|
+
*/
|
|
1290
|
+
queryAggregate(stateJson: string, paramsJson: string): string;
|
|
1291
|
+
/** Serialize all internal state for persistence */
|
|
1292
|
+
serializeState(): string;
|
|
1293
|
+
/** Restore internal state from previously serialized data */
|
|
1294
|
+
restoreState(stateJson: string): void;
|
|
1295
|
+
/** Engine statistics (definition count, total groups, etc.) */
|
|
1296
|
+
readonly stats: string;
|
|
1297
|
+
}
|
|
1298
|
+
//# sourceMappingURL=types.d.ts.map
|