@soulcraft/cor 3.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +16 -0
- package/README.md +328 -0
- package/dist/aggregation/NativeAggregationEngine.d.ts +118 -0
- package/dist/aggregation/NativeAggregationEngine.js +186 -0
- package/dist/cli.d.ts +12 -0
- package/dist/cli.js +236 -0
- package/dist/graph/GraphAccelerationAdapter.d.ts +153 -0
- package/dist/graph/GraphAccelerationAdapter.js +326 -0
- package/dist/graph/NativeGraphAdjacencyIndex.d.ts +286 -0
- package/dist/graph/NativeGraphAdjacencyIndex.js +944 -0
- package/dist/hnsw/AdaptiveDiskAnnModeSelector.d.ts +187 -0
- package/dist/hnsw/AdaptiveDiskAnnModeSelector.js +313 -0
- package/dist/hnsw/NativeDiskAnnWrapper.d.ts +489 -0
- package/dist/hnsw/NativeDiskAnnWrapper.js +1456 -0
- package/dist/index.d.ts +28 -0
- package/dist/index.js +29 -0
- package/dist/legacyLayoutGuard.d.ts +93 -0
- package/dist/legacyLayoutGuard.js +237 -0
- package/dist/license/constants.d.ts +30 -0
- package/dist/license/constants.js +30 -0
- package/dist/license/onlineValidator.d.ts +43 -0
- package/dist/license/onlineValidator.js +225 -0
- package/dist/license/types.d.ts +39 -0
- package/dist/license/types.js +5 -0
- package/dist/license.d.ts +64 -0
- package/dist/license.js +165 -0
- package/dist/migration/MigrationCoordinator.d.ts +171 -0
- package/dist/migration/MigrationCoordinator.js +249 -0
- package/dist/migration/indexEpochGuard.d.ts +163 -0
- package/dist/migration/indexEpochGuard.js +202 -0
- package/dist/native/NativeEmbeddingEngine.d.ts +79 -0
- package/dist/native/NativeEmbeddingEngine.js +318 -0
- package/dist/native/NativeRoaringBitmap32.d.ts +114 -0
- package/dist/native/NativeRoaringBitmap32.js +221 -0
- package/dist/native/ffi.d.ts +20 -0
- package/dist/native/ffi.js +48 -0
- package/dist/native/idMapperTestSupport.d.ts +67 -0
- package/dist/native/idMapperTestSupport.js +112 -0
- package/dist/native/index.d.ts +49 -0
- package/dist/native/index.js +87 -0
- package/dist/native/napi.d.ts +21 -0
- package/dist/native/napi.js +88 -0
- package/dist/native/types.d.ts +1298 -0
- package/dist/native/types.js +16 -0
- package/dist/plugin.d.ts +50 -0
- package/dist/plugin.js +388 -0
- package/dist/providerContracts.d.ts +277 -0
- package/dist/providerContracts.js +38 -0
- package/dist/resource/OsMemoryProbe.d.ts +175 -0
- package/dist/resource/OsMemoryProbe.js +206 -0
- package/dist/resource/ResourceManager.d.ts +491 -0
- package/dist/resource/ResourceManager.js +960 -0
- package/dist/utils/ColumnManifest.d.ts +97 -0
- package/dist/utils/ColumnManifest.js +129 -0
- package/dist/utils/NativeColumnStore.d.ts +284 -0
- package/dist/utils/NativeColumnStore.js +685 -0
- package/dist/utils/NativeMetadataIndex.d.ts +882 -0
- package/dist/utils/NativeMetadataIndex.js +2631 -0
- package/dist/utils/NativeUnifiedCache.d.ts +87 -0
- package/dist/utils/NativeUnifiedCache.js +273 -0
- package/dist/utils/binaryIdMapperFactory.d.ts +59 -0
- package/dist/utils/binaryIdMapperFactory.js +94 -0
- package/dist/utils/collation.d.ts +30 -0
- package/dist/utils/collation.js +40 -0
- package/dist/utils/columnStoreTypes.d.ts +161 -0
- package/dist/utils/columnStoreTypes.js +29 -0
- package/dist/utils/nativeBinaryEntityIdMapper.d.ts +211 -0
- package/dist/utils/nativeBinaryEntityIdMapper.js +381 -0
- package/dist/utils/nativeEntityIdMapper.d.ts +111 -0
- package/dist/utils/nativeEntityIdMapper.js +170 -0
- package/docs/ADR-002-diskann-100-percent-rust.md +294 -0
- package/docs/ADR-003-semantic-time-travel.md +100 -0
- package/docs/aggregation.md +251 -0
- package/docs/comparison.md +162 -0
- package/docs/deployment-limits.md +87 -0
- package/docs/diskann.md +184 -0
- package/docs/migration-3.0.md +396 -0
- package/docs/performance-budget.md +274 -0
- package/docs/performance.md +117 -0
- package/docs/scaling.md +439 -0
- package/docs/snapshot-safety.md +246 -0
- package/docs/u64-id-space.md +214 -0
- package/docs/verification-report.md +670 -0
- package/native/brainy-native.node +0 -0
- package/native/index.d.ts +3916 -0
- package/package.json +92 -0
- package/scripts/migrate-cortex-2x-to-3x.mjs +970 -0
|
@@ -0,0 +1,3916 @@
|
|
|
1
|
+
/* auto-generated by NAPI-RS */
|
|
2
|
+
/* eslint-disable */
|
|
3
|
+
/**
|
|
4
|
+
* NativeAggregationEngine — napi wrapper for the Rust aggregation engine.
|
|
5
|
+
*
|
|
6
|
+
* Cor owns compute, Brainy owns storage. All metric calculations happen
|
|
7
|
+
* in Rust; results are returned to TS for state merging and persistence.
|
|
8
|
+
*/
|
|
9
|
+
export declare class NativeAggregationEngine {
|
|
10
|
+
constructor()
|
|
11
|
+
/** Register an aggregate definition (compiles and caches for hot path). */
|
|
12
|
+
defineAggregate(definitionJson: string): void
|
|
13
|
+
/** Remove a registered aggregate definition and its state. */
|
|
14
|
+
removeAggregate(name: string): void
|
|
15
|
+
/**
|
|
16
|
+
* Incrementally update aggregation state when an entity changes.
|
|
17
|
+
*
|
|
18
|
+
* Returns JSON array of modified AggregateGroupState objects.
|
|
19
|
+
* Entity/prev are passed as serde_json::Value (NAPI converts JS objects directly).
|
|
20
|
+
*/
|
|
21
|
+
incrementalUpdate(name: string, entity: any, op: string, prev?: any | undefined | null): string
|
|
22
|
+
/**
|
|
23
|
+
* Compute the group key for an entity given groupBy dimensions.
|
|
24
|
+
* Returns JSON string of the group key object.
|
|
25
|
+
*/
|
|
26
|
+
computeGroupKey(entity: any, groupByJson: string): string
|
|
27
|
+
/**
|
|
28
|
+
* Rebuild an entire aggregate from scratch using Rayon parallelism.
|
|
29
|
+
*
|
|
30
|
+
* Takes definition and entities as JSON strings.
|
|
31
|
+
* Returns JSON string of Map<serializedGroupKey, AggregateGroupState>.
|
|
32
|
+
*/
|
|
33
|
+
rebuildAggregate(definitionJson: string, entitiesJson: string): string
|
|
34
|
+
/**
|
|
35
|
+
* Query aggregate state with filtering/sorting/pagination.
|
|
36
|
+
*
|
|
37
|
+
* Takes state and params as JSON strings.
|
|
38
|
+
* Returns JSON array of AggregateResult objects.
|
|
39
|
+
*/
|
|
40
|
+
queryAggregate(stateJson: string, paramsJson: string): string
|
|
41
|
+
/** Serialize all internal state for persistence. */
|
|
42
|
+
serializeState(): string
|
|
43
|
+
/** Restore internal state from previously serialized data. */
|
|
44
|
+
restoreState(stateJson: string): void
|
|
45
|
+
/** Get engine statistics as JSON string. */
|
|
46
|
+
get stats(): string
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** JS-exposed binary EntityIdMapper. */
|
|
50
|
+
export declare class NativeBinaryIdMapper {
|
|
51
|
+
constructor()
|
|
52
|
+
/** Create a fresh mapper (truncates existing files at the paths). */
|
|
53
|
+
static create(config: BinaryIdMapperConfig): NativeBinaryIdMapper
|
|
54
|
+
/**
|
|
55
|
+
* Open an existing mapper. Runs the consistency check between
|
|
56
|
+
* uuid_to_int and int_to_uuid; surfaces crash-induced
|
|
57
|
+
* inconsistencies as errors.
|
|
58
|
+
*/
|
|
59
|
+
static openExisting(config: BinaryIdMapperConfig): NativeBinaryIdMapper
|
|
60
|
+
/**
|
|
61
|
+
* Current id-mapper generation across both substrates
|
|
62
|
+
* (uuid_to_int + int_to_uuid). Returned as BigInt — fits the
|
|
63
|
+
* brainy generation contract; lossless across the JS bridge
|
|
64
|
+
* regardless of value range.
|
|
65
|
+
*/
|
|
66
|
+
currentGeneration(): bigint
|
|
67
|
+
/**
|
|
68
|
+
* Pin a generation across BOTH substrates. The returned
|
|
69
|
+
* `NativeBinaryIdMapperPin` MUST be `release()`d (or dropped
|
|
70
|
+
* by the JS GC) for compaction to resume; while alive, both
|
|
71
|
+
* substrates retain the on-disk base files at that
|
|
72
|
+
* generation.
|
|
73
|
+
*
|
|
74
|
+
* Use this to back `brain.db().asOf(g)` /
|
|
75
|
+
* `brain.db().with(...)` time-travel views (Datomic-style
|
|
76
|
+
* immutable Db API, Piece 17). brainy holds one Pin per
|
|
77
|
+
* substrate per view; when the view drops, every Pin
|
|
78
|
+
* releases together.
|
|
79
|
+
*
|
|
80
|
+
* Throws if `generation` is strictly newer than the current
|
|
81
|
+
* generation (the brain hasn't advanced to that point yet).
|
|
82
|
+
*/
|
|
83
|
+
pin(generation: bigint): NativeBinaryIdMapperPin
|
|
84
|
+
/**
|
|
85
|
+
* Smallest currently-pinned generation across both
|
|
86
|
+
* substrates, or `null` when no pins are held. Diagnostic;
|
|
87
|
+
* compaction queries each substrate independently.
|
|
88
|
+
*/
|
|
89
|
+
minPinnedGeneration(): bigint | null
|
|
90
|
+
/**
|
|
91
|
+
* **Phase D step 5b — point-in-time lookup.** Returns the
|
|
92
|
+
* int assigned to `uuid` as it existed at brainy generation
|
|
93
|
+
* `at_gen`. Caller must hold a [`NativeBinaryIdMapperPin`]
|
|
94
|
+
* at `at_gen` for the result to be stable across concurrent
|
|
95
|
+
* compaction. Throws in U64 mode — use `getIntAtGenerationBig`.
|
|
96
|
+
*/
|
|
97
|
+
getIntAtGeneration(uuid: Buffer, atGen: bigint): number | null
|
|
98
|
+
/**
|
|
99
|
+
* **Phase D step 5b — BigInt sibling.** Returns the int
|
|
100
|
+
* assigned to `uuid` at brainy generation `at_gen` as a
|
|
101
|
+
* `BigInt`. Use in U64 mode; the u32 sibling is
|
|
102
|
+
* `getIntAtGeneration`.
|
|
103
|
+
*/
|
|
104
|
+
getIntAtGenerationBig(uuid: Buffer, atGen: bigint): bigint | null
|
|
105
|
+
/**
|
|
106
|
+
* **Phase D step 5d — int → UUID point-in-time lookup.**
|
|
107
|
+
* Returns the UUID assigned to `int` as it existed at
|
|
108
|
+
* brainy generation `at_gen`. Throws in U64 mode — use
|
|
109
|
+
* `getUuidAtGenerationBig`.
|
|
110
|
+
*/
|
|
111
|
+
getUuidAtGeneration(int: number, atGen: bigint): Buffer | null
|
|
112
|
+
/** **Phase D step 5d — BigInt sibling for U64 mode.** */
|
|
113
|
+
getUuidAtGenerationBig(int: bigint, atGen: bigint): Buffer | null
|
|
114
|
+
/**
|
|
115
|
+
* Retrieve the int previously assigned to `uuid`, or allocate a
|
|
116
|
+
* fresh one. Idempotent across concurrent callers for the same
|
|
117
|
+
* UUID (256 sharded per-UUID mutexes enforce single assignment).
|
|
118
|
+
*
|
|
119
|
+
* Returns a JS `number`. Throws if this brain is `IdSpace::U64`
|
|
120
|
+
* — use [`get_or_assign_big`] in that case.
|
|
121
|
+
*/
|
|
122
|
+
getOrAssign(uuid: Buffer): number
|
|
123
|
+
/**
|
|
124
|
+
* Lookup the int for `uuid`. Returns null if not present. Throws
|
|
125
|
+
* in `IdSpace::U64` mode — use [`get_int_big`].
|
|
126
|
+
*/
|
|
127
|
+
getInt(uuid: Buffer): number | null
|
|
128
|
+
/**
|
|
129
|
+
* Lookup the UUID for `int`. Returns a 16-byte Buffer or null.
|
|
130
|
+
* Accepts a JS `number`. Throws in `IdSpace::U64` mode — use
|
|
131
|
+
* [`get_uuid_big`].
|
|
132
|
+
*/
|
|
133
|
+
getUuid(int: number): Buffer | null
|
|
134
|
+
/**
|
|
135
|
+
* Live (non-tombstone) entry count as a JS `number`. Throws in
|
|
136
|
+
* `IdSpace::U64` mode — use [`size_big`].
|
|
137
|
+
*/
|
|
138
|
+
size(): number
|
|
139
|
+
/**
|
|
140
|
+
* Largest int ever assigned + 1, as a JS `number`. Throws in
|
|
141
|
+
* `IdSpace::U64` mode — use [`next_int_big`].
|
|
142
|
+
*/
|
|
143
|
+
nextInt(): number
|
|
144
|
+
/**
|
|
145
|
+
* Materialise every live int into a single Vec. **Avoid at
|
|
146
|
+
* billion scale** — at U32 the cap is 100 M (~400 MB array on
|
|
147
|
+
* the V8 heap); above the cap throws and points at the streaming
|
|
148
|
+
* `iterator` API. U64 mode disables this method unconditionally;
|
|
149
|
+
* use `iterator()` instead.
|
|
150
|
+
*/
|
|
151
|
+
getAllIntIds(): Array<number>
|
|
152
|
+
/**
|
|
153
|
+
* BigInt sibling of [`get_or_assign`]. Works in both `IdSpace`
|
|
154
|
+
* modes — U32 brains return a BigInt that fits in 4 bytes; U64
|
|
155
|
+
* brains may return values > 2^53 (lossless via BigInt, lossy as
|
|
156
|
+
* a JS number).
|
|
157
|
+
*/
|
|
158
|
+
getOrAssignBig(uuid: Buffer): bigint
|
|
159
|
+
/** BigInt sibling of [`get_int`]. */
|
|
160
|
+
getIntBig(uuid: Buffer): bigint | null
|
|
161
|
+
/** BigInt sibling of [`get_uuid`]. */
|
|
162
|
+
getUuidBig(int: bigint): Buffer | null
|
|
163
|
+
/**
|
|
164
|
+
* **Batch sibling of [`get_uuid_big`] (cor 3.0 Phase A, ADR-006 Q5).**
|
|
165
|
+
* Resolve many entity ints to their UUID bytes in ONE boundary
|
|
166
|
+
* crossing — the `entityIntsToUuids` provider path brainy uses to
|
|
167
|
+
* hydrate a `Subgraph`'s `nodes` column (one crossing beats N
|
|
168
|
+
* per-int `getUuidBig` calls when a viz/export renders many rows).
|
|
169
|
+
* Returns a parallel vector: `Some(16-byte UUID Buffer)` for a
|
|
170
|
+
* mapped int, `None` for an unmapped one, so the result stays
|
|
171
|
+
* index-aligned with the input (the TS wrapper stringifies each /
|
|
172
|
+
* fills gaps).
|
|
173
|
+
*/
|
|
174
|
+
intsToUuidsBig(ints: Array<bigint>): Array<Buffer | undefined | null>
|
|
175
|
+
/** BigInt sibling of [`size`]. */
|
|
176
|
+
sizeBig(): bigint
|
|
177
|
+
/** BigInt sibling of [`next_int`]. */
|
|
178
|
+
nextIntBig(): bigint
|
|
179
|
+
/**
|
|
180
|
+
* Construct a streaming iterator over every live `(int, uuid)`
|
|
181
|
+
* pair. The iterator snapshots `next_int` at construction time —
|
|
182
|
+
* allocations performed after the iterator was created are NOT
|
|
183
|
+
* visible. Tombstoned slots are skipped.
|
|
184
|
+
*
|
|
185
|
+
* Returns a [`NativeBinaryIdMapperIter`] with [`next_batch`]
|
|
186
|
+
* yielding BigInt ints (uniform across IdSpace modes).
|
|
187
|
+
*/
|
|
188
|
+
iterator(): NativeBinaryIdMapperIter
|
|
189
|
+
/** Tombstone an entry. The int is NEVER reused (stability contract). */
|
|
190
|
+
remove(uuid: Buffer): boolean
|
|
191
|
+
/** Flush dirty mmap pages to disk. */
|
|
192
|
+
flush(): void
|
|
193
|
+
/** Convenience: filesystem paths to the underlying files. */
|
|
194
|
+
paths(): Array<string>
|
|
195
|
+
/**
|
|
196
|
+
* IdSpace as a string — `"u32"` or `"u64"`. The TS layer uses
|
|
197
|
+
* this for the dispatch in [`BinaryIdMapperConfig::id_space`].
|
|
198
|
+
*/
|
|
199
|
+
idSpace(): string
|
|
200
|
+
/**
|
|
201
|
+
* Algorithmic capabilities — exposed for diagnostics and parity
|
|
202
|
+
* tests against the JS reference mapper.
|
|
203
|
+
*
|
|
204
|
+
* The u32-typed `size` and `next_int` fields throw in
|
|
205
|
+
* `IdSpace::U64` mode (use `size_big` / `next_int_big`); the
|
|
206
|
+
* BigInt fields work in both modes.
|
|
207
|
+
*/
|
|
208
|
+
diagnostics(): BinaryIdMapperDiagnostics
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Streaming iterator over `(int, uuid)` pairs. Snapshots `next_int`
|
|
213
|
+
* at construction time; tombstoned slots are skipped during
|
|
214
|
+
* iteration. Returned by [`NativeBinaryIdMapper::iterator`].
|
|
215
|
+
*
|
|
216
|
+
* JS callers drive iteration via [`next_batch`]. Each batch returns
|
|
217
|
+
* up to `max_batch` BigInt ints. The iterator is exhausted when
|
|
218
|
+
* `next_batch` returns `null` (the snapshot range has been fully
|
|
219
|
+
* walked).
|
|
220
|
+
*/
|
|
221
|
+
export declare class NativeBinaryIdMapperIter {
|
|
222
|
+
/**
|
|
223
|
+
* Read the next batch of live ints (tombstones skipped). Returns
|
|
224
|
+
* `null` when the iterator is exhausted. `max_batch` is an upper
|
|
225
|
+
* bound on the number of SLOTS scanned per call, not the number
|
|
226
|
+
* of live ints returned — heavily tombstoned regions yield small
|
|
227
|
+
* batches.
|
|
228
|
+
*/
|
|
229
|
+
nextBatch(maxBatch: number): Array<bigint> | null
|
|
230
|
+
/**
|
|
231
|
+
* Snapshot upper bound (exclusive) of the iteration range, fixed
|
|
232
|
+
* at iterator construction time.
|
|
233
|
+
*/
|
|
234
|
+
rangeEnd(): bigint
|
|
235
|
+
/**
|
|
236
|
+
* Current cursor position. Useful for resumable iteration and
|
|
237
|
+
* progress reporting.
|
|
238
|
+
*/
|
|
239
|
+
cursor(): bigint
|
|
240
|
+
/** True when the iterator is exhausted (cursor >= range_end). */
|
|
241
|
+
isDone(): boolean
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* JS handle to a BinaryIdMapperPin. Holds both substrates' pins
|
|
246
|
+
* (uuid_to_int + int_to_uuid) at the same generation; releasing
|
|
247
|
+
* drops both atomically.
|
|
248
|
+
*
|
|
249
|
+
* JS has no RAII, so callers must call `release()` explicitly to
|
|
250
|
+
* allow compaction to resume. If the JS GC reaps the handle
|
|
251
|
+
* first, Drop runs the same release path — but relying on GC
|
|
252
|
+
* timing is not recommended for production code.
|
|
253
|
+
*/
|
|
254
|
+
export declare class NativeBinaryIdMapperPin {
|
|
255
|
+
/**
|
|
256
|
+
* The pinned generation. Stable for the lifetime of the
|
|
257
|
+
* handle.
|
|
258
|
+
*/
|
|
259
|
+
generation(): bigint
|
|
260
|
+
/**
|
|
261
|
+
* Release the pin. Idempotent: calling twice is a no-op.
|
|
262
|
+
* After release, the brain may compact past the pinned
|
|
263
|
+
* generation; any read against the pinned view becomes
|
|
264
|
+
* undefined.
|
|
265
|
+
*/
|
|
266
|
+
release(): void
|
|
267
|
+
/**
|
|
268
|
+
* Whether the pin is still held. `false` after `release()`
|
|
269
|
+
* or after the JS GC has reaped a backing Rust handle that
|
|
270
|
+
* internally released.
|
|
271
|
+
*/
|
|
272
|
+
isHeld(): boolean
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
/**
|
|
276
|
+
* Native column store exposed to TypeScript via napi-rs.
|
|
277
|
+
*
|
|
278
|
+
* Each instance manages segments for one or more fields. The TypeScript
|
|
279
|
+
* wrapper (`NativeColumnStore.ts`) bridges storage I/O: it loads segment
|
|
280
|
+
* data from the storage adapter and passes it to Rust for mmap-based
|
|
281
|
+
* querying. All four brainy value types are fully native: `Number` and
|
|
282
|
+
* `Boolean` share the i64 path, `Float` uses an f64 path, and `String`
|
|
283
|
+
* uses length-prefixed UTF-8. A field's type is inferred from its tail
|
|
284
|
+
* buffer or loaded segment headers.
|
|
285
|
+
*/
|
|
286
|
+
export declare class NativeColumnStore {
|
|
287
|
+
constructor()
|
|
288
|
+
/**
|
|
289
|
+
* **Piece 6 generation counter — accessor.** See
|
|
290
|
+
* `native/src/metadata_index/core.rs` for the ordering
|
|
291
|
+
* rationale shared across all native indexes.
|
|
292
|
+
*/
|
|
293
|
+
currentGeneration(): bigint
|
|
294
|
+
/** Load a segment from a buffer (in-memory storage). */
|
|
295
|
+
loadSegmentFromBuffer(field: string, segmentId: string, data: Buffer): void
|
|
296
|
+
/** Load a segment by memory-mapping a file path (zero-copy reads). */
|
|
297
|
+
loadSegmentFromPath(field: string, segmentId: string, path: string): void
|
|
298
|
+
/**
|
|
299
|
+
* Add an entry to the in-memory tail buffer for a numeric (i64) field.
|
|
300
|
+
*
|
|
301
|
+
* **Cor 3.0 Piece E**: consolidated to a single BigInt API.
|
|
302
|
+
* The legacy `addNumericBig` alias was removed; this method now
|
|
303
|
+
* accepts the full u64 range. Cortex 2.x callers passing a JS
|
|
304
|
+
* number must wrap as BigInt (`BigInt(n)`) at the call site.
|
|
305
|
+
*/
|
|
306
|
+
addNumeric(field: string, value: number, entityIntId: bigint): void
|
|
307
|
+
/**
|
|
308
|
+
* Add an entry to the in-memory tail buffer for a float (f64) field.
|
|
309
|
+
*
|
|
310
|
+
* Float values are stored as full IEEE 754 doubles (not rounded to
|
|
311
|
+
* i64), so non-integer values like `19.99` are preserved and ordered
|
|
312
|
+
* correctly. On flush this becomes a [`ValueType::Float`] `.cidx` segment.
|
|
313
|
+
*/
|
|
314
|
+
addFloat(field: string, value: number, entityIntId: bigint): void
|
|
315
|
+
/** Add an entry to the in-memory tail buffer for a string field. */
|
|
316
|
+
addString(field: string, value: string, entityIntId: bigint): void
|
|
317
|
+
/** Mark an entity as deleted for a specific field. */
|
|
318
|
+
deleteEntity(field: string, entityIntId: bigint): void
|
|
319
|
+
/**
|
|
320
|
+
* Sort top-K entity ints by a field (numeric, float, or string —
|
|
321
|
+
* auto-routed by field type). Returns BigInts for full u64
|
|
322
|
+
* fidelity at billion scale.
|
|
323
|
+
*
|
|
324
|
+
* The k-way merge requires each cursor to be sorted, so the live
|
|
325
|
+
* tail buffer (in insertion order) is sorted into a per-query
|
|
326
|
+
* copy first. Segments are already sorted on disk; the tail is
|
|
327
|
+
* bounded by the flush threshold, so the copy+sort is cheap.
|
|
328
|
+
*/
|
|
329
|
+
sortTopK(field: string, order: string, k: number): Array<bigint>
|
|
330
|
+
/** Filter: entity IDs where the numeric field equals the value (serialized roaring bitmap). */
|
|
331
|
+
filterNumeric(field: string, value: number): Buffer
|
|
332
|
+
/** Range query: entity IDs where numeric value is in [lo, hi] (serialized roaring bitmap). */
|
|
333
|
+
rangeQueryNumeric(field: string, lo: number, hi: number): Buffer
|
|
334
|
+
/** Filter: entity IDs where the float (f64) field equals the value (serialized roaring bitmap). */
|
|
335
|
+
filterFloat(field: string, value: number): Buffer
|
|
336
|
+
/** Range query: entity IDs where float (f64) value is in [lo, hi] (serialized roaring bitmap). */
|
|
337
|
+
rangeQueryFloat(field: string, lo: number, hi: number): Buffer
|
|
338
|
+
/** Filter: entity IDs where the string field equals the value (serialized roaring bitmap). */
|
|
339
|
+
filterString(field: string, value: string): Buffer
|
|
340
|
+
/** Range query: entity IDs where string value is in [lo, hi] (serialized roaring bitmap). */
|
|
341
|
+
rangeQueryString(field: string, lo: string, hi: string): Buffer
|
|
342
|
+
/**
|
|
343
|
+
* Write a field's tail buffer to a sorted `.cidx` segment buffer (numeric, float, or string).
|
|
344
|
+
* Returns the segment content as Buffer, or None if empty.
|
|
345
|
+
*
|
|
346
|
+
* **Cor 3.0 Piece E**: now always writes the u64-keyed segment
|
|
347
|
+
* variant (`write_*_segment_u64`). The pre-E u32-keyed writers
|
|
348
|
+
* (`write_*_segment`) stay available for crash-recovery readers
|
|
349
|
+
* of legacy data, but new flushes are exclusively u64. Tail-
|
|
350
|
+
* buffer ids are u64 throughout post-E.
|
|
351
|
+
*/
|
|
352
|
+
flushTailBuffer(field: string, level: number, segmentId: number): Buffer | null
|
|
353
|
+
/** Get tail buffer entry count for a field (numeric, float, or string). */
|
|
354
|
+
tailBufferSize(field: string): number
|
|
355
|
+
/** Check if any segments or tail data exist for a field. */
|
|
356
|
+
hasField(field: string): boolean
|
|
357
|
+
/**
|
|
358
|
+
* All distinct values for a field across segments + tail buffer, as
|
|
359
|
+
* strings in ascending order (numeric by value, string by code point).
|
|
360
|
+
* Backs the `getFilterValues` provider method. Deleted entities are not
|
|
361
|
+
* excluded — values are reported if any (live or tombstoned) entity holds
|
|
362
|
+
* them, matching brainy's segment-derived value enumeration.
|
|
363
|
+
*/
|
|
364
|
+
distinctValues(field: string): Array<string>
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
/** Napi-exposed DiskANN index. */
|
|
368
|
+
export declare class NativeDiskAnn {
|
|
369
|
+
/** Construct via static factories. */
|
|
370
|
+
constructor()
|
|
371
|
+
/**
|
|
372
|
+
* Build a fresh DiskANN index from a flat f32 vector buffer and
|
|
373
|
+
* write it to `output_path`. Returns the opened searcher.
|
|
374
|
+
*
|
|
375
|
+
* `vectors` is a Buffer of `node_count × dim × 4` little-endian
|
|
376
|
+
* f32 bytes. For datasets that don't fit in a single Buffer, use
|
|
377
|
+
* [`build_from_file`](Self::build_from_file).
|
|
378
|
+
*/
|
|
379
|
+
static build(vectors: Buffer, dim: number, outputPath: string, config?: DiskAnnConfig | undefined | null): NativeDiskAnn
|
|
380
|
+
/**
|
|
381
|
+
* Build a fresh DiskANN index, streaming vectors from a flat file
|
|
382
|
+
* at `vectors_path`. The file contains `header_skip` arbitrary
|
|
383
|
+
* leading bytes followed by `node_count × dim × 4` LE f32 bytes.
|
|
384
|
+
*
|
|
385
|
+
* This is the billion-scale entry point: the source dataset stays
|
|
386
|
+
* mmap'd, so 1.5 TB of vectors never touches the JS heap.
|
|
387
|
+
*/
|
|
388
|
+
static buildFromFile(vectorsPath: string, headerSkip: number, dim: number, outputPath: string, config?: DiskAnnConfig | undefined | null): NativeDiskAnn
|
|
389
|
+
/**
|
|
390
|
+
* Build a fresh DiskANN index from a BIGANN/SIFT `.bvecs` base file
|
|
391
|
+
* — the SIFT100M / SIFT1B verification entry point.
|
|
392
|
+
*
|
|
393
|
+
* `.bvecs` stores each vector as a 4-byte LE dimension prefix + `dim`
|
|
394
|
+
* `u8` components, the canonical billion-scale benchmark layout that
|
|
395
|
+
* [`build_from_file`](Self::build_from_file) (flat f32) can't read
|
|
396
|
+
* directly. This factory streams the first `node_count` records of
|
|
397
|
+
* `bvecs_path` into a flat-f32 sidecar at `f32_cache_path`, then
|
|
398
|
+
* builds from that sidecar via the mmap'd source — so the 512 GB of
|
|
399
|
+
* SIFT1B f32 vectors never touch the JS heap or physical RAM.
|
|
400
|
+
*
|
|
401
|
+
* The sidecar is reused when it already exists at the exact expected
|
|
402
|
+
* size (`node_count × dim × 4` bytes); converting the 132 GB SIFT1B
|
|
403
|
+
* base is slow, so re-runs at the same scale skip it. Pass distinct
|
|
404
|
+
* `f32_cache_path`s for distinct scales (the SIFT100M sidecar is a
|
|
405
|
+
* strict prefix of the SIFT1B one, but the harness keys them by
|
|
406
|
+
* subset for clarity).
|
|
407
|
+
*
|
|
408
|
+
* `node_count` is `u32`: a single DiskANN shard addresses at most
|
|
409
|
+
* `u32::MAX − 1` slots by construction (see the module-level slot-ID
|
|
410
|
+
* note), and SIFT1B (1e9) fits comfortably.
|
|
411
|
+
*/
|
|
412
|
+
static buildFromBvecsFile(bvecsPath: string, dim: number, nodeCount: number, f32CachePath: string, outputPath: string, config?: DiskAnnConfig | undefined | null): NativeDiskAnn
|
|
413
|
+
/**
|
|
414
|
+
* Rebuild a DiskANN index by merging a subset of an existing index
|
|
415
|
+
* (the "live" slots — typically existing minus tombstones) with a
|
|
416
|
+
* freshly-inserted delta buffer. **Critical for billion-scale
|
|
417
|
+
* rebuild correctness**: the existing vectors stay mmap'd; only
|
|
418
|
+
* the delta crosses the FFI boundary. At 1B × 1536 × 4 bytes
|
|
419
|
+
* (= ~6 TB) of existing vector data, this is the difference
|
|
420
|
+
* between "rebuild works" and "rebuild OOMs every time".
|
|
421
|
+
*
|
|
422
|
+
* ## Slot layout in the new index
|
|
423
|
+
*
|
|
424
|
+
* New slot `i` maps to:
|
|
425
|
+
* - `live_old_slots[i]` in the OLD index (for `i < live_old_slots.len()`)
|
|
426
|
+
* - `delta_vectors[i - live_old_slots.len()]` (for the tail)
|
|
427
|
+
*
|
|
428
|
+
* The caller is responsible for rebuilding the UUID↔slot maps
|
|
429
|
+
* using this invariant; the napi layer doesn't carry UUIDs.
|
|
430
|
+
*
|
|
431
|
+
* ## Path handling
|
|
432
|
+
*
|
|
433
|
+
* If `output_path == existing_path` (in-place rebuild), the new
|
|
434
|
+
* index is written to `<output_path>.rebuild-tmp` first, the old
|
|
435
|
+
* mmap is dropped, then the temp file is atomically renamed over
|
|
436
|
+
* the original. This avoids mmap'ing a file we're concurrently
|
|
437
|
+
* writing.
|
|
438
|
+
*/
|
|
439
|
+
static rebuildFromExisting(params: DiskAnnRebuildFromExistingParams): NativeDiskAnn
|
|
440
|
+
/**
|
|
441
|
+
* Async sibling of [`Self::rebuild_from_existing`] — identical build
|
|
442
|
+
* semantics, but the fold runs on a Node worker thread
|
|
443
|
+
* ([`AsyncTask`] → the libuv pool) so the JS event loop keeps
|
|
444
|
+
* serving reads/writes for the entire build. This is the "serving
|
|
445
|
+
* never freezes" primitive: at billion scale the synchronous
|
|
446
|
+
* factory blocks the loop for hours-to-days; this variant blocks
|
|
447
|
+
* nothing. Resolves to the reopened new index once the on-disk
|
|
448
|
+
* artifacts (index + slotmap sidecars) are durably in place.
|
|
449
|
+
*
|
|
450
|
+
* Input buffers are copied out of JS memory on the calling thread
|
|
451
|
+
* (bounded by the wrapper's delta budget), so the task owns plain
|
|
452
|
+
* `Send` data and the JS side may freely mutate its buffers after
|
|
453
|
+
* the call returns.
|
|
454
|
+
*/
|
|
455
|
+
static rebuildFromExistingAsync(params: DiskAnnRebuildFromExistingParams): Promise<NativeDiskAnn>
|
|
456
|
+
/**
|
|
457
|
+
* Read a contiguous chunk of vectors from the underlying mmap'd
|
|
458
|
+
* index. Zero-copy on the Rust side (the returned Buffer is a
|
|
459
|
+
* fresh allocation, but the read from the mmap is direct). Useful
|
|
460
|
+
* for diagnostic tooling + small-scale rebuilds that don't go
|
|
461
|
+
* through [`rebuild_from_existing`].
|
|
462
|
+
*
|
|
463
|
+
* Returns a Buffer of `count * dim * 4` LE f32 bytes. If
|
|
464
|
+
* `start + count` exceeds the index size, the trailing slots are
|
|
465
|
+
* silently truncated.
|
|
466
|
+
*/
|
|
467
|
+
readVectorChunk(start: number, count: number): Buffer
|
|
468
|
+
/**
|
|
469
|
+
* Open an existing DiskANN file for search-only access.
|
|
470
|
+
*
|
|
471
|
+
* `mode` selects the per-section madvise profile applied at open
|
|
472
|
+
* time — one of `"in-memory" | "hybrid" | "on-disk"`. When
|
|
473
|
+
* omitted, the file header drives the choice: `pq_m == 0`
|
|
474
|
+
* (Mode 1 marker) → in-memory hints; PQ-enabled → hybrid hints.
|
|
475
|
+
* The mode never changes correctness; it only biases the OS
|
|
476
|
+
* page cache toward the right residency for the operating
|
|
477
|
+
* regime. See [`OpenMode`] for the full hint table.
|
|
478
|
+
*/
|
|
479
|
+
static openExisting(path: string, mode?: string | undefined | null): NativeDiskAnn
|
|
480
|
+
/**
|
|
481
|
+
* k-NN search. `l_search` controls the greedy-walk candidate set;
|
|
482
|
+
* the DiskANN paper recommends `l_search ≥ 2k`. `padding_factor`
|
|
483
|
+
* is the **optional** re-rank cap: when omitted, the searcher
|
|
484
|
+
* re-ranks the **full walked candidate list** with exact vectors
|
|
485
|
+
* (canonical DiskANN, best recall — the mmap-resident vectors make
|
|
486
|
+
* it near-free). Pass an explicit value to bound the re-rank to
|
|
487
|
+
* `ceil(k × padding_factor)` for the cold-NVMe random-read case.
|
|
488
|
+
*/
|
|
489
|
+
search(query: Array<number>, k: number, lSearch?: number | undefined | null, paddingFactor?: number | undefined | null): Array<DiskAnnHit>
|
|
490
|
+
/**
|
|
491
|
+
* Filtered k-NN search — **predicate pushdown** (#10). `allowed_slots`
|
|
492
|
+
* are the graph-internal slot IDs the result may contain; the TS
|
|
493
|
+
* wrapper translates the metadata∩graph entity universe → slots and
|
|
494
|
+
* inflates `l_search` by the predicate selectivity. The walk traverses
|
|
495
|
+
* through all nodes but only collects allowed slots, so this returns
|
|
496
|
+
* far higher recall than post-filtering the unfiltered top-k when the
|
|
497
|
+
* predicate is selective. An empty `allowed_slots` returns no hits.
|
|
498
|
+
*/
|
|
499
|
+
searchFiltered(query: Array<number>, k: number, allowedSlots: Array<number>, lSearch?: number | undefined | null, paddingFactor?: number | undefined | null): Array<DiskAnnHit>
|
|
500
|
+
/**
|
|
501
|
+
* Exact top-k over an explicit slot set — **brute-force pushdown** for
|
|
502
|
+
* extreme-selectivity filters. The TS wrapper routes here (instead of
|
|
503
|
+
* `searchFiltered`) when `k / selectivity` would exceed the walk's
|
|
504
|
+
* `l_search` cap: a bounded walk can't reach scattered allowed points, so
|
|
505
|
+
* scanning the (tiny) allowed set is both faster AND exact. `slots` are
|
|
506
|
+
* graph-internal slot ids; out-of-range ids are skipped.
|
|
507
|
+
*/
|
|
508
|
+
searchExactSlots(query: Array<number>, k: number, slots: Array<number>): Array<DiskAnnHit>
|
|
509
|
+
/** Number of vectors in the index. */
|
|
510
|
+
size(): number
|
|
511
|
+
/** Header introspection. */
|
|
512
|
+
header(): DiskAnnHeader
|
|
513
|
+
/** On-disk file path. */
|
|
514
|
+
filePath(): string
|
|
515
|
+
/**
|
|
516
|
+
* Whether a valid forward (`slot → int`) slotmap is attached. The TS
|
|
517
|
+
* wrapper discards a cold-opened index whose slotmap failed to load
|
|
518
|
+
* (missing / stale) so it never resolves slot ids as entity ids.
|
|
519
|
+
*/
|
|
520
|
+
slotmapReady(): boolean
|
|
521
|
+
/**
|
|
522
|
+
* Read the entity ints for a set of slots from the forward slotmap.
|
|
523
|
+
* Index-aligned with the input (out-of-range slots fall back to the
|
|
524
|
+
* slot value). The TS rebuild path uses this to recover the stable
|
|
525
|
+
* entity ints of retained ("live old") slots without a resident
|
|
526
|
+
* `slot → int` map — they pass straight back into the new slotmap.
|
|
527
|
+
*/
|
|
528
|
+
slotIdsForSlots(slots: Array<number>): Array<bigint>
|
|
529
|
+
/**
|
|
530
|
+
* Translate a set of entity ints to the slots that hold them in this
|
|
531
|
+
* shard via the reverse (`int → slot`) slotmap. Entity ints not in
|
|
532
|
+
* this shard are dropped (the result is the present subset, NOT
|
|
533
|
+
* index-aligned) — exactly the `allowedSlots` set the predicate-
|
|
534
|
+
* pushdown walk consumes. Empty when no slotmap is attached.
|
|
535
|
+
*/
|
|
536
|
+
slotsForInts(ints: Array<bigint>): Array<number>
|
|
537
|
+
/**
|
|
538
|
+
* Whether an entity int is present in this shard (reverse slotmap
|
|
539
|
+
* membership). Backs `removeItem` + the tombstone count. `false`
|
|
540
|
+
* when no slotmap is attached.
|
|
541
|
+
*/
|
|
542
|
+
hasInt(int: bigint): boolean
|
|
543
|
+
}
|
|
544
|
+
export type NativeDiskANN = NativeDiskAnn
|
|
545
|
+
|
|
546
|
+
/**
|
|
547
|
+
* Native embedding engine (napi-rs class).
|
|
548
|
+
*
|
|
549
|
+
* Pooling is always mean over token embeddings, weighted by the
|
|
550
|
+
* attention mask — the canonical sentence-embedding shape for the
|
|
551
|
+
* MiniLM model family. We hardcode it rather than carry a
|
|
552
|
+
* strategy enum because (1) only mean-pooling is ever used and
|
|
553
|
+
* (2) adding CLS / max-pool later costs a few lines plus a
|
|
554
|
+
* matching test, not a field that lies dormant in every engine.
|
|
555
|
+
*/
|
|
556
|
+
export declare class NativeEmbeddingEngine {
|
|
557
|
+
/** Create a new embedding engine instance (not yet loaded). */
|
|
558
|
+
constructor()
|
|
559
|
+
/**
|
|
560
|
+
* Load the model and tokenizer from byte buffers.
|
|
561
|
+
*
|
|
562
|
+
* Arguments:
|
|
563
|
+
* - model_bytes: SafeTensors format model weights
|
|
564
|
+
* - tokenizer_bytes: tokenizer.json contents
|
|
565
|
+
* - config_bytes: config.json contents
|
|
566
|
+
*/
|
|
567
|
+
load(modelBytes: Buffer, tokenizerBytes: Buffer, configBytes: Buffer): void
|
|
568
|
+
/** Check if the engine has a loaded model and tokenizer. */
|
|
569
|
+
isReady(): boolean
|
|
570
|
+
/** Generate embedding for a single text. Returns Vec<f64> of 384 dimensions. */
|
|
571
|
+
embed(text: string): Array<number>
|
|
572
|
+
/**
|
|
573
|
+
* Generate embeddings for multiple texts.
|
|
574
|
+
* Returns Vec<Vec<f64>>.
|
|
575
|
+
*/
|
|
576
|
+
embedBatch(texts: Array<string>): Array<Array<number>>
|
|
577
|
+
/** Get the embedding dimension (384 for all-MiniLM-L6-v2). */
|
|
578
|
+
dimension(): number
|
|
579
|
+
/** Get the maximum sequence length. */
|
|
580
|
+
maxSequenceLength(): number
|
|
581
|
+
/** Get engine statistics. */
|
|
582
|
+
getStats(): EngineStats
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
/**
|
|
586
|
+
* Bidirectional UUID ↔ integer mapper for roaring bitmap integration.
|
|
587
|
+
*
|
|
588
|
+
* All methods are synchronous (O(1) HashMap lookups).
|
|
589
|
+
* Persistence is handled by the TypeScript wrapper via load_from_json/save_to_json.
|
|
590
|
+
*/
|
|
591
|
+
export declare class NativeEntityIdMapper {
|
|
592
|
+
/** Create a new empty mapper in `IdSpace::U32` mode. */
|
|
593
|
+
constructor(idSpace?: string | undefined | null)
|
|
594
|
+
/**
|
|
595
|
+
* Load state from JSON string (storage format). Replaces all
|
|
596
|
+
* current state. Sets dirty = false. The loader auto-detects
|
|
597
|
+
* IdSpace from the envelope (v1 / missing → U32).
|
|
598
|
+
*/
|
|
599
|
+
loadFromJson(json: string): void
|
|
600
|
+
/**
|
|
601
|
+
* Serialize state to JSON string for storage persistence.
|
|
602
|
+
* Format depends on this mapper's IdSpace — see the module-level
|
|
603
|
+
* docstring for the v1 / v2 envelopes.
|
|
604
|
+
*/
|
|
605
|
+
saveToJson(): string
|
|
606
|
+
/** IdSpace as a string — `"u32"` or `"u64"`. */
|
|
607
|
+
idSpace(): string
|
|
608
|
+
/**
|
|
609
|
+
* Get integer ID for UUID, assigning a new one if not mapped.
|
|
610
|
+
* Throws in `IdSpace::U64` mode — use [`get_or_assign_big`].
|
|
611
|
+
*/
|
|
612
|
+
getOrAssign(uuid: string): number
|
|
613
|
+
/**
|
|
614
|
+
* Get UUID for integer ID. Returns null if not found. Throws in
|
|
615
|
+
* `IdSpace::U64` mode — use [`get_uuid_big`].
|
|
616
|
+
*/
|
|
617
|
+
getUuid(intId: number): string | null
|
|
618
|
+
/**
|
|
619
|
+
* Get integer ID for UUID without assigning. Throws in
|
|
620
|
+
* `IdSpace::U64` mode — use [`get_int_big`].
|
|
621
|
+
*/
|
|
622
|
+
getInt(uuid: string): number | null
|
|
623
|
+
/**
|
|
624
|
+
* Total number of mappings as a JS `number`. Throws in
|
|
625
|
+
* `IdSpace::U64` mode — use [`size_big`].
|
|
626
|
+
*/
|
|
627
|
+
get size(): number
|
|
628
|
+
/**
|
|
629
|
+
* Convert UUIDs → ints, assigning new IDs as needed. Throws in
|
|
630
|
+
* `IdSpace::U64` mode — use [`uuids_to_ints_big`].
|
|
631
|
+
*/
|
|
632
|
+
uuidsToInts(uuids: Array<string>): Array<number>
|
|
633
|
+
/**
|
|
634
|
+
* Convert ints → UUIDs. Throws in `IdSpace::U64` mode — use
|
|
635
|
+
* [`ints_to_uuids_big`].
|
|
636
|
+
*/
|
|
637
|
+
intsToUuids(ints: Array<number>): Array<string>
|
|
638
|
+
/** BigInt sibling of [`get_or_assign`]. Works in both IdSpace modes. */
|
|
639
|
+
getOrAssignBig(uuid: string): bigint
|
|
640
|
+
/** BigInt sibling of [`get_uuid`]. */
|
|
641
|
+
getUuidBig(intId: bigint): string | null
|
|
642
|
+
/** BigInt sibling of [`get_int`]. */
|
|
643
|
+
getIntBig(uuid: string): bigint | null
|
|
644
|
+
/** BigInt sibling of [`size`]. */
|
|
645
|
+
sizeBig(): bigint
|
|
646
|
+
/** BigInt sibling of [`uuids_to_ints`]. */
|
|
647
|
+
uuidsToIntsBig(uuids: Array<string>): Array<bigint>
|
|
648
|
+
/** BigInt sibling of [`ints_to_uuids`]. */
|
|
649
|
+
intsToUuidsBig(ints: Array<bigint>): Array<string>
|
|
650
|
+
/** Check if UUID has been assigned an integer ID. */
|
|
651
|
+
has(uuid: string): boolean
|
|
652
|
+
/** Remove mapping for UUID. Returns true if it existed. */
|
|
653
|
+
remove(uuid: string): boolean
|
|
654
|
+
/** Whether there are unsaved changes. */
|
|
655
|
+
isDirty(): boolean
|
|
656
|
+
/** Clear all mappings and reset counter. Preserves IdSpace. */
|
|
657
|
+
clear(): void
|
|
658
|
+
/**
|
|
659
|
+
* Get statistics about the mapper. The legacy u32 fields cap at
|
|
660
|
+
* `u32::MAX`; the `*_big` fields are overflow-safe.
|
|
661
|
+
*/
|
|
662
|
+
getStats(): NativeEntityIdMapperStats
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
export declare class NativeGraphAdjacencyIndex {
|
|
666
|
+
constructor(config: GraphAdjacencyConfig)
|
|
667
|
+
/**
|
|
668
|
+
* **Piece 6 generation counter — accessor.** Returns the current
|
|
669
|
+
* generation as a BigInt (u64). The TS wrapper polls this on
|
|
670
|
+
* every read entry and invalidates its caches when the value
|
|
671
|
+
* changes. See `native/src/metadata_index/core.rs` for the
|
|
672
|
+
* memory-ordering rationale.
|
|
673
|
+
*/
|
|
674
|
+
currentGeneration(): bigint
|
|
675
|
+
/**
|
|
676
|
+
* **Cor 3.0 + brainy 8.0.** Add a verb to the graph with its
|
|
677
|
+
* `(source_int, target_int)` endpoint pair. The pair indexes:
|
|
678
|
+
* - the `verb_endpoints` store (`verb_int → (source_int, target_int)`),
|
|
679
|
+
* - the `source` LSM tree (`source_int → target_int`),
|
|
680
|
+
* - the `target` LSM tree (`target_int → source_int`),
|
|
681
|
+
* - the `verbs_source` LSM tree (`source_int → (verb_int, target_int)`),
|
|
682
|
+
* - the `verbs_target` LSM tree (`target_int → (verb_int, source_int)`).
|
|
683
|
+
*
|
|
684
|
+
* `subtype` is optional — when `Some(value)` the verb also lands
|
|
685
|
+
* in the per-`(type, subtype)` membership bitmap + count slot.
|
|
686
|
+
*
|
|
687
|
+
* **Endpoints are u64 BigInts** for fidelity at 10B+ scale (U64
|
|
688
|
+
* brains). U32 brains pass the same channel with zeroed high
|
|
689
|
+
* bits — no separate API needed.
|
|
690
|
+
*
|
|
691
|
+
* `generation` is brainy's commit generation for this write —
|
|
692
|
+
* the same value brainy threads into the LSM `insert(...,gen)`.
|
|
693
|
+
* It records the endpoint pair in the per-`verb_id` version
|
|
694
|
+
* chain at that generation so `getVerbEndpointsAtGeneration`
|
|
695
|
+
* resolves historical edges for `db.asOf(g)` (graph time travel).
|
|
696
|
+
*/
|
|
697
|
+
addVerbWithEndpoints(verbId: string, sourceInt: bigint, targetInt: bigint, verbType: string, subtype: string, generation: bigint): GraphAddVerbResult
|
|
698
|
+
/**
|
|
699
|
+
* **Strict variant.** Combines pre-resolved verb-type index with
|
|
700
|
+
* the canonical u64 endpoint pair. Happy path for brainy 8.0
|
|
701
|
+
* callers — they have the index from `TypeUtils.getVerbIndex` and
|
|
702
|
+
* the ints from the brain's `BinaryIdMapper`.
|
|
703
|
+
*/
|
|
704
|
+
addVerbWithEndpointsByIndex(verbId: string, sourceInt: bigint, targetInt: bigint, verbTypeIndex: number, subtype: string, generation: bigint): GraphAddVerbResult
|
|
705
|
+
/**
|
|
706
|
+
* Remove a verb (tombstone deletion). LSM-tree edges persist —
|
|
707
|
+
* removed verbs are filtered out by the membership bitmap on
|
|
708
|
+
* query. Endpoint slot is tombstoned too so a future
|
|
709
|
+
* `getVerbEndpoints` returns `None`.
|
|
710
|
+
*
|
|
711
|
+
* `verb_type_index` and `subtype` are required for the per-
|
|
712
|
+
* `(type, subtype)` bookkeeping (decrement the count slot, drop
|
|
713
|
+
* from the per-pair membership bitmap). Pass an empty string for
|
|
714
|
+
* `subtype` to skip the subtype-specific bookkeeping (legacy
|
|
715
|
+
* pre-brainy-8.0 verbs without a subtype). Brainy 8.0's
|
|
716
|
+
* `verbSubtypeByIdCache` is the source of truth for "which subtype
|
|
717
|
+
* did this verb have?" and passes the value back on remove.
|
|
718
|
+
*
|
|
719
|
+
* **Cor 3.0 / brainy 8.0 lockstep tightening (handoff I.1):**
|
|
720
|
+
* `subtype` is `String` (not `Option<String>`) at the napi boundary.
|
|
721
|
+
* Brainy 8.0's enforcement layer guarantees every verb write
|
|
722
|
+
* supplies a non-empty subtype; the empty-string escape exists
|
|
723
|
+
* solely for legacy-data remove paths.
|
|
724
|
+
*/
|
|
725
|
+
removeVerb(verbId: string, verbTypeIndex: number, subtype: string, generation: bigint): void
|
|
726
|
+
/**
|
|
727
|
+
* **Look up a verb's endpoints by verb-ID.** Returns
|
|
728
|
+
* `{sourceInt, targetInt}` (as u64 BigInts) when the verb was
|
|
729
|
+
* added via one of the `addVerbWithEndpoints*` variants AND is
|
|
730
|
+
* currently live (not tombstoned via `removeVerb`). Returns
|
|
731
|
+
* `None` (JS `null`) when:
|
|
732
|
+
*
|
|
733
|
+
* - The verb-ID was never interned (`addVerb` never called for
|
|
734
|
+
* it).
|
|
735
|
+
* - The verb was added via a back-compat `addVerb` /
|
|
736
|
+
* `addVerbBy*` / `addVerb*WithSubtype` variant that didn't
|
|
737
|
+
* carry endpoint info — those verbs are in the membership
|
|
738
|
+
* bitmap and LSM trees but not in the endpoint map.
|
|
739
|
+
* - The verb was removed.
|
|
740
|
+
*
|
|
741
|
+
* Brainy 8.0's multi-hop subtype BFS uses this lookup to resolve
|
|
742
|
+
* each filtered verb-ID to its (source, target) for the next hop;
|
|
743
|
+
* verbs with no endpoint info are skipped (the only correct
|
|
744
|
+
* fallback — guessing endpoints could pollute traversal results
|
|
745
|
+
* at billion-scale).
|
|
746
|
+
*/
|
|
747
|
+
getVerbEndpoints(verbId: string): VerbEndpointsResult | null
|
|
748
|
+
/**
|
|
749
|
+
* **Phase D step 5f/5g — point-in-time endpoint lookup
|
|
750
|
+
* (membership-at-generation correct).**
|
|
751
|
+
* Returns the (source, target) pair for `verb_id` as it
|
|
752
|
+
* existed at brainy generation `at_gen`. Caller must hold
|
|
753
|
+
* the verb-endpoints substrate's `PinGuard` at `at_gen`
|
|
754
|
+
* for the result to be stable across concurrent compaction.
|
|
755
|
+
*
|
|
756
|
+
* **Membership-at-generation semantics (5g).** This path
|
|
757
|
+
* trusts the endpoint store's per-`verb_id` version chain
|
|
758
|
+
* as the single source of truth for *both* the endpoints
|
|
759
|
+
* *and* the liveness of the verb at `at_gen`. The chain
|
|
760
|
+
* records `Some((src,tgt))` at the generation a verb was
|
|
761
|
+
* (re)written and `None` at the generation it was
|
|
762
|
+
* tombstoned; `get_at_generation` walks
|
|
763
|
+
* `range(..=at_gen).next_back()`, so:
|
|
764
|
+
*
|
|
765
|
+
* - added g1, removed g5, read g3 → `Some` (live then) ✓
|
|
766
|
+
* - added g1, removed g5, read g7 → `None` (gone then) ✓
|
|
767
|
+
* - added g5, read g3 → chain-miss → falls to
|
|
768
|
+
* the compacted base, valid only while a pin at `at_gen`
|
|
769
|
+
* keeps history from collapsing below it (the contract).
|
|
770
|
+
*
|
|
771
|
+
* The current-live `verb_id_membership` bitmap is
|
|
772
|
+
* deliberately **not** consulted here — it reflects the
|
|
773
|
+
* present, not `at_gen`. (It still gates the hot-path
|
|
774
|
+
* [`get_verb_endpoints`], which is a current-state read.)
|
|
775
|
+
* Consulting it here was the old 5f limitation: a verb
|
|
776
|
+
* removed after `at_gen` would wrongly read as absent at
|
|
777
|
+
* `at_gen`. The chain already encodes liveness per
|
|
778
|
+
* generation, so the bitmap check is both unnecessary and
|
|
779
|
+
* incorrect for historical reads.
|
|
780
|
+
*
|
|
781
|
+
* Note: this resolves a single edge's endpoints at `at_gen`.
|
|
782
|
+
* Historical *enumeration* of a node's edges
|
|
783
|
+
* ([`get_neighbors`]/[`get_verb_ids_by_source`]) is
|
|
784
|
+
* current-state; brainy's `db.asOf(g)` composition
|
|
785
|
+
* intersects the graph hop with the generation-filtered
|
|
786
|
+
* metadata candidate set (the LSM, which IS at-generation),
|
|
787
|
+
* so edges to entities that did not exist at `at_gen` are
|
|
788
|
+
* dropped by that intersection.
|
|
789
|
+
*/
|
|
790
|
+
getVerbEndpointsAtGeneration(verbId: string, atGen: bigint): VerbEndpointsResult | null
|
|
791
|
+
/**
|
|
792
|
+
* **Diagnostic.** Returns the number of verb-IDs currently
|
|
793
|
+
* carrying endpoint info. Useful for tests + ops dashboards
|
|
794
|
+
* (counts how many verbs are "brainy-8.0-ready" vs back-compat
|
|
795
|
+
* without endpoints).
|
|
796
|
+
*
|
|
797
|
+
* At billion-scale this is a single HashMap.len() — O(1).
|
|
798
|
+
*/
|
|
799
|
+
verbEndpointsCount(): bigint
|
|
800
|
+
/**
|
|
801
|
+
* Get neighbor entity ints for an entity.
|
|
802
|
+
*
|
|
803
|
+
* `direction`: `"in"`, `"out"`, or `"both"`.
|
|
804
|
+
*
|
|
805
|
+
* **Cor 3.0 Piece D**: all entity ids cross as `BigInt` (the
|
|
806
|
+
* canonical u64 from brainy's `BinaryIdMapper`). Brainy 8.0
|
|
807
|
+
* handles UUID ↔ int conversion on its side via the same mapper.
|
|
808
|
+
*/
|
|
809
|
+
getNeighbors(id: bigint, direction: string, limit?: number | undefined | null, offset?: number | undefined | null): Array<bigint>
|
|
810
|
+
/**
|
|
811
|
+
* Get verb ints by source entity, filtered by the live-membership
|
|
812
|
+
* bitmap (tombstone filtering).
|
|
813
|
+
*
|
|
814
|
+
* **Cor 3.0 Piece D**: the verbs_source LSM tree carries
|
|
815
|
+
* `(verb_int, target_int)` pairs. This method extracts the verb_int
|
|
816
|
+
* from each pair, dedupes (a verb_int can show up under the same
|
|
817
|
+
* source if multiple add cycles tombstoned and re-added it), then
|
|
818
|
+
* filters by the membership bitmap.
|
|
819
|
+
*
|
|
820
|
+
* The return type is `Vec<BigInt>` of verb ints — brainy 8.0
|
|
821
|
+
* owns the verb_int ↔ verb_id_string mapping on its side
|
|
822
|
+
* (cor's `verb_id_namespace` interns via SHA-256 fingerprint
|
|
823
|
+
* so the original string is unrecoverable from cor).
|
|
824
|
+
*/
|
|
825
|
+
getVerbIdsBySource(sourceInt: bigint, limit?: number | undefined | null, offset?: number | undefined | null): Array<bigint>
|
|
826
|
+
/**
|
|
827
|
+
* Get verb ints by target entity, filtered by the live-membership
|
|
828
|
+
* bitmap. See [`get_verb_ids_by_source`].
|
|
829
|
+
*/
|
|
830
|
+
getVerbIdsByTarget(targetInt: bigint, limit?: number | undefined | null, offset?: number | undefined | null): Array<bigint>
|
|
831
|
+
/**
|
|
832
|
+
* Get verb ints by source entity, filtered by both the
|
|
833
|
+
* live-membership bitmap AND the per-`(verb_type, subtype)`
|
|
834
|
+
* membership bitmap. When `verb_type_index` or `subtype` is
|
|
835
|
+
* `None` that dimension is unfiltered. Both `None` reduces to
|
|
836
|
+
* [`get_verb_ids_by_source`](#method.get_verb_ids_by_source).
|
|
837
|
+
*/
|
|
838
|
+
getVerbIdsBySourceWithSubtype(sourceInt: bigint, verbTypeIndex?: number | undefined | null, subtype?: string | undefined | null, limit?: number | undefined | null, offset?: number | undefined | null): Array<bigint>
|
|
839
|
+
/**
|
|
840
|
+
* Get verb ints by target entity, filtered by both the
|
|
841
|
+
* live-membership bitmap AND the per-`(verb_type, subtype)`
|
|
842
|
+
* membership bitmap. See
|
|
843
|
+
* [`get_verb_ids_by_source_with_subtype`](#method.get_verb_ids_by_source_with_subtype).
|
|
844
|
+
*/
|
|
845
|
+
getVerbIdsByTargetWithSubtype(targetInt: bigint, verbTypeIndex?: number | undefined | null, subtype?: string | undefined | null, limit?: number | undefined | null, offset?: number | undefined | null): Array<bigint>
|
|
846
|
+
/**
|
|
847
|
+
* Per-`(verb_type, subtype)` relationship count. Returns the
|
|
848
|
+
* current value of the `relationship_subtype_counts` slot (0 if
|
|
849
|
+
* the pair has no entries). Used by brainy 8.0's
|
|
850
|
+
* `brain.counts.byRelationshipSubtype(verb, subtype)` to read
|
|
851
|
+
* the native rollup instead of brainy's TS map — same
|
|
852
|
+
* architectural transition as the noun-side
|
|
853
|
+
* `getEntityCountByType` already does for noun counts.
|
|
854
|
+
*/
|
|
855
|
+
getRelationshipSubtypeCount(verbTypeIndex: number, subtype: string): bigint
|
|
856
|
+
/**
|
|
857
|
+
* List the distinct subtype values currently observed for the
|
|
858
|
+
* given verb-type. Order is unspecified. Returns empty when the
|
|
859
|
+
* verb-type has no subtyped edges.
|
|
860
|
+
*
|
|
861
|
+
* Mirrors brainy 7.30's `brain.relationshipSubtypesOf(verb)` —
|
|
862
|
+
* brainy 8.0 can route the call through cor once the byte-for-
|
|
863
|
+
* byte JSON parity work lands.
|
|
864
|
+
*/
|
|
865
|
+
listVerbSubtypes(verbTypeIndex: number): Array<string>
|
|
866
|
+
/**
|
|
867
|
+
* **Cor 3.0 Piece D.3 — multi-hop subtype-filtered BFS.**
|
|
868
|
+
*
|
|
869
|
+
* Walks the graph starting from `source_int`, expanding through
|
|
870
|
+
* edges that match `(verb_type_index, subtype)`, up to `depth`
|
|
871
|
+
* hops, returning the deduplicated set of reachable neighbor ints.
|
|
872
|
+
*
|
|
873
|
+
* # Algorithm
|
|
874
|
+
*
|
|
875
|
+
* Standard layer-by-layer BFS using the `verbs_source` LSM tree's
|
|
876
|
+
* new pair value (`(verb_int, target_int)`) — D.2's whole reason
|
|
877
|
+
* for being. Each frontier `s` queries `verbs_source.get(s)` →
|
|
878
|
+
* `Vec<(verb_int, target_int)>`; surviving verb_ints (live + matching
|
|
879
|
+
* the type/subtype filter) contribute their target_int to the next
|
|
880
|
+
* frontier. A visited HashSet prevents cycle re-traversal.
|
|
881
|
+
*
|
|
882
|
+
* # Filter semantics
|
|
883
|
+
*
|
|
884
|
+
* - `verb_type_index` is required. Out-of-range indices return
|
|
885
|
+
* an empty result (strict — matches `getRelationshipCountByIndex`).
|
|
886
|
+
* - `subtype = Some(value)`: edges must be in the specific
|
|
887
|
+
* `(verb_type, subtype)` PostingList bitmap.
|
|
888
|
+
* - `subtype = None`: edges must be in SOME subtype bitmap of
|
|
889
|
+
* `verb_type_index`. Non-subtyped verbs are excluded — same
|
|
890
|
+
* pattern as [`filter_verb_ints_by_subtype`]'s `(Some(ti), None)`
|
|
891
|
+
* arm. A future piece can add a per-type "all verbs" bitmap if
|
|
892
|
+
* we want non-subtyped edges to participate in BFS.
|
|
893
|
+
*
|
|
894
|
+
* # Bounds
|
|
895
|
+
*
|
|
896
|
+
* - `depth = 0` returns an empty result (no expansion requested).
|
|
897
|
+
* - `depth > 0` runs the standard frontier expansion. Each iteration
|
|
898
|
+
* processes the previous frontier and builds the next.
|
|
899
|
+
* - `limit = Some(n)` short-circuits as soon as `n` distinct
|
|
900
|
+
* neighbors have been added — partial results in BFS order.
|
|
901
|
+
* - `source_int` is NEVER included in the result, even if a cycle
|
|
902
|
+
* would visit it on a later hop (the visited set guards it from
|
|
903
|
+
* the start).
|
|
904
|
+
*/
|
|
905
|
+
findConnectedSubtype(sourceInt: bigint, verbTypeIndex: number, subtype: string | undefined | null, depth: number, limit?: number | undefined | null): Array<bigint>
|
|
906
|
+
/**
|
|
907
|
+
* **Native multi-hop traversal from explicit id seeds (ADR-006 Phase A).**
|
|
908
|
+
* `seeds` are entity ints (BigInts). The optional trailing
|
|
909
|
+
* `generation` selects as-of-`g` time-travel (omit for a live read).
|
|
910
|
+
* See [`traverse_core`] for the traversal semantics and the columnar
|
|
911
|
+
* [`NativeSubgraph`] result.
|
|
912
|
+
*/
|
|
913
|
+
traverse(seeds: Array<bigint>, opts: TraverseOptions, generation?: bigint | undefined | null): NativeSubgraph
|
|
914
|
+
/**
|
|
915
|
+
* **Native traversal seeded from a `find()`-result `OpaqueIdSet`
|
|
916
|
+
* (ADR-006 Phase A — the query→expand fusion).**
|
|
917
|
+
*
|
|
918
|
+
* The seed node set crosses as cor's opaque, version-tagged id-set
|
|
919
|
+
* envelope (see [`crate::opaque_id_set`]); brainy forwards it straight
|
|
920
|
+
* from `find()` without ever materializing the ids in TS, so "find
|
|
921
|
+
* entities matching X, then everything they're related to" stays one
|
|
922
|
+
* boundary crossing. Fails loud (`BufferFormatMismatch`) on a
|
|
923
|
+
* stale/foreign buffer. See [`traverse_core`] for traversal semantics.
|
|
924
|
+
*/
|
|
925
|
+
traverseFromOpaqueIdSet(seed: Buffer, opts: TraverseOptions, generation?: bigint | undefined | null): NativeSubgraph
|
|
926
|
+
/**
|
|
927
|
+
* **Open a streaming graph cursor (ADR-006 Phase B).** Pins a
|
|
928
|
+
* generation (`options.generation`, else the current generation) and
|
|
929
|
+
* returns a handle for [`graph_cursor_next`] / [`graph_cursor_close`].
|
|
930
|
+
* Omit `seeds`/`seeds_opaque` for a whole-graph export; supply them
|
|
931
|
+
* for an incident-edge export of a seeded region. Pass
|
|
932
|
+
* `options.cursor` (a prior chunk's resume token) to resume after a
|
|
933
|
+
* `SnapshotExpired`.
|
|
934
|
+
*/
|
|
935
|
+
graphCursorOpen(options: GraphCursorOpenOptions): string
|
|
936
|
+
/**
|
|
937
|
+
* **Pull the next chunk from an open cursor (ADR-006 Phase B).**
|
|
938
|
+
* Returns up to `chunk_size` edges (a single high-degree node may
|
|
939
|
+
* span chunks) plus a resume token. `done: true` once exhausted. A
|
|
940
|
+
* handle idle past the TTL is reaped and rejected with
|
|
941
|
+
* `SnapshotExpired` — reopen with the last chunk's token to resume.
|
|
942
|
+
*/
|
|
943
|
+
graphCursorNext(handle: string, chunkSize: number): NativeGraphCursorChunk
|
|
944
|
+
/**
|
|
945
|
+
* **Release a cursor's pinned generation + server-side state
|
|
946
|
+
* (ADR-006 Phase B).** Idempotent — closing an unknown / already-
|
|
947
|
+
* closed handle is a no-op. Dropping the registry entry drops its
|
|
948
|
+
* `PinGuard`, unblocking compaction reclamation of that view.
|
|
949
|
+
*/
|
|
950
|
+
graphCursorClose(handle: string): void
|
|
951
|
+
/**
|
|
952
|
+
* **PageRank over the as-of-`g` graph (ADR-006 Phase C).** Power iteration
|
|
953
|
+
* with damping, dangling-mass redistribution, L1 convergence. Returns
|
|
954
|
+
* per-node scores (descending; `top_k` truncates). `excludeVisibility` is
|
|
955
|
+
* intentionally absent — brainy post-filters on hydrate.
|
|
956
|
+
*/
|
|
957
|
+
pageRank(options: NativePageRankOptions, generation?: bigint | undefined | null): NativeGraphScores
|
|
958
|
+
/**
|
|
959
|
+
* **Connected components over the as-of-`g` graph (ADR-006 Phase C).**
|
|
960
|
+
* `"weak"` (default) = union-find on the undirected CSR; `"strong"` =
|
|
961
|
+
* iterative Tarjan (explicit stacks — no recursion) on the directed CSR.
|
|
962
|
+
*/
|
|
963
|
+
connectedComponents(options: NativeComponentsOptions, generation?: bigint | undefined | null): NativeGraphComponents
|
|
964
|
+
/**
|
|
965
|
+
* **Shortest path between two nodes as-of-`g` (ADR-006 Phase C).** BFS
|
|
966
|
+
* (hops — cor stores no edge weights), bounded by `max_depth`; honors
|
|
967
|
+
* direction + `verb_types`. `None` when unreachable. No CSR (frontier
|
|
968
|
+
* walk via `out_edges_at_g`/`in_edges_at_g`).
|
|
969
|
+
*/
|
|
970
|
+
shortestPath(fromInt: bigint, toInt: bigint, options: NativeShortestPathOptions, generation?: bigint | undefined | null): NativeGraphPath | null
|
|
971
|
+
/**
|
|
972
|
+
* **Deterministic, seeded neighborhood sample as-of-`g` (ADR-006 Phase C).**
|
|
973
|
+
* Bounded BFS from `seeds` (ints or an OpaqueIdSet) with a per-node fanout
|
|
974
|
+
* cap, depth + node caps; a seeded SplitMix64 (keyed per node) makes a
|
|
975
|
+
* given `(seeds, opts, seed)` byte-reproducible. Returns a columnar
|
|
976
|
+
* `NativeSubgraph` (no CSR).
|
|
977
|
+
*/
|
|
978
|
+
neighborhoodSample(seeds: Array<bigint>, seedsOpaque: Buffer | undefined | null, options: NativeNeighborhoodSampleOptions, generation?: bigint | undefined | null): NativeSubgraph
|
|
979
|
+
/**
|
|
980
|
+
* **Top-K nodes by degree as-of-`g` (ADR-006 Phase C).** Degree-only CSR
|
|
981
|
+
* fast path (no `out_targets`) + a bounded min-heap. `direction` selects
|
|
982
|
+
* out / in / both degree. Returns per-node degree scores, descending.
|
|
983
|
+
*/
|
|
984
|
+
topByDegree(options: NativeTopByDegreeOptions, generation?: bigint | undefined | null): NativeGraphScores
|
|
985
|
+
/**
|
|
986
|
+
* Track a verb ID without adding to LSM-trees. Used during
|
|
987
|
+
* rebuild when LSM-trees already have the data on disk.
|
|
988
|
+
*
|
|
989
|
+
* Brainy walks its own persisted verbs (which carry their original
|
|
990
|
+
* verb-type strings + verb-IDs) and calls this once per verb to
|
|
991
|
+
* repopulate the membership bitmap + per-type counts. The LSM
|
|
992
|
+
* trees themselves don't need to be touched — they're already
|
|
993
|
+
* loaded via `loadSstableFromBinary` / `loadSstableFromFile`.
|
|
994
|
+
*
|
|
995
|
+
* **Cor 3.0 Piece D**: the legacy `trackVerbIdByIndex` variant
|
|
996
|
+
* is deleted along with the other strict-index variants. Brainy
|
|
997
|
+
* 8.0's rebuild calls this with the verb-type string from storage.
|
|
998
|
+
* The fallback-to-`relatedTo` behavior for unknown strings is
|
|
999
|
+
* acceptable for rebuild (matches the addVerb-time behavior so
|
|
1000
|
+
* counts converge to the same totals).
|
|
1001
|
+
*/
|
|
1002
|
+
trackVerbId(verbId: string, verbType: string): void
|
|
1003
|
+
/**
|
|
1004
|
+
* Clear membership bitmap + relationship counts. Used before
|
|
1005
|
+
* rebuild to ensure clean state. The verb-ID namespace's
|
|
1006
|
+
* interning assignments are intentionally preserved — Piece 12
|
|
1007
|
+
* stability contract: an int once assigned is never reused.
|
|
1008
|
+
* Re-adding the same verb-ID after a clear returns the same
|
|
1009
|
+
* interned id (intern is idempotent), so the bitmap rebuilds to
|
|
1010
|
+
* identical contents.
|
|
1011
|
+
*
|
|
1012
|
+
* **Why this DOESN'T clear endpoints / subtypes / subtype
|
|
1013
|
+
* counts**: the rebuild path (`populateVerbIdSetFromStorage` →
|
|
1014
|
+
* `trackVerbIdByIndex`) walks the LSM trees and re-tracks each
|
|
1015
|
+
* verb. The track call only carries `(verb_id, verb_type_index)`
|
|
1016
|
+
* — it does NOT carry endpoint or subtype info, which only
|
|
1017
|
+
* arrive via explicit `addVerbWithEndpoints` / `addVerbWithSubtype`
|
|
1018
|
+
* calls. So if `clearVerbTracking` flushed endpoints + subtypes,
|
|
1019
|
+
* the rebuild couldn't restore them and the brain would
|
|
1020
|
+
* permanently lose that data.
|
|
1021
|
+
*
|
|
1022
|
+
* The endpoint / subtype state IS managed correctly for
|
|
1023
|
+
* per-verb removes (see [`remove_verb_internal`]) — that path
|
|
1024
|
+
* tombstones the endpoint slot + drops the subtype bitmap entry.
|
|
1025
|
+
* Stale slots that survive a `clear_verb_tracking` rebuild are
|
|
1026
|
+
* invisible to readers because [`get_verb_endpoints`] and the
|
|
1027
|
+
* subtype-filtering paths check the membership bitmap first.
|
|
1028
|
+
* They consume a bit of extra disk for the lifetime of the
|
|
1029
|
+
* orphan; explicit compaction is a future piece.
|
|
1030
|
+
*/
|
|
1031
|
+
clearVerbTracking(): void
|
|
1032
|
+
/**
|
|
1033
|
+
* Check if a specific tree's level needs compaction.
|
|
1034
|
+
* `tree_name`: `"source"`, `"target"`, `"verbs-source"`, `"verbs-target"`.
|
|
1035
|
+
*/
|
|
1036
|
+
needsCompaction(treeName: string, level: number): boolean
|
|
1037
|
+
/**
|
|
1038
|
+
* Flush a tree's MemTable to binary .sst format (no disk I/O).
|
|
1039
|
+
* Returns binary data for the adapter to persist.
|
|
1040
|
+
*/
|
|
1041
|
+
flushTreeToBinary(treeName: string): NativeBinaryFlushResult | null
|
|
1042
|
+
/**
|
|
1043
|
+
* Compact a tree's level to binary .sst format (no disk I/O).
|
|
1044
|
+
* Returns binary data + old IDs for the adapter to handle.
|
|
1045
|
+
*/
|
|
1046
|
+
compactTreeToBinary(treeName: string, level: number): NativeBinaryCompactResult | null
|
|
1047
|
+
/**
|
|
1048
|
+
* Mmap an adapter-written file, replacing the matching InMemory
|
|
1049
|
+
* variant with a zero-copy `Mapped` variant.
|
|
1050
|
+
*/
|
|
1051
|
+
registerMmapSstable(treeName: string, sstableId: string, level: number, filePath: string): void
|
|
1052
|
+
/**
|
|
1053
|
+
* Load an SSTable from binary `.sst` data (for adapters that
|
|
1054
|
+
* round-trip bytes rather than file paths).
|
|
1055
|
+
*/
|
|
1056
|
+
loadSstableFromBinary(treeName: string, sstableId: string, level: number, data: Buffer): void
|
|
1057
|
+
/** Load an SSTable from a `.sst` file via mmap. */
|
|
1058
|
+
loadSstableFromFile(treeName: string, sstableId: string, level: number, filePath: string): void
|
|
1059
|
+
/** Get manifest JSON for a specific tree. */
|
|
1060
|
+
getManifestJson(treeName: string): string
|
|
1061
|
+
/** Load manifest JSON for a specific tree. */
|
|
1062
|
+
loadManifestJson(treeName: string, json: string): void
|
|
1063
|
+
/** Get SSTable IDs from a tree's manifest. */
|
|
1064
|
+
getManifestSstableIds(treeName: string): Array<string>
|
|
1065
|
+
/** Get the level for an SSTable ID in a tree's manifest. */
|
|
1066
|
+
getManifestSstableLevel(treeName: string, id: string): number | null
|
|
1067
|
+
/** Check if a specific tree's MemTable is empty. */
|
|
1068
|
+
memTableIsEmpty(treeName: string): boolean
|
|
1069
|
+
/**
|
|
1070
|
+
* Total relationship count from the source tree (narrowed u32 —
|
|
1071
|
+
* caps at [`u32::MAX`]; use [`size_big`] at billion scale).
|
|
1072
|
+
*/
|
|
1073
|
+
size(): number
|
|
1074
|
+
/**
|
|
1075
|
+
* BigInt sibling of [`size`]. Overflow-safe at billion scale.
|
|
1076
|
+
* Cor 3.0 Piece D widened `LSMTreeCoreU64::total_relationships()`
|
|
1077
|
+
* to u64 (was u32 in the legacy `LSMTreeCore`), so this returns
|
|
1078
|
+
* the full u64 without further widening.
|
|
1079
|
+
*/
|
|
1080
|
+
sizeBig(): bigint
|
|
1081
|
+
/**
|
|
1082
|
+
* Number of tracked verb IDs (narrowed u32 — caps at
|
|
1083
|
+
* [`u32::MAX`]; use [`verb_id_count_big`] at billion scale).
|
|
1084
|
+
* Piece 12: reads the Roaring32 membership bitmap's cardinality,
|
|
1085
|
+
* constant-time regardless of the bitmap's contents.
|
|
1086
|
+
*/
|
|
1087
|
+
verbIdCount(): number
|
|
1088
|
+
/**
|
|
1089
|
+
* BigInt sibling of [`verb_id_count`]. Returns the full u64
|
|
1090
|
+
* cardinality of the live-verb bitmap — overflow-safe at
|
|
1091
|
+
* billion scale.
|
|
1092
|
+
*/
|
|
1093
|
+
verbIdCountBig(): bigint
|
|
1094
|
+
/**
|
|
1095
|
+
* **Batch verb-int reverse resolver** (brainy 8.0 lockstep
|
|
1096
|
+
* `GraphIndexProvider.verbIntsToIds`, handoff L.7).
|
|
1097
|
+
*
|
|
1098
|
+
* Given a batch of `verb_int` BigInts, return the corresponding
|
|
1099
|
+
* canonical UUID string for each — `null` for any int that has
|
|
1100
|
+
* no entry in the namespace's persistent reverse map.
|
|
1101
|
+
*
|
|
1102
|
+
* This is brainy 8.0's single source of truth for "what was this
|
|
1103
|
+
* verb-int's original verb-id?" — brainy keeps only a bounded
|
|
1104
|
+
* in-memory LRU warm cache fed by `addVerb` returns + resolver
|
|
1105
|
+
* hits; the durable mapping lives in cor's `verb-int-to-uuid.bin`
|
|
1106
|
+
* (a direct-indexed sparse file, O(1) random access per lookup).
|
|
1107
|
+
*
|
|
1108
|
+
* **Identity-fingerprint design** (L.7): brainy enforces that
|
|
1109
|
+
* every verb-id is a canonical UUID, so cor's namespace
|
|
1110
|
+
* stores the raw 16 UUID bytes (identity, not a hash). The
|
|
1111
|
+
* reverse lookup format the 16 bytes back to the canonical
|
|
1112
|
+
* lowercase 8-4-4-4-12 hex string — round-trips exactly.
|
|
1113
|
+
*
|
|
1114
|
+
* The in-memory namespace variant doesn't persist a reverse map
|
|
1115
|
+
* (it's transient + test-only); returns `null` for every input
|
|
1116
|
+
* when the namespace is in-memory.
|
|
1117
|
+
*/
|
|
1118
|
+
verbIntsToUuids(verbInts: Array<bigint>): Array<string | undefined | null>
|
|
1119
|
+
/**
|
|
1120
|
+
* Get relationship count for a specific type (narrowed u32 — see
|
|
1121
|
+
* [`get_relationship_count_by_type_big`] for the overflow-safe
|
|
1122
|
+
* sibling that does NOT cap at `u32::MAX`).
|
|
1123
|
+
*
|
|
1124
|
+
* Unknown verb-type strings collapse to `VerbType.RelatedTo`'s
|
|
1125
|
+
* counter (brainy contract), so a caller querying a custom type
|
|
1126
|
+
* will see the count of `relatedTo` edges. Use
|
|
1127
|
+
* [`get_relationship_count_by_index`] when you have the
|
|
1128
|
+
* canonical index and want strict semantics.
|
|
1129
|
+
*/
|
|
1130
|
+
getRelationshipCountByType(verbType: string): number
|
|
1131
|
+
/**
|
|
1132
|
+
* BigInt sibling of [`get_relationship_count_by_type`]. Returns
|
|
1133
|
+
* the full u64 per-type count. Required at scale: a single hot
|
|
1134
|
+
* verb type ("follows", "likes") in a billion-entity graph can
|
|
1135
|
+
* exceed `u32::MAX` even though the count of distinct types
|
|
1136
|
+
* stays small.
|
|
1137
|
+
*/
|
|
1138
|
+
getRelationshipCountByTypeBig(verbType: string): bigint
|
|
1139
|
+
/**
|
|
1140
|
+
* **Piece 12.** Strict count lookup by canonical verb-type index.
|
|
1141
|
+
* Out-of-range indices return `0n` (treated as "no edges of an
|
|
1142
|
+
* unknown type") rather than falling back to the `relatedTo`
|
|
1143
|
+
* slot — strict callers want to see the violation, not silently
|
|
1144
|
+
* read someone else's count.
|
|
1145
|
+
*/
|
|
1146
|
+
getRelationshipCountByIndex(verbTypeIndex: number): bigint
|
|
1147
|
+
/**
|
|
1148
|
+
* Get all relationship counts as JSON. Returns a `{type_name:
|
|
1149
|
+
* count}` map keyed by canonical verb-type strings. Zero-count
|
|
1150
|
+
* types are omitted to keep the payload small in sparse graphs
|
|
1151
|
+
* (a brain that only ever uses `follows` shouldn't have 127
|
|
1152
|
+
* entries of zero in its stats).
|
|
1153
|
+
*/
|
|
1154
|
+
getAllRelationshipCountsJson(): string
|
|
1155
|
+
/**
|
|
1156
|
+
* Get relationship statistics as JSON. `totalRelationships` is a
|
|
1157
|
+
* u64 (cor 3.0 Piece D widening); JS consumers must parse it
|
|
1158
|
+
* as a string for billion-scale fidelity (BigInt isn't valid JSON).
|
|
1159
|
+
*/
|
|
1160
|
+
getRelationshipStatsJson(): string
|
|
1161
|
+
/** Whether the index is healthy */
|
|
1162
|
+
isHealthy(): boolean
|
|
1163
|
+
/**
|
|
1164
|
+
* Whether the graph's edges are loaded and queryable WITHOUT a rebuild.
|
|
1165
|
+
*
|
|
1166
|
+
* Brainy 8.0's rebuild gate (#36) consults this right after `init()`:
|
|
1167
|
+
* `true` means the durable cold-load already restored the adjacency, so
|
|
1168
|
+
* brainy SKIPS its O(E) rebuild-from-storage; `false` means the graph is
|
|
1169
|
+
* empty (a fresh brain with nothing persisted) and brainy should build
|
|
1170
|
+
* it.
|
|
1171
|
+
*
|
|
1172
|
+
* **Edges-loaded semantics — never membership.** Returns `true` iff the
|
|
1173
|
+
* graph actually holds edges through a durable or live path: the
|
|
1174
|
+
* cold-load latch (set when an SSTable loads from binary/file, a manifest
|
|
1175
|
+
* names ≥1 SSTable, or a memtable flushes to a persisted SSTable), or a
|
|
1176
|
+
* currently non-empty memtable on any of the four trees. The verb-id
|
|
1177
|
+
* membership bitmap — whose population is the deferred O(N) fast-follow —
|
|
1178
|
+
* is deliberately NOT consulted, so a slow membership walk can never make
|
|
1179
|
+
* a graph with queryable edges report "not ready".
|
|
1180
|
+
*/
|
|
1181
|
+
isReady(): boolean
|
|
1182
|
+
/**
|
|
1183
|
+
* Get comprehensive statistics. Both the legacy u32 fields and
|
|
1184
|
+
* the Piece-10 BigInt sibling fields are populated; callers
|
|
1185
|
+
* should prefer the `*_big` fields at billion scale and treat
|
|
1186
|
+
* the u32 fields as best-effort summaries that cap at
|
|
1187
|
+
* [`u32::MAX`].
|
|
1188
|
+
*/
|
|
1189
|
+
getStats(): GraphAdjacencyStats
|
|
1190
|
+
/** List all tree names (for iteration in TS wrapper) */
|
|
1191
|
+
treeNames(): Array<string>
|
|
1192
|
+
/**
|
|
1193
|
+
* **Piece B.** Wire width of the verb-ID interning namespace +
|
|
1194
|
+
* the live-verb / per-`(type, subtype)` membership bitmaps.
|
|
1195
|
+
* Returns `"U32"` (cortex 2.x-compatible, ~4.29 B verb-ID
|
|
1196
|
+
* ceiling) or `"U64"` (cor 3.0, ~1.8 × 10^19 verb-ID
|
|
1197
|
+
* ceiling).
|
|
1198
|
+
*
|
|
1199
|
+
* JS callers introspect this for diagnostics — e.g. a brainy 8.0
|
|
1200
|
+
* `brain.diagnostics()` panel showing whether the brain is on
|
|
1201
|
+
* the wide wire width. Set at construction via
|
|
1202
|
+
* [`GraphAdjacencyConfig::verb_id_space`]; immutable for the
|
|
1203
|
+
* life of the index.
|
|
1204
|
+
*/
|
|
1205
|
+
verbIdSpace(): string
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
/** napi-exposed engine. */
|
|
1209
|
+
export declare class NativeLsmEngine {
|
|
1210
|
+
/** Construct a fresh engine. */
|
|
1211
|
+
constructor(options?: NativeLsmEngineOptions | undefined | null)
|
|
1212
|
+
/**
|
|
1213
|
+
* Construct an engine and seed the epoch counter from an existing
|
|
1214
|
+
* manifest. Used on engine restart so new mutations always outrank
|
|
1215
|
+
* every persisted epoch.
|
|
1216
|
+
*/
|
|
1217
|
+
static openWithManifest(manifestJson: string, options?: NativeLsmEngineOptions | undefined | null): NativeLsmEngine
|
|
1218
|
+
/**
|
|
1219
|
+
* **Piece 7.** Construct an engine whose memtable is backed by
|
|
1220
|
+
* mmap'd per-shard append logs under `dir/memtable/`. On open,
|
|
1221
|
+
* every surviving record in each shard's log replays into the
|
|
1222
|
+
* BTreeMap before the engine accepts new writes — so a process
|
|
1223
|
+
* crash between inserts and the next `flush()` is recovered
|
|
1224
|
+
* transparently. Optionally seeds the epoch counter from a
|
|
1225
|
+
* manifest (when reopening alongside persisted SSTables); when
|
|
1226
|
+
* omitted, the post-replay epoch is `max(log_max_epoch) + 1`.
|
|
1227
|
+
*/
|
|
1228
|
+
static openWithDir(dir: string, manifestJson?: string | undefined | null, options?: NativeLsmEngineOptions | undefined | null): NativeLsmEngine
|
|
1229
|
+
/**
|
|
1230
|
+
* G4.2 redo step 3: snapshot-replay opener. Equivalent to
|
|
1231
|
+
* [`openWithDir`](Self::open_with_dir) except every shard's
|
|
1232
|
+
* memtable log replay filters records to
|
|
1233
|
+
* `generation <= atGeneration`. The caller supplies the
|
|
1234
|
+
* brainy snapshot generation from the snapshot's manifest;
|
|
1235
|
+
* any post-snapshot writes that leaked through the
|
|
1236
|
+
* hard-linked active-log inode are excluded.
|
|
1237
|
+
*
|
|
1238
|
+
* Use this when opening cor against a brainy
|
|
1239
|
+
* `db.persist()` snapshot directory; use `openWithDir`
|
|
1240
|
+
* (no filter) for live-restart recovery where every
|
|
1241
|
+
* durable record must replay regardless of generation.
|
|
1242
|
+
*/
|
|
1243
|
+
static openWithDirAtGeneration(dir: string, atGeneration: bigint, manifestJson?: string | undefined | null, options?: NativeLsmEngineOptions | undefined | null): NativeLsmEngine
|
|
1244
|
+
/**
|
|
1245
|
+
* **Piece 7.** Whether this engine's memtable is backed by
|
|
1246
|
+
* persistent logs (i.e. constructed via
|
|
1247
|
+
* [`openWithDir`](#method.open_with_dir)). Diagnostic surface
|
|
1248
|
+
* for tests + brainy callers wanting to confirm the durability
|
|
1249
|
+
* path is wired.
|
|
1250
|
+
*/
|
|
1251
|
+
isPersistent(): boolean
|
|
1252
|
+
/**
|
|
1253
|
+
* **Piece 7.** Advance the flush watermark on every shard's
|
|
1254
|
+
* memtable log so records with `epoch <= throughEpoch` are
|
|
1255
|
+
* dropped from future replay. The wrapper calls this AFTER it
|
|
1256
|
+
* has durably persisted the SSTable bytes returned by the
|
|
1257
|
+
* preceding `flush()` — at that point the data is in two
|
|
1258
|
+
* places (the log + the persisted SSTable) and the log copy
|
|
1259
|
+
* is safe to retire.
|
|
1260
|
+
*
|
|
1261
|
+
* No-op for in-memory engines (constructed without
|
|
1262
|
+
* `openWithDir`).
|
|
1263
|
+
*/
|
|
1264
|
+
commitFlushThrough(throughEpoch: bigint): void
|
|
1265
|
+
/** `"u32"` or `"u64"`. */
|
|
1266
|
+
idSpace(): string
|
|
1267
|
+
/**
|
|
1268
|
+
* Add an entity int to `(field, value)`. JS `number` path; throws
|
|
1269
|
+
* on U64 mode if you exceed `Number.MAX_SAFE_INTEGER`. Prefer
|
|
1270
|
+
* [`insert_big`](Self::insert_big) for U64 mode.
|
|
1271
|
+
*
|
|
1272
|
+
* **G4.2 redo**: optional `generation` argument carries the
|
|
1273
|
+
* caller's MVCC generation. When omitted (the historical
|
|
1274
|
+
* signature), the record is tagged with generation = 0 —
|
|
1275
|
+
* the no-coordination default that matches every
|
|
1276
|
+
* `at_generation >= 0` snapshot replay filter. Brainy
|
|
1277
|
+
* callers can flip the field on when their Phase 3b
|
|
1278
|
+
* generation plumbing is ready; cor's behaviour is
|
|
1279
|
+
* unchanged for callers that don't pass it.
|
|
1280
|
+
*/
|
|
1281
|
+
insert(field: string, value: string, intId: number, generation?: bigint | undefined | null): void
|
|
1282
|
+
/** BigInt variant of [`insert`](Self::insert). */
|
|
1283
|
+
insertBig(field: string, value: string, intId: bigint, generation?: bigint | undefined | null): void
|
|
1284
|
+
/**
|
|
1285
|
+
* Tombstone an entity int at `(field, value)`. See
|
|
1286
|
+
* [`insert`](Self::insert) re the optional `generation` arg.
|
|
1287
|
+
*/
|
|
1288
|
+
delete(field: string, value: string, intId: number, generation?: bigint | undefined | null): void
|
|
1289
|
+
/** BigInt variant of [`delete`](Self::delete). */
|
|
1290
|
+
deleteBig(field: string, value: string, intId: bigint, generation?: bigint | undefined | null): void
|
|
1291
|
+
/** Drain the memtable. Returns `null` if there was nothing to flush. */
|
|
1292
|
+
flush(): NativeLsmFlushResult | null
|
|
1293
|
+
/**
|
|
1294
|
+
* Update the budget ceiling. Phase 6's ResourceManager subscriber
|
|
1295
|
+
* drives this from the TS wrapper layer. Pass `Number.MAX_SAFE_INTEGER`
|
|
1296
|
+
* or higher (mapped to `u64::MAX`) to disable admission control.
|
|
1297
|
+
*/
|
|
1298
|
+
setAllowedBytes(bytes: bigint): void
|
|
1299
|
+
/** Current budget ceiling as `bigint`. */
|
|
1300
|
+
allowedBytes(): bigint
|
|
1301
|
+
/** Memtable footprint estimate. */
|
|
1302
|
+
memtableBytes(): bigint
|
|
1303
|
+
/**
|
|
1304
|
+
* Bytes currently reserved by in-flight inserts (the atomic
|
|
1305
|
+
* admission counter). Diagnostic / telemetry — production code
|
|
1306
|
+
* usually queries [`isOverAdmissionThreshold`](Self::is_over_admission_threshold)
|
|
1307
|
+
* which already includes this value.
|
|
1308
|
+
*/
|
|
1309
|
+
reservedBytes(): bigint
|
|
1310
|
+
/**
|
|
1311
|
+
* **Test-only** harness exposing the engine's internal multi-
|
|
1312
|
+
* thread admission stress: spawn `nThreads` real OS threads, each
|
|
1313
|
+
* performing `perThread` inserts, with a background flusher
|
|
1314
|
+
* running at a 50% trigger. Returns `{ peakMemtableBytes,
|
|
1315
|
+
* nBlocked }`.
|
|
1316
|
+
*
|
|
1317
|
+
* The atomic admission invariant is verified externally by the
|
|
1318
|
+
* caller: `peakMemtableBytes ≤ budget × 1.05`. The 5% margin
|
|
1319
|
+
* covers the in-flight reservations during the burst. With the
|
|
1320
|
+
* pre-fix soft threshold, the same workload would observe peaks
|
|
1321
|
+
* well above 2× budget under heavy contention.
|
|
1322
|
+
*/
|
|
1323
|
+
_testStressConcurrentAdmission(nThreads: number, perThread: number, budget: bigint): NativeLsmStressResult
|
|
1324
|
+
/** Distinct `(field, value)` keys currently buffered. */
|
|
1325
|
+
memtableEntryCount(): bigint
|
|
1326
|
+
/** Next epoch the engine will allocate. */
|
|
1327
|
+
currentEpoch(): bigint
|
|
1328
|
+
/** Whether the next insert would block on admission control. */
|
|
1329
|
+
isOverAdmissionThreshold(): boolean
|
|
1330
|
+
/**
|
|
1331
|
+
* **Test-only** — probe an SSTable's bloom filter directly.
|
|
1332
|
+
* Returns the fraction of `(field, value)` probes the bloom
|
|
1333
|
+
* rejected (a value in `[0.0, 1.0]`). Used by the Phase 4
|
|
1334
|
+
* closeout A1 test to prove the bloom short-circuit is wired:
|
|
1335
|
+
* for keys NEVER inserted into the SSTable, the rejection
|
|
1336
|
+
* fraction should approach 1.0 (modulo the bloom's false-
|
|
1337
|
+
* positive rate, ~1% at 10 bits/element). For keys that ARE in
|
|
1338
|
+
* the SSTable, the fraction should be 0.0 (bloom must never
|
|
1339
|
+
* false-negative).
|
|
1340
|
+
*
|
|
1341
|
+
* Accepts a single SSTable Buffer and a list of `(field, value)`
|
|
1342
|
+
* pairs to probe.
|
|
1343
|
+
*/
|
|
1344
|
+
_testBloomRejectionRate(sstable: Buffer, field: string, values: Array<string>): number
|
|
1345
|
+
/**
|
|
1346
|
+
* **Test-only** — whether the SSTable has a bloom filter wired.
|
|
1347
|
+
* Used by the Phase 4 closeout test to assert that `emitBloom:
|
|
1348
|
+
* true` actually produces a reader with a bloom (and that
|
|
1349
|
+
* `emitBloom: false` does not).
|
|
1350
|
+
*/
|
|
1351
|
+
_testHasBloom(sstable: Buffer): boolean
|
|
1352
|
+
/**
|
|
1353
|
+
* Read a posting list for `(field, value)` across the live
|
|
1354
|
+
* memtable + the supplied SSTable bytes.
|
|
1355
|
+
*
|
|
1356
|
+
* SSTables are looked up through the engine's content-keyed
|
|
1357
|
+
* LRU reader cache. The first call for a given SSTable pays
|
|
1358
|
+
* the full parse + SHA-256 + bloom-deserialization cost;
|
|
1359
|
+
* subsequent calls return the cached parsed reader after a
|
|
1360
|
+
* re-verification SHA-256 over the candidate body (forge-proof,
|
|
1361
|
+
* per A4 INV-1).
|
|
1362
|
+
*
|
|
1363
|
+
* Returns the merged posting list as croaring portable bytes.
|
|
1364
|
+
* The TS wrapper materializes them via
|
|
1365
|
+
* `NativeRoaringBitmap32.deserialize` (U32 mode) or
|
|
1366
|
+
* `NativeRoaringBitmap64.deserialize` (U64 mode). Returns an
|
|
1367
|
+
* empty Buffer when no entries match (NOT null — an empty
|
|
1368
|
+
* PostingList is the correct semantic "no matches").
|
|
1369
|
+
*
|
|
1370
|
+
* Each `sstable` Buffer MUST be a complete, undamaged SSTable
|
|
1371
|
+
* produced by [`flush`](Self::flush) (or a Phase 5 compaction).
|
|
1372
|
+
* SHA-256 mismatches throw.
|
|
1373
|
+
*/
|
|
1374
|
+
readPostingList(field: string, value: string, sstables: Array<Buffer>): Buffer
|
|
1375
|
+
/**
|
|
1376
|
+
* Zero-copy variant of [`read_posting_list`]: SSTables are
|
|
1377
|
+
* opened directly from filesystem paths via `mmap`. The body
|
|
1378
|
+
* bytes live in the OS page cache (`RssFile`, kernel-reclaimable)
|
|
1379
|
+
* and do NOT grow process anonymous RSS — the production path
|
|
1380
|
+
* for billion-entity scale where SSTables exceed RAM.
|
|
1381
|
+
*
|
|
1382
|
+
* Identical semantics to [`read_posting_list`] otherwise:
|
|
1383
|
+
* memtable + SSTable merge, returns the merged PostingList as
|
|
1384
|
+
* croaring portable bytes.
|
|
1385
|
+
*
|
|
1386
|
+
* Path strings MUST be UTF-8. Cor stores SSTables under
|
|
1387
|
+
* sha256-hex names so this is not a practical limitation
|
|
1388
|
+
* (per spec test #25 + minor concern #11).
|
|
1389
|
+
*/
|
|
1390
|
+
readPostingListFromPaths(field: string, value: string, paths: Array<string>): Buffer
|
|
1391
|
+
/**
|
|
1392
|
+
* Open an SSTable from the given filesystem path through the
|
|
1393
|
+
* reader cache (mmap zero-copy). The cache retains its own
|
|
1394
|
+
* `Arc<LsmSstableReader>`; the next `readPostingListFromPaths`
|
|
1395
|
+
* for the same path hits the warm cache without re-stat'ing
|
|
1396
|
+
* the file beyond the path_hint validation.
|
|
1397
|
+
*
|
|
1398
|
+
* Idempotent: repeated calls for the same path are warm-cache
|
|
1399
|
+
* hits and do not re-validate SHA-256.
|
|
1400
|
+
*/
|
|
1401
|
+
openSstablePath(path: string): void
|
|
1402
|
+
/**
|
|
1403
|
+
* Open the SSTable at `path` through the reader cache (mmap +
|
|
1404
|
+
* SHA-256 verify) and register the resulting reader into the
|
|
1405
|
+
* engine's live-SSTable set, so every subsequent read snapshot
|
|
1406
|
+
* includes it. Distinct from [`open_sstable_path`], which only
|
|
1407
|
+
* warms the evictable reader cache without growing the live set.
|
|
1408
|
+
*
|
|
1409
|
+
* The TS wrapper calls this on cold start (once per manifest
|
|
1410
|
+
* SSTable entry) and after each flush, AFTER the `.sst` bytes and
|
|
1411
|
+
* the manifest entry that references them are durably persisted.
|
|
1412
|
+
*/
|
|
1413
|
+
registerSstablePath(path: string): void
|
|
1414
|
+
/**
|
|
1415
|
+
* Number of SSTable readers currently in the engine's live set.
|
|
1416
|
+
* Diagnostic used by the integration tests to assert that a
|
|
1417
|
+
* `flush` + `registerSstablePath` cycle actually grew the read
|
|
1418
|
+
* view.
|
|
1419
|
+
*/
|
|
1420
|
+
liveSstableCount(): number
|
|
1421
|
+
/**
|
|
1422
|
+
* Phase 5 hook (exposed now for A5 test coverage of INV-A5-12):
|
|
1423
|
+
* drops the path_hint + cache entry for the given path WITHOUT
|
|
1424
|
+
* invalidating snapshot Arcs already handed out. The caller may
|
|
1425
|
+
* `unlink(path)` after this returns; held Arc clones continue
|
|
1426
|
+
* to serve reads from the unlink'd mmap until the last reader
|
|
1427
|
+
* drops, per POSIX unlink-while-mapped semantics.
|
|
1428
|
+
*/
|
|
1429
|
+
evictReaderCacheByPath(path: string): void
|
|
1430
|
+
/**
|
|
1431
|
+
* Combined native memory footprint reported to ResourceManager
|
|
1432
|
+
* (memtable + reader cache). See `LsmEngine::native_memory_bytes`.
|
|
1433
|
+
*/
|
|
1434
|
+
nativeMemoryBytes(): bigint
|
|
1435
|
+
/**
|
|
1436
|
+
* **Test-only.** Cache hit counter — incremented on every
|
|
1437
|
+
* `get_or_open` that returns a previously-inserted entry.
|
|
1438
|
+
*/
|
|
1439
|
+
_testReaderCacheHits(): bigint
|
|
1440
|
+
/**
|
|
1441
|
+
* **Test-only.** Cache miss counter — incremented when a parse
|
|
1442
|
+
* + insert occurs. Passthrough mode (sub-floor budget) does NOT
|
|
1443
|
+
* increment.
|
|
1444
|
+
*/
|
|
1445
|
+
_testReaderCacheMisses(): bigint
|
|
1446
|
+
/** **Test-only.** Current resident bytes across cached entries. */
|
|
1447
|
+
_testReaderCacheBytes(): bigint
|
|
1448
|
+
/** **Test-only.** Current cache budget after the INV-8 clamp. */
|
|
1449
|
+
_testReaderCacheBudget(): bigint
|
|
1450
|
+
/** **Test-only.** Number of cached entries. */
|
|
1451
|
+
_testReaderCacheLen(): number
|
|
1452
|
+
/**
|
|
1453
|
+
* **Test-only.** Wipes every cached entry. Snapshots holding
|
|
1454
|
+
* Arc clones survive.
|
|
1455
|
+
*/
|
|
1456
|
+
_testReaderCacheClear(): void
|
|
1457
|
+
/**
|
|
1458
|
+
* **Test-only.** Delta of `LsmSstableReader::open()` invocations
|
|
1459
|
+
* since this engine's cache was constructed. Used to prove the
|
|
1460
|
+
* cache short-circuits repeat parses without wall-clock
|
|
1461
|
+
* measurements (replaces the flaky perf test from earlier drafts).
|
|
1462
|
+
*/
|
|
1463
|
+
_testReaderCacheOpenInvocations(): bigint
|
|
1464
|
+
/**
|
|
1465
|
+
* **Test-only.** Current pinned overshoot (bytes by which
|
|
1466
|
+
* pinned entries exceed the budget). Cleared after eviction
|
|
1467
|
+
* succeeds.
|
|
1468
|
+
*/
|
|
1469
|
+
_testReaderCachePinnedOvershootBytes(): bigint
|
|
1470
|
+
/**
|
|
1471
|
+
* **Test-only.** Highest pinned overshoot ever observed
|
|
1472
|
+
* (diagnostic high-water mark).
|
|
1473
|
+
*/
|
|
1474
|
+
_testReaderCachePinnedOvershootHighWater(): bigint
|
|
1475
|
+
/**
|
|
1476
|
+
* **Test-only.** Number of SHA-256 walks the cache has executed
|
|
1477
|
+
* across all entry points. Bumped on the Buffer-path body re-hash
|
|
1478
|
+
* (every `readPostingList` call) and on the mmap cold path
|
|
1479
|
+
* (`openSstablePath` / `readPostingListFromPaths` miss). Warm
|
|
1480
|
+
* mmap hits do NOT bump it — that's the observable the A5
|
|
1481
|
+
* closeout gate uses to prove the warm path skipped validation
|
|
1482
|
+
* (spec test #13).
|
|
1483
|
+
*/
|
|
1484
|
+
_testReaderCacheSha256Walks(): bigint
|
|
1485
|
+
/**
|
|
1486
|
+
* **Test-only.** Whether the reader for the given path is mmap-
|
|
1487
|
+
* backed (`true`) or heap-resident (`false`). Used by A5 spec
|
|
1488
|
+
* tests #15 + #16 (variant-of-winner accounting) and #19 + #20
|
|
1489
|
+
* (RSS contribution check). Returns `false` if the path is not
|
|
1490
|
+
* in the cache.
|
|
1491
|
+
*/
|
|
1492
|
+
_testReaderCacheIsMmap(path: string): boolean
|
|
1493
|
+
/**
|
|
1494
|
+
* **Test-only.** Measure writer throughput at `nWriters`
|
|
1495
|
+
* concurrent threads. Closes P3-1 (≥ 4× speedup from N=1 to
|
|
1496
|
+
* N=16). Returns `{elapsedNs, totalInserts}`; the caller
|
|
1497
|
+
* computes inserts/sec.
|
|
1498
|
+
*/
|
|
1499
|
+
_testWriterThroughput(nWriters: number, perWriter: number): NativeLsmThroughputResult
|
|
1500
|
+
/**
|
|
1501
|
+
* **Test-only.** Per-insert latency distribution under
|
|
1502
|
+
* admission pressure. Closes P3-2 (admission p99 ≤ 200 ms).
|
|
1503
|
+
* Returns the SORTED distribution in MICROSECONDS so the
|
|
1504
|
+
* caller can compute any percentile.
|
|
1505
|
+
*/
|
|
1506
|
+
_testAdmissionLatencyDistribution(nWriters: number, perWriter: number, budget: bigint): Array<bigint>
|
|
1507
|
+
/**
|
|
1508
|
+
* **Test-only.** Measure SSTable flush throughput including
|
|
1509
|
+
* write + fsync to a real filesystem path. Closes P3-4
|
|
1510
|
+
* (≥ 150 MB/s on NVMe).
|
|
1511
|
+
*/
|
|
1512
|
+
_testFlushThroughputWithFsync(targetPayloadMb: number, path: string): NativeLsmFlushThroughputResult
|
|
1513
|
+
/**
|
|
1514
|
+
* **Test-only.** Drive concurrent readers + writers + a
|
|
1515
|
+
* flusher to validate snapshot point-in-time isolation under
|
|
1516
|
+
* contention. Closes Phase 3 gate P3-3 + Phase 4 gate P4-4.
|
|
1517
|
+
*
|
|
1518
|
+
* Workers run on real OS threads (`std::thread`) so the CAS +
|
|
1519
|
+
* mutex contention paths are actually exercised. The hard
|
|
1520
|
+
* invariant: `torn_reads == 0` — a snapshot's cardinality for
|
|
1521
|
+
* its target key must NEVER change for the snapshot's lifetime,
|
|
1522
|
+
* regardless of concurrent writes or flushes.
|
|
1523
|
+
*/
|
|
1524
|
+
_testStressReaderWriterFlusher(nReaders: number, readsPerReader: number, nWriters: number, writesPerWriter: number, nPreloadKeys: number, flushIntervalMicros: number): NativeLsmReaderWriterStressResult
|
|
1525
|
+
/**
|
|
1526
|
+
* Walk every directory entry in an SSTable in sorted `(field,
|
|
1527
|
+
* value)` order, returning one summary per entry. This is the
|
|
1528
|
+
* production-quality SSTable inspection API: useful for
|
|
1529
|
+
* diagnostics, manifest validation, backup verification,
|
|
1530
|
+
* migration tooling, and dump-and-reload flows.
|
|
1531
|
+
*
|
|
1532
|
+
* Each summary carries:
|
|
1533
|
+
* - `field` / `value` — the directory key
|
|
1534
|
+
* - `additionsBytes` / `deletionsBytes` — croaring portable
|
|
1535
|
+
* serializations of the per-entry PostingLists (deserialize
|
|
1536
|
+
* with `NativeRoaringBitmap32` in U32 mode or
|
|
1537
|
+
* `NativeRoaringBitmap64` in U64 mode)
|
|
1538
|
+
* - `additionCount` / `deletionCount` — cardinalities
|
|
1539
|
+
* (cheaper to consume than re-deserializing the Buffers)
|
|
1540
|
+
* - `epoch` — the SSTable's per-entry epoch
|
|
1541
|
+
* - `isTombstone` — `true` when the entry is a pure delete
|
|
1542
|
+
*
|
|
1543
|
+
* SHA-256 + bloom + length checks all run inside the cache (or
|
|
1544
|
+
* `open` on miss), so a malformed Buffer throws before any
|
|
1545
|
+
* entry is materialized.
|
|
1546
|
+
*/
|
|
1547
|
+
scanSstable(sstable: Buffer): Array<NativeLsmScanEntry>
|
|
1548
|
+
/**
|
|
1549
|
+
* Take a point-in-time snapshot of the engine for reading.
|
|
1550
|
+
* The returned handle holds Arc clones of the memtable shards +
|
|
1551
|
+
* the supplied SSTable readers (opened through the content-keyed
|
|
1552
|
+
* reader cache). Reads through this handle are valid for up to
|
|
1553
|
+
* 30 seconds from creation; after that the snapshot returns
|
|
1554
|
+
* `SnapshotExpired` errors and the caller must re-acquire.
|
|
1555
|
+
*
|
|
1556
|
+
* The 30-second lifetime is hardcoded per the zero-config
|
|
1557
|
+
* mandate — operator knobs introduce footgun surface for almost
|
|
1558
|
+
* no benefit. See [`SNAPSHOT_LIFETIME`](super::snapshot::SNAPSHOT_LIFETIME).
|
|
1559
|
+
*/
|
|
1560
|
+
takeSnapshot(sstables: Array<Buffer>): NativeLsmSnapshot
|
|
1561
|
+
}
|
|
1562
|
+
|
|
1563
|
+
/**
|
|
1564
|
+
* A point-in-time read handle on an `LsmEngine`. Cheap to clone
|
|
1565
|
+
* (`Arc<LsmSnapshotInner>` internally). Reads return
|
|
1566
|
+
* `SnapshotExpired` after the 30-second lifetime; callers re-acquire
|
|
1567
|
+
* via [`NativeLsmEngine::take_snapshot`].
|
|
1568
|
+
*
|
|
1569
|
+
* The snapshot model mirrors LMDB read transactions: writes happen
|
|
1570
|
+
* against the live engine while the snapshot continues to see the
|
|
1571
|
+
* memtable + SSTable state at the moment of acquisition.
|
|
1572
|
+
*/
|
|
1573
|
+
export declare class NativeLsmSnapshot {
|
|
1574
|
+
/**
|
|
1575
|
+
* Read the posting list for `(field, value)` at this snapshot's
|
|
1576
|
+
* point in time. Throws `SnapshotExpired` if the snapshot was
|
|
1577
|
+
* created more than 30 seconds ago.
|
|
1578
|
+
*/
|
|
1579
|
+
readPostingList(field: string, value: string): Buffer
|
|
1580
|
+
/**
|
|
1581
|
+
* Whether the snapshot has aged past its 30-second lifetime.
|
|
1582
|
+
* Returns `true` once it is no longer usable for reads.
|
|
1583
|
+
*/
|
|
1584
|
+
isExpired(): boolean
|
|
1585
|
+
/** Number of SSTables this snapshot holds (diagnostic). */
|
|
1586
|
+
sstableCount(): number
|
|
1587
|
+
}
|
|
1588
|
+
|
|
1589
|
+
/**
|
|
1590
|
+
* NativeMetadataIndex — napi wrapper for the Rust metadata index.
|
|
1591
|
+
*
|
|
1592
|
+
* All query and mutation operations execute synchronously in Rust against
|
|
1593
|
+
* the attached LSM engine. The TS wrapper handles async storage I/O (the LSM
|
|
1594
|
+
* SSTable/manifest durability, the value-enumeration field indexes, and the
|
|
1595
|
+
* field registry).
|
|
1596
|
+
*/
|
|
1597
|
+
export declare class NativeMetadataIndex {
|
|
1598
|
+
/**
|
|
1599
|
+
* Create a new metadata index with optional configuration.
|
|
1600
|
+
*
|
|
1601
|
+
* `idSpace` opts into the entity-int wire width: `"u32"` (the
|
|
1602
|
+
* brainy 7.x-compatible 4.29 B-capped mode) or `"u64"` (cor
|
|
1603
|
+
* 3.0 billion-scale mode). Omitted defaults to U32 at the
|
|
1604
|
+
* raw napi surface to keep this constructor's API stable;
|
|
1605
|
+
* production callers (`MetadataIndexManager` in
|
|
1606
|
+
* `src/utils/NativeMetadataIndex.ts`) pass `"u64"` explicitly
|
|
1607
|
+
* per the cor 3.0 zero-config production default. Every
|
|
1608
|
+
* chunk created via the mutate path inherits this brain's
|
|
1609
|
+
* IdSpace so the entire index stays variant-consistent.
|
|
1610
|
+
*/
|
|
1611
|
+
constructor(configJson?: string | undefined | null, idSpace?: string | undefined | null)
|
|
1612
|
+
/**
|
|
1613
|
+
* IdSpace as a string — `"u32"` or `"u64"`. Mirrors the field on
|
|
1614
|
+
* [`MetadataIndexCore`].
|
|
1615
|
+
*/
|
|
1616
|
+
idSpace(): string
|
|
1617
|
+
/**
|
|
1618
|
+
* **The durable-engine swap.** Replace this index's LSM engine with the
|
|
1619
|
+
* durable, mmap-backed engine the TS wrapper opened for the brain's
|
|
1620
|
+
* storage path. The index is constructed with a non-persistent in-RAM
|
|
1621
|
+
* engine so it is usable immediately; production calls this once during
|
|
1622
|
+
* `init()` (after seeding the engine's epoch from the on-disk manifest)
|
|
1623
|
+
* to swap in the persistent engine before any mutation.
|
|
1624
|
+
*
|
|
1625
|
+
* The engine's `IdSpace` MUST match the metadata index's (both
|
|
1626
|
+
* `"u32"` or both `"u64"`) — a mismatch would corrupt the
|
|
1627
|
+
* `PostingList` algebra at the LSM ↔ metadata boundary. The
|
|
1628
|
+
* method rejects mismatched pairings with an explicit error.
|
|
1629
|
+
*
|
|
1630
|
+
* The previous engine's Arc is released on replacement (its memtable +
|
|
1631
|
+
* reader cache stay alive as long as anyone else holds an Arc). The
|
|
1632
|
+
* index always has a working engine, so there is no "detached" state.
|
|
1633
|
+
*/
|
|
1634
|
+
setLsmEngine(engine: NativeLsmEngine): void
|
|
1635
|
+
/**
|
|
1636
|
+
* Whether mutations + queries route through the LSM engine. Always
|
|
1637
|
+
* `true` in cor 3.0 — the LSM engine is the sole metadata engine and is
|
|
1638
|
+
* attached at construction. Retained as a diagnostic surface for tests
|
|
1639
|
+
* + TS-side telemetry that assert the dispatch is engaged.
|
|
1640
|
+
*/
|
|
1641
|
+
isLsmEnabled(): boolean
|
|
1642
|
+
/**
|
|
1643
|
+
* **#72 Phase C.** Inject the shared mmap `BinaryIdMapper` as this
|
|
1644
|
+
* metadata index's canonical UUID ↔ entity-int allocator. The napi
|
|
1645
|
+
* core is constructed with no storage path (the `id_mapper` starts
|
|
1646
|
+
* empty); the TS layer derives the `_id_mapper/*` paths from
|
|
1647
|
+
* storage, builds the mapper, and calls this once during `init()`
|
|
1648
|
+
* before any mutation, and again (with a freshly-`create`d, i.e.
|
|
1649
|
+
* truncated, mapper) at the start of a full rebuild.
|
|
1650
|
+
*
|
|
1651
|
+
* The mapper's `IdSpace` MUST match the metadata index's (both
|
|
1652
|
+
* `"u32"` or both `"u64"`) — a mismatch would corrupt the
|
|
1653
|
+
* `PostingList` algebra, which keys bitmaps by the int this mapper
|
|
1654
|
+
* assigns. Rejected with an explicit error.
|
|
1655
|
+
*
|
|
1656
|
+
* This replaces the legacy JSON entity-id-mapper path
|
|
1657
|
+
* (`loadEntityIdMapper` / `saveEntityIdMapper`), whose two resident
|
|
1658
|
+
* HashMaps cost ~56 GB at 1 B entities and whose serialize/parse
|
|
1659
|
+
* threw at the V8 ~512 MB / ~13 M-UUID cliff.
|
|
1660
|
+
*/
|
|
1661
|
+
setIdMapper(mapper: NativeBinaryIdMapper): void
|
|
1662
|
+
/**
|
|
1663
|
+
* Get entity ID mapper size (saturates at u32::MAX; use
|
|
1664
|
+
* [`entity_id_mapper_size_big`] for the overflow-safe BigInt
|
|
1665
|
+
* sibling).
|
|
1666
|
+
*/
|
|
1667
|
+
entityIdMapperSize(): number
|
|
1668
|
+
/**
|
|
1669
|
+
* BigInt sibling of [`entity_id_mapper_size`]. Overflow-safe at
|
|
1670
|
+
* billion scale; preferred by brainy 8.0 consumers.
|
|
1671
|
+
*/
|
|
1672
|
+
entityIdMapperSizeBig(): bigint
|
|
1673
|
+
/**
|
|
1674
|
+
* **Piece 6 generation counter — accessor.** Returns the current
|
|
1675
|
+
* generation as a BigInt (u64). Bumped by every read-visible
|
|
1676
|
+
* mutation (`add_to_index` / `remove_from_index`). The TS
|
|
1677
|
+
* wrapper caches the last-seen value and compares on every
|
|
1678
|
+
* query; an observed bump invalidates `loadedFields` so the
|
|
1679
|
+
* next query re-reads the current state.
|
|
1680
|
+
*
|
|
1681
|
+
* Cheap call: a single atomic `load(Acquire)` — fine to invoke
|
|
1682
|
+
* before every read without measurable overhead.
|
|
1683
|
+
*/
|
|
1684
|
+
currentGeneration(): bigint
|
|
1685
|
+
/**
|
|
1686
|
+
* **Piece 15 — per-field generation accessor.** Returns the
|
|
1687
|
+
* current generation for a single field as a BigInt (u64).
|
|
1688
|
+
* Bumped by every mutation in `add_to_index` / `remove_from_index`
|
|
1689
|
+
* that touches the field's data. Returns `0n` for fields that
|
|
1690
|
+
* have never been registered (the cold-boot semantic — a TS
|
|
1691
|
+
* reader probing a never-seen field gets `0n` and records it
|
|
1692
|
+
* without invalidating anything).
|
|
1693
|
+
*
|
|
1694
|
+
* Brainy 8.0+ wrappers use this to ask "did field X change since
|
|
1695
|
+
* I last looked?" without invalidating every loaded field on
|
|
1696
|
+
* every mutation. The index-wide
|
|
1697
|
+
* [`currentGeneration`](#method.current_generation) still
|
|
1698
|
+
* bumps on every mutation — brainy 7.x callers keep working
|
|
1699
|
+
* unchanged.
|
|
1700
|
+
*
|
|
1701
|
+
* Cheap call: HashMap lookup + atomic `load(Acquire)` — fine to
|
|
1702
|
+
* invoke per-field on every read entry point. See `MetadataIndexCore::
|
|
1703
|
+
* field_generation` for the memory-ordering rationale.
|
|
1704
|
+
*/
|
|
1705
|
+
fieldGeneration(field: string): bigint
|
|
1706
|
+
/**
|
|
1707
|
+
* Resolve an entity's integer id from its UUID (no assignment). Used to
|
|
1708
|
+
* populate the column store after `add_to_index` has assigned the id, and
|
|
1709
|
+
* to locate an entity on removal. O(1).
|
|
1710
|
+
*/
|
|
1711
|
+
uuidToInt(uuid: string): number | null
|
|
1712
|
+
/**
|
|
1713
|
+
* BigInt sibling of [`uuid_to_int`]. Works for both U32 and U64
|
|
1714
|
+
* brains; preferred by brainy 8.0 consumers.
|
|
1715
|
+
*/
|
|
1716
|
+
uuidToIntBig(uuid: string): bigint | null
|
|
1717
|
+
/**
|
|
1718
|
+
* Resolve a UUID from an entity's integer id. Backs `getIdMapper().getUuid`
|
|
1719
|
+
* for the `orderBy` path. O(1).
|
|
1720
|
+
*/
|
|
1721
|
+
intToUuid(intId: number): string | null
|
|
1722
|
+
/**
|
|
1723
|
+
* BigInt sibling of [`int_to_uuid`]. Works for both U32 and U64
|
|
1724
|
+
* brains.
|
|
1725
|
+
*/
|
|
1726
|
+
intToUuidBig(intId: bigint): string | null
|
|
1727
|
+
/**
|
|
1728
|
+
* Batch-resolve UUIDs from integer ids (skips unknown ids). Backs
|
|
1729
|
+
* `getIdMapper().intsIterableToUuids` for paginating sorted results.
|
|
1730
|
+
*/
|
|
1731
|
+
intsToUuids(ints: Array<number>): Array<string>
|
|
1732
|
+
/**
|
|
1733
|
+
* BigInt sibling of [`ints_to_uuids`]. Accepts a Vec<BigInt> so
|
|
1734
|
+
* the dispatch is uniform across U32 and U64 brains.
|
|
1735
|
+
*/
|
|
1736
|
+
intsToUuidsBig(ints: Array<bigint>): Array<string>
|
|
1737
|
+
/** Load field registry from JSON string. */
|
|
1738
|
+
loadFieldRegistry(json: string): void
|
|
1739
|
+
/** Save field registry to JSON string. */
|
|
1740
|
+
saveFieldRegistry(): string
|
|
1741
|
+
/** Load a field index from JSON string. */
|
|
1742
|
+
loadFieldIndex(field: string, json: string): void
|
|
1743
|
+
/** Save a field index to JSON string. */
|
|
1744
|
+
saveFieldIndex(field: string): string | null
|
|
1745
|
+
/** Get IDs for a single field-value pair. */
|
|
1746
|
+
getIds(field: string, valueJson: string): Array<string>
|
|
1747
|
+
/**
|
|
1748
|
+
* Count IDs for a single field-value pair WITHOUT materializing the UUID
|
|
1749
|
+
* strings. Equals `get_ids(field, value).length` exactly (counts only
|
|
1750
|
+
* mapped ints), but allocates nothing — the count-only fast path used by
|
|
1751
|
+
* `NativeMetadataIndex` at 10B scale (#78 pt2). Returned as i64 (napi has
|
|
1752
|
+
* no u64); counts are non-negative and well under 2^53.
|
|
1753
|
+
*/
|
|
1754
|
+
getIdsCount(field: string, valueJson: string): number
|
|
1755
|
+
/**
|
|
1756
|
+
* Get IDs for a filter expression.
|
|
1757
|
+
*
|
|
1758
|
+
* - `all_ids_json`: optional JSON array of all entity IDs (needed for
|
|
1759
|
+
* ne/exists:false).
|
|
1760
|
+
* - `limit` / `offset`: optional **page bound** (#75). When `limit` is
|
|
1761
|
+
* supplied, only the `[offset, offset + limit)` window of the sorted
|
|
1762
|
+
* match set is produced — the posting-list → UUID conversion
|
|
1763
|
+
* short-circuits at `offset + limit` mapped ids instead of
|
|
1764
|
+
* materializing the full list and slicing, removing the O(N) FFI
|
|
1765
|
+
* marshal for broad-filter single-page `find()` at billion scale. This
|
|
1766
|
+
* is the producer half of brainy's
|
|
1767
|
+
* `MetadataIndexProvider.getIdsForFilter(filter, {limit, offset})`
|
|
1768
|
+
* contract; brainy passes `{limit: pageEnd, offset: 0}` and re-windows
|
|
1769
|
+
* the prefix itself. Omitting `limit` returns the FULL match set
|
|
1770
|
+
* (cortex 2.x back-compat).
|
|
1771
|
+
*/
|
|
1772
|
+
getIdsForFilter(filterJson: string, allIdsJson?: string | undefined | null, limit?: number | undefined | null, offset?: number | undefined | null): Array<string>
|
|
1773
|
+
/**
|
|
1774
|
+
* Cumulative count of id-mapper resolutions performed by the filter
|
|
1775
|
+
* producer ([`get_ids_for_filter`](#method.get_ids_for_filter)) since
|
|
1776
|
+
* construction (#75 observability).
|
|
1777
|
+
*
|
|
1778
|
+
* The page-bounded path keeps this at `offset + limit` per call instead
|
|
1779
|
+
* of the full match cardinality — so a delta after a windowed call proves
|
|
1780
|
+
* the producer short-circuited at production time rather than
|
|
1781
|
+
* materializing the whole list then slicing. Returned as i64 (napi has no
|
|
1782
|
+
* u64); monotonic and well under 2^53 in practice.
|
|
1783
|
+
*/
|
|
1784
|
+
filterIdsResolved(): number
|
|
1785
|
+
/** Get IDs for multiple field-value pairs (AND intersection). */
|
|
1786
|
+
getIdsForMultipleFields(pairsJson: string): Array<string>
|
|
1787
|
+
/** Get IDs for a range query on a field. */
|
|
1788
|
+
getIdsForRange(field: string, minJson: string | undefined | null, maxJson: string | undefined | null, includeMin: boolean, includeMax: boolean): Array<string>
|
|
1789
|
+
/** Get IDs for a text query. Returns JSON array of {id, matchCount}. */
|
|
1790
|
+
getIdsForTextQuery(queryStr: string): string
|
|
1791
|
+
/** Get all unique values for a field (for filter discovery). */
|
|
1792
|
+
getFilterValues(field: string): Array<string>
|
|
1793
|
+
/** Get all indexed field names (for filter discovery). */
|
|
1794
|
+
getFilterFields(): Array<string>
|
|
1795
|
+
/** Extract field names from entity JSON (for lazy loading). */
|
|
1796
|
+
extractFieldNames(entityJson: string): Array<string>
|
|
1797
|
+
/** Add an entity to the index. Returns MutationResult JSON. */
|
|
1798
|
+
addToIndex(id: string, entityJson: string): string
|
|
1799
|
+
/** Remove an entity from the index. Returns MutationResult JSON. */
|
|
1800
|
+
removeFromIndex(id: string, metadataJson?: string | undefined | null): string
|
|
1801
|
+
/**
|
|
1802
|
+
* **Brainy rc.6 cold-open self-heal (matrix 7.7).** An O(1)-ish
|
|
1803
|
+
* consistency probe brainy calls ONCE on the first read; on `false` it
|
|
1804
|
+
* runs `detectAndRepairCorruption()`.
|
|
1805
|
+
*
|
|
1806
|
+
* The check is fully internal — it needs NO entity re-read. It samples a
|
|
1807
|
+
* bounded set of fields (`noun` first — the canonical single-valued type
|
|
1808
|
+
* field — then a few others, skipping the inherently multi-valued
|
|
1809
|
+
* `__words__` text index), draws a bounded sample of ints from each
|
|
1810
|
+
* field's value buckets, and counts how many buckets each sampled int
|
|
1811
|
+
* occupies. A single-valued field carrying a phantom shows the tell-tale
|
|
1812
|
+
* MIX of single-bucket ints (the healthy majority) AND ≥1 multi-bucket
|
|
1813
|
+
* int (the phantom) → returns `false`. A field whose sampled ints are
|
|
1814
|
+
* UNIFORMLY multi-bucket is genuinely multi-valued (e.g. `tags`) and is
|
|
1815
|
+
* NOT flagged, so the probe does not false-positive on array fields.
|
|
1816
|
+
*
|
|
1817
|
+
* Panic-free and never throws: returns `true` on an empty index or when
|
|
1818
|
+
* no field is confidently inconsistent. The TS wrapper additionally wraps
|
|
1819
|
+
* the call so any error degrades to `true` — a probe must never break the
|
|
1820
|
+
* read path.
|
|
1821
|
+
*/
|
|
1822
|
+
probeConsistency(): boolean
|
|
1823
|
+
/**
|
|
1824
|
+
* **Phantom triad — state-level scan (matrix 7.8b).** Find every
|
|
1825
|
+
* query-visible int that sits in MORE THAN ONE value bucket of a field —
|
|
1826
|
+
* the cross-bucket phantom signature — and return them as a JSON array of
|
|
1827
|
+
* `{ uuid, field }` for the TS wrapper to repair against each entity's
|
|
1828
|
+
* stored value(s).
|
|
1829
|
+
*
|
|
1830
|
+
* Enumerates each field's value buckets from the value-enumeration
|
|
1831
|
+
* surface (`field_indexes`) and counts per-int bucket membership through
|
|
1832
|
+
* the LSM snapshot. The inherently multi-valued `__words__` text index is
|
|
1833
|
+
* skipped. Only ints the id mapper still resolves to a UUID are returned:
|
|
1834
|
+
* an unmapped int is invisible to `getIds` (it cannot produce a wrong
|
|
1835
|
+
* result), so it needs no repair. `max_candidates` (0 = unlimited) caps
|
|
1836
|
+
* the result so a massively corrupt index can be handled by the wrapper's
|
|
1837
|
+
* rebuild fallback instead of an unbounded list. This is the heavyweight
|
|
1838
|
+
* repair path — only reached after `probeConsistency()` flagged the index.
|
|
1839
|
+
*/
|
|
1840
|
+
detectCrossBucketPhantoms(maxCandidates: number): string
|
|
1841
|
+
/**
|
|
1842
|
+
* **Phantom triad — state-level purge (matrix 7.8b).** Tombstone the
|
|
1843
|
+
* entity `uuid`'s int out of every value bucket of `field` EXCEPT the
|
|
1844
|
+
* buckets matching its CURRENT stored value(s), supplied as a JSON array
|
|
1845
|
+
* in `current_values_json` (brainy's source of truth, read from storage by
|
|
1846
|
+
* the wrapper — never from the index being repaired).
|
|
1847
|
+
*
|
|
1848
|
+
* The raw values are normalized exactly as `add_to_index` would, forming
|
|
1849
|
+
* the `keep` set, then [`mutate::evict_stale_postings`] removes the rest.
|
|
1850
|
+
* An empty `current_values_json` (entity gone, or the field dropped)
|
|
1851
|
+
* evicts the int from EVERY bucket of the field — purging an orphan
|
|
1852
|
+
* phantom entirely. Returns `true` if any stale posting was tombstoned.
|
|
1853
|
+
* Unmapped `uuid` (already removed) is a no-op returning `false`.
|
|
1854
|
+
*/
|
|
1855
|
+
repairEntityFieldPhantom(uuid: string, field: string, currentValuesJson: string): boolean
|
|
1856
|
+
/** Get entity count by type name. */
|
|
1857
|
+
getEntityCountByType(typeName: string): number
|
|
1858
|
+
/** Get total entity count. */
|
|
1859
|
+
getTotalEntityCount(): number
|
|
1860
|
+
/** Get top N noun types by count. Returns JSON. */
|
|
1861
|
+
getTopNounTypes(n: number): string
|
|
1862
|
+
/** Get top N verb types by count. Returns JSON. */
|
|
1863
|
+
getTopVerbTypes(n: number): string
|
|
1864
|
+
/** Get comprehensive statistics. Returns JSON. */
|
|
1865
|
+
getStats(): string
|
|
1866
|
+
/** Clear all in-memory state. */
|
|
1867
|
+
clear(): void
|
|
1868
|
+
}
|
|
1869
|
+
|
|
1870
|
+
/**
|
|
1871
|
+
* A trained PQ codebook.
|
|
1872
|
+
*
|
|
1873
|
+
* Contains M×ksub centroids, each of dimension dsub = dim/M.
|
|
1874
|
+
* Stored as a flat f32 array: centroids[m][k][d] = data[m * ksub * dsub + k * dsub + d]
|
|
1875
|
+
*/
|
|
1876
|
+
export declare class NativePqCodebook {
|
|
1877
|
+
/** Create a PQ codebook. Use `NativePQCodebook.train()` or `NativePQCodebook.deserialize()` instead. */
|
|
1878
|
+
constructor()
|
|
1879
|
+
/**
|
|
1880
|
+
* Train a PQ codebook from a set of training vectors.
|
|
1881
|
+
*
|
|
1882
|
+
* Parameters:
|
|
1883
|
+
* - vectors: training data (flat array, each vector is `dim` f64 values)
|
|
1884
|
+
* - dim: vector dimension
|
|
1885
|
+
* - m: number of subspaces (must divide dim evenly)
|
|
1886
|
+
* - ksub: centroids per subspace (typically 256)
|
|
1887
|
+
* - iterations: k-means iterations (typically 20-50)
|
|
1888
|
+
*/
|
|
1889
|
+
static train(vectors: Array<number>, dim: number, m: number, ksub: number, iterations: number): NativePqCodebook
|
|
1890
|
+
/**
|
|
1891
|
+
* Encode a single vector to a PQ code (M bytes).
|
|
1892
|
+
*
|
|
1893
|
+
* Each byte is the index of the nearest centroid in the corresponding subspace.
|
|
1894
|
+
*/
|
|
1895
|
+
encode(vector: Array<number>): Buffer
|
|
1896
|
+
/**
|
|
1897
|
+
* Encode multiple vectors to PQ codes.
|
|
1898
|
+
* Returns flat buffer: n * m bytes.
|
|
1899
|
+
*/
|
|
1900
|
+
encodeBatch(vectors: Array<number>, dim: number): Buffer
|
|
1901
|
+
/** Decode a PQ code back to an approximate vector. */
|
|
1902
|
+
decode(code: Buffer): Array<number>
|
|
1903
|
+
/**
|
|
1904
|
+
* Precompute the ADC (Asymmetric Distance Computation) lookup table.
|
|
1905
|
+
*
|
|
1906
|
+
* Returns a flat f64 array of m * ksub distances.
|
|
1907
|
+
* table[sub_idx * ksub + k] = squared distance from query subvector to centroid k.
|
|
1908
|
+
*/
|
|
1909
|
+
computeAdcTable(query: Array<number>): Array<number>
|
|
1910
|
+
/**
|
|
1911
|
+
* Compute approximate distance from a precomputed ADC table to a single PQ code.
|
|
1912
|
+
*
|
|
1913
|
+
* Returns squared Euclidean distance (sum of M table lookups).
|
|
1914
|
+
*/
|
|
1915
|
+
distanceAdc(table: Array<number>, code: Buffer): number
|
|
1916
|
+
/**
|
|
1917
|
+
* Batch compute ADC distances from a precomputed table to multiple PQ codes.
|
|
1918
|
+
*
|
|
1919
|
+
* codes: flat buffer of n * m bytes.
|
|
1920
|
+
* Returns n distances.
|
|
1921
|
+
*/
|
|
1922
|
+
distanceAdcBatch(table: Array<number>, codes: Buffer, n: number): Array<number>
|
|
1923
|
+
/**
|
|
1924
|
+
* Serialize the codebook to a binary format for persistence.
|
|
1925
|
+
*
|
|
1926
|
+
* Format: [m:u32][ksub:u32][dsub:u32][dim:u32][centroids:f32×(m×ksub×dsub)]
|
|
1927
|
+
*/
|
|
1928
|
+
serialize(): Buffer
|
|
1929
|
+
/** Deserialize a codebook from binary format. */
|
|
1930
|
+
static deserialize(data: Buffer): NativePqCodebook
|
|
1931
|
+
/** Get codebook parameters. */
|
|
1932
|
+
get params(): PQParams
|
|
1933
|
+
}
|
|
1934
|
+
export type NativePQCodebook = NativePqCodebook
|
|
1935
|
+
|
|
1936
|
+
/** Native roaring bitmap (drop-in replacement for roaring-wasm RoaringBitmap32). */
|
|
1937
|
+
export declare class NativeRoaringBitmap32 {
|
|
1938
|
+
/** Create a new empty bitmap, or from an array of u32 values. */
|
|
1939
|
+
constructor(values?: Array<number> | undefined | null)
|
|
1940
|
+
/** Number of values in the bitmap. */
|
|
1941
|
+
get size(): number
|
|
1942
|
+
/** Whether the bitmap is empty. */
|
|
1943
|
+
get isEmpty(): boolean
|
|
1944
|
+
/** Add a value to the bitmap. */
|
|
1945
|
+
add(value: number): void
|
|
1946
|
+
/** Remove a value from the bitmap (called `delete` in roaring-wasm API). */
|
|
1947
|
+
delete(value: number): void
|
|
1948
|
+
/** Add a value, return true if it was newly added (false if already present). */
|
|
1949
|
+
tryAdd(value: number): boolean
|
|
1950
|
+
/** Check if the bitmap contains a value. */
|
|
1951
|
+
has(value: number): boolean
|
|
1952
|
+
/** Convert bitmap to an array of u32 values. */
|
|
1953
|
+
toArray(): Array<number>
|
|
1954
|
+
/** Serialize to portable format (binary-identical to roaring-wasm portable format). */
|
|
1955
|
+
serialize(): Buffer
|
|
1956
|
+
/** Get serialization size in bytes (portable format). */
|
|
1957
|
+
serializationSizeInBytes(): number
|
|
1958
|
+
/** Deserialize from portable format buffer. */
|
|
1959
|
+
static deserialize(buffer: Buffer): NativeRoaringBitmap32
|
|
1960
|
+
/** Minimum value in the bitmap. Returns undefined if empty. */
|
|
1961
|
+
minimum(): number | null
|
|
1962
|
+
/** Maximum value in the bitmap. Returns undefined if empty. */
|
|
1963
|
+
maximum(): number | null
|
|
1964
|
+
/** Compute intersection (AND) of two bitmaps. */
|
|
1965
|
+
static and(a: NativeRoaringBitmap32, b: NativeRoaringBitmap32): NativeRoaringBitmap32
|
|
1966
|
+
/** Compute union (OR) of two bitmaps. */
|
|
1967
|
+
static or(a: NativeRoaringBitmap32, b: NativeRoaringBitmap32): NativeRoaringBitmap32
|
|
1968
|
+
/** Compute union (OR) of multiple bitmaps. */
|
|
1969
|
+
static orMany(bitmaps: Array<NativeRoaringBitmap32>): NativeRoaringBitmap32
|
|
1970
|
+
/** Compute difference (AND NOT) of two bitmaps. */
|
|
1971
|
+
static andNot(a: NativeRoaringBitmap32, b: NativeRoaringBitmap32): NativeRoaringBitmap32
|
|
1972
|
+
/** Compute symmetric difference (XOR) of two bitmaps. */
|
|
1973
|
+
static xor(a: NativeRoaringBitmap32, b: NativeRoaringBitmap32): NativeRoaringBitmap32
|
|
1974
|
+
}
|
|
1975
|
+
|
|
1976
|
+
/**
|
|
1977
|
+
* 64-bit roaring bitmap backed by `croaring::Treemap`. Used by
|
|
1978
|
+
* `IdSpace::U64` brains where entity ints may exceed `u32::MAX`.
|
|
1979
|
+
*
|
|
1980
|
+
* The JS surface uses `BigInt` exclusively so values above 2^53 cross
|
|
1981
|
+
* the napi boundary losslessly. The Portable serialization format is
|
|
1982
|
+
* the croaring `Treemap` Portable format — **NOT** compatible with
|
|
1983
|
+
* `NativeRoaringBitmap32`'s Portable buffers. Callers select between
|
|
1984
|
+
* the two types via the enclosing index's `IdSpace` discriminator.
|
|
1985
|
+
*/
|
|
1986
|
+
export declare class NativeRoaringBitmap64 {
|
|
1987
|
+
/** Create a new empty bitmap, or from an array of BigInt values. */
|
|
1988
|
+
constructor(values?: Array<bigint> | undefined | null)
|
|
1989
|
+
/**
|
|
1990
|
+
* Number of values in the bitmap (BigInt because cardinality may
|
|
1991
|
+
* exceed u32 in a 64-bit-keyed bitmap).
|
|
1992
|
+
*/
|
|
1993
|
+
get size(): bigint
|
|
1994
|
+
/** Whether the bitmap is empty. */
|
|
1995
|
+
get isEmpty(): boolean
|
|
1996
|
+
/** Add a value to the bitmap. */
|
|
1997
|
+
add(value: bigint): void
|
|
1998
|
+
/**
|
|
1999
|
+
* Add many values in one call. Cheaper than repeated `add` for
|
|
2000
|
+
* bulk insertion because the napi crossing is amortized.
|
|
2001
|
+
*/
|
|
2002
|
+
addMany(values: Array<bigint>): void
|
|
2003
|
+
/**
|
|
2004
|
+
* Remove a value from the bitmap. Named `delete` to mirror the
|
|
2005
|
+
* 32-bit sibling and roaring-wasm's API.
|
|
2006
|
+
*/
|
|
2007
|
+
delete(value: bigint): void
|
|
2008
|
+
/**
|
|
2009
|
+
* Add a value, return true if it was newly added (false if
|
|
2010
|
+
* already present).
|
|
2011
|
+
*/
|
|
2012
|
+
tryAdd(value: bigint): boolean
|
|
2013
|
+
/** Check if the bitmap contains a value. */
|
|
2014
|
+
has(value: bigint): boolean
|
|
2015
|
+
/**
|
|
2016
|
+
* Materialise the bitmap as a sorted `BigInt[]`. **Avoid at
|
|
2017
|
+
* billion scale** — `toArray` allocates one BigInt per entry.
|
|
2018
|
+
* For large bitmaps prefer streaming consumption from the
|
|
2019
|
+
* enclosing posting-list API.
|
|
2020
|
+
*/
|
|
2021
|
+
toArray(): Array<bigint>
|
|
2022
|
+
/**
|
|
2023
|
+
* Serialize to the Treemap Portable format. Not interchangeable
|
|
2024
|
+
* with `NativeRoaringBitmap32` Portable buffers (different
|
|
2025
|
+
* underlying container layout).
|
|
2026
|
+
*/
|
|
2027
|
+
serialize(): Buffer
|
|
2028
|
+
/** Get serialization size in bytes. */
|
|
2029
|
+
serializationSizeInBytes(): bigint
|
|
2030
|
+
/** Deserialize from a Treemap Portable buffer. */
|
|
2031
|
+
static deserialize(buffer: Buffer): NativeRoaringBitmap64
|
|
2032
|
+
/** Minimum value in the bitmap. Returns undefined if empty. */
|
|
2033
|
+
minimum(): bigint | null
|
|
2034
|
+
/** Maximum value in the bitmap. Returns undefined if empty. */
|
|
2035
|
+
maximum(): bigint | null
|
|
2036
|
+
/** Clear all values (in-place). */
|
|
2037
|
+
clear(): void
|
|
2038
|
+
/** Compute intersection (AND) of two bitmaps. */
|
|
2039
|
+
static and(a: NativeRoaringBitmap64, b: NativeRoaringBitmap64): NativeRoaringBitmap64
|
|
2040
|
+
/** Compute union (OR) of two bitmaps. */
|
|
2041
|
+
static or(a: NativeRoaringBitmap64, b: NativeRoaringBitmap64): NativeRoaringBitmap64
|
|
2042
|
+
/**
|
|
2043
|
+
* Compute union (OR) of multiple bitmaps. `Treemap` has no
|
|
2044
|
+
* dedicated `or_many` primitive in croaring 2.5; we fold pairwise.
|
|
2045
|
+
*/
|
|
2046
|
+
static orMany(bitmaps: Array<NativeRoaringBitmap64>): NativeRoaringBitmap64
|
|
2047
|
+
/** Compute difference (AND NOT) of two bitmaps. */
|
|
2048
|
+
static andNot(a: NativeRoaringBitmap64, b: NativeRoaringBitmap64): NativeRoaringBitmap64
|
|
2049
|
+
/** Compute symmetric difference (XOR) of two bitmaps. */
|
|
2050
|
+
static xor(a: NativeRoaringBitmap64, b: NativeRoaringBitmap64): NativeRoaringBitmap64
|
|
2051
|
+
}
|
|
2052
|
+
|
|
2053
|
+
/**
|
|
2054
|
+
* JS-side handle to a [`ShardMigrationManifest`]. Owns the Rust
|
|
2055
|
+
* value; state transitions mutate in place. Use
|
|
2056
|
+
* [`to_bytes`](NativeShardMigrationManifest::to_bytes) to get JSON
|
|
2057
|
+
* bytes for atomic persistence via brainy's storage adapter.
|
|
2058
|
+
*/
|
|
2059
|
+
export declare class NativeShardMigrationManifest {
|
|
2060
|
+
/**
|
|
2061
|
+
* Construct a new manifest in `Idle` state for a brain with
|
|
2062
|
+
* `currentDepth` (2 or 3).
|
|
2063
|
+
*/
|
|
2064
|
+
constructor(currentDepth: number)
|
|
2065
|
+
/**
|
|
2066
|
+
* Load an existing manifest from its JSON bytes (the canonical
|
|
2067
|
+
* form written via [`to_bytes`]).
|
|
2068
|
+
*/
|
|
2069
|
+
static fromBytes(bytes: Buffer): NativeShardMigrationManifest
|
|
2070
|
+
/**
|
|
2071
|
+
* Serialize the current manifest state to pretty-printed JSON
|
|
2072
|
+
* bytes for atomic-write via brainy's storage adapter
|
|
2073
|
+
* (`saveBinaryBlob(path, buffer)` uses tmp+rename inside the
|
|
2074
|
+
* adapter).
|
|
2075
|
+
*/
|
|
2076
|
+
toBytes(): Buffer
|
|
2077
|
+
/**
|
|
2078
|
+
* Manifest format version. Bumped when the on-disk JSON shape
|
|
2079
|
+
* changes incompatibly.
|
|
2080
|
+
*/
|
|
2081
|
+
get version(): number
|
|
2082
|
+
/**
|
|
2083
|
+
* Current shard depth — the layout entities are stored under.
|
|
2084
|
+
* Equals the migration target after `finalizeCommit`.
|
|
2085
|
+
*/
|
|
2086
|
+
get currentDepth(): number
|
|
2087
|
+
/**
|
|
2088
|
+
* Read-only summary of the migration state. Returns
|
|
2089
|
+
* `{stateKind: 'idle'}` when no migration is in flight.
|
|
2090
|
+
*/
|
|
2091
|
+
stateSummary(): ShardMigrationStateSummary
|
|
2092
|
+
/**
|
|
2093
|
+
* Begin a migration to `targetDepth`. Throws if a migration is
|
|
2094
|
+
* already in progress or if `targetDepth` equals `currentDepth`.
|
|
2095
|
+
*/
|
|
2096
|
+
startMigration(targetDepth: number, totalUuids: bigint, nowMs: bigint): void
|
|
2097
|
+
/**
|
|
2098
|
+
* Update the cumulative `scanned` count and the
|
|
2099
|
+
* last-progress timestamp. Throws if state is not `Migrating`.
|
|
2100
|
+
*/
|
|
2101
|
+
recordProgress(scanned: bigint, nowMs: bigint): void
|
|
2102
|
+
/**
|
|
2103
|
+
* Enter the `Committing` barrier — all entities are moved;
|
|
2104
|
+
* about to flip `currentDepth` on disk. Throws if state is not
|
|
2105
|
+
* `Migrating`.
|
|
2106
|
+
*/
|
|
2107
|
+
beginCommit(nowMs: bigint): void
|
|
2108
|
+
/**
|
|
2109
|
+
* Atomic flip: `currentDepth` becomes the migration's target,
|
|
2110
|
+
* state returns to `Idle`. Throws if state is not `Committing`.
|
|
2111
|
+
* Caller must then serialize + atomically write the new
|
|
2112
|
+
* manifest bytes.
|
|
2113
|
+
*/
|
|
2114
|
+
finalizeCommit(): void
|
|
2115
|
+
/**
|
|
2116
|
+
* Abort an in-flight migration. State returns to `Idle` with
|
|
2117
|
+
* `currentDepth` unchanged. Brainy is responsible for cleaning
|
|
2118
|
+
* up any already-migrated entity files at the target paths.
|
|
2119
|
+
* No-op when state is already `Idle`.
|
|
2120
|
+
*/
|
|
2121
|
+
cancelMigration(): void
|
|
2122
|
+
/**
|
|
2123
|
+
* Storage paths brainy should consult for `uuid`. Returns ONE
|
|
2124
|
+
* path when `Idle`, TWO paths when `Migrating` / `Committing`
|
|
2125
|
+
* (target layout first, current as fallback). Caller tries
|
|
2126
|
+
* them in order.
|
|
2127
|
+
*/
|
|
2128
|
+
resolveLayoutPaths(uuid: string): Array<string>
|
|
2129
|
+
}
|
|
2130
|
+
|
|
2131
|
+
/**
|
|
2132
|
+
* Native unified cache engine.
|
|
2133
|
+
* Manages eviction decisions and metadata; TS wrapper holds actual JS data.
|
|
2134
|
+
*/
|
|
2135
|
+
export declare class NativeUnifiedCache {
|
|
2136
|
+
/** Create a new cache with the given maximum size in bytes. */
|
|
2137
|
+
constructor(maxSize: number)
|
|
2138
|
+
/** Record a cache hit — increment access count for the key. */
|
|
2139
|
+
recordAccess(key: string): void
|
|
2140
|
+
/**
|
|
2141
|
+
* Insert a cache entry. Returns keys that were evicted to make room.
|
|
2142
|
+
* cache_type: 0=hnsw, 1=metadata, 2=embedding, 3=other
|
|
2143
|
+
*/
|
|
2144
|
+
insert(key: string, cacheType: number, size: number, rebuildCost: number): NativeEvictionResult
|
|
2145
|
+
/** Remove a specific key. Returns freed bytes. */
|
|
2146
|
+
remove(key: string): number
|
|
2147
|
+
/** Remove all entries matching a key prefix. Returns list of removed keys. */
|
|
2148
|
+
removeByPrefix(prefix: string): Array<string>
|
|
2149
|
+
/**
|
|
2150
|
+
* Clear all entries or entries of a specific type. Returns removed keys.
|
|
2151
|
+
* Pass null/undefined to clear all.
|
|
2152
|
+
*/
|
|
2153
|
+
clear(cacheType?: number | undefined | null): Array<string>
|
|
2154
|
+
/** Evict entries to free at least `bytes_needed` bytes. */
|
|
2155
|
+
evictForSize(bytesNeeded: number): NativeEvictionResult
|
|
2156
|
+
/**
|
|
2157
|
+
* Fairness check: detect types using >70% cache with <15% access.
|
|
2158
|
+
* Returns keys evicted during fairness correction.
|
|
2159
|
+
*/
|
|
2160
|
+
checkFairness(): NativeEvictionResult
|
|
2161
|
+
/** Report native subsystem memory usage (called periodically by TS). */
|
|
2162
|
+
reportNativeMemory(hnsw: number, metadata: number, graph: number): void
|
|
2163
|
+
/** Get comprehensive statistics. */
|
|
2164
|
+
getStats(): NativeCacheStats
|
|
2165
|
+
/** Check if a key exists in the cache metadata. */
|
|
2166
|
+
has(key: string): boolean
|
|
2167
|
+
/** Number of entries in the cache. */
|
|
2168
|
+
itemCount(): number
|
|
2169
|
+
/** Save access patterns as JSON for warm start persistence. */
|
|
2170
|
+
saveAccessPatterns(): string
|
|
2171
|
+
/** Load access patterns from JSON for warm start. */
|
|
2172
|
+
loadAccessPatterns(json: string): void
|
|
2173
|
+
}
|
|
2174
|
+
|
|
2175
|
+
/**
|
|
2176
|
+
* JS-exposed configuration. All fields optional; defaults match the
|
|
2177
|
+
* cor 3.0 billion-scale target (32 GB sparse files per side).
|
|
2178
|
+
*/
|
|
2179
|
+
export interface BinaryIdMapperConfig {
|
|
2180
|
+
/** Path for `uuid_to_int.mkv` (extendible-hash KV file). REQUIRED. */
|
|
2181
|
+
uuidToIntPath: string
|
|
2182
|
+
/** Path for `int_to_uuid.bin` (direct-indexed sparse file). REQUIRED. */
|
|
2183
|
+
intToUuidPath: string
|
|
2184
|
+
/** Sparse file size for int_to_uuid in bytes. Default 32 GB. */
|
|
2185
|
+
intToUuidSize?: bigint
|
|
2186
|
+
/** Sparse file size for uuid_to_int in bytes. Default 32 GB. */
|
|
2187
|
+
uuidToIntSize?: bigint
|
|
2188
|
+
/** MmapKv bucket_capacity (entries per bucket). Default 16. */
|
|
2189
|
+
bucketCapacity?: number
|
|
2190
|
+
/**
|
|
2191
|
+
* MmapKv max_global_depth (extendible-hash directory depth cap).
|
|
2192
|
+
* Default 28 (supports ~12 B entries).
|
|
2193
|
+
*/
|
|
2194
|
+
maxGlobalDepth?: number
|
|
2195
|
+
/**
|
|
2196
|
+
* Entity-int wire-width: `"u32"` (default — cortex 2.x compatible,
|
|
2197
|
+
* caps at 4.29 B entries) or `"u64"` (Piece 10 — required for
|
|
2198
|
+
* >4.29 B brains, persists 8-byte ints + bumps `int_to_uuid.bin`
|
|
2199
|
+
* to v2 header). Ignored at open time: the on-disk format
|
|
2200
|
+
* dictates the IdSpace and any user-provided value is
|
|
2201
|
+
* cross-checked against the header (mismatch is a hard error).
|
|
2202
|
+
*/
|
|
2203
|
+
idSpace?: string
|
|
2204
|
+
}
|
|
2205
|
+
|
|
2206
|
+
/**
|
|
2207
|
+
* Diagnostic snapshot returned by [`NativeBinaryIdMapper::diagnostics`].
|
|
2208
|
+
*
|
|
2209
|
+
* The u32-typed `size` / `next_int` fields are zero in `IdSpace::U64`
|
|
2210
|
+
* mode (use the BigInt siblings); `int_size` and `id_space` always
|
|
2211
|
+
* reflect the brain's actual IdSpace.
|
|
2212
|
+
*/
|
|
2213
|
+
export interface BinaryIdMapperDiagnostics {
|
|
2214
|
+
uuidSize: number
|
|
2215
|
+
intSize: number
|
|
2216
|
+
intToUuidEntrySize: number
|
|
2217
|
+
assignShards: number
|
|
2218
|
+
size: number
|
|
2219
|
+
nextInt: number
|
|
2220
|
+
sizeBig: bigint
|
|
2221
|
+
nextIntBig: bigint
|
|
2222
|
+
idSpace: string
|
|
2223
|
+
}
|
|
2224
|
+
|
|
2225
|
+
/**
|
|
2226
|
+
* Cosine distance between two vectors.
|
|
2227
|
+
* Range: 0 (identical) to 2 (opposite).
|
|
2228
|
+
* Lower values indicate higher similarity.
|
|
2229
|
+
*/
|
|
2230
|
+
export declare function cosineDistance(a: Array<number>, b: Array<number>): number
|
|
2231
|
+
|
|
2232
|
+
/**
|
|
2233
|
+
* Batch cosine distance: compute distances from a query to multiple vectors.
|
|
2234
|
+
* Uses Rayon parallelism for large batches.
|
|
2235
|
+
* Returns array of distances.
|
|
2236
|
+
*/
|
|
2237
|
+
export declare function cosineDistanceBatch(query: Array<number>, vectors: Array<Array<number>>): Array<number>
|
|
2238
|
+
|
|
2239
|
+
/**
|
|
2240
|
+
* Compute approximate cosine distance between two SQ4-quantized vectors.
|
|
2241
|
+
*
|
|
2242
|
+
* Unpacks nibbles to u8 values, then uses the same integer accumulation
|
|
2243
|
+
* approach as SQ8 but with 4-bit precision.
|
|
2244
|
+
*/
|
|
2245
|
+
export declare function cosineDistanceSq4(a: Buffer, aMin: number, aMax: number, aDim: number, b: Buffer, bMin: number, bMax: number, bDim: number): number
|
|
2246
|
+
|
|
2247
|
+
/** Batch cosine distance from an SQ4 query to multiple SQ4 vectors. */
|
|
2248
|
+
export declare function cosineDistanceSq4Batch(query: Buffer, queryMin: number, queryMax: number, queryDim: number, vectors: Array<Buffer>, mins: Array<number>, maxs: Array<number>, dims: Array<number>): Array<number>
|
|
2249
|
+
|
|
2250
|
+
/**
|
|
2251
|
+
* Compute approximate cosine distance between two SQ8-quantized vectors.
|
|
2252
|
+
*
|
|
2253
|
+
* Operates directly on uint8 values using integer arithmetic in the inner loop.
|
|
2254
|
+
* Avoids full dequantization by reconstructing the dot product and norms from
|
|
2255
|
+
* integer sums and the per-vector scale factors.
|
|
2256
|
+
*
|
|
2257
|
+
* The accumulation loop auto-vectorizes with AVX2/NEON.
|
|
2258
|
+
*/
|
|
2259
|
+
export declare function cosineDistanceSq8(a: Buffer, aMin: number, aMax: number, b: Buffer, bMin: number, bMax: number): number
|
|
2260
|
+
|
|
2261
|
+
/**
|
|
2262
|
+
* Batch cosine distance from an SQ8 query to multiple SQ8 vectors.
|
|
2263
|
+
* Uses Rayon parallelism for large batches.
|
|
2264
|
+
*/
|
|
2265
|
+
export declare function cosineDistanceSq8Batch(query: Buffer, queryMin: number, queryMax: number, vectors: Array<Buffer>, mins: Array<number>, maxs: Array<number>): Array<number>
|
|
2266
|
+
|
|
2267
|
+
/** Calculate cosine similarity between two embedding vectors. */
|
|
2268
|
+
export declare function cosineSimilarity(a: Array<number>, b: Array<number>): number
|
|
2269
|
+
|
|
2270
|
+
/** Decode delta-varint compressed bytes back to connection list. */
|
|
2271
|
+
export declare function decodeConnections(data: Buffer): Array<number>
|
|
2272
|
+
|
|
2273
|
+
/**
|
|
2274
|
+
* **Piece 3.** Number of shard buckets at the given depth.
|
|
2275
|
+
*
|
|
2276
|
+
* - `shardDepth = 2` → `256` (16² distinct two-hex prefixes)
|
|
2277
|
+
* - `shardDepth = 3` → `16777216` (256³ three-level prefixes)
|
|
2278
|
+
*
|
|
2279
|
+
* Returns the value as a JS `number`. Returns `0` for unsupported
|
|
2280
|
+
* depths (the napi surface validates strictly elsewhere — this
|
|
2281
|
+
* accessor is loose to keep the JS-side branchless).
|
|
2282
|
+
*/
|
|
2283
|
+
export declare function deepShardBucketCount(shardDepth: number): number
|
|
2284
|
+
|
|
2285
|
+
/**
|
|
2286
|
+
* **Piece 3.** Compute the filesystem-relative path for a UUID at
|
|
2287
|
+
* the given shard depth. Returns the path as a `/`-separated
|
|
2288
|
+
* string ready to pass to the storage adapter.
|
|
2289
|
+
*
|
|
2290
|
+
* Examples:
|
|
2291
|
+
* - `layoutPath("ab123456-1234-5678-9abc-def012345678", 2)` →
|
|
2292
|
+
* `"ab/ab12345612345678 9abcdef012345678"` (single shard dir)
|
|
2293
|
+
* - `layoutPath(same, 3)` →
|
|
2294
|
+
* `"ab/12/34/ab12345612345678 9abcdef012345678"` (three shard dirs)
|
|
2295
|
+
*
|
|
2296
|
+
* Throws on invalid input — see {@link deep_shard_normalize_uuid}.
|
|
2297
|
+
*/
|
|
2298
|
+
export declare function deepShardLayoutPath(uuid: string, shardDepth: number): string
|
|
2299
|
+
|
|
2300
|
+
/**
|
|
2301
|
+
* **Piece 3.** Normalize a UUID — strip hyphens, lowercase, validate
|
|
2302
|
+
* as 32 hex chars. Mirrors brainy's `getShardIdFromUuid`
|
|
2303
|
+
* normalization so cor-side callers (verb-ID interning, future
|
|
2304
|
+
* Piece 14 migration worker) and brainy-side callers see the
|
|
2305
|
+
* identical normalized form.
|
|
2306
|
+
*
|
|
2307
|
+
* Throws on empty input, wrong length after normalization, or
|
|
2308
|
+
* non-hex characters — same error shape as
|
|
2309
|
+
* `layoutPath`/`shardPrefix`.
|
|
2310
|
+
*/
|
|
2311
|
+
export declare function deepShardNormalizeUuid(uuid: string): string
|
|
2312
|
+
|
|
2313
|
+
/**
|
|
2314
|
+
* **Piece 3.** Compute JUST the shard-prefix portion (no filename
|
|
2315
|
+
* component). Useful for `mkdir -p`-style preparation before
|
|
2316
|
+
* writing.
|
|
2317
|
+
*/
|
|
2318
|
+
export declare function deepShardPrefix(uuid: string, shardDepth: number): string
|
|
2319
|
+
|
|
2320
|
+
/**
|
|
2321
|
+
* **Piece 3.** Number of hex characters the prefix consumes —
|
|
2322
|
+
* `2` for two-level layouts, `6` for three-level.
|
|
2323
|
+
*/
|
|
2324
|
+
export declare function deepShardPrefixLen(shardDepth: number): number
|
|
2325
|
+
|
|
2326
|
+
/** Dequantize SQ4 packed bytes back to float64 vector. */
|
|
2327
|
+
export declare function dequantizeSq4(quantized: Buffer, min: number, max: number, dim: number): Array<number>
|
|
2328
|
+
|
|
2329
|
+
/** Dequantize SQ8 bytes back to float64 vector. */
|
|
2330
|
+
export declare function dequantizeSq8(quantized: Buffer, min: number, max: number): Array<number>
|
|
2331
|
+
|
|
2332
|
+
/** Deserialize SQ8 data from compact binary format. */
|
|
2333
|
+
export declare function deserializeSq8(data: Buffer): Sq8QuantizeResult
|
|
2334
|
+
|
|
2335
|
+
/** Device information returned to JavaScript */
|
|
2336
|
+
export interface DeviceInfo {
|
|
2337
|
+
/** Active device name: "cpu", "cuda", or "metal" */
|
|
2338
|
+
device: string
|
|
2339
|
+
/** Whether CUDA is available (compiled with cuda feature and GPU detected) */
|
|
2340
|
+
cudaAvailable: boolean
|
|
2341
|
+
/** Whether Metal is available (compiled with metal feature and GPU detected) */
|
|
2342
|
+
metalAvailable: boolean
|
|
2343
|
+
}
|
|
2344
|
+
|
|
2345
|
+
/**
|
|
2346
|
+
* Choice of adjacency backend at build time. The mmap variant is
|
|
2347
|
+
* required for >~100 M node builds; the RAM variant is faster for
|
|
2348
|
+
* smaller workloads.
|
|
2349
|
+
*/
|
|
2350
|
+
export interface DiskAnnAdjacencyBackend {
|
|
2351
|
+
/** `"ram"` or `"mmap"`. Defaults to `"ram"` when omitted. */
|
|
2352
|
+
kind: string
|
|
2353
|
+
/**
|
|
2354
|
+
* Required when `kind = "mmap"`: scratch file path for the build's
|
|
2355
|
+
* concurrent adjacency. Recommended location is a fast local SSD.
|
|
2356
|
+
*/
|
|
2357
|
+
mmapPath?: string
|
|
2358
|
+
}
|
|
2359
|
+
|
|
2360
|
+
/**
|
|
2361
|
+
* Top-level build config bundle. Any field can be omitted; defaults
|
|
2362
|
+
* are filled in to match the DiskANN paper's published parameters.
|
|
2363
|
+
*
|
|
2364
|
+
* ## Operating modes
|
|
2365
|
+
*
|
|
2366
|
+
* `mode` selects which DiskANN flavour to build:
|
|
2367
|
+
*
|
|
2368
|
+
* - `"in-memory"` — **Mode 1**: skip PQ entirely. The codebook +
|
|
2369
|
+
* codes sections collapse to zero bytes; search walks the graph
|
|
2370
|
+
* using exact full-vector distance. RAM-resident vectors give
|
|
2371
|
+
* sub-ms latency. Pieces G of the cor 3.0 plan.
|
|
2372
|
+
* - `"hybrid"` (default) — **Mode 2**: PQ codes in RAM, full vectors
|
|
2373
|
+
* mmap'd, PQ-greedy walk with full-vector re-rank. Today's
|
|
2374
|
+
* default behaviour.
|
|
2375
|
+
* - `"on-disk"` — **Mode 3**: same as hybrid but with explicit
|
|
2376
|
+
* madvise hints biased toward SSD-resident vectors. Piece H.
|
|
2377
|
+
*
|
|
2378
|
+
* When `mode = "in-memory"`, the `pq` and `adjacency` fields are
|
|
2379
|
+
* ignored — Mode 1 forces no PQ and RAM-resident build adjacency.
|
|
2380
|
+
*/
|
|
2381
|
+
export interface DiskAnnConfig {
|
|
2382
|
+
vamana?: DiskAnnVamanaConfig
|
|
2383
|
+
pq?: DiskAnnPqConfig
|
|
2384
|
+
adjacency?: DiskAnnAdjacencyBackend
|
|
2385
|
+
/**
|
|
2386
|
+
* `"in-memory" | "hybrid" | "on-disk"`. Defaults to `"hybrid"`
|
|
2387
|
+
* (today's behaviour) when omitted.
|
|
2388
|
+
*/
|
|
2389
|
+
mode?: string
|
|
2390
|
+
}
|
|
2391
|
+
|
|
2392
|
+
/**
|
|
2393
|
+
* Header introspection — exposes the on-disk metadata so JS can
|
|
2394
|
+
* reason about index shape (dim, node count, PQ params).
|
|
2395
|
+
*/
|
|
2396
|
+
export interface DiskAnnHeader {
|
|
2397
|
+
version: number
|
|
2398
|
+
dim: number
|
|
2399
|
+
nodeCount: number
|
|
2400
|
+
pqM: number
|
|
2401
|
+
pqKsub: number
|
|
2402
|
+
pqDsub: number
|
|
2403
|
+
maxDegree: number
|
|
2404
|
+
entryPoint: number
|
|
2405
|
+
}
|
|
2406
|
+
|
|
2407
|
+
/**
|
|
2408
|
+
* Single search hit returned to JS.
|
|
2409
|
+
*
|
|
2410
|
+
* IMPORTANT: `slot` is a **per-shard graph-internal slot ID**, NOT a
|
|
2411
|
+
* brain-wide entity ID. Slots are NOT stable across rebuilds; the
|
|
2412
|
+
* TS wrapper translates `slot ↔ entity-id` through the brain's
|
|
2413
|
+
* `BinaryIdMapper` (see the module-level doc + Step 14 commit for
|
|
2414
|
+
* the layering rationale). Slot stays `u32` in both `IdSpace::U32`
|
|
2415
|
+
* and `IdSpace::U64` brains.
|
|
2416
|
+
*/
|
|
2417
|
+
export interface DiskAnnHit {
|
|
2418
|
+
/**
|
|
2419
|
+
* **Per-shard graph-internal slot ID** — NOT an entity ID. Translate
|
|
2420
|
+
* through `BinaryIdMapper` in the TS wrapper before exposing to
|
|
2421
|
+
* brainy / consumer code. Stays `u32` even in U64 brains because
|
|
2422
|
+
* a single DiskANN shard addresses at most `u32::MAX − 1` nodes
|
|
2423
|
+
* by construction.
|
|
2424
|
+
*/
|
|
2425
|
+
slot: number
|
|
2426
|
+
/** Exact (full-vector) squared distance to the query. */
|
|
2427
|
+
distance: number
|
|
2428
|
+
/**
|
|
2429
|
+
* **#72 Phase A** — the brain-wide **entity int** for this hit,
|
|
2430
|
+
* resolved O(1) from the `main.slotmap` sidecar (or `slot` itself
|
|
2431
|
+
* when the searcher was opened without a slotmap). The TS wrapper
|
|
2432
|
+
* resolves this int → UUID through the canonical id-mapper, so it
|
|
2433
|
+
* never needs a resident `slot → uuid` map. BigInt so it stays
|
|
2434
|
+
* exact above `u32::MAX` in U64 brains.
|
|
2435
|
+
*/
|
|
2436
|
+
id: bigint
|
|
2437
|
+
}
|
|
2438
|
+
|
|
2439
|
+
/**
|
|
2440
|
+
* PQ knobs exposed to JS. Sensible defaults are filled in by
|
|
2441
|
+
* [`NativeDiskANN.build`] when fields are absent.
|
|
2442
|
+
*/
|
|
2443
|
+
export interface DiskAnnPqConfig {
|
|
2444
|
+
/**
|
|
2445
|
+
* Number of PQ subspaces. dim must be divisible by m. **`0` =
|
|
2446
|
+
* auto-derive from the observed dimension** (target dsub ≈ 8, via
|
|
2447
|
+
* `pqDeriveM`) — the zero-config default the wrapper sends. Any
|
|
2448
|
+
* non-zero value is an explicit pin and is used verbatim.
|
|
2449
|
+
*/
|
|
2450
|
+
m: number
|
|
2451
|
+
/** Centroids per subspace. 256 = 8-bit codes (the standard). Default 256. */
|
|
2452
|
+
ksub: number
|
|
2453
|
+
/** k-means iterations during codebook training. Default 25. */
|
|
2454
|
+
iterations: number
|
|
2455
|
+
/** How many training vectors to sample. 0 = use all of them. Default 200_000. */
|
|
2456
|
+
trainingSample: number
|
|
2457
|
+
}
|
|
2458
|
+
|
|
2459
|
+
/**
|
|
2460
|
+
* Parameters for [`NativeDiskANN::rebuild_from_existing`]. Lets the TS
|
|
2461
|
+
* wrapper drive an in-place rebuild without ever copying the existing
|
|
2462
|
+
* vector dataset across the FFI boundary.
|
|
2463
|
+
*
|
|
2464
|
+
* (No `Clone`/`Debug` derives because `napi::Buffer` doesn't implement
|
|
2465
|
+
* them — the napi runtime owns the buffer's lifetime, not Rust.)
|
|
2466
|
+
*/
|
|
2467
|
+
export interface DiskAnnRebuildFromExistingParams {
|
|
2468
|
+
/**
|
|
2469
|
+
* Path to the prior index on disk. Omit (or pass `None`) for a
|
|
2470
|
+
* delta-only first-time build via this path.
|
|
2471
|
+
*/
|
|
2472
|
+
existingPath?: string
|
|
2473
|
+
/**
|
|
2474
|
+
* **Graph-internal slot IDs** (NOT entity IDs — see the module
|
|
2475
|
+
* doc) in the existing index to retain. Caller filters out
|
|
2476
|
+
* tombstoned slots; this list IS the live set. Each entry must
|
|
2477
|
+
* be `< existing.header().node_count`. Empty when rebuilding
|
|
2478
|
+
* from delta only.
|
|
2479
|
+
*/
|
|
2480
|
+
liveOldSlots: Array<number>
|
|
2481
|
+
/**
|
|
2482
|
+
* Flat f32 buffer of newly-inserted vectors,
|
|
2483
|
+
* `delta_count × dim × 4` bytes LE. Omit when there are no new
|
|
2484
|
+
* inserts (compaction-only rebuild).
|
|
2485
|
+
*/
|
|
2486
|
+
deltaVectors?: Buffer
|
|
2487
|
+
/**
|
|
2488
|
+
* Number of vectors in `delta_vectors`. Must be `0` when
|
|
2489
|
+
* `delta_vectors` is omitted.
|
|
2490
|
+
*/
|
|
2491
|
+
deltaCount: number
|
|
2492
|
+
/**
|
|
2493
|
+
* Vector dimensionality. Must match the existing index when one
|
|
2494
|
+
* is provided.
|
|
2495
|
+
*/
|
|
2496
|
+
dim: number
|
|
2497
|
+
/**
|
|
2498
|
+
* Where to write the new index. May equal `existing_path` for
|
|
2499
|
+
* in-place rebuild — the binding writes to a temp file and
|
|
2500
|
+
* atomic-renames over the original.
|
|
2501
|
+
*/
|
|
2502
|
+
outputPath: string
|
|
2503
|
+
/** Build knobs. Defaults match the DiskANN paper when omitted. */
|
|
2504
|
+
cfg?: DiskAnnConfig
|
|
2505
|
+
/**
|
|
2506
|
+
* **#72 Phase A+B** — per-slot entity ints for the new index, as a
|
|
2507
|
+
* flat little-endian `u64` buffer of exactly `node_count × 8` bytes
|
|
2508
|
+
* (`node_count = live_old_slots.len() + delta_count`). Element `i`
|
|
2509
|
+
* is the brain-wide entity int of new slot `i`. When present, the
|
|
2510
|
+
* binding writes the `main.slotmap` (slot → int) + `main.slotrev`
|
|
2511
|
+
* (int → slot) sidecars atomically with the new index, so the TS
|
|
2512
|
+
* wrapper never keeps a resident `slot ↔ uuid` map. Omit only for
|
|
2513
|
+
* the slot-only benchmark paths that don't carry entity ids.
|
|
2514
|
+
*/
|
|
2515
|
+
slotIds?: Buffer
|
|
2516
|
+
}
|
|
2517
|
+
|
|
2518
|
+
/** Vamana graph build knobs exposed to JS. */
|
|
2519
|
+
export interface DiskAnnVamanaConfig {
|
|
2520
|
+
/** Max out-degree per node (R). Default 64. */
|
|
2521
|
+
maxDegree: number
|
|
2522
|
+
/** Candidate list size during build (L). Default 100. */
|
|
2523
|
+
searchListSize: number
|
|
2524
|
+
/** Density parameter for the second pass. Default 1.2. */
|
|
2525
|
+
alpha: number
|
|
2526
|
+
/** RNG seed for deterministic builds. */
|
|
2527
|
+
seed: bigint
|
|
2528
|
+
/** Build with rayon parallelism. Default true. */
|
|
2529
|
+
parallel: boolean
|
|
2530
|
+
/** Nodes per rayon batch. Default 64. */
|
|
2531
|
+
parallelBatch: number
|
|
2532
|
+
}
|
|
2533
|
+
|
|
2534
|
+
/**
|
|
2535
|
+
* Dot product distance between two vectors.
|
|
2536
|
+
* Converted to distance metric (lower is better): returns -dot_product.
|
|
2537
|
+
*/
|
|
2538
|
+
export declare function dotProductDistance(a: Array<number>, b: Array<number>): number
|
|
2539
|
+
|
|
2540
|
+
/**
|
|
2541
|
+
* Encode a connection list (neighbor IDs) as delta-varint compressed bytes.
|
|
2542
|
+
*
|
|
2543
|
+
* 1. Sort the IDs
|
|
2544
|
+
* 2. Compute deltas between consecutive values
|
|
2545
|
+
* 3. Encode each delta as a variable-length integer (LEB128)
|
|
2546
|
+
*
|
|
2547
|
+
* Achieves 2-3x compression for typical HNSW neighbor lists.
|
|
2548
|
+
*/
|
|
2549
|
+
export declare function encodeConnections(ids: Array<number>): Buffer
|
|
2550
|
+
|
|
2551
|
+
/** Engine statistics returned to JavaScript. */
|
|
2552
|
+
export interface EngineStats {
|
|
2553
|
+
initialized: boolean
|
|
2554
|
+
embedCount: number
|
|
2555
|
+
totalProcessingTimeMs: number
|
|
2556
|
+
avgProcessingTimeMs: number
|
|
2557
|
+
modelName: string
|
|
2558
|
+
}
|
|
2559
|
+
|
|
2560
|
+
/**
|
|
2561
|
+
* **Piece 4.** Standalone sort-latency estimator. Returns the
|
|
2562
|
+
* estimated microseconds to sort `n` items using the documented
|
|
2563
|
+
* per-comparison constant. Useful for ad-hoc cost queries from
|
|
2564
|
+
* brainy (e.g. "should I sort this list of N candidates before
|
|
2565
|
+
* issuing a deep-shard read?").
|
|
2566
|
+
*
|
|
2567
|
+
* Returns `0` for `n` of 0 or 1.
|
|
2568
|
+
*/
|
|
2569
|
+
export declare function estimateSortLatencyUs(n: number): number
|
|
2570
|
+
|
|
2571
|
+
/**
|
|
2572
|
+
* Euclidean distance between two vectors.
|
|
2573
|
+
* Lower values indicate higher similarity.
|
|
2574
|
+
*/
|
|
2575
|
+
export declare function euclideanDistance(a: Array<number>, b: Array<number>): number
|
|
2576
|
+
|
|
2577
|
+
/** Batch euclidean distance: compute distances from a query to multiple vectors. */
|
|
2578
|
+
export declare function euclideanDistanceBatch(query: Array<number>, vectors: Array<Array<number>>): Array<number>
|
|
2579
|
+
|
|
2580
|
+
/**
|
|
2581
|
+
* Outcome of raising the open-file limit. Field names cross to JS as
|
|
2582
|
+
* camelCase (`softBefore`, `softAfter`, `hard`, `raised`).
|
|
2583
|
+
*/
|
|
2584
|
+
export interface FileLimitInfo {
|
|
2585
|
+
/** Soft `RLIMIT_NOFILE` before the call. */
|
|
2586
|
+
softBefore: number
|
|
2587
|
+
/** Soft `RLIMIT_NOFILE` after the call (== `hard` on success). */
|
|
2588
|
+
softAfter: number
|
|
2589
|
+
/**
|
|
2590
|
+
* Hard cap — the ceiling this process cannot exceed without the
|
|
2591
|
+
* deployment raising it.
|
|
2592
|
+
*/
|
|
2593
|
+
hard: number
|
|
2594
|
+
/** Whether the soft limit was actually raised. */
|
|
2595
|
+
raised: boolean
|
|
2596
|
+
/**
|
|
2597
|
+
* Linux `vm.max_map_count` (max mmap regions per process), read from
|
|
2598
|
+
* `/proc/sys/vm/max_map_count`. A process CANNOT raise this itself
|
|
2599
|
+
* (global sysctl) — it must be set by the deployment. `0` if not
|
|
2600
|
+
* readable (non-Linux). At 1000+ small brains, each mmapping its
|
|
2601
|
+
* SSTables/index/id-mappers, this can bind as brains scale up.
|
|
2602
|
+
*/
|
|
2603
|
+
vmMaxMapCount: number
|
|
2604
|
+
}
|
|
2605
|
+
|
|
2606
|
+
/** Get information about the active compute device. */
|
|
2607
|
+
export declare function getDeviceInfo(): DeviceInfo
|
|
2608
|
+
|
|
2609
|
+
/**
|
|
2610
|
+
* Result from addVerb — indicates which trees need flushing
|
|
2611
|
+
* and (cor 3.0 brainy-8.0 lockstep) returns the verb's
|
|
2612
|
+
* interned `u64` integer ID.
|
|
2613
|
+
*/
|
|
2614
|
+
export interface GraphAddVerbResult {
|
|
2615
|
+
/** Whether the source tree MemTable needs flush */
|
|
2616
|
+
needsFlushSource: boolean
|
|
2617
|
+
/** Whether the target tree MemTable needs flush */
|
|
2618
|
+
needsFlushTarget: boolean
|
|
2619
|
+
/** Whether the verbs-source tree MemTable needs flush */
|
|
2620
|
+
needsFlushVerbsSource: boolean
|
|
2621
|
+
/** Whether the verbs-target tree MemTable needs flush */
|
|
2622
|
+
needsFlushVerbsTarget: boolean
|
|
2623
|
+
/**
|
|
2624
|
+
* **Brainy 8.0 lockstep** (PLATFORM-HANDOFF
|
|
2625
|
+
* BRAINY-8.0-RENAME-COORDINATION A.3): the verb's interned
|
|
2626
|
+
* `u64` integer ID, returned here so brainy 8.0's sidecar
|
|
2627
|
+
* `verb_int ↔ verb_id_string` map can be populated without
|
|
2628
|
+
* the prior extra FFI round-trip through
|
|
2629
|
+
* `getVerbIntForString`. Saves one boundary crossing per
|
|
2630
|
+
* add at billion-scale ingest.
|
|
2631
|
+
*/
|
|
2632
|
+
verbInt: bigint
|
|
2633
|
+
}
|
|
2634
|
+
|
|
2635
|
+
/** Configuration for NativeGraphAdjacencyIndex */
|
|
2636
|
+
export interface GraphAdjacencyConfig {
|
|
2637
|
+
/** MemTable flush threshold per tree. Default: 100_000 */
|
|
2638
|
+
memTableThreshold?: number
|
|
2639
|
+
/** Max SSTables per level per tree. Default: 10 */
|
|
2640
|
+
maxSstablesPerLevel?: number
|
|
2641
|
+
/**
|
|
2642
|
+
* **Piece 12.** Optional directory for the verb-ID interning
|
|
2643
|
+
* namespace's mmap-backed files (`verb-uuid-to-int.mkv` +
|
|
2644
|
+
* `verb-int-to-uuid.bin`). When set, verb-ID assignments survive
|
|
2645
|
+
* process restart and the namespace shares the directory with
|
|
2646
|
+
* any other artifacts the wrapper persists there. When omitted,
|
|
2647
|
+
* the namespace falls back to a 64-shard in-memory interner —
|
|
2648
|
+
* adequate for tests + transient brains, but tombstones are lost
|
|
2649
|
+
* on exit. Production callers always provide this.
|
|
2650
|
+
*/
|
|
2651
|
+
persistenceDir?: string
|
|
2652
|
+
/**
|
|
2653
|
+
* **Piece B (cor 3.0).** Wire width of the verb-ID interning
|
|
2654
|
+
* namespace + the live-verb / per-(type, subtype) membership
|
|
2655
|
+
* bitmaps. `"U32"` (default, cortex 2.x-compatible) caps at
|
|
2656
|
+
* ~4.29 B unique verb-IDs and uses Roaring32 bitmaps. `"U64"`
|
|
2657
|
+
* (cor 3.0 widening) lifts the cap to `u64::MAX - 1` and
|
|
2658
|
+
* uses Treemap (Roaring64) bitmaps.
|
|
2659
|
+
*
|
|
2660
|
+
* The choice is independent of the brain's *entity* IdSpace —
|
|
2661
|
+
* a U32-entity brain can run U64-verb if its edge-to-noun
|
|
2662
|
+
* fan-out is unusually high, and vice versa.
|
|
2663
|
+
*
|
|
2664
|
+
* **Persistence contract**: when `persistence_dir` is set, the
|
|
2665
|
+
* chosen width is stamped into the underlying `BinaryIdMapper`'s
|
|
2666
|
+
* file header and is immutable post-create. Re-opening a
|
|
2667
|
+
* directory with a mismatched value raises an error rather than
|
|
2668
|
+
* silently mis-reading the bitmap geometry — wire-width
|
|
2669
|
+
* migrations need the offline migrator (Piece E).
|
|
2670
|
+
*
|
|
2671
|
+
* Accepted values: `"U32"`, `"U64"` (case-sensitive). Anything
|
|
2672
|
+
* else raises a config error at construction.
|
|
2673
|
+
*/
|
|
2674
|
+
verbIdSpace?: string
|
|
2675
|
+
}
|
|
2676
|
+
|
|
2677
|
+
/**
|
|
2678
|
+
* Statistics for the graph adjacency index.
|
|
2679
|
+
*
|
|
2680
|
+
* The legacy u32 counters (`total_relationships`, `verb_id_count`,
|
|
2681
|
+
* `relationship_type_count`) are preserved for cortex 2.x-compatible
|
|
2682
|
+
* callers — they narrow internally via `as u32` and cap at
|
|
2683
|
+
* [`u32::MAX`]. New BigInt sibling fields (`*_big`) carry the same
|
|
2684
|
+
* values losslessly at billion scale and should be preferred by any
|
|
2685
|
+
* brainy 8.0 / cor 3.0 consumer.
|
|
2686
|
+
*
|
|
2687
|
+
* Per-tree fields (`source_mem_table_size`, `*_sstable_count`,
|
|
2688
|
+
* `*_mem_table_memory`) stay u32 because they are bounded by the
|
|
2689
|
+
* LSM flush threshold (default 100 K entries) — they cannot reach
|
|
2690
|
+
* the u32 ceiling by construction.
|
|
2691
|
+
*/
|
|
2692
|
+
export interface GraphAdjacencyStats {
|
|
2693
|
+
/**
|
|
2694
|
+
* Total relationships in source tree (narrowed u32 — see
|
|
2695
|
+
* `total_relationships_big` for the BigInt sibling).
|
|
2696
|
+
*/
|
|
2697
|
+
totalRelationships: number
|
|
2698
|
+
/**
|
|
2699
|
+
* Total verb IDs tracked (narrowed u32 — see
|
|
2700
|
+
* `verb_id_count_big`).
|
|
2701
|
+
*/
|
|
2702
|
+
verbIdCount: number
|
|
2703
|
+
/**
|
|
2704
|
+
* Number of relationship types (narrowed u32 — see
|
|
2705
|
+
* `relationship_type_count_big`).
|
|
2706
|
+
*/
|
|
2707
|
+
relationshipTypeCount: number
|
|
2708
|
+
/** Source tree MemTable size */
|
|
2709
|
+
sourceMemTableSize: number
|
|
2710
|
+
/** Target tree MemTable size */
|
|
2711
|
+
targetMemTableSize: number
|
|
2712
|
+
/** Source tree SSTable count */
|
|
2713
|
+
sourceSstableCount: number
|
|
2714
|
+
/** Target tree SSTable count */
|
|
2715
|
+
targetSstableCount: number
|
|
2716
|
+
/** Source tree MemTable memory estimate */
|
|
2717
|
+
sourceMemTableMemory: number
|
|
2718
|
+
/** Target tree MemTable memory estimate */
|
|
2719
|
+
targetMemTableMemory: number
|
|
2720
|
+
/**
|
|
2721
|
+
* Total relationships in source tree as a `BigInt`. Use this
|
|
2722
|
+
* field instead of `total_relationships` at billion scale.
|
|
2723
|
+
*/
|
|
2724
|
+
totalRelationshipsBig: bigint
|
|
2725
|
+
/** Total verb IDs tracked as a `BigInt`. */
|
|
2726
|
+
verbIdCountBig: bigint
|
|
2727
|
+
/** Number of relationship types as a `BigInt`. */
|
|
2728
|
+
relationshipTypeCountBig: bigint
|
|
2729
|
+
}
|
|
2730
|
+
|
|
2731
|
+
/**
|
|
2732
|
+
* Options for opening a streaming graph cursor
|
|
2733
|
+
* ([`graph_cursor_open`](NativeGraphAdjacencyIndex::graph_cursor_open)).
|
|
2734
|
+
* Mirrors brainy's `GraphCursorOptions` (+ the trailing `generation`),
|
|
2735
|
+
* flattened so the seed set can arrive either as explicit ints or as
|
|
2736
|
+
* cor's opaque id-set envelope (the `find()`→export fusion).
|
|
2737
|
+
*/
|
|
2738
|
+
export interface GraphCursorOpenOptions {
|
|
2739
|
+
/**
|
|
2740
|
+
* `"out"`, `"in"`, or `"both"` (default). Honored for a **seeded**
|
|
2741
|
+
* cursor; ignored for a whole-graph walk, which emits every edge
|
|
2742
|
+
* exactly once via its source's out-list regardless.
|
|
2743
|
+
*/
|
|
2744
|
+
direction?: string
|
|
2745
|
+
/**
|
|
2746
|
+
* `"light"` (default) omits the `edge_verb_ints` column — topology
|
|
2747
|
+
* + edge-type only, the viz default. `"full"` includes it (brainy
|
|
2748
|
+
* resolves verb-id strings from the ints lazily).
|
|
2749
|
+
*/
|
|
2750
|
+
projection?: string
|
|
2751
|
+
/**
|
|
2752
|
+
* Seed set as explicit entity ints. Omitted **and** `seeds_opaque`
|
|
2753
|
+
* omitted = a whole-graph walk. Mutually exclusive with
|
|
2754
|
+
* `seeds_opaque` (if both are set, `seeds_opaque` wins).
|
|
2755
|
+
*/
|
|
2756
|
+
seeds?: Array<bigint>
|
|
2757
|
+
/**
|
|
2758
|
+
* Seed set as cor's [`OpaqueIdSet`](crate::opaque_id_set) envelope
|
|
2759
|
+
* — a `find()` universe forwarded with no id materialization.
|
|
2760
|
+
*/
|
|
2761
|
+
seedsOpaque?: Buffer
|
|
2762
|
+
/**
|
|
2763
|
+
* Resume token from a prior chunk — reopen after a `SnapshotExpired`
|
|
2764
|
+
* to continue the walk from where it left off (at the same pinned
|
|
2765
|
+
* generation). Overrides `generation` and the seed/whole-graph mode.
|
|
2766
|
+
*/
|
|
2767
|
+
cursor?: string
|
|
2768
|
+
/**
|
|
2769
|
+
* Explicit as-of generation to pin. Omitted = pin the current
|
|
2770
|
+
* generation at open.
|
|
2771
|
+
*/
|
|
2772
|
+
generation?: bigint
|
|
2773
|
+
}
|
|
2774
|
+
|
|
2775
|
+
/**
|
|
2776
|
+
* Fallback noun-type index (`NounType.Thing`, 3). Used when an unknown
|
|
2777
|
+
* string is coerced via the loose lookup.
|
|
2778
|
+
*/
|
|
2779
|
+
export declare function graphNounFallbackIndex(): number
|
|
2780
|
+
|
|
2781
|
+
/** Strict noun-index → string lookup. Returns `null` on out-of-range. */
|
|
2782
|
+
export declare function graphNounFromIndex(i: number): string | null
|
|
2783
|
+
|
|
2784
|
+
/** Index → string with brainy-parity fallback to `"thing"`. */
|
|
2785
|
+
export declare function graphNounFromIndexOrFallback(i: number): string
|
|
2786
|
+
|
|
2787
|
+
/**
|
|
2788
|
+
* Strict noun-string → index lookup. Returns `null` on unknown
|
|
2789
|
+
* strings — caller decides whether to surface a vocabulary violation
|
|
2790
|
+
* or coerce. Mirror of brainy's `TypeUtils.getNounIndex` minus the
|
|
2791
|
+
* silent fallback.
|
|
2792
|
+
*/
|
|
2793
|
+
export declare function graphNounIndex(s: string): number | null
|
|
2794
|
+
|
|
2795
|
+
/**
|
|
2796
|
+
* Lookup with brainy-parity fallback to `NounType.Thing` (3). Mirrors
|
|
2797
|
+
* `TypeUtils.getNounIndex`'s runtime behavior.
|
|
2798
|
+
*/
|
|
2799
|
+
export declare function graphNounIndexOrFallback(s: string): number
|
|
2800
|
+
|
|
2801
|
+
/** Number of standard noun types (42). Mirrors brainy's `NOUN_TYPE_COUNT`. */
|
|
2802
|
+
export declare function graphNounTypeCount(): number
|
|
2803
|
+
|
|
2804
|
+
/** Fallback verb-type index (`VerbType.RelatedTo`, 3). */
|
|
2805
|
+
export declare function graphVerbFallbackIndex(): number
|
|
2806
|
+
|
|
2807
|
+
/** Strict verb-index → string lookup. Returns `null` on out-of-range. */
|
|
2808
|
+
export declare function graphVerbFromIndex(i: number): string | null
|
|
2809
|
+
|
|
2810
|
+
/** Index → string with brainy-parity fallback to `"relatedTo"`. */
|
|
2811
|
+
export declare function graphVerbFromIndexOrFallback(i: number): string
|
|
2812
|
+
|
|
2813
|
+
/** Strict verb-string → index lookup. Returns `null` on unknown. */
|
|
2814
|
+
export declare function graphVerbIndex(s: string): number | null
|
|
2815
|
+
|
|
2816
|
+
/** Lookup with brainy-parity fallback to `VerbType.RelatedTo` (3). */
|
|
2817
|
+
export declare function graphVerbIndexOrFallback(s: string): number
|
|
2818
|
+
|
|
2819
|
+
/** Number of standard verb types (127). Mirrors brainy's `VERB_TYPE_COUNT`. */
|
|
2820
|
+
export declare function graphVerbTypeCount(): number
|
|
2821
|
+
|
|
2822
|
+
/**
|
|
2823
|
+
* **Piece 4.** Migration-recommended threshold (microseconds). Echoed
|
|
2824
|
+
* so wrappers + monitoring don't have to hardcode the value
|
|
2825
|
+
* independently.
|
|
2826
|
+
*/
|
|
2827
|
+
export declare function hydrationMigrationThresholdUs(): number
|
|
2828
|
+
|
|
2829
|
+
/**
|
|
2830
|
+
* **Piece 4.** Per-comparison nanosecond cost constant used by the
|
|
2831
|
+
* sort estimate. Exposed for observability (so wrappers can
|
|
2832
|
+
* surface the assumption in dashboards) — NOT a runtime knob.
|
|
2833
|
+
*/
|
|
2834
|
+
export declare function hydrationNsPerCompare(): number
|
|
2835
|
+
|
|
2836
|
+
/**
|
|
2837
|
+
* Top-level hydration plan, JS-friendly form. See
|
|
2838
|
+
* [`crate::io_batch_entities::HydrationPlan`].
|
|
2839
|
+
*/
|
|
2840
|
+
export interface HydrationPlan {
|
|
2841
|
+
/** Echo of the input shard depth. */
|
|
2842
|
+
shardDepth: number
|
|
2843
|
+
/** Echo of `uuids.len()`. */
|
|
2844
|
+
totalUuidCount: number
|
|
2845
|
+
/**
|
|
2846
|
+
* Per-shard breakdown. Vec ordering is unspecified — the
|
|
2847
|
+
* wrapper iterates to issue reads in whatever order makes
|
|
2848
|
+
* sense for its workload.
|
|
2849
|
+
*/
|
|
2850
|
+
shards: Array<ShardHydrationPlan>
|
|
2851
|
+
/** Highest single-shard estimated sort latency, in microseconds. */
|
|
2852
|
+
maxShardSortUs: number
|
|
2853
|
+
/**
|
|
2854
|
+
* Threshold (in microseconds) the wrapper compared against —
|
|
2855
|
+
* echoed so JS-side logs can include it without a separate
|
|
2856
|
+
* lookup.
|
|
2857
|
+
*/
|
|
2858
|
+
migrationThresholdUs: number
|
|
2859
|
+
/**
|
|
2860
|
+
* `true` when `maxShardSortUs > migrationThresholdUs`. The
|
|
2861
|
+
* wrapper passes this hint to Piece 14's migration worker
|
|
2862
|
+
* asynchronously; it does NOT block the current query.
|
|
2863
|
+
*/
|
|
2864
|
+
migrationRecommended: boolean
|
|
2865
|
+
}
|
|
2866
|
+
|
|
2867
|
+
export interface LicenseResult {
|
|
2868
|
+
valid: boolean
|
|
2869
|
+
tier: string
|
|
2870
|
+
email: string
|
|
2871
|
+
expiresAt: number
|
|
2872
|
+
error?: string
|
|
2873
|
+
}
|
|
2874
|
+
|
|
2875
|
+
/** JSON shape for a directory entry on the napi boundary. */
|
|
2876
|
+
export interface LsmDirectoryEntryJson {
|
|
2877
|
+
fieldLen: number
|
|
2878
|
+
valueLen: number
|
|
2879
|
+
fieldOffset: number
|
|
2880
|
+
valueOffset: number
|
|
2881
|
+
payloadOffset: bigint
|
|
2882
|
+
payloadLen: bigint
|
|
2883
|
+
epoch: bigint
|
|
2884
|
+
flags: number
|
|
2885
|
+
}
|
|
2886
|
+
|
|
2887
|
+
/**
|
|
2888
|
+
* Assemble a full SSTable byte buffer from structured inputs. Returns
|
|
2889
|
+
* the bytes ready to be hashed + asserted against a golden SHA-256.
|
|
2890
|
+
*
|
|
2891
|
+
* The directory entries' `field_offset` / `value_offset` are computed
|
|
2892
|
+
* automatically as cumulative offsets into the keys section. The
|
|
2893
|
+
* `payload_offset` / `payload_len` are computed from the cumulative
|
|
2894
|
+
* per-entry payload sizes. The header's SHA-256 is computed over the
|
|
2895
|
+
* concatenation of directory + keys + payload + bloom sections.
|
|
2896
|
+
*/
|
|
2897
|
+
export declare function lsmFormatBuildSstable(spec: LsmSstableBuildSpec): Buffer
|
|
2898
|
+
|
|
2899
|
+
/** Parse a 40-byte directory entry. */
|
|
2900
|
+
export declare function lsmFormatDecodeDirectoryEntry(bytes: Buffer): LsmDirectoryEntryJson
|
|
2901
|
+
|
|
2902
|
+
/** Parse a 128-byte buffer into structured header fields. */
|
|
2903
|
+
export declare function lsmFormatDecodeHeader(bytes: Buffer): LsmHeaderJson
|
|
2904
|
+
|
|
2905
|
+
/**
|
|
2906
|
+
* Inverse of `lsmFormatEncodePayload`. Splits a payload into its
|
|
2907
|
+
* additions + deletions bitmap byte slices.
|
|
2908
|
+
*/
|
|
2909
|
+
export declare function lsmFormatDecodePayload(bytes: Buffer): LsmPayloadParts
|
|
2910
|
+
|
|
2911
|
+
/** Build a 40-byte directory entry buffer. */
|
|
2912
|
+
export declare function lsmFormatEncodeDirectoryEntry(spec: LsmDirectoryEntryJson): Buffer
|
|
2913
|
+
|
|
2914
|
+
/** Build a 128-byte SSTable header buffer from structured inputs. */
|
|
2915
|
+
export declare function lsmFormatEncodeHeader(spec: LsmHeaderJson): Buffer
|
|
2916
|
+
|
|
2917
|
+
/**
|
|
2918
|
+
* Encode a per-entry payload: `additions_len:u64 LE | additions bytes
|
|
2919
|
+
* | deletions_len:u64 LE | deletions bytes`.
|
|
2920
|
+
*/
|
|
2921
|
+
export declare function lsmFormatEncodePayload(additions: Buffer, deletions: Buffer): Buffer
|
|
2922
|
+
|
|
2923
|
+
/**
|
|
2924
|
+
* Parse a full SSTable byte buffer. Verifies the SHA-256 implicitly
|
|
2925
|
+
* by returning the computed hash alongside the stored hash; callers
|
|
2926
|
+
* (golden tests) assert equality.
|
|
2927
|
+
*/
|
|
2928
|
+
export declare function lsmFormatParseSstable(bytes: Buffer): LsmSstableParseResult
|
|
2929
|
+
|
|
2930
|
+
/** JSON shape for an LSM header on the napi boundary. */
|
|
2931
|
+
export interface LsmHeaderJson {
|
|
2932
|
+
/** `"u32"` or `"u64"`. */
|
|
2933
|
+
idSpace: string
|
|
2934
|
+
/** Defaults to `0`. */
|
|
2935
|
+
headerFlags: number
|
|
2936
|
+
entryCount: bigint
|
|
2937
|
+
keysSize: bigint
|
|
2938
|
+
payloadSize: bigint
|
|
2939
|
+
bloomSize: bigint
|
|
2940
|
+
minEpoch: bigint
|
|
2941
|
+
maxEpoch: bigint
|
|
2942
|
+
/** Lowercase hex (64 chars). */
|
|
2943
|
+
sha256Hex: string
|
|
2944
|
+
}
|
|
2945
|
+
|
|
2946
|
+
/**
|
|
2947
|
+
* Decoded payload split (used as the napi return of
|
|
2948
|
+
* [`lsm_format_decode_payload`]).
|
|
2949
|
+
*/
|
|
2950
|
+
export interface LsmPayloadParts {
|
|
2951
|
+
additions: Buffer
|
|
2952
|
+
deletions: Buffer
|
|
2953
|
+
}
|
|
2954
|
+
|
|
2955
|
+
/**
|
|
2956
|
+
* Input for [`lsm_format_build_sstable`]. Each entry's `(field, value)`
|
|
2957
|
+
* is appended to a flat keys section in directory order; the
|
|
2958
|
+
* `field_offset` and `value_offset` are computed automatically so the
|
|
2959
|
+
* caller doesn't have to track them.
|
|
2960
|
+
*/
|
|
2961
|
+
export interface LsmSstableBuildEntry {
|
|
2962
|
+
field: string
|
|
2963
|
+
value: string
|
|
2964
|
+
additions: Buffer
|
|
2965
|
+
deletions: Buffer
|
|
2966
|
+
epoch: bigint
|
|
2967
|
+
/**
|
|
2968
|
+
* Set to `true` to mark this entry as a posting-list-wide
|
|
2969
|
+
* tombstone (sets the `FLAG_ENTRY_TOMBSTONE` bit). `additions`
|
|
2970
|
+
* should be empty in this case; the deletion bitmap carries the
|
|
2971
|
+
* removed ints.
|
|
2972
|
+
*/
|
|
2973
|
+
tombstone: boolean
|
|
2974
|
+
}
|
|
2975
|
+
|
|
2976
|
+
/**
|
|
2977
|
+
* Inputs for assembling a whole SSTable. Entries are NOT pre-sorted by
|
|
2978
|
+
* this helper — golden fixtures must pass them already in
|
|
2979
|
+
* `(field, value)` order so the on-disk layout is deterministic.
|
|
2980
|
+
*/
|
|
2981
|
+
export interface LsmSstableBuildSpec {
|
|
2982
|
+
idSpace: string
|
|
2983
|
+
entries: Array<LsmSstableBuildEntry>
|
|
2984
|
+
/**
|
|
2985
|
+
* Optional bloom-filter bytes — appended verbatim after the
|
|
2986
|
+
* payload section. Pass an empty Buffer to skip the bloom section
|
|
2987
|
+
* entirely (`bloom_size = 0`).
|
|
2988
|
+
*/
|
|
2989
|
+
bloom: Buffer
|
|
2990
|
+
minEpoch: bigint
|
|
2991
|
+
maxEpoch: bigint
|
|
2992
|
+
}
|
|
2993
|
+
|
|
2994
|
+
/** One parsed entry from an SSTable. */
|
|
2995
|
+
export interface LsmSstableParsedEntry {
|
|
2996
|
+
field: string
|
|
2997
|
+
value: string
|
|
2998
|
+
additions: Buffer
|
|
2999
|
+
deletions: Buffer
|
|
3000
|
+
epoch: bigint
|
|
3001
|
+
tombstone: boolean
|
|
3002
|
+
}
|
|
3003
|
+
|
|
3004
|
+
/** Decoded SSTable representation returned by [`lsm_format_parse_sstable`]. */
|
|
3005
|
+
export interface LsmSstableParseResult {
|
|
3006
|
+
header: LsmHeaderJson
|
|
3007
|
+
entries: Array<LsmSstableParsedEntry>
|
|
3008
|
+
bloom: Buffer
|
|
3009
|
+
/**
|
|
3010
|
+
* SHA-256 of the body (directory + keys + payload + bloom) as the
|
|
3011
|
+
* reader recomputed it — must equal `header.sha256_hex` for an
|
|
3012
|
+
* uncorrupted file.
|
|
3013
|
+
*/
|
|
3014
|
+
computedSha256Hex: string
|
|
3015
|
+
}
|
|
3016
|
+
|
|
3017
|
+
/**
|
|
3018
|
+
* Manhattan (L1) distance between two vectors.
|
|
3019
|
+
* Lower values indicate higher similarity.
|
|
3020
|
+
*/
|
|
3021
|
+
export declare function manhattanDistance(a: Array<number>, b: Array<number>): number
|
|
3022
|
+
|
|
3023
|
+
/** Decode msgpack bytes to a JSON value. */
|
|
3024
|
+
export declare function msgpackDecode(buffer: Buffer): any
|
|
3025
|
+
|
|
3026
|
+
/**
|
|
3027
|
+
* Batch decode multiple msgpack buffers to JSON values.
|
|
3028
|
+
* Uses Rayon parallelism for large batches.
|
|
3029
|
+
*/
|
|
3030
|
+
export declare function msgpackDecodeBatch(buffers: Array<Buffer>): Array<any>
|
|
3031
|
+
|
|
3032
|
+
/** Encode a JSON value to msgpack bytes. */
|
|
3033
|
+
export declare function msgpackEncode(value: any): Buffer
|
|
3034
|
+
|
|
3035
|
+
/**
|
|
3036
|
+
* Batch encode multiple JSON values to msgpack.
|
|
3037
|
+
* Uses Rayon parallelism for large batches.
|
|
3038
|
+
*/
|
|
3039
|
+
export declare function msgpackEncodeBatch(values: Array<any>): Array<Buffer>
|
|
3040
|
+
|
|
3041
|
+
/** Result from compacting a tree to binary .sst format (adapter-controlled) */
|
|
3042
|
+
export interface NativeBinaryCompactResult {
|
|
3043
|
+
/** Which tree was compacted */
|
|
3044
|
+
treeName: string
|
|
3045
|
+
/** New merged SSTable ID */
|
|
3046
|
+
newSstableId: string
|
|
3047
|
+
/** Binary .sst data for the adapter to persist */
|
|
3048
|
+
data: Buffer
|
|
3049
|
+
/** Target level */
|
|
3050
|
+
newLevel: number
|
|
3051
|
+
/** Old SSTable IDs to remove from manifest/storage */
|
|
3052
|
+
oldIds: Array<string>
|
|
3053
|
+
}
|
|
3054
|
+
|
|
3055
|
+
/**
|
|
3056
|
+
* Result from flushing a tree's MemTable to binary .sst format (adapter-controlled).
|
|
3057
|
+
*
|
|
3058
|
+
* **Cor 3.0 Piece D**: the legacy msgpack-based `GraphFlushResult` /
|
|
3059
|
+
* `GraphCompactResult` types were dropped along with the msgpack-only
|
|
3060
|
+
* `LSMTreeCore.flush_mem_table` / `compact` API path. The binary `.sst`
|
|
3061
|
+
* format ([`crate::mmap_sstable_u64::MmapSSTableU64`]) is the only
|
|
3062
|
+
* on-disk shape now — same durability bar (SHA-256 + fsync), faster
|
|
3063
|
+
* reads, smaller files (no string table).
|
|
3064
|
+
*
|
|
3065
|
+
* `entry_count` / `value_count` are narrowed `u32` for napi-visible
|
|
3066
|
+
* per-SSTable totals. The underlying `LSMTreeCoreU64` tracks them as
|
|
3067
|
+
* `u64`; a single SSTable would need >4B entries to overflow which is
|
|
3068
|
+
* impractical (4 B × 20-byte entry index = 80 GB per SSTable).
|
|
3069
|
+
*/
|
|
3070
|
+
export interface NativeBinaryFlushResult {
|
|
3071
|
+
/** Which tree was flushed */
|
|
3072
|
+
treeName: string
|
|
3073
|
+
/** SSTable ID */
|
|
3074
|
+
sstableId: string
|
|
3075
|
+
/** Binary .sst data for the adapter to persist */
|
|
3076
|
+
data: Buffer
|
|
3077
|
+
/** Compaction level (always 0 for flush) */
|
|
3078
|
+
level: number
|
|
3079
|
+
/** Number of unique entries */
|
|
3080
|
+
entryCount: number
|
|
3081
|
+
/** Total value count (sum of values across all entries) */
|
|
3082
|
+
valueCount: number
|
|
3083
|
+
}
|
|
3084
|
+
|
|
3085
|
+
/** Cache statistics including native memory tracking. */
|
|
3086
|
+
export interface NativeCacheStats {
|
|
3087
|
+
totalSize: number
|
|
3088
|
+
maxSize: number
|
|
3089
|
+
itemCount: number
|
|
3090
|
+
typeSizes: Array<number>
|
|
3091
|
+
typeCounts: Array<number>
|
|
3092
|
+
typeAccessCounts: Array<number>
|
|
3093
|
+
totalAccessCount: number
|
|
3094
|
+
nativeMemory: Array<number>
|
|
3095
|
+
totalManagedMemory: number
|
|
3096
|
+
}
|
|
3097
|
+
|
|
3098
|
+
/** Options for `connectedComponents`. */
|
|
3099
|
+
export interface NativeComponentsOptions {
|
|
3100
|
+
/**
|
|
3101
|
+
* `"weak"` (default; ignores direction) or `"strong"` (SCCs, honors
|
|
3102
|
+
* direction). Any other value is treated as `"weak"` (lenient).
|
|
3103
|
+
*/
|
|
3104
|
+
mode?: string
|
|
3105
|
+
/**
|
|
3106
|
+
* **Contextual scoping.** When set, label only the induced subgraph
|
|
3107
|
+
* reachable from these seeds (O(scope)). Omitted = global.
|
|
3108
|
+
*/
|
|
3109
|
+
seeds?: Array<bigint>
|
|
3110
|
+
/** Scope seeds as cor's OpaqueIdSet envelope. */
|
|
3111
|
+
seedsOpaque?: Buffer
|
|
3112
|
+
/** Hop bound for the scope neighborhood (default 2). */
|
|
3113
|
+
scopeDepth?: number
|
|
3114
|
+
}
|
|
3115
|
+
|
|
3116
|
+
/**
|
|
3117
|
+
* Statistics about the mapper state. Legacy u32 fields (`mappings`,
|
|
3118
|
+
* `next_id`, `memory_estimate`) cap at [`u32::MAX`]; Piece-10 BigInt
|
|
3119
|
+
* fields (`*_big`) carry the same values losslessly and should be
|
|
3120
|
+
* preferred at billion scale.
|
|
3121
|
+
*/
|
|
3122
|
+
export interface NativeEntityIdMapperStats {
|
|
3123
|
+
/** Number of active UUID ↔ integer mappings (caps at u32::MAX). */
|
|
3124
|
+
mappings: number
|
|
3125
|
+
/** Next integer ID to assign (caps at u32::MAX). */
|
|
3126
|
+
nextId: number
|
|
3127
|
+
/** Whether there are unsaved changes */
|
|
3128
|
+
dirty: boolean
|
|
3129
|
+
/** Estimated memory usage in bytes (caps at u32::MAX). */
|
|
3130
|
+
memoryEstimate: number
|
|
3131
|
+
/** IdSpace as a string — `"u32"` or `"u64"`. */
|
|
3132
|
+
idSpace: string
|
|
3133
|
+
/** Number of mappings as BigInt (overflow-safe). */
|
|
3134
|
+
mappingsBig: bigint
|
|
3135
|
+
/** Next integer ID as BigInt (overflow-safe). */
|
|
3136
|
+
nextIdBig: bigint
|
|
3137
|
+
/** Estimated memory usage as BigInt (overflow-safe). */
|
|
3138
|
+
memoryEstimateBig: bigint
|
|
3139
|
+
}
|
|
3140
|
+
|
|
3141
|
+
/** Result from eviction or clear operations. */
|
|
3142
|
+
export interface NativeEvictionResult {
|
|
3143
|
+
evictedKeys: Array<string>
|
|
3144
|
+
bytesFreed: number
|
|
3145
|
+
}
|
|
3146
|
+
|
|
3147
|
+
/**
|
|
3148
|
+
* Connected-component labelling — the columnar result of
|
|
3149
|
+
* `connectedComponents`. `component_ids[i]` is the (dense `0..count`)
|
|
3150
|
+
* component of `node_ints[i]`. Decoded to brainy's `GraphComponents`.
|
|
3151
|
+
*/
|
|
3152
|
+
export interface NativeGraphComponents {
|
|
3153
|
+
/** Entity ints, `u64` LE. */
|
|
3154
|
+
nodeInts: Buffer
|
|
3155
|
+
/**
|
|
3156
|
+
* Component label per node, `u32` LE (4 bytes each), parallel to
|
|
3157
|
+
* `node_ints`. Labels are arbitrary-but-stable for a fixed
|
|
3158
|
+
* `(graph, generation, mode)` — only the PARTITION is meaningful;
|
|
3159
|
+
* values renumber across mutations (do not cache as a key).
|
|
3160
|
+
*/
|
|
3161
|
+
componentIds: Buffer
|
|
3162
|
+
/** Distinct component total (a SEPARATE field, not `max(component_ids)`). */
|
|
3163
|
+
componentCount: number
|
|
3164
|
+
}
|
|
3165
|
+
|
|
3166
|
+
/**
|
|
3167
|
+
* One streamed chunk from a graph cursor
|
|
3168
|
+
* ([`graph_cursor_next`](NativeGraphAdjacencyIndex::graph_cursor_next)).
|
|
3169
|
+
*/
|
|
3170
|
+
export interface NativeGraphCursorChunk {
|
|
3171
|
+
/**
|
|
3172
|
+
* This chunk's nodes + edges, columnar (same wire format as
|
|
3173
|
+
* [`NativeSubgraph`]; `node_depth` is always empty for a flat
|
|
3174
|
+
* export, and `edge_verb_ints` is empty under `"light"`).
|
|
3175
|
+
*/
|
|
3176
|
+
subgraph: NativeSubgraph
|
|
3177
|
+
/**
|
|
3178
|
+
* Opaque resume token — pass to `graphCursorOpen({ cursor })` after
|
|
3179
|
+
* a `SnapshotExpired` to resume. `None` once the walk is exhausted.
|
|
3180
|
+
*/
|
|
3181
|
+
cursor?: string
|
|
3182
|
+
/**
|
|
3183
|
+
* `true` once the walk is exhausted; do not call `graphCursorNext`
|
|
3184
|
+
* again (a redundant call returns an empty, `done: true` chunk).
|
|
3185
|
+
*/
|
|
3186
|
+
done: boolean
|
|
3187
|
+
}
|
|
3188
|
+
|
|
3189
|
+
/**
|
|
3190
|
+
* A shortest path — the result of `shortestPath` (`None` ⇒ unreachable).
|
|
3191
|
+
* Decoded to brainy's `GraphPath`.
|
|
3192
|
+
*/
|
|
3193
|
+
export interface NativeGraphPath {
|
|
3194
|
+
/** The path's node sequence (from → to), `u64` LE. */
|
|
3195
|
+
nodeInts: Buffer
|
|
3196
|
+
/**
|
|
3197
|
+
* Edge verb ints, `u64` LE. CONTRACT: `len == node_ints.len() - 1`
|
|
3198
|
+
* (parallel to the GAPS between nodes, not to the nodes).
|
|
3199
|
+
*/
|
|
3200
|
+
edgeVerbInts: Buffer
|
|
3201
|
+
/** Hop count (cor stores no edge weights → hops-only weighting). */
|
|
3202
|
+
cost: number
|
|
3203
|
+
}
|
|
3204
|
+
|
|
3205
|
+
/**
|
|
3206
|
+
* Per-node scalar scores — the columnar result of `pageRank` and
|
|
3207
|
+
* `topByDegree` (ADR-006 Phase C). Two parallel little-endian columns,
|
|
3208
|
+
* descending by score; decoded zero-copy to brainy's `GraphScores`
|
|
3209
|
+
* (`nodeInts: BigInt64Array`, `scores: Float64Array`).
|
|
3210
|
+
*/
|
|
3211
|
+
export interface NativeGraphScores {
|
|
3212
|
+
/** Entity ints, `u64` LE (8 bytes each). */
|
|
3213
|
+
nodeInts: Buffer
|
|
3214
|
+
/** Per-node score, `f64` LE (8 bytes each), parallel to `node_ints`. */
|
|
3215
|
+
scores: Buffer
|
|
3216
|
+
}
|
|
3217
|
+
|
|
3218
|
+
/**
|
|
3219
|
+
* JS options bag for the engine constructor. Every field optional with
|
|
3220
|
+
* sensible defaults per the [[zero-config-dynamic-adaptation]] mandate.
|
|
3221
|
+
*/
|
|
3222
|
+
export interface NativeLsmEngineOptions {
|
|
3223
|
+
/** `"u32"` (default) or `"u64"`. */
|
|
3224
|
+
idSpace?: string
|
|
3225
|
+
/** Emit bloom filters in flushed SSTables. Default `true`. */
|
|
3226
|
+
emitBloom?: boolean
|
|
3227
|
+
/** Bits per element for the bloom filter. Default `10` (~1% FPR). */
|
|
3228
|
+
bloomBitsPerElement?: number
|
|
3229
|
+
/**
|
|
3230
|
+
* Admission threshold ratio (0.0..1.0). Default `0.85`. Below
|
|
3231
|
+
* `threshold × allowed_bytes` the engine accepts writes
|
|
3232
|
+
* immediately; at or above, new writes block on the flush
|
|
3233
|
+
* condvar. Per the plan this is hardcoded at 0.85 — exposing
|
|
3234
|
+
* it here is for TEST DRIVERS only (legitimate production use
|
|
3235
|
+
* always sticks with the default).
|
|
3236
|
+
*/
|
|
3237
|
+
admissionThresholdRatio?: number
|
|
3238
|
+
}
|
|
3239
|
+
|
|
3240
|
+
/** One flushed SSTable's structured result, paired with its raw bytes. */
|
|
3241
|
+
export interface NativeLsmFlushResult {
|
|
3242
|
+
/** Raw SSTable bytes — hand to `storage.saveBinaryBlob`. */
|
|
3243
|
+
sstable: Buffer
|
|
3244
|
+
/**
|
|
3245
|
+
* `LsmSstableManifestEntry` JSON — the wrapper splices this into
|
|
3246
|
+
* the manifest after a successful disk write.
|
|
3247
|
+
*/
|
|
3248
|
+
manifestEntryJson: string
|
|
3249
|
+
/** Number of `(field, value)` keys in the flushed SSTable. */
|
|
3250
|
+
entryCount: bigint
|
|
3251
|
+
/** New `last_durable_epoch` value to write into the manifest. */
|
|
3252
|
+
lastDurableEpoch: bigint
|
|
3253
|
+
}
|
|
3254
|
+
|
|
3255
|
+
/**
|
|
3256
|
+
* Result of the test-only `_testFlushThroughputWithFsync` harness.
|
|
3257
|
+
* Backs Phase 3 gate P3-4.
|
|
3258
|
+
*/
|
|
3259
|
+
export interface NativeLsmFlushThroughputResult {
|
|
3260
|
+
/**
|
|
3261
|
+
* Wall-clock elapsed nanoseconds for: `flush()` + `write_all` +
|
|
3262
|
+
* `fsync`. The measured "flush throughput" is `sstable_bytes /
|
|
3263
|
+
* elapsed_ns × 1e9`.
|
|
3264
|
+
*/
|
|
3265
|
+
elapsedNs: bigint
|
|
3266
|
+
/**
|
|
3267
|
+
* Size of the SSTable bytes produced by `flush()` and written
|
|
3268
|
+
* to disk.
|
|
3269
|
+
*/
|
|
3270
|
+
sstableBytes: bigint
|
|
3271
|
+
/** Memtable resident bytes just before the flush — diagnostic. */
|
|
3272
|
+
memtableBytesPre: bigint
|
|
3273
|
+
}
|
|
3274
|
+
|
|
3275
|
+
/**
|
|
3276
|
+
* Result of the test-only `_testStressReaderWriterFlusher` harness.
|
|
3277
|
+
* Closes Phase 3 gate P3-3 (flush during concurrent reader → no
|
|
3278
|
+
* torn reads) and Phase 4 gate P4-4 (4r/1w/1f stress 100/100 → no
|
|
3279
|
+
* panics, no torn reads).
|
|
3280
|
+
*/
|
|
3281
|
+
export interface NativeLsmReaderWriterStressResult {
|
|
3282
|
+
/** Total reads completed across all reader threads. */
|
|
3283
|
+
readsDone: bigint
|
|
3284
|
+
/** Total inserts completed across all writer threads. */
|
|
3285
|
+
writesDone: bigint
|
|
3286
|
+
/** Total flushes the flusher executed. */
|
|
3287
|
+
flushesDone: bigint
|
|
3288
|
+
/**
|
|
3289
|
+
* Number of torn-read events — MUST be 0 for the gate to PASS.
|
|
3290
|
+
* A torn read is one where a snapshot's cardinality for its
|
|
3291
|
+
* target key changed mid-snapshot. Point-in-time isolation
|
|
3292
|
+
* guarantees this cannot happen.
|
|
3293
|
+
*/
|
|
3294
|
+
tornReads: bigint
|
|
3295
|
+
}
|
|
3296
|
+
|
|
3297
|
+
/**
|
|
3298
|
+
* One entry materialized from an SSTable scan. Returned by
|
|
3299
|
+
* [`NativeLsmEngine::scan_sstable`].
|
|
3300
|
+
*
|
|
3301
|
+
* `additionsBytes` / `deletionsBytes` are croaring portable
|
|
3302
|
+
* serializations of the per-entry PostingLists. Cardinalities
|
|
3303
|
+
* are also surfaced so consumers that only need counts don't
|
|
3304
|
+
* pay the deserialize cost.
|
|
3305
|
+
*/
|
|
3306
|
+
export interface NativeLsmScanEntry {
|
|
3307
|
+
field: string
|
|
3308
|
+
value: string
|
|
3309
|
+
additionsBytes: Buffer
|
|
3310
|
+
deletionsBytes: Buffer
|
|
3311
|
+
additionCount: bigint
|
|
3312
|
+
deletionCount: bigint
|
|
3313
|
+
epoch: bigint
|
|
3314
|
+
isTombstone: boolean
|
|
3315
|
+
}
|
|
3316
|
+
|
|
3317
|
+
/** Result of the test-only `_testStressConcurrentAdmission` harness. */
|
|
3318
|
+
export interface NativeLsmStressResult {
|
|
3319
|
+
/**
|
|
3320
|
+
* Highest memtable byte count observed across the run. Compare
|
|
3321
|
+
* against `budget × 1.05` to verify the atomic admission
|
|
3322
|
+
* guarantee held.
|
|
3323
|
+
*/
|
|
3324
|
+
peakMemtableBytes: bigint
|
|
3325
|
+
/**
|
|
3326
|
+
* How many writer iterations OBSERVED `isOverAdmissionThreshold`
|
|
3327
|
+
* returning true (telemetry; meant to confirm backpressure
|
|
3328
|
+
* actually engaged under load).
|
|
3329
|
+
*/
|
|
3330
|
+
nBlocked: bigint
|
|
3331
|
+
}
|
|
3332
|
+
|
|
3333
|
+
/**
|
|
3334
|
+
* Result of the test-only `_testWriterThroughput` harness.
|
|
3335
|
+
* Backs Phase 3 gate P3-1.
|
|
3336
|
+
*/
|
|
3337
|
+
export interface NativeLsmThroughputResult {
|
|
3338
|
+
/** Wall-clock elapsed nanoseconds for the workload. */
|
|
3339
|
+
elapsedNs: bigint
|
|
3340
|
+
/**
|
|
3341
|
+
* `n_writers × per_writer` — the total number of inserts
|
|
3342
|
+
* completed.
|
|
3343
|
+
*/
|
|
3344
|
+
totalInserts: bigint
|
|
3345
|
+
}
|
|
3346
|
+
|
|
3347
|
+
/** Options for `neighborhoodSample`. */
|
|
3348
|
+
export interface NativeNeighborhoodSampleOptions {
|
|
3349
|
+
/** `"out"`, `"in"`, or `"both"` (default). */
|
|
3350
|
+
direction?: string
|
|
3351
|
+
/** Max hop distance from any seed (required). */
|
|
3352
|
+
depth: number
|
|
3353
|
+
/** Max neighbors sampled per node (required; deterministic, seeded). */
|
|
3354
|
+
fanout: number
|
|
3355
|
+
/**
|
|
3356
|
+
* RNG seed so `(seeds, opts, seed)` yields a stable sample. Narrowed
|
|
3357
|
+
* to `u64` (NaN/negative → 0); brainy passes a JS `number`.
|
|
3358
|
+
*/
|
|
3359
|
+
seed?: number
|
|
3360
|
+
/** Cap on returned nodes (sets `truncated`). */
|
|
3361
|
+
maxNodes?: number
|
|
3362
|
+
/** Populate `node_depth` (default true). */
|
|
3363
|
+
includeDepth?: boolean
|
|
3364
|
+
}
|
|
3365
|
+
|
|
3366
|
+
/**
|
|
3367
|
+
* Options for `pageRank` (ADR-006 Phase C). Mirrors brainy's `PageRankOptions`
|
|
3368
|
+
* (minus `excludeVisibility`, which brainy applies on hydrate — cor's graph
|
|
3369
|
+
* layer has no node-visibility data).
|
|
3370
|
+
*/
|
|
3371
|
+
export interface NativePageRankOptions {
|
|
3372
|
+
/** Damping factor (default 0.85). */
|
|
3373
|
+
damping?: number
|
|
3374
|
+
/** Max power-iterations (default 20). */
|
|
3375
|
+
maxIterations?: number
|
|
3376
|
+
/** L1 convergence tolerance (default 1e-6). */
|
|
3377
|
+
tolerance?: number
|
|
3378
|
+
/** Treat the graph as undirected (default false). */
|
|
3379
|
+
undirected?: boolean
|
|
3380
|
+
/** Return only the top-K by score (default: all, descending). */
|
|
3381
|
+
topK?: number
|
|
3382
|
+
/**
|
|
3383
|
+
* **Contextual scoping (ADR-006 Phase C).** When set, rank only the
|
|
3384
|
+
* induced subgraph reachable from these seed ints — "importance within
|
|
3385
|
+
* this context" in O(scope) instead of O(whole graph). Omitted = global.
|
|
3386
|
+
*/
|
|
3387
|
+
seeds?: Array<bigint>
|
|
3388
|
+
/** Scope seeds as cor's OpaqueIdSet envelope (the `find()`→scope fusion). */
|
|
3389
|
+
seedsOpaque?: Buffer
|
|
3390
|
+
/** Hop bound for the scope neighborhood (default 2). Ignored when global. */
|
|
3391
|
+
scopeDepth?: number
|
|
3392
|
+
}
|
|
3393
|
+
|
|
3394
|
+
/**
|
|
3395
|
+
* Options for `shortestPath`. **No `weight` field** — cor stores no edge
|
|
3396
|
+
* weights, so native shortest-path is hops-only; brainy keeps `weight:'edge'`
|
|
3397
|
+
* TS-side and routes only `weight:'hops'`/omitted here.
|
|
3398
|
+
*/
|
|
3399
|
+
export interface NativeShortestPathOptions {
|
|
3400
|
+
/** `"out"`, `"in"`, or `"both"` (default). */
|
|
3401
|
+
direction?: string
|
|
3402
|
+
/** Restrict to these `VerbType` indices (empty/omitted = all). */
|
|
3403
|
+
verbTypes?: Array<number>
|
|
3404
|
+
/** Abandon the search past this many hops. */
|
|
3405
|
+
maxDepth?: number
|
|
3406
|
+
}
|
|
3407
|
+
|
|
3408
|
+
/**
|
|
3409
|
+
* Columnar subgraph result — parallel typed columns packed as
|
|
3410
|
+
* little-endian [`Buffer`]s, never an array of objects (ADR-006 wire
|
|
3411
|
+
* format). The TS wrapper views each Buffer as a typed array
|
|
3412
|
+
* (`BigInt64Array` / `Uint16Array` / `Uint8Array`) with zero element
|
|
3413
|
+
* copies; brainy resolves the u64 ints to UUIDs lazily, only for the
|
|
3414
|
+
* rows it renders.
|
|
3415
|
+
*/
|
|
3416
|
+
export interface NativeSubgraph {
|
|
3417
|
+
/**
|
|
3418
|
+
* Distinct node ints in deterministic BFS-discovery order (seeds
|
|
3419
|
+
* first), `u64` little-endian (8 bytes each).
|
|
3420
|
+
*/
|
|
3421
|
+
nodes: Buffer
|
|
3422
|
+
/**
|
|
3423
|
+
* Per-node BFS depth (`0` = seed), `u8`, parallel to `nodes`. Empty
|
|
3424
|
+
* when `include_depth` is `false`.
|
|
3425
|
+
*/
|
|
3426
|
+
nodeDepth: Buffer
|
|
3427
|
+
/**
|
|
3428
|
+
* Per-edge source node int (`u64` LE), parallel to the other edge
|
|
3429
|
+
* columns. Empty when `include_edges` is `false`.
|
|
3430
|
+
*/
|
|
3431
|
+
edgeSources: Buffer
|
|
3432
|
+
/** Per-edge target node int (`u64` LE). */
|
|
3433
|
+
edgeTargets: Buffer
|
|
3434
|
+
/**
|
|
3435
|
+
* Per-edge interned verb int (`u64` LE) — brainy maps these to
|
|
3436
|
+
* verb-id strings via the existing `verbIntsToIds`.
|
|
3437
|
+
*/
|
|
3438
|
+
edgeVerbInts: Buffer
|
|
3439
|
+
/** Per-edge verb-type index (`u16` LE; `VerbType` ≪ 65535). */
|
|
3440
|
+
edgeTypes: Buffer
|
|
3441
|
+
/**
|
|
3442
|
+
* `true` if a `max_nodes`/`max_edges` cap stopped expansion before
|
|
3443
|
+
* the full `depth` was reached — the returned columns are a
|
|
3444
|
+
* deterministic BFS-order prefix (page the rest via the cursor).
|
|
3445
|
+
*/
|
|
3446
|
+
truncated: boolean
|
|
3447
|
+
}
|
|
3448
|
+
|
|
3449
|
+
/** Options for `topByDegree`. */
|
|
3450
|
+
export interface NativeTopByDegreeOptions {
|
|
3451
|
+
/** `"out"` (default), `"in"`, or `"both"`. */
|
|
3452
|
+
direction?: string
|
|
3453
|
+
/** Number of top nodes to return (required). */
|
|
3454
|
+
topK: number
|
|
3455
|
+
/**
|
|
3456
|
+
* **Contextual scoping.** When set, rank degrees only within the induced
|
|
3457
|
+
* subgraph reachable from these seeds (O(scope)). Omitted = global.
|
|
3458
|
+
*/
|
|
3459
|
+
seeds?: Array<bigint>
|
|
3460
|
+
/** Scope seeds as cor's OpaqueIdSet envelope. */
|
|
3461
|
+
seedsOpaque?: Buffer
|
|
3462
|
+
/** Hop bound for the scope neighborhood (default 2). */
|
|
3463
|
+
scopeDepth?: number
|
|
3464
|
+
}
|
|
3465
|
+
|
|
3466
|
+
/**
|
|
3467
|
+
* **napi: build an `OpaqueIdSet` from explicit entity ints.** Constructs
|
|
3468
|
+
* the `find()` → `traverse`/`search` payload from a materialized id list.
|
|
3469
|
+
* `id_space` selects the bitmap width — `"U64"` for billion-scale brains,
|
|
3470
|
+
* `"U32"` for ≤4 B (U32 errors on an int above `u32::MAX`). Used by cor's
|
|
3471
|
+
* own emission paths and by tests; the production fusion path composes the
|
|
3472
|
+
* payload via cor's roaring provider, but the envelope is identical.
|
|
3473
|
+
*/
|
|
3474
|
+
export declare function opaqueIdSetFromInts(ints: Array<bigint>, idSpace: string): Buffer
|
|
3475
|
+
|
|
3476
|
+
/**
|
|
3477
|
+
* **napi: decode an `OpaqueIdSet` back to its entity ints.** Inverse of
|
|
3478
|
+
* [`opaque_id_set_from_ints`]; fails loud on a malformed envelope. For
|
|
3479
|
+
* round-trip tests and any caller that needs the materialized ids.
|
|
3480
|
+
*/
|
|
3481
|
+
export declare function opaqueIdSetToInts(buf: Buffer): Array<bigint>
|
|
3482
|
+
|
|
3483
|
+
/**
|
|
3484
|
+
* Parity helper: compare two strings by UTF-8 byte order (== code-point order
|
|
3485
|
+
* == brainy's `compareCodePoints`). Returns `-1`, `0`, or `1`.
|
|
3486
|
+
*/
|
|
3487
|
+
export declare function parityCompareCodePoints(a: string, b: string): number
|
|
3488
|
+
|
|
3489
|
+
/**
|
|
3490
|
+
* Parity helper: long-value hash, identical to brainy's `hashValue`
|
|
3491
|
+
* (`__HASH_<base36>`). See [`hash_value`].
|
|
3492
|
+
*/
|
|
3493
|
+
export declare function parityHashValue(value: string): string
|
|
3494
|
+
|
|
3495
|
+
/**
|
|
3496
|
+
* Parity helper: FNV-1a word hash, identical to brainy's `hashWord` (signed i32).
|
|
3497
|
+
* See [`hash_word`].
|
|
3498
|
+
*/
|
|
3499
|
+
export declare function parityHashWord(word: string): number
|
|
3500
|
+
|
|
3501
|
+
/**
|
|
3502
|
+
* Parity helper: normalize a JSON-encoded value exactly as brainy's
|
|
3503
|
+
* `normalizeValue` with a fresh manager (default config, empty field stats so the
|
|
3504
|
+
* precision strategy never fires). `value_json` is the value as JSON (e.g. `42`,
|
|
3505
|
+
* `"café"`, `true`, `null`, `[1,2,3]`), matching how brainy receives raw values.
|
|
3506
|
+
* See [`normalize_value`].
|
|
3507
|
+
*/
|
|
3508
|
+
export declare function parityNormalizeValue(valueJson: string): string
|
|
3509
|
+
|
|
3510
|
+
/**
|
|
3511
|
+
* Parity helper: load a `.cidx` segment from a Buffer and read
|
|
3512
|
+
* every entity ID via `entity_id_big` (the IdSpace-agnostic
|
|
3513
|
+
* accessor). Returns BigInt[] so values above 2^53 survive the
|
|
3514
|
+
* napi crossing losslessly. The returned IDs are in the segment's
|
|
3515
|
+
* stored order (the same order the writer received them).
|
|
3516
|
+
*/
|
|
3517
|
+
export declare function parityReadSegmentEntityIdsBig(buffer: Buffer): Array<bigint>
|
|
3518
|
+
|
|
3519
|
+
/**
|
|
3520
|
+
* Parity helper: load a segment, inspect its header flag, and
|
|
3521
|
+
* report whether it was written with `FLAG_U64_IDS`. Used by the
|
|
3522
|
+
* gap-C wire-format fixture to assert the on-disk bit assignment
|
|
3523
|
+
* matches the brainy 8.0 reader's expectation.
|
|
3524
|
+
*/
|
|
3525
|
+
export declare function paritySegmentIsU64Ids(buffer: Buffer): boolean
|
|
3526
|
+
|
|
3527
|
+
/**
|
|
3528
|
+
* Parity helper: sort strings by the native column-store order (UTF-8 byte /
|
|
3529
|
+
* code-point order). The returned order must equal a JS sort by
|
|
3530
|
+
* `compareCodePoints`.
|
|
3531
|
+
*/
|
|
3532
|
+
export declare function paritySortStrings(values: Array<string>): Array<string>
|
|
3533
|
+
|
|
3534
|
+
/**
|
|
3535
|
+
* Parity helper: tokenize text exactly as brainy's `MetadataIndexManager.tokenize`.
|
|
3536
|
+
* Returns the ordered, deduped, ASCII-only token set. See [`tokenize`].
|
|
3537
|
+
*/
|
|
3538
|
+
export declare function parityTokenize(text: string): Array<string>
|
|
3539
|
+
|
|
3540
|
+
/**
|
|
3541
|
+
* Parity helper: write a U64 float `.cidx` segment from a list
|
|
3542
|
+
* of (f64 value, BigInt entity-id) pairs.
|
|
3543
|
+
*/
|
|
3544
|
+
export declare function parityWriteU64FloatSegment(field: string, values: Array<number>, entityIds: Array<bigint>): Buffer
|
|
3545
|
+
|
|
3546
|
+
/**
|
|
3547
|
+
* Parity helper: write a U64 numeric `.cidx` segment from a list
|
|
3548
|
+
* of (timestamp, BigInt entity-id) pairs. Returns the segment
|
|
3549
|
+
* bytes for direct inspection / round-trip verification.
|
|
3550
|
+
*/
|
|
3551
|
+
export declare function parityWriteU64NumericSegment(field: string, timestamps: Array<number>, entityIds: Array<bigint>): Buffer
|
|
3552
|
+
|
|
3553
|
+
/**
|
|
3554
|
+
* Parity helper: write a U64 string `.cidx` segment from a list
|
|
3555
|
+
* of (UTF-8 string, BigInt entity-id) pairs.
|
|
3556
|
+
*/
|
|
3557
|
+
export declare function parityWriteU64StringSegment(field: string, values: Array<string>, entityIds: Array<bigint>): Buffer
|
|
3558
|
+
|
|
3559
|
+
/**
|
|
3560
|
+
* **Piece 4.** Plan a batched entity hydration for `uuids` at the
|
|
3561
|
+
* given shard depth. Returns a [`HydrationPlanJs`] grouping UUIDs
|
|
3562
|
+
* by shard + per-shard estimated sort cost + the
|
|
3563
|
+
* migration-recommended hint.
|
|
3564
|
+
*
|
|
3565
|
+
* Inputs:
|
|
3566
|
+
* - `uuids` — entity UUIDs to hydrate (any format normalize_uuid
|
|
3567
|
+
* accepts — hyphenated or compact, mixed case).
|
|
3568
|
+
* - `shardDepth` — `2` (256 buckets, brainy default) or `3`
|
|
3569
|
+
* (16.7M buckets, cor 3.0 deep-shard layout).
|
|
3570
|
+
*
|
|
3571
|
+
* Throws on:
|
|
3572
|
+
* - Invalid `shardDepth` (anything other than 2 or 3).
|
|
3573
|
+
* - Any UUID failing normalization (empty, wrong length, non-hex).
|
|
3574
|
+
*
|
|
3575
|
+
* Cost: O(n) HashMap inserts where n = `uuids.len()`. The per-shard
|
|
3576
|
+
* sort estimate is constant-time arithmetic. Plan cost is well under
|
|
3577
|
+
* 1 µs per UUID, so brainy can call this on every query without
|
|
3578
|
+
* concern.
|
|
3579
|
+
*/
|
|
3580
|
+
export declare function planHydration(uuids: Array<string>, shardDepth: number): HydrationPlan
|
|
3581
|
+
|
|
3582
|
+
/**
|
|
3583
|
+
* **Piece 13.** Plan a PQ configuration for the given entity count.
|
|
3584
|
+
* Returns the recommended config — typically the input unchanged
|
|
3585
|
+
* for sub-billion brains; for billion-scale brains the autotune
|
|
3586
|
+
* downshifts to 4-bit codes (and lowers `ksub` to 16 accordingly).
|
|
3587
|
+
* `maxBytes = 0n` uses the cor 3.0 plan's
|
|
3588
|
+
* `PQ_AUTO_TUNE_MAX_CODES_BYTES` (8 GiB) default; pass an explicit
|
|
3589
|
+
* value to override (e.g., for tighter / looser per-deployment
|
|
3590
|
+
* budgets).
|
|
3591
|
+
*
|
|
3592
|
+
* **Important**: the recommended config is a PLANNING-TIME signal.
|
|
3593
|
+
* Today's `NativeDiskANN.build` only implements 8-bit codes; when
|
|
3594
|
+
* the autotune recommends 4-bit, brainy should surface a warning
|
|
3595
|
+
* + build with 8-bit until Piece 13B's encoder lands. The
|
|
3596
|
+
* `downshifted_to_4bit` field makes this branch trivial.
|
|
3597
|
+
*/
|
|
3598
|
+
export declare function pqAutoTuneConfig(numVectors: bigint, m: number, ksub: number, iterations: number, trainingSample: number, currentBitsPerCode: number, maxCodesBytes: bigint): PqAutoTuneResult
|
|
3599
|
+
|
|
3600
|
+
/**
|
|
3601
|
+
* **Piece 13.** Expose the cor 3.0 plan's autotune threshold —
|
|
3602
|
+
* the codes-section size above which the autotune downshifts to
|
|
3603
|
+
* 4-bit codes. Default 8 GiB. Exposed so brainy callers can
|
|
3604
|
+
* surface the active threshold in operator diagnostics.
|
|
3605
|
+
*/
|
|
3606
|
+
export declare function pqAutoTuneMaxCodesBytes(): bigint
|
|
3607
|
+
|
|
3608
|
+
/**
|
|
3609
|
+
* **Piece 13.** Result of a `pqAutoTuneConfig` call — the
|
|
3610
|
+
* recommended PQ configuration plus the estimated codes-section
|
|
3611
|
+
* size at that configuration. Brainy callers consume this BEFORE
|
|
3612
|
+
* `NativeDiskANN.build` to decide whether the recommended config
|
|
3613
|
+
* is acceptable (e.g., 4-bit when the 8-bit encoder is the only
|
|
3614
|
+
* one available).
|
|
3615
|
+
*/
|
|
3616
|
+
export interface PqAutoTuneResult {
|
|
3617
|
+
/**
|
|
3618
|
+
* `m` — number of PQ subspaces. Preserved from the input
|
|
3619
|
+
* (autotune never adjusts m; see commit 1's rationale).
|
|
3620
|
+
*/
|
|
3621
|
+
m: number
|
|
3622
|
+
/**
|
|
3623
|
+
* `ksub` — adjusted down to 16 when the autotune recommended
|
|
3624
|
+
* 4-bit codes.
|
|
3625
|
+
*/
|
|
3626
|
+
ksub: number
|
|
3627
|
+
/** k-means iterations (preserved from input). */
|
|
3628
|
+
iterations: number
|
|
3629
|
+
/** Training sample (preserved from input). */
|
|
3630
|
+
trainingSample: number
|
|
3631
|
+
/**
|
|
3632
|
+
* **Piece 13.** `4` or `8`. Bit width of the per-subspace
|
|
3633
|
+
* codes. `8` is the only width the current encoder
|
|
3634
|
+
* implements; `4` is the planning-time recommendation for
|
|
3635
|
+
* configurations that would exceed the codes-section budget.
|
|
3636
|
+
*/
|
|
3637
|
+
bitsPerCode: number
|
|
3638
|
+
/**
|
|
3639
|
+
* Estimated codes-section bytes at the returned configuration.
|
|
3640
|
+
* Brainy uses this to decide whether to admit the build.
|
|
3641
|
+
*/
|
|
3642
|
+
estimatedCodesBytes: bigint
|
|
3643
|
+
/**
|
|
3644
|
+
* Whether the autotune downshifted to 4-bit codes. When
|
|
3645
|
+
* `true`, the recommended config is the 4-bit one; brainy
|
|
3646
|
+
* today should still build with 8-bit and surface a warning
|
|
3647
|
+
* (the encoder lands in Piece 13B).
|
|
3648
|
+
*/
|
|
3649
|
+
downshiftedTo4Bit: boolean
|
|
3650
|
+
/**
|
|
3651
|
+
* Whether the recommended config STILL exceeds `maxBytes`.
|
|
3652
|
+
* Happens at 10B+ scale where even 4-bit codes don't fit; the
|
|
3653
|
+
* caller must reduce `m` or scale up the hardware. Surface,
|
|
3654
|
+
* don't auto-adjust.
|
|
3655
|
+
*/
|
|
3656
|
+
stillOverBudget: boolean
|
|
3657
|
+
}
|
|
3658
|
+
|
|
3659
|
+
/**
|
|
3660
|
+
* **Piece 14 (zero-config PQ).** Derive the recommended PQ subspace
|
|
3661
|
+
* count `m` for a given vector `dim`, targeting `dsub ≈ 8` — the same
|
|
3662
|
+
* derivation the build path applies when `m` is left at the `0`
|
|
3663
|
+
* auto-sentinel. `m` always divides `dim`; for the common
|
|
3664
|
+
* multiple-of-8 embedding sizes this is `dim / 8` (a clean 32×
|
|
3665
|
+
* compression at 8-bit codes). Exposed as a pure planning surface so
|
|
3666
|
+
* brainy can preview / log the `m` a build will choose without
|
|
3667
|
+
* triggering one. Errors when `dim == 0`.
|
|
3668
|
+
*/
|
|
3669
|
+
export declare function pqDeriveM(dim: number): number
|
|
3670
|
+
|
|
3671
|
+
/**
|
|
3672
|
+
* **Piece 13.** Estimate the codes-section size in bytes for a
|
|
3673
|
+
* given PQ configuration + entity count. Cheap call: integer
|
|
3674
|
+
* arithmetic only. Used by brainy's pre-build admission gate +
|
|
3675
|
+
* the autotune planning surface.
|
|
3676
|
+
*/
|
|
3677
|
+
export declare function pqEstimateCodesBytes(numVectors: bigint, m: number, bitsPerCode: number): bigint
|
|
3678
|
+
|
|
3679
|
+
/** PQ codebook parameters. */
|
|
3680
|
+
export interface PqParams {
|
|
3681
|
+
m: number
|
|
3682
|
+
ksub: number
|
|
3683
|
+
dsub: number
|
|
3684
|
+
dim: number
|
|
3685
|
+
}
|
|
3686
|
+
|
|
3687
|
+
/**
|
|
3688
|
+
* Quantize a float64 vector to SQ4 (4-bit, two values per byte).
|
|
3689
|
+
*
|
|
3690
|
+
* Each dimension is mapped to [0, 15]. Pairs of values are packed into
|
|
3691
|
+
* single bytes (high nibble first). 8x compression vs float32.
|
|
3692
|
+
*/
|
|
3693
|
+
export declare function quantizeSq4(vector: Array<number>): Sq4QuantizeResult
|
|
3694
|
+
|
|
3695
|
+
/**
|
|
3696
|
+
* Quantize a float64 vector to SQ8 (uint8 with per-vector min/max).
|
|
3697
|
+
*
|
|
3698
|
+
* Each dimension is mapped to [0, 255] using the vector's value range.
|
|
3699
|
+
* Returns the quantized bytes plus min/max for reconstruction.
|
|
3700
|
+
*/
|
|
3701
|
+
export declare function quantizeSq8(vector: Array<number>): Sq8QuantizeResult
|
|
3702
|
+
|
|
3703
|
+
/**
|
|
3704
|
+
* Raise the soft `RLIMIT_NOFILE` to the hard limit. Idempotent + safe
|
|
3705
|
+
* (no privilege needed to raise the soft limit toward the hard cap).
|
|
3706
|
+
* Called once by the TS native loader on startup so the multi-tenant
|
|
3707
|
+
* pool gets the full available fd headroom with zero configuration.
|
|
3708
|
+
* No-op on non-Unix platforms (returns `raised: false`).
|
|
3709
|
+
*/
|
|
3710
|
+
export declare function raiseOpenFileLimit(): FileLimitInfo
|
|
3711
|
+
|
|
3712
|
+
/**
|
|
3713
|
+
* Serialize SQ8 data to compact binary format: [min:f32][max:f32][quantized bytes].
|
|
3714
|
+
* Total size: 8 + dim bytes.
|
|
3715
|
+
*/
|
|
3716
|
+
export declare function serializeSq8(quantized: Buffer, min: number, max: number): Buffer
|
|
3717
|
+
|
|
3718
|
+
/**
|
|
3719
|
+
* One shard's slice of the hydration plan, JS-friendly form. See
|
|
3720
|
+
* [`crate::io_batch_entities::ShardPlan`].
|
|
3721
|
+
*/
|
|
3722
|
+
export interface ShardHydrationPlan {
|
|
3723
|
+
/**
|
|
3724
|
+
* Filesystem-relative shard prefix. JS callers concatenate
|
|
3725
|
+
* this with the storage adapter's root to produce the
|
|
3726
|
+
* directory to read from.
|
|
3727
|
+
*/
|
|
3728
|
+
shardPrefix: string
|
|
3729
|
+
/**
|
|
3730
|
+
* UUIDs in this shard, in original input order. The wrapper
|
|
3731
|
+
* is expected to sort these before calling
|
|
3732
|
+
* `storage.getNounsBatch` if the adapter benefits from
|
|
3733
|
+
* sorted-key reads.
|
|
3734
|
+
*/
|
|
3735
|
+
uuids: Array<string>
|
|
3736
|
+
/**
|
|
3737
|
+
* Estimated nanoseconds to sort `uuids.len()` items in this
|
|
3738
|
+
* shard. `0` for shards of size 0 or 1.
|
|
3739
|
+
*/
|
|
3740
|
+
estimatedSortNs: number
|
|
3741
|
+
/**
|
|
3742
|
+
* Same value, microseconds (echoed for convenience —
|
|
3743
|
+
* wrappers commonly format in microseconds).
|
|
3744
|
+
*/
|
|
3745
|
+
estimatedSortUs: number
|
|
3746
|
+
}
|
|
3747
|
+
|
|
3748
|
+
/**
|
|
3749
|
+
* On-disk manifest format version. Echoed so callers can pin the
|
|
3750
|
+
* version in tests + diagnostics without hardcoding.
|
|
3751
|
+
*/
|
|
3752
|
+
export declare function shardMigrationManifestVersion(): number
|
|
3753
|
+
|
|
3754
|
+
/**
|
|
3755
|
+
* Read-only view of [`ShardMigrationState`] for JS callers.
|
|
3756
|
+
*
|
|
3757
|
+
* `stateKind` discriminates: `"idle"` / `"migrating"` /
|
|
3758
|
+
* `"committing"`. Other fields are populated only when relevant to
|
|
3759
|
+
* the state.
|
|
3760
|
+
*/
|
|
3761
|
+
export interface ShardMigrationStateSummary {
|
|
3762
|
+
stateKind: string
|
|
3763
|
+
targetDepth?: number
|
|
3764
|
+
scanned?: bigint
|
|
3765
|
+
total?: bigint
|
|
3766
|
+
startedAtMs?: bigint
|
|
3767
|
+
lastProgressAtMs?: bigint
|
|
3768
|
+
}
|
|
3769
|
+
|
|
3770
|
+
/**
|
|
3771
|
+
* Number of consecutive `migrationRecommended=true` hints required
|
|
3772
|
+
* before [`shouldStartShardMigration`] returns `true`. Cor 3.0
|
|
3773
|
+
* default = 10.
|
|
3774
|
+
*/
|
|
3775
|
+
export declare function shardMigrationSustainedThreshold(): number
|
|
3776
|
+
|
|
3777
|
+
/**
|
|
3778
|
+
* Anti-flap trigger. Returns `true` only when the tail of
|
|
3779
|
+
* `recentHints` contains at least
|
|
3780
|
+
* [`shardMigrationSustainedThreshold`] consecutive `true` values.
|
|
3781
|
+
* Any single `false` resets the consecutive counter.
|
|
3782
|
+
*
|
|
3783
|
+
* Brainy maintains the sliding window (typically the last N hints
|
|
3784
|
+
* from per-query `HydrationPlan.migrationRecommended` values) and
|
|
3785
|
+
* calls this when deciding whether to schedule migration.
|
|
3786
|
+
*/
|
|
3787
|
+
export declare function shouldStartShardMigration(recentHints: Array<boolean>): boolean
|
|
3788
|
+
|
|
3789
|
+
/**
|
|
3790
|
+
* Select the indices of the top-K elements of `scores`, in ranked order.
|
|
3791
|
+
*
|
|
3792
|
+
* Runs in **O(N log K)** via a bounded max-heap of the K best candidates seen
|
|
3793
|
+
* so far (vs. O(N log N) for a full sort). When `k >= scores.len()` it performs
|
|
3794
|
+
* a single full sort instead, which is optimal in that regime.
|
|
3795
|
+
*
|
|
3796
|
+
* # Arguments
|
|
3797
|
+
* * `scores` - The score for each result row, indexed 0..N.
|
|
3798
|
+
* * `k` - How many top indices to return. `0` → empty; values `>= N` → all N.
|
|
3799
|
+
* * `descending` - `true` ranks the highest score first; `false` the lowest.
|
|
3800
|
+
*
|
|
3801
|
+
* # Returns
|
|
3802
|
+
* A vector of length `min(k, scores.len())` containing indices into `scores`,
|
|
3803
|
+
* ordered best-first per `descending`. Ties broken by index ascending; `NaN`
|
|
3804
|
+
* scores treated as worst (see the module docs for the full contract).
|
|
3805
|
+
*
|
|
3806
|
+
* # Example
|
|
3807
|
+
* ```ignore
|
|
3808
|
+
* // scores: a=0.1, b=0.9, c=0.5 ; want the best 2 (descending)
|
|
3809
|
+
* let top = sort_top_k_scores(vec![0.1, 0.9, 0.5], 2, true);
|
|
3810
|
+
* assert_eq!(top, vec![1, 2]); // index 1 (0.9) then index 2 (0.5)
|
|
3811
|
+
* ```
|
|
3812
|
+
*/
|
|
3813
|
+
export declare function sortTopKScores(scores: Array<number>, k: number, descending: boolean): Array<number>
|
|
3814
|
+
|
|
3815
|
+
/** Result of SQ4 quantization. */
|
|
3816
|
+
export interface Sq4QuantizeResult {
|
|
3817
|
+
/** Packed nibbles: two 4-bit values per byte. Length = ceil(dim / 2). */
|
|
3818
|
+
quantized: Buffer
|
|
3819
|
+
min: number
|
|
3820
|
+
max: number
|
|
3821
|
+
/** Original vector dimension (needed because packed length rounds up). */
|
|
3822
|
+
dim: number
|
|
3823
|
+
}
|
|
3824
|
+
|
|
3825
|
+
/** Result of SQ8 quantization. */
|
|
3826
|
+
export interface Sq8QuantizeResult {
|
|
3827
|
+
quantized: Buffer
|
|
3828
|
+
min: number
|
|
3829
|
+
max: number
|
|
3830
|
+
}
|
|
3831
|
+
|
|
3832
|
+
/**
|
|
3833
|
+
* Options for [`NativeGraphAdjacencyIndex::traverse`] — one native
|
|
3834
|
+
* multi-hop BFS over the live u64 adjacency that returns the whole
|
|
3835
|
+
* subgraph (nodes + edges + per-node depth) in a single boundary
|
|
3836
|
+
* crossing, replacing brainy's per-node-per-hop TS BFS. See
|
|
3837
|
+
* `cor/docs/ADR-006`.
|
|
3838
|
+
*/
|
|
3839
|
+
export interface TraverseOptions {
|
|
3840
|
+
/**
|
|
3841
|
+
* `"out"` (source→target), `"in"` (target→source), or `"both"`
|
|
3842
|
+
* (union). Any other value is treated as `"both"`.
|
|
3843
|
+
*/
|
|
3844
|
+
direction: string
|
|
3845
|
+
/**
|
|
3846
|
+
* Maximum hop count from the seed set. `0` returns just the seeds
|
|
3847
|
+
* (no edges). Each hop expands the current frontier one level.
|
|
3848
|
+
*/
|
|
3849
|
+
depth: number
|
|
3850
|
+
/**
|
|
3851
|
+
* Restrict to these `verb_type` indices (brainy's `VerbTypeEnum`
|
|
3852
|
+
* index, `0..VERB_TYPE_COUNT`). `None`/empty = every verb type.
|
|
3853
|
+
* Out-of-range indices are dropped. A captured edge's `edge_types`
|
|
3854
|
+
* entry is the matched type.
|
|
3855
|
+
*/
|
|
3856
|
+
verbTypes?: Array<number>
|
|
3857
|
+
/**
|
|
3858
|
+
* Cap on distinct nodes in the result (seeds included). When hit,
|
|
3859
|
+
* expansion stops and `truncated` is set.
|
|
3860
|
+
*/
|
|
3861
|
+
maxNodes?: number
|
|
3862
|
+
/**
|
|
3863
|
+
* Cap on edges in the result. When hit, expansion stops and
|
|
3864
|
+
* `truncated` is set.
|
|
3865
|
+
*/
|
|
3866
|
+
maxEdges?: number
|
|
3867
|
+
/**
|
|
3868
|
+
* Capture edges (default `true`). `false` returns nodes only — the
|
|
3869
|
+
* cheap "who is reachable" projection.
|
|
3870
|
+
*/
|
|
3871
|
+
includeEdges?: boolean
|
|
3872
|
+
/** Populate `node_depth` (default `true`). */
|
|
3873
|
+
includeDepth?: boolean
|
|
3874
|
+
}
|
|
3875
|
+
|
|
3876
|
+
/**
|
|
3877
|
+
* Validate a v2 license key.
|
|
3878
|
+
*
|
|
3879
|
+
* Steps:
|
|
3880
|
+
* 1. Strip `sc_cor_` prefix (or legacy `sc_cortex_`)
|
|
3881
|
+
* 2. Split JWT into header.payload.signature
|
|
3882
|
+
* 3. Base64url-decode each part
|
|
3883
|
+
* 4. Verify Ed25519 signature over "header.payload"
|
|
3884
|
+
* 5. Parse payload JSON
|
|
3885
|
+
* 6. Check product == "cor" (or legacy "cortex")
|
|
3886
|
+
* 7. Check expiry (0 = perpetual, else compare to now)
|
|
3887
|
+
* 8. Return result with tier info
|
|
3888
|
+
*/
|
|
3889
|
+
export declare function validateLicenseKey(token: string): LicenseResult
|
|
3890
|
+
|
|
3891
|
+
/**
|
|
3892
|
+
* **Cor 3.0.** Result from
|
|
3893
|
+
* [`get_verb_endpoints`](NativeGraphAdjacencyIndex::get_verb_endpoints).
|
|
3894
|
+
* Carries the verb's (source, target) entity ints as u64 BigInts so
|
|
3895
|
+
* brainy 8.0 callers can round-trip them via the brain's canonical
|
|
3896
|
+
* `BinaryIdMapper` without precision loss at 10B+ scale.
|
|
3897
|
+
*/
|
|
3898
|
+
export interface VerbEndpointsResult {
|
|
3899
|
+
/**
|
|
3900
|
+
* Source entity's canonical u64 int (from the brain's
|
|
3901
|
+
* `BinaryIdMapper`). Same value brainy passed to
|
|
3902
|
+
* `addVerbWithEndpoints*` at add time.
|
|
3903
|
+
*/
|
|
3904
|
+
sourceInt: bigint
|
|
3905
|
+
/**
|
|
3906
|
+
* Target entity's canonical u64 int (from the brain's
|
|
3907
|
+
* `BinaryIdMapper`).
|
|
3908
|
+
*/
|
|
3909
|
+
targetInt: bigint
|
|
3910
|
+
/**
|
|
3911
|
+
* The verb's `VerbType` index (generation-stable; ADR-006 Phase A).
|
|
3912
|
+
* Carried on the endpoint record so as-of-`g` reads can type an
|
|
3913
|
+
* edge whose verb was removed after `g`.
|
|
3914
|
+
*/
|
|
3915
|
+
verbType: number
|
|
3916
|
+
}
|