@soulcraft/cor 3.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +16 -0
- package/README.md +328 -0
- package/dist/aggregation/NativeAggregationEngine.d.ts +118 -0
- package/dist/aggregation/NativeAggregationEngine.js +186 -0
- package/dist/cli.d.ts +12 -0
- package/dist/cli.js +236 -0
- package/dist/graph/GraphAccelerationAdapter.d.ts +153 -0
- package/dist/graph/GraphAccelerationAdapter.js +326 -0
- package/dist/graph/NativeGraphAdjacencyIndex.d.ts +286 -0
- package/dist/graph/NativeGraphAdjacencyIndex.js +944 -0
- package/dist/hnsw/AdaptiveDiskAnnModeSelector.d.ts +187 -0
- package/dist/hnsw/AdaptiveDiskAnnModeSelector.js +313 -0
- package/dist/hnsw/NativeDiskAnnWrapper.d.ts +489 -0
- package/dist/hnsw/NativeDiskAnnWrapper.js +1456 -0
- package/dist/index.d.ts +28 -0
- package/dist/index.js +29 -0
- package/dist/legacyLayoutGuard.d.ts +93 -0
- package/dist/legacyLayoutGuard.js +237 -0
- package/dist/license/constants.d.ts +30 -0
- package/dist/license/constants.js +30 -0
- package/dist/license/onlineValidator.d.ts +43 -0
- package/dist/license/onlineValidator.js +225 -0
- package/dist/license/types.d.ts +39 -0
- package/dist/license/types.js +5 -0
- package/dist/license.d.ts +64 -0
- package/dist/license.js +165 -0
- package/dist/migration/MigrationCoordinator.d.ts +171 -0
- package/dist/migration/MigrationCoordinator.js +249 -0
- package/dist/migration/indexEpochGuard.d.ts +163 -0
- package/dist/migration/indexEpochGuard.js +202 -0
- package/dist/native/NativeEmbeddingEngine.d.ts +79 -0
- package/dist/native/NativeEmbeddingEngine.js +318 -0
- package/dist/native/NativeRoaringBitmap32.d.ts +114 -0
- package/dist/native/NativeRoaringBitmap32.js +221 -0
- package/dist/native/ffi.d.ts +20 -0
- package/dist/native/ffi.js +48 -0
- package/dist/native/idMapperTestSupport.d.ts +67 -0
- package/dist/native/idMapperTestSupport.js +112 -0
- package/dist/native/index.d.ts +49 -0
- package/dist/native/index.js +87 -0
- package/dist/native/napi.d.ts +21 -0
- package/dist/native/napi.js +88 -0
- package/dist/native/types.d.ts +1298 -0
- package/dist/native/types.js +16 -0
- package/dist/plugin.d.ts +50 -0
- package/dist/plugin.js +388 -0
- package/dist/providerContracts.d.ts +277 -0
- package/dist/providerContracts.js +38 -0
- package/dist/resource/OsMemoryProbe.d.ts +175 -0
- package/dist/resource/OsMemoryProbe.js +206 -0
- package/dist/resource/ResourceManager.d.ts +491 -0
- package/dist/resource/ResourceManager.js +960 -0
- package/dist/utils/ColumnManifest.d.ts +97 -0
- package/dist/utils/ColumnManifest.js +129 -0
- package/dist/utils/NativeColumnStore.d.ts +284 -0
- package/dist/utils/NativeColumnStore.js +685 -0
- package/dist/utils/NativeMetadataIndex.d.ts +882 -0
- package/dist/utils/NativeMetadataIndex.js +2631 -0
- package/dist/utils/NativeUnifiedCache.d.ts +87 -0
- package/dist/utils/NativeUnifiedCache.js +273 -0
- package/dist/utils/binaryIdMapperFactory.d.ts +59 -0
- package/dist/utils/binaryIdMapperFactory.js +94 -0
- package/dist/utils/collation.d.ts +30 -0
- package/dist/utils/collation.js +40 -0
- package/dist/utils/columnStoreTypes.d.ts +161 -0
- package/dist/utils/columnStoreTypes.js +29 -0
- package/dist/utils/nativeBinaryEntityIdMapper.d.ts +211 -0
- package/dist/utils/nativeBinaryEntityIdMapper.js +381 -0
- package/dist/utils/nativeEntityIdMapper.d.ts +111 -0
- package/dist/utils/nativeEntityIdMapper.js +170 -0
- package/docs/ADR-002-diskann-100-percent-rust.md +294 -0
- package/docs/ADR-003-semantic-time-travel.md +100 -0
- package/docs/aggregation.md +251 -0
- package/docs/comparison.md +162 -0
- package/docs/deployment-limits.md +87 -0
- package/docs/diskann.md +184 -0
- package/docs/migration-3.0.md +396 -0
- package/docs/performance-budget.md +274 -0
- package/docs/performance.md +117 -0
- package/docs/scaling.md +439 -0
- package/docs/snapshot-safety.md +246 -0
- package/docs/u64-id-space.md +214 -0
- package/docs/verification-report.md +670 -0
- package/native/brainy-native.node +0 -0
- package/native/index.d.ts +3916 -0
- package/package.json +92 -0
- package/scripts/migrate-cortex-2x-to-3x.mjs +970 -0
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module graph/GraphAccelerationAdapter
|
|
3
|
+
* @description The `'graphAcceleration'` provider — cor's native graph
|
|
4
|
+
* engine behind brainy 8.0's optional {@link GraphAccelerationProvider}
|
|
5
|
+
* contract (ADR-006). When brainy feature-detects this provider, the
|
|
6
|
+
* public `brain.graph.*` surface (`subgraph` / `export`) and the
|
|
7
|
+
* `related()` / `find({ connected })` paths route here instead of brainy's
|
|
8
|
+
* pure-TS adjacency fallback.
|
|
9
|
+
*
|
|
10
|
+
* **It wraps the SAME native handle the `graphIndex` provider already
|
|
11
|
+
* holds** — never a second `NativeGraphAdjacencyIndex` (which would split
|
|
12
|
+
* state + double RAM). brainy's `activate(context)` hook has no storage and
|
|
13
|
+
* its provider factories are called with storage only (not the graphIndex
|
|
14
|
+
* instance), and the `graphAcceleration` value is feature-detected raw
|
|
15
|
+
* (the factory is never invoked with storage), so this adapter is
|
|
16
|
+
* registered at `activate()` time bound to a *resolver* that the
|
|
17
|
+
* `graphIndex` factory fills once it constructs the native engine. Until
|
|
18
|
+
* the engine is built + loaded, {@link isInitialized} reports `false` and
|
|
19
|
+
* brainy keeps serving graph ops from TS — the contract's safety valve, so
|
|
20
|
+
* a read is never routed here before the data is loaded.
|
|
21
|
+
*
|
|
22
|
+
* **Scope (cor 3.0 / brainy 8.0):** the full `GraphAccelerationProvider`
|
|
23
|
+
* surface — `traverse`, `edgesForNode`, the streaming cursor
|
|
24
|
+
* (`graphCursorOpen` / `graphCursorNext` / `graphCursorClose`), AND the Phase C
|
|
25
|
+
* analytics (`pageRank`, `connectedComponents`, `shortestPath`,
|
|
26
|
+
* `neighborhoodSample`, `topByDegree`). All five analytics ship together (the
|
|
27
|
+
* feature-detect anchors on `traverse` + `graphCursorOpen`, but brainy does not
|
|
28
|
+
* per-method-guard the analytics — a partial set would throw). **`shortestPath`
|
|
29
|
+
* is hops-only** — cor stores no edge weights, so the adapter refuses
|
|
30
|
+
* `weight: 'edge'` and brainy must serve weighted paths from TS.
|
|
31
|
+
*
|
|
32
|
+
* **Zero-copy decode:** native reads return columnar little-endian
|
|
33
|
+
* {@link Buffer}s; this adapter views each as the matching typed array over
|
|
34
|
+
* the same memory (no element-wise copy) on the little-endian hosts cor
|
|
35
|
+
* targets, falling back to a copy only on the (never-expected) misaligned
|
|
36
|
+
* Buffer. brainy resolves the u64 node/verb ints to UUIDs lazily.
|
|
37
|
+
*
|
|
38
|
+
* **Not applied here:** `excludeVisibility` (the graph index carries no
|
|
39
|
+
* node-visibility data — that lives in the metadata/vector layer; brainy
|
|
40
|
+
* applies visibility on reassembly) and `subtypes` (cor's traverse filters
|
|
41
|
+
* by verb-type index only). Both are dropped at this seam, matching Phase A
|
|
42
|
+
* `traverse`.
|
|
43
|
+
*/
|
|
44
|
+
/** View a packed little-endian `u64` column as a `BigInt64Array` (the wire
|
|
45
|
+
* format brainy's {@link Subgraph} expects), zero-copy when the Buffer is
|
|
46
|
+
* 8-aligned (the napi `Vec`-backed norm), else via a one-time aligned copy. */
|
|
47
|
+
function viewBigInt64(buf) {
|
|
48
|
+
if (buf.byteLength === 0)
|
|
49
|
+
return new BigInt64Array(0);
|
|
50
|
+
if (buf.byteOffset % 8 === 0 && buf.byteLength % 8 === 0) {
|
|
51
|
+
return new BigInt64Array(buf.buffer, buf.byteOffset, buf.byteLength / 8);
|
|
52
|
+
}
|
|
53
|
+
const copy = new ArrayBuffer(buf.byteLength);
|
|
54
|
+
new Uint8Array(copy).set(buf);
|
|
55
|
+
return new BigInt64Array(copy);
|
|
56
|
+
}
|
|
57
|
+
/** View a packed little-endian `u16` column as a `Uint16Array`. */
|
|
58
|
+
function viewUint16(buf) {
|
|
59
|
+
if (buf.byteLength === 0)
|
|
60
|
+
return new Uint16Array(0);
|
|
61
|
+
if (buf.byteOffset % 2 === 0 && buf.byteLength % 2 === 0) {
|
|
62
|
+
return new Uint16Array(buf.buffer, buf.byteOffset, buf.byteLength / 2);
|
|
63
|
+
}
|
|
64
|
+
const copy = new ArrayBuffer(buf.byteLength);
|
|
65
|
+
new Uint8Array(copy).set(buf);
|
|
66
|
+
return new Uint16Array(copy);
|
|
67
|
+
}
|
|
68
|
+
/** View a `u8` column as a `Uint8Array` (always byte-aligned → zero-copy). */
|
|
69
|
+
function viewUint8(buf) {
|
|
70
|
+
if (buf.byteLength === 0)
|
|
71
|
+
return new Uint8Array(0);
|
|
72
|
+
return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
73
|
+
}
|
|
74
|
+
/** View a packed little-endian `f64` column as a `Float64Array` (brainy's
|
|
75
|
+
* `GraphScores.scores`), zero-copy when 8-aligned else via an aligned copy. */
|
|
76
|
+
function viewFloat64(buf) {
|
|
77
|
+
if (buf.byteLength === 0)
|
|
78
|
+
return new Float64Array(0);
|
|
79
|
+
if (buf.byteOffset % 8 === 0 && buf.byteLength % 8 === 0) {
|
|
80
|
+
return new Float64Array(buf.buffer, buf.byteOffset, buf.byteLength / 8);
|
|
81
|
+
}
|
|
82
|
+
const copy = new ArrayBuffer(buf.byteLength);
|
|
83
|
+
new Uint8Array(copy).set(buf);
|
|
84
|
+
return new Float64Array(copy);
|
|
85
|
+
}
|
|
86
|
+
/** View a packed little-endian `u32` column as a `Uint32Array` (brainy's
|
|
87
|
+
* `GraphComponents.componentIds`), zero-copy when 4-aligned else via a copy. */
|
|
88
|
+
function viewUint32(buf) {
|
|
89
|
+
if (buf.byteLength === 0)
|
|
90
|
+
return new Uint32Array(0);
|
|
91
|
+
if (buf.byteOffset % 4 === 0 && buf.byteLength % 4 === 0) {
|
|
92
|
+
return new Uint32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
|
|
93
|
+
}
|
|
94
|
+
const copy = new ArrayBuffer(buf.byteLength);
|
|
95
|
+
new Uint8Array(copy).set(buf);
|
|
96
|
+
return new Uint32Array(copy);
|
|
97
|
+
}
|
|
98
|
+
/** Map a native columnar {@link NativeSubgraph} to brainy's {@link Subgraph}
|
|
99
|
+
* typed-array shape. `nodeDepth` is omitted when the native column is empty
|
|
100
|
+
* (the cursor's flat export, or `includeDepth: false`). */
|
|
101
|
+
function decodeSubgraph(sg) {
|
|
102
|
+
const out = {
|
|
103
|
+
nodes: viewBigInt64(sg.nodes),
|
|
104
|
+
edgeSources: viewBigInt64(sg.edgeSources),
|
|
105
|
+
edgeTargets: viewBigInt64(sg.edgeTargets),
|
|
106
|
+
edgeVerbInts: viewBigInt64(sg.edgeVerbInts),
|
|
107
|
+
edgeTypes: viewUint16(sg.edgeTypes),
|
|
108
|
+
truncated: sg.truncated,
|
|
109
|
+
};
|
|
110
|
+
if (sg.nodeDepth.byteLength > 0) {
|
|
111
|
+
out.nodeDepth = viewUint8(sg.nodeDepth);
|
|
112
|
+
}
|
|
113
|
+
return out;
|
|
114
|
+
}
|
|
115
|
+
/** Wrap a `Uint8Array` ({@link OpaqueIdSet}) as a `Buffer` over the same
|
|
116
|
+
* memory (zero-copy) for the napi seed-set parameter. */
|
|
117
|
+
function asBuffer(u8) {
|
|
118
|
+
return Buffer.from(u8.buffer, u8.byteOffset, u8.byteLength);
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* @description cor's `'graphAcceleration'` provider. Registered structurally
|
|
122
|
+
* (NOT via `implements GraphAccelerationProvider`, which would force the 5
|
|
123
|
+
* Phase-C analytics methods to exist) — brainy validates it at runtime via
|
|
124
|
+
* the two-anchor feature-detect (`traverse` + `graphCursorOpen`).
|
|
125
|
+
*/
|
|
126
|
+
export class GraphAccelerationAdapter {
|
|
127
|
+
resolveEngine;
|
|
128
|
+
/**
|
|
129
|
+
* @param resolveEngine - Resolver for the shared `graphIndex` provider
|
|
130
|
+
* instance. Returns `undefined` until the `graphIndex` factory has
|
|
131
|
+
* constructed the native engine; the adapter reports
|
|
132
|
+
* {@link isInitialized} `false` until then so brainy doesn't route here.
|
|
133
|
+
*/
|
|
134
|
+
constructor(resolveEngine) {
|
|
135
|
+
this.resolveEngine = resolveEngine;
|
|
136
|
+
}
|
|
137
|
+
/** The shared native graph engine, or `undefined` before construction. */
|
|
138
|
+
get engine() {
|
|
139
|
+
return this.resolveEngine();
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* `false` until the shared native graph engine is constructed AND has
|
|
143
|
+
* loaded its persisted state. brainy probes this before routing, so a
|
|
144
|
+
* read is never served from native before the data is present.
|
|
145
|
+
*/
|
|
146
|
+
get isInitialized() {
|
|
147
|
+
const engine = this.engine;
|
|
148
|
+
return engine !== undefined && engine.isInitialized;
|
|
149
|
+
}
|
|
150
|
+
/** Resolve the native instance or throw a clear error (defensive — brainy
|
|
151
|
+
* gates on {@link isInitialized}, so this should never fire in practice). */
|
|
152
|
+
native() {
|
|
153
|
+
const engine = this.engine;
|
|
154
|
+
if (engine === undefined) {
|
|
155
|
+
throw new Error('GraphAccelerationAdapter: native graph engine not yet constructed ' +
|
|
156
|
+
'(graphIndex provider not initialized) — brainy should gate on isInitialized');
|
|
157
|
+
}
|
|
158
|
+
return engine.nativeInstance;
|
|
159
|
+
}
|
|
160
|
+
/** Translate brainy's {@link TraverseOptions} to the native shape (drops
|
|
161
|
+
* `subtypes` / `excludeVisibility`, which this seam does not apply). */
|
|
162
|
+
toNativeTraverse(options) {
|
|
163
|
+
return {
|
|
164
|
+
direction: options.direction ?? 'both',
|
|
165
|
+
depth: options.depth,
|
|
166
|
+
verbTypes: options.verbTypes,
|
|
167
|
+
maxNodes: options.maxNodes,
|
|
168
|
+
maxEdges: options.maxEdges,
|
|
169
|
+
includeEdges: options.includeEdges,
|
|
170
|
+
includeDepth: options.includeDepth,
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* @description Bounded multi-hop expansion from `seeds` (entity ints or a
|
|
175
|
+
* `find()` {@link OpaqueIdSet}), returning the reachable subgraph in one
|
|
176
|
+
* native call.
|
|
177
|
+
* @param seeds - Start nodes as `bigint[]`, or an OpaqueIdSet envelope
|
|
178
|
+
* (forwarded with no id materialization — the query→expand fusion).
|
|
179
|
+
* @param options - Depth, direction, verb-type filter, caps.
|
|
180
|
+
* @param generation - Optional as-of generation (omitted = now).
|
|
181
|
+
*/
|
|
182
|
+
async traverse(seeds, options, generation) {
|
|
183
|
+
const opts = this.toNativeTraverse(options);
|
|
184
|
+
const sg = Array.isArray(seeds)
|
|
185
|
+
? this.native().traverse(seeds, opts, generation ?? null)
|
|
186
|
+
: this.native().traverseFromOpaqueIdSet(asBuffer(seeds), opts, generation ?? null);
|
|
187
|
+
return decodeSubgraph(sg);
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* @description All edges incident to one node, both directions — a
|
|
191
|
+
* depth-1 {@link traverse} from the single node.
|
|
192
|
+
* @param nodeInt - The node's interned entity int.
|
|
193
|
+
* @param options - Direction, verb-type filter, limit.
|
|
194
|
+
* @param generation - Optional as-of generation.
|
|
195
|
+
*/
|
|
196
|
+
async edgesForNode(nodeInt, options, generation) {
|
|
197
|
+
const opts = {
|
|
198
|
+
direction: options.direction ?? 'both',
|
|
199
|
+
depth: 1,
|
|
200
|
+
verbTypes: options.verbTypes,
|
|
201
|
+
maxEdges: options.limit,
|
|
202
|
+
includeEdges: true,
|
|
203
|
+
includeDepth: false,
|
|
204
|
+
};
|
|
205
|
+
return decodeSubgraph(this.native().traverse([nodeInt], opts, generation ?? null));
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* @description Open a snapshot-consistent streaming walk of the whole
|
|
209
|
+
* graph (or a seeded region), pinned to a generation. Returns a
|
|
210
|
+
* TTL-bounded handle for {@link graphCursorNext} / {@link graphCursorClose}.
|
|
211
|
+
* @param options - Direction, projection, optional seeds / resume cursor.
|
|
212
|
+
* @param generation - Optional explicit as-of generation to pin.
|
|
213
|
+
*/
|
|
214
|
+
async graphCursorOpen(options, generation) {
|
|
215
|
+
const nativeOpts = {
|
|
216
|
+
direction: options.direction,
|
|
217
|
+
projection: options.projection,
|
|
218
|
+
cursor: options.cursor,
|
|
219
|
+
generation: generation ?? undefined,
|
|
220
|
+
};
|
|
221
|
+
if (options.seeds !== undefined) {
|
|
222
|
+
if (Array.isArray(options.seeds)) {
|
|
223
|
+
nativeOpts.seeds = options.seeds;
|
|
224
|
+
}
|
|
225
|
+
else {
|
|
226
|
+
nativeOpts.seedsOpaque = asBuffer(options.seeds);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
return this.native().graphCursorOpen(nativeOpts);
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* @description Pull the next chunk from an open cursor.
|
|
233
|
+
* @param handle - From {@link graphCursorOpen}.
|
|
234
|
+
* @param chunkSize - Target number of edges in this chunk.
|
|
235
|
+
* @throws An error whose message starts `SnapshotExpired:` if the handle
|
|
236
|
+
* sat idle past its TTL — reopen with the last chunk's `cursor` to resume.
|
|
237
|
+
*/
|
|
238
|
+
async graphCursorNext(handle, chunkSize) {
|
|
239
|
+
const chunk = this.native().graphCursorNext(handle, chunkSize);
|
|
240
|
+
return {
|
|
241
|
+
subgraph: decodeSubgraph(chunk.subgraph),
|
|
242
|
+
cursor: chunk.cursor,
|
|
243
|
+
done: chunk.done,
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* @description Release an open cursor's pinned generation + server-side
|
|
248
|
+
* state. Idempotent.
|
|
249
|
+
* @param handle - From {@link graphCursorOpen}.
|
|
250
|
+
*/
|
|
251
|
+
async graphCursorClose(handle) {
|
|
252
|
+
this.native().graphCursorClose(handle);
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* @description `rank` — importance/influence ranking over the visible graph.
|
|
256
|
+
* Intent-level: cor implements it natively via PageRank (algorithm is the
|
|
257
|
+
* provider's choice). `excludeVisibility` is dropped at this seam — brainy
|
|
258
|
+
* post-filters (and renormalizes/re-top-Ks) on hydrate; the native rank is
|
|
259
|
+
* over all nodes. (Renamed from `pageRank` for brainy rc.3 intent contract, #71.)
|
|
260
|
+
*/
|
|
261
|
+
async rank(options, generation) {
|
|
262
|
+
const r = this.native().pageRank({ topK: options.topK }, generation ?? null);
|
|
263
|
+
return { nodeInts: viewBigInt64(r.nodeInts), scores: viewFloat64(r.scores) };
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* @description `communities` — group related nodes. Intent-level: cor
|
|
267
|
+
* implements it natively via connected components (`directed` → strong/SCC,
|
|
268
|
+
* else weak). The labels are arbitrary-but-stable — only the partition is
|
|
269
|
+
* meaningful; do not cache them as keys. (Renamed from `connectedComponents`, #71.)
|
|
270
|
+
*/
|
|
271
|
+
async communities(options, generation) {
|
|
272
|
+
const r = this.native().connectedComponents({ mode: options.directed ? 'strong' : 'weak' }, generation ?? null);
|
|
273
|
+
return {
|
|
274
|
+
nodeInts: viewBigInt64(r.nodeInts),
|
|
275
|
+
communityIds: viewUint32(r.componentIds),
|
|
276
|
+
communityCount: r.componentCount,
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
/**
|
|
280
|
+
* @description `path` — best route between two nodes. Intent-level: cor
|
|
281
|
+
* implements hops natively. cor stores no edge weights, so `by: 'weight'` is
|
|
282
|
+
* unsupported here — brainy serves least-weight paths from its TS fallback.
|
|
283
|
+
* Returns `null` when unreachable. (Renamed from `shortestPath`; `weight` →
|
|
284
|
+
* `by` for brainy rc.3, #71.)
|
|
285
|
+
* @throws If `options.by === 'weight'`.
|
|
286
|
+
*/
|
|
287
|
+
async path(fromInt, toInt, options, generation) {
|
|
288
|
+
if (options.by === 'weight') {
|
|
289
|
+
throw new Error('GraphAccelerationAdapter.path: by:"weight" is unsupported — cor stores no edge ' +
|
|
290
|
+
'weights; brainy must serve least-weight paths from its TS fallback');
|
|
291
|
+
}
|
|
292
|
+
const r = this.native().shortestPath(fromInt, toInt, { direction: options.direction, verbTypes: options.verbTypes, maxDepth: options.maxDepth }, generation ?? null);
|
|
293
|
+
return r === null
|
|
294
|
+
? null
|
|
295
|
+
: { nodeInts: viewBigInt64(r.nodeInts), edgeVerbInts: viewBigInt64(r.edgeVerbInts), cost: r.cost };
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* @description `sample` — deterministic, seeded bounded neighborhood sample for
|
|
299
|
+
* dense-graph viz. Intent-level; cor implements it natively. (Renamed from
|
|
300
|
+
* `neighborhoodSample`, #71.)
|
|
301
|
+
* @param seeds - `bigint[]` or a `find()` OpaqueIdSet.
|
|
302
|
+
*/
|
|
303
|
+
async sample(seeds, options, generation) {
|
|
304
|
+
const opts = {
|
|
305
|
+
direction: options.direction,
|
|
306
|
+
depth: options.depth,
|
|
307
|
+
fanout: options.fanout,
|
|
308
|
+
seed: options.seed,
|
|
309
|
+
maxNodes: options.maxNodes,
|
|
310
|
+
};
|
|
311
|
+
const r = Array.isArray(seeds)
|
|
312
|
+
? this.native().neighborhoodSample(seeds, null, opts, generation ?? null)
|
|
313
|
+
: this.native().neighborhoodSample([], asBuffer(seeds), opts, generation ?? null);
|
|
314
|
+
return decodeSubgraph(r);
|
|
315
|
+
}
|
|
316
|
+
/**
|
|
317
|
+
* @description `mostConnected` — the top-K most-connected nodes. Intent-level:
|
|
318
|
+
* cor implements it natively via degree. `excludeVisibility` dropped at this
|
|
319
|
+
* seam (brainy post-filters). (Renamed from `topByDegree`, #71.)
|
|
320
|
+
*/
|
|
321
|
+
async mostConnected(options, generation) {
|
|
322
|
+
const r = this.native().topByDegree({ direction: options.direction, topK: options.topK }, generation ?? null);
|
|
323
|
+
return { nodeInts: viewBigInt64(r.nodeInts), scores: viewFloat64(r.scores) };
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
//# sourceMappingURL=GraphAccelerationAdapter.js.map
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NativeGraphAdjacencyIndex — Thin TS wrapper around the Rust graph engine
|
|
3
|
+
*
|
|
4
|
+
* Handles async storage I/O while the Rust engine handles all in-memory
|
|
5
|
+
* data operations (4 LSM-trees, verb tracking, relationship counts).
|
|
6
|
+
*
|
|
7
|
+
* Storage pattern: adapter-controlled I/O with 3-tier fallback.
|
|
8
|
+
* - Tier 1 (mmap): saveBinaryBlob + getBinaryBlobPath → zero-copy queries
|
|
9
|
+
* - Tier 2 (binary blob): saveBinaryBlob only → binary format in memory
|
|
10
|
+
* - Tier 3 (legacy): saveMetadata → base64 JSON
|
|
11
|
+
*
|
|
12
|
+
* UnifiedCache integration stays in TS (getVerbCached, getVerbsBatchCached)
|
|
13
|
+
* since it coordinates cross-subsystem caching and requires async storage access.
|
|
14
|
+
*/
|
|
15
|
+
import type { GraphVerb, StorageAdapter } from '@soulcraft/brainy';
|
|
16
|
+
import type { NativeGraphAdjacencyInstance } from '../native/types.js';
|
|
17
|
+
import type { GraphIndexProvider } from '../providerContracts.js';
|
|
18
|
+
import type { EntityIdMapperLike } from '../utils/columnStoreTypes.js';
|
|
19
|
+
import type { MigrationCoordinator, MigrationStatus } from '../migration/MigrationCoordinator.js';
|
|
20
|
+
export interface GraphIndexConfig {
|
|
21
|
+
maxIndexSize?: number;
|
|
22
|
+
rebuildThreshold?: number;
|
|
23
|
+
autoOptimize?: boolean;
|
|
24
|
+
flushInterval?: number;
|
|
25
|
+
memTableThreshold?: number;
|
|
26
|
+
maxSSTablesPerLevel?: number;
|
|
27
|
+
/**
|
|
28
|
+
* **Piece 12.** Optional directory for the native verb-ID
|
|
29
|
+
* interning namespace. When set, the `VerbIdNamespace` writes
|
|
30
|
+
* `verb-uuid-to-int.mkv` + `verb-int-to-uuid.bin` under this
|
|
31
|
+
* directory so verb-ID interning survives process restart — a
|
|
32
|
+
* verb-ID re-encountered after reopen lands at the same internal
|
|
33
|
+
* u32, which keeps the Roaring32 membership bitmap consistent
|
|
34
|
+
* across restarts.
|
|
35
|
+
*
|
|
36
|
+
* When omitted, the constructor auto-derives it from the storage
|
|
37
|
+
* adapter for filesystem-backed brains (see
|
|
38
|
+
* {@link GRAPH_VERB_NS_PROBE_KEY}), so the durable mmap namespace is
|
|
39
|
+
* the default whenever a real on-disk path exists — required for
|
|
40
|
+
* `related()` / `getVerbsBySource()` to resolve verb ids (the
|
|
41
|
+
* in-memory reverse map is a no-op) and for O(working-set) RAM at
|
|
42
|
+
* billion-edge scale. It only falls back to the 64-shard in-memory
|
|
43
|
+
* interner when the adapter has no local path (cloud / memory
|
|
44
|
+
* storage — task #26 materializes a scratch dir there) or for a
|
|
45
|
+
* transient in-process index built directly against `MemoryStorage`.
|
|
46
|
+
*/
|
|
47
|
+
persistenceDir?: string;
|
|
48
|
+
}
|
|
49
|
+
export interface GraphIndexStats {
|
|
50
|
+
totalRelationships: number;
|
|
51
|
+
sourceNodes: number;
|
|
52
|
+
targetNodes: number;
|
|
53
|
+
memoryUsage: number;
|
|
54
|
+
lastRebuild: number;
|
|
55
|
+
rebuildTime: number;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Native 4-LSM-tree graph adjacency index. Implements {@link GraphIndexProvider}
|
|
59
|
+
* so a compile error fires the moment this wrapper drops or drifts a member
|
|
60
|
+
* brainy calls via `this.graphIndex.*` (e.g. the `isInitialized` getter).
|
|
61
|
+
*/
|
|
62
|
+
export declare class GraphAdjacencyIndex implements GraphIndexProvider {
|
|
63
|
+
/**
|
|
64
|
+
* Lazy resolver to the shared {@link MigrationCoordinator} — the same getter
|
|
65
|
+
* the metadata + vector providers receive, so Brainy's feature-detected lock
|
|
66
|
+
* methods return a consistent answer across all three. Undefined when no
|
|
67
|
+
* migration path is wired.
|
|
68
|
+
*/
|
|
69
|
+
private readonly migrationCoordinatorGetter?;
|
|
70
|
+
private storage;
|
|
71
|
+
private native;
|
|
72
|
+
private unifiedCache;
|
|
73
|
+
private config;
|
|
74
|
+
private initialized;
|
|
75
|
+
private isRebuilding;
|
|
76
|
+
private flushTimer?;
|
|
77
|
+
private rebuildStartTime;
|
|
78
|
+
private totalRelationshipsIndexed;
|
|
79
|
+
private isCompacting;
|
|
80
|
+
private hasBinaryBlobs;
|
|
81
|
+
private hasMmapPaths;
|
|
82
|
+
/**
|
|
83
|
+
* Lazy resolver to the shared entity-id mapper (the metadata index's mapper —
|
|
84
|
+
* the one brainy resolves verb endpoints through at write time). Used ONLY by
|
|
85
|
+
* `rebuild()` to turn persisted `verb.sourceId`/`targetId` UUIDs back into the
|
|
86
|
+
* entity ints the 4 LSM adjacency trees are keyed on. Lazy because the
|
|
87
|
+
* metadata-index instance may be constructed after this one; it is resolved at
|
|
88
|
+
* rebuild time (post-init), by when both providers exist. `undefined` when cor
|
|
89
|
+
* is used without a metadata index — `rebuild()` then degrades to the
|
|
90
|
+
* membership-only pass and says so. (#68)
|
|
91
|
+
*/
|
|
92
|
+
private mapperResolver?;
|
|
93
|
+
get isInitialized(): boolean;
|
|
94
|
+
/**
|
|
95
|
+
* The underlying native graph engine handle. Exposed so the
|
|
96
|
+
* `graphAcceleration` provider adapter ({@link GraphAccelerationAdapter})
|
|
97
|
+
* can drive `traverse` / the streaming cursor against the SAME native
|
|
98
|
+
* instance this provider already holds — never a second index (which
|
|
99
|
+
* would split state + double RAM). Read-only by contract: the adapter
|
|
100
|
+
* only calls read/cursor methods on it.
|
|
101
|
+
*/
|
|
102
|
+
get nativeInstance(): NativeGraphAdjacencyInstance;
|
|
103
|
+
constructor(storage: StorageAdapter, config?: GraphIndexConfig, mapperResolver?: () => EntityIdMapperLike | undefined,
|
|
104
|
+
/**
|
|
105
|
+
* Lazy resolver to the shared {@link MigrationCoordinator} — the same getter
|
|
106
|
+
* the metadata + vector providers receive, so Brainy's feature-detected lock
|
|
107
|
+
* methods return a consistent answer across all three. Undefined when no
|
|
108
|
+
* migration path is wired.
|
|
109
|
+
*/
|
|
110
|
+
migrationCoordinatorGetter?: (() => MigrationCoordinator | undefined) | undefined);
|
|
111
|
+
/**
|
|
112
|
+
* **Eager cold-load entry — brainy 8.0 rebuild gate (#36).** Brainy
|
|
113
|
+
* awaits this right after `metadataIndex.init()` and BEFORE it consults
|
|
114
|
+
* {@link isReady}, so the durable graph is restored (the four u64 LSM
|
|
115
|
+
* trees cold-loaded from their `.sst` blobs) before the gate reads
|
|
116
|
+
* readiness — otherwise `isReady()` would be `false` until the first
|
|
117
|
+
* lazy op and brainy would needlessly rebuild a graph that is already on
|
|
118
|
+
* disk. Idempotent and a no-op for an in-memory index; mirrors
|
|
119
|
+
* {@link MetadataIndexProvider.init}. Delegates to the same
|
|
120
|
+
* {@link ensureInitialized} guard the lazy read/mutation paths use, so
|
|
121
|
+
* an index that is never `init()`-ed still cold-loads on first touch.
|
|
122
|
+
*/
|
|
123
|
+
init(): Promise<void>;
|
|
124
|
+
/**
|
|
125
|
+
* Idempotent lazy-init guard. Runs the cold-load
|
|
126
|
+
* ({@link openGraphLsmEngine}) exactly once, then arms auto-flush. Both
|
|
127
|
+
* the eager {@link init} and every read/mutation path funnel through
|
|
128
|
+
* here, so a caller that skips `init()` still gets a fully cold-loaded
|
|
129
|
+
* graph on first touch.
|
|
130
|
+
*/
|
|
131
|
+
private ensureInitialized;
|
|
132
|
+
/**
|
|
133
|
+
* Cold-load the four u64 LSM trees from persistence: load each tree's
|
|
134
|
+
* manifest and register every persisted SSTable (mmap tier, then binary
|
|
135
|
+
* blob tier) so the adjacency is queryable with NO rebuild. Each SSTable
|
|
136
|
+
* load latches the native `is_ready()` flag, so {@link isReady} reports
|
|
137
|
+
* `true` the instant durable edges are mapped in.
|
|
138
|
+
*
|
|
139
|
+
* The trailing membership-consistency walk is the **deferred O(N)
|
|
140
|
+
* fast-follow** (#19): it repopulates the verb-id membership bitmap when
|
|
141
|
+
* the trees hold edges but membership came back empty. It is explicitly
|
|
142
|
+
* NOT part of readiness — `isReady()` is already `true` from the SSTable
|
|
143
|
+
* loads above, so a slow membership walk can never make a graph with
|
|
144
|
+
* queryable edges report "not ready".
|
|
145
|
+
*/
|
|
146
|
+
private openGraphLsmEngine;
|
|
147
|
+
private populateVerbIdSetFromStorage;
|
|
148
|
+
getNeighbors(id: bigint, optionsOrDirection?: {
|
|
149
|
+
direction?: 'in' | 'out' | 'both';
|
|
150
|
+
limit?: number;
|
|
151
|
+
offset?: number;
|
|
152
|
+
} | 'in' | 'out' | 'both'): Promise<bigint[]>;
|
|
153
|
+
getVerbIdsBySource(sourceInt: bigint, options?: {
|
|
154
|
+
limit?: number;
|
|
155
|
+
offset?: number;
|
|
156
|
+
}): Promise<bigint[]>;
|
|
157
|
+
getVerbIdsByTarget(targetInt: bigint, options?: {
|
|
158
|
+
limit?: number;
|
|
159
|
+
offset?: number;
|
|
160
|
+
}): Promise<bigint[]>;
|
|
161
|
+
getVerbCached(verbId: string): Promise<GraphVerb | null>;
|
|
162
|
+
getVerbsBatchCached(verbIds: string[]): Promise<Map<string, GraphVerb>>;
|
|
163
|
+
/**
|
|
164
|
+
* Add a verb to the graph. Cor 3.0 Piece D requires the brain's
|
|
165
|
+
* canonical `(sourceInt, targetInt)` entity-int pair as explicit
|
|
166
|
+
* parameters — brainy 8.0 resolves these via its `BinaryIdMapper`
|
|
167
|
+
* before calling. The `verb` object provides the verb-ID, verb-type,
|
|
168
|
+
* and optional subtype; entity UUIDs on the verb (`verb.sourceId`,
|
|
169
|
+
* `verb.targetId`) are no longer used by cor.
|
|
170
|
+
*/
|
|
171
|
+
/**
|
|
172
|
+
* Index a new verb (relationship). Returns the verb's interned
|
|
173
|
+
* `u64` integer ID so brainy 8.0's `verb_int ↔ verb_id_string`
|
|
174
|
+
* sidecar map populates without a follow-up
|
|
175
|
+
* `getVerbIntForString` FFI roundtrip — one boundary crossing
|
|
176
|
+
* saved per add at billion-scale ingest (PLATFORM-HANDOFF
|
|
177
|
+
* BRAINY-8.0-RENAME-COORDINATION A.3).
|
|
178
|
+
*
|
|
179
|
+
* The wrapper's previous brainy 7.x signature was
|
|
180
|
+
* `Promise<void>`; brainy 7.x callers that ignore the new
|
|
181
|
+
* return value continue to work unchanged.
|
|
182
|
+
*/
|
|
183
|
+
addVerb(verb: GraphVerb, sourceInt: bigint, targetInt: bigint, generation: bigint): Promise<bigint>;
|
|
184
|
+
removeVerb(verbId: string, generation: bigint): Promise<void>;
|
|
185
|
+
size(): number;
|
|
186
|
+
getRelationshipCountByType(type: string): number;
|
|
187
|
+
getTotalRelationshipCount(): number;
|
|
188
|
+
getAllRelationshipCounts(): Map<string, number>;
|
|
189
|
+
/**
|
|
190
|
+
* **Batch verb-int reverse resolver** — brainy 8.0
|
|
191
|
+
* `GraphIndexProvider.verbIntsToIds` contract (handoff Section L.7).
|
|
192
|
+
*
|
|
193
|
+
* Given a batch of verb-int BigInts, return the canonical UUID
|
|
194
|
+
* string for each — `null` for any int that has no entry in
|
|
195
|
+
* cor's persistent reverse map.
|
|
196
|
+
*
|
|
197
|
+
* Cor's namespace is the single source of truth for the
|
|
198
|
+
* `verb-int ↔ verb-id` mapping (the persisted `verb-int-to-uuid.bin`
|
|
199
|
+
* is direct-indexed for O(1) random reads). Brainy keeps only a
|
|
200
|
+
* bounded in-memory LRU warm cache fed by `addVerb` returns and
|
|
201
|
+
* resolver hits — no full reverse map on the JS side.
|
|
202
|
+
*
|
|
203
|
+
* The identity-fingerprint design (handoff L.7) makes this
|
|
204
|
+
* lossless: brainy's `relate()` validates verb-ids are canonical
|
|
205
|
+
* UUIDs, cor stores the raw 16 UUID bytes, and `verbIntsToIds`
|
|
206
|
+
* formats them back to the canonical lowercase 8-4-4-4-12 form.
|
|
207
|
+
*/
|
|
208
|
+
verbIntsToIds(verbInts: bigint[]): Promise<(string | null)[]>;
|
|
209
|
+
getRelationshipStats(): {
|
|
210
|
+
totalRelationships: number;
|
|
211
|
+
relationshipsByType: Record<string, number>;
|
|
212
|
+
uniqueSourceNodes: number;
|
|
213
|
+
uniqueTargetNodes: number;
|
|
214
|
+
totalNodes: number;
|
|
215
|
+
};
|
|
216
|
+
getStats(): GraphIndexStats;
|
|
217
|
+
isHealthy(): boolean;
|
|
218
|
+
/**
|
|
219
|
+
* Whether the graph's edges are loaded and queryable WITHOUT a rebuild —
|
|
220
|
+
* brainy 8.0's rebuild gate (#36). Delegates straight to the native
|
|
221
|
+
* edges-ready latch (set on SSTable cold-load, a non-empty manifest, or a
|
|
222
|
+
* memtable flush) ORed with a live non-empty memtable. Synchronous (no
|
|
223
|
+
* await) and **never gated on the verb-id membership walk**, so the
|
|
224
|
+
* deferred O(N) membership fast-follow can never depress readiness for a
|
|
225
|
+
* graph whose edges are already mapped in.
|
|
226
|
+
*/
|
|
227
|
+
isReady(): boolean;
|
|
228
|
+
/**
|
|
229
|
+
* @description **#18 migration lock — feature-detected by Brainy.** `true` while
|
|
230
|
+
* the brain is undergoing (or has failed) the coordinated 7.x → 8.0 rebuild, so
|
|
231
|
+
* Brainy holds reads+writes behind the lock (and, in rc.9, `verifyGraphAdjacencyLive`
|
|
232
|
+
* short-circuits to `'live'` rather than firing its own rebuild-on-read).
|
|
233
|
+
* Delegates to the shared {@link MigrationCoordinator}; `false` when none is wired.
|
|
234
|
+
* @returns `true` when the brain must not be read from or written to.
|
|
235
|
+
*/
|
|
236
|
+
isMigrating(): boolean;
|
|
237
|
+
/**
|
|
238
|
+
* @description **#18 migration lock — structured progress** Brainy relays into
|
|
239
|
+
* `getIndexStatus().migration`. `null` unless a migration is in flight.
|
|
240
|
+
* @returns The current {@link MigrationStatus}, or `null`.
|
|
241
|
+
*/
|
|
242
|
+
migrationStatus(): MigrationStatus | null;
|
|
243
|
+
/**
|
|
244
|
+
* Rebuild the graph adjacency from brainy's persisted verbs — used after a
|
|
245
|
+
* restore/migration that wiped the native subdirs (#68).
|
|
246
|
+
*
|
|
247
|
+
* **Full adjacency reconstruction** when a shared entity-id mapper resolver is
|
|
248
|
+
* wired (the metadata index's mapper — the SAME one brainy resolves verb
|
|
249
|
+
* endpoints through at write time, `metadataIndex.getIdMapper().getOrAssign`).
|
|
250
|
+
* For each persisted verb this resolves `sourceId`/`targetId` → entity ints via
|
|
251
|
+
* that mapper and replays `addVerbWithEndpointsByIndex`, rebuilding all four
|
|
252
|
+
* u64-keyed LSM trees exactly as the live write path does (which also
|
|
253
|
+
* repopulates the membership bitmap + per-type counts — so `trackVerbId` is NOT
|
|
254
|
+
* also called for a reconstructed edge, or membership/counts would double). Brainy
|
|
255
|
+
* 8.0 `restore()` awaits `metadataIndex.getIdMapper().rebuild()` BEFORE the
|
|
256
|
+
* `Promise.all` that runs this, so the mapper is fully populated with the
|
|
257
|
+
* snapshot's assignments before endpoints are resolved here.
|
|
258
|
+
*
|
|
259
|
+
* Cursor-paginated (#74) — O(batch) RAM at any edge count; memtable flush
|
|
260
|
+
* thresholds are respected mid-walk so RAM stays bounded at billion-edge scale.
|
|
261
|
+
* Endpoints whose UUID does not resolve in the mapper (an entity that is a graph
|
|
262
|
+
* node but was not walked as a noun) are tracked for membership, counted, and
|
|
263
|
+
* warned — never silently dropped. Restored edges reconstruct at the floor
|
|
264
|
+
* generation (0): present-state is exact; historical as-of-g for pre-restore
|
|
265
|
+
* edges is not reconstructable from canonical records (the same limit the
|
|
266
|
+
* metadata + vector rebuilds carry).
|
|
267
|
+
*
|
|
268
|
+
* Without a mapper resolver (cor used without a metadata index), it degrades to
|
|
269
|
+
* the membership-only pass (bitmap + per-type counts via `trackVerbId`) and logs
|
|
270
|
+
* that adjacency was not reconstructed — the on-disk `.sst` trees loaded in
|
|
271
|
+
* {@link ensureInitialized} remain the adjacency source in that case.
|
|
272
|
+
*/
|
|
273
|
+
rebuild(): Promise<void>;
|
|
274
|
+
flush(): Promise<void>;
|
|
275
|
+
close(): Promise<void>;
|
|
276
|
+
/** Blob key for an SSTable: "graph-lsm/{treeName}/{sstableId}" */
|
|
277
|
+
private blobKey;
|
|
278
|
+
private loadTreeFromStorage;
|
|
279
|
+
private loadTreeSSTables;
|
|
280
|
+
private flushTree;
|
|
281
|
+
private compactTree;
|
|
282
|
+
private saveTreeManifest;
|
|
283
|
+
private startAutoFlush;
|
|
284
|
+
private calculateMemoryUsage;
|
|
285
|
+
}
|
|
286
|
+
//# sourceMappingURL=NativeGraphAdjacencyIndex.d.ts.map
|