@soulcraft/cor 3.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +16 -0
- package/README.md +328 -0
- package/dist/aggregation/NativeAggregationEngine.d.ts +118 -0
- package/dist/aggregation/NativeAggregationEngine.js +186 -0
- package/dist/cli.d.ts +12 -0
- package/dist/cli.js +236 -0
- package/dist/graph/GraphAccelerationAdapter.d.ts +153 -0
- package/dist/graph/GraphAccelerationAdapter.js +326 -0
- package/dist/graph/NativeGraphAdjacencyIndex.d.ts +286 -0
- package/dist/graph/NativeGraphAdjacencyIndex.js +944 -0
- package/dist/hnsw/AdaptiveDiskAnnModeSelector.d.ts +187 -0
- package/dist/hnsw/AdaptiveDiskAnnModeSelector.js +313 -0
- package/dist/hnsw/NativeDiskAnnWrapper.d.ts +489 -0
- package/dist/hnsw/NativeDiskAnnWrapper.js +1456 -0
- package/dist/index.d.ts +28 -0
- package/dist/index.js +29 -0
- package/dist/legacyLayoutGuard.d.ts +93 -0
- package/dist/legacyLayoutGuard.js +237 -0
- package/dist/license/constants.d.ts +30 -0
- package/dist/license/constants.js +30 -0
- package/dist/license/onlineValidator.d.ts +43 -0
- package/dist/license/onlineValidator.js +225 -0
- package/dist/license/types.d.ts +39 -0
- package/dist/license/types.js +5 -0
- package/dist/license.d.ts +64 -0
- package/dist/license.js +165 -0
- package/dist/migration/MigrationCoordinator.d.ts +171 -0
- package/dist/migration/MigrationCoordinator.js +249 -0
- package/dist/migration/indexEpochGuard.d.ts +163 -0
- package/dist/migration/indexEpochGuard.js +202 -0
- package/dist/native/NativeEmbeddingEngine.d.ts +79 -0
- package/dist/native/NativeEmbeddingEngine.js +318 -0
- package/dist/native/NativeRoaringBitmap32.d.ts +114 -0
- package/dist/native/NativeRoaringBitmap32.js +221 -0
- package/dist/native/ffi.d.ts +20 -0
- package/dist/native/ffi.js +48 -0
- package/dist/native/idMapperTestSupport.d.ts +67 -0
- package/dist/native/idMapperTestSupport.js +112 -0
- package/dist/native/index.d.ts +49 -0
- package/dist/native/index.js +87 -0
- package/dist/native/napi.d.ts +21 -0
- package/dist/native/napi.js +88 -0
- package/dist/native/types.d.ts +1298 -0
- package/dist/native/types.js +16 -0
- package/dist/plugin.d.ts +50 -0
- package/dist/plugin.js +388 -0
- package/dist/providerContracts.d.ts +277 -0
- package/dist/providerContracts.js +38 -0
- package/dist/resource/OsMemoryProbe.d.ts +175 -0
- package/dist/resource/OsMemoryProbe.js +206 -0
- package/dist/resource/ResourceManager.d.ts +491 -0
- package/dist/resource/ResourceManager.js +960 -0
- package/dist/utils/ColumnManifest.d.ts +97 -0
- package/dist/utils/ColumnManifest.js +129 -0
- package/dist/utils/NativeColumnStore.d.ts +284 -0
- package/dist/utils/NativeColumnStore.js +685 -0
- package/dist/utils/NativeMetadataIndex.d.ts +882 -0
- package/dist/utils/NativeMetadataIndex.js +2631 -0
- package/dist/utils/NativeUnifiedCache.d.ts +87 -0
- package/dist/utils/NativeUnifiedCache.js +273 -0
- package/dist/utils/binaryIdMapperFactory.d.ts +59 -0
- package/dist/utils/binaryIdMapperFactory.js +94 -0
- package/dist/utils/collation.d.ts +30 -0
- package/dist/utils/collation.js +40 -0
- package/dist/utils/columnStoreTypes.d.ts +161 -0
- package/dist/utils/columnStoreTypes.js +29 -0
- package/dist/utils/nativeBinaryEntityIdMapper.d.ts +211 -0
- package/dist/utils/nativeBinaryEntityIdMapper.js +381 -0
- package/dist/utils/nativeEntityIdMapper.d.ts +111 -0
- package/dist/utils/nativeEntityIdMapper.js +170 -0
- package/docs/ADR-002-diskann-100-percent-rust.md +294 -0
- package/docs/ADR-003-semantic-time-travel.md +100 -0
- package/docs/aggregation.md +251 -0
- package/docs/comparison.md +162 -0
- package/docs/deployment-limits.md +87 -0
- package/docs/diskann.md +184 -0
- package/docs/migration-3.0.md +396 -0
- package/docs/performance-budget.md +274 -0
- package/docs/performance.md +117 -0
- package/docs/scaling.md +439 -0
- package/docs/snapshot-safety.md +246 -0
- package/docs/u64-id-space.md +214 -0
- package/docs/verification-report.md +670 -0
- package/native/brainy-native.node +0 -0
- package/native/index.d.ts +3916 -0
- package/package.json +92 -0
- package/scripts/migrate-cortex-2x-to-3x.mjs +970 -0
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module utils/nativeBinaryEntityIdMapper
|
|
3
|
+
* @description TypeScript wrapper around cor's native binary
|
|
4
|
+
* `BinaryIdMapper`. Implements brainy's `EntityIdMapperProvider` so the
|
|
5
|
+
* mmap-backed billion-scale mapper is a drop-in for the existing
|
|
6
|
+
* JSON-persisted one.
|
|
7
|
+
*
|
|
8
|
+
* ## When this engages
|
|
9
|
+
*
|
|
10
|
+
* The cor plugin registers this wrapper as the `'entityIdMapper'`
|
|
11
|
+
* provider when the storage adapter exposes `getBinaryBlobPath()` (i.e.
|
|
12
|
+
* filesystem-backed storage with cor's 2.4.0 #2 mmap-vector layer).
|
|
13
|
+
* Cloud-storage adapters fall back to the JSON variant
|
|
14
|
+
* (`NativeEntityIdMapperWrapper`) since they have no local-path concept.
|
|
15
|
+
*
|
|
16
|
+
* ## UUID format conversion
|
|
17
|
+
*
|
|
18
|
+
* Brainy passes UUIDs as strings (typically the canonical 36-char
|
|
19
|
+
* `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`). The native side works in
|
|
20
|
+
* 16-byte Buffers. This wrapper converts at the boundary. Non-canonical
|
|
21
|
+
* UUID strings (any other 32-hex-digit form) are also accepted.
|
|
22
|
+
*
|
|
23
|
+
* ## Concurrency
|
|
24
|
+
*
|
|
25
|
+
* `getOrAssign` is atomic across concurrent callers for the same UUID
|
|
26
|
+
* (256 sharded per-UUID mutexes in the native layer). Lookups are
|
|
27
|
+
* lock-free. The wrapper holds no JS-side mutable state besides the
|
|
28
|
+
* native handle.
|
|
29
|
+
*
|
|
30
|
+
* ## IdSpace (Piece 10)
|
|
31
|
+
*
|
|
32
|
+
* The wrapper supports two entity-int wire widths:
|
|
33
|
+
*
|
|
34
|
+
* - `'u32'` (default): cortex 2.x compatible — JS `number` throughout,
|
|
35
|
+
* capped at 4.29 B entities. Persists in the legacy `int_to_uuid.bin`
|
|
36
|
+
* v1 header.
|
|
37
|
+
* - `'u64'`: opt-in via `idSpace: 'u64'`. The native layer's `number`
|
|
38
|
+
* surface throws in this mode, so the wrapper transparently routes
|
|
39
|
+
* the `EntityIdMapperProvider` methods through the BigInt napi
|
|
40
|
+
* siblings and converts BigInt → number at the boundary. Entity ints
|
|
41
|
+
* above `Number.MAX_SAFE_INTEGER` (2^53 - 1 = ~9 PB of entities)
|
|
42
|
+
* throw a clear `EntityIdSpaceExceeded` error so callers get a loud
|
|
43
|
+
* failure rather than a silent precision loss.
|
|
44
|
+
*
|
|
45
|
+
* The `getOrAssignBig` / `getIntBig` / `getUuidBig` / `sizeBig`
|
|
46
|
+
* sibling methods are available on both modes and always return
|
|
47
|
+
* `bigint` losslessly — use them in u64-aware code paths that need
|
|
48
|
+
* the full u64 range.
|
|
49
|
+
*/
|
|
50
|
+
import { existsSync } from 'node:fs';
|
|
51
|
+
import { loadNativeModule } from '../native/index.js';
|
|
52
|
+
import { prodLog } from '@soulcraft/brainy/internals';
|
|
53
|
+
const UUID_BYTES = 16;
|
|
54
|
+
/**
|
|
55
|
+
* Thrown when a U64-mode mapper allocates or returns an entity int
|
|
56
|
+
* above `Number.MAX_SAFE_INTEGER` (2^53 - 1). At this point a JS
|
|
57
|
+
* `number` can no longer represent the value losslessly; callers must
|
|
58
|
+
* switch to the BigInt sibling methods (`getOrAssignBig`,
|
|
59
|
+
* `getIntBig`, `getUuidBig`).
|
|
60
|
+
*/
|
|
61
|
+
export class EntityIdSpaceExceeded extends Error {
|
|
62
|
+
/** The u64 entity int that exceeded the safe-integer ceiling. */
|
|
63
|
+
value;
|
|
64
|
+
/** The method that was called (`'getOrAssign'`, `'getInt'`, etc.). */
|
|
65
|
+
method;
|
|
66
|
+
constructor(method, value) {
|
|
67
|
+
super(`${method}: entity int ${value} exceeds Number.MAX_SAFE_INTEGER ` +
|
|
68
|
+
`(2^53 - 1). Switch to the BigInt sibling method (${method}Big) ` +
|
|
69
|
+
`for entity ints above 9.007 PB.`);
|
|
70
|
+
this.name = 'EntityIdSpaceExceeded';
|
|
71
|
+
this.method = method;
|
|
72
|
+
this.value = value;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
const MAX_SAFE_INTEGER_BIG = BigInt(Number.MAX_SAFE_INTEGER);
|
|
76
|
+
/**
|
|
77
|
+
* Convert a `bigint` entity int to a JS `number`. Throws
|
|
78
|
+
* {@link EntityIdSpaceExceeded} if the value exceeds the JS safe-integer
|
|
79
|
+
* range.
|
|
80
|
+
*/
|
|
81
|
+
function bigToSafeNumber(value, method) {
|
|
82
|
+
if (value > MAX_SAFE_INTEGER_BIG) {
|
|
83
|
+
throw new EntityIdSpaceExceeded(method, value);
|
|
84
|
+
}
|
|
85
|
+
return Number(value);
|
|
86
|
+
}
|
|
87
|
+
const DEFAULT_UUID_TO_INT_KEY = '_id_mapper/uuid_to_int.mkv';
|
|
88
|
+
const DEFAULT_INT_TO_UUID_KEY = '_id_mapper/int_to_uuid.bin';
|
|
89
|
+
/**
|
|
90
|
+
* Drop-in `EntityIdMapperProvider` backed by the native `BinaryIdMapper`.
|
|
91
|
+
*
|
|
92
|
+
* @example
|
|
93
|
+
* ```typescript
|
|
94
|
+
* const mapper = new NativeBinaryEntityIdMapperWrapper({ storage })
|
|
95
|
+
* await mapper.init()
|
|
96
|
+
* const intId = mapper.getOrAssign('12345678-1234-5678-1234-567812345678')
|
|
97
|
+
* const uuid = mapper.getUuid(intId)
|
|
98
|
+
* ```
|
|
99
|
+
*/
|
|
100
|
+
export class NativeBinaryEntityIdMapperWrapper {
|
|
101
|
+
storage;
|
|
102
|
+
uuidToIntKey;
|
|
103
|
+
intToUuidKey;
|
|
104
|
+
intToUuidSize;
|
|
105
|
+
uuidToIntSize;
|
|
106
|
+
bucketCapacity;
|
|
107
|
+
maxGlobalDepth;
|
|
108
|
+
requestedIdSpace;
|
|
109
|
+
/**
|
|
110
|
+
* The actual IdSpace of the open mapper, sourced from the native
|
|
111
|
+
* binding's `idSpace()` reflection after `init()`. The on-disk
|
|
112
|
+
* header wins over `requestedIdSpace` (which may be ignored at
|
|
113
|
+
* `openExisting` time).
|
|
114
|
+
*/
|
|
115
|
+
resolvedIdSpace = 'u32';
|
|
116
|
+
native = null;
|
|
117
|
+
initialized = false;
|
|
118
|
+
constructor(options) {
|
|
119
|
+
this.storage = options.storage;
|
|
120
|
+
this.uuidToIntKey = options.uuidToIntKey ?? DEFAULT_UUID_TO_INT_KEY;
|
|
121
|
+
this.intToUuidKey = options.intToUuidKey ?? DEFAULT_INT_TO_UUID_KEY;
|
|
122
|
+
this.intToUuidSize = options.intToUuidSize ?? BigInt(32) * BigInt(1024) ** BigInt(3);
|
|
123
|
+
this.uuidToIntSize = options.uuidToIntSize ?? BigInt(32) * BigInt(1024) ** BigInt(3);
|
|
124
|
+
this.bucketCapacity = options.bucketCapacity ?? 16;
|
|
125
|
+
this.maxGlobalDepth = options.maxGlobalDepth ?? 28;
|
|
126
|
+
this.requestedIdSpace = options.idSpace ?? 'u64';
|
|
127
|
+
}
|
|
128
|
+
async init() {
|
|
129
|
+
if (this.initialized)
|
|
130
|
+
return;
|
|
131
|
+
const storage = this.storage;
|
|
132
|
+
if (!storage.getBinaryBlobPath) {
|
|
133
|
+
throw new Error('NativeBinaryEntityIdMapperWrapper requires a storage adapter that ' +
|
|
134
|
+
'exposes getBinaryBlobPath() (filesystem-backed). For cloud adapters, ' +
|
|
135
|
+
'use NativeEntityIdMapperWrapper (JSON variant) instead.');
|
|
136
|
+
}
|
|
137
|
+
const uuidToIntPath = storage.getBinaryBlobPath(this.uuidToIntKey);
|
|
138
|
+
const intToUuidPath = storage.getBinaryBlobPath(this.intToUuidKey);
|
|
139
|
+
if (!uuidToIntPath || !intToUuidPath) {
|
|
140
|
+
throw new Error(`NativeBinaryEntityIdMapperWrapper: getBinaryBlobPath returned null for ` +
|
|
141
|
+
`${this.uuidToIntKey} or ${this.intToUuidKey}`);
|
|
142
|
+
}
|
|
143
|
+
const bindings = loadNativeModule();
|
|
144
|
+
const NativeBinaryIdMapper = bindings.NativeBinaryIdMapper;
|
|
145
|
+
if (!NativeBinaryIdMapper) {
|
|
146
|
+
throw new Error('NativeBinaryIdMapper binding missing from cor native module — ' +
|
|
147
|
+
'this build of cor is older than the BinaryIdMapper feature');
|
|
148
|
+
}
|
|
149
|
+
const config = {
|
|
150
|
+
uuidToIntPath,
|
|
151
|
+
intToUuidPath,
|
|
152
|
+
intToUuidSize: this.intToUuidSize,
|
|
153
|
+
uuidToIntSize: this.uuidToIntSize,
|
|
154
|
+
bucketCapacity: this.bucketCapacity,
|
|
155
|
+
maxGlobalDepth: this.maxGlobalDepth,
|
|
156
|
+
idSpace: this.requestedIdSpace,
|
|
157
|
+
};
|
|
158
|
+
// Explicitly distinguish "fresh install" from "existing files".
|
|
159
|
+
// Both files must exist together (paired write semantics) — a
|
|
160
|
+
// half-present state is corruption from a crash between file
|
|
161
|
+
// creations and is surfaced as an error rather than silently
|
|
162
|
+
// recreated.
|
|
163
|
+
const uuidFileExists = existsSync(uuidToIntPath);
|
|
164
|
+
const intFileExists = existsSync(intToUuidPath);
|
|
165
|
+
if (uuidFileExists && intFileExists) {
|
|
166
|
+
this.native = NativeBinaryIdMapper.openExisting(config);
|
|
167
|
+
}
|
|
168
|
+
else if (!uuidFileExists && !intFileExists) {
|
|
169
|
+
this.native = NativeBinaryIdMapper.create(config);
|
|
170
|
+
}
|
|
171
|
+
else {
|
|
172
|
+
throw new Error(`NativeBinaryEntityIdMapperWrapper: half-present file pair — ` +
|
|
173
|
+
`${this.uuidToIntKey} ${uuidFileExists ? 'exists' : 'missing'}, ` +
|
|
174
|
+
`${this.intToUuidKey} ${intFileExists ? 'exists' : 'missing'}. ` +
|
|
175
|
+
`Refusing to silently recreate; investigate manually.`);
|
|
176
|
+
}
|
|
177
|
+
// Reflect the on-disk IdSpace — authoritative over the requested
|
|
178
|
+
// value when openExisting opens a file with a different mode.
|
|
179
|
+
this.resolvedIdSpace = this.native.idSpace();
|
|
180
|
+
this.initialized = true;
|
|
181
|
+
if (prodLog?.debug) {
|
|
182
|
+
prodLog.debug(`[cor] BinaryIdMapper wired: paths=[${uuidToIntPath}, ${intToUuidPath}], idSpace=${this.resolvedIdSpace}`);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Report the mapper's actual IdSpace mode. Returns `'u32'` before
|
|
187
|
+
* `init()` (the default the wrapper assumes); after init, returns the
|
|
188
|
+
* mode reported by the native binding (which is authoritative).
|
|
189
|
+
*/
|
|
190
|
+
getIdSpace() {
|
|
191
|
+
return this.resolvedIdSpace;
|
|
192
|
+
}
|
|
193
|
+
// -- EntityIdMapperProvider surface (number-typed, brainy contract) --
|
|
194
|
+
/**
|
|
195
|
+
* Allocate or retrieve the entity int for `uuid`. Returns a JS
|
|
196
|
+
* `number`. In U64 mode, routes through the BigInt sibling and
|
|
197
|
+
* throws {@link EntityIdSpaceExceeded} if the allocated int exceeds
|
|
198
|
+
* `Number.MAX_SAFE_INTEGER` — at that point the caller MUST switch
|
|
199
|
+
* to `getOrAssignBig` for the full u64 range.
|
|
200
|
+
*/
|
|
201
|
+
getOrAssign(uuid) {
|
|
202
|
+
const native = this.ensure();
|
|
203
|
+
if (this.resolvedIdSpace === 'u64') {
|
|
204
|
+
return bigToSafeNumber(native.getOrAssignBig(this.encode(uuid)), 'getOrAssign');
|
|
205
|
+
}
|
|
206
|
+
return native.getOrAssign(this.encode(uuid));
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Look up the UUID for `intId`. Accepts a JS `number` — in U64 mode
|
|
210
|
+
* this is a lossy conversion above 2^53; use {@link getUuidBig} for
|
|
211
|
+
* the full u64 range.
|
|
212
|
+
*/
|
|
213
|
+
getUuid(intId) {
|
|
214
|
+
const native = this.ensure();
|
|
215
|
+
const buf = this.resolvedIdSpace === 'u64'
|
|
216
|
+
? native.getUuidBig(BigInt(intId))
|
|
217
|
+
: native.getUuid(intId);
|
|
218
|
+
if (!buf)
|
|
219
|
+
return undefined;
|
|
220
|
+
return this.decode(buf);
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Look up the entity int for `uuid`. Returns a JS `number`. In U64
|
|
224
|
+
* mode throws {@link EntityIdSpaceExceeded} if the int exceeds
|
|
225
|
+
* `Number.MAX_SAFE_INTEGER`.
|
|
226
|
+
*/
|
|
227
|
+
getInt(uuid) {
|
|
228
|
+
const native = this.ensure();
|
|
229
|
+
if (this.resolvedIdSpace === 'u64') {
|
|
230
|
+
const big = native.getIntBig(this.encode(uuid));
|
|
231
|
+
return big == null ? undefined : bigToSafeNumber(big, 'getInt');
|
|
232
|
+
}
|
|
233
|
+
const out = native.getInt(this.encode(uuid));
|
|
234
|
+
return out == null ? undefined : out;
|
|
235
|
+
}
|
|
236
|
+
remove(uuid) {
|
|
237
|
+
const native = this.ensure();
|
|
238
|
+
return native.remove(this.encode(uuid));
|
|
239
|
+
}
|
|
240
|
+
async flush() {
|
|
241
|
+
const native = this.ensure();
|
|
242
|
+
native.flush();
|
|
243
|
+
}
|
|
244
|
+
async clear() {
|
|
245
|
+
// Reset by recreating the files. Atomicity caveat: any concurrent
|
|
246
|
+
// reader holds a stale mmap. Brainy calls clear() during clear()
|
|
247
|
+
// operations that already block other access; this is fine.
|
|
248
|
+
this.initialized = false;
|
|
249
|
+
this.native = null;
|
|
250
|
+
await this.init();
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Materialise every live int id into a JS `number[]`. **U32 mode
|
|
254
|
+
* only.** U64 mode throws — the native binding refuses to allocate
|
|
255
|
+
* a giant JS array at the scale a U64 brain implies. Iterate via
|
|
256
|
+
* the BigInt sibling iterator (TBD — a follow-up surfaces it on the
|
|
257
|
+
* wrapper) for U64 brains.
|
|
258
|
+
*/
|
|
259
|
+
getAllIntIds() {
|
|
260
|
+
const native = this.ensure();
|
|
261
|
+
if (this.resolvedIdSpace === 'u64') {
|
|
262
|
+
throw new Error('getAllIntIds: not supported in U64 mode — the materialised ' +
|
|
263
|
+
'number[] would risk OOM at billion scale. Use the BigInt ' +
|
|
264
|
+
'streaming iterator on the underlying native binding instead.');
|
|
265
|
+
}
|
|
266
|
+
return native.getAllIntIds();
|
|
267
|
+
}
|
|
268
|
+
intsIterableToUuids(ints) {
|
|
269
|
+
const native = this.ensure();
|
|
270
|
+
// ONE native call via the batched binding (which works in both u32 and u64
|
|
271
|
+
// modes), not a per-row `getUuid`/`getUuidBig` FFI crossing. The per-row
|
|
272
|
+
// loop is the pattern performance-budget.md forbids — it cost ~5ms to
|
|
273
|
+
// hydrate a 1K-row find() result (10× the batched path) on EVERY query, at
|
|
274
|
+
// every scale. Unmapped ints (`null`) are dropped, matching prior behaviour.
|
|
275
|
+
const bigInts = [];
|
|
276
|
+
for (const i of ints)
|
|
277
|
+
bigInts.push(BigInt(i));
|
|
278
|
+
const out = [];
|
|
279
|
+
for (const buf of native.intsToUuidsBig(bigInts)) {
|
|
280
|
+
if (buf)
|
|
281
|
+
out.push(this.decode(buf));
|
|
282
|
+
}
|
|
283
|
+
return out;
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* @description Batch-resolve entity ints to their UUIDs in one native
|
|
287
|
+
* call (ADR-006 Q5) — the provider path brainy uses to hydrate a
|
|
288
|
+
* `Subgraph`'s `nodes` column. One boundary crossing instead of N
|
|
289
|
+
* per-int `getUuidBig` calls when a viz/export renders many rows. An
|
|
290
|
+
* unmapped int yields `''` so the result stays index-aligned with the
|
|
291
|
+
* input.
|
|
292
|
+
* @param ints - The entity ints to resolve (a `BigInt64Array`, the
|
|
293
|
+
* `Subgraph.nodes` column).
|
|
294
|
+
* @returns UUID strings index-aligned with `ints` (`''` for unmapped).
|
|
295
|
+
*/
|
|
296
|
+
entityIntsToUuids(ints) {
|
|
297
|
+
const native = this.ensure();
|
|
298
|
+
return native
|
|
299
|
+
.intsToUuidsBig(Array.from(ints))
|
|
300
|
+
.map((buf) => (buf ? this.decode(buf) : ''));
|
|
301
|
+
}
|
|
302
|
+
get size() {
|
|
303
|
+
if (!this.initialized || !this.native)
|
|
304
|
+
return 0;
|
|
305
|
+
if (this.resolvedIdSpace === 'u64') {
|
|
306
|
+
return bigToSafeNumber(this.native.sizeBig(), 'size');
|
|
307
|
+
}
|
|
308
|
+
return this.native.size();
|
|
309
|
+
}
|
|
310
|
+
// -- BigInt sibling surface (u64-safe, works in both modes) -------
|
|
311
|
+
/**
|
|
312
|
+
* Allocate or retrieve the entity int for `uuid` as a `bigint`.
|
|
313
|
+
* Lossless across the full u64 range; safe to call in either mode.
|
|
314
|
+
*/
|
|
315
|
+
getOrAssignBig(uuid) {
|
|
316
|
+
const native = this.ensure();
|
|
317
|
+
return native.getOrAssignBig(this.encode(uuid));
|
|
318
|
+
}
|
|
319
|
+
/** Look up the entity int for `uuid` as a `bigint`. */
|
|
320
|
+
getIntBig(uuid) {
|
|
321
|
+
const native = this.ensure();
|
|
322
|
+
const out = native.getIntBig(this.encode(uuid));
|
|
323
|
+
return out == null ? undefined : out;
|
|
324
|
+
}
|
|
325
|
+
/** Look up the UUID for `int` (passed as a `bigint`). */
|
|
326
|
+
getUuidBig(int) {
|
|
327
|
+
const native = this.ensure();
|
|
328
|
+
const buf = native.getUuidBig(int);
|
|
329
|
+
if (!buf)
|
|
330
|
+
return undefined;
|
|
331
|
+
return this.decode(buf);
|
|
332
|
+
}
|
|
333
|
+
/** Live (non-tombstone) entry count as a `bigint`. */
|
|
334
|
+
sizeBig() {
|
|
335
|
+
if (!this.initialized || !this.native)
|
|
336
|
+
return 0n;
|
|
337
|
+
return this.native.sizeBig();
|
|
338
|
+
}
|
|
339
|
+
/** Largest int ever assigned + 1, as a `bigint`. */
|
|
340
|
+
nextIntBig() {
|
|
341
|
+
return this.ensure().nextIntBig();
|
|
342
|
+
}
|
|
343
|
+
// ---------------------------------------------------------------
|
|
344
|
+
// UUID string ↔ Buffer conversion
|
|
345
|
+
// ---------------------------------------------------------------
|
|
346
|
+
/**
|
|
347
|
+
* Encode a UUID string into a 16-byte Buffer. Accepts canonical
|
|
348
|
+
* 36-char form (with hyphens) or any 32-hex-digit form. Throws on
|
|
349
|
+
* malformed input.
|
|
350
|
+
*/
|
|
351
|
+
encode(uuid) {
|
|
352
|
+
const hex = uuid.replace(/-/g, '').toLowerCase();
|
|
353
|
+
if (hex.length !== 32 || !/^[0-9a-f]{32}$/.test(hex)) {
|
|
354
|
+
throw new Error(`NativeBinaryEntityIdMapperWrapper: invalid UUID string "${uuid}"`);
|
|
355
|
+
}
|
|
356
|
+
return Buffer.from(hex, 'hex');
|
|
357
|
+
}
|
|
358
|
+
/** Decode a 16-byte Buffer back to canonical UUID string. */
|
|
359
|
+
decode(buf) {
|
|
360
|
+
if (buf.length !== UUID_BYTES) {
|
|
361
|
+
throw new Error(`NativeBinaryEntityIdMapperWrapper: native returned ${buf.length}-byte uuid (expected ${UUID_BYTES})`);
|
|
362
|
+
}
|
|
363
|
+
const hex = buf.toString('hex');
|
|
364
|
+
return (hex.slice(0, 8) +
|
|
365
|
+
'-' +
|
|
366
|
+
hex.slice(8, 12) +
|
|
367
|
+
'-' +
|
|
368
|
+
hex.slice(12, 16) +
|
|
369
|
+
'-' +
|
|
370
|
+
hex.slice(16, 20) +
|
|
371
|
+
'-' +
|
|
372
|
+
hex.slice(20, 32));
|
|
373
|
+
}
|
|
374
|
+
ensure() {
|
|
375
|
+
if (!this.initialized || !this.native) {
|
|
376
|
+
throw new Error('NativeBinaryEntityIdMapperWrapper: call init() before any operation');
|
|
377
|
+
}
|
|
378
|
+
return this.native;
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
//# sourceMappingURL=nativeBinaryEntityIdMapper.js.map
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EntityIdMapper — Bidirectional UUID ↔ integer mapping for roaring bitmaps
|
|
3
|
+
*
|
|
4
|
+
* Thin TypeScript wrapper around the native Rust NativeEntityIdMapper.
|
|
5
|
+
* All mapping operations execute synchronously in Rust (O(1) HashMap lookups).
|
|
6
|
+
* This wrapper handles async storage I/O (load from storage, persist to storage).
|
|
7
|
+
*/
|
|
8
|
+
import type { StorageAdapter } from '@soulcraft/brainy';
|
|
9
|
+
import type { EntityIdMapperProvider } from '../providerContracts.js';
|
|
10
|
+
export interface NativeEntityIdMapperWrapperOptions {
|
|
11
|
+
storage: StorageAdapter;
|
|
12
|
+
storageKey?: string;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Bidirectional UUID ↔ integer mapping for roaring bitmap integration.
|
|
16
|
+
* Delegates to native Rust implementation for all mapping operations.
|
|
17
|
+
*
|
|
18
|
+
* Implements {@link EntityIdMapperProvider} so a compile error fires the moment
|
|
19
|
+
* this wrapper drops or drifts a member brainy's `MetadataIndexManager` calls on
|
|
20
|
+
* an injected native mapper (incl. `getAllIntIds`).
|
|
21
|
+
*/
|
|
22
|
+
export declare class NativeEntityIdMapperWrapper implements EntityIdMapperProvider {
|
|
23
|
+
private storage;
|
|
24
|
+
private storageKey;
|
|
25
|
+
private native;
|
|
26
|
+
constructor(options: NativeEntityIdMapperWrapperOptions);
|
|
27
|
+
/**
|
|
28
|
+
* Initialize the mapper by loading from storage
|
|
29
|
+
*/
|
|
30
|
+
init(): Promise<void>;
|
|
31
|
+
/**
|
|
32
|
+
* Get integer ID for UUID, assigning a new ID if not exists
|
|
33
|
+
*/
|
|
34
|
+
getOrAssign(uuid: string): number;
|
|
35
|
+
/**
|
|
36
|
+
* Get integer ID for UUID with immediate persistence guarantee
|
|
37
|
+
*/
|
|
38
|
+
getOrAssignSync(uuid: string): Promise<number>;
|
|
39
|
+
/**
|
|
40
|
+
* Get UUID for integer ID
|
|
41
|
+
*/
|
|
42
|
+
getUuid(intId: number): string | undefined;
|
|
43
|
+
/**
|
|
44
|
+
* Get integer ID for UUID (without assigning if not exists)
|
|
45
|
+
*/
|
|
46
|
+
getInt(uuid: string): number | undefined;
|
|
47
|
+
/**
|
|
48
|
+
* Check if UUID has been assigned an integer ID
|
|
49
|
+
*/
|
|
50
|
+
has(uuid: string): boolean;
|
|
51
|
+
/**
|
|
52
|
+
* Remove mapping for UUID
|
|
53
|
+
*/
|
|
54
|
+
remove(uuid: string): boolean;
|
|
55
|
+
/**
|
|
56
|
+
* Get total number of mappings
|
|
57
|
+
*/
|
|
58
|
+
get size(): number;
|
|
59
|
+
/**
|
|
60
|
+
* Convert array of UUIDs to array of integers
|
|
61
|
+
*/
|
|
62
|
+
uuidsToInts(uuids: string[]): number[];
|
|
63
|
+
/**
|
|
64
|
+
* Convert array of integers to array of UUIDs
|
|
65
|
+
*/
|
|
66
|
+
intsToUuids(ints: number[]): string[];
|
|
67
|
+
/**
|
|
68
|
+
* Convert iterable of integers to array of UUIDs (for roaring bitmap iteration)
|
|
69
|
+
*/
|
|
70
|
+
intsIterableToUuids(ints: Iterable<number>): string[];
|
|
71
|
+
/**
|
|
72
|
+
* @description Batch-resolve entity ints to their UUIDs in one native
|
|
73
|
+
* call (ADR-006 Q5) — the provider path brainy uses to hydrate a
|
|
74
|
+
* `Subgraph`'s `nodes` column. One boundary crossing instead of N
|
|
75
|
+
* per-int `getUuid` calls when a viz/export renders many rows.
|
|
76
|
+
* @param ints - The entity ints to resolve (a `BigInt64Array`, the
|
|
77
|
+
* `Subgraph.nodes` column).
|
|
78
|
+
* @returns UUID strings index-aligned with `ints`.
|
|
79
|
+
*/
|
|
80
|
+
entityIntsToUuids(ints: BigInt64Array): string[];
|
|
81
|
+
/**
|
|
82
|
+
* @description Return every assigned integer ID. This is the "all-ids universe"
|
|
83
|
+
* brainy's `MetadataIndexManager` uses to evaluate negation / `exists:false` /
|
|
84
|
+
* `missing:true` filters (the set of all known entities minus the matching set)
|
|
85
|
+
* when a native EntityIdMapper is injected into the TypeScript metadata index.
|
|
86
|
+
* Ids come from the native `int → uuid` map, so the universe is sourced from the
|
|
87
|
+
* same canonical assignment `getOrAssign` produces — no storage scan.
|
|
88
|
+
* @returns All assigned integer IDs, in unspecified order.
|
|
89
|
+
* @example
|
|
90
|
+
* const allUuids = mapper.intsIterableToUuids(mapper.getAllIntIds())
|
|
91
|
+
*/
|
|
92
|
+
getAllIntIds(): number[];
|
|
93
|
+
/**
|
|
94
|
+
* Flush mappings to storage
|
|
95
|
+
*/
|
|
96
|
+
flush(): Promise<void>;
|
|
97
|
+
/**
|
|
98
|
+
* Clear all mappings
|
|
99
|
+
*/
|
|
100
|
+
clear(): Promise<void>;
|
|
101
|
+
/**
|
|
102
|
+
* Get statistics about the mapper
|
|
103
|
+
*/
|
|
104
|
+
getStats(): {
|
|
105
|
+
mappings: number;
|
|
106
|
+
nextId: number;
|
|
107
|
+
dirty: boolean;
|
|
108
|
+
memoryEstimate: number;
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
//# sourceMappingURL=nativeEntityIdMapper.d.ts.map
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EntityIdMapper — Bidirectional UUID ↔ integer mapping for roaring bitmaps
|
|
3
|
+
*
|
|
4
|
+
* Thin TypeScript wrapper around the native Rust NativeEntityIdMapper.
|
|
5
|
+
* All mapping operations execute synchronously in Rust (O(1) HashMap lookups).
|
|
6
|
+
* This wrapper handles async storage I/O (load from storage, persist to storage).
|
|
7
|
+
*/
|
|
8
|
+
import { loadNativeModule } from '../native/index.js';
|
|
9
|
+
/**
|
|
10
|
+
* Bidirectional UUID ↔ integer mapping for roaring bitmap integration.
|
|
11
|
+
* Delegates to native Rust implementation for all mapping operations.
|
|
12
|
+
*
|
|
13
|
+
* Implements {@link EntityIdMapperProvider} so a compile error fires the moment
|
|
14
|
+
* this wrapper drops or drifts a member brainy's `MetadataIndexManager` calls on
|
|
15
|
+
* an injected native mapper (incl. `getAllIntIds`).
|
|
16
|
+
*/
|
|
17
|
+
export class NativeEntityIdMapperWrapper {
|
|
18
|
+
storage;
|
|
19
|
+
storageKey;
|
|
20
|
+
native;
|
|
21
|
+
constructor(options) {
|
|
22
|
+
this.storage = options.storage;
|
|
23
|
+
this.storageKey = options.storageKey || 'brainy:entityIdMapper';
|
|
24
|
+
const bindings = loadNativeModule();
|
|
25
|
+
this.native = new bindings.NativeEntityIdMapper();
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Initialize the mapper by loading from storage
|
|
29
|
+
*/
|
|
30
|
+
async init() {
|
|
31
|
+
try {
|
|
32
|
+
const metadata = await this.storage.getMetadata(this.storageKey);
|
|
33
|
+
if (metadata && metadata.nextId !== undefined) {
|
|
34
|
+
this.native.loadFromJson(JSON.stringify(metadata));
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
// First time initialization — native mapper starts empty
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Get integer ID for UUID, assigning a new ID if not exists
|
|
43
|
+
*/
|
|
44
|
+
getOrAssign(uuid) {
|
|
45
|
+
return this.native.getOrAssign(uuid);
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Get integer ID for UUID with immediate persistence guarantee
|
|
49
|
+
*/
|
|
50
|
+
async getOrAssignSync(uuid) {
|
|
51
|
+
const id = this.native.getOrAssign(uuid);
|
|
52
|
+
if (this.native.isDirty()) {
|
|
53
|
+
await this.flush();
|
|
54
|
+
}
|
|
55
|
+
return id;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Get UUID for integer ID
|
|
59
|
+
*/
|
|
60
|
+
getUuid(intId) {
|
|
61
|
+
return this.native.getUuid(intId) ?? undefined;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Get integer ID for UUID (without assigning if not exists)
|
|
65
|
+
*/
|
|
66
|
+
getInt(uuid) {
|
|
67
|
+
return this.native.getInt(uuid) ?? undefined;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Check if UUID has been assigned an integer ID
|
|
71
|
+
*/
|
|
72
|
+
has(uuid) {
|
|
73
|
+
return this.native.has(uuid);
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Remove mapping for UUID
|
|
77
|
+
*/
|
|
78
|
+
remove(uuid) {
|
|
79
|
+
return this.native.remove(uuid);
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Get total number of mappings
|
|
83
|
+
*/
|
|
84
|
+
get size() {
|
|
85
|
+
return this.native.size;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Convert array of UUIDs to array of integers
|
|
89
|
+
*/
|
|
90
|
+
uuidsToInts(uuids) {
|
|
91
|
+
return this.native.uuidsToInts(uuids);
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Convert array of integers to array of UUIDs
|
|
95
|
+
*/
|
|
96
|
+
intsToUuids(ints) {
|
|
97
|
+
return this.native.intsToUuids(ints);
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Convert iterable of integers to array of UUIDs (for roaring bitmap iteration)
|
|
101
|
+
*/
|
|
102
|
+
intsIterableToUuids(ints) {
|
|
103
|
+
return this.native.intsToUuids(Array.from(ints));
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* @description Batch-resolve entity ints to their UUIDs in one native
|
|
107
|
+
* call (ADR-006 Q5) — the provider path brainy uses to hydrate a
|
|
108
|
+
* `Subgraph`'s `nodes` column. One boundary crossing instead of N
|
|
109
|
+
* per-int `getUuid` calls when a viz/export renders many rows.
|
|
110
|
+
* @param ints - The entity ints to resolve (a `BigInt64Array`, the
|
|
111
|
+
* `Subgraph.nodes` column).
|
|
112
|
+
* @returns UUID strings index-aligned with `ints`.
|
|
113
|
+
*/
|
|
114
|
+
entityIntsToUuids(ints) {
|
|
115
|
+
return this.native.intsToUuidsBig(Array.from(ints));
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* @description Return every assigned integer ID. This is the "all-ids universe"
|
|
119
|
+
* brainy's `MetadataIndexManager` uses to evaluate negation / `exists:false` /
|
|
120
|
+
* `missing:true` filters (the set of all known entities minus the matching set)
|
|
121
|
+
* when a native EntityIdMapper is injected into the TypeScript metadata index.
|
|
122
|
+
* Ids come from the native `int → uuid` map, so the universe is sourced from the
|
|
123
|
+
* same canonical assignment `getOrAssign` produces — no storage scan.
|
|
124
|
+
* @returns All assigned integer IDs, in unspecified order.
|
|
125
|
+
* @example
|
|
126
|
+
* const allUuids = mapper.intsIterableToUuids(mapper.getAllIntIds())
|
|
127
|
+
*/
|
|
128
|
+
getAllIntIds() {
|
|
129
|
+
const snapshot = JSON.parse(this.native.saveToJson());
|
|
130
|
+
const intToUuid = snapshot.intToUuid ?? {};
|
|
131
|
+
const ids = [];
|
|
132
|
+
for (const key of Object.keys(intToUuid)) {
|
|
133
|
+
ids.push(Number(key));
|
|
134
|
+
}
|
|
135
|
+
return ids;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Flush mappings to storage
|
|
139
|
+
*/
|
|
140
|
+
async flush() {
|
|
141
|
+
if (!this.native.isDirty()) {
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
const json = this.native.saveToJson();
|
|
145
|
+
const data = JSON.parse(json);
|
|
146
|
+
await this.storage.saveMetadata(this.storageKey, data);
|
|
147
|
+
// Reload to clear dirty flag (native sets dirty=false on load)
|
|
148
|
+
this.native.loadFromJson(json);
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Clear all mappings
|
|
152
|
+
*/
|
|
153
|
+
async clear() {
|
|
154
|
+
this.native.clear();
|
|
155
|
+
await this.flush();
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Get statistics about the mapper
|
|
159
|
+
*/
|
|
160
|
+
getStats() {
|
|
161
|
+
const stats = this.native.getStats();
|
|
162
|
+
return {
|
|
163
|
+
mappings: stats.mappings,
|
|
164
|
+
nextId: stats.nextId,
|
|
165
|
+
dirty: stats.dirty,
|
|
166
|
+
memoryEstimate: stats.memoryEstimate,
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
//# sourceMappingURL=nativeEntityIdMapper.js.map
|