@soulcraft/cor 3.0.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/LICENSE +16 -0
  2. package/README.md +328 -0
  3. package/dist/aggregation/NativeAggregationEngine.d.ts +118 -0
  4. package/dist/aggregation/NativeAggregationEngine.js +186 -0
  5. package/dist/cli.d.ts +12 -0
  6. package/dist/cli.js +236 -0
  7. package/dist/graph/GraphAccelerationAdapter.d.ts +153 -0
  8. package/dist/graph/GraphAccelerationAdapter.js +326 -0
  9. package/dist/graph/NativeGraphAdjacencyIndex.d.ts +286 -0
  10. package/dist/graph/NativeGraphAdjacencyIndex.js +944 -0
  11. package/dist/hnsw/AdaptiveDiskAnnModeSelector.d.ts +187 -0
  12. package/dist/hnsw/AdaptiveDiskAnnModeSelector.js +313 -0
  13. package/dist/hnsw/NativeDiskAnnWrapper.d.ts +489 -0
  14. package/dist/hnsw/NativeDiskAnnWrapper.js +1456 -0
  15. package/dist/index.d.ts +28 -0
  16. package/dist/index.js +29 -0
  17. package/dist/legacyLayoutGuard.d.ts +93 -0
  18. package/dist/legacyLayoutGuard.js +237 -0
  19. package/dist/license/constants.d.ts +30 -0
  20. package/dist/license/constants.js +30 -0
  21. package/dist/license/onlineValidator.d.ts +43 -0
  22. package/dist/license/onlineValidator.js +225 -0
  23. package/dist/license/types.d.ts +39 -0
  24. package/dist/license/types.js +5 -0
  25. package/dist/license.d.ts +64 -0
  26. package/dist/license.js +165 -0
  27. package/dist/migration/MigrationCoordinator.d.ts +171 -0
  28. package/dist/migration/MigrationCoordinator.js +249 -0
  29. package/dist/migration/indexEpochGuard.d.ts +163 -0
  30. package/dist/migration/indexEpochGuard.js +202 -0
  31. package/dist/native/NativeEmbeddingEngine.d.ts +79 -0
  32. package/dist/native/NativeEmbeddingEngine.js +318 -0
  33. package/dist/native/NativeRoaringBitmap32.d.ts +114 -0
  34. package/dist/native/NativeRoaringBitmap32.js +221 -0
  35. package/dist/native/ffi.d.ts +20 -0
  36. package/dist/native/ffi.js +48 -0
  37. package/dist/native/idMapperTestSupport.d.ts +67 -0
  38. package/dist/native/idMapperTestSupport.js +112 -0
  39. package/dist/native/index.d.ts +49 -0
  40. package/dist/native/index.js +87 -0
  41. package/dist/native/napi.d.ts +21 -0
  42. package/dist/native/napi.js +88 -0
  43. package/dist/native/types.d.ts +1298 -0
  44. package/dist/native/types.js +16 -0
  45. package/dist/plugin.d.ts +50 -0
  46. package/dist/plugin.js +388 -0
  47. package/dist/providerContracts.d.ts +277 -0
  48. package/dist/providerContracts.js +38 -0
  49. package/dist/resource/OsMemoryProbe.d.ts +175 -0
  50. package/dist/resource/OsMemoryProbe.js +206 -0
  51. package/dist/resource/ResourceManager.d.ts +491 -0
  52. package/dist/resource/ResourceManager.js +960 -0
  53. package/dist/utils/ColumnManifest.d.ts +97 -0
  54. package/dist/utils/ColumnManifest.js +129 -0
  55. package/dist/utils/NativeColumnStore.d.ts +284 -0
  56. package/dist/utils/NativeColumnStore.js +685 -0
  57. package/dist/utils/NativeMetadataIndex.d.ts +882 -0
  58. package/dist/utils/NativeMetadataIndex.js +2631 -0
  59. package/dist/utils/NativeUnifiedCache.d.ts +87 -0
  60. package/dist/utils/NativeUnifiedCache.js +273 -0
  61. package/dist/utils/binaryIdMapperFactory.d.ts +59 -0
  62. package/dist/utils/binaryIdMapperFactory.js +94 -0
  63. package/dist/utils/collation.d.ts +30 -0
  64. package/dist/utils/collation.js +40 -0
  65. package/dist/utils/columnStoreTypes.d.ts +161 -0
  66. package/dist/utils/columnStoreTypes.js +29 -0
  67. package/dist/utils/nativeBinaryEntityIdMapper.d.ts +211 -0
  68. package/dist/utils/nativeBinaryEntityIdMapper.js +381 -0
  69. package/dist/utils/nativeEntityIdMapper.d.ts +111 -0
  70. package/dist/utils/nativeEntityIdMapper.js +170 -0
  71. package/docs/ADR-002-diskann-100-percent-rust.md +294 -0
  72. package/docs/ADR-003-semantic-time-travel.md +100 -0
  73. package/docs/aggregation.md +251 -0
  74. package/docs/comparison.md +162 -0
  75. package/docs/deployment-limits.md +87 -0
  76. package/docs/diskann.md +184 -0
  77. package/docs/migration-3.0.md +396 -0
  78. package/docs/performance-budget.md +274 -0
  79. package/docs/performance.md +117 -0
  80. package/docs/scaling.md +439 -0
  81. package/docs/snapshot-safety.md +246 -0
  82. package/docs/u64-id-space.md +214 -0
  83. package/docs/verification-report.md +670 -0
  84. package/native/brainy-native.node +0 -0
  85. package/native/index.d.ts +3916 -0
  86. package/package.json +92 -0
  87. package/scripts/migrate-cortex-2x-to-3x.mjs +970 -0
@@ -0,0 +1,87 @@
1
+ /**
2
+ * NativeUnifiedCacheWrapper — Native Rust eviction engine with JS data storage
3
+ *
4
+ * The Rust NativeUnifiedCache manages eviction metadata (key, type, size, cost,
5
+ * access counts) but does NOT store actual JS data. This wrapper:
6
+ * - Keeps a Map<string, any> for actual cached data
7
+ * - Delegates eviction decisions to the native engine
8
+ * - Implements request coalescing in JS (same as brainy's UnifiedCache)
9
+ * - Drop-in replacement for brainy's UnifiedCache
10
+ */
11
+ import type { UnifiedCacheConfig } from '@soulcraft/brainy/internals';
12
+ import type { CacheProvider } from '../providerContracts.js';
13
+ /**
14
+ * Implements {@link CacheProvider} so a compile error fires the moment this
15
+ * wrapper drops or drifts a member brainy calls on the installed global cache
16
+ * (`getSync` / `set` / `delete` / `deleteByPrefix` / `clear`).
17
+ */
18
+ export declare class NativeUnifiedCacheWrapper implements CacheProvider {
19
+ private data;
20
+ private native;
21
+ private loadingPromises;
22
+ private readonly maxSize;
23
+ private readonly config;
24
+ private readonly memoryInfo;
25
+ private readonly allocationStrategy;
26
+ private memoryPressureCheckTimer;
27
+ private lastMemoryWarning;
28
+ constructor(config?: UnifiedCacheConfig);
29
+ get(key: string, loadFn?: () => Promise<any>): Promise<any>;
30
+ getSync(key: string): any | undefined;
31
+ set(key: string, data: any, type: 'vectors' | 'metadata' | 'embedding' | 'other', size: number, rebuildCost?: number): void;
32
+ delete(key: string): boolean;
33
+ deleteByPrefix(prefix: string): number;
34
+ clear(type?: 'vectors' | 'metadata' | 'embedding' | 'other'): void;
35
+ evictForSize(bytesNeeded: number): boolean;
36
+ private startFairnessMonitor;
37
+ private checkFairness;
38
+ private startMemoryPressureMonitor;
39
+ private checkMemoryPressure;
40
+ getStats(): {
41
+ totalSize: number;
42
+ maxSize: number;
43
+ utilization: number;
44
+ itemCount: number;
45
+ typeSizes: Record<string, number>;
46
+ typeCounts: Record<string, number>;
47
+ typeAccessCounts: Record<string, number>;
48
+ totalAccessCount: number;
49
+ hitRate: number;
50
+ memory: {
51
+ available: number;
52
+ source: "cgroup-v2" | "cgroup-v1" | "system" | "fallback";
53
+ isContainer: boolean;
54
+ systemTotal: number;
55
+ allocationRatio: number;
56
+ environment: "unknown" | "production" | "development" | "container";
57
+ };
58
+ };
59
+ getMemoryInfo(): {
60
+ memoryInfo: {
61
+ available: number;
62
+ source: "cgroup-v2" | "cgroup-v1" | "system" | "fallback";
63
+ isContainer: boolean;
64
+ systemTotal: number;
65
+ free: number;
66
+ warnings: string[];
67
+ };
68
+ allocationStrategy: {
69
+ cacheSize: number;
70
+ ratio: number;
71
+ minSize: number;
72
+ maxSize: number | null;
73
+ environment: "production" | "development" | "container" | "unknown";
74
+ modelMemory: number;
75
+ modelPrecision: "q8" | "fp32";
76
+ availableForCache: number;
77
+ reasoning: string;
78
+ };
79
+ currentPressure: {
80
+ pressure: "none" | "moderate" | "high" | "critical";
81
+ warnings: string[];
82
+ };
83
+ };
84
+ saveAccessPatterns(): Promise<any>;
85
+ loadAccessPatterns(patterns: any): Promise<void>;
86
+ }
87
+ //# sourceMappingURL=NativeUnifiedCache.d.ts.map
@@ -0,0 +1,273 @@
1
+ /**
2
+ * NativeUnifiedCacheWrapper — Native Rust eviction engine with JS data storage
3
+ *
4
+ * The Rust NativeUnifiedCache manages eviction metadata (key, type, size, cost,
5
+ * access counts) but does NOT store actual JS data. This wrapper:
6
+ * - Keeps a Map<string, any> for actual cached data
7
+ * - Delegates eviction decisions to the native engine
8
+ * - Implements request coalescing in JS (same as brainy's UnifiedCache)
9
+ * - Drop-in replacement for brainy's UnifiedCache
10
+ */
11
+ import { loadNativeModule } from '../native/index.js';
12
+ import { prodLog } from '@soulcraft/brainy/internals';
13
+ import { getRecommendedCacheConfig, formatBytes, checkMemoryPressure, } from '@soulcraft/brainy/internals';
14
+ // Cache type enum matching Rust (0-3)
15
+ const CACHE_TYPE_MAP = {
16
+ vectors: 0,
17
+ metadata: 1,
18
+ embedding: 2,
19
+ other: 3,
20
+ };
21
+ /**
22
+ * Implements {@link CacheProvider} so a compile error fires the moment this
23
+ * wrapper drops or drifts a member brainy calls on the installed global cache
24
+ * (`getSync` / `set` / `delete` / `deleteByPrefix` / `clear`).
25
+ */
26
+ export class NativeUnifiedCacheWrapper {
27
+ data = new Map();
28
+ native;
29
+ loadingPromises = new Map();
30
+ maxSize;
31
+ config;
32
+ memoryInfo;
33
+ allocationStrategy;
34
+ memoryPressureCheckTimer = null;
35
+ lastMemoryWarning = 0;
36
+ constructor(config = {}) {
37
+ const recommendation = getRecommendedCacheConfig({
38
+ manualSize: config.maxSize,
39
+ minSize: config.minSize,
40
+ developmentMode: config.developmentMode,
41
+ });
42
+ this.memoryInfo = recommendation.memoryInfo;
43
+ this.allocationStrategy = recommendation.allocation;
44
+ this.maxSize = recommendation.allocation.cacheSize;
45
+ prodLog.info(`NativeUnifiedCache initialized: ${formatBytes(this.maxSize)} ` +
46
+ `(${this.allocationStrategy.environment} mode, ` +
47
+ `${(this.allocationStrategy.ratio * 100).toFixed(0)}% of ${formatBytes(this.allocationStrategy.availableForCache)} ` +
48
+ `after ${formatBytes(this.allocationStrategy.modelMemory)} ${this.allocationStrategy.modelPrecision.toUpperCase()} model)`);
49
+ for (const warning of recommendation.warnings) {
50
+ prodLog.warn(`NativeUnifiedCache: ${warning}`);
51
+ }
52
+ this.config = {
53
+ enableRequestCoalescing: true,
54
+ enableFairnessCheck: true,
55
+ fairnessCheckInterval: 30000,
56
+ persistPatterns: true,
57
+ enableMemoryMonitoring: true,
58
+ memoryCheckInterval: 30000,
59
+ ...config,
60
+ };
61
+ const bindings = loadNativeModule();
62
+ this.native = new bindings.NativeUnifiedCache(this.maxSize);
63
+ if (this.config.enableFairnessCheck) {
64
+ this.startFairnessMonitor();
65
+ }
66
+ if (this.config.enableMemoryMonitoring) {
67
+ this.startMemoryPressureMonitor();
68
+ }
69
+ }
70
+ // ---------------------------------------------------------------------------
71
+ // Core API
72
+ // ---------------------------------------------------------------------------
73
+ async get(key, loadFn) {
74
+ // Check JS data map
75
+ const cached = this.data.get(key);
76
+ if (cached !== undefined) {
77
+ this.native.recordAccess(key);
78
+ return cached;
79
+ }
80
+ if (!loadFn)
81
+ return undefined;
82
+ // Request coalescing
83
+ if (this.config.enableRequestCoalescing && this.loadingPromises.has(key)) {
84
+ return this.loadingPromises.get(key);
85
+ }
86
+ const loadPromise = loadFn();
87
+ if (this.config.enableRequestCoalescing) {
88
+ this.loadingPromises.set(key, loadPromise);
89
+ }
90
+ try {
91
+ const result = await loadPromise;
92
+ return result;
93
+ }
94
+ finally {
95
+ if (this.config.enableRequestCoalescing) {
96
+ this.loadingPromises.delete(key);
97
+ }
98
+ }
99
+ }
100
+ getSync(key) {
101
+ const cached = this.data.get(key);
102
+ if (cached !== undefined) {
103
+ this.native.recordAccess(key);
104
+ return cached;
105
+ }
106
+ return undefined;
107
+ }
108
+ set(key, data, type, size, rebuildCost = 1) {
109
+ const cacheType = CACHE_TYPE_MAP[type] ?? 3;
110
+ // Remove existing entry if present
111
+ if (this.data.has(key)) {
112
+ this.native.remove(key);
113
+ }
114
+ // Insert into native (may trigger eviction)
115
+ const evictionResult = this.native.insert(key, cacheType, size, rebuildCost);
116
+ // Remove evicted keys from JS data
117
+ for (const evictedKey of evictionResult.evictedKeys) {
118
+ this.data.delete(evictedKey);
119
+ }
120
+ // Store actual data in JS
121
+ this.data.set(key, data);
122
+ }
123
+ delete(key) {
124
+ const existed = this.data.has(key);
125
+ if (existed) {
126
+ this.native.remove(key);
127
+ this.data.delete(key);
128
+ }
129
+ return existed;
130
+ }
131
+ deleteByPrefix(prefix) {
132
+ const removedKeys = this.native.removeByPrefix(prefix);
133
+ for (const key of removedKeys) {
134
+ this.data.delete(key);
135
+ }
136
+ return removedKeys.length;
137
+ }
138
+ clear(type) {
139
+ if (!type) {
140
+ const clearedKeys = this.native.clear();
141
+ for (const key of clearedKeys) {
142
+ this.data.delete(key);
143
+ }
144
+ // Also clear any remaining data entries
145
+ this.data.clear();
146
+ }
147
+ else {
148
+ const cacheType = CACHE_TYPE_MAP[type] ?? 3;
149
+ const clearedKeys = this.native.clear(cacheType);
150
+ for (const key of clearedKeys) {
151
+ this.data.delete(key);
152
+ }
153
+ }
154
+ }
155
+ // ---------------------------------------------------------------------------
156
+ // Eviction
157
+ // ---------------------------------------------------------------------------
158
+ evictForSize(bytesNeeded) {
159
+ const result = this.native.evictForSize(bytesNeeded);
160
+ for (const key of result.evictedKeys) {
161
+ this.data.delete(key);
162
+ }
163
+ return result.bytesFreed >= bytesNeeded;
164
+ }
165
+ // ---------------------------------------------------------------------------
166
+ // Fairness
167
+ // ---------------------------------------------------------------------------
168
+ startFairnessMonitor() {
169
+ const timer = setInterval(() => {
170
+ this.checkFairness();
171
+ }, this.config.fairnessCheckInterval);
172
+ if (timer.unref)
173
+ timer.unref();
174
+ }
175
+ checkFairness() {
176
+ const result = this.native.checkFairness();
177
+ for (const key of result.evictedKeys) {
178
+ this.data.delete(key);
179
+ }
180
+ }
181
+ // ---------------------------------------------------------------------------
182
+ // Memory pressure
183
+ // ---------------------------------------------------------------------------
184
+ startMemoryPressureMonitor() {
185
+ const checkInterval = this.config.memoryCheckInterval || 30000;
186
+ this.memoryPressureCheckTimer = setInterval(() => {
187
+ this.checkMemoryPressure();
188
+ }, checkInterval);
189
+ if (this.memoryPressureCheckTimer.unref) {
190
+ this.memoryPressureCheckTimer.unref();
191
+ }
192
+ }
193
+ checkMemoryPressure() {
194
+ const stats = this.native.getStats();
195
+ const pressure = checkMemoryPressure(stats.totalSize, this.memoryInfo);
196
+ const now = Date.now();
197
+ const fiveMinutes = 5 * 60 * 1000;
198
+ if (pressure.warnings.length > 0 && now - this.lastMemoryWarning > fiveMinutes) {
199
+ for (const warning of pressure.warnings) {
200
+ prodLog.warn(`NativeUnifiedCache: ${warning}`);
201
+ }
202
+ this.lastMemoryWarning = now;
203
+ }
204
+ if (pressure.pressure === 'critical') {
205
+ const targetSize = Math.floor(this.maxSize * 0.7);
206
+ const bytesToFree = stats.totalSize - targetSize;
207
+ if (bytesToFree > 0) {
208
+ prodLog.warn(`NativeUnifiedCache: Critical memory pressure - forcing eviction of ${formatBytes(bytesToFree)}`);
209
+ this.evictForSize(bytesToFree);
210
+ }
211
+ }
212
+ }
213
+ // ---------------------------------------------------------------------------
214
+ // Statistics
215
+ // ---------------------------------------------------------------------------
216
+ getStats() {
217
+ const nativeStats = this.native.getStats();
218
+ const typeNames = ['vectors', 'metadata', 'embedding', 'other'];
219
+ const typeSizes = {};
220
+ const typeCounts = {};
221
+ const typeAccessCounts = {};
222
+ for (let i = 0; i < typeNames.length; i++) {
223
+ typeSizes[typeNames[i]] = nativeStats.typeSizes[i] || 0;
224
+ typeCounts[typeNames[i]] = nativeStats.typeCounts[i] || 0;
225
+ typeAccessCounts[typeNames[i]] = nativeStats.typeAccessCounts[i] || 0;
226
+ }
227
+ const hitRate = nativeStats.totalAccessCount > 0
228
+ ? nativeStats.itemCount / nativeStats.totalAccessCount
229
+ : 0;
230
+ return {
231
+ totalSize: nativeStats.totalSize,
232
+ maxSize: nativeStats.maxSize,
233
+ utilization: nativeStats.maxSize > 0 ? nativeStats.totalSize / nativeStats.maxSize : 0,
234
+ itemCount: nativeStats.itemCount,
235
+ typeSizes,
236
+ typeCounts,
237
+ typeAccessCounts,
238
+ totalAccessCount: nativeStats.totalAccessCount,
239
+ hitRate,
240
+ memory: {
241
+ available: this.memoryInfo.available,
242
+ source: this.memoryInfo.source,
243
+ isContainer: this.memoryInfo.isContainer,
244
+ systemTotal: this.memoryInfo.systemTotal,
245
+ allocationRatio: this.allocationStrategy.ratio,
246
+ environment: this.allocationStrategy.environment,
247
+ },
248
+ };
249
+ }
250
+ getMemoryInfo() {
251
+ const stats = this.native.getStats();
252
+ return {
253
+ memoryInfo: { ...this.memoryInfo },
254
+ allocationStrategy: { ...this.allocationStrategy },
255
+ currentPressure: checkMemoryPressure(stats.totalSize, this.memoryInfo),
256
+ };
257
+ }
258
+ // ---------------------------------------------------------------------------
259
+ // Access pattern persistence
260
+ // ---------------------------------------------------------------------------
261
+ async saveAccessPatterns() {
262
+ if (!this.config.persistPatterns)
263
+ return;
264
+ const json = this.native.saveAccessPatterns();
265
+ return JSON.parse(json);
266
+ }
267
+ async loadAccessPatterns(patterns) {
268
+ if (!patterns)
269
+ return;
270
+ this.native.loadAccessPatterns(JSON.stringify(patterns));
271
+ }
272
+ }
273
+ //# sourceMappingURL=NativeUnifiedCache.js.map
@@ -0,0 +1,59 @@
1
+ /**
2
+ * @module utils/binaryIdMapperFactory
3
+ * @description Single canonical constructor for cor's native mmap
4
+ * `BinaryIdMapper`. Derives the `_id_mapper/*` paths from a
5
+ * filesystem-backed storage adapter and opens (or creates / truncates)
6
+ * the paired `uuid_to_int.mkv` + `int_to_uuid.bin` files.
7
+ *
8
+ * Both the metadata index (the canonical UUID ↔ entity-int allocator
9
+ * since #72 Phase C) and the legacy `entityIdMapper` provider go through
10
+ * here so there is exactly one place that knows the file layout, the
11
+ * default sizes, and the paired-write / half-present invariant.
12
+ *
13
+ * The mapper is mmap-backed and demand-paged: a 32 GB default file is
14
+ * SPARSE on disk (only touched pages occupy blocks), so a fresh small
15
+ * brain pays only for what it uses. This is the constant-RAM,
16
+ * billion-scale store that replaces the resident JSON id-maps (#72).
17
+ */
18
+ import type { NativeBinaryIdMapperInstance } from '../native/types.js';
19
+ /** Default storage key for the extendible-hash UUID → int file. */
20
+ export declare const DEFAULT_UUID_TO_INT_KEY = "_id_mapper/uuid_to_int.mkv";
21
+ /** Default storage key for the direct-indexed int → UUID file. */
22
+ export declare const DEFAULT_INT_TO_UUID_KEY = "_id_mapper/int_to_uuid.bin";
23
+ /** Minimal storage surface this factory needs. */
24
+ export interface StorageWithBlobPath {
25
+ getBinaryBlobPath?: (key: string) => string | null;
26
+ }
27
+ /** Options controlling file layout, sizing, and id width. */
28
+ export interface BinaryIdMapperFactoryOptions {
29
+ uuidToIntKey?: string;
30
+ intToUuidKey?: string;
31
+ intToUuidSize?: bigint;
32
+ uuidToIntSize?: bigint;
33
+ bucketCapacity?: number;
34
+ maxGlobalDepth?: number;
35
+ idSpace?: 'u32' | 'u64';
36
+ }
37
+ /** What {@link openOrCreateBinaryIdMapper} resolves the mapper with. */
38
+ export interface BinaryIdMapperHandle {
39
+ mapper: NativeBinaryIdMapperInstance;
40
+ uuidToIntPath: string;
41
+ intToUuidPath: string;
42
+ /** The mapper's actual on-disk IdSpace (authoritative over requested). */
43
+ idSpace: 'u32' | 'u64';
44
+ /** Whether the files were freshly created (vs opened existing). */
45
+ created: boolean;
46
+ }
47
+ /**
48
+ * Open (or create / truncate) the native `BinaryIdMapper` for a brain.
49
+ *
50
+ * @param storage - Filesystem-backed storage exposing `getBinaryBlobPath`.
51
+ * @param opts - File layout, sizing, id width, and the `forceCreate`
52
+ * rebuild flag.
53
+ * @returns The native mapper handle plus its resolved paths + IdSpace.
54
+ * @throws If storage has no `getBinaryBlobPath`, if a path resolves null,
55
+ * if the native `NativeBinaryIdMapper` binding is missing, or if the
56
+ * two files are in a half-present (crash-torn) state.
57
+ */
58
+ export declare function openOrCreateBinaryIdMapper(storage: StorageWithBlobPath, opts?: BinaryIdMapperFactoryOptions): BinaryIdMapperHandle;
59
+ //# sourceMappingURL=binaryIdMapperFactory.d.ts.map
@@ -0,0 +1,94 @@
1
+ /**
2
+ * @module utils/binaryIdMapperFactory
3
+ * @description Single canonical constructor for cor's native mmap
4
+ * `BinaryIdMapper`. Derives the `_id_mapper/*` paths from a
5
+ * filesystem-backed storage adapter and opens (or creates / truncates)
6
+ * the paired `uuid_to_int.mkv` + `int_to_uuid.bin` files.
7
+ *
8
+ * Both the metadata index (the canonical UUID ↔ entity-int allocator
9
+ * since #72 Phase C) and the legacy `entityIdMapper` provider go through
10
+ * here so there is exactly one place that knows the file layout, the
11
+ * default sizes, and the paired-write / half-present invariant.
12
+ *
13
+ * The mapper is mmap-backed and demand-paged: a 32 GB default file is
14
+ * SPARSE on disk (only touched pages occupy blocks), so a fresh small
15
+ * brain pays only for what it uses. This is the constant-RAM,
16
+ * billion-scale store that replaces the resident JSON id-maps (#72).
17
+ */
18
+ import { existsSync } from 'node:fs';
19
+ import { loadNativeModule } from '../native/index.js';
20
+ /** Default storage key for the extendible-hash UUID → int file. */
21
+ export const DEFAULT_UUID_TO_INT_KEY = '_id_mapper/uuid_to_int.mkv';
22
+ /** Default storage key for the direct-indexed int → UUID file. */
23
+ export const DEFAULT_INT_TO_UUID_KEY = '_id_mapper/int_to_uuid.bin';
24
+ /** 32 GB default sparse-file size for each mapper file. */
25
+ const DEFAULT_FILE_SIZE = BigInt(32) * BigInt(1024) ** BigInt(3);
26
+ /**
27
+ * Open (or create / truncate) the native `BinaryIdMapper` for a brain.
28
+ *
29
+ * @param storage - Filesystem-backed storage exposing `getBinaryBlobPath`.
30
+ * @param opts - File layout, sizing, id width, and the `forceCreate`
31
+ * rebuild flag.
32
+ * @returns The native mapper handle plus its resolved paths + IdSpace.
33
+ * @throws If storage has no `getBinaryBlobPath`, if a path resolves null,
34
+ * if the native `NativeBinaryIdMapper` binding is missing, or if the
35
+ * two files are in a half-present (crash-torn) state.
36
+ */
37
+ export function openOrCreateBinaryIdMapper(storage, opts = {}) {
38
+ if (!storage.getBinaryBlobPath) {
39
+ throw new Error('openOrCreateBinaryIdMapper requires a storage adapter that exposes ' +
40
+ 'getBinaryBlobPath() (filesystem-backed).');
41
+ }
42
+ const uuidToIntKey = opts.uuidToIntKey ?? DEFAULT_UUID_TO_INT_KEY;
43
+ const intToUuidKey = opts.intToUuidKey ?? DEFAULT_INT_TO_UUID_KEY;
44
+ const uuidToIntPath = storage.getBinaryBlobPath(uuidToIntKey);
45
+ const intToUuidPath = storage.getBinaryBlobPath(intToUuidKey);
46
+ if (!uuidToIntPath || !intToUuidPath) {
47
+ throw new Error(`openOrCreateBinaryIdMapper: getBinaryBlobPath returned null for ` +
48
+ `${uuidToIntKey} or ${intToUuidKey}`);
49
+ }
50
+ const bindings = loadNativeModule();
51
+ const NativeBinaryIdMapper = bindings.NativeBinaryIdMapper;
52
+ if (!NativeBinaryIdMapper) {
53
+ throw new Error('NativeBinaryIdMapper binding missing from cor native module — ' +
54
+ 'this build of cor is older than the BinaryIdMapper feature');
55
+ }
56
+ const config = {
57
+ uuidToIntPath,
58
+ intToUuidPath,
59
+ intToUuidSize: opts.intToUuidSize ?? DEFAULT_FILE_SIZE,
60
+ uuidToIntSize: opts.uuidToIntSize ?? DEFAULT_FILE_SIZE,
61
+ bucketCapacity: opts.bucketCapacity ?? 16,
62
+ maxGlobalDepth: opts.maxGlobalDepth ?? 28,
63
+ idSpace: opts.idSpace ?? 'u64',
64
+ };
65
+ // Both files must exist together (paired write semantics). A
66
+ // half-present state is crash corruption surfaced as an error,
67
+ // never silently recreated.
68
+ let mapper;
69
+ let created;
70
+ const uuidFileExists = existsSync(uuidToIntPath);
71
+ const intFileExists = existsSync(intToUuidPath);
72
+ if (uuidFileExists && intFileExists) {
73
+ mapper = NativeBinaryIdMapper.openExisting(config);
74
+ created = false;
75
+ }
76
+ else if (!uuidFileExists && !intFileExists) {
77
+ mapper = NativeBinaryIdMapper.create(config);
78
+ created = true;
79
+ }
80
+ else {
81
+ throw new Error(`openOrCreateBinaryIdMapper: half-present file pair — ` +
82
+ `${uuidToIntKey} ${uuidFileExists ? 'exists' : 'missing'}, ` +
83
+ `${intToUuidKey} ${intFileExists ? 'exists' : 'missing'}. ` +
84
+ `Refusing to silently recreate; investigate manually.`);
85
+ }
86
+ return {
87
+ mapper,
88
+ uuidToIntPath,
89
+ intToUuidPath,
90
+ idSpace: mapper.idSpace(),
91
+ created,
92
+ };
93
+ }
94
+ //# sourceMappingURL=binaryIdMapperFactory.js.map
@@ -0,0 +1,30 @@
1
+ /**
2
+ * @module utils/collation
3
+ * @description Deterministic string ordering for cor's TypeScript fallback paths,
4
+ * byte-for-byte identical to `@soulcraft/brainy`'s `compareCodePoints` and to the
5
+ * native (Rust) column-store / aggregation engine.
6
+ *
7
+ * Strings are compared by **UTF-8 byte order**, which equals Unicode code-point order
8
+ * and Rust's `str::cmp` (`as_bytes().cmp()`). This is deterministic across
9
+ * environments (unlike `localeCompare`, whose default-locale ordering varies by
10
+ * OS / Node / ICU build) and consistent with the native engine, so sorted query
11
+ * results are identical with or without the native plugin.
12
+ *
13
+ * Cor keeps a local copy (rather than importing brainy's) so it never depends on
14
+ * a brainy internal export and is guaranteed available against any supported brainy
15
+ * version. The implementation must stay in lockstep with brainy `src/utils/collation.ts`.
16
+ * See `docs/ADR-001`.
17
+ */
18
+ /**
19
+ * @description Compare two strings by UTF-8 byte order (== Unicode code-point order
20
+ * == Rust `str::cmp`). Use instead of `String.localeCompare` and the `<`/`>`
21
+ * operators (which compare UTF-16 code units and diverge for supplementary-plane
22
+ * characters) wherever ordering must match the native engine or be deterministic.
23
+ * @param a - First string.
24
+ * @param b - Second string.
25
+ * @returns Negative if `a < b`, positive if `a > b`, `0` if equal — by UTF-8 byte order.
26
+ * @example
27
+ * ['b', 'A', 'a'].sort(compareCodePoints) // ['A', 'a', 'b'] (A=65, a=97, b=98)
28
+ */
29
+ export declare function compareCodePoints(a: string, b: string): number;
30
+ //# sourceMappingURL=collation.d.ts.map
@@ -0,0 +1,40 @@
1
+ /**
2
+ * @module utils/collation
3
+ * @description Deterministic string ordering for cor's TypeScript fallback paths,
4
+ * byte-for-byte identical to `@soulcraft/brainy`'s `compareCodePoints` and to the
5
+ * native (Rust) column-store / aggregation engine.
6
+ *
7
+ * Strings are compared by **UTF-8 byte order**, which equals Unicode code-point order
8
+ * and Rust's `str::cmp` (`as_bytes().cmp()`). This is deterministic across
9
+ * environments (unlike `localeCompare`, whose default-locale ordering varies by
10
+ * OS / Node / ICU build) and consistent with the native engine, so sorted query
11
+ * results are identical with or without the native plugin.
12
+ *
13
+ * Cor keeps a local copy (rather than importing brainy's) so it never depends on
14
+ * a brainy internal export and is guaranteed available against any supported brainy
15
+ * version. The implementation must stay in lockstep with brainy `src/utils/collation.ts`.
16
+ * See `docs/ADR-001`.
17
+ */
18
+ const utf8 = new TextEncoder();
19
+ /**
20
+ * @description Compare two strings by UTF-8 byte order (== Unicode code-point order
21
+ * == Rust `str::cmp`). Use instead of `String.localeCompare` and the `<`/`>`
22
+ * operators (which compare UTF-16 code units and diverge for supplementary-plane
23
+ * characters) wherever ordering must match the native engine or be deterministic.
24
+ * @param a - First string.
25
+ * @param b - Second string.
26
+ * @returns Negative if `a < b`, positive if `a > b`, `0` if equal — by UTF-8 byte order.
27
+ * @example
28
+ * ['b', 'A', 'a'].sort(compareCodePoints) // ['A', 'a', 'b'] (A=65, a=97, b=98)
29
+ */
30
+ export function compareCodePoints(a, b) {
31
+ const ea = utf8.encode(a);
32
+ const eb = utf8.encode(b);
33
+ const len = ea.length < eb.length ? ea.length : eb.length;
34
+ for (let i = 0; i < len; i++) {
35
+ if (ea[i] !== eb[i])
36
+ return ea[i] - eb[i];
37
+ }
38
+ return ea.length - eb.length;
39
+ }
40
+ //# sourceMappingURL=collation.js.map