@soulcraft/cor 3.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +16 -0
- package/README.md +328 -0
- package/dist/aggregation/NativeAggregationEngine.d.ts +118 -0
- package/dist/aggregation/NativeAggregationEngine.js +186 -0
- package/dist/cli.d.ts +12 -0
- package/dist/cli.js +236 -0
- package/dist/graph/GraphAccelerationAdapter.d.ts +153 -0
- package/dist/graph/GraphAccelerationAdapter.js +326 -0
- package/dist/graph/NativeGraphAdjacencyIndex.d.ts +286 -0
- package/dist/graph/NativeGraphAdjacencyIndex.js +944 -0
- package/dist/hnsw/AdaptiveDiskAnnModeSelector.d.ts +187 -0
- package/dist/hnsw/AdaptiveDiskAnnModeSelector.js +313 -0
- package/dist/hnsw/NativeDiskAnnWrapper.d.ts +489 -0
- package/dist/hnsw/NativeDiskAnnWrapper.js +1456 -0
- package/dist/index.d.ts +28 -0
- package/dist/index.js +29 -0
- package/dist/legacyLayoutGuard.d.ts +93 -0
- package/dist/legacyLayoutGuard.js +237 -0
- package/dist/license/constants.d.ts +30 -0
- package/dist/license/constants.js +30 -0
- package/dist/license/onlineValidator.d.ts +43 -0
- package/dist/license/onlineValidator.js +225 -0
- package/dist/license/types.d.ts +39 -0
- package/dist/license/types.js +5 -0
- package/dist/license.d.ts +64 -0
- package/dist/license.js +165 -0
- package/dist/migration/MigrationCoordinator.d.ts +171 -0
- package/dist/migration/MigrationCoordinator.js +249 -0
- package/dist/migration/indexEpochGuard.d.ts +163 -0
- package/dist/migration/indexEpochGuard.js +202 -0
- package/dist/native/NativeEmbeddingEngine.d.ts +79 -0
- package/dist/native/NativeEmbeddingEngine.js +318 -0
- package/dist/native/NativeRoaringBitmap32.d.ts +114 -0
- package/dist/native/NativeRoaringBitmap32.js +221 -0
- package/dist/native/ffi.d.ts +20 -0
- package/dist/native/ffi.js +48 -0
- package/dist/native/idMapperTestSupport.d.ts +67 -0
- package/dist/native/idMapperTestSupport.js +112 -0
- package/dist/native/index.d.ts +49 -0
- package/dist/native/index.js +87 -0
- package/dist/native/napi.d.ts +21 -0
- package/dist/native/napi.js +88 -0
- package/dist/native/types.d.ts +1298 -0
- package/dist/native/types.js +16 -0
- package/dist/plugin.d.ts +50 -0
- package/dist/plugin.js +388 -0
- package/dist/providerContracts.d.ts +277 -0
- package/dist/providerContracts.js +38 -0
- package/dist/resource/OsMemoryProbe.d.ts +175 -0
- package/dist/resource/OsMemoryProbe.js +206 -0
- package/dist/resource/ResourceManager.d.ts +491 -0
- package/dist/resource/ResourceManager.js +960 -0
- package/dist/utils/ColumnManifest.d.ts +97 -0
- package/dist/utils/ColumnManifest.js +129 -0
- package/dist/utils/NativeColumnStore.d.ts +284 -0
- package/dist/utils/NativeColumnStore.js +685 -0
- package/dist/utils/NativeMetadataIndex.d.ts +882 -0
- package/dist/utils/NativeMetadataIndex.js +2631 -0
- package/dist/utils/NativeUnifiedCache.d.ts +87 -0
- package/dist/utils/NativeUnifiedCache.js +273 -0
- package/dist/utils/binaryIdMapperFactory.d.ts +59 -0
- package/dist/utils/binaryIdMapperFactory.js +94 -0
- package/dist/utils/collation.d.ts +30 -0
- package/dist/utils/collation.js +40 -0
- package/dist/utils/columnStoreTypes.d.ts +161 -0
- package/dist/utils/columnStoreTypes.js +29 -0
- package/dist/utils/nativeBinaryEntityIdMapper.d.ts +211 -0
- package/dist/utils/nativeBinaryEntityIdMapper.js +381 -0
- package/dist/utils/nativeEntityIdMapper.d.ts +111 -0
- package/dist/utils/nativeEntityIdMapper.js +170 -0
- package/docs/ADR-002-diskann-100-percent-rust.md +294 -0
- package/docs/ADR-003-semantic-time-travel.md +100 -0
- package/docs/aggregation.md +251 -0
- package/docs/comparison.md +162 -0
- package/docs/deployment-limits.md +87 -0
- package/docs/diskann.md +184 -0
- package/docs/migration-3.0.md +396 -0
- package/docs/performance-budget.md +274 -0
- package/docs/performance.md +117 -0
- package/docs/scaling.md +439 -0
- package/docs/snapshot-safety.md +246 -0
- package/docs/u64-id-space.md +214 -0
- package/docs/verification-report.md +670 -0
- package/native/brainy-native.node +0 -0
- package/native/index.d.ts +3916 -0
- package/package.json +92 -0
- package/scripts/migrate-cortex-2x-to-3x.mjs +970 -0
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NativeUnifiedCacheWrapper — Native Rust eviction engine with JS data storage
|
|
3
|
+
*
|
|
4
|
+
* The Rust NativeUnifiedCache manages eviction metadata (key, type, size, cost,
|
|
5
|
+
* access counts) but does NOT store actual JS data. This wrapper:
|
|
6
|
+
* - Keeps a Map<string, any> for actual cached data
|
|
7
|
+
* - Delegates eviction decisions to the native engine
|
|
8
|
+
* - Implements request coalescing in JS (same as brainy's UnifiedCache)
|
|
9
|
+
* - Drop-in replacement for brainy's UnifiedCache
|
|
10
|
+
*/
|
|
11
|
+
import type { UnifiedCacheConfig } from '@soulcraft/brainy/internals';
|
|
12
|
+
import type { CacheProvider } from '../providerContracts.js';
|
|
13
|
+
/**
|
|
14
|
+
* Implements {@link CacheProvider} so a compile error fires the moment this
|
|
15
|
+
* wrapper drops or drifts a member brainy calls on the installed global cache
|
|
16
|
+
* (`getSync` / `set` / `delete` / `deleteByPrefix` / `clear`).
|
|
17
|
+
*/
|
|
18
|
+
export declare class NativeUnifiedCacheWrapper implements CacheProvider {
|
|
19
|
+
private data;
|
|
20
|
+
private native;
|
|
21
|
+
private loadingPromises;
|
|
22
|
+
private readonly maxSize;
|
|
23
|
+
private readonly config;
|
|
24
|
+
private readonly memoryInfo;
|
|
25
|
+
private readonly allocationStrategy;
|
|
26
|
+
private memoryPressureCheckTimer;
|
|
27
|
+
private lastMemoryWarning;
|
|
28
|
+
constructor(config?: UnifiedCacheConfig);
|
|
29
|
+
get(key: string, loadFn?: () => Promise<any>): Promise<any>;
|
|
30
|
+
getSync(key: string): any | undefined;
|
|
31
|
+
set(key: string, data: any, type: 'vectors' | 'metadata' | 'embedding' | 'other', size: number, rebuildCost?: number): void;
|
|
32
|
+
delete(key: string): boolean;
|
|
33
|
+
deleteByPrefix(prefix: string): number;
|
|
34
|
+
clear(type?: 'vectors' | 'metadata' | 'embedding' | 'other'): void;
|
|
35
|
+
evictForSize(bytesNeeded: number): boolean;
|
|
36
|
+
private startFairnessMonitor;
|
|
37
|
+
private checkFairness;
|
|
38
|
+
private startMemoryPressureMonitor;
|
|
39
|
+
private checkMemoryPressure;
|
|
40
|
+
getStats(): {
|
|
41
|
+
totalSize: number;
|
|
42
|
+
maxSize: number;
|
|
43
|
+
utilization: number;
|
|
44
|
+
itemCount: number;
|
|
45
|
+
typeSizes: Record<string, number>;
|
|
46
|
+
typeCounts: Record<string, number>;
|
|
47
|
+
typeAccessCounts: Record<string, number>;
|
|
48
|
+
totalAccessCount: number;
|
|
49
|
+
hitRate: number;
|
|
50
|
+
memory: {
|
|
51
|
+
available: number;
|
|
52
|
+
source: "cgroup-v2" | "cgroup-v1" | "system" | "fallback";
|
|
53
|
+
isContainer: boolean;
|
|
54
|
+
systemTotal: number;
|
|
55
|
+
allocationRatio: number;
|
|
56
|
+
environment: "unknown" | "production" | "development" | "container";
|
|
57
|
+
};
|
|
58
|
+
};
|
|
59
|
+
getMemoryInfo(): {
|
|
60
|
+
memoryInfo: {
|
|
61
|
+
available: number;
|
|
62
|
+
source: "cgroup-v2" | "cgroup-v1" | "system" | "fallback";
|
|
63
|
+
isContainer: boolean;
|
|
64
|
+
systemTotal: number;
|
|
65
|
+
free: number;
|
|
66
|
+
warnings: string[];
|
|
67
|
+
};
|
|
68
|
+
allocationStrategy: {
|
|
69
|
+
cacheSize: number;
|
|
70
|
+
ratio: number;
|
|
71
|
+
minSize: number;
|
|
72
|
+
maxSize: number | null;
|
|
73
|
+
environment: "production" | "development" | "container" | "unknown";
|
|
74
|
+
modelMemory: number;
|
|
75
|
+
modelPrecision: "q8" | "fp32";
|
|
76
|
+
availableForCache: number;
|
|
77
|
+
reasoning: string;
|
|
78
|
+
};
|
|
79
|
+
currentPressure: {
|
|
80
|
+
pressure: "none" | "moderate" | "high" | "critical";
|
|
81
|
+
warnings: string[];
|
|
82
|
+
};
|
|
83
|
+
};
|
|
84
|
+
saveAccessPatterns(): Promise<any>;
|
|
85
|
+
loadAccessPatterns(patterns: any): Promise<void>;
|
|
86
|
+
}
|
|
87
|
+
//# sourceMappingURL=NativeUnifiedCache.d.ts.map
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NativeUnifiedCacheWrapper — Native Rust eviction engine with JS data storage
|
|
3
|
+
*
|
|
4
|
+
* The Rust NativeUnifiedCache manages eviction metadata (key, type, size, cost,
|
|
5
|
+
* access counts) but does NOT store actual JS data. This wrapper:
|
|
6
|
+
* - Keeps a Map<string, any> for actual cached data
|
|
7
|
+
* - Delegates eviction decisions to the native engine
|
|
8
|
+
* - Implements request coalescing in JS (same as brainy's UnifiedCache)
|
|
9
|
+
* - Drop-in replacement for brainy's UnifiedCache
|
|
10
|
+
*/
|
|
11
|
+
import { loadNativeModule } from '../native/index.js';
|
|
12
|
+
import { prodLog } from '@soulcraft/brainy/internals';
|
|
13
|
+
import { getRecommendedCacheConfig, formatBytes, checkMemoryPressure, } from '@soulcraft/brainy/internals';
|
|
14
|
+
// Cache type enum matching Rust (0-3)
|
|
15
|
+
const CACHE_TYPE_MAP = {
|
|
16
|
+
vectors: 0,
|
|
17
|
+
metadata: 1,
|
|
18
|
+
embedding: 2,
|
|
19
|
+
other: 3,
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* Implements {@link CacheProvider} so a compile error fires the moment this
|
|
23
|
+
* wrapper drops or drifts a member brainy calls on the installed global cache
|
|
24
|
+
* (`getSync` / `set` / `delete` / `deleteByPrefix` / `clear`).
|
|
25
|
+
*/
|
|
26
|
+
export class NativeUnifiedCacheWrapper {
|
|
27
|
+
data = new Map();
|
|
28
|
+
native;
|
|
29
|
+
loadingPromises = new Map();
|
|
30
|
+
maxSize;
|
|
31
|
+
config;
|
|
32
|
+
memoryInfo;
|
|
33
|
+
allocationStrategy;
|
|
34
|
+
memoryPressureCheckTimer = null;
|
|
35
|
+
lastMemoryWarning = 0;
|
|
36
|
+
constructor(config = {}) {
|
|
37
|
+
const recommendation = getRecommendedCacheConfig({
|
|
38
|
+
manualSize: config.maxSize,
|
|
39
|
+
minSize: config.minSize,
|
|
40
|
+
developmentMode: config.developmentMode,
|
|
41
|
+
});
|
|
42
|
+
this.memoryInfo = recommendation.memoryInfo;
|
|
43
|
+
this.allocationStrategy = recommendation.allocation;
|
|
44
|
+
this.maxSize = recommendation.allocation.cacheSize;
|
|
45
|
+
prodLog.info(`NativeUnifiedCache initialized: ${formatBytes(this.maxSize)} ` +
|
|
46
|
+
`(${this.allocationStrategy.environment} mode, ` +
|
|
47
|
+
`${(this.allocationStrategy.ratio * 100).toFixed(0)}% of ${formatBytes(this.allocationStrategy.availableForCache)} ` +
|
|
48
|
+
`after ${formatBytes(this.allocationStrategy.modelMemory)} ${this.allocationStrategy.modelPrecision.toUpperCase()} model)`);
|
|
49
|
+
for (const warning of recommendation.warnings) {
|
|
50
|
+
prodLog.warn(`NativeUnifiedCache: ${warning}`);
|
|
51
|
+
}
|
|
52
|
+
this.config = {
|
|
53
|
+
enableRequestCoalescing: true,
|
|
54
|
+
enableFairnessCheck: true,
|
|
55
|
+
fairnessCheckInterval: 30000,
|
|
56
|
+
persistPatterns: true,
|
|
57
|
+
enableMemoryMonitoring: true,
|
|
58
|
+
memoryCheckInterval: 30000,
|
|
59
|
+
...config,
|
|
60
|
+
};
|
|
61
|
+
const bindings = loadNativeModule();
|
|
62
|
+
this.native = new bindings.NativeUnifiedCache(this.maxSize);
|
|
63
|
+
if (this.config.enableFairnessCheck) {
|
|
64
|
+
this.startFairnessMonitor();
|
|
65
|
+
}
|
|
66
|
+
if (this.config.enableMemoryMonitoring) {
|
|
67
|
+
this.startMemoryPressureMonitor();
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
// ---------------------------------------------------------------------------
|
|
71
|
+
// Core API
|
|
72
|
+
// ---------------------------------------------------------------------------
|
|
73
|
+
async get(key, loadFn) {
|
|
74
|
+
// Check JS data map
|
|
75
|
+
const cached = this.data.get(key);
|
|
76
|
+
if (cached !== undefined) {
|
|
77
|
+
this.native.recordAccess(key);
|
|
78
|
+
return cached;
|
|
79
|
+
}
|
|
80
|
+
if (!loadFn)
|
|
81
|
+
return undefined;
|
|
82
|
+
// Request coalescing
|
|
83
|
+
if (this.config.enableRequestCoalescing && this.loadingPromises.has(key)) {
|
|
84
|
+
return this.loadingPromises.get(key);
|
|
85
|
+
}
|
|
86
|
+
const loadPromise = loadFn();
|
|
87
|
+
if (this.config.enableRequestCoalescing) {
|
|
88
|
+
this.loadingPromises.set(key, loadPromise);
|
|
89
|
+
}
|
|
90
|
+
try {
|
|
91
|
+
const result = await loadPromise;
|
|
92
|
+
return result;
|
|
93
|
+
}
|
|
94
|
+
finally {
|
|
95
|
+
if (this.config.enableRequestCoalescing) {
|
|
96
|
+
this.loadingPromises.delete(key);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
getSync(key) {
|
|
101
|
+
const cached = this.data.get(key);
|
|
102
|
+
if (cached !== undefined) {
|
|
103
|
+
this.native.recordAccess(key);
|
|
104
|
+
return cached;
|
|
105
|
+
}
|
|
106
|
+
return undefined;
|
|
107
|
+
}
|
|
108
|
+
set(key, data, type, size, rebuildCost = 1) {
|
|
109
|
+
const cacheType = CACHE_TYPE_MAP[type] ?? 3;
|
|
110
|
+
// Remove existing entry if present
|
|
111
|
+
if (this.data.has(key)) {
|
|
112
|
+
this.native.remove(key);
|
|
113
|
+
}
|
|
114
|
+
// Insert into native (may trigger eviction)
|
|
115
|
+
const evictionResult = this.native.insert(key, cacheType, size, rebuildCost);
|
|
116
|
+
// Remove evicted keys from JS data
|
|
117
|
+
for (const evictedKey of evictionResult.evictedKeys) {
|
|
118
|
+
this.data.delete(evictedKey);
|
|
119
|
+
}
|
|
120
|
+
// Store actual data in JS
|
|
121
|
+
this.data.set(key, data);
|
|
122
|
+
}
|
|
123
|
+
delete(key) {
|
|
124
|
+
const existed = this.data.has(key);
|
|
125
|
+
if (existed) {
|
|
126
|
+
this.native.remove(key);
|
|
127
|
+
this.data.delete(key);
|
|
128
|
+
}
|
|
129
|
+
return existed;
|
|
130
|
+
}
|
|
131
|
+
deleteByPrefix(prefix) {
|
|
132
|
+
const removedKeys = this.native.removeByPrefix(prefix);
|
|
133
|
+
for (const key of removedKeys) {
|
|
134
|
+
this.data.delete(key);
|
|
135
|
+
}
|
|
136
|
+
return removedKeys.length;
|
|
137
|
+
}
|
|
138
|
+
clear(type) {
|
|
139
|
+
if (!type) {
|
|
140
|
+
const clearedKeys = this.native.clear();
|
|
141
|
+
for (const key of clearedKeys) {
|
|
142
|
+
this.data.delete(key);
|
|
143
|
+
}
|
|
144
|
+
// Also clear any remaining data entries
|
|
145
|
+
this.data.clear();
|
|
146
|
+
}
|
|
147
|
+
else {
|
|
148
|
+
const cacheType = CACHE_TYPE_MAP[type] ?? 3;
|
|
149
|
+
const clearedKeys = this.native.clear(cacheType);
|
|
150
|
+
for (const key of clearedKeys) {
|
|
151
|
+
this.data.delete(key);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
// ---------------------------------------------------------------------------
|
|
156
|
+
// Eviction
|
|
157
|
+
// ---------------------------------------------------------------------------
|
|
158
|
+
evictForSize(bytesNeeded) {
|
|
159
|
+
const result = this.native.evictForSize(bytesNeeded);
|
|
160
|
+
for (const key of result.evictedKeys) {
|
|
161
|
+
this.data.delete(key);
|
|
162
|
+
}
|
|
163
|
+
return result.bytesFreed >= bytesNeeded;
|
|
164
|
+
}
|
|
165
|
+
// ---------------------------------------------------------------------------
|
|
166
|
+
// Fairness
|
|
167
|
+
// ---------------------------------------------------------------------------
|
|
168
|
+
startFairnessMonitor() {
|
|
169
|
+
const timer = setInterval(() => {
|
|
170
|
+
this.checkFairness();
|
|
171
|
+
}, this.config.fairnessCheckInterval);
|
|
172
|
+
if (timer.unref)
|
|
173
|
+
timer.unref();
|
|
174
|
+
}
|
|
175
|
+
checkFairness() {
|
|
176
|
+
const result = this.native.checkFairness();
|
|
177
|
+
for (const key of result.evictedKeys) {
|
|
178
|
+
this.data.delete(key);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
// ---------------------------------------------------------------------------
|
|
182
|
+
// Memory pressure
|
|
183
|
+
// ---------------------------------------------------------------------------
|
|
184
|
+
startMemoryPressureMonitor() {
|
|
185
|
+
const checkInterval = this.config.memoryCheckInterval || 30000;
|
|
186
|
+
this.memoryPressureCheckTimer = setInterval(() => {
|
|
187
|
+
this.checkMemoryPressure();
|
|
188
|
+
}, checkInterval);
|
|
189
|
+
if (this.memoryPressureCheckTimer.unref) {
|
|
190
|
+
this.memoryPressureCheckTimer.unref();
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
checkMemoryPressure() {
|
|
194
|
+
const stats = this.native.getStats();
|
|
195
|
+
const pressure = checkMemoryPressure(stats.totalSize, this.memoryInfo);
|
|
196
|
+
const now = Date.now();
|
|
197
|
+
const fiveMinutes = 5 * 60 * 1000;
|
|
198
|
+
if (pressure.warnings.length > 0 && now - this.lastMemoryWarning > fiveMinutes) {
|
|
199
|
+
for (const warning of pressure.warnings) {
|
|
200
|
+
prodLog.warn(`NativeUnifiedCache: ${warning}`);
|
|
201
|
+
}
|
|
202
|
+
this.lastMemoryWarning = now;
|
|
203
|
+
}
|
|
204
|
+
if (pressure.pressure === 'critical') {
|
|
205
|
+
const targetSize = Math.floor(this.maxSize * 0.7);
|
|
206
|
+
const bytesToFree = stats.totalSize - targetSize;
|
|
207
|
+
if (bytesToFree > 0) {
|
|
208
|
+
prodLog.warn(`NativeUnifiedCache: Critical memory pressure - forcing eviction of ${formatBytes(bytesToFree)}`);
|
|
209
|
+
this.evictForSize(bytesToFree);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
// ---------------------------------------------------------------------------
|
|
214
|
+
// Statistics
|
|
215
|
+
// ---------------------------------------------------------------------------
|
|
216
|
+
getStats() {
|
|
217
|
+
const nativeStats = this.native.getStats();
|
|
218
|
+
const typeNames = ['vectors', 'metadata', 'embedding', 'other'];
|
|
219
|
+
const typeSizes = {};
|
|
220
|
+
const typeCounts = {};
|
|
221
|
+
const typeAccessCounts = {};
|
|
222
|
+
for (let i = 0; i < typeNames.length; i++) {
|
|
223
|
+
typeSizes[typeNames[i]] = nativeStats.typeSizes[i] || 0;
|
|
224
|
+
typeCounts[typeNames[i]] = nativeStats.typeCounts[i] || 0;
|
|
225
|
+
typeAccessCounts[typeNames[i]] = nativeStats.typeAccessCounts[i] || 0;
|
|
226
|
+
}
|
|
227
|
+
const hitRate = nativeStats.totalAccessCount > 0
|
|
228
|
+
? nativeStats.itemCount / nativeStats.totalAccessCount
|
|
229
|
+
: 0;
|
|
230
|
+
return {
|
|
231
|
+
totalSize: nativeStats.totalSize,
|
|
232
|
+
maxSize: nativeStats.maxSize,
|
|
233
|
+
utilization: nativeStats.maxSize > 0 ? nativeStats.totalSize / nativeStats.maxSize : 0,
|
|
234
|
+
itemCount: nativeStats.itemCount,
|
|
235
|
+
typeSizes,
|
|
236
|
+
typeCounts,
|
|
237
|
+
typeAccessCounts,
|
|
238
|
+
totalAccessCount: nativeStats.totalAccessCount,
|
|
239
|
+
hitRate,
|
|
240
|
+
memory: {
|
|
241
|
+
available: this.memoryInfo.available,
|
|
242
|
+
source: this.memoryInfo.source,
|
|
243
|
+
isContainer: this.memoryInfo.isContainer,
|
|
244
|
+
systemTotal: this.memoryInfo.systemTotal,
|
|
245
|
+
allocationRatio: this.allocationStrategy.ratio,
|
|
246
|
+
environment: this.allocationStrategy.environment,
|
|
247
|
+
},
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
getMemoryInfo() {
|
|
251
|
+
const stats = this.native.getStats();
|
|
252
|
+
return {
|
|
253
|
+
memoryInfo: { ...this.memoryInfo },
|
|
254
|
+
allocationStrategy: { ...this.allocationStrategy },
|
|
255
|
+
currentPressure: checkMemoryPressure(stats.totalSize, this.memoryInfo),
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
// ---------------------------------------------------------------------------
|
|
259
|
+
// Access pattern persistence
|
|
260
|
+
// ---------------------------------------------------------------------------
|
|
261
|
+
async saveAccessPatterns() {
|
|
262
|
+
if (!this.config.persistPatterns)
|
|
263
|
+
return;
|
|
264
|
+
const json = this.native.saveAccessPatterns();
|
|
265
|
+
return JSON.parse(json);
|
|
266
|
+
}
|
|
267
|
+
async loadAccessPatterns(patterns) {
|
|
268
|
+
if (!patterns)
|
|
269
|
+
return;
|
|
270
|
+
this.native.loadAccessPatterns(JSON.stringify(patterns));
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
//# sourceMappingURL=NativeUnifiedCache.js.map
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module utils/binaryIdMapperFactory
|
|
3
|
+
* @description Single canonical constructor for cor's native mmap
|
|
4
|
+
* `BinaryIdMapper`. Derives the `_id_mapper/*` paths from a
|
|
5
|
+
* filesystem-backed storage adapter and opens (or creates / truncates)
|
|
6
|
+
* the paired `uuid_to_int.mkv` + `int_to_uuid.bin` files.
|
|
7
|
+
*
|
|
8
|
+
* Both the metadata index (the canonical UUID ↔ entity-int allocator
|
|
9
|
+
* since #72 Phase C) and the legacy `entityIdMapper` provider go through
|
|
10
|
+
* here so there is exactly one place that knows the file layout, the
|
|
11
|
+
* default sizes, and the paired-write / half-present invariant.
|
|
12
|
+
*
|
|
13
|
+
* The mapper is mmap-backed and demand-paged: a 32 GB default file is
|
|
14
|
+
* SPARSE on disk (only touched pages occupy blocks), so a fresh small
|
|
15
|
+
* brain pays only for what it uses. This is the constant-RAM,
|
|
16
|
+
* billion-scale store that replaces the resident JSON id-maps (#72).
|
|
17
|
+
*/
|
|
18
|
+
import type { NativeBinaryIdMapperInstance } from '../native/types.js';
|
|
19
|
+
/** Default storage key for the extendible-hash UUID → int file. */
|
|
20
|
+
export declare const DEFAULT_UUID_TO_INT_KEY = "_id_mapper/uuid_to_int.mkv";
|
|
21
|
+
/** Default storage key for the direct-indexed int → UUID file. */
|
|
22
|
+
export declare const DEFAULT_INT_TO_UUID_KEY = "_id_mapper/int_to_uuid.bin";
|
|
23
|
+
/** Minimal storage surface this factory needs. */
|
|
24
|
+
export interface StorageWithBlobPath {
|
|
25
|
+
getBinaryBlobPath?: (key: string) => string | null;
|
|
26
|
+
}
|
|
27
|
+
/** Options controlling file layout, sizing, and id width. */
|
|
28
|
+
export interface BinaryIdMapperFactoryOptions {
|
|
29
|
+
uuidToIntKey?: string;
|
|
30
|
+
intToUuidKey?: string;
|
|
31
|
+
intToUuidSize?: bigint;
|
|
32
|
+
uuidToIntSize?: bigint;
|
|
33
|
+
bucketCapacity?: number;
|
|
34
|
+
maxGlobalDepth?: number;
|
|
35
|
+
idSpace?: 'u32' | 'u64';
|
|
36
|
+
}
|
|
37
|
+
/** What {@link openOrCreateBinaryIdMapper} resolves the mapper with. */
|
|
38
|
+
export interface BinaryIdMapperHandle {
|
|
39
|
+
mapper: NativeBinaryIdMapperInstance;
|
|
40
|
+
uuidToIntPath: string;
|
|
41
|
+
intToUuidPath: string;
|
|
42
|
+
/** The mapper's actual on-disk IdSpace (authoritative over requested). */
|
|
43
|
+
idSpace: 'u32' | 'u64';
|
|
44
|
+
/** Whether the files were freshly created (vs opened existing). */
|
|
45
|
+
created: boolean;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Open (or create / truncate) the native `BinaryIdMapper` for a brain.
|
|
49
|
+
*
|
|
50
|
+
* @param storage - Filesystem-backed storage exposing `getBinaryBlobPath`.
|
|
51
|
+
* @param opts - File layout, sizing, id width, and the `forceCreate`
|
|
52
|
+
* rebuild flag.
|
|
53
|
+
* @returns The native mapper handle plus its resolved paths + IdSpace.
|
|
54
|
+
* @throws If storage has no `getBinaryBlobPath`, if a path resolves null,
|
|
55
|
+
* if the native `NativeBinaryIdMapper` binding is missing, or if the
|
|
56
|
+
* two files are in a half-present (crash-torn) state.
|
|
57
|
+
*/
|
|
58
|
+
export declare function openOrCreateBinaryIdMapper(storage: StorageWithBlobPath, opts?: BinaryIdMapperFactoryOptions): BinaryIdMapperHandle;
|
|
59
|
+
//# sourceMappingURL=binaryIdMapperFactory.d.ts.map
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module utils/binaryIdMapperFactory
|
|
3
|
+
* @description Single canonical constructor for cor's native mmap
|
|
4
|
+
* `BinaryIdMapper`. Derives the `_id_mapper/*` paths from a
|
|
5
|
+
* filesystem-backed storage adapter and opens (or creates / truncates)
|
|
6
|
+
* the paired `uuid_to_int.mkv` + `int_to_uuid.bin` files.
|
|
7
|
+
*
|
|
8
|
+
* Both the metadata index (the canonical UUID ↔ entity-int allocator
|
|
9
|
+
* since #72 Phase C) and the legacy `entityIdMapper` provider go through
|
|
10
|
+
* here so there is exactly one place that knows the file layout, the
|
|
11
|
+
* default sizes, and the paired-write / half-present invariant.
|
|
12
|
+
*
|
|
13
|
+
* The mapper is mmap-backed and demand-paged: a 32 GB default file is
|
|
14
|
+
* SPARSE on disk (only touched pages occupy blocks), so a fresh small
|
|
15
|
+
* brain pays only for what it uses. This is the constant-RAM,
|
|
16
|
+
* billion-scale store that replaces the resident JSON id-maps (#72).
|
|
17
|
+
*/
|
|
18
|
+
import { existsSync } from 'node:fs';
|
|
19
|
+
import { loadNativeModule } from '../native/index.js';
|
|
20
|
+
/** Default storage key for the extendible-hash UUID → int file. */
|
|
21
|
+
export const DEFAULT_UUID_TO_INT_KEY = '_id_mapper/uuid_to_int.mkv';
|
|
22
|
+
/** Default storage key for the direct-indexed int → UUID file. */
|
|
23
|
+
export const DEFAULT_INT_TO_UUID_KEY = '_id_mapper/int_to_uuid.bin';
|
|
24
|
+
/** 32 GB default sparse-file size for each mapper file. */
|
|
25
|
+
const DEFAULT_FILE_SIZE = BigInt(32) * BigInt(1024) ** BigInt(3);
|
|
26
|
+
/**
|
|
27
|
+
* Open (or create / truncate) the native `BinaryIdMapper` for a brain.
|
|
28
|
+
*
|
|
29
|
+
* @param storage - Filesystem-backed storage exposing `getBinaryBlobPath`.
|
|
30
|
+
* @param opts - File layout, sizing, id width, and the `forceCreate`
|
|
31
|
+
* rebuild flag.
|
|
32
|
+
* @returns The native mapper handle plus its resolved paths + IdSpace.
|
|
33
|
+
* @throws If storage has no `getBinaryBlobPath`, if a path resolves null,
|
|
34
|
+
* if the native `NativeBinaryIdMapper` binding is missing, or if the
|
|
35
|
+
* two files are in a half-present (crash-torn) state.
|
|
36
|
+
*/
|
|
37
|
+
export function openOrCreateBinaryIdMapper(storage, opts = {}) {
|
|
38
|
+
if (!storage.getBinaryBlobPath) {
|
|
39
|
+
throw new Error('openOrCreateBinaryIdMapper requires a storage adapter that exposes ' +
|
|
40
|
+
'getBinaryBlobPath() (filesystem-backed).');
|
|
41
|
+
}
|
|
42
|
+
const uuidToIntKey = opts.uuidToIntKey ?? DEFAULT_UUID_TO_INT_KEY;
|
|
43
|
+
const intToUuidKey = opts.intToUuidKey ?? DEFAULT_INT_TO_UUID_KEY;
|
|
44
|
+
const uuidToIntPath = storage.getBinaryBlobPath(uuidToIntKey);
|
|
45
|
+
const intToUuidPath = storage.getBinaryBlobPath(intToUuidKey);
|
|
46
|
+
if (!uuidToIntPath || !intToUuidPath) {
|
|
47
|
+
throw new Error(`openOrCreateBinaryIdMapper: getBinaryBlobPath returned null for ` +
|
|
48
|
+
`${uuidToIntKey} or ${intToUuidKey}`);
|
|
49
|
+
}
|
|
50
|
+
const bindings = loadNativeModule();
|
|
51
|
+
const NativeBinaryIdMapper = bindings.NativeBinaryIdMapper;
|
|
52
|
+
if (!NativeBinaryIdMapper) {
|
|
53
|
+
throw new Error('NativeBinaryIdMapper binding missing from cor native module — ' +
|
|
54
|
+
'this build of cor is older than the BinaryIdMapper feature');
|
|
55
|
+
}
|
|
56
|
+
const config = {
|
|
57
|
+
uuidToIntPath,
|
|
58
|
+
intToUuidPath,
|
|
59
|
+
intToUuidSize: opts.intToUuidSize ?? DEFAULT_FILE_SIZE,
|
|
60
|
+
uuidToIntSize: opts.uuidToIntSize ?? DEFAULT_FILE_SIZE,
|
|
61
|
+
bucketCapacity: opts.bucketCapacity ?? 16,
|
|
62
|
+
maxGlobalDepth: opts.maxGlobalDepth ?? 28,
|
|
63
|
+
idSpace: opts.idSpace ?? 'u64',
|
|
64
|
+
};
|
|
65
|
+
// Both files must exist together (paired write semantics). A
|
|
66
|
+
// half-present state is crash corruption surfaced as an error,
|
|
67
|
+
// never silently recreated.
|
|
68
|
+
let mapper;
|
|
69
|
+
let created;
|
|
70
|
+
const uuidFileExists = existsSync(uuidToIntPath);
|
|
71
|
+
const intFileExists = existsSync(intToUuidPath);
|
|
72
|
+
if (uuidFileExists && intFileExists) {
|
|
73
|
+
mapper = NativeBinaryIdMapper.openExisting(config);
|
|
74
|
+
created = false;
|
|
75
|
+
}
|
|
76
|
+
else if (!uuidFileExists && !intFileExists) {
|
|
77
|
+
mapper = NativeBinaryIdMapper.create(config);
|
|
78
|
+
created = true;
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
throw new Error(`openOrCreateBinaryIdMapper: half-present file pair — ` +
|
|
82
|
+
`${uuidToIntKey} ${uuidFileExists ? 'exists' : 'missing'}, ` +
|
|
83
|
+
`${intToUuidKey} ${intFileExists ? 'exists' : 'missing'}. ` +
|
|
84
|
+
`Refusing to silently recreate; investigate manually.`);
|
|
85
|
+
}
|
|
86
|
+
return {
|
|
87
|
+
mapper,
|
|
88
|
+
uuidToIntPath,
|
|
89
|
+
intToUuidPath,
|
|
90
|
+
idSpace: mapper.idSpace(),
|
|
91
|
+
created,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
//# sourceMappingURL=binaryIdMapperFactory.js.map
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module utils/collation
|
|
3
|
+
* @description Deterministic string ordering for cor's TypeScript fallback paths,
|
|
4
|
+
* byte-for-byte identical to `@soulcraft/brainy`'s `compareCodePoints` and to the
|
|
5
|
+
* native (Rust) column-store / aggregation engine.
|
|
6
|
+
*
|
|
7
|
+
* Strings are compared by **UTF-8 byte order**, which equals Unicode code-point order
|
|
8
|
+
* and Rust's `str::cmp` (`as_bytes().cmp()`). This is deterministic across
|
|
9
|
+
* environments (unlike `localeCompare`, whose default-locale ordering varies by
|
|
10
|
+
* OS / Node / ICU build) and consistent with the native engine, so sorted query
|
|
11
|
+
* results are identical with or without the native plugin.
|
|
12
|
+
*
|
|
13
|
+
* Cor keeps a local copy (rather than importing brainy's) so it never depends on
|
|
14
|
+
* a brainy internal export and is guaranteed available against any supported brainy
|
|
15
|
+
* version. The implementation must stay in lockstep with brainy `src/utils/collation.ts`.
|
|
16
|
+
* See `docs/ADR-001`.
|
|
17
|
+
*/
|
|
18
|
+
/**
|
|
19
|
+
* @description Compare two strings by UTF-8 byte order (== Unicode code-point order
|
|
20
|
+
* == Rust `str::cmp`). Use instead of `String.localeCompare` and the `<`/`>`
|
|
21
|
+
* operators (which compare UTF-16 code units and diverge for supplementary-plane
|
|
22
|
+
* characters) wherever ordering must match the native engine or be deterministic.
|
|
23
|
+
* @param a - First string.
|
|
24
|
+
* @param b - Second string.
|
|
25
|
+
* @returns Negative if `a < b`, positive if `a > b`, `0` if equal — by UTF-8 byte order.
|
|
26
|
+
* @example
|
|
27
|
+
* ['b', 'A', 'a'].sort(compareCodePoints) // ['A', 'a', 'b'] (A=65, a=97, b=98)
|
|
28
|
+
*/
|
|
29
|
+
export declare function compareCodePoints(a: string, b: string): number;
|
|
30
|
+
//# sourceMappingURL=collation.d.ts.map
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module utils/collation
|
|
3
|
+
* @description Deterministic string ordering for cor's TypeScript fallback paths,
|
|
4
|
+
* byte-for-byte identical to `@soulcraft/brainy`'s `compareCodePoints` and to the
|
|
5
|
+
* native (Rust) column-store / aggregation engine.
|
|
6
|
+
*
|
|
7
|
+
* Strings are compared by **UTF-8 byte order**, which equals Unicode code-point order
|
|
8
|
+
* and Rust's `str::cmp` (`as_bytes().cmp()`). This is deterministic across
|
|
9
|
+
* environments (unlike `localeCompare`, whose default-locale ordering varies by
|
|
10
|
+
* OS / Node / ICU build) and consistent with the native engine, so sorted query
|
|
11
|
+
* results are identical with or without the native plugin.
|
|
12
|
+
*
|
|
13
|
+
* Cor keeps a local copy (rather than importing brainy's) so it never depends on
|
|
14
|
+
* a brainy internal export and is guaranteed available against any supported brainy
|
|
15
|
+
* version. The implementation must stay in lockstep with brainy `src/utils/collation.ts`.
|
|
16
|
+
* See `docs/ADR-001`.
|
|
17
|
+
*/
|
|
18
|
+
const utf8 = new TextEncoder();
|
|
19
|
+
/**
|
|
20
|
+
* @description Compare two strings by UTF-8 byte order (== Unicode code-point order
|
|
21
|
+
* == Rust `str::cmp`). Use instead of `String.localeCompare` and the `<`/`>`
|
|
22
|
+
* operators (which compare UTF-16 code units and diverge for supplementary-plane
|
|
23
|
+
* characters) wherever ordering must match the native engine or be deterministic.
|
|
24
|
+
* @param a - First string.
|
|
25
|
+
* @param b - Second string.
|
|
26
|
+
* @returns Negative if `a < b`, positive if `a > b`, `0` if equal — by UTF-8 byte order.
|
|
27
|
+
* @example
|
|
28
|
+
* ['b', 'A', 'a'].sort(compareCodePoints) // ['A', 'a', 'b'] (A=65, a=97, b=98)
|
|
29
|
+
*/
|
|
30
|
+
export function compareCodePoints(a, b) {
|
|
31
|
+
const ea = utf8.encode(a);
|
|
32
|
+
const eb = utf8.encode(b);
|
|
33
|
+
const len = ea.length < eb.length ? ea.length : eb.length;
|
|
34
|
+
for (let i = 0; i < len; i++) {
|
|
35
|
+
if (ea[i] !== eb[i])
|
|
36
|
+
return ea[i] - eb[i];
|
|
37
|
+
}
|
|
38
|
+
return ea.length - eb.length;
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=collation.js.map
|