@soulcraft/brainy 6.2.9 → 6.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/brainy.d.ts +55 -0
- package/dist/brainy.js +104 -7
- package/dist/graph/graphAdjacencyIndex.d.ts +7 -0
- package/dist/graph/graphAdjacencyIndex.js +42 -0
- package/dist/storage/baseStorage.d.ts +7 -0
- package/dist/storage/baseStorage.js +31 -38
- package/dist/versioning/VersionIndex.d.ts +42 -47
- package/dist/versioning/VersionIndex.js +141 -166
- package/dist/versioning/VersionManager.d.ts +12 -6
- package/dist/versioning/VersionManager.js +26 -8
- package/dist/versioning/VersionStorage.d.ts +25 -15
- package/dist/versioning/VersionStorage.js +49 -65
- package/dist/vfs/PathResolver.d.ts +6 -0
- package/dist/vfs/PathResolver.js +20 -0
- package/dist/vfs/semantic/SemanticPathResolver.d.ts +6 -0
- package/dist/vfs/semantic/SemanticPathResolver.js +11 -0
- package/package.json +1 -1
- package/dist/augmentations/KnowledgeAugmentation.d.ts +0 -40
- package/dist/augmentations/KnowledgeAugmentation.js +0 -251
- package/dist/importManager.d.ts +0 -78
- package/dist/importManager.js +0 -267
- package/dist/query/typeInference.d.ts +0 -158
- package/dist/query/typeInference.js +0 -760
- package/dist/storage/adapters/typeAwareStorageAdapter.d.ts +0 -252
- package/dist/storage/adapters/typeAwareStorageAdapter.js +0 -814
- package/dist/types/brainyDataInterface.d.ts +0 -52
- package/dist/types/brainyDataInterface.js +0 -10
- package/dist/vfs/ConceptSystem.d.ts +0 -203
- package/dist/vfs/ConceptSystem.js +0 -545
- package/dist/vfs/EntityManager.d.ts +0 -75
- package/dist/vfs/EntityManager.js +0 -216
- package/dist/vfs/EventRecorder.d.ts +0 -84
- package/dist/vfs/EventRecorder.js +0 -269
- package/dist/vfs/GitBridge.d.ts +0 -167
- package/dist/vfs/GitBridge.js +0 -537
- package/dist/vfs/KnowledgeLayer.d.ts +0 -35
- package/dist/vfs/KnowledgeLayer.js +0 -443
- package/dist/vfs/PersistentEntitySystem.d.ts +0 -165
- package/dist/vfs/PersistentEntitySystem.js +0 -503
- package/dist/vfs/SemanticVersioning.d.ts +0 -105
- package/dist/vfs/SemanticVersioning.js +0 -309
|
@@ -1,814 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Type-Aware Storage Adapter
|
|
3
|
-
*
|
|
4
|
-
* Implements type-first storage architecture for billion-scale optimization
|
|
5
|
-
*
|
|
6
|
-
* Key Features:
|
|
7
|
-
* - Type-first paths: entities/nouns/{type}/vectors/{shard}/{uuid}.json
|
|
8
|
-
* - Fixed-size type tracking: Uint32Array(31) for nouns, Uint32Array(40) for verbs
|
|
9
|
-
* - O(1) type filtering: Can list entities by type via directory structure
|
|
10
|
-
* - Zero technical debt: Clean implementation, no legacy paths
|
|
11
|
-
*
|
|
12
|
-
* Memory Impact @ 1B Scale:
|
|
13
|
-
* - Type tracking: 284 bytes (vs ~120KB with Maps) = -99.76%
|
|
14
|
-
* - Metadata index: 3GB (vs 5GB) = -40% (when combined with TypeFirstMetadataIndex)
|
|
15
|
-
* - Total system: 69GB (vs 557GB) = -88%
|
|
16
|
-
*
|
|
17
|
-
* @version 3.45.0
|
|
18
|
-
* @since Phase 1 - Type-First Implementation
|
|
19
|
-
*/
|
|
20
|
-
import { BaseStorage } from '../baseStorage.js';
|
|
21
|
-
import { TypeUtils, NOUN_TYPE_COUNT, VERB_TYPE_COUNT } from '../../types/graphTypes.js';
|
|
22
|
-
import { getShardIdFromUuid } from '../sharding.js';
|
|
23
|
-
/**
|
|
24
|
-
* Type-first storage paths
|
|
25
|
-
* Beautiful, self-documenting structure
|
|
26
|
-
*/
|
|
27
|
-
const SYSTEM_DIR = '_system';
|
|
28
|
-
/**
|
|
29
|
-
* Get type-first path for noun vectors
|
|
30
|
-
*/
|
|
31
|
-
function getNounVectorPath(type, id) {
|
|
32
|
-
const shard = getShardIdFromUuid(id);
|
|
33
|
-
return `entities/nouns/${type}/vectors/${shard}/${id}.json`;
|
|
34
|
-
}
|
|
35
|
-
/**
|
|
36
|
-
* Get type-first path for noun metadata
|
|
37
|
-
*/
|
|
38
|
-
function getNounMetadataPath(type, id) {
|
|
39
|
-
const shard = getShardIdFromUuid(id);
|
|
40
|
-
return `entities/nouns/${type}/metadata/${shard}/${id}.json`;
|
|
41
|
-
}
|
|
42
|
-
/**
|
|
43
|
-
* Get type-first path for verb vectors
|
|
44
|
-
*/
|
|
45
|
-
function getVerbVectorPath(type, id) {
|
|
46
|
-
const shard = getShardIdFromUuid(id);
|
|
47
|
-
return `entities/verbs/${type}/vectors/${shard}/${id}.json`;
|
|
48
|
-
}
|
|
49
|
-
/**
|
|
50
|
-
* Get type-first path for verb metadata
|
|
51
|
-
*/
|
|
52
|
-
function getVerbMetadataPath(type, id) {
|
|
53
|
-
const shard = getShardIdFromUuid(id);
|
|
54
|
-
return `entities/verbs/${type}/metadata/${shard}/${id}.json`;
|
|
55
|
-
}
|
|
56
|
-
/**
|
|
57
|
-
* Type-Aware Storage Adapter
|
|
58
|
-
*
|
|
59
|
-
* Wraps an underlying storage adapter and adds type-first routing
|
|
60
|
-
* Tracks types with fixed-size arrays for billion-scale efficiency
|
|
61
|
-
*/
|
|
62
|
-
export class TypeAwareStorageAdapter extends BaseStorage {
|
|
63
|
-
constructor(options) {
|
|
64
|
-
super();
|
|
65
|
-
// Fixed-size type tracking (99.76% memory reduction vs Maps)
|
|
66
|
-
this.nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT); // 124 bytes
|
|
67
|
-
this.verbCountsByType = new Uint32Array(VERB_TYPE_COUNT); // 160 bytes
|
|
68
|
-
// Total: 284 bytes (vs ~120KB with Maps)
|
|
69
|
-
// Type cache for fast lookups (id -> type)
|
|
70
|
-
// Only for entities we've seen this session (bounded size)
|
|
71
|
-
this.nounTypeCache = new Map();
|
|
72
|
-
this.verbTypeCache = new Map();
|
|
73
|
-
this.underlying = options.underlyingStorage;
|
|
74
|
-
this.verbose = options.verbose || false;
|
|
75
|
-
}
|
|
76
|
-
/**
|
|
77
|
-
* Helper to access protected methods on underlying storage
|
|
78
|
-
* TypeScript doesn't allow calling protected methods across instances,
|
|
79
|
-
* so we cast to any to bypass this restriction
|
|
80
|
-
*/
|
|
81
|
-
get u() {
|
|
82
|
-
return this.underlying;
|
|
83
|
-
}
|
|
84
|
-
/**
|
|
85
|
-
* Initialize storage adapter
|
|
86
|
-
*/
|
|
87
|
-
async init() {
|
|
88
|
-
if (this.verbose) {
|
|
89
|
-
console.log('[TypeAwareStorage] Initializing...');
|
|
90
|
-
}
|
|
91
|
-
// Initialize underlying storage
|
|
92
|
-
if (typeof this.underlying.init === 'function') {
|
|
93
|
-
await this.underlying.init();
|
|
94
|
-
}
|
|
95
|
-
// Load type statistics from storage (if they exist)
|
|
96
|
-
await this.loadTypeStatistics();
|
|
97
|
-
this.isInitialized = true;
|
|
98
|
-
if (this.verbose) {
|
|
99
|
-
console.log('[TypeAwareStorage] Initialized successfully');
|
|
100
|
-
console.log(`[TypeAwareStorage] Noun counts:`, Array.from(this.nounCountsByType));
|
|
101
|
-
console.log(`[TypeAwareStorage] Verb counts:`, Array.from(this.verbCountsByType));
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
/**
|
|
105
|
-
* Load type statistics from storage
|
|
106
|
-
* Rebuilds type counts if needed
|
|
107
|
-
*/
|
|
108
|
-
async loadTypeStatistics() {
|
|
109
|
-
try {
|
|
110
|
-
const stats = await this.u.readObjectFromPath(`${SYSTEM_DIR}/type-statistics.json`);
|
|
111
|
-
if (stats) {
|
|
112
|
-
// Restore counts from saved statistics
|
|
113
|
-
if (stats.nounCounts && stats.nounCounts.length === NOUN_TYPE_COUNT) {
|
|
114
|
-
this.nounCountsByType = new Uint32Array(stats.nounCounts);
|
|
115
|
-
}
|
|
116
|
-
if (stats.verbCounts && stats.verbCounts.length === VERB_TYPE_COUNT) {
|
|
117
|
-
this.verbCountsByType = new Uint32Array(stats.verbCounts);
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
catch (error) {
|
|
122
|
-
if (this.verbose) {
|
|
123
|
-
console.log('[TypeAwareStorage] No existing type statistics, starting fresh');
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
/**
|
|
128
|
-
* Save type statistics to storage
|
|
129
|
-
*/
|
|
130
|
-
async saveTypeStatistics() {
|
|
131
|
-
const stats = {
|
|
132
|
-
nounCounts: Array.from(this.nounCountsByType),
|
|
133
|
-
verbCounts: Array.from(this.verbCountsByType),
|
|
134
|
-
updatedAt: Date.now()
|
|
135
|
-
};
|
|
136
|
-
await this.u.writeObjectToPath(`${SYSTEM_DIR}/type-statistics.json`, stats);
|
|
137
|
-
}
|
|
138
|
-
/**
|
|
139
|
-
* Get noun type from cache
|
|
140
|
-
*
|
|
141
|
-
* v4.0.0: Metadata is stored separately, so we rely on the cache
|
|
142
|
-
* which is populated when saveNounMetadata is called
|
|
143
|
-
*/
|
|
144
|
-
getNounType(noun) {
|
|
145
|
-
// Check cache (populated when metadata is saved)
|
|
146
|
-
const cached = this.nounTypeCache.get(noun.id);
|
|
147
|
-
if (cached) {
|
|
148
|
-
return cached;
|
|
149
|
-
}
|
|
150
|
-
// Default to 'thing' if unknown
|
|
151
|
-
// This should only happen if saveNoun_internal is called before saveNounMetadata
|
|
152
|
-
console.warn(`[TypeAwareStorage] Unknown noun type for ${noun.id}, defaulting to 'thing'`);
|
|
153
|
-
return 'thing';
|
|
154
|
-
}
|
|
155
|
-
/**
|
|
156
|
-
* Get verb type from verb object
|
|
157
|
-
*
|
|
158
|
-
* ARCHITECTURAL FIX (v3.50.1): Simplified - verb field is now always present
|
|
159
|
-
*/
|
|
160
|
-
getVerbType(verb) {
|
|
161
|
-
// v3.50.1+: verb is a required field in HNSWVerb
|
|
162
|
-
if ('verb' in verb && verb.verb) {
|
|
163
|
-
return verb.verb;
|
|
164
|
-
}
|
|
165
|
-
// Fallback for GraphVerb (type alias)
|
|
166
|
-
if ('type' in verb && verb.type) {
|
|
167
|
-
return verb.type;
|
|
168
|
-
}
|
|
169
|
-
// This should never happen with v3.50.1+ data
|
|
170
|
-
console.warn(`[TypeAwareStorage] Verb missing type field for ${verb.id}, defaulting to 'relatedTo'`);
|
|
171
|
-
return 'relatedTo';
|
|
172
|
-
}
|
|
173
|
-
// ============================================================================
|
|
174
|
-
// ABSTRACT METHOD IMPLEMENTATIONS
|
|
175
|
-
// ============================================================================
|
|
176
|
-
/**
|
|
177
|
-
* Save noun (type-first path)
|
|
178
|
-
*/
|
|
179
|
-
async saveNoun_internal(noun) {
|
|
180
|
-
const type = this.getNounType(noun);
|
|
181
|
-
const path = getNounVectorPath(type, noun.id);
|
|
182
|
-
// Update type tracking
|
|
183
|
-
const typeIndex = TypeUtils.getNounIndex(type);
|
|
184
|
-
this.nounCountsByType[typeIndex]++;
|
|
185
|
-
this.nounTypeCache.set(noun.id, type);
|
|
186
|
-
// Delegate to underlying storage
|
|
187
|
-
await this.u.writeObjectToPath(path, noun);
|
|
188
|
-
// Periodically save statistics (every 100 saves)
|
|
189
|
-
if (this.nounCountsByType[typeIndex] % 100 === 0) {
|
|
190
|
-
await this.saveTypeStatistics();
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
/**
|
|
194
|
-
* Get noun (type-first path)
|
|
195
|
-
*/
|
|
196
|
-
async getNoun_internal(id) {
|
|
197
|
-
// Try cache first
|
|
198
|
-
const cachedType = this.nounTypeCache.get(id);
|
|
199
|
-
if (cachedType) {
|
|
200
|
-
const path = getNounVectorPath(cachedType, id);
|
|
201
|
-
return await this.u.readObjectFromPath(path);
|
|
202
|
-
}
|
|
203
|
-
// Need to search across all types (expensive, but cached after first access)
|
|
204
|
-
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
|
|
205
|
-
const type = TypeUtils.getNounFromIndex(i);
|
|
206
|
-
const path = getNounVectorPath(type, id);
|
|
207
|
-
try {
|
|
208
|
-
const noun = await this.u.readObjectFromPath(path);
|
|
209
|
-
if (noun) {
|
|
210
|
-
// Cache the type for next time
|
|
211
|
-
this.nounTypeCache.set(id, type);
|
|
212
|
-
return noun;
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
catch (error) {
|
|
216
|
-
// Not in this type, continue searching
|
|
217
|
-
}
|
|
218
|
-
}
|
|
219
|
-
return null;
|
|
220
|
-
}
|
|
221
|
-
/**
|
|
222
|
-
* Get nouns by noun type (O(1) with type-first paths!)
|
|
223
|
-
*/
|
|
224
|
-
async getNounsByNounType_internal(nounType) {
|
|
225
|
-
const type = nounType;
|
|
226
|
-
const prefix = `entities/nouns/${type}/vectors/`;
|
|
227
|
-
// List all files under this type's directory
|
|
228
|
-
const paths = await this.u.listObjectsUnderPath(prefix);
|
|
229
|
-
// Load all nouns of this type
|
|
230
|
-
const nouns = [];
|
|
231
|
-
for (const path of paths) {
|
|
232
|
-
try {
|
|
233
|
-
const noun = await this.u.readObjectFromPath(path);
|
|
234
|
-
if (noun) {
|
|
235
|
-
nouns.push(noun);
|
|
236
|
-
// Cache the type
|
|
237
|
-
this.nounTypeCache.set(noun.id, type);
|
|
238
|
-
}
|
|
239
|
-
}
|
|
240
|
-
catch (error) {
|
|
241
|
-
console.warn(`[TypeAwareStorage] Failed to load noun from ${path}:`, error);
|
|
242
|
-
}
|
|
243
|
-
}
|
|
244
|
-
return nouns;
|
|
245
|
-
}
|
|
246
|
-
/**
|
|
247
|
-
* Delete noun (type-first path)
|
|
248
|
-
*/
|
|
249
|
-
async deleteNoun_internal(id) {
|
|
250
|
-
// Try cache first
|
|
251
|
-
const cachedType = this.nounTypeCache.get(id);
|
|
252
|
-
if (cachedType) {
|
|
253
|
-
const path = getNounVectorPath(cachedType, id);
|
|
254
|
-
await this.u.deleteObjectFromPath(path);
|
|
255
|
-
// Update counts
|
|
256
|
-
const typeIndex = TypeUtils.getNounIndex(cachedType);
|
|
257
|
-
if (this.nounCountsByType[typeIndex] > 0) {
|
|
258
|
-
this.nounCountsByType[typeIndex]--;
|
|
259
|
-
}
|
|
260
|
-
this.nounTypeCache.delete(id);
|
|
261
|
-
return;
|
|
262
|
-
}
|
|
263
|
-
// Search across all types
|
|
264
|
-
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
|
|
265
|
-
const type = TypeUtils.getNounFromIndex(i);
|
|
266
|
-
const path = getNounVectorPath(type, id);
|
|
267
|
-
try {
|
|
268
|
-
await this.u.deleteObjectFromPath(path);
|
|
269
|
-
// Update counts
|
|
270
|
-
if (this.nounCountsByType[i] > 0) {
|
|
271
|
-
this.nounCountsByType[i]--;
|
|
272
|
-
}
|
|
273
|
-
this.nounTypeCache.delete(id);
|
|
274
|
-
return;
|
|
275
|
-
}
|
|
276
|
-
catch (error) {
|
|
277
|
-
// Not in this type, continue
|
|
278
|
-
}
|
|
279
|
-
}
|
|
280
|
-
}
|
|
281
|
-
/**
|
|
282
|
-
* Save verb (type-first path)
|
|
283
|
-
*
|
|
284
|
-
* ARCHITECTURAL FIX (v3.50.1): No more caching hack needed!
|
|
285
|
-
* HNSWVerb now includes verb field, so type is always available
|
|
286
|
-
*/
|
|
287
|
-
async saveVerb_internal(verb) {
|
|
288
|
-
// Type is now a first-class field in HNSWVerb - no caching needed!
|
|
289
|
-
const type = verb.verb;
|
|
290
|
-
const path = getVerbVectorPath(type, verb.id);
|
|
291
|
-
// Update type tracking
|
|
292
|
-
const typeIndex = TypeUtils.getVerbIndex(type);
|
|
293
|
-
this.verbCountsByType[typeIndex]++;
|
|
294
|
-
this.verbTypeCache.set(verb.id, type);
|
|
295
|
-
// Delegate to underlying storage
|
|
296
|
-
await this.u.writeObjectToPath(path, verb);
|
|
297
|
-
// Periodically save statistics
|
|
298
|
-
if (this.verbCountsByType[typeIndex] % 100 === 0) {
|
|
299
|
-
await this.saveTypeStatistics();
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
/**
|
|
303
|
-
* Get verb (type-first path)
|
|
304
|
-
*
|
|
305
|
-
* ARCHITECTURAL FIX (v3.50.1): Cache still useful for performance
|
|
306
|
-
* Once we know where a verb is, we can retrieve it O(1) instead of searching all types
|
|
307
|
-
*/
|
|
308
|
-
async getVerb_internal(id) {
|
|
309
|
-
// Try cache first for O(1) retrieval
|
|
310
|
-
const cachedType = this.verbTypeCache.get(id);
|
|
311
|
-
if (cachedType) {
|
|
312
|
-
const path = getVerbVectorPath(cachedType, id);
|
|
313
|
-
const verb = await this.u.readObjectFromPath(path);
|
|
314
|
-
return verb;
|
|
315
|
-
}
|
|
316
|
-
// Search across all types (only on first access)
|
|
317
|
-
for (let i = 0; i < VERB_TYPE_COUNT; i++) {
|
|
318
|
-
const type = TypeUtils.getVerbFromIndex(i);
|
|
319
|
-
const path = getVerbVectorPath(type, id);
|
|
320
|
-
try {
|
|
321
|
-
const verb = await this.u.readObjectFromPath(path);
|
|
322
|
-
if (verb) {
|
|
323
|
-
// Cache the type for next time (read from verb.verb field)
|
|
324
|
-
this.verbTypeCache.set(id, verb.verb);
|
|
325
|
-
return verb;
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
catch (error) {
|
|
329
|
-
// Not in this type, continue
|
|
330
|
-
}
|
|
331
|
-
}
|
|
332
|
-
return null;
|
|
333
|
-
}
|
|
334
|
-
/**
|
|
335
|
-
* Get verbs by source
|
|
336
|
-
*/
|
|
337
|
-
async getVerbsBySource_internal(sourceId) {
|
|
338
|
-
// Need to search across all verb types
|
|
339
|
-
// TODO: Optimize with metadata index in Phase 1b
|
|
340
|
-
const verbs = [];
|
|
341
|
-
for (let i = 0; i < VERB_TYPE_COUNT; i++) {
|
|
342
|
-
const type = TypeUtils.getVerbFromIndex(i);
|
|
343
|
-
const prefix = `entities/verbs/${type}/vectors/`;
|
|
344
|
-
const paths = await this.u.listObjectsUnderPath(prefix);
|
|
345
|
-
for (const path of paths) {
|
|
346
|
-
try {
|
|
347
|
-
const id = path.split('/').pop()?.replace('.json', '');
|
|
348
|
-
if (!id)
|
|
349
|
-
continue;
|
|
350
|
-
// Load the HNSWVerb
|
|
351
|
-
const hnswVerb = await this.u.readObjectFromPath(path);
|
|
352
|
-
if (!hnswVerb)
|
|
353
|
-
continue;
|
|
354
|
-
// Check sourceId from HNSWVerb (v4.0.0: core fields are in HNSWVerb)
|
|
355
|
-
if (hnswVerb.sourceId !== sourceId)
|
|
356
|
-
continue;
|
|
357
|
-
// Load metadata separately
|
|
358
|
-
const metadata = await this.getVerbMetadata(id);
|
|
359
|
-
if (!metadata)
|
|
360
|
-
continue;
|
|
361
|
-
// Create HNSWVerbWithMetadata (verbs don't have level field)
|
|
362
|
-
const verbWithMetadata = {
|
|
363
|
-
id: hnswVerb.id,
|
|
364
|
-
vector: [...hnswVerb.vector],
|
|
365
|
-
connections: new Map(hnswVerb.connections),
|
|
366
|
-
verb: hnswVerb.verb,
|
|
367
|
-
sourceId: hnswVerb.sourceId,
|
|
368
|
-
targetId: hnswVerb.targetId,
|
|
369
|
-
metadata: metadata
|
|
370
|
-
};
|
|
371
|
-
verbs.push(verbWithMetadata);
|
|
372
|
-
}
|
|
373
|
-
catch (error) {
|
|
374
|
-
// Continue searching
|
|
375
|
-
}
|
|
376
|
-
}
|
|
377
|
-
}
|
|
378
|
-
return verbs;
|
|
379
|
-
}
|
|
380
|
-
/**
|
|
381
|
-
* Get verbs by target
|
|
382
|
-
*/
|
|
383
|
-
async getVerbsByTarget_internal(targetId) {
|
|
384
|
-
// Similar to getVerbsBySource_internal
|
|
385
|
-
const verbs = [];
|
|
386
|
-
for (let i = 0; i < VERB_TYPE_COUNT; i++) {
|
|
387
|
-
const type = TypeUtils.getVerbFromIndex(i);
|
|
388
|
-
const prefix = `entities/verbs/${type}/vectors/`;
|
|
389
|
-
const paths = await this.u.listObjectsUnderPath(prefix);
|
|
390
|
-
for (const path of paths) {
|
|
391
|
-
try {
|
|
392
|
-
const id = path.split('/').pop()?.replace('.json', '');
|
|
393
|
-
if (!id)
|
|
394
|
-
continue;
|
|
395
|
-
// Load the HNSWVerb
|
|
396
|
-
const hnswVerb = await this.u.readObjectFromPath(path);
|
|
397
|
-
if (!hnswVerb)
|
|
398
|
-
continue;
|
|
399
|
-
// Check targetId from HNSWVerb (v4.0.0: core fields are in HNSWVerb)
|
|
400
|
-
if (hnswVerb.targetId !== targetId)
|
|
401
|
-
continue;
|
|
402
|
-
// Load metadata separately
|
|
403
|
-
const metadata = await this.getVerbMetadata(id);
|
|
404
|
-
if (!metadata)
|
|
405
|
-
continue;
|
|
406
|
-
// Create HNSWVerbWithMetadata (verbs don't have level field)
|
|
407
|
-
const verbWithMetadata = {
|
|
408
|
-
id: hnswVerb.id,
|
|
409
|
-
vector: [...hnswVerb.vector],
|
|
410
|
-
connections: new Map(hnswVerb.connections),
|
|
411
|
-
verb: hnswVerb.verb,
|
|
412
|
-
sourceId: hnswVerb.sourceId,
|
|
413
|
-
targetId: hnswVerb.targetId,
|
|
414
|
-
metadata: metadata
|
|
415
|
-
};
|
|
416
|
-
verbs.push(verbWithMetadata);
|
|
417
|
-
}
|
|
418
|
-
catch (error) {
|
|
419
|
-
// Continue
|
|
420
|
-
}
|
|
421
|
-
}
|
|
422
|
-
}
|
|
423
|
-
return verbs;
|
|
424
|
-
}
|
|
425
|
-
/**
|
|
426
|
-
* Get verbs by type (O(1) with type-first paths!)
|
|
427
|
-
*
|
|
428
|
-
* v4.0.0: Load verbs and combine with metadata
|
|
429
|
-
*/
|
|
430
|
-
async getVerbsByType_internal(verbType) {
|
|
431
|
-
const type = verbType;
|
|
432
|
-
const prefix = `entities/verbs/${type}/vectors/`;
|
|
433
|
-
const paths = await this.u.listObjectsUnderPath(prefix);
|
|
434
|
-
const verbs = [];
|
|
435
|
-
for (const path of paths) {
|
|
436
|
-
try {
|
|
437
|
-
const hnswVerb = await this.u.readObjectFromPath(path);
|
|
438
|
-
if (!hnswVerb)
|
|
439
|
-
continue;
|
|
440
|
-
// Cache type from HNSWVerb for future O(1) retrievals
|
|
441
|
-
this.verbTypeCache.set(hnswVerb.id, hnswVerb.verb);
|
|
442
|
-
// Load metadata separately
|
|
443
|
-
const metadata = await this.getVerbMetadata(hnswVerb.id);
|
|
444
|
-
if (!metadata)
|
|
445
|
-
continue;
|
|
446
|
-
// Create HNSWVerbWithMetadata (verbs don't have level field)
|
|
447
|
-
const verbWithMetadata = {
|
|
448
|
-
id: hnswVerb.id,
|
|
449
|
-
vector: [...hnswVerb.vector],
|
|
450
|
-
connections: new Map(hnswVerb.connections),
|
|
451
|
-
verb: hnswVerb.verb,
|
|
452
|
-
sourceId: hnswVerb.sourceId,
|
|
453
|
-
targetId: hnswVerb.targetId,
|
|
454
|
-
metadata: metadata
|
|
455
|
-
};
|
|
456
|
-
verbs.push(verbWithMetadata);
|
|
457
|
-
}
|
|
458
|
-
catch (error) {
|
|
459
|
-
console.warn(`[TypeAwareStorage] Failed to load verb from ${path}:`, error);
|
|
460
|
-
}
|
|
461
|
-
}
|
|
462
|
-
return verbs;
|
|
463
|
-
}
|
|
464
|
-
/**
|
|
465
|
-
* Delete verb (type-first path)
|
|
466
|
-
*/
|
|
467
|
-
async deleteVerb_internal(id) {
|
|
468
|
-
// Try cache first
|
|
469
|
-
const cachedType = this.verbTypeCache.get(id);
|
|
470
|
-
if (cachedType) {
|
|
471
|
-
const path = getVerbVectorPath(cachedType, id);
|
|
472
|
-
await this.u.deleteObjectFromPath(path);
|
|
473
|
-
const typeIndex = TypeUtils.getVerbIndex(cachedType);
|
|
474
|
-
if (this.verbCountsByType[typeIndex] > 0) {
|
|
475
|
-
this.verbCountsByType[typeIndex]--;
|
|
476
|
-
}
|
|
477
|
-
this.verbTypeCache.delete(id);
|
|
478
|
-
return;
|
|
479
|
-
}
|
|
480
|
-
// Search across all types
|
|
481
|
-
for (let i = 0; i < VERB_TYPE_COUNT; i++) {
|
|
482
|
-
const type = TypeUtils.getVerbFromIndex(i);
|
|
483
|
-
const path = getVerbVectorPath(type, id);
|
|
484
|
-
try {
|
|
485
|
-
await this.u.deleteObjectFromPath(path);
|
|
486
|
-
if (this.verbCountsByType[i] > 0) {
|
|
487
|
-
this.verbCountsByType[i]--;
|
|
488
|
-
}
|
|
489
|
-
this.verbTypeCache.delete(id);
|
|
490
|
-
return;
|
|
491
|
-
}
|
|
492
|
-
catch (error) {
|
|
493
|
-
// Continue
|
|
494
|
-
}
|
|
495
|
-
}
|
|
496
|
-
}
|
|
497
|
-
/**
|
|
498
|
-
* Save noun metadata (override to cache type for type-aware routing)
|
|
499
|
-
*
|
|
500
|
-
* v4.0.0: Extract and cache noun type when metadata is saved
|
|
501
|
-
*/
|
|
502
|
-
async saveNounMetadata(id, metadata) {
|
|
503
|
-
// Extract and cache the type
|
|
504
|
-
const type = (metadata.noun || 'thing');
|
|
505
|
-
this.nounTypeCache.set(id, type);
|
|
506
|
-
// Save to type-aware path
|
|
507
|
-
const path = getNounMetadataPath(type, id);
|
|
508
|
-
await this.u.writeObjectToPath(path, metadata);
|
|
509
|
-
}
|
|
510
|
-
/**
|
|
511
|
-
* Get noun metadata (override to use type-aware paths)
|
|
512
|
-
*/
|
|
513
|
-
async getNounMetadata(id) {
|
|
514
|
-
// Try cache first
|
|
515
|
-
const cachedType = this.nounTypeCache.get(id);
|
|
516
|
-
if (cachedType) {
|
|
517
|
-
const path = getNounMetadataPath(cachedType, id);
|
|
518
|
-
return await this.u.readObjectFromPath(path);
|
|
519
|
-
}
|
|
520
|
-
// Search across all types
|
|
521
|
-
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
|
|
522
|
-
const type = TypeUtils.getNounFromIndex(i);
|
|
523
|
-
const path = getNounMetadataPath(type, id);
|
|
524
|
-
try {
|
|
525
|
-
const metadata = await this.u.readObjectFromPath(path);
|
|
526
|
-
if (metadata) {
|
|
527
|
-
// Cache the type for next time
|
|
528
|
-
const metadataType = (metadata.noun || 'thing');
|
|
529
|
-
this.nounTypeCache.set(id, metadataType);
|
|
530
|
-
return metadata;
|
|
531
|
-
}
|
|
532
|
-
}
|
|
533
|
-
catch (error) {
|
|
534
|
-
// Not in this type, continue searching
|
|
535
|
-
}
|
|
536
|
-
}
|
|
537
|
-
return null;
|
|
538
|
-
}
|
|
539
|
-
/**
|
|
540
|
-
* Delete noun metadata (override to use type-aware paths)
|
|
541
|
-
*/
|
|
542
|
-
async deleteNounMetadata(id) {
|
|
543
|
-
const cachedType = this.nounTypeCache.get(id);
|
|
544
|
-
if (cachedType) {
|
|
545
|
-
const path = getNounMetadataPath(cachedType, id);
|
|
546
|
-
await this.u.deleteObjectFromPath(path);
|
|
547
|
-
return;
|
|
548
|
-
}
|
|
549
|
-
// Search across all types
|
|
550
|
-
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
|
|
551
|
-
const type = TypeUtils.getNounFromIndex(i);
|
|
552
|
-
const path = getNounMetadataPath(type, id);
|
|
553
|
-
try {
|
|
554
|
-
await this.u.deleteObjectFromPath(path);
|
|
555
|
-
return;
|
|
556
|
-
}
|
|
557
|
-
catch (error) {
|
|
558
|
-
// Not in this type, continue
|
|
559
|
-
}
|
|
560
|
-
}
|
|
561
|
-
}
|
|
562
|
-
/**
|
|
563
|
-
* Save verb metadata (override to use type-aware paths)
|
|
564
|
-
*
|
|
565
|
-
* Note: Verb type comes from HNSWVerb.verb field, not metadata
|
|
566
|
-
* We need to read the verb to get the type for path routing
|
|
567
|
-
*/
|
|
568
|
-
async saveVerbMetadata(id, metadata) {
|
|
569
|
-
// Get verb type from cache or by reading the verb
|
|
570
|
-
let type = this.verbTypeCache.get(id);
|
|
571
|
-
if (!type) {
|
|
572
|
-
// Need to read the verb to get its type
|
|
573
|
-
const verb = await this.getVerb_internal(id);
|
|
574
|
-
if (verb) {
|
|
575
|
-
type = verb.verb;
|
|
576
|
-
this.verbTypeCache.set(id, type);
|
|
577
|
-
}
|
|
578
|
-
else {
|
|
579
|
-
type = 'relatedTo';
|
|
580
|
-
}
|
|
581
|
-
}
|
|
582
|
-
// Save to type-aware path
|
|
583
|
-
const path = getVerbMetadataPath(type, id);
|
|
584
|
-
await this.u.writeObjectToPath(path, metadata);
|
|
585
|
-
}
|
|
586
|
-
/**
|
|
587
|
-
* Get verb metadata (override to use type-aware paths)
|
|
588
|
-
*/
|
|
589
|
-
async getVerbMetadata(id) {
|
|
590
|
-
// Try cache first
|
|
591
|
-
const cachedType = this.verbTypeCache.get(id);
|
|
592
|
-
if (cachedType) {
|
|
593
|
-
const path = getVerbMetadataPath(cachedType, id);
|
|
594
|
-
return await this.u.readObjectFromPath(path);
|
|
595
|
-
}
|
|
596
|
-
// Search across all types
|
|
597
|
-
for (let i = 0; i < VERB_TYPE_COUNT; i++) {
|
|
598
|
-
const type = TypeUtils.getVerbFromIndex(i);
|
|
599
|
-
const path = getVerbMetadataPath(type, id);
|
|
600
|
-
try {
|
|
601
|
-
const metadata = await this.u.readObjectFromPath(path);
|
|
602
|
-
if (metadata) {
|
|
603
|
-
// Cache the type for next time
|
|
604
|
-
this.verbTypeCache.set(id, type);
|
|
605
|
-
return metadata;
|
|
606
|
-
}
|
|
607
|
-
}
|
|
608
|
-
catch (error) {
|
|
609
|
-
// Not in this type, continue
|
|
610
|
-
}
|
|
611
|
-
}
|
|
612
|
-
return null;
|
|
613
|
-
}
|
|
614
|
-
/**
|
|
615
|
-
* Delete verb metadata (override to use type-aware paths)
|
|
616
|
-
*/
|
|
617
|
-
async deleteVerbMetadata(id) {
|
|
618
|
-
const cachedType = this.verbTypeCache.get(id);
|
|
619
|
-
if (cachedType) {
|
|
620
|
-
const path = getVerbMetadataPath(cachedType, id);
|
|
621
|
-
await this.u.deleteObjectFromPath(path);
|
|
622
|
-
return;
|
|
623
|
-
}
|
|
624
|
-
// Search across all types
|
|
625
|
-
for (let i = 0; i < VERB_TYPE_COUNT; i++) {
|
|
626
|
-
const type = TypeUtils.getVerbFromIndex(i);
|
|
627
|
-
const path = getVerbMetadataPath(type, id);
|
|
628
|
-
try {
|
|
629
|
-
await this.u.deleteObjectFromPath(path);
|
|
630
|
-
return;
|
|
631
|
-
}
|
|
632
|
-
catch (error) {
|
|
633
|
-
// Not in this type, continue
|
|
634
|
-
}
|
|
635
|
-
}
|
|
636
|
-
}
|
|
637
|
-
/**
|
|
638
|
-
* Write object to path (delegate to underlying storage)
|
|
639
|
-
*/
|
|
640
|
-
async writeObjectToPath(path, data) {
|
|
641
|
-
return this.u.writeObjectToPath(path, data);
|
|
642
|
-
}
|
|
643
|
-
/**
|
|
644
|
-
* Read object from path (delegate to underlying storage)
|
|
645
|
-
*/
|
|
646
|
-
async readObjectFromPath(path) {
|
|
647
|
-
return this.u.readObjectFromPath(path);
|
|
648
|
-
}
|
|
649
|
-
/**
|
|
650
|
-
* Delete object from path (delegate to underlying storage)
|
|
651
|
-
*/
|
|
652
|
-
async deleteObjectFromPath(path) {
|
|
653
|
-
return this.u.deleteObjectFromPath(path);
|
|
654
|
-
}
|
|
655
|
-
/**
|
|
656
|
-
* List objects under path (delegate to underlying storage)
|
|
657
|
-
*/
|
|
658
|
-
async listObjectsUnderPath(prefix) {
|
|
659
|
-
return this.u.listObjectsUnderPath(prefix);
|
|
660
|
-
}
|
|
661
|
-
/**
|
|
662
|
-
* Save statistics data
|
|
663
|
-
*/
|
|
664
|
-
async saveStatisticsData(statistics) {
|
|
665
|
-
return this.u.writeObjectToPath(`${SYSTEM_DIR}/statistics.json`, statistics);
|
|
666
|
-
}
|
|
667
|
-
/**
|
|
668
|
-
* Get statistics data
|
|
669
|
-
*/
|
|
670
|
-
async getStatisticsData() {
|
|
671
|
-
return this.u.readObjectFromPath(`${SYSTEM_DIR}/statistics.json`);
|
|
672
|
-
}
|
|
673
|
-
/**
|
|
674
|
-
* Clear all data
|
|
675
|
-
*/
|
|
676
|
-
async clear() {
|
|
677
|
-
// Clear type tracking
|
|
678
|
-
this.nounCountsByType.fill(0);
|
|
679
|
-
this.verbCountsByType.fill(0);
|
|
680
|
-
this.nounTypeCache.clear();
|
|
681
|
-
this.verbTypeCache.clear();
|
|
682
|
-
// Delegate to underlying storage
|
|
683
|
-
if (typeof this.underlying.clear === 'function') {
|
|
684
|
-
await this.underlying.clear();
|
|
685
|
-
}
|
|
686
|
-
}
|
|
687
|
-
/**
|
|
688
|
-
* Get storage status
|
|
689
|
-
*/
|
|
690
|
-
async getStorageStatus() {
|
|
691
|
-
const underlyingStatus = await this.underlying.getStorageStatus();
|
|
692
|
-
return {
|
|
693
|
-
...underlyingStatus,
|
|
694
|
-
type: 'type-aware',
|
|
695
|
-
details: {
|
|
696
|
-
...underlyingStatus.details,
|
|
697
|
-
typeTracking: {
|
|
698
|
-
nounTypes: NOUN_TYPE_COUNT,
|
|
699
|
-
verbTypes: VERB_TYPE_COUNT,
|
|
700
|
-
memoryBytes: 284, // 124 + 160
|
|
701
|
-
nounCounts: Array.from(this.nounCountsByType),
|
|
702
|
-
verbCounts: Array.from(this.verbCountsByType),
|
|
703
|
-
cacheSize: this.nounTypeCache.size + this.verbTypeCache.size
|
|
704
|
-
}
|
|
705
|
-
}
|
|
706
|
-
};
|
|
707
|
-
}
|
|
708
|
-
/**
|
|
709
|
-
* Initialize counts from storage
|
|
710
|
-
*/
|
|
711
|
-
async initializeCounts() {
|
|
712
|
-
// TypeAwareStorageAdapter maintains its own type-based counts
|
|
713
|
-
// which are loaded in loadTypeStatistics()
|
|
714
|
-
// But we should also initialize the underlying storage's counts
|
|
715
|
-
if (this.u.initializeCounts) {
|
|
716
|
-
await this.u.initializeCounts();
|
|
717
|
-
}
|
|
718
|
-
}
|
|
719
|
-
/**
|
|
720
|
-
* Persist counts to storage
|
|
721
|
-
*/
|
|
722
|
-
async persistCounts() {
|
|
723
|
-
// Persist our type statistics
|
|
724
|
-
await this.saveTypeStatistics();
|
|
725
|
-
// Also persist underlying storage counts
|
|
726
|
-
if (this.u.persistCounts) {
|
|
727
|
-
await this.u.persistCounts();
|
|
728
|
-
}
|
|
729
|
-
}
|
|
730
|
-
/**
|
|
731
|
-
* Get noun vector (delegate to underlying storage)
|
|
732
|
-
*/
|
|
733
|
-
async getNounVector(id) {
|
|
734
|
-
const noun = await this.getNoun_internal(id);
|
|
735
|
-
return noun?.vector || null;
|
|
736
|
-
}
|
|
737
|
-
/**
|
|
738
|
-
* Save HNSW data for a noun
|
|
739
|
-
*/
|
|
740
|
-
async saveHNSWData(nounId, hnswData) {
|
|
741
|
-
// Get noun type for type-first path
|
|
742
|
-
const cachedType = this.nounTypeCache.get(nounId);
|
|
743
|
-
const type = cachedType || 'thing'; // Default if not cached
|
|
744
|
-
const shard = getShardIdFromUuid(nounId);
|
|
745
|
-
const path = `entities/nouns/${type}/hnsw/${shard}/${nounId}.json`;
|
|
746
|
-
await this.u.writeObjectToPath(path, hnswData);
|
|
747
|
-
}
|
|
748
|
-
/**
|
|
749
|
-
* Get HNSW data for a noun
|
|
750
|
-
*/
|
|
751
|
-
async getHNSWData(nounId) {
|
|
752
|
-
// Try cache first
|
|
753
|
-
const cachedType = this.nounTypeCache.get(nounId);
|
|
754
|
-
if (cachedType) {
|
|
755
|
-
const shard = getShardIdFromUuid(nounId);
|
|
756
|
-
const path = `entities/nouns/${cachedType}/hnsw/${shard}/${nounId}.json`;
|
|
757
|
-
return await this.u.readObjectFromPath(path);
|
|
758
|
-
}
|
|
759
|
-
// Search across all types
|
|
760
|
-
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
|
|
761
|
-
const type = TypeUtils.getNounFromIndex(i);
|
|
762
|
-
const shard = getShardIdFromUuid(nounId);
|
|
763
|
-
const path = `entities/nouns/${type}/hnsw/${shard}/${nounId}.json`;
|
|
764
|
-
try {
|
|
765
|
-
const data = await this.u.readObjectFromPath(path);
|
|
766
|
-
if (data) {
|
|
767
|
-
return data;
|
|
768
|
-
}
|
|
769
|
-
}
|
|
770
|
-
catch (error) {
|
|
771
|
-
// Not in this type, continue
|
|
772
|
-
}
|
|
773
|
-
}
|
|
774
|
-
return null;
|
|
775
|
-
}
|
|
776
|
-
/**
|
|
777
|
-
* Save HNSW system data (entry point, max level)
|
|
778
|
-
*/
|
|
779
|
-
async saveHNSWSystem(systemData) {
|
|
780
|
-
await this.u.writeObjectToPath(`${SYSTEM_DIR}/hnsw-system.json`, systemData);
|
|
781
|
-
}
|
|
782
|
-
/**
|
|
783
|
-
* Get HNSW system data
|
|
784
|
-
*/
|
|
785
|
-
async getHNSWSystem() {
|
|
786
|
-
return await this.u.readObjectFromPath(`${SYSTEM_DIR}/hnsw-system.json`);
|
|
787
|
-
}
|
|
788
|
-
/**
|
|
789
|
-
* Get type statistics
|
|
790
|
-
* Useful for analytics and optimization
|
|
791
|
-
*/
|
|
792
|
-
getTypeStatistics() {
|
|
793
|
-
const nouns = [];
|
|
794
|
-
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
|
|
795
|
-
const count = this.nounCountsByType[i];
|
|
796
|
-
if (count > 0) {
|
|
797
|
-
nouns.push({ type: TypeUtils.getNounFromIndex(i), count });
|
|
798
|
-
}
|
|
799
|
-
}
|
|
800
|
-
const verbs = [];
|
|
801
|
-
for (let i = 0; i < VERB_TYPE_COUNT; i++) {
|
|
802
|
-
const count = this.verbCountsByType[i];
|
|
803
|
-
if (count > 0) {
|
|
804
|
-
verbs.push({ type: TypeUtils.getVerbFromIndex(i), count });
|
|
805
|
-
}
|
|
806
|
-
}
|
|
807
|
-
return {
|
|
808
|
-
nouns: nouns.sort((a, b) => b.count - a.count),
|
|
809
|
-
verbs: verbs.sort((a, b) => b.count - a.count),
|
|
810
|
-
totalMemory: 284 // bytes
|
|
811
|
-
};
|
|
812
|
-
}
|
|
813
|
-
}
|
|
814
|
-
//# sourceMappingURL=typeAwareStorageAdapter.js.map
|