@soulcraft/brainy 3.50.2 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +201 -0
- package/README.md +358 -658
- package/dist/api/ConfigAPI.js +56 -19
- package/dist/api/DataAPI.js +24 -18
- package/dist/augmentations/storageAugmentations.d.ts +24 -0
- package/dist/augmentations/storageAugmentations.js +22 -0
- package/dist/brainy.js +32 -9
- package/dist/cli/commands/core.d.ts +20 -10
- package/dist/cli/commands/core.js +384 -82
- package/dist/cli/commands/import.d.ts +41 -0
- package/dist/cli/commands/import.js +456 -0
- package/dist/cli/commands/insights.d.ts +34 -0
- package/dist/cli/commands/insights.js +300 -0
- package/dist/cli/commands/neural.d.ts +6 -12
- package/dist/cli/commands/neural.js +113 -10
- package/dist/cli/commands/nlp.d.ts +28 -0
- package/dist/cli/commands/nlp.js +246 -0
- package/dist/cli/commands/storage.d.ts +64 -0
- package/dist/cli/commands/storage.js +730 -0
- package/dist/cli/index.js +210 -24
- package/dist/coreTypes.d.ts +206 -34
- package/dist/distributed/configManager.js +8 -6
- package/dist/distributed/shardMigration.js +2 -0
- package/dist/distributed/storageDiscovery.js +6 -4
- package/dist/embeddings/EmbeddingManager.d.ts +2 -2
- package/dist/embeddings/EmbeddingManager.js +5 -1
- package/dist/graph/lsm/LSMTree.js +32 -20
- package/dist/hnsw/typeAwareHNSWIndex.js +6 -2
- package/dist/storage/adapters/azureBlobStorage.d.ts +545 -0
- package/dist/storage/adapters/azureBlobStorage.js +1809 -0
- package/dist/storage/adapters/baseStorageAdapter.d.ts +16 -13
- package/dist/storage/adapters/fileSystemStorage.d.ts +21 -9
- package/dist/storage/adapters/fileSystemStorage.js +204 -127
- package/dist/storage/adapters/gcsStorage.d.ts +119 -9
- package/dist/storage/adapters/gcsStorage.js +317 -62
- package/dist/storage/adapters/memoryStorage.d.ts +30 -18
- package/dist/storage/adapters/memoryStorage.js +99 -94
- package/dist/storage/adapters/opfsStorage.d.ts +48 -10
- package/dist/storage/adapters/opfsStorage.js +201 -80
- package/dist/storage/adapters/r2Storage.d.ts +12 -5
- package/dist/storage/adapters/r2Storage.js +63 -15
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +164 -17
- package/dist/storage/adapters/s3CompatibleStorage.js +472 -80
- package/dist/storage/adapters/typeAwareStorageAdapter.d.ts +38 -6
- package/dist/storage/adapters/typeAwareStorageAdapter.js +218 -39
- package/dist/storage/baseStorage.d.ts +41 -38
- package/dist/storage/baseStorage.js +110 -134
- package/dist/storage/storageFactory.d.ts +29 -2
- package/dist/storage/storageFactory.js +30 -1
- package/dist/utils/entityIdMapper.js +5 -2
- package/dist/utils/fieldTypeInference.js +8 -1
- package/dist/utils/metadataFilter.d.ts +3 -2
- package/dist/utils/metadataFilter.js +1 -0
- package/dist/utils/metadataIndex.js +2 -0
- package/dist/utils/metadataIndexChunking.js +9 -4
- package/dist/utils/periodicCleanup.js +1 -0
- package/package.json +3 -1
|
@@ -138,45 +138,41 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
138
138
|
}
|
|
139
139
|
}
|
|
140
140
|
/**
|
|
141
|
-
* Save a noun to storage
|
|
141
|
+
* Save a noun to storage (v4.0.0: vector only, metadata saved separately)
|
|
142
|
+
* @param noun Pure HNSW vector data (no metadata)
|
|
142
143
|
*/
|
|
143
144
|
async saveNoun(noun) {
|
|
144
145
|
await this.ensureInitialized();
|
|
145
|
-
//
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
}
|
|
149
|
-
// Save both the HNSWNoun vector data and metadata separately (2-file system)
|
|
150
|
-
try {
|
|
151
|
-
// Save the lightweight HNSWNoun vector file first
|
|
152
|
-
await this.saveNoun_internal(noun);
|
|
153
|
-
// Then save the metadata to separate file (if present)
|
|
154
|
-
if (noun.metadata) {
|
|
155
|
-
await this.saveNounMetadata(noun.id, noun.metadata);
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
catch (error) {
|
|
159
|
-
console.error(`[ERROR] Failed to save noun ${noun.id}:`, error);
|
|
160
|
-
// Attempt cleanup - remove noun file if metadata failed
|
|
161
|
-
try {
|
|
162
|
-
const nounExists = await this.getNoun_internal(noun.id);
|
|
163
|
-
if (nounExists) {
|
|
164
|
-
console.log(`[CLEANUP] Attempting to remove orphaned noun file ${noun.id}`);
|
|
165
|
-
await this.deleteNoun_internal(noun.id);
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
catch (cleanupError) {
|
|
169
|
-
console.error(`[ERROR] Failed to cleanup orphaned noun ${noun.id}:`, cleanupError);
|
|
170
|
-
}
|
|
171
|
-
throw new Error(`Failed to save noun ${noun.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
172
|
-
}
|
|
146
|
+
// Save the HNSWNoun vector data only
|
|
147
|
+
// Metadata must be saved separately via saveNounMetadata()
|
|
148
|
+
await this.saveNoun_internal(noun);
|
|
173
149
|
}
|
|
174
150
|
/**
|
|
175
|
-
* Get a noun from storage
|
|
151
|
+
* Get a noun from storage (v4.0.0: returns combined HNSWNounWithMetadata)
|
|
152
|
+
* @param id Entity ID
|
|
153
|
+
* @returns Combined vector + metadata or null
|
|
176
154
|
*/
|
|
177
155
|
async getNoun(id) {
|
|
178
156
|
await this.ensureInitialized();
|
|
179
|
-
|
|
157
|
+
// Load vector and metadata separately
|
|
158
|
+
const vector = await this.getNoun_internal(id);
|
|
159
|
+
if (!vector) {
|
|
160
|
+
return null;
|
|
161
|
+
}
|
|
162
|
+
// Load metadata
|
|
163
|
+
const metadata = await this.getNounMetadata(id);
|
|
164
|
+
if (!metadata) {
|
|
165
|
+
console.warn(`[Storage] Noun ${id} has vector but no metadata - this should not happen in v4.0.0`);
|
|
166
|
+
return null;
|
|
167
|
+
}
|
|
168
|
+
// Combine into HNSWNounWithMetadata
|
|
169
|
+
return {
|
|
170
|
+
id: vector.id,
|
|
171
|
+
vector: vector.vector,
|
|
172
|
+
connections: vector.connections,
|
|
173
|
+
level: vector.level,
|
|
174
|
+
metadata
|
|
175
|
+
};
|
|
180
176
|
}
|
|
181
177
|
/**
|
|
182
178
|
* Get nouns by noun type
|
|
@@ -185,7 +181,20 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
185
181
|
*/
|
|
186
182
|
async getNounsByNounType(nounType) {
|
|
187
183
|
await this.ensureInitialized();
|
|
188
|
-
|
|
184
|
+
// Internal method returns HNSWNoun[], need to combine with metadata
|
|
185
|
+
const nouns = await this.getNounsByNounType_internal(nounType);
|
|
186
|
+
// Combine each noun with its metadata
|
|
187
|
+
const nounsWithMetadata = [];
|
|
188
|
+
for (const noun of nouns) {
|
|
189
|
+
const metadata = await this.getNounMetadata(noun.id);
|
|
190
|
+
if (metadata) {
|
|
191
|
+
nounsWithMetadata.push({
|
|
192
|
+
...noun,
|
|
193
|
+
metadata
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
return nounsWithMetadata;
|
|
189
198
|
}
|
|
190
199
|
/**
|
|
191
200
|
* Delete a noun from storage
|
|
@@ -204,90 +213,58 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
204
213
|
}
|
|
205
214
|
}
|
|
206
215
|
/**
|
|
207
|
-
* Save a verb to storage
|
|
216
|
+
* Save a verb to storage (v4.0.0: verb only, metadata saved separately)
|
|
208
217
|
*
|
|
209
|
-
*
|
|
210
|
-
* These are core relational fields, not metadata. They're stored in the vector
|
|
211
|
-
* file for fast access and to align with actual usage patterns.
|
|
218
|
+
* @param verb Pure HNSW verb with core relational fields (verb, sourceId, targetId)
|
|
212
219
|
*/
|
|
213
220
|
async saveVerb(verb) {
|
|
214
221
|
await this.ensureInitialized();
|
|
215
222
|
// Validate verb type before saving - storage boundary protection
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
const hnswVerb = {
|
|
221
|
-
id: verb.id,
|
|
222
|
-
vector: verb.vector,
|
|
223
|
-
connections: verb.connections || new Map(),
|
|
224
|
-
// CORE RELATIONAL DATA (v3.50.1+)
|
|
225
|
-
verb: (verb.verb || verb.type || 'relatedTo'),
|
|
226
|
-
sourceId: verb.sourceId || verb.source || '',
|
|
227
|
-
targetId: verb.targetId || verb.target || '',
|
|
228
|
-
// User metadata (if any)
|
|
229
|
-
metadata: verb.metadata
|
|
230
|
-
};
|
|
231
|
-
// Extract lightweight metadata for separate file (optional fields only)
|
|
232
|
-
const metadata = {
|
|
233
|
-
weight: verb.weight,
|
|
234
|
-
data: verb.data,
|
|
235
|
-
createdAt: verb.createdAt,
|
|
236
|
-
updatedAt: verb.updatedAt,
|
|
237
|
-
createdBy: verb.createdBy,
|
|
238
|
-
// Legacy aliases for backward compatibility
|
|
239
|
-
source: verb.source || verb.sourceId,
|
|
240
|
-
target: verb.target || verb.targetId,
|
|
241
|
-
type: verb.type || verb.verb
|
|
242
|
-
};
|
|
243
|
-
// Save both the HNSWVerb and metadata atomically
|
|
244
|
-
try {
|
|
245
|
-
console.log(`[DEBUG] Saving verb ${verb.id}: sourceId=${verb.sourceId}, targetId=${verb.targetId}`);
|
|
246
|
-
// Save the HNSWVerb first
|
|
247
|
-
await this.saveVerb_internal(hnswVerb);
|
|
248
|
-
console.log(`[DEBUG] Successfully saved HNSWVerb file for ${verb.id}`);
|
|
249
|
-
// Then save the metadata
|
|
250
|
-
await this.saveVerbMetadata(verb.id, metadata);
|
|
251
|
-
console.log(`[DEBUG] Successfully saved metadata file for ${verb.id}`);
|
|
252
|
-
}
|
|
253
|
-
catch (error) {
|
|
254
|
-
console.error(`[ERROR] Failed to save verb ${verb.id}:`, error);
|
|
255
|
-
// Attempt cleanup - remove verb file if metadata failed
|
|
256
|
-
try {
|
|
257
|
-
const verbExists = await this.getVerb_internal(verb.id);
|
|
258
|
-
if (verbExists) {
|
|
259
|
-
console.log(`[CLEANUP] Attempting to remove orphaned verb file ${verb.id}`);
|
|
260
|
-
await this.deleteVerb_internal(verb.id);
|
|
261
|
-
}
|
|
262
|
-
}
|
|
263
|
-
catch (cleanupError) {
|
|
264
|
-
console.error(`[ERROR] Failed to cleanup orphaned verb ${verb.id}:`, cleanupError);
|
|
265
|
-
}
|
|
266
|
-
throw new Error(`Failed to save verb ${verb.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
267
|
-
}
|
|
223
|
+
validateVerbType(verb.verb);
|
|
224
|
+
// Save the HNSWVerb vector and core fields only
|
|
225
|
+
// Metadata must be saved separately via saveVerbMetadata()
|
|
226
|
+
await this.saveVerb_internal(verb);
|
|
268
227
|
}
|
|
269
228
|
/**
|
|
270
|
-
* Get a verb from storage
|
|
229
|
+
* Get a verb from storage (v4.0.0: returns combined HNSWVerbWithMetadata)
|
|
230
|
+
* @param id Entity ID
|
|
231
|
+
* @returns Combined verb + metadata or null
|
|
271
232
|
*/
|
|
272
233
|
async getVerb(id) {
|
|
273
234
|
await this.ensureInitialized();
|
|
274
|
-
|
|
275
|
-
|
|
235
|
+
// Load verb vector and core fields
|
|
236
|
+
const verb = await this.getVerb_internal(id);
|
|
237
|
+
if (!verb) {
|
|
276
238
|
return null;
|
|
277
239
|
}
|
|
278
|
-
|
|
240
|
+
// Load metadata
|
|
241
|
+
const metadata = await this.getVerbMetadata(id);
|
|
242
|
+
if (!metadata) {
|
|
243
|
+
console.warn(`[Storage] Verb ${id} has vector but no metadata - this should not happen in v4.0.0`);
|
|
244
|
+
return null;
|
|
245
|
+
}
|
|
246
|
+
// Combine into HNSWVerbWithMetadata
|
|
247
|
+
return {
|
|
248
|
+
id: verb.id,
|
|
249
|
+
vector: verb.vector,
|
|
250
|
+
connections: verb.connections,
|
|
251
|
+
verb: verb.verb,
|
|
252
|
+
sourceId: verb.sourceId,
|
|
253
|
+
targetId: verb.targetId,
|
|
254
|
+
metadata
|
|
255
|
+
};
|
|
279
256
|
}
|
|
280
257
|
/**
|
|
281
258
|
* Convert HNSWVerb to GraphVerb by combining with metadata
|
|
259
|
+
* DEPRECATED: For backward compatibility only. Use getVerb() which returns HNSWVerbWithMetadata.
|
|
282
260
|
*
|
|
283
|
-
*
|
|
284
|
-
* Only optional fields (weight, timestamps, etc.) come from metadata file
|
|
261
|
+
* @deprecated Use getVerb() instead which returns HNSWVerbWithMetadata
|
|
285
262
|
*/
|
|
286
263
|
async convertHNSWVerbToGraphVerb(hnswVerb) {
|
|
287
264
|
try {
|
|
288
|
-
//
|
|
265
|
+
// Load metadata
|
|
289
266
|
const metadata = await this.getVerbMetadata(hnswVerb.id);
|
|
290
|
-
// Create default timestamp
|
|
267
|
+
// Create default timestamp in Firestore format
|
|
291
268
|
const defaultTimestamp = {
|
|
292
269
|
seconds: Math.floor(Date.now() / 1000),
|
|
293
270
|
nanoseconds: (Date.now() % 1000) * 1000000
|
|
@@ -297,10 +274,22 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
297
274
|
augmentation: 'unknown',
|
|
298
275
|
version: '1.0'
|
|
299
276
|
};
|
|
277
|
+
// Convert flexible timestamp to Firestore format for GraphVerb
|
|
278
|
+
const normalizeTimestamp = (ts) => {
|
|
279
|
+
if (!ts)
|
|
280
|
+
return defaultTimestamp;
|
|
281
|
+
if (typeof ts === 'number') {
|
|
282
|
+
return {
|
|
283
|
+
seconds: Math.floor(ts / 1000),
|
|
284
|
+
nanoseconds: (ts % 1000) * 1000000
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
return ts;
|
|
288
|
+
};
|
|
300
289
|
return {
|
|
301
290
|
id: hnswVerb.id,
|
|
302
291
|
vector: hnswVerb.vector,
|
|
303
|
-
// CORE FIELDS from HNSWVerb
|
|
292
|
+
// CORE FIELDS from HNSWVerb
|
|
304
293
|
verb: hnswVerb.verb,
|
|
305
294
|
sourceId: hnswVerb.sourceId,
|
|
306
295
|
targetId: hnswVerb.targetId,
|
|
@@ -310,9 +299,9 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
310
299
|
target: hnswVerb.targetId,
|
|
311
300
|
// Optional fields from metadata file
|
|
312
301
|
weight: metadata?.weight || 1.0,
|
|
313
|
-
metadata:
|
|
314
|
-
createdAt: metadata?.createdAt
|
|
315
|
-
updatedAt: metadata?.updatedAt
|
|
302
|
+
metadata: metadata || {},
|
|
303
|
+
createdAt: normalizeTimestamp(metadata?.createdAt),
|
|
304
|
+
updatedAt: normalizeTimestamp(metadata?.updatedAt),
|
|
316
305
|
createdBy: metadata?.createdBy || defaultCreatedBy,
|
|
317
306
|
data: metadata?.data,
|
|
318
307
|
embedding: hnswVerb.vector
|
|
@@ -333,23 +322,15 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
333
322
|
const result = await this.getVerbs({
|
|
334
323
|
pagination: { limit: Number.MAX_SAFE_INTEGER }
|
|
335
324
|
});
|
|
336
|
-
// Convert
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
verb: (graphVerb.verb || graphVerb.type || 'relatedTo'),
|
|
346
|
-
sourceId: graphVerb.sourceId || graphVerb.source || '',
|
|
347
|
-
targetId: graphVerb.targetId || graphVerb.target || '',
|
|
348
|
-
// User metadata
|
|
349
|
-
metadata: graphVerb.metadata
|
|
350
|
-
};
|
|
351
|
-
hnswVerbs.push(hnswVerb);
|
|
352
|
-
}
|
|
325
|
+
// v4.0.0: Convert HNSWVerbWithMetadata to HNSWVerb (strip metadata)
|
|
326
|
+
const hnswVerbs = result.items.map(verbWithMetadata => ({
|
|
327
|
+
id: verbWithMetadata.id,
|
|
328
|
+
vector: verbWithMetadata.vector,
|
|
329
|
+
connections: verbWithMetadata.connections,
|
|
330
|
+
verb: verbWithMetadata.verb,
|
|
331
|
+
sourceId: verbWithMetadata.sourceId,
|
|
332
|
+
targetId: verbWithMetadata.targetId
|
|
333
|
+
}));
|
|
353
334
|
return hnswVerbs;
|
|
354
335
|
}
|
|
355
336
|
/**
|
|
@@ -423,8 +404,8 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
423
404
|
const nounType = Array.isArray(options.filter.nounType)
|
|
424
405
|
? options.filter.nounType[0]
|
|
425
406
|
: options.filter.nounType;
|
|
426
|
-
// Get nouns by type directly
|
|
427
|
-
const nounsByType = await this.
|
|
407
|
+
// Get nouns by type directly (already combines with metadata)
|
|
408
|
+
const nounsByType = await this.getNounsByNounType(nounType);
|
|
428
409
|
// Apply pagination
|
|
429
410
|
const paginatedNouns = nounsByType.slice(offset, offset + limit);
|
|
430
411
|
const hasMore = offset + limit < nounsByType.length;
|
|
@@ -696,7 +677,7 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
696
677
|
return this.graphIndex;
|
|
697
678
|
}
|
|
698
679
|
/**
|
|
699
|
-
* Save metadata to storage
|
|
680
|
+
* Save metadata to storage (v4.0.0: now typed)
|
|
700
681
|
* Routes to correct location (system or entity) based on key format
|
|
701
682
|
*/
|
|
702
683
|
async saveMetadata(id, metadata) {
|
|
@@ -705,7 +686,7 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
705
686
|
return this.writeObjectToPath(keyInfo.fullPath, metadata);
|
|
706
687
|
}
|
|
707
688
|
/**
|
|
708
|
-
* Get metadata from storage
|
|
689
|
+
* Get metadata from storage (v4.0.0: now typed)
|
|
709
690
|
* Routes to correct location (system or entity) based on key format
|
|
710
691
|
*/
|
|
711
692
|
async getMetadata(id) {
|
|
@@ -714,18 +695,16 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
714
695
|
return this.readObjectFromPath(keyInfo.fullPath);
|
|
715
696
|
}
|
|
716
697
|
/**
|
|
717
|
-
* Save noun metadata to storage
|
|
698
|
+
* Save noun metadata to storage (v4.0.0: now typed)
|
|
718
699
|
* Routes to correct sharded location based on UUID
|
|
719
700
|
*/
|
|
720
701
|
async saveNounMetadata(id, metadata) {
|
|
721
702
|
// Validate noun type in metadata - storage boundary protection
|
|
722
|
-
|
|
723
|
-
validateNounType(metadata.noun);
|
|
724
|
-
}
|
|
703
|
+
validateNounType(metadata.noun);
|
|
725
704
|
return this.saveNounMetadata_internal(id, metadata);
|
|
726
705
|
}
|
|
727
706
|
/**
|
|
728
|
-
* Internal method for saving noun metadata
|
|
707
|
+
* Internal method for saving noun metadata (v4.0.0: now typed)
|
|
729
708
|
* Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
|
|
730
709
|
* @protected
|
|
731
710
|
*/
|
|
@@ -735,7 +714,7 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
735
714
|
return this.writeObjectToPath(keyInfo.fullPath, metadata);
|
|
736
715
|
}
|
|
737
716
|
/**
|
|
738
|
-
* Get noun metadata from storage
|
|
717
|
+
* Get noun metadata from storage (v4.0.0: now typed)
|
|
739
718
|
* Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
|
|
740
719
|
*/
|
|
741
720
|
async getNounMetadata(id) {
|
|
@@ -753,18 +732,15 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
753
732
|
return this.deleteObjectFromPath(keyInfo.fullPath);
|
|
754
733
|
}
|
|
755
734
|
/**
|
|
756
|
-
* Save verb metadata to storage
|
|
735
|
+
* Save verb metadata to storage (v4.0.0: now typed)
|
|
757
736
|
* Routes to correct sharded location based on UUID
|
|
758
737
|
*/
|
|
759
738
|
async saveVerbMetadata(id, metadata) {
|
|
760
|
-
//
|
|
761
|
-
if (metadata?.verb) {
|
|
762
|
-
validateVerbType(metadata.verb);
|
|
763
|
-
}
|
|
739
|
+
// Note: verb type is in HNSWVerb, not metadata
|
|
764
740
|
return this.saveVerbMetadata_internal(id, metadata);
|
|
765
741
|
}
|
|
766
742
|
/**
|
|
767
|
-
* Internal method for saving verb metadata
|
|
743
|
+
* Internal method for saving verb metadata (v4.0.0: now typed)
|
|
768
744
|
* Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
|
|
769
745
|
* @protected
|
|
770
746
|
*/
|
|
@@ -774,7 +750,7 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
774
750
|
return this.writeObjectToPath(keyInfo.fullPath, metadata);
|
|
775
751
|
}
|
|
776
752
|
/**
|
|
777
|
-
* Get verb metadata from storage
|
|
753
|
+
* Get verb metadata from storage (v4.0.0: now typed)
|
|
778
754
|
* Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
|
|
779
755
|
*/
|
|
780
756
|
async getVerbMetadata(id) {
|
|
@@ -8,6 +8,7 @@ import { OPFSStorage } from './adapters/opfsStorage.js';
|
|
|
8
8
|
import { S3CompatibleStorage } from './adapters/s3CompatibleStorage.js';
|
|
9
9
|
import { R2Storage } from './adapters/r2Storage.js';
|
|
10
10
|
import { GcsStorage } from './adapters/gcsStorage.js';
|
|
11
|
+
import { AzureBlobStorage } from './adapters/azureBlobStorage.js';
|
|
11
12
|
import { TypeAwareStorageAdapter } from './adapters/typeAwareStorageAdapter.js';
|
|
12
13
|
import { OperationConfig } from '../utils/operationUtils.js';
|
|
13
14
|
/**
|
|
@@ -24,9 +25,10 @@ export interface StorageOptions {
|
|
|
24
25
|
* - 'r2': Use Cloudflare R2 storage
|
|
25
26
|
* - 'gcs': Use Google Cloud Storage (S3-compatible with HMAC keys)
|
|
26
27
|
* - 'gcs-native': Use Google Cloud Storage (native SDK with ADC)
|
|
28
|
+
* - 'azure': Use Azure Blob Storage (native SDK with Managed Identity)
|
|
27
29
|
* - 'type-aware': Use type-first storage adapter (wraps another adapter)
|
|
28
30
|
*/
|
|
29
|
-
type?: 'auto' | 'memory' | 'opfs' | 'filesystem' | 's3' | 'r2' | 'gcs' | 'gcs-native' | 'type-aware';
|
|
31
|
+
type?: 'auto' | 'memory' | 'opfs' | 'filesystem' | 's3' | 'r2' | 'gcs' | 'gcs-native' | 'azure' | 'type-aware';
|
|
30
32
|
/**
|
|
31
33
|
* Force the use of memory storage even if other storage types are available
|
|
32
34
|
*/
|
|
@@ -148,6 +150,31 @@ export interface StorageOptions {
|
|
|
148
150
|
*/
|
|
149
151
|
secretAccessKey?: string;
|
|
150
152
|
};
|
|
153
|
+
/**
|
|
154
|
+
* Configuration for Azure Blob Storage (native SDK with Managed Identity)
|
|
155
|
+
*/
|
|
156
|
+
azureStorage?: {
|
|
157
|
+
/**
|
|
158
|
+
* Azure container name
|
|
159
|
+
*/
|
|
160
|
+
containerName: string;
|
|
161
|
+
/**
|
|
162
|
+
* Azure Storage account name (for Managed Identity or SAS)
|
|
163
|
+
*/
|
|
164
|
+
accountName?: string;
|
|
165
|
+
/**
|
|
166
|
+
* Azure Storage account key (optional, uses Managed Identity if not provided)
|
|
167
|
+
*/
|
|
168
|
+
accountKey?: string;
|
|
169
|
+
/**
|
|
170
|
+
* Azure connection string (highest priority if provided)
|
|
171
|
+
*/
|
|
172
|
+
connectionString?: string;
|
|
173
|
+
/**
|
|
174
|
+
* SAS token (optional, alternative to account key)
|
|
175
|
+
*/
|
|
176
|
+
sasToken?: string;
|
|
177
|
+
};
|
|
151
178
|
/**
|
|
152
179
|
* Configuration for Type-Aware Storage (type-first architecture)
|
|
153
180
|
* Wraps another storage adapter and adds type-first routing
|
|
@@ -254,4 +281,4 @@ export declare function createStorage(options?: StorageOptions): Promise<Storage
|
|
|
254
281
|
/**
|
|
255
282
|
* Export storage adapters
|
|
256
283
|
*/
|
|
257
|
-
export { MemoryStorage, OPFSStorage, S3CompatibleStorage, R2Storage, GcsStorage, TypeAwareStorageAdapter };
|
|
284
|
+
export { MemoryStorage, OPFSStorage, S3CompatibleStorage, R2Storage, GcsStorage, AzureBlobStorage, TypeAwareStorageAdapter };
|
|
@@ -7,6 +7,7 @@ import { OPFSStorage } from './adapters/opfsStorage.js';
|
|
|
7
7
|
import { S3CompatibleStorage } from './adapters/s3CompatibleStorage.js';
|
|
8
8
|
import { R2Storage } from './adapters/r2Storage.js';
|
|
9
9
|
import { GcsStorage } from './adapters/gcsStorage.js';
|
|
10
|
+
import { AzureBlobStorage } from './adapters/azureBlobStorage.js';
|
|
10
11
|
import { TypeAwareStorageAdapter } from './adapters/typeAwareStorageAdapter.js';
|
|
11
12
|
// FileSystemStorage is dynamically imported to avoid issues in browser environments
|
|
12
13
|
import { isBrowser } from '../utils/environment.js';
|
|
@@ -157,6 +158,22 @@ export async function createStorage(options = {}) {
|
|
|
157
158
|
console.warn('GCS native storage configuration is missing, falling back to memory storage');
|
|
158
159
|
return new MemoryStorage();
|
|
159
160
|
}
|
|
161
|
+
case 'azure':
|
|
162
|
+
if (options.azureStorage) {
|
|
163
|
+
console.log('Using Azure Blob Storage (native SDK)');
|
|
164
|
+
return new AzureBlobStorage({
|
|
165
|
+
containerName: options.azureStorage.containerName,
|
|
166
|
+
accountName: options.azureStorage.accountName,
|
|
167
|
+
accountKey: options.azureStorage.accountKey,
|
|
168
|
+
connectionString: options.azureStorage.connectionString,
|
|
169
|
+
sasToken: options.azureStorage.sasToken,
|
|
170
|
+
cacheConfig: options.cacheConfig
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
else {
|
|
174
|
+
console.warn('Azure storage configuration is missing, falling back to memory storage');
|
|
175
|
+
return new MemoryStorage();
|
|
176
|
+
}
|
|
160
177
|
case 'type-aware': {
|
|
161
178
|
console.log('Using Type-Aware Storage (type-first architecture)');
|
|
162
179
|
// Create underlying storage adapter
|
|
@@ -241,6 +258,18 @@ export async function createStorage(options = {}) {
|
|
|
241
258
|
cacheConfig: options.cacheConfig
|
|
242
259
|
});
|
|
243
260
|
}
|
|
261
|
+
// If Azure storage is specified, use it
|
|
262
|
+
if (options.azureStorage) {
|
|
263
|
+
console.log('Using Azure Blob Storage (native SDK)');
|
|
264
|
+
return new AzureBlobStorage({
|
|
265
|
+
containerName: options.azureStorage.containerName,
|
|
266
|
+
accountName: options.azureStorage.accountName,
|
|
267
|
+
accountKey: options.azureStorage.accountKey,
|
|
268
|
+
connectionString: options.azureStorage.connectionString,
|
|
269
|
+
sasToken: options.azureStorage.sasToken,
|
|
270
|
+
cacheConfig: options.cacheConfig
|
|
271
|
+
});
|
|
272
|
+
}
|
|
244
273
|
// Auto-detect the best storage adapter based on the environment
|
|
245
274
|
// First, check if we're in Node.js (prioritize for test environments)
|
|
246
275
|
if (!isBrowser()) {
|
|
@@ -286,7 +315,7 @@ export async function createStorage(options = {}) {
|
|
|
286
315
|
/**
|
|
287
316
|
* Export storage adapters
|
|
288
317
|
*/
|
|
289
|
-
export { MemoryStorage, OPFSStorage, S3CompatibleStorage, R2Storage, GcsStorage, TypeAwareStorageAdapter };
|
|
318
|
+
export { MemoryStorage, OPFSStorage, S3CompatibleStorage, R2Storage, GcsStorage, AzureBlobStorage, TypeAwareStorageAdapter };
|
|
290
319
|
// Export FileSystemStorage conditionally
|
|
291
320
|
// NOTE: FileSystemStorage is now only imported dynamically to avoid fs imports in browser builds
|
|
292
321
|
// export { FileSystemStorage } from './adapters/fileSystemStorage.js'
|
|
@@ -32,8 +32,9 @@ export class EntityIdMapper {
|
|
|
32
32
|
*/
|
|
33
33
|
async init() {
|
|
34
34
|
try {
|
|
35
|
-
const
|
|
36
|
-
if (data) {
|
|
35
|
+
const metadata = await this.storage.getMetadata(this.storageKey);
|
|
36
|
+
if (metadata && metadata.data) {
|
|
37
|
+
const data = metadata.data;
|
|
37
38
|
this.nextId = data.nextId;
|
|
38
39
|
// Rebuild maps from serialized data
|
|
39
40
|
this.uuidToInt = new Map(Object.entries(data.uuidToInt).map(([k, v]) => [k, Number(v)]));
|
|
@@ -136,7 +137,9 @@ export class EntityIdMapper {
|
|
|
136
137
|
return;
|
|
137
138
|
}
|
|
138
139
|
// Convert maps to plain objects for serialization
|
|
140
|
+
// v4.0.0: Add required 'noun' property for NounMetadata
|
|
139
141
|
const data = {
|
|
142
|
+
noun: 'EntityIdMapper',
|
|
140
143
|
nextId: this.nextId,
|
|
141
144
|
uuidToInt: Object.fromEntries(this.uuidToInt),
|
|
142
145
|
intToUuid: Object.fromEntries(this.intToUuid)
|
|
@@ -298,6 +298,7 @@ export class FieldTypeInference {
|
|
|
298
298
|
const cacheKey = `${this.CACHE_STORAGE_PREFIX}${field}`;
|
|
299
299
|
const data = await this.storage.getMetadata(cacheKey);
|
|
300
300
|
if (data) {
|
|
301
|
+
// v4.0.0: Double cast for type boundary crossing
|
|
301
302
|
return data;
|
|
302
303
|
}
|
|
303
304
|
}
|
|
@@ -313,8 +314,13 @@ export class FieldTypeInference {
|
|
|
313
314
|
// Save to in-memory cache
|
|
314
315
|
this.typeCache.set(field, typeInfo);
|
|
315
316
|
// Save to persistent storage (async, non-blocking)
|
|
317
|
+
// v4.0.0: Add required 'noun' property for NounMetadata
|
|
316
318
|
const cacheKey = `${this.CACHE_STORAGE_PREFIX}${field}`;
|
|
317
|
-
|
|
319
|
+
const metadataObj = {
|
|
320
|
+
noun: 'FieldTypeCache',
|
|
321
|
+
...typeInfo
|
|
322
|
+
};
|
|
323
|
+
await this.storage.saveMetadata(cacheKey, metadataObj).catch(error => {
|
|
318
324
|
prodLog.warn(`Failed to save field type cache for '${field}':`, error);
|
|
319
325
|
});
|
|
320
326
|
}
|
|
@@ -377,6 +383,7 @@ export class FieldTypeInference {
|
|
|
377
383
|
if (field) {
|
|
378
384
|
this.typeCache.delete(field);
|
|
379
385
|
const cacheKey = `${this.CACHE_STORAGE_PREFIX}${field}`;
|
|
386
|
+
// v4.0.0: null signals deletion to storage adapter
|
|
380
387
|
await this.storage.saveMetadata(cacheKey, null);
|
|
381
388
|
}
|
|
382
389
|
else {
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Filters DURING search to ensure relevant results
|
|
4
4
|
* Simple API that just works without configuration
|
|
5
5
|
*/
|
|
6
|
-
import { SearchResult,
|
|
6
|
+
import { SearchResult, HNSWNounWithMetadata } from '../coreTypes.js';
|
|
7
7
|
/**
|
|
8
8
|
* Brainy Field Operators (BFO) - Our own field query system
|
|
9
9
|
* Designed for performance, clarity, and patent independence
|
|
@@ -74,8 +74,9 @@ export declare function applyCompoundScoring<T>(results: SearchResult<T>[], filt
|
|
|
74
74
|
export declare function filterSearchResultsByMetadata<T>(results: SearchResult<T>[], filter: MetadataFilter): SearchResult<T>[];
|
|
75
75
|
/**
|
|
76
76
|
* Filter nouns by metadata before search
|
|
77
|
+
* v4.0.0: Takes HNSWNounWithMetadata which includes metadata field
|
|
77
78
|
*/
|
|
78
|
-
export declare function filterNounsByMetadata(nouns:
|
|
79
|
+
export declare function filterNounsByMetadata(nouns: HNSWNounWithMetadata[], filter: MetadataFilter): HNSWNounWithMetadata[];
|
|
79
80
|
/**
|
|
80
81
|
* Aggregate search results for faceted search
|
|
81
82
|
*/
|
|
@@ -237,6 +237,7 @@ export function filterSearchResultsByMetadata(results, filter) {
|
|
|
237
237
|
}
|
|
238
238
|
/**
|
|
239
239
|
* Filter nouns by metadata before search
|
|
240
|
+
* v4.0.0: Takes HNSWNounWithMetadata which includes metadata field
|
|
240
241
|
*/
|
|
241
242
|
export function filterNounsByMetadata(nouns, filter) {
|
|
242
243
|
if (!filter || Object.keys(filter).length === 0) {
|
|
@@ -1462,7 +1462,9 @@ export class MetadataIndexManager {
|
|
|
1462
1462
|
try {
|
|
1463
1463
|
const indexId = `__metadata_field_index__${filename}`;
|
|
1464
1464
|
const unifiedKey = `metadata:field:${filename}`;
|
|
1465
|
+
// v4.0.0: Add required 'noun' property for NounMetadata
|
|
1465
1466
|
await this.storage.saveMetadata(indexId, {
|
|
1467
|
+
noun: 'MetadataFieldIndex',
|
|
1466
1468
|
values: fieldIndex.values,
|
|
1467
1469
|
lastUpdated: fieldIndex.lastUpdated
|
|
1468
1470
|
});
|
|
@@ -460,11 +460,13 @@ export class ChunkManager {
|
|
|
460
460
|
const chunkPath = this.getChunkPath(field, chunkId);
|
|
461
461
|
const data = await this.storage.getMetadata(chunkPath);
|
|
462
462
|
if (data) {
|
|
463
|
+
// v4.0.0: Cast NounMetadata to chunk data structure
|
|
464
|
+
const chunkData = data;
|
|
463
465
|
// Deserialize: convert serialized roaring bitmaps back to RoaringBitmap32 objects
|
|
464
466
|
const chunk = {
|
|
465
|
-
chunkId:
|
|
466
|
-
field:
|
|
467
|
-
entries: new Map(Object.entries(
|
|
467
|
+
chunkId: chunkData.chunkId,
|
|
468
|
+
field: chunkData.field,
|
|
469
|
+
entries: new Map(Object.entries(chunkData.entries).map(([value, serializedBitmap]) => {
|
|
468
470
|
// Deserialize roaring bitmap from portable format
|
|
469
471
|
const bitmap = new RoaringBitmap32();
|
|
470
472
|
if (serializedBitmap && typeof serializedBitmap === 'object' && serializedBitmap.buffer) {
|
|
@@ -473,7 +475,7 @@ export class ChunkManager {
|
|
|
473
475
|
}
|
|
474
476
|
return [value, bitmap];
|
|
475
477
|
})),
|
|
476
|
-
lastUpdated:
|
|
478
|
+
lastUpdated: chunkData.lastUpdated
|
|
477
479
|
};
|
|
478
480
|
this.chunkCache.set(cacheKey, chunk);
|
|
479
481
|
return chunk;
|
|
@@ -492,7 +494,9 @@ export class ChunkManager {
|
|
|
492
494
|
// Update cache
|
|
493
495
|
this.chunkCache.set(cacheKey, chunk);
|
|
494
496
|
// Serialize: convert RoaringBitmap32 to portable format (Buffer)
|
|
497
|
+
// v4.0.0: Add required 'noun' property for NounMetadata
|
|
495
498
|
const serializable = {
|
|
499
|
+
noun: 'IndexChunk', // Required by NounMetadata interface
|
|
496
500
|
chunkId: chunk.chunkId,
|
|
497
501
|
field: chunk.field,
|
|
498
502
|
entries: Object.fromEntries(Array.from(chunk.entries.entries()).map(([value, bitmap]) => [
|
|
@@ -652,6 +656,7 @@ export class ChunkManager {
|
|
|
652
656
|
const cacheKey = `${field}:${chunkId}`;
|
|
653
657
|
this.chunkCache.delete(cacheKey);
|
|
654
658
|
const chunkPath = this.getChunkPath(field, chunkId);
|
|
659
|
+
// v4.0.0: null signals deletion to storage adapter
|
|
655
660
|
await this.storage.saveMetadata(chunkPath, null);
|
|
656
661
|
}
|
|
657
662
|
/**
|
|
@@ -153,6 +153,7 @@ export class PeriodicCleanup {
|
|
|
153
153
|
});
|
|
154
154
|
for (const noun of nounsResult.items) {
|
|
155
155
|
try {
|
|
156
|
+
// v4.0.0: Cast NounMetadata to NamespacedMetadata for isDeleted check
|
|
156
157
|
if (!noun.metadata || !isDeleted(noun.metadata)) {
|
|
157
158
|
continue; // Not deleted, skip
|
|
158
159
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@soulcraft/brainy",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "4.0.0",
|
|
4
4
|
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.js",
|
|
@@ -163,6 +163,8 @@
|
|
|
163
163
|
},
|
|
164
164
|
"dependencies": {
|
|
165
165
|
"@aws-sdk/client-s3": "^3.540.0",
|
|
166
|
+
"@azure/identity": "^4.0.0",
|
|
167
|
+
"@azure/storage-blob": "^12.17.0",
|
|
166
168
|
"@google-cloud/storage": "^7.14.0",
|
|
167
169
|
"@huggingface/transformers": "^3.7.2",
|
|
168
170
|
"@msgpack/msgpack": "^3.1.2",
|