@soulcraft/brainy 3.50.2 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/CHANGELOG.md +201 -0
  2. package/README.md +358 -658
  3. package/dist/api/ConfigAPI.js +56 -19
  4. package/dist/api/DataAPI.js +24 -18
  5. package/dist/augmentations/storageAugmentations.d.ts +24 -0
  6. package/dist/augmentations/storageAugmentations.js +22 -0
  7. package/dist/brainy.js +32 -9
  8. package/dist/cli/commands/core.d.ts +20 -10
  9. package/dist/cli/commands/core.js +384 -82
  10. package/dist/cli/commands/import.d.ts +41 -0
  11. package/dist/cli/commands/import.js +456 -0
  12. package/dist/cli/commands/insights.d.ts +34 -0
  13. package/dist/cli/commands/insights.js +300 -0
  14. package/dist/cli/commands/neural.d.ts +6 -12
  15. package/dist/cli/commands/neural.js +113 -10
  16. package/dist/cli/commands/nlp.d.ts +28 -0
  17. package/dist/cli/commands/nlp.js +246 -0
  18. package/dist/cli/commands/storage.d.ts +64 -0
  19. package/dist/cli/commands/storage.js +730 -0
  20. package/dist/cli/index.js +210 -24
  21. package/dist/coreTypes.d.ts +206 -34
  22. package/dist/distributed/configManager.js +8 -6
  23. package/dist/distributed/shardMigration.js +2 -0
  24. package/dist/distributed/storageDiscovery.js +6 -4
  25. package/dist/embeddings/EmbeddingManager.d.ts +2 -2
  26. package/dist/embeddings/EmbeddingManager.js +5 -1
  27. package/dist/graph/lsm/LSMTree.js +32 -20
  28. package/dist/hnsw/typeAwareHNSWIndex.js +6 -2
  29. package/dist/storage/adapters/azureBlobStorage.d.ts +545 -0
  30. package/dist/storage/adapters/azureBlobStorage.js +1809 -0
  31. package/dist/storage/adapters/baseStorageAdapter.d.ts +16 -13
  32. package/dist/storage/adapters/fileSystemStorage.d.ts +21 -9
  33. package/dist/storage/adapters/fileSystemStorage.js +204 -127
  34. package/dist/storage/adapters/gcsStorage.d.ts +119 -9
  35. package/dist/storage/adapters/gcsStorage.js +317 -62
  36. package/dist/storage/adapters/memoryStorage.d.ts +30 -18
  37. package/dist/storage/adapters/memoryStorage.js +99 -94
  38. package/dist/storage/adapters/opfsStorage.d.ts +48 -10
  39. package/dist/storage/adapters/opfsStorage.js +201 -80
  40. package/dist/storage/adapters/r2Storage.d.ts +12 -5
  41. package/dist/storage/adapters/r2Storage.js +63 -15
  42. package/dist/storage/adapters/s3CompatibleStorage.d.ts +164 -17
  43. package/dist/storage/adapters/s3CompatibleStorage.js +472 -80
  44. package/dist/storage/adapters/typeAwareStorageAdapter.d.ts +38 -6
  45. package/dist/storage/adapters/typeAwareStorageAdapter.js +218 -39
  46. package/dist/storage/baseStorage.d.ts +41 -38
  47. package/dist/storage/baseStorage.js +110 -134
  48. package/dist/storage/storageFactory.d.ts +29 -2
  49. package/dist/storage/storageFactory.js +30 -1
  50. package/dist/utils/entityIdMapper.js +5 -2
  51. package/dist/utils/fieldTypeInference.js +8 -1
  52. package/dist/utils/metadataFilter.d.ts +3 -2
  53. package/dist/utils/metadataFilter.js +1 -0
  54. package/dist/utils/metadataIndex.js +2 -0
  55. package/dist/utils/metadataIndexChunking.js +9 -4
  56. package/dist/utils/periodicCleanup.js +1 -0
  57. package/package.json +3 -1
@@ -138,45 +138,41 @@ export class BaseStorage extends BaseStorageAdapter {
138
138
  }
139
139
  }
140
140
  /**
141
- * Save a noun to storage
141
+ * Save a noun to storage (v4.0.0: vector only, metadata saved separately)
142
+ * @param noun Pure HNSW vector data (no metadata)
142
143
  */
143
144
  async saveNoun(noun) {
144
145
  await this.ensureInitialized();
145
- // Validate noun type before saving - storage boundary protection
146
- if (noun.metadata?.noun) {
147
- validateNounType(noun.metadata.noun);
148
- }
149
- // Save both the HNSWNoun vector data and metadata separately (2-file system)
150
- try {
151
- // Save the lightweight HNSWNoun vector file first
152
- await this.saveNoun_internal(noun);
153
- // Then save the metadata to separate file (if present)
154
- if (noun.metadata) {
155
- await this.saveNounMetadata(noun.id, noun.metadata);
156
- }
157
- }
158
- catch (error) {
159
- console.error(`[ERROR] Failed to save noun ${noun.id}:`, error);
160
- // Attempt cleanup - remove noun file if metadata failed
161
- try {
162
- const nounExists = await this.getNoun_internal(noun.id);
163
- if (nounExists) {
164
- console.log(`[CLEANUP] Attempting to remove orphaned noun file ${noun.id}`);
165
- await this.deleteNoun_internal(noun.id);
166
- }
167
- }
168
- catch (cleanupError) {
169
- console.error(`[ERROR] Failed to cleanup orphaned noun ${noun.id}:`, cleanupError);
170
- }
171
- throw new Error(`Failed to save noun ${noun.id}: ${error instanceof Error ? error.message : String(error)}`);
172
- }
146
+ // Save the HNSWNoun vector data only
147
+ // Metadata must be saved separately via saveNounMetadata()
148
+ await this.saveNoun_internal(noun);
173
149
  }
174
150
  /**
175
- * Get a noun from storage
151
+ * Get a noun from storage (v4.0.0: returns combined HNSWNounWithMetadata)
152
+ * @param id Entity ID
153
+ * @returns Combined vector + metadata or null
176
154
  */
177
155
  async getNoun(id) {
178
156
  await this.ensureInitialized();
179
- return this.getNoun_internal(id);
157
+ // Load vector and metadata separately
158
+ const vector = await this.getNoun_internal(id);
159
+ if (!vector) {
160
+ return null;
161
+ }
162
+ // Load metadata
163
+ const metadata = await this.getNounMetadata(id);
164
+ if (!metadata) {
165
+ console.warn(`[Storage] Noun ${id} has vector but no metadata - this should not happen in v4.0.0`);
166
+ return null;
167
+ }
168
+ // Combine into HNSWNounWithMetadata
169
+ return {
170
+ id: vector.id,
171
+ vector: vector.vector,
172
+ connections: vector.connections,
173
+ level: vector.level,
174
+ metadata
175
+ };
180
176
  }
181
177
  /**
182
178
  * Get nouns by noun type
@@ -185,7 +181,20 @@ export class BaseStorage extends BaseStorageAdapter {
185
181
  */
186
182
  async getNounsByNounType(nounType) {
187
183
  await this.ensureInitialized();
188
- return this.getNounsByNounType_internal(nounType);
184
+ // Internal method returns HNSWNoun[], need to combine with metadata
185
+ const nouns = await this.getNounsByNounType_internal(nounType);
186
+ // Combine each noun with its metadata
187
+ const nounsWithMetadata = [];
188
+ for (const noun of nouns) {
189
+ const metadata = await this.getNounMetadata(noun.id);
190
+ if (metadata) {
191
+ nounsWithMetadata.push({
192
+ ...noun,
193
+ metadata
194
+ });
195
+ }
196
+ }
197
+ return nounsWithMetadata;
189
198
  }
190
199
  /**
191
200
  * Delete a noun from storage
@@ -204,90 +213,58 @@ export class BaseStorage extends BaseStorageAdapter {
204
213
  }
205
214
  }
206
215
  /**
207
- * Save a verb to storage
216
+ * Save a verb to storage (v4.0.0: verb only, metadata saved separately)
208
217
  *
209
- * ARCHITECTURAL FIX (v3.50.1): HNSWVerb now includes verb/sourceId/targetId
210
- * These are core relational fields, not metadata. They're stored in the vector
211
- * file for fast access and to align with actual usage patterns.
218
+ * @param verb Pure HNSW verb with core relational fields (verb, sourceId, targetId)
212
219
  */
213
220
  async saveVerb(verb) {
214
221
  await this.ensureInitialized();
215
222
  // Validate verb type before saving - storage boundary protection
216
- if (verb.verb) {
217
- validateVerbType(verb.verb);
218
- }
219
- // Extract HNSWVerb with CORE relational fields included
220
- const hnswVerb = {
221
- id: verb.id,
222
- vector: verb.vector,
223
- connections: verb.connections || new Map(),
224
- // CORE RELATIONAL DATA (v3.50.1+)
225
- verb: (verb.verb || verb.type || 'relatedTo'),
226
- sourceId: verb.sourceId || verb.source || '',
227
- targetId: verb.targetId || verb.target || '',
228
- // User metadata (if any)
229
- metadata: verb.metadata
230
- };
231
- // Extract lightweight metadata for separate file (optional fields only)
232
- const metadata = {
233
- weight: verb.weight,
234
- data: verb.data,
235
- createdAt: verb.createdAt,
236
- updatedAt: verb.updatedAt,
237
- createdBy: verb.createdBy,
238
- // Legacy aliases for backward compatibility
239
- source: verb.source || verb.sourceId,
240
- target: verb.target || verb.targetId,
241
- type: verb.type || verb.verb
242
- };
243
- // Save both the HNSWVerb and metadata atomically
244
- try {
245
- console.log(`[DEBUG] Saving verb ${verb.id}: sourceId=${verb.sourceId}, targetId=${verb.targetId}`);
246
- // Save the HNSWVerb first
247
- await this.saveVerb_internal(hnswVerb);
248
- console.log(`[DEBUG] Successfully saved HNSWVerb file for ${verb.id}`);
249
- // Then save the metadata
250
- await this.saveVerbMetadata(verb.id, metadata);
251
- console.log(`[DEBUG] Successfully saved metadata file for ${verb.id}`);
252
- }
253
- catch (error) {
254
- console.error(`[ERROR] Failed to save verb ${verb.id}:`, error);
255
- // Attempt cleanup - remove verb file if metadata failed
256
- try {
257
- const verbExists = await this.getVerb_internal(verb.id);
258
- if (verbExists) {
259
- console.log(`[CLEANUP] Attempting to remove orphaned verb file ${verb.id}`);
260
- await this.deleteVerb_internal(verb.id);
261
- }
262
- }
263
- catch (cleanupError) {
264
- console.error(`[ERROR] Failed to cleanup orphaned verb ${verb.id}:`, cleanupError);
265
- }
266
- throw new Error(`Failed to save verb ${verb.id}: ${error instanceof Error ? error.message : String(error)}`);
267
- }
223
+ validateVerbType(verb.verb);
224
+ // Save the HNSWVerb vector and core fields only
225
+ // Metadata must be saved separately via saveVerbMetadata()
226
+ await this.saveVerb_internal(verb);
268
227
  }
269
228
  /**
270
- * Get a verb from storage
229
+ * Get a verb from storage (v4.0.0: returns combined HNSWVerbWithMetadata)
230
+ * @param id Entity ID
231
+ * @returns Combined verb + metadata or null
271
232
  */
272
233
  async getVerb(id) {
273
234
  await this.ensureInitialized();
274
- const hnswVerb = await this.getVerb_internal(id);
275
- if (!hnswVerb) {
235
+ // Load verb vector and core fields
236
+ const verb = await this.getVerb_internal(id);
237
+ if (!verb) {
276
238
  return null;
277
239
  }
278
- return this.convertHNSWVerbToGraphVerb(hnswVerb);
240
+ // Load metadata
241
+ const metadata = await this.getVerbMetadata(id);
242
+ if (!metadata) {
243
+ console.warn(`[Storage] Verb ${id} has vector but no metadata - this should not happen in v4.0.0`);
244
+ return null;
245
+ }
246
+ // Combine into HNSWVerbWithMetadata
247
+ return {
248
+ id: verb.id,
249
+ vector: verb.vector,
250
+ connections: verb.connections,
251
+ verb: verb.verb,
252
+ sourceId: verb.sourceId,
253
+ targetId: verb.targetId,
254
+ metadata
255
+ };
279
256
  }
280
257
  /**
281
258
  * Convert HNSWVerb to GraphVerb by combining with metadata
259
+ * DEPRECATED: For backward compatibility only. Use getVerb() which returns HNSWVerbWithMetadata.
282
260
  *
283
- * ARCHITECTURAL FIX (v3.50.1): Core fields (verb/sourceId/targetId) are now in HNSWVerb
284
- * Only optional fields (weight, timestamps, etc.) come from metadata file
261
+ * @deprecated Use getVerb() instead which returns HNSWVerbWithMetadata
285
262
  */
286
263
  async convertHNSWVerbToGraphVerb(hnswVerb) {
287
264
  try {
288
- // Metadata file is now optional - contains only weight, timestamps, etc.
265
+ // Load metadata
289
266
  const metadata = await this.getVerbMetadata(hnswVerb.id);
290
- // Create default timestamp if not present
267
+ // Create default timestamp in Firestore format
291
268
  const defaultTimestamp = {
292
269
  seconds: Math.floor(Date.now() / 1000),
293
270
  nanoseconds: (Date.now() % 1000) * 1000000
@@ -297,10 +274,22 @@ export class BaseStorage extends BaseStorageAdapter {
297
274
  augmentation: 'unknown',
298
275
  version: '1.0'
299
276
  };
277
+ // Convert flexible timestamp to Firestore format for GraphVerb
278
+ const normalizeTimestamp = (ts) => {
279
+ if (!ts)
280
+ return defaultTimestamp;
281
+ if (typeof ts === 'number') {
282
+ return {
283
+ seconds: Math.floor(ts / 1000),
284
+ nanoseconds: (ts % 1000) * 1000000
285
+ };
286
+ }
287
+ return ts;
288
+ };
300
289
  return {
301
290
  id: hnswVerb.id,
302
291
  vector: hnswVerb.vector,
303
- // CORE FIELDS from HNSWVerb (v3.50.1+)
292
+ // CORE FIELDS from HNSWVerb
304
293
  verb: hnswVerb.verb,
305
294
  sourceId: hnswVerb.sourceId,
306
295
  targetId: hnswVerb.targetId,
@@ -310,9 +299,9 @@ export class BaseStorage extends BaseStorageAdapter {
310
299
  target: hnswVerb.targetId,
311
300
  // Optional fields from metadata file
312
301
  weight: metadata?.weight || 1.0,
313
- metadata: hnswVerb.metadata || {},
314
- createdAt: metadata?.createdAt || defaultTimestamp,
315
- updatedAt: metadata?.updatedAt || defaultTimestamp,
302
+ metadata: metadata || {},
303
+ createdAt: normalizeTimestamp(metadata?.createdAt),
304
+ updatedAt: normalizeTimestamp(metadata?.updatedAt),
316
305
  createdBy: metadata?.createdBy || defaultCreatedBy,
317
306
  data: metadata?.data,
318
307
  embedding: hnswVerb.vector
@@ -333,23 +322,15 @@ export class BaseStorage extends BaseStorageAdapter {
333
322
  const result = await this.getVerbs({
334
323
  pagination: { limit: Number.MAX_SAFE_INTEGER }
335
324
  });
336
- // Convert GraphVerbs back to HNSWVerbs for internal use
337
- // ARCHITECTURAL FIX (v3.50.1): Include core relational fields
338
- const hnswVerbs = [];
339
- for (const graphVerb of result.items) {
340
- const hnswVerb = {
341
- id: graphVerb.id,
342
- vector: graphVerb.vector,
343
- connections: new Map(),
344
- // CORE RELATIONAL DATA
345
- verb: (graphVerb.verb || graphVerb.type || 'relatedTo'),
346
- sourceId: graphVerb.sourceId || graphVerb.source || '',
347
- targetId: graphVerb.targetId || graphVerb.target || '',
348
- // User metadata
349
- metadata: graphVerb.metadata
350
- };
351
- hnswVerbs.push(hnswVerb);
352
- }
325
+ // v4.0.0: Convert HNSWVerbWithMetadata to HNSWVerb (strip metadata)
326
+ const hnswVerbs = result.items.map(verbWithMetadata => ({
327
+ id: verbWithMetadata.id,
328
+ vector: verbWithMetadata.vector,
329
+ connections: verbWithMetadata.connections,
330
+ verb: verbWithMetadata.verb,
331
+ sourceId: verbWithMetadata.sourceId,
332
+ targetId: verbWithMetadata.targetId
333
+ }));
353
334
  return hnswVerbs;
354
335
  }
355
336
  /**
@@ -423,8 +404,8 @@ export class BaseStorage extends BaseStorageAdapter {
423
404
  const nounType = Array.isArray(options.filter.nounType)
424
405
  ? options.filter.nounType[0]
425
406
  : options.filter.nounType;
426
- // Get nouns by type directly
427
- const nounsByType = await this.getNounsByNounType_internal(nounType);
407
+ // Get nouns by type directly (already combines with metadata)
408
+ const nounsByType = await this.getNounsByNounType(nounType);
428
409
  // Apply pagination
429
410
  const paginatedNouns = nounsByType.slice(offset, offset + limit);
430
411
  const hasMore = offset + limit < nounsByType.length;
@@ -696,7 +677,7 @@ export class BaseStorage extends BaseStorageAdapter {
696
677
  return this.graphIndex;
697
678
  }
698
679
  /**
699
- * Save metadata to storage
680
+ * Save metadata to storage (v4.0.0: now typed)
700
681
  * Routes to correct location (system or entity) based on key format
701
682
  */
702
683
  async saveMetadata(id, metadata) {
@@ -705,7 +686,7 @@ export class BaseStorage extends BaseStorageAdapter {
705
686
  return this.writeObjectToPath(keyInfo.fullPath, metadata);
706
687
  }
707
688
  /**
708
- * Get metadata from storage
689
+ * Get metadata from storage (v4.0.0: now typed)
709
690
  * Routes to correct location (system or entity) based on key format
710
691
  */
711
692
  async getMetadata(id) {
@@ -714,18 +695,16 @@ export class BaseStorage extends BaseStorageAdapter {
714
695
  return this.readObjectFromPath(keyInfo.fullPath);
715
696
  }
716
697
  /**
717
- * Save noun metadata to storage
698
+ * Save noun metadata to storage (v4.0.0: now typed)
718
699
  * Routes to correct sharded location based on UUID
719
700
  */
720
701
  async saveNounMetadata(id, metadata) {
721
702
  // Validate noun type in metadata - storage boundary protection
722
- if (metadata?.noun) {
723
- validateNounType(metadata.noun);
724
- }
703
+ validateNounType(metadata.noun);
725
704
  return this.saveNounMetadata_internal(id, metadata);
726
705
  }
727
706
  /**
728
- * Internal method for saving noun metadata
707
+ * Internal method for saving noun metadata (v4.0.0: now typed)
729
708
  * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
730
709
  * @protected
731
710
  */
@@ -735,7 +714,7 @@ export class BaseStorage extends BaseStorageAdapter {
735
714
  return this.writeObjectToPath(keyInfo.fullPath, metadata);
736
715
  }
737
716
  /**
738
- * Get noun metadata from storage
717
+ * Get noun metadata from storage (v4.0.0: now typed)
739
718
  * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
740
719
  */
741
720
  async getNounMetadata(id) {
@@ -753,18 +732,15 @@ export class BaseStorage extends BaseStorageAdapter {
753
732
  return this.deleteObjectFromPath(keyInfo.fullPath);
754
733
  }
755
734
  /**
756
- * Save verb metadata to storage
735
+ * Save verb metadata to storage (v4.0.0: now typed)
757
736
  * Routes to correct sharded location based on UUID
758
737
  */
759
738
  async saveVerbMetadata(id, metadata) {
760
- // Validate verb type in metadata - storage boundary protection
761
- if (metadata?.verb) {
762
- validateVerbType(metadata.verb);
763
- }
739
+ // Note: verb type is in HNSWVerb, not metadata
764
740
  return this.saveVerbMetadata_internal(id, metadata);
765
741
  }
766
742
  /**
767
- * Internal method for saving verb metadata
743
+ * Internal method for saving verb metadata (v4.0.0: now typed)
768
744
  * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
769
745
  * @protected
770
746
  */
@@ -774,7 +750,7 @@ export class BaseStorage extends BaseStorageAdapter {
774
750
  return this.writeObjectToPath(keyInfo.fullPath, metadata);
775
751
  }
776
752
  /**
777
- * Get verb metadata from storage
753
+ * Get verb metadata from storage (v4.0.0: now typed)
778
754
  * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded)
779
755
  */
780
756
  async getVerbMetadata(id) {
@@ -8,6 +8,7 @@ import { OPFSStorage } from './adapters/opfsStorage.js';
8
8
  import { S3CompatibleStorage } from './adapters/s3CompatibleStorage.js';
9
9
  import { R2Storage } from './adapters/r2Storage.js';
10
10
  import { GcsStorage } from './adapters/gcsStorage.js';
11
+ import { AzureBlobStorage } from './adapters/azureBlobStorage.js';
11
12
  import { TypeAwareStorageAdapter } from './adapters/typeAwareStorageAdapter.js';
12
13
  import { OperationConfig } from '../utils/operationUtils.js';
13
14
  /**
@@ -24,9 +25,10 @@ export interface StorageOptions {
24
25
  * - 'r2': Use Cloudflare R2 storage
25
26
  * - 'gcs': Use Google Cloud Storage (S3-compatible with HMAC keys)
26
27
  * - 'gcs-native': Use Google Cloud Storage (native SDK with ADC)
28
+ * - 'azure': Use Azure Blob Storage (native SDK with Managed Identity)
27
29
  * - 'type-aware': Use type-first storage adapter (wraps another adapter)
28
30
  */
29
- type?: 'auto' | 'memory' | 'opfs' | 'filesystem' | 's3' | 'r2' | 'gcs' | 'gcs-native' | 'type-aware';
31
+ type?: 'auto' | 'memory' | 'opfs' | 'filesystem' | 's3' | 'r2' | 'gcs' | 'gcs-native' | 'azure' | 'type-aware';
30
32
  /**
31
33
  * Force the use of memory storage even if other storage types are available
32
34
  */
@@ -148,6 +150,31 @@ export interface StorageOptions {
148
150
  */
149
151
  secretAccessKey?: string;
150
152
  };
153
+ /**
154
+ * Configuration for Azure Blob Storage (native SDK with Managed Identity)
155
+ */
156
+ azureStorage?: {
157
+ /**
158
+ * Azure container name
159
+ */
160
+ containerName: string;
161
+ /**
162
+ * Azure Storage account name (for Managed Identity or SAS)
163
+ */
164
+ accountName?: string;
165
+ /**
166
+ * Azure Storage account key (optional, uses Managed Identity if not provided)
167
+ */
168
+ accountKey?: string;
169
+ /**
170
+ * Azure connection string (highest priority if provided)
171
+ */
172
+ connectionString?: string;
173
+ /**
174
+ * SAS token (optional, alternative to account key)
175
+ */
176
+ sasToken?: string;
177
+ };
151
178
  /**
152
179
  * Configuration for Type-Aware Storage (type-first architecture)
153
180
  * Wraps another storage adapter and adds type-first routing
@@ -254,4 +281,4 @@ export declare function createStorage(options?: StorageOptions): Promise<Storage
254
281
  /**
255
282
  * Export storage adapters
256
283
  */
257
- export { MemoryStorage, OPFSStorage, S3CompatibleStorage, R2Storage, GcsStorage, TypeAwareStorageAdapter };
284
+ export { MemoryStorage, OPFSStorage, S3CompatibleStorage, R2Storage, GcsStorage, AzureBlobStorage, TypeAwareStorageAdapter };
@@ -7,6 +7,7 @@ import { OPFSStorage } from './adapters/opfsStorage.js';
7
7
  import { S3CompatibleStorage } from './adapters/s3CompatibleStorage.js';
8
8
  import { R2Storage } from './adapters/r2Storage.js';
9
9
  import { GcsStorage } from './adapters/gcsStorage.js';
10
+ import { AzureBlobStorage } from './adapters/azureBlobStorage.js';
10
11
  import { TypeAwareStorageAdapter } from './adapters/typeAwareStorageAdapter.js';
11
12
  // FileSystemStorage is dynamically imported to avoid issues in browser environments
12
13
  import { isBrowser } from '../utils/environment.js';
@@ -157,6 +158,22 @@ export async function createStorage(options = {}) {
157
158
  console.warn('GCS native storage configuration is missing, falling back to memory storage');
158
159
  return new MemoryStorage();
159
160
  }
161
+ case 'azure':
162
+ if (options.azureStorage) {
163
+ console.log('Using Azure Blob Storage (native SDK)');
164
+ return new AzureBlobStorage({
165
+ containerName: options.azureStorage.containerName,
166
+ accountName: options.azureStorage.accountName,
167
+ accountKey: options.azureStorage.accountKey,
168
+ connectionString: options.azureStorage.connectionString,
169
+ sasToken: options.azureStorage.sasToken,
170
+ cacheConfig: options.cacheConfig
171
+ });
172
+ }
173
+ else {
174
+ console.warn('Azure storage configuration is missing, falling back to memory storage');
175
+ return new MemoryStorage();
176
+ }
160
177
  case 'type-aware': {
161
178
  console.log('Using Type-Aware Storage (type-first architecture)');
162
179
  // Create underlying storage adapter
@@ -241,6 +258,18 @@ export async function createStorage(options = {}) {
241
258
  cacheConfig: options.cacheConfig
242
259
  });
243
260
  }
261
+ // If Azure storage is specified, use it
262
+ if (options.azureStorage) {
263
+ console.log('Using Azure Blob Storage (native SDK)');
264
+ return new AzureBlobStorage({
265
+ containerName: options.azureStorage.containerName,
266
+ accountName: options.azureStorage.accountName,
267
+ accountKey: options.azureStorage.accountKey,
268
+ connectionString: options.azureStorage.connectionString,
269
+ sasToken: options.azureStorage.sasToken,
270
+ cacheConfig: options.cacheConfig
271
+ });
272
+ }
244
273
  // Auto-detect the best storage adapter based on the environment
245
274
  // First, check if we're in Node.js (prioritize for test environments)
246
275
  if (!isBrowser()) {
@@ -286,7 +315,7 @@ export async function createStorage(options = {}) {
286
315
  /**
287
316
  * Export storage adapters
288
317
  */
289
- export { MemoryStorage, OPFSStorage, S3CompatibleStorage, R2Storage, GcsStorage, TypeAwareStorageAdapter };
318
+ export { MemoryStorage, OPFSStorage, S3CompatibleStorage, R2Storage, GcsStorage, AzureBlobStorage, TypeAwareStorageAdapter };
290
319
  // Export FileSystemStorage conditionally
291
320
  // NOTE: FileSystemStorage is now only imported dynamically to avoid fs imports in browser builds
292
321
  // export { FileSystemStorage } from './adapters/fileSystemStorage.js'
@@ -32,8 +32,9 @@ export class EntityIdMapper {
32
32
  */
33
33
  async init() {
34
34
  try {
35
- const data = await this.storage.getMetadata(this.storageKey);
36
- if (data) {
35
+ const metadata = await this.storage.getMetadata(this.storageKey);
36
+ if (metadata && metadata.data) {
37
+ const data = metadata.data;
37
38
  this.nextId = data.nextId;
38
39
  // Rebuild maps from serialized data
39
40
  this.uuidToInt = new Map(Object.entries(data.uuidToInt).map(([k, v]) => [k, Number(v)]));
@@ -136,7 +137,9 @@ export class EntityIdMapper {
136
137
  return;
137
138
  }
138
139
  // Convert maps to plain objects for serialization
140
+ // v4.0.0: Add required 'noun' property for NounMetadata
139
141
  const data = {
142
+ noun: 'EntityIdMapper',
140
143
  nextId: this.nextId,
141
144
  uuidToInt: Object.fromEntries(this.uuidToInt),
142
145
  intToUuid: Object.fromEntries(this.intToUuid)
@@ -298,6 +298,7 @@ export class FieldTypeInference {
298
298
  const cacheKey = `${this.CACHE_STORAGE_PREFIX}${field}`;
299
299
  const data = await this.storage.getMetadata(cacheKey);
300
300
  if (data) {
301
+ // v4.0.0: Double cast for type boundary crossing
301
302
  return data;
302
303
  }
303
304
  }
@@ -313,8 +314,13 @@ export class FieldTypeInference {
313
314
  // Save to in-memory cache
314
315
  this.typeCache.set(field, typeInfo);
315
316
  // Save to persistent storage (async, non-blocking)
317
+ // v4.0.0: Add required 'noun' property for NounMetadata
316
318
  const cacheKey = `${this.CACHE_STORAGE_PREFIX}${field}`;
317
- await this.storage.saveMetadata(cacheKey, typeInfo).catch(error => {
319
+ const metadataObj = {
320
+ noun: 'FieldTypeCache',
321
+ ...typeInfo
322
+ };
323
+ await this.storage.saveMetadata(cacheKey, metadataObj).catch(error => {
318
324
  prodLog.warn(`Failed to save field type cache for '${field}':`, error);
319
325
  });
320
326
  }
@@ -377,6 +383,7 @@ export class FieldTypeInference {
377
383
  if (field) {
378
384
  this.typeCache.delete(field);
379
385
  const cacheKey = `${this.CACHE_STORAGE_PREFIX}${field}`;
386
+ // v4.0.0: null signals deletion to storage adapter
380
387
  await this.storage.saveMetadata(cacheKey, null);
381
388
  }
382
389
  else {
@@ -3,7 +3,7 @@
3
3
  * Filters DURING search to ensure relevant results
4
4
  * Simple API that just works without configuration
5
5
  */
6
- import { SearchResult, HNSWNoun } from '../coreTypes.js';
6
+ import { SearchResult, HNSWNounWithMetadata } from '../coreTypes.js';
7
7
  /**
8
8
  * Brainy Field Operators (BFO) - Our own field query system
9
9
  * Designed for performance, clarity, and patent independence
@@ -74,8 +74,9 @@ export declare function applyCompoundScoring<T>(results: SearchResult<T>[], filt
74
74
  export declare function filterSearchResultsByMetadata<T>(results: SearchResult<T>[], filter: MetadataFilter): SearchResult<T>[];
75
75
  /**
76
76
  * Filter nouns by metadata before search
77
+ * v4.0.0: Takes HNSWNounWithMetadata which includes metadata field
77
78
  */
78
- export declare function filterNounsByMetadata(nouns: HNSWNoun[], filter: MetadataFilter): HNSWNoun[];
79
+ export declare function filterNounsByMetadata(nouns: HNSWNounWithMetadata[], filter: MetadataFilter): HNSWNounWithMetadata[];
79
80
  /**
80
81
  * Aggregate search results for faceted search
81
82
  */
@@ -237,6 +237,7 @@ export function filterSearchResultsByMetadata(results, filter) {
237
237
  }
238
238
  /**
239
239
  * Filter nouns by metadata before search
240
+ * v4.0.0: Takes HNSWNounWithMetadata which includes metadata field
240
241
  */
241
242
  export function filterNounsByMetadata(nouns, filter) {
242
243
  if (!filter || Object.keys(filter).length === 0) {
@@ -1462,7 +1462,9 @@ export class MetadataIndexManager {
1462
1462
  try {
1463
1463
  const indexId = `__metadata_field_index__${filename}`;
1464
1464
  const unifiedKey = `metadata:field:${filename}`;
1465
+ // v4.0.0: Add required 'noun' property for NounMetadata
1465
1466
  await this.storage.saveMetadata(indexId, {
1467
+ noun: 'MetadataFieldIndex',
1466
1468
  values: fieldIndex.values,
1467
1469
  lastUpdated: fieldIndex.lastUpdated
1468
1470
  });
@@ -460,11 +460,13 @@ export class ChunkManager {
460
460
  const chunkPath = this.getChunkPath(field, chunkId);
461
461
  const data = await this.storage.getMetadata(chunkPath);
462
462
  if (data) {
463
+ // v4.0.0: Cast NounMetadata to chunk data structure
464
+ const chunkData = data;
463
465
  // Deserialize: convert serialized roaring bitmaps back to RoaringBitmap32 objects
464
466
  const chunk = {
465
- chunkId: data.chunkId,
466
- field: data.field,
467
- entries: new Map(Object.entries(data.entries).map(([value, serializedBitmap]) => {
467
+ chunkId: chunkData.chunkId,
468
+ field: chunkData.field,
469
+ entries: new Map(Object.entries(chunkData.entries).map(([value, serializedBitmap]) => {
468
470
  // Deserialize roaring bitmap from portable format
469
471
  const bitmap = new RoaringBitmap32();
470
472
  if (serializedBitmap && typeof serializedBitmap === 'object' && serializedBitmap.buffer) {
@@ -473,7 +475,7 @@ export class ChunkManager {
473
475
  }
474
476
  return [value, bitmap];
475
477
  })),
476
- lastUpdated: data.lastUpdated
478
+ lastUpdated: chunkData.lastUpdated
477
479
  };
478
480
  this.chunkCache.set(cacheKey, chunk);
479
481
  return chunk;
@@ -492,7 +494,9 @@ export class ChunkManager {
492
494
  // Update cache
493
495
  this.chunkCache.set(cacheKey, chunk);
494
496
  // Serialize: convert RoaringBitmap32 to portable format (Buffer)
497
+ // v4.0.0: Add required 'noun' property for NounMetadata
495
498
  const serializable = {
499
+ noun: 'IndexChunk', // Required by NounMetadata interface
496
500
  chunkId: chunk.chunkId,
497
501
  field: chunk.field,
498
502
  entries: Object.fromEntries(Array.from(chunk.entries.entries()).map(([value, bitmap]) => [
@@ -652,6 +656,7 @@ export class ChunkManager {
652
656
  const cacheKey = `${field}:${chunkId}`;
653
657
  this.chunkCache.delete(cacheKey);
654
658
  const chunkPath = this.getChunkPath(field, chunkId);
659
+ // v4.0.0: null signals deletion to storage adapter
655
660
  await this.storage.saveMetadata(chunkPath, null);
656
661
  }
657
662
  /**
@@ -153,6 +153,7 @@ export class PeriodicCleanup {
153
153
  });
154
154
  for (const noun of nounsResult.items) {
155
155
  try {
156
+ // v4.0.0: Cast NounMetadata to NamespacedMetadata for isDeleted check
156
157
  if (!noun.metadata || !isDeleted(noun.metadata)) {
157
158
  continue; // Not deleted, skip
158
159
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@soulcraft/brainy",
3
- "version": "3.50.2",
3
+ "version": "4.0.0",
4
4
  "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
5
5
  "main": "dist/index.js",
6
6
  "module": "dist/index.js",
@@ -163,6 +163,8 @@
163
163
  },
164
164
  "dependencies": {
165
165
  "@aws-sdk/client-s3": "^3.540.0",
166
+ "@azure/identity": "^4.0.0",
167
+ "@azure/storage-blob": "^12.17.0",
166
168
  "@google-cloud/storage": "^7.14.0",
167
169
  "@huggingface/transformers": "^3.7.2",
168
170
  "@msgpack/msgpack": "^3.1.2",