@soulcraft/brainy 3.50.2 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/CHANGELOG.md +201 -0
  2. package/README.md +358 -658
  3. package/dist/api/ConfigAPI.js +56 -19
  4. package/dist/api/DataAPI.js +24 -18
  5. package/dist/augmentations/storageAugmentations.d.ts +24 -0
  6. package/dist/augmentations/storageAugmentations.js +22 -0
  7. package/dist/brainy.js +32 -9
  8. package/dist/cli/commands/core.d.ts +20 -10
  9. package/dist/cli/commands/core.js +384 -82
  10. package/dist/cli/commands/import.d.ts +41 -0
  11. package/dist/cli/commands/import.js +456 -0
  12. package/dist/cli/commands/insights.d.ts +34 -0
  13. package/dist/cli/commands/insights.js +300 -0
  14. package/dist/cli/commands/neural.d.ts +6 -12
  15. package/dist/cli/commands/neural.js +113 -10
  16. package/dist/cli/commands/nlp.d.ts +28 -0
  17. package/dist/cli/commands/nlp.js +246 -0
  18. package/dist/cli/commands/storage.d.ts +64 -0
  19. package/dist/cli/commands/storage.js +730 -0
  20. package/dist/cli/index.js +210 -24
  21. package/dist/coreTypes.d.ts +206 -34
  22. package/dist/distributed/configManager.js +8 -6
  23. package/dist/distributed/shardMigration.js +2 -0
  24. package/dist/distributed/storageDiscovery.js +6 -4
  25. package/dist/embeddings/EmbeddingManager.d.ts +2 -2
  26. package/dist/embeddings/EmbeddingManager.js +5 -1
  27. package/dist/graph/lsm/LSMTree.js +32 -20
  28. package/dist/hnsw/typeAwareHNSWIndex.js +6 -2
  29. package/dist/storage/adapters/azureBlobStorage.d.ts +545 -0
  30. package/dist/storage/adapters/azureBlobStorage.js +1809 -0
  31. package/dist/storage/adapters/baseStorageAdapter.d.ts +16 -13
  32. package/dist/storage/adapters/fileSystemStorage.d.ts +21 -9
  33. package/dist/storage/adapters/fileSystemStorage.js +204 -127
  34. package/dist/storage/adapters/gcsStorage.d.ts +119 -9
  35. package/dist/storage/adapters/gcsStorage.js +317 -62
  36. package/dist/storage/adapters/memoryStorage.d.ts +30 -18
  37. package/dist/storage/adapters/memoryStorage.js +99 -94
  38. package/dist/storage/adapters/opfsStorage.d.ts +48 -10
  39. package/dist/storage/adapters/opfsStorage.js +201 -80
  40. package/dist/storage/adapters/r2Storage.d.ts +12 -5
  41. package/dist/storage/adapters/r2Storage.js +63 -15
  42. package/dist/storage/adapters/s3CompatibleStorage.d.ts +164 -17
  43. package/dist/storage/adapters/s3CompatibleStorage.js +472 -80
  44. package/dist/storage/adapters/typeAwareStorageAdapter.d.ts +38 -6
  45. package/dist/storage/adapters/typeAwareStorageAdapter.js +218 -39
  46. package/dist/storage/baseStorage.d.ts +41 -38
  47. package/dist/storage/baseStorage.js +110 -134
  48. package/dist/storage/storageFactory.d.ts +29 -2
  49. package/dist/storage/storageFactory.js +30 -1
  50. package/dist/utils/entityIdMapper.js +5 -2
  51. package/dist/utils/fieldTypeInference.js +8 -1
  52. package/dist/utils/metadataFilter.d.ts +3 -2
  53. package/dist/utils/metadataFilter.js +1 -0
  54. package/dist/utils/metadataIndex.js +2 -0
  55. package/dist/utils/metadataIndexChunking.js +9 -4
  56. package/dist/utils/periodicCleanup.js +1 -0
  57. package/package.json +3 -1
@@ -33,9 +33,16 @@ export class ConfigAPI {
33
33
  };
34
34
  // Store in cache
35
35
  this.configCache.set(key, entry);
36
- // Persist to storage
36
+ // v4.0.0: Persist to storage as NounMetadata
37
37
  const configId = this.CONFIG_NOUN_PREFIX + key;
38
- await this.storage.saveMetadata(configId, entry);
38
+ const nounMetadata = {
39
+ noun: 'config',
40
+ ...entry,
41
+ service: 'config',
42
+ createdAt: entry.createdAt,
43
+ updatedAt: entry.updatedAt
44
+ };
45
+ await this.storage.saveNounMetadata(configId, nounMetadata);
39
46
  }
40
47
  /**
41
48
  * Get a configuration value with optional decryption
@@ -47,11 +54,24 @@ export class ConfigAPI {
47
54
  // If not in cache, load from storage
48
55
  if (!entry) {
49
56
  const configId = this.CONFIG_NOUN_PREFIX + key;
50
- const metadata = await this.storage.getMetadata(configId);
57
+ const metadata = await this.storage.getNounMetadata(configId);
51
58
  if (!metadata) {
52
59
  return defaultValue;
53
60
  }
54
- entry = metadata;
61
+ // v4.0.0: Extract ConfigEntry from NounMetadata
62
+ const createdAtVal = typeof metadata.createdAt === 'object' && metadata.createdAt !== null && 'seconds' in metadata.createdAt
63
+ ? metadata.createdAt.seconds * 1000 + Math.floor((metadata.createdAt.nanoseconds || 0) / 1000000)
64
+ : (metadata.createdAt || Date.now());
65
+ const updatedAtVal = typeof metadata.updatedAt === 'object' && metadata.updatedAt !== null && 'seconds' in metadata.updatedAt
66
+ ? metadata.updatedAt.seconds * 1000 + Math.floor((metadata.updatedAt.nanoseconds || 0) / 1000000)
67
+ : (metadata.updatedAt || createdAtVal);
68
+ entry = {
69
+ key: metadata.key,
70
+ value: metadata.value,
71
+ encrypted: metadata.encrypted,
72
+ createdAt: createdAtVal,
73
+ updatedAt: updatedAtVal
74
+ };
55
75
  this.configCache.set(key, entry);
56
76
  }
57
77
  let value = entry.value;
@@ -77,24 +97,20 @@ export class ConfigAPI {
77
97
  async delete(key) {
78
98
  // Remove from cache
79
99
  this.configCache.delete(key);
80
- // Remove from storage
100
+ // v4.0.0: Remove from storage
81
101
  const configId = this.CONFIG_NOUN_PREFIX + key;
82
- await this.storage.saveMetadata(configId, null);
102
+ await this.storage.deleteNounMetadata(configId);
83
103
  }
84
104
  /**
85
105
  * List all configuration keys
86
106
  */
87
107
  async list() {
88
- // Get all metadata keys from storage
89
- const allMetadata = await this.storage.getMetadata('');
90
- if (!allMetadata || typeof allMetadata !== 'object') {
91
- return [];
92
- }
93
- // Filter for config keys
108
+ // v4.0.0: Get all nouns and filter for config entries
109
+ const result = await this.storage.getNouns({ pagination: { limit: 10000 } });
94
110
  const configKeys = [];
95
- for (const key of Object.keys(allMetadata)) {
96
- if (key.startsWith(this.CONFIG_NOUN_PREFIX)) {
97
- configKeys.push(key.substring(this.CONFIG_NOUN_PREFIX.length));
111
+ for (const noun of result.items) {
112
+ if (noun.id.startsWith(this.CONFIG_NOUN_PREFIX)) {
113
+ configKeys.push(noun.id.substring(this.CONFIG_NOUN_PREFIX.length));
98
114
  }
99
115
  }
100
116
  return configKeys;
@@ -107,7 +123,7 @@ export class ConfigAPI {
107
123
  return true;
108
124
  }
109
125
  const configId = this.CONFIG_NOUN_PREFIX + key;
110
- const metadata = await this.storage.getMetadata(configId);
126
+ const metadata = await this.storage.getNounMetadata(configId);
111
127
  return metadata !== null && metadata !== undefined;
112
128
  }
113
129
  /**
@@ -143,7 +159,15 @@ export class ConfigAPI {
143
159
  for (const [key, entry] of Object.entries(config)) {
144
160
  this.configCache.set(key, entry);
145
161
  const configId = this.CONFIG_NOUN_PREFIX + key;
146
- await this.storage.saveMetadata(configId, entry);
162
+ // v4.0.0: Convert ConfigEntry to NounMetadata
163
+ const nounMetadata = {
164
+ noun: 'config',
165
+ ...entry,
166
+ service: 'config',
167
+ createdAt: entry.createdAt,
168
+ updatedAt: entry.updatedAt
169
+ };
170
+ await this.storage.saveNounMetadata(configId, nounMetadata);
147
171
  }
148
172
  }
149
173
  /**
@@ -154,11 +178,24 @@ export class ConfigAPI {
154
178
  return this.configCache.get(key);
155
179
  }
156
180
  const configId = this.CONFIG_NOUN_PREFIX + key;
157
- const metadata = await this.storage.getMetadata(configId);
181
+ const metadata = await this.storage.getNounMetadata(configId);
158
182
  if (!metadata) {
159
183
  return null;
160
184
  }
161
- const entry = metadata;
185
+ // v4.0.0: Extract ConfigEntry from NounMetadata
186
+ const createdAtVal = typeof metadata.createdAt === 'object' && metadata.createdAt !== null && 'seconds' in metadata.createdAt
187
+ ? metadata.createdAt.seconds * 1000 + Math.floor((metadata.createdAt.nanoseconds || 0) / 1000000)
188
+ : (metadata.createdAt || Date.now());
189
+ const updatedAtVal = typeof metadata.updatedAt === 'object' && metadata.updatedAt !== null && 'seconds' in metadata.updatedAt
190
+ ? metadata.updatedAt.seconds * 1000 + Math.floor((metadata.updatedAt.nanoseconds || 0) / 1000000)
191
+ : (metadata.updatedAt || createdAtVal);
192
+ const entry = {
193
+ key: metadata.key,
194
+ value: metadata.value,
195
+ encrypted: metadata.encrypted,
196
+ createdAt: createdAtVal,
197
+ updatedAt: updatedAtVal
198
+ };
162
199
  this.configCache.set(key, entry);
163
200
  return entry;
164
201
  }
@@ -41,8 +41,8 @@ export class DataAPI {
41
41
  id: verb.id,
42
42
  from: verb.sourceId,
43
43
  to: verb.targetId,
44
- type: (verb.verb || verb.type),
45
- weight: verb.weight || 1.0,
44
+ type: verb.verb,
45
+ weight: verb.metadata?.weight || 1.0,
46
46
  metadata: verb.metadata
47
47
  });
48
48
  }
@@ -91,16 +91,18 @@ export class DataAPI {
91
91
  // Restore entities
92
92
  for (const entity of backup.entities) {
93
93
  try {
94
+ // v4.0.0: Prepare noun and metadata separately
94
95
  const noun = {
95
96
  id: entity.id,
96
97
  vector: entity.vector || new Array(384).fill(0), // Default vector if missing
97
98
  connections: new Map(),
98
- level: 0,
99
- metadata: {
100
- ...entity.metadata,
101
- noun: entity.type,
102
- service: entity.service
103
- }
99
+ level: 0
100
+ };
101
+ const metadata = {
102
+ ...entity.metadata,
103
+ noun: entity.type,
104
+ service: entity.service,
105
+ createdAt: Date.now()
104
106
  };
105
107
  // Check if entity exists when merging
106
108
  if (merge) {
@@ -110,6 +112,7 @@ export class DataAPI {
110
112
  }
111
113
  }
112
114
  await this.storage.saveNoun(noun);
115
+ await this.storage.saveNounMetadata(entity.id, metadata);
113
116
  }
114
117
  catch (error) {
115
118
  console.error(`Failed to restore entity ${entity.id}:`, error);
@@ -127,17 +130,18 @@ export class DataAPI {
127
130
  }
128
131
  // Compute relation vector as average of source and target
129
132
  const relationVector = sourceNoun.vector.map((v, i) => (v + targetNoun.vector[i]) / 2);
133
+ // v4.0.0: Prepare verb and metadata separately
130
134
  const verb = {
131
135
  id: relation.id,
132
136
  vector: relationVector,
133
- sourceId: relation.from,
134
- targetId: relation.to,
135
- source: sourceNoun.metadata?.noun || NounType.Thing,
136
- target: targetNoun.metadata?.noun || NounType.Thing,
137
+ connections: new Map(),
137
138
  verb: relation.type,
138
- type: relation.type,
139
+ sourceId: relation.from,
140
+ targetId: relation.to
141
+ };
142
+ const verbMetadata = {
139
143
  weight: relation.weight,
140
- metadata: relation.metadata,
144
+ ...relation.metadata,
141
145
  createdAt: Date.now()
142
146
  };
143
147
  // Check if relation exists when merging
@@ -148,6 +152,7 @@ export class DataAPI {
148
152
  }
149
153
  }
150
154
  await this.storage.saveVerb(verb);
155
+ await this.storage.saveVerbMetadata(relation.id, verbMetadata);
151
156
  }
152
157
  catch (error) {
153
158
  console.error(`Failed to restore relation ${relation.id}:`, error);
@@ -253,15 +258,16 @@ export class DataAPI {
253
258
  if (validate) {
254
259
  this.validateImportItem(mapped);
255
260
  }
256
- // Save as entity
261
+ // v4.0.0: Save entity - separate vector and metadata
262
+ const id = mapped.id || this.generateId();
257
263
  const noun = {
258
- id: mapped.id || this.generateId(),
264
+ id,
259
265
  vector: mapped.vector || new Array(384).fill(0),
260
266
  connections: new Map(),
261
- level: 0,
262
- metadata: mapped
267
+ level: 0
263
268
  };
264
269
  await this.storage.saveNoun(noun);
270
+ await this.storage.saveNounMetadata(id, { ...mapped, createdAt: Date.now() });
265
271
  result.successful++;
266
272
  }
267
273
  catch (error) {
@@ -115,6 +115,30 @@ export declare class GCSStorageAugmentation extends StorageAugmentation {
115
115
  provideStorage(): Promise<StorageAdapter>;
116
116
  protected onInitialize(): Promise<void>;
117
117
  }
118
+ /**
119
+ * Azure Blob Storage Augmentation - Microsoft Azure
120
+ */
121
+ export declare class AzureStorageAugmentation extends StorageAugmentation {
122
+ readonly name = "azure-storage";
123
+ protected config: {
124
+ containerName: string;
125
+ accountName?: string;
126
+ accountKey?: string;
127
+ connectionString?: string;
128
+ sasToken?: string;
129
+ cacheConfig?: any;
130
+ };
131
+ constructor(config: {
132
+ containerName: string;
133
+ accountName?: string;
134
+ accountKey?: string;
135
+ connectionString?: string;
136
+ sasToken?: string;
137
+ cacheConfig?: any;
138
+ });
139
+ provideStorage(): Promise<StorageAdapter>;
140
+ protected onInitialize(): Promise<void>;
141
+ }
118
142
  /**
119
143
  * Auto-select the best storage augmentation for the environment
120
144
  * Maintains zero-config philosophy
@@ -9,6 +9,7 @@ import { MemoryStorage } from '../storage/adapters/memoryStorage.js';
9
9
  import { OPFSStorage } from '../storage/adapters/opfsStorage.js';
10
10
  import { S3CompatibleStorage } from '../storage/adapters/s3CompatibleStorage.js';
11
11
  import { R2Storage } from '../storage/adapters/r2Storage.js';
12
+ import { AzureBlobStorage } from '../storage/adapters/azureBlobStorage.js';
12
13
  /**
13
14
  * Memory Storage Augmentation - Fast in-memory storage
14
15
  */
@@ -338,6 +339,27 @@ export class GCSStorageAugmentation extends StorageAugmentation {
338
339
  this.log(`GCS storage initialized with bucket ${this.config.bucketName}`);
339
340
  }
340
341
  }
342
+ /**
343
+ * Azure Blob Storage Augmentation - Microsoft Azure
344
+ */
345
+ export class AzureStorageAugmentation extends StorageAugmentation {
346
+ constructor(config) {
347
+ super();
348
+ this.name = 'azure-storage';
349
+ this.config = config;
350
+ }
351
+ async provideStorage() {
352
+ const storage = new AzureBlobStorage({
353
+ ...this.config
354
+ });
355
+ this.storageAdapter = storage;
356
+ return storage;
357
+ }
358
+ async onInitialize() {
359
+ await this.storageAdapter.init();
360
+ this.log(`Azure Blob Storage initialized with container ${this.config.containerName}`);
361
+ }
362
+ }
341
363
  /**
342
364
  * Auto-select the best storage augmentation for the environment
343
365
  * Maintains zero-config philosophy
package/dist/brainy.js CHANGED
@@ -282,14 +282,14 @@ export class Brainy {
282
282
  service: params.service,
283
283
  createdAt: Date.now()
284
284
  };
285
- // Save to storage
285
+ // v4.0.0: Save vector and metadata separately
286
286
  await this.storage.saveNoun({
287
287
  id,
288
288
  vector,
289
289
  connections: new Map(),
290
- level: 0,
291
- metadata
290
+ level: 0
292
291
  });
292
+ await this.storage.saveNounMetadata(id, metadata);
293
293
  // Add to metadata index for fast filtering
294
294
  await this.metadataIndex.addToIndex(id, metadata);
295
295
  return id;
@@ -446,13 +446,14 @@ export class Brainy {
446
446
  createdAt: existing.createdAt,
447
447
  updatedAt: Date.now()
448
448
  };
449
+ // v4.0.0: Save vector and metadata separately
449
450
  await this.storage.saveNoun({
450
451
  id: params.id,
451
452
  vector,
452
453
  connections: new Map(),
453
- level: 0,
454
- metadata: updatedMetadata
454
+ level: 0
455
455
  });
456
+ await this.storage.saveNounMetadata(params.id, updatedMetadata);
456
457
  // Update metadata index - remove old entry and add new one
457
458
  await this.metadataIndex.removeFromIndex(params.id, existing.metadata);
458
459
  await this.metadataIndex.addToIndex(params.id, updatedMetadata);
@@ -635,7 +636,7 @@ export class Brainy {
635
636
  // Bug #1 showed incrementing verb counts (7→8→9...) indicating duplicates
636
637
  const existingVerbs = await this.storage.getVerbsBySource(params.from);
637
638
  const duplicate = existingVerbs.find(v => v.targetId === params.to &&
638
- v.type === params.type);
639
+ v.verb === params.type);
639
640
  if (duplicate) {
640
641
  // Relationship already exists - return existing ID instead of creating duplicate
641
642
  console.log(`[DEBUG] Skipping duplicate relationship: ${params.from} → ${params.to} (${params.type})`);
@@ -646,7 +647,13 @@ export class Brainy {
646
647
  // Compute relationship vector (average of entities)
647
648
  const relationVector = fromEntity.vector.map((v, i) => (v + toEntity.vector[i]) / 2);
648
649
  return this.augmentationRegistry.execute('relate', params, async () => {
649
- // Save to storage
650
+ // v4.0.0: Prepare verb metadata
651
+ const verbMetadata = {
652
+ weight: params.weight ?? 1.0,
653
+ ...(params.metadata || {}),
654
+ createdAt: Date.now()
655
+ };
656
+ // Save to storage (v4.0.0: vector and metadata separately)
650
657
  const verb = {
651
658
  id,
652
659
  vector: relationVector,
@@ -660,7 +667,15 @@ export class Brainy {
660
667
  metadata: params.metadata,
661
668
  createdAt: Date.now()
662
669
  };
663
- await this.storage.saveVerb(verb);
670
+ await this.storage.saveVerb({
671
+ id,
672
+ vector: relationVector,
673
+ connections: new Map(),
674
+ verb: params.type,
675
+ sourceId: params.from,
676
+ targetId: params.to
677
+ });
678
+ await this.storage.saveVerbMetadata(id, verbMetadata);
664
679
  // Add to graph index for O(1) lookups
665
680
  await this.graphIndex.addVerb(verb);
666
681
  // Create bidirectional if requested
@@ -674,7 +689,15 @@ export class Brainy {
674
689
  source: toEntity.type,
675
690
  target: fromEntity.type
676
691
  };
677
- await this.storage.saveVerb(reverseVerb);
692
+ await this.storage.saveVerb({
693
+ id: reverseId,
694
+ vector: relationVector,
695
+ connections: new Map(),
696
+ verb: params.type,
697
+ sourceId: params.to,
698
+ targetId: params.from
699
+ });
700
+ await this.storage.saveVerbMetadata(reverseId, verbMetadata);
678
701
  // Add reverse relationship to graph index too
679
702
  await this.graphIndex.addVerb(reverseVerb);
680
703
  }
@@ -37,10 +37,6 @@ interface RelateOptions extends CoreOptions {
37
37
  weight?: string;
38
38
  metadata?: string;
39
39
  }
40
- interface ImportOptions extends CoreOptions {
41
- format?: 'json' | 'csv' | 'jsonl';
42
- batchSize?: string;
43
- }
44
40
  interface ExportOptions extends CoreOptions {
45
41
  format?: 'json' | 'csv' | 'jsonl';
46
42
  }
@@ -48,23 +44,37 @@ export declare const coreCommands: {
48
44
  /**
49
45
  * Add data to the neural database
50
46
  */
51
- add(text: string, options: AddOptions): Promise<void>;
47
+ add(text: string | undefined, options: AddOptions): Promise<void>;
52
48
  /**
53
49
  * Search the neural database with Triple Intelligence™
54
50
  */
55
- search(query: string, options: SearchOptions): Promise<void>;
51
+ search(query: string | undefined, options: SearchOptions): Promise<void>;
56
52
  /**
57
53
  * Get item by ID
58
54
  */
59
- get(id: string, options: GetOptions): Promise<void>;
55
+ get(id: string | undefined, options: GetOptions): Promise<void>;
60
56
  /**
61
57
  * Create relationship between items
62
58
  */
63
- relate(source: string, verb: string, target: string, options: RelateOptions): Promise<void>;
59
+ relate(source: string | undefined, verb: string | undefined, target: string | undefined, options: RelateOptions): Promise<void>;
60
+ /**
61
+ * Update an existing entity
62
+ */
63
+ update(id: string | undefined, options: AddOptions & {
64
+ content?: string;
65
+ }): Promise<void>;
66
+ /**
67
+ * Delete an entity
68
+ */
69
+ deleteEntity(id: string | undefined, options: CoreOptions & {
70
+ force?: boolean;
71
+ }): Promise<void>;
64
72
  /**
65
- * Import data from file
73
+ * Remove a relationship
66
74
  */
67
- import(file: string, options: ImportOptions): Promise<void>;
75
+ unrelate(id: string | undefined, options: CoreOptions & {
76
+ force?: boolean;
77
+ }): Promise<void>;
68
78
  /**
69
79
  * Export database
70
80
  */