@soulcraft/brainy 3.50.1 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +242 -0
  2. package/README.md +358 -658
  3. package/dist/api/ConfigAPI.js +56 -19
  4. package/dist/api/DataAPI.js +24 -18
  5. package/dist/augmentations/storageAugmentations.d.ts +24 -0
  6. package/dist/augmentations/storageAugmentations.js +22 -0
  7. package/dist/brainy.js +32 -9
  8. package/dist/cli/commands/core.d.ts +20 -10
  9. package/dist/cli/commands/core.js +384 -82
  10. package/dist/cli/commands/import.d.ts +41 -0
  11. package/dist/cli/commands/import.js +456 -0
  12. package/dist/cli/commands/insights.d.ts +34 -0
  13. package/dist/cli/commands/insights.js +300 -0
  14. package/dist/cli/commands/neural.d.ts +6 -12
  15. package/dist/cli/commands/neural.js +113 -10
  16. package/dist/cli/commands/nlp.d.ts +28 -0
  17. package/dist/cli/commands/nlp.js +246 -0
  18. package/dist/cli/commands/storage.d.ts +64 -0
  19. package/dist/cli/commands/storage.js +730 -0
  20. package/dist/cli/index.js +210 -24
  21. package/dist/coreTypes.d.ts +206 -34
  22. package/dist/distributed/configManager.js +8 -6
  23. package/dist/distributed/shardMigration.js +2 -0
  24. package/dist/distributed/storageDiscovery.js +6 -4
  25. package/dist/embeddings/EmbeddingManager.d.ts +2 -2
  26. package/dist/embeddings/EmbeddingManager.js +5 -1
  27. package/dist/graph/lsm/LSMTree.js +32 -20
  28. package/dist/hnsw/typeAwareHNSWIndex.js +6 -2
  29. package/dist/storage/adapters/azureBlobStorage.d.ts +545 -0
  30. package/dist/storage/adapters/azureBlobStorage.js +1809 -0
  31. package/dist/storage/adapters/baseStorageAdapter.d.ts +16 -13
  32. package/dist/storage/adapters/fileSystemStorage.d.ts +21 -9
  33. package/dist/storage/adapters/fileSystemStorage.js +204 -127
  34. package/dist/storage/adapters/gcsStorage.d.ts +119 -9
  35. package/dist/storage/adapters/gcsStorage.js +317 -62
  36. package/dist/storage/adapters/memoryStorage.d.ts +30 -18
  37. package/dist/storage/adapters/memoryStorage.js +99 -94
  38. package/dist/storage/adapters/opfsStorage.d.ts +48 -10
  39. package/dist/storage/adapters/opfsStorage.js +201 -80
  40. package/dist/storage/adapters/r2Storage.d.ts +12 -5
  41. package/dist/storage/adapters/r2Storage.js +63 -15
  42. package/dist/storage/adapters/s3CompatibleStorage.d.ts +164 -17
  43. package/dist/storage/adapters/s3CompatibleStorage.js +472 -80
  44. package/dist/storage/adapters/typeAwareStorageAdapter.d.ts +38 -6
  45. package/dist/storage/adapters/typeAwareStorageAdapter.js +218 -39
  46. package/dist/storage/baseStorage.d.ts +41 -38
  47. package/dist/storage/baseStorage.js +110 -134
  48. package/dist/storage/storageFactory.d.ts +29 -2
  49. package/dist/storage/storageFactory.js +30 -1
  50. package/dist/utils/entityIdMapper.js +5 -2
  51. package/dist/utils/fieldTypeInference.js +8 -1
  52. package/dist/utils/metadataFilter.d.ts +3 -2
  53. package/dist/utils/metadataFilter.js +1 -0
  54. package/dist/utils/metadataIndex.d.ts +2 -1
  55. package/dist/utils/metadataIndex.js +9 -1
  56. package/dist/utils/metadataIndexChunking.js +9 -4
  57. package/dist/utils/periodicCleanup.js +1 -0
  58. package/package.json +3 -1
@@ -5,9 +5,10 @@
5
5
  */
6
6
  import chalk from 'chalk';
7
7
  import ora from 'ora';
8
- import { readFileSync, writeFileSync } from 'node:fs';
8
+ import inquirer from 'inquirer';
9
+ import { writeFileSync } from 'node:fs';
9
10
  import { Brainy } from '../../brainy.js';
10
- import { BrainyTypes, NounType } from '../../index.js';
11
+ import { BrainyTypes } from '../../index.js';
11
12
  let brainyInstance = null;
12
13
  const getBrainy = () => {
13
14
  if (!brainyInstance) {
@@ -25,8 +26,50 @@ export const coreCommands = {
25
26
  * Add data to the neural database
26
27
  */
27
28
  async add(text, options) {
28
- const spinner = ora('Adding to neural database...').start();
29
+ let spinner = null;
29
30
  try {
31
+ // Interactive mode if no text provided
32
+ if (!text) {
33
+ const answers = await inquirer.prompt([
34
+ {
35
+ type: 'input',
36
+ name: 'content',
37
+ message: 'Enter content:',
38
+ validate: (input) => input.trim().length > 0 || 'Content cannot be empty'
39
+ },
40
+ {
41
+ type: 'input',
42
+ name: 'nounType',
43
+ message: 'Noun type (optional, press Enter to auto-detect):',
44
+ default: ''
45
+ },
46
+ {
47
+ type: 'input',
48
+ name: 'metadata',
49
+ message: 'Metadata (JSON, optional):',
50
+ default: '',
51
+ validate: (input) => {
52
+ if (!input.trim())
53
+ return true;
54
+ try {
55
+ JSON.parse(input);
56
+ return true;
57
+ }
58
+ catch {
59
+ return 'Invalid JSON format';
60
+ }
61
+ }
62
+ }
63
+ ]);
64
+ text = answers.content;
65
+ if (answers.nounType) {
66
+ options.type = answers.nounType;
67
+ }
68
+ if (answers.metadata) {
69
+ options.metadata = answers.metadata;
70
+ }
71
+ }
72
+ const spinner = ora('Adding to neural database...').start();
30
73
  const brain = getBrainy();
31
74
  let metadata = {};
32
75
  if (options.metadata) {
@@ -86,8 +129,9 @@ export const coreCommands = {
86
129
  }
87
130
  }
88
131
  catch (error) {
89
- spinner.fail('Failed to add data');
90
- console.error(chalk.red(error.message));
132
+ if (spinner)
133
+ spinner.fail('Failed to add data');
134
+ console.error(chalk.red('Failed to add data:', error.message));
91
135
  process.exit(1);
92
136
  }
93
137
  },
@@ -95,8 +139,68 @@ export const coreCommands = {
95
139
  * Search the neural database with Triple Intelligence™
96
140
  */
97
141
  async search(query, options) {
98
- const spinner = ora('Searching with Triple Intelligence™...').start();
142
+ let spinner = null;
99
143
  try {
144
+ // Interactive mode if no query provided
145
+ if (!query) {
146
+ const answers = await inquirer.prompt([
147
+ {
148
+ type: 'input',
149
+ name: 'query',
150
+ message: 'What are you looking for?',
151
+ validate: (input) => input.trim().length > 0 || 'Query cannot be empty'
152
+ },
153
+ {
154
+ type: 'number',
155
+ name: 'limit',
156
+ message: 'Number of results:',
157
+ default: 10
158
+ },
159
+ {
160
+ type: 'confirm',
161
+ name: 'useAdvanced',
162
+ message: 'Use advanced filters?',
163
+ default: false
164
+ }
165
+ ]);
166
+ query = answers.query;
167
+ if (!options.limit) {
168
+ options.limit = answers.limit.toString();
169
+ }
170
+ // Advanced filters
171
+ if (answers.useAdvanced) {
172
+ const advancedAnswers = await inquirer.prompt([
173
+ {
174
+ type: 'input',
175
+ name: 'type',
176
+ message: 'Filter by type (optional):',
177
+ default: ''
178
+ },
179
+ {
180
+ type: 'input',
181
+ name: 'threshold',
182
+ message: 'Similarity threshold (0-1, default 0.7):',
183
+ default: '0.7',
184
+ validate: (input) => {
185
+ const num = parseFloat(input);
186
+ return (num >= 0 && num <= 1) || 'Must be between 0 and 1';
187
+ }
188
+ },
189
+ {
190
+ type: 'confirm',
191
+ name: 'explain',
192
+ message: 'Show scoring breakdown?',
193
+ default: false
194
+ }
195
+ ]);
196
+ if (advancedAnswers.type)
197
+ options.type = advancedAnswers.type;
198
+ if (advancedAnswers.threshold)
199
+ options.threshold = advancedAnswers.threshold;
200
+ options.explain = advancedAnswers.explain;
201
+ }
202
+ }
203
+ const spinner = ora('Searching with Triple Intelligence™...').start();
100
204
  const brain = getBrainy();
101
205
  // Build comprehensive search params
102
206
  const searchParams = {
@@ -250,8 +354,9 @@ export const coreCommands = {
250
354
  }
251
355
  }
252
356
  catch (error) {
253
- spinner.fail('Search failed');
254
- console.error(chalk.red(error.message));
357
+ if (spinner)
358
+ spinner.fail('Search failed');
359
+ console.error(chalk.red('Search failed:', error.message));
255
360
  if (options.verbose) {
256
361
  console.error(chalk.dim(error.stack));
257
362
  }
@@ -262,8 +367,28 @@ export const coreCommands = {
262
367
  * Get item by ID
263
368
  */
264
369
  async get(id, options) {
265
- const spinner = ora('Fetching item...').start();
370
+ let spinner = null;
266
371
  try {
372
+ // Interactive mode if no ID provided
373
+ if (!id) {
374
+ const answers = await inquirer.prompt([
375
+ {
376
+ type: 'input',
377
+ name: 'id',
378
+ message: 'Enter item ID:',
379
+ validate: (input) => input.trim().length > 0 || 'ID cannot be empty'
380
+ },
381
+ {
382
+ type: 'confirm',
383
+ name: 'withConnections',
384
+ message: 'Include connections?',
385
+ default: false
386
+ }
387
+ ]);
388
+ id = answers.id;
389
+ options.withConnections = answers.withConnections;
390
+ }
391
+ const spinner = ora('Fetching item...').start();
267
392
  const brain = getBrainy();
268
393
  // Try to get the item
269
394
  const item = await brain.get(id);
@@ -297,8 +422,9 @@ export const coreCommands = {
297
422
  }
298
423
  }
299
424
  catch (error) {
300
- spinner.fail('Failed to get item');
301
- console.error(chalk.red(error.message));
425
+ if (spinner)
426
+ spinner.fail('Failed to get item');
427
+ console.error(chalk.red('Failed to get item:', error.message));
302
428
  process.exit(1);
303
429
  }
304
430
  },
@@ -306,8 +432,53 @@ export const coreCommands = {
306
432
  * Create relationship between items
307
433
  */
308
434
  async relate(source, verb, target, options) {
309
- const spinner = ora('Creating relationship...').start();
435
+ let spinner = null;
310
436
  try {
437
+ // Interactive mode if parameters missing
438
+ if (!source || !verb || !target) {
439
+ const answers = await inquirer.prompt([
440
+ {
441
+ type: 'input',
442
+ name: 'source',
443
+ message: 'Source entity ID:',
444
+ default: source || '',
445
+ validate: (input) => input.trim().length > 0 || 'Source ID cannot be empty'
446
+ },
447
+ {
448
+ type: 'input',
449
+ name: 'verb',
450
+ message: 'Relationship type (verb):',
451
+ default: verb || '',
452
+ validate: (input) => input.trim().length > 0 || 'Verb cannot be empty'
453
+ },
454
+ {
455
+ type: 'input',
456
+ name: 'target',
457
+ message: 'Target entity ID:',
458
+ default: target || '',
459
+ validate: (input) => input.trim().length > 0 || 'Target ID cannot be empty'
460
+ },
461
+ {
462
+ type: 'input',
463
+ name: 'weight',
464
+ message: 'Relationship weight (0-1, optional):',
465
+ default: '',
466
+ validate: (input) => {
467
+ if (!input.trim())
468
+ return true;
469
+ const num = parseFloat(input);
470
+ return (num >= 0 && num <= 1) || 'Must be between 0 and 1';
471
+ }
472
+ }
473
+ ]);
474
+ source = answers.source;
475
+ verb = answers.verb;
476
+ target = answers.target;
477
+ if (answers.weight) {
478
+ options.weight = answers.weight;
479
+ }
480
+ }
481
+ const spinner = ora('Creating relationship...').start();
311
482
  const brain = getBrainy();
312
483
  let metadata = {};
313
484
  if (options.metadata) {
@@ -342,94 +513,225 @@ export const coreCommands = {
342
513
  }
343
514
  }
344
515
  catch (error) {
345
- spinner.fail('Failed to create relationship');
346
- console.error(chalk.red(error.message));
516
+ if (spinner)
517
+ spinner.fail('Failed to create relationship');
518
+ console.error(chalk.red('Failed to create relationship:', error.message));
347
519
  process.exit(1);
348
520
  }
349
521
  },
350
522
  /**
351
- * Import data from file
523
+ * Update an existing entity
352
524
  */
353
- async import(file, options) {
354
- const spinner = ora('Importing data...').start();
525
+ async update(id, options) {
526
+ let spinner = null;
355
527
  try {
356
- const brain = getBrainy();
357
- const format = options.format || 'json';
358
- const batchSize = options.batchSize ? parseInt(options.batchSize) : 100;
359
- // Read file content
360
- const content = readFileSync(file, 'utf-8');
361
- let items = [];
362
- switch (format) {
363
- case 'json':
364
- items = JSON.parse(content);
365
- if (!Array.isArray(items)) {
366
- items = [items];
528
+ // Interactive mode if no ID provided
529
+ if (!id) {
530
+ const answers = await inquirer.prompt([
531
+ {
532
+ type: 'input',
533
+ name: 'id',
534
+ message: 'Entity ID to update:',
535
+ validate: (input) => input.trim().length > 0 || 'ID cannot be empty'
536
+ },
537
+ {
538
+ type: 'input',
539
+ name: 'content',
540
+ message: 'New content (optional, press Enter to skip):',
541
+ default: ''
542
+ },
543
+ {
544
+ type: 'input',
545
+ name: 'metadata',
546
+ message: 'Metadata to merge (JSON, optional):',
547
+ default: '',
548
+ validate: (input) => {
549
+ if (!input.trim())
550
+ return true;
551
+ try {
552
+ JSON.parse(input);
553
+ return true;
554
+ }
555
+ catch {
556
+ return 'Invalid JSON format';
557
+ }
558
+ }
367
559
  }
368
- break;
369
- case 'jsonl':
370
- items = content.split('\n')
371
- .filter(line => line.trim())
372
- .map(line => JSON.parse(line));
373
- break;
374
- case 'csv':
375
- // Simple CSV parsing (first line is headers)
376
- const lines = content.split('\n').filter(line => line.trim());
377
- const headers = lines[0].split(',').map(h => h.trim());
378
- items = lines.slice(1).map(line => {
379
- const values = line.split(',').map(v => v.trim());
380
- const obj = {};
381
- headers.forEach((h, i) => {
382
- obj[h] = values[i];
383
- });
384
- return obj;
385
- });
386
- break;
560
+ ]);
561
+ id = answers.id;
562
+ if (answers.content) {
563
+ options.content = answers.content;
564
+ }
565
+ if (answers.metadata) {
566
+ options.metadata = answers.metadata;
567
+ }
387
568
  }
388
- spinner.text = `Importing ${items.length} items...`;
389
- // Process in batches
390
- let imported = 0;
391
- for (let i = 0; i < items.length; i += batchSize) {
392
- const batch = items.slice(i, i + batchSize);
393
- for (const item of batch) {
394
- let content;
395
- let metadata = {};
396
- if (typeof item === 'string') {
397
- content = item;
398
- }
399
- else if (item.content || item.text) {
400
- content = item.content || item.text;
401
- metadata = item.metadata || item;
569
+ spinner = ora('Updating entity...').start();
570
+ const brain = getBrainy();
571
+ // Get existing entity first
572
+ const existing = await brain.get(id);
573
+ if (!existing) {
574
+ spinner.fail('Entity not found');
575
+ console.log(chalk.yellow(`No entity found with ID: ${id}`));
576
+ process.exit(1);
577
+ }
578
+ // Build update params
579
+ const updateParams = { id };
580
+ if (options.content) {
581
+ updateParams.data = options.content;
582
+ }
583
+ if (options.metadata) {
584
+ try {
585
+ const newMetadata = JSON.parse(options.metadata);
586
+ updateParams.metadata = {
587
+ ...existing.metadata,
588
+ ...newMetadata
589
+ };
590
+ }
591
+ catch {
592
+ spinner.fail('Invalid metadata JSON');
593
+ process.exit(1);
594
+ }
595
+ }
596
+ if (options.type) {
597
+ updateParams.type = options.type;
598
+ }
599
+ await brain.update(updateParams);
600
+ spinner.succeed('Entity updated successfully');
601
+ if (!options.json) {
602
+ console.log(chalk.green(`✓ Updated entity: ${id}`));
603
+ if (options.content) {
604
+ console.log(chalk.dim(` New content: ${options.content.substring(0, 80)}...`));
605
+ }
606
+ if (updateParams.metadata) {
607
+ console.log(chalk.dim(` Metadata: ${JSON.stringify(updateParams.metadata)}`));
608
+ }
609
+ }
610
+ else {
611
+ formatOutput({ id, updated: true }, options);
612
+ }
613
+ }
614
+ catch (error) {
615
+ if (spinner)
616
+ spinner.fail('Failed to update entity');
617
+ console.error(chalk.red('Update failed:', error.message));
618
+ process.exit(1);
619
+ }
620
+ },
621
+ /**
622
+ * Delete an entity
623
+ */
624
+ async deleteEntity(id, options) {
625
+ let spinner = null;
626
+ try {
627
+ // Interactive mode if no ID provided
628
+ if (!id) {
629
+ const answers = await inquirer.prompt([
630
+ {
631
+ type: 'input',
632
+ name: 'id',
633
+ message: 'Entity ID to delete:',
634
+ validate: (input) => input.trim().length > 0 || 'ID cannot be empty'
635
+ },
636
+ {
637
+ type: 'confirm',
638
+ name: 'confirm',
639
+ message: 'Are you sure? This cannot be undone.',
640
+ default: false
402
641
  }
403
- else {
404
- content = JSON.stringify(item);
405
- metadata = { originalData: item };
642
+ ]);
643
+ if (!answers.confirm) {
644
+ console.log(chalk.yellow('Delete cancelled'));
645
+ return;
646
+ }
647
+ id = answers.id;
648
+ }
649
+ else if (!options.force) {
650
+ // Confirmation for non-interactive mode
651
+ const answer = await inquirer.prompt([{
652
+ type: 'confirm',
653
+ name: 'confirm',
654
+ message: `Delete entity ${id}? This cannot be undone.`,
655
+ default: false
656
+ }]);
657
+ if (!answer.confirm) {
658
+ console.log(chalk.yellow('Delete cancelled'));
659
+ return;
660
+ }
661
+ }
662
+ spinner = ora('Deleting entity...').start();
663
+ const brain = getBrainy();
664
+ await brain.delete(id);
665
+ spinner.succeed('Entity deleted successfully');
666
+ if (!options.json) {
667
+ console.log(chalk.green(`✓ Deleted entity: ${id}`));
668
+ }
669
+ else {
670
+ formatOutput({ id, deleted: true }, options);
671
+ }
672
+ }
673
+ catch (error) {
674
+ if (spinner)
675
+ spinner.fail('Failed to delete entity');
676
+ console.error(chalk.red('Delete failed:', error.message));
677
+ process.exit(1);
678
+ }
679
+ },
680
+ /**
681
+ * Remove a relationship
682
+ */
683
+ async unrelate(id, options) {
684
+ let spinner = null;
685
+ try {
686
+ // Interactive mode if no ID provided
687
+ if (!id) {
688
+ const answers = await inquirer.prompt([
689
+ {
690
+ type: 'input',
691
+ name: 'id',
692
+ message: 'Relationship ID to remove:',
693
+ validate: (input) => input.trim().length > 0 || 'ID cannot be empty'
694
+ },
695
+ {
696
+ type: 'confirm',
697
+ name: 'confirm',
698
+ message: 'Remove this relationship?',
699
+ default: false
406
700
  }
407
- // Use AI to detect type for each item
408
- const suggestion = await BrainyTypes.suggestNoun(typeof content === 'string' ? { content, ...metadata } : content);
409
- // Use suggested type or default to Content if low confidence
410
- const nounType = suggestion.confidence >= 0.5 ? suggestion.type : NounType.Content;
411
- await brain.add({
412
- data: content,
413
- type: nounType,
414
- metadata
415
- });
416
- imported++;
701
+ ]);
702
+ if (!answers.confirm) {
703
+ console.log(chalk.yellow('Operation cancelled'));
704
+ return;
705
+ }
706
+ id = answers.id;
707
+ }
708
+ else if (!options.force) {
709
+ const answer = await inquirer.prompt([{
710
+ type: 'confirm',
711
+ name: 'confirm',
712
+ message: `Remove relationship ${id}?`,
713
+ default: false
714
+ }]);
715
+ if (!answer.confirm) {
716
+ console.log(chalk.yellow('Operation cancelled'));
717
+ return;
417
718
  }
418
- spinner.text = `Imported ${imported}/${items.length} items...`;
419
719
  }
420
- spinner.succeed(`Imported ${imported} items`);
720
+ spinner = ora('Removing relationship...').start();
721
+ const brain = getBrainy();
722
+ await brain.unrelate(id);
723
+ spinner.succeed('Relationship removed successfully');
421
724
  if (!options.json) {
422
- console.log(chalk.green(`✓ Successfully imported ${imported} items from ${file}`));
423
- console.log(chalk.dim(` Format: ${format}`));
424
- console.log(chalk.dim(` Batch size: ${batchSize}`));
725
+ console.log(chalk.green(`✓ Removed relationship: ${id}`));
425
726
  }
426
727
  else {
427
- formatOutput({ imported, file, format, batchSize }, options);
728
+ formatOutput({ id, removed: true }, options);
428
729
  }
429
730
  }
430
731
  catch (error) {
431
- spinner.fail('Import failed');
432
- console.error(chalk.red(error.message));
732
+ if (spinner)
733
+ spinner.fail('Failed to remove relationship');
734
+ console.error(chalk.red('Unrelate failed:', error.message));
433
735
  process.exit(1);
434
736
  }
435
737
  },
@@ -0,0 +1,41 @@
1
+ /**
2
+ * Import Commands - Neural Import & Data Import
3
+ *
4
+ * Complete import system exposing ALL Brainy import capabilities:
5
+ * - UniversalImportAPI: Neural import with AI type matching
6
+ * - DirectoryImporter: VFS directory imports
7
+ * - DataAPI: Backup/restore
8
+ *
9
+ * Supports: files, directories, URLs, all formats
10
+ */
11
+ interface ImportOptions {
12
+ verbose?: boolean;
13
+ json?: boolean;
14
+ pretty?: boolean;
15
+ quiet?: boolean;
16
+ format?: 'json' | 'csv' | 'jsonl' | 'yaml' | 'markdown' | 'html' | 'xml' | 'text';
17
+ recursive?: boolean;
18
+ batchSize?: string;
19
+ extractConcepts?: boolean;
20
+ extractEntities?: boolean;
21
+ detectRelationships?: boolean;
22
+ confidence?: string;
23
+ progress?: boolean;
24
+ skipHidden?: boolean;
25
+ skipNodeModules?: boolean;
26
+ target?: string;
27
+ generateEmbeddings?: boolean;
28
+ extractMetadata?: boolean;
29
+ }
30
+ export declare const importCommands: {
31
+ /**
32
+ * Enhanced import using UniversalImportAPI
33
+ * Supports files, directories, URLs, all formats
34
+ */
35
+ import(source: string | undefined, options: ImportOptions): Promise<void>;
36
+ /**
37
+ * VFS-specific import (files/directories into VFS)
38
+ */
39
+ vfsImport(source: string | undefined, options: ImportOptions): Promise<void>;
40
+ };
41
+ export {};