@soulcraft/brainy 3.50.1 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +242 -0
  2. package/README.md +358 -658
  3. package/dist/api/ConfigAPI.js +56 -19
  4. package/dist/api/DataAPI.js +24 -18
  5. package/dist/augmentations/storageAugmentations.d.ts +24 -0
  6. package/dist/augmentations/storageAugmentations.js +22 -0
  7. package/dist/brainy.js +32 -9
  8. package/dist/cli/commands/core.d.ts +20 -10
  9. package/dist/cli/commands/core.js +384 -82
  10. package/dist/cli/commands/import.d.ts +41 -0
  11. package/dist/cli/commands/import.js +456 -0
  12. package/dist/cli/commands/insights.d.ts +34 -0
  13. package/dist/cli/commands/insights.js +300 -0
  14. package/dist/cli/commands/neural.d.ts +6 -12
  15. package/dist/cli/commands/neural.js +113 -10
  16. package/dist/cli/commands/nlp.d.ts +28 -0
  17. package/dist/cli/commands/nlp.js +246 -0
  18. package/dist/cli/commands/storage.d.ts +64 -0
  19. package/dist/cli/commands/storage.js +730 -0
  20. package/dist/cli/index.js +210 -24
  21. package/dist/coreTypes.d.ts +206 -34
  22. package/dist/distributed/configManager.js +8 -6
  23. package/dist/distributed/shardMigration.js +2 -0
  24. package/dist/distributed/storageDiscovery.js +6 -4
  25. package/dist/embeddings/EmbeddingManager.d.ts +2 -2
  26. package/dist/embeddings/EmbeddingManager.js +5 -1
  27. package/dist/graph/lsm/LSMTree.js +32 -20
  28. package/dist/hnsw/typeAwareHNSWIndex.js +6 -2
  29. package/dist/storage/adapters/azureBlobStorage.d.ts +545 -0
  30. package/dist/storage/adapters/azureBlobStorage.js +1809 -0
  31. package/dist/storage/adapters/baseStorageAdapter.d.ts +16 -13
  32. package/dist/storage/adapters/fileSystemStorage.d.ts +21 -9
  33. package/dist/storage/adapters/fileSystemStorage.js +204 -127
  34. package/dist/storage/adapters/gcsStorage.d.ts +119 -9
  35. package/dist/storage/adapters/gcsStorage.js +317 -62
  36. package/dist/storage/adapters/memoryStorage.d.ts +30 -18
  37. package/dist/storage/adapters/memoryStorage.js +99 -94
  38. package/dist/storage/adapters/opfsStorage.d.ts +48 -10
  39. package/dist/storage/adapters/opfsStorage.js +201 -80
  40. package/dist/storage/adapters/r2Storage.d.ts +12 -5
  41. package/dist/storage/adapters/r2Storage.js +63 -15
  42. package/dist/storage/adapters/s3CompatibleStorage.d.ts +164 -17
  43. package/dist/storage/adapters/s3CompatibleStorage.js +472 -80
  44. package/dist/storage/adapters/typeAwareStorageAdapter.d.ts +38 -6
  45. package/dist/storage/adapters/typeAwareStorageAdapter.js +218 -39
  46. package/dist/storage/baseStorage.d.ts +41 -38
  47. package/dist/storage/baseStorage.js +110 -134
  48. package/dist/storage/storageFactory.d.ts +29 -2
  49. package/dist/storage/storageFactory.js +30 -1
  50. package/dist/utils/entityIdMapper.js +5 -2
  51. package/dist/utils/fieldTypeInference.js +8 -1
  52. package/dist/utils/metadataFilter.d.ts +3 -2
  53. package/dist/utils/metadataFilter.js +1 -0
  54. package/dist/utils/metadataIndex.d.ts +2 -1
  55. package/dist/utils/metadataIndex.js +9 -1
  56. package/dist/utils/metadataIndexChunking.js +9 -4
  57. package/dist/utils/periodicCleanup.js +1 -0
  58. package/package.json +3 -1
package/dist/cli/index.js CHANGED
@@ -11,6 +11,10 @@ import { coreCommands } from './commands/core.js';
11
11
  import { utilityCommands } from './commands/utility.js';
12
12
  import { vfsCommands } from './commands/vfs.js';
13
13
  import { dataCommands } from './commands/data.js';
14
+ import { storageCommands } from './commands/storage.js';
15
+ import { nlpCommands } from './commands/nlp.js';
16
+ import { insightsCommands } from './commands/insights.js';
17
+ import { importCommands } from './commands/import.js';
14
18
  import { readFileSync } from 'fs';
15
19
  import { fileURLToPath } from 'url';
16
20
  import { dirname, join } from 'path';
@@ -21,28 +25,74 @@ const version = packageJson.version;
21
25
  const program = new Command();
22
26
  program
23
27
  .name('brainy')
24
- .description('🧠 Enterprise Neural Intelligence Database')
25
- .version(version)
28
+ .description('🧠 Brainy - The Knowledge Operating System')
29
+ .version(version, '-V, --version', 'Show version number')
26
30
  .option('-v, --verbose', 'Verbose output')
27
31
  .option('--json', 'JSON output format')
28
32
  .option('--pretty', 'Pretty JSON output')
29
- .option('--no-color', 'Disable colored output');
33
+ .option('--no-color', 'Disable colored output')
34
+ .option('-q, --quiet', 'Suppress non-essential output')
35
+ .addHelpText('after', `
36
+ ${chalk.cyan('Examples:')}
37
+ ${chalk.dim('# Core operations')}
38
+ $ brainy add "React is a JavaScript library"
39
+ $ brainy find "JavaScript frameworks"
40
+ $ brainy update <id> --content "Updated content"
41
+ $ brainy delete <id> ${chalk.dim('# Requires confirmation')}
42
+ $ brainy search "react" --type Component --where '{"tested":true}'
43
+
44
+ ${chalk.dim('# Neural API')}
45
+ $ brainy similar "react" "vue"
46
+ $ brainy cluster --algorithm kmeans
47
+ $ brainy related <id> --limit 10
48
+
49
+ ${chalk.dim('# NLP & Entity Extraction')}
50
+ $ brainy extract "Apple announced new iPhone in California"
51
+ $ brainy extract-concepts "Machine learning enables AI"
52
+ $ brainy analyze "Full text analysis with sentiment"
53
+
54
+ ${chalk.dim('# Insights & Analytics')}
55
+ $ brainy insights ${chalk.dim('# Database analytics')}
56
+ $ brainy fields ${chalk.dim('# All metadata fields')}
57
+ $ brainy field-values status ${chalk.dim('# Values for a field')}
58
+ $ brainy query-plan --filters '{"status":"active"}'
59
+
60
+ ${chalk.dim('# VFS operations')}
61
+ $ brainy vfs ls /projects
62
+ $ brainy vfs search "React components"
63
+ $ brainy vfs similar /code/Button.tsx
64
+
65
+ ${chalk.dim('# Storage management (v4.0.0)')}
66
+ $ brainy storage status --quota
67
+ $ brainy storage lifecycle set ${chalk.dim('# Interactive mode')}
68
+ $ brainy storage cost-estimate
69
+ $ brainy storage batch-delete old-ids.txt
70
+
71
+ ${chalk.dim('# Interactive mode')}
72
+ $ brainy interactive
73
+
74
+ ${chalk.cyan('Documentation:')}
75
+ ${chalk.dim('Full docs:')} https://github.com/soulcraftlabs/brainy
76
+ ${chalk.dim('Report issues:')} https://github.com/soulcraftlabs/brainy/issues
77
+
78
+ ${chalk.yellow('💡 Tip:')} All commands work interactively if you omit parameters!
79
+ `);
30
80
  // ===== Core Commands =====
31
81
  program
32
- .command('add <text>')
33
- .description('Add text or JSON to the neural database')
82
+ .command('add [text]')
83
+ .description('Add text or JSON to the neural database (interactive if no text)')
34
84
  .option('-i, --id <id>', 'Specify custom ID')
35
85
  .option('-m, --metadata <json>', 'Add metadata')
36
86
  .option('-t, --type <type>', 'Specify noun type')
37
87
  .action(coreCommands.add);
38
88
  program
39
- .command('find <query>')
40
- .description('Simple NLP search (just like code: brain.find("query"))')
89
+ .command('find [query]')
90
+ .description('Simple NLP search (interactive if no query)')
41
91
  .option('-k, --limit <number>', 'Number of results', '10')
42
92
  .action(coreCommands.search);
43
93
  program
44
- .command('search <query>')
45
- .description('Advanced search with Triple Intelligence™ (vector + graph + field)')
94
+ .command('search [query]')
95
+ .description('Advanced search with Triple Intelligence™ (interactive if no query)')
46
96
  .option('-k, --limit <number>', 'Number of results', '10')
47
97
  .option('--offset <number>', 'Skip N results (pagination)')
48
98
  .option('-t, --threshold <number>', 'Similarity threshold (0-1)', '0.7')
@@ -60,22 +110,47 @@ program
60
110
  .option('--field-weight <n>', 'Field search weight (0-1)')
61
111
  .action(coreCommands.search);
62
112
  program
63
- .command('get <id>')
64
- .description('Get item by ID')
113
+ .command('get [id]')
114
+ .description('Get item by ID (interactive if no ID)')
65
115
  .option('--with-connections', 'Include connections')
66
116
  .action(coreCommands.get);
67
117
  program
68
- .command('relate <source> <verb> <target>')
69
- .description('Create a relationship between items')
118
+ .command('relate [source] [verb] [target]')
119
+ .description('Create a relationship between items (interactive if parameters missing)')
70
120
  .option('-w, --weight <number>', 'Relationship weight')
71
121
  .option('-m, --metadata <json>', 'Relationship metadata')
72
122
  .action(coreCommands.relate);
73
123
  program
74
- .command('import <file>')
75
- .description('Import data from file')
76
- .option('-f, --format <format>', 'Input format (json|csv|jsonl)', 'json')
124
+ .command('update [id]')
125
+ .description('Update an existing entity (interactive if no ID)')
126
+ .option('-c, --content <text>', 'New content')
127
+ .option('-m, --metadata <json>', 'Metadata to merge')
128
+ .option('-t, --type <type>', 'New type')
129
+ .action(coreCommands.update);
130
+ program
131
+ .command('delete [id]')
132
+ .description('Delete an entity (interactive if no ID, requires confirmation)')
133
+ .option('-f, --force', 'Skip confirmation prompt')
134
+ .action(coreCommands.deleteEntity);
135
+ program
136
+ .command('unrelate [id]')
137
+ .description('Remove a relationship (interactive if no ID, requires confirmation)')
138
+ .option('-f, --force', 'Skip confirmation prompt')
139
+ .action(coreCommands.unrelate);
140
+ program
141
+ .command('import [source]')
142
+ .description('Neural import from file, directory, or URL (interactive if no source)')
143
+ .option('-f, --format <format>', 'Format (json|csv|jsonl|yaml|markdown|html|xml|text)')
144
+ .option('--recursive', 'Import directories recursively')
77
145
  .option('--batch-size <number>', 'Batch size for import', '100')
78
- .action(coreCommands.import);
146
+ .option('--extract-concepts', 'Extract concepts as entities')
147
+ .option('--extract-entities', 'Extract named entities (NLP)')
148
+ .option('--detect-relationships', 'Auto-detect relationships', true)
149
+ .option('--confidence <n>', 'Confidence threshold (0-1)', '0.5')
150
+ .option('--progress', 'Show progress')
151
+ .option('--skip-hidden', 'Skip hidden files')
152
+ .option('--skip-node-modules', 'Skip node_modules', true)
153
+ .action(importCommands.import);
79
154
  program
80
155
  .command('export [file]')
81
156
  .description('Export database')
@@ -83,16 +158,16 @@ program
83
158
  .action(coreCommands.export);
84
159
  // ===== Neural Commands =====
85
160
  program
86
- .command('similar <a> <b>')
161
+ .command('similar [a] [b]')
87
162
  .alias('sim')
88
- .description('Calculate similarity between two items')
163
+ .description('Calculate similarity between two items (interactive if parameters missing)')
89
164
  .option('--explain', 'Show detailed explanation')
90
165
  .option('--breakdown', 'Show similarity breakdown')
91
166
  .action(neuralCommands.similar);
92
167
  program
93
168
  .command('cluster')
94
169
  .alias('clusters')
95
- .description('Find semantic clusters in the data')
170
+ .description('Find semantic clusters in the data (interactive mode available)')
96
171
  .option('--algorithm <type>', 'Clustering algorithm (hierarchical|kmeans|dbscan)', 'hierarchical')
97
172
  .option('--threshold <number>', 'Similarity threshold', '0.7')
98
173
  .option('--min-size <number>', 'Minimum cluster size', '2')
@@ -101,18 +176,18 @@ program
101
176
  .option('--show', 'Show visual representation')
102
177
  .action(neuralCommands.cluster);
103
178
  program
104
- .command('related <id>')
179
+ .command('related [id]')
105
180
  .alias('neighbors')
106
- .description('Find semantically related items')
181
+ .description('Find semantically related items (interactive if no ID)')
107
182
  .option('-l, --limit <number>', 'Number of results', '10')
108
183
  .option('-r, --radius <number>', 'Semantic radius', '0.3')
109
184
  .option('--with-scores', 'Include similarity scores')
110
185
  .option('--with-edges', 'Include connections')
111
186
  .action(neuralCommands.related);
112
187
  program
113
- .command('hierarchy <id>')
188
+ .command('hierarchy [id]')
114
189
  .alias('tree')
115
- .description('Show semantic hierarchy for an item')
190
+ .description('Show semantic hierarchy for an item (interactive if no ID)')
116
191
  .option('-d, --depth <number>', 'Hierarchy depth', '3')
117
192
  .option('--parents-only', 'Show only parent hierarchy')
118
193
  .option('--children-only', 'Show only child hierarchy')
@@ -217,6 +292,20 @@ program
217
292
  .option('-d, --depth <number>', 'Max depth', '3')
218
293
  .action((path, options) => {
219
294
  vfsCommands.tree(path, options);
295
+ }))
296
+ .addCommand(new Command('import')
297
+ .argument('[source]', 'File or directory to import')
298
+ .description('Import files/directories into VFS (interactive if no source)')
299
+ .option('--target <path>', 'VFS target path', '/')
300
+ .option('--recursive', 'Import directories recursively', true)
301
+ .option('--generate-embeddings', 'Generate file embeddings', true)
302
+ .option('--extract-metadata', 'Extract file metadata', true)
303
+ .option('--skip-hidden', 'Skip hidden files')
304
+ .option('--skip-node-modules', 'Skip node_modules', true)
305
+ .option('--batch-size <number>', 'Batch size', '100')
306
+ .option('--progress', 'Show progress')
307
+ .action((source, options) => {
308
+ importCommands.vfsImport(source, options);
220
309
  }));
221
310
  // ===== VFS Commands (Backward Compatibility - Deprecated) =====
222
311
  program
@@ -299,6 +388,70 @@ program
299
388
  console.log(chalk.yellow('⚠️ Command "vfs-tree" is deprecated. Use: brainy vfs tree'));
300
389
  vfsCommands.tree(path, options);
301
390
  });
391
+ // ===== Storage Management Commands (v4.0.0) =====
392
+ program
393
+ .command('storage')
394
+ .description('💾 Storage management and cost optimization')
395
+ .addCommand(new Command('status')
396
+ .description('Show storage status and health')
397
+ .option('--detailed', 'Show detailed information')
398
+ .option('--quota', 'Show quota information (OPFS)')
399
+ .action((options) => {
400
+ storageCommands.status(options);
401
+ }))
402
+ .addCommand(new Command('lifecycle')
403
+ .description('Lifecycle policy management')
404
+ .addCommand(new Command('set')
405
+ .argument('[config-file]', 'Policy configuration file (JSON)')
406
+ .description('Set lifecycle policy (interactive if no file)')
407
+ .option('--validate', 'Validate before applying')
408
+ .action((configFile, options) => {
409
+ storageCommands.lifecycle.set(configFile, options);
410
+ }))
411
+ .addCommand(new Command('get')
412
+ .description('Get current lifecycle policy')
413
+ .option('-f, --format <type>', 'Output format (json|yaml)', 'json')
414
+ .action((options) => {
415
+ storageCommands.lifecycle.get(options);
416
+ }))
417
+ .addCommand(new Command('remove')
418
+ .description('Remove lifecycle policy')
419
+ .action((options) => {
420
+ storageCommands.lifecycle.remove(options);
421
+ })))
422
+ .addCommand(new Command('compression')
423
+ .description('Compression management (FileSystem)')
424
+ .addCommand(new Command('enable')
425
+ .description('Enable gzip compression')
426
+ .action((options) => {
427
+ storageCommands.compression.enable(options);
428
+ }))
429
+ .addCommand(new Command('disable')
430
+ .description('Disable compression')
431
+ .action((options) => {
432
+ storageCommands.compression.disable(options);
433
+ }))
434
+ .addCommand(new Command('status')
435
+ .description('Show compression status')
436
+ .action((options) => {
437
+ storageCommands.compression.status(options);
438
+ })))
439
+ .addCommand(new Command('batch-delete')
440
+ .argument('<file>', 'File containing entity IDs (one per line)')
441
+ .description('Batch delete with retry logic')
442
+ .option('--max-retries <n>', 'Maximum retry attempts', '3')
443
+ .option('--continue-on-error', 'Continue if some deletes fail')
444
+ .action((file, options) => {
445
+ storageCommands.batchDelete(file, options);
446
+ }))
447
+ .addCommand(new Command('cost-estimate')
448
+ .description('Estimate cloud storage costs')
449
+ .option('--provider <type>', 'Cloud provider (aws|gcs|azure|r2)')
450
+ .option('--size <gb>', 'Data size in GB')
451
+ .option('--operations <n>', 'Monthly operations')
452
+ .action((options) => {
453
+ storageCommands.costEstimate(options);
454
+ }));
302
455
  // ===== Data Management Commands =====
303
456
  program
304
457
  .command('backup <file>')
@@ -314,6 +467,39 @@ program
314
467
  .command('data-stats')
315
468
  .description('Show detailed database statistics')
316
469
  .action(dataCommands.stats);
470
+ // ===== NLP Commands =====
471
+ program
472
+ .command('extract [text]')
473
+ .description('Extract entities from text using neural NLP (interactive if no text)')
474
+ .action(nlpCommands.extract);
475
+ program
476
+ .command('extract-concepts [text]')
477
+ .description('Extract concepts from text with neural analysis (interactive if no text)')
478
+ .option('--threshold <n>', 'Minimum confidence threshold (0-1)', '0.5')
479
+ .action(nlpCommands.extractConcepts);
480
+ program
481
+ .command('analyze [text]')
482
+ .description('Full NLP analysis: entities, sentiment, topics (interactive if no text)')
483
+ .action(nlpCommands.analyze);
484
+ // ===== Insights & Analytics Commands =====
485
+ program
486
+ .command('insights')
487
+ .description('Get comprehensive database insights and analytics')
488
+ .action(insightsCommands.insights);
489
+ program
490
+ .command('fields')
491
+ .description('List all metadata fields with statistics')
492
+ .action(insightsCommands.fields);
493
+ program
494
+ .command('field-values [field]')
495
+ .description('Get all values for a specific metadata field (interactive if no field)')
496
+ .option('--limit <n>', 'Limit number of values shown', '100')
497
+ .action(insightsCommands.fieldValues);
498
+ program
499
+ .command('query-plan')
500
+ .description('Get optimal query plan for filters')
501
+ .option('--filters <json>', 'Filter JSON to analyze')
502
+ .action(insightsCommands.queryPlan);
317
503
  // ===== Utility Commands =====
318
504
  program
319
505
  .command('stats')