@soulcraft/brainy 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +325 -307
- package/dist/brainyData.d.ts +28 -0
- package/dist/unified.js +102 -2
- package/dist/unified.min.js +1 -1
- package/package.json +1 -4
package/README.md
CHANGED
|
@@ -61,7 +61,8 @@ GitHub Pages that showcases Brainy's main features.
|
|
|
61
61
|
npm install @soulcraft/brainy
|
|
62
62
|
```
|
|
63
63
|
|
|
64
|
-
TensorFlow.js packages are included as bundled dependencies and will be automatically installed without any additional
|
|
64
|
+
TensorFlow.js packages are included as bundled dependencies and will be automatically installed without any additional
|
|
65
|
+
configuration.
|
|
65
66
|
|
|
66
67
|
### Additional Packages
|
|
67
68
|
|
|
@@ -84,7 +85,7 @@ REST API web service wrapper that provides HTTP endpoints for search operations
|
|
|
84
85
|
Brainy uses a unified build that automatically adapts to your environment (Node.js, browser, or serverless):
|
|
85
86
|
|
|
86
87
|
```typescript
|
|
87
|
-
import {
|
|
88
|
+
import {BrainyData, NounType, VerbType} from '@soulcraft/brainy'
|
|
88
89
|
|
|
89
90
|
// Create and initialize the database
|
|
90
91
|
const db = new BrainyData()
|
|
@@ -92,13 +93,13 @@ await db.init()
|
|
|
92
93
|
|
|
93
94
|
// Add data (automatically converted to vectors)
|
|
94
95
|
const catId = await db.add("Cats are independent pets", {
|
|
95
|
-
|
|
96
|
-
|
|
96
|
+
noun: NounType.Thing,
|
|
97
|
+
category: 'animal'
|
|
97
98
|
})
|
|
98
99
|
|
|
99
100
|
const dogId = await db.add("Dogs are loyal companions", {
|
|
100
|
-
|
|
101
|
-
|
|
101
|
+
noun: NounType.Thing,
|
|
102
|
+
category: 'animal'
|
|
102
103
|
})
|
|
103
104
|
|
|
104
105
|
// Search for similar items
|
|
@@ -107,8 +108,8 @@ console.log(results)
|
|
|
107
108
|
|
|
108
109
|
// Add a relationship between items
|
|
109
110
|
await db.addVerb(catId, dogId, {
|
|
110
|
-
|
|
111
|
-
|
|
111
|
+
verb: VerbType.RelatedTo,
|
|
112
|
+
description: 'Both are common household pets'
|
|
112
113
|
})
|
|
113
114
|
```
|
|
114
115
|
|
|
@@ -116,28 +117,30 @@ await db.addVerb(catId, dogId, {
|
|
|
116
117
|
|
|
117
118
|
```typescript
|
|
118
119
|
// Standard import - automatically adapts to any environment
|
|
119
|
-
import {
|
|
120
|
+
import {BrainyData} from '@soulcraft/brainy'
|
|
120
121
|
|
|
121
122
|
// Minified version for production
|
|
122
|
-
import {
|
|
123
|
+
import {BrainyData} from '@soulcraft/brainy/min'
|
|
123
124
|
```
|
|
124
125
|
|
|
125
|
-
> **Note**: The CLI functionality is available as a separate package `@soulcraft/brainy-cli` to reduce the bundle size
|
|
126
|
+
> **Note**: The CLI functionality is available as a separate package `@soulcraft/brainy-cli` to reduce the bundle size
|
|
127
|
+
> of the main package. Install it globally with `npm install -g @soulcraft/brainy-cli` to use the command-line
|
|
128
|
+
> interface.
|
|
126
129
|
|
|
127
130
|
### Browser Usage
|
|
128
131
|
|
|
129
132
|
```html
|
|
130
133
|
|
|
131
134
|
<script type="module">
|
|
132
|
-
|
|
133
|
-
|
|
135
|
+
// Use local files instead of CDN
|
|
136
|
+
import {BrainyData} from './dist/unified.js'
|
|
134
137
|
|
|
135
|
-
|
|
136
|
-
|
|
138
|
+
// Or minified version
|
|
139
|
+
// import { BrainyData } from './dist/unified.min.js'
|
|
137
140
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
+
const db = new BrainyData()
|
|
142
|
+
await db.init()
|
|
143
|
+
// ...
|
|
141
144
|
</script>
|
|
142
145
|
```
|
|
143
146
|
|
|
@@ -292,13 +295,13 @@ The pipeline runs automatically when you:
|
|
|
292
295
|
|
|
293
296
|
```typescript
|
|
294
297
|
// Add data (runs embedding → indexing → storage)
|
|
295
|
-
const id = await db.add("Your text data here", {
|
|
298
|
+
const id = await db.add("Your text data here", {metadata})
|
|
296
299
|
|
|
297
300
|
// Search (runs embedding → similarity search)
|
|
298
301
|
const results = await db.searchText("Your query here", 5)
|
|
299
302
|
|
|
300
303
|
// Connect entities (runs graph construction → storage)
|
|
301
|
-
await db.addVerb(sourceId, targetId, {
|
|
304
|
+
await db.addVerb(sourceId, targetId, {verb: VerbType.RelatedTo})
|
|
302
305
|
```
|
|
303
306
|
|
|
304
307
|
Using the CLI:
|
|
@@ -423,7 +426,8 @@ brainy visualize --root <id> --depth 3
|
|
|
423
426
|
|
|
424
427
|
### Using the CLI in Your Code
|
|
425
428
|
|
|
426
|
-
The CLI functionality is available as a separate package `@soulcraft/brainy-cli`. If you need CLI functionality in your
|
|
429
|
+
The CLI functionality is available as a separate package `@soulcraft/brainy-cli`. If you need CLI functionality in your
|
|
430
|
+
application, install the CLI package:
|
|
427
431
|
|
|
428
432
|
```bash
|
|
429
433
|
npm install @soulcraft/brainy-cli
|
|
@@ -481,7 +485,7 @@ const status = await db.status()
|
|
|
481
485
|
const backupData = await db.backup()
|
|
482
486
|
|
|
483
487
|
// Restore data into the database
|
|
484
|
-
const restoreResult = await db.restore(backupData, {
|
|
488
|
+
const restoreResult = await db.restore(backupData, {clearExisting: true})
|
|
485
489
|
```
|
|
486
490
|
|
|
487
491
|
### Working with Nouns (Entities)
|
|
@@ -489,25 +493,25 @@ const restoreResult = await db.restore(backupData, { clearExisting: true })
|
|
|
489
493
|
```typescript
|
|
490
494
|
// Add a noun (automatically vectorized)
|
|
491
495
|
const id = await db.add(textOrVector, {
|
|
492
|
-
|
|
493
|
-
|
|
496
|
+
noun: NounType.Thing,
|
|
497
|
+
// other metadata...
|
|
494
498
|
})
|
|
495
499
|
|
|
496
500
|
// Add multiple nouns in parallel (with multithreading and batch embedding)
|
|
497
501
|
const ids = await db.addBatch([
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
502
|
+
{
|
|
503
|
+
vectorOrData: "First item to add",
|
|
504
|
+
metadata: {noun: NounType.Thing, category: 'example'}
|
|
505
|
+
},
|
|
506
|
+
{
|
|
507
|
+
vectorOrData: "Second item to add",
|
|
508
|
+
metadata: {noun: NounType.Thing, category: 'example'}
|
|
509
|
+
},
|
|
510
|
+
// More items...
|
|
507
511
|
], {
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
512
|
+
forceEmbed: false,
|
|
513
|
+
concurrency: 4, // Control the level of parallelism (default: 4)
|
|
514
|
+
batchSize: 50 // Control the number of items to process in a single batch (default: 50)
|
|
511
515
|
})
|
|
512
516
|
|
|
513
517
|
// Retrieve a noun
|
|
@@ -515,8 +519,8 @@ const noun = await db.get(id)
|
|
|
515
519
|
|
|
516
520
|
// Update noun metadata
|
|
517
521
|
await db.updateMetadata(id, {
|
|
518
|
-
|
|
519
|
-
|
|
522
|
+
noun: NounType.Thing,
|
|
523
|
+
// updated metadata...
|
|
520
524
|
})
|
|
521
525
|
|
|
522
526
|
// Delete a noun
|
|
@@ -535,8 +539,21 @@ const thingNouns = await db.searchByNounTypes([NounType.Thing], numResults)
|
|
|
535
539
|
```typescript
|
|
536
540
|
// Add a relationship between nouns
|
|
537
541
|
await db.addVerb(sourceId, targetId, {
|
|
538
|
-
|
|
539
|
-
|
|
542
|
+
verb: VerbType.RelatedTo,
|
|
543
|
+
// other metadata...
|
|
544
|
+
})
|
|
545
|
+
|
|
546
|
+
// Add a relationship with auto-creation of missing nouns
|
|
547
|
+
// This is useful when the target noun might not exist yet
|
|
548
|
+
await db.addVerb(sourceId, targetId, {
|
|
549
|
+
verb: VerbType.RelatedTo,
|
|
550
|
+
// Enable auto-creation of missing nouns
|
|
551
|
+
autoCreateMissingNouns: true,
|
|
552
|
+
// Optional metadata for auto-created nouns
|
|
553
|
+
missingNounMetadata: {
|
|
554
|
+
noun: NounType.Concept,
|
|
555
|
+
description: 'Auto-created noun'
|
|
556
|
+
}
|
|
540
557
|
})
|
|
541
558
|
|
|
542
559
|
// Get all relationships
|
|
@@ -564,20 +581,20 @@ await db.deleteVerb(verbId)
|
|
|
564
581
|
|
|
565
582
|
```typescript
|
|
566
583
|
import {
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
584
|
+
BrainyData,
|
|
585
|
+
createTensorFlowEmbeddingFunction,
|
|
586
|
+
createThreadedEmbeddingFunction
|
|
570
587
|
} from '@soulcraft/brainy'
|
|
571
588
|
|
|
572
589
|
// Use the standard TensorFlow Universal Sentence Encoder embedding function
|
|
573
590
|
const db = new BrainyData({
|
|
574
|
-
|
|
591
|
+
embeddingFunction: createTensorFlowEmbeddingFunction()
|
|
575
592
|
})
|
|
576
593
|
await db.init()
|
|
577
594
|
|
|
578
595
|
// Or use the threaded embedding function for better performance
|
|
579
596
|
const threadedDb = new BrainyData({
|
|
580
|
-
|
|
597
|
+
embeddingFunction: createThreadedEmbeddingFunction()
|
|
581
598
|
})
|
|
582
599
|
await threadedDb.init()
|
|
583
600
|
|
|
@@ -618,42 +635,42 @@ Brainy includes comprehensive multithreading support to improve performance acro
|
|
|
618
635
|
7. **Automatic Environment Detection**: Adapts to browser (Web Workers) and Node.js (Worker Threads) environments
|
|
619
636
|
|
|
620
637
|
```typescript
|
|
621
|
-
import {
|
|
638
|
+
import {BrainyData, euclideanDistance} from '@soulcraft/brainy'
|
|
622
639
|
|
|
623
640
|
// Configure with custom options
|
|
624
641
|
const db = new BrainyData({
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
642
|
+
// Use Euclidean distance instead of default cosine distance
|
|
643
|
+
distanceFunction: euclideanDistance,
|
|
644
|
+
|
|
645
|
+
// HNSW index configuration for search performance
|
|
646
|
+
hnsw: {
|
|
647
|
+
M: 16, // Max connections per noun
|
|
648
|
+
efConstruction: 200, // Construction candidate list size
|
|
649
|
+
efSearch: 50, // Search candidate list size
|
|
650
|
+
},
|
|
651
|
+
|
|
652
|
+
// Performance optimization options
|
|
653
|
+
performance: {
|
|
654
|
+
useParallelization: true, // Enable multithreaded search operations
|
|
655
|
+
},
|
|
656
|
+
|
|
657
|
+
// Noun and Verb type validation
|
|
658
|
+
typeValidation: {
|
|
659
|
+
enforceNounTypes: true, // Validate noun types against NounType enum
|
|
660
|
+
enforceVerbTypes: true, // Validate verb types against VerbType enum
|
|
661
|
+
},
|
|
662
|
+
|
|
663
|
+
// Storage configuration
|
|
664
|
+
storage: {
|
|
665
|
+
requestPersistentStorage: true,
|
|
666
|
+
// Example configuration for cloud storage (replace with your own values):
|
|
667
|
+
// s3Storage: {
|
|
668
|
+
// bucketName: 'your-s3-bucket-name',
|
|
669
|
+
// region: 'your-aws-region'
|
|
670
|
+
// // Credentials should be provided via environment variables
|
|
671
|
+
// // AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
|
|
672
|
+
// }
|
|
673
|
+
}
|
|
657
674
|
})
|
|
658
675
|
```
|
|
659
676
|
|
|
@@ -667,34 +684,34 @@ hybrid approach:
|
|
|
667
684
|
3. **Memory-Efficient Indexing** - Optimizes memory usage for large-scale vector collections
|
|
668
685
|
|
|
669
686
|
```typescript
|
|
670
|
-
import {
|
|
687
|
+
import {BrainyData} from '@soulcraft/brainy'
|
|
671
688
|
|
|
672
689
|
// Configure with optimized HNSW index for large datasets
|
|
673
690
|
const db = new BrainyData({
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
691
|
+
hnswOptimized: {
|
|
692
|
+
// Standard HNSW parameters
|
|
693
|
+
M: 16, // Max connections per noun
|
|
694
|
+
efConstruction: 200, // Construction candidate list size
|
|
695
|
+
efSearch: 50, // Search candidate list size
|
|
696
|
+
|
|
697
|
+
// Memory threshold in bytes - when exceeded, will use disk-based approach
|
|
698
|
+
memoryThreshold: 1024 * 1024 * 1024, // 1GB default threshold
|
|
699
|
+
|
|
700
|
+
// Product quantization settings for dimensionality reduction
|
|
701
|
+
productQuantization: {
|
|
702
|
+
enabled: true, // Enable product quantization
|
|
703
|
+
numSubvectors: 16, // Number of subvectors to split the vector into
|
|
704
|
+
numCentroids: 256 // Number of centroids per subvector
|
|
705
|
+
},
|
|
706
|
+
|
|
707
|
+
// Whether to use disk-based storage for the index
|
|
708
|
+
useDiskBasedIndex: true // Enable disk-based storage
|
|
688
709
|
},
|
|
689
710
|
|
|
690
|
-
//
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
// Storage configuration (required for disk-based index)
|
|
695
|
-
storage: {
|
|
696
|
-
requestPersistentStorage: true
|
|
697
|
-
}
|
|
711
|
+
// Storage configuration (required for disk-based index)
|
|
712
|
+
storage: {
|
|
713
|
+
requestPersistentStorage: true
|
|
714
|
+
}
|
|
698
715
|
})
|
|
699
716
|
|
|
700
717
|
// The optimized index automatically adapts based on dataset size:
|
|
@@ -759,24 +776,24 @@ Brainy's restore functionality can handle:
|
|
|
759
776
|
```typescript
|
|
760
777
|
// Restore data with all options
|
|
761
778
|
const restoreResult = await db.restore(backupData, {
|
|
762
|
-
|
|
779
|
+
clearExisting: true // Whether to clear existing data before restore
|
|
763
780
|
})
|
|
764
781
|
|
|
765
782
|
// Import sparse data (without vectors)
|
|
766
783
|
// Vectors will be automatically created using the embedding function
|
|
767
784
|
const sparseData = {
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
785
|
+
nouns: [
|
|
786
|
+
{
|
|
787
|
+
id: '123',
|
|
788
|
+
// No vector field - will be created during import
|
|
789
|
+
metadata: {
|
|
790
|
+
noun: 'Thing',
|
|
791
|
+
text: 'This text will be used to generate a vector'
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
],
|
|
795
|
+
verbs: [],
|
|
796
|
+
version: '1.0.0'
|
|
780
797
|
}
|
|
781
798
|
|
|
782
799
|
const sparseImportResult = await db.importSparseData(sparseData)
|
|
@@ -823,82 +840,82 @@ boilerplate:
|
|
|
823
840
|
|
|
824
841
|
```typescript
|
|
825
842
|
import {
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
843
|
+
createMemoryAugmentation,
|
|
844
|
+
createConduitAugmentation,
|
|
845
|
+
createSenseAugmentation,
|
|
846
|
+
addWebSocketSupport,
|
|
847
|
+
executeStreamlined,
|
|
848
|
+
processStaticData,
|
|
849
|
+
processStreamingData,
|
|
850
|
+
createPipeline
|
|
834
851
|
} from '@soulcraft/brainy'
|
|
835
852
|
|
|
836
853
|
// Create a memory augmentation with minimal code
|
|
837
854
|
const memoryAug = createMemoryAugmentation({
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
855
|
+
name: 'simple-memory',
|
|
856
|
+
description: 'A simple in-memory storage augmentation',
|
|
857
|
+
autoRegister: true,
|
|
858
|
+
autoInitialize: true,
|
|
859
|
+
|
|
860
|
+
// Implement only the methods you need
|
|
861
|
+
storeData: async (key, data) => {
|
|
862
|
+
// Your implementation here
|
|
863
|
+
return {
|
|
864
|
+
success: true,
|
|
865
|
+
data: true
|
|
866
|
+
}
|
|
867
|
+
},
|
|
851
868
|
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
869
|
+
retrieveData: async (key) => {
|
|
870
|
+
// Your implementation here
|
|
871
|
+
return {
|
|
872
|
+
success: true,
|
|
873
|
+
data: {example: 'data', key}
|
|
874
|
+
}
|
|
857
875
|
}
|
|
858
|
-
}
|
|
859
876
|
})
|
|
860
877
|
|
|
861
878
|
// Add WebSocket support to any augmentation
|
|
862
879
|
const wsAugmentation = addWebSocketSupport(memoryAug, {
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
880
|
+
connectWebSocket: async (url) => {
|
|
881
|
+
// Your implementation here
|
|
882
|
+
return {
|
|
883
|
+
connectionId: 'ws-1',
|
|
884
|
+
url,
|
|
885
|
+
status: 'connected'
|
|
886
|
+
}
|
|
869
887
|
}
|
|
870
|
-
}
|
|
871
888
|
})
|
|
872
889
|
|
|
873
890
|
// Process static data through a pipeline
|
|
874
891
|
const result = await processStaticData(
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
892
|
+
'Input data',
|
|
893
|
+
[
|
|
894
|
+
{
|
|
895
|
+
augmentation: senseAug,
|
|
896
|
+
method: 'processRawData',
|
|
897
|
+
transformArgs: (data) => [data, 'text']
|
|
898
|
+
},
|
|
899
|
+
{
|
|
900
|
+
augmentation: memoryAug,
|
|
901
|
+
method: 'storeData',
|
|
902
|
+
transformArgs: (data) => ['processed-data', data]
|
|
903
|
+
}
|
|
904
|
+
]
|
|
888
905
|
)
|
|
889
906
|
|
|
890
907
|
// Create a reusable pipeline
|
|
891
908
|
const pipeline = createPipeline([
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
909
|
+
{
|
|
910
|
+
augmentation: senseAug,
|
|
911
|
+
method: 'processRawData',
|
|
912
|
+
transformArgs: (data) => [data, 'text']
|
|
913
|
+
},
|
|
914
|
+
{
|
|
915
|
+
augmentation: memoryAug,
|
|
916
|
+
method: 'storeData',
|
|
917
|
+
transformArgs: (data) => ['processed-data', data]
|
|
918
|
+
}
|
|
902
919
|
])
|
|
903
920
|
|
|
904
921
|
// Use the pipeline
|
|
@@ -906,11 +923,11 @@ const result = await pipeline('New input data')
|
|
|
906
923
|
|
|
907
924
|
// Dynamically load augmentations at runtime
|
|
908
925
|
const loadedAugmentations = await loadAugmentationModule(
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
926
|
+
import('./my-augmentations.js'),
|
|
927
|
+
{
|
|
928
|
+
autoRegister: true,
|
|
929
|
+
autoInitialize: true
|
|
930
|
+
}
|
|
914
931
|
)
|
|
915
932
|
```
|
|
916
933
|
|
|
@@ -924,60 +941,61 @@ The simplified augmentation system provides:
|
|
|
924
941
|
|
|
925
942
|
#### WebSocket Augmentation Types
|
|
926
943
|
|
|
927
|
-
Brainy exports several WebSocket augmentation types that can be used by augmentation creators to add WebSocket
|
|
944
|
+
Brainy exports several WebSocket augmentation types that can be used by augmentation creators to add WebSocket
|
|
945
|
+
capabilities to their augmentations:
|
|
928
946
|
|
|
929
947
|
```typescript
|
|
930
948
|
import {
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
949
|
+
// Base WebSocket support interface
|
|
950
|
+
IWebSocketSupport,
|
|
951
|
+
|
|
952
|
+
// Combined WebSocket augmentation types
|
|
953
|
+
IWebSocketSenseAugmentation,
|
|
954
|
+
IWebSocketConduitAugmentation,
|
|
955
|
+
IWebSocketCognitionAugmentation,
|
|
956
|
+
IWebSocketMemoryAugmentation,
|
|
957
|
+
IWebSocketPerceptionAugmentation,
|
|
958
|
+
IWebSocketDialogAugmentation,
|
|
959
|
+
IWebSocketActivationAugmentation,
|
|
960
|
+
|
|
961
|
+
// Function to add WebSocket support to any augmentation
|
|
962
|
+
addWebSocketSupport
|
|
945
963
|
} from '@soulcraft/brainy'
|
|
946
964
|
|
|
947
965
|
// Example: Creating a typed WebSocket-enabled sense augmentation
|
|
948
966
|
const mySenseAug = createSenseAugmentation({
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
967
|
+
name: 'my-sense',
|
|
968
|
+
processRawData: async (data, dataType) => {
|
|
969
|
+
// Implementation
|
|
970
|
+
return {
|
|
971
|
+
success: true,
|
|
972
|
+
data: {nouns: [], verbs: []}
|
|
973
|
+
}
|
|
955
974
|
}
|
|
956
|
-
}
|
|
957
975
|
}) as IWebSocketSenseAugmentation
|
|
958
976
|
|
|
959
977
|
// Add WebSocket support
|
|
960
978
|
addWebSocketSupport(mySenseAug, {
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
979
|
+
connectWebSocket: async (url) => {
|
|
980
|
+
// WebSocket implementation
|
|
981
|
+
return {
|
|
982
|
+
connectionId: 'ws-1',
|
|
983
|
+
url,
|
|
984
|
+
status: 'connected'
|
|
985
|
+
}
|
|
986
|
+
},
|
|
987
|
+
sendWebSocketMessage: async (connectionId, data) => {
|
|
988
|
+
// Send message implementation
|
|
989
|
+
},
|
|
990
|
+
onWebSocketMessage: async (connectionId, callback) => {
|
|
991
|
+
// Register callback implementation
|
|
992
|
+
},
|
|
993
|
+
offWebSocketMessage: async (connectionId, callback) => {
|
|
994
|
+
// Remove callback implementation
|
|
995
|
+
},
|
|
996
|
+
closeWebSocket: async (connectionId, code, reason) => {
|
|
997
|
+
// Close connection implementation
|
|
967
998
|
}
|
|
968
|
-
},
|
|
969
|
-
sendWebSocketMessage: async (connectionId, data) => {
|
|
970
|
-
// Send message implementation
|
|
971
|
-
},
|
|
972
|
-
onWebSocketMessage: async (connectionId, callback) => {
|
|
973
|
-
// Register callback implementation
|
|
974
|
-
},
|
|
975
|
-
offWebSocketMessage: async (connectionId, callback) => {
|
|
976
|
-
// Remove callback implementation
|
|
977
|
-
},
|
|
978
|
-
closeWebSocket: async (connectionId, code, reason) => {
|
|
979
|
-
// Close connection implementation
|
|
980
|
-
}
|
|
981
999
|
})
|
|
982
1000
|
|
|
983
1001
|
// Now mySenseAug has both sense augmentation methods and WebSocket methods
|
|
@@ -985,7 +1003,8 @@ await mySenseAug.processRawData('data', 'text')
|
|
|
985
1003
|
await mySenseAug.connectWebSocket('wss://example.com')
|
|
986
1004
|
```
|
|
987
1005
|
|
|
988
|
-
These WebSocket augmentation types combine the base augmentation interfaces with the `IWebSocketSupport` interface,
|
|
1006
|
+
These WebSocket augmentation types combine the base augmentation interfaces with the `IWebSocketSupport` interface,
|
|
1007
|
+
providing type safety and autocompletion for augmentations with WebSocket capabilities.
|
|
989
1008
|
|
|
990
1009
|
### Model Control Protocol (MCP)
|
|
991
1010
|
|
|
@@ -1014,13 +1033,13 @@ everywhere.
|
|
|
1014
1033
|
Brainy automatically detects the environment it's running in:
|
|
1015
1034
|
|
|
1016
1035
|
```typescript
|
|
1017
|
-
import {
|
|
1036
|
+
import {environment} from '@soulcraft/brainy'
|
|
1018
1037
|
|
|
1019
1038
|
// Check which environment we're running in
|
|
1020
1039
|
console.log(`Running in ${
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1040
|
+
environment.isBrowser ? 'browser' :
|
|
1041
|
+
environment.isNode ? 'Node.js' :
|
|
1042
|
+
'serverless/unknown'
|
|
1024
1043
|
} environment`)
|
|
1025
1044
|
```
|
|
1026
1045
|
|
|
@@ -1049,7 +1068,6 @@ Works in all modern browsers:
|
|
|
1049
1068
|
|
|
1050
1069
|
For browsers without OPFS support, falls back to in-memory storage.
|
|
1051
1070
|
|
|
1052
|
-
|
|
1053
1071
|
## Related Projects
|
|
1054
1072
|
|
|
1055
1073
|
- **[Cartographer](https://github.com/sodal-project/cartographer)** - A companion project that provides standardized
|
|
@@ -1094,9 +1112,9 @@ You can use the conduit augmentations to sync Brainy instances:
|
|
|
1094
1112
|
|
|
1095
1113
|
```typescript
|
|
1096
1114
|
import {
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1115
|
+
BrainyData,
|
|
1116
|
+
pipeline,
|
|
1117
|
+
createConduitAugmentation
|
|
1100
1118
|
} from '@soulcraft/brainy'
|
|
1101
1119
|
|
|
1102
1120
|
// Create and initialize the database
|
|
@@ -1112,36 +1130,36 @@ pipeline.register(wsConduit)
|
|
|
1112
1130
|
// Connect to another Brainy instance (server or browser)
|
|
1113
1131
|
// Replace the example URL below with your actual WebSocket server URL
|
|
1114
1132
|
const connectionResult = await pipeline.executeConduitPipeline(
|
|
1115
|
-
|
|
1116
|
-
|
|
1133
|
+
'establishConnection',
|
|
1134
|
+
['wss://example-websocket-server.com/brainy-sync', {protocols: 'brainy-sync'}]
|
|
1117
1135
|
)
|
|
1118
1136
|
|
|
1119
1137
|
if (connectionResult[0] && (await connectionResult[0]).success) {
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
}
|
|
1135
|
-
|
|
1136
|
-
// Set up real-time sync by monitoring the stream
|
|
1137
|
-
await wsConduit.monitorStream(connection.connectionId, async (data) => {
|
|
1138
|
-
// Handle incoming data (e.g., new nouns, verbs, updates)
|
|
1139
|
-
if (data.type === 'newNoun') {
|
|
1140
|
-
await db.add(data.vector, data.metadata)
|
|
1141
|
-
} else if (data.type === 'newVerb') {
|
|
1142
|
-
await db.addVerb(data.sourceId, data.targetId, data.vector, data.options)
|
|
1138
|
+
const connection = (await connectionResult[0]).data
|
|
1139
|
+
|
|
1140
|
+
// Read data from the remote instance
|
|
1141
|
+
const readResult = await pipeline.executeConduitPipeline(
|
|
1142
|
+
'readData',
|
|
1143
|
+
[{connectionId: connection.connectionId, query: {type: 'getAllNouns'}}]
|
|
1144
|
+
)
|
|
1145
|
+
|
|
1146
|
+
// Process and add the received data to the local instance
|
|
1147
|
+
if (readResult[0] && (await readResult[0]).success) {
|
|
1148
|
+
const remoteNouns = (await readResult[0]).data
|
|
1149
|
+
for (const noun of remoteNouns) {
|
|
1150
|
+
await db.add(noun.vector, noun.metadata)
|
|
1151
|
+
}
|
|
1143
1152
|
}
|
|
1144
|
-
|
|
1153
|
+
|
|
1154
|
+
// Set up real-time sync by monitoring the stream
|
|
1155
|
+
await wsConduit.monitorStream(connection.connectionId, async (data) => {
|
|
1156
|
+
// Handle incoming data (e.g., new nouns, verbs, updates)
|
|
1157
|
+
if (data.type === 'newNoun') {
|
|
1158
|
+
await db.add(data.vector, data.metadata)
|
|
1159
|
+
} else if (data.type === 'newVerb') {
|
|
1160
|
+
await db.addVerb(data.sourceId, data.targetId, data.vector, data.options)
|
|
1161
|
+
}
|
|
1162
|
+
})
|
|
1145
1163
|
}
|
|
1146
1164
|
```
|
|
1147
1165
|
|
|
@@ -1149,9 +1167,9 @@ if (connectionResult[0] && (await connectionResult[0]).success) {
|
|
|
1149
1167
|
|
|
1150
1168
|
```typescript
|
|
1151
1169
|
import {
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1170
|
+
BrainyData,
|
|
1171
|
+
pipeline,
|
|
1172
|
+
createConduitAugmentation
|
|
1155
1173
|
} from '@soulcraft/brainy'
|
|
1156
1174
|
|
|
1157
1175
|
// Create and initialize the database
|
|
@@ -1167,48 +1185,48 @@ pipeline.register(webrtcConduit)
|
|
|
1167
1185
|
// Connect to a peer using a signaling server
|
|
1168
1186
|
// Replace the example values below with your actual configuration
|
|
1169
1187
|
const connectionResult = await pipeline.executeConduitPipeline(
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1188
|
+
'establishConnection',
|
|
1189
|
+
[
|
|
1190
|
+
'peer-id-to-connect-to', // Replace with actual peer ID
|
|
1191
|
+
{
|
|
1192
|
+
signalServerUrl: 'wss://example-signal-server.com', // Replace with your signal server
|
|
1193
|
+
localPeerId: 'my-local-peer-id', // Replace with your local peer ID
|
|
1194
|
+
iceServers: [{urls: 'stun:stun.l.google.com:19302'}] // Public STUN server
|
|
1195
|
+
}
|
|
1196
|
+
]
|
|
1179
1197
|
)
|
|
1180
1198
|
|
|
1181
1199
|
if (connectionResult[0] && (await connectionResult[0]).success) {
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
}
|
|
1192
|
-
})
|
|
1193
|
-
|
|
1194
|
-
// When adding new data locally, also send to the peer
|
|
1195
|
-
const nounId = await db.add("New data to sync", { noun: "Thing" })
|
|
1196
|
-
|
|
1197
|
-
// Send the new noun to the peer
|
|
1198
|
-
await pipeline.executeConduitPipeline(
|
|
1199
|
-
'writeData',
|
|
1200
|
-
[
|
|
1201
|
-
{
|
|
1202
|
-
connectionId: connection.connectionId,
|
|
1203
|
-
data: {
|
|
1204
|
-
type: 'newNoun',
|
|
1205
|
-
id: nounId,
|
|
1206
|
-
vector: (await db.get(nounId)).vector,
|
|
1207
|
-
metadata: (await db.get(nounId)).metadata
|
|
1200
|
+
const connection = (await connectionResult[0]).data
|
|
1201
|
+
|
|
1202
|
+
// Set up real-time sync by monitoring the stream
|
|
1203
|
+
await webrtcConduit.monitorStream(connection.connectionId, async (data) => {
|
|
1204
|
+
// Handle incoming data (e.g., new nouns, verbs, updates)
|
|
1205
|
+
if (data.type === 'newNoun') {
|
|
1206
|
+
await db.add(data.vector, data.metadata)
|
|
1207
|
+
} else if (data.type === 'newVerb') {
|
|
1208
|
+
await db.addVerb(data.sourceId, data.targetId, data.vector, data.options)
|
|
1208
1209
|
}
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1210
|
+
})
|
|
1211
|
+
|
|
1212
|
+
// When adding new data locally, also send to the peer
|
|
1213
|
+
const nounId = await db.add("New data to sync", {noun: "Thing"})
|
|
1214
|
+
|
|
1215
|
+
// Send the new noun to the peer
|
|
1216
|
+
await pipeline.executeConduitPipeline(
|
|
1217
|
+
'writeData',
|
|
1218
|
+
[
|
|
1219
|
+
{
|
|
1220
|
+
connectionId: connection.connectionId,
|
|
1221
|
+
data: {
|
|
1222
|
+
type: 'newNoun',
|
|
1223
|
+
id: nounId,
|
|
1224
|
+
vector: (await db.get(nounId)).vector,
|
|
1225
|
+
metadata: (await db.get(nounId)).metadata
|
|
1226
|
+
}
|
|
1227
|
+
}
|
|
1228
|
+
]
|
|
1229
|
+
)
|
|
1212
1230
|
}
|
|
1213
1231
|
```
|
|
1214
1232
|
|
|
@@ -1218,39 +1236,39 @@ Brainy supports searching a server-hosted instance from a browser, storing resul
|
|
|
1218
1236
|
searches against the local instance:
|
|
1219
1237
|
|
|
1220
1238
|
```typescript
|
|
1221
|
-
import {
|
|
1239
|
+
import {BrainyData} from '@soulcraft/brainy'
|
|
1222
1240
|
|
|
1223
1241
|
// Create and initialize the database with remote server configuration
|
|
1224
1242
|
// Replace the example URL below with your actual Brainy server URL
|
|
1225
1243
|
const db = new BrainyData({
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1244
|
+
remoteServer: {
|
|
1245
|
+
url: 'wss://example-brainy-server.com/ws', // Replace with your server URL
|
|
1246
|
+
protocols: 'brainy-sync',
|
|
1247
|
+
autoConnect: true // Connect automatically during initialization
|
|
1248
|
+
}
|
|
1231
1249
|
})
|
|
1232
1250
|
await db.init()
|
|
1233
1251
|
|
|
1234
1252
|
// Or connect manually after initialization
|
|
1235
1253
|
if (!db.isConnectedToRemoteServer()) {
|
|
1236
|
-
|
|
1237
|
-
|
|
1254
|
+
// Replace the example URL below with your actual Brainy server URL
|
|
1255
|
+
await db.connectToRemoteServer('wss://example-brainy-server.com/ws', 'brainy-sync')
|
|
1238
1256
|
}
|
|
1239
1257
|
|
|
1240
1258
|
// Search the remote server (results are stored locally)
|
|
1241
|
-
const remoteResults = await db.searchText('machine learning', 5, {
|
|
1259
|
+
const remoteResults = await db.searchText('machine learning', 5, {searchMode: 'remote'})
|
|
1242
1260
|
|
|
1243
1261
|
// Search the local database (includes previously stored results)
|
|
1244
|
-
const localResults = await db.searchText('machine learning', 5, {
|
|
1262
|
+
const localResults = await db.searchText('machine learning', 5, {searchMode: 'local'})
|
|
1245
1263
|
|
|
1246
1264
|
// Perform a combined search (local first, then remote if needed)
|
|
1247
|
-
const combinedResults = await db.searchText('neural networks', 5, {
|
|
1265
|
+
const combinedResults = await db.searchText('neural networks', 5, {searchMode: 'combined'})
|
|
1248
1266
|
|
|
1249
1267
|
// Add data to both local and remote instances
|
|
1250
1268
|
const id = await db.addToBoth('Deep learning is a subset of machine learning', {
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1269
|
+
noun: 'Concept',
|
|
1270
|
+
category: 'AI',
|
|
1271
|
+
tags: ['deep learning', 'neural networks']
|
|
1254
1272
|
})
|
|
1255
1273
|
|
|
1256
1274
|
// Clean up when done (this also cleans up worker pools)
|