@soulcraft/brainy 0.62.3 → 1.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/bin/brainy.js +903 -1153
- package/dist/augmentationPipeline.d.ts +60 -0
- package/dist/augmentationPipeline.js +94 -0
- package/dist/augmentations/{cortexSense.d.ts → neuralImport.d.ts} +14 -11
- package/dist/augmentations/{cortexSense.js → neuralImport.js} +14 -11
- package/dist/brainyData.d.ts +199 -18
- package/dist/brainyData.js +601 -18
- package/dist/chat/BrainyChat.d.ts +113 -0
- package/dist/chat/BrainyChat.js +368 -0
- package/dist/chat/ChatCLI.d.ts +61 -0
- package/dist/chat/ChatCLI.js +351 -0
- package/dist/connectors/interfaces/IConnector.d.ts +3 -3
- package/dist/connectors/interfaces/IConnector.js +1 -1
- package/dist/cortex/neuralImport.js +1 -3
- package/dist/index.d.ts +4 -6
- package/dist/index.js +6 -7
- package/dist/pipeline.d.ts +15 -271
- package/dist/pipeline.js +25 -586
- package/dist/shared/default-augmentations.d.ts +3 -3
- package/dist/shared/default-augmentations.js +10 -10
- package/package.json +3 -1
- package/dist/chat/brainyChat.d.ts +0 -42
- package/dist/chat/brainyChat.js +0 -340
- package/dist/cortex/cliWrapper.d.ts +0 -32
- package/dist/cortex/cliWrapper.js +0 -209
- package/dist/cortex/cortex-legacy.d.ts +0 -264
- package/dist/cortex/cortex-legacy.js +0 -2463
- package/dist/cortex/cortex.d.ts +0 -264
- package/dist/cortex/cortex.js +0 -2463
- package/dist/cortex/serviceIntegration.d.ts +0 -156
- package/dist/cortex/serviceIntegration.js +0 -384
- package/dist/sequentialPipeline.d.ts +0 -113
- package/dist/sequentialPipeline.js +0 -417
- package/dist/utils/modelLoader.d.ts +0 -12
- package/dist/utils/modelLoader.js +0 -88
package/dist/brainyData.js
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import { v4 as uuidv4 } from './universal/uuid.js';
|
|
6
6
|
import { HNSWIndex } from './hnsw/hnswIndex.js';
|
|
7
|
+
import { ExecutionMode } from './augmentationPipeline.js';
|
|
7
8
|
import { HNSWIndexOptimized } from './hnsw/hnswIndexOptimized.js';
|
|
8
9
|
import { createStorage } from './storage/storageFactory.js';
|
|
9
10
|
import { cosineDistance, defaultEmbeddingFunction, cleanupWorkerPools, batchEmbed } from './utils/index.js';
|
|
@@ -983,12 +984,24 @@ export class BrainyData {
|
|
|
983
984
|
}
|
|
984
985
|
}
|
|
985
986
|
/**
|
|
986
|
-
* Add
|
|
987
|
-
*
|
|
987
|
+
* Add data to the database with intelligent processing
|
|
988
|
+
*
|
|
988
989
|
* @param vectorOrData Vector or data to add
|
|
989
|
-
* @param metadata Optional metadata to associate with the
|
|
990
|
-
* @param options Additional options
|
|
991
|
-
* @returns The ID of the added
|
|
990
|
+
* @param metadata Optional metadata to associate with the data
|
|
991
|
+
* @param options Additional options for processing
|
|
992
|
+
* @returns The ID of the added data
|
|
993
|
+
*
|
|
994
|
+
* @example
|
|
995
|
+
* // Auto mode - intelligently decides processing
|
|
996
|
+
* await brainy.add("Customer feedback: Great product!")
|
|
997
|
+
*
|
|
998
|
+
* @example
|
|
999
|
+
* // Explicit literal mode for sensitive data
|
|
1000
|
+
* await brainy.add("API_KEY=secret123", null, { process: 'literal' })
|
|
1001
|
+
*
|
|
1002
|
+
* @example
|
|
1003
|
+
* // Force neural processing
|
|
1004
|
+
* await brainy.add("John works at Acme Corp", null, { process: 'neural' })
|
|
992
1005
|
*/
|
|
993
1006
|
async add(vectorOrData, metadata, options = {}) {
|
|
994
1007
|
await this.ensureInitialized();
|
|
@@ -1246,6 +1259,31 @@ export class BrainyData {
|
|
|
1246
1259
|
}
|
|
1247
1260
|
// Invalidate search cache since data has changed
|
|
1248
1261
|
this.searchCache.invalidateOnDataChange('add');
|
|
1262
|
+
// Determine processing mode
|
|
1263
|
+
const processingMode = options.process || 'auto';
|
|
1264
|
+
let shouldProcessNeurally = false;
|
|
1265
|
+
if (processingMode === 'neural') {
|
|
1266
|
+
shouldProcessNeurally = true;
|
|
1267
|
+
}
|
|
1268
|
+
else if (processingMode === 'auto') {
|
|
1269
|
+
// Auto-detect whether to use neural processing
|
|
1270
|
+
shouldProcessNeurally = this.shouldAutoProcessNeurally(vectorOrData, metadata);
|
|
1271
|
+
}
|
|
1272
|
+
// 'literal' mode means no neural processing
|
|
1273
|
+
// 🧠 AI Processing (Neural Import) - Based on processing mode
|
|
1274
|
+
if (shouldProcessNeurally) {
|
|
1275
|
+
try {
|
|
1276
|
+
// Execute SENSE pipeline (includes Neural Import and other AI augmentations)
|
|
1277
|
+
await augmentationPipeline.executeSensePipeline('processRawData', [vectorOrData, typeof vectorOrData === 'string' ? 'text' : 'data'], { mode: ExecutionMode.SEQUENTIAL });
|
|
1278
|
+
if (this.loggingConfig?.verbose) {
|
|
1279
|
+
console.log(`🧠 AI processing completed for data: ${id}`);
|
|
1280
|
+
}
|
|
1281
|
+
}
|
|
1282
|
+
catch (processingError) {
|
|
1283
|
+
// Don't fail the add operation if processing fails
|
|
1284
|
+
console.warn(`🧠 AI processing failed for ${id}:`, processingError);
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1249
1287
|
return id;
|
|
1250
1288
|
}
|
|
1251
1289
|
catch (error) {
|
|
@@ -2255,6 +2293,13 @@ export class BrainyData {
|
|
|
2255
2293
|
* @returns Promise that resolves to true if the vector was deleted, false otherwise
|
|
2256
2294
|
*/
|
|
2257
2295
|
async delete(id, options = {}) {
|
|
2296
|
+
const opts = {
|
|
2297
|
+
service: undefined,
|
|
2298
|
+
soft: true, // Soft delete is default - preserves indexes
|
|
2299
|
+
cascade: false,
|
|
2300
|
+
force: false,
|
|
2301
|
+
...options
|
|
2302
|
+
};
|
|
2258
2303
|
// Validate id parameter first, before any other logic
|
|
2259
2304
|
if (id === null || id === undefined) {
|
|
2260
2305
|
throw new Error('ID cannot be null or undefined');
|
|
@@ -2280,7 +2325,16 @@ export class BrainyData {
|
|
|
2280
2325
|
}
|
|
2281
2326
|
}
|
|
2282
2327
|
}
|
|
2283
|
-
//
|
|
2328
|
+
// Handle soft delete vs hard delete
|
|
2329
|
+
if (opts.soft) {
|
|
2330
|
+
// Soft delete: just mark as deleted - metadata filter will exclude from search
|
|
2331
|
+
return await this.updateMetadata(actualId, {
|
|
2332
|
+
deleted: true,
|
|
2333
|
+
deletedAt: new Date().toISOString(),
|
|
2334
|
+
deletedBy: opts.service || 'user'
|
|
2335
|
+
});
|
|
2336
|
+
}
|
|
2337
|
+
// Hard delete: Remove from index
|
|
2284
2338
|
const removed = this.index.removeItem(actualId);
|
|
2285
2339
|
if (!removed) {
|
|
2286
2340
|
return false;
|
|
@@ -2288,7 +2342,7 @@ export class BrainyData {
|
|
|
2288
2342
|
// Remove from storage
|
|
2289
2343
|
await this.storage.deleteNoun(actualId);
|
|
2290
2344
|
// Track deletion statistics
|
|
2291
|
-
const service = this.getServiceName(
|
|
2345
|
+
const service = this.getServiceName({ service: opts.service });
|
|
2292
2346
|
await this.storage.decrementStatistic('noun', service);
|
|
2293
2347
|
// Try to remove metadata (ignore errors)
|
|
2294
2348
|
try {
|
|
@@ -2422,7 +2476,7 @@ export class BrainyData {
|
|
|
2422
2476
|
if (relationType === null || relationType === undefined) {
|
|
2423
2477
|
throw new Error('Relation type cannot be null or undefined');
|
|
2424
2478
|
}
|
|
2425
|
-
return this.
|
|
2479
|
+
return this._addVerbInternal(sourceId, targetId, undefined, {
|
|
2426
2480
|
type: relationType,
|
|
2427
2481
|
metadata: metadata
|
|
2428
2482
|
});
|
|
@@ -2455,7 +2509,7 @@ export class BrainyData {
|
|
|
2455
2509
|
*
|
|
2456
2510
|
* @throws Error if source or target nouns don't exist and autoCreateMissingNouns is false or auto-creation fails
|
|
2457
2511
|
*/
|
|
2458
|
-
async
|
|
2512
|
+
async _addVerbInternal(sourceId, targetId, vector, options = {}) {
|
|
2459
2513
|
await this.ensureInitialized();
|
|
2460
2514
|
// Check if database is in read-only mode
|
|
2461
2515
|
this.checkReadOnly();
|
|
@@ -3145,6 +3199,14 @@ export class BrainyData {
|
|
|
3145
3199
|
}
|
|
3146
3200
|
}
|
|
3147
3201
|
}
|
|
3202
|
+
/**
|
|
3203
|
+
* @deprecated Use add() instead - it's smart by default now
|
|
3204
|
+
* @hidden
|
|
3205
|
+
*/
|
|
3206
|
+
async addSmart(vectorOrData, metadata, options = {}) {
|
|
3207
|
+
console.warn('⚠️ addSmart() is deprecated. Use add() instead - it\'s smart by default now!');
|
|
3208
|
+
return this.add(vectorOrData, metadata, { ...options, process: 'auto' });
|
|
3209
|
+
}
|
|
3148
3210
|
/**
|
|
3149
3211
|
* Get the number of nouns in the database (excluding verbs)
|
|
3150
3212
|
* This is used for statistics reporting to match the expected behavior in tests
|
|
@@ -4440,7 +4502,7 @@ export class BrainyData {
|
|
|
4440
4502
|
}
|
|
4441
4503
|
}
|
|
4442
4504
|
// Add the verb
|
|
4443
|
-
await this.
|
|
4505
|
+
await this._addVerbInternal(verb.sourceId, verb.targetId, verb.vector, {
|
|
4444
4506
|
id: verb.id,
|
|
4445
4507
|
type: verb.metadata?.verb || VerbType.RelatedTo,
|
|
4446
4508
|
metadata: verb.metadata
|
|
@@ -4614,7 +4676,7 @@ export class BrainyData {
|
|
|
4614
4676
|
}
|
|
4615
4677
|
};
|
|
4616
4678
|
// Add the verb
|
|
4617
|
-
const id = await this.
|
|
4679
|
+
const id = await this._addVerbInternal(sourceId, targetId, undefined, {
|
|
4618
4680
|
type: verbType,
|
|
4619
4681
|
weight: metadata.weight,
|
|
4620
4682
|
metadata
|
|
@@ -4751,24 +4813,488 @@ export class BrainyData {
|
|
|
4751
4813
|
prodLog.debug('Cortex integration coming soon');
|
|
4752
4814
|
}
|
|
4753
4815
|
/**
|
|
4754
|
-
* Set a configuration value
|
|
4816
|
+
* Set a configuration value with optional encryption
|
|
4755
4817
|
* @param key Configuration key
|
|
4756
4818
|
* @param value Configuration value
|
|
4757
4819
|
* @param options Options including encryption
|
|
4758
4820
|
*/
|
|
4759
4821
|
async setConfig(key, value, options) {
|
|
4760
|
-
|
|
4761
|
-
|
|
4822
|
+
const configNoun = {
|
|
4823
|
+
configKey: key,
|
|
4824
|
+
configValue: options?.encrypt ? await this.encryptData(JSON.stringify(value)) : value,
|
|
4825
|
+
encrypted: !!options?.encrypt,
|
|
4826
|
+
timestamp: new Date().toISOString()
|
|
4827
|
+
};
|
|
4828
|
+
await this.add(configNoun, {
|
|
4829
|
+
nounType: NounType.State,
|
|
4830
|
+
configKey: key,
|
|
4831
|
+
encrypted: !!options?.encrypt
|
|
4832
|
+
});
|
|
4762
4833
|
}
|
|
4763
4834
|
/**
|
|
4764
|
-
* Get a configuration value
|
|
4835
|
+
* Get a configuration value with automatic decryption
|
|
4765
4836
|
* @param key Configuration key
|
|
4766
4837
|
* @returns Configuration value or undefined
|
|
4767
4838
|
*/
|
|
4768
4839
|
async getConfig(key) {
|
|
4769
|
-
|
|
4770
|
-
|
|
4771
|
-
|
|
4840
|
+
try {
|
|
4841
|
+
const results = await this.search('', 1, {
|
|
4842
|
+
nounTypes: [NounType.State],
|
|
4843
|
+
metadata: { configKey: key }
|
|
4844
|
+
});
|
|
4845
|
+
if (results.length === 0)
|
|
4846
|
+
return undefined;
|
|
4847
|
+
const configNoun = results[0];
|
|
4848
|
+
const value = configNoun.data?.configValue || configNoun.metadata?.configValue;
|
|
4849
|
+
const encrypted = configNoun.data?.encrypted || configNoun.metadata?.encrypted;
|
|
4850
|
+
if (encrypted && typeof value === 'string') {
|
|
4851
|
+
const decrypted = await this.decryptData(value);
|
|
4852
|
+
return JSON.parse(decrypted);
|
|
4853
|
+
}
|
|
4854
|
+
return value;
|
|
4855
|
+
}
|
|
4856
|
+
catch (error) {
|
|
4857
|
+
prodLog.debug('Config retrieval failed:', error);
|
|
4858
|
+
return undefined;
|
|
4859
|
+
}
|
|
4860
|
+
}
|
|
4861
|
+
/**
|
|
4862
|
+
* Encrypt data using universal crypto utilities
|
|
4863
|
+
*/
|
|
4864
|
+
async encryptData(data) {
|
|
4865
|
+
const crypto = await import('./universal/crypto.js');
|
|
4866
|
+
const key = crypto.randomBytes(32);
|
|
4867
|
+
const iv = crypto.randomBytes(16);
|
|
4868
|
+
const cipher = crypto.createCipheriv('aes-256-cbc', key, iv);
|
|
4869
|
+
let encrypted = cipher.update(data, 'utf8', 'hex');
|
|
4870
|
+
encrypted += cipher.final('hex');
|
|
4871
|
+
// Store key and iv with encrypted data (in production, manage keys separately)
|
|
4872
|
+
return JSON.stringify({
|
|
4873
|
+
encrypted,
|
|
4874
|
+
key: Array.from(key).map(b => b.toString(16).padStart(2, '0')).join(''),
|
|
4875
|
+
iv: Array.from(iv).map(b => b.toString(16).padStart(2, '0')).join('')
|
|
4876
|
+
});
|
|
4877
|
+
}
|
|
4878
|
+
/**
|
|
4879
|
+
* Decrypt data using universal crypto utilities
|
|
4880
|
+
*/
|
|
4881
|
+
async decryptData(encryptedData) {
|
|
4882
|
+
const crypto = await import('./universal/crypto.js');
|
|
4883
|
+
const { encrypted, key: keyHex, iv: ivHex } = JSON.parse(encryptedData);
|
|
4884
|
+
const key = new Uint8Array(keyHex.match(/.{1,2}/g).map((byte) => parseInt(byte, 16)));
|
|
4885
|
+
const iv = new Uint8Array(ivHex.match(/.{1,2}/g).map((byte) => parseInt(byte, 16)));
|
|
4886
|
+
const decipher = crypto.createDecipheriv('aes-256-cbc', key, iv);
|
|
4887
|
+
let decrypted = decipher.update(encrypted, 'hex', 'utf8');
|
|
4888
|
+
decrypted += decipher.final('utf8');
|
|
4889
|
+
return decrypted;
|
|
4890
|
+
}
|
|
4891
|
+
// ========================================
|
|
4892
|
+
// UNIFIED API - Core Methods (7 total)
|
|
4893
|
+
// ONE way to do everything! 🧠⚛️
|
|
4894
|
+
//
|
|
4895
|
+
// 1. add() - Smart data addition (auto/guided/explicit/literal)
|
|
4896
|
+
// 2. search() - Triple-power search (vector + graph + facets)
|
|
4897
|
+
// 3. import() - Neural import with semantic type detection
|
|
4898
|
+
// 4. addNoun() - Explicit noun creation with NounType
|
|
4899
|
+
// 5. addVerb() - Relationship creation between nouns
|
|
4900
|
+
// 6. update() - Update noun data/metadata with index sync
|
|
4901
|
+
// 7. delete() - Smart delete with soft delete default (enhanced original)
|
|
4902
|
+
// ========================================
|
|
4903
|
+
/**
|
|
4904
|
+
* Neural Import - Smart bulk data import with semantic type detection
|
|
4905
|
+
* Uses transformer embeddings to automatically detect and classify data types
|
|
4906
|
+
* @param data Array of data items or single item to import
|
|
4907
|
+
* @param options Import options including type hints and processing mode
|
|
4908
|
+
* @returns Array of created IDs
|
|
4909
|
+
*/
|
|
4910
|
+
async import(data, options) {
|
|
4911
|
+
const items = Array.isArray(data) ? data : [data];
|
|
4912
|
+
const results = [];
|
|
4913
|
+
const batchSize = options?.batchSize || 50;
|
|
4914
|
+
// Process in batches to avoid memory issues
|
|
4915
|
+
for (let i = 0; i < items.length; i += batchSize) {
|
|
4916
|
+
const batch = items.slice(i, i + batchSize);
|
|
4917
|
+
for (const item of batch) {
|
|
4918
|
+
try {
|
|
4919
|
+
// Auto-detect type using semantic schema if enabled
|
|
4920
|
+
let detectedType = options?.typeHint;
|
|
4921
|
+
if (options?.autoDetect !== false && !detectedType) {
|
|
4922
|
+
detectedType = await this.detectNounType(item);
|
|
4923
|
+
}
|
|
4924
|
+
// Create metadata with detected type
|
|
4925
|
+
const metadata = {};
|
|
4926
|
+
if (detectedType) {
|
|
4927
|
+
metadata.nounType = detectedType;
|
|
4928
|
+
}
|
|
4929
|
+
// Import item using standard add method
|
|
4930
|
+
const id = await this.add(item, metadata, {
|
|
4931
|
+
process: options?.process || 'auto'
|
|
4932
|
+
});
|
|
4933
|
+
results.push(id);
|
|
4934
|
+
}
|
|
4935
|
+
catch (error) {
|
|
4936
|
+
prodLog.warn(`Failed to import item:`, error);
|
|
4937
|
+
// Continue with next item rather than failing entire batch
|
|
4938
|
+
}
|
|
4939
|
+
}
|
|
4940
|
+
}
|
|
4941
|
+
prodLog.info(`📦 Neural import completed: ${results.length}/${items.length} items imported`);
|
|
4942
|
+
return results;
|
|
4943
|
+
}
|
|
4944
|
+
/**
|
|
4945
|
+
* Add Noun - Explicit noun creation with strongly-typed NounType
|
|
4946
|
+
* For when you know exactly what type of noun you're creating
|
|
4947
|
+
* @param data The noun data
|
|
4948
|
+
* @param nounType The explicit noun type from NounType enum
|
|
4949
|
+
* @param metadata Additional metadata
|
|
4950
|
+
* @returns Created noun ID
|
|
4951
|
+
*/
|
|
4952
|
+
async addNoun(data, nounType, metadata) {
|
|
4953
|
+
const nounMetadata = {
|
|
4954
|
+
nounType,
|
|
4955
|
+
...metadata
|
|
4956
|
+
};
|
|
4957
|
+
return await this.add(data, nounMetadata, {
|
|
4958
|
+
process: 'neural' // Neural mode since type is already known
|
|
4959
|
+
});
|
|
4960
|
+
}
|
|
4961
|
+
/**
|
|
4962
|
+
* Add Verb - Unified relationship creation between nouns
|
|
4963
|
+
* Creates typed relationships with proper vector embeddings from metadata
|
|
4964
|
+
* @param sourceId Source noun ID
|
|
4965
|
+
* @param targetId Target noun ID
|
|
4966
|
+
* @param verbType Relationship type from VerbType enum
|
|
4967
|
+
* @param metadata Additional metadata for the relationship (will be embedded for searchability)
|
|
4968
|
+
* @param weight Relationship weight/strength (0-1, default: 0.5)
|
|
4969
|
+
* @returns Created verb ID
|
|
4970
|
+
*/
|
|
4971
|
+
async addVerb(sourceId, targetId, verbType, metadata, weight) {
|
|
4972
|
+
// Validate that source and target nouns exist
|
|
4973
|
+
const sourceNoun = this.index.getNouns().get(sourceId);
|
|
4974
|
+
const targetNoun = this.index.getNouns().get(targetId);
|
|
4975
|
+
if (!sourceNoun) {
|
|
4976
|
+
throw new Error(`Source noun with ID ${sourceId} does not exist`);
|
|
4977
|
+
}
|
|
4978
|
+
if (!targetNoun) {
|
|
4979
|
+
throw new Error(`Target noun with ID ${targetId} does not exist`);
|
|
4980
|
+
}
|
|
4981
|
+
// Create embeddable text from verb type and metadata for searchability
|
|
4982
|
+
let embeddingText = `${verbType} relationship`;
|
|
4983
|
+
// Include meaningful metadata in embedding
|
|
4984
|
+
if (metadata) {
|
|
4985
|
+
const metadataStrings = [];
|
|
4986
|
+
// Add text-based metadata fields for better searchability
|
|
4987
|
+
for (const [key, value] of Object.entries(metadata)) {
|
|
4988
|
+
if (typeof value === 'string' && value.length > 0) {
|
|
4989
|
+
metadataStrings.push(`${key}: ${value}`);
|
|
4990
|
+
}
|
|
4991
|
+
else if (typeof value === 'number' || typeof value === 'boolean') {
|
|
4992
|
+
metadataStrings.push(`${key}: ${value}`);
|
|
4993
|
+
}
|
|
4994
|
+
}
|
|
4995
|
+
if (metadataStrings.length > 0) {
|
|
4996
|
+
embeddingText += ` with ${metadataStrings.join(', ')}`;
|
|
4997
|
+
}
|
|
4998
|
+
}
|
|
4999
|
+
// Generate embedding for the relationship including metadata
|
|
5000
|
+
const vector = await this.embeddingFunction(embeddingText);
|
|
5001
|
+
// Create complete verb metadata
|
|
5002
|
+
const verbMetadata = {
|
|
5003
|
+
verb: verbType,
|
|
5004
|
+
sourceId,
|
|
5005
|
+
targetId,
|
|
5006
|
+
weight: weight || 0.5,
|
|
5007
|
+
embeddingText, // Include the text used for embedding for debugging
|
|
5008
|
+
...metadata
|
|
5009
|
+
};
|
|
5010
|
+
// Use existing internal addVerb method with proper parameters
|
|
5011
|
+
return await this._addVerbInternal(sourceId, targetId, vector, {
|
|
5012
|
+
type: verbType,
|
|
5013
|
+
weight: weight || 0.5,
|
|
5014
|
+
metadata: verbMetadata,
|
|
5015
|
+
forceEmbed: false // We already have the vector
|
|
5016
|
+
});
|
|
5017
|
+
}
|
|
5018
|
+
/**
|
|
5019
|
+
* Auto-detect whether to use neural processing for data
|
|
5020
|
+
* @private
|
|
5021
|
+
*/
|
|
5022
|
+
shouldAutoProcessNeurally(data, metadata) {
|
|
5023
|
+
// Simple heuristics for auto-detection
|
|
5024
|
+
if (typeof data === 'string') {
|
|
5025
|
+
// Long text likely benefits from neural processing
|
|
5026
|
+
if (data.length > 50)
|
|
5027
|
+
return true;
|
|
5028
|
+
// Short text with meaningful content
|
|
5029
|
+
if (data.includes(' ') && data.length > 10)
|
|
5030
|
+
return true;
|
|
5031
|
+
}
|
|
5032
|
+
if (typeof data === 'object' && data !== null) {
|
|
5033
|
+
// Complex objects usually benefit from neural processing
|
|
5034
|
+
if (Object.keys(data).length > 2)
|
|
5035
|
+
return true;
|
|
5036
|
+
// Objects with text content
|
|
5037
|
+
if (data.content || data.text || data.description)
|
|
5038
|
+
return true;
|
|
5039
|
+
}
|
|
5040
|
+
// Check metadata hints
|
|
5041
|
+
if (metadata?.nounType)
|
|
5042
|
+
return true;
|
|
5043
|
+
if (metadata?.needsProcessing)
|
|
5044
|
+
return metadata.needsProcessing;
|
|
5045
|
+
// Default to neural processing for rich data
|
|
5046
|
+
return true;
|
|
5047
|
+
}
|
|
5048
|
+
/**
|
|
5049
|
+
* Detect noun type using semantic analysis
|
|
5050
|
+
* @private
|
|
5051
|
+
*/
|
|
5052
|
+
async detectNounType(data) {
|
|
5053
|
+
// Simple heuristic-based detection (could be enhanced with ML)
|
|
5054
|
+
if (typeof data === 'string') {
|
|
5055
|
+
if (data.includes('@') && data.includes('.')) {
|
|
5056
|
+
return NounType.Person; // Email indicates person
|
|
5057
|
+
}
|
|
5058
|
+
if (data.startsWith('http')) {
|
|
5059
|
+
return NounType.Document; // URL indicates document
|
|
5060
|
+
}
|
|
5061
|
+
if (data.length < 100) {
|
|
5062
|
+
return NounType.Concept; // Short text as concept
|
|
5063
|
+
}
|
|
5064
|
+
return NounType.Content; // Default for longer text
|
|
5065
|
+
}
|
|
5066
|
+
if (typeof data === 'object' && data !== null) {
|
|
5067
|
+
if (data.name || data.title) {
|
|
5068
|
+
return NounType.Concept;
|
|
5069
|
+
}
|
|
5070
|
+
if (data.email || data.phone || data.firstName) {
|
|
5071
|
+
return NounType.Person;
|
|
5072
|
+
}
|
|
5073
|
+
if (data.url || data.content || data.body) {
|
|
5074
|
+
return NounType.Document;
|
|
5075
|
+
}
|
|
5076
|
+
if (data.message || data.text) {
|
|
5077
|
+
return NounType.Message;
|
|
5078
|
+
}
|
|
5079
|
+
}
|
|
5080
|
+
return NounType.Content; // Safe default
|
|
5081
|
+
}
|
|
5082
|
+
/**
|
|
5083
|
+
* Get Noun with Connected Verbs - Retrieve noun and all its relationships
|
|
5084
|
+
* Provides complete traversal view of a noun and its connections using existing searchVerbs
|
|
5085
|
+
* @param nounId The noun ID to retrieve
|
|
5086
|
+
* @param options Traversal options
|
|
5087
|
+
* @returns Noun data with connected verbs and related nouns
|
|
5088
|
+
*/
|
|
5089
|
+
async getNounWithVerbs(nounId, options) {
|
|
5090
|
+
const opts = {
|
|
5091
|
+
includeIncoming: true,
|
|
5092
|
+
includeOutgoing: true,
|
|
5093
|
+
verbLimit: 50,
|
|
5094
|
+
...options
|
|
5095
|
+
};
|
|
5096
|
+
// Get the noun
|
|
5097
|
+
const noun = this.index.getNouns().get(nounId);
|
|
5098
|
+
if (!noun) {
|
|
5099
|
+
return null;
|
|
5100
|
+
}
|
|
5101
|
+
const result = {
|
|
5102
|
+
noun: {
|
|
5103
|
+
id: nounId,
|
|
5104
|
+
data: noun.metadata || {}, // Use metadata as data for consistency
|
|
5105
|
+
metadata: noun.metadata || {},
|
|
5106
|
+
nounType: noun.metadata?.nounType
|
|
5107
|
+
},
|
|
5108
|
+
incomingVerbs: [],
|
|
5109
|
+
outgoingVerbs: [],
|
|
5110
|
+
totalConnections: 0
|
|
5111
|
+
};
|
|
5112
|
+
// Use existing searchVerbs functionality - it searches by target/source filters
|
|
5113
|
+
try {
|
|
5114
|
+
if (opts.includeIncoming) {
|
|
5115
|
+
// Search for verbs where this noun is the target
|
|
5116
|
+
const incomingVerbOptions = {
|
|
5117
|
+
verbTypes: opts.verbTypes
|
|
5118
|
+
};
|
|
5119
|
+
const incomingResults = await this.searchVerbs(nounId, opts.verbLimit, incomingVerbOptions);
|
|
5120
|
+
result.incomingVerbs = incomingResults.filter(verb => verb.targetId === nounId || verb.sourceId === nounId);
|
|
5121
|
+
}
|
|
5122
|
+
if (opts.includeOutgoing) {
|
|
5123
|
+
// Search for verbs where this noun is the source
|
|
5124
|
+
const outgoingVerbOptions = {
|
|
5125
|
+
verbTypes: opts.verbTypes
|
|
5126
|
+
};
|
|
5127
|
+
const outgoingResults = await this.searchVerbs(nounId, opts.verbLimit, outgoingVerbOptions);
|
|
5128
|
+
result.outgoingVerbs = outgoingResults.filter(verb => verb.sourceId === nounId || verb.targetId === nounId);
|
|
5129
|
+
}
|
|
5130
|
+
}
|
|
5131
|
+
catch (error) {
|
|
5132
|
+
prodLog.warn(`Error searching verbs for noun ${nounId}:`, error);
|
|
5133
|
+
// Continue with empty arrays
|
|
5134
|
+
}
|
|
5135
|
+
result.totalConnections = result.incomingVerbs.length + result.outgoingVerbs.length;
|
|
5136
|
+
prodLog.debug(`🔍 Retrieved noun ${nounId} with ${result.totalConnections} connections`);
|
|
5137
|
+
return result;
|
|
5138
|
+
}
|
|
5139
|
+
/**
|
|
5140
|
+
* Update - Smart noun update with automatic index synchronization
|
|
5141
|
+
* Updates both data and metadata while maintaining search index integrity
|
|
5142
|
+
* @param id The noun ID to update
|
|
5143
|
+
* @param data New data (optional - if not provided, only metadata is updated)
|
|
5144
|
+
* @param metadata New metadata (merged with existing)
|
|
5145
|
+
* @param options Update options
|
|
5146
|
+
* @returns Success boolean
|
|
5147
|
+
*/
|
|
5148
|
+
async update(id, data, metadata, options) {
|
|
5149
|
+
const opts = {
|
|
5150
|
+
merge: true,
|
|
5151
|
+
reindex: true,
|
|
5152
|
+
cascade: false,
|
|
5153
|
+
...options
|
|
5154
|
+
};
|
|
5155
|
+
// Update data if provided
|
|
5156
|
+
if (data !== undefined) {
|
|
5157
|
+
// For data updates, we need to regenerate the vector
|
|
5158
|
+
const existingNoun = this.index.getNouns().get(id);
|
|
5159
|
+
if (!existingNoun) {
|
|
5160
|
+
throw new Error(`Noun with ID ${id} does not exist`);
|
|
5161
|
+
}
|
|
5162
|
+
// Create new vector for updated data
|
|
5163
|
+
const vector = await this.embeddingFunction(data);
|
|
5164
|
+
// Update the noun with new data and vector
|
|
5165
|
+
const updatedNoun = {
|
|
5166
|
+
...existingNoun,
|
|
5167
|
+
vector,
|
|
5168
|
+
metadata: opts.merge ? { ...existingNoun.metadata, ...metadata } : metadata
|
|
5169
|
+
};
|
|
5170
|
+
// Update in index
|
|
5171
|
+
this.index.getNouns().set(id, updatedNoun);
|
|
5172
|
+
// Note: HNSW index will be updated automatically on next search
|
|
5173
|
+
// Reindexing happens lazily for performance
|
|
5174
|
+
}
|
|
5175
|
+
else if (metadata !== undefined) {
|
|
5176
|
+
// Metadata-only update using existing updateMetadata method
|
|
5177
|
+
return await this.updateMetadata(id, metadata);
|
|
5178
|
+
}
|
|
5179
|
+
// Update related verbs if cascade enabled
|
|
5180
|
+
if (opts.cascade) {
|
|
5181
|
+
// TODO: Implement cascade verb updates when verb access methods are clarified
|
|
5182
|
+
prodLog.debug(`Cascade update requested for ${id} - feature pending implementation`);
|
|
5183
|
+
}
|
|
5184
|
+
prodLog.debug(`✅ Updated noun ${id} (data: ${data !== undefined}, metadata: ${metadata !== undefined})`);
|
|
5185
|
+
return true;
|
|
5186
|
+
}
|
|
5187
|
+
/**
|
|
5188
|
+
* Preload Transformer Model - Essential for container deployments
|
|
5189
|
+
* Downloads and caches models during initialization to avoid runtime delays
|
|
5190
|
+
* @param options Preload options
|
|
5191
|
+
* @returns Success boolean and model info
|
|
5192
|
+
*/
|
|
5193
|
+
static async preloadModel(options) {
|
|
5194
|
+
const opts = {
|
|
5195
|
+
model: 'Xenova/all-MiniLM-L6-v2',
|
|
5196
|
+
cacheDir: './models',
|
|
5197
|
+
device: 'auto',
|
|
5198
|
+
force: false,
|
|
5199
|
+
...options
|
|
5200
|
+
};
|
|
5201
|
+
try {
|
|
5202
|
+
// Import embedding utilities
|
|
5203
|
+
const { TransformerEmbedding, resolveDevice } = await import('./utils/embedding.js');
|
|
5204
|
+
// Resolve optimal device
|
|
5205
|
+
const device = await resolveDevice(opts.device);
|
|
5206
|
+
prodLog.info(`🤖 Preloading transformer model: ${opts.model}`);
|
|
5207
|
+
prodLog.info(`📁 Cache directory: ${opts.cacheDir}`);
|
|
5208
|
+
prodLog.info(`⚡ Target device: ${device}`);
|
|
5209
|
+
// Create embedder instance with preload settings
|
|
5210
|
+
const embedder = new TransformerEmbedding({
|
|
5211
|
+
model: opts.model,
|
|
5212
|
+
cacheDir: opts.cacheDir,
|
|
5213
|
+
device: device,
|
|
5214
|
+
localFilesOnly: false, // Allow downloads during preload
|
|
5215
|
+
verbose: true
|
|
5216
|
+
});
|
|
5217
|
+
// Initialize and warm up the model
|
|
5218
|
+
await embedder.init();
|
|
5219
|
+
// Test with a small input to fully load the model
|
|
5220
|
+
await embedder.embed('test initialization');
|
|
5221
|
+
// Get model info for container deployments
|
|
5222
|
+
const modelInfo = {
|
|
5223
|
+
success: true,
|
|
5224
|
+
modelPath: opts.cacheDir,
|
|
5225
|
+
modelSize: await this.getModelSize(opts.cacheDir, opts.model),
|
|
5226
|
+
device: device
|
|
5227
|
+
};
|
|
5228
|
+
prodLog.info(`✅ Model preloaded successfully`);
|
|
5229
|
+
prodLog.info(`📊 Model size: ${(modelInfo.modelSize / 1024 / 1024).toFixed(2)}MB`);
|
|
5230
|
+
return modelInfo;
|
|
5231
|
+
}
|
|
5232
|
+
catch (error) {
|
|
5233
|
+
prodLog.error(`❌ Model preload failed:`, error);
|
|
5234
|
+
return {
|
|
5235
|
+
success: false,
|
|
5236
|
+
modelPath: '',
|
|
5237
|
+
modelSize: 0,
|
|
5238
|
+
device: 'cpu'
|
|
5239
|
+
};
|
|
5240
|
+
}
|
|
5241
|
+
}
|
|
5242
|
+
/**
|
|
5243
|
+
* Warmup - Initialize BrainyData with preloaded models (container-optimized)
|
|
5244
|
+
* For production deployments where models should be ready immediately
|
|
5245
|
+
* @param config BrainyData configuration
|
|
5246
|
+
* @param options Warmup options
|
|
5247
|
+
*/
|
|
5248
|
+
static async warmup(config, options) {
|
|
5249
|
+
const opts = {
|
|
5250
|
+
preloadModel: true,
|
|
5251
|
+
testEmbedding: true,
|
|
5252
|
+
...options
|
|
5253
|
+
};
|
|
5254
|
+
prodLog.info(`🚀 Starting Brainy warmup for container deployment`);
|
|
5255
|
+
// Preload transformer models if requested
|
|
5256
|
+
if (opts.preloadModel) {
|
|
5257
|
+
const modelInfo = await BrainyData.preloadModel(opts.modelOptions);
|
|
5258
|
+
if (!modelInfo.success) {
|
|
5259
|
+
prodLog.warn(`⚠️ Model preload failed, continuing with lazy loading`);
|
|
5260
|
+
}
|
|
5261
|
+
}
|
|
5262
|
+
// Create and initialize BrainyData instance
|
|
5263
|
+
const brainy = new BrainyData(config);
|
|
5264
|
+
await brainy.init();
|
|
5265
|
+
// Test embedding to ensure everything works
|
|
5266
|
+
if (opts.testEmbedding) {
|
|
5267
|
+
try {
|
|
5268
|
+
await brainy.embeddingFunction('test warmup embedding');
|
|
5269
|
+
prodLog.info(`✅ Embedding test successful`);
|
|
5270
|
+
}
|
|
5271
|
+
catch (error) {
|
|
5272
|
+
prodLog.warn(`⚠️ Embedding test failed:`, error);
|
|
5273
|
+
}
|
|
5274
|
+
}
|
|
5275
|
+
prodLog.info(`🎉 Brainy warmup complete - ready for production!`);
|
|
5276
|
+
return brainy;
|
|
5277
|
+
}
|
|
5278
|
+
/**
|
|
5279
|
+
* Get model size for deployment info
|
|
5280
|
+
* @private
|
|
5281
|
+
*/
|
|
5282
|
+
static async getModelSize(cacheDir, modelName) {
|
|
5283
|
+
try {
|
|
5284
|
+
const fs = await import('fs');
|
|
5285
|
+
const path = await import('path');
|
|
5286
|
+
// Estimate model size (actual implementation would scan cache directory)
|
|
5287
|
+
// For now, return known sizes for common models
|
|
5288
|
+
const modelSizes = {
|
|
5289
|
+
'Xenova/all-MiniLM-L6-v2': 90 * 1024 * 1024, // ~90MB
|
|
5290
|
+
'Xenova/all-mpnet-base-v2': 420 * 1024 * 1024, // ~420MB
|
|
5291
|
+
'Xenova/distilbert-base-uncased': 250 * 1024 * 1024 // ~250MB
|
|
5292
|
+
};
|
|
5293
|
+
return modelSizes[modelName] || 100 * 1024 * 1024; // Default 100MB
|
|
5294
|
+
}
|
|
5295
|
+
catch {
|
|
5296
|
+
return 0;
|
|
5297
|
+
}
|
|
4772
5298
|
}
|
|
4773
5299
|
/**
|
|
4774
5300
|
* Coordinate storage migration across distributed services
|
|
@@ -4817,6 +5343,63 @@ export class BrainyData {
|
|
|
4817
5343
|
await this.metadataIndex.rebuild();
|
|
4818
5344
|
}
|
|
4819
5345
|
}
|
|
5346
|
+
// ===== Augmentation Control Methods =====
|
|
5347
|
+
/**
|
|
5348
|
+
* Enable an augmentation by name
|
|
5349
|
+
* Universal control for built-in, community, and premium augmentations
|
|
5350
|
+
*
|
|
5351
|
+
* @param name The name of the augmentation to enable
|
|
5352
|
+
* @returns True if augmentation was found and enabled
|
|
5353
|
+
*/
|
|
5354
|
+
enableAugmentation(name) {
|
|
5355
|
+
return augmentationPipeline.enableAugmentation(name);
|
|
5356
|
+
}
|
|
5357
|
+
/**
|
|
5358
|
+
* Disable an augmentation by name
|
|
5359
|
+
* Universal control for built-in, community, and premium augmentations
|
|
5360
|
+
*
|
|
5361
|
+
* @param name The name of the augmentation to disable
|
|
5362
|
+
* @returns True if augmentation was found and disabled
|
|
5363
|
+
*/
|
|
5364
|
+
disableAugmentation(name) {
|
|
5365
|
+
return augmentationPipeline.disableAugmentation(name);
|
|
5366
|
+
}
|
|
5367
|
+
/**
|
|
5368
|
+
* Check if an augmentation is enabled
|
|
5369
|
+
*
|
|
5370
|
+
* @param name The name of the augmentation to check
|
|
5371
|
+
* @returns True if augmentation is found and enabled, false otherwise
|
|
5372
|
+
*/
|
|
5373
|
+
isAugmentationEnabled(name) {
|
|
5374
|
+
return augmentationPipeline.isAugmentationEnabled(name);
|
|
5375
|
+
}
|
|
5376
|
+
/**
|
|
5377
|
+
* Get all augmentations with their enabled status
|
|
5378
|
+
* Shows built-in, community, and premium augmentations
|
|
5379
|
+
*
|
|
5380
|
+
* @returns Array of augmentations with name, type, and enabled status
|
|
5381
|
+
*/
|
|
5382
|
+
listAugmentations() {
|
|
5383
|
+
return augmentationPipeline.listAugmentationsWithStatus();
|
|
5384
|
+
}
|
|
5385
|
+
/**
|
|
5386
|
+
* Enable all augmentations of a specific type
|
|
5387
|
+
*
|
|
5388
|
+
* @param type The type of augmentations to enable (sense, conduit, cognition, etc.)
|
|
5389
|
+
* @returns Number of augmentations enabled
|
|
5390
|
+
*/
|
|
5391
|
+
enableAugmentationType(type) {
|
|
5392
|
+
return augmentationPipeline.enableAugmentationType(type);
|
|
5393
|
+
}
|
|
5394
|
+
/**
|
|
5395
|
+
* Disable all augmentations of a specific type
|
|
5396
|
+
*
|
|
5397
|
+
* @param type The type of augmentations to disable (sense, conduit, cognition, etc.)
|
|
5398
|
+
* @returns Number of augmentations disabled
|
|
5399
|
+
*/
|
|
5400
|
+
disableAugmentationType(type) {
|
|
5401
|
+
return augmentationPipeline.disableAugmentationType(type);
|
|
5402
|
+
}
|
|
4820
5403
|
}
|
|
4821
5404
|
// Export distance functions for convenience
|
|
4822
5405
|
export { euclideanDistance, cosineDistance, manhattanDistance, dotProductDistance } from './utils/index.js';
|