@soulcraft/brainy 4.0.2 → 4.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -1
- package/dist/brainy.js +13 -14
- package/dist/storage/adapters/gcsStorage.d.ts +5 -0
- package/dist/storage/adapters/gcsStorage.js +83 -14
- package/dist/storage/storageFactory.d.ts +22 -3
- package/dist/storage/storageFactory.js +67 -48
- package/dist/utils/embedding.js +1 -1
- package/dist/utils/environment.js +2 -2
- package/dist/utils/workerUtils.d.ts +1 -1
- package/dist/utils/workerUtils.js +2 -2
- package/package.json +3 -3
- package/dist/augmentations/KnowledgeAugmentation.d.ts +0 -40
- package/dist/augmentations/KnowledgeAugmentation.js +0 -251
- package/dist/query/typeInference.d.ts +0 -158
- package/dist/query/typeInference.js +0 -760
- package/dist/types/brainyDataInterface.d.ts +0 -52
- package/dist/types/brainyDataInterface.js +0 -10
- package/dist/vfs/ConceptSystem.d.ts +0 -203
- package/dist/vfs/ConceptSystem.js +0 -545
- package/dist/vfs/EntityManager.d.ts +0 -75
- package/dist/vfs/EntityManager.js +0 -216
- package/dist/vfs/EventRecorder.d.ts +0 -84
- package/dist/vfs/EventRecorder.js +0 -269
- package/dist/vfs/GitBridge.d.ts +0 -167
- package/dist/vfs/GitBridge.js +0 -537
- package/dist/vfs/KnowledgeLayer.d.ts +0 -35
- package/dist/vfs/KnowledgeLayer.js +0 -443
- package/dist/vfs/PersistentEntitySystem.d.ts +0 -165
- package/dist/vfs/PersistentEntitySystem.js +0 -503
- package/dist/vfs/SemanticVersioning.d.ts +0 -105
- package/dist/vfs/SemanticVersioning.js +0 -309
package/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,25 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
-
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/
|
|
3
|
+
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
|
4
|
+
|
|
5
|
+
### [4.1.1](https://github.com/soulcraftlabs/brainy/compare/v4.1.0...v4.1.1) (2025-10-20)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
### 🐛 Bug Fixes
|
|
9
|
+
|
|
10
|
+
* correct Node.js version references from 24 to 22 in comments and code ([22513ff](https://github.com/soulcraftlabs/brainy/commit/22513ffcb40cc6498898400ac5d1bae19c5d02ed))
|
|
11
|
+
|
|
12
|
+
## [4.1.0](https://github.com/soulcraftlabs/brainy/compare/v4.0.1...v4.1.0) (2025-10-20)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
### 📚 Documentation
|
|
16
|
+
|
|
17
|
+
* restructure README for clarity and engagement ([26c5c78](https://github.com/soulcraftlabs/brainy/commit/26c5c784293293e2d922e0822b553b860262af1c))
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
### ✨ Features
|
|
21
|
+
|
|
22
|
+
* simplify GCS storage naming and add Cloud Run deployment options ([38343c0](https://github.com/soulcraftlabs/brainy/commit/38343c012846f0bdf70dc7402be0ef7ad93d7179))
|
|
4
23
|
|
|
5
24
|
## [4.0.0](https://github.com/soulcraftlabs/brainy/compare/v3.50.2...v4.0.0) (2025-10-17)
|
|
6
25
|
|
package/dist/brainy.js
CHANGED
|
@@ -2483,24 +2483,23 @@ export class Brainy {
|
|
|
2483
2483
|
*/
|
|
2484
2484
|
normalizeConfig(config) {
|
|
2485
2485
|
// Validate storage configuration
|
|
2486
|
-
if (config?.storage?.type && !['auto', 'memory', 'filesystem', 'opfs', 'remote', 's3', 'r2', 'gcs', 'gcs-native'].includes(config.storage.type)) {
|
|
2487
|
-
throw new Error(`Invalid storage type: ${config.storage.type}. Must be one of: auto, memory, filesystem, opfs, remote, s3, r2, gcs, gcs-native`);
|
|
2486
|
+
if (config?.storage?.type && !['auto', 'memory', 'filesystem', 'opfs', 'remote', 's3', 'r2', 'gcs', 'gcs-native', 'azure'].includes(config.storage.type)) {
|
|
2487
|
+
throw new Error(`Invalid storage type: ${config.storage.type}. Must be one of: auto, memory, filesystem, opfs, remote, s3, r2, gcs, gcs-native, azure`);
|
|
2488
2488
|
}
|
|
2489
|
-
//
|
|
2489
|
+
// Warn about deprecated gcs-native
|
|
2490
|
+
if (config?.storage?.type === 'gcs-native') {
|
|
2491
|
+
console.warn('⚠️ DEPRECATED: type "gcs-native" is deprecated. Use type "gcs" instead.');
|
|
2492
|
+
console.warn(' This will continue to work but may be removed in a future version.');
|
|
2493
|
+
}
|
|
2494
|
+
// Validate storage type/config pairing (now more lenient)
|
|
2490
2495
|
if (config?.storage) {
|
|
2491
2496
|
const storage = config.storage;
|
|
2492
|
-
//
|
|
2493
|
-
if (storage.
|
|
2494
|
-
|
|
2495
|
-
`You provided 'gcsNativeStorage' which requires type 'gcs-native'. ` +
|
|
2496
|
-
`Either change type to 'gcs-native' or use 'gcsStorage' instead of 'gcsNativeStorage'.`);
|
|
2497
|
-
}
|
|
2498
|
-
// Check for gcs-native/gcsStorage mismatch
|
|
2499
|
-
if (storage.type === 'gcs-native' && storage.gcsStorage) {
|
|
2500
|
-
throw new Error(`Storage type/config mismatch: type 'gcs-native' requires 'gcsNativeStorage' config object. ` +
|
|
2501
|
-
`You provided 'gcsStorage' which requires type 'gcs' (S3-compatible). ` +
|
|
2502
|
-
`Either change type to 'gcs' or use 'gcsNativeStorage' instead of 'gcsStorage'.`);
|
|
2497
|
+
// Warn about legacy gcsStorage config with HMAC keys
|
|
2498
|
+
if (storage.gcsStorage && storage.gcsStorage.accessKeyId && storage.gcsStorage.secretAccessKey) {
|
|
2499
|
+
console.warn('⚠️ GCS with HMAC keys (gcsStorage) is legacy. Consider migrating to native GCS (gcsNativeStorage) with ADC.');
|
|
2503
2500
|
}
|
|
2501
|
+
// No longer throw errors for mismatches - storageFactory now handles this intelligently
|
|
2502
|
+
// Both 'gcs' and 'gcs-native' can now use either gcsStorage or gcsNativeStorage
|
|
2504
2503
|
}
|
|
2505
2504
|
// Validate model configuration
|
|
2506
2505
|
if (config?.model?.type && !['fast', 'accurate', 'custom'].includes(config.model.type)) {
|
|
@@ -55,6 +55,8 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
55
55
|
private nounCacheManager;
|
|
56
56
|
private verbCacheManager;
|
|
57
57
|
private logger;
|
|
58
|
+
private skipInitialScan;
|
|
59
|
+
private skipCountsFile;
|
|
58
60
|
/**
|
|
59
61
|
* Initialize the storage adapter
|
|
60
62
|
* @param options Configuration options for Google Cloud Storage
|
|
@@ -65,6 +67,8 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
65
67
|
credentials?: object;
|
|
66
68
|
accessKeyId?: string;
|
|
67
69
|
secretAccessKey?: string;
|
|
70
|
+
skipInitialScan?: boolean;
|
|
71
|
+
skipCountsFile?: boolean;
|
|
68
72
|
cacheConfig?: {
|
|
69
73
|
hotCacheMaxSize?: number;
|
|
70
74
|
hotCacheEvictionThreshold?: number;
|
|
@@ -347,6 +351,7 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
347
351
|
protected initializeCounts(): Promise<void>;
|
|
348
352
|
/**
|
|
349
353
|
* Initialize counts from storage scan (expensive - only for first-time init)
|
|
354
|
+
* Includes timeout handling to prevent Cloud Run startup failures
|
|
350
355
|
*/
|
|
351
356
|
private initializeCountsFromScan;
|
|
352
357
|
/**
|
|
@@ -64,11 +64,16 @@ export class GcsStorage extends BaseStorage {
|
|
|
64
64
|
this.forceHighVolumeMode = false; // Environment variable override
|
|
65
65
|
// Module logger
|
|
66
66
|
this.logger = createModuleLogger('GcsStorage');
|
|
67
|
+
// Configuration options
|
|
68
|
+
this.skipInitialScan = false;
|
|
69
|
+
this.skipCountsFile = false;
|
|
67
70
|
this.bucketName = options.bucketName;
|
|
68
71
|
this.keyFilename = options.keyFilename;
|
|
69
72
|
this.credentials = options.credentials;
|
|
70
73
|
this.accessKeyId = options.accessKeyId;
|
|
71
74
|
this.secretAccessKey = options.secretAccessKey;
|
|
75
|
+
this.skipInitialScan = options.skipInitialScan || false;
|
|
76
|
+
this.skipCountsFile = options.skipCountsFile || false;
|
|
72
77
|
this.readOnly = options.readOnly || false;
|
|
73
78
|
// Set up prefixes for different types of data using entity-based structure
|
|
74
79
|
this.nounPrefix = `${getDirectoryPath('noun', 'vector')}/`;
|
|
@@ -1215,21 +1220,22 @@ export class GcsStorage extends BaseStorage {
|
|
|
1215
1220
|
// Get bucket metadata
|
|
1216
1221
|
const [metadata] = await this.bucket.getMetadata();
|
|
1217
1222
|
return {
|
|
1218
|
-
type: 'gcs',
|
|
1223
|
+
type: 'gcs', // Consistent with new naming (native SDK is just 'gcs')
|
|
1219
1224
|
used: 0, // GCS doesn't provide usage info easily
|
|
1220
1225
|
quota: null, // No quota in GCS
|
|
1221
1226
|
details: {
|
|
1222
1227
|
bucket: this.bucketName,
|
|
1223
1228
|
location: metadata.location,
|
|
1224
1229
|
storageClass: metadata.storageClass,
|
|
1225
|
-
created: metadata.timeCreated
|
|
1230
|
+
created: metadata.timeCreated,
|
|
1231
|
+
sdk: 'native' // Indicate we're using native SDK
|
|
1226
1232
|
}
|
|
1227
1233
|
};
|
|
1228
1234
|
}
|
|
1229
1235
|
catch (error) {
|
|
1230
1236
|
this.logger.error('Failed to get storage status:', error);
|
|
1231
1237
|
return {
|
|
1232
|
-
type: 'gcs',
|
|
1238
|
+
type: 'gcs', // Consistent with new naming
|
|
1233
1239
|
used: 0,
|
|
1234
1240
|
quota: null
|
|
1235
1241
|
};
|
|
@@ -1301,6 +1307,15 @@ export class GcsStorage extends BaseStorage {
|
|
|
1301
1307
|
* Initialize counts from storage
|
|
1302
1308
|
*/
|
|
1303
1309
|
async initializeCounts() {
|
|
1310
|
+
// Skip counts file entirely if configured
|
|
1311
|
+
if (this.skipCountsFile) {
|
|
1312
|
+
prodLog.info('📊 Skipping counts file (skipCountsFile: true)');
|
|
1313
|
+
this.totalNounCount = 0;
|
|
1314
|
+
this.totalVerbCount = 0;
|
|
1315
|
+
this.entityCounts = new Map();
|
|
1316
|
+
this.verbCounts = new Map();
|
|
1317
|
+
return;
|
|
1318
|
+
}
|
|
1304
1319
|
const key = `${this.systemPrefix}counts.json`;
|
|
1305
1320
|
try {
|
|
1306
1321
|
const file = this.bucket.file(key);
|
|
@@ -1314,38 +1329,68 @@ export class GcsStorage extends BaseStorage {
|
|
|
1314
1329
|
}
|
|
1315
1330
|
catch (error) {
|
|
1316
1331
|
if (error.code === 404) {
|
|
1317
|
-
// No counts file yet
|
|
1318
|
-
|
|
1319
|
-
|
|
1332
|
+
// No counts file yet
|
|
1333
|
+
if (this.skipInitialScan) {
|
|
1334
|
+
prodLog.info('📊 No counts file found - starting with zero counts (skipInitialScan: true)');
|
|
1335
|
+
this.totalNounCount = 0;
|
|
1336
|
+
this.totalVerbCount = 0;
|
|
1337
|
+
this.entityCounts = new Map();
|
|
1338
|
+
this.verbCounts = new Map();
|
|
1339
|
+
}
|
|
1340
|
+
else {
|
|
1341
|
+
// Initialize from scan (first-time setup or counts not persisted)
|
|
1342
|
+
prodLog.info('📊 No counts file found - scanning bucket to initialize counts');
|
|
1343
|
+
await this.initializeCountsFromScan();
|
|
1344
|
+
}
|
|
1320
1345
|
}
|
|
1321
1346
|
else {
|
|
1322
1347
|
// CRITICAL FIX: Don't silently fail on network/permission errors
|
|
1323
1348
|
this.logger.error('❌ CRITICAL: Failed to load counts from GCS:', error);
|
|
1324
1349
|
prodLog.error(`❌ Error loading ${key}: ${error.message}`);
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1350
|
+
if (this.skipInitialScan) {
|
|
1351
|
+
prodLog.warn('⚠️ Starting with zero counts due to error (skipInitialScan: true)');
|
|
1352
|
+
this.totalNounCount = 0;
|
|
1353
|
+
this.totalVerbCount = 0;
|
|
1354
|
+
this.entityCounts = new Map();
|
|
1355
|
+
this.verbCounts = new Map();
|
|
1356
|
+
}
|
|
1357
|
+
else {
|
|
1358
|
+
// Try to recover by scanning the bucket
|
|
1359
|
+
prodLog.warn('⚠️ Attempting recovery by scanning GCS bucket...');
|
|
1360
|
+
await this.initializeCountsFromScan();
|
|
1361
|
+
}
|
|
1328
1362
|
}
|
|
1329
1363
|
}
|
|
1330
1364
|
}
|
|
1331
1365
|
/**
|
|
1332
1366
|
* Initialize counts from storage scan (expensive - only for first-time init)
|
|
1367
|
+
* Includes timeout handling to prevent Cloud Run startup failures
|
|
1333
1368
|
*/
|
|
1334
1369
|
async initializeCountsFromScan() {
|
|
1370
|
+
const SCAN_TIMEOUT_MS = 120000; // 2 minutes timeout
|
|
1335
1371
|
try {
|
|
1336
1372
|
prodLog.info('📊 Scanning GCS bucket to initialize counts...');
|
|
1337
1373
|
prodLog.info(`🔍 Noun prefix: ${this.nounPrefix}`);
|
|
1338
1374
|
prodLog.info(`🔍 Verb prefix: ${this.verbPrefix}`);
|
|
1339
|
-
|
|
1340
|
-
|
|
1375
|
+
prodLog.info(`⏱️ Timeout: ${SCAN_TIMEOUT_MS / 1000}s (configure skipInitialScan to avoid this)`);
|
|
1376
|
+
// Create timeout promise
|
|
1377
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
1378
|
+
setTimeout(() => {
|
|
1379
|
+
reject(new Error(`Bucket scan timeout after ${SCAN_TIMEOUT_MS / 1000}s`));
|
|
1380
|
+
}, SCAN_TIMEOUT_MS);
|
|
1381
|
+
});
|
|
1382
|
+
// Count nouns with timeout
|
|
1383
|
+
const nounScanPromise = this.bucket.getFiles({ prefix: this.nounPrefix });
|
|
1384
|
+
const [nounFiles] = await Promise.race([nounScanPromise, timeoutPromise]);
|
|
1341
1385
|
prodLog.info(`🔍 Found ${nounFiles?.length || 0} total files under noun prefix`);
|
|
1342
1386
|
const jsonNounFiles = nounFiles?.filter((f) => f.name?.endsWith('.json')) || [];
|
|
1343
1387
|
this.totalNounCount = jsonNounFiles.length;
|
|
1344
1388
|
if (jsonNounFiles.length > 0 && jsonNounFiles.length <= 5) {
|
|
1345
1389
|
prodLog.info(`📄 Sample noun files: ${jsonNounFiles.slice(0, 5).map((f) => f.name).join(', ')}`);
|
|
1346
1390
|
}
|
|
1347
|
-
// Count verbs
|
|
1348
|
-
const
|
|
1391
|
+
// Count verbs with timeout
|
|
1392
|
+
const verbScanPromise = this.bucket.getFiles({ prefix: this.verbPrefix });
|
|
1393
|
+
const [verbFiles] = await Promise.race([verbScanPromise, timeoutPromise]);
|
|
1349
1394
|
prodLog.info(`🔍 Found ${verbFiles?.length || 0} total files under verb prefix`);
|
|
1350
1395
|
const jsonVerbFiles = verbFiles?.filter((f) => f.name?.endsWith('.json')) || [];
|
|
1351
1396
|
this.totalVerbCount = jsonVerbFiles.length;
|
|
@@ -1362,15 +1407,39 @@ export class GcsStorage extends BaseStorage {
|
|
|
1362
1407
|
}
|
|
1363
1408
|
}
|
|
1364
1409
|
catch (error) {
|
|
1410
|
+
// Handle timeout specifically
|
|
1411
|
+
if (error.message?.includes('Bucket scan timeout')) {
|
|
1412
|
+
prodLog.error(`❌ TIMEOUT: Bucket scan exceeded ${SCAN_TIMEOUT_MS / 1000}s limit`);
|
|
1413
|
+
prodLog.error(` This typically happens with large buckets in Cloud Run deployments.`);
|
|
1414
|
+
prodLog.error(` Solutions:`);
|
|
1415
|
+
prodLog.error(` 1. Increase Cloud Run timeout: timeoutSeconds: 600`);
|
|
1416
|
+
prodLog.error(` 2. Use skipInitialScan: true in gcsNativeStorage config`);
|
|
1417
|
+
prodLog.error(` 3. Pre-create counts file before deployment`);
|
|
1418
|
+
prodLog.warn(`⚠️ Starting with zero counts due to timeout`);
|
|
1419
|
+
this.totalNounCount = 0;
|
|
1420
|
+
this.totalVerbCount = 0;
|
|
1421
|
+
this.entityCounts = new Map();
|
|
1422
|
+
this.verbCounts = new Map();
|
|
1423
|
+
return;
|
|
1424
|
+
}
|
|
1365
1425
|
// CRITICAL FIX: Don't silently fail - this prevents data loss scenarios
|
|
1366
1426
|
this.logger.error('❌ CRITICAL: Failed to initialize counts from GCS bucket scan:', error);
|
|
1367
|
-
|
|
1427
|
+
prodLog.error(` Error: ${error.message || String(error)}`);
|
|
1428
|
+
prodLog.warn(`⚠️ Starting with zero counts due to error`);
|
|
1429
|
+
this.totalNounCount = 0;
|
|
1430
|
+
this.totalVerbCount = 0;
|
|
1431
|
+
this.entityCounts = new Map();
|
|
1432
|
+
this.verbCounts = new Map();
|
|
1368
1433
|
}
|
|
1369
1434
|
}
|
|
1370
1435
|
/**
|
|
1371
1436
|
* Persist counts to storage
|
|
1372
1437
|
*/
|
|
1373
1438
|
async persistCounts() {
|
|
1439
|
+
// Skip if skipCountsFile is enabled
|
|
1440
|
+
if (this.skipCountsFile) {
|
|
1441
|
+
return;
|
|
1442
|
+
}
|
|
1374
1443
|
try {
|
|
1375
1444
|
const key = `${this.systemPrefix}counts.json`;
|
|
1376
1445
|
const counts = {
|
|
@@ -23,8 +23,8 @@ export interface StorageOptions {
|
|
|
23
23
|
* - 'filesystem': Use file system storage (Node.js only)
|
|
24
24
|
* - 's3': Use Amazon S3 storage
|
|
25
25
|
* - 'r2': Use Cloudflare R2 storage
|
|
26
|
-
* - 'gcs': Use Google Cloud Storage (
|
|
27
|
-
* - 'gcs-native':
|
|
26
|
+
* - 'gcs': Use Google Cloud Storage (native SDK with ADC)
|
|
27
|
+
* - 'gcs-native': DEPRECATED - Use 'gcs' instead
|
|
28
28
|
* - 'azure': Use Azure Blob Storage (native SDK with Managed Identity)
|
|
29
29
|
* - 'type-aware': Use type-first storage adapter (wraps another adapter)
|
|
30
30
|
*/
|
|
@@ -101,7 +101,9 @@ export interface StorageOptions {
|
|
|
101
101
|
secretAccessKey: string;
|
|
102
102
|
};
|
|
103
103
|
/**
|
|
104
|
-
* Configuration for Google Cloud Storage (S3-compatible with HMAC keys)
|
|
104
|
+
* Configuration for Google Cloud Storage (Legacy S3-compatible with HMAC keys)
|
|
105
|
+
* @deprecated Use gcsNativeStorage instead for better performance with ADC
|
|
106
|
+
* This is only needed if you must use HMAC keys for backward compatibility
|
|
105
107
|
*/
|
|
106
108
|
gcsStorage?: {
|
|
107
109
|
/**
|
|
@@ -127,6 +129,8 @@ export interface StorageOptions {
|
|
|
127
129
|
};
|
|
128
130
|
/**
|
|
129
131
|
* Configuration for Google Cloud Storage (native SDK with ADC)
|
|
132
|
+
* This is the recommended way to use GCS with Brainy
|
|
133
|
+
* Supports Application Default Credentials for zero-config cloud deployments
|
|
130
134
|
*/
|
|
131
135
|
gcsNativeStorage?: {
|
|
132
136
|
/**
|
|
@@ -143,12 +147,27 @@ export interface StorageOptions {
|
|
|
143
147
|
credentials?: object;
|
|
144
148
|
/**
|
|
145
149
|
* HMAC access key ID (backward compatibility, not recommended)
|
|
150
|
+
* @deprecated Use ADC, keyFilename, or credentials instead
|
|
146
151
|
*/
|
|
147
152
|
accessKeyId?: string;
|
|
148
153
|
/**
|
|
149
154
|
* HMAC secret access key (backward compatibility, not recommended)
|
|
155
|
+
* @deprecated Use ADC, keyFilename, or credentials instead
|
|
150
156
|
*/
|
|
151
157
|
secretAccessKey?: string;
|
|
158
|
+
/**
|
|
159
|
+
* Skip initial bucket scan for counting entities
|
|
160
|
+
* Useful for large buckets where the scan would timeout
|
|
161
|
+
* If true, counts start at 0 and are updated incrementally
|
|
162
|
+
* @default false
|
|
163
|
+
*/
|
|
164
|
+
skipInitialScan?: boolean;
|
|
165
|
+
/**
|
|
166
|
+
* Skip loading and saving the counts file entirely
|
|
167
|
+
* Useful for very large datasets where counts aren't critical
|
|
168
|
+
* @default false
|
|
169
|
+
*/
|
|
170
|
+
skipCountsFile?: boolean;
|
|
152
171
|
};
|
|
153
172
|
/**
|
|
154
173
|
* Configuration for Azure Blob Storage (native SDK with Managed Identity)
|
|
@@ -125,39 +125,48 @@ export async function createStorage(options = {}) {
|
|
|
125
125
|
console.warn('R2 storage configuration is missing, falling back to memory storage');
|
|
126
126
|
return new MemoryStorage();
|
|
127
127
|
}
|
|
128
|
-
case 'gcs':
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
cacheConfig: options.cacheConfig
|
|
139
|
-
});
|
|
140
|
-
}
|
|
141
|
-
else {
|
|
128
|
+
case 'gcs-native':
|
|
129
|
+
// DEPRECATED: gcs-native is deprecated in favor of just 'gcs'
|
|
130
|
+
console.warn('⚠️ DEPRECATED: type "gcs-native" is deprecated. Use type "gcs" instead.');
|
|
131
|
+
console.warn(' This will continue to work but may be removed in a future version.');
|
|
132
|
+
// Fall through to 'gcs' case
|
|
133
|
+
case 'gcs': {
|
|
134
|
+
// Prefer gcsNativeStorage, but also accept gcsStorage for backward compatibility
|
|
135
|
+
const gcsNative = options.gcsNativeStorage;
|
|
136
|
+
const gcsLegacy = options.gcsStorage;
|
|
137
|
+
if (!gcsNative && !gcsLegacy) {
|
|
142
138
|
console.warn('GCS storage configuration is missing, falling back to memory storage');
|
|
143
139
|
return new MemoryStorage();
|
|
144
140
|
}
|
|
145
|
-
|
|
146
|
-
if (
|
|
147
|
-
console.
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
141
|
+
// If using legacy gcsStorage with HMAC keys, use S3-compatible adapter
|
|
142
|
+
if (gcsLegacy && gcsLegacy.accessKeyId && gcsLegacy.secretAccessKey) {
|
|
143
|
+
console.warn('⚠️ GCS with HMAC keys detected. Consider using gcsNativeStorage with ADC instead.');
|
|
144
|
+
console.warn(' Native GCS with Application Default Credentials is recommended for better performance and security.');
|
|
145
|
+
// Use S3-compatible storage for HMAC keys
|
|
146
|
+
return new S3CompatibleStorage({
|
|
147
|
+
bucketName: gcsLegacy.bucketName,
|
|
148
|
+
region: gcsLegacy.region,
|
|
149
|
+
endpoint: gcsLegacy.endpoint || 'https://storage.googleapis.com',
|
|
150
|
+
accessKeyId: gcsLegacy.accessKeyId,
|
|
151
|
+
secretAccessKey: gcsLegacy.secretAccessKey,
|
|
152
|
+
serviceType: 'gcs',
|
|
154
153
|
cacheConfig: options.cacheConfig
|
|
155
154
|
});
|
|
156
155
|
}
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
156
|
+
// Use native GCS SDK (the correct default!)
|
|
157
|
+
const config = gcsNative || gcsLegacy;
|
|
158
|
+
console.log('Using Google Cloud Storage (native SDK)');
|
|
159
|
+
return new GcsStorage({
|
|
160
|
+
bucketName: config.bucketName,
|
|
161
|
+
keyFilename: gcsNative?.keyFilename,
|
|
162
|
+
credentials: gcsNative?.credentials,
|
|
163
|
+
accessKeyId: config.accessKeyId,
|
|
164
|
+
secretAccessKey: config.secretAccessKey,
|
|
165
|
+
skipInitialScan: gcsNative?.skipInitialScan,
|
|
166
|
+
skipCountsFile: gcsNative?.skipCountsFile,
|
|
167
|
+
cacheConfig: options.cacheConfig
|
|
168
|
+
});
|
|
169
|
+
}
|
|
161
170
|
case 'azure':
|
|
162
171
|
if (options.azureStorage) {
|
|
163
172
|
console.log('Using Azure Blob Storage (native SDK)');
|
|
@@ -233,28 +242,38 @@ export async function createStorage(options = {}) {
|
|
|
233
242
|
cacheConfig: options.cacheConfig
|
|
234
243
|
});
|
|
235
244
|
}
|
|
236
|
-
// If GCS
|
|
237
|
-
|
|
238
|
-
|
|
245
|
+
// If GCS storage is specified (native or legacy S3-compatible)
|
|
246
|
+
// Prefer gcsNativeStorage, but also accept gcsStorage for backward compatibility
|
|
247
|
+
const gcsNative = options.gcsNativeStorage;
|
|
248
|
+
const gcsLegacy = options.gcsStorage;
|
|
249
|
+
if (gcsNative || gcsLegacy) {
|
|
250
|
+
// If using legacy gcsStorage with HMAC keys, use S3-compatible adapter
|
|
251
|
+
if (gcsLegacy && gcsLegacy.accessKeyId && gcsLegacy.secretAccessKey) {
|
|
252
|
+
console.warn('⚠️ GCS with HMAC keys detected. Consider using gcsNativeStorage with ADC instead.');
|
|
253
|
+
console.warn(' Native GCS with Application Default Credentials is recommended for better performance and security.');
|
|
254
|
+
// Use S3-compatible storage for HMAC keys
|
|
255
|
+
console.log('Using Google Cloud Storage (S3-compatible with HMAC - auto-detected)');
|
|
256
|
+
return new S3CompatibleStorage({
|
|
257
|
+
bucketName: gcsLegacy.bucketName,
|
|
258
|
+
region: gcsLegacy.region,
|
|
259
|
+
endpoint: gcsLegacy.endpoint || 'https://storage.googleapis.com',
|
|
260
|
+
accessKeyId: gcsLegacy.accessKeyId,
|
|
261
|
+
secretAccessKey: gcsLegacy.secretAccessKey,
|
|
262
|
+
serviceType: 'gcs',
|
|
263
|
+
cacheConfig: options.cacheConfig
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
// Use native GCS SDK (the correct default!)
|
|
267
|
+
const config = gcsNative || gcsLegacy;
|
|
268
|
+
console.log('Using Google Cloud Storage (native SDK - auto-detected)');
|
|
239
269
|
return new GcsStorage({
|
|
240
|
-
bucketName:
|
|
241
|
-
keyFilename:
|
|
242
|
-
credentials:
|
|
243
|
-
accessKeyId:
|
|
244
|
-
secretAccessKey:
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
}
|
|
248
|
-
// If GCS storage is specified, use it (S3-compatible)
|
|
249
|
-
if (options.gcsStorage) {
|
|
250
|
-
console.log('Using Google Cloud Storage (S3-compatible)');
|
|
251
|
-
return new S3CompatibleStorage({
|
|
252
|
-
bucketName: options.gcsStorage.bucketName,
|
|
253
|
-
region: options.gcsStorage.region,
|
|
254
|
-
endpoint: options.gcsStorage.endpoint || 'https://storage.googleapis.com',
|
|
255
|
-
accessKeyId: options.gcsStorage.accessKeyId,
|
|
256
|
-
secretAccessKey: options.gcsStorage.secretAccessKey,
|
|
257
|
-
serviceType: 'gcs',
|
|
270
|
+
bucketName: config.bucketName,
|
|
271
|
+
keyFilename: gcsNative?.keyFilename,
|
|
272
|
+
credentials: gcsNative?.credentials,
|
|
273
|
+
accessKeyId: config.accessKeyId,
|
|
274
|
+
secretAccessKey: config.secretAccessKey,
|
|
275
|
+
skipInitialScan: gcsNative?.skipInitialScan,
|
|
276
|
+
skipCountsFile: gcsNative?.skipCountsFile,
|
|
258
277
|
cacheConfig: options.cacheConfig
|
|
259
278
|
});
|
|
260
279
|
}
|
package/dist/utils/embedding.js
CHANGED
|
@@ -10,7 +10,7 @@ import { pipeline, env } from '@huggingface/transformers';
|
|
|
10
10
|
if (typeof process !== 'undefined' && process.env) {
|
|
11
11
|
process.env.ORT_DISABLE_MEMORY_ARENA = '1';
|
|
12
12
|
process.env.ORT_DISABLE_MEMORY_PATTERN = '1';
|
|
13
|
-
// Force single-threaded operation for maximum stability (Node.js
|
|
13
|
+
// Force single-threaded operation for maximum stability (Node.js 22 LTS)
|
|
14
14
|
process.env.ORT_INTRA_OP_NUM_THREADS = '1'; // Single thread for operators
|
|
15
15
|
process.env.ORT_INTER_OP_NUM_THREADS = '1'; // Single thread for sessions
|
|
16
16
|
process.env.ORT_NUM_THREADS = '1'; // Additional safety override
|
|
@@ -56,8 +56,8 @@ export async function areWorkerThreadsAvailable() {
|
|
|
56
56
|
export function areWorkerThreadsAvailableSync() {
|
|
57
57
|
if (!isNode())
|
|
58
58
|
return false;
|
|
59
|
-
// In Node.js
|
|
60
|
-
return parseInt(process.versions.node.split('.')[0]) >=
|
|
59
|
+
// In Node.js 22+, worker_threads is always available
|
|
60
|
+
return parseInt(process.versions.node.split('.')[0]) >= 22;
|
|
61
61
|
}
|
|
62
62
|
/**
|
|
63
63
|
* Determine if threading is available in the current environment
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Utility functions for executing functions in Worker Threads (Node.js) or Web Workers (Browser)
|
|
3
|
-
* This implementation leverages Node.js
|
|
3
|
+
* This implementation leverages Node.js 22's stable Worker Threads API for maximum reliability
|
|
4
4
|
*/
|
|
5
5
|
/**
|
|
6
6
|
* Execute a function in a separate thread
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Utility functions for executing functions in Worker Threads (Node.js) or Web Workers (Browser)
|
|
3
|
-
* This implementation leverages Node.js
|
|
3
|
+
* This implementation leverages Node.js 22's stable Worker Threads API for maximum reliability
|
|
4
4
|
*/
|
|
5
5
|
import { isBrowser, isNode } from './environment.js';
|
|
6
6
|
// Worker pool to reuse workers
|
|
@@ -64,7 +64,7 @@ export function executeInThread(fnString, args) {
|
|
|
64
64
|
}
|
|
65
65
|
/**
|
|
66
66
|
* Execute a function in a Node.js Worker Thread
|
|
67
|
-
* Optimized for Node.js
|
|
67
|
+
* Optimized for Node.js 22 LTS with stable Worker Threads performance
|
|
68
68
|
*/
|
|
69
69
|
function executeInNodeWorker(fnString, args) {
|
|
70
70
|
return new Promise((resolve, reject) => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@soulcraft/brainy",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.1.1",
|
|
4
4
|
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.js",
|
|
@@ -163,8 +163,8 @@
|
|
|
163
163
|
},
|
|
164
164
|
"dependencies": {
|
|
165
165
|
"@aws-sdk/client-s3": "^3.540.0",
|
|
166
|
-
"@azure/identity": "^4.
|
|
167
|
-
"@azure/storage-blob": "^12.
|
|
166
|
+
"@azure/identity": "^4.0.0",
|
|
167
|
+
"@azure/storage-blob": "^12.17.0",
|
|
168
168
|
"@google-cloud/storage": "^7.14.0",
|
|
169
169
|
"@huggingface/transformers": "^3.7.2",
|
|
170
170
|
"@msgpack/msgpack": "^3.1.2",
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Knowledge Layer Augmentation for VFS
|
|
3
|
-
*
|
|
4
|
-
* Adds intelligent features to VFS without modifying core functionality:
|
|
5
|
-
* - Event recording for all operations
|
|
6
|
-
* - Semantic versioning based on content changes
|
|
7
|
-
* - Entity and concept extraction
|
|
8
|
-
* - Git bridge for import/export
|
|
9
|
-
*
|
|
10
|
-
* This is a TRUE augmentation - VFS works perfectly without it
|
|
11
|
-
*/
|
|
12
|
-
import { Brainy } from '../brainy.js';
|
|
13
|
-
import { BaseAugmentation } from './brainyAugmentation.js';
|
|
14
|
-
export declare class KnowledgeAugmentation extends BaseAugmentation {
|
|
15
|
-
name: string;
|
|
16
|
-
timing: 'after';
|
|
17
|
-
metadata: 'none';
|
|
18
|
-
operations: any;
|
|
19
|
-
priority: number;
|
|
20
|
-
constructor(config?: any);
|
|
21
|
-
execute<T = any>(operation: string, params: any, next: () => Promise<T>): Promise<T>;
|
|
22
|
-
private eventRecorder?;
|
|
23
|
-
private semanticVersioning?;
|
|
24
|
-
private entitySystem?;
|
|
25
|
-
private conceptSystem?;
|
|
26
|
-
private gitBridge?;
|
|
27
|
-
private originalMethods;
|
|
28
|
-
initialize(context: any): Promise<void>;
|
|
29
|
-
augment(brain: Brainy): Promise<void>;
|
|
30
|
-
/**
|
|
31
|
-
* Wrap a VFS method to add Knowledge Layer functionality
|
|
32
|
-
*/
|
|
33
|
-
private wrapMethod;
|
|
34
|
-
/**
|
|
35
|
-
* Add Knowledge Layer methods to VFS
|
|
36
|
-
*/
|
|
37
|
-
private addKnowledgeMethods;
|
|
38
|
-
private isSemanticChange;
|
|
39
|
-
cleanup(brain: Brainy): Promise<void>;
|
|
40
|
-
}
|