@soulcraft/brainy 4.9.1 → 4.9.2

package/dist/storage/adapters/s3CompatibleStorage.js

@@ -3074,51 +3074,74 @@ export class S3CompatibleStorage extends BaseStorage {
      */
     async saveHNSWData(nounId, hnswData) {
         await this.ensureInitialized();
-        try {
-            const { PutObjectCommand, GetObjectCommand } = await import('@aws-sdk/client-s3');
-            // CRITICAL FIX (v4.7.3): Must preserve existing node data (id, vector) when updating HNSW metadata
-            const shard = getShardIdFromUuid(nounId);
-            const key = `entities/nouns/hnsw/${shard}/${nounId}.json`;
+        const { PutObjectCommand, GetObjectCommand } = await import('@aws-sdk/client-s3');
+        // CRITICAL FIX (v4.7.3): Must preserve existing node data (id, vector) when updating HNSW metadata
+        // Previous implementation overwrote the entire file, destroying vector data
+        // Now we READ the existing node, UPDATE only connections/level, then WRITE back the complete node
+        // CRITICAL FIX (v4.10.1): Optimistic locking with ETags to prevent race conditions
+        // Uses S3 IfMatch preconditions - retries with exponential backoff on conflicts
+        // Prevents data corruption when multiple entities connect to same neighbor simultaneously
+        const shard = getShardIdFromUuid(nounId);
+        const key = `entities/nouns/hnsw/${shard}/${nounId}.json`;
+        const maxRetries = 5;
+        for (let attempt = 0; attempt < maxRetries; attempt++) {
             try {
-                // Read existing node data
-                const getResponse = await this.s3Client.send(new GetObjectCommand({
-                    Bucket: this.bucketName,
-                    Key: key
-                }));
-                const existingData = await getResponse.Body.transformToString();
-                const existingNode = JSON.parse(existingData);
+                // Get current ETag and data
+                let currentETag;
+                let existingNode = {};
+                try {
+                    const getResponse = await this.s3Client.send(new GetObjectCommand({
+                        Bucket: this.bucketName,
+                        Key: key
+                    }));
+                    const existingData = await getResponse.Body.transformToString();
+                    existingNode = JSON.parse(existingData);
+                    currentETag = getResponse.ETag;
+                }
+                catch (error) {
+                    // File doesn't exist yet - will create new
+                    if (error.name !== 'NoSuchKey' && error.Code !== 'NoSuchKey') {
+                        throw error;
+                    }
+                }
                 // Preserve id and vector, update only HNSW graph metadata
                 const updatedNode = {
-                    ...existingNode,
+                    ...existingNode, // Preserve all existing fields (id, vector, etc.)
                     level: hnswData.level,
                     connections: hnswData.connections
                 };
+                // ATOMIC WRITE: Use ETag precondition
+                // If currentETag exists, only write if ETag matches (no concurrent modification)
+                // If no ETag, only write if file doesn't exist (IfNoneMatch: *)
                 await this.s3Client.send(new PutObjectCommand({
                     Bucket: this.bucketName,
                     Key: key,
                     Body: JSON.stringify(updatedNode, null, 2),
-                    ContentType: 'application/json'
+                    ContentType: 'application/json',
+                    ...(currentETag
+                        ? { IfMatch: currentETag }
+                        : { IfNoneMatch: '*' }) // Only create if doesn't exist
                 }));
+                // Success! Exit retry loop
+                return;
             }
             catch (error) {
-                // If node doesn't exist yet, create it with just HNSW data
-                if (error.name === 'NoSuchKey' || error.Code === 'NoSuchKey') {
-                    await this.s3Client.send(new PutObjectCommand({
-                        Bucket: this.bucketName,
-                        Key: key,
-                        Body: JSON.stringify(hnswData, null, 2),
-                        ContentType: 'application/json'
-                    }));
-                }
-                else {
-                    throw error;
+                // Precondition failed - concurrent modification detected
+                if (error.name === 'PreconditionFailed' || error.Code === 'PreconditionFailed') {
+                    if (attempt === maxRetries - 1) {
+                        this.logger.error(`Max retries (${maxRetries}) exceeded for ${nounId} - concurrent modification conflict`);
+                        throw new Error(`Failed to save HNSW data for ${nounId}: max retries exceeded due to concurrent modifications`);
+                    }
+                    // Exponential backoff: 50ms, 100ms, 200ms, 400ms, 800ms
+                    const backoffMs = 50 * Math.pow(2, attempt);
+                    await new Promise(resolve => setTimeout(resolve, backoffMs));
+                    continue;
                 }
+                // Other error - rethrow
+                this.logger.error(`Failed to save HNSW data for ${nounId}:`, error);
+                throw new Error(`Failed to save HNSW data for ${nounId}: ${error}`);
             }
         }
-        catch (error) {
-            this.logger.error(`Failed to save HNSW data for ${nounId}:`, error);
-            throw new Error(`Failed to save HNSW data for ${nounId}: ${error}`);
-        }
     }
     /**
      * Get HNSW graph data for a noun
@@ -3153,22 +3176,59 @@ export class S3CompatibleStorage extends BaseStorage {
     /**
      * Save HNSW system data (entry point, max level)
      * Storage path: system/hnsw-system.json
+     *
+     * CRITICAL FIX (v4.10.1): Optimistic locking with ETags to prevent race conditions
      */
     async saveHNSWSystem(systemData) {
         await this.ensureInitialized();
-        try {
-            const { PutObjectCommand } = await import('@aws-sdk/client-s3');
-            const key = `${this.systemPrefix}hnsw-system.json`;
-            await this.s3Client.send(new PutObjectCommand({
-                Bucket: this.bucketName,
-                Key: key,
-                Body: JSON.stringify(systemData, null, 2),
-                ContentType: 'application/json'
-            }));
-        }
-        catch (error) {
-            this.logger.error('Failed to save HNSW system data:', error);
-            throw new Error(`Failed to save HNSW system data: ${error}`);
+        const { PutObjectCommand, HeadObjectCommand } = await import('@aws-sdk/client-s3');
+        const key = `${this.systemPrefix}hnsw-system.json`;
+        const maxRetries = 5;
+        for (let attempt = 0; attempt < maxRetries; attempt++) {
+            try {
+                // Get current ETag (use HEAD to avoid downloading data)
+                let currentETag;
+                try {
+                    const headResponse = await this.s3Client.send(new HeadObjectCommand({
+                        Bucket: this.bucketName,
+                        Key: key
+                    }));
+                    currentETag = headResponse.ETag;
+                }
+                catch (error) {
+                    // File doesn't exist yet
+                    if (error.name !== 'NotFound' && error.name !== 'NoSuchKey' && error.Code !== 'NoSuchKey') {
+                        throw error;
+                    }
+                }
+                // ATOMIC WRITE: Use ETag precondition
+                await this.s3Client.send(new PutObjectCommand({
+                    Bucket: this.bucketName,
+                    Key: key,
+                    Body: JSON.stringify(systemData, null, 2),
+                    ContentType: 'application/json',
+                    ...(currentETag
+                        ? { IfMatch: currentETag }
+                        : { IfNoneMatch: '*' })
+                }));
+                // Success!
+                return;
+            }
+            catch (error) {
+                // Precondition failed - concurrent modification
+                if (error.name === 'PreconditionFailed' || error.Code === 'PreconditionFailed') {
+                    if (attempt === maxRetries - 1) {
+                        this.logger.error(`Max retries (${maxRetries}) exceeded for HNSW system data`);
+                        throw new Error('Failed to save HNSW system data: max retries exceeded due to concurrent modifications');
+                    }
+                    const backoffMs = 50 * Math.pow(2, attempt);
+                    await new Promise(resolve => setTimeout(resolve, backoffMs));
+                    continue;
+                }
+                // Other error - rethrow
+                this.logger.error('Failed to save HNSW system data:', error);
+                throw new Error(`Failed to save HNSW system data: ${error}`);
+            }
         }
     }
     /**

package/dist/vfs/importers/DirectoryImporter.d.ts

@@ -19,6 +19,9 @@ export interface ImportOptions {
     extractMetadata?: boolean;
     showProgress?: boolean;
     filter?: (path: string) => boolean;
+    importId?: string;
+    projectId?: string;
+    customMetadata?: Record<string, any>;
 }
 export interface ImportResult {
     imported: string[];
@@ -47,6 +50,10 @@ export declare class DirectoryImporter {
      * Import a directory or file into VFS
      */
     import(sourcePath: string, options?: ImportOptions): Promise<ImportResult>;
+    /**
+     * Derive project ID from target path
+     */
+    private deriveProjectId;
     /**
      * Import with progress tracking (generator)
      */

package/dist/vfs/importers/DirectoryImporter.js

@@ -9,6 +9,7 @@
  */
 import { promises as fs } from 'fs';
 import * as path from 'path';
+import { v4 as uuidv4 } from '../../universal/uuid.js';
 export class DirectoryImporter {
     constructor(vfs, brain) {
         this.vfs = vfs;
@@ -19,6 +20,18 @@ export class DirectoryImporter {
      */
     async import(sourcePath, options = {}) {
         const startTime = Date.now();
+        // v4.10.0: Generate tracking metadata
+        const importId = options.importId || uuidv4();
+        const projectId = options.projectId || this.deriveProjectId(options.targetPath || '/');
+        const trackingMetadata = {
+            importIds: [importId],
+            projectId,
+            importedAt: Date.now(),
+            importSource: sourcePath,
+            ...(options.customMetadata || {})
+        };
+        // Store tracking metadata in options for use in helper methods
+        const enhancedOptions = { ...options, _trackingMetadata: trackingMetadata };
         const result = {
             imported: [],
             failed: [],
@@ -34,7 +47,7 @@ export class DirectoryImporter {
                 await this.importFile(sourcePath, options.targetPath || '/', result);
             }
             else if (stats.isDirectory()) {
-                await this.importDirectory(sourcePath, options, result);
+                await this.importDirectory(sourcePath, enhancedOptions, result);
             }
         }
         catch (error) {
@@ -46,6 +59,13 @@ export class DirectoryImporter {
         result.duration = Date.now() - startTime;
         return result;
     }
+    /**
+     * Derive project ID from target path
+     */
+    deriveProjectId(targetPath) {
+        const segments = targetPath.split('/').filter(s => s.length > 0);
+        return segments.length > 0 ? segments[0] : 'default_project';
+    }
     /**
      * Import with progress tracking (generator)
      */
@@ -125,9 +145,13 @@ export class DirectoryImporter {
         };
         await collectDirs(sourcePath, targetPath);
         // Create all directories
+        const trackingMetadata = options._trackingMetadata || {};
         for (const dirPath of dirsToCreate) {
             try {
-                await this.vfs.mkdir(dirPath, { recursive: true });
+                await this.vfs.mkdir(dirPath, {
+                    recursive: true,
+                    metadata: trackingMetadata // v4.10.0: Add tracking metadata
+                });
                 result.directoriesCreated++;
             }
             catch (error) {
@@ -207,14 +231,15 @@ export class DirectoryImporter {
             }
         }
         // Write to VFS
+        const trackingMetadata = options._trackingMetadata || {};
         await this.vfs.writeFile(vfsPath, content, {
             generateEmbedding: options.generateEmbeddings,
             extractMetadata: options.extractMetadata,
             metadata: {
                 originalPath: filePath,
-                importedAt: Date.now(),
                 originalSize: stats.size,
-                originalModified: stats.mtime.getTime()
+                originalModified: stats.mtime.getTime(),
+                ...trackingMetadata // v4.10.0: Add tracking metadata
             }
         });
         return { vfsPath, size: stats.size };

package/dist/vfs/types.d.ts

@@ -54,6 +54,11 @@ export interface VFSMetadata {
     dependencies?: string[];
     exports?: string[];
     language?: string;
+    importIds?: string[];
+    projectId?: string;
+    importedAt?: number;
+    importFormat?: string;
+    importSource?: string;
     lineCount?: number;
     wordCount?: number;
     charset?: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "4.9.1",
+  "version": "4.9.2",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",