@soulcraft/brainy 3.50.1 → 4.0.0

package/dist/storage/adapters/fileSystemStorage.js

@@ -7,16 +7,19 @@ import { StorageCompatibilityLayer } from '../backwardCompatibility.js';
 // Node.js modules - dynamically imported to avoid issues in browser environments
 let fs;
 let path;
+let zlib;
 let moduleLoadingPromise = null;
 // Try to load Node.js modules
 try {
     // Using dynamic imports to avoid issues in browser environments
     const fsPromise = import('node:fs');
     const pathPromise = import('node:path');
-    moduleLoadingPromise = Promise.all([fsPromise, pathPromise])
-        .then(([fsModule, pathModule]) => {
+    const zlibPromise = import('node:zlib');
+    moduleLoadingPromise = Promise.all([fsPromise, pathPromise, zlibPromise])
+        .then(([fsModule, pathModule, zlibModule]) => {
         fs = fsModule;
         path = pathModule.default;
+        zlib = zlibModule;
     })
         .catch((error) => {
         console.error('Failed to load Node.js modules:', error);
@@ -34,8 +37,9 @@ export class FileSystemStorage extends BaseStorage {
     /**
      * Initialize the storage adapter
      * @param rootDirectory The root directory for storage
+     * @param options Optional configuration
      */
-    constructor(rootDirectory) {
+    constructor(rootDirectory, options) {
         super();
         // Fixed sharding configuration for optimal balance of simplicity and performance
         // Single-level sharding (depth=1) provides excellent performance for 1-2.5M entities
@@ -50,7 +54,17 @@ export class FileSystemStorage extends BaseStorage {
         this.activeLocks = new Set();
         this.lockTimers = new Map(); // Track timers for cleanup
         this.allTimers = new Set(); // Track all timers for cleanup
+        // Compression configuration (v4.0.0)
+        this.compressionEnabled = true; // Enable gzip compression by default for 60-80% disk savings
+        this.compressionLevel = 6; // zlib compression level (1-9, default: 6 = balanced)
         this.rootDir = rootDirectory;
+        // Configure compression
+        if (options?.compression !== undefined) {
+            this.compressionEnabled = options.compression;
+        }
+        if (options?.compressionLevel !== undefined) {
+            this.compressionLevel = Math.min(9, Math.max(1, options.compressionLevel));
+        }
         // Defer path operations until init() when path module is guaranteed to be loaded
     }
     /**
@@ -180,9 +194,11 @@ export class FileSystemStorage extends BaseStorage {
         const filePath = this.getNodePath(node.id);
         await this.ensureDirectoryExists(path.dirname(filePath));
         await fs.promises.writeFile(filePath, JSON.stringify(serializableNode, null, 2));
-        // Update counts for new nodes (intelligent type detection)
+        // Update counts for new nodes (v4.0.0: load metadata separately)
         if (isNew) {
-            const type = node.metadata?.type || node.metadata?.nounType || 'default';
+            // v4.0.0: Get type from separate metadata storage
+            const metadata = await this.getNounMetadata(node.id);
+            const type = metadata?.noun || 'default';
             this.incrementEntityCount(type);
             // Persist counts periodically (every 10 operations for efficiency)
             if (this.totalNounCount % 10 === 0) {
@@ -308,16 +324,16 @@ export class FileSystemStorage extends BaseStorage {
     async deleteNode(id) {
         await this.ensureInitialized();
         const filePath = this.getNodePath(id);
-        // Load node to get type for count update
+        // Load metadata to get type for count update (v4.0.0: separate storage)
         try {
-            const node = await this.getNode(id);
-            if (node) {
-                const type = node.metadata?.type || node.metadata?.nounType || 'default';
+            const metadata = await this.getNounMetadata(id);
+            if (metadata) {
+                const type = metadata.noun || 'default';
                 this.decrementEntityCount(type);
             }
         }
         catch {
-            // Node might not exist, that's ok
+            // Metadata might not exist, that's ok
         }
         try {
             await fs.promises.unlink(filePath);
@@ -380,7 +396,7 @@ export class FileSystemStorage extends BaseStorage {
             for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
                 connections.set(Number(level), new Set(nodeIds));
             }
-            // ARCHITECTURAL FIX (v3.50.1): Return HNSWVerb with core relational fields
+            // v4.0.0: Return HNSWVerb with core relational fields (NO metadata field)
             return {
                 id: parsedEdge.id,
                 vector: parsedEdge.vector,
@@ -388,9 +404,9 @@ export class FileSystemStorage extends BaseStorage {
                 // CORE RELATIONAL DATA (read from vector file)
                 verb: parsedEdge.verb,
                 sourceId: parsedEdge.sourceId,
-                targetId: parsedEdge.targetId,
-                // User metadata (retrieved separately via getVerbMetadata())
-                metadata: parsedEdge.metadata
+                targetId: parsedEdge.targetId
+                // ✅ NO metadata field in v4.0.0
+                // User metadata retrieved separately via getVerbMetadata()
             };
         }
         catch (error) {
@@ -423,7 +439,7 @@ export class FileSystemStorage extends BaseStorage {
                 for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
                     connections.set(Number(level), new Set(nodeIds));
                 }
-                // ARCHITECTURAL FIX (v3.50.1): Include core relational fields
+                // v4.0.0: Include core relational fields (NO metadata field)
                 allEdges.push({
                     id: parsedEdge.id,
                     vector: parsedEdge.vector,
@@ -431,9 +447,9 @@ export class FileSystemStorage extends BaseStorage {
                     // CORE RELATIONAL DATA
                     verb: parsedEdge.verb,
                     sourceId: parsedEdge.sourceId,
-                    targetId: parsedEdge.targetId,
-                    // User metadata
-                    metadata: parsedEdge.metadata
+                    targetId: parsedEdge.targetId
+                    // ✅ NO metadata field in v4.0.0
+                    // User metadata retrieved separately via getVerbMetadata()
                 });
             }
         }
@@ -492,7 +508,7 @@ export class FileSystemStorage extends BaseStorage {
         try {
             const metadata = await this.getVerbMetadata(id);
             if (metadata) {
-                const verbType = metadata.verb || metadata.type || 'default';
+                const verbType = (metadata.verb || metadata.type || 'default');
                 this.decrementVerbCount(verbType);
                 await this.deleteVerbMetadata(id);
             }
@@ -505,21 +521,71 @@ export class FileSystemStorage extends BaseStorage {
     /**
      * Primitive operation: Write object to path
      * All metadata operations use this internally via base class routing
+     * v4.0.0: Supports gzip compression for 60-80% disk savings
      */
     async writeObjectToPath(pathStr, data) {
         await this.ensureInitialized();
         const fullPath = path.join(this.rootDir, pathStr);
         await this.ensureDirectoryExists(path.dirname(fullPath));
-        await fs.promises.writeFile(fullPath, JSON.stringify(data, null, 2));
+        if (this.compressionEnabled) {
+            // Write compressed data with .gz extension
+            const compressedPath = `${fullPath}.gz`;
+            const jsonString = JSON.stringify(data, null, 2);
+            const compressed = await new Promise((resolve, reject) => {
+                zlib.gzip(Buffer.from(jsonString, 'utf-8'), { level: this.compressionLevel }, (err, result) => {
+                    if (err)
+                        reject(err);
+                    else
+                        resolve(result);
+                });
+            });
+            await fs.promises.writeFile(compressedPath, compressed);
+            // Clean up uncompressed file if it exists (migration from uncompressed)
+            try {
+                await fs.promises.unlink(fullPath);
+            }
+            catch (error) {
+                // Ignore if file doesn't exist
+                if (error.code !== 'ENOENT') {
+                    console.warn(`Failed to remove uncompressed file ${fullPath}:`, error);
+                }
+            }
+        }
+        else {
+            // Write uncompressed data
+            await fs.promises.writeFile(fullPath, JSON.stringify(data, null, 2));
+        }
     }
     /**
      * Primitive operation: Read object from path
      * All metadata operations use this internally via base class routing
      * Enhanced error handling for corrupted metadata files (Bug #3 mitigation)
+     * v4.0.0: Supports reading both compressed (.gz) and uncompressed files for backward compatibility
      */
     async readObjectFromPath(pathStr) {
         await this.ensureInitialized();
         const fullPath = path.join(this.rootDir, pathStr);
+        const compressedPath = `${fullPath}.gz`;
+        // Try reading compressed file first (if compression is enabled or file exists)
+        try {
+            const compressedData = await fs.promises.readFile(compressedPath);
+            const decompressed = await new Promise((resolve, reject) => {
+                zlib.gunzip(compressedData, (err, result) => {
+                    if (err)
+                        reject(err);
+                    else
+                        resolve(result);
+                });
+            });
+            return JSON.parse(decompressed.toString('utf-8'));
+        }
+        catch (error) {
+            // If compressed file doesn't exist, fall back to uncompressed
+            if (error.code !== 'ENOENT') {
+                console.warn(`Failed to read compressed file ${compressedPath}:`, error);
+            }
+        }
+        // Fall back to reading uncompressed file (for backward compatibility)
         try {
             const data = await fs.promises.readFile(fullPath, 'utf-8');
             return JSON.parse(data);
@@ -542,33 +608,71 @@ export class FileSystemStorage extends BaseStorage {
     /**
      * Primitive operation: Delete object from path
      * All metadata operations use this internally via base class routing
+     * v4.0.0: Deletes both compressed and uncompressed versions (for cleanup)
      */
     async deleteObjectFromPath(pathStr) {
         await this.ensureInitialized();
         const fullPath = path.join(this.rootDir, pathStr);
+        const compressedPath = `${fullPath}.gz`;
+        // Try deleting both compressed and uncompressed files (for cleanup during migration)
+        let deletedCount = 0;
+        // Delete compressed file
+        try {
+            await fs.promises.unlink(compressedPath);
+            deletedCount++;
+        }
+        catch (error) {
+            if (error.code !== 'ENOENT') {
+                console.warn(`Error deleting compressed file ${compressedPath}:`, error);
+            }
+        }
+        // Delete uncompressed file
         try {
             await fs.promises.unlink(fullPath);
+            deletedCount++;
         }
         catch (error) {
             if (error.code !== 'ENOENT') {
-                console.error(`Error deleting object from ${pathStr}:`, error);
+                console.error(`Error deleting uncompressed file ${pathStr}:`, error);
                 throw error;
             }
         }
+        // If neither file existed, it's not an error (already deleted)
+        if (deletedCount === 0) {
+            // File doesn't exist - this is fine
+        }
     }
     /**
      * Primitive operation: List objects under path prefix
      * All metadata operations use this internally via base class routing
+     * v4.0.0: Handles both .json and .json.gz files, normalizes paths
      */
     async listObjectsUnderPath(prefix) {
         await this.ensureInitialized();
         const fullPath = path.join(this.rootDir, prefix);
         const paths = [];
+        const seen = new Set(); // Track files to avoid duplicates (both .json and .json.gz)
         try {
             const entries = await fs.promises.readdir(fullPath, { withFileTypes: true });
             for (const entry of entries) {
-                if (entry.isFile() && entry.name.endsWith('.json')) {
-                    paths.push(path.join(prefix, entry.name));
+                if (entry.isFile()) {
+                    // Handle both .json and .json.gz files
+                    if (entry.name.endsWith('.json.gz')) {
+                        // Strip .gz extension and add the .json path
+                        const normalizedName = entry.name.slice(0, -3); // Remove .gz
+                        const normalizedPath = path.join(prefix, normalizedName);
+                        if (!seen.has(normalizedPath)) {
+                            paths.push(normalizedPath);
+                            seen.add(normalizedPath);
+                        }
+                    }
+                    else if (entry.name.endsWith('.json')) {
+                        const filePath = path.join(prefix, entry.name);
+                        if (!seen.has(filePath)) {
+                            paths.push(filePath);
+                            seen.add(filePath);
+                        }
+                    }
                 }
                 else if (entry.isDirectory()) {
                     const subdirPaths = await this.listObjectsUnderPath(path.join(prefix, entry.name));
@@ -646,7 +750,7 @@ export class FileSystemStorage extends BaseStorage {
             }
             // Get page of files
             const pageFiles = nounFiles.slice(startIndex, startIndex + limit);
-            // Load nouns - count actual successfully loaded items
+            // v4.0.0: Load nouns and combine with metadata
             const items = [];
             let successfullyLoaded = 0;
             let totalValidFiles = 0;
@@ -654,18 +758,21 @@ export class FileSystemStorage extends BaseStorage {
             totalValidFiles = this.totalNounCount;
             // No need to count files anymore - we maintain accurate counts
             // This eliminates the O(n) operation completely
-            // Second pass: load the current page
+            // Second pass: load the current page with metadata
             for (const file of pageFiles) {
                 try {
                     const id = file.replace('.json', '');
                     const data = await fs.promises.readFile(this.getNodePath(id), 'utf-8');
-                    const noun = JSON.parse(data);
+                    const parsedNoun = JSON.parse(data);
+                    // v4.0.0: Load metadata from separate storage
+                    const metadata = await this.getNounMetadata(id);
+                    if (!metadata)
+                        continue;
                     // Apply filter if provided
                     if (options.filter) {
-                        // Simple filter implementation
                         let matches = true;
                         for (const [key, value] of Object.entries(options.filter)) {
-                            if (noun.metadata && noun.metadata[key] !== value) {
+                            if (metadata[key] !== value) {
                                 matches = false;
                                 break;
                             }
@@ -673,7 +780,24 @@ export class FileSystemStorage extends BaseStorage {
                         if (!matches)
                             continue;
                     }
-                    items.push(noun);
+                    // Convert connections if needed
+                    let connections = parsedNoun.connections;
+                    if (connections && typeof connections === 'object' && !(connections instanceof Map)) {
+                        const connectionsMap = new Map();
+                        for (const [level, nodeIds] of Object.entries(connections)) {
+                            connectionsMap.set(Number(level), new Set(nodeIds));
+                        }
+                        connections = connectionsMap;
+                    }
+                    // v4.0.0: Create HNSWNounWithMetadata by combining noun with metadata
+                    const nounWithMetadata = {
+                        id: parsedNoun.id,
+                        vector: parsedNoun.vector,
+                        connections: connections,
+                        level: parsedNoun.level || 0,
+                        metadata: metadata
+                    };
+                    items.push(nounWithMetadata);
                     successfullyLoaded++;
                 }
                 catch (error) {
@@ -895,21 +1019,17 @@ export class FileSystemStorage extends BaseStorage {
     }
     /**
      * Get a noun from storage (internal implementation)
-     * Combines vector data from getNode() with metadata from getNounMetadata()
+     * v4.0.0: Returns ONLY vector data (no metadata field)
+     * Base class combines with metadata via getNoun() -> HNSWNounWithMetadata
      */
     async getNoun_internal(id) {
-        // Get vector data (lightweight)
+        // v4.0.0: Return ONLY vector data (no metadata field)
         const node = await this.getNode(id);
         if (!node) {
             return null;
         }
-        // Get metadata (entity data in 2-file system)
-        const metadata = await this.getNounMetadata(id);
-        // Combine into complete noun object
-        return {
-            ...node,
-            metadata: metadata || {}
-        };
+        // Return pure vector structure
+        return node;
     }
     /**
      * Get nouns by noun type
@@ -931,21 +1051,17 @@ export class FileSystemStorage extends BaseStorage {
     }
     /**
      * Get a verb from storage (internal implementation)
-     * Combines vector data from getEdge() with metadata from getVerbMetadata()
+     * v4.0.0: Returns ONLY vector + core relational fields (no metadata field)
+     * Base class combines with metadata via getVerb() -> HNSWVerbWithMetadata
      */
     async getVerb_internal(id) {
-        // Get vector data (lightweight)
+        // v4.0.0: Return ONLY vector + core relational data (no metadata field)
         const edge = await this.getEdge(id);
         if (!edge) {
             return null;
         }
-        // Get metadata (relationship data in 2-file system)
-        const metadata = await this.getVerbMetadata(id);
-        // Combine into complete verb object
-        return {
-            ...edge,
-            metadata: metadata || {}
-        };
+        // Return pure vector + core fields structure
+        return edge;
     }
     /**
      * Get verbs by source
@@ -1029,22 +1145,9 @@ export class FileSystemStorage extends BaseStorage {
                     const edge = JSON.parse(data);
                     // Get metadata which contains the actual verb information
                     const metadata = await this.getVerbMetadata(id);
-                    // If no metadata exists, try to reconstruct basic metadata from filename
+                    // v4.0.0: No fallbacks - skip verbs without metadata
                     if (!metadata) {
-                        console.warn(`Verb ${id} has no metadata, trying to create minimal verb`);
-                        // Create minimal GraphVerb without full metadata
-                        const minimalVerb = {
-                            id: edge.id,
-                            vector: edge.vector,
-                            connections: edge.connections || new Map(),
-                            sourceId: 'unknown',
-                            targetId: 'unknown',
-                            source: 'unknown',
-                            target: 'unknown',
-                            type: 'relationship',
-                            verb: 'relatedTo'
-                        };
-                        verbs.push(minimalVerb);
+                        console.warn(`Verb ${id} has no metadata, skipping`);
                         continue;
                     }
                     // Convert connections Map to proper format if needed
@@ -1056,24 +1159,15 @@ export class FileSystemStorage extends BaseStorage {
                         }
                         connections = connectionsMap;
                     }
-                    // Properly reconstruct GraphVerb from HNSWVerb + metadata
-                    const verb = {
+                    // v4.0.0: Clean HNSWVerbWithMetadata construction
+                    const verbWithMetadata = {
                         id: edge.id,
-                        vector: edge.vector, // Include the vector field!
+                        vector: edge.vector,
                         connections: connections,
-                        sourceId: metadata.sourceId || metadata.source,
-                        targetId: metadata.targetId || metadata.target,
-                        source: metadata.source || metadata.sourceId,
-                        target: metadata.target || metadata.targetId,
-                        verb: metadata.verb || metadata.type,
-                        type: metadata.type || metadata.verb,
-                        weight: metadata.weight,
-                        metadata: metadata.metadata || metadata,
-                        data: metadata.data,
-                        createdAt: metadata.createdAt,
-                        updatedAt: metadata.updatedAt,
-                        createdBy: metadata.createdBy,
-                        embedding: metadata.embedding || edge.vector
+                        verb: edge.verb,
+                        sourceId: edge.sourceId,
+                        targetId: edge.targetId,
+                        metadata: metadata
                     };
                     // Apply filters if provided
                     if (options.filter) {
@@ -1081,22 +1175,19 @@ export class FileSystemStorage extends BaseStorage {
                         // Check verbType filter
                         if (filter.verbType) {
                             const types = Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType];
-                            const verbType = verb.type || verb.verb;
-                            if (verbType && !types.includes(verbType))
+                            if (!types.includes(verbWithMetadata.verb))
                                 continue;
                         }
                         // Check sourceId filter
                         if (filter.sourceId) {
                             const sources = Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId];
-                            const sourceId = verb.sourceId || verb.source;
-                            if (!sourceId || !sources.includes(sourceId))
+                            if (!sources.includes(verbWithMetadata.sourceId))
                                 continue;
                         }
                         // Check targetId filter
                         if (filter.targetId) {
                             const targets = Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId];
-                            const targetId = verb.targetId || verb.target;
-                            if (!targetId || !targets.includes(targetId))
+                            if (!targets.includes(verbWithMetadata.targetId))
                                 continue;
                         }
                         // Check service filter
@@ -1106,7 +1197,7 @@ export class FileSystemStorage extends BaseStorage {
                                 continue;
                         }
                     }
-                    verbs.push(verb);
+                    verbs.push(verbWithMetadata);
                     successfullyLoaded++;
                 }
                 catch (error) {
@@ -1485,33 +1576,21 @@ export class FileSystemStorage extends BaseStorage {
             const validVerbFiles = await this.getAllFilesAtDepth(this.verbsDir, depthToUse);
             this.totalVerbCount = validVerbFiles.length;
             // Sample some files to get type distribution (don't read all)
+            // v4.0.0: Load metadata separately for type information
             const sampleSize = Math.min(100, validNounFiles.length);
             for (let i = 0; i < sampleSize; i++) {
                 try {
                     const file = validNounFiles[i];
                     const id = file.replace('.json', '');
-                    // Construct path using detected depth (not cached depth which may be wrong)
-                    let filePath;
-                    switch (depthToUse) {
-                        case 0:
-                            filePath = path.join(this.nounsDir, `${id}.json`);
-                            break;
-                        case 1:
-                            filePath = path.join(this.nounsDir, id.substring(0, 2), `${id}.json`);
-                            break;
-                        case 2:
-                            filePath = path.join(this.nounsDir, id.substring(0, 2), id.substring(2, 4), `${id}.json`);
-                            break;
-                        default:
-                            throw new Error(`Unsupported depth: ${depthToUse}`);
+                    // v4.0.0: Load metadata from separate storage for type info
+                    const metadata = await this.getNounMetadata(id);
+                    if (metadata) {
+                        const type = metadata.noun || 'default';
+                        this.entityCounts.set(type, (this.entityCounts.get(type) || 0) + 1);
                     }
-                    const data = await fs.promises.readFile(filePath, 'utf-8');
-                    const noun = JSON.parse(data);
-                    const type = noun.metadata?.type || noun.metadata?.nounType || 'default';
-                    this.entityCounts.set(type, (this.entityCounts.get(type) || 0) + 1);
                 }
                 catch {
-                    // Skip invalid files
+                    // Skip invalid files or missing metadata
                 }
             }
             // Extrapolate counts if we sampled
@@ -2044,52 +2123,50 @@ export class FileSystemStorage extends BaseStorage {
                     const data = await fs.promises.readFile(filePath, 'utf-8');
                     const edge = JSON.parse(data);
                     const metadata = await this.getVerbMetadata(id);
+                    // v4.0.0: No fallbacks - skip verbs without metadata
                     if (!metadata) {
                         processedCount++;
                         return true; // continue, skip this verb
                     }
-                    // Reconstruct GraphVerb
-                    const verb = {
+                    // Convert connections if needed
+                    let connections = edge.connections;
+                    if (connections && typeof connections === 'object' && !(connections instanceof Map)) {
+                        const connectionsMap = new Map();
+                        for (const [level, nodeIds] of Object.entries(connections)) {
+                            connectionsMap.set(Number(level), new Set(nodeIds));
+                        }
+                        connections = connectionsMap;
+                    }
+                    // v4.0.0: Clean HNSWVerbWithMetadata construction
+                    const verbWithMetadata = {
                         id: edge.id,
                         vector: edge.vector,
-                        connections: edge.connections || new Map(),
-                        sourceId: metadata.sourceId || metadata.source,
-                        targetId: metadata.targetId || metadata.target,
-                        source: metadata.source || metadata.sourceId,
-                        target: metadata.target || metadata.targetId,
-                        verb: metadata.verb || metadata.type,
-                        type: metadata.type || metadata.verb,
-                        weight: metadata.weight,
-                        metadata: metadata.metadata || metadata,
-                        data: metadata.data,
-                        createdAt: metadata.createdAt,
-                        updatedAt: metadata.updatedAt,
-                        createdBy: metadata.createdBy,
-                        embedding: metadata.embedding || edge.vector
+                        connections: connections || new Map(),
+                        verb: edge.verb,
+                        sourceId: edge.sourceId,
+                        targetId: edge.targetId,
+                        metadata: metadata
                     };
                     // Apply filters
                     if (options.filter) {
                         const filter = options.filter;
                         if (filter.verbType) {
                             const types = Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType];
-                            const verbType = verb.type || verb.verb;
-                            if (verbType && !types.includes(verbType))
+                            if (!types.includes(verbWithMetadata.verb))
                                 return true; // continue
                         }
                         if (filter.sourceId) {
                             const sources = Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId];
-                            const sourceId = verb.sourceId || verb.source;
-                            if (!sourceId || !sources.includes(sourceId))
+                            if (!sources.includes(verbWithMetadata.sourceId))
                                 return true; // continue
                         }
                         if (filter.targetId) {
                             const targets = Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId];
-                            const targetId = verb.targetId || verb.target;
-                            if (!targetId || !targets.includes(targetId))
+                            if (!targets.includes(verbWithMetadata.targetId))
                                 return true; // continue
                         }
                     }
-                    verbs.push(verb);
+                    verbs.push(verbWithMetadata);
                     resultCount++;
                     processedCount++;
                     return true; // continue