@soulcraft/brainy 5.1.1 → 5.2.0

package/dist/vfs/VirtualFileSystem.js

@@ -7,6 +7,7 @@
 import crypto from 'crypto';
 import { Brainy } from '../brainy.js';
 import { NounType, VerbType } from '../types/graphTypes.js';
+import { mimeDetector } from './MimeTypeDetector.js';
 import { SemanticPathResolver, ProjectionRegistry, ConceptProjection, AuthorProjection, TemporalProjection, RelationshipProjection, SimilarityProjection, TagProjection } from './semantic/index.js';
 // Knowledge Layer can remain as optional augmentation for now
 import { VFSError, VFSErrorCode } from './types.js';
@@ -34,6 +35,17 @@ export class VirtualFileSystem {
         // Default configuration (will be overridden in init)
         this.config = this.getDefaultConfig();
     }
+    /**
+     * v5.2.0: Access to BlobStorage for unified file storage
+     */
+    get blobStorage() {
+        // TypeScript doesn't know about blobStorage on storage, use type assertion
+        const storage = this.brain['storage'];
+        if (!storage || !('blobStorage' in storage)) {
+            throw new Error('BlobStorage not available. Requires COW-enabled storage adapter.');
+        }
+        return storage.blobStorage;
+    }
     /**
      * Initialize the VFS
      */
@@ -176,47 +188,12 @@ export class VirtualFileSystem {
         if (entity.metadata.vfsType !== 'file') {
             throw new VFSError(VFSErrorCode.EISDIR, `Is a directory: ${path}`, path, 'readFile');
         }
-        // Get content based on storage type
-        let content;
-        let isCompressed = false;
-        if (!entity.metadata.storage || entity.metadata.storage.type === 'inline') {
-            // Content stored in metadata for new files, or try entity data for compatibility
-            if (entity.metadata.rawData) {
-                // rawData is ALWAYS stored uncompressed as base64
-                content = Buffer.from(entity.metadata.rawData, 'base64');
-                isCompressed = false; // rawData is never compressed
-            }
-            else if (!entity.data) {
-                content = Buffer.alloc(0);
-            }
-            else if (Buffer.isBuffer(entity.data)) {
-                content = entity.data;
-                isCompressed = entity.metadata.storage?.compressed || false;
-            }
-            else if (typeof entity.data === 'string') {
-                content = Buffer.from(entity.data);
-            }
-            else {
-                content = Buffer.from(JSON.stringify(entity.data));
-            }
-        }
-        else if (entity.metadata.storage.type === 'reference') {
-            // Content stored in external storage
-            content = await this.readExternalContent(entity.metadata.storage.key);
-            isCompressed = entity.metadata.storage.compressed || false;
-        }
-        else if (entity.metadata.storage.type === 'chunked') {
-            // Content stored in chunks
-            content = await this.readChunkedContent(entity.metadata.storage.chunks);
-            isCompressed = entity.metadata.storage.compressed || false;
-        }
-        else {
-            throw new VFSError(VFSErrorCode.EIO, `Unknown storage type: ${entity.metadata.storage.type}`, path, 'readFile');
-        }
-        // Decompress if needed (but NOT for rawData which is never compressed)
-        if (isCompressed && options?.decompress !== false) {
-            content = await this.decompress(content);
+        // v5.2.0: Unified blob storage - ONE path only
+        if (!entity.metadata.storage?.type || entity.metadata.storage.type !== 'blob') {
+            throw new VFSError(VFSErrorCode.EIO, `File has no blob storage: ${path}. Requires v5.2.0+ storage format.`, path, 'readFile');
         }
+        // Read from BlobStorage (handles decompression automatically)
+        const content = await this.blobStorage.read(entity.metadata.storage.hash);
         // Update access time
         await this.updateAccessTime(entityId);
         // Cache the content
@@ -259,36 +236,19 @@ export class VirtualFileSystem {
             // File doesn't exist, which is fine
             existingId = null;
         }
-        // Determine storage strategy based on size
-        let storageStrategy;
-        let entityData = null;
-        if (buffer.length <= (this.config.storage?.inline?.maxSize || 100000)) {
-            // Store inline for small files
-            storageStrategy = { type: 'inline' };
-            entityData = buffer;
-        }
-        else if (buffer.length <= 10000000) {
-            // Store as reference for medium files
-            const key = await this.storeExternalContent(buffer);
-            storageStrategy = { type: 'reference', key };
-        }
-        else {
-            // Store as chunks for large files
-            const chunks = await this.storeChunkedContent(buffer);
-            storageStrategy = { type: 'chunked', chunks };
-        }
-        // Compress if beneficial
-        if (this.shouldCompress(buffer) && options?.compress !== false) {
-            const compressed = await this.compress(buffer);
-            if (compressed.length < buffer.length * 0.9) { // Only if >10% savings
-                storageStrategy.compressed = true;
-                if (storageStrategy.type === 'inline') {
-                    entityData = compressed;
-                }
-            }
-        }
-        // Detect MIME type
-        const mimeType = this.detectMimeType(name, buffer);
+        // v5.2.0: Unified blob storage for ALL files (no size-based branching)
+        // Store in BlobStorage (content-addressable, auto-deduplication, streaming)
+        const blobHash = await this.blobStorage.write(buffer);
+        // Get blob metadata (size, compression info)
+        const blobMetadata = await this.blobStorage.getMetadata(blobHash);
+        const storageStrategy = {
+            type: 'blob',
+            hash: blobHash,
+            size: buffer.length,
+            compressed: blobMetadata?.compressed
+        };
+        // Detect MIME type (v5.2.0: using comprehensive MimeTypeDetector)
+        const mimeType = mimeDetector.detectMimeType(name, buffer);
         // Create metadata
         const metadata = {
             path,
@@ -304,9 +264,9 @@ export class VirtualFileSystem {
             group: 'users',
             accessed: Date.now(),
             modified: Date.now(),
-            storage: storageStrategy,
-            // Store raw buffer data for retrieval
-            rawData: buffer.toString('base64') // Store as base64 for safe serialization
+            storage: storageStrategy
+            // v5.2.0: No rawData - content is in BlobStorage
+            // Backward compatibility: readFile() checks for rawData for legacy files
         };
         // Extract additional metadata if enabled
         if (this.config.intelligence?.autoExtract && options?.extractMetadata !== false) {
@@ -314,9 +274,9 @@ export class VirtualFileSystem {
         }
         if (existingId) {
             // Update existing file
+            // v5.2.0: No entity.data - content is in BlobStorage
             await this.brain.update({
                 id: existingId,
-                data: entityData,
                 metadata
             });
             // Ensure Contains relationship exists (fix for missing relationships)
@@ -338,7 +298,7 @@ export class VirtualFileSystem {
         else {
             // Create new file entity
             // For embedding: use text content, for storage: use raw data
-            const embeddingData = this.isTextFile(mimeType) ? buffer.toString('utf-8') : `File: ${name} (${mimeType}, ${buffer.length} bytes)`;
+            const embeddingData = mimeDetector.isTextFile(mimeType) ? buffer.toString('utf-8') : `File: ${name} (${mimeType}, ${buffer.length} bytes)`;
             const entity = await this.brain.add({
                 data: embeddingData, // Always provide string for embeddings
                 type: this.getFileNounType(mimeType),
@@ -392,14 +352,9 @@ export class VirtualFileSystem {
         if (entity.metadata.vfsType !== 'file') {
             throw new VFSError(VFSErrorCode.EISDIR, `Is a directory: ${path}`, path, 'unlink');
         }
-        // Delete external content if needed
-        if (entity.metadata.storage) {
-            if (entity.metadata.storage.type === 'reference') {
-                await this.deleteExternalContent(entity.metadata.storage.key);
-            }
-            else if (entity.metadata.storage.type === 'chunked') {
-                await this.deleteChunkedContent(entity.metadata.storage.chunks);
-            }
+        // v5.2.0: Delete blob from BlobStorage (decrements ref count)
+        if (entity.metadata.storage?.type === 'blob') {
+            await this.blobStorage.delete(entity.metadata.storage.hash);
         }
         // Delete the entity
         await this.brain.delete(entityId);
@@ -938,34 +893,8 @@ export class VirtualFileSystem {
             return undefined;
         return filename.substring(lastDot + 1).toLowerCase();
     }
-    detectMimeType(filename, content) {
-        const ext = this.getExtension(filename);
-        // Common MIME types by extension
-        const mimeTypes = {
-            txt: 'text/plain',
-            html: 'text/html',
-            css: 'text/css',
-            js: 'application/javascript',
-            json: 'application/json',
-            pdf: 'application/pdf',
-            jpg: 'image/jpeg',
-            jpeg: 'image/jpeg',
-            png: 'image/png',
-            gif: 'image/gif',
-            mp3: 'audio/mpeg',
-            mp4: 'video/mp4',
-            zip: 'application/zip'
-        };
-        return mimeTypes[ext || ''] || 'application/octet-stream';
-    }
-    isTextFile(mimeType) {
-        return mimeType.startsWith('text/') ||
-            mimeType.includes('json') ||
-            mimeType.includes('javascript') ||
-            mimeType.includes('xml') ||
-            mimeType.includes('yaml') ||
-            mimeType === 'application/json';
-    }
+    // v5.2.0: MIME detection moved to MimeTypeDetector service
+    // Removed detectMimeType() and isTextFile() - now using mimeDetector singleton
     getFileNounType(mimeType) {
         if (mimeType.startsWith('text/') || mimeType.includes('json')) {
             return NounType.Document;
@@ -975,122 +904,13 @@ export class VirtualFileSystem {
         }
         return NounType.File;
     }
-    shouldCompress(buffer) {
-        if (!this.config.storage?.compression?.enabled)
-            return false;
-        if (buffer.length < (this.config.storage.compression.minSize || 10000))
-            return false;
-        // Don't compress already compressed formats
-        const firstBytes = buffer.slice(0, 4).toString('hex');
-        const compressedSignatures = [
-            '504b0304', // ZIP
-            '1f8b', // GZIP
-            '425a', // BZIP2
-            '89504e47', // PNG
-            'ffd8ff' // JPEG
-        ];
-        return !compressedSignatures.some(sig => firstBytes.startsWith(sig));
-    }
-    // External storage methods - leverages Brainy's storage adapters (memory, file, S3, R2)
-    async readExternalContent(key) {
-        // Read from Brainy - Brainy's storage adapter handles retrieval
-        const entity = await this.brain.get(key);
-        if (!entity) {
-            throw new Error(`External content not found: ${key}`);
-        }
-        // Content is stored in the data field
-        // Brainy handles storage/retrieval through its adapters (memory, file, S3, R2)
-        return Buffer.isBuffer(entity.data) ? entity.data : Buffer.from(entity.data);
-    }
-    async storeExternalContent(buffer) {
-        // Store as Brainy entity - let Brainy's storage adapter handle it
-        // Brainy automatically handles large data through its storage adapters (memory, file, S3, R2)
-        const entityId = await this.brain.add({
-            data: buffer, // Store actual buffer - Brainy will handle it efficiently
-            type: NounType.File,
-            metadata: {
-                vfsType: 'external-storage',
-                size: buffer.length,
-                created: Date.now()
-            }
-        });
-        return entityId;
-    }
-    async deleteExternalContent(key) {
-        // Delete the external storage entity
-        try {
-            await this.brain.delete(key);
-        }
-        catch (error) {
-            console.debug('Failed to delete external content:', key, error);
-        }
-    }
-    async readChunkedContent(chunks) {
-        // Read all chunk entities and combine
-        const buffers = [];
-        for (const chunkId of chunks) {
-            const entity = await this.brain.get(chunkId);
-            if (!entity) {
-                throw new Error(`Chunk not found: ${chunkId}`);
-            }
-            // Read actual data from entity - Brainy handles storage
-            const chunkBuffer = Buffer.isBuffer(entity.data) ? entity.data : Buffer.from(entity.data);
-            buffers.push(chunkBuffer);
-        }
-        return Buffer.concat(buffers);
-    }
-    async storeChunkedContent(buffer) {
-        const chunkSize = this.config.storage?.chunking?.chunkSize || 5000000; // 5MB chunks
-        const chunks = [];
-        for (let i = 0; i < buffer.length; i += chunkSize) {
-            const chunk = buffer.slice(i, Math.min(i + chunkSize, buffer.length));
-            // Store each chunk as a separate entity
-            // Let Brainy handle the chunk data efficiently
-            const chunkId = await this.brain.add({
-                data: chunk, // Store actual chunk - Brainy handles it
-                type: NounType.File,
-                metadata: {
-                    vfsType: 'chunk',
-                    chunkIndex: chunks.length,
-                    size: chunk.length,
-                    created: Date.now()
-                }
-            });
-            chunks.push(chunkId);
-        }
-        return chunks;
-    }
-    async deleteChunkedContent(chunks) {
-        // Delete all chunk entities
-        await Promise.all(chunks.map(chunkId => this.brain.delete(chunkId).catch(err => console.debug('Failed to delete chunk:', chunkId, err))));
-    }
-    async compress(buffer) {
-        const zlib = await import('zlib');
-        return new Promise((resolve, reject) => {
-            zlib.gzip(buffer, (err, compressed) => {
-                if (err)
-                    reject(err);
-                else
-                    resolve(compressed);
-            });
-        });
-    }
-    async decompress(buffer) {
-        const zlib = await import('zlib');
-        return new Promise((resolve, reject) => {
-            zlib.gunzip(buffer, (err, decompressed) => {
-                if (err)
-                    reject(err);
-                else
-                    resolve(decompressed);
-            });
-        });
-    }
+    // v5.2.0: Removed compression methods (shouldCompress, compress, decompress)
+    // BlobStorage handles all compression automatically with zstd
     async generateEmbedding(buffer, mimeType) {
         try {
             // Use text content for text files, description for binary
             let content;
-            if (this.isTextFile(mimeType)) {
+            if (mimeDetector.isTextFile(mimeType)) {
                 // Use first 10KB for embedding
                 content = buffer.toString('utf8', 0, Math.min(10240, buffer.length));
             }
@@ -1119,7 +939,7 @@ export class VirtualFileSystem {
     async extractMetadata(buffer, mimeType) {
         const metadata = {};
         // Extract basic metadata based on content type
-        if (this.isTextFile(mimeType)) {
+        if (mimeDetector.isTextFile(mimeType)) {
             const text = buffer.toString('utf8');
             metadata.lineCount = text.split('\n').length;
             metadata.wordCount = text.split(/\s+/).filter(w => w).length;

package/dist/vfs/importers/DirectoryImporter.d.ts

@@ -86,8 +86,4 @@ export declare class DirectoryImporter {
      * Check if a path should be skipped
      */
     private shouldSkip;
-    /**
-     * Detect MIME type from file content and extension
-     */
-    private detectMimeType;
 }

package/dist/vfs/importers/DirectoryImporter.js

@@ -279,45 +279,5 @@ export class DirectoryImporter {
         }
         return false;
     }
-    /**
-     * Detect MIME type from file content and extension
-     */
-    detectMimeType(filePath, content) {
-        const ext = path.extname(filePath).toLowerCase();
-        // Common extensions
-        const mimeTypes = {
-            '.js': 'application/javascript',
-            '.ts': 'application/typescript',
-            '.jsx': 'application/javascript',
-            '.tsx': 'application/typescript',
-            '.json': 'application/json',
-            '.md': 'text/markdown',
-            '.html': 'text/html',
-            '.css': 'text/css',
-            '.py': 'text/x-python',
-            '.go': 'text/x-go',
-            '.rs': 'text/x-rust',
-            '.java': 'text/x-java',
-            '.cpp': 'text/x-c++',
-            '.c': 'text/x-c',
-            '.h': 'text/x-c',
-            '.txt': 'text/plain',
-            '.xml': 'application/xml',
-            '.yaml': 'text/yaml',
-            '.yml': 'text/yaml',
-            '.toml': 'text/toml',
-            '.sh': 'text/x-shellscript',
-            '.pdf': 'application/pdf',
-            '.jpg': 'image/jpeg',
-            '.jpeg': 'image/jpeg',
-            '.png': 'image/png',
-            '.gif': 'image/gif',
-            '.svg': 'image/svg+xml',
-            '.mp3': 'audio/mpeg',
-            '.mp4': 'video/mp4',
-            '.zip': 'application/zip'
-        };
-        return mimeTypes[ext] || 'application/octet-stream';
-    }
 }
 //# sourceMappingURL=DirectoryImporter.js.map

package/dist/vfs/index.d.ts

@@ -7,6 +7,7 @@
 export { VirtualFileSystem } from './VirtualFileSystem.js';
 export { PathResolver } from './PathResolver.js';
 export * from './types.js';
+export { MimeTypeDetector, mimeDetector } from './MimeTypeDetector.js';
 export { FSCompat, createFS } from './FSCompat.js';
 export { DirectoryImporter } from './importers/DirectoryImporter.js';
 export { VFSReadStream } from './streams/VFSReadStream.js';

package/dist/vfs/index.js

@@ -8,6 +8,8 @@
 export { VirtualFileSystem } from './VirtualFileSystem.js';
 export { PathResolver } from './PathResolver.js';
 export * from './types.js';
+// MIME Type Detection (v5.2.0)
+export { MimeTypeDetector, mimeDetector } from './MimeTypeDetector.js';
 // fs compatibility layer
 export { FSCompat, createFS } from './FSCompat.js';
 // Directory import

package/dist/vfs/types.d.ts

@@ -37,9 +37,9 @@ export interface VFSMetadata {
     accessed: number;
     modified: number;
     storage?: {
-        type: 'inline' | 'reference' | 'chunked';
-        key?: string;
-        chunks?: string[];
+        type: 'blob';
+        hash: string;
+        size: number;
         compressed?: boolean;
     };
     attributes?: Record<string, any>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "5.1.1",
+  "version": "5.2.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. 31 nouns × 40 verbs for infinite expressiveness.",
   "main": "dist/index.js",
   "module": "dist/index.js",
@@ -147,7 +147,9 @@
     "@rollup/plugin-replace": "^6.0.2",
     "@rollup/plugin-terser": "^0.4.4",
     "@testcontainers/redis": "^11.5.1",
+    "@types/mime": "^3.0.4",
     "@types/node": "^20.11.30",
+    "@types/sharp": "^0.31.1",
     "@types/uuid": "^10.0.0",
     "@types/ws": "^8.18.1",
     "@typescript-eslint/eslint-plugin": "^8.0.0",
@@ -175,13 +177,16 @@
     "cli-table3": "^0.6.5",
     "commander": "^11.1.0",
     "csv-parse": "^6.1.0",
+    "exifr": "^7.1.3",
     "inquirer": "^12.9.3",
     "js-yaml": "^4.1.0",
     "mammoth": "^1.11.0",
+    "mime": "^4.1.0",
     "ora": "^8.2.0",
     "pdfjs-dist": "^4.0.379",
     "prompts": "^2.4.2",
     "roaring-wasm": "^1.1.0",
+    "sharp": "^0.33.5",
     "uuid": "^9.0.1",
     "ws": "^8.18.3",
     "xlsx": "^0.18.5"