@soulcraft/brainy 3.9.1 → 3.10.0

package/dist/vfs/importers/DirectoryImporter.d.ts

@@ -0,0 +1,86 @@
+/**
+ * Directory Importer for VFS
+ *
+ * Efficiently imports real directories into VFS with:
+ * - Batch processing for performance
+ * - Progress tracking
+ * - Error recovery
+ * - Parallel processing
+ */
+import { VirtualFileSystem } from '../VirtualFileSystem.js';
+import { Brainy } from '../../brainy.js';
+export interface ImportOptions {
+    targetPath?: string;
+    recursive?: boolean;
+    skipHidden?: boolean;
+    skipNodeModules?: boolean;
+    batchSize?: number;
+    generateEmbeddings?: boolean;
+    extractMetadata?: boolean;
+    showProgress?: boolean;
+    filter?: (path: string) => boolean;
+}
+export interface ImportResult {
+    imported: string[];
+    failed: Array<{
+        path: string;
+        error: Error;
+    }>;
+    skipped: string[];
+    totalSize: number;
+    duration: number;
+    filesProcessed: number;
+    directoriesCreated: number;
+}
+export interface ImportProgress {
+    type: 'progress' | 'complete' | 'error';
+    processed: number;
+    total?: number;
+    current?: string;
+    error?: Error;
+}
+export declare class DirectoryImporter {
+    private vfs;
+    private brain;
+    constructor(vfs: VirtualFileSystem, brain: Brainy);
+    /**
+     * Import a directory or file into VFS
+     */
+    import(sourcePath: string, options?: ImportOptions): Promise<ImportResult>;
+    /**
+     * Import with progress tracking (generator)
+     */
+    importStream(sourcePath: string, options?: ImportOptions): AsyncGenerator<ImportProgress>;
+    /**
+     * Import a directory recursively
+     */
+    private importDirectory;
+    /**
+     * Create directory structure in VFS
+     */
+    private createDirectoryStructure;
+    /**
+     * Collect all files to be imported
+     */
+    private collectFiles;
+    /**
+     * Process a batch of files
+     */
+    private processBatch;
+    /**
+     * Import a single file
+     */
+    private importSingleFile;
+    /**
+     * Import a single file (for non-directory imports)
+     */
+    private importFile;
+    /**
+     * Check if a path should be skipped
+     */
+    private shouldSkip;
+    /**
+     * Detect MIME type from file content and extension
+     */
+    private detectMimeType;
+}

package/dist/vfs/importers/DirectoryImporter.js

@@ -0,0 +1,298 @@
+/**
+ * Directory Importer for VFS
+ *
+ * Efficiently imports real directories into VFS with:
+ * - Batch processing for performance
+ * - Progress tracking
+ * - Error recovery
+ * - Parallel processing
+ */
+import { promises as fs } from 'fs';
+import * as path from 'path';
+export class DirectoryImporter {
+    constructor(vfs, brain) {
+        this.vfs = vfs;
+        this.brain = brain;
+    }
+    /**
+     * Import a directory or file into VFS
+     */
+    async import(sourcePath, options = {}) {
+        const startTime = Date.now();
+        const result = {
+            imported: [],
+            failed: [],
+            skipped: [],
+            totalSize: 0,
+            duration: 0,
+            filesProcessed: 0,
+            directoriesCreated: 0
+        };
+        try {
+            const stats = await fs.stat(sourcePath);
+            if (stats.isFile()) {
+                await this.importFile(sourcePath, options.targetPath || '/', result);
+            }
+            else if (stats.isDirectory()) {
+                await this.importDirectory(sourcePath, options, result);
+            }
+        }
+        catch (error) {
+            result.failed.push({
+                path: sourcePath,
+                error: error
+            });
+        }
+        result.duration = Date.now() - startTime;
+        return result;
+    }
+    /**
+     * Import with progress tracking (generator)
+     */
+    async *importStream(sourcePath, options = {}) {
+        const files = await this.collectFiles(sourcePath, options);
+        const total = files.length;
+        const batchSize = options.batchSize || 100;
+        let processed = 0;
+        // Process in batches
+        for (let i = 0; i < files.length; i += batchSize) {
+            const batch = files.slice(i, i + batchSize);
+            try {
+                await this.processBatch(batch, options);
+                processed += batch.length;
+                yield {
+                    type: 'progress',
+                    processed,
+                    total,
+                    current: batch[batch.length - 1]
+                };
+            }
+            catch (error) {
+                yield {
+                    type: 'error',
+                    processed,
+                    total,
+                    error: error
+                };
+            }
+        }
+        yield {
+            type: 'complete',
+            processed,
+            total
+        };
+    }
+    /**
+     * Import a directory recursively
+     */
+    async importDirectory(dirPath, options, result) {
+        const targetPath = options.targetPath || '/';
+        // Create VFS directory structure
+        await this.createDirectoryStructure(dirPath, targetPath, options, result);
+        // Collect all files
+        const files = await this.collectFiles(dirPath, options);
+        // Process files in batches
+        const batchSize = options.batchSize || 100;
+        for (let i = 0; i < files.length; i += batchSize) {
+            const batch = files.slice(i, i + batchSize);
+            await this.processBatch(batch, options, result);
+            if (options.showProgress && i % (batchSize * 10) === 0) {
+                console.log(`Imported ${i} / ${files.length} files...`);
+            }
+        }
+    }
+    /**
+     * Create directory structure in VFS
+     */
+    async createDirectoryStructure(sourcePath, targetPath, options, result) {
+        // Walk directory tree and create all directories first
+        const dirsToCreate = [];
+        const collectDirs = async (dir, vfsPath) => {
+            dirsToCreate.push(vfsPath);
+            const entries = await fs.readdir(dir, { withFileTypes: true });
+            for (const entry of entries) {
+                if (entry.isDirectory()) {
+                    if (this.shouldSkip(entry.name, path.join(dir, entry.name), options)) {
+                        continue;
+                    }
+                    const childPath = path.join(dir, entry.name);
+                    const childVfsPath = path.posix.join(vfsPath, entry.name);
+                    if (options.recursive !== false) {
+                        await collectDirs(childPath, childVfsPath);
+                    }
+                }
+            }
+        };
+        await collectDirs(sourcePath, targetPath);
+        // Create all directories
+        for (const dirPath of dirsToCreate) {
+            try {
+                await this.vfs.mkdir(dirPath, { recursive: true });
+                result.directoriesCreated++;
+            }
+            catch (error) {
+                if (error.code !== 'EEXIST') {
+                    result.failed.push({ path: dirPath, error });
+                }
+            }
+        }
+    }
+    /**
+     * Collect all files to be imported
+     */
+    async collectFiles(dirPath, options) {
+        const files = [];
+        const walk = async (dir) => {
+            const entries = await fs.readdir(dir, { withFileTypes: true });
+            for (const entry of entries) {
+                const fullPath = path.join(dir, entry.name);
+                if (this.shouldSkip(entry.name, fullPath, options)) {
+                    continue;
+                }
+                if (entry.isFile()) {
+                    files.push(fullPath);
+                }
+                else if (entry.isDirectory() && options.recursive !== false) {
+                    await walk(fullPath);
+                }
+            }
+        };
+        await walk(dirPath);
+        return files;
+    }
+    /**
+     * Process a batch of files
+     */
+    async processBatch(files, options, result) {
+        const imports = await Promise.allSettled(files.map(filePath => this.importSingleFile(filePath, options)));
+        if (result) {
+            for (let i = 0; i < imports.length; i++) {
+                const importResult = imports[i];
+                const filePath = files[i];
+                if (importResult.status === 'fulfilled') {
+                    result.imported.push(importResult.value.vfsPath);
+                    result.totalSize += importResult.value.size;
+                    result.filesProcessed++;
+                }
+                else {
+                    result.failed.push({
+                        path: filePath,
+                        error: importResult.reason
+                    });
+                }
+            }
+        }
+    }
+    /**
+     * Import a single file
+     */
+    async importSingleFile(filePath, options) {
+        const stats = await fs.stat(filePath);
+        const content = await fs.readFile(filePath);
+        // Calculate VFS path
+        const relativePath = path.relative(process.cwd(), filePath);
+        const vfsPath = path.posix.join(options.targetPath || '/', relativePath);
+        // Generate embedding if requested
+        let embedding;
+        if (options.generateEmbeddings !== false) {
+            try {
+                // Use first 10KB for embedding
+                const text = content.toString('utf8', 0, Math.min(10240, content.length));
+                // Generate embedding using brain's embed method
+                const embedResult = await this.brain.embed({ data: text });
+                embedding = embedResult;
+            }
+            catch {
+                // Continue without embedding if generation fails
+            }
+        }
+        // Write to VFS
+        await this.vfs.writeFile(vfsPath, content, {
+            generateEmbedding: options.generateEmbeddings,
+            extractMetadata: options.extractMetadata,
+            metadata: {
+                originalPath: filePath,
+                importedAt: Date.now(),
+                originalSize: stats.size,
+                originalModified: stats.mtime.getTime()
+            }
+        });
+        return { vfsPath, size: stats.size };
+    }
+    /**
+     * Import a single file (for non-directory imports)
+     */
+    async importFile(filePath, targetPath, result) {
+        try {
+            const imported = await this.importSingleFile(filePath, { targetPath });
+            result.imported.push(imported.vfsPath);
+            result.totalSize += imported.size;
+            result.filesProcessed++;
+        }
+        catch (error) {
+            result.failed.push({
+                path: filePath,
+                error: error
+            });
+        }
+    }
+    /**
+     * Check if a path should be skipped
+     */
+    shouldSkip(name, fullPath, options) {
+        // Skip hidden files if requested
+        if (options.skipHidden && name.startsWith('.')) {
+            return true;
+        }
+        // Skip node_modules by default
+        if (name === 'node_modules' && options.skipNodeModules !== false) {
+            return true;
+        }
+        // Apply custom filter
+        if (options.filter && !options.filter(fullPath)) {
+            return true;
+        }
+        return false;
+    }
+    /**
+     * Detect MIME type from file content and extension
+     */
+    detectMimeType(filePath, content) {
+        const ext = path.extname(filePath).toLowerCase();
+        // Common extensions
+        const mimeTypes = {
+            '.js': 'application/javascript',
+            '.ts': 'application/typescript',
+            '.jsx': 'application/javascript',
+            '.tsx': 'application/typescript',
+            '.json': 'application/json',
+            '.md': 'text/markdown',
+            '.html': 'text/html',
+            '.css': 'text/css',
+            '.py': 'text/x-python',
+            '.go': 'text/x-go',
+            '.rs': 'text/x-rust',
+            '.java': 'text/x-java',
+            '.cpp': 'text/x-c++',
+            '.c': 'text/x-c',
+            '.h': 'text/x-c',
+            '.txt': 'text/plain',
+            '.xml': 'application/xml',
+            '.yaml': 'text/yaml',
+            '.yml': 'text/yaml',
+            '.toml': 'text/toml',
+            '.sh': 'text/x-shellscript',
+            '.pdf': 'application/pdf',
+            '.jpg': 'image/jpeg',
+            '.jpeg': 'image/jpeg',
+            '.png': 'image/png',
+            '.gif': 'image/gif',
+            '.svg': 'image/svg+xml',
+            '.mp3': 'audio/mpeg',
+            '.mp4': 'video/mp4',
+            '.zip': 'application/zip'
+        };
+        return mimeTypes[ext] || 'application/octet-stream';
+    }
+}
+//# sourceMappingURL=DirectoryImporter.js.map

package/dist/vfs/index.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Brainy Virtual Filesystem
+ *
+ * A simplified fs-compatible filesystem that stores data in Brainy
+ * Works across all storage adapters and scales to millions of files
+ */
+export { VirtualFileSystem } from './VirtualFileSystem.js';
+export { PathResolver } from './PathResolver.js';
+export * from './types.js';
+export { FSCompat, createFS } from './FSCompat.js';
+export { DirectoryImporter } from './importers/DirectoryImporter.js';
+export { VFSReadStream } from './streams/VFSReadStream.js';
+export { VFSWriteStream } from './streams/VFSWriteStream.js';
+export { EventRecorder } from './EventRecorder.js';
+export { SemanticVersioning } from './SemanticVersioning.js';
+export { PersistentEntitySystem } from './PersistentEntitySystem.js';
+export { ConceptSystem } from './ConceptSystem.js';
+export { GitBridge } from './GitBridge.js';
+export { VirtualFileSystem as VFS } from './VirtualFileSystem.js';

package/dist/vfs/index.js

@@ -0,0 +1,26 @@
+/**
+ * Brainy Virtual Filesystem
+ *
+ * A simplified fs-compatible filesystem that stores data in Brainy
+ * Works across all storage adapters and scales to millions of files
+ */
+// Core VFS
+export { VirtualFileSystem } from './VirtualFileSystem.js';
+export { PathResolver } from './PathResolver.js';
+export * from './types.js';
+// fs compatibility layer
+export { FSCompat, createFS } from './FSCompat.js';
+// Directory import
+export { DirectoryImporter } from './importers/DirectoryImporter.js';
+// Streaming
+export { VFSReadStream } from './streams/VFSReadStream.js';
+export { VFSWriteStream } from './streams/VFSWriteStream.js';
+// Knowledge Layer Components (optional via augmentation)
+export { EventRecorder } from './EventRecorder.js';
+export { SemanticVersioning } from './SemanticVersioning.js';
+export { PersistentEntitySystem } from './PersistentEntitySystem.js';
+export { ConceptSystem } from './ConceptSystem.js';
+export { GitBridge } from './GitBridge.js';
+// Convenience alias
+export { VirtualFileSystem as VFS } from './VirtualFileSystem.js';
+//# sourceMappingURL=index.js.map

package/dist/vfs/streams/VFSReadStream.d.ts

@@ -0,0 +1,19 @@
+/**
+ * VFS Read Stream Implementation
+ *
+ * Real streaming support for large files
+ */
+import { Readable } from 'stream';
+import { VirtualFileSystem } from '../VirtualFileSystem.js';
+import { ReadStreamOptions } from '../types.js';
+export declare class VFSReadStream extends Readable {
+    private vfs;
+    private path;
+    private options;
+    private position;
+    private entity;
+    private data;
+    constructor(vfs: VirtualFileSystem, path: string, options?: ReadStreamOptions);
+    _read(size: number): Promise<void>;
+    _destroy(error: Error | null, callback: (error?: Error | null) => void): void;
+}

package/dist/vfs/streams/VFSReadStream.js

@@ -0,0 +1,54 @@
+/**
+ * VFS Read Stream Implementation
+ *
+ * Real streaming support for large files
+ */
+import { Readable } from 'stream';
+export class VFSReadStream extends Readable {
+    constructor(vfs, path, options = {}) {
+        super({
+            highWaterMark: options.highWaterMark || 64 * 1024 // 64KB chunks
+        });
+        this.vfs = vfs;
+        this.path = path;
+        this.options = options;
+        this.entity = null;
+        this.data = null;
+        this.position = options.start || 0;
+    }
+    async _read(size) {
+        try {
+            // Lazy load entity
+            if (!this.entity) {
+                this.entity = await this.vfs.getEntity(this.path);
+                this.data = this.entity.data;
+                if (!Buffer.isBuffer(this.data)) {
+                    // Convert string to buffer if needed
+                    this.data = Buffer.from(this.data);
+                }
+            }
+            // Check if we've reached the end
+            const end = this.options.end || this.data.length;
+            if (this.position >= end) {
+                this.push(null); // Signal EOF
+                return;
+            }
+            // Calculate chunk size
+            const chunkEnd = Math.min(this.position + size, end);
+            const chunk = this.data.slice(this.position, chunkEnd);
+            // Update position and push chunk
+            this.position = chunkEnd;
+            this.push(chunk);
+        }
+        catch (error) {
+            this.destroy(error);
+        }
+    }
+    _destroy(error, callback) {
+        // Clean up resources
+        this.entity = null;
+        this.data = null;
+        callback(error);
+    }
+}
+//# sourceMappingURL=VFSReadStream.js.map

package/dist/vfs/streams/VFSWriteStream.d.ts

@@ -0,0 +1,21 @@
+/**
+ * VFS Write Stream Implementation
+ *
+ * Real streaming write support for large files
+ */
+import { Writable } from 'stream';
+import { VirtualFileSystem } from '../VirtualFileSystem.js';
+import { WriteStreamOptions } from '../types.js';
+export declare class VFSWriteStream extends Writable {
+    private vfs;
+    private path;
+    private options;
+    private chunks;
+    private size;
+    private _closed;
+    constructor(vfs: VirtualFileSystem, path: string, options?: WriteStreamOptions);
+    _write(chunk: any, encoding: BufferEncoding, callback: (error?: Error | null) => void): Promise<void>;
+    _final(callback: (error?: Error | null) => void): Promise<void>;
+    private _flush;
+    _destroy(error: Error | null, callback: (error?: Error | null) => void): void;
+}

package/dist/vfs/streams/VFSWriteStream.js

@@ -0,0 +1,70 @@
+/**
+ * VFS Write Stream Implementation
+ *
+ * Real streaming write support for large files
+ */
+import { Writable } from 'stream';
+export class VFSWriteStream extends Writable {
+    constructor(vfs, path, options = {}) {
+        super({
+            highWaterMark: 64 * 1024 // 64KB chunks
+        });
+        this.vfs = vfs;
+        this.path = path;
+        this.options = options;
+        this.chunks = [];
+        this.size = 0;
+        this._closed = false;
+        // Handle autoClose option
+        if (options.autoClose !== false) {
+            this.once('finish', () => this._flush());
+        }
+    }
+    async _write(chunk, encoding, callback) {
+        try {
+            // Convert to buffer if needed
+            const buffer = Buffer.isBuffer(chunk)
+                ? chunk
+                : Buffer.from(chunk, encoding);
+            // Store chunk
+            this.chunks.push(buffer);
+            this.size += buffer.length;
+            // For very large files, we could flush periodically
+            // to avoid memory issues, but for now we accumulate
+            callback();
+        }
+        catch (error) {
+            callback(error);
+        }
+    }
+    async _final(callback) {
+        try {
+            await this._flush();
+            callback();
+        }
+        catch (error) {
+            callback(error);
+        }
+    }
+    async _flush() {
+        if (this._closed)
+            return;
+        this._closed = true;
+        // Combine all chunks
+        const data = Buffer.concat(this.chunks, this.size);
+        // Write to VFS
+        await this.vfs.writeFile(this.path, data, {
+            mode: this.options.mode,
+            encoding: this.options.encoding
+        });
+        // Clear chunks to free memory
+        this.chunks = [];
+    }
+    _destroy(error, callback) {
+        // Clean up resources
+        this.chunks = [];
+        this._closed = true;
+        callback(error);
+    }
+}
+//# sourceMappingURL=VFSWriteStream.js.map