@soulcraft/brainy 3.0.0 → 3.0.1

package/dist/augmentations/walAugmentation.d.ts

@@ -1,109 +0,0 @@
-/**
- * Write-Ahead Log (WAL) Augmentation
- *
- * Provides file-based durability and atomicity for storage operations
- * Automatically enabled for all critical storage operations
- *
- * Features:
- * - True file-based persistence for crash recovery
- * - Operation replay after startup
- * - Automatic log rotation and cleanup
- * - Cross-platform compatibility (filesystem, OPFS, cloud)
- */
-import { BaseAugmentation } from './brainyAugmentation.js';
-interface WALConfig {
-    enabled?: boolean;
-    immediateWrites?: boolean;
-    adaptivePersistence?: boolean;
-    walPrefix?: string;
-    maxSize?: number;
-    checkpointInterval?: number;
-    autoRecover?: boolean;
-    maxRetries?: number;
-}
-export declare class WALAugmentation extends BaseAugmentation {
-    name: string;
-    timing: "around";
-    metadata: "readonly";
-    operations: ("addNoun" | "addVerb" | "saveNoun" | "saveVerb" | "updateMetadata" | "delete" | "deleteVerb" | "clear")[];
-    priority: number;
-    private config;
-    private currentLogId;
-    private operationCounter;
-    private checkpointTimer?;
-    private isRecovering;
-    constructor(config?: WALConfig);
-    protected onInitialize(): Promise<void>;
-    shouldExecute(operation: string, params: any): boolean;
-    execute<T = any>(operation: string, params: any, next: () => Promise<T>): Promise<T>;
-    /**
-     * Asynchronous WAL entry logging (fire-and-forget for immediate writes)
-     */
-    private logAsyncWALEntry;
-    /**
-     * Write WAL entry to persistent storage using storage adapter
-     */
-    private writeWALEntry;
-    /**
-     * Recover pending operations from all existing WAL files
-     */
-    private recoverPendingOperations;
-    /**
-     * Find all WAL files in storage
-     */
-    private findWALFiles;
-    /**
-     * Read WAL entries from a file
-     */
-    private readWALEntries;
-    /**
-     * Find operations that were started but not completed
-     */
-    private findPendingOperations;
-    /**
-     * Replay an operation during recovery
-     */
-    private replayOperation;
-    /**
-     * Create a checkpoint to mark a point in time
-     */
-    private createCheckpoint;
-    /**
-     * Check if log rotation is needed
-     */
-    private checkLogRotation;
-    /**
-     * Sanitize parameters for logging (remove large objects)
-     */
-    private sanitizeParams;
-    /**
-     * Get WAL statistics
-     */
-    getStats(): {
-        enabled: boolean;
-        currentLogId: string;
-        operationCount: number;
-        logSize: number;
-        pendingOperations: number;
-        failedOperations: number;
-    };
-    /**
-     * Manually trigger checkpoint
-     */
-    checkpoint(): Promise<void>;
-    /**
-     * Manually trigger log rotation
-     */
-    rotate(): Promise<void>;
-    /**
-     * Write WAL data directly to storage without embedding
-     * This bypasses the brain's AI processing pipeline for raw WAL data
-     */
-    private writeWALFileDirectly;
-    /**
-     * Read WAL data directly from storage without embedding
-     */
-    private readWALFileDirectly;
-    protected onShutdown(): Promise<void>;
-}
-export {};

package/dist/augmentations/walAugmentation.js

@@ -1,516 +0,0 @@
-/**
- * Write-Ahead Log (WAL) Augmentation
- *
- * Provides file-based durability and atomicity for storage operations
- * Automatically enabled for all critical storage operations
- *
- * Features:
- * - True file-based persistence for crash recovery
- * - Operation replay after startup
- * - Automatic log rotation and cleanup
- * - Cross-platform compatibility (filesystem, OPFS, cloud)
- */
-import { BaseAugmentation } from './brainyAugmentation.js';
-import { v4 as uuidv4 } from '../universal/uuid.js';
-export class WALAugmentation extends BaseAugmentation {
-    constructor(config = {}) {
-        super();
-        this.name = 'WAL';
-        this.timing = 'around';
-        this.metadata = 'readonly'; // Reads metadata for logging/recovery
-        this.operations = ['addNoun', 'addVerb', 'saveNoun', 'saveVerb', 'updateMetadata', 'delete', 'deleteVerb', 'clear'];
-        this.priority = 100; // Critical system operation - highest priority
-        this.operationCounter = 0;
-        this.isRecovering = false;
-        this.config = {
-            enabled: config.enabled ?? true,
-            immediateWrites: config.immediateWrites ?? true, // Zero-config: immediate by default
-            adaptivePersistence: config.adaptivePersistence ?? true, // Zero-config: adaptive by default
-            walPrefix: config.walPrefix ?? 'wal',
-            maxSize: config.maxSize ?? 10 * 1024 * 1024, // 10MB
-            checkpointInterval: config.checkpointInterval ?? 60 * 1000, // 1 minute
-            autoRecover: config.autoRecover ?? true,
-            maxRetries: config.maxRetries ?? 3
-        };
-        // Create unique log ID for this session
-        this.currentLogId = `${this.config.walPrefix}_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
-    }
-    async onInitialize() {
-        if (!this.config.enabled) {
-            this.log('Write-Ahead Log disabled');
-            return;
-        }
-        this.log('Write-Ahead Log initializing with file-based persistence');
-        // Recover any pending operations from previous sessions
-        if (this.config.autoRecover) {
-            await this.recoverPendingOperations();
-        }
-        // Start checkpoint timer
-        if (this.config.checkpointInterval > 0) {
-            this.checkpointTimer = setInterval(() => this.createCheckpoint(), this.config.checkpointInterval);
-        }
-        this.log('Write-Ahead Log initialized with file-based durability');
-    }
-    shouldExecute(operation, params) {
-        // Only execute if enabled and for write operations that modify data
-        return this.config.enabled && !this.isRecovering && (operation === 'saveNoun' ||
-            operation === 'saveVerb' ||
-            operation === 'addNoun' ||
-            operation === 'addVerb' ||
-            operation === 'updateMetadata' ||
-            operation === 'delete');
-    }
-    async execute(operation, params, next) {
-        if (!this.shouldExecute(operation, params)) {
-            return next();
-        }
-        const entry = {
-            id: uuidv4(),
-            operation,
-            params: this.sanitizeParams(params),
-            timestamp: Date.now(),
-            status: 'pending'
-        };
-        // ZERO-CONFIG INTELLIGENT ADAPTATION:
-        // If immediate writes are enabled, execute first then log asynchronously
-        if (this.config.immediateWrites) {
-            try {
-                // Step 1: Execute operation immediately for user responsiveness
-                const result = await next();
-                // Step 2: Log completion asynchronously (non-blocking)
-                entry.status = 'completed';
-                this.logAsyncWALEntry(entry); // Fire-and-forget logging
-                this.operationCounter++;
-                // Step 3: Background log maintenance (non-blocking)
-                setImmediate(() => this.checkLogRotation());
-                return result;
-            }
-            catch (error) {
-                // Log failure asynchronously (non-blocking)
-                entry.status = 'failed';
-                entry.error = error.message;
-                this.logAsyncWALEntry(entry); // Fire-and-forget logging
-                this.log(`Operation ${operation} failed: ${entry.error}`, 'error');
-                throw error;
-            }
-        }
-        else {
-            // Traditional WAL: durability first (for high-reliability scenarios)
-            // Step 1: Write operation to WAL (durability first!)
-            await this.writeWALEntry(entry);
-            try {
-                // Step 2: Execute the actual operation
-                const result = await next();
-                // Step 3: Mark as completed in WAL
-                entry.status = 'completed';
-                await this.writeWALEntry(entry);
-                this.operationCounter++;
-                // Check if we need to rotate log
-                await this.checkLogRotation();
-                return result;
-            }
-            catch (error) {
-                // Mark as failed in WAL
-                entry.status = 'failed';
-                entry.error = error.message;
-                await this.writeWALEntry(entry);
-                this.log(`Operation ${operation} failed: ${entry.error}`, 'error');
-                throw error;
-            }
-        }
-    }
-    /**
-     * Asynchronous WAL entry logging (fire-and-forget for immediate writes)
-     */
-    logAsyncWALEntry(entry) {
-        // Use setImmediate to defer logging without blocking the main operation
-        setImmediate(async () => {
-            try {
-                await this.writeWALEntry(entry);
-            }
-            catch (error) {
-                // Log WAL write failures but don't throw (fire-and-forget)
-                this.log(`Background WAL write failed: ${error.message}`, 'warn');
-            }
-        });
-    }
-    /**
-     * Write WAL entry to persistent storage using storage adapter
-     */
-    async writeWALEntry(entry) {
-        try {
-            if (!this.context?.brain?.storage) {
-                throw new Error('Storage adapter not available');
-            }
-            const line = JSON.stringify(entry) + '\n';
-            // Read existing log content directly from WAL file
-            let existingContent = '';
-            try {
-                existingContent = await this.readWALFileDirectly(this.currentLogId);
-            }
-            catch {
-                // No existing log, start fresh
-            }
-            const newContent = existingContent + line;
-            // Write WAL directly to storage without going through embedding pipeline
-            // WAL files should be raw text, not embedded vectors
-            await this.writeWALFileDirectly(this.currentLogId, newContent);
-        }
-        catch (error) {
-            // WAL write failure is critical - but don't block operations
-            this.log(`WAL write failed: ${error}`, 'error');
-            console.error('WAL write failure:', error);
-        }
-    }
-    /**
-     * Recover pending operations from all existing WAL files
-     */
-    async recoverPendingOperations() {
-        if (!this.context?.brain?.storage)
-            return;
-        this.isRecovering = true;
-        try {
-            // Find all WAL files by searching for nouns with walType metadata
-            const walFiles = await this.findWALFiles();
-            if (walFiles.length === 0) {
-                this.log('No WAL files found for recovery');
-                return;
-            }
-            this.log(`Found ${walFiles.length} WAL files for recovery`);
-            let totalRecovered = 0;
-            for (const walFile of walFiles) {
-                const entries = await this.readWALEntries(walFile.id);
-                const pending = this.findPendingOperations(entries);
-                if (pending.length > 0) {
-                    this.log(`Recovering ${pending.length} pending operations from ${walFile.id}`);
-                    for (const entry of pending) {
-                        try {
-                            // Attempt to replay the operation
-                            await this.replayOperation(entry);
-                            // Mark as recovered
-                            entry.status = 'completed';
-                            await this.writeWALEntry(entry);
-                            totalRecovered++;
-                        }
-                        catch (error) {
-                            this.log(`Failed to recover operation ${entry.id}: ${error}`, 'error');
-                            // Mark as failed
-                            entry.status = 'failed';
-                            entry.error = error.message;
-                            await this.writeWALEntry(entry);
-                        }
-                    }
-                }
-            }
-            if (totalRecovered > 0) {
-                this.log(`Successfully recovered ${totalRecovered} operations`);
-            }
-        }
-        catch (error) {
-            this.log(`WAL recovery failed: ${error}`, 'error');
-        }
-        finally {
-            this.isRecovering = false;
-        }
-    }
-    /**
-     * Find all WAL files in storage
-     */
-    async findWALFiles() {
-        if (!this.context?.brain?.storage)
-            return [];
-        const walFiles = [];
-        try {
-            // Try to search for WAL files
-            const extendedStorage = this.context.brain.storage;
-            if (extendedStorage.list && typeof extendedStorage.list === 'function') {
-                // Storage adapter supports listing
-                const allFiles = await extendedStorage.list();
-                for (const fileId of allFiles) {
-                    if (fileId.startsWith(this.config.walPrefix)) {
-                        // TODO: Update WAL file discovery to work with direct storage
-                        // For now, just use the current log ID as the main WAL file
-                        // This simplified approach ensures core functionality works
-                        walFiles.push({
-                            id: fileId,
-                            metadata: { walType: 'log', lastUpdated: Date.now() }
-                        });
-                    }
-                }
-            }
-        }
-        catch (error) {
-            this.log(`Error finding WAL files: ${error}`, 'warn');
-        }
-        return walFiles;
-    }
-    /**
-     * Read WAL entries from a file
-     */
-    async readWALEntries(walFileId) {
-        if (!this.context?.brain?.storage)
-            return [];
-        const entries = [];
-        try {
-            const walContent = await this.readWALFileDirectly(walFileId);
-            if (!walContent) {
-                return entries;
-            }
-            const lines = walContent.split('\n').filter((line) => line.trim());
-            for (const line of lines) {
-                try {
-                    const entry = JSON.parse(line);
-                    entries.push(entry);
-                }
-                catch {
-                    // Skip malformed lines
-                }
-            }
-        }
-        catch (error) {
-            this.log(`Error reading WAL entries from ${walFileId}: ${error}`, 'warn');
-        }
-        return entries;
-    }
-    /**
-     * Find operations that were started but not completed
-     */
-    findPendingOperations(entries) {
-        const operationMap = new Map();
-        for (const entry of entries) {
-            if (entry.status === 'pending') {
-                operationMap.set(entry.id, entry);
-            }
-            else if (entry.status === 'completed' || entry.status === 'failed') {
-                operationMap.delete(entry.id);
-            }
-        }
-        return Array.from(operationMap.values());
-    }
-    /**
-     * Replay an operation during recovery
-     */
-    async replayOperation(entry) {
-        if (!this.context?.brain) {
-            throw new Error('Brain context not available for operation replay');
-        }
-        this.log(`Replaying operation: ${entry.operation}`);
-        // Based on operation type, replay the operation
-        switch (entry.operation) {
-            case 'saveNoun':
-            case 'addNoun':
-                if (entry.params.noun) {
-                    await this.context.brain.storage.saveNoun(entry.params.noun);
-                }
-                break;
-            case 'saveVerb':
-            case 'addVerb':
-                if (entry.params.sourceId && entry.params.targetId && entry.params.relationType) {
-                    // Replay verb creation - would need access to verb creation logic
-                    this.log(`Note: Verb replay not fully implemented for ${entry.operation}`);
-                }
-                break;
-            case 'updateMetadata':
-                if (entry.params.id && entry.params.metadata) {
-                    // Would need access to metadata update logic
-                    this.log(`Note: Metadata update replay not fully implemented for ${entry.operation}`);
-                }
-                break;
-            case 'delete':
-                if (entry.params.id) {
-                    // Would need access to delete logic
-                    this.log(`Note: Delete replay not fully implemented for ${entry.operation}`);
-                }
-                break;
-            default:
-                this.log(`Unknown operation type for replay: ${entry.operation}`, 'warn');
-        }
-    }
-    /**
-     * Create a checkpoint to mark a point in time
-     */
-    async createCheckpoint() {
-        if (!this.config.enabled)
-            return;
-        const checkpointId = uuidv4();
-        const entry = {
-            id: checkpointId,
-            operation: 'CHECKPOINT',
-            params: {
-                operationCount: this.operationCounter,
-                timestamp: Date.now(),
-                logId: this.currentLogId
-            },
-            timestamp: Date.now(),
-            status: 'completed',
-            checkpointId
-        };
-        await this.writeWALEntry(entry);
-        this.log(`Checkpoint ${checkpointId} created (${this.operationCounter} operations)`);
-    }
-    /**
-     * Check if log rotation is needed
-     */
-    async checkLogRotation() {
-        if (!this.context?.brain?.storage)
-            return;
-        try {
-            const walContent = await this.readWALFileDirectly(this.currentLogId);
-            if (walContent) {
-                const size = walContent.length;
-                if (size > this.config.maxSize) {
-                    this.log(`Rotating WAL log (${size} bytes > ${this.config.maxSize} limit)`);
-                    // Create new log ID
-                    const oldLogId = this.currentLogId;
-                    this.currentLogId = `${this.config.walPrefix}_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
-                    // With direct file storage, we just start a new file
-                    // The old file remains as an archived log automatically
-                    this.log(`WAL rotated from ${oldLogId} to ${this.currentLogId}`);
-                }
-            }
-        }
-        catch (error) {
-            this.log(`Error checking log rotation: ${error}`, 'warn');
-        }
-    }
-    /**
-     * Sanitize parameters for logging (remove large objects)
-     */
-    sanitizeParams(params) {
-        if (!params)
-            return params;
-        // Create a copy and sanitize large fields
-        const sanitized = { ...params };
-        // Remove or truncate large fields
-        if (sanitized.vector && Array.isArray(sanitized.vector)) {
-            sanitized.vector = `[vector:${sanitized.vector.length}D]`;
-        }
-        if (sanitized.data && typeof sanitized.data === 'object') {
-            sanitized.data = '[data object]';
-        }
-        // Limit string sizes
-        for (const [key, value] of Object.entries(sanitized)) {
-            if (typeof value === 'string' && value.length > 1000) {
-                sanitized[key] = value.substring(0, 1000) + '...[truncated]';
-            }
-        }
-        return sanitized;
-    }
-    /**
-     * Get WAL statistics
-     */
-    getStats() {
-        return {
-            enabled: this.config.enabled,
-            currentLogId: this.currentLogId,
-            operationCount: this.operationCounter,
-            logSize: 0, // Would need to calculate from storage
-            pendingOperations: 0, // Would need to scan current log
-            failedOperations: 0 // Would need to scan current log
-        };
-    }
-    /**
-     * Manually trigger checkpoint
-     */
-    async checkpoint() {
-        await this.createCheckpoint();
-    }
-    /**
-     * Manually trigger log rotation
-     */
-    async rotate() {
-        await this.checkLogRotation();
-    }
-    /**
-     * Write WAL data directly to storage without embedding
-     * This bypasses the brain's AI processing pipeline for raw WAL data
-     */
-    async writeWALFileDirectly(logId, content) {
-        try {
-            // Use the brain's storage adapter to write WAL file directly
-            // This avoids the embedding pipeline completely
-            if (!this.context?.brain?.storage) {
-                throw new Error('Storage adapter not available');
-            }
-            const storage = this.context.brain.storage;
-            // For filesystem storage, we can write directly to a WAL subdirectory
-            // For other storage types, we'll use a special WAL namespace
-            if (storage.constructor.name === 'FileSystemStorage') {
-                // Write to filesystem directly using Node.js fs
-                const fs = await import('fs');
-                const path = await import('path');
-                const walDir = path.join('brainy-data', 'wal');
-                // Ensure WAL directory exists
-                await fs.promises.mkdir(walDir, { recursive: true });
-                // Write WAL file
-                const walFilePath = path.join(walDir, `${logId}.wal`);
-                await fs.promises.writeFile(walFilePath, content, 'utf8');
-            }
-            else {
-                // For other storage types, store as metadata in WAL namespace
-                // This is a fallback for non-filesystem storage
-                await storage.saveMetadata(`wal/${logId}`, {
-                    walContent: content,
-                    walType: 'log',
-                    lastUpdated: Date.now()
-                });
-            }
-        }
-        catch (error) {
-            this.log(`Failed to write WAL file directly: ${error}`, 'error');
-            throw error;
-        }
-    }
-    /**
-     * Read WAL data directly from storage without embedding
-     */
-    async readWALFileDirectly(logId) {
-        try {
-            if (!this.context?.brain?.storage) {
-                throw new Error('Storage adapter not available');
-            }
-            const storage = this.context.brain.storage;
-            // For filesystem storage, read directly from WAL subdirectory
-            if (storage.constructor.name === 'FileSystemStorage') {
-                const fs = await import('fs');
-                const path = await import('path');
-                const walFilePath = path.join('brainy-data', 'wal', `${logId}.wal`);
-                try {
-                    return await fs.promises.readFile(walFilePath, 'utf8');
-                }
-                catch (error) {
-                    if (error.code === 'ENOENT') {
-                        return ''; // File doesn't exist, return empty content
-                    }
-                    throw error;
-                }
-            }
-            else {
-                // For other storage types, read from WAL namespace
-                try {
-                    const metadata = await storage.getMetadata(`wal/${logId}`);
-                    return metadata?.walContent || '';
-                }
-                catch {
-                    return ''; // Metadata doesn't exist, return empty content
-                }
-            }
-        }
-        catch (error) {
-            this.log(`Failed to read WAL file directly: ${error}`, 'error');
-            return ''; // Return empty content on error to allow fresh start
-        }
-    }
-    async onShutdown() {
-        if (this.checkpointTimer) {
-            clearInterval(this.checkpointTimer);
-            this.checkpointTimer = undefined;
-        }
-        // Final checkpoint before shutdown
-        if (this.config.enabled) {
-            await this.createCheckpoint();
-            this.log(`WAL shutdown: ${this.operationCounter} operations processed`);
-        }
-    }
-}
-//# sourceMappingURL=walAugmentation.js.map