@exaudeus/workrail 0.1.0 → 0.1.2

package/dist/infrastructure/storage/hybrid-context-storage.js

@@ -0,0 +1,400 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HybridContextStorage = void 0;
+exports.createDefaultHybridContextStorage = createDefaultHybridContextStorage;
+exports.createCustomHybridContextStorage = createCustomHybridContextStorage;
+const crypto_1 = __importDefault(require("crypto"));
+const error_handler_1 = require("../../core/error-handler");
+class HybridContextStorage {
+    constructor(metadataStorage, blobStorage, config) {
+        this.isInitialized = false;
+        this.metadataStorage = metadataStorage;
+        this.blobStorage = blobStorage;
+        this.config = config;
+    }
+    async initialize() {
+        if (this.isInitialized) {
+            return;
+        }
+        try {
+            await Promise.all([
+                this.metadataStorage.initialize(),
+                this.blobStorage.initialize()
+            ]);
+            this.isInitialized = true;
+            console.log('✅ Hybrid context storage initialized');
+        }
+        catch (error) {
+            throw new Error(`Failed to initialize hybrid context storage: ${error}`);
+        }
+    }
+    async saveCheckpoint(metadata, blob) {
+        this.ensureInitialized();
+        let operationId = null;
+        try {
+            const checkpointId = metadata.id || crypto_1.default.randomUUID();
+            const fullMetadata = {
+                ...metadata,
+                id: checkpointId,
+                createdAt: metadata.createdAt || new Date().toISOString(),
+                status: metadata.status || 'active'
+            };
+            await this.metadataStorage.saveCheckpointMetadata(fullMetadata);
+            const blobMetadata = await this.blobStorage.saveBlob(metadata.sessionId, checkpointId, blob);
+            await this.metadataStorage.upsertSessionInfo({
+                id: metadata.sessionId,
+                lastAccessedAt: new Date().toISOString(),
+                totalSizeBytes: blob.originalSize,
+                createdAt: new Date().toISOString()
+            });
+            console.log(`✅ Saved checkpoint: ${checkpointId} (${blobMetadata.sizeBytes} bytes)`);
+        }
+        catch (error) {
+            try {
+                if (operationId) {
+                    console.warn(`Warning: Checkpoint save failed, blob may be orphaned: ${metadata.id}`);
+                }
+            }
+            catch (cleanupError) {
+                console.warn('Failed to cleanup after save failure:', cleanupError);
+            }
+            throw new Error(`Failed to save checkpoint: ${error}`);
+        }
+        finally {
+            if (operationId) {
+                try {
+                    await this.metadataStorage.releaseSessionLock(operationId);
+                }
+                catch (lockError) {
+                    console.warn('Failed to release session lock:', lockError);
+                }
+            }
+        }
+    }
+    async loadCheckpoint(checkpointId) {
+        this.ensureInitialized();
+        try {
+            const metadata = await this.metadataStorage.getCheckpointMetadata(checkpointId);
+            if (!metadata) {
+                throw new error_handler_1.CheckpointNotFoundError(checkpointId);
+            }
+            const blobMetadata = {
+                checkpointId: metadata.id,
+                sessionId: metadata.sessionId,
+                path: metadata.blobPath,
+                sizeBytes: metadata.contextSizeBytes,
+                hash: metadata.contextHash,
+                encrypted: false
+            };
+            const blob = await this.blobStorage.loadBlob(blobMetadata);
+            if (!blob) {
+                throw new error_handler_1.CheckpointNotFoundError(checkpointId);
+            }
+            await this.updateSessionAccess(metadata.sessionId);
+            console.log(`✅ Loaded checkpoint: ${checkpointId}`);
+            return { metadata, blob };
+        }
+        catch (error) {
+            throw new Error(`Failed to load checkpoint: ${error}`);
+        }
+    }
+    async listCheckpoints(sessionId, limit = 20, offset = 0) {
+        this.ensureInitialized();
+        try {
+            const checkpoints = await this.metadataStorage.listCheckpointMetadata(sessionId, limit, offset);
+            if (checkpoints.length > 0) {
+                await this.updateSessionAccess(sessionId);
+            }
+            return checkpoints;
+        }
+        catch (error) {
+            throw new Error(`Failed to list checkpoints: ${error}`);
+        }
+    }
+    async deleteCheckpoint(checkpointId) {
+        this.ensureInitialized();
+        let operationId = null;
+        try {
+            const metadata = await this.metadataStorage.getCheckpointMetadata(checkpointId);
+            if (!metadata) {
+                throw new error_handler_1.CheckpointNotFoundError(checkpointId);
+            }
+            operationId = await this.metadataStorage.acquireSessionLock(metadata.sessionId, 'delete', this.config.concurrency.operationTimeoutMs);
+            await this.metadataStorage.deleteCheckpointMetadata(checkpointId);
+            const blobMetadata = {
+                checkpointId: metadata.id,
+                sessionId: metadata.sessionId,
+                path: metadata.blobPath,
+                sizeBytes: metadata.contextSizeBytes,
+                hash: metadata.contextHash,
+                encrypted: false
+            };
+            await this.blobStorage.deleteBlob(blobMetadata);
+            console.log(`🗑️ Deleted checkpoint: ${checkpointId}`);
+        }
+        catch (error) {
+            throw new Error(`Failed to delete checkpoint: ${error}`);
+        }
+        finally {
+            if (operationId) {
+                try {
+                    await this.metadataStorage.releaseSessionLock(operationId);
+                }
+                catch (lockError) {
+                    console.warn('Failed to release session lock:', lockError);
+                }
+            }
+        }
+    }
+    async getSession(sessionId) {
+        this.ensureInitialized();
+        try {
+            return await this.metadataStorage.getSessionInfo(sessionId);
+        }
+        catch (error) {
+            throw new Error(`Failed to get session: ${error}`);
+        }
+    }
+    async upsertSession(session) {
+        this.ensureInitialized();
+        try {
+            await this.metadataStorage.upsertSessionInfo(session);
+            console.log(`📋 Upserted session: ${session.id}`);
+        }
+        catch (error) {
+            throw new Error(`Failed to upsert session: ${error}`);
+        }
+    }
+    async deleteSession(sessionId) {
+        this.ensureInitialized();
+        let operationId = null;
+        try {
+            operationId = await this.metadataStorage.acquireSessionLock(sessionId, 'delete', this.config.concurrency.operationTimeoutMs);
+            const checkpoints = await this.metadataStorage.listCheckpointMetadata(sessionId, 1000, 0);
+            for (const checkpoint of checkpoints) {
+                try {
+                    const blobMetadata = {
+                        checkpointId: checkpoint.id,
+                        sessionId: checkpoint.sessionId,
+                        path: checkpoint.blobPath,
+                        sizeBytes: checkpoint.contextSizeBytes,
+                        hash: checkpoint.contextHash,
+                        encrypted: false
+                    };
+                    await this.blobStorage.deleteBlob(blobMetadata);
+                }
+                catch (blobError) {
+                    console.warn(`Warning: Failed to delete blob for checkpoint ${checkpoint.id}:`, blobError);
+                }
+            }
+            await this.metadataStorage.deleteSessionInfo(sessionId);
+            console.log(`🗑️ Deleted session: ${sessionId} (${checkpoints.length} checkpoints)`);
+        }
+        catch (error) {
+            throw new Error(`Failed to delete session: ${error}`);
+        }
+        finally {
+            if (operationId) {
+                try {
+                    await this.metadataStorage.releaseSessionLock(operationId);
+                }
+                catch (lockError) {
+                    console.warn('Failed to release session lock:', lockError);
+                }
+            }
+        }
+    }
+    async getStorageStats() {
+        this.ensureInitialized();
+        try {
+            const [metadataStats, blobStats] = await Promise.all([
+                this.metadataStorage.getMetadataStats(),
+                this.blobStorage.getBlobStats()
+            ]);
+            const totalSizeBytes = blobStats.totalSizeBytes;
+            const maxTotalSize = this.config.quotas.maxTotalSize;
+            const storageUtilization = maxTotalSize > 0 ? totalSizeBytes / maxTotalSize : 0;
+            return {
+                totalSessions: metadataStats.totalSessions,
+                totalCheckpoints: metadataStats.totalCheckpoints,
+                totalSizeBytes: blobStats.totalSizeBytes,
+                averageCheckpointSize: metadataStats.averageCheckpointSize,
+                oldestCheckpoint: undefined,
+                newestCheckpoint: undefined,
+                storageUtilization
+            };
+        }
+        catch (error) {
+            throw new Error(`Failed to get storage stats: ${error}`);
+        }
+    }
+    async getCheckpoint(checkpointId) {
+        this.ensureInitialized();
+        return this.metadataStorage.getCheckpointMetadata(checkpointId);
+    }
+    async validateIntegrity() {
+        this.ensureInitialized();
+        const result = {
+            isValid: true,
+            errors: [],
+            warnings: [],
+            repairSuggestions: []
+        };
+        try {
+            const blobValidation = await this.blobStorage.validateBlobIntegrity();
+            if (!blobValidation.isValid) {
+                result.isValid = false;
+                if (blobValidation.corruptedFiles.length > 0) {
+                    result.errors.push({
+                        type: 'CORRUPTION',
+                        description: `Found ${blobValidation.corruptedFiles.length} corrupted blob files`,
+                        affectedItems: blobValidation.corruptedFiles,
+                        severity: 'HIGH'
+                    });
+                }
+                if (blobValidation.missingFiles.length > 0) {
+                    result.errors.push({
+                        type: 'MISSING_DATA',
+                        description: `Found ${blobValidation.missingFiles.length} missing blob files`,
+                        affectedItems: blobValidation.missingFiles,
+                        severity: 'CRITICAL'
+                    });
+                }
+                if (blobValidation.orphanedFiles.length > 0) {
+                    result.warnings.push({
+                        type: 'CLEANUP_NEEDED',
+                        description: `Found ${blobValidation.orphanedFiles.length} orphaned blob files`,
+                        suggestion: 'Run orphaned file cleanup to reclaim disk space'
+                    });
+                }
+            }
+            const stats = await this.getStorageStats();
+            if (stats.storageUtilization > this.config.quotas.warningThreshold) {
+                const severity = stats.storageUtilization > this.config.quotas.cleanupThreshold ? 'HIGH' : 'MEDIUM';
+                result.warnings.push({
+                    type: 'QUOTA_WARNING',
+                    description: `Storage usage at ${(stats.storageUtilization * 100).toFixed(1)}% of quota`,
+                    suggestion: severity === 'HIGH' ? 'Immediate cleanup recommended' : 'Consider cleanup soon'
+                });
+            }
+            if (result.errors.length > 0) {
+                result.repairSuggestions.push('Run storage repair to fix corrupted or missing data');
+            }
+            if (result.warnings.some(w => w.type === 'CLEANUP_NEEDED')) {
+                result.repairSuggestions.push('Run orphaned file cleanup to reclaim disk space');
+            }
+            return result;
+        }
+        catch (error) {
+            result.isValid = false;
+            result.errors.push({
+                type: 'CORRUPTION',
+                description: `Storage validation failed: ${error}`,
+                affectedItems: ['storage-system'],
+                severity: 'CRITICAL'
+            });
+            return result;
+        }
+    }
+    async close() {
+        try {
+            await Promise.all([
+                this.metadataStorage.close(),
+                this.blobStorage.close()
+            ]);
+            this.isInitialized = false;
+            console.log('🔒 Hybrid context storage closed');
+        }
+        catch (error) {
+            console.warn('Warning during hybrid storage close:', error);
+        }
+    }
+    async cleanupOrphanedBlobs() {
+        this.ensureInitialized();
+        try {
+            const allCheckpoints = await this.metadataStorage.listCheckpointMetadata('', 10000, 0);
+            const referencedPaths = allCheckpoints.map(checkpoint => checkpoint.blobPath);
+            const cleanedCount = await this.blobStorage.cleanupOrphanedBlobs(referencedPaths);
+            if (cleanedCount > 0) {
+                console.log(`🧹 Cleaned up ${cleanedCount} orphaned blobs`);
+            }
+            return cleanedCount;
+        }
+        catch (error) {
+            throw new Error(`Failed to cleanup orphaned blobs: ${error}`);
+        }
+    }
+    async updateSessionAccess(sessionId) {
+        try {
+            const session = await this.metadataStorage.getSessionInfo(sessionId);
+            if (session) {
+                const updatedSession = {
+                    ...session,
+                    lastAccessedAt: new Date().toISOString()
+                };
+                await this.metadataStorage.upsertSessionInfo(updatedSession);
+            }
+        }
+        catch (error) {
+            console.warn(`Warning: Failed to update session access time for ${sessionId}:`, error);
+        }
+    }
+    ensureInitialized() {
+        if (!this.isInitialized) {
+            throw new Error('HybridContextStorage not initialized');
+        }
+    }
+}
+exports.HybridContextStorage = HybridContextStorage;
+async function createDefaultHybridContextStorage(config) {
+    const { createDefaultContextStorageConfig } = await Promise.resolve().then(() => __importStar(require('./context-storage')));
+    const { SqliteMetadataStorage } = await Promise.resolve().then(() => __importStar(require('./sqlite-metadata-storage')));
+    const { FileSystemBlobStorage } = await Promise.resolve().then(() => __importStar(require('./filesystem-blob-storage')));
+    const fullConfig = config ? { ...createDefaultContextStorageConfig(), ...config } : createDefaultContextStorageConfig();
+    const metadataStorage = new SqliteMetadataStorage(fullConfig);
+    const blobStorage = new FileSystemBlobStorage(fullConfig);
+    const hybridStorage = new HybridContextStorage(metadataStorage, blobStorage, fullConfig);
+    await hybridStorage.initialize();
+    return hybridStorage;
+}
+async function createCustomHybridContextStorage(metadataStorage, blobStorage, config) {
+    const hybridStorage = new HybridContextStorage(metadataStorage, blobStorage, config);
+    await hybridStorage.initialize();
+    return hybridStorage;
+}

package/dist/infrastructure/storage/migrations/001_initial_schema.sql

@@ -0,0 +1,38 @@
+-- Migration: Initial schema for context management
+-- Version: 1
+-- Description: Creates the initial tables for context persistence
+
+-- Sessions table
+CREATE TABLE IF NOT EXISTS sessions (
+  id TEXT PRIMARY KEY,
+  created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+  last_accessed_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+  workflow_id TEXT,
+  tags TEXT, -- JSON array stored as text
+  total_size_bytes INTEGER DEFAULT 0
+);
+
+-- Checkpoint metadata table
+CREATE TABLE IF NOT EXISTS checkpoint_metadata (
+  id TEXT PRIMARY KEY,
+  session_id TEXT NOT NULL,
+  name TEXT,
+  agent_id TEXT,
+  created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+  tags TEXT, -- JSON array stored as text
+  context_size_bytes INTEGER NOT NULL,
+  context_hash TEXT NOT NULL,
+  blob_path TEXT NOT NULL,
+  status TEXT DEFAULT 'active',
+  classification_info TEXT, -- JSON object stored as text
+  created_by_operation TEXT DEFAULT NULL,
+  compression_ratio REAL DEFAULT 1.0,
+  FOREIGN KEY (session_id) REFERENCES sessions(id) ON DELETE CASCADE
+);
+
+-- Indexes for performance
+CREATE INDEX IF NOT EXISTS idx_checkpoint_session ON checkpoint_metadata(session_id);
+CREATE INDEX IF NOT EXISTS idx_checkpoint_created ON checkpoint_metadata(created_at);
+CREATE INDEX IF NOT EXISTS idx_checkpoint_status ON checkpoint_metadata(status);
+CREATE INDEX IF NOT EXISTS idx_session_workflow ON sessions(workflow_id);
+CREATE INDEX IF NOT EXISTS idx_session_accessed ON sessions(last_accessed_at); 

package/dist/infrastructure/storage/migrations/002_context_concurrency_enhancements.sql

@@ -0,0 +1,234 @@
+-- Enhanced Concurrency Safety for Native Context Management
+-- Version 2: Adds active operations tracking and lock management
+-- Based on Devil's Advocate Review recommendations
+
+-- Update schema version
+INSERT OR IGNORE INTO schema_version (version, description) VALUES (2, 'Enhanced concurrency safety with active operations tracking');
+
+-- =============================================================================
+-- ACTIVE OPERATIONS TRACKING
+-- =============================================================================
+
+-- Table to track active operations for concurrency safety
+CREATE TABLE IF NOT EXISTS active_operations (
+    id TEXT PRIMARY KEY,                                    -- Operation UUID
+    session_id TEXT NOT NULL,                              -- Session being operated on
+    operation_type TEXT NOT NULL CHECK (operation_type IN ('save', 'load', 'delete', 'cleanup')), -- Type of operation
+    started_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, -- When operation started
+    heartbeat_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, -- Last heartbeat timestamp
+    timeout_ms INTEGER NOT NULL DEFAULT 5000,              -- Timeout in milliseconds
+    metadata TEXT,                                          -- JSON metadata about operation
+    FOREIGN KEY (session_id) REFERENCES sessions(id) ON DELETE CASCADE
+);
+
+-- Indexes for efficient lock management
+CREATE INDEX IF NOT EXISTS idx_active_operations_session ON active_operations(session_id);
+CREATE INDEX IF NOT EXISTS idx_active_operations_heartbeat ON active_operations(heartbeat_at);
+CREATE INDEX IF NOT EXISTS idx_active_operations_started ON active_operations(started_at);
+CREATE INDEX IF NOT EXISTS idx_active_operations_type ON active_operations(operation_type);
+
+-- =============================================================================
+-- AUTOMATIC CLEANUP TRIGGERS
+-- =============================================================================
+
+-- Trigger to cleanup stale operations when new operations are inserted
+CREATE TRIGGER IF NOT EXISTS cleanup_stale_operations_on_insert
+AFTER INSERT ON active_operations
+FOR EACH ROW
+BEGIN
+    -- Remove operations older than 10 minutes or with stale heartbeats (>5 minutes)
+    DELETE FROM active_operations 
+    WHERE (
+        heartbeat_at < datetime('now', '-5 minutes') OR 
+        started_at < datetime('now', '-10 minutes')
+    ) AND id != NEW.id;
+END;
+
+-- Trigger to cleanup operations when sessions are deleted
+CREATE TRIGGER IF NOT EXISTS cleanup_operations_on_session_delete
+AFTER DELETE ON sessions
+FOR EACH ROW
+BEGIN
+    DELETE FROM active_operations WHERE session_id = OLD.id;
+END;
+
+-- =============================================================================
+-- SESSION LOCK MANAGEMENT
+-- =============================================================================
+
+-- Add session-level lock information to sessions table
+ALTER TABLE sessions ADD COLUMN locked_at DATETIME DEFAULT NULL;
+ALTER TABLE sessions ADD COLUMN locked_by TEXT DEFAULT NULL;  -- Operation ID holding the lock
+ALTER TABLE sessions ADD COLUMN lock_timeout_at DATETIME DEFAULT NULL;
+
+-- Index for efficient lock queries
+CREATE INDEX IF NOT EXISTS idx_sessions_locked_at ON sessions(locked_at);
+CREATE INDEX IF NOT EXISTS idx_sessions_lock_timeout ON sessions(lock_timeout_at);
+
+-- Trigger to automatically release expired locks
+CREATE TRIGGER IF NOT EXISTS release_expired_session_locks
+AFTER INSERT ON active_operations
+FOR EACH ROW
+BEGIN
+    -- Release locks that have timed out
+    UPDATE sessions 
+    SET locked_at = NULL, locked_by = NULL, lock_timeout_at = NULL
+    WHERE lock_timeout_at IS NOT NULL AND lock_timeout_at < CURRENT_TIMESTAMP;
+END;
+
+-- =============================================================================
+-- ENHANCED CHECKPOINT METADATA
+-- =============================================================================
+
+-- Add operation tracking to checkpoint metadata
+ALTER TABLE checkpoint_metadata ADD COLUMN created_by_operation TEXT DEFAULT NULL;
+ALTER TABLE checkpoint_metadata ADD COLUMN compression_ratio REAL DEFAULT 1.0;
+ALTER TABLE checkpoint_metadata ADD COLUMN classification_info TEXT DEFAULT NULL; -- JSON with layer counts
+
+-- Index for operation tracking
+CREATE INDEX IF NOT EXISTS idx_checkpoints_created_by_operation ON checkpoint_metadata(created_by_operation);
+
+-- =============================================================================
+-- STORAGE QUOTA TRACKING
+-- =============================================================================
+
+-- Table for tracking storage quotas and usage
+CREATE TABLE IF NOT EXISTS storage_quotas (
+    id TEXT PRIMARY KEY DEFAULT 'global',                  -- Usually 'global' or session_id
+    max_total_size INTEGER NOT NULL DEFAULT 10737418240,   -- 10GB default
+    max_checkpoints INTEGER NOT NULL DEFAULT 1000,         -- 1000 checkpoints default
+    warning_threshold REAL NOT NULL DEFAULT 0.8,           -- Warn at 80%
+    cleanup_threshold REAL NOT NULL DEFAULT 0.9,           -- Auto-cleanup at 90%
+    last_cleanup_at DATETIME DEFAULT NULL,
+    created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Insert default global quota
+INSERT OR IGNORE INTO storage_quotas (id, max_total_size, max_checkpoints) 
+VALUES ('global', 10737418240, 1000);
+
+-- =============================================================================
+-- PERFORMANCE MONITORING
+-- =============================================================================
+
+-- Table for tracking operation performance metrics
+CREATE TABLE IF NOT EXISTS operation_metrics (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    operation_type TEXT NOT NULL,                          -- save, load, classify, compress
+    duration_ms INTEGER NOT NULL,                          -- Operation duration
+    context_size_bytes INTEGER DEFAULT NULL,               -- Input size
+    compressed_size_bytes INTEGER DEFAULT NULL,            -- Output size (if applicable)
+    compression_ratio REAL DEFAULT NULL,                   -- Compression ratio achieved
+    session_id TEXT DEFAULT NULL,                          -- Associated session
+    timestamp DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    metadata TEXT DEFAULT NULL                             -- JSON with additional metrics
+);
+
+-- Indexes for performance analysis
+CREATE INDEX IF NOT EXISTS idx_metrics_operation_type ON operation_metrics(operation_type);
+CREATE INDEX IF NOT EXISTS idx_metrics_timestamp ON operation_metrics(timestamp);
+CREATE INDEX IF NOT EXISTS idx_metrics_session ON operation_metrics(session_id);
+
+-- Trigger to automatically cleanup old metrics (keep last 30 days)
+CREATE TRIGGER IF NOT EXISTS cleanup_old_metrics
+AFTER INSERT ON operation_metrics
+FOR EACH ROW
+WHEN (NEW.id % 100 = 0)  -- Only run cleanup every 100 inserts for performance
+BEGIN
+    DELETE FROM operation_metrics 
+    WHERE timestamp < datetime('now', '-30 days');
+END;
+
+-- =============================================================================
+-- ENHANCED VIEWS FOR MONITORING
+-- =============================================================================
+
+-- View for active sessions with lock status
+CREATE VIEW IF NOT EXISTS active_sessions_with_locks AS
+SELECT 
+    s.*,
+    ao.operation_type as active_operation,
+    ao.started_at as operation_started_at,
+    ao.heartbeat_at as last_heartbeat,
+    CASE 
+        WHEN s.locked_at IS NOT NULL AND s.lock_timeout_at > CURRENT_TIMESTAMP THEN 'locked'
+        WHEN s.locked_at IS NOT NULL AND s.lock_timeout_at <= CURRENT_TIMESTAMP THEN 'expired_lock'
+        WHEN ao.id IS NOT NULL THEN 'operation_active'
+        ELSE 'available'
+    END as lock_status
+FROM sessions s
+LEFT JOIN active_operations ao ON s.id = ao.session_id
+WHERE s.last_accessed_at > datetime('now', '-7 days');  -- Only recent sessions
+
+-- View for storage usage summary
+CREATE VIEW IF NOT EXISTS storage_usage_summary AS
+SELECT 
+    'global' as scope,
+    COUNT(DISTINCT s.id) as total_sessions,
+    COUNT(c.id) as total_checkpoints,
+    COALESCE(SUM(s.total_size_bytes), 0) as total_size_bytes,
+    COALESCE(AVG(c.context_size_bytes), 0) as avg_checkpoint_size,
+    MAX(c.created_at) as latest_checkpoint,
+    q.max_total_size,
+    q.max_checkpoints,
+    CAST(COALESCE(SUM(s.total_size_bytes), 0) AS REAL) / q.max_total_size as size_utilization,
+    CAST(COUNT(c.id) AS REAL) / q.max_checkpoints as checkpoint_utilization
+FROM storage_quotas q
+LEFT JOIN sessions s ON q.id = 'global'
+LEFT JOIN checkpoint_metadata c ON s.id = c.session_id
+WHERE q.id = 'global'
+GROUP BY q.id, q.max_total_size, q.max_checkpoints;
+
+-- View for recent operation performance
+CREATE VIEW IF NOT EXISTS recent_operation_performance AS
+SELECT 
+    operation_type,
+    COUNT(*) as operation_count,
+    AVG(duration_ms) as avg_duration_ms,
+    MIN(duration_ms) as min_duration_ms,
+    MAX(duration_ms) as max_duration_ms,
+    PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY duration_ms) as p50_duration_ms,
+    PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY duration_ms) as p95_duration_ms,
+    AVG(CASE WHEN compression_ratio IS NOT NULL THEN compression_ratio END) as avg_compression_ratio,
+    COUNT(CASE WHEN duration_ms > 1000 THEN 1 END) as slow_operations_count
+FROM operation_metrics 
+WHERE timestamp > datetime('now', '-24 hours')
+GROUP BY operation_type;
+
+-- =============================================================================
+-- INTEGRITY CONSTRAINTS
+-- =============================================================================
+
+-- Ensure no duplicate active operations per session for save operations
+CREATE UNIQUE INDEX IF NOT EXISTS idx_unique_save_operations 
+ON active_operations(session_id) 
+WHERE operation_type = 'save';
+
+-- Ensure checkpoint metadata integrity
+CREATE INDEX IF NOT EXISTS idx_checkpoints_hash_unique ON checkpoint_metadata(context_hash, session_id);
+
+-- =============================================================================
+-- CLEANUP PROCEDURES
+-- =============================================================================
+
+-- Create a trigger for automatic maintenance
+CREATE TRIGGER IF NOT EXISTS automatic_maintenance
+AFTER INSERT ON sessions
+FOR EACH ROW
+WHEN (NEW.rowid % 50 = 0)  -- Run maintenance every 50 new sessions
+BEGIN
+    -- Update storage quota usage tracking
+    UPDATE storage_quotas 
+    SET updated_at = CURRENT_TIMESTAMP 
+    WHERE id = 'global';
+    
+    -- Clean up orphaned operations
+    DELETE FROM active_operations 
+    WHERE session_id NOT IN (SELECT id FROM sessions);
+    
+    -- Clean up very old sessions with no checkpoints
+    DELETE FROM sessions 
+    WHERE id NOT IN (SELECT DISTINCT session_id FROM checkpoint_metadata)
+    AND last_accessed_at < datetime('now', '-90 days');
+END; 

package/dist/infrastructure/storage/migrations/003_classification_overrides.sql

@@ -0,0 +1,20 @@
+-- Migration: Classification Overrides
+-- Version: 3
+-- Description: Adds table for session-specific classification overrides
+
+-- Update schema version
+INSERT OR IGNORE INTO schema_version (version, description) VALUES (3, 'Classification overrides');
+
+-- Classification overrides table
+CREATE TABLE IF NOT EXISTS classification_overrides (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  session_id TEXT NOT NULL,
+  context_key TEXT NOT NULL,
+  classification TEXT NOT NULL,
+  created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+  FOREIGN KEY (session_id) REFERENCES sessions(id) ON DELETE CASCADE,
+  UNIQUE(session_id, context_key)
+);
+
+-- Index for performance
+CREATE INDEX IF NOT EXISTS idx_classification_session ON classification_overrides(session_id);