@dynamicu/chromedebug-mcp 2.2.0

package/src/capture/architecture.js

@@ -0,0 +1,426 @@
+/**
+ * Unified Interaction Capture System - Data Flow Architecture
+ *
+ * This module defines the complete data flow architecture and module boundaries
+ * for the unified interaction capture system.
+ *
+ * SYSTEM OVERVIEW:
+ * ================
+ * The capture system replaces three separate legacy systems:
+ * 1. Frame recording (screenshots + console logs)
+ * 2. Workflow recording (user actions + DOM state)
+ * 3. Function tracing (execution traces)
+ *
+ * UNIFIED APPROACH:
+ * ================
+ * All interactions are captured as CaptureEvents with a unified format.
+ * Each event has a type (screen, dom, console, user, function, network, state)
+ * and type-specific data payload.
+ *
+ * DATA FLOW DIAGRAM:
+ * ==================
+ *
+ * ┌─────────────────────────────────────────────────────────────────────┐
+ * │                         CAPTURE SOURCES                            │
+ * └─────────────────────────────────────────────────────────────────────┘
+ *           │                    │                    │
+ *           ▼                    ▼                    ▼
+ * ┌───────────────┐    ┌─────────────────┐    ┌──────────────────┐
+ * │ Chrome        │    │ Puppeteer       │    │ DevTools         │
+ * │ Extension     │    │ Source          │    │ Source           │
+ * │               │    │                 │    │                  │
+ * │ • Screenshots │    │ • DOM Snapshots │    │ • Console Logs   │
+ * │ • User Events │    │ • Network Data  │    │ • Function Trace │
+ * │ • Console     │    │ • Performance   │    │ • Browser State  │
+ * └───────────────┘    └─────────────────┘    └──────────────────┘
+ *           │                    │                    │
+ *           └────────────────────┼────────────────────┘
+ *                                ▼
+ * ┌─────────────────────────────────────────────────────────────────────┐
+ * │                      CAPTURE COLLECTOR                             │
+ * │                                                                     │
+ * │ • Event Routing & Coordination                                      │
+ * │ • Source Registration                                               │
+ * │ • Event Sequencing                                                  │
+ * │ • Memory Pressure Management                                        │
+ * └─────────────────────────────────────────────────────────────────────┘
+ *                                ▼
+ * ┌─────────────────────────────────────────────────────────────────────┐
+ * │                    CAPTURE PROCESSORS                               │
+ * │                                                                     │
+ * │ ┌─────────────┐  ┌─────────────┐  ┌─────────────┐  ┌─────────────┐ │
+ * │ │ Filter      │  │ Transform   │  │ Enrichment  │  │ Compression │ │
+ * │ │ Processor   │  │ Processor   │  │ Processor   │  │ Processor   │ │
+ * │ │             │  │             │  │             │  │             │ │
+ * │ │ • Dedupe    │  │ • Normalize │  │ • Add Meta  │  │ • Image     │ │
+ * │ │ • Priority  │  │ • Sanitize  │  │ • Correlate │  │ • JSON      │ │
+ * │ │ • Rate Limit│  │ • Format    │  │ • Context   │  │ • Binary    │ │
+ * │ └─────────────┘  └─────────────┘  └─────────────┘  └─────────────┘ │
+ * └─────────────────────────────────────────────────────────────────────┘
+ *                                ▼
+ * ┌─────────────────────────────────────────────────────────────────────┐
+ * │                      CAPTURE STORAGE                                │
+ * │                                                                     │
+ * │ ┌─────────────┐  ┌─────────────┐  ┌─────────────┐  ┌─────────────┐ │
+ * │ │ Database    │  │ File System │  │ Memory      │  │ Network     │ │
+ * │ │ Storage     │  │ Storage     │  │ Storage     │  │ Storage     │ │
+ * │ │             │  │             │  │             │  │             │ │
+ * │ │ • SQLite    │  │ • Images    │  │ • Recent    │  │ • Cloud     │ │
+ * │ │ • Metadata  │  │ • Videos    │  │ • Cache     │  │ • Backup    │ │
+ * │ │ • Indexes   │  │ • Logs      │  │ • Temp      │  │ • Sync      │ │
+ * │ └─────────────┘  └─────────────┘  └─────────────┘  └─────────────┘ │
+ * └─────────────────────────────────────────────────────────────────────┘
+ *                                ▼
+ * ┌─────────────────────────────────────────────────────────────────────┐
+ * │                    CAPTURE QUERY & ANALYSIS                        │
+ * │                                                                     │
+ * │ ┌─────────────┐  ┌─────────────┐  ┌─────────────┐  ┌─────────────┐ │
+ * │ │ Query       │  │ Search      │  │ Analytics   │  │ Export      │ │
+ * │ │ Engine      │  │ Engine      │  │ Engine      │  │ Engine      │ │
+ * │ │             │  │             │  │             │  │             │ │
+ * │ │ • Filter    │  │ • Text      │  │ • Patterns  │  │ • JSON      │ │
+ * │ │ • Sort      │  │ • Semantic  │  │ • Metrics   │  │ • CSV       │ │
+ * │ │ • Paginate  │  │ • Fuzzy     │  │ • Reports   │  │ • Video     │ │
+ * │ └─────────────┘  └─────────────┘  └─────────────┘  └─────────────┘ │
+ * └─────────────────────────────────────────────────────────────────────┘
+ *
+ * MEMORY MANAGEMENT STRATEGY:
+ * ===========================
+ *
+ * 1. TIERED STORAGE:
+ *    • Hot Data (Recent): Memory cache for instant access
+ *    • Warm Data (Session): Database for quick queries
+ *    • Cold Data (Archive): File system for long-term storage
+ *
+ * 2. SIZE-BASED OPTIMIZATION:
+ *    • Small Events (<1KB): Keep in memory, batch to database
+ *    • Medium Events (1KB-100KB): Stream to database immediately
+ *    • Large Events (100KB-1MB): Stream to file system, reference in database
+ *    • Huge Events (>1MB): Compress, stream to file system
+ *
+ * 3. MEMORY PRESSURE HANDLING:
+ *    • Monitor memory usage continuously
+ *    • Trigger cleanup when limits approached
+ *    • Use backpressure to slow capture if needed
+ *    • Implement circular buffers for high-frequency events
+ *
+ * 4. PERFORMANCE OPTIMIZATION:
+ *    • Async processing pipelines
+ *    • Batch operations for database writes
+ *    • Streaming for large data
+ *    • Lazy loading for queries
+ *
+ * MODULE BOUNDARIES & DEPENDENCIES:
+ * =================================
+ *
+ * 1. CAPTURE SOURCES (Independent):
+ *    - No dependencies on other capture modules
+ *    - Only depends on types.js for event format
+ *    - Can be developed and tested in isolation
+ *    - Examples: ChromeExtensionSource, PuppeteerSource, DevToolsSource
+ *
+ * 2. CAPTURE COLLECTOR (Orchestrator):
+ *    - Depends on ICaptureSource interface
+ *    - Manages source lifecycle
+ *    - Routes events to processors
+ *    - Handles backpressure and flow control
+ *
+ * 3. CAPTURE PROCESSORS (Transformers):
+ *    - No dependencies on sources or storage
+ *    - Only depends on types.js for event format
+ *    - Can be chained in pipeline
+ *    - Examples: FilterProcessor, CompressionProcessor, EnrichmentProcessor
+ *
+ * 4. CAPTURE STORAGE (Persistence):
+ *    - No dependencies on sources or processors
+ *    - Only depends on types.js for event format
+ *    - Multiple storage backends possible
+ *    - Examples: SQLiteStorage, FileSystemStorage, MemoryStorage
+ *
+ * 5. CAPTURE QUERY (Read Layer):
+ *    - Depends only on ICaptureStorage interface
+ *    - Provides unified query interface
+ *    - Can work with any storage backend
+ *
+ * 6. CAPTURE ANALYZER (Intelligence):
+ *    - Depends only on ICaptureQuery interface
+ *    - Provides insights and analytics
+ *    - Can be extended with new analysis types
+ *
+ * ERROR HANDLING STRATEGY:
+ * ========================
+ *
+ * 1. GRACEFUL DEGRADATION:
+ *    • If screenshot capture fails, continue with other events
+ *    • If database unavailable, fallback to file storage
+ *    • If memory full, switch to streaming mode
+ *
+ * 2. ERROR ISOLATION:
+ *    • Errors in one source don't affect others
+ *    • Processor errors don't stop the pipeline
+ *    • Storage errors trigger fallback mechanisms
+ *
+ * 3. RECOVERY MECHANISMS:
+ *    • Automatic retry with exponential backoff
+ *    • Circuit breaker pattern for failing components
+ *    • Health checks for system components
+ *
+ * 4. ERROR REPORTING:
+ *    • Structured error events in capture stream
+ *    • Error metrics and monitoring
+ *    • Debug logs with context
+ *
+ * EXTENSIBILITY DESIGN:
+ * =====================
+ *
+ * 1. NEW CAPTURE TYPES:
+ *    • Add new type to CAPTURE_TYPES enum
+ *    • Define data format in EVENT_DATA_TYPES
+ *    • Implement source for new type
+ *    • System automatically handles storage/query
+ *
+ * 2. NEW SOURCES:
+ *    • Implement ICaptureSource interface
+ *    • Register with collector
+ *    • No changes needed to other modules
+ *
+ * 3. NEW PROCESSORS:
+ *    • Implement ICaptureProcessor interface
+ *    • Register in processing pipeline
+ *    • Can transform any event type
+ *
+ * 4. NEW STORAGE BACKENDS:
+ *    • Implement ICaptureStorage interface
+ *    • Register with system
+ *    • Query layer works automatically
+ *
+ * CHROME EXTENSION INTEGRATION:
+ * ==============================
+ *
+ * 1. MESSAGE PASSING:
+ *    • Extension uses chrome.runtime.sendMessage
+ *    • Background script receives and forwards to collector
+ *    • Efficient binary data transfer for screenshots
+ *
+ * 2. CONTENT SCRIPT COORDINATION:
+ *    • DOM events captured by content script
+ *    • User interactions tracked with event listeners
+ *    • Coordinates with background script via messaging
+ *
+ * 3. PERFORMANCE CONSIDERATIONS:
+ *    • Batch small events to reduce message overhead
+ *    • Stream large events (screenshots) efficiently
+ *    • Use transfer objects for binary data
+ *
+ * TESTING STRATEGY:
+ * =================
+ *
+ * 1. UNIT TESTING:
+ *    • Each module can be tested in isolation
+ *    • Mock interfaces for dependencies
+ *    • Test data generators for event types
+ *
+ * 2. INTEGRATION TESTING:
+ *    • Test module interactions
+ *    • End-to-end capture scenarios
+ *    • Performance testing with large datasets
+ *
+ * 3. INCREMENTAL TESTING WITH TRUSTBUTVERIFY:
+ *    • Test individual modules as developed
+ *    • Verify interfaces work correctly
+ *    • Ensure performance meets requirements
+ *
+ * MIGRATION FROM LEGACY SYSTEMS:
+ * ===============================
+ *
+ * 1. PARALLEL OPERATION:
+ *    • New system runs alongside legacy
+ *    • Compare outputs for compatibility
+ *    • Gradual migration of features
+ *
+ * 2. DATA MIGRATION:
+ *    • Convert legacy frame recordings to new format
+ *    • Convert workflow recordings to capture events
+ *    • Preserve existing analysis capabilities
+ *
+ * 3. API COMPATIBILITY:
+ *    • Maintain existing MCP tool interfaces
+ *    • New tools use unified capture format
+ *    • Legacy tools get compatibility layer
+ */
+
+/**
+ * System configuration constants
+ */
+export const ARCHITECTURE_CONFIG = {
+  // Memory management thresholds
+  MEMORY: {
+    // Maximum memory usage before triggering cleanup (MB)
+    MAX_MEMORY_MB: 100,
+    // Memory usage that triggers warnings (MB)
+    WARNING_THRESHOLD_MB: 75,
+    // Memory usage that triggers aggressive cleanup (MB)
+    CRITICAL_THRESHOLD_MB: 90,
+    // Cleanup interval in milliseconds
+    CLEANUP_INTERVAL_MS: 30000
+  },
+
+  // Event processing limits
+  EVENTS: {
+    // Maximum events in memory before flushing
+    MAX_MEMORY_EVENTS: 1000,
+    // Batch size for database writes
+    BATCH_SIZE: 100,
+    // Maximum event age in memory (ms)
+    MAX_AGE_MS: 60000,
+    // Circuit breaker failure threshold
+    ERROR_THRESHOLD: 5
+  },
+
+  // Storage configuration
+  STORAGE: {
+    // File size thresholds for storage decisions
+    SMALL_EVENT_BYTES: 1024,        // < 1KB
+    MEDIUM_EVENT_BYTES: 102400,     // < 100KB
+    LARGE_EVENT_BYTES: 1048576,     // < 1MB
+    // Database page size optimization
+    DB_PAGE_SIZE: 4096,
+    // File system chunk size
+    FS_CHUNK_SIZE: 65536
+  },
+
+  // Performance tuning
+  PERFORMANCE: {
+    // Processing pipeline worker count
+    PROCESSOR_WORKERS: 2,
+    // Storage writer worker count
+    STORAGE_WORKERS: 1,
+    // Query engine cache size (entries)
+    QUERY_CACHE_SIZE: 1000,
+    // Network timeout for remote storage
+    NETWORK_TIMEOUT_MS: 30000
+  }
+};
+
+/**
+ * Data flow validation rules
+ */
+export const FLOW_VALIDATION = {
+  // Required fields for each event type
+  REQUIRED_FIELDS: {
+    screen: ['imageData', 'format', 'width', 'height'],
+    dom: ['html'],
+    console: ['level', 'message'],
+    user: ['action', 'target'],
+    function: ['name', 'status'],
+    network: ['url', 'method'],
+    state: ['type', 'from', 'to']
+  },
+
+  // Maximum sizes for different data types
+  MAX_SIZES: {
+    // Maximum image size in bytes (10MB)
+    IMAGE_BYTES: 10485760,
+    // Maximum HTML size in bytes (1MB)
+    HTML_BYTES: 1048576,
+    // Maximum log message length
+    MESSAGE_LENGTH: 10000,
+    // Maximum function trace depth
+    TRACE_DEPTH: 100
+  },
+
+  // Event rate limits
+  RATE_LIMITS: {
+    // Maximum events per second by type
+    screen: 10,      // 10 screenshots per second max
+    dom: 5,          // 5 DOM snapshots per second max
+    console: 100,    // 100 log messages per second max
+    user: 50,        // 50 user interactions per second max
+    function: 1000,  // 1000 function calls per second max
+    network: 100,    // 100 network requests per second max
+    state: 20        // 20 state changes per second max
+  }
+};
+
+/**
+ * Error recovery strategies
+ */
+export const ERROR_RECOVERY = {
+  // Retry configuration
+  RETRY: {
+    // Maximum retry attempts
+    MAX_ATTEMPTS: 3,
+    // Initial delay in milliseconds
+    INITIAL_DELAY_MS: 1000,
+    // Backoff multiplier
+    BACKOFF_MULTIPLIER: 2,
+    // Maximum delay in milliseconds
+    MAX_DELAY_MS: 30000
+  },
+
+  // Circuit breaker configuration
+  CIRCUIT_BREAKER: {
+    // Failure count threshold
+    FAILURE_THRESHOLD: 5,
+    // Time window for failure count (ms)
+    FAILURE_WINDOW_MS: 60000,
+    // Circuit open duration (ms)
+    OPEN_DURATION_MS: 30000,
+    // Half-open test interval (ms)
+    HALF_OPEN_INTERVAL_MS: 10000
+  },
+
+  // Fallback strategies
+  FALLBACK: {
+    // Storage fallback chain
+    STORAGE_CHAIN: ['database', 'filesystem', 'memory'],
+    // Processing fallback (skip on error)
+    PROCESSING_SKIP: true,
+    // Source failure handling (continue with others)
+    SOURCE_ISOLATION: true
+  }
+};
+
+/**
+ * Module dependency graph for initialization order
+ */
+export const DEPENDENCY_GRAPH = {
+  // Modules with no dependencies (can start first)
+  LEVEL_0: [
+    'ICaptureErrorHandler',
+    'ICaptureMemoryManager'
+  ],
+
+  // Modules depending on level 0
+  LEVEL_1: [
+    'ICaptureStorage',
+    'ICaptureSource'
+  ],
+
+  // Modules depending on level 1
+  LEVEL_2: [
+    'ICaptureProcessor',
+    'ICaptureQuery'
+  ],
+
+  // Modules depending on level 2
+  LEVEL_3: [
+    'ICaptureCollector',
+    'ICaptureAnalyzer'
+  ],
+
+  // Top-level orchestrator
+  LEVEL_4: [
+    'ICaptureSystem'
+  ]
+};
+
+/**
+ * Export architecture configuration
+ */
+export default {
+  ARCHITECTURE_CONFIG,
+  FLOW_VALIDATION,
+  ERROR_RECOVERY,
+  DEPENDENCY_GRAPH
+};

package/src/capture/error-handling-tests.md

@@ -0,0 +1,38 @@
+# Error Handling System - Testing Strategy
+
+## Testing Framework Requirements
+
+### Unit Tests
+- **Storage Layer Tests:** Validate chrome.storage.local and IndexedDB operations
+- **State Machine Tests:** Verify all state transitions and actions
+- **Circuit Breaker Tests:** Test failure threshold and recovery behavior
+- **Compression Tests:** Validate data compression/decompression accuracy
+
+### Integration Tests
+- **End-to-End Error Scenarios:** Simulate network failures, server errors, storage quota exceeded
+- **Cross-Component Communication:** Test message passing between background script and content scripts
+- **Backup/Recovery Workflows:** Validate complete backup creation and restoration processes
+
+### Performance Tests
+- **Memory Usage:** Monitor memory consumption during error scenarios
+- **Storage Efficiency:** Measure compression ratios and storage utilization
+- **Response Times:** Validate error handling doesn't impact normal operation performance
+
+### Error Simulation Framework
+```javascript
+const ERROR_SIMULATION = {
+  networkFailure: () => { /* Mock network timeout */ },
+  serverError: () => { /* Mock HTTP 500 responses */ },
+  storageQuotaExceeded: () => { /* Mock storage quota errors */ },
+  browserCrash: () => { /* Simulate tab close events */ }
+};
+```
+
+### Validation Checklist
+- [ ] All data protection tiers function correctly
+- [ ] State machine handles all defined transitions
+- [ ] Circuit breaker prevents cascading failures
+- [ ] Backup/recovery processes are reliable
+- [ ] User notifications work across all scenarios
+- [ ] Performance impact is within acceptable limits
+- [ ] Memory leaks are prevented during extended error periods