agentgui 1.0.38 → 1.0.40

package/static/sync-manager.js

@@ -0,0 +1,273 @@
+/**
+ * Sync Manager - Handles real-time synchronization with automatic reconnection
+ * Guarantees: No lost data, perfect recovery, consistent state
+ */
+class SyncManager {
+  constructor() {
+    this.ws = null;
+    this.clientId = null;
+    this.subscriptions = new Map();
+    this.reconnectAttempts = 0;
+    this.maxReconnectAttempts = 10;
+    this.reconnectDelay = 1000;
+    this.isConnected = false;
+    this.handlers = new Map();
+    this.lastCheckpoint = new Map();
+  }
+
+  /**
+   * Connect to sync server with automatic reconnection
+   */
+  connect() {
+    return new Promise((resolve, reject) => {
+      const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
+      const url = `${protocol}//${window.location.host}${window.__BASE_URL || '/gm'}/sync`;
+
+      try {
+        this.ws = new WebSocket(url);
+
+        this.ws.onopen = () => {
+          console.log('[SyncManager] Connected to server');
+          this.isConnected = true;
+          this.reconnectAttempts = 0;
+          this.emit('connected', { clientId: this.clientId });
+
+          // Resubscribe to all previously subscribed sessions
+          for (const [sessionId, handlers] of this.subscriptions) {
+            this.subscribe(sessionId, handlers.onUpdate, handlers.onRecover);
+          }
+
+          resolve();
+        };
+
+        this.ws.onmessage = (event) => {
+          this.handleMessage(JSON.parse(event.data));
+        };
+
+        this.ws.onclose = () => {
+          console.log('[SyncManager] Disconnected from server');
+          this.isConnected = false;
+          this.attemptReconnect();
+        };
+
+        this.ws.onerror = (error) => {
+          console.error('[SyncManager] WebSocket error:', error);
+          reject(error);
+        };
+      } catch (err) {
+        console.error('[SyncManager] Failed to create WebSocket:', err);
+        reject(err);
+      }
+    });
+  }
+
+  /**
+   * Handle incoming messages
+   */
+  handleMessage(message) {
+    const { type, sessionId, clientId } = message;
+
+    if (type === 'sync_connected') {
+      this.clientId = message.clientId;
+      console.log(`[SyncManager] Assigned client ID: ${this.clientId}`);
+    } else if (type === 'state_snapshot') {
+      // Received state after subscription
+      console.log(`[SyncManager] Received state snapshot for ${sessionId}`);
+      this.lastCheckpoint.set(sessionId, message.state.checkpoint);
+
+      const handlers = this.subscriptions.get(sessionId);
+      if (handlers?.onRecover) {
+        handlers.onRecover(message.state);
+      }
+    } else if (type === 'recovery_response') {
+      // Received full state recovery
+      console.log(`[SyncManager] Received recovery response for ${sessionId}`);
+      this.lastCheckpoint.set(sessionId, message.state.checkpoint);
+
+      const handlers = this.subscriptions.get(sessionId);
+      if (handlers?.onRecover) {
+        handlers.onRecover(message.state);
+      }
+    } else if (type === 'stream_update') {
+      // Real-time update from server
+      this.lastCheckpoint.set(sessionId, message.timestamp);
+
+      const handlers = this.subscriptions.get(sessionId);
+      if (handlers?.onUpdate) {
+        try {
+          handlers.onUpdate(message);
+        } catch (err) {
+          console.error(`[SyncManager] Error in update handler: ${err.message}`);
+        }
+      }
+    }
+  }
+
+  /**
+   * Subscribe to session updates with callbacks
+   * @param {string} sessionId
+   * @param {Function} onUpdate - Called for each real-time update
+   * @param {Function} onRecover - Called with full state on subscribe/reconnect
+   */
+  subscribe(sessionId, onUpdate, onRecover) {
+    if (!this.subscriptions.has(sessionId)) {
+      this.subscriptions.set(sessionId, { onUpdate, onRecover });
+    }
+
+    if (this.isConnected && this.ws?.readyState === WebSocket.OPEN) {
+      this.ws.send(JSON.stringify({
+        type: 'subscribe',
+        sessionId
+      }));
+    }
+  }
+
+  /**
+   * Unsubscribe from session
+   */
+  unsubscribe(sessionId) {
+    this.subscriptions.delete(sessionId);
+    this.lastCheckpoint.delete(sessionId);
+
+    if (this.isConnected && this.ws?.readyState === WebSocket.OPEN) {
+      this.ws.send(JSON.stringify({
+        type: 'unsubscribe',
+        sessionId
+      }));
+    }
+  }
+
+  /**
+   * Request recovery from a specific checkpoint
+   * Called when client detects missing data
+   */
+  requestRecovery(sessionId) {
+    console.log(`[SyncManager] Requesting recovery for ${sessionId}`);
+
+    if (this.isConnected && this.ws?.readyState === WebSocket.OPEN) {
+      this.ws.send(JSON.stringify({
+        type: 'recovery_request',
+        sessionId
+      }));
+    } else {
+      // If not connected, recover when connection is restored
+      this.connect().then(() => {
+        this.ws.send(JSON.stringify({
+          type: 'recovery_request',
+          sessionId
+        }));
+      });
+    }
+  }
+
+  /**
+   * Verify data consistency by querying server
+   */
+  async validateSession(sessionId) {
+    const baseUrl = window.__BASE_URL || '/gm';
+    try {
+      const response = await fetch(`${baseUrl}/api/sessions/${sessionId}/validate`);
+      const validation = await response.json();
+      return validation;
+    } catch (err) {
+      console.error(`[SyncManager] Validation failed: ${err.message}`);
+      return null;
+    }
+  }
+
+  /**
+   * Fetch full state for recovery
+   */
+  async fetchSessionState(sessionId) {
+    const baseUrl = window.__BASE_URL || '/gm';
+    try {
+      const response = await fetch(`${baseUrl}/api/sessions/${sessionId}/state-recovery`);
+      if (!response.ok) return null;
+      return await response.json();
+    } catch (err) {
+      console.error(`[SyncManager] Failed to fetch session state: ${err.message}`);
+      return null;
+    }
+  }
+
+  /**
+   * Automatic reconnection with exponential backoff
+   */
+  attemptReconnect() {
+    if (this.reconnectAttempts >= this.maxReconnectAttempts) {
+      console.error('[SyncManager] Max reconnection attempts reached');
+      this.emit('reconnect_failed');
+      return;
+    }
+
+    this.reconnectAttempts++;
+    const delay = this.reconnectDelay * Math.pow(2, this.reconnectAttempts - 1);
+    console.log(`[SyncManager] Reconnecting in ${delay}ms (attempt ${this.reconnectAttempts}/${this.maxReconnectAttempts})`);
+
+    setTimeout(() => {
+      this.connect().catch(err => {
+        console.error('[SyncManager] Reconnection failed:', err);
+        this.attemptReconnect();
+      });
+    }, delay);
+  }
+
+  /**
+   * Detect missing updates by checking sequence gaps
+   */
+  detectMissingUpdates(updates) {
+    const gaps = [];
+    for (let i = 0; i < updates.length - 1; i++) {
+      if (updates[i + 1].sequence !== updates[i].sequence + 1) {
+        gaps.push({
+          expected: updates[i].sequence + 1,
+          actual: updates[i + 1].sequence
+        });
+      }
+    }
+    return gaps;
+  }
+
+  /**
+   * Register event listener
+   */
+  on(event, callback) {
+    if (!this.handlers.has(event)) {
+      this.handlers.set(event, []);
+    }
+    this.handlers.get(event).push(callback);
+  }
+
+  /**
+   * Emit event
+   */
+  emit(event, data) {
+    const callbacks = this.handlers.get(event) || [];
+    for (const callback of callbacks) {
+      try {
+        callback(data);
+      } catch (err) {
+        console.error(`[SyncManager] Error in ${event} handler: ${err.message}`);
+      }
+    }
+  }
+
+  /**
+   * Close connection gracefully
+   */
+  disconnect() {
+    this.subscriptions.clear();
+    this.lastCheckpoint.clear();
+    if (this.ws) {
+      this.ws.close();
+      this.ws = null;
+    }
+  }
+}
+
+// Export as global for browser use
+if (typeof window !== 'undefined') {
+  window.SyncManager = SyncManager;
+}
+
+export default SyncManager;

package/stream-handler.js

@@ -0,0 +1,101 @@
+import { queries } from './database.js';
+import { StateValidator } from './state-validator.js';
+
+export class StreamHandler {
+  constructor(sessionId, conversationId, broadcastFn) {
+    this.sessionId = sessionId;
+    this.conversationId = conversationId;
+    this.broadcastFn = broadcastFn;
+    this.updateCount = 0;
+    this.sequence = -1;
+    this.hasText = false;
+    this.hasBlocks = false;
+    this.blocks = [];
+    this.stateCheckpoint = StateValidator.getSessionState(sessionId);
+  }
+
+  handleUpdate(params, baseUrl) {
+    const u = params.update;
+    if (!u) return;
+
+    const kind = u.sessionUpdate;
+    if (kind === 'agent_message_chunk' && u.content?.text) {
+      this.hasText = true;
+      const update = {
+        type: 'text',
+        content: u.content.text,
+        timestamp: Date.now()
+      };
+      this.persistAndBroadcast('text', update, baseUrl);
+    } else if (kind === 'html_content' && u.content?.html) {
+      this.hasBlocks = true;
+      const update = {
+        type: 'html',
+        html: u.content.html,
+        title: u.content.title,
+        id: u.content.id,
+        timestamp: Date.now()
+      };
+      this.blocks.push({ type: 'html', html: u.content.html, title: u.content.title, id: u.content.id });
+      this.persistAndBroadcast('html', update, baseUrl);
+    } else if (kind === 'image_content' && u.content?.path) {
+      this.hasBlocks = true;
+      const imageUrl = baseUrl + '/api/image/' + encodeURIComponent(u.content.path);
+      const update = {
+        type: 'image',
+        path: u.content.path,
+        url: imageUrl,
+        title: u.content.title,
+        alt: u.content.alt,
+        timestamp: Date.now()
+      };
+      this.blocks.push({ type: 'image', path: u.content.path, url: imageUrl, title: u.content.title, alt: u.content.alt });
+      this.persistAndBroadcast('image', update, baseUrl);
+    }
+  }
+
+  persistAndBroadcast(updateType, update, baseUrl) {
+    try {
+      // CRITICAL: Database write MUST complete before broadcast
+      // This guarantees database is source of truth
+      const persistedUpdate = queries.createStreamUpdate(this.sessionId, this.conversationId, updateType, update);
+      this.sequence = persistedUpdate.sequence;
+      this.updateCount++;
+
+      // Validate consistency after write
+      const validation = StateValidator.validateSession(this.sessionId);
+      if (!validation.valid) {
+        console.error(`[StreamHandler] State validation failed after update:`, validation);
+        // Log but continue - database is still source of truth
+      }
+
+      // CRITICAL: Broadcast happens AFTER database write confirms
+      // This ensures clients see data that's already persisted
+      this.broadcastFn({
+        type: 'stream_update',
+        sessionId: this.sessionId,
+        conversationId: this.conversationId,
+        updateType,
+        update: persistedUpdate.content,
+        sequence: this.sequence,
+        persisted: true,
+        timestamp: persistedUpdate.created_at,
+        validation: validation.valid ? undefined : { error: validation.error }
+      });
+    } catch (err) {
+      console.error(`[StreamHandler] Error persisting update: ${err.message}`);
+      // On persistence failure, do NOT broadcast - maintain consistency
+      throw err;
+    }
+  }
+
+  getBlocks() {
+    return this.blocks;
+  }
+
+  getUpdateCount() {
+    return this.updateCount;
+  }
+}
+
+export default StreamHandler;

package/AUTOMATIC_IMPORT.md

@@ -1,284 +0,0 @@
-# Automatic Continuous Importing Feature
-
-## Overview
-AgentGUI now automatically and continuously imports Claude Code conversations **without requiring any user action**. This ensures conversations are always available and up-to-date.
-
-## How It Works
-
-### Server-Side (Every 30 Seconds)
-```
-Server startup
-    ↓
-[IMMEDIATE] Import Claude Code conversations (first run)
-    ↓
-[EVERY 30 SECONDS] Auto-import new conversations
-    ↓
-If new conversations found:
-  • Add them to database
-  • Broadcast 'conversations_updated' event to all connected clients
-  • Log: "[AUTO-IMPORT] Imported X new Claude Code conversations"
-```
-
-### Frontend-Side
-```
-Page loads
-    ↓
-[IMMEDIATE] Fetch conversations from API
-    ↓
-[EVERY 10 SECONDS] Fetch conversations again (as fallback)
-    ↓
-[ON SYNC EVENT] Receive 'conversations_updated' from server
-    ↓
-Immediately refresh conversation list
-    ↓
-Users see new conversations appear in real-time
-```
-
-## Key Features
-
-✅ **Automatic**: No user action needed  
-✅ **Continuous**: Runs every 30 seconds (server) and 10 seconds (client)  
-✅ **Real-time**: New conversations appear immediately via WebSocket  
-✅ **No Duplicates**: Skips conversations already imported  
-✅ **Cross-tab**: Broadcasts via BroadcastChannel API  
-✅ **Resilient**: Fallback mechanism if WebSocket fails  
-✅ **Logging**: All imports logged for debugging  
-
-## Where Conversations Come From
-
-### Automatically Discovered From:
-1. **Claude Code Projects** (~/.claude/projects/)
-   - Scans sessions-index.json files
-   - Reads .jsonl message files
-   - Imports with "[project] title" format
-
-2. **Created in AgentGUI**
-   - New conversations created via UI
-   - Automatically stored in database
-
-## Example Flow
-
-### Scenario: User Uses Claude Code, Then Opens AgentGUI
-
-```
-11:00:00 - User creates conversation in Claude Code
-11:00:15 - AgentGUI server detects new conversation in ~/.claude/projects/
-11:00:20 - Server imports conversation automatically
-11:00:20 - Server broadcasts 'conversations_updated' event
-11:00:21 - All connected browser tabs receive update
-11:00:21 - Users see new conversation appear in sidebar
-```
-
-### Scenario: Multiple Tabs Open
-
-```
-Tab 1 opens AgentGUI
-Tab 2 opens AgentGUI (few seconds later)
-
-Tab 1 receives 'conversations_updated' from server
-Tab 1 uses BroadcastChannel to notify Tab 2
-Tab 2 also refreshes conversation list
-Both tabs show latest conversations in sync
-```
-
-## Server Logs
-
-You'll see logs like:
-```
-[AUTO-IMPORT] Imported 2 new Claude Code conversations (42 already exist)
-[AUTO-IMPORT] Imported 1 new Claude Code conversation (43 already exist)
-[AUTO-IMPORT] (nothing new this cycle)
-```
-
-## Client Logs
-
-In browser console:
-```
-[SYNC] Server imported 3 new conversations, refreshing...
-[DEBUG] Init: Auto-imported Claude Code conversations
-[DEBUG] Loaded conversations, total: 86
-```
-
-## Configuration
-
-### Import Frequency (Server)
-Current: **30 seconds**
-Location: `server.js` line `setInterval(performAutoImport, 30000);`
-
-To change:
-```javascript
-setInterval(performAutoImport, 60000); // 60 seconds
-setInterval(performAutoImport, 5000);  // 5 seconds
-```
-
-### Refresh Frequency (Client)
-Current: **10 seconds**
-Location: `app.js` line `setInterval(() => { ... }, 10000);`
-
-To change:
-```javascript
-}, 60000);  // 60 seconds
-}, 5000);   // 5 seconds
-```
-
-## Data Flow Diagram
-
-```
-┌─────────────────────────────────────────────────────┐
-│ Claude Code                                          │
-│ ~/.claude/projects/*/sessions-index.json            │
-└─────────────────┬───────────────────────────────────┘
-                  │ (monitors every 30s)
-                  ↓
-┌─────────────────────────────────────────────────────┐
-│ AgentGUI Server                                      │
-│ • queries.importClaudeCodeConversations()           │
-│ • Stores in ~/.gmgui/data.db                        │
-│ • Broadcasts via WebSocket                          │
-└──────┬──────────────────────────┬────────────────────┘
-       │                          │
-       │ (WebSocket event)        │ (HTTP API)
-       ↓                          ↓
-┌─────────────────────────────────────────────────────┐
-│ Browser (Frontend)                                   │
-│ • Listens on sync WebSocket                         │
-│ • Receives 'conversations_updated' event            │
-│ • Calls fetchConversations()                        │
-│ • Calls renderChatHistory()                         │
-└──────┬──────────────────────────────────────────────┘
-       │
-       ↓
-┌─────────────────────────────────────────────────────┐
-│ Chat Sidebar                                         │
-│ • Displays conversation list                        │
-│ • User can click to view conversation               │
-└─────────────────────────────────────────────────────┘
-```
-
-## Troubleshooting
-
-### Conversations Not Appearing
-
-**Check 1: Server Auto-Import**
-```bash
-# Look for these logs in server output
-grep "\[AUTO-IMPORT\]" server.log
-
-# If no logs, auto-import might not be running
-```
-
-**Check 2: Claude Code Availability**
-```bash
-# Check if Claude Code projects exist
-ls -la ~/.claude/projects/
-
-# Count projects with conversations
-find ~/.claude/projects -name "sessions-index.json" | wc -l
-```
-
-**Check 3: Browser Sync**
-```javascript
-// In browser console
-// Check if WebSocket is connected
-console.log('WebSocket state:', app.syncWs.ws?.readyState);
-
-// Try manual refresh
-await app.fetchConversations();
-app.renderChatHistory();
-```
-
-**Check 4: Database**
-```bash
-# Check total conversations in DB
-node -e "
-const DB = require('better-sqlite3');
-const db = new DB(process.env.HOME + '/.gmgui/data.db');
-const count = db.prepare('SELECT COUNT(*) as c FROM conversations').get();
-console.log('Database has:', count.c, 'conversations');
-db.close();
-"
-```
-
-### Too Many Import Logs
-
-If server logs are too verbose:
-1. Increase `setInterval` time (30000 → 60000 or more)
-2. Add log level filtering
-
-### Conversations Take Too Long to Appear
-
-If new conversations take > 1 minute:
-1. Check server auto-import interval (default 30s)
-2. Check client refresh interval (default 10s)
-3. Check network connectivity
-4. Check browser console for errors
-
-## Performance Considerations
-
-### Import Impact
-- **Minimal**: Import skips existing conversations
-- **Fast**: Only processes new conversations
-- **Efficient**: Uses database transactions
-
-### Client Impact
-- **WebSocket**: Real-time updates, low bandwidth
-- **Polling**: Every 10 seconds, minimal traffic
-- **Rendering**: Only updates when conversations change
-
-## Security Notes
-
-- Only imports from user's local `.claude/projects/`
-- No external network access needed
-- All conversations stored locally
-- Respects filesystem permissions
-
-## Future Enhancements
-
-Potential improvements:
-- Configurable import interval via UI
-- Import from multiple sources
-- Batch import optimization
-- Import history/logs viewer
-- Import statistics dashboard
-- Per-project import settings
-
-## Testing
-
-### Manual Test: Add Claude Code Conversation
-```bash
-# 1. Use Claude Code (creates ~/.claude/projects/*/sessions-index.json)
-# 2. Wait up to 30 seconds
-# 3. Check browser - should see new conversation appear
-# 4. Confirm console logs show "[AUTO-IMPORT] Imported X..."
-```
-
-### Manual Test: Force Import
-```javascript
-// In browser console
-await fetch('/gm/api/import/claude-code')
-  .then(r => r.json())
-  .then(d => console.log('Manual import result:', d));
-
-// Then refresh
-await app.fetchConversations();
-app.renderChatHistory();
-```
-
-## Related Files
-
-- `server.js` - Server auto-import implementation
-- `app.js` - Frontend sync handling
-- `database.js` - Query functions
-- `acp-launcher.js` - Claude Code connection
-
-## Changelog
-
-### Version 1.1.0 (Current)
-- ✅ Added automatic server-side importing every 30 seconds
-- ✅ Added WebSocket broadcast for instant updates
-- ✅ Added client-side sync event handler
-- ✅ Integrated with existing periodic sync
-
-### Version 1.0.0 (Previous)
-- Manual import on demand via `/api/import/claude-code`
-- No automatic background importing