draft-board 0.1.0-beta.0

package/app/backend/app/services/log_normalizer.py

@@ -0,0 +1,342 @@
+"""Service for parsing raw agent logs into structured entries."""
+
+import difflib
+import re
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.normalized_log import LogEntryType, NormalizedLogEntry
+
+
+@dataclass
+class ParsedEntry:
+    """Temporary structure before DB insert."""
+
+    entry_type: LogEntryType
+    content: str
+    metadata: dict[str, Any]
+    timestamp: datetime | None = None
+
+
+class ClaudeLogParser:
+    """Parser for Claude Code CLI output format."""
+
+    # Regex patterns for Claude's output format
+    THINKING_PATTERN = re.compile(r"<thinking>(.*?)</thinking>", re.DOTALL)
+    TOOL_USE_PATTERN = re.compile(r"<tool_use>(.*?)</tool_use>", re.DOTALL)
+    FILE_EDIT_PATTERN = re.compile(
+        r"<file_edit>\s*<path>(.*?)</path>\s*<content>(.*?)</content>\s*</file_edit>",
+        re.DOTALL,
+    )
+    FILE_CREATE_PATTERN = re.compile(
+        r"<file_create>\s*<path>(.*?)</path>\s*<content>(.*?)</content>\s*</file_create>",
+        re.DOTALL,
+    )
+    FILE_DELETE_PATTERN = re.compile(
+        r"<file_delete>\s*<path>(.*?)</path>\s*</file_delete>", re.DOTALL
+    )
+    COMMAND_PATTERN = re.compile(r"<command>(.*?)</command>", re.DOTALL)
+    ERROR_PATTERN = re.compile(r"Error:(.*?)(?=\n\n|\Z)", re.DOTALL)
+
+    def parse(self, raw_log: str) -> list[ParsedEntry]:
+        """Parse raw log into structured entries."""
+        entries: list[ParsedEntry] = []
+        position = 0
+
+        # Split log by major sections
+        while position < len(raw_log):
+            # Try to match patterns in order of priority
+
+            # 1. Tool use (contains file edits, commands, etc.)
+            tool_match = self.TOOL_USE_PATTERN.search(raw_log, position)
+            if tool_match and tool_match.start() == position:
+                tool_content = tool_match.group(1)
+                entries.extend(self._parse_tool_use(tool_content))
+                position = tool_match.end()
+                continue
+
+            # 2. Thinking blocks
+            thinking_match = self.THINKING_PATTERN.search(raw_log, position)
+            if thinking_match and thinking_match.start() == position:
+                thinking = thinking_match.group(1).strip()
+                entries.append(
+                    ParsedEntry(
+                        entry_type=LogEntryType.THINKING,
+                        content=thinking,
+                        metadata={"collapsed": True},  # Start collapsed
+                    )
+                )
+                position = thinking_match.end()
+                continue
+
+            # 3. Errors
+            error_match = self.ERROR_PATTERN.search(raw_log, position)
+            if error_match and error_match.start() == position:
+                error_text = error_match.group(1).strip()
+                entries.append(
+                    ParsedEntry(
+                        entry_type=LogEntryType.ERROR,
+                        content=error_text,
+                        metadata={"highlight": True},
+                    )
+                )
+                position = error_match.end()
+                continue
+
+            # 4. Plain text (system messages, outputs, etc.)
+            next_special = self._find_next_special(raw_log, position)
+            if next_special > position:
+                plain_text = raw_log[position:next_special].strip()
+                if plain_text:
+                    entries.append(
+                        ParsedEntry(
+                            entry_type=LogEntryType.SYSTEM_MESSAGE,
+                            content=plain_text,
+                            metadata={},
+                        )
+                    )
+                position = next_special
+            else:
+                # No more special patterns, capture remaining text
+                remaining = raw_log[position:].strip()
+                if remaining:
+                    entries.append(
+                        ParsedEntry(
+                            entry_type=LogEntryType.SYSTEM_MESSAGE,
+                            content=remaining,
+                            metadata={},
+                        )
+                    )
+                break
+
+        return entries
+
+    def _parse_tool_use(self, tool_content: str) -> list[ParsedEntry]:
+        """Parse tool use block (may contain multiple operations)."""
+        entries: list[ParsedEntry] = []
+
+        # File edits
+        for match in self.FILE_EDIT_PATTERN.finditer(tool_content):
+            file_path = match.group(1).strip()
+            content = match.group(2).strip()
+
+            # Generate diff if we have original file
+            diff = self._generate_diff(file_path, content)
+
+            entries.append(
+                ParsedEntry(
+                    entry_type=LogEntryType.FILE_EDIT,
+                    content=f"Edited {file_path}",
+                    metadata={
+                        "file_path": file_path,
+                        "new_content": content,
+                        "diff": diff,
+                        "language": self._detect_language(file_path),
+                    },
+                )
+            )
+
+        # File creates
+        for match in self.FILE_CREATE_PATTERN.finditer(tool_content):
+            file_path = match.group(1).strip()
+            content = match.group(2).strip()
+
+            entries.append(
+                ParsedEntry(
+                    entry_type=LogEntryType.FILE_CREATE,
+                    content=f"Created {file_path}",
+                    metadata={
+                        "file_path": file_path,
+                        "new_content": content,
+                        "language": self._detect_language(file_path),
+                    },
+                )
+            )
+
+        # File deletes
+        for match in self.FILE_DELETE_PATTERN.finditer(tool_content):
+            file_path = match.group(1).strip()
+
+            entries.append(
+                ParsedEntry(
+                    entry_type=LogEntryType.FILE_DELETE,
+                    content=f"Deleted {file_path}",
+                    metadata={"file_path": file_path},
+                )
+            )
+
+        # Commands
+        for match in self.COMMAND_PATTERN.finditer(tool_content):
+            command = match.group(1).strip()
+
+            # Try to extract result if present
+            result_pattern = re.compile(
+                rf"<command>{re.escape(command)}</command>.*?<result>(.*?)</result>",
+                re.DOTALL,
+            )
+            result_match = result_pattern.search(tool_content)
+            output = result_match.group(1).strip() if result_match else None
+
+            # Simple heuristic for exit code
+            exit_code = (
+                0 if output and "error" not in output.lower() else 1 if output else 0
+            )
+
+            entries.append(
+                ParsedEntry(
+                    entry_type=LogEntryType.COMMAND_RUN,
+                    content=command,
+                    metadata={
+                        "command": command,
+                        "output": output,
+                        "exit_code": exit_code,
+                    },
+                )
+            )
+
+        return entries
+
+    def _generate_diff(self, file_path: str, new_content: str) -> str | None:
+        """Generate unified diff if original file exists."""
+        try:
+            path = Path(file_path)
+            if not path.exists():
+                return None
+
+            original = path.read_text()
+            diff_lines = difflib.unified_diff(
+                original.splitlines(keepends=True),
+                new_content.splitlines(keepends=True),
+                fromfile=f"a/{file_path}",
+                tofile=f"b/{file_path}",
+                lineterm="",
+            )
+            return "".join(diff_lines)
+        except Exception:
+            return None
+
+    def _detect_language(self, file_path: str) -> str:
+        """Detect programming language from file extension."""
+        ext_map = {
+            ".py": "python",
+            ".js": "javascript",
+            ".ts": "typescript",
+            ".jsx": "javascript",
+            ".tsx": "typescript",
+            ".rs": "rust",
+            ".go": "go",
+            ".java": "java",
+            ".cpp": "cpp",
+            ".c": "c",
+            ".rb": "ruby",
+            ".php": "php",
+            ".sql": "sql",
+            ".sh": "bash",
+            ".yaml": "yaml",
+            ".yml": "yaml",
+            ".json": "json",
+            ".md": "markdown",
+        }
+        ext = Path(file_path).suffix.lower()
+        return ext_map.get(ext, "text")
+
+    def _find_next_special(self, text: str, start: int) -> int:
+        """Find the next position of any special pattern."""
+        patterns = [self.THINKING_PATTERN, self.TOOL_USE_PATTERN, self.ERROR_PATTERN]
+
+        min_pos = len(text)
+        for pattern in patterns:
+            match = pattern.search(text, start)
+            if match:
+                min_pos = min(min_pos, match.start())
+
+        return min_pos
+
+
+class CursorLogParser:
+    """Parser for Cursor Agent CLI output."""
+
+    def parse(self, raw_log: str) -> list[ParsedEntry]:
+        """Parse Cursor Agent output (similar structure to Claude)."""
+        # For now, use similar parsing logic as Claude
+        # Can be customized later for Cursor-specific format
+        return ClaudeLogParser().parse(raw_log)
+
+
+class LogNormalizerService:
+    """Main service for log normalization."""
+
+    def __init__(self):
+        self.parsers = {
+            "claude": ClaudeLogParser(),
+            "cursor": CursorLogParser(),
+            # Add more as needed
+        }
+
+    async def normalize_and_store(
+        self,
+        db: AsyncSession,
+        job_id: str,
+        raw_log: str,
+        agent_type: str = "claude",
+    ) -> list[NormalizedLogEntry]:
+        """
+        Parse raw log and store normalized entries.
+
+        Args:
+            db: Database session
+            job_id: The job ID
+            raw_log: Raw log content
+            agent_type: Type of agent (determines parser)
+
+        Returns:
+            List of created NormalizedLogEntry objects
+        """
+        parser = self.parsers.get(agent_type)
+        if not parser:
+            # Fallback to Claude parser
+            parser = self.parsers["claude"]
+
+        # Parse
+        parsed_entries = parser.parse(raw_log)
+
+        # Store in DB
+        db_entries = []
+        for i, parsed in enumerate(parsed_entries):
+            entry = NormalizedLogEntry(
+                job_id=job_id,
+                sequence=i,
+                entry_type=parsed.entry_type,
+                content=parsed.content,
+                entry_metadata=parsed.metadata,
+                timestamp=parsed.timestamp or datetime.utcnow(),
+                collapsed=parsed.metadata.get("collapsed", False),
+                highlight=parsed.metadata.get("highlight", False),
+            )
+            db.add(entry)
+            db_entries.append(entry)
+
+        await db.commit()
+
+        # Refresh to get IDs
+        for entry in db_entries:
+            await db.refresh(entry)
+
+        return db_entries
+
+    async def get_normalized_logs(
+        self, db: AsyncSession, job_id: str
+    ) -> list[NormalizedLogEntry]:
+        """Retrieve all normalized logs for a job."""
+        from sqlalchemy import select
+
+        result = await db.execute(
+            select(NormalizedLogEntry)
+            .where(NormalizedLogEntry.job_id == job_id)
+            .order_by(NormalizedLogEntry.sequence)
+        )
+        return list(result.scalars().all())

package/app/backend/app/services/log_stream_service.py

@@ -0,0 +1,276 @@
+"""Real-time log streaming service with ultra-low latency.
+
+Uses in-memory broadcast for same-process subscribers (<1ms latency).
+Worker runs in-process, so cross-process communication is not needed.
+"""
+
+import asyncio
+import logging
+import threading
+import time
+from collections import defaultdict
+from collections.abc import AsyncIterator
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from enum import StrEnum
+from weakref import WeakSet
+
+logger = logging.getLogger(__name__)
+
+
+class LogLevel(StrEnum):
+    """Log message levels."""
+
+    STDOUT = "stdout"
+    STDERR = "stderr"
+    INFO = "info"
+    ERROR = "error"
+    PROGRESS = "progress"  # Progress updates
+    NORMALIZED = "normalized"  # Normalized agent log entry (parsed JSON)
+    FINISHED = "finished"
+
+
+@dataclass
+class LogMessage:
+    """A single log message with optional metadata."""
+
+    level: LogLevel
+    content: str
+    timestamp: datetime = field(default_factory=lambda: datetime.now(UTC))
+    # Optional progress metadata
+    progress_pct: int | None = None  # 0-100
+    stage: str | None = None  # e.g., "parsing", "generating", "applying"
+
+    def to_dict(self) -> dict:
+        """Serialize for Redis/JSON."""
+        d = {
+            "level": self.level.value,
+            "content": self.content,
+            "timestamp": self.timestamp.isoformat(),
+        }
+        if self.progress_pct is not None:
+            d["progress_pct"] = self.progress_pct
+        if self.stage:
+            d["stage"] = self.stage
+        return d
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "LogMessage":
+        """Deserialize from Redis/JSON."""
+        return cls(
+            level=LogLevel(data["level"]),
+            content=data["content"],
+            timestamp=datetime.fromisoformat(data["timestamp"]),
+            progress_pct=data.get("progress_pct"),
+            stage=data.get("stage"),
+        )
+
+
+def _get_channel_name(job_id: str) -> str:
+    return f"job_logs:{job_id}"
+
+
+def _get_history_key(job_id: str) -> str:
+    return f"job_logs_history:{job_id}"
+
+
+class InMemoryBroadcaster:
+    """In-memory pub/sub for same-process subscribers.
+
+    Provides <1ms latency for local subscribers.
+    Thread-safe for worker threads.
+    """
+
+    # History is retained for 5 minutes after a job finishes, then cleaned up.
+    _HISTORY_RETENTION_SECONDS = 300
+    # Run stale history cleanup every N pushes to amortize cost.
+    _CLEANUP_INTERVAL_PUSHES = 100
+
+    def __init__(self):
+        self._lock = threading.Lock()
+        # job_id -> set of async queues
+        self._subscribers: dict[str, WeakSet[asyncio.Queue]] = defaultdict(WeakSet)
+        # job_id -> list of messages (for history/catch-up)
+        self._history: dict[str, list[LogMessage]] = defaultdict(list)
+        self._max_history = 500
+        # job_id -> monotonic timestamp when the job finished
+        self._finished_at: dict[str, float] = {}
+        # Counter for amortized cleanup
+        self._push_count: int = 0
+
+    def push(self, job_id: str, msg: LogMessage) -> None:
+        """Push message to all local subscribers (thread-safe)."""
+        with self._lock:
+            # Store in history
+            history = self._history[job_id]
+            history.append(msg)
+            if len(history) > self._max_history:
+                self._history[job_id] = history[-self._max_history :]
+
+            # Broadcast to subscribers
+            subscribers = self._subscribers.get(job_id, set())
+            for queue in list(subscribers):
+                try:
+                    queue.put_nowait(msg)
+                except asyncio.QueueFull:
+                    pass  # Drop if subscriber is slow
+
+            # Periodic cleanup of stale history
+            self._push_count += 1
+            if self._push_count >= self._CLEANUP_INTERVAL_PUSHES:
+                self._push_count = 0
+                self._cleanup_stale_history()
+
+    def subscribe(self, job_id: str) -> asyncio.Queue:
+        """Create a subscription queue for a job."""
+        queue: asyncio.Queue = asyncio.Queue(maxsize=1000)
+        with self._lock:
+            self._subscribers[job_id].add(queue)
+        return queue
+
+    def unsubscribe(self, job_id: str, queue: asyncio.Queue) -> None:
+        """Remove a subscription."""
+        with self._lock:
+            if job_id in self._subscribers:
+                self._subscribers[job_id].discard(queue)
+
+    def get_history(self, job_id: str) -> list[LogMessage]:
+        """Get message history for catch-up."""
+        with self._lock:
+            return list(self._history.get(job_id, []))
+
+    def cleanup(self, job_id: str) -> None:
+        """Clean up subscribers and schedule history for deferred removal."""
+        with self._lock:
+            self._subscribers.pop(job_id, None)
+            # Mark job as finished so history is cleaned up after retention period
+            self._finished_at[job_id] = time.monotonic()
+
+    def _cleanup_stale_history(self) -> None:
+        """Remove history entries whose jobs finished more than retention seconds ago.
+
+        Must be called while self._lock is held.
+        """
+        now = time.monotonic()
+        stale_job_ids = [
+            job_id
+            for job_id, finished_ts in self._finished_at.items()
+            if (now - finished_ts) >= self._HISTORY_RETENTION_SECONDS
+        ]
+        for job_id in stale_job_ids:
+            self._history.pop(job_id, None)
+            del self._finished_at[job_id]
+
+
+# Global in-memory broadcaster
+_broadcaster = InMemoryBroadcaster()
+
+
+class LogStreamPublisher:
+    """In-memory log publisher for same-process workers.
+
+    Used by workers to push logs to subscribers via InMemoryBroadcaster.
+    """
+
+    def push(
+        self,
+        job_id: str,
+        level: LogLevel,
+        content: str,
+        progress_pct: int | None = None,
+        stage: str | None = None,
+    ) -> None:
+        """Push a log message via in-memory broadcast (<1ms)."""
+        msg = LogMessage(
+            level=level,
+            content=content,
+            progress_pct=progress_pct,
+            stage=stage,
+        )
+        _broadcaster.push(job_id, msg)
+
+    def flush_all(self, job_id: str) -> None:
+        """No-op — in-memory broadcast is instant."""
+        pass
+
+    def push_stdout(self, job_id: str, content: str) -> None:
+        self.push(job_id, LogLevel.STDOUT, content)
+
+    def push_stderr(self, job_id: str, content: str) -> None:
+        self.push(job_id, LogLevel.STDERR, content)
+
+    def push_info(self, job_id: str, content: str) -> None:
+        self.push(job_id, LogLevel.INFO, content)
+
+    def push_error(self, job_id: str, content: str) -> None:
+        self.push(job_id, LogLevel.ERROR, content)
+
+    def push_progress(
+        self, job_id: str, pct: int, stage: str, content: str = ""
+    ) -> None:
+        """Push a progress update."""
+        self.push(job_id, LogLevel.PROGRESS, content, progress_pct=pct, stage=stage)
+
+    def push_finished(self, job_id: str) -> None:
+        """Signal job completion."""
+        self.push(job_id, LogLevel.FINISHED, "")
+        self.flush_all(job_id)  # Ensure all messages are flushed
+        _broadcaster.cleanup(job_id)
+
+
+class LogStreamSubscriber:
+    """In-memory subscriber for log streams.
+
+    Used by FastAPI SSE endpoints. Worker runs in-process so
+    in-memory broadcast provides instant delivery.
+    """
+
+    def get_history(self, job_id: str) -> list[LogMessage]:
+        """Get message history from in-memory broadcaster."""
+        return _broadcaster.get_history(job_id)
+
+    async def subscribe(
+        self, job_id: str, max_wait_seconds: int = 1800
+    ) -> AsyncIterator[LogMessage]:
+        """Subscribe to log stream with minimal latency.
+
+        1. Yield history for catch-up
+        2. Use in-memory broadcast (<1ms latency)
+        """
+        # First yield history
+        for msg in self.get_history(job_id):
+            yield msg
+            if msg.level == LogLevel.FINISHED:
+                return
+
+        queue = _broadcaster.subscribe(job_id)
+        start_time = time.monotonic()
+
+        try:
+            while (time.monotonic() - start_time) < max_wait_seconds:
+                try:
+                    msg = queue.get_nowait()
+                    yield msg
+                    if msg.level == LogLevel.FINISHED:
+                        return
+                    continue
+                except asyncio.QueueEmpty:
+                    pass
+
+                await asyncio.sleep(0.01)  # 10ms poll interval
+
+            logger.warning(
+                f"Log stream subscription for job {job_id} timed out after {max_wait_seconds}s"
+            )
+            yield LogMessage(
+                level=LogLevel.INFO,
+                content=f"[Stream timeout after {max_wait_seconds}s - connection closed]",
+            )
+
+        finally:
+            _broadcaster.unsubscribe(job_id, queue)
+
+
+# Global instances
+log_stream_publisher = LogStreamPublisher()
+log_stream_service = LogStreamSubscriber()