autoforge-ai 0.1.14 → 0.1.16

package/agent.py

@@ -74,46 +74,65 @@ async def run_agent_session(
         await client.query(message)
 
         # Collect response text and show tool use
+        # Retry receive_response() on MessageParseError — the SDK raises this for
+        # unknown CLI message types (e.g. "rate_limit_event") which kills the async
+        # generator.  The subprocess is still alive so we restart to read remaining
+        # messages from the buffered channel.
         response_text = ""
-        async for msg in client.receive_response():
-            msg_type = type(msg).__name__
-
-            # Handle AssistantMessage (text and tool use)
-            if msg_type == "AssistantMessage" and hasattr(msg, "content"):
-                for block in msg.content:
-                    block_type = type(block).__name__
-
-                    if block_type == "TextBlock" and hasattr(block, "text"):
-                        response_text += block.text
-                        print(block.text, end="", flush=True)
-                    elif block_type == "ToolUseBlock" and hasattr(block, "name"):
-                        print(f"\n[Tool: {block.name}]", flush=True)
-                        if hasattr(block, "input"):
-                            input_str = str(block.input)
-                            if len(input_str) > 200:
-                                print(f"   Input: {input_str[:200]}...", flush=True)
-                            else:
-                                print(f"   Input: {input_str}", flush=True)
-
-            # Handle UserMessage (tool results)
-            elif msg_type == "UserMessage" and hasattr(msg, "content"):
-                for block in msg.content:
-                    block_type = type(block).__name__
-
-                    if block_type == "ToolResultBlock":
-                        result_content = getattr(block, "content", "")
-                        is_error = getattr(block, "is_error", False)
-
-                        # Check if command was blocked by security hook
-                        if "blocked" in str(result_content).lower():
-                            print(f"   [BLOCKED] {result_content}", flush=True)
-                        elif is_error:
-                            # Show errors (truncated)
-                            error_str = str(result_content)[:500]
-                            print(f"   [Error] {error_str}", flush=True)
-                        else:
-                            # Tool succeeded - just show brief confirmation
-                            print("   [Done]", flush=True)
+        max_parse_retries = 50
+        parse_retries = 0
+        while True:
+            try:
+                async for msg in client.receive_response():
+                    msg_type = type(msg).__name__
+
+                    # Handle AssistantMessage (text and tool use)
+                    if msg_type == "AssistantMessage" and hasattr(msg, "content"):
+                        for block in msg.content:
+                            block_type = type(block).__name__
+
+                            if block_type == "TextBlock" and hasattr(block, "text"):
+                                response_text += block.text
+                                print(block.text, end="", flush=True)
+                            elif block_type == "ToolUseBlock" and hasattr(block, "name"):
+                                print(f"\n[Tool: {block.name}]", flush=True)
+                                if hasattr(block, "input"):
+                                    input_str = str(block.input)
+                                    if len(input_str) > 200:
+                                        print(f"   Input: {input_str[:200]}...", flush=True)
+                                    else:
+                                        print(f"   Input: {input_str}", flush=True)
+
+                    # Handle UserMessage (tool results)
+                    elif msg_type == "UserMessage" and hasattr(msg, "content"):
+                        for block in msg.content:
+                            block_type = type(block).__name__
+
+                            if block_type == "ToolResultBlock":
+                                result_content = getattr(block, "content", "")
+                                is_error = getattr(block, "is_error", False)
+
+                                # Check if command was blocked by security hook
+                                if "blocked" in str(result_content).lower():
+                                    print(f"   [BLOCKED] {result_content}", flush=True)
+                                elif is_error:
+                                    # Show errors (truncated)
+                                    error_str = str(result_content)[:500]
+                                    print(f"   [Error] {error_str}", flush=True)
+                                else:
+                                    # Tool succeeded - just show brief confirmation
+                                    print("   [Done]", flush=True)
+
+                break  # Normal completion
+            except Exception as inner_exc:
+                if type(inner_exc).__name__ == "MessageParseError":
+                    parse_retries += 1
+                    if parse_retries > max_parse_retries:
+                        print(f"Too many unrecognized CLI messages ({parse_retries}), stopping")
+                        break
+                    print(f"Ignoring unrecognized message from Claude CLI: {inner_exc}")
+                    continue
+                raise  # Re-raise to outer except
 
         print("\n" + "-" * 70 + "\n")
         return "continue", response_text

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "autoforge-ai",
-  "version": "0.1.14",
+  "version": "0.1.16",
   "description": "Autonomous coding agent with web UI - build complete apps with AI",
   "license": "AGPL-3.0",
   "bin": {

package/server/main.py

@@ -36,6 +36,7 @@ from .routers import (
     features_router,
     filesystem_router,
     projects_router,
+    scaffold_router,
     schedules_router,
     settings_router,
     spec_creation_router,
@@ -169,6 +170,7 @@ app.include_router(filesystem_router)
 app.include_router(assistant_chat_router)
 app.include_router(settings_router)
 app.include_router(terminal_router)
+app.include_router(scaffold_router)
 
 
 # ============================================================================

package/server/routers/__init__.py

@@ -12,6 +12,7 @@ from .expand_project import router as expand_project_router
 from .features import router as features_router
 from .filesystem import router as filesystem_router
 from .projects import router as projects_router
+from .scaffold import router as scaffold_router
 from .schedules import router as schedules_router
 from .settings import router as settings_router
 from .spec_creation import router as spec_creation_router
@@ -29,4 +30,5 @@ __all__ = [
     "assistant_chat_router",
     "settings_router",
     "terminal_router",
+    "scaffold_router",
 ]

package/server/routers/scaffold.py

@@ -0,0 +1,136 @@
+"""
+Scaffold Router
+================
+
+SSE streaming endpoint for running project scaffold commands.
+Supports templated project creation (e.g., Next.js agentic starter).
+"""
+
+import asyncio
+import json
+import logging
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+from fastapi import APIRouter, Request
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel
+
+from .filesystem import is_path_blocked
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api/scaffold", tags=["scaffold"])
+
+# Hardcoded templates — no arbitrary commands allowed
+TEMPLATES: dict[str, list[str]] = {
+    "agentic-starter": ["npx", "create-agentic-app@latest", ".", "-y", "-p", "npm", "--skip-git"],
+}
+
+
+class ScaffoldRequest(BaseModel):
+    template: str
+    target_path: str
+
+
+def _sse_event(data: dict) -> str:
+    """Format a dict as an SSE data line."""
+    return f"data: {json.dumps(data)}\n\n"
+
+
+async def _stream_scaffold(template: str, target_path: str, request: Request):
+    """Run the scaffold command and yield SSE events."""
+    # Validate template
+    if template not in TEMPLATES:
+        yield _sse_event({"type": "error", "message": f"Unknown template: {template}"})
+        return
+
+    # Validate path
+    path = Path(target_path)
+    try:
+        path = path.resolve()
+    except (OSError, ValueError) as e:
+        yield _sse_event({"type": "error", "message": f"Invalid path: {e}"})
+        return
+
+    if is_path_blocked(path):
+        yield _sse_event({"type": "error", "message": "Access to this directory is not allowed"})
+        return
+
+    if not path.exists() or not path.is_dir():
+        yield _sse_event({"type": "error", "message": "Target directory does not exist"})
+        return
+
+    # Check npx is available
+    npx_name = "npx"
+    if sys.platform == "win32":
+        npx_name = "npx.cmd"
+
+    if not shutil.which(npx_name):
+        yield _sse_event({"type": "error", "message": "npx is not available. Please install Node.js."})
+        return
+
+    # Build command
+    argv = list(TEMPLATES[template])
+    if sys.platform == "win32" and not argv[0].lower().endswith(".cmd"):
+        argv[0] = argv[0] + ".cmd"
+
+    process = None
+    try:
+        popen_kwargs: dict = {
+            "stdout": subprocess.PIPE,
+            "stderr": subprocess.STDOUT,
+            "stdin": subprocess.DEVNULL,
+            "cwd": str(path),
+        }
+        if sys.platform == "win32":
+            popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
+
+        process = subprocess.Popen(argv, **popen_kwargs)
+        logger.info("Scaffold process started: pid=%s, template=%s, path=%s", process.pid, template, target_path)
+
+        # Stream stdout lines
+        assert process.stdout is not None
+        for raw_line in iter(process.stdout.readline, b""):
+            # Check if client disconnected
+            if await request.is_disconnected():
+                logger.info("Client disconnected during scaffold, terminating process")
+                break
+
+            line = raw_line.decode("utf-8", errors="replace").rstrip("\n\r")
+            yield _sse_event({"type": "output", "line": line})
+            # Yield control to event loop so disconnect checks work
+            await asyncio.sleep(0)
+
+        process.wait()
+        exit_code = process.returncode
+        success = exit_code == 0
+        logger.info("Scaffold process completed: exit_code=%s, template=%s", exit_code, template)
+        yield _sse_event({"type": "complete", "success": success, "exit_code": exit_code})
+
+    except Exception as e:
+        logger.error("Scaffold error: %s", e)
+        yield _sse_event({"type": "error", "message": str(e)})
+
+    finally:
+        if process and process.poll() is None:
+            try:
+                process.terminate()
+                process.wait(timeout=5)
+            except Exception:
+                process.kill()
+
+
+@router.post("/run")
+async def run_scaffold(body: ScaffoldRequest, request: Request):
+    """Run a scaffold template command with SSE streaming output."""
+    return StreamingResponse(
+        _stream_scaffold(body.template, body.target_path, request),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "X-Accel-Buffering": "no",
+        },
+    )

package/server/services/assistant_chat_session.py

@@ -7,7 +7,6 @@ The assistant can answer questions about the codebase and features
 but cannot modify any files.
 """
 
-import asyncio
 import json
 import logging
 import os
@@ -27,10 +26,9 @@ from .assistant_database import (
     get_messages,
 )
 from .chat_constants import (
-    MAX_CHAT_RATE_LIMIT_RETRIES,
     ROOT_DIR,
-    calculate_rate_limit_backoff,
     check_rate_limit_error,
+    safe_receive_response,
 )
 
 # Load environment variables from .env file if present
@@ -399,66 +397,47 @@ class AssistantChatSession:
 
         full_response = ""
 
-        # Stream the response (with rate-limit retry)
-        for _attempt in range(MAX_CHAT_RATE_LIMIT_RETRIES + 1):
-            try:
-                async for msg in self.client.receive_response():
-                    msg_type = type(msg).__name__
-
-                    if msg_type == "AssistantMessage" and hasattr(msg, "content"):
-                        for block in msg.content:
-                            block_type = type(block).__name__
-
-                            if block_type == "TextBlock" and hasattr(block, "text"):
-                                text = block.text
-                                if text:
-                                    full_response += text
-                                    yield {"type": "text", "content": text}
-
-                            elif block_type == "ToolUseBlock" and hasattr(block, "name"):
-                                tool_name = block.name
-                                tool_input = getattr(block, "input", {})
-
-                                # Intercept ask_user tool calls -> yield as question message
-                                if tool_name == "mcp__features__ask_user":
-                                    questions = tool_input.get("questions", [])
-                                    if questions:
-                                        yield {
-                                            "type": "question",
-                                            "questions": questions,
-                                        }
-                                        continue
-
-                                yield {
-                                    "type": "tool_call",
-                                    "tool": tool_name,
-                                    "input": tool_input,
-                                }
-                # Completed successfully — break out of retry loop
-                break
-            except Exception as exc:
-                is_rate_limit, retry_secs = check_rate_limit_error(exc)
-                if is_rate_limit and _attempt < MAX_CHAT_RATE_LIMIT_RETRIES:
-                    delay = retry_secs if retry_secs else calculate_rate_limit_backoff(_attempt)
-                    logger.warning(f"Rate limited (attempt {_attempt + 1}/{MAX_CHAT_RATE_LIMIT_RETRIES}), retrying in {delay}s")
-                    yield {
-                        "type": "rate_limited",
-                        "retry_in": delay,
-                        "attempt": _attempt + 1,
-                        "max_attempts": MAX_CHAT_RATE_LIMIT_RETRIES,
-                    }
-                    await asyncio.sleep(delay)
-                    await self.client.query(message)
-                    continue
-                if is_rate_limit:
-                    logger.error("Rate limit retries exhausted for assistant chat")
-                    yield {"type": "error", "content": "Rate limited. Please try again later."}
-                    return
-                # Non-rate-limit MessageParseError: log and break (don't crash)
-                if type(exc).__name__ == "MessageParseError":
-                    logger.warning(f"Ignoring unrecognized message from Claude CLI: {exc}")
-                    break
-                raise
+        # Stream the response
+        try:
+            async for msg in safe_receive_response(self.client, logger):
+                msg_type = type(msg).__name__
+
+                if msg_type == "AssistantMessage" and hasattr(msg, "content"):
+                    for block in msg.content:
+                        block_type = type(block).__name__
+
+                        if block_type == "TextBlock" and hasattr(block, "text"):
+                            text = block.text
+                            if text:
+                                full_response += text
+                                yield {"type": "text", "content": text}
+
+                        elif block_type == "ToolUseBlock" and hasattr(block, "name"):
+                            tool_name = block.name
+                            tool_input = getattr(block, "input", {})
+
+                            # Intercept ask_user tool calls -> yield as question message
+                            if tool_name == "mcp__features__ask_user":
+                                questions = tool_input.get("questions", [])
+                                if questions:
+                                    yield {
+                                        "type": "question",
+                                        "questions": questions,
+                                    }
+                                    continue
+
+                            yield {
+                                "type": "tool_call",
+                                "tool": tool_name,
+                                "input": tool_input,
+                            }
+        except Exception as exc:
+            is_rate_limit, _ = check_rate_limit_error(exc)
+            if is_rate_limit:
+                logger.warning(f"Rate limited: {exc}")
+                yield {"type": "error", "content": "Rate limited. Please try again later."}
+                return
+            raise
 
         # Store the complete response in the database
         if full_response and self.conversation_id:

package/server/services/chat_constants.py

@@ -12,7 +12,7 @@ imports (``from .chat_constants import API_ENV_VARS``) continue to work.
 import logging
 import sys
 from pathlib import Path
-from typing import AsyncGenerator
+from typing import Any, AsyncGenerator
 
 # -------------------------------------------------------------------
 # Root directory of the autoforge project (repository root).
@@ -33,15 +33,10 @@ if _root_str not in sys.path:
 # imports continue to work unchanged.
 # -------------------------------------------------------------------
 from env_constants import API_ENV_VARS  # noqa: E402, F401
-from rate_limit_utils import calculate_rate_limit_backoff, is_rate_limit_error, parse_retry_after  # noqa: E402, F401
+from rate_limit_utils import is_rate_limit_error, parse_retry_after  # noqa: E402, F401
 
 logger = logging.getLogger(__name__)
 
-# -------------------------------------------------------------------
-# Rate-limit handling for chat sessions
-# -------------------------------------------------------------------
-MAX_CHAT_RATE_LIMIT_RETRIES = 3
-
 
 def check_rate_limit_error(exc: Exception) -> tuple[bool, int | None]:
     """Inspect an exception and determine if it represents a rate-limit.
@@ -49,24 +44,15 @@ def check_rate_limit_error(exc: Exception) -> tuple[bool, int | None]:
     Returns ``(is_rate_limit, retry_seconds)``.  ``retry_seconds`` is the
     parsed Retry-After value when available, otherwise ``None`` (caller
     should use exponential backoff).
-
-    Handles:
-    - ``MessageParseError`` whose raw *data* dict has
-      ``type == "rate_limit_event"`` (Claude CLI sends this).
-    - Any exception whose string representation matches known rate-limit
-      patterns (via ``rate_limit_utils.is_rate_limit_error``).
     """
-    exc_str = str(exc)
-
-    # Check for MessageParseError with a rate_limit_event payload
-    cls_name = type(exc).__name__
-    if cls_name == "MessageParseError":
-        raw_data = getattr(exc, "data", None)
-        if isinstance(raw_data, dict) and raw_data.get("type") == "rate_limit_event":
-            retry = parse_retry_after(str(raw_data)) if raw_data else None
-            return True, retry
+    # MessageParseError = unknown CLI message type (e.g. "rate_limit_event").
+    # These are informational events, NOT actual rate limit errors.
+    # The word "rate_limit" in the type name would false-positive the regex.
+    if type(exc).__name__ == "MessageParseError":
+        return False, None
 
-    # Fallback: match error text against known rate-limit patterns
+    # For all other exceptions: match error text against known rate-limit patterns
+    exc_str = str(exc)
     if is_rate_limit_error(exc_str):
         retry = parse_retry_after(exc_str)
         return True, retry
@@ -74,6 +60,34 @@ def check_rate_limit_error(exc: Exception) -> tuple[bool, int | None]:
     return False, None
 
 
+async def safe_receive_response(client: Any, log: logging.Logger) -> AsyncGenerator:
+    """Wrap ``client.receive_response()`` to skip ``MessageParseError``.
+
+    The Claude Code CLI may emit message types (e.g. ``rate_limit_event``)
+    that the installed Python SDK does not recognise, causing
+    ``MessageParseError`` which kills the async generator.  The CLI
+    subprocess is still alive and the SDK uses a buffered memory channel,
+    so we restart ``receive_response()`` to continue reading remaining
+    messages without losing data.
+    """
+    max_retries = 50
+    retries = 0
+    while True:
+        try:
+            async for msg in client.receive_response():
+                yield msg
+            return  # Normal completion
+        except Exception as exc:
+            if type(exc).__name__ == "MessageParseError":
+                retries += 1
+                if retries > max_retries:
+                    log.error(f"Too many unrecognized CLI messages ({retries}), stopping")
+                    return
+                log.warning(f"Ignoring unrecognized message from Claude CLI: {exc}")
+                continue
+            raise
+
+
 async def make_multimodal_message(content_blocks: list[dict]) -> AsyncGenerator[dict, None]:
     """Yield a single multimodal user message in Claude Agent SDK format.
 

package/server/services/expand_chat_session.py

@@ -23,11 +23,10 @@ from dotenv import load_dotenv
 
 from ..schemas import ImageAttachment
 from .chat_constants import (
-    MAX_CHAT_RATE_LIMIT_RETRIES,
     ROOT_DIR,
-    calculate_rate_limit_backoff,
     check_rate_limit_error,
     make_multimodal_message,
+    safe_receive_response,
 )
 
 # Load environment variables from .env file if present
@@ -304,67 +303,32 @@ class ExpandChatSession:
         else:
             await self.client.query(message)
 
-        # Stream the response (with rate-limit retry)
-        for _attempt in range(MAX_CHAT_RATE_LIMIT_RETRIES + 1):
-            try:
-                async for msg in self.client.receive_response():
-                    msg_type = type(msg).__name__
-
-                    if msg_type == "AssistantMessage" and hasattr(msg, "content"):
-                        for block in msg.content:
-                            block_type = type(block).__name__
-
-                            if block_type == "TextBlock" and hasattr(block, "text"):
-                                text = block.text
-                                if text:
-                                    yield {"type": "text", "content": text}
-
-                                    self.messages.append({
-                                        "role": "assistant",
-                                        "content": text,
-                                        "timestamp": datetime.now().isoformat()
-                                    })
-                # Completed successfully — break out of retry loop
-                break
-            except Exception as exc:
-                is_rate_limit, retry_secs = check_rate_limit_error(exc)
-                if is_rate_limit and _attempt < MAX_CHAT_RATE_LIMIT_RETRIES:
-                    delay = retry_secs if retry_secs else calculate_rate_limit_backoff(_attempt)
-                    logger.warning(f"Rate limited (attempt {_attempt + 1}/{MAX_CHAT_RATE_LIMIT_RETRIES}), retrying in {delay}s")
-                    yield {
-                        "type": "rate_limited",
-                        "retry_in": delay,
-                        "attempt": _attempt + 1,
-                        "max_attempts": MAX_CHAT_RATE_LIMIT_RETRIES,
-                    }
-                    await asyncio.sleep(delay)
-                    # Re-send the query before retrying receive_response
-                    if attachments and len(attachments) > 0:
-                        content_blocks_retry: list[dict[str, Any]] = []
-                        if message:
-                            content_blocks_retry.append({"type": "text", "text": message})
-                        for att in attachments:
-                            content_blocks_retry.append({
-                                "type": "image",
-                                "source": {
-                                    "type": "base64",
-                                    "media_type": att.mimeType,
-                                    "data": att.base64Data,
-                                }
-                            })
-                        await self.client.query(make_multimodal_message(content_blocks_retry))
-                    else:
-                        await self.client.query(message)
-                    continue
-                if is_rate_limit:
-                    logger.error("Rate limit retries exhausted for expand chat")
-                    yield {"type": "error", "content": "Rate limited. Please try again later."}
-                    return
-                # Non-rate-limit MessageParseError: log and break (don't crash)
-                if type(exc).__name__ == "MessageParseError":
-                    logger.warning(f"Ignoring unrecognized message from Claude CLI: {exc}")
-                    break
-                raise
+        # Stream the response
+        try:
+            async for msg in safe_receive_response(self.client, logger):
+                msg_type = type(msg).__name__
+
+                if msg_type == "AssistantMessage" and hasattr(msg, "content"):
+                    for block in msg.content:
+                        block_type = type(block).__name__
+
+                        if block_type == "TextBlock" and hasattr(block, "text"):
+                            text = block.text
+                            if text:
+                                yield {"type": "text", "content": text}
+
+                                self.messages.append({
+                                    "role": "assistant",
+                                    "content": text,
+                                    "timestamp": datetime.now().isoformat()
+                                })
+        except Exception as exc:
+            is_rate_limit, _ = check_rate_limit_error(exc)
+            if is_rate_limit:
+                logger.warning(f"Rate limited: {exc}")
+                yield {"type": "error", "content": "Rate limited. Please try again later."}
+                return
+            raise
 
     def get_features_created(self) -> int:
         """Get the total number of features created in this session."""