feed-the-machine 1.1.0 → 1.3.0

package/ftm-inbox/backend/main.py

@@ -0,0 +1,103 @@
+"""
+ftm-inbox FastAPI application entry point.
+
+Default port: 8042 (override via FTM_INBOX_PORT env var).
+
+Startup sequence:
+  1. Open SQLite connection (WAL mode)
+  2. Initialize schema (idempotent CREATE TABLE IF NOT EXISTS)
+  3. Load adapter registry from config.yml (warn if missing, don't crash)
+  4. Register API routes
+
+CORS allows all localhost origins for development.
+"""
+
+import logging
+import os
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+from backend.db.connection import get_connection
+from backend.db.schema import initialize_schema
+from backend.routes.health import router as health_router
+from backend.routes.inbox import router as inbox_router
+from backend.routes.plan import router as plan_router
+from backend.routes.execute import router as execute_router
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s  %(levelname)-8s  %(name)s  %(message)s",
+)
+logger = logging.getLogger("ftm-inbox")
+
+app = FastAPI(
+    title="FTM Inbox",
+    description="Operator Cockpit backend — aggregates tasks from all connected sources.",
+    version="0.1.0",
+)
+
+# CORS: allow all localhost origins (frontend dev server + any local tooling)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=[
+        "http://localhost:5173",
+        "http://localhost:3000",
+        "http://127.0.0.1:5173",
+        "http://127.0.0.1:3000",
+    ],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# ---------------------------------------------------------------------------
+# Startup
+# ---------------------------------------------------------------------------
+
+@app.on_event("startup")
+async def startup() -> None:
+    logger.info("Starting ftm-inbox backend…")
+
+    # 1. Open DB and initialize schema
+    conn = get_connection()
+    initialize_schema(conn)
+    logger.info("Database initialized.")
+
+    # 2. Load adapter registry (non-fatal if config.yml is missing)
+    try:
+        from backend.adapters.registry import AdapterRegistry
+        registry = AdapterRegistry.from_config()
+        app.state.adapter_registry = registry
+        logger.info("Adapter registry loaded: %s", registry)
+    except Exception as exc:
+        logger.warning("Adapter registry failed to load: %s", exc)
+        app.state.adapter_registry = None
+
+
+@app.on_event("shutdown")
+async def shutdown() -> None:
+    from backend.db.connection import close_connection
+    close_connection()
+    logger.info("Database connection closed.")
+
+
+# ---------------------------------------------------------------------------
+# Routes
+# ---------------------------------------------------------------------------
+
+app.include_router(health_router)
+app.include_router(inbox_router)
+app.include_router(plan_router)
+app.include_router(execute_router)
+
+
+# ---------------------------------------------------------------------------
+# Dev entrypoint
+# ---------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    import uvicorn
+
+    port = int(os.environ.get("FTM_INBOX_PORT", "8042"))
+    uvicorn.run("backend.main:app", host="0.0.0.0", port=port, reload=True)

package/ftm-inbox/backend/models/__init__.py

@@ -0,0 +1 @@
+"""ftm-inbox data models."""

package/ftm-inbox/backend/models/unified_task.py

@@ -0,0 +1,36 @@
+"""
+UnifiedTask — canonical Pydantic model for a task regardless of source.
+
+This is the read/API-facing model. The write path uses NormalizedItem (a dataclass)
+for speed; UnifiedTask is the model returned to callers and stored as JSON.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class UnifiedTask(BaseModel):
+    """Single unified representation of a task from any integrated source."""
+
+    id: int | None = None
+    source: str
+    source_id: str
+    title: str
+    body: str = ""
+    status: str = "open"
+    priority: str = "medium"
+    assignee: str | None = None
+    requester: str | None = None
+    created_at: str | None = None
+    updated_at: str | None = None
+    tags: list[str] = Field(default_factory=list)
+    custom_fields: dict[str, Any] = Field(default_factory=dict)
+    raw_payload: dict[str, Any] = Field(default_factory=dict)
+    source_url: str | None = None
+    content_hash: str | None = None
+    ingested_at: str | None = None
+
+    model_config = {"populate_by_name": True}

package/ftm-inbox/backend/planner/__init__.py

@@ -0,0 +1,6 @@
+"""
+ftm-inbox planner package.
+
+Generates structured YAML execution plans from task payloads via Claude CLI,
+and persists them to SQLite for per-step approval workflows.
+"""

package/ftm-inbox/backend/planner/generator.py

@@ -0,0 +1,127 @@
+"""
+Plan generator — invokes Claude CLI to produce a structured YAML execution plan.
+
+The generator calls `claude -p <prompt> --output-format json`, parses the JSON
+envelope to extract the text result, then extracts the embedded YAML block.
+
+On timeout or any subprocess error the function returns a safe error dict so
+callers can surface the failure without crashing.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+
+import yaml
+
+
+def generate_plan(task_data: dict, capabilities: dict | None = None) -> dict:
+    """Invoke Claude CLI to generate a structured plan for a task.
+
+    Returns a dict with keys:
+      - steps: list[dict]  — parsed plan steps (empty on failure)
+      - yaml_content: str  — raw YAML string
+      - raw_response: str  — full text from Claude (useful for debugging)
+      - error: str | None  — set only on failure
+    """
+    cap_note = ""
+    if capabilities:
+        available = ", ".join(
+            k for k, v in capabilities.items() if v
+        )
+        if available:
+            cap_note = f"\nAvailable integrations: {available}"
+
+    prompt = f"""Generate a structured execution plan for this task.
+
+Task: {task_data.get('title', '')}
+Source: {task_data.get('source', '')}
+Body: {task_data.get('body', '')}{cap_note}
+
+Return a YAML object with this structure:
+steps:
+  - id: 1
+    title: "Step description"
+    target_system: "jira|freshservice|slack|gmail|local"
+    method_primary: "tool or action"
+    method_fallback: "alternative if primary unavailable"
+    risk_level: "low|medium|high"
+    approval_required: false
+    rollback: "how to undo"
+
+Only return the YAML, no other text."""
+
+    try:
+        result = subprocess.run(
+            ["claude", "-p", prompt, "--output-format", "json"],
+            capture_output=True,
+            text=True,
+            timeout=120,
+        )
+        if result.returncode != 0:
+            return {"error": result.stderr or "Claude CLI returned non-zero exit", "steps": [], "yaml_content": "", "raw_response": ""}
+
+        # Claude CLI JSON envelope: {"result": "<text>", ...}
+        try:
+            output = json.loads(result.stdout)
+            text = output.get("result", result.stdout)
+        except json.JSONDecodeError:
+            text = result.stdout
+
+        yaml_content = _extract_yaml(text)
+        try:
+            plan_data = yaml.safe_load(yaml_content) or {}
+        except yaml.YAMLError as exc:
+            return {
+                "error": f"YAML parse error: {exc}",
+                "steps": [],
+                "yaml_content": yaml_content,
+                "raw_response": text,
+            }
+
+        raw_steps = plan_data.get("steps", [])
+        # Normalise: ensure each step has all expected keys with safe defaults
+        steps = [_normalise_step(i + 1, s) for i, s in enumerate(raw_steps)]
+
+        return {
+            "steps": steps,
+            "yaml_content": yaml_content,
+            "raw_response": text,
+            "error": None,
+        }
+
+    except subprocess.TimeoutExpired:
+        return {"error": "Plan generation timed out after 120s", "steps": [], "yaml_content": "", "raw_response": ""}
+    except FileNotFoundError:
+        return {"error": "Claude CLI not found — ensure 'claude' is on PATH", "steps": [], "yaml_content": "", "raw_response": ""}
+    except Exception as exc:
+        return {"error": str(exc), "steps": [], "yaml_content": "", "raw_response": ""}
+
+
+def _extract_yaml(text: str) -> str:
+    """Extract YAML content from Claude's response, handling optional code fences."""
+    if "```yaml" in text:
+        start = text.index("```yaml") + 7
+        end = text.index("```", start)
+        return text[start:end].strip()
+    if "```" in text:
+        start = text.index("```") + 3
+        end = text.index("```", start)
+        return text[start:end].strip()
+    return text.strip()
+
+
+def _normalise_step(index: int, raw: dict) -> dict:
+    """Ensure a raw step dict has all required keys."""
+    return {
+        "id": raw.get("id", index),
+        "title": raw.get("title", f"Step {index}"),
+        "target_system": raw.get("target_system", "local"),
+        "method_primary": raw.get("method_primary", ""),
+        "method_fallback": raw.get("method_fallback", ""),
+        "risk_level": raw.get("risk_level", "low"),
+        "approval_required": bool(raw.get("approval_required", False)),
+        "rollback": raw.get("rollback", ""),
+        "status": "pending",
+    }

package/ftm-inbox/backend/planner/schema.py

@@ -0,0 +1,34 @@
+"""
+Pydantic models for structured execution plans.
+
+A Plan contains an ordered list of PlanSteps. Each step tracks its approval
+state independently so the operator can approve individual steps before
+the executor runs them.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+from typing import Any
+
+
+class PlanStep(BaseModel):
+    id: int
+    title: str
+    target_system: str = ""
+    method_primary: str = ""
+    method_fallback: str = ""
+    risk_level: str = "low"       # low | medium | high
+    approval_required: bool = False
+    rollback: str = ""
+    status: str = "pending"       # pending | approved | rejected | running | completed | failed
+
+
+class Plan(BaseModel):
+    id: int | None = None
+    task_id: int
+    steps: list[PlanStep] = Field(default_factory=list)
+    status: str = "draft"         # draft | approved | executing | completed | failed
+    yaml_content: str = ""
+    created_at: str | None = None
+    updated_at: str | None = None

package/ftm-inbox/backend/requirements.txt

@@ -0,0 +1,5 @@
+fastapi>=0.111.0
+uvicorn[standard]>=0.29.0
+pyyaml>=6.0.1
+requests>=2.31.0
+pydantic>=2.6.0

package/ftm-inbox/backend/routes/execute.py

@@ -0,0 +1,186 @@
+"""
+Execution API routes — start, pause, resume, retry, audit log, SSE streaming.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+
+from fastapi import APIRouter, HTTPException, Query
+from fastapi.responses import StreamingResponse
+
+from backend.db.connection import get_connection
+from backend.executor.engine import ExecutionEngine
+
+router = APIRouter(prefix="/api", tags=["execution"])
+
+# In-memory registry of active engines (single-process; swap for Redis in production)
+_active_engines: dict[int, ExecutionEngine] = {}
+
+
+@router.post("/tasks/{task_id}/execute")
+async def start_execution(task_id: int):
+    """Start execution of approved plan steps."""
+    conn = get_connection()
+    plan_row = conn.execute(
+        "SELECT id, status FROM plans WHERE task_id = ? ORDER BY id DESC LIMIT 1",
+        (task_id,),
+    ).fetchone()
+
+    if not plan_row:
+        raise HTTPException(404, "No plan found for this task")
+
+    plan_id = plan_row["id"]
+    engine = ExecutionEngine(task_id, plan_id)
+    _active_engines[task_id] = engine
+
+    result = await engine.execute()
+    _active_engines.pop(task_id, None)
+    return result
+
+
+@router.get("/tasks/{task_id}/execution-stream")
+async def execution_stream(task_id: int):
+    """SSE stream of execution output."""
+    conn = get_connection()
+    plan_row = conn.execute(
+        "SELECT id FROM plans WHERE task_id = ? ORDER BY id DESC LIMIT 1",
+        (task_id,),
+    ).fetchone()
+
+    if not plan_row:
+        raise HTTPException(404, "No plan found for this task")
+
+    plan_id = plan_row["id"]
+    engine = ExecutionEngine(task_id, plan_id)
+    _active_engines[task_id] = engine
+
+    output_queue: asyncio.Queue[str] = asyncio.Queue()
+
+    def on_output(text: str):
+        output_queue.put_nowait(text)
+
+    engine.on_output(on_output)
+
+    async def event_generator():
+        # Start execution in background
+        task = asyncio.create_task(engine.execute())
+
+        try:
+            while not task.done():
+                try:
+                    text = await asyncio.wait_for(output_queue.get(), timeout=1.0)
+                    yield f"data: {json.dumps({'type': 'chunk', 'text': text})}\n\n"
+                except asyncio.TimeoutError:
+                    yield f"data: {json.dumps({'type': 'heartbeat'})}\n\n"
+
+            # Drain remaining messages
+            while not output_queue.empty():
+                text = output_queue.get_nowait()
+                yield f"data: {json.dumps({'type': 'chunk', 'text': text})}\n\n"
+
+            result = task.result()
+            yield f"data: {json.dumps({'type': 'done', 'result': result})}\n\n"
+        except Exception as exc:
+            yield f"data: {json.dumps({'type': 'error', 'message': str(exc)})}\n\n"
+        finally:
+            _active_engines.pop(task_id, None)
+
+    return StreamingResponse(
+        event_generator(),
+        media_type="text/event-stream",
+        headers={"Cache-Control": "no-cache", "Connection": "keep-alive"},
+    )
+
+
+@router.post("/tasks/{task_id}/pause")
+async def pause_execution(task_id: int):
+    """Pause execution after the current step completes."""
+    engine = _active_engines.get(task_id)
+    if not engine:
+        raise HTTPException(404, "No active execution for this task")
+    engine.pause()
+    return {"status": "pausing", "message": "Execution will pause after current step"}
+
+
+@router.post("/tasks/{task_id}/resume")
+async def resume_execution(task_id: int):
+    """Resume a paused execution."""
+    engine = _active_engines.get(task_id)
+    if engine:
+        engine.resume()
+        result = await engine.execute()
+        return result
+
+    # No active engine — restart from where we left off
+    conn = get_connection()
+    plan_row = conn.execute(
+        "SELECT id FROM plans WHERE task_id = ? ORDER BY id DESC LIMIT 1",
+        (task_id,),
+    ).fetchone()
+    if not plan_row:
+        raise HTTPException(404, "No plan found for this task")
+
+    engine = ExecutionEngine(task_id, plan_row["id"])
+    _active_engines[task_id] = engine
+    result = await engine.execute()
+    _active_engines.pop(task_id, None)
+    return result
+
+
+@router.post("/tasks/{task_id}/steps/{step_id}/retry")
+async def retry_step(task_id: int, step_id: int):
+    """Retry a failed step."""
+    conn = get_connection()
+    plan_row = conn.execute(
+        "SELECT id, yaml_content FROM plans WHERE task_id = ? ORDER BY id DESC LIMIT 1",
+        (task_id,),
+    ).fetchone()
+    if not plan_row:
+        raise HTTPException(404, "No plan found")
+
+    import yaml
+    plan_data = yaml.safe_load(plan_row["yaml_content"]) or {}
+    for step in plan_data.get("steps", []):
+        if step.get("id") == step_id:
+            step["status"] = "approved"
+
+    updated = yaml.dump(plan_data, default_flow_style=False)
+    conn.execute(
+        "UPDATE plans SET yaml_content = ?, updated_at = datetime('now') WHERE id = ?",
+        (updated, plan_row["id"]),
+    )
+    conn.commit()
+
+    return {"status": "reset", "step_id": step_id, "message": "Step reset to approved, ready for re-execution"}
+
+
+@router.get("/tasks/{task_id}/audit-log")
+async def get_audit_log(task_id: int, limit: int = Query(100, ge=1, le=1000)):
+    """Get audit log entries for a task's execution."""
+    conn = get_connection()
+
+    # Get all plan IDs for this task
+    plan_rows = conn.execute(
+        "SELECT id FROM plans WHERE task_id = ?", (task_id,)
+    ).fetchall()
+    if not plan_rows:
+        return {"entries": []}
+
+    plan_ids = [r["id"] for r in plan_rows]
+
+    # Get step_ids from the plans to filter audit_log
+    rows = conn.execute(
+        f"SELECT * FROM audit_log ORDER BY created_at DESC LIMIT ?",
+        (limit,),
+    ).fetchall()
+
+    entries = []
+    for row in rows:
+        entry = dict(row)
+        if entry.get("result") and isinstance(entry["result"], str):
+            entry["result"] = json.loads(entry["result"])
+        entries.append(entry)
+
+    return {"entries": entries}

package/ftm-inbox/backend/routes/health.py

@@ -0,0 +1,52 @@
+"""
+Health check endpoint for ftm-inbox backend.
+
+GET /health
+  Returns 200 with status, version, and DB connectivity check.
+"""
+
+import sqlite3
+from datetime import datetime, timezone
+
+from fastapi import APIRouter, Depends
+from fastapi.responses import JSONResponse
+
+router = APIRouter()
+
+
+def _check_db(conn: sqlite3.Connection) -> bool:
+    try:
+        conn.execute("SELECT 1").fetchone()
+        return True
+    except Exception:
+        return False
+
+
+@router.get("/health")
+async def health_check() -> JSONResponse:
+    """
+    Returns backend health status.
+
+    Does not depend on the adapter registry — just confirms the process
+    is alive and the database is reachable.
+    """
+    from backend.db.connection import get_connection
+
+    db_ok = False
+    try:
+        conn = get_connection()
+        db_ok = _check_db(conn)
+    except Exception:
+        db_ok = False
+
+    status = "ok" if db_ok else "degraded"
+    code = 200 if db_ok else 503
+
+    return JSONResponse(
+        status_code=code,
+        content={
+            "status": status,
+            "db": "ok" if db_ok else "unreachable",
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+        },
+    )

package/ftm-inbox/backend/routes/inbox.py

@@ -0,0 +1,68 @@
+"""
+Inbox API routes — paginated task listing with source/status filtering.
+"""
+
+from __future__ import annotations
+
+import json
+
+from fastapi import APIRouter, Query
+
+from backend.db.connection import get_connection
+
+router = APIRouter(prefix="/api", tags=["inbox"])
+
+_JSON_FIELDS = ("tags", "custom_fields", "raw_payload")
+
+
+@router.get("/inbox")
+async def list_inbox(
+    source: str | None = Query(None, description="Filter by source"),
+    status: str | None = Query(None, description="Filter by status"),
+    page: int = Query(1, ge=1),
+    per_page: int = Query(50, ge=1, le=200),
+):
+    """Return paginated inbox tasks with optional filters."""
+    conn = get_connection()
+    conditions: list[str] = []
+    params: list[str] = []
+
+    if source:
+        conditions.append("source = ?")
+        params.append(source)
+    if status:
+        conditions.append("status = ?")
+        params.append(status)
+
+    where = (" WHERE " + " AND ".join(conditions)) if conditions else ""
+    offset = (page - 1) * per_page
+
+    rows = conn.execute(
+        f"SELECT * FROM inbox{where} ORDER BY ingested_at DESC LIMIT ? OFFSET ?",
+        params + [per_page, offset],
+    ).fetchall()
+
+    tasks = []
+    for row in rows:
+        task = dict(row)
+        for field in _JSON_FIELDS:
+            val = task.get(field)
+            if val and isinstance(val, str):
+                task[field] = json.loads(val)
+        tasks.append(task)
+
+    total = conn.execute(
+        f"SELECT COUNT(*) as total FROM inbox{where}", params
+    ).fetchone()["total"]
+
+    return {"tasks": tasks, "total": total, "page": page, "per_page": per_page}
+
+
+@router.get("/inbox/sources")
+async def list_sources():
+    """Return distinct source names with task counts."""
+    conn = get_connection()
+    rows = conn.execute(
+        "SELECT source, COUNT(*) as count FROM inbox GROUP BY source ORDER BY count DESC"
+    ).fetchall()
+    return {"sources": [{"name": r["source"], "count": r["count"]} for r in rows]}