feed-the-machine 1.1.0 → 1.3.0

package/ftm-inbox/backend/adapters/registry.py

@@ -0,0 +1,192 @@
+"""
+AdapterRegistry — loads and manages poller adapters from config.yml.
+
+Config format (see config.example.yml):
+
+    adapters:
+      - name: jira
+        class: ftm_inbox.adapters.jira.JiraAdapter
+        interval_seconds: 60
+        credentials:
+          api_token: "..."
+          email: "..."
+        config:
+          base_url: "https://myorg.atlassian.net"
+          project_key: "OPS"
+
+The registry uses importlib to load adapter classes by dotted module path,
+validates that each class is a subclass of BaseAdapter, and confirms that
+credentials are present (non-empty) before registering the adapter.
+"""
+
+import importlib
+import logging
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from .base import BaseAdapter
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_CONFIG_PATH = (
+    Path(__file__).resolve().parent.parent.parent / "config.yml"
+)
+
+
+class AdapterRegistryError(Exception):
+    """Raised when an adapter cannot be loaded or validated."""
+
+
+class AdapterRegistry:
+    """
+    Loads adapter definitions from config.yml and instantiates each one.
+
+    Usage:
+        registry = AdapterRegistry.from_config()
+        for adapter in registry.adapters:
+            adapter.run_cycle(conn)
+    """
+
+    def __init__(self) -> None:
+        self._adapters: list[BaseAdapter] = []
+
+    @property
+    def adapters(self) -> list[BaseAdapter]:
+        return list(self._adapters)
+
+    @classmethod
+    def from_config(
+        cls, config_path: Path | None = None
+    ) -> "AdapterRegistry":
+        """
+        Build a registry from a YAML config file.
+
+        Args:
+            config_path: Path to config.yml. Defaults to <project_root>/config.yml.
+
+        Returns:
+            Populated AdapterRegistry.
+
+        Raises:
+            AdapterRegistryError: If the config file is missing or an adapter
+                                  fails validation.
+        """
+        path = config_path or _DEFAULT_CONFIG_PATH
+
+        if not path.exists():
+            logger.warning(
+                "config.yml not found at %s — no adapters registered.", path
+            )
+            return cls()
+
+        with path.open() as fh:
+            raw = yaml.safe_load(fh) or {}
+
+        adapter_defs: list[dict[str, Any]] = raw.get("adapters", [])
+        registry = cls()
+
+        for definition in adapter_defs:
+            try:
+                adapter = _load_adapter(definition)
+                registry._adapters.append(adapter)
+                logger.info(
+                    "Registered adapter '%s' (%s)",
+                    definition.get("name"),
+                    definition.get("class"),
+                )
+            except AdapterRegistryError as exc:
+                logger.error("Skipping adapter '%s': %s", definition.get("name"), exc)
+
+        return registry
+
+    def get(self, name: str) -> BaseAdapter | None:
+        """Return a registered adapter by name, or None."""
+        for adapter in self._adapters:
+            if adapter.source_name == name:
+                return adapter
+        return None
+
+    def __len__(self) -> int:
+        return len(self._adapters)
+
+    def __repr__(self) -> str:
+        names = [a.source_name for a in self._adapters]
+        return f"AdapterRegistry(adapters={names})"
+
+
+def _load_adapter(definition: dict[str, Any]) -> BaseAdapter:
+    """
+    Instantiate a single adapter from its config definition.
+
+    Validates:
+      - 'class' key is present and is a valid dotted path
+      - The resolved class is a subclass of BaseAdapter
+      - Required credentials keys are present and non-empty
+
+    Raises:
+        AdapterRegistryError on any validation failure.
+    """
+    dotted_path: str = definition.get("class", "")
+    if not dotted_path:
+        raise AdapterRegistryError("Missing 'class' key in adapter definition.")
+
+    module_path, _, class_name = dotted_path.rpartition(".")
+    if not module_path:
+        raise AdapterRegistryError(
+            f"'class' must be a dotted path (got '{dotted_path}')."
+        )
+
+    try:
+        module = importlib.import_module(module_path)
+    except ImportError as exc:
+        raise AdapterRegistryError(
+            f"Cannot import module '{module_path}': {exc}"
+        ) from exc
+
+    klass = getattr(module, class_name, None)
+    if klass is None:
+        raise AdapterRegistryError(
+            f"Class '{class_name}' not found in module '{module_path}'."
+        )
+
+    if not (isinstance(klass, type) and issubclass(klass, BaseAdapter)):
+        raise AdapterRegistryError(
+            f"'{dotted_path}' is not a subclass of BaseAdapter."
+        )
+
+    credentials: dict[str, Any] = definition.get("credentials", {}) or {}
+    config: dict[str, Any] = definition.get("config", {}) or {}
+
+    _validate_credentials(definition.get("name", dotted_path), credentials, klass)
+
+    instance = klass(credentials=credentials, config=config)
+    # Allow the class to declare its source_name via class attribute;
+    # fall back to the 'name' key in the config definition.
+    if not instance.source_name:
+        instance.source_name = definition.get("name", class_name.lower())
+
+    return instance
+
+
+def _validate_credentials(
+    adapter_name: str,
+    credentials: dict[str, Any],
+    klass: type,
+) -> None:
+    """
+    Check that all keys declared in klass.required_credentials are present
+    and non-empty in the credentials dict.
+
+    Adapters opt into this by setting a class-level list:
+        required_credentials = ["api_token", "email"]
+
+    If the class doesn't declare required_credentials, skip validation.
+    """
+    required: list[str] = getattr(klass, "required_credentials", [])
+    missing = [key for key in required if not credentials.get(key)]
+    if missing:
+        raise AdapterRegistryError(
+            f"Adapter '{adapter_name}' missing required credentials: {missing}"
+        )

package/ftm-inbox/backend/adapters/slack.py

@@ -0,0 +1,110 @@
+"""
+SlackAdapter — polls Slack channels for recent messages.
+
+Required credentials:
+    bot_token   Slack bot token (xoxb-...)
+
+Config keys:
+    channels        List of channel IDs to poll
+    lookback_hours  How far back to fetch messages, default 24
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any
+
+import requests
+
+from backend.adapters._retry import retry
+from backend.adapters.base import BaseAdapter, NormalizedItem
+
+logger = logging.getLogger(__name__)
+
+
+class SlackAdapter(BaseAdapter):
+    """Polls Slack channels for recent messages."""
+
+    source_name = "slack"
+    required_credentials = ["bot_token"]
+
+    def __init__(self, credentials: dict, config: dict) -> None:
+        super().__init__(credentials, config)
+        self._token = credentials["bot_token"]
+        self._channels: list[str] = config.get("channels", [])
+        self._lookback_hours = int(config.get("lookback_hours", 24))
+        self._headers = {
+            "Authorization": f"Bearer {self._token}",
+            "Content-Type": "application/json",
+        }
+
+    @retry(max_attempts=3, base_delay=1.0, exceptions=(requests.RequestException,))
+    def poll(self) -> list[dict]:
+        """Fetch recent messages from configured Slack channels."""
+        oldest = str(time.time() - self._lookback_hours * 3600)
+        all_messages: list[dict] = []
+
+        for channel_id in self._channels:
+            url = "https://slack.com/api/conversations.history"
+            params: dict[str, Any] = {
+                "channel": channel_id,
+                "oldest": oldest,
+                "limit": 200,
+            }
+            response = requests.get(
+                url, headers=self._headers, params=params, timeout=30
+            )
+            response.raise_for_status()
+            data = response.json()
+
+            if not data.get("ok"):
+                logger.warning(
+                    "Slack API error for channel %s: %s",
+                    channel_id,
+                    data.get("error", "unknown"),
+                )
+                continue
+
+            messages = data.get("messages", [])
+            for msg in messages:
+                msg["_channel_id"] = channel_id
+            all_messages.extend(messages)
+
+        return all_messages
+
+    def normalize(self, raw_item: dict) -> NormalizedItem:
+        """Map a Slack message dict to NormalizedItem."""
+        ts = raw_item.get("ts", "")
+        channel_id = raw_item.get("_channel_id", "")
+        text = raw_item.get("text") or ""
+
+        title = text[:100].replace("\n", " ") if text else "(no text)"
+        body = text
+
+        user = raw_item.get("user") or raw_item.get("bot_id") or "unknown"
+
+        created_at = ts
+
+        source_url = (
+            f"https://slack.com/archives/{channel_id}/p{ts.replace('.', '')}"
+            if channel_id and ts
+            else None
+        )
+
+        return NormalizedItem(
+            source=self.source_name,
+            source_id=f"{channel_id}:{ts}",
+            title=title,
+            body=body,
+            status="open",
+            priority="medium",
+            assignee=None,
+            requester=user,
+            created_at=created_at,
+            updated_at=None,
+            tags=[],
+            custom_fields={"channel_id": channel_id},
+            raw_payload=raw_item,
+            source_url=source_url,
+        )

package/ftm-inbox/backend/db/connection.py

@@ -0,0 +1,54 @@
+"""
+SQLite connection helper for ftm-inbox.
+
+Uses WAL journal mode for better read concurrency and sets a sensible
+busy timeout so concurrent writer contention fails gracefully rather
+than immediately.
+"""
+
+import sqlite3
+import threading
+from pathlib import Path
+
+_local = threading.local()
+
+DEFAULT_DB_PATH = Path(__file__).resolve().parent.parent.parent / "ftm-inbox.db"
+
+
+def get_connection(db_path: Path | None = None) -> sqlite3.Connection:
+    """
+    Return a thread-local SQLite connection.
+
+    The connection is configured with:
+      - WAL journal mode for concurrent reads during writes
+      - 5-second busy timeout to handle writer contention
+      - Row factory set to sqlite3.Row for dict-like access
+      - Foreign keys enabled
+    """
+    path = db_path or DEFAULT_DB_PATH
+
+    if not hasattr(_local, "conn") or _local.conn is None:
+        _local.conn = _open_connection(path)
+
+    return _local.conn
+
+
+def _open_connection(path: Path) -> sqlite3.Connection:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(str(path), check_same_thread=False)
+    conn.row_factory = sqlite3.Row
+
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA busy_timeout=5000")
+    conn.execute("PRAGMA foreign_keys=ON")
+    conn.execute("PRAGMA synchronous=NORMAL")
+    conn.commit()
+
+    return conn
+
+
+def close_connection() -> None:
+    """Close the thread-local connection if open."""
+    if hasattr(_local, "conn") and _local.conn is not None:
+        _local.conn.close()
+        _local.conn = None

package/ftm-inbox/backend/db/schema.py

@@ -0,0 +1,78 @@
+"""
+SQLite schema definitions for ftm-inbox.
+
+Tables:
+  - inbox       : normalized items from all pollers
+  - events      : raw event log from all sources
+  - plans       : structured YAML plans linked to inbox tasks
+  - audit_log   : immutable record of every mutation performed by the executor
+"""
+
+SCHEMA_SQL = """
+CREATE TABLE IF NOT EXISTS inbox (
+    id              INTEGER PRIMARY KEY AUTOINCREMENT,
+    source          TEXT NOT NULL,
+    source_id       TEXT NOT NULL,
+    title           TEXT NOT NULL DEFAULT '',
+    body            TEXT NOT NULL DEFAULT '',
+    status          TEXT NOT NULL DEFAULT 'open',
+    priority        TEXT NOT NULL DEFAULT 'medium',
+    assignee        TEXT,
+    requester       TEXT,
+    created_at      TEXT,
+    updated_at      TEXT,
+    tags            TEXT NOT NULL DEFAULT '[]',
+    custom_fields   TEXT NOT NULL DEFAULT '{}',
+    raw_payload     TEXT NOT NULL DEFAULT '{}',
+    source_url      TEXT,
+    content_hash    TEXT NOT NULL UNIQUE,
+    ingested_at     TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+CREATE INDEX IF NOT EXISTS idx_inbox_source ON inbox(source);
+CREATE INDEX IF NOT EXISTS idx_inbox_status ON inbox(status);
+CREATE INDEX IF NOT EXISTS idx_inbox_content_hash ON inbox(content_hash);
+
+CREATE TABLE IF NOT EXISTS events (
+    id          INTEGER PRIMARY KEY AUTOINCREMENT,
+    type        TEXT NOT NULL,
+    payload     TEXT NOT NULL DEFAULT '{}',
+    created_at  TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+CREATE INDEX IF NOT EXISTS idx_events_type ON events(type);
+CREATE INDEX IF NOT EXISTS idx_events_created_at ON events(created_at);
+
+CREATE TABLE IF NOT EXISTS plans (
+    id              INTEGER PRIMARY KEY AUTOINCREMENT,
+    task_id         INTEGER NOT NULL REFERENCES inbox(id) ON DELETE CASCADE,
+    yaml_content    TEXT NOT NULL,
+    status          TEXT NOT NULL DEFAULT 'draft',
+    created_at      TEXT NOT NULL DEFAULT (datetime('now')),
+    updated_at      TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+CREATE INDEX IF NOT EXISTS idx_plans_task_id ON plans(task_id);
+CREATE INDEX IF NOT EXISTS idx_plans_status ON plans(status);
+
+CREATE TABLE IF NOT EXISTS audit_log (
+    id                  INTEGER PRIMARY KEY AUTOINCREMENT,
+    step_id             TEXT NOT NULL,
+    action_type         TEXT NOT NULL,
+    target_system       TEXT NOT NULL,
+    target_object       TEXT NOT NULL,
+    mutation_performed  TEXT NOT NULL,
+    result              TEXT NOT NULL DEFAULT '{}',
+    rollback_available  INTEGER NOT NULL DEFAULT 0,
+    created_at          TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+CREATE INDEX IF NOT EXISTS idx_audit_step_id ON audit_log(step_id);
+CREATE INDEX IF NOT EXISTS idx_audit_action_type ON audit_log(action_type);
+"""
+
+
+def initialize_schema(conn) -> None:
+    """Execute all CREATE TABLE and CREATE INDEX statements."""
+    conn.executescript(SCHEMA_SQL)
+    conn.commit()

package/ftm-inbox/backend/executor/__init__.py

@@ -0,0 +1,7 @@
+"""
+Execution engine package for ftm-inbox.
+
+Components:
+  - engine.py     : ExecutionEngine — orchestrates sequential step execution
+  - step_runner.py: StepRunner — executes a single plan step via Claude CLI
+"""

package/ftm-inbox/backend/executor/engine.py

@@ -0,0 +1,149 @@
+"""
+ExecutionEngine — orchestrates sequential execution of approved plan steps.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from datetime import datetime, timezone
+from typing import Any, Callable
+
+from backend.db.connection import get_connection
+from backend.executor.step_runner import StepRunner
+
+logger = logging.getLogger(__name__)
+
+
+class ExecutionEngine:
+    """Executes approved plan steps in sequence with pause/resume support."""
+
+    def __init__(self, task_id: int, plan_id: int) -> None:
+        self.task_id = task_id
+        self.plan_id = plan_id
+        self._paused = False
+        self._callbacks: list[Callable[[str], None]] = []
+        self._invocation_count = 0
+
+    def on_output(self, callback: Callable[[str], None]) -> None:
+        """Register a callback for streaming output."""
+        self._callbacks.append(callback)
+
+    def pause(self) -> None:
+        self._paused = True
+
+    def resume(self) -> None:
+        self._paused = False
+
+    def _emit(self, text: str) -> None:
+        for cb in self._callbacks:
+            try:
+                cb(text)
+            except Exception:
+                pass
+
+    async def execute(self) -> dict[str, Any]:
+        """Execute all approved steps in sequence."""
+        conn = get_connection()
+        steps = self._load_approved_steps(conn)
+        task_context = self._load_task_context(conn)
+
+        if not steps:
+            return {"status": "no_steps", "message": "No approved steps to execute"}
+
+        self._update_plan_status(conn, "executing")
+        results: list[dict] = []
+
+        for step in steps:
+            if self._paused:
+                self._emit("Execution paused. Use resume to continue.")
+                return {"status": "paused", "last_step_id": step["id"], "results": results}
+
+            step_id = step["id"]
+            self._update_step_status(conn, step_id, "running")
+            self._emit(f"Step {step_id}: {step.get('title', '...')}")
+
+            result = await StepRunner.run(step, task_context)
+            self._invocation_count += 1
+            results.append({"step_id": step_id, **result})
+
+            if result["status"] == "completed":
+                self._update_step_status(conn, step_id, "completed")
+                self._log_audit(conn, step, result)
+                self._emit(f"Step {step_id} completed ({result['duration_ms']}ms)")
+            else:
+                self._update_step_status(conn, step_id, "failed")
+                self._mark_dependents_blocked(conn, step_id, steps)
+                self._log_audit(conn, step, result)
+                self._emit(f"Step {step_id} failed: {result.get('error', 'unknown')}")
+                self._update_plan_status(conn, "failed")
+                return {"status": "failed", "failed_step": step_id, "results": results}
+
+        self._update_plan_status(conn, "completed")
+        self._emit(f"Execution complete. {len(steps)} steps, {self._invocation_count} CLI invocations.")
+        return {"status": "completed", "results": results, "invocations": self._invocation_count}
+
+    def _load_approved_steps(self, conn) -> list[dict]:
+        row = conn.execute(
+            "SELECT yaml_content FROM plans WHERE id = ?", (self.plan_id,)
+        ).fetchone()
+        if not row:
+            return []
+        import yaml
+        plan_data = yaml.safe_load(row["yaml_content"]) or {}
+        steps = plan_data.get("steps", [])
+        return [s for s in steps if s.get("status") in ("approved", "pending")]
+
+    def _load_task_context(self, conn) -> dict:
+        row = conn.execute(
+            "SELECT * FROM inbox WHERE id = ?", (self.task_id,)
+        ).fetchone()
+        return dict(row) if row else {}
+
+    def _update_step_status(self, conn, step_id: int, status: str) -> None:
+        row = conn.execute(
+            "SELECT yaml_content FROM plans WHERE id = ?", (self.plan_id,)
+        ).fetchone()
+        if not row:
+            return
+        import yaml
+        plan_data = yaml.safe_load(row["yaml_content"]) or {}
+        for step in plan_data.get("steps", []):
+            if step.get("id") == step_id:
+                step["status"] = status
+        updated = yaml.dump(plan_data, default_flow_style=False)
+        conn.execute(
+            "UPDATE plans SET yaml_content = ?, updated_at = datetime('now') WHERE id = ?",
+            (updated, self.plan_id),
+        )
+        conn.commit()
+
+    def _update_plan_status(self, conn, status: str) -> None:
+        conn.execute(
+            "UPDATE plans SET status = ?, updated_at = datetime('now') WHERE id = ?",
+            (status, self.plan_id),
+        )
+        conn.commit()
+
+    def _mark_dependents_blocked(self, conn, failed_step_id: int, all_steps: list[dict]) -> None:
+        for step in all_steps:
+            if step.get("id", 0) > failed_step_id and step.get("status") != "completed":
+                self._update_step_status(conn, step["id"], "blocked")
+
+    def _log_audit(self, conn, step: dict, result: dict) -> None:
+        conn.execute(
+            """INSERT INTO audit_log
+               (step_id, action_type, target_system, target_object,
+                mutation_performed, result, rollback_available)
+            VALUES (?, ?, ?, ?, ?, ?, ?)""",
+            (
+                str(step.get("id", "")),
+                "execute_step",
+                step.get("target_system", ""),
+                step.get("title", ""),
+                result.get("output", "")[:500],
+                json.dumps({"status": result["status"], "duration_ms": result.get("duration_ms", 0)}),
+                1 if step.get("rollback") else 0,
+            ),
+        )
+        conn.commit()

package/ftm-inbox/backend/executor/step_runner.py

@@ -0,0 +1,98 @@
+"""
+StepRunner — executes a single plan step via Claude CLI.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import subprocess
+import time
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_STEP_TIMEOUT = 120  # seconds
+
+
+class StepRunner:
+    """Runs a single plan step by invoking Claude CLI."""
+
+    @staticmethod
+    async def run(step: dict, task_context: dict) -> dict[str, Any]:
+        """
+        Execute a plan step and return structured result.
+
+        Returns:
+            {"status": "completed"|"failed", "output": str, "error": str, "duration_ms": int}
+        """
+        prompt = (
+            f"Execute this plan step:\n\n"
+            f"Step: {step.get('title', '')}\n"
+            f"Target system: {step.get('target_system', 'local')}\n"
+            f"Method: {step.get('method_primary', '')}\n"
+            f"Fallback: {step.get('method_fallback', '')}\n\n"
+            f"Task context:\n"
+            f"Title: {task_context.get('title', '')}\n"
+            f"Source: {task_context.get('source', '')}\n"
+            f"Body: {task_context.get('body', '')}\n\n"
+            f"Execute the step and report what was done. "
+            f"If the action cannot be performed, explain why."
+        )
+
+        start = time.monotonic()
+        try:
+            result = subprocess.run(
+                ["claude", "-p", prompt, "--output-format", "json"],
+                capture_output=True,
+                text=True,
+                timeout=_STEP_TIMEOUT,
+            )
+            duration_ms = int((time.monotonic() - start) * 1000)
+
+            if result.returncode != 0:
+                return {
+                    "status": "failed",
+                    "output": result.stdout,
+                    "error": result.stderr or f"Exit code {result.returncode}",
+                    "duration_ms": duration_ms,
+                }
+
+            # Parse Claude JSON output
+            try:
+                data = json.loads(result.stdout)
+                text = data.get("result", result.stdout)
+            except json.JSONDecodeError:
+                text = result.stdout
+
+            return {
+                "status": "completed",
+                "output": text,
+                "error": "",
+                "duration_ms": duration_ms,
+            }
+
+        except subprocess.TimeoutExpired:
+            duration_ms = int((time.monotonic() - start) * 1000)
+            return {
+                "status": "failed",
+                "output": "",
+                "error": f"Step timed out after {_STEP_TIMEOUT}s",
+                "duration_ms": duration_ms,
+            }
+        except FileNotFoundError:
+            duration_ms = int((time.monotonic() - start) * 1000)
+            return {
+                "status": "failed",
+                "output": "",
+                "error": "Claude CLI not found on PATH",
+                "duration_ms": duration_ms,
+            }
+        except Exception as exc:
+            duration_ms = int((time.monotonic() - start) * 1000)
+            return {
+                "status": "failed",
+                "output": "",
+                "error": str(exc),
+                "duration_ms": duration_ms,
+            }