feed-the-machine 1.6.1 → 1.7.0

package/bin/playbook_engine/test_models.py

@@ -0,0 +1,166 @@
+import pytest
+import yaml
+import tempfile
+import os
+from models import Playbook, PlaybookStep, ActionBinding, ParamSource
+
+def test_playbook_from_yaml():
+    raw = {
+        "id": "sso-onboarding",
+        "name": "SSO Onboarding",
+        "version": 1,
+        "confidence": "low",
+        "trigger_patterns": [
+            {"ticket_type": "Service Request", "keywords": ["SSO", "SAML"], "source": ["freshservice"]}
+        ],
+        "created_from": "session",
+        "executions": 0,
+        "steps": [
+            {
+                "id": 1,
+                "name": "Create Jira ticket",
+                "action": {
+                    "tool": "mcp__mcp-atlassian__jira_create_issue",
+                    "params": {"project": "ITWORK2", "summary": "[SSO] {{app_name}}"},
+                    "param_sources": {"app_name": {"from": "trigger_ticket", "field": "subject", "extract": "app name"}}
+                },
+                "auth_required": False,
+                "human_required": False,
+            }
+        ],
+    }
+    pb = Playbook.from_dict(raw)
+    assert pb.id == "sso-onboarding"
+    assert pb.confidence == "low"
+    assert len(pb.steps) == 1
+    assert pb.steps[0].action.tool == "mcp__mcp-atlassian__jira_create_issue"
+    assert pb.steps[0].action.param_sources["app_name"].field == "subject"
+
+def test_playbook_round_trip_yaml(tmp_path):
+    raw = {
+        "id": "test-pb",
+        "name": "Test Playbook",
+        "version": 1,
+        "confidence": "medium",
+        "trigger_patterns": [],
+        "created_from": "dictated",
+        "executions": 0,
+        "steps": [],
+    }
+    pb = Playbook.from_dict(raw)
+    path = tmp_path / "test-pb.yml"
+    pb.save(str(path))
+    loaded = Playbook.load(str(path))
+    assert loaded.id == pb.id
+    assert loaded.version == pb.version
+
+def test_playbook_matches_ticket():
+    raw = {
+        "id": "sso",
+        "name": "SSO",
+        "version": 1,
+        "confidence": "high",
+        "trigger_patterns": [
+            {"ticket_type": "Service Request", "keywords": ["SSO", "SAML"], "source": ["freshservice"]}
+        ],
+        "created_from": "session",
+        "executions": 3,
+        "steps": [],
+    }
+    pb = Playbook.from_dict(raw)
+    assert pb.matches(ticket_type="Service Request", text="Set up SSO for Linear", source="freshservice")
+    assert not pb.matches(ticket_type="Incident", text="Server is down", source="freshservice")
+    assert not pb.matches(ticket_type="Service Request", text="New laptop request", source="freshservice")
+
+def test_confidence_progression():
+    raw = {
+        "id": "t",
+        "name": "T",
+        "version": 1,
+        "confidence": "low",
+        "trigger_patterns": [],
+        "created_from": "session",
+        "executions": 0,
+        "steps": [],
+    }
+    pb = Playbook.from_dict(raw)
+    pb.record_execution(success=True)
+    assert pb.confidence == "medium"
+    assert pb.executions == 1
+    pb.record_execution(success=True)
+    pb.record_execution(success=True)
+    assert pb.confidence == "high"
+    assert pb.executions == 3
+
+def test_confidence_degradation_on_failure():
+    raw = {
+        "id": "t",
+        "name": "T",
+        "version": 1,
+        "confidence": "high",
+        "trigger_patterns": [],
+        "created_from": "session",
+        "executions": 5,
+        "steps": [],
+    }
+    pb = Playbook.from_dict(raw)
+    pb.record_execution(success=False)
+    assert pb.confidence == "medium"
+    pb.record_execution(success=False)
+    assert pb.confidence == "low"
+    # Already at low — should stay at low
+    pb.record_execution(success=False)
+    assert pb.confidence == "low"
+
+def test_empty_trigger_patterns_match_everything():
+    """Playbook with no trigger patterns should NOT match any ticket."""
+    raw = {
+        "id": "t",
+        "name": "T",
+        "version": 1,
+        "confidence": "low",
+        "trigger_patterns": [],
+        "created_from": "session",
+        "executions": 0,
+        "steps": [],
+    }
+    pb = Playbook.from_dict(raw)
+    # Empty trigger list means no patterns match (any() on empty returns False)
+    assert not pb.matches(text="anything", source="freshservice")
+
+def test_to_dict_from_dict_round_trip():
+    """Verify full round-trip serialization symmetry."""
+    raw = {
+        "id": "round-trip",
+        "name": "Round Trip Test",
+        "version": 2,
+        "confidence": "medium",
+        "trigger_patterns": [
+            {"ticket_type": "Service Request", "keywords": ["SSO"], "source": ["freshservice"]}
+        ],
+        "created_from": "dictated",
+        "executions": 5,
+        "steps": [
+            {
+                "id": 1,
+                "name": "Do something",
+                "action": {
+                    "tool": "test_tool",
+                    "params": {"key": "value"},
+                    "param_sources": {"key": {"from": "trigger_ticket", "field": "subject"}},
+                },
+                "auth_required": True,
+                "auth_method": "stored_session",
+                "human_required": False,
+                "optional": True,
+            }
+        ],
+        "last_executed": "2026-01-01T00:00:00Z",
+        "last_updated": "2026-01-02T00:00:00Z",
+        "update_history": [{"version": 2, "reason": "test"}],
+    }
+    pb = Playbook.from_dict(raw)
+    d = pb.to_dict()
+    pb2 = Playbook.from_dict(d)
+    d2 = pb2.to_dict()
+    assert d == d2

package/bin/playbook_engine/test_registry.py

@@ -0,0 +1,67 @@
+import pytest
+import yaml
+import tempfile
+import os
+from registry import ToolRegistry
+
+def test_load_registry(tmp_path):
+    reg_dir = tmp_path / "tool-registry"
+    reg_dir.mkdir()
+
+    (reg_dir / "_registry.yml").write_text(yaml.dump({
+        "tools": {
+            "jira": {
+                "type": "mcp",
+                "prefix": "mcp__mcp-atlassian__jira_",
+                "capabilities": ["create_issue"],
+                "auth": "persistent",
+                "domains": ["ticket_management"],
+            }
+        }
+    }))
+
+    (reg_dir / "jira.defaults.yml").write_text(yaml.dump({
+        "create_issue": {"assignee": "test@test.com", "board_id": 70},
+        "field_mappings": {"sprint_field": "customfield_10020"},
+    }))
+
+    reg = ToolRegistry(str(reg_dir))
+    assert "jira" in reg.tools
+    assert reg.tools["jira"]["type"] == "mcp"
+    assert reg.get_defaults("jira", "create_issue")["assignee"] == "test@test.com"
+
+def test_get_defaults_missing_tool(tmp_path):
+    reg_dir = tmp_path / "tool-registry"
+    reg_dir.mkdir()
+    (reg_dir / "_registry.yml").write_text(yaml.dump({"tools": {}}))
+    reg = ToolRegistry(str(reg_dir))
+    assert reg.get_defaults("nonexistent", "action") == {}
+
+def test_merge_params(tmp_path):
+    reg_dir = tmp_path / "tool-registry"
+    reg_dir.mkdir()
+    (reg_dir / "_registry.yml").write_text(yaml.dump({
+        "tools": {"jira": {"type": "mcp", "prefix": "p_", "capabilities": [], "auth": "persistent", "domains": []}}
+    }))
+    (reg_dir / "jira.defaults.yml").write_text(yaml.dump({
+        "create_issue": {"assignee": "default@test.com", "board_id": 70},
+    }))
+    reg = ToolRegistry(str(reg_dir))
+    merged = reg.merge_params("jira", "create_issue", {"summary": "Test", "assignee": "override@test.com"})
+    assert merged["assignee"] == "override@test.com"  # playbook overrides default
+    assert merged["board_id"] == 70  # default fills in
+    assert merged["summary"] == "Test"  # playbook-specific preserved
+
+def test_resolve_tool_from_mcp_name(tmp_path):
+    reg_dir = tmp_path / "tool-registry"
+    reg_dir.mkdir()
+    (reg_dir / "_registry.yml").write_text(yaml.dump({
+        "tools": {
+            "jira": {"type": "mcp", "prefix": "mcp__mcp-atlassian__jira_", "capabilities": [], "auth": "persistent", "domains": []},
+            "slack": {"type": "mcp", "prefix": "mcp__slack__", "capabilities": [], "auth": "persistent", "domains": []},
+        }
+    }))
+    reg = ToolRegistry(str(reg_dir))
+    assert reg.resolve_tool_name("mcp__mcp-atlassian__jira_create_issue") == ("jira", "create_issue")
+    assert reg.resolve_tool_name("mcp__slack__post_message") == ("slack", "post_message")
+    assert reg.resolve_tool_name("unknown_tool") == (None, None)

package/bin/playbook_engine/test_tracer.py

@@ -0,0 +1,86 @@
+import pytest
+import json
+import os
+from tracer import WorkflowTracer, TraceEvent
+
+def test_add_tool_call_event(tmp_path):
+    tracer = WorkflowTracer(traces_dir=str(tmp_path))
+    tracer.start_trace("ITWORK2-1234")
+    tracer.add_event(TraceEvent(
+        type="tool_call",
+        tool="mcp__mcp-atlassian__jira_create_issue",
+        params={"project": "ITWORK2", "summary": "Test"},
+    ))
+    trace = tracer.get_trace("ITWORK2-1234")
+    assert len(trace["events"]) == 1
+    assert trace["events"][0]["type"] == "tool_call"
+
+def test_add_user_instruction_event(tmp_path):
+    tracer = WorkflowTracer(traces_dir=str(tmp_path))
+    tracer.start_trace("ITWORK2-1234")
+    tracer.add_event(TraceEvent(
+        type="user_instruction",
+        content="Always set due date to 30 days out",
+        applies_to=["jira.create_issue"],
+        persist=True,
+    ))
+    trace = tracer.get_trace("ITWORK2-1234")
+    assert trace["events"][0]["type"] == "user_instruction"
+    assert trace["events"][0]["persist"] is True
+
+def test_add_user_correction_event(tmp_path):
+    tracer = WorkflowTracer(traces_dir=str(tmp_path))
+    tracer.start_trace("ITWORK2-1234")
+    tracer.add_event(TraceEvent(
+        type="user_correction",
+        content="No, assign to next sprint",
+        corrects="tool_defaults.jira.create_issue.sprint",
+        new_value="next",
+    ))
+    trace = tracer.get_trace("ITWORK2-1234")
+    assert trace["events"][0]["corrects"] == "tool_defaults.jira.create_issue.sprint"
+
+def test_add_manual_action_event(tmp_path):
+    tracer = WorkflowTracer(traces_dir=str(tmp_path))
+    tracer.start_trace("ITWORK2-1234")
+    tracer.add_event(TraceEvent(
+        type="user_manual_action",
+        content="I configured SAML in Okta",
+        inferred_step="Configure SAML in IdP",
+        action_binding="playwright",
+        auth_note="needs Okta admin",
+    ))
+    trace = tracer.get_trace("ITWORK2-1234")
+    assert trace["events"][0]["action_binding"] == "playwright"
+
+def test_trace_persists_to_disk(tmp_path):
+    tracer = WorkflowTracer(traces_dir=str(tmp_path))
+    tracer.start_trace("ITWORK2-5678")
+    tracer.add_event(TraceEvent(type="tool_call", tool="test_tool"))
+    tracer.flush("ITWORK2-5678")
+    path = tmp_path / "active" / "ITWORK2-5678.json"
+    assert path.exists()
+    with open(path) as f:
+        data = json.load(f)
+    assert len(data["events"]) == 1
+
+def test_similarity_score(tmp_path):
+    tracer = WorkflowTracer(traces_dir=str(tmp_path))
+    tracer.start_trace("t1")
+    tracer.add_event(TraceEvent(type="tool_call", tool="jira_create"))
+    tracer.add_event(TraceEvent(type="tool_call", tool="slack_post"))
+    tracer.add_event(TraceEvent(type="tool_call", tool="freshservice_update"))
+
+    tracer.start_trace("t2")
+    tracer.add_event(TraceEvent(type="tool_call", tool="jira_create"))
+    tracer.add_event(TraceEvent(type="tool_call", tool="slack_post"))
+    tracer.add_event(TraceEvent(type="tool_call", tool="freshservice_update"))
+
+    score = tracer.similarity("t1", "t2")
+    assert score >= 0.9  # nearly identical tool sequences
+
+def test_from_dict_ignores_unknown_keys(tmp_path):
+    """TraceEvent.from_dict should ignore unknown keys gracefully."""
+    event = TraceEvent.from_dict({"type": "tool_call", "tool": "test", "unknown_field": "ignored"})
+    assert event.type == "tool_call"
+    assert event.tool == "test"

package/bin/playbook_engine/tracer.py

@@ -0,0 +1,93 @@
+"""WorkflowTracer - captures tool call traces for playbook extraction."""
+
+import json
+import os
+import time
+from dataclasses import dataclass, field, asdict
+from typing import List, Optional
+
+
+@dataclass
+class TraceEvent:
+    """A single event in a workflow trace."""
+    timestamp: float = field(default_factory=time.time)
+    tool: str = ""
+    action: str = ""
+    params: dict = field(default_factory=dict)
+    result_summary: str = ""
+    human_instruction: str = ""
+    correction: str = ""
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "TraceEvent":
+        return cls(**{k: v for k, v in d.items() if k in cls.__dataclass_fields__})
+
+
+@dataclass
+class Trace:
+    """A complete workflow trace."""
+    id: str = ""
+    events: List[TraceEvent] = field(default_factory=list)
+    started: float = field(default_factory=time.time)
+    completed: float = 0.0
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "events": [e.to_dict() for e in self.events],
+            "started": self.started,
+            "completed": self.completed,
+        }
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "Trace":
+        return cls(
+            id=d["id"],
+            events=[TraceEvent.from_dict(e) for e in d.get("events", [])],
+            started=d.get("started", 0),
+            completed=d.get("completed", 0),
+        )
+
+
+class WorkflowTracer:
+    """Manages workflow traces on disk."""
+
+    def __init__(self, traces_dir: str):
+        self.traces_dir = traces_dir
+        os.makedirs(traces_dir, exist_ok=True)
+        self._traces: dict = {}
+
+    def _path(self, trace_id: str) -> str:
+        return os.path.join(self.traces_dir, f"{trace_id}.json")
+
+    def start_trace(self, trace_id: str) -> Trace:
+        trace = Trace(id=trace_id)
+        self._traces[trace_id] = trace
+        return trace
+
+    def load_trace(self, trace_id: str) -> Optional[Trace]:
+        if trace_id in self._traces:
+            return self._traces[trace_id]
+        path = self._path(trace_id)
+        if not os.path.exists(path):
+            return None
+        with open(path) as fh:
+            trace = Trace.from_dict(json.load(fh))
+            self._traces[trace_id] = trace
+            return trace
+
+    def add_event(self, event: TraceEvent):
+        # Add to the most recent trace
+        for trace in reversed(list(self._traces.values())):
+            trace.events.append(event)
+            return
+
+    def flush(self, trace_id: str):
+        trace = self._traces.get(trace_id)
+        if not trace:
+            return
+        with open(self._path(trace_id), "w") as fh:
+            json.dump(trace.to_dict(), fh, indent=2)