@openlucaskaka/kagent 0.1.7

package/src/kagent/core/tools.py

@@ -0,0 +1,196 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Callable, Dict, List, Match, Optional, Pattern
+
+ToolResult = Dict[str, str]
+ToolHandler = Callable[[Match[str]], ToolResult]
+
+
+@dataclass(frozen=True)
+class ToolSpec:
+    name: str
+    command: str
+    description: str
+    example: str
+    pattern: Pattern[str]
+    handler: ToolHandler
+
+
+def execute_step(step: str) -> Optional[ToolResult]:
+    for tool in TOOLS:
+        match = tool.pattern.fullmatch(step)
+        if match:
+            return tool.handler(match)
+    return None
+
+
+def registered_tool_names() -> List[str]:
+    return [tool.name for tool in TOOLS]
+
+
+def registered_tool_metadata() -> List[Dict[str, str]]:
+    return [
+        {
+            "name": tool.name,
+            "command": tool.command,
+            "description": tool.description,
+            "example": tool.example,
+        }
+        for tool in TOOLS
+    ]
+
+
+def matching_tool_name(step: str) -> Optional[str]:
+    for tool in TOOLS:
+        if tool.pattern.fullmatch(step):
+            return tool.name
+    return None
+
+
+def expected_answer(step: Optional[str]) -> Optional[str]:
+    if step is None:
+        return None
+
+    execution = execute_step(step)
+    if execution is None:
+        return None
+    return execution["output"]
+
+
+def _calculate(match: Match[str]) -> ToolResult:
+    left, right = match.groups()
+    return _numeric_result("calculate_sum", match, int(left) + int(right))
+
+
+def _multiply(match: Match[str]) -> ToolResult:
+    left, right = match.groups()
+    return _numeric_result("multiply_numbers", match, int(left) * int(right))
+
+
+def _subtract(match: Match[str]) -> ToolResult:
+    left, right = match.groups()
+    return _numeric_result("subtract_numbers", match, int(left) - int(right))
+
+
+def _numeric_result(tool: str, match: Match[str], output: int) -> ToolResult:
+    return {
+        "tool": tool,
+        "input": match.group(0),
+        "output": str(output),
+    }
+
+
+def _count_words(match: Match[str]) -> ToolResult:
+    text = match.group(1)
+    return _text_result("count_words", text, str(len([word for word in text.split() if word])))
+
+
+def _lowercase_text(match: Match[str]) -> ToolResult:
+    text = match.group(1)
+    return _text_result("lowercase_text", text, text.lower())
+
+
+def _uppercase_text(match: Match[str]) -> ToolResult:
+    text = match.group(1)
+    return _text_result("uppercase_text", text, text.upper())
+
+
+def _reverse_text(match: Match[str]) -> ToolResult:
+    text = match.group(1)
+    return _text_result("reverse_text", text, text[::-1])
+
+
+def _trim_text(match: Match[str]) -> ToolResult:
+    text = match.group(1)
+    return _text_result("trim_text", text, text.strip())
+
+
+def _text_result(tool: str, text: str, output: str) -> ToolResult:
+    return {
+        "tool": tool,
+        "input": text,
+        "output": output,
+    }
+
+
+def _text_tool(
+    name: str,
+    command: str,
+    description: str,
+    example: str,
+    handler: ToolHandler,
+) -> ToolSpec:
+    return ToolSpec(
+        name=name,
+        command=f"{command} text in 'text'",
+        description=description,
+        example=example,
+        pattern=re.compile(rf"{command}\s+text\s+in\s+['\"](.*)['\"]"),
+        handler=handler,
+    )
+
+
+TOOLS = [
+    ToolSpec(
+        name="calculate_sum",
+        command="calculate N + M",
+        description="Add two integers.",
+        example="calculate 2 + 3",
+        pattern=re.compile(r"calculate\s+(-?\d+)\s*\+\s*(-?\d+)"),
+        handler=_calculate,
+    ),
+    ToolSpec(
+        name="count_words",
+        command="count words in 'text'",
+        description="Count whitespace-separated words in quoted text.",
+        example="count words in 'ship small reliable agents'",
+        pattern=re.compile(r"count\s+words\s+in\s+['\"](.*)['\"]"),
+        handler=_count_words,
+    ),
+    _text_tool(
+        "lowercase_text",
+        "lowercase",
+        "Convert quoted text to lowercase.",
+        "lowercase text in 'Agent Loop'",
+        _lowercase_text,
+    ),
+    ToolSpec(
+        name="multiply_numbers",
+        command="multiply N * M",
+        description="Multiply two integers.",
+        example="multiply 6 * 7",
+        pattern=re.compile(r"multiply\s+(-?\d+)\s*\*\s*(-?\d+)"),
+        handler=_multiply,
+    ),
+    _text_tool(
+        "reverse_text",
+        "reverse",
+        "Reverse quoted text while preserving character case.",
+        "reverse text in 'Agent Loop'",
+        _reverse_text,
+    ),
+    ToolSpec(
+        name="subtract_numbers",
+        command="subtract N - M",
+        description="Subtract the right integer from the left integer.",
+        example="subtract 10 - 4",
+        pattern=re.compile(r"subtract\s+(-?\d+)\s*-\s*(-?\d+)"),
+        handler=_subtract,
+    ),
+    _text_tool(
+        "trim_text",
+        "trim",
+        "Remove surrounding whitespace from quoted text.",
+        "trim text in '  agent loop  '",
+        _trim_text,
+    ),
+    _text_tool(
+        "uppercase_text",
+        "uppercase",
+        "Convert quoted text to uppercase.",
+        "uppercase text in 'agent loop'",
+        _uppercase_text,
+    ),
+]

package/src/kagent/core/trace.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from kagent.core.state import AgentState
+
+
+def record_node_event(state: AgentState, node: str) -> None:
+    state["events"].append(
+        {
+            "node": node,
+            "status": state["status"].value,
+            "retry_count": state["retry_count"],
+            "step_retry_count": state.get("step_retry_count", 0),
+        }
+    )
+
+
+def record_execution_attempt(
+    state: AgentState,
+    step: str,
+    tool: str,
+    output: str,
+    fault: str,
+) -> None:
+    state["execution_attempts"].append(
+        {
+            "step": step,
+            "tool": tool,
+            "output": output,
+            "fault": fault,
+            "retry": str(state["step_retry_count"]),
+        }
+    )
+
+
+def copy_agent_state(state: AgentState) -> AgentState:
+    copied = dict(state)
+    copied["plan"] = list(state.get("plan", []))
+    copied["plan_validations"] = [
+        dict(item) for item in state.get("plan_validations", [])
+    ]
+    copied["step_results"] = list(state.get("step_results", []))
+    copied["tool_calls"] = [dict(item) for item in state.get("tool_calls", [])]
+    copied["execution_attempts"] = [
+        dict(item) for item in state.get("execution_attempts", [])
+    ]
+    copied["errors"] = list(state.get("errors", []))
+    copied["events"] = [dict(item) for item in state.get("events", [])]
+    copied["reflection_notes"] = list(state.get("reflection_notes", []))
+    copied["reflections"] = [dict(item) for item in state.get("reflections", [])]
+    copied["verification_results"] = [
+        dict(item) for item in state.get("verification_results", [])
+    ]
+    copied["fault_plan"] = {
+        key: list(value) for key, value in state.get("fault_plan", {}).items()
+    }
+    copied["fault_counters"] = dict(state.get("fault_counters", {}))
+    return copied

package/src/kagent/eval/__init__.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+__all__ = ["evaluate_agent", "registered_evaluation_cases"]
+
+
+def __getattr__(name: str):
+    if name in __all__:
+        from kagent.eval import evaluator
+
+        return getattr(evaluator, name)
+    raise AttributeError(name)

package/src/kagent/eval/cases.py

@@ -0,0 +1,229 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, List
+
+CaseCheck = Callable[[Dict[str, Any]], bool]
+
+
+@dataclass(frozen=True)
+class EvaluationCase:
+    name: str
+    category: str
+    run: Callable[[], Dict[str, Any]]
+    check: CaseCheck
+
+
+def build_evaluation_cases(AgentConfig, AgentStatus, run_agent) -> List[EvaluationCase]:
+    return [
+        EvaluationCase(
+            "multi_step_success",
+            "workflow",
+            lambda: run_agent(
+                "calculate 2 + 3 then count words in 'ship small reliable agents'",
+                config=AgentConfig(max_steps=4, max_retries=1),
+            ),
+            lambda result: (
+                result["status"] == AgentStatus.DONE
+                and result["step_results"] == ["5", "4"]
+                and result["retry_count"] == 0
+                and [item["passed"] for item in result["verification_results"]]
+                == ["true", "true"]
+                and [item["tool"] for item in result["plan_validations"]]
+                == ["calculate_sum", "count_words"]
+            ),
+        ),
+        EvaluationCase(
+            "self_correction_success",
+            "recovery",
+            lambda: run_agent(
+                "calculate 2 + 3",
+                config=AgentConfig(max_steps=3, max_retries=2),
+                fault_plan={"calculate 2 + 3": ["wrong-answer"]},
+            ),
+            lambda result: (
+                result["status"] == AgentStatus.DONE
+                and result["answer"] == "5"
+                and result["retry_count"] == 1
+                and result["reflections"][-1]["actual"] == "6"
+                and result["reflections"][-1]["expected"] == "5"
+                and [item["passed"] for item in result["verification_results"]]
+                == ["false", "true"]
+            ),
+        ),
+        EvaluationCase(
+            "retry_budget_failure",
+            "failure",
+            lambda: run_agent(
+                "calculate 2 + 3",
+                config=AgentConfig(max_steps=3, max_retries=1),
+                fault_plan={"calculate 2 + 3": ["wrong-answer", "wrong-answer"]},
+            ),
+            lambda result: (
+                result["status"] == AgentStatus.FAILED
+                and result["errors"][-1] == "retry budget exhausted"
+            ),
+        ),
+        EvaluationCase(
+            "uppercase_tool_success",
+            "tool",
+            lambda: run_agent(
+                "uppercase text in 'agent loop'",
+                config=AgentConfig(max_steps=3, max_retries=1),
+            ),
+            lambda result: (
+                result["status"] == AgentStatus.DONE
+                and result["answer"] == "AGENT LOOP"
+                and result["tool_calls"][-1]["tool"] == "uppercase_text"
+            ),
+        ),
+        EvaluationCase(
+            "unsupported_plan_failure",
+            "failure",
+            lambda: run_agent(
+                "calculate 1 + 1 then search the web",
+                config=AgentConfig(max_steps=4, max_retries=2),
+            ),
+            lambda result: (
+                result["status"] == AgentStatus.FAILED
+                and result["errors"] == ["unsupported planned step: search the web"]
+                and result["retry_count"] == 0
+                and [event["node"] for event in result["events"]] == ["planner"]
+                and result["plan_validations"][-1]["supported"] == "false"
+            ),
+        ),
+        EvaluationCase(
+            "empty_answer_recovery",
+            "recovery",
+            lambda: run_agent(
+                "uppercase text in 'agent loop'",
+                config=AgentConfig(max_steps=4, max_retries=2),
+                fault_plan={"uppercase text in 'agent loop'": ["empty-answer"]},
+            ),
+            lambda result: (
+                result["status"] == AgentStatus.DONE
+                and result["answer"] == "AGENT LOOP"
+                and result["reflections"][-1]["actual"] == ""
+                and result["reflections"][-1]["reason"] == "answer was empty"
+                and [item["passed"] for item in result["verification_results"]]
+                == ["false", "true"]
+                and [item["fault"] for item in result["execution_attempts"]]
+                == ["empty-answer", ""]
+                and result["execution_attempts"][-1]["tool"] == "uppercase_text"
+            ),
+        ),
+        EvaluationCase(
+            "per_step_retry_budget_success",
+            "recovery",
+            lambda: run_agent(
+                "calculate 2 + 3 then uppercase text in 'agent loop'",
+                config=AgentConfig(max_steps=4, max_retries=1),
+                fault_plan={
+                    "calculate 2 + 3": ["wrong-answer"],
+                    "uppercase text in 'agent loop'": ["empty-answer"],
+                },
+            ),
+            lambda result: (
+                result["status"] == AgentStatus.DONE
+                and result["retry_count"] == 2
+                and result["step_retry_count"] == 0
+                and result["step_results"] == ["5", "AGENT LOOP"]
+                and [item["retry"] for item in result["verification_results"]]
+                == ["0", "1", "0", "1"]
+            ),
+        ),
+        EvaluationCase(
+            "multiplication_tool_success",
+            "tool",
+            lambda: run_agent(
+                "multiply 6 * 7",
+                config=AgentConfig(max_steps=3, max_retries=1),
+            ),
+            lambda result: (
+                result["status"] == AgentStatus.DONE
+                and result["answer"] == "42"
+                and result["tool_calls"][-1]["tool"] == "multiply_numbers"
+            ),
+        ),
+        EvaluationCase(
+            "reverse_text_tool_success",
+            "tool",
+            lambda: run_agent(
+                "Reverse Text in 'Agent Loop'",
+                config=AgentConfig(max_steps=3, max_retries=1),
+            ),
+            lambda result: (
+                result["status"] == AgentStatus.DONE
+                and result["answer"] == "pooL tnegA"
+                and result["tool_calls"][-1]["tool"] == "reverse_text"
+            ),
+        ),
+        EvaluationCase(
+            "lowercase_text_tool_success",
+            "tool",
+            lambda: run_agent(
+                "Lowercase Text in 'Agent Loop'",
+                config=AgentConfig(max_steps=3, max_retries=1),
+            ),
+            lambda result: (
+                result["status"] == AgentStatus.DONE
+                and result["answer"] == "agent loop"
+                and result["tool_calls"][-1]["tool"] == "lowercase_text"
+            ),
+        ),
+        EvaluationCase(
+            "trim_text_tool_success",
+            "tool",
+            lambda: run_agent(
+                "Trim Text in '  Agent Loop  '",
+                config=AgentConfig(max_steps=3, max_retries=1),
+            ),
+            lambda result: (
+                result["status"] == AgentStatus.DONE
+                and result["answer"] == "Agent Loop"
+                and result["tool_calls"][-1]["tool"] == "trim_text"
+            ),
+        ),
+        EvaluationCase(
+            "subtraction_tool_success",
+            "tool",
+            lambda: run_agent(
+                "subtract 10 - 4",
+                config=AgentConfig(max_steps=3, max_retries=1),
+            ),
+            lambda result: (
+                result["status"] == AgentStatus.DONE
+                and result["answer"] == "6"
+                and result["tool_calls"][-1]["tool"] == "subtract_numbers"
+            ),
+        ),
+        EvaluationCase(
+            "tool_error_recovery",
+            "recovery",
+            lambda: run_agent(
+                "reverse text in 'Agent Loop'",
+                config=AgentConfig(max_steps=4, max_retries=2),
+                fault_plan={"reverse text in 'Agent Loop'": ["tool-error"]},
+            ),
+            lambda result: (
+                result["status"] == AgentStatus.DONE
+                and result["answer"] == "pooL tnegA"
+                and result["reflections"][-1]["reason"] == "tool execution failed"
+                and [item["fault"] for item in result["execution_attempts"]]
+                == ["tool-error", ""]
+            ),
+        ),
+        EvaluationCase(
+            "empty_plan_failure",
+            "failure",
+            lambda: run_agent(
+                "   ",
+                config=AgentConfig(max_steps=3, max_retries=1),
+            ),
+            lambda result: (
+                result["status"] == AgentStatus.FAILED
+                and result["errors"] == ["empty plan"]
+                and [event["node"] for event in result["events"]] == ["planner"]
+            ),
+        ),
+    ]