@openlucaskaka/kagent 0.1.7

package/src/kagent/eval/evaluator.py

@@ -0,0 +1,216 @@
+from __future__ import annotations
+
+import argparse
+import contextlib
+import io
+import time
+import warnings
+from typing import Any, Callable, Dict, List
+
+from kagent.core.invariants import validate_run_invariants
+from kagent.core.summary import summarize_run
+from kagent.eval.cases import (
+    CaseCheck,
+    EvaluationCase,
+    build_evaluation_cases,
+)
+from kagent.utils.json_output import format_and_write_json
+
+
+def evaluate_agent(category: str = "", case_name: str = "") -> Dict[str, Any]:
+    started_at = time.perf_counter()
+    warning_sink = io.StringIO()
+    with contextlib.redirect_stderr(warning_sink), warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        from kagent.core.agent import AgentConfig, AgentStatus, run_agent
+
+        all_cases = build_evaluation_cases(AgentConfig, AgentStatus, run_agent)
+        _validate_case_filters(all_cases, category=category, case_name=case_name)
+        selected_cases = _filter_cases(
+            all_cases,
+            category=category,
+            case_name=case_name,
+        )
+        cases = [
+            _run_case(item.name, item.category, item.run, item.check)
+            for item in selected_cases
+        ]
+    passed = sum(1 for case in cases if case["passed"])
+    recovered_cases = _recovered_case_count(cases)
+    return {
+        "passed": passed,
+        "failed": len(cases) - passed,
+        "filters": {"category": category, "case": case_name},
+        "recovered_cases": str(recovered_cases),
+        "recovery_rate": f"{recovered_cases / len(cases):.2f}" if cases else "0.00",
+        "category_counts": _category_counts(cases),
+        "duration_seconds": _duration_since(started_at),
+        "slowest_case": _slowest_case_name(cases),
+        "cases": cases,
+    }
+
+
+def registered_evaluation_cases() -> List[Dict[str, str]]:
+    warning_sink = io.StringIO()
+    with contextlib.redirect_stderr(warning_sink), warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        from kagent.core.agent import AgentConfig, AgentStatus, run_agent
+
+        return [
+            {"name": case.name, "category": case.category}
+            for case in build_evaluation_cases(AgentConfig, AgentStatus, run_agent)
+        ]
+
+
+def _validate_case_filters(
+    cases: List[EvaluationCase],
+    *,
+    category: str,
+    case_name: str,
+) -> None:
+    if category and category not in {case.category for case in cases}:
+        raise ValueError(f"unknown evaluator category: {category}")
+    if case_name and case_name not in {case.name for case in cases}:
+        raise ValueError(f"unknown evaluator case: {case_name}")
+
+
+def _filter_cases(
+    cases: List[EvaluationCase],
+    *,
+    category: str,
+    case_name: str,
+) -> List[EvaluationCase]:
+    selected = cases
+    if category:
+        selected = [case for case in selected if case.category == category]
+    if case_name:
+        selected = [case for case in selected if case.name == case_name]
+    return selected
+
+
+def _run_case(
+    name: str,
+    category: str,
+    run: Callable[[], Dict[str, Any]],
+    check: CaseCheck,
+) -> Dict[str, Any]:
+    started_at = time.perf_counter()
+    try:
+        result = run()
+    except Exception as exc:
+        return {
+            "name": name,
+            "category": category,
+            "passed": False,
+            "status": "error",
+            "answer": None,
+            "retry_count": 0,
+            "events": [],
+            "invariant_errors": [],
+            "summary": {},
+            "error": str(exc),
+            "duration_seconds": _duration_since(started_at),
+        }
+
+    invariant_errors = validate_run_invariants(result)
+    return {
+        "name": name,
+        "category": category,
+        "passed": check(result) and not invariant_errors,
+        "status": result["status"].value,
+        "answer": result.get("answer"),
+        "retry_count": result.get("retry_count", 0),
+        "events": [event["node"] for event in result.get("events", [])],
+        "invariant_errors": invariant_errors,
+        "summary": summarize_run(result),
+        "error": "",
+        "duration_seconds": _duration_since(started_at),
+    }
+
+
+def _duration_since(started_at: float) -> str:
+    return f"{time.perf_counter() - started_at:.4f}"
+
+
+def _slowest_case_name(cases: List[Dict[str, Any]]) -> str:
+    if not cases:
+        return ""
+    return max(cases, key=lambda case: float(case["duration_seconds"]))["name"]
+
+
+def _recovered_case_count(cases: List[Dict[str, Any]]) -> int:
+    return sum(
+        1
+        for case in cases
+        if case.get("summary", {}).get("recovered") == "true"
+    )
+
+
+def _category_counts(cases: List[Dict[str, Any]]) -> Dict[str, str]:
+    counts: Dict[str, int] = {}
+    for case in cases:
+        category = case.get("category", "")
+        counts[category] = counts.get(category, 0) + 1
+    return {category: str(counts[category]) for category in sorted(counts)}
+
+
+def _exit_code_for_report(report: Dict[str, Any], *, fail_on_failure: bool) -> int:
+    if fail_on_failure and int(report.get("failed", 0)) > 0:
+        return 1
+    return 0
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Evaluate the kagent.")
+    parser.add_argument(
+        "--category",
+        default="",
+        help="Run only evaluator cases in this category, such as recovery or tool.",
+    )
+    parser.add_argument(
+        "--case",
+        default="",
+        help="Run only the evaluator case with this exact name.",
+    )
+    parser.add_argument(
+        "--list-cases",
+        action="store_true",
+        help="List evaluator case names and categories without running them.",
+    )
+    parser.add_argument(
+        "--output",
+        default="",
+        metavar="PATH",
+        help="Write the JSON payload to PATH as well as stdout.",
+    )
+    parser.add_argument(
+        "--fail-on-failure",
+        action="store_true",
+        help="Exit with code 1 when the evaluator report contains failed cases.",
+    )
+    args = parser.parse_args()
+    if args.list_cases:
+        _emit_json_payload({"cases": registered_evaluation_cases()}, args.output, parser)
+        return
+    try:
+        report = evaluate_agent(category=args.category, case_name=args.case)
+    except ValueError as exc:
+        parser.error(str(exc))
+    _emit_json_payload(report, args.output, parser)
+    raise SystemExit(_exit_code_for_report(report, fail_on_failure=args.fail_on_failure))
+
+
+def _emit_json_payload(
+    payload: Dict[str, Any],
+    output_path: str,
+    parser: argparse.ArgumentParser,
+) -> None:
+    try:
+        json_payload = format_and_write_json(payload, output_path)
+    except OSError as exc:
+        parser.error(f"could not write --output file: {exc}")
+    print(json_payload)
+
+
+if __name__ == "__main__":
+    main()

package/src/kagent/integrations/__init__.py

@@ -0,0 +1,3 @@
+from __future__ import annotations
+
+__all__ = []

package/src/kagent/integrations/audit.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import json
+import urllib.request
+from dataclasses import dataclass
+from typing import Any, Dict
+
+_AUDIT_EVENT_FIELDS = (
+    "run_id",
+    "type",
+    "node",
+    "iteration",
+    "action_id",
+    "tool",
+    "status",
+    "error_code",
+    "duration_seconds",
+    "action_count",
+    "iteration_count",
+)
+
+
+@dataclass
+class KafkaRestAuditHook:
+    url: str
+    topic: str
+    timeout_seconds: float = 2.0
+    fail_closed: bool = False
+    failure_count: int = 0
+
+    def on_run_end(self, context: Dict[str, Any]) -> None:
+        payload = {
+            "topic": self.topic,
+            "event": {
+                "type": "run_end",
+                "run_id": str(context.get("run_id", "")),
+                "goal": str(context.get("goal", "")),
+                "status": str(context.get("status", "")),
+                "duration_seconds": str(context.get("duration_seconds", "")),
+            },
+        }
+        self._post(payload)
+
+    def _post(self, payload: Dict[str, Any]) -> None:
+        request = urllib.request.Request(
+            self.url,
+            data=json.dumps(payload, sort_keys=True).encode("utf-8"),
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        try:
+            with urllib.request.urlopen(request, timeout=self.timeout_seconds) as response:
+                if int(response.status) < 200 or int(response.status) >= 300:
+                    raise RuntimeError("audit sink rejected event")
+        except Exception:
+            self.failure_count += 1
+            if self.fail_closed:
+                raise
+
+
+@dataclass
+class KafkaRestProgressEventSink:
+    url: str
+    topic: str
+    timeout_seconds: float = 2.0
+    fail_closed: bool = False
+    failure_count: int = 0
+
+    def __call__(self, event: Dict[str, Any]) -> None:
+        payload = {
+            "topic": self.topic,
+            "event": _redacted_progress_event(event),
+        }
+        request = urllib.request.Request(
+            self.url,
+            data=json.dumps(payload, sort_keys=True).encode("utf-8"),
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        try:
+            with urllib.request.urlopen(request, timeout=self.timeout_seconds) as response:
+                if int(response.status) < 200 or int(response.status) >= 300:
+                    raise RuntimeError("audit sink rejected event")
+        except Exception:
+            self.failure_count += 1
+            if self.fail_closed:
+                raise
+
+
+def _redacted_progress_event(event: Dict[str, Any]) -> Dict[str, str]:
+    return {
+        field: str(event[field])
+        for field in _AUDIT_EVENT_FIELDS
+        if field in event and event[field] is not None
+    }

package/src/kagent/integrations/backends.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+import socket
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from os import environ
+from typing import Dict, Mapping, Optional
+
+
+@dataclass(frozen=True)
+class ExternalBackendConfig:
+    redis_url: str = ""
+    milvus_url: str = ""
+    kafka_audit_url: str = ""
+    kafka_audit_topic: str = ""
+    timeout_seconds: float = 2.0
+
+    @classmethod
+    def from_env(
+        cls,
+        env: Optional[Mapping[str, str]] = None,
+    ) -> "ExternalBackendConfig":
+        source = env if env is not None else environ
+        return cls(
+            redis_url=source.get("KAGENT_REDIS_URL", cls.redis_url),
+            milvus_url=source.get("KAGENT_MILVUS_URL", cls.milvus_url),
+            kafka_audit_url=source.get(
+                "KAGENT_KAFKA_AUDIT_URL",
+                cls.kafka_audit_url,
+            ),
+            kafka_audit_topic=source.get(
+                "KAGENT_KAFKA_AUDIT_TOPIC",
+                cls.kafka_audit_topic,
+            ),
+            timeout_seconds=_env_float(
+                source,
+                "KAGENT_EXTERNAL_BACKEND_TIMEOUT_SECONDS",
+                cls.timeout_seconds,
+            ),
+        )
+
+    def __post_init__(self) -> None:
+        if self.timeout_seconds <= 0:
+            raise ValueError("external backend timeout_seconds must be positive")
+
+    def redacted_snapshot(self) -> Dict[str, str]:
+        return {
+            "redis_short_term_memory": "enabled" if self.redis_url else "disabled",
+            "milvus_long_term_memory": "enabled" if self.milvus_url else "disabled",
+            "kafka_audit_sink": "enabled" if self.kafka_audit_url else "disabled",
+            "kafka_audit_topic_configured": str(bool(self.kafka_audit_topic)).lower(),
+            "external_backend_timeout_seconds": str(self.timeout_seconds),
+        }
+
+
+def check_external_backends(config: ExternalBackendConfig) -> Dict[str, str]:
+    checks: Dict[str, str] = {}
+    if config.redis_url:
+        checks["redis_short_term_memory"] = _check(
+            lambda: _redis_ping(config.redis_url, timeout_seconds=config.timeout_seconds),
+            "redis_unavailable",
+        )
+    if config.milvus_url:
+        checks["milvus_long_term_memory"] = _check(
+            lambda: _http_get_ok(
+                config.milvus_url,
+                timeout_seconds=config.timeout_seconds,
+            ),
+            "milvus_unavailable",
+        )
+    if config.kafka_audit_url:
+        checks["kafka_audit_sink"] = _check(
+            lambda: _http_get_ok(
+                config.kafka_audit_url,
+                timeout_seconds=config.timeout_seconds,
+            ),
+            "kafka_audit_unavailable",
+        )
+    return checks
+
+
+def _check(check, failure_code: str) -> str:
+    try:
+        check()
+    except Exception:
+        return f"failed: {failure_code}"
+    return "ok"
+
+
+def _redis_ping(redis_url: str, *, timeout_seconds: float) -> None:
+    parsed = urllib.parse.urlparse(redis_url)
+    if parsed.scheme != "redis":
+        raise ValueError("redis url must use redis://")
+    if not parsed.hostname:
+        raise ValueError("redis url host is required")
+    port = parsed.port or 6379
+    with socket.create_connection(
+        (parsed.hostname, port),
+        timeout=timeout_seconds,
+    ) as conn:
+        conn.settimeout(timeout_seconds)
+        conn.sendall(b"*1\r\n$4\r\nPING\r\n")
+        response = conn.recv(64)
+    if not response.startswith(b"+PONG"):
+        raise RuntimeError("redis ping failed")
+
+
+def _http_get_ok(url: str, *, timeout_seconds: float) -> None:
+    parsed = urllib.parse.urlparse(url)
+    if parsed.scheme not in {"http", "https"}:
+        raise ValueError("backend url must use http:// or https://")
+    request = urllib.request.Request(url, method="GET")
+    try:
+        with urllib.request.urlopen(request, timeout=timeout_seconds) as response:
+            status_code = int(response.status)
+    except urllib.error.HTTPError as exc:
+        status_code = int(exc.code)
+    if status_code < 200 or status_code >= 300:
+        raise RuntimeError("backend health check failed")
+
+
+def _env_float(source: Mapping[str, str], name: str, default: float) -> float:
+    value = source.get(name, "")
+    if value == "":
+        return default
+    try:
+        return float(value)
+    except ValueError as exc:
+        raise ValueError(f"{name} must be a float") from exc