@openlucaskaka/kagent 0.1.7

package/src/kagent/ops/doctor.py

@@ -0,0 +1,255 @@
+from __future__ import annotations
+
+import argparse
+from typing import Any, Dict, List, Optional
+
+from kagent import __version__
+from kagent.core.tools import registered_tool_names
+from kagent.providers.llm import LLMProviderConfig
+from kagent.service.runtime import ServiceConfig
+from kagent.service.runtime_policy import (
+    execute_runtime_policy_request,
+)
+from kagent.service.safety import safe_header_value
+from kagent.service.status import (
+    readiness_payload,
+    service_config_snapshot,
+)
+from kagent.utils.json_output import format_and_write_json
+
+MIN_PRODUCTION_AUTH_TOKEN_CHARS = 16
+MIN_RUNTIME_PROVIDER_ITERATIONS = 2
+PLACEHOLDER_AUTH_TOKENS = {
+    "change-me",
+    "changeme",
+    "placeholder",
+    "replace-me",
+    "replace-with-a-long-random-token",
+    "secret",
+    "token",
+}
+
+
+def doctor_payload(
+    config: Optional[ServiceConfig] = None,
+    *,
+    require_auth: bool = False,
+    require_production_controls: bool = False,
+    require_runtime_provider: bool = False,
+    llm_config: Optional[LLMProviderConfig] = None,
+) -> Dict[str, Any]:
+    active_config = config or ServiceConfig.from_env()
+    active_llm_config = llm_config or LLMProviderConfig.from_sources()
+    readiness = readiness_payload(active_config)
+    policy = _policy_payload(
+        active_config,
+        active_llm_config,
+        require_auth=require_auth,
+        require_production_controls=require_production_controls,
+        require_runtime_provider=require_runtime_provider,
+    )
+    status = (
+        "ready"
+        if readiness["status"] == "ready" and policy["status"] != "failed"
+        else "not_ready"
+    )
+    config_snapshot = service_config_snapshot(active_config)
+    config_snapshot.update(active_llm_config.redacted_snapshot())
+    runtime_policy = _runtime_policy_summary(active_config)
+    return {
+        "status": status,
+        "version": __version__,
+        "readiness": readiness,
+        "policy": policy,
+        "runtime_policy": runtime_policy,
+        "config": config_snapshot,
+        "tool_count": str(len(registered_tool_names())),
+    }
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    parser = argparse.ArgumentParser(
+        description="Run deployment self-checks for the kagent."
+    )
+    try:
+        defaults = ServiceConfig.from_env()
+    except ValueError as exc:
+        parser.error(str(exc))
+    parser.add_argument("--trace-dir", default=defaults.trace_dir)
+    parser.add_argument(
+        "--runtime-workspace-dir",
+        default=defaults.runtime_workspace_dir,
+        help="Optional runtime virtual workspace root to validate in readiness.",
+    )
+    parser.add_argument(
+        "--require-auth",
+        action="store_true",
+        help="Fail the self-check when POST /run bearer auth is disabled.",
+    )
+    parser.add_argument(
+        "--production",
+        action="store_true",
+        help=(
+            "Fail the self-check unless production controls are configured: "
+            "strong auth, diagnostic protection, trace persistence, rate "
+            "limiting, and bounded concurrency."
+        ),
+    )
+    parser.add_argument(
+        "--require-runtime-provider",
+        action="store_true",
+        help=(
+            "Fail the self-check unless the OpenAI-compatible runtime provider "
+            "environment is configured and runtime replanning has at least two "
+            "iterations available."
+        ),
+    )
+    parser.add_argument("--output", default="", help="Optional JSON artifact path.")
+    args = parser.parse_args(argv)
+
+    config = ServiceConfig(
+        host=defaults.host,
+        port=defaults.port,
+        max_request_bytes=defaults.max_request_bytes,
+        max_goal_chars=defaults.max_goal_chars,
+        auth_token=defaults.auth_token,
+        auth_tokens=defaults.auth_tokens,
+        rate_limit_per_minute=defaults.rate_limit_per_minute,
+        max_concurrent_runs=defaults.max_concurrent_runs,
+        idempotency_cache_size=defaults.idempotency_cache_size,
+        idempotency_cache_path=defaults.idempotency_cache_path,
+        runtime_allowed_tools=defaults.runtime_allowed_tools,
+        runtime_allowed_tools_by_subject=defaults.runtime_allowed_tools_by_subject,
+        runtime_max_iterations=defaults.runtime_max_iterations,
+        allow_full_trace_response=defaults.allow_full_trace_response,
+        protect_diagnostics=defaults.protect_diagnostics,
+        trust_forwarded_for=defaults.trust_forwarded_for,
+        trace_dir=args.trace_dir,
+        runtime_workspace_dir=args.runtime_workspace_dir,
+        redis_url=defaults.redis_url,
+        milvus_url=defaults.milvus_url,
+        kafka_audit_url=defaults.kafka_audit_url,
+        kafka_audit_topic=defaults.kafka_audit_topic,
+        external_backend_timeout_seconds=defaults.external_backend_timeout_seconds,
+        run_timeout_seconds=defaults.run_timeout_seconds,
+        request_timeout_seconds=defaults.request_timeout_seconds,
+    )
+    try:
+        payload = doctor_payload(
+            config,
+            require_auth=args.require_auth,
+            require_production_controls=args.production,
+            require_runtime_provider=args.require_runtime_provider,
+        )
+    except ValueError as exc:
+        parser.error(str(exc))
+    try:
+        json_payload = format_and_write_json(payload, args.output)
+    except OSError as exc:
+        parser.error(f"could not write --output file: {exc}")
+    print(json_payload)
+    return 0 if payload["status"] == "ready" else 1
+
+
+def _policy_payload(
+    config: ServiceConfig,
+    llm_config: LLMProviderConfig,
+    *,
+    require_auth: bool,
+    require_production_controls: bool,
+    require_runtime_provider: bool,
+) -> Dict[str, Any]:
+    warnings = []
+    failures = []
+    configured_auth_tokens = _configured_auth_tokens(config)
+    if _is_public_bind(config.host) and not config.auth_required:
+        warnings.append("public_bind_without_auth")
+    if (require_auth or require_production_controls) and not config.auth_required:
+        failures.append("auth_required")
+    if require_auth or require_production_controls:
+        if any(not safe_header_value(f"Bearer {token}") for token in configured_auth_tokens):
+            failures.append("auth_token_unsafe")
+        if any(_is_placeholder_auth_token(token) for token in configured_auth_tokens):
+            failures.append("auth_token_placeholder")
+    if require_production_controls:
+        if any(len(token) < MIN_PRODUCTION_AUTH_TOKEN_CHARS for token in configured_auth_tokens):
+            failures.append("auth_token_too_short")
+        if not config.trace_dir:
+            failures.append("trace_dir_required")
+        if config.rate_limit_per_minute <= 0:
+            failures.append("rate_limit_required")
+        if config.max_concurrent_runs <= 0:
+            failures.append("concurrency_limit_required")
+        if not config.protect_diagnostics:
+            failures.append("diagnostics_protection_required")
+        if config.allow_full_trace_response:
+            failures.append("full_trace_response_must_be_disabled")
+    if require_runtime_provider:
+        if not llm_config.base_url:
+            failures.append("llm_base_url_required")
+        if not llm_config.model:
+            failures.append("llm_model_required")
+        if not llm_config.api_key:
+            failures.append("llm_api_key_required")
+        if config.runtime_max_iterations < MIN_RUNTIME_PROVIDER_ITERATIONS:
+            failures.append("runtime_iterations_too_low")
+    if failures:
+        status = "failed"
+    elif warnings:
+        status = "warning"
+    else:
+        status = "ok"
+    return {
+        "status": status,
+        "warnings": warnings,
+        "failures": failures,
+    }
+
+
+def _is_public_bind(host: str) -> bool:
+    return host in {"0.0.0.0", "::", ""}
+
+
+def _is_placeholder_auth_token(auth_token: str) -> bool:
+    normalized = auth_token.strip().lower()
+    return normalized in PLACEHOLDER_AUTH_TOKENS
+
+
+def _configured_auth_tokens(config: ServiceConfig) -> list[str]:
+    tokens = []
+    if config.auth_token:
+        tokens.append(config.auth_token)
+    tokens.extend(config.auth_tokens.values())
+    return tokens
+
+
+def _runtime_policy_summary(config: ServiceConfig) -> Dict[str, Any]:
+    _status_code, payload = execute_runtime_policy_request(
+        config,
+        request_auth_subject="default",
+        request_auth_is_admin=True,
+    )
+    effective_tool_policy = payload.get("effective_tool_policy", [])
+    approval_required_count = sum(
+        1
+        for item in effective_tool_policy
+        if isinstance(item, dict)
+        and item.get("approval_required") == "true"
+    )
+    return {
+        "trace_type": payload.get("trace_type", ""),
+        "effective_policy_source": payload.get("effective_policy_source", ""),
+        "effective_allowed_tools": payload.get("effective_allowed_tools", []),
+        "effective_allowed_tool_count": str(
+            len(payload.get("effective_allowed_tools", []))
+        ),
+        "approval_required_tool_count": str(approval_required_count),
+        "subject_policy_count": payload.get("subject_policy_count", "0"),
+        "effective_tool_policy_sha256": payload.get(
+            "effective_tool_policy_sha256", ""
+        ),
+    }
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/src/kagent/ops/metrics.py

@@ -0,0 +1,214 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any, Dict, List, Tuple
+
+from kagent.utils.json_output import format_and_write_json
+
+
+def summarize_metrics_file(path: Path) -> Dict[str, Any]:
+    metrics_file_found = path.exists()
+    records, malformed_lines = _read_jsonl(path)
+    total = len(records)
+    passed_records = [record for record in records if record.get("status") == "passed"]
+    failed_records = [record for record in records if record.get("status") != "passed"]
+    durations = [_duration_value(record.get("duration_seconds", 0)) for record in records]
+    latest = records[-1] if records else {}
+
+    pass_rate = len(passed_records) / total if total else 0.0
+    average_duration = sum(durations) / total if total else 0.0
+    failed_iterations = [str(record.get("iteration")) for record in failed_records]
+    latest_evaluator_failed = _string_or_empty(latest.get("evaluator_failed"))
+    latest_status = _string_or_empty(latest.get("status"))
+    latest_category_counts = _category_counts(latest.get("evaluator_category_counts"))
+    recent_statuses = _recent_statuses(records)
+    health = _health(total, len(passed_records), latest_evaluator_failed, malformed_lines)
+
+    return {
+        "iterations": str(total),
+        "passed": str(len(passed_records)),
+        "failed": str(len(failed_records)),
+        "pass_rate": f"{pass_rate:.2f}",
+        "health": health,
+        "metrics_file_found": str(metrics_file_found).lower(),
+        "average_duration_seconds": f"{average_duration:.2f}",
+        "latest_status": latest_status,
+        "recent_health": _recent_health(recent_statuses),
+        "consecutive_passes": str(_consecutive_pass_count(records)),
+        "recent_statuses": recent_statuses,
+        "failed_iterations": failed_iterations,
+        "malformed_lines": malformed_lines,
+        "latest_evaluator_passed": _string_or_empty(latest.get("evaluator_passed")),
+        "latest_evaluator_failed": latest_evaluator_failed,
+        "latest_slowest_case": _string_or_empty(latest.get("evaluator_slowest_case")),
+        "latest_recovered_cases": _string_or_empty(
+            latest.get("evaluator_recovered_cases")
+        ),
+        "latest_recovery_rate": _string_or_empty(latest.get("evaluator_recovery_rate")),
+        "latest_category_counts": latest_category_counts,
+        "recommendations": _recommendations(
+            failed_iterations,
+            latest_evaluator_failed,
+            malformed_lines,
+            latest_status,
+            _string_or_empty(latest.get("evaluator_passed")),
+            latest_category_counts,
+            metrics_file_found,
+            path,
+        ),
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Summarize continuous iteration JSONL metrics."
+    )
+    parser.add_argument("metrics_jsonl", help="Path to the metrics JSONL file.")
+    parser.add_argument(
+        "--output",
+        default="",
+        metavar="PATH",
+        help="Write the JSON payload to PATH as well as stdout.",
+    )
+    parser.add_argument(
+        "--require-recent-health",
+        choices=["healthy", "recovering", "failing", "unknown"],
+        default="",
+        help="Exit with code 1 unless recent_health matches this value.",
+    )
+    args = parser.parse_args()
+    summary = summarize_metrics_file(Path(args.metrics_jsonl))
+    try:
+        json_payload = format_and_write_json(summary, args.output)
+    except OSError as exc:
+        parser.error(f"could not write --output file: {exc}")
+    print(json_payload)
+    if args.require_recent_health and summary["recent_health"] != args.require_recent_health:
+        raise SystemExit(1)
+
+
+def _read_jsonl(path: Path) -> Tuple[List[Dict[str, Any]], List[str]]:
+    if not path.exists():
+        return [], []
+    records = []
+    malformed_lines = []
+    for line_number, line in enumerate(path.read_text().splitlines(), start=1):
+        if not line.strip():
+            continue
+        try:
+            payload = json.loads(line)
+        except json.JSONDecodeError:
+            malformed_lines.append(str(line_number))
+            continue
+        if not isinstance(payload, dict):
+            malformed_lines.append(str(line_number))
+            continue
+        records.append(payload)
+    return records, malformed_lines
+
+
+def _string_or_empty(value: Any) -> str:
+    if value is None:
+        return ""
+    return str(value)
+
+
+def _duration_value(value: Any) -> float:
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return 0.0
+
+
+def _consecutive_pass_count(records: List[Dict[str, Any]]) -> int:
+    count = 0
+    for record in reversed(records):
+        if record.get("status") != "passed":
+            break
+        count += 1
+    return count
+
+
+def _recent_statuses(records: List[Dict[str, Any]], limit: int = 5) -> List[str]:
+    return [
+        _string_or_empty(record.get("status"))
+        for record in records[-limit:]
+    ]
+
+
+def _recent_health(recent_statuses: List[str]) -> str:
+    if not recent_statuses:
+        return "unknown"
+    if recent_statuses[-1] != "passed":
+        return "failing"
+    if all(status == "passed" for status in recent_statuses):
+        return "healthy"
+    return "recovering"
+
+
+def _category_counts(value: Any) -> Dict[str, str]:
+    if not isinstance(value, dict):
+        return {}
+    return {str(key): str(value[key]) for key in sorted(value)}
+
+
+def _health(
+    total: int,
+    passed: int,
+    latest_evaluator_failed: str,
+    malformed_lines: List[str],
+) -> str:
+    if total == 0:
+        return "failing" if malformed_lines else "unknown"
+    if malformed_lines:
+        return "degraded"
+    if passed == total and latest_evaluator_failed in {"", "0"}:
+        return "healthy"
+    if passed == 0:
+        return "failing"
+    return "degraded"
+
+
+def _recommendations(
+    failed_iterations: List[str],
+    latest_evaluator_failed: str,
+    malformed_lines: List[str],
+    latest_status: str,
+    latest_evaluator_passed: str,
+    latest_category_counts: Dict[str, str],
+    metrics_file_found: bool,
+    metrics_path: Path,
+) -> List[str]:
+    recommendations = []
+    if not metrics_file_found:
+        recommendations.append(f"metrics file not found: {metrics_path}")
+        return recommendations
+    if failed_iterations:
+        recommendations.append(f"inspect failed iterations: {', '.join(failed_iterations)}")
+        if latest_status == "passed":
+            recommendations.append("latest run is passing after previous failures")
+        elif (
+            latest_status == "failed"
+            and not latest_evaluator_passed
+            and not latest_evaluator_failed
+        ):
+            recommendations.append(
+                "inspect check log; latest run failed before a fresh evaluator report"
+            )
+    if latest_evaluator_failed not in {"", "0"}:
+        recommendations.append("review evaluator failures from the latest run")
+    if malformed_lines:
+        recommendations.append(
+            f"inspect malformed metrics lines: {', '.join(malformed_lines)}"
+        )
+    elif latest_evaluator_passed and not latest_category_counts:
+        recommendations.append(
+            "review continuous metrics wiring; latest evaluator category counts are missing"
+        )
+    return recommendations
+
+
+if __name__ == "__main__":
+    main()