@openlucaskaka/kagent 0.1.7

package/src/kagent/service/runtime.py

@@ -0,0 +1,1915 @@
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass, field
+from math import ceil
+from os import environ
+from threading import BoundedSemaphore, Lock
+from typing import Any, Callable, Dict, Mapping, Optional, Tuple
+
+from kagent.service.idempotency import (
+    ServiceIdempotencyCache as ServiceIdempotencyCache,
+)
+from kagent.service.idempotency import (
+    SqliteServiceIdempotencyCache as SqliteServiceIdempotencyCache,
+)
+
+_KNOWN_HTTP_METHODS = {"GET", "HEAD", "OPTIONS", "POST"}
+_UNKNOWN_METRICS_LABEL = "__unknown__"
+_DURATION_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0)
+_RUNTIME_RECONCILIATION_STATUSES = {"ok", "unavailable"}
+_RUNTIME_RECONCILIATION_OUTCOMES = (
+    "recovered_running",
+    "completed_resumes",
+    "reopened_approvals",
+    "protected_live",
+    "skipped_unowned",
+    "skipped_locked",
+)
+
+
+@dataclass(frozen=True)
+class ServiceConfig:
+    host: str = "127.0.0.1"
+    port: int = 8000
+    max_request_bytes: int = 65536
+    max_goal_chars: int = 4096
+    auth_token: str = ""
+    auth_tokens: Dict[str, str] = field(default_factory=dict)
+    rate_limit_per_minute: int = 0
+    max_concurrent_runs: int = 4
+    idempotency_cache_size: int = 0
+    idempotency_cache_path: str = ""
+    runtime_allowed_tools: Tuple[str, ...] = ()
+    runtime_allowed_tools_by_subject: Dict[str, Tuple[str, ...]] = field(
+        default_factory=dict
+    )
+    runtime_max_iterations: int = 10
+    runtime_pending_approval_stale_seconds: int = 3600
+    runtime_instance_heartbeat_seconds: float = 10.0
+    runtime_orphaned_run_stale_seconds: float = 60.0
+    allow_full_trace_response: bool = False
+    protect_diagnostics: bool = False
+    trust_forwarded_for: bool = False
+    trace_dir: str = ""
+    runtime_workspace_dir: str = ""
+    redis_url: str = ""
+    milvus_url: str = ""
+    embedding_base_url: str = ""
+    embedding_api_key: str = ""
+    embedding_model: str = ""
+    embedding_timeout_seconds: float = 30.0
+    embedding_max_retries: int = 2
+    embedding_retry_backoff_seconds: float = 0.25
+    kafka_audit_url: str = ""
+    kafka_audit_topic: str = ""
+    external_backend_timeout_seconds: float = 2.0
+    run_timeout_seconds: float = 30.0
+    request_timeout_seconds: float = 10.0
+
+    @classmethod
+    def from_env(cls, env: Optional[Mapping[str, str]] = None) -> "ServiceConfig":
+        source = env if env is not None else environ
+        return cls(
+            host=source.get("KAGENT_SERVICE_HOST", cls.host),
+            port=_env_int(source, "KAGENT_SERVICE_PORT", cls.port),
+            max_request_bytes=_env_int(
+                source,
+                "KAGENT_SERVICE_MAX_REQUEST_BYTES",
+                cls.max_request_bytes,
+            ),
+            max_goal_chars=_env_int(
+                source,
+                "KAGENT_SERVICE_MAX_GOAL_CHARS",
+                cls.max_goal_chars,
+            ),
+            auth_token=source.get("KAGENT_SERVICE_AUTH_TOKEN", cls.auth_token),
+            auth_tokens=_env_auth_tokens(
+                source,
+                "KAGENT_SERVICE_AUTH_TOKENS",
+            ),
+            rate_limit_per_minute=_env_int(
+                source,
+                "KAGENT_SERVICE_RATE_LIMIT_PER_MINUTE",
+                cls.rate_limit_per_minute,
+            ),
+            max_concurrent_runs=_env_int(
+                source,
+                "KAGENT_SERVICE_MAX_CONCURRENT_RUNS",
+                cls.max_concurrent_runs,
+            ),
+            idempotency_cache_size=_env_int(
+                source,
+                "KAGENT_SERVICE_IDEMPOTENCY_CACHE_SIZE",
+                cls.idempotency_cache_size,
+            ),
+            idempotency_cache_path=source.get(
+                "KAGENT_SERVICE_IDEMPOTENCY_CACHE_PATH",
+                cls.idempotency_cache_path,
+            ),
+            runtime_allowed_tools=_env_csv_tuple(
+                source,
+                "KAGENT_SERVICE_RUNTIME_ALLOWED_TOOLS",
+            ),
+            runtime_allowed_tools_by_subject=_env_subject_csv_tuple_map(
+                source,
+                "KAGENT_SERVICE_RUNTIME_ALLOWED_TOOLS_BY_SUBJECT",
+            ),
+            runtime_max_iterations=_env_int(
+                source,
+                "KAGENT_SERVICE_RUNTIME_MAX_ITERATIONS",
+                cls.runtime_max_iterations,
+            ),
+            runtime_pending_approval_stale_seconds=_env_int(
+                source,
+                "KAGENT_SERVICE_RUNTIME_PENDING_APPROVAL_STALE_SECONDS",
+                cls.runtime_pending_approval_stale_seconds,
+            ),
+            runtime_instance_heartbeat_seconds=_env_float(
+                source,
+                "KAGENT_SERVICE_RUNTIME_INSTANCE_HEARTBEAT_SECONDS",
+                cls.runtime_instance_heartbeat_seconds,
+            ),
+            runtime_orphaned_run_stale_seconds=_env_float(
+                source,
+                "KAGENT_SERVICE_RUNTIME_ORPHANED_RUN_STALE_SECONDS",
+                cls.runtime_orphaned_run_stale_seconds,
+            ),
+            allow_full_trace_response=_env_bool(
+                source,
+                "KAGENT_SERVICE_ALLOW_FULL_TRACE_RESPONSE",
+                cls.allow_full_trace_response,
+            ),
+            protect_diagnostics=_env_bool(
+                source,
+                "KAGENT_SERVICE_PROTECT_DIAGNOSTICS",
+                cls.protect_diagnostics,
+            ),
+            trust_forwarded_for=_env_bool(
+                source,
+                "KAGENT_SERVICE_TRUST_FORWARDED_FOR",
+                cls.trust_forwarded_for,
+            ),
+            trace_dir=source.get("KAGENT_SERVICE_TRACE_DIR", cls.trace_dir),
+            runtime_workspace_dir=source.get(
+                "KAGENT_SERVICE_RUNTIME_WORKSPACE_DIR",
+                cls.runtime_workspace_dir,
+            ),
+            redis_url=source.get("KAGENT_REDIS_URL", cls.redis_url),
+            milvus_url=source.get("KAGENT_MILVUS_URL", cls.milvus_url),
+            embedding_base_url=source.get(
+                "KAGENT_EMBEDDING_BASE_URL",
+                source.get("KAGENT_LLM_BASE_URL", cls.embedding_base_url),
+            ),
+            embedding_api_key=source.get(
+                "KAGENT_EMBEDDING_API_KEY",
+                source.get("KAGENT_LLM_API_KEY", cls.embedding_api_key),
+            ),
+            embedding_model=source.get(
+                "KAGENT_EMBEDDING_MODEL",
+                cls.embedding_model,
+            ),
+            embedding_timeout_seconds=_env_float(
+                source,
+                "KAGENT_EMBEDDING_TIMEOUT_SECONDS",
+                cls.embedding_timeout_seconds,
+            ),
+            embedding_max_retries=_env_int(
+                source,
+                "KAGENT_EMBEDDING_MAX_RETRIES",
+                cls.embedding_max_retries,
+            ),
+            embedding_retry_backoff_seconds=_env_float(
+                source,
+                "KAGENT_EMBEDDING_RETRY_BACKOFF_SECONDS",
+                cls.embedding_retry_backoff_seconds,
+            ),
+            kafka_audit_url=source.get(
+                "KAGENT_KAFKA_AUDIT_URL",
+                cls.kafka_audit_url,
+            ),
+            kafka_audit_topic=source.get(
+                "KAGENT_KAFKA_AUDIT_TOPIC",
+                cls.kafka_audit_topic,
+            ),
+            external_backend_timeout_seconds=_env_float(
+                source,
+                "KAGENT_EXTERNAL_BACKEND_TIMEOUT_SECONDS",
+                cls.external_backend_timeout_seconds,
+            ),
+            run_timeout_seconds=_env_float(
+                source,
+                "KAGENT_SERVICE_RUN_TIMEOUT_SECONDS",
+                cls.run_timeout_seconds,
+            ),
+            request_timeout_seconds=_env_float(
+                source,
+                "KAGENT_SERVICE_REQUEST_TIMEOUT_SECONDS",
+                cls.request_timeout_seconds,
+            ),
+        )
+
+    def __post_init__(self) -> None:
+        if self.port < 1 or self.port > 65535:
+            raise ValueError("port must be between 1 and 65535")
+        if self.max_request_bytes < 1:
+            raise ValueError("max_request_bytes must be at least 1")
+        if self.max_goal_chars < 1:
+            raise ValueError("max_goal_chars must be at least 1")
+        if self.rate_limit_per_minute < 0:
+            raise ValueError("rate_limit_per_minute must be non-negative")
+        if self.max_concurrent_runs < 0:
+            raise ValueError("max_concurrent_runs must be non-negative")
+        if self.idempotency_cache_size < 0:
+            raise ValueError("idempotency_cache_size must be non-negative")
+        if self.idempotency_cache_path and self.idempotency_cache_size == 0:
+            raise ValueError("idempotency_cache_path requires idempotency_cache_size")
+        _validate_runtime_allowed_tools(self.runtime_allowed_tools)
+        _validate_runtime_allowed_tools_by_subject(
+            self.runtime_allowed_tools_by_subject
+        )
+        if self.runtime_max_iterations < 1:
+            raise ValueError("runtime_max_iterations must be at least 1")
+        if self.runtime_pending_approval_stale_seconds < 0:
+            raise ValueError(
+                "runtime_pending_approval_stale_seconds must be non-negative"
+            )
+        if self.runtime_instance_heartbeat_seconds <= 0:
+            raise ValueError("runtime_instance_heartbeat_seconds must be positive")
+        if (
+            self.runtime_orphaned_run_stale_seconds
+            <= self.runtime_instance_heartbeat_seconds
+        ):
+            raise ValueError(
+                "runtime_orphaned_run_stale_seconds must be greater than "
+                "runtime_instance_heartbeat_seconds"
+            )
+        if self.protect_diagnostics and not self.auth_required:
+            raise ValueError("protect_diagnostics requires auth_token")
+        _validate_auth_tokens(self.auth_tokens)
+        if self.run_timeout_seconds <= 0:
+            raise ValueError("run_timeout_seconds must be positive")
+        if self.request_timeout_seconds <= 0:
+            raise ValueError("request_timeout_seconds must be positive")
+        if self.external_backend_timeout_seconds <= 0:
+            raise ValueError("external_backend_timeout_seconds must be positive")
+        if self.embedding_timeout_seconds <= 0:
+            raise ValueError("embedding_timeout_seconds must be positive")
+        if self.embedding_max_retries < 0:
+            raise ValueError("embedding_max_retries must be non-negative")
+        if self.embedding_retry_backoff_seconds < 0:
+            raise ValueError(
+                "embedding_retry_backoff_seconds must be non-negative"
+            )
+
+    @property
+    def auth_required(self) -> bool:
+        return bool(self.auth_token or self.auth_tokens)
+
+    def runtime_allowed_tools_for_subject(
+        self,
+        auth_subject: str,
+    ) -> Optional[Tuple[str, ...]]:
+        if auth_subject and auth_subject in self.runtime_allowed_tools_by_subject:
+            return self.runtime_allowed_tools_by_subject[auth_subject]
+        if self.runtime_allowed_tools:
+            return self.runtime_allowed_tools
+        return None
+
+
+class ServiceMetrics:
+    def __init__(self, *, started_at: Optional[float] = None) -> None:
+        self._lock = Lock()
+        self._started_at = time.monotonic() if started_at is None else started_at
+        self._requests_total = 0
+        self._total_duration_seconds = 0.0
+        self._max_duration_seconds = 0.0
+        self._request_duration_buckets: Dict[str, int] = _empty_duration_buckets()
+        self._responses_by_status: Dict[str, int] = {}
+        self._requests_by_method: Dict[str, int] = {}
+        self._requests_by_path: Dict[str, int] = {}
+        self._requests_by_auth_subject: Dict[str, int] = {}
+        self._error_responses_by_code: Dict[str, int] = {}
+        self._agent_runs_total = 0
+        self._agent_run_duration_seconds = 0.0
+        self._max_agent_run_duration_seconds = 0.0
+        self._agent_run_duration_buckets: Dict[str, int] = _empty_duration_buckets()
+        self._agent_runs_by_status: Dict[str, int] = {}
+        self._runtime_runs_total = 0
+        self._runtime_runs_by_status: Dict[str, int] = {}
+        self._runtime_runs_by_lifecycle_state: Dict[str, int] = {}
+        self._runtime_runs_by_auth_subject: Dict[str, int] = {}
+        self._runtime_runs_by_auth_subject_status: Dict[str, int] = {}
+        self._runtime_runs_by_auth_subject_lifecycle_state: Dict[str, int] = {}
+        self._runtime_resumes_by_auth_subject: Dict[str, int] = {}
+        self._runtime_approvals_by_auth_subject: Dict[str, int] = {}
+        self._runtime_failed_observations_total = 0
+        self._runtime_progress_event_sink_failures_total = 0
+        self._runtime_hook_failures_total = 0
+        self._runtime_reconciliation_runs_total = 0
+        self._runtime_reconciliation_runs_by_status: Dict[str, int] = {}
+        self._runtime_reconciliation_traces_scanned_total = 0
+        self._runtime_reconciliation_outcomes: Dict[str, int] = {}
+        self._runtime_reconciliation_errors_total = 0
+        self._runtime_observation_errors_by_code: Dict[str, int] = {}
+        self._runtime_tool_executions_by_tool_status: Dict[str, int] = {}
+        self._runtime_planner_attempts_by_status: Dict[str, int] = {}
+        self._runtime_planner_failures_total = 0
+        self._runtime_planner_failures_by_error_code: Dict[str, int] = {}
+        self._runtime_llm_provider_requests_total = 0
+        self._runtime_llm_provider_request_attempts_total = 0
+        self._runtime_llm_provider_request_retries_total = 0
+        self._runtime_llm_provider_requests_by_status: Dict[str, int] = {}
+        self._runtime_llm_provider_request_errors_by_type: Dict[str, int] = {}
+        self._runtime_llm_provider_request_http_status: Dict[str, int] = {}
+        self._runtime_llm_provider_request_retryable_reason: Dict[str, int] = {}
+        self._runtime_llm_provider_request_duration_seconds = 0.0
+        self._max_runtime_llm_provider_request_duration_seconds = 0.0
+        self._runtime_llm_provider_request_duration_buckets: Dict[str, int] = (
+            _empty_duration_buckets()
+        )
+        self._runtime_approval_required_total = 0
+        self._runtime_failed_budget_exhaustions_total = 0
+        self._runtime_run_duration_seconds = 0.0
+        self._max_runtime_run_duration_seconds = 0.0
+        self._runtime_run_duration_buckets: Dict[str, int] = _empty_duration_buckets()
+
+    def record(
+        self,
+        *,
+        path: str,
+        status_code: int,
+        method: str = "UNKNOWN",
+        duration_seconds: float = 0.0,
+        error_code: str = "",
+        auth_subject: str = "",
+    ) -> None:
+        with self._lock:
+            self._requests_total += 1
+            self._total_duration_seconds += duration_seconds
+            self._max_duration_seconds = max(self._max_duration_seconds, duration_seconds)
+            for bucket, label in _duration_bucket_labels():
+                if duration_seconds <= bucket:
+                    self._request_duration_buckets[label] += 1
+            self._request_duration_buckets["+Inf"] += 1
+            status_key = str(status_code)
+            method_key = _method_metrics_label(method)
+            self._responses_by_status[status_key] = self._responses_by_status.get(status_key, 0) + 1
+            self._requests_by_method[method_key] = self._requests_by_method.get(method_key, 0) + 1
+            self._requests_by_path[path] = self._requests_by_path.get(path, 0) + 1
+            if auth_subject:
+                self._requests_by_auth_subject[auth_subject] = (
+                    self._requests_by_auth_subject.get(auth_subject, 0) + 1
+                )
+            if error_code:
+                self._error_responses_by_code[error_code] = (
+                    self._error_responses_by_code.get(error_code, 0) + 1
+                )
+
+    def record_agent_run(self, *, status: str, duration_seconds: float = 0.0) -> None:
+        with self._lock:
+            self._agent_runs_total += 1
+            self._agent_run_duration_seconds += duration_seconds
+            self._max_agent_run_duration_seconds = max(
+                self._max_agent_run_duration_seconds,
+                duration_seconds,
+            )
+            for bucket, label in _duration_bucket_labels():
+                if duration_seconds <= bucket:
+                    self._agent_run_duration_buckets[label] += 1
+            self._agent_run_duration_buckets["+Inf"] += 1
+            self._agent_runs_by_status[status] = self._agent_runs_by_status.get(status, 0) + 1
+
+    def record_runtime_run(
+        self,
+        *,
+        status: str,
+        lifecycle_state: str = "",
+        failed_observation_count: int = 0,
+        approval_required_count: int = 0,
+        budget_exhausted: bool = False,
+        duration_seconds: float = 0.0,
+        error_code_counts: Optional[Mapping[str, int]] = None,
+        tool_status_counts: Optional[Mapping[str, int]] = None,
+        planner_attempt_status_counts: Optional[Mapping[str, int]] = None,
+        planner_failure_count: int = 0,
+        planner_error_code_counts: Optional[Mapping[str, int]] = None,
+        llm_provider_request: Optional[Mapping[str, Any]] = None,
+        auth_subject: str = "",
+        resumed_by_auth_subject: str = "",
+        approved_by_auth_subject: str = "",
+        progress_event_sink_failure_count: int = 0,
+        hook_failure_count: int = 0,
+    ) -> None:
+        with self._lock:
+            self._runtime_runs_total += 1
+            self._runtime_runs_by_status[status] = (
+                self._runtime_runs_by_status.get(status, 0) + 1
+            )
+            normalized_lifecycle_state = lifecycle_state or _runtime_lifecycle_state(
+                status
+            )
+            self._runtime_runs_by_lifecycle_state[normalized_lifecycle_state] = (
+                self._runtime_runs_by_lifecycle_state.get(
+                    normalized_lifecycle_state,
+                    0,
+                )
+                + 1
+            )
+            if auth_subject:
+                self._runtime_runs_by_auth_subject[auth_subject] = (
+                    self._runtime_runs_by_auth_subject.get(auth_subject, 0) + 1
+                )
+                subject_status_key = _combined_metrics_key(auth_subject, status)
+                self._runtime_runs_by_auth_subject_status[subject_status_key] = (
+                    self._runtime_runs_by_auth_subject_status.get(subject_status_key, 0)
+                    + 1
+                )
+                subject_lifecycle_key = _combined_metrics_key(
+                    auth_subject,
+                    normalized_lifecycle_state,
+                )
+                self._runtime_runs_by_auth_subject_lifecycle_state[
+                    subject_lifecycle_key
+                ] = (
+                    self._runtime_runs_by_auth_subject_lifecycle_state.get(
+                        subject_lifecycle_key,
+                        0,
+                    )
+                    + 1
+                )
+            if resumed_by_auth_subject:
+                self._runtime_resumes_by_auth_subject[resumed_by_auth_subject] = (
+                    self._runtime_resumes_by_auth_subject.get(
+                        resumed_by_auth_subject,
+                        0,
+                    )
+                    + 1
+                )
+            if approved_by_auth_subject:
+                self._runtime_approvals_by_auth_subject[approved_by_auth_subject] = (
+                    self._runtime_approvals_by_auth_subject.get(
+                        approved_by_auth_subject,
+                        0,
+                    )
+                    + 1
+                )
+            self._runtime_run_duration_seconds += duration_seconds
+            self._max_runtime_run_duration_seconds = max(
+                self._max_runtime_run_duration_seconds,
+                duration_seconds,
+            )
+            for bucket, label in _duration_bucket_labels():
+                if duration_seconds <= bucket:
+                    self._runtime_run_duration_buckets[label] += 1
+            self._runtime_run_duration_buckets["+Inf"] += 1
+            self._runtime_failed_observations_total += max(0, failed_observation_count)
+            self._runtime_progress_event_sink_failures_total += max(
+                0,
+                progress_event_sink_failure_count,
+            )
+            self._runtime_hook_failures_total += max(0, hook_failure_count)
+            for error_code, count in (error_code_counts or {}).items():
+                normalized_error_code = str(error_code)
+                if not normalized_error_code:
+                    continue
+                self._runtime_observation_errors_by_code[normalized_error_code] = (
+                    self._runtime_observation_errors_by_code.get(
+                        normalized_error_code,
+                        0,
+                    )
+                    + max(0, int(count))
+                )
+            for tool_status, count in (tool_status_counts or {}).items():
+                tool, status_label = _split_combined_metrics_key(str(tool_status))
+                normalized_tool = _runtime_tool_metrics_label(tool)
+                normalized_status = _runtime_observation_status_metrics_label(
+                    status_label
+                )
+                key = _combined_metrics_key(normalized_tool, normalized_status)
+                self._runtime_tool_executions_by_tool_status[key] = (
+                    self._runtime_tool_executions_by_tool_status.get(key, 0)
+                    + max(0, int(count))
+                )
+            for status_label, count in (planner_attempt_status_counts or {}).items():
+                normalized_status = _runtime_planner_attempt_status_metrics_label(
+                    str(status_label)
+                )
+                self._runtime_planner_attempts_by_status[normalized_status] = (
+                    self._runtime_planner_attempts_by_status.get(
+                        normalized_status,
+                        0,
+                    )
+                    + max(0, int(count))
+                )
+            self._runtime_planner_failures_total += max(0, planner_failure_count)
+            for error_code, count in (planner_error_code_counts or {}).items():
+                normalized_error_code = str(error_code)
+                if not normalized_error_code:
+                    continue
+                self._runtime_planner_failures_by_error_code[
+                    normalized_error_code
+                ] = (
+                    self._runtime_planner_failures_by_error_code.get(
+                        normalized_error_code,
+                        0,
+                    )
+                    + max(0, int(count))
+                )
+            if llm_provider_request:
+                attempt_count = _non_negative_int_value(
+                    llm_provider_request.get("attempt_count", 0)
+                )
+                retry_count = _non_negative_int_value(
+                    llm_provider_request.get("retry_count", 0)
+                )
+                provider_duration = _non_negative_float_value(
+                    llm_provider_request.get("duration_seconds", 0.0)
+                )
+                provider_status = _runtime_llm_provider_status_metrics_label(
+                    str(llm_provider_request.get("status", ""))
+                )
+                provider_error_type = _runtime_llm_provider_error_type_metrics_label(
+                    str(llm_provider_request.get("error_type", ""))
+                )
+                provider_http_status = _runtime_llm_provider_http_status_metrics_label(
+                    str(llm_provider_request.get("http_status", ""))
+                )
+                provider_retryable_reason = (
+                    _runtime_llm_provider_retryable_reason_metrics_label(
+                        str(llm_provider_request.get("retryable_reason", ""))
+                    )
+                )
+                self._runtime_llm_provider_requests_total += 1
+                self._runtime_llm_provider_request_attempts_total += attempt_count
+                self._runtime_llm_provider_request_retries_total += retry_count
+                self._runtime_llm_provider_requests_by_status[provider_status] = (
+                    self._runtime_llm_provider_requests_by_status.get(
+                        provider_status,
+                        0,
+                    )
+                    + 1
+                )
+                if provider_error_type:
+                    self._runtime_llm_provider_request_errors_by_type[
+                        provider_error_type
+                    ] = (
+                        self._runtime_llm_provider_request_errors_by_type.get(
+                            provider_error_type,
+                            0,
+                        )
+                        + 1
+                    )
+                if provider_http_status:
+                    self._runtime_llm_provider_request_http_status[
+                        provider_http_status
+                    ] = (
+                        self._runtime_llm_provider_request_http_status.get(
+                            provider_http_status,
+                            0,
+                        )
+                        + 1
+                    )
+                if provider_retryable_reason:
+                    self._runtime_llm_provider_request_retryable_reason[
+                        provider_retryable_reason
+                    ] = (
+                        self._runtime_llm_provider_request_retryable_reason.get(
+                            provider_retryable_reason,
+                            0,
+                        )
+                        + 1
+                    )
+                self._runtime_llm_provider_request_duration_seconds += provider_duration
+                self._max_runtime_llm_provider_request_duration_seconds = max(
+                    self._max_runtime_llm_provider_request_duration_seconds,
+                    provider_duration,
+                )
+                for bucket, label in _duration_bucket_labels():
+                    if provider_duration <= bucket:
+                        self._runtime_llm_provider_request_duration_buckets[label] += 1
+                self._runtime_llm_provider_request_duration_buckets["+Inf"] += 1
+            self._runtime_approval_required_total += max(0, approval_required_count)
+            if budget_exhausted:
+                self._runtime_failed_budget_exhaustions_total += 1
+
+    def record_runtime_reconciliation(
+        self,
+        summary: Mapping[str, Any],
+        *,
+        status: str = "ok",
+    ) -> None:
+        status_key = (
+            status
+            if status in _RUNTIME_RECONCILIATION_STATUSES
+            else _UNKNOWN_METRICS_LABEL
+        )
+        errors = summary.get("errors", ())
+        error_count = (
+            len(errors)
+            if isinstance(errors, (list, tuple))
+            else _non_negative_int_value(errors)
+        )
+        with self._lock:
+            self._runtime_reconciliation_runs_total += 1
+            self._runtime_reconciliation_runs_by_status[status_key] = (
+                self._runtime_reconciliation_runs_by_status.get(status_key, 0) + 1
+            )
+            self._runtime_reconciliation_traces_scanned_total += (
+                _non_negative_int_value(summary.get("scanned", 0))
+            )
+            for outcome in _RUNTIME_RECONCILIATION_OUTCOMES:
+                count = _non_negative_int_value(summary.get(outcome, 0))
+                if count:
+                    self._runtime_reconciliation_outcomes[outcome] = (
+                        self._runtime_reconciliation_outcomes.get(outcome, 0) + count
+                    )
+            self._runtime_reconciliation_errors_total += error_count
+
+    def snapshot(self, *, now: Optional[float] = None) -> Dict[str, Any]:
+        with self._lock:
+            current_time = time.monotonic() if now is None else now
+            average_duration = (
+                self._total_duration_seconds / self._requests_total
+                if self._requests_total
+                else 0.0
+            )
+            average_agent_run_duration = (
+                self._agent_run_duration_seconds / self._agent_runs_total
+                if self._agent_runs_total
+                else 0.0
+            )
+            average_runtime_run_duration = (
+                self._runtime_run_duration_seconds / self._runtime_runs_total
+                if self._runtime_runs_total
+                else 0.0
+            )
+            average_runtime_llm_provider_request_duration = (
+                self._runtime_llm_provider_request_duration_seconds
+                / self._runtime_llm_provider_requests_total
+                if self._runtime_llm_provider_requests_total
+                else 0.0
+            )
+            return {
+                "requests_total": str(self._requests_total),
+                "responses_by_status": _string_counts(self._responses_by_status),
+                "requests_by_method": _string_counts(self._requests_by_method),
+                "requests_by_path": _string_counts(self._requests_by_path),
+                "requests_by_auth_subject": _string_counts(
+                    self._requests_by_auth_subject
+                ),
+                "error_responses_by_code": _string_counts(self._error_responses_by_code),
+                "request_duration_seconds_bucket": _string_counts(
+                    self._request_duration_buckets
+                ),
+                "request_duration_seconds_count": str(self._requests_total),
+                "request_duration_seconds_sum": f"{self._total_duration_seconds:.4f}",
+                "average_duration_seconds": f"{average_duration:.4f}",
+                "max_duration_seconds": f"{self._max_duration_seconds:.4f}",
+                "agent_runs_total": str(self._agent_runs_total),
+                "agent_runs_by_status": _string_counts(self._agent_runs_by_status),
+                "agent_run_duration_seconds_bucket": _string_counts(
+                    self._agent_run_duration_buckets
+                ),
+                "agent_run_duration_seconds_count": str(self._agent_runs_total),
+                "agent_run_duration_seconds_sum": f"{self._agent_run_duration_seconds:.4f}",
+                "average_agent_run_duration_seconds": f"{average_agent_run_duration:.4f}",
+                "max_agent_run_duration_seconds": f"{self._max_agent_run_duration_seconds:.4f}",
+                "runtime_runs_total": str(self._runtime_runs_total),
+                "runtime_runs_by_status": _string_counts(self._runtime_runs_by_status),
+                "runtime_runs_by_lifecycle_state": _string_counts(
+                    self._runtime_runs_by_lifecycle_state
+                ),
+                "runtime_runs_by_auth_subject": _string_counts(
+                    self._runtime_runs_by_auth_subject
+                ),
+                "runtime_runs_by_auth_subject_status": _string_counts(
+                    self._runtime_runs_by_auth_subject_status
+                ),
+                "runtime_runs_by_auth_subject_lifecycle_state": _string_counts(
+                    self._runtime_runs_by_auth_subject_lifecycle_state
+                ),
+                "runtime_resumes_by_auth_subject": _string_counts(
+                    self._runtime_resumes_by_auth_subject
+                ),
+                "runtime_approvals_by_auth_subject": _string_counts(
+                    self._runtime_approvals_by_auth_subject
+                ),
+                "runtime_failed_observations_total": str(
+                    self._runtime_failed_observations_total
+                ),
+                "runtime_progress_event_sink_failures_total": str(
+                    self._runtime_progress_event_sink_failures_total
+                ),
+                "runtime_hook_failures_total": str(
+                    self._runtime_hook_failures_total
+                ),
+                "runtime_reconciliation_runs_total": str(
+                    self._runtime_reconciliation_runs_total
+                ),
+                "runtime_reconciliation_runs_by_status": _string_counts(
+                    self._runtime_reconciliation_runs_by_status
+                ),
+                "runtime_reconciliation_traces_scanned_total": str(
+                    self._runtime_reconciliation_traces_scanned_total
+                ),
+                "runtime_reconciliation_outcomes": _string_counts(
+                    self._runtime_reconciliation_outcomes
+                ),
+                "runtime_reconciliation_errors_total": str(
+                    self._runtime_reconciliation_errors_total
+                ),
+                "runtime_observation_errors_by_code": _string_counts(
+                    self._runtime_observation_errors_by_code
+                ),
+                "runtime_tool_executions_by_tool_status": _string_counts(
+                    self._runtime_tool_executions_by_tool_status
+                ),
+                "runtime_planner_attempts_by_status": _string_counts(
+                    self._runtime_planner_attempts_by_status
+                ),
+                "runtime_planner_failures_total": str(
+                    self._runtime_planner_failures_total
+                ),
+                "runtime_planner_failures_by_error_code": _string_counts(
+                    self._runtime_planner_failures_by_error_code
+                ),
+                "runtime_llm_provider_requests_total": str(
+                    self._runtime_llm_provider_requests_total
+                ),
+                "runtime_llm_provider_request_attempts_total": str(
+                    self._runtime_llm_provider_request_attempts_total
+                ),
+                "runtime_llm_provider_request_retries_total": str(
+                    self._runtime_llm_provider_request_retries_total
+                ),
+                "runtime_llm_provider_requests_by_status": _string_counts(
+                    self._runtime_llm_provider_requests_by_status
+                ),
+                "runtime_llm_provider_request_errors_by_type": _string_counts(
+                    self._runtime_llm_provider_request_errors_by_type
+                ),
+                "runtime_llm_provider_request_http_status": _string_counts(
+                    self._runtime_llm_provider_request_http_status
+                ),
+                "runtime_llm_provider_request_retryable_reason": _string_counts(
+                    self._runtime_llm_provider_request_retryable_reason
+                ),
+                "runtime_llm_provider_request_duration_seconds_bucket": _string_counts(
+                    self._runtime_llm_provider_request_duration_buckets
+                ),
+                "runtime_llm_provider_request_duration_seconds_count": str(
+                    self._runtime_llm_provider_requests_total
+                ),
+                "runtime_llm_provider_request_duration_seconds_sum": (
+                    f"{self._runtime_llm_provider_request_duration_seconds:.4f}"
+                ),
+                "average_runtime_llm_provider_request_duration_seconds": (
+                    f"{average_runtime_llm_provider_request_duration:.4f}"
+                ),
+                "max_runtime_llm_provider_request_duration_seconds": (
+                    f"{self._max_runtime_llm_provider_request_duration_seconds:.4f}"
+                ),
+                "runtime_approval_required_total": str(
+                    self._runtime_approval_required_total
+                ),
+                "runtime_failed_budget_exhaustions_total": str(
+                    self._runtime_failed_budget_exhaustions_total
+                ),
+                "runtime_run_duration_seconds_bucket": _string_counts(
+                    self._runtime_run_duration_buckets
+                ),
+                "runtime_run_duration_seconds_count": str(self._runtime_runs_total),
+                "runtime_run_duration_seconds_sum": (
+                    f"{self._runtime_run_duration_seconds:.4f}"
+                ),
+                "average_runtime_run_duration_seconds": (
+                    f"{average_runtime_run_duration:.4f}"
+                ),
+                "max_runtime_run_duration_seconds": (
+                    f"{self._max_runtime_run_duration_seconds:.4f}"
+                ),
+                "uptime_seconds": f"{max(0.0, current_time - self._started_at):.4f}",
+            }
+
+
+class ServiceRateLimiter:
+    def __init__(self, *, limit_per_minute: int) -> None:
+        if limit_per_minute < 0:
+            raise ValueError("limit_per_minute must be non-negative")
+        self._limit_per_minute = limit_per_minute
+        self._lock = Lock()
+        self._windows: Dict[str, Tuple[float, int]] = {}
+
+    def allow(self, key: str, *, now: Optional[float] = None) -> bool:
+        if self._limit_per_minute == 0:
+            return True
+        current_time = time.monotonic() if now is None else now
+        with self._lock:
+            self._prune_expired_windows(current_time)
+            window_start, count = self._windows.get(key, (current_time, 0))
+            if current_time - window_start >= 60:
+                self._windows[key] = (current_time, 1)
+                return True
+            if count >= self._limit_per_minute:
+                return False
+            self._windows[key] = (window_start, count + 1)
+            return True
+
+    def snapshot(self, *, now: Optional[float] = None) -> Dict[str, str]:
+        current_time = time.monotonic() if now is None else now
+        with self._lock:
+            self._prune_expired_windows(current_time)
+            return {
+                "active_rate_limit_windows": str(len(self._windows)),
+                "rate_limit_per_minute": str(self._limit_per_minute),
+            }
+
+    def retry_after_seconds(self, key: str, *, now: Optional[float] = None) -> int:
+        if self._limit_per_minute == 0:
+            return 0
+        current_time = time.monotonic() if now is None else now
+        with self._lock:
+            self._prune_expired_windows(current_time)
+            window = self._windows.get(key)
+            if window is None:
+                return 0
+            window_start, _count = window
+            remaining = 60 - (current_time - window_start)
+            return max(1, int(ceil(remaining)))
+
+    def _prune_expired_windows(self, current_time: float) -> None:
+        expired_keys = [
+            key
+            for key, (window_start, _count) in self._windows.items()
+            if current_time - window_start >= 60
+        ]
+        for key in expired_keys:
+            del self._windows[key]
+
+
+class ServiceConcurrencyLimiter:
+    def __init__(self, *, max_concurrent_runs: int) -> None:
+        if max_concurrent_runs < 0:
+            raise ValueError("max_concurrent_runs must be non-negative")
+        self._lock = Lock()
+        self._max_concurrent_runs = max_concurrent_runs
+        self._active_runs = 0
+        self._disabled = max_concurrent_runs == 0
+        self._semaphore = BoundedSemaphore(max_concurrent_runs or 1)
+
+    def try_acquire(self) -> Optional[Callable[[], None]]:
+        if self._disabled:
+            return _noop_release
+        if not self._semaphore.acquire(blocking=False):
+            return None
+        with self._lock:
+            self._active_runs += 1
+
+        released = False
+
+        def release() -> None:
+            nonlocal released
+            if released:
+                return
+            released = True
+            with self._lock:
+                self._active_runs -= 1
+            self._semaphore.release()
+
+        return release
+
+    def snapshot(self) -> Dict[str, str]:
+        with self._lock:
+            return {
+                "active_concurrent_runs": str(self._active_runs),
+                "max_concurrent_runs": str(self._max_concurrent_runs),
+            }
+
+
+def access_log_record(
+    *,
+    method: str,
+    path: str,
+    status_code: int,
+    duration_seconds: float,
+    request_id: str,
+    remote_addr: str,
+    error_code: str = "",
+    run_id: str = "",
+    trace_path: str = "",
+    idempotency_key_present: Optional[bool] = None,
+    request_body_bytes: Optional[int] = None,
+    auth_subject: str = "",
+    runtime_owner_auth_subject: str = "",
+    resumed_by_auth_subject: str = "",
+    approved_by_auth_subject: str = "",
+) -> Dict[str, Any]:
+    record = {
+        "event": "http_request",
+        "method": method,
+        "path": path,
+        "status_code": status_code,
+        "duration_seconds": f"{duration_seconds:.4f}",
+        "request_id": request_id,
+        "remote_addr": remote_addr,
+    }
+    if error_code:
+        record["error_code"] = error_code
+    if run_id:
+        record["run_id"] = run_id
+    if trace_path:
+        record["trace_path"] = trace_path
+    if idempotency_key_present is not None:
+        record["idempotency_key_present"] = idempotency_key_present
+    if request_body_bytes is not None:
+        record["request_body_bytes"] = request_body_bytes
+    if auth_subject:
+        record["auth_subject"] = auth_subject
+    if runtime_owner_auth_subject:
+        record["runtime_owner_auth_subject"] = runtime_owner_auth_subject
+    if resumed_by_auth_subject:
+        record["resumed_by_auth_subject"] = resumed_by_auth_subject
+    if approved_by_auth_subject:
+        record["approved_by_auth_subject"] = approved_by_auth_subject
+    return record
+
+
+def access_log_schema() -> Dict[str, Any]:
+    return {
+        "type": "object",
+        "required": [
+            "event",
+            "method",
+            "path",
+            "status_code",
+            "duration_seconds",
+            "request_id",
+            "remote_addr",
+        ],
+        "properties": {
+            "event": {"type": "string", "const": "http_request"},
+            "method": {"type": "string"},
+            "path": {"type": "string"},
+            "status_code": {"type": "integer"},
+            "duration_seconds": {"type": "string", "pattern": r"^\d+\.\d{4}$"},
+            "request_id": {"type": "string"},
+            "remote_addr": {"type": "string"},
+            "error_code": {"type": "string"},
+            "run_id": {"type": "string"},
+            "trace_path": {"type": "string"},
+            "idempotency_key_present": {"type": "boolean"},
+            "request_body_bytes": {"type": "integer"},
+            "auth_subject": {"type": "string"},
+            "runtime_owner_auth_subject": {"type": "string"},
+            "resumed_by_auth_subject": {"type": "string"},
+            "approved_by_auth_subject": {"type": "string"},
+        },
+        "additionalProperties": False,
+    }
+
+
+def prometheus_metrics_text(snapshot: Mapping[str, Any]) -> str:
+    lines = [
+        "# HELP kagent_requests_total Total HTTP requests handled.",
+        "# TYPE kagent_requests_total counter",
+        f"kagent_requests_total {snapshot.get('requests_total', '0')}",
+        "# HELP kagent_responses_total HTTP responses by status code.",
+        "# TYPE kagent_responses_total counter",
+    ]
+    for status, count in _mapping_value(snapshot, "responses_by_status").items():
+        lines.append(
+            f'kagent_responses_total{{status="{_prometheus_label(status)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_requests_by_method_total HTTP requests by method.",
+            "# TYPE kagent_requests_by_method_total counter",
+        ]
+    )
+    for method, count in _mapping_value(snapshot, "requests_by_method").items():
+        lines.append(
+            "kagent_requests_by_method_total"
+            f'{{method="{_prometheus_label(method)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_requests_by_path_total HTTP requests by route.",
+            "# TYPE kagent_requests_by_path_total counter",
+        ]
+    )
+    for path, count in _mapping_value(snapshot, "requests_by_path").items():
+        lines.append(
+            f'kagent_requests_by_path_total{{path="{_prometheus_label(path)}"}} '
+            f"{count}"
+        )
+    lines.extend(
+        [
+            "# HELP kagent_requests_by_auth_subject_total "
+            "HTTP requests by authenticated internal subject.",
+            "# TYPE kagent_requests_by_auth_subject_total counter",
+        ]
+    )
+    for auth_subject, count in _mapping_value(snapshot, "requests_by_auth_subject").items():
+        lines.append(
+            "kagent_requests_by_auth_subject_total"
+            f'{{auth_subject="{_prometheus_label(auth_subject)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_error_responses_total Errors by stable code.",
+            "# TYPE kagent_error_responses_total counter",
+        ]
+    )
+    for error_code, count in _mapping_value(snapshot, "error_responses_by_code").items():
+        lines.append(
+            "kagent_error_responses_total"
+            f'{{error_code="{_prometheus_label(error_code)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_request_duration_seconds "
+            "HTTP request duration in seconds.",
+            "# TYPE kagent_request_duration_seconds histogram",
+        ]
+    )
+    for bucket, count in _mapping_value(snapshot, "request_duration_seconds_bucket").items():
+        lines.append(
+            "kagent_request_duration_seconds_bucket"
+            f'{{le="{_prometheus_label(bucket)}"}} {count}'
+        )
+    lines.append(
+        "kagent_request_duration_seconds_count "
+        f"{snapshot.get('request_duration_seconds_count', '0')}"
+    )
+    lines.append(
+        "kagent_request_duration_seconds_sum "
+        f"{snapshot.get('request_duration_seconds_sum', '0.0000')}"
+    )
+    lines.extend(
+        [
+            "# HELP kagent_runs_total Total agent runs handled by /run.",
+            "# TYPE kagent_runs_total counter",
+            f"kagent_runs_total {snapshot.get('agent_runs_total', '0')}",
+            "# HELP kagent_run_status_total Agent runs by final status.",
+            "# TYPE kagent_run_status_total counter",
+        ]
+    )
+    for status, count in _mapping_value(snapshot, "agent_runs_by_status").items():
+        lines.append(
+            f'kagent_run_status_total{{status="{_prometheus_label(status)}"}} '
+            f"{count}"
+        )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_runs_total "
+            "Total Codex-style runtime runs handled.",
+            "# TYPE kagent_runtime_runs_total counter",
+            f"kagent_runtime_runs_total "
+            f"{snapshot.get('runtime_runs_total', '0')}",
+            "# HELP kagent_runtime_run_status_total "
+            "Codex-style runtime runs by final status.",
+            "# TYPE kagent_runtime_run_status_total counter",
+        ]
+    )
+    for status, count in _mapping_value(snapshot, "runtime_runs_by_status").items():
+        lines.append(
+            "kagent_runtime_run_status_total"
+            f'{{status="{_prometheus_label(status)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_run_lifecycle_state_total "
+            "Codex-style runtime runs by derived operator lifecycle state.",
+            "# TYPE kagent_runtime_run_lifecycle_state_total counter",
+        ]
+    )
+    for lifecycle_state, count in _mapping_value(
+        snapshot,
+        "runtime_runs_by_lifecycle_state",
+    ).items():
+        lines.append(
+            "kagent_runtime_run_lifecycle_state_total"
+            f'{{lifecycle_state="{_prometheus_label(lifecycle_state)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_runs_by_auth_subject_total "
+            "Codex-style runtime runs by authenticated internal subject.",
+            "# TYPE kagent_runtime_runs_by_auth_subject_total counter",
+        ]
+    )
+    for auth_subject, count in _mapping_value(
+        snapshot,
+        "runtime_runs_by_auth_subject",
+    ).items():
+        lines.append(
+            "kagent_runtime_runs_by_auth_subject_total"
+            f'{{auth_subject="{_prometheus_label(auth_subject)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_run_status_by_auth_subject_total "
+            "Codex-style runtime runs by authenticated internal subject and final status.",
+            "# TYPE kagent_runtime_run_status_by_auth_subject_total counter",
+        ]
+    )
+    for subject_status, count in _mapping_value(
+        snapshot,
+        "runtime_runs_by_auth_subject_status",
+    ).items():
+        auth_subject, status = _split_combined_metrics_key(subject_status)
+        lines.append(
+            "kagent_runtime_run_status_by_auth_subject_total"
+            f'{{auth_subject="{_prometheus_label(auth_subject)}",'
+            f'status="{_prometheus_label(status)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_run_lifecycle_state_by_auth_subject_total "
+            "Codex-style runtime runs by authenticated internal subject and lifecycle state.",
+            "# TYPE kagent_runtime_run_lifecycle_state_by_auth_subject_total counter",
+        ]
+    )
+    for subject_lifecycle, count in _mapping_value(
+        snapshot,
+        "runtime_runs_by_auth_subject_lifecycle_state",
+    ).items():
+        auth_subject, lifecycle_state = _split_combined_metrics_key(subject_lifecycle)
+        lines.append(
+            "kagent_runtime_run_lifecycle_state_by_auth_subject_total"
+            f'{{auth_subject="{_prometheus_label(auth_subject)}",'
+            f'lifecycle_state="{_prometheus_label(lifecycle_state)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_resumes_by_auth_subject_total "
+            "Codex-style runtime resumes by authenticated resume subject.",
+            "# TYPE kagent_runtime_resumes_by_auth_subject_total counter",
+        ]
+    )
+    for auth_subject, count in _mapping_value(
+        snapshot,
+        "runtime_resumes_by_auth_subject",
+    ).items():
+        lines.append(
+            "kagent_runtime_resumes_by_auth_subject_total"
+            f'{{auth_subject="{_prometheus_label(auth_subject)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_approvals_by_auth_subject_total "
+            "Codex-style runtime approvals by authenticated approver subject.",
+            "# TYPE kagent_runtime_approvals_by_auth_subject_total counter",
+        ]
+    )
+    for auth_subject, count in _mapping_value(
+        snapshot,
+        "runtime_approvals_by_auth_subject",
+    ).items():
+        lines.append(
+            "kagent_runtime_approvals_by_auth_subject_total"
+            f'{{auth_subject="{_prometheus_label(auth_subject)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_failed_observations_total "
+            "Failed observations produced by Codex-style runtime runs.",
+            "# TYPE kagent_runtime_failed_observations_total counter",
+            "kagent_runtime_failed_observations_total "
+            f"{snapshot.get('runtime_failed_observations_total', '0')}",
+            "# HELP kagent_runtime_observation_errors_total "
+            "Runtime observation failures by stable error code.",
+            "# TYPE kagent_runtime_observation_errors_total counter",
+        ]
+    )
+    for error_code, count in _mapping_value(
+        snapshot,
+        "runtime_observation_errors_by_code",
+    ).items():
+        lines.append(
+            "kagent_runtime_observation_errors_total"
+            f'{{error_code="{_prometheus_label(error_code)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_tool_executions_total "
+            "Runtime tool observations by bounded tool and status labels.",
+            "# TYPE kagent_runtime_tool_executions_total counter",
+        ]
+    )
+    for tool_status, count in _mapping_value(
+        snapshot,
+        "runtime_tool_executions_by_tool_status",
+    ).items():
+        tool, status = _split_combined_metrics_key(tool_status)
+        lines.append(
+            "kagent_runtime_tool_executions_total"
+            f'{{tool="{_prometheus_label(tool)}",'
+            f'status="{_prometheus_label(status)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_planner_attempts_total "
+            "Runtime planner attempts by bounded status.",
+            "# TYPE kagent_runtime_planner_attempts_total counter",
+        ]
+    )
+    for status, count in _mapping_value(
+        snapshot,
+        "runtime_planner_attempts_by_status",
+    ).items():
+        lines.append(
+            "kagent_runtime_planner_attempts_total"
+            f'{{status="{_prometheus_label(status)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_planner_failures_total "
+            "Runtime planner failures before tool execution.",
+            "# TYPE kagent_runtime_planner_failures_total counter",
+            "kagent_runtime_planner_failures_total "
+            f"{snapshot.get('runtime_planner_failures_total', '0')}",
+            "# HELP kagent_runtime_planner_failures_by_error_code_total "
+            "Runtime planner failures by stable error code.",
+            "# TYPE kagent_runtime_planner_failures_by_error_code_total counter",
+        ]
+    )
+    for error_code, count in _mapping_value(
+        snapshot,
+        "runtime_planner_failures_by_error_code",
+    ).items():
+        lines.append(
+            "kagent_runtime_planner_failures_by_error_code_total"
+            f'{{error_code="{_prometheus_label(error_code)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_llm_provider_requests_total "
+            "Runtime LLM provider requests with redacted diagnostics.",
+            "# TYPE kagent_runtime_llm_provider_requests_total counter",
+            "kagent_runtime_llm_provider_requests_total "
+            f"{snapshot.get('runtime_llm_provider_requests_total', '0')}",
+            "# HELP kagent_runtime_llm_provider_request_attempts_total "
+            "Runtime LLM provider HTTP attempts including retries.",
+            "# TYPE kagent_runtime_llm_provider_request_attempts_total counter",
+            "kagent_runtime_llm_provider_request_attempts_total "
+            f"{snapshot.get('runtime_llm_provider_request_attempts_total', '0')}",
+            "# HELP kagent_runtime_llm_provider_request_retries_total "
+            "Runtime LLM provider retry attempts.",
+            "# TYPE kagent_runtime_llm_provider_request_retries_total counter",
+            "kagent_runtime_llm_provider_request_retries_total "
+            f"{snapshot.get('runtime_llm_provider_request_retries_total', '0')}",
+            "# HELP kagent_runtime_llm_provider_requests_by_status_total "
+            "Runtime LLM provider requests by bounded status.",
+            "# TYPE kagent_runtime_llm_provider_requests_by_status_total counter",
+        ]
+    )
+    for status, count in _mapping_value(
+        snapshot,
+        "runtime_llm_provider_requests_by_status",
+    ).items():
+        lines.append(
+            "kagent_runtime_llm_provider_requests_by_status_total"
+            f'{{status="{_prometheus_label(status)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_llm_provider_request_errors_by_type_total "
+            "Runtime LLM provider request failures by bounded error type.",
+            "# TYPE kagent_runtime_llm_provider_request_errors_by_type_total counter",
+        ]
+    )
+    for error_type, count in _mapping_value(
+        snapshot,
+        "runtime_llm_provider_request_errors_by_type",
+    ).items():
+        lines.append(
+            "kagent_runtime_llm_provider_request_errors_by_type_total"
+            f'{{error_type="{_prometheus_label(error_type)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_llm_provider_request_http_status_total "
+            "Runtime LLM provider HTTP failures by status code.",
+            "# TYPE kagent_runtime_llm_provider_request_http_status_total counter",
+        ]
+    )
+    for http_status, count in _mapping_value(
+        snapshot,
+        "runtime_llm_provider_request_http_status",
+    ).items():
+        lines.append(
+            "kagent_runtime_llm_provider_request_http_status_total"
+            f'{{http_status="{_prometheus_label(http_status)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_llm_provider_request_retryable_reason_total "
+            "Runtime LLM provider retryable failures by bounded reason.",
+            "# TYPE kagent_runtime_llm_provider_request_retryable_reason_total counter",
+        ]
+    )
+    for reason, count in _mapping_value(
+        snapshot,
+        "runtime_llm_provider_request_retryable_reason",
+    ).items():
+        lines.append(
+            "kagent_runtime_llm_provider_request_retryable_reason_total"
+            f'{{retryable_reason="{_prometheus_label(reason)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_llm_provider_request_duration_seconds "
+            "Runtime LLM provider request duration in seconds.",
+            "# TYPE kagent_runtime_llm_provider_request_duration_seconds histogram",
+        ]
+    )
+    for bucket, count in _mapping_value(
+        snapshot,
+        "runtime_llm_provider_request_duration_seconds_bucket",
+    ).items():
+        lines.append(
+            "kagent_runtime_llm_provider_request_duration_seconds_bucket"
+            f'{{le="{_prometheus_label(bucket)}"}} {count}'
+        )
+    lines.append(
+        "kagent_runtime_llm_provider_request_duration_seconds_count "
+        f"{snapshot.get('runtime_llm_provider_request_duration_seconds_count', '0')}"
+    )
+    lines.append(
+        "kagent_runtime_llm_provider_request_duration_seconds_sum "
+        f"{snapshot.get('runtime_llm_provider_request_duration_seconds_sum', '0.0000')}"
+    )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_progress_event_sink_failures_total "
+            "Runtime progress event sink delivery failures.",
+            "# TYPE kagent_runtime_progress_event_sink_failures_total counter",
+            "kagent_runtime_progress_event_sink_failures_total "
+            f"{snapshot.get('runtime_progress_event_sink_failures_total', '0')}",
+            "# HELP kagent_runtime_hook_failures_total "
+            "Runtime hook callback failures.",
+            "# TYPE kagent_runtime_hook_failures_total counter",
+            "kagent_runtime_hook_failures_total "
+            f"{snapshot.get('runtime_hook_failures_total', '0')}",
+            "# HELP kagent_runtime_reconciliation_runs_total "
+            "Runtime startup reconciliation passes by bounded status.",
+            "# TYPE kagent_runtime_reconciliation_runs_total counter",
+        ]
+    )
+    for status, count in _mapping_value(
+        snapshot,
+        "runtime_reconciliation_runs_by_status",
+    ).items():
+        lines.append(
+            "kagent_runtime_reconciliation_runs_total"
+            f'{{status="{_prometheus_label(status)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_reconciliation_traces_scanned_total "
+            "Persisted runtime traces scanned during startup reconciliation.",
+            "# TYPE kagent_runtime_reconciliation_traces_scanned_total counter",
+            "kagent_runtime_reconciliation_traces_scanned_total "
+            f"{snapshot.get('runtime_reconciliation_traces_scanned_total', '0')}",
+            "# HELP kagent_runtime_reconciliation_outcomes_total "
+            "Startup reconciliation outcomes by bounded outcome.",
+            "# TYPE kagent_runtime_reconciliation_outcomes_total counter",
+        ]
+    )
+    for outcome, count in _mapping_value(
+        snapshot,
+        "runtime_reconciliation_outcomes",
+    ).items():
+        lines.append(
+            "kagent_runtime_reconciliation_outcomes_total"
+            f'{{outcome="{_prometheus_label(outcome)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_reconciliation_errors_total "
+            "Errors encountered while reconciling persisted runtime traces.",
+            "# TYPE kagent_runtime_reconciliation_errors_total counter",
+            "kagent_runtime_reconciliation_errors_total "
+            f"{snapshot.get('runtime_reconciliation_errors_total', '0')}",
+            "# HELP kagent_runtime_approval_required_total "
+            "Approval-required observations produced by Codex-style runtime runs.",
+            "# TYPE kagent_runtime_approval_required_total counter",
+            "kagent_runtime_approval_required_total "
+            f"{snapshot.get('runtime_approval_required_total', '0')}",
+            "# HELP kagent_runtime_final_answer_guardrails_total "
+            "Final answers corrected by runtime guardrails.",
+            "# TYPE kagent_runtime_final_answer_guardrails_total counter",
+            "kagent_runtime_final_answer_guardrails_total "
+            f"{snapshot.get('runtime_final_answer_guardrails_total', '0')}",
+            "# HELP kagent_runtime_final_answer_guardrails_by_reason_total "
+            "Final answer guardrail corrections by stable reason.",
+            "# TYPE kagent_runtime_final_answer_guardrails_by_reason_total counter",
+        ]
+    )
+    for reason, count in _mapping_value(
+        snapshot,
+        "runtime_final_answer_guardrails_by_reason",
+    ).items():
+        lines.append(
+            "kagent_runtime_final_answer_guardrails_by_reason_total"
+            f'{{reason="{_prometheus_label(reason)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "# HELP kagent_runtime_pending_approvals_current "
+            "Current persisted pending approval queue size.",
+            "# TYPE kagent_runtime_pending_approvals_current gauge",
+            "kagent_runtime_pending_approvals_current "
+            f"{snapshot.get('runtime_pending_approvals_current', '0')}",
+            "# HELP kagent_runtime_stale_pending_approvals_current "
+            "Current pending approvals older than the configured stale threshold.",
+            "# TYPE kagent_runtime_stale_pending_approvals_current gauge",
+            "kagent_runtime_stale_pending_approvals_current "
+            f"{snapshot.get('runtime_stale_pending_approvals_current', '0')}",
+            "# HELP kagent_runtime_max_pending_approval_age_seconds "
+            "Maximum current pending approval age in seconds.",
+            "# TYPE kagent_runtime_max_pending_approval_age_seconds gauge",
+            "kagent_runtime_max_pending_approval_age_seconds "
+            f"{snapshot.get('runtime_max_pending_approval_age_seconds', '0')}",
+            "# HELP kagent_runtime_pending_approval_stale_seconds "
+            "Configured stale pending approval age threshold in seconds.",
+            "# TYPE kagent_runtime_pending_approval_stale_seconds gauge",
+            "kagent_runtime_pending_approval_stale_seconds "
+            f"{snapshot.get('runtime_pending_approval_stale_seconds', '3600')}",
+            "# HELP kagent_runtime_failed_budget_exhaustions_total "
+            "Failed Codex-style runtime runs that exhausted their iteration budget.",
+            "# TYPE kagent_runtime_failed_budget_exhaustions_total counter",
+            "kagent_runtime_failed_budget_exhaustions_total "
+            f"{snapshot.get('runtime_failed_budget_exhaustions_total', '0')}",
+            "# HELP kagent_runtime_run_duration_seconds "
+            "Codex-style runtime run duration in seconds.",
+            "# TYPE kagent_runtime_run_duration_seconds histogram",
+        ]
+    )
+    for bucket, count in _mapping_value(
+        snapshot,
+        "runtime_run_duration_seconds_bucket",
+    ).items():
+        lines.append(
+            "kagent_runtime_run_duration_seconds_bucket"
+            f'{{le="{_prometheus_label(bucket)}"}} {count}'
+        )
+    lines.extend(
+        [
+            "kagent_runtime_run_duration_seconds_count "
+            f"{snapshot.get('runtime_run_duration_seconds_count', '0')}",
+            "kagent_runtime_run_duration_seconds_sum "
+            f"{snapshot.get('runtime_run_duration_seconds_sum', '0.0000')}",
+        ]
+    )
+    lines.extend(
+        [
+            "# HELP kagent_agent_run_duration_seconds "
+            "Agent run duration in seconds.",
+            "# TYPE kagent_agent_run_duration_seconds histogram",
+        ]
+    )
+    for bucket, count in _mapping_value(snapshot, "agent_run_duration_seconds_bucket").items():
+        lines.append(
+            "kagent_agent_run_duration_seconds_bucket"
+            f'{{le="{_prometheus_label(bucket)}"}} {count}'
+        )
+    lines.append(
+        "kagent_agent_run_duration_seconds_count "
+        f"{snapshot.get('agent_run_duration_seconds_count', '0')}"
+    )
+    lines.append(
+        "kagent_agent_run_duration_seconds_sum "
+        f"{snapshot.get('agent_run_duration_seconds_sum', '0.0000')}"
+    )
+    if "service_version" in snapshot:
+        build_labels = {
+            "auth_required": snapshot.get("auth_required", "false"),
+            "auth_subject_count": snapshot.get("auth_subject_count", "0"),
+            "allow_full_trace_response": snapshot.get("allow_full_trace_response", "false"),
+            "bind_host": snapshot.get("bind_host", ""),
+            "bind_port": snapshot.get("bind_port", "0"),
+            "idempotency_cache_backend": snapshot.get(
+                "idempotency_cache_backend",
+                "memory",
+            ),
+            "idempotency_cache_path_configured": snapshot.get(
+                "idempotency_cache_path_configured",
+                "false",
+            ),
+            "idempotency_cache_size": snapshot.get("idempotency_cache_size", "0"),
+            "runtime_allowed_tools": snapshot.get("runtime_allowed_tools", "default"),
+            "runtime_allowed_tools_by_subject_count": snapshot.get(
+                "runtime_allowed_tools_by_subject_count",
+                "0",
+            ),
+            "runtime_max_iterations": snapshot.get("runtime_max_iterations", "0"),
+            "max_concurrent_runs": snapshot.get("max_concurrent_runs", "0"),
+            "max_goal_chars": snapshot.get("max_goal_chars", "0"),
+            "max_request_bytes": snapshot.get("max_request_bytes", "0"),
+            "protect_diagnostics": snapshot.get("protect_diagnostics", "false"),
+            "rate_limit_per_minute": snapshot.get("rate_limit_per_minute", "0"),
+            "request_timeout_seconds": snapshot.get("request_timeout_seconds", "0"),
+            "run_timeout_seconds": snapshot.get("run_timeout_seconds", "0"),
+            "trace_persistence": snapshot.get("trace_persistence", "disabled"),
+            "trust_forwarded_for": snapshot.get("trust_forwarded_for", "false"),
+            "version": snapshot["service_version"],
+            "trace_directory_permissions": snapshot.get(
+                "trace_directory_permissions",
+                "",
+            ),
+            "trace_file_permissions": snapshot.get("trace_file_permissions", ""),
+            "trace_probe_file_permissions": snapshot.get(
+                "trace_probe_file_permissions",
+                "",
+            ),
+            "embedding_provider": snapshot.get("embedding_provider", "unconfigured"),
+            "embedding_base_url": snapshot.get("embedding_base_url", ""),
+            "embedding_base_url_configured": snapshot.get(
+                "embedding_base_url_configured",
+                "false",
+            ),
+            "embedding_model": snapshot.get("embedding_model", ""),
+            "embedding_api_key_configured": snapshot.get(
+                "embedding_api_key_configured",
+                "false",
+            ),
+            "embedding_timeout_seconds": snapshot.get("embedding_timeout_seconds", "0"),
+            "embedding_max_retries": snapshot.get("embedding_max_retries", "0"),
+            "embedding_retry_backoff_seconds": snapshot.get(
+                "embedding_retry_backoff_seconds",
+                "0",
+            ),
+            "llm_provider": snapshot.get("llm_provider", "unconfigured"),
+            "llm_provider_display_name": snapshot.get(
+                "llm_provider_display_name",
+                "Unconfigured",
+            ),
+            "llm_base_url": snapshot.get("llm_base_url", ""),
+            "llm_base_url_configured": snapshot.get(
+                "llm_base_url_configured",
+                "false",
+            ),
+            "llm_model": snapshot.get("llm_model", ""),
+            "llm_api_key_configured": snapshot.get("llm_api_key_configured", "false"),
+            "llm_timeout_seconds": snapshot.get("llm_timeout_seconds", "0"),
+            "llm_max_retries": snapshot.get("llm_max_retries", "0"),
+            "llm_retry_backoff_seconds": snapshot.get(
+                "llm_retry_backoff_seconds",
+                "0",
+            ),
+            "security_response_headers": snapshot.get("security_response_headers", "unknown"),
+            "cache_control_header": snapshot.get("cache_control_header", ""),
+            "content_security_policy_header": snapshot.get(
+                "content_security_policy_header",
+                "",
+            ),
+            "referrer_policy_header": snapshot.get("referrer_policy_header", ""),
+            "x_frame_options_header": snapshot.get("x_frame_options_header", ""),
+            "x_content_type_options_header": snapshot.get(
+                "x_content_type_options_header",
+                "",
+            ),
+        }
+        build_label_text = ",".join(
+            f'{name}="{_prometheus_label(value)}"'
+            for name, value in build_labels.items()
+        )
+        lines.extend(
+            [
+                "# HELP kagent_build_info Service build and runtime controls.",
+                "# TYPE kagent_build_info gauge",
+                f"kagent_build_info{{{build_label_text}}} 1",
+            ]
+        )
+    scalar_metrics = [
+        (
+            "average_duration_seconds",
+            "gauge",
+            "Average HTTP request duration in seconds.",
+        ),
+        ("max_duration_seconds", "gauge", "Maximum observed HTTP request duration in seconds."),
+        (
+            "average_agent_run_duration_seconds",
+            "gauge",
+            "Average agent run duration in seconds.",
+        ),
+        (
+            "max_agent_run_duration_seconds",
+            "gauge",
+            "Maximum observed agent run duration in seconds.",
+        ),
+        ("uptime_seconds", "gauge", "Service process uptime in seconds."),
+        ("active_concurrent_runs", "gauge", "Currently active agent runs."),
+        ("max_concurrent_runs", "gauge", "Configured maximum concurrent agent runs."),
+        ("max_goal_chars", "gauge", "Configured maximum accepted goal length in characters."),
+        (
+            "runtime_max_iterations",
+            "gauge",
+            "Configured maximum Codex-style runtime planner iterations per request.",
+        ),
+        ("max_request_bytes", "gauge", "Configured maximum accepted request body size in bytes."),
+        ("active_rate_limit_windows", "gauge", "Currently tracked rate limit windows."),
+        ("rate_limit_per_minute", "gauge", "Configured per-client run rate limit per minute."),
+        ("idempotency_cache_entries", "gauge", "Current idempotency cache entry count."),
+        ("idempotency_cache_size", "gauge", "Configured idempotency cache entry limit."),
+        ("idempotency_cache_hits", "counter", "Total idempotency cache hits."),
+        ("idempotency_cache_misses", "counter", "Total idempotency cache misses."),
+        ("idempotency_cache_conflicts", "counter", "Total idempotency key conflicts."),
+        ("idempotency_cache_stores", "counter", "Total idempotency cache stores."),
+        ("idempotency_cache_evictions", "counter", "Total idempotency cache evictions."),
+        ("idempotency_cache_claims", "counter", "Total idempotency execution claims."),
+        ("idempotency_cache_waits", "counter", "Total idempotency claim waits."),
+        (
+            "idempotency_cache_wait_timeouts",
+            "counter",
+            "Total idempotency claim wait timeouts.",
+        ),
+        ("idempotency_cache_takeovers", "counter", "Total expired claim takeovers."),
+    ]
+    for name, metric_type, help_text in scalar_metrics:
+        if name in snapshot:
+            lines.append(f"# HELP kagent_{name} {help_text}")
+            lines.append(f"# TYPE kagent_{name} {metric_type}")
+            lines.append(f"kagent_{name} {snapshot[name]}")
+    return "\n".join(lines) + "\n"
+
+
+def _env_int(env: Mapping[str, str], name: str, default: int) -> int:
+    value = env.get(name)
+    if value in {None, ""}:
+        return default
+    try:
+        return int(value)
+    except ValueError as exc:
+        raise ValueError(f"{name} must be an integer") from exc
+
+
+def _env_float(env: Mapping[str, str], name: str, default: float) -> float:
+    value = env.get(name)
+    if value in {None, ""}:
+        return default
+    try:
+        return float(value)
+    except ValueError as exc:
+        raise ValueError(f"{name} must be a number") from exc
+
+
+def _env_bool(env: Mapping[str, str], name: str, default: bool) -> bool:
+    value = env.get(name)
+    if value in {None, ""}:
+        return default
+    normalized = value.strip().lower()
+    if normalized in {"1", "true", "yes", "on"}:
+        return True
+    if normalized in {"0", "false", "no", "off"}:
+        return False
+    raise ValueError(f"{name} must be a boolean")
+
+
+def _env_auth_tokens(env: Mapping[str, str], name: str) -> Dict[str, str]:
+    value = env.get(name)
+    if value in {None, ""}:
+        return {}
+    try:
+        payload = json.loads(str(value))
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"{name} must be a JSON object") from exc
+    if not isinstance(payload, dict):
+        raise ValueError(f"{name} must be a JSON object")
+    tokens: Dict[str, str] = {}
+    for subject, token in payload.items():
+        if not isinstance(subject, str) or not isinstance(token, str):
+            raise ValueError(f"{name} subjects and tokens must be strings")
+        tokens[subject] = token
+    return tokens
+
+
+def _env_csv_tuple(env: Mapping[str, str], name: str) -> Tuple[str, ...]:
+    value = env.get(name)
+    if value in {None, ""}:
+        return ()
+    items = {item.strip() for item in str(value).split(",") if item.strip()}
+    return tuple(sorted(items))
+
+
+def _env_subject_csv_tuple_map(
+    env: Mapping[str, str],
+    name: str,
+) -> Dict[str, Tuple[str, ...]]:
+    value = env.get(name)
+    if value in {None, ""}:
+        return {}
+    try:
+        payload = json.loads(str(value))
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"{name} must be a JSON object") from exc
+    if not isinstance(payload, dict):
+        raise ValueError(f"{name} must be a JSON object")
+    result: Dict[str, Tuple[str, ...]] = {}
+    for subject in sorted(payload):
+        tools = payload[subject]
+        if not isinstance(subject, str):
+            raise ValueError(f"{name} subjects must be strings")
+        if isinstance(tools, str):
+            result[subject] = tuple(
+                sorted({item.strip() for item in tools.split(",") if item.strip()})
+            )
+        elif isinstance(tools, list) and all(isinstance(item, str) for item in tools):
+            result[subject] = tuple(
+                sorted({item.strip() for item in tools if item.strip()})
+            )
+        else:
+            raise ValueError(f"{name} values must be strings or arrays of strings")
+    return result
+
+
+def _validate_auth_tokens(auth_tokens: Mapping[str, str]) -> None:
+    for subject, token in auth_tokens.items():
+        if not subject or not _safe_header_value(subject):
+            raise ValueError("auth token subjects must be printable ASCII")
+        if not token or not _safe_header_value(f"Bearer {token}"):
+            raise ValueError("auth tokens must be printable ASCII")
+
+
+def _validate_runtime_allowed_tools(runtime_allowed_tools: Tuple[str, ...]) -> None:
+    if not runtime_allowed_tools:
+        return
+    from kagent.runtime.tools import default_runtime_tools
+
+    known_tools = set(default_runtime_tools())
+    unknown_tools = sorted(set(runtime_allowed_tools) - known_tools)
+    if unknown_tools:
+        raise ValueError(
+            "runtime_allowed_tools contains unknown tools: "
+            + ", ".join(unknown_tools)
+        )
+
+
+def _validate_runtime_allowed_tools_by_subject(
+    runtime_allowed_tools_by_subject: Mapping[str, Tuple[str, ...]],
+) -> None:
+    if not runtime_allowed_tools_by_subject:
+        return
+    from kagent.runtime.tools import default_runtime_tools
+
+    known_tools = set(default_runtime_tools())
+    for subject, tools in runtime_allowed_tools_by_subject.items():
+        if not subject or not _safe_header_value(subject):
+            raise ValueError(
+                "runtime_allowed_tools_by_subject subjects must be printable ASCII"
+            )
+        unknown_tools = sorted(set(tools) - known_tools)
+        if unknown_tools:
+            raise ValueError(
+                "runtime_allowed_tools_by_subject contains unknown tools for "
+                + subject
+                + ": "
+                + ", ".join(unknown_tools)
+            )
+
+
+def _safe_header_value(value: str) -> bool:
+    return bool(value) and all(32 <= ord(character) <= 126 for character in value)
+
+
+def _string_counts(counts: Mapping[str, int]) -> Dict[str, str]:
+    return {key: str(counts[key]) for key in sorted(counts)}
+
+
+def _combined_metrics_key(left: str, right: str) -> str:
+    return f"{left}:{right}"
+
+
+def _split_combined_metrics_key(value: str) -> Tuple[str, str]:
+    left, separator, right = value.partition(":")
+    if not separator:
+        return value, ""
+    return left, right
+
+
+def _runtime_tool_metrics_label(value: str) -> str:
+    normalized = str(value).strip()
+    if not normalized:
+        return "unknown"
+    from kagent.runtime.tools import default_runtime_tools
+
+    if normalized in default_runtime_tools():
+        return normalized
+    return "unknown"
+
+
+def _runtime_observation_status_metrics_label(value: str) -> str:
+    normalized = str(value).strip()
+    if normalized in {"failed", "ok", "requires_approval"}:
+        return normalized
+    return "other"
+
+
+def _runtime_planner_attempt_status_metrics_label(value: str) -> str:
+    normalized = str(value).strip()
+    if normalized in {"failed", "ok"}:
+        return normalized
+    return "other"
+
+
+def _runtime_llm_provider_status_metrics_label(value: str) -> str:
+    normalized = str(value).strip()
+    if normalized in {"failed", "ok"}:
+        return normalized
+    return "other"
+
+
+def _runtime_llm_provider_error_type_metrics_label(value: str) -> str:
+    normalized = str(value).strip()
+    if not normalized:
+        return ""
+    if normalized in {
+        "exhausted",
+        "http_error",
+        "provider_error",
+        "response_error",
+        "timeout",
+        "url_error",
+    }:
+        return normalized
+    return "other"
+
+
+def _runtime_llm_provider_http_status_metrics_label(value: str) -> str:
+    normalized = str(value).strip()
+    if len(normalized) == 3 and normalized.isdigit():
+        return normalized
+    return ""
+
+
+def _runtime_llm_provider_retryable_reason_metrics_label(value: str) -> str:
+    normalized = str(value).strip()
+    if normalized in {"model_unloaded"}:
+        return normalized
+    if normalized:
+        return "other"
+    return ""
+
+
+def _runtime_lifecycle_state(status: str) -> str:
+    if status == "cancelled":
+        return "cancelled"
+    if status == "done":
+        return "succeeded"
+    if status == "failed":
+        return "failed"
+    if status == "requires_approval":
+        return "waiting_approval"
+    if status in {"running", "resuming"}:
+        return "running"
+    if status == "resumed":
+        return "succeeded"
+    return "unknown"
+
+
+def _duration_bucket_labels() -> Tuple[Tuple[float, str], ...]:
+    return tuple((bucket, f"{bucket:g}") for bucket in _DURATION_BUCKETS)
+
+
+def _empty_duration_buckets() -> Dict[str, int]:
+    buckets = {label: 0 for _bucket, label in _duration_bucket_labels()}
+    buckets["+Inf"] = 0
+    return buckets
+
+
+def _non_negative_int_value(value: Any) -> int:
+    try:
+        return max(0, int(value))
+    except (TypeError, ValueError):
+        return 0
+
+
+def _non_negative_float_value(value: Any) -> float:
+    try:
+        return max(0.0, float(value))
+    except (TypeError, ValueError):
+        return 0.0
+
+
+def _method_metrics_label(method: str) -> str:
+    method_key = method.upper()
+    if method_key in _KNOWN_HTTP_METHODS:
+        return method_key
+    return _UNKNOWN_METRICS_LABEL
+
+
+def _mapping_value(snapshot: Mapping[str, Any], name: str) -> Mapping[str, Any]:
+    value = snapshot.get(name)
+    return value if isinstance(value, dict) else {}
+
+
+def _prometheus_label(value: Any) -> str:
+    return str(value).replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
+
+
+def _noop_release() -> None:
+    return None