mcp-efficiency-engine 0.1.0 → 0.1.2

package/telemetry/exporters/langsmith/exporter.py

@@ -0,0 +1,415 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from datetime import datetime
+from datetime import timezone
+from uuid import UUID
+from uuid import uuid5
+from uuid import NAMESPACE_URL
+from typing import Any
+
+
+class LangSmithExporter:
+    name = "langsmith"
+
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        project: str,
+        endpoint: str | None = None,
+        workspace_id: str | None = None,
+        high_signal_only: bool = True,
+        min_span_duration_ms: float = 100.0,
+        emit_execution_summary: bool = True,
+    ) -> None:
+        self.api_key = api_key.strip()
+        self.project = project.strip()
+        self.endpoint = (endpoint or "").strip() or None
+        self.workspace_id = (workspace_id or "").strip() or None
+        self.high_signal_only = bool(high_signal_only)
+        self.min_span_duration_ms = max(0.0, float(min_span_duration_ms))
+        self.emit_execution_summary = bool(emit_execution_summary)
+        self._client: Any | None = None
+        self._usage_by_execution: dict[str, dict[str, float]] = defaultdict(dict)
+        self._execution_state: dict[str, dict[str, Any]] = defaultdict(dict)
+        if not self.api_key or not self.project:
+            raise ValueError("LangSmith exporter requires api_key and project")
+
+    def _update_execution_state(self, record: dict[str, Any], timestamp: datetime) -> None:
+        context = record.get("context", {})
+        payload = record.get("payload", {})
+        execution_id = str(context.get("execution_id") or "")
+        if not execution_id:
+            return
+
+        state = self._execution_state[execution_id]
+        state.setdefault("warnings", 0)
+        state.setdefault("errors", 0)
+        state.setdefault("operation", payload.get("operation"))
+        state.setdefault("provider", context.get("provider"))
+        state.setdefault("model", context.get("model"))
+
+        event_name = str(record.get("event_name", ""))
+        record_type = str(record.get("type", "event"))
+        level = str(record.get("level", "INFO")).upper()
+
+        if event_name == "ExecutionStarted":
+            started_at = payload.get("started_at") or record.get("timestamp")
+            state["started_at"] = self._parse_timestamp(started_at)
+            if payload.get("operation"):
+                state["operation"] = payload.get("operation")
+        elif event_name == "RoutingResolved":
+            if payload.get("agent"):
+                state["agent"] = payload.get("agent")
+            if payload.get("engine"):
+                state["engine"] = payload.get("engine")
+        elif event_name in {"WarningGenerated"} or level == "WARNING":
+            state["warnings"] = int(state.get("warnings", 0)) + 1
+        elif event_name in {"ExceptionThrown"} or level == "ERROR":
+            state["errors"] = int(state.get("errors", 0)) + 1
+            if payload.get("error"):
+                state["last_error"] = payload.get("error")
+
+        if record_type == "span":
+            status = str(payload.get("status", "OK")).upper()
+            if status != "OK":
+                state["errors"] = int(state.get("errors", 0)) + 1
+            state["last_span_duration_ms"] = payload.get("duration_ms")
+
+    def _build_execution_summary(self, execution_id: str, context: dict[str, Any], timestamp: datetime) -> dict[str, Any]:
+        state = self._execution_state.get(execution_id, {})
+        started_at = state.get("started_at")
+        if isinstance(started_at, datetime):
+            duration_ms = max(0.0, (timestamp - started_at).total_seconds() * 1000.0)
+        else:
+            duration_ms = 0.0
+
+        warnings = int(state.get("warnings", 0))
+        errors = int(state.get("errors", 0))
+        status = "error" if errors > 0 else "ok"
+
+        usage = self._usage_by_execution.get(execution_id, {})
+        usage_summary = {
+            "input_tokens": int(usage.get("tokens_input", 0.0)),
+            "output_tokens": int(usage.get("tokens_output", 0.0)),
+            "total_tokens": int(usage.get("total_tokens", 0.0)),
+            "total_cost": float(usage.get("estimated_cost", 0.0)),
+        }
+
+        summary = {
+            "name": "ExecutionSummary",
+            "run_type": "chain",
+            "inputs": {
+                "execution_id": execution_id,
+                "operation": state.get("operation"),
+                "agent": state.get("agent"),
+                "engine": state.get("engine"),
+            },
+            "outputs": {
+                "status": status,
+                "duration_ms": duration_ms,
+                "warning_count": warnings,
+                "error_count": errors,
+                "usage_metadata": usage_summary,
+                "provider": context.get("provider") or state.get("provider"),
+                "model": context.get("model") or state.get("model"),
+            },
+            "extra": {
+                "metadata": {
+                    "execution_id": execution_id,
+                    "telemetry_kind": "execution_summary",
+                    "last_error": state.get("last_error"),
+                    "usage_metadata": usage_summary,
+                    "ls_provider": context.get("provider") or state.get("provider"),
+                    "ls_model_name": context.get("model") or state.get("model"),
+                }
+            },
+            "id": self._to_uuid(f"{execution_id}:execution-summary", salt="run"),
+            "start_time": started_at if isinstance(started_at, datetime) else timestamp,
+            "end_time": timestamp,
+            "total_tokens": usage_summary["total_tokens"],
+            "total_cost": usage_summary["total_cost"],
+            "tags": ["telemetry", "execution", "summary"],
+        }
+        return summary
+
+    def _is_high_signal_record(self, record: dict[str, Any]) -> bool:
+        record_type = str(record.get("type", "event"))
+        event_name = str(record.get("event_name", "TelemetryEvent"))
+        payload = record.get("payload", {})
+        level = str(record.get("level", "INFO")).upper()
+
+        if level in {"ERROR", "WARNING"}:
+            return True
+
+        if record_type == "metric":
+            # Emit only the per-execution summary for usage/cost to avoid noise.
+            return False
+
+        if record_type == "span":
+            duration_ms = float(payload.get("duration_ms") or 0.0)
+            status = str(payload.get("status", "OK")).upper()
+            return status != "OK" or duration_ms >= self.min_span_duration_ms
+
+        return event_name in {
+            "ExecutionStarted",
+            "ExecutionFinished",
+            "RoutingResolved",
+            "ExceptionThrown",
+            "WarningGenerated",
+        }
+
+    def _to_uuid(self, value: Any, *, salt: str = "") -> str | None:
+        if value is None:
+            return None
+        raw = str(value).strip()
+        if not raw:
+            return None
+        try:
+            return str(UUID(raw))
+        except Exception:
+            try:
+                return str(UUID(hex=raw))
+            except Exception:
+                return str(uuid5(NAMESPACE_URL, f"mcpee:{salt}:{raw}"))
+
+    def _parse_timestamp(self, value: Any) -> datetime:
+        if isinstance(value, datetime):
+            return value.astimezone(timezone.utc)
+        if isinstance(value, str) and value:
+            try:
+                return datetime.fromisoformat(value).astimezone(timezone.utc)
+            except Exception:
+                pass
+        return datetime.now(timezone.utc)
+
+    def _build_usage_summary(self, execution_id: str, context: dict[str, Any], timestamp: datetime) -> dict[str, Any]:
+        usage = self._usage_by_execution.get(execution_id, {})
+        model = context.get("model")
+        provider = context.get("provider")
+        input_tokens = int(usage.get("tokens_input", 0.0))
+        output_tokens = int(usage.get("tokens_output", 0.0))
+        total_tokens = int(usage.get("total_tokens", 0.0))
+        total_cost = float(usage.get("estimated_cost", 0.0))
+        usage_metadata = {
+            "input_tokens": input_tokens,
+            "output_tokens": output_tokens,
+            "total_tokens": total_tokens,
+            "total_cost": total_cost,
+        }
+        return {
+            "name": "UsageSummary",
+            "run_type": "llm",
+            "inputs": {
+                "execution_id": execution_id,
+                "provider": provider,
+                "model": model,
+            },
+            "outputs": {
+                "model": model,
+                "provider": provider,
+                "usage_metadata": usage_metadata,
+                "status": "recorded",
+            },
+            "extra": {
+                "metadata": {
+                    "execution_id": execution_id,
+                    "trace_id": context.get("trace_id"),
+                    "provider": provider,
+                    "model": model,
+                    "ls_provider": provider,
+                    "ls_model_name": model,
+                    "usage_metadata": usage_metadata,
+                    "telemetry_kind": "usage_summary",
+                }
+            },
+            "id": self._to_uuid(f"{execution_id}:usage-summary", salt="run"),
+            "start_time": timestamp,
+            "end_time": timestamp,
+            "prompt_tokens": input_tokens,
+            "completion_tokens": output_tokens,
+            "total_tokens": total_tokens,
+            "total_cost": total_cost,
+            "tags": ["telemetry", "usage", "cost"],
+        }
+
+    def _run_payload(self, record: dict[str, Any], *, index: int) -> dict[str, Any]:
+        event_name = str(record.get("event_name", "TelemetryEvent"))
+        context = record.get("context", {})
+        payload = record.get("payload", {})
+        record_type = str(record.get("type", "event"))
+        level = str(record.get("level", "INFO"))
+        execution_id = str(context.get("execution_id") or "")
+        provider = context.get("provider")
+        model = context.get("model")
+        timestamp = self._parse_timestamp(record.get("timestamp"))
+
+        inputs: dict[str, Any] = {"event": event_name, "type": record_type}
+        outputs: dict[str, Any] = {"status": "recorded", "level": level}
+        run_type = "chain"
+        prompt_tokens: int | None = None
+        completion_tokens: int | None = None
+        total_tokens: int | None = None
+        total_cost: float | None = None
+
+        if record_type == "metric":
+            metric_name = str(payload.get("name", "unknown"))
+            metric_value = payload.get("value")
+            metric_unit = payload.get("unit")
+            inputs = {
+                "metric": metric_name,
+                "value": metric_value,
+                "unit": metric_unit,
+                "execution_id": execution_id,
+            }
+            outputs = {
+                "metric": metric_name,
+                "value": metric_value,
+                "unit": metric_unit,
+                "provider": provider,
+                "model": model,
+                "status": "recorded",
+            }
+            if metric_name in {"tokens_input", "tokens_output", "total_tokens", "estimated_cost"} and execution_id:
+                self._usage_by_execution[execution_id][metric_name] = float(metric_value or 0.0)
+                run_type = "llm"
+                if metric_name == "tokens_input":
+                    prompt_tokens = int(float(metric_value or 0.0))
+                elif metric_name == "tokens_output":
+                    completion_tokens = int(float(metric_value or 0.0))
+                elif metric_name == "total_tokens":
+                    total_tokens = int(float(metric_value or 0.0))
+                elif metric_name == "estimated_cost":
+                    total_cost = float(metric_value or 0.0)
+        elif record_type == "span":
+            run_type = "tool"
+            inputs = {
+                "span": payload.get("name"),
+                "kind": payload.get("kind"),
+                "execution_id": execution_id,
+            }
+            outputs = {
+                "status": payload.get("status", "recorded"),
+                "duration_ms": payload.get("duration_ms"),
+                "error": payload.get("error"),
+                "provider": provider,
+                "model": model,
+            }
+        else:
+            inputs = {
+                "event": event_name,
+                "operation": payload.get("operation"),
+                "execution_id": execution_id,
+            }
+            outputs = {
+                "status": payload.get("status", "recorded"),
+                "provider": provider,
+                "model": model,
+            }
+
+        metadata = {
+            "execution_id": context.get("execution_id"),
+            "trace_id": context.get("trace_id"),
+            "span_id": context.get("span_id"),
+            "parent_span_id": context.get("parent_span_id"),
+            "request_id": context.get("request_id"),
+            "session_id": context.get("session_id"),
+            "correlation_id": context.get("correlation_id"),
+            "user": context.get("user"),
+            "provider": provider,
+            "model": model,
+            "ls_provider": provider,
+            "ls_model_name": model,
+            "event_name": event_name,
+            "type": record_type,
+            "level": level,
+            "payload": payload,
+        }
+
+        if prompt_tokens is not None or completion_tokens is not None or total_tokens is not None or total_cost is not None:
+            metadata["usage_metadata"] = {
+                "input_tokens": prompt_tokens or 0,
+                "output_tokens": completion_tokens or 0,
+                "total_tokens": total_tokens or 0,
+                "total_cost": total_cost or 0.0,
+            }
+
+        run_id = self._to_uuid(
+            f"{execution_id or 'no-exec'}:{context.get('span_id') or 'no-span'}:{event_name}:{record_type}:{index}",
+            salt="run",
+        )
+
+        run_payload: dict[str, Any] = {
+            "name": event_name,
+            "run_type": run_type,
+            "inputs": inputs,
+            "outputs": outputs,
+            "project_name": self.project,
+            "extra": {"metadata": metadata},
+            "id": run_id,
+            "start_time": timestamp,
+            "end_time": timestamp,
+            "tags": ["telemetry", record_type],
+            "_usage_ready": record_type == "metric" and str(payload.get("name", "")) == "estimated_cost" and bool(execution_id),
+            "_execution_id": execution_id,
+            "_context": context,
+        }
+        if prompt_tokens is not None:
+            run_payload["prompt_tokens"] = prompt_tokens
+        if completion_tokens is not None:
+            run_payload["completion_tokens"] = completion_tokens
+        if total_tokens is not None:
+            run_payload["total_tokens"] = total_tokens
+        if total_cost is not None:
+            run_payload["total_cost"] = total_cost
+        return run_payload
+
+    def _get_client(self) -> Any:
+        if self._client is not None:
+            return self._client
+        try:
+            from langsmith import Client  # type: ignore
+        except Exception as exc:
+            raise RuntimeError("langsmith SDK is not installed") from exc
+
+        kwargs: dict[str, Any] = {"api_key": self.api_key}
+        if self.endpoint:
+            kwargs["api_url"] = self.endpoint
+        if self.workspace_id:
+            kwargs["workspace_id"] = self.workspace_id
+        self._client = Client(**kwargs)
+        return self._client
+
+    def export(self, records: list[dict]) -> None:
+        client = self._get_client()
+        for index, record in enumerate(records):
+            run = self._run_payload(record, index=index)
+            usage_ready = bool(run.pop("_usage_ready", False))
+            execution_id = str(run.pop("_execution_id", "") or "")
+            context = run.pop("_context", {})
+            timestamp = run["end_time"]
+
+            self._update_execution_state(record, timestamp)
+
+            should_emit = True
+            if self.high_signal_only:
+                should_emit = self._is_high_signal_record(record)
+
+            if should_emit:
+                client.create_run(**run)
+
+            if usage_ready and execution_id:
+                summary = self._build_usage_summary(execution_id, context, run["end_time"])
+                client.create_run(project_name=self.project, **summary)
+
+            event_name = str(record.get("event_name", ""))
+            if self.emit_execution_summary and execution_id and event_name == "ExecutionFinished":
+                execution_summary = self._build_execution_summary(execution_id, context, timestamp)
+                client.create_run(project_name=self.project, **execution_summary)
+
+    def flush(self) -> None:
+        return
+
+    def shutdown(self) -> None:
+        return

package/telemetry/metrics/aggregator.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class MetricsSnapshot:
+    execution_time_ms: float = 0.0
+    tool_duration_ms: float = 0.0
+    average_duration_ms: float = 0.0
+    parallel_tasks: int = 0
+    sequential_tasks: int = 0
+    retry_count: int = 0
+    error_count: int = 0
+    warning_count: int = 0
+    llm_calls: int = 0
+    provider_calls: int = 0
+    mcp_calls: int = 0
+    tool_invocations: int = 0
+    tokens_input: int = 0
+    tokens_output: int = 0
+    total_tokens: int = 0
+    estimated_cost: float = 0.0
+    cache_hits: int = 0
+    cache_misses: int = 0
+    compression_ratio: float = 0.0
+    parallel_efficiency: float = 0.0
+    prompt_reduction: float = 0.0
+    context_reduction: float = 0.0
+
+
+@dataclass
+class MetricsAggregator:
+    durations: list[float] = field(default_factory=list)
+    snapshot: MetricsSnapshot = field(default_factory=MetricsSnapshot)
+
+    def add_duration(self, value_ms: float) -> None:
+        safe = max(0.0, float(value_ms))
+        self.durations.append(safe)
+        self.snapshot.tool_duration_ms += safe
+        if self.durations:
+            self.snapshot.average_duration_ms = self.snapshot.tool_duration_ms / len(self.durations)
+
+    def add_tokens(self, input_tokens: int, output_tokens: int) -> None:
+        self.snapshot.tokens_input += max(0, int(input_tokens))
+        self.snapshot.tokens_output += max(0, int(output_tokens))
+        self.snapshot.total_tokens = self.snapshot.tokens_input + self.snapshot.tokens_output
+
+    def add_cost(self, estimated_cost: float) -> None:
+        self.snapshot.estimated_cost += max(0.0, float(estimated_cost))
+
+    def bump(self, metric_name: str, value: int = 1) -> None:
+        if not hasattr(self.snapshot, metric_name):
+            return
+        current = getattr(self.snapshot, metric_name)
+        if isinstance(current, int):
+            setattr(self.snapshot, metric_name, max(0, current + int(value)))
+
+    def to_dict(self) -> dict[str, float | int]:
+        return self.snapshot.__dict__.copy()

package/telemetry/scoring/efficiency.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from typing import Any
+
+
+DEFAULT_WEIGHTS = {
+    "time": 0.18,
+    "cost": 0.16,
+    "tokens": 0.14,
+    "cache": 0.12,
+    "parallelism": 0.10,
+    "reuse": 0.10,
+    "calls": 0.10,
+    "compression": 0.10,
+}
+
+
+def _clamp(value: float, lo: float = 0.0, hi: float = 1.0) -> float:
+    return max(lo, min(hi, value))
+
+
+def _safe_ratio(numerator: float, denominator: float) -> float:
+    if denominator <= 0:
+        return 0.0
+    return numerator / denominator
+
+
+def compute_efficiency_score(metrics: dict[str, Any], weights: dict[str, float] | None = None) -> float:
+    w = dict(DEFAULT_WEIGHTS)
+    if weights:
+        w.update(weights)
+
+    duration = float(metrics.get("execution_time_ms", 0.0))
+    cost = float(metrics.get("estimated_cost", 0.0))
+    total_tokens = float(metrics.get("total_tokens", 0.0))
+    cache_hits = float(metrics.get("cache_hits", 0.0))
+    cache_misses = float(metrics.get("cache_misses", 0.0))
+    parallel_eff = float(metrics.get("parallel_efficiency", 0.0))
+    prompt_reduction = float(metrics.get("prompt_reduction", 0.0))
+    context_reduction = float(metrics.get("context_reduction", 0.0))
+    calls = float(
+        metrics.get("tool_invocations", 0)
+        + metrics.get("provider_calls", 0)
+        + metrics.get("llm_calls", 0)
+        + metrics.get("mcp_calls", 0)
+    )
+    compression = float(metrics.get("compression_ratio", 0.0))
+
+    time_score = 1.0 - _clamp(duration / 120000.0)
+    cost_score = 1.0 - _clamp(cost / 2.0)
+    token_score = 1.0 - _clamp(total_tokens / 200000.0)
+    cache_score = _clamp(_safe_ratio(cache_hits, cache_hits + cache_misses))
+    parallel_score = _clamp(parallel_eff)
+    reuse_score = _clamp((prompt_reduction + context_reduction) / 2.0)
+    call_score = 1.0 - _clamp(calls / 200.0)
+    compression_score = _clamp(compression)
+
+    weighted = (
+        time_score * w["time"]
+        + cost_score * w["cost"]
+        + token_score * w["tokens"]
+        + cache_score * w["cache"]
+        + parallel_score * w["parallelism"]
+        + reuse_score * w["reuse"]
+        + call_score * w["calls"]
+        + compression_score * w["compression"]
+    )
+    return round(_clamp(weighted) * 100.0, 2)

package/telemetry/tracing/span.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+@dataclass
+class SpanRecord:
+    name: str
+    kind: str
+    span_id: str
+    parent_span_id: str | None
+    trace_id: str
+    execution_id: str
+    started_at: str
+    ended_at: str | None = None
+    duration_ms: float | None = None
+    status: str = "UNSET"
+    attributes: dict[str, Any] = field(default_factory=dict)
+    events: list[dict[str, Any]] = field(default_factory=list)
+    error: str | None = None
+
+    def add_event(self, name: str, attributes: dict[str, Any] | None = None) -> None:
+        self.events.append(
+            {
+                "name": name,
+                "timestamp": utc_now(),
+                "attributes": attributes or {},
+            }
+        )
+
+    def end(self, *, status: str = "OK", error: str | None = None) -> None:
+        self.ended_at = utc_now()
+        self.status = status
+        self.error = error
+        start_dt = datetime.fromisoformat(self.started_at)
+        end_dt = datetime.fromisoformat(self.ended_at)
+        self.duration_ms = max(0.0, (end_dt - start_dt).total_seconds() * 1000.0)

package/observability/evals/chat-token-usage-report.json

@@ -1,64 +0,0 @@
-{
-  "timestamp": "2026-07-05T09:54:53.886533+00:00",
-  "source": "observability/logs/iteration-metrics.jsonl",
-  "filters": {
-    "source": "copilot-session",
-    "from": "",
-    "to": ""
-  },
-  "totals": {
-    "events": 36,
-    "input_tokens": 3893402,
-    "output_tokens": 126892,
-    "total_tokens": 4020294,
-    "copilot_credits": 816.9227
-  },
-  "breakdown": {
-    "model": {
-      "copilot/auto": {
-        "count": 36,
-        "input_tokens": 3893402,
-        "output_tokens": 126892,
-        "total_tokens": 4020294,
-        "copilot_credits": 816.9227
-      }
-    },
-    "day": {
-      "2026-07-03": {
-        "count": 19,
-        "input_tokens": 2219137,
-        "output_tokens": 67846,
-        "total_tokens": 2286983,
-        "copilot_credits": 492.07088600000003
-      },
-      "2026-07-04": {
-        "count": 9,
-        "input_tokens": 876490,
-        "output_tokens": 21560,
-        "total_tokens": 898050,
-        "copilot_credits": 160.161499
-      },
-      "2026-07-05": {
-        "count": 8,
-        "input_tokens": 797775,
-        "output_tokens": 37486,
-        "total_tokens": 835261,
-        "copilot_credits": 164.690315
-      }
-    }
-  },
-  "license": {
-    "plan": "business",
-    "seats": 1,
-    "included_credits_per_user_month": 19.0,
-    "included_credits_total_month": 19.0,
-    "official_plans_url": "https://github.com/features/copilot/plans?ref_cta=View+pricing+and+plans&ref_loc=hero&ref_page=%2Ffeatures_copilot_copilot_business&plans=business"
-  },
-  "budget": {
-    "used_copilot_credits": 816.9227,
-    "budget_utilization_rate": 42.995932,
-    "overage_credits": 797.9227,
-    "equivalent_seats_used": 42.995932,
-    "percent_of_one_seat": 4299.59
-  }
-}

package/observability/evals/chat-token-usage-report.md

@@ -1,33 +0,0 @@
-# Chat Token Usage Report
-
-- timestamp: 2026-07-05T09:54:53.886533+00:00
-- source: observability/logs/iteration-metrics.jsonl
-- filter.source: copilot-session
-- filter.from: 
-- filter.to: 
-- plan: business
-- seats: 1
-- official_plans_url: https://github.com/features/copilot/plans?ref_cta=View+pricing+and+plans&ref_loc=hero&ref_page=%2Ffeatures_copilot_copilot_business&plans=business
-
-## Totals
-
-- events: 36
-- input_tokens: 3893402
-- output_tokens: 126892
-- total_tokens: 4020294
-- copilot_credits: 816.9227
-- included_credits_total_month: 19.0
-- budget_utilization_rate: 42.995932
-- overage_credits: 797.9227
-- equivalent_seats_used: 42.995932
-- percent_of_one_seat: 4299.59
-
-## Breakdown by Model
-
-- copilot/auto: count=36, input=3893402, output=126892, total=4020294, credits=816.9227
-
-## Breakdown by Day
-
-- 2026-07-03: count=19, input=2219137, output=67846, total=2286983, credits=492.070886
-- 2026-07-04: count=9, input=876490, output=21560, total=898050, credits=160.161499
-- 2026-07-05: count=8, input=797775, output=37486, total=835261, credits=164.690315