mcp-efficiency-engine 0.1.0 → 0.1.1

package/telemetry/tracing/span.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+@dataclass
+class SpanRecord:
+    name: str
+    kind: str
+    span_id: str
+    parent_span_id: str | None
+    trace_id: str
+    execution_id: str
+    started_at: str
+    ended_at: str | None = None
+    duration_ms: float | None = None
+    status: str = "UNSET"
+    attributes: dict[str, Any] = field(default_factory=dict)
+    events: list[dict[str, Any]] = field(default_factory=list)
+    error: str | None = None
+
+    def add_event(self, name: str, attributes: dict[str, Any] | None = None) -> None:
+        self.events.append(
+            {
+                "name": name,
+                "timestamp": utc_now(),
+                "attributes": attributes or {},
+            }
+        )
+
+    def end(self, *, status: str = "OK", error: str | None = None) -> None:
+        self.ended_at = utc_now()
+        self.status = status
+        self.error = error
+        start_dt = datetime.fromisoformat(self.started_at)
+        end_dt = datetime.fromisoformat(self.ended_at)
+        self.duration_ms = max(0.0, (end_dt - start_dt).total_seconds() * 1000.0)

package/observability/evals/chat-token-usage-report.json

@@ -1,64 +0,0 @@
-{
-  "timestamp": "2026-07-05T09:54:53.886533+00:00",
-  "source": "observability/logs/iteration-metrics.jsonl",
-  "filters": {
-    "source": "copilot-session",
-    "from": "",
-    "to": ""
-  },
-  "totals": {
-    "events": 36,
-    "input_tokens": 3893402,
-    "output_tokens": 126892,
-    "total_tokens": 4020294,
-    "copilot_credits": 816.9227
-  },
-  "breakdown": {
-    "model": {
-      "copilot/auto": {
-        "count": 36,
-        "input_tokens": 3893402,
-        "output_tokens": 126892,
-        "total_tokens": 4020294,
-        "copilot_credits": 816.9227
-      }
-    },
-    "day": {
-      "2026-07-03": {
-        "count": 19,
-        "input_tokens": 2219137,
-        "output_tokens": 67846,
-        "total_tokens": 2286983,
-        "copilot_credits": 492.07088600000003
-      },
-      "2026-07-04": {
-        "count": 9,
-        "input_tokens": 876490,
-        "output_tokens": 21560,
-        "total_tokens": 898050,
-        "copilot_credits": 160.161499
-      },
-      "2026-07-05": {
-        "count": 8,
-        "input_tokens": 797775,
-        "output_tokens": 37486,
-        "total_tokens": 835261,
-        "copilot_credits": 164.690315
-      }
-    }
-  },
-  "license": {
-    "plan": "business",
-    "seats": 1,
-    "included_credits_per_user_month": 19.0,
-    "included_credits_total_month": 19.0,
-    "official_plans_url": "https://github.com/features/copilot/plans?ref_cta=View+pricing+and+plans&ref_loc=hero&ref_page=%2Ffeatures_copilot_copilot_business&plans=business"
-  },
-  "budget": {
-    "used_copilot_credits": 816.9227,
-    "budget_utilization_rate": 42.995932,
-    "overage_credits": 797.9227,
-    "equivalent_seats_used": 42.995932,
-    "percent_of_one_seat": 4299.59
-  }
-}

package/observability/evals/chat-token-usage-report.md

@@ -1,33 +0,0 @@
-# Chat Token Usage Report
-
-- timestamp: 2026-07-05T09:54:53.886533+00:00
-- source: observability/logs/iteration-metrics.jsonl
-- filter.source: copilot-session
-- filter.from: 
-- filter.to: 
-- plan: business
-- seats: 1
-- official_plans_url: https://github.com/features/copilot/plans?ref_cta=View+pricing+and+plans&ref_loc=hero&ref_page=%2Ffeatures_copilot_copilot_business&plans=business
-
-## Totals
-
-- events: 36
-- input_tokens: 3893402
-- output_tokens: 126892
-- total_tokens: 4020294
-- copilot_credits: 816.9227
-- included_credits_total_month: 19.0
-- budget_utilization_rate: 42.995932
-- overage_credits: 797.9227
-- equivalent_seats_used: 42.995932
-- percent_of_one_seat: 4299.59
-
-## Breakdown by Model
-
-- copilot/auto: count=36, input=3893402, output=126892, total=4020294, credits=816.9227
-
-## Breakdown by Day
-
-- 2026-07-03: count=19, input=2219137, output=67846, total=2286983, credits=492.070886
-- 2026-07-04: count=9, input=876490, output=21560, total=898050, credits=160.161499
-- 2026-07-05: count=8, input=797775, output=37486, total=835261, credits=164.690315

package/observability/evals/iteration-value-report.json

@@ -1,76 +0,0 @@
-{
-  "timestamp": "2026-07-05T09:54:54.568327+00:00",
-  "sources": {
-    "routing": "observability/logs/routing-decisions.jsonl",
-    "feedback": "observability/logs/learning-feedback.jsonl",
-    "metrics": "observability/logs/iteration-metrics.jsonl"
-  },
-  "totals": {
-    "events": 305,
-    "events_with_event_id": 247,
-    "metrics_covered": 7,
-    "feedback_confirmed": 101,
-    "confirmed_success": 101
-  },
-  "kpis": {
-    "metrics_coverage_rate": 0.0283,
-    "confirmed_feedback_rate": 0.4089,
-    "confirmed_success_rate": 1.0,
-    "total_tokens": 6474,
-    "total_cost_usd": 0.0542,
-    "tokens_per_confirmed_success": 64.1,
-    "cost_per_confirmed_success_usd": 0.000537
-  },
-  "breakdown": {
-    "tool_mode": {
-      "hybrid": {
-        "count": 2,
-        "tokens": 2400,
-        "cost_usd": 0.02
-      },
-      "local-only": {
-        "count": 4,
-        "tokens": 4044,
-        "cost_usd": 0.0342
-      },
-      "model-only": {
-        "count": 1,
-        "tokens": 30,
-        "cost_usd": 0.0
-      }
-    },
-    "model": {
-      "GPT-5.3-Codex": {
-        "count": 7,
-        "tokens": 6474,
-        "cost_usd": 0.054200000000000005
-      }
-    },
-    "engine": {
-      "Azure RAG Builder": {
-        "count": 1,
-        "tokens": 1200,
-        "cost_usd": 0.01
-      },
-      "CodeGraph": {
-        "count": 4,
-        "tokens": 2874,
-        "cost_usd": 0.0242
-      },
-      "GitNexus": {
-        "count": 1,
-        "tokens": 1200,
-        "cost_usd": 0.01
-      },
-      "Graphify": {
-        "count": 1,
-        "tokens": 1200,
-        "cost_usd": 0.01
-      }
-    }
-  },
-  "assessment": {
-    "has_value_signal": true,
-    "notes": "Value signal available. Track trend, not one-point absolute values."
-  }
-}

package/observability/evals/iteration-value-report.md

@@ -1,39 +0,0 @@
-# Iteration Value Report
-
-- timestamp: 2026-07-05T09:54:54.568327+00:00
-- events: 305
-- events_with_event_id: 247
-- metrics_covered: 7
-- feedback_confirmed: 101
-- confirmed_success: 101
-
-## KPI
-
-- metrics_coverage_rate: 2.83%
-- confirmed_feedback_rate: 40.89%
-- confirmed_success_rate: 100.0%
-- total_tokens: 6474
-- total_cost_usd: 0.0542
-- tokens_per_confirmed_success: 64.1
-- cost_per_confirmed_success_usd: 0.000537
-
-## Breakdown
-
-### Tool mode
-- hybrid: count=2, tokens=2400, cost_usd=0.02
-- local-only: count=4, tokens=4044, cost_usd=0.0342
-- model-only: count=1, tokens=30, cost_usd=0.0
-
-### Model
-- GPT-5.3-Codex: count=7, tokens=6474, cost_usd=0.0542
-
-### Engine
-- Azure RAG Builder: count=1, tokens=1200, cost_usd=0.01
-- CodeGraph: count=4, tokens=2874, cost_usd=0.0242
-- GitNexus: count=1, tokens=1200, cost_usd=0.01
-- Graphify: count=1, tokens=1200, cost_usd=0.01
-
-## Assessment
-
-- has_value_signal: True
-- notes: Value signal available. Track trend, not one-point absolute values.

package/observability/evals/learning-loop-report.json

@@ -1,268 +0,0 @@
-{
-  "timestamp": "2026-07-05T09:54:51.179894+00:00",
-  "source": "C:\\repo\\mcp-efficiency-engine\\observability\\logs\\routing-decisions.jsonl",
-  "feedback_source": "C:\\repo\\mcp-efficiency-engine\\observability\\logs\\learning-feedback.jsonl",
-  "total_events": 305,
-  "pending_events": 197,
-  "confirmed_events": 108,
-  "kpis": {
-    "fallback_rate": 0.1475,
-    "grounded_rate": 0.8492,
-    "success_rate": 1.0,
-    "confidence_avg": 0.945
-  },
-  "thresholds": {
-    "fallback_rate_max": 0.2,
-    "grounded_rate_min": 0.8,
-    "confidence_avg_min": 0.8
-  },
-  "health": {
-    "fallback_ok": true,
-    "grounded_ok": true,
-    "confidence_ok": true
-  },
-  "patterns": [
-    {
-      "pattern": "azure-rag+info",
-      "total": 4,
-      "fallback_rate": 0.0,
-      "grounded_rate": 1.0,
-      "pending_count": 4,
-      "confirmed_count": 0,
-      "success_rate": 0.0,
-      "confidence_avg": 0.0,
-      "engines": {
-        "Azure RAG Builder": 4
-      },
-      "agents": {
-        "rag-azure": 4
-      }
-    },
-    {
-      "pattern": "azure-rag+knowledge-grounded",
-      "total": 52,
-      "fallback_rate": 0.0,
-      "grounded_rate": 1.0,
-      "pending_count": 38,
-      "confirmed_count": 14,
-      "success_rate": 1.0,
-      "confidence_avg": 0.946,
-      "engines": {
-        "Azure RAG Builder": 52
-      },
-      "agents": {
-        "rag-azure": 52
-      }
-    },
-    {
-      "pattern": "backend+bug-fix",
-      "total": 12,
-      "fallback_rate": 0.0,
-      "grounded_rate": 1.0,
-      "pending_count": 0,
-      "confirmed_count": 12,
-      "success_rate": 1.0,
-      "confidence_avg": 0.95,
-      "engines": {
-        "CodeGraph": 12
-      },
-      "agents": {
-        "backend": 12
-      }
-    },
-    {
-      "pattern": "dba+analysis",
-      "total": 51,
-      "fallback_rate": 0.0,
-      "grounded_rate": 1.0,
-      "pending_count": 37,
-      "confirmed_count": 14,
-      "success_rate": 1.0,
-      "confidence_avg": 0.946,
-      "engines": {
-        "Graphify": 51
-      },
-      "agents": {
-        "dba": 51
-      }
-    },
-    {
-      "pattern": "dba+sql",
-      "total": 18,
-      "fallback_rate": 0.0,
-      "grounded_rate": 1.0,
-      "pending_count": 11,
-      "confirmed_count": 7,
-      "success_rate": 1.0,
-      "confidence_avg": 0.95,
-      "engines": {
-        "Graphify": 18
-      },
-      "agents": {
-        "dba": 18
-      }
-    },
-    {
-      "pattern": "dev+bug-fix",
-      "total": 50,
-      "fallback_rate": 0.06,
-      "grounded_rate": 1.0,
-      "pending_count": 47,
-      "confirmed_count": 3,
-      "success_rate": 1.0,
-      "confidence_avg": 0.917,
-      "engines": {
-        "CodeGraph": 50
-      },
-      "agents": {
-        "backend": 50
-      }
-    },
-    {
-      "pattern": "dev+info",
-      "total": 2,
-      "fallback_rate": 0.0,
-      "grounded_rate": 0.5,
-      "pending_count": 2,
-      "confirmed_count": 0,
-      "success_rate": 0.0,
-      "confidence_avg": 0.0,
-      "engines": {
-        "CodeGraph": 2
-      },
-      "agents": {
-        "backend": 2
-      }
-    },
-    {
-      "pattern": "frontend+bug-fix",
-      "total": 1,
-      "fallback_rate": 1.0,
-      "grounded_rate": 0.0,
-      "pending_count": 1,
-      "confirmed_count": 0,
-      "success_rate": 0.0,
-      "confidence_avg": 0.0,
-      "engines": {
-        "CodeGraph": 1
-      },
-      "agents": {
-        "frontend-agent": 1
-      }
-    },
-    {
-      "pattern": "frontend+feature",
-      "total": 14,
-      "fallback_rate": 0.0,
-      "grounded_rate": 1.0,
-      "pending_count": 2,
-      "confirmed_count": 12,
-      "success_rate": 1.0,
-      "confidence_avg": 0.95,
-      "engines": {
-        "CodeGraph": 14
-      },
-      "agents": {
-        "frontend-agent": 14
-      }
-    },
-    {
-      "pattern": "incremental-json-projection",
-      "total": 7,
-      "fallback_rate": 0.0,
-      "grounded_rate": 1.0,
-      "pending_count": 0,
-      "confirmed_count": 7,
-      "success_rate": 1.0,
-      "confidence_avg": 0.91,
-      "engines": {
-        "CodeGraph": 7
-      },
-      "agents": {
-        "autodocs-agent": 7
-      }
-    },
-    {
-      "pattern": "iot+architecture",
-      "total": 1,
-      "fallback_rate": 0.0,
-      "grounded_rate": 1.0,
-      "pending_count": 1,
-      "confirmed_count": 0,
-      "success_rate": 0.0,
-      "confidence_avg": 0.0,
-      "engines": {
-        "GitNexus/CodeGraph + Graphify": 1
-      },
-      "agents": {
-        "iot": 1
-      }
-    },
-    {
-      "pattern": "legacy+migration",
-      "total": 53,
-      "fallback_rate": 0.7358,
-      "grounded_rate": 0.3208,
-      "pending_count": 39,
-      "confirmed_count": 14,
-      "success_rate": 1.0,
-      "confidence_avg": 0.946,
-      "engines": {
-        "GitNexus": 53
-      },
-      "agents": {
-        "legacy": 53
-      }
-    },
-    {
-      "pattern": "rag+info",
-      "total": 21,
-      "fallback_rate": 0.0952,
-      "grounded_rate": 0.9048,
-      "pending_count": 8,
-      "confirmed_count": 13,
-      "success_rate": 1.0,
-      "confidence_avg": 0.95,
-      "engines": {
-        "CodeGraph": 1,
-        "Graphify": 20
-      },
-      "agents": {
-        "backend": 1,
-        "rag-local": 20
-      }
-    },
-    {
-      "pattern": "unknown-domain+info",
-      "total": 6,
-      "fallback_rate": 0.0,
-      "grounded_rate": 0.0,
-      "pending_count": 6,
-      "confirmed_count": 0,
-      "success_rate": 0.0,
-      "confidence_avg": 0.0,
-      "engines": {
-        "CodeGraph": 6
-      },
-      "agents": {
-        "backend": 6
-      }
-    },
-    {
-      "pattern": "ux-ui+review",
-      "total": 13,
-      "fallback_rate": 0.0,
-      "grounded_rate": 1.0,
-      "pending_count": 1,
-      "confirmed_count": 12,
-      "success_rate": 1.0,
-      "confidence_avg": 0.95,
-      "engines": {
-        "Graphify": 13
-      },
-      "agents": {
-        "ux-ui": 13
-      }
-    }
-  ]
-}

package/observability/evals/learning-loop-report.md

@@ -1,37 +0,0 @@
-# Learning Loop Report
-
-- timestamp: 2026-07-05T09:54:51.179894+00:00
-- total_events: 305
-- pending_events: 197
-- confirmed_events: 108
-
-## Global KPIs
-
-- fallback_rate: 14.75%
-- grounded_rate: 84.92%
-- success_rate: 100.0%
-- confidence_avg: 0.945
-
-## Health
-
-- fallback_ok: True
-- grounded_ok: True
-- confidence_ok: True
-
-## Patterns
-
-- azure-rag+info: total=4, pending=4, confirmed=0, fallback=0.0%, grounded=100.0%, confidence=0.0
-- azure-rag+knowledge-grounded: total=52, pending=38, confirmed=14, fallback=0.0%, grounded=100.0%, confidence=0.946
-- backend+bug-fix: total=12, pending=0, confirmed=12, fallback=0.0%, grounded=100.0%, confidence=0.95
-- dba+analysis: total=51, pending=37, confirmed=14, fallback=0.0%, grounded=100.0%, confidence=0.946
-- dba+sql: total=18, pending=11, confirmed=7, fallback=0.0%, grounded=100.0%, confidence=0.95
-- dev+bug-fix: total=50, pending=47, confirmed=3, fallback=6.0%, grounded=100.0%, confidence=0.917
-- dev+info: total=2, pending=2, confirmed=0, fallback=0.0%, grounded=50.0%, confidence=0.0
-- frontend+bug-fix: total=1, pending=1, confirmed=0, fallback=100.0%, grounded=0.0%, confidence=0.0
-- frontend+feature: total=14, pending=2, confirmed=12, fallback=0.0%, grounded=100.0%, confidence=0.95
-- incremental-json-projection: total=7, pending=0, confirmed=7, fallback=0.0%, grounded=100.0%, confidence=0.91
-- iot+architecture: total=1, pending=1, confirmed=0, fallback=0.0%, grounded=100.0%, confidence=0.0
-- legacy+migration: total=53, pending=39, confirmed=14, fallback=73.58%, grounded=32.08%, confidence=0.946
-- rag+info: total=21, pending=8, confirmed=13, fallback=9.52%, grounded=90.48%, confidence=0.95
-- unknown-domain+info: total=6, pending=6, confirmed=0, fallback=0.0%, grounded=0.0%, confidence=0.0
-- ux-ui+review: total=13, pending=1, confirmed=12, fallback=0.0%, grounded=100.0%, confidence=0.95

package/observability/evals/repo-intake-eval-cases.md

@@ -1,23 +0,0 @@
-# Repo Intake Eval Cases
-
-Casos minimos para validar que el intake registra capacidades y dominios esperados.
-
-## Matriz base
-
-1. `boostDBA` -> `domain=dba`, motor esperado `Graphify`.
-1. `boost_azure_rag` -> `domain=azure-rag`, motor esperado `Azure RAG Builder`.
-1. `boost_iot` -> `domain=iot`, agente esperado `iot`.
-
-## Criterios de aprobacion
-
-1. Repo aparece en `repo-registry/repos.yml` con `approval.status=approved`.
-1. Se genera capability en `repo-intake/generated/v2/<slug>/<version>/capabilities/capability.json`.
-1. No hay errores en `validate-repo-registry.ps1 -Strict`.
-
-## Comandos
-
-```powershell
-pwsh -NoProfile -ExecutionPolicy Bypass -File .\scripts\intake\validate-repo-registry.ps1 -Strict
-.\scripts\intake\run-repo-intake.ps1
-```
-

package/observability/evals/routing-eval-cases.json

@@ -1,68 +0,0 @@
-{
-  "cases": [
-    {
-      "id": "case-login-bug",
-      "input": "Arregla bug de login",
-      "intent": "bug-fix",
-      "domain": "backend",
-      "source_type": "code",
-      "capability": "backend-coding"
-    },
-    {
-      "id": "case-sla-grounding",
-      "input": "Necesito fuentes SLA reales",
-      "intent": "knowledge-grounded",
-      "domain": "azure-rag",
-      "source_type": "corporate-docs",
-      "capability": "azure-rag-enterprise"
-    },
-    {
-      "id": "case-dba-analysis",
-      "input": "Analiza schema SQL",
-      "intent": "analysis",
-      "domain": "dba",
-      "source_type": "technical-docs",
-      "capability": "database-analysis"
-    },
-    {
-      "id": "case-dba-sql-intent",
-      "input": "Analiza esquema SQL de la tabla de pedidos",
-      "intent": "sql",
-      "domain": "dba",
-      "source_type": "technical-docs",
-      "capability": "database-analysis"
-    },
-    {
-      "id": "case-legacy-migration",
-      "input": "Plan de migracion legacy",
-      "intent": "migration",
-      "domain": "legacy",
-      "source_type": "code",
-      "capability": "legacy-migration"
-    },
-    {
-      "id": "case-rag-tech-docs-local",
-      "input": "explica guia tecnica local",
-      "intent": "info",
-      "domain": "rag",
-      "source_type": "technical-docs",
-      "capability": "rag-knowledge"
-    },
-    {
-      "id": "case-frontend-feature",
-      "input": "Implementar nuevo componente de checkout",
-      "intent": "feature",
-      "domain": "frontend",
-      "source_type": "code",
-      "capability": "frontend-coding"
-    },
-    {
-      "id": "case-ux-ui-review",
-      "input": "Revisar consistencia y accesibilidad del design system",
-      "intent": "review",
-      "domain": "ux-ui",
-      "source_type": "technical-docs",
-      "capability": "ux-ui-governance"
-    }
-  ]
-}

package/observability/evals/routing-eval-cases.md

@@ -1,25 +0,0 @@
-# Routing Eval Cases
-
-## Casos base
-
-1. `bug login` -> `backend` + `CodeGraph` + Caveman `full`.
-1. `SLA` -> `rag-azure` + `Azure RAG Builder` + `sources` no vacio.
-1. `arquitectura` -> `rag-local`/`dba` segun dominio con `Graphify`.
-1. `legacy migration` -> `legacy` + `GitNexus` + `hitl.required=true` cuando haya alto impacto/fallback.
-
-## Suite ejecutable JSON-first
-
-- `observability/evals/routing-eval-cases.json`
-
-## Runner
-
-```powershell
-py -3 .\scripts\intake\run-routing-evals.py
-```
-
-## Criterios de pase
-
-1. `cases_failed = 0`.
-1. Coincidencia exacta de `agent` y `engine` por caso.
-1. `prompt.exists = true` en todos los casos.
-