mcp-efficiency-engine 0.1.0 → 0.1.1

package/orchestrator/wiki/plugin_manager.py

@@ -1,40 +1,105 @@
 from datetime import datetime, timezone
 from typing import Any, Dict, List
 
+from telemetry import build_telemetry_collector
+
 
 class PluginManager:
-    def __init__(self) -> None:
+    def __init__(self, telemetry_collector: Any | None = None) -> None:
         self._providers: List[Any] = []
+        self._telemetry = telemetry_collector
+
+    def _collector(self) -> Any:
+        if self._telemetry is None:
+            from pathlib import Path
+
+            repo_root = Path(__file__).resolve().parents[2]
+            self._telemetry = build_telemetry_collector(repo_root)
+        return self._telemetry
 
     def register_provider(self, provider: Any) -> None:
+        collector = self._collector()
+        with collector.start_span(name="plugin.register_provider", kind="INTERNAL"):
+            collector.record_event(
+                "ProviderRequest",
+                {
+                    "provider_id": str(getattr(provider, "provider_id", "unknown")),
+                    "operation": "register_provider",
+                },
+            )
         if not hasattr(provider, "provider_id"):
             raise ValueError("Provider must define provider_id")
         if not hasattr(provider, "gather_knowledge"):
             raise ValueError("Provider must implement gather_knowledge()")
         self._providers.append(provider)
+        collector.record_event(
+            "ProviderResponse",
+            {
+                "provider_id": str(getattr(provider, "provider_id", "unknown")),
+                "status": "registered",
+            },
+        )
 
     def gather_all(self) -> Dict[str, List[Dict[str, Any]]]:
+        collector = self._collector()
         contracts: List[Dict[str, Any]] = []
         errors: List[Dict[str, Any]] = []
-        for provider in self._providers:
-            try:
-                contract = provider.gather_knowledge()
-                if isinstance(contract, dict):
-                    contracts.append(contract)
-                else:
-                    errors.append(
-                        {
-                            "timestamp": datetime.now(timezone.utc).isoformat(),
-                            "provider_id": str(getattr(provider, "provider_id", "unknown")),
-                            "error": "Provider returned non-dict contract",
-                        }
-                    )
-            except Exception as exc:
-                errors.append(
-                    {
-                        "timestamp": datetime.now(timezone.utc).isoformat(),
-                        "provider_id": str(getattr(provider, "provider_id", "unknown")),
-                        "error": str(exc),
-                    }
-                )
+        with collector.start_execution(operation="plugin-manager-gather-all", session_id="wiki-orchestrator"):
+            for provider in self._providers:
+                provider_id = str(getattr(provider, "provider_id", "unknown"))
+                with collector.start_span(name="plugin.gather_knowledge", kind="INTERNAL", attributes={"provider_id": provider_id}):
+                    try:
+                        collector.record_event(
+                            "ProviderRequest",
+                            {
+                                "provider_id": provider_id,
+                                "operation": "gather_knowledge",
+                            },
+                        )
+                        contract = provider.gather_knowledge()
+                        if isinstance(contract, dict):
+                            contracts.append(contract)
+                            collector.record_event(
+                                "ProviderResponse",
+                                {
+                                    "provider_id": provider_id,
+                                    "status": "ok",
+                                    "contract_kind": str(contract.get("kind", "unknown")),
+                                },
+                            )
+                        else:
+                            errors.append(
+                                {
+                                    "timestamp": datetime.now(timezone.utc).isoformat(),
+                                    "provider_id": provider_id,
+                                    "error": "Provider returned non-dict contract",
+                                }
+                            )
+                            collector.record_event(
+                                "WarningGenerated",
+                                {
+                                    "provider_id": provider_id,
+                                    "warning": "Provider returned non-dict contract",
+                                },
+                                level="WARNING",
+                            )
+                    except Exception as exc:
+                        errors.append(
+                            {
+                                "timestamp": datetime.now(timezone.utc).isoformat(),
+                                "provider_id": provider_id,
+                                "error": str(exc),
+                            }
+                        )
+                        collector.record_event(
+                            "ExceptionThrown",
+                            {
+                                "provider_id": provider_id,
+                                "error": str(exc),
+                            },
+                            level="ERROR",
+                        )
+
+            collector.record_metric("provider_calls", float(len(self._providers)), unit="count")
+            collector.record_metric("error_count", float(len(errors)), unit="count")
         return {"contracts": contracts, "errors": errors}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-efficiency-engine",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Motor de orquestacion para agentes MCP con routing por dominio y bootstrap portable.",
   "license": "MIT",
   "type": "commonjs",
@@ -10,6 +10,7 @@
   },
   "files": [
     "bin/",
+    "telemetry/",
     "autodocs/README.md",
     "autodocs/schema/",
     "autolearning/",
@@ -20,7 +21,12 @@
     "repo-registry/repos.schema.json",
     "repo-registry/repos.template.json",
     "orchestrator/",
-    "observability/",
+    "observability/evaluation.md",
+    "observability/logs.schema.json",
+    "observability/logs.schema.md",
+    "observability/metrics.md",
+    "observability/routing-evaluator.md",
+    "observability/scoring-rubric.md",
     "optimization/",
     "policies/",
     "specs/",

package/requirements.txt

@@ -1 +1,2 @@
 graphifyy[mcp]
+langsmith

package/scripts/intake/resolve-routing.py

@@ -2,11 +2,18 @@ from __future__ import annotations
 
 import argparse
 import json
+import sys
 import uuid
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any
 
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(_REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(_REPO_ROOT))
+
+from telemetry import build_telemetry_collector
+
 try:
     import yaml  # type: ignore
 except Exception:
@@ -477,6 +484,7 @@ def main() -> int:
         return 1
 
     repo_root = Path(__file__).resolve().parents[2]
+    collector = build_telemetry_collector(repo_root)
     registry = load_registry((repo_root / args.registry).resolve())
     capabilities = load_capability_index((repo_root / args.generated_root).resolve())
     manifests = load_manifest_index((repo_root / args.generated_root).resolve())
@@ -525,93 +533,122 @@ def main() -> int:
         fallback=fallback,
         notes=notes,
     )
-    event = {
-        "event_id": event_id,
-        "timestamp": utc_now(),
-        "input": args.input,
-        "intent": args.intent,
-        "domain": normalized_domain,
-        "source_type": args.source_type,
-        "agent": route["agent"],
-        "engine": route["engine"],
-        "optimization_profile": f"{token_saver_profile}+{caveman_profile}",
-        "fallback": fallback,
-        "grounded": grounded,
-        "sources": sources,
-        "notes": ";".join(notes) if notes else "",
-        "prompt": {
-            "selected": selected_prompt,
-            "exists": prompt_exists,
-            "selection_mode": "auto",
-        },
-        "skill": {
-            "selected": selected_skill,
-            "exists": skill_exists,
-            "selection_mode": "auto",
-        },
-        "optimization": {
-            "token_saver": "always_on",
-            "token_saver_profile": token_saver_profile,
-            "caveman": "always_on",
-            "caveman_profile": caveman_profile,
-            "sources_preserved": True,
-            "context_reduction_strategy": context_strategy,
-        },
-        "memory": {
-            "selected": selected_memories,
-            "reason": memory_reason,
-        },
-        "requirements": runtime_requirements_for_route(args.source_type, route["engine"]),
-        "learning": {
-            "used_pattern": f"{normalized_domain}+{args.intent}",
-            # Real success must be written post-execution by the feedback updater.
-            "success": None,
-            "outcome_status": "pending",
-            "fallback": fallback,
-            "confidence": 0.9 if not fallback else 0.7,
-        },
-        "hitl": hitl,
-    }
-
-    if has_input_tokens and has_output_tokens:
-        total_tokens = args.input_tokens + args.output_tokens
-        event["usage"] = {
-            "input_tokens": args.input_tokens,
-            "output_tokens": args.output_tokens,
-            "total_tokens": total_tokens,
-            "estimated_cost_usd": round(float(args.estimated_cost_usd), 6),
-        }
-
-    metrics_recorded = False
-    metrics_output_path = ""
-    if has_input_tokens and has_output_tokens:
-        metric = build_iteration_metric(
-            event_id=event_id,
-            model=str(args.model),
-            input_tokens=args.input_tokens,
-            output_tokens=args.output_tokens,
-            estimated_cost_usd=float(args.estimated_cost_usd),
-            local_tools=args.local_tools,
-            remote_tools=args.remote_tools,
-            notes="auto-recorded by resolve-routing",
-        )
-        metrics_path = (repo_root / args.metrics_output).resolve()
-        append_jsonl(metrics_path, metric)
-        metrics_recorded = True
-        metrics_output_path = args.metrics_output.replace("\\", "/")
-
-    event["telemetry"] = {
-        "iteration_metrics_recorded": metrics_recorded,
-        "metrics_output": metrics_output_path,
-    }
-
-    output_path = (repo_root / args.output).resolve()
-    append_jsonl(output_path, event)
-
-    print(json.dumps(event, indent=2, ensure_ascii=False))
-    print(f"Event appended to: {output_path}")
-    if metrics_recorded:
-        print(f"Metrics appended to: {metrics_output_path}")
+    with collector.start_execution(
+        operation="resolve-routing",
+        request_id=event_id,
+        session_id="intake-routing",
+        correlation_id=event_id,
+        execution_id=event_id,
+        model=str(args.model),
+    ):
+        with collector.start_span(name="routing-resolution", kind="INTERNAL", attributes={"intent": args.intent, "domain": normalized_domain}):
+            event = {
+                "event_id": event_id,
+                "timestamp": utc_now(),
+                "input": args.input,
+                "intent": args.intent,
+                "domain": normalized_domain,
+                "source_type": args.source_type,
+                "agent": route["agent"],
+                "engine": route["engine"],
+                "optimization_profile": f"{token_saver_profile}+{caveman_profile}",
+                "fallback": fallback,
+                "grounded": grounded,
+                "sources": sources,
+                "notes": ";".join(notes) if notes else "",
+                "prompt": {
+                    "selected": selected_prompt,
+                    "exists": prompt_exists,
+                    "selection_mode": "auto",
+                },
+                "skill": {
+                    "selected": selected_skill,
+                    "exists": skill_exists,
+                    "selection_mode": "auto",
+                },
+                "optimization": {
+                    "token_saver": "always_on",
+                    "token_saver_profile": token_saver_profile,
+                    "caveman": "always_on",
+                    "caveman_profile": caveman_profile,
+                    "sources_preserved": True,
+                    "context_reduction_strategy": context_strategy,
+                },
+                "memory": {
+                    "selected": selected_memories,
+                    "reason": memory_reason,
+                },
+                "requirements": runtime_requirements_for_route(args.source_type, route["engine"]),
+                "learning": {
+                    "used_pattern": f"{normalized_domain}+{args.intent}",
+                    # Real success must be written post-execution by the feedback updater.
+                    "success": None,
+                    "outcome_status": "pending",
+                    "fallback": fallback,
+                    "confidence": 0.9 if not fallback else 0.7,
+                },
+                "hitl": hitl,
+            }
+
+            if has_input_tokens and has_output_tokens:
+                total_tokens = args.input_tokens + args.output_tokens
+                event["usage"] = {
+                    "input_tokens": args.input_tokens,
+                    "output_tokens": args.output_tokens,
+                    "total_tokens": total_tokens,
+                    "estimated_cost_usd": round(float(args.estimated_cost_usd), 6),
+                }
+                collector.record_usage(
+                    input_tokens=args.input_tokens,
+                    output_tokens=args.output_tokens,
+                    estimated_cost_usd=float(args.estimated_cost_usd),
+                )
+
+            metrics_recorded = False
+            metrics_output_path = ""
+            if has_input_tokens and has_output_tokens:
+                metric = build_iteration_metric(
+                    event_id=event_id,
+                    model=str(args.model),
+                    input_tokens=args.input_tokens,
+                    output_tokens=args.output_tokens,
+                    estimated_cost_usd=float(args.estimated_cost_usd),
+                    local_tools=args.local_tools,
+                    remote_tools=args.remote_tools,
+                    notes="auto-recorded by resolve-routing",
+                )
+                metrics_path = (repo_root / args.metrics_output).resolve()
+                append_jsonl(metrics_path, metric)
+                metrics_recorded = True
+                metrics_output_path = args.metrics_output.replace("\\", "/")
+
+            event["telemetry"] = {
+                "iteration_metrics_recorded": metrics_recorded,
+                "metrics_output": metrics_output_path,
+            }
+
+            collector.record_event(
+                "RoutingResolved",
+                {
+                    "event_id": event_id,
+                    "intent": args.intent,
+                    "domain": normalized_domain,
+                    "agent": route["agent"],
+                    "engine": route["engine"],
+                    "fallback": fallback,
+                    "grounded": grounded,
+                },
+            )
+
+            output_path = (repo_root / args.output).resolve()
+            append_jsonl(output_path, event)
+
+            print(json.dumps(event, indent=2, ensure_ascii=False))
+            print(f"Event appended to: {output_path}")
+            if metrics_recorded:
+                print(f"Metrics appended to: {metrics_output_path}")
+
+    collector.shutdown()
     return 0
 
 

package/scripts/intake/validate-database-routing.py

@@ -31,13 +31,22 @@ def run_routing(**kwargs: str) -> dict:
         encoding="utf-8",
         errors="replace"
     )
-    # Output contains a JSON block followed by a trailing log line; extract JSON.
+    # Output can include telemetry lines after the main event JSON.
+    # Decode only the first valid JSON object from stdout.
     stdout = proc.stdout or ""
     brace = stdout.find("{")
-    last_brace = stdout.rfind("}")
-    if brace == -1 or last_brace == -1:
+    if brace == -1:
         raise ValueError(f"No JSON in output: {stdout[:200]}")
-    return json.loads(stdout[brace : last_brace + 1])
+
+    decoder = json.JSONDecoder()
+    try:
+        payload, _ = decoder.raw_decode(stdout[brace:])
+    except Exception as exc:
+        raise ValueError(f"Invalid JSON payload in output: {stdout[:200]}") from exc
+
+    if not isinstance(payload, dict):
+        raise ValueError(f"Expected JSON object in output, got: {type(payload).__name__}")
+    return payload
 
 
 def main() -> int:

package/scripts/intake/validate-rag-routing.py

@@ -33,10 +33,11 @@ def run_routing(**kwargs: str) -> dict:
     )
     stdout = proc.stdout or ""
     brace = stdout.find("{")
-    last_brace = stdout.rfind("}")
-    if brace == -1 or last_brace == -1:
+    if brace == -1:
         raise ValueError(f"No JSON in output: {stdout[:200]}")
-    return json.loads(stdout[brace : last_brace + 1])
+    decoder = json.JSONDecoder()
+    payload, _ = decoder.raw_decode(stdout[brace:])
+    return payload
 
 
 def main() -> int:

package/scripts/learning/learning-loop-report.py

@@ -2,11 +2,18 @@ from __future__ import annotations
 
 import argparse
 import json
+import sys
 from collections import defaultdict
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any
 
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(_REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(_REPO_ROOT))
+
+from telemetry import build_telemetry_collector
+
 
 def utc_now() -> str:
     return datetime.now(timezone.utc).isoformat()
@@ -247,24 +254,35 @@ def main() -> int:
     args = parser.parse_args()
 
     repo_root = Path(__file__).resolve().parents[2]
+    collector = build_telemetry_collector(repo_root)
     in_path = (repo_root / args.input).resolve()
     feedback_path = (repo_root / args.feedback).resolve()
     out_json = (repo_root / args.out_json).resolve()
     out_md = (repo_root / args.out_md).resolve()
 
-    events = parse_events(in_path)
-    feedback = parse_feedback(feedback_path)
-    report = build_report(events, str(in_path), feedback, str(feedback_path))
+    with collector.start_execution(operation="learning-loop-report", session_id="learning"):
+        with collector.start_span(name="learning.report.build", kind="INTERNAL"):
+            events = parse_events(in_path)
+            feedback = parse_feedback(feedback_path)
+            report = build_report(events, str(in_path), feedback, str(feedback_path))
+
+            out_json.parent.mkdir(parents=True, exist_ok=True)
+            out_md.parent.mkdir(parents=True, exist_ok=True)
+
+            out_json.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+            out_md.write_text(report_to_markdown(report), encoding="utf-8")
 
-    out_json.parent.mkdir(parents=True, exist_ok=True)
-    out_md.parent.mkdir(parents=True, exist_ok=True)
+            kpis = report.get("kpis", {}) if isinstance(report.get("kpis", {}), dict) else {}
+            collector.record_metric("tool_invocations", float(report.get("total_events", 0)), unit="count")
+            collector.record_metric("fallback_rate", float(kpis.get("fallback_rate", 0.0)), unit="ratio")
+            collector.record_metric("grounded_rate", float(kpis.get("grounded_rate", 0.0)), unit="ratio")
+            collector.record_metric("confidence_avg", float(kpis.get("confidence_avg", 0.0)), unit="score")
 
-    out_json.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
-    out_md.write_text(report_to_markdown(report), encoding="utf-8")
+            print(f"Learning events processed: {len(events)}")
+            print(f"Report JSON: {out_json}")
+            print(f"Report MD: {out_md}")
 
-    print(f"Learning events processed: {len(events)}")
-    print(f"Report JSON: {out_json}")
-    print(f"Report MD: {out_md}")
+    collector.shutdown()
     return 0