mcp-efficiency-engine 0.1.0 → 0.1.2

package/orchestrator/wiki/graph_consolidator.py

@@ -11,12 +11,21 @@ SECTIONS = section_catalog()
 
 
 class GraphConsolidator:
-    def __init__(self, output_graph_path: Path, markdown_output_dir: Path) -> None:
+    def __init__(self, output_graph_path: Path, markdown_output_dir: Path, telemetry_collector: Any | None = None) -> None:
         self.output_graph_path = output_graph_path
         self.markdown_output_dir = markdown_output_dir
         self.generated_output_dir = output_graph_path.parent
+        self._telemetry = telemetry_collector
 
     def consolidate(self, provider_contracts: List[Dict[str, Any]]) -> Dict[str, Any]:
+        if self._telemetry is not None:
+            self._telemetry.record_event(
+                "KnowledgeGenerated",
+                {
+                    "operation": "graph_consolidate_start",
+                    "provider_contracts": len(provider_contracts),
+                },
+            )
         raw_nodes: Dict[str, Dict[str, Any]] = {}
         wiki_nodes: Dict[str, Dict[str, Any]] = {}
         for contract in provider_contracts:
@@ -54,6 +63,17 @@ class GraphConsolidator:
         relations_index = self._build_relations_index(wiki_nodes)
         section_manifest = self._build_section_manifest(wiki_nodes)
 
+        if self._telemetry is not None:
+            self._telemetry.record_metric("tool_invocations", float(len(provider_contracts)), unit="count")
+            self._telemetry.record_metric("cache_misses", 0.0, unit="count")
+            self._telemetry.record_event(
+                "KnowledgeGenerated",
+                {
+                    "operation": "graph_consolidate_finish",
+                    "nodes": len(wiki_nodes),
+                },
+            )
+
         return {
             "last_updated": self._deterministic_marker(wiki_nodes),
             "nodes": wiki_nodes,
@@ -71,11 +91,23 @@ class GraphConsolidator:
         return f"sha256:{digest}"
 
     def persist_graph(self, unified_graph: Dict[str, Any]) -> None:
+        if self._telemetry is not None:
+            self._telemetry.record_event("ToolStarted", {"tool": "persist_graph"})
         self.output_graph_path.parent.mkdir(parents=True, exist_ok=True)
         with self.output_graph_path.open("w", encoding="utf-8") as handle:
             json.dump(unified_graph, handle, indent=2, ensure_ascii=True)
+        if self._telemetry is not None:
+            self._telemetry.record_event(
+                "ToolFinished",
+                {
+                    "tool": "persist_graph",
+                    "path": str(self.output_graph_path),
+                },
+            )
 
     def persist_manifests(self, unified_graph: Dict[str, Any]) -> Dict[str, int]:
+        if self._telemetry is not None:
+            self._telemetry.record_event("ToolStarted", {"tool": "persist_manifests"})
         self.generated_output_dir.mkdir(parents=True, exist_ok=True)
         artifacts = {
             "search-index.json": unified_graph.get("search_index", []),
@@ -87,6 +119,14 @@ class GraphConsolidator:
             output_path = self.generated_output_dir / file_name
             with output_path.open("w", encoding="utf-8") as handle:
                 json.dump(payload, handle, indent=2, ensure_ascii=True)
+        if self._telemetry is not None:
+            self._telemetry.record_event(
+                "ToolFinished",
+                {
+                    "tool": "persist_manifests",
+                    "manifests": len(artifacts),
+                },
+            )
         return {"manifests": len(artifacts)}
 
     def project_dirty_nodes(
@@ -96,6 +136,15 @@ class GraphConsolidator:
         all_nodes: Dict[str, Dict[str, Any]],
         cached_nodes: Dict[str, Dict[str, Any]],
     ) -> Dict[str, int]:
+        if self._telemetry is not None:
+            self._telemetry.record_event(
+                "ToolStarted",
+                {
+                    "tool": "project_dirty_nodes",
+                    "dirty_nodes": len(dirty_nodes),
+                    "deleted_nodes": len(deleted_nodes),
+                },
+            )
         self.markdown_output_dir.mkdir(parents=True, exist_ok=True)
 
         rendered = 0
@@ -124,6 +173,16 @@ class GraphConsolidator:
                 deleted += 1
 
         indexes = self._render_index_pages(all_nodes)
+        if self._telemetry is not None:
+            self._telemetry.record_event(
+                "ToolFinished",
+                {
+                    "tool": "project_dirty_nodes",
+                    "rendered": rendered,
+                    "deleted": deleted,
+                    "indexes": indexes,
+                },
+            )
         return {
             "rendered": rendered,
             "deleted": deleted,

package/orchestrator/wiki/incremental_engine.py

@@ -4,8 +4,9 @@ from typing import Any, Dict
 
 
 class IncrementalEngine:
-    def __init__(self, cache_path: Path) -> None:
+    def __init__(self, cache_path: Path, telemetry_collector: Any | None = None) -> None:
         self.cache_path = cache_path
+        self._telemetry = telemetry_collector
 
     def _extract_nodes(self, loaded: Dict[str, Any]) -> Dict[str, Dict[str, Any]]:
         preferred = loaded.get("wiki_nodes")
@@ -37,6 +38,14 @@ class IncrementalEngine:
             return {"last_updated": "", "nodes": {}}
 
     def diff(self, incoming_nodes: Dict[str, Dict[str, Any]]) -> Dict[str, Any]:
+        if self._telemetry is not None:
+            self._telemetry.record_event(
+                "ToolStarted",
+                {
+                    "tool": "incremental_diff",
+                    "incoming_nodes": len(incoming_nodes),
+                },
+            )
         cached = self._load_cached_graph()
         cached_nodes = cached.get("nodes", {})
 
@@ -50,6 +59,19 @@ class IncrementalEngine:
                 dirty_nodes[node_key] = incoming
 
         deleted_nodes = [key for key in cached_nodes.keys() if key not in incoming_nodes]
+        if self._telemetry is not None:
+            cache_hits = max(0, len(incoming_nodes) - len(dirty_nodes))
+            cache_misses = len(dirty_nodes)
+            self._telemetry.record_event(
+                "ToolFinished",
+                {
+                    "tool": "incremental_diff",
+                    "dirty_nodes": len(dirty_nodes),
+                    "deleted_nodes": len(deleted_nodes),
+                },
+            )
+            self._telemetry.record_metric("cache_hits", float(cache_hits), unit="count")
+            self._telemetry.record_metric("cache_misses", float(cache_misses), unit="count")
         return {
             "dirty_nodes": dirty_nodes,
             "deleted_nodes": deleted_nodes,

package/orchestrator/wiki/plugin_manager.py

@@ -1,40 +1,105 @@
 from datetime import datetime, timezone
 from typing import Any, Dict, List
 
+from telemetry import build_telemetry_collector
+
 
 class PluginManager:
-    def __init__(self) -> None:
+    def __init__(self, telemetry_collector: Any | None = None) -> None:
         self._providers: List[Any] = []
+        self._telemetry = telemetry_collector
+
+    def _collector(self) -> Any:
+        if self._telemetry is None:
+            from pathlib import Path
+
+            repo_root = Path(__file__).resolve().parents[2]
+            self._telemetry = build_telemetry_collector(repo_root)
+        return self._telemetry
 
     def register_provider(self, provider: Any) -> None:
+        collector = self._collector()
+        with collector.start_span(name="plugin.register_provider", kind="INTERNAL"):
+            collector.record_event(
+                "ProviderRequest",
+                {
+                    "provider_id": str(getattr(provider, "provider_id", "unknown")),
+                    "operation": "register_provider",
+                },
+            )
         if not hasattr(provider, "provider_id"):
             raise ValueError("Provider must define provider_id")
         if not hasattr(provider, "gather_knowledge"):
             raise ValueError("Provider must implement gather_knowledge()")
         self._providers.append(provider)
+        collector.record_event(
+            "ProviderResponse",
+            {
+                "provider_id": str(getattr(provider, "provider_id", "unknown")),
+                "status": "registered",
+            },
+        )
 
     def gather_all(self) -> Dict[str, List[Dict[str, Any]]]:
+        collector = self._collector()
         contracts: List[Dict[str, Any]] = []
         errors: List[Dict[str, Any]] = []
-        for provider in self._providers:
-            try:
-                contract = provider.gather_knowledge()
-                if isinstance(contract, dict):
-                    contracts.append(contract)
-                else:
-                    errors.append(
-                        {
-                            "timestamp": datetime.now(timezone.utc).isoformat(),
-                            "provider_id": str(getattr(provider, "provider_id", "unknown")),
-                            "error": "Provider returned non-dict contract",
-                        }
-                    )
-            except Exception as exc:
-                errors.append(
-                    {
-                        "timestamp": datetime.now(timezone.utc).isoformat(),
-                        "provider_id": str(getattr(provider, "provider_id", "unknown")),
-                        "error": str(exc),
-                    }
-                )
+        with collector.start_execution(operation="plugin-manager-gather-all", session_id="wiki-orchestrator"):
+            for provider in self._providers:
+                provider_id = str(getattr(provider, "provider_id", "unknown"))
+                with collector.start_span(name="plugin.gather_knowledge", kind="INTERNAL", attributes={"provider_id": provider_id}):
+                    try:
+                        collector.record_event(
+                            "ProviderRequest",
+                            {
+                                "provider_id": provider_id,
+                                "operation": "gather_knowledge",
+                            },
+                        )
+                        contract = provider.gather_knowledge()
+                        if isinstance(contract, dict):
+                            contracts.append(contract)
+                            collector.record_event(
+                                "ProviderResponse",
+                                {
+                                    "provider_id": provider_id,
+                                    "status": "ok",
+                                    "contract_kind": str(contract.get("kind", "unknown")),
+                                },
+                            )
+                        else:
+                            errors.append(
+                                {
+                                    "timestamp": datetime.now(timezone.utc).isoformat(),
+                                    "provider_id": provider_id,
+                                    "error": "Provider returned non-dict contract",
+                                }
+                            )
+                            collector.record_event(
+                                "WarningGenerated",
+                                {
+                                    "provider_id": provider_id,
+                                    "warning": "Provider returned non-dict contract",
+                                },
+                                level="WARNING",
+                            )
+                    except Exception as exc:
+                        errors.append(
+                            {
+                                "timestamp": datetime.now(timezone.utc).isoformat(),
+                                "provider_id": provider_id,
+                                "error": str(exc),
+                            }
+                        )
+                        collector.record_event(
+                            "ExceptionThrown",
+                            {
+                                "provider_id": provider_id,
+                                "error": str(exc),
+                            },
+                            level="ERROR",
+                        )
+
+            collector.record_metric("provider_calls", float(len(self._providers)), unit="count")
+            collector.record_metric("error_count", float(len(errors)), unit="count")
         return {"contracts": contracts, "errors": errors}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-efficiency-engine",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Motor de orquestacion para agentes MCP con routing por dominio y bootstrap portable.",
   "license": "MIT",
   "type": "commonjs",
@@ -9,7 +9,9 @@
     "mcp-efficiency-engine": "./bin/mcpee.js"
   },
   "files": [
+    ".githooks/",
     "bin/",
+    "telemetry/",
     "autodocs/README.md",
     "autodocs/schema/",
     "autolearning/",
@@ -20,7 +22,12 @@
     "repo-registry/repos.schema.json",
     "repo-registry/repos.template.json",
     "orchestrator/",
-    "observability/",
+    "observability/evaluation.md",
+    "observability/logs.schema.json",
+    "observability/logs.schema.md",
+    "observability/metrics.md",
+    "observability/routing-evaluator.md",
+    "observability/scoring-rubric.md",
     "optimization/",
     "policies/",
     "specs/",
@@ -44,6 +51,7 @@
     "hi": "node ./bin/mcpee.js hi",
     "bye": "node ./bin/mcpee.js bye",
     "intake": "node ./bin/mcpee.js intake",
+    "langsmith:kpis": "py -3 ./scripts/ops/publish-langsmith-kpis.py",
     "pack:check": "npm pack --dry-run"
   },
   "engines": {

package/requirements.txt

@@ -1 +1,3 @@
 graphifyy[mcp]
+langsmith
+mcp

package/scripts/intake/resolve-routing.py

@@ -2,11 +2,18 @@ from __future__ import annotations
 
 import argparse
 import json
+import sys
 import uuid
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any
 
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(_REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(_REPO_ROOT))
+
+from telemetry import build_telemetry_collector
+
 try:
     import yaml  # type: ignore
 except Exception:
@@ -477,6 +484,7 @@ def main() -> int:
         return 1
 
     repo_root = Path(__file__).resolve().parents[2]
+    collector = build_telemetry_collector(repo_root)
     registry = load_registry((repo_root / args.registry).resolve())
     capabilities = load_capability_index((repo_root / args.generated_root).resolve())
     manifests = load_manifest_index((repo_root / args.generated_root).resolve())
@@ -525,93 +533,122 @@ def main() -> int:
         fallback=fallback,
         notes=notes,
     )
-    event = {
-        "event_id": event_id,
-        "timestamp": utc_now(),
-        "input": args.input,
-        "intent": args.intent,
-        "domain": normalized_domain,
-        "source_type": args.source_type,
-        "agent": route["agent"],
-        "engine": route["engine"],
-        "optimization_profile": f"{token_saver_profile}+{caveman_profile}",
-        "fallback": fallback,
-        "grounded": grounded,
-        "sources": sources,
-        "notes": ";".join(notes) if notes else "",
-        "prompt": {
-            "selected": selected_prompt,
-            "exists": prompt_exists,
-            "selection_mode": "auto",
-        },
-        "skill": {
-            "selected": selected_skill,
-            "exists": skill_exists,
-            "selection_mode": "auto",
-        },
-        "optimization": {
-            "token_saver": "always_on",
-            "token_saver_profile": token_saver_profile,
-            "caveman": "always_on",
-            "caveman_profile": caveman_profile,
-            "sources_preserved": True,
-            "context_reduction_strategy": context_strategy,
-        },
-        "memory": {
-            "selected": selected_memories,
-            "reason": memory_reason,
-        },
-        "requirements": runtime_requirements_for_route(args.source_type, route["engine"]),
-        "learning": {
-            "used_pattern": f"{normalized_domain}+{args.intent}",
-            # Real success must be written post-execution by the feedback updater.
-            "success": None,
-            "outcome_status": "pending",
-            "fallback": fallback,
-            "confidence": 0.9 if not fallback else 0.7,
-        },
-        "hitl": hitl,
-    }
-
-    if has_input_tokens and has_output_tokens:
-        total_tokens = args.input_tokens + args.output_tokens
-        event["usage"] = {
-            "input_tokens": args.input_tokens,
-            "output_tokens": args.output_tokens,
-            "total_tokens": total_tokens,
-            "estimated_cost_usd": round(float(args.estimated_cost_usd), 6),
-        }
-
-    metrics_recorded = False
-    metrics_output_path = ""
-    if has_input_tokens and has_output_tokens:
-        metric = build_iteration_metric(
-            event_id=event_id,
-            model=str(args.model),
-            input_tokens=args.input_tokens,
-            output_tokens=args.output_tokens,
-            estimated_cost_usd=float(args.estimated_cost_usd),
-            local_tools=args.local_tools,
-            remote_tools=args.remote_tools,
-            notes="auto-recorded by resolve-routing",
-        )
-        metrics_path = (repo_root / args.metrics_output).resolve()
-        append_jsonl(metrics_path, metric)
-        metrics_recorded = True
-        metrics_output_path = args.metrics_output.replace("\\", "/")
-
-    event["telemetry"] = {
-        "iteration_metrics_recorded": metrics_recorded,
-        "metrics_output": metrics_output_path,
-    }
-
-    output_path = (repo_root / args.output).resolve()
-    append_jsonl(output_path, event)
-
-    print(json.dumps(event, indent=2, ensure_ascii=False))
-    print(f"Event appended to: {output_path}")
-    if metrics_recorded:
-        print(f"Metrics appended to: {metrics_output_path}")
+    with collector.start_execution(
+        operation="resolve-routing",
+        request_id=event_id,
+        session_id="intake-routing",
+        correlation_id=event_id,
+        execution_id=event_id,
+        model=str(args.model),
+    ):
+        with collector.start_span(name="routing-resolution", kind="INTERNAL", attributes={"intent": args.intent, "domain": normalized_domain}):
+            event = {
+                "event_id": event_id,
+                "timestamp": utc_now(),
+                "input": args.input,
+                "intent": args.intent,
+                "domain": normalized_domain,
+                "source_type": args.source_type,
+                "agent": route["agent"],
+                "engine": route["engine"],
+                "optimization_profile": f"{token_saver_profile}+{caveman_profile}",
+                "fallback": fallback,
+                "grounded": grounded,
+                "sources": sources,
+                "notes": ";".join(notes) if notes else "",
+                "prompt": {
+                    "selected": selected_prompt,
+                    "exists": prompt_exists,
+                    "selection_mode": "auto",
+                },
+                "skill": {
+                    "selected": selected_skill,
+                    "exists": skill_exists,
+                    "selection_mode": "auto",
+                },
+                "optimization": {
+                    "token_saver": "always_on",
+                    "token_saver_profile": token_saver_profile,
+                    "caveman": "always_on",
+                    "caveman_profile": caveman_profile,
+                    "sources_preserved": True,
+                    "context_reduction_strategy": context_strategy,
+                },
+                "memory": {
+                    "selected": selected_memories,
+                    "reason": memory_reason,
+                },
+                "requirements": runtime_requirements_for_route(args.source_type, route["engine"]),
+                "learning": {
+                    "used_pattern": f"{normalized_domain}+{args.intent}",
+                    # Real success must be written post-execution by the feedback updater.
+                    "success": None,
+                    "outcome_status": "pending",
+                    "fallback": fallback,
+                    "confidence": 0.9 if not fallback else 0.7,
+                },
+                "hitl": hitl,
+            }
+
+            if has_input_tokens and has_output_tokens:
+                total_tokens = args.input_tokens + args.output_tokens
+                event["usage"] = {
+                    "input_tokens": args.input_tokens,
+                    "output_tokens": args.output_tokens,
+                    "total_tokens": total_tokens,
+                    "estimated_cost_usd": round(float(args.estimated_cost_usd), 6),
+                }
+                collector.record_usage(
+                    input_tokens=args.input_tokens,
+                    output_tokens=args.output_tokens,
+                    estimated_cost_usd=float(args.estimated_cost_usd),
+                )
+
+            metrics_recorded = False
+            metrics_output_path = ""
+            if has_input_tokens and has_output_tokens:
+                metric = build_iteration_metric(
+                    event_id=event_id,
+                    model=str(args.model),
+                    input_tokens=args.input_tokens,
+                    output_tokens=args.output_tokens,
+                    estimated_cost_usd=float(args.estimated_cost_usd),
+                    local_tools=args.local_tools,
+                    remote_tools=args.remote_tools,
+                    notes="auto-recorded by resolve-routing",
+                )
+                metrics_path = (repo_root / args.metrics_output).resolve()
+                append_jsonl(metrics_path, metric)
+                metrics_recorded = True
+                metrics_output_path = args.metrics_output.replace("\\", "/")
+
+            event["telemetry"] = {
+                "iteration_metrics_recorded": metrics_recorded,
+                "metrics_output": metrics_output_path,
+            }
+
+            collector.record_event(
+                "RoutingResolved",
+                {
+                    "event_id": event_id,
+                    "intent": args.intent,
+                    "domain": normalized_domain,
+                    "agent": route["agent"],
+                    "engine": route["engine"],
+                    "fallback": fallback,
+                    "grounded": grounded,
+                },
+            )
+
+            output_path = (repo_root / args.output).resolve()
+            append_jsonl(output_path, event)
+
+            print(json.dumps(event, indent=2, ensure_ascii=False))
+            print(f"Event appended to: {output_path}")
+            if metrics_recorded:
+                print(f"Metrics appended to: {metrics_output_path}")
+
+    collector.shutdown()
     return 0
 
 

package/scripts/intake/validate-database-routing.py

@@ -31,13 +31,22 @@ def run_routing(**kwargs: str) -> dict:
         encoding="utf-8",
         errors="replace"
     )
-    # Output contains a JSON block followed by a trailing log line; extract JSON.
+    # Output can include telemetry lines after the main event JSON.
+    # Decode only the first valid JSON object from stdout.
     stdout = proc.stdout or ""
     brace = stdout.find("{")
-    last_brace = stdout.rfind("}")
-    if brace == -1 or last_brace == -1:
+    if brace == -1:
         raise ValueError(f"No JSON in output: {stdout[:200]}")
-    return json.loads(stdout[brace : last_brace + 1])
+
+    decoder = json.JSONDecoder()
+    try:
+        payload, _ = decoder.raw_decode(stdout[brace:])
+    except Exception as exc:
+        raise ValueError(f"Invalid JSON payload in output: {stdout[:200]}") from exc
+
+    if not isinstance(payload, dict):
+        raise ValueError(f"Expected JSON object in output, got: {type(payload).__name__}")
+    return payload
 
 
 def main() -> int:

package/scripts/intake/validate-rag-routing.py

@@ -33,10 +33,11 @@ def run_routing(**kwargs: str) -> dict:
     )
     stdout = proc.stdout or ""
     brace = stdout.find("{")
-    last_brace = stdout.rfind("}")
-    if brace == -1 or last_brace == -1:
+    if brace == -1:
         raise ValueError(f"No JSON in output: {stdout[:200]}")
-    return json.loads(stdout[brace : last_brace + 1])
+    decoder = json.JSONDecoder()
+    payload, _ = decoder.raw_decode(stdout[brace:])
+    return payload
 
 
 def main() -> int: