mcp-efficiency-engine 0.1.0 → 0.1.2

package/scripts/ops/publish-langsmith-kpis.py

@@ -0,0 +1,302 @@
+from __future__ import annotations
+
+import json
+import os
+import sys
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from telemetry.config import load_config
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    if not path.exists():
+        return {}
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return {}
+    return payload if isinstance(payload, dict) else {}
+
+
+def _iso_now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _slug(value: str) -> str:
+    return "".join(ch.lower() if ch.isalnum() else "-" for ch in value).strip("-")
+
+
+def _resolve_host_scope(repo_root: Path) -> tuple[str, str]:
+    host_project = (os.getenv("MCPEE_HOST_PROJECT") or repo_root.name or "unknown-project").strip()
+    host_slug = _slug(host_project) or "unknown-project"
+    platform_slugs = {"mcp-efficiency-engine", "efficiency", "mcpee"}
+    scope = "platform" if host_slug in platform_slugs else "consumer"
+    return host_project, scope
+
+
+def _flatten_kpis(payload: dict[str, Any]) -> dict[str, Any]:
+    flat: dict[str, Any] = {}
+    for key, value in payload.items():
+        flat[f"kpi_{key}"] = value
+    return flat
+
+
+def _publish_run(
+    client: Any,
+    *,
+    project: str,
+    name: str,
+    inputs: dict[str, Any],
+    outputs: dict[str, Any],
+    tags: list[str],
+    metadata: dict[str, Any] | None = None,
+    total_cost: float | None = None,
+    total_tokens: int | None = None,
+    run_type: str = "chain",
+) -> str:
+    now = _iso_now()
+    run_id = str(uuid.uuid4())
+    payload: dict[str, Any] = {
+        "id": run_id,
+        "project_name": project,
+        "name": name,
+        "run_type": run_type,
+        "inputs": inputs,
+        "outputs": outputs,
+        "start_time": now,
+        "end_time": now,
+        "tags": tags,
+        "extra": {"metadata": metadata or {}},
+    }
+
+    if total_cost is not None:
+        payload["total_cost"] = float(total_cost)
+        payload["prompt_cost"] = 0.0
+        payload["completion_cost"] = float(total_cost)
+    if total_tokens is not None:
+        payload["total_tokens"] = int(total_tokens)
+        # For LLM-compatible dashboard metrics, expose token fields explicitly.
+        payload["prompt_tokens"] = 0
+        payload["completion_tokens"] = int(total_tokens)
+
+    usage_metadata: dict[str, Any] = {
+        "input_tokens": int(payload.get("prompt_tokens", 0) or 0),
+        "output_tokens": int(payload.get("completion_tokens", 0) or 0),
+        "total_tokens": int(payload.get("total_tokens", 0) or 0),
+        "total_cost": float(payload.get("total_cost", 0.0) or 0.0),
+    }
+
+    outputs_kpi = payload.setdefault("outputs", {})
+    outputs_kpi["usage_metadata"] = usage_metadata
+
+    meta = payload.setdefault("extra", {}).setdefault("metadata", {})
+    meta["usage_metadata"] = usage_metadata
+
+    client.create_run(**payload)
+    return run_id
+
+
+def _publish_feedback_metrics(client: Any, *, run_id: str, metrics: dict[str, Any], source: str) -> None:
+    for key, raw_value in metrics.items():
+        try:
+            value = float(raw_value)
+        except Exception:
+            continue
+
+        try:
+            client.create_feedback(
+                run_id=run_id,
+                key=str(key),
+                score=value,
+                comment=f"mcpee kpi:{source}",
+            )
+        except Exception:
+            # Feedback is optional. Keep publishing resilient.
+            continue
+
+
+def main() -> int:
+    repo_root = REPO_ROOT
+    cfg = load_config(repo_root)
+    host_project, telemetry_scope = _resolve_host_scope(repo_root)
+    host_slug = _slug(host_project)
+
+    if not cfg.langsmith.enabled:
+        print("LangSmith disabled in effective config. Nothing to publish.")
+        return 0
+
+    if not cfg.langsmith.api_key.strip() or not cfg.langsmith.project.strip():
+        print("LangSmith config missing api_key or project. Nothing to publish.")
+        return 0
+
+    try:
+        from langsmith import Client  # type: ignore
+    except Exception as exc:
+        print(f"langsmith SDK is not available: {exc}")
+        return 1
+
+    kwargs: dict[str, Any] = {"api_key": cfg.langsmith.api_key}
+    if cfg.langsmith.endpoint:
+        kwargs["api_url"] = cfg.langsmith.endpoint
+    if cfg.langsmith.workspace_id:
+        kwargs["workspace_id"] = cfg.langsmith.workspace_id
+
+    client = Client(**kwargs)
+
+    learning_path = repo_root / "observability" / "evals" / "learning-loop-report.json"
+    value_path = repo_root / "observability" / "evals" / "iteration-value-report.json"
+
+    learning = _read_json(learning_path)
+    value = _read_json(value_path)
+
+    published = 0
+
+    if learning:
+        learning_kpis = learning.get("kpis", {}) if isinstance(learning.get("kpis", {}), dict) else {}
+        learning_flat_kpis = _flatten_kpis(learning_kpis)
+        run_id = _publish_run(
+            client,
+            project=cfg.langsmith.project,
+            name="KPI::LearningLoop",
+            inputs={
+                "source_file": str(learning_path),
+                "timestamp": learning.get("timestamp"),
+            },
+            outputs={
+                "kpis": learning_kpis,
+                **learning_flat_kpis,
+                "health": learning.get("health", {}),
+                "totals": {
+                    "total_events": learning.get("total_events"),
+                    "confirmed_events": learning.get("confirmed_events"),
+                    "pending_events": learning.get("pending_events"),
+                },
+            },
+            tags=[
+                "mcpee",
+                "kpi",
+                "learning-loop",
+                "dashboard",
+                f"scope:{telemetry_scope}",
+                f"host:{host_slug}",
+            ],
+            metadata={
+                "kpi_source": "learning-loop",
+                "kpi_kind": "snapshot",
+                "host_project": host_project,
+                "host_project_slug": host_slug,
+                "telemetry_scope": telemetry_scope,
+                **learning_flat_kpis,
+            },
+        )
+        _publish_feedback_metrics(client, run_id=run_id, metrics=learning_flat_kpis, source="learning-loop")
+        published += 1
+
+    if value:
+        value_kpis = value.get("kpis", {}) if isinstance(value.get("kpis", {}), dict) else {}
+        value_flat_kpis = _flatten_kpis(value_kpis)
+        run_id = _publish_run(
+            client,
+            project=cfg.langsmith.project,
+            name="KPI::IterationValue",
+            inputs={
+                "source_file": str(value_path),
+                "timestamp": value.get("timestamp"),
+            },
+            outputs={
+                "kpis": value_kpis,
+                **value_flat_kpis,
+                "totals": value.get("totals", {}),
+                "assessment": value.get("assessment", {}),
+            },
+            tags=[
+                "mcpee",
+                "kpi",
+                "iteration-value",
+                "dashboard",
+                f"scope:{telemetry_scope}",
+                f"host:{host_slug}",
+            ],
+            metadata={
+                "kpi_source": "iteration-value",
+                "kpi_kind": "snapshot",
+                "host_project": host_project,
+                "host_project_slug": host_slug,
+                "telemetry_scope": telemetry_scope,
+                **value_flat_kpis,
+            },
+            total_cost=float(value_kpis.get("total_cost_usd", 0.0) or 0.0),
+            total_tokens=int(value_kpis.get("total_tokens", 0) or 0),
+            run_type="llm",
+        )
+        _publish_feedback_metrics(client, run_id=run_id, metrics=value_flat_kpis, source="iteration-value")
+        published += 1
+
+    if learning or value:
+        learning_kpis = learning.get("kpis", {}) if isinstance(learning.get("kpis", {}), dict) else {}
+        value_kpis = value.get("kpis", {}) if isinstance(value.get("kpis", {}), dict) else {}
+        learning_flat_kpis = _flatten_kpis(learning_kpis)
+        value_flat_kpis = _flatten_kpis(value_kpis)
+        run_id = _publish_run(
+            client,
+            project=cfg.langsmith.project,
+            name="KPI::AlignmentSnapshot",
+            inputs={
+                "source": "local-observability-evals",
+                "learning_present": bool(learning),
+                "value_present": bool(value),
+            },
+            outputs={
+                "learning_kpis": learning_kpis,
+                "value_kpis": value_kpis,
+                **{f"learning_{k}": v for k, v in learning_flat_kpis.items()},
+                **{f"value_{k}": v for k, v in value_flat_kpis.items()},
+                "published_at": _iso_now().isoformat(),
+            },
+            tags=[
+                "mcpee",
+                "kpi",
+                "alignment",
+                "dashboard",
+                f"scope:{telemetry_scope}",
+                f"host:{host_slug}",
+            ],
+            metadata={
+                "kpi_source": "alignment",
+                "kpi_kind": "snapshot",
+                "host_project": host_project,
+                "host_project_slug": host_slug,
+                "telemetry_scope": telemetry_scope,
+                **{f"learning_{k}": v for k, v in learning_flat_kpis.items()},
+                **{f"value_{k}": v for k, v in value_flat_kpis.items()},
+            },
+            total_cost=float(value_kpis.get("total_cost_usd", 0.0) or 0.0),
+            total_tokens=int(value_kpis.get("total_tokens", 0) or 0),
+            run_type="llm",
+        )
+        alignment_feedback = {
+            **{f"learning_{k}": v for k, v in learning_flat_kpis.items()},
+            **{f"value_{k}": v for k, v in value_flat_kpis.items()},
+        }
+        _publish_feedback_metrics(client, run_id=run_id, metrics=alignment_feedback, source="alignment")
+        published += 1
+
+    print(
+        f"Published KPI runs: {published} to project '{cfg.langsmith.project}' "
+        f"(host_project='{host_project}', scope='{telemetry_scope}')."
+    )
+    if published == 0:
+        print("No local KPI report JSON files were found in observability/evals.")
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/ops/telemetry-benchmark.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import statistics
+import subprocess
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def build_command(python_exe: str) -> list[str]:
+    return [
+        python_exe,
+        "scripts/intake/resolve-routing.py",
+        "--input",
+        "telemetry benchmark",
+        "--intent",
+        "feature",
+        "--domain",
+        "backend",
+        "--source-type",
+        "code",
+        "--input-tokens",
+        "120",
+        "--output-tokens",
+        "45",
+        "--estimated-cost-usd",
+        "0.0032",
+        "--model",
+        "GPT-5.3-Codex",
+        "--local-tools",
+        "2",
+        "--remote-tools",
+        "1",
+    ]
+
+
+def run_variant(*, repo_root: Path, iterations: int, telemetry_enabled: bool, python_exe: str) -> dict[str, Any]:
+    durations_ms: list[float] = []
+    failures = 0
+    for _ in range(iterations):
+        env = os.environ.copy()
+        env["TELEMETRY_ENABLED"] = "true" if telemetry_enabled else "false"
+        env["TELEMETRY_EXPORTERS"] = "console,json"
+
+        start = time.perf_counter()
+        proc = subprocess.run(
+            build_command(python_exe),
+            cwd=str(repo_root),
+            env=env,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            check=False,
+        )
+        elapsed_ms = (time.perf_counter() - start) * 1000.0
+        durations_ms.append(elapsed_ms)
+        if proc.returncode != 0:
+            failures += 1
+
+    avg_ms = statistics.mean(durations_ms) if durations_ms else 0.0
+    p95_ms = statistics.quantiles(durations_ms, n=20)[18] if len(durations_ms) >= 20 else max(durations_ms, default=0.0)
+
+    return {
+        "telemetry_enabled": telemetry_enabled,
+        "iterations": iterations,
+        "failures": failures,
+        "avg_ms": round(avg_ms, 3),
+        "min_ms": round(min(durations_ms) if durations_ms else 0.0, 3),
+        "max_ms": round(max(durations_ms) if durations_ms else 0.0, 3),
+        "p95_ms": round(p95_ms, 3),
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Benchmark telemetry overhead on/off using resolve-routing as workload.")
+    parser.add_argument("--iterations", type=int, default=10)
+    parser.add_argument("--python", default="py")
+    parser.add_argument("--python-arg", default="-3")
+    parser.add_argument("--out", default="observability/evals/telemetry-overhead-benchmark.json")
+    args = parser.parse_args()
+
+    repo_root = Path(__file__).resolve().parents[2]
+    iterations = max(2, int(args.iterations))
+    python_exe = args.python
+
+    # Windows default launcher pattern: py -3
+    if args.python == "py" and args.python_arg:
+        python_exe = f"{args.python} {args.python_arg}"  # handled below
+
+    if " " in python_exe:
+        tokens = python_exe.split()
+        base = tokens[0]
+        extra = tokens[1:]
+    else:
+        base = python_exe
+        extra = []
+
+    def build_python() -> str:
+        if not extra:
+            return base
+        return base
+
+    # Probe python command in current environment.
+    probe = subprocess.run([base, *extra, "--version"], cwd=str(repo_root), stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
+    if probe.returncode != 0:
+        print("Unable to resolve python interpreter for benchmark")
+        return 1
+
+    def command_python() -> str:
+        return base
+
+    # Wrap variant runner to include optional extra interpreter args.
+    def run_with_variant(telemetry_enabled: bool) -> dict[str, Any]:
+        durations_ms: list[float] = []
+        failures = 0
+        for _ in range(iterations):
+            env = os.environ.copy()
+            env["TELEMETRY_ENABLED"] = "true" if telemetry_enabled else "false"
+            env["TELEMETRY_EXPORTERS"] = "console,json"
+            cmd = [base, *extra, *build_command(command_python())[1:]]
+
+            start = time.perf_counter()
+            proc = subprocess.run(
+                cmd,
+                cwd=str(repo_root),
+                env=env,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                check=False,
+            )
+            elapsed_ms = (time.perf_counter() - start) * 1000.0
+            durations_ms.append(elapsed_ms)
+            if proc.returncode != 0:
+                failures += 1
+
+        avg_ms = statistics.mean(durations_ms) if durations_ms else 0.0
+        p95_ms = statistics.quantiles(durations_ms, n=20)[18] if len(durations_ms) >= 20 else max(durations_ms, default=0.0)
+
+        return {
+            "telemetry_enabled": telemetry_enabled,
+            "iterations": iterations,
+            "failures": failures,
+            "avg_ms": round(avg_ms, 3),
+            "min_ms": round(min(durations_ms) if durations_ms else 0.0, 3),
+            "max_ms": round(max(durations_ms) if durations_ms else 0.0, 3),
+            "p95_ms": round(p95_ms, 3),
+        }
+
+    enabled = run_with_variant(True)
+    disabled = run_with_variant(False)
+
+    overhead_ms = enabled["avg_ms"] - disabled["avg_ms"]
+    overhead_pct = ((overhead_ms / disabled["avg_ms"]) * 100.0) if disabled["avg_ms"] > 0 else 0.0
+
+    report = {
+        "timestamp": utc_now(),
+        "workload": "scripts/intake/resolve-routing.py",
+        "iterations": iterations,
+        "with_telemetry": enabled,
+        "without_telemetry": disabled,
+        "overhead": {
+            "avg_ms": round(overhead_ms, 3),
+            "avg_pct": round(overhead_pct, 3),
+        },
+    }
+
+    out_path = (repo_root / args.out).resolve()
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    out_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+    print(json.dumps(report, indent=2, ensure_ascii=False))
+    print(f"Benchmark report: {out_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/setup/install-project-hooks.ps1

@@ -0,0 +1,33 @@
+param(
+    [switch]$Force
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+$repoRoot = (& git rev-parse --show-toplevel 2>$null)
+if (-not $repoRoot) {
+    throw 'Not inside a git repository.'
+}
+
+Set-Location $repoRoot
+$hooksDir = Join-Path $repoRoot '.githooks'
+$postCommitHook = Join-Path $hooksDir 'post-commit'
+
+if (-not (Test-Path $postCommitHook)) {
+    throw "Missing hook template: $postCommitHook"
+}
+
+$currentHooksPath = (& git config --local --get core.hooksPath 2>$null)
+if (-not $Force -and $currentHooksPath -and $currentHooksPath -ne '.githooks') {
+    Write-Warning "core.hooksPath is already set to '$currentHooksPath'. Skipping update. Use -Force to override."
+    exit 0
+}
+
+& git config --local core.hooksPath .githooks
+if ($LASTEXITCODE -ne 0) {
+    throw 'Failed to configure core.hooksPath.'
+}
+
+Write-Host '[mcpee] Git hooks configured: core.hooksPath=.githooks'
+Write-Host '[mcpee] Hook active: post-commit -> scripts/ops/post-commit-refresh.ps1'

package/scripts/setup/setup-prerequisites.ps1

@@ -111,12 +111,36 @@ function Install-PythonRequirements {
 
 function Resolve-PythonCommand {
     if (Test-Command 'py') {
-        return @('py', '-3.14')
+        return @('py', '-3')
     }
     if (Test-Command 'python') {
         return @('python')
     }
-    throw 'Python 3.14 is required for graphify MCP. Install Python first: https://www.python.org/downloads/'
+    throw 'Python 3 is required for MCP scripts. Install Python first: https://www.python.org/downloads/'
+}
+
+function Ensure-PythonModuleInstalled {
+    param(
+        [Parameter(Mandatory = $true)][string]$PythonCommand,
+        [string[]]$PythonArgs = @(),
+        [Parameter(Mandatory = $true)][string]$Module,
+        [Parameter(Mandatory = $true)][string]$Package
+    )
+
+    if (Test-PythonImport -PythonCommand $PythonCommand -PythonArgs $PythonArgs -Module $Module) {
+        Write-Host "[ok] Python module '$Module' already available"
+        return
+    }
+
+    Write-Host "[install] $PythonCommand $($PythonArgs -join ' ') -m pip install $Package"
+    & $PythonCommand @PythonArgs -m pip install $Package
+    if ($LASTEXITCODE -ne 0) {
+        throw "Failed to install Python package '$Package'"
+    }
+
+    if (-not (Test-PythonImport -PythonCommand $PythonCommand -PythonArgs $PythonArgs -Module $Module)) {
+        throw "Python module '$Module' is still unavailable after installing '$Package'"
+    }
 }
 
 Write-Host '== MCP platform prerequisites setup =='
@@ -159,21 +183,24 @@ if (-not $SkipRepomix) {
     Install-ToolFromManifest -Tool $toolByCommand['repomix']
 }
 
-if (-not $SkipGraphify) {
-    $py = Resolve-PythonCommand
-    $pyCmd = $py[0]
-    $pyArgs = @()
-    if ($py.Length -gt 1) {
-        $pyArgs = $py[1..($py.Length - 1)]
-    }
+$py = Resolve-PythonCommand
+$pyCmd = $py[0]
+$pyArgs = @()
+if ($py.Length -gt 1) {
+    $pyArgs = $py[1..($py.Length - 1)]
+}
 
-    if ($py.Length -gt 1) {
-        Write-Host "[setup] Using Python launcher: $($py -join ' ')"
-    }
-    else {
-        Write-Host "[setup] Using Python launcher: $pyCmd"
-    }
+if ($py.Length -gt 1) {
+    Write-Host "[setup] Using Python launcher: $($py -join ' ')"
+}
+else {
+    Write-Host "[setup] Using Python launcher: $pyCmd"
+}
+
+# Core observability dependency used by KPI publishing and telemetry export.
+Ensure-PythonModuleInstalled -PythonCommand $pyCmd -PythonArgs $pyArgs -Module 'langsmith' -Package 'langsmith'
 
+if (-not $SkipGraphify) {
     Install-PythonRequirements -PythonCommand $pyCmd -PythonArgs $pyArgs -RequirementsPath 'requirements.txt'
 
     if (-not (Test-PythonImport -PythonCommand $pyCmd -PythonArgs $pyArgs -Module 'graphify.serve')) {