mcp-efficiency-engine 0.1.0

package/scripts/intake/run-routing-evals.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def load_cases(path: Path) -> list[dict[str, Any]]:
+    data = json.loads(path.read_text(encoding="utf-8"))
+    cases = data.get("cases", [])
+    if not isinstance(cases, list):
+        return []
+    return [c for c in cases if isinstance(c, dict)]
+
+
+def parse_event_ids(path: Path) -> set[str]:
+    if not path.exists():
+        return set()
+    ids: set[str] = set()
+    for line in path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            obj = json.loads(line)
+        except Exception:
+            continue
+        if not isinstance(obj, dict):
+            continue
+        event_id = str(obj.get("event_id", "")).strip()
+        if event_id:
+            ids.add(event_id)
+    return ids
+
+
+def run_case(repo_root: Path, case: dict[str, Any], routing_log: Path) -> tuple[bool, str, str | None]:
+    before_ids = parse_event_ids(routing_log)
+    cmd = [
+        sys.executable,
+        str((repo_root / "scripts/intake/resolve-routing.py").resolve()),
+        "--input",
+        str(case.get("input", "")),
+        "--intent",
+        str(case.get("intent", "")),
+        "--domain",
+        str(case.get("domain", "")),
+        "--source-type",
+        str(case.get("source_type", "technical-docs")),
+        "--capability",
+        str(case.get("capability", "")),
+    ]
+    proc = subprocess.run(cmd, cwd=str(repo_root), capture_output=True, text=True)
+    output = (proc.stdout or "") + (proc.stderr or "")
+
+    event_id: str | None = None
+    after_ids = parse_event_ids(routing_log)
+    new_ids = after_ids - before_ids
+    if new_ids:
+        event_id = sorted(new_ids)[-1]
+
+    return proc.returncode == 0, output, event_id
+
+
+def record_feedback(repo_root: Path, event_id: str, case_id: str) -> tuple[bool, str]:
+    cmd = [
+        sys.executable,
+        str((repo_root / "scripts/learning/record-learning-feedback.py").resolve()),
+        "--event-id",
+        event_id,
+        "--success",
+        "true",
+        "--confidence",
+        "0.95",
+        "--source",
+        "ci",
+        "--notes",
+        f"routing-eval-pass:{case_id}",
+    ]
+    proc = subprocess.run(cmd, cwd=str(repo_root), capture_output=True, text=True)
+    output = (proc.stdout or "") + (proc.stderr or "")
+    return proc.returncode == 0, output
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Run routing eval cases and write JSON report.")
+    parser.add_argument("--cases", default="observability/evals/routing-eval-cases.json")
+    parser.add_argument("--report", default="observability/evals/routing-eval-report.json")
+    args = parser.parse_args()
+
+    repo_root = Path(__file__).resolve().parents[2]
+    routing_log = (repo_root / "observability/logs/routing-decisions.jsonl").resolve()
+    cases_path = (repo_root / args.cases).resolve()
+    if not cases_path.exists():
+        print(f"Missing eval cases file: {cases_path}")
+        return 1
+
+    cases = load_cases(cases_path)
+    results: list[dict[str, Any]] = []
+    passed = 0
+
+    for case in cases:
+        ok, output, event_id = run_case(repo_root, case, routing_log)
+        feedback_ok = False
+        feedback_output = ""
+        case_id = str(case.get("id", "unknown"))
+
+        if ok and event_id:
+            feedback_ok, feedback_output = record_feedback(repo_root, event_id, case_id)
+        if ok:
+            passed += 1
+        results.append(
+            {
+                "id": case_id,
+                "ok": ok,
+                "event_id": event_id,
+                "feedback_recorded": feedback_ok,
+                "feedback_output": feedback_output,
+                "output": output,
+            }
+        )
+
+    report = {
+        "timestamp": utc_now(),
+        "cases_total": len(cases),
+        "cases_passed": passed,
+        "cases_failed": len(cases) - passed,
+        "results": results,
+    }
+    report_path = (repo_root / args.report).resolve()
+    report_path.parent.mkdir(parents=True, exist_ok=True)
+    report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+    print(f"Routing evals completed: {passed}/{len(cases)} passed")
+    print(f"Report: {report_path}")
+    return 0 if passed == len(cases) else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/intake/validate-database-routing.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+"""Validate database routing spec requirements.
+
+Assertions:
+  1. intent=sql with domain=dba selects agent=dba.
+  2. An event with no resolvable source declares grounded=false.
+
+Exit 0 = pass.  Exit 1 = fail.
+"""
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+RESOLVE_SCRIPT = REPO_ROOT / "scripts" / "intake" / "resolve-routing.py"
+
+
+def run_routing(**kwargs: str) -> dict:
+    cmd = [sys.executable, str(RESOLVE_SCRIPT)]
+    for k, v in kwargs.items():
+        cmd += [f"--{k.replace('_', '-')}", v]
+    # Use utf-8 encoding and errors='replace' to handle non-ASCII chars
+    proc = subprocess.run(
+        cmd,
+        cwd=str(REPO_ROOT),
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        errors="replace"
+    )
+    # Output contains a JSON block followed by a trailing log line; extract JSON.
+    stdout = proc.stdout or ""
+    brace = stdout.find("{")
+    last_brace = stdout.rfind("}")
+    if brace == -1 or last_brace == -1:
+        raise ValueError(f"No JSON in output: {stdout[:200]}")
+    return json.loads(stdout[brace : last_brace + 1])
+
+
+def main() -> int:
+    errors: list[str] = []
+    print("=== validate-database-routing ===")
+
+    # --- Assertion 1: intent=sql routes to dba ---
+    try:
+        event = run_routing(
+            input="analiza esquema sql",
+            intent="sql",
+            domain="dba",
+            source_type="technical-docs",
+        )
+        agent = event.get("agent", "")
+        if agent != "dba":
+            errors.append(f"[FAIL] intent=sql expected agent=dba, got agent={agent!r}")
+        else:
+            print(f"[OK] intent=sql → agent={agent}")
+    except Exception as exc:
+        errors.append(f"[FAIL] routing error for intent=sql: {exc}")
+
+    # --- Assertion 2: no resolvable source → grounded=false ---
+    try:
+        event = run_routing(
+            input="test grounding",
+            intent="info",
+            domain="unknown-domain",
+            source_type="code",
+        )
+        grounded = event.get("grounded")
+        if grounded is not False:
+            errors.append(f"[FAIL] no-source event expected grounded=false, got grounded={grounded!r}")
+        else:
+            print("[OK] no-source event → grounded=false")
+    except Exception as exc:
+        errors.append(f"[FAIL] routing error for grounded check: {exc}")
+
+    print()
+    if errors:
+        for e in errors:
+            print(e)
+        print(f"validate-database-routing FAILED: {len(errors)} issue(s)")
+        return 1
+
+    print("validate-database-routing PASSED")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/intake/validate-rag-routing.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+"""Validate RAG local routing spec requirements.
+
+Assertions:
+  1. domain=rag with source_type=technical-docs selects agent=rag-local.
+  2. The routing event declares grounded=true with explicit sources from the repo.
+
+Exit 0 = pass.  Exit 1 = fail.
+"""
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+RESOLVE_SCRIPT = REPO_ROOT / "scripts" / "intake" / "resolve-routing.py"
+
+
+def run_routing(**kwargs: str) -> dict:
+    cmd = [sys.executable, str(RESOLVE_SCRIPT)]
+    for k, v in kwargs.items():
+        cmd += [f"--{k.replace('_', '-')}", v]
+    # Use utf-8 encoding and errors='replace' to handle non-ASCII chars
+    proc = subprocess.run(
+        cmd,
+        cwd=str(REPO_ROOT),
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        errors="replace"
+    )
+    stdout = proc.stdout or ""
+    brace = stdout.find("{")
+    last_brace = stdout.rfind("}")
+    if brace == -1 or last_brace == -1:
+        raise ValueError(f"No JSON in output: {stdout[:200]}")
+    return json.loads(stdout[brace : last_brace + 1])
+
+
+def main() -> int:
+    errors: list[str] = []
+    print("=== validate-rag-routing ===")
+
+    # --- Assertion 1: rag domain + technical-docs → rag-local ---
+    try:
+        event = run_routing(
+            input="explica guia tecnica local",
+            intent="info",
+            domain="rag",
+            source_type="technical-docs",
+        )
+        agent = event.get("agent", "")
+        if agent != "rag-local":
+            errors.append(f"[FAIL] expected agent=rag-local, got agent={agent!r}")
+        else:
+            print(f"[OK] domain=rag + technical-docs → agent={agent}")
+
+        # --- Assertion 2: grounded=true with explicit sources ---
+        grounded = event.get("grounded")
+        sources = event.get("sources", [])
+        if not grounded:
+            errors.append(f"[FAIL] rag-local event expected grounded=true, got grounded={grounded!r}")
+        elif not sources:
+            errors.append("[FAIL] rag-local event grounded=true but sources list is empty")
+        else:
+            print(f"[OK] grounded=true with sources: {sources}")
+
+    except Exception as exc:
+        errors.append(f"[FAIL] routing error: {exc}")
+
+    print()
+    if errors:
+        for e in errors:
+            print(e)
+        print(f"validate-rag-routing FAILED: {len(errors)} issue(s)")
+        return 1
+
+    print("validate-rag-routing PASSED")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/intake/validate-repo-registry.ps1

@@ -0,0 +1,20 @@
+param(
+  [string]$RegistryPath = "repo-registry/repos.yml",
+  [switch]$Strict
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = "Stop"
+
+$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path
+Set-Location $repoRoot
+
+$args = @(".\scripts\intake\validate-repo-registry.py", "--registry", $RegistryPath)
+if ($Strict) {
+  $args += "--strict"
+}
+
+python @args
+if ($LASTEXITCODE -ne 0) {
+  exit $LASTEXITCODE
+}

package/scripts/intake/validate-repo-registry.py

@@ -0,0 +1,286 @@
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import subprocess
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+try:
+    import yaml  # type: ignore
+except Exception:
+    yaml = None
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def parse_simple_yml(path: Path) -> dict[str, Any]:
+    repos: list[dict[str, Any]] = []
+    current: dict[str, Any] | None = None
+    for line in path.read_text(encoding="utf-8").splitlines():
+        st = line.strip()
+        if st.startswith("- name:"):
+            if current:
+                repos.append(current)
+            current = {"name": st.split(":", 1)[1].strip()}
+        elif current and ":" in st and not st.startswith("#"):
+            k, v = st.split(":", 1)
+            k = k.strip()
+            v = v.strip().strip('"')
+            if k in {"domain", "location", "type", "repo_url", "branch", "cache_location"}:
+                current[k] = v
+    if current:
+        repos.append(current)
+    return {"schema_version": "1.0", "repos": repos}
+
+
+def load_registry(path: Path) -> dict[str, Any]:
+    content = path.read_text(encoding="utf-8")
+
+    # JSON-first support (JSON is valid YAML, but this path avoids extra deps).
+    if content.lstrip().startswith("{"):
+        return json.loads(content)
+
+    if yaml is not None:
+        data = yaml.safe_load(content) or {}
+        if isinstance(data, dict):
+            return data
+
+    return parse_simple_yml(path)
+
+
+@dataclass
+class ValidationResult:
+    errors: list[str]
+    warnings: list[str]
+
+
+def command_exists(name: str) -> bool:
+    try:
+        completed = subprocess.run(
+            [name, "--version"],
+            check=False,
+            capture_output=True,
+            text=True,
+        )
+    except OSError:
+        return False
+    return completed.returncode == 0
+
+
+def validate(registry: dict[str, Any], strict: bool, repo_root: Path) -> ValidationResult:
+    errors: list[str] = []
+    warnings: list[str] = []
+
+    registry_mode = str(registry.get("registry_mode", "enterprise")).strip().lower() or "enterprise"
+    repos = registry.get("repos")
+    if not isinstance(repos, list):
+        return ValidationResult(errors=["Registry must define a repos list."], warnings=[])
+
+    if not repos:
+        if strict or registry_mode != "template":
+            return ValidationResult(errors=["Registry must define a non-empty repos list."], warnings=[])
+        warnings.append("Template registry has no repos yet.")
+        return ValidationResult(errors=[], warnings=warnings)
+
+    schema_version = str(registry.get("schema_version", "1.0"))
+    is_v2 = schema_version >= "2.0"
+    governance = registry.get("governance", {}) if isinstance(registry.get("governance", {}), dict) else {}
+    repo_name_prefix = str(governance.get("repo_name_prefix", "boost_")).strip() or "boost_"
+
+    names: set[str] = set()
+    graph: dict[str, list[str]] = {}
+
+    for repo in repos:
+        if not isinstance(repo, dict):
+            errors.append("Each repo entry must be an object.")
+            continue
+
+        name = str(repo.get("name", "")).strip()
+        if not name:
+            errors.append("Repo is missing required field: name")
+            continue
+
+        if strict or is_v2:
+            if not name.startswith(repo_name_prefix):
+                errors.append(
+                    f"Repo '{name}' does not satisfy naming contract. Expected prefix '{repo_name_prefix}'."
+                )
+
+        if name in names:
+            errors.append(f"Duplicate repo name: {name}")
+            continue
+        names.add(name)
+
+        for field in ("domain", "type"):
+            if not str(repo.get(field, "")).strip():
+                errors.append(f"Repo '{name}' is missing required field: {field}")
+
+        repo_type = str(repo.get("type", "")).strip().lower()
+        if repo_type not in {"local", "github"}:
+            errors.append(f"Repo '{name}' type must be 'local' or 'github'.")
+
+        location = str(repo.get("location", "")).strip()
+        if repo_type == "local":
+            if not location:
+                errors.append(f"Repo '{name}' is missing required field: location")
+            elif location:
+                resolved = (repo_root / location).resolve()
+                if not resolved.exists():
+                    if bool(repo.get("optional", False)):
+                        warnings.append(f"Optional repo '{name}' location does not exist: {location}")
+                    else:
+                        errors.append(f"Repo '{name}' location does not exist: {location}")
+        elif repo_type == "github":
+            repo_url = str(repo.get("repo_url", "")).strip()
+            if not repo_url:
+                errors.append(f"Repo '{name}' is missing required field: repo_url")
+            elif not re.match(r"^https://github\.com/[^/]+/[^/]+(?:\.git)?$", repo_url):
+                errors.append(f"Repo '{name}' repo_url must be a valid https GitHub repository URL.")
+            branch = str(repo.get("branch", "")).strip()
+            if branch and not re.match(r"^[A-Za-z0-9._\/-]+$", branch):
+                errors.append(f"Repo '{name}' branch contains invalid characters: {branch}")
+            cache_location = str(repo.get("cache_location", "")).strip()
+            if cache_location:
+                cache_resolved = (repo_root / cache_location).resolve()
+                cache_parent = cache_resolved.parent
+                if not cache_parent.exists():
+                    warnings.append(f"GitHub repo '{name}' cache parent does not exist yet: {cache_parent}")
+            if strict and not command_exists("git"):
+                errors.append(f"Repo '{name}' requires git command for type 'github'.")
+
+        deps = repo.get("dependencies", [])
+        dep_refs: list[str] = []
+        if deps:
+            if not isinstance(deps, list):
+                errors.append(f"Repo '{name}' dependencies must be a list.")
+            else:
+                for dep in deps:
+                    if not isinstance(dep, dict) or not str(dep.get("ref", "")).strip():
+                        errors.append(f"Repo '{name}' has a dependency without ref.")
+                        continue
+                    dep_refs.append(str(dep["ref"]))
+        graph[name] = dep_refs
+
+        if strict or is_v2:
+            approval = repo.get("approval")
+            if not isinstance(approval, dict):
+                errors.append(f"Repo '{name}' requires approval block in v2/strict mode.")
+            else:
+                if approval.get("status") != "approved":
+                    errors.append(f"Repo '{name}' is not approved (approval.status must be 'approved').")
+                if not str(approval.get("approved_by", "")).strip():
+                    errors.append(f"Repo '{name}' missing approval.approved_by")
+                if not str(approval.get("approved_date", "")).strip():
+                    errors.append(f"Repo '{name}' missing approval.approved_date")
+                if not str(approval.get("review_ticket", "")).strip():
+                    warnings.append(f"Repo '{name}' missing approval.review_ticket")
+
+    for src, dep_refs in graph.items():
+        for dep in dep_refs:
+            if dep not in names:
+                errors.append(f"Repo '{src}' depends on unknown repo '{dep}'.")
+
+    # Cycle detection via DFS colors: 0=unseen, 1=visiting, 2=done
+    state: dict[str, int] = {name: 0 for name in names}
+
+    def visit(node: str) -> None:
+        if state[node] == 1:
+            errors.append(f"Dependency cycle detected at '{node}'.")
+            return
+        if state[node] == 2:
+            return
+        state[node] = 1
+        for nxt in graph.get(node, []):
+            if nxt in state:
+                visit(nxt)
+        state[node] = 2
+
+    for name in names:
+        visit(name)
+
+    return ValidationResult(errors=errors, warnings=warnings)
+
+
+def write_report(
+    path: Path,
+    registry_path: Path,
+    registry_mode: str,
+    schema_version: str,
+    strict: bool,
+    result: ValidationResult,
+    repos_count: int,
+) -> None:
+    report = {
+        "timestamp": utc_now(),
+        "registry_path": str(registry_path).replace("\\", "/"),
+        "registry_mode": str(registry_mode),
+        "schema_version": schema_version,
+        "strict_mode": strict,
+        "repos_count": repos_count,
+        "errors_count": len(result.errors),
+        "warnings_count": len(result.warnings),
+        "errors": result.errors,
+        "warnings": result.warnings,
+    }
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Validate repo-registry with strict governance checks.")
+    parser.add_argument("--registry", default="repo-registry/repos.yml", help="Registry file path")
+    parser.add_argument("--strict", action="store_true", help="Enable strict governance checks")
+    args = parser.parse_args()
+
+    repo_root = Path(__file__).resolve().parents[2]
+    registry_path = (repo_root / args.registry).resolve()
+    if not registry_path.exists():
+        print(f"Missing registry file: {registry_path}")
+        return 1
+
+    try:
+        registry = load_registry(registry_path)
+    except Exception as exc:
+        print(f"Failed to parse registry file: {exc}")
+        return 1
+
+    registry_mode = str(registry.get("registry_mode", "enterprise")).strip().lower() or "enterprise"
+    schema_version = str(registry.get("schema_version", "1.0"))
+    repos_count = len(registry.get("repos", [])) if isinstance(registry.get("repos", []), list) else 0
+    result = validate(registry=registry, strict=args.strict, repo_root=repo_root)
+
+    report_path = repo_root / "repo-intake/generated/reports/repo-registry-validation.json"
+    write_report(
+        path=report_path,
+        registry_path=registry_path,
+        registry_mode=registry_mode,
+        schema_version=schema_version,
+        strict=args.strict,
+        result=result,
+        repos_count=repos_count,
+    )
+
+    if result.errors:
+        print("Repo registry validation failed.")
+        for err in result.errors:
+            print(f"- {err}")
+        print(f"Report: {report_path}")
+        return 1
+
+    print("Repo registry validation OK.")
+    if result.warnings:
+        print("Warnings:")
+        for warn in result.warnings:
+            print(f"- {warn}")
+    print(f"Report: {report_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())