mcp-efficiency-engine 0.1.3 → 0.1.5

package/.github/workflows/release-npm.yml

@@ -26,7 +26,7 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: 20
+          node-version: 22
           registry-url: https://registry.npmjs.org
 
       - name: Resolve publish version
@@ -94,14 +94,23 @@ jobs:
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
 
-      - name: Sync package.json version to repository
+      - name: Record package.json sync requirement
         if: steps.version.outputs.changed == 'true'
         run: |
-          git config user.name "github-actions[bot]"
-          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git add package.json
-          git commit -m "chore(release): sync npm version to ${{ steps.version.outputs.tag }} [skip ci]"
-          git push
+          echo "package.json was bumped in CI to publish an available npm version (${{ steps.version.outputs.tag }})."
+          echo "Repository sync is skipped because main is protected and requires PR-based updates."
+          echo "Open a follow-up PR to align package.json with npm published version."
+
+      - name: Release summary
+        run: |
+          echo "## NPM Release" >> "$GITHUB_STEP_SUMMARY"
+          echo "- mode: ${{ steps.version.outputs.mode }}" >> "$GITHUB_STEP_SUMMARY"
+          echo "- published: ${{ steps.version.outputs.tag }}" >> "$GITHUB_STEP_SUMMARY"
+          if [ "${{ steps.version.outputs.changed }}" = "true" ]; then
+            echo "- package.json sync: skipped (protected branch, requires PR)" >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "- package.json sync: not required" >> "$GITHUB_STEP_SUMMARY"
+          fi
 
       - name: Create GitHub release
         uses: softprops/action-gh-release@v2

package/ARCHITECTURE.md

@@ -7,21 +7,27 @@ flowchart TD
     O --> A[Agents Layer]
 
     A --> DEV[backend]
+    A --> FE[frontend-agent]
     A --> LEG[legacy]
     A --> DBA[dba]
+    A --> UX[ux-ui]
     A --> RAG[rag-local]
     A --> AZ[rag-azure]
     A --> IOT[iot]
     A --> COM[community-manager]
+    A --> WIKI[wiki-agent]
     A --> SNAP[snapshot]
 
     DEV --> TS[Token Saver Policy]
+    FE --> TS
     LEG --> TS
     DBA --> TS
+    UX --> TS
     RAG --> TS
     AZ --> TS
     IOT --> TS
     COM --> TS
+    WIKI --> TS
     SNAP --> TS
 
     TS --> CG[CodeGraph]
@@ -55,3 +61,12 @@ Token Saver optimiza qué contexto se usa.
 Routing decide qué motor se usa.
 Observability mide si todo funciona.
 ```
+
+## Routing base (resumen)
+
+- `backend` y `frontend-agent` -> `CodeGraph`
+- `legacy` -> `GitNexus`
+- `dba`, `ux-ui`, `rag-local`, `community-manager` -> `Graphify`
+- `rag-azure` -> `Azure RAG Builder`
+- `wiki-agent` -> `CodeGraph` (fallback `Graphify`)
+- `snapshot` -> `Repomix`

package/FINAL_USAGE_GUIDE.md

@@ -85,7 +85,30 @@ Esto significa:
 - las fuentes se conservan cuando sean obligatorias,
 - el sistema solo aumenta detalle si el usuario o el caso lo requiere.
 
-## 7. Telemetry Engine
+## 7. Instalación desde npm (rápido)
+
+Instalación recomendada en proyecto host:
+
+```powershell
+npm install mcp-efficiency-engine
+```
+
+Si el entorno bloquea scripts de npm (`allow-scripts`), ejecutar manualmente:
+
+```powershell
+npx mcp-efficiency-engine install
+npx mcp-efficiency-engine validate -PortableMode
+```
+
+Validación de contenido publicado en npm:
+
+```powershell
+npm pack --dry-run
+```
+
+Debe listar, como mínimo, `bin/`, `scripts/`, `.github/`, `README.md`, `AGENTS.md`, `ARCHITECTURE.md` y `FINAL_USAGE_GUIDE.md`.
+
+## 8. Telemetry Engine
 
 El sistema incluye un engine de telemetría desacoplado con collector único.
 

package/README.md

@@ -118,6 +118,7 @@ Contrato global en `AGENTS.md`:
 - `rag-azure` -> `Azure RAG Builder`
 - `iot` -> `GitNexus/CodeGraph + Graphify`
 - `community-manager` -> `Graphify`
+- `wiki-agent` -> `CodeGraph` (fallback `Graphify`)
 - `snapshot` -> `Repomix`
 
 ## Motores Y Herramientas
@@ -252,6 +253,11 @@ Comportamiento del hook:
   - `scripts/learning/iteration-value-report.py`
   - `scripts/ops/publish-langsmith-kpis.py` (best effort)
 
+Notas operativas recientes:
+
+- `mcpee` envuelve scripts PowerShell con `scripts/ops/trace-command.py` para emitir trazas por comando (`mcpee.*`) en el collector.
+- `scripts/ops/publish-langsmith-kpis.py` agrega snapshots locales de flujos, coste y tokens antes de publicar KPI runs en LangSmith.
+
 Artefactos/resultados:
 
 - AutoDocs actualizado en `autodocs/generated` y `autodocs/site`

package/bin/mcpee.js

@@ -60,6 +60,7 @@ function resolveInvocationCandidates() {
 function runPowerShellScript(scriptRelativePath, forwardedArgs) {
   const executionRoot = resolveExecutionRoot(scriptRelativePath);
   const scriptPath = path.join(executionRoot, scriptRelativePath);
+  const tracerScriptPath = path.join(repoRoot, "scripts", "ops", "trace-command.py");
   if (!fs.existsSync(scriptPath)) {
     process.stderr.write(`Script no encontrado: ${scriptRelativePath}\n`);
     return 1;
@@ -67,6 +68,43 @@ function runPowerShellScript(scriptRelativePath, forwardedArgs) {
 
   const shellCandidates = resolveInvocationCandidates();
   for (const shellCommand of shellCandidates) {
+    if (fs.existsSync(tracerScriptPath)) {
+      const operation = `mcpee.${path.basename(scriptRelativePath, path.extname(scriptRelativePath))}`;
+      const tracedResult = spawnSync(
+        "py",
+        [
+          "-3",
+          tracerScriptPath,
+          "--operation",
+          operation,
+          "--session-id",
+          "mcpee-cli",
+          "--cwd",
+          executionRoot,
+          "--",
+          shellCommand,
+          "-NoProfile",
+          "-ExecutionPolicy",
+          "Bypass",
+          "-File",
+          scriptPath,
+          ...forwardedArgs,
+        ],
+        {
+          cwd: repoRoot,
+          stdio: "inherit",
+        },
+      );
+
+      if (!tracedResult.error || tracedResult.error.code !== "ENOENT") {
+        if (tracedResult.error) {
+          process.stderr.write(`${tracedResult.error.message}\n`);
+          return 1;
+        }
+        return tracedResult.status ?? 0;
+      }
+    }
+
     const result = spawnSync(
       shellCommand,
       ["-NoProfile", "-ExecutionPolicy", "Bypass", "-File", scriptPath, ...forwardedArgs],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-efficiency-engine",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "description": "Motor de orquestacion para agentes MCP con routing por dominio y bootstrap portable.",
   "license": "MIT",
   "type": "commonjs",
@@ -51,7 +51,7 @@
     "hi": "node ./bin/mcpee.js hi",
     "bye": "node ./bin/mcpee.js bye",
     "intake": "node ./bin/mcpee.js intake",
-    "langsmith:kpis": "py -3 ./scripts/ops/publish-langsmith-kpis.py",
+    "langsmith:kpis": "py -3 ./scripts/ops/trace-command.py --operation mcpee.langsmith-kpis --session-id mcpee-cli --cwd . -- py -3 ./scripts/ops/publish-langsmith-kpis.py",
     "pack:check": "npm pack --dry-run"
   },
   "engines": {

package/scripts/ops/publish-langsmith-kpis.py

@@ -4,6 +4,7 @@ import json
 import os
 import sys
 import uuid
+from collections import Counter
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any
@@ -25,6 +26,28 @@ def _read_json(path: Path) -> dict[str, Any]:
     return payload if isinstance(payload, dict) else {}
 
 
+def _read_jsonl(path: Path) -> list[dict[str, Any]]:
+    if not path.exists():
+        return []
+    rows: list[dict[str, Any]] = []
+    for line in path.read_text(encoding="utf-8").splitlines():
+        raw = line.strip()
+        if not raw:
+            continue
+        try:
+            obj = json.loads(raw)
+        except Exception:
+            continue
+        if isinstance(obj, dict):
+            rows.append(obj)
+    return rows
+
+
+def _write_json(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
 def _iso_now() -> datetime:
     return datetime.now(timezone.utc)
 
@@ -48,6 +71,161 @@ def _flatten_kpis(payload: dict[str, Any]) -> dict[str, Any]:
     return flat
 
 
+def _build_telemetry_snapshot(repo_root: Path) -> dict[str, Any]:
+    traces_path = repo_root / ".telemetry" / "traces.jsonl"
+    metrics_path = repo_root / ".telemetry" / "metrics.jsonl"
+    traces = _read_jsonl(traces_path)
+    metrics = _read_jsonl(metrics_path)
+
+    execution_ids: set[str] = set()
+    operations = Counter()
+    sessions = Counter()
+    engines = Counter()
+    agents = Counter()
+    warnings = 0
+    errors = 0
+    fallback_count = 0
+    grounded_count = 0
+    routing_count = 0
+    durations: list[float] = []
+    exporter_failures = Counter()
+
+    for row in traces:
+        context = row.get("context", {}) if isinstance(row.get("context"), dict) else {}
+        payload = row.get("payload", {}) if isinstance(row.get("payload"), dict) else {}
+        event_name = str(row.get("event_name", ""))
+        level = str(row.get("level", "INFO")).upper()
+
+        execution_id = str(context.get("execution_id", "")).strip()
+        if execution_id:
+            execution_ids.add(execution_id)
+
+        session_id = str(context.get("session_id", "")).strip()
+        if session_id:
+            sessions[session_id] += 1
+
+        if level == "WARNING":
+            warnings += 1
+        if level == "ERROR":
+            errors += 1
+
+        if event_name == "ExecutionStarted":
+            op = str(payload.get("operation", "")).strip() or "unknown"
+            operations[op] += 1
+
+        if event_name == "RoutingResolved":
+            routing_count += 1
+            engine = str(payload.get("engine", "")).strip() or "unknown"
+            agent = str(payload.get("agent", "")).strip() or "unknown"
+            engines[engine] += 1
+            agents[agent] += 1
+            if bool(payload.get("fallback", False)):
+                fallback_count += 1
+            if bool(payload.get("grounded", False)):
+                grounded_count += 1
+
+        if event_name == "SpanFinished":
+            try:
+                durations.append(float(payload.get("duration_ms") or 0.0))
+            except Exception:
+                pass
+
+        if event_name == "WarningGenerated" and str(payload.get("warning", "")) == "exporter_failed":
+            exporter = str(payload.get("exporter", "unknown")).strip() or "unknown"
+            exporter_failures[exporter] += 1
+
+    metric_totals = {
+        "tokens_input": 0.0,
+        "tokens_output": 0.0,
+        "total_tokens": 0.0,
+        "estimated_cost": 0.0,
+        "efficiency_score_samples": 0,
+        "efficiency_score_avg": 0.0,
+    }
+    efficiency_sum = 0.0
+    for row in metrics:
+        payload = row.get("payload", {}) if isinstance(row.get("payload"), dict) else {}
+        if str(row.get("event_name", "")) != "MetricCalculated":
+            continue
+        name = str(payload.get("name", "")).strip()
+        try:
+            value = float(payload.get("value") or 0.0)
+        except Exception:
+            continue
+
+        if name in {"tokens_input", "tokens_output", "total_tokens", "estimated_cost"}:
+            metric_totals[name] += value
+        elif name == "efficiency_score":
+            efficiency_sum += value
+            metric_totals["efficiency_score_samples"] += 1
+
+    samples = int(metric_totals["efficiency_score_samples"])
+    if samples > 0:
+        metric_totals["efficiency_score_avg"] = round(efficiency_sum / samples, 4)
+
+    avg_duration = round(sum(durations) / len(durations), 4) if durations else 0.0
+    p95_duration = 0.0
+    if durations:
+        sorted_d = sorted(durations)
+        idx = max(0, min(len(sorted_d) - 1, int(round(0.95 * (len(sorted_d) - 1)))))
+        p95_duration = round(sorted_d[idx], 4)
+
+    fallback_rate = round((fallback_count / routing_count), 4) if routing_count else 0.0
+    grounded_rate = round((grounded_count / routing_count), 4) if routing_count else 0.0
+
+    return {
+        "timestamp": _iso_now().isoformat(),
+        "source": {
+            "traces": str(traces_path),
+            "metrics": str(metrics_path),
+        },
+        "totals": {
+            "trace_records": len(traces),
+            "metric_records": len(metrics),
+            "executions": len(execution_ids),
+            "warnings": warnings,
+            "errors": errors,
+        },
+        "kpis": {
+            "routing_fallback_rate": fallback_rate,
+            "routing_grounded_rate": grounded_rate,
+            "avg_span_duration_ms": avg_duration,
+            "p95_span_duration_ms": p95_duration,
+            "tokens_input": int(metric_totals["tokens_input"]),
+            "tokens_output": int(metric_totals["tokens_output"]),
+            "total_tokens": int(metric_totals["total_tokens"]),
+            "total_cost_usd": round(float(metric_totals["estimated_cost"]), 6),
+            "efficiency_score_avg": metric_totals["efficiency_score_avg"],
+        },
+        "breakdown": {
+            "operation": dict(operations),
+            "session": dict(sessions),
+            "engine": dict(engines),
+            "agent": dict(agents),
+            "exporter_failures": dict(exporter_failures),
+        },
+    }
+
+
+def _build_chat_usage_snapshot(repo_root: Path) -> dict[str, Any]:
+    chat_path = repo_root / "observability" / "evals" / "chat-token-usage-report.json"
+    payload = _read_json(chat_path)
+    totals = payload.get("totals", {}) if isinstance(payload.get("totals", {}), dict) else {}
+    budget = payload.get("budget", {}) if isinstance(payload.get("budget", {}), dict) else {}
+    return {
+        "timestamp": payload.get("timestamp"),
+        "source": str(chat_path),
+        "kpis": {
+            "input_tokens": int(totals.get("input_tokens", 0) or 0),
+            "output_tokens": int(totals.get("output_tokens", 0) or 0),
+            "total_tokens": int(totals.get("total_tokens", 0) or 0),
+            "copilot_credits": float(totals.get("copilot_credits", 0.0) or 0.0),
+            "budget_utilization_rate": float(budget.get("budget_utilization_rate", 0.0) or 0.0),
+        },
+        "raw": payload,
+    }
+
+
 def _publish_run(
     client: Any,
     *,
@@ -104,12 +282,18 @@ def _publish_run(
 
 
 def _publish_feedback_metrics(client: Any, *, run_id: str, metrics: dict[str, Any], source: str) -> None:
+    min_score = -99999.9999
+    max_score = 99999.9999
     for key, raw_value in metrics.items():
         try:
             value = float(raw_value)
         except Exception:
             continue
 
+        if value < min_score or value > max_score:
+            # LangSmith feedback score has bounded range.
+            continue
+
         try:
             client.create_feedback(
                 run_id=run_id,
@@ -152,9 +336,38 @@ def main() -> int:
 
     learning_path = repo_root / "observability" / "evals" / "learning-loop-report.json"
     value_path = repo_root / "observability" / "evals" / "iteration-value-report.json"
+    telemetry_snapshot_path = repo_root / "observability" / "evals" / "telemetry-flow-cost-token-report.json"
+    chat_snapshot_path = repo_root / "observability" / "evals" / "chat-token-usage-report.json"
 
     learning = _read_json(learning_path)
     value = _read_json(value_path)
+    telemetry_snapshot = _build_telemetry_snapshot(repo_root)
+    _write_json(telemetry_snapshot_path, telemetry_snapshot)
+    chat_snapshot = _build_chat_usage_snapshot(repo_root)
+
+    project_traceability_path = (
+        repo_root
+        / "projects"
+        / "techriders"
+        / "analysis_mcpee"
+        / f"telemetry-recovery-{datetime.now(timezone.utc).strftime('%Y-%m-%d')}.json"
+    )
+    unified_snapshot = {
+        "timestamp": _iso_now().isoformat(),
+        "host_project": host_project,
+        "telemetry_scope": telemetry_scope,
+        "sources": {
+            "learning": str(learning_path),
+            "iteration_value": str(value_path),
+            "telemetry_snapshot": str(telemetry_snapshot_path),
+            "chat_usage": str(chat_snapshot_path),
+        },
+        "learning": learning,
+        "iteration_value": value,
+        "telemetry": telemetry_snapshot,
+        "chat_usage": chat_snapshot,
+    }
+    _write_json(project_traceability_path, unified_snapshot)
 
     published = 0
 
@@ -288,10 +501,92 @@ def main() -> int:
         _publish_feedback_metrics(client, run_id=run_id, metrics=alignment_feedback, source="alignment")
         published += 1
 
+    telemetry_kpis = telemetry_snapshot.get("kpis", {}) if isinstance(telemetry_snapshot.get("kpis", {}), dict) else {}
+    if telemetry_kpis:
+        telemetry_flat_kpis = _flatten_kpis(telemetry_kpis)
+        run_id = _publish_run(
+            client,
+            project=cfg.langsmith.project,
+            name="KPI::TelemetryFlowCostToken",
+            inputs={
+                "source_file": str(telemetry_snapshot_path),
+                "timestamp": telemetry_snapshot.get("timestamp"),
+            },
+            outputs={
+                "kpis": telemetry_kpis,
+                **telemetry_flat_kpis,
+                "totals": telemetry_snapshot.get("totals", {}),
+                "breakdown": telemetry_snapshot.get("breakdown", {}),
+            },
+            tags=[
+                "mcpee",
+                "kpi",
+                "telemetry",
+                "cost-token-flow",
+                "dashboard",
+                f"scope:{telemetry_scope}",
+                f"host:{host_slug}",
+            ],
+            metadata={
+                "kpi_source": "telemetry-flow-cost-token",
+                "kpi_kind": "snapshot",
+                "host_project": host_project,
+                "host_project_slug": host_slug,
+                "telemetry_scope": telemetry_scope,
+                **telemetry_flat_kpis,
+            },
+            total_cost=float(telemetry_kpis.get("total_cost_usd", 0.0) or 0.0),
+            total_tokens=int(telemetry_kpis.get("total_tokens", 0) or 0),
+            run_type="llm",
+        )
+        _publish_feedback_metrics(client, run_id=run_id, metrics=telemetry_flat_kpis, source="telemetry-flow-cost-token")
+        published += 1
+
+    chat_kpis = chat_snapshot.get("kpis", {}) if isinstance(chat_snapshot.get("kpis", {}), dict) else {}
+    if chat_kpis:
+        chat_flat_kpis = _flatten_kpis(chat_kpis)
+        run_id = _publish_run(
+            client,
+            project=cfg.langsmith.project,
+            name="KPI::ChatTokenUsage",
+            inputs={
+                "source_file": str(chat_snapshot_path),
+                "timestamp": chat_snapshot.get("timestamp"),
+            },
+            outputs={
+                "kpis": chat_kpis,
+                **chat_flat_kpis,
+                "budget": (chat_snapshot.get("raw", {}) or {}).get("budget", {}),
+            },
+            tags=[
+                "mcpee",
+                "kpi",
+                "chat-usage",
+                "cost-token-flow",
+                "dashboard",
+                f"scope:{telemetry_scope}",
+                f"host:{host_slug}",
+            ],
+            metadata={
+                "kpi_source": "chat-token-usage",
+                "kpi_kind": "snapshot",
+                "host_project": host_project,
+                "host_project_slug": host_slug,
+                "telemetry_scope": telemetry_scope,
+                **chat_flat_kpis,
+            },
+            total_tokens=int(chat_kpis.get("total_tokens", 0) or 0),
+            run_type="llm",
+        )
+        _publish_feedback_metrics(client, run_id=run_id, metrics=chat_flat_kpis, source="chat-token-usage")
+        published += 1
+
     print(
         f"Published KPI runs: {published} to project '{cfg.langsmith.project}' "
         f"(host_project='{host_project}', scope='{telemetry_scope}')."
     )
+    print(f"Local telemetry snapshot: {telemetry_snapshot_path}")
+    print(f"Project traceability snapshot: {project_traceability_path}")
     if published == 0:
         print("No local KPI report JSON files were found in observability/evals.")
 

package/scripts/ops/trace-command.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import argparse
+import subprocess
+import sys
+from pathlib import Path
+
+
+def _bootstrap_repo_root() -> Path:
+    repo_root = Path(__file__).resolve().parents[2]
+    if str(repo_root) not in sys.path:
+        sys.path.insert(0, str(repo_root))
+    return repo_root
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Run a command wrapped with telemetry execution/span.")
+    parser.add_argument("--operation", required=True, help="Telemetry operation name.")
+    parser.add_argument("--session-id", default="mcpee-cli", help="Telemetry session identifier.")
+    parser.add_argument("--cwd", default=".", help="Working directory for the command.")
+    parser.add_argument("command", nargs=argparse.REMAINDER, help="Command to execute after --.")
+    return parser.parse_args()
+
+
+def _normalize_command(raw: list[str]) -> list[str]:
+    cmd = list(raw)
+    if cmd and cmd[0] == "--":
+        cmd = cmd[1:]
+    return cmd
+
+
+def main() -> int:
+    args = _parse_args()
+    command = _normalize_command(args.command)
+    if not command:
+        print("trace-command.py: missing command to execute", file=sys.stderr)
+        return 2
+
+    repo_root = _bootstrap_repo_root()
+    from telemetry import build_telemetry_collector
+
+    collector = build_telemetry_collector(repo_root)
+    cwd = Path(args.cwd).resolve()
+    exit_code = 0
+
+    try:
+        with collector.start_execution(operation=args.operation, session_id=args.session_id, model="mcpee-cli"):
+            with collector.start_span(
+                name="subprocess",
+                kind="TOOL",
+                attributes={
+                    "cwd": str(cwd),
+                    "command": command[0],
+                    "args_count": len(command) - 1,
+                },
+            ):
+                completed = subprocess.run(command, cwd=str(cwd), check=False)
+                exit_code = int(completed.returncode)
+                collector.record_metric("subprocess_exit_code", float(exit_code), unit="code")
+                if exit_code != 0:
+                    raise RuntimeError(f"command failed with exit code {exit_code}")
+    except RuntimeError:
+        pass
+
+    return exit_code
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())