mcp-efficiency-engine 0.1.0 → 0.1.2

package/scripts/learning/learning-loop-report.py

@@ -2,11 +2,18 @@ from __future__ import annotations
 
 import argparse
 import json
+import sys
 from collections import defaultdict
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any
 
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(_REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(_REPO_ROOT))
+
+from telemetry import build_telemetry_collector
+
 
 def utc_now() -> str:
     return datetime.now(timezone.utc).isoformat()
@@ -247,24 +254,35 @@ def main() -> int:
     args = parser.parse_args()
 
     repo_root = Path(__file__).resolve().parents[2]
+    collector = build_telemetry_collector(repo_root)
     in_path = (repo_root / args.input).resolve()
     feedback_path = (repo_root / args.feedback).resolve()
     out_json = (repo_root / args.out_json).resolve()
     out_md = (repo_root / args.out_md).resolve()
 
-    events = parse_events(in_path)
-    feedback = parse_feedback(feedback_path)
-    report = build_report(events, str(in_path), feedback, str(feedback_path))
+    with collector.start_execution(operation="learning-loop-report", session_id="learning"):
+        with collector.start_span(name="learning.report.build", kind="INTERNAL"):
+            events = parse_events(in_path)
+            feedback = parse_feedback(feedback_path)
+            report = build_report(events, str(in_path), feedback, str(feedback_path))
+
+            out_json.parent.mkdir(parents=True, exist_ok=True)
+            out_md.parent.mkdir(parents=True, exist_ok=True)
+
+            out_json.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+            out_md.write_text(report_to_markdown(report), encoding="utf-8")
 
-    out_json.parent.mkdir(parents=True, exist_ok=True)
-    out_md.parent.mkdir(parents=True, exist_ok=True)
+            kpis = report.get("kpis", {}) if isinstance(report.get("kpis", {}), dict) else {}
+            collector.record_metric("tool_invocations", float(report.get("total_events", 0)), unit="count")
+            collector.record_metric("fallback_rate", float(kpis.get("fallback_rate", 0.0)), unit="ratio")
+            collector.record_metric("grounded_rate", float(kpis.get("grounded_rate", 0.0)), unit="ratio")
+            collector.record_metric("confidence_avg", float(kpis.get("confidence_avg", 0.0)), unit="score")
 
-    out_json.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
-    out_md.write_text(report_to_markdown(report), encoding="utf-8")
+            print(f"Learning events processed: {len(events)}")
+            print(f"Report JSON: {out_json}")
+            print(f"Report MD: {out_md}")
 
-    print(f"Learning events processed: {len(events)}")
-    print(f"Report JSON: {out_json}")
-    print(f"Report MD: {out_md}")
+    collector.shutdown()
     return 0
 
 

package/scripts/ops/bye.ps1

@@ -73,6 +73,26 @@ function Test-CommandAvailable {
     return [bool](Get-Command $Name -ErrorAction SilentlyContinue)
 }
 
+function Ensure-SetupPrerequisites {
+    if ($script:SetupAttempted) {
+        if (-not $script:SetupSucceeded) {
+            throw 'setup-prerequisites was already attempted and failed earlier in this run'
+        }
+        return $true
+    }
+
+    $script:SetupAttempted = $true
+    Write-Host '[info] Missing prerequisite detected. Running setup-prerequisites automatically...' -ForegroundColor DarkYellow
+    & pwsh -NoProfile -ExecutionPolicy Bypass -File .\scripts\setup\setup-prerequisites.ps1 -PortableMode
+    if ($LASTEXITCODE -ne 0) {
+        $script:SetupSucceeded = $false
+        throw "setup-prerequisites failed with exit code $LASTEXITCODE"
+    }
+
+    $script:SetupSucceeded = $true
+    return $true
+}
+
 function Invoke-ObservabilityRetentionCleanup {
     param(
         [string]$RepoRoot,
@@ -320,6 +340,8 @@ Set-Location $repoRoot
 
 $script:Steps = @()
 $script:HasFailures = $false
+$script:SetupAttempted = $false
+$script:SetupSucceeded = $false
 $script:StepLogs = [ordered]@{}
 $script:RetentionCleanupReport = [ordered]@{
     applied = $false
@@ -367,6 +389,10 @@ else {
 
 if (-not $SkipGraphRefresh) {
     Invoke-Step -Name 'Refresh codegraph index sync' -Action {
+        if (-not (Get-Command codegraph -ErrorAction SilentlyContinue)) {
+            [void](Ensure-SetupPrerequisites)
+        }
+
         if (-not (Get-Command codegraph -ErrorAction SilentlyContinue)) {
             throw 'codegraph command not found'
         }
@@ -548,6 +574,10 @@ else {
 
 if (-not $SkipCodegraphStatus) {
     Invoke-Step -Name 'Check codegraph status' -Action {
+        if (-not (Get-Command codegraph -ErrorAction SilentlyContinue)) {
+            [void](Ensure-SetupPrerequisites)
+        }
+
         if (-not (Get-Command codegraph -ErrorAction SilentlyContinue)) {
             throw 'codegraph command not found'
         }

package/scripts/ops/hi.ps1

@@ -835,6 +835,19 @@ else {
 
 if (-not $SkipMcpStartupChecks) {
     Invoke-Step -Name 'Validate MCP servers start/listen capability' -Action {
+        $requiredCommands = @(
+            'token-saver-mcp',
+            'codebase-memory-mcp',
+            'codegraph',
+            'npx'
+        )
+
+        $missingCommands = @($requiredCommands | Where-Object { -not (Test-CommandAvailable -Name $_) })
+        if ($missingCommands.Count -gt 0) {
+            Write-Host ("[info] Missing MCP prerequisites detected ({0}). Running setup-prerequisites automatically..." -f ($missingCommands -join ', ')) -ForegroundColor DarkYellow
+            [void](Ensure-SetupPrerequisites)
+        }
+
         $py = Get-PythonCommand
         $pyCmd = $py[0]
         $pyParts = @()

package/scripts/ops/llm-token-probe.py

@@ -17,9 +17,16 @@ from __future__ import annotations
 
 import argparse
 import os
+import sys
 from pathlib import Path
 from typing import List
 
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(_REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(_REPO_ROOT))
+
+from telemetry import build_telemetry_collector
+
 try:
     from openai import AzureOpenAI
 except Exception:
@@ -81,7 +88,10 @@ def main() -> int:
     args = parser.parse_args()
 
     repo_root = Path(args.repo_root)
+    collector = build_telemetry_collector(_REPO_ROOT)
     if not repo_root.exists():
+        collector.record_event("WarningGenerated", {"warning": "repo_root_missing", "repo_root": str(repo_root)}, level="WARNING")
+        collector.shutdown()
         print("prompt_tokens=0")
         print("completion_tokens=0")
         print("total_tokens=0")
@@ -99,6 +109,17 @@ def main() -> int:
     model = os.getenv("OPENAI_CHAT_MODEL", "gpt-4o-mini")
 
     if AzureOpenAI is None or not api_key or not endpoint:
+        collector.record_event(
+            "WarningGenerated",
+            {
+                "warning": "missing_credentials_or_openai_pkg",
+                "has_openai_pkg": AzureOpenAI is not None,
+                "has_api_key": bool(api_key),
+                "has_endpoint": bool(endpoint),
+            },
+            level="WARNING",
+        )
+        collector.shutdown()
         print("prompt_tokens=0")
         print("completion_tokens=0")
         print("total_tokens=0")
@@ -119,45 +140,88 @@ def main() -> int:
         "Devuelve un resumen tecnico corto con riesgos y evidencia."
     )
 
-    try:
-        response = client.chat.completions.create(
-            model=model,
-            messages=[
-                {"role": "system", "content": system_prompt},
-                {"role": "user", "content": user_prompt},
-            ],
-            temperature=0,
-            max_completion_tokens=400,
-        )
-
-        usage = response.usage
-        prompt_tokens = int(getattr(usage, "prompt_tokens", 0) or 0)
-        completion_tokens = int(getattr(usage, "completion_tokens", 0) or 0)
-        total_tokens = int(getattr(usage, "total_tokens", prompt_tokens + completion_tokens) or 0)
-
-        price_in = to_float(os.getenv("LLM_PRICE_INPUT_USD_PER_1K"), 0.0)
-        price_out = to_float(os.getenv("LLM_PRICE_OUTPUT_USD_PER_1K"), 0.0)
-        if price_in > 0 or price_out > 0:
-            cost = (prompt_tokens / 1000.0) * price_in + (completion_tokens / 1000.0) * price_out
-        else:
-            # fallback simple when split pricing is unknown
-            flat = to_float(os.getenv("LLM_PRICE_USD_PER_1K_TOKENS"), 0.0)
-            cost = (total_tokens / 1000.0) * flat if flat > 0 else 0.0
-
-        print(f"prompt_tokens={prompt_tokens}")
-        print(f"completion_tokens={completion_tokens}")
-        print(f"total_tokens={total_tokens}")
-        print(f"total_cost_usd={cost:.6f}")
-        print("probe_status=ok")
-        return 0
-
-    except Exception as ex:
-        print("prompt_tokens=0")
-        print("completion_tokens=0")
-        print("total_tokens=0")
-        print("total_cost_usd=0")
-        print(f"probe_status=error:{str(ex).replace(os.linesep, ' ')[:200]}")
-        return 0
+    with collector.start_execution(
+        operation="llm-token-probe",
+        session_id="ops",
+        provider="azure-openai",
+        model=model,
+    ):
+        with collector.start_span(name="provider.azure_openai_call", kind="CLIENT", attributes={"profile": args.profile}):
+            collector.record_event(
+                "LLMRequest",
+                {
+                    "provider": "azure-openai",
+                    "model": model,
+                    "profile": args.profile,
+                    "selected_files": len(files),
+                },
+            )
+            try:
+                response = client.chat.completions.create(
+                    model=model,
+                    messages=[
+                        {"role": "system", "content": system_prompt},
+                        {"role": "user", "content": user_prompt},
+                    ],
+                    temperature=0,
+                    max_completion_tokens=400,
+                )
+
+                usage = response.usage
+                prompt_tokens = int(getattr(usage, "prompt_tokens", 0) or 0)
+                completion_tokens = int(getattr(usage, "completion_tokens", 0) or 0)
+                total_tokens = int(getattr(usage, "total_tokens", prompt_tokens + completion_tokens) or 0)
+
+                price_in = to_float(os.getenv("LLM_PRICE_INPUT_USD_PER_1K"), 0.0)
+                price_out = to_float(os.getenv("LLM_PRICE_OUTPUT_USD_PER_1K"), 0.0)
+                if price_in > 0 or price_out > 0:
+                    cost = (prompt_tokens / 1000.0) * price_in + (completion_tokens / 1000.0) * price_out
+                else:
+                    # fallback simple when split pricing is unknown
+                    flat = to_float(os.getenv("LLM_PRICE_USD_PER_1K_TOKENS"), 0.0)
+                    cost = (total_tokens / 1000.0) * flat if flat > 0 else 0.0
+
+                collector.record_usage(
+                    input_tokens=prompt_tokens,
+                    output_tokens=completion_tokens,
+                    estimated_cost_usd=cost,
+                )
+                collector.record_event(
+                    "LLMResponse",
+                    {
+                        "provider": "azure-openai",
+                        "model": model,
+                        "prompt_tokens": prompt_tokens,
+                        "completion_tokens": completion_tokens,
+                        "total_tokens": total_tokens,
+                        "estimated_cost_usd": round(cost, 6),
+                    },
+                )
+
+                print(f"prompt_tokens={prompt_tokens}")
+                print(f"completion_tokens={completion_tokens}")
+                print(f"total_tokens={total_tokens}")
+                print(f"total_cost_usd={cost:.6f}")
+                print("probe_status=ok")
+                collector.shutdown()
+                return 0
+
+            except Exception as ex:
+                collector.record_event(
+                    "ExceptionThrown",
+                    {
+                        "provider": "azure-openai",
+                        "error": str(ex),
+                    },
+                    level="ERROR",
+                )
+                print("prompt_tokens=0")
+                print("completion_tokens=0")
+                print("total_tokens=0")
+                print("total_cost_usd=0")
+                print(f"probe_status=error:{str(ex).replace(os.linesep, ' ')[:200]}")
+                collector.shutdown()
+                return 0
 
 
 if __name__ == "__main__":

package/scripts/ops/post-commit-refresh.ps1

@@ -0,0 +1,110 @@
+param(
+    [switch]$Force,
+    [switch]$SkipLangSmith,
+    [switch]$SkipLearningReports,
+    [switch]$SkipAutoDocs
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+function Get-PythonCommand {
+    if (Get-Command py -ErrorAction SilentlyContinue) {
+        return @('py', '-3')
+    }
+    if (Get-Command python -ErrorAction SilentlyContinue) {
+        return @('python')
+    }
+    throw 'Python is required (py or python not found in PATH).'
+}
+
+function Invoke-PythonScript {
+    param(
+        [string]$ScriptPath,
+        [string[]]$Args = @(),
+        [switch]$IgnoreErrors
+    )
+
+    $python = Get-PythonCommand
+    & $python[0] $python[1] $ScriptPath @Args
+    $exitCode = $LASTEXITCODE
+    if ($exitCode -ne 0 -and -not $IgnoreErrors) {
+        throw "Command failed ($exitCode): $ScriptPath"
+    }
+    return $exitCode
+}
+
+$repoRoot = (& git rev-parse --show-toplevel 2>$null)
+if (-not $repoRoot) {
+    Write-Host '[mcpee] post-commit-refresh skipped: git repo not detected.'
+    exit 0
+}
+
+Set-Location $repoRoot
+
+$changedFiles = (& git diff-tree --no-commit-id --name-only -r HEAD 2>$null)
+if ($LASTEXITCODE -ne 0) {
+    Write-Host '[mcpee] post-commit-refresh skipped: unable to inspect last commit.'
+    exit 0
+}
+
+$projectChanges = @($changedFiles | Where-Object { $_ -match '^projects/' })
+if (-not $Force -and $projectChanges.Count -eq 0) {
+    Write-Host '[mcpee] post-commit-refresh skipped: no changes under projects/ in last commit.'
+    exit 0
+}
+
+$started = Get-Date
+$status = 'ok'
+$errors = @()
+$steps = @()
+
+try {
+    if (-not $SkipAutoDocs) {
+        Write-Host '[mcpee] AutoDocs: compiling incremental projection...'
+        Invoke-PythonScript -ScriptPath '.\scripts\wiki\compiler_main.py'
+        $steps += 'autodocs'
+    }
+
+    if (-not $SkipLearningReports) {
+        Write-Host '[mcpee] Observability: refreshing learning reports...'
+        Invoke-PythonScript -ScriptPath '.\scripts\learning\learning-loop-report.py'
+        Invoke-PythonScript -ScriptPath '.\scripts\learning\iteration-value-report.py'
+        $steps += 'learning-reports'
+    }
+
+    if (-not $SkipLangSmith) {
+        Write-Host '[mcpee] LangSmith: publishing KPI snapshots...'
+        Invoke-PythonScript -ScriptPath '.\scripts\ops\publish-langsmith-kpis.py' -IgnoreErrors
+        $steps += 'langsmith-kpis'
+    }
+}
+catch {
+    $status = 'failed'
+    $errors += $_.Exception.Message
+    Write-Warning ("[mcpee] post-commit-refresh failed: {0}" -f $_.Exception.Message)
+}
+
+$finished = Get-Date
+$duration = [math]::Round((($finished - $started).TotalSeconds), 2)
+
+$summary = [ordered]@{
+    timestamp = $finished.ToString('o')
+    operation = 'post-commit-refresh'
+    status = $status
+    duration_sec = $duration
+    changed_projects_files = $projectChanges.Count
+    steps = $steps
+    errors = $errors
+}
+
+$sessionDir = Join-Path $repoRoot 'observability/logs/session'
+New-Item -ItemType Directory -Path $sessionDir -Force | Out-Null
+$stamp = Get-Date -Format 'yyyyMMdd-HHmmss'
+$reportPath = Join-Path $sessionDir ("post-commit-refresh-$stamp.json")
+$summary | ConvertTo-Json -Depth 8 | Set-Content -Path $reportPath -Encoding UTF8
+
+Write-Host ("[mcpee] post-commit-refresh {0}. report: {1}" -f $status, $reportPath)
+
+# Never block commit history after it was created.
+exit 0