mcp-efficiency-engine 0.1.0

package/scripts/ops/llm-token-probe.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+"""
+llm-token-probe.py
+
+Ejecuta una consulta LLM real (Azure OpenAI) y emite metricas parseables
+por run-demo-session.ps1:
+- prompt_tokens=...
+- completion_tokens=...
+- total_tokens=...
+- total_cost_usd=...
+
+Uso:
+  python llm-token-probe.py --profile baseline --repo-root C:/repo/TSS2026 --question "..."
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+from pathlib import Path
+from typing import List
+
+try:
+    from openai import AzureOpenAI
+except Exception:
+    AzureOpenAI = None
+
+PROFILES = {
+    "baseline": {"max_files": 8, "chars_per_file": 1800},
+    "agentes": {"max_files": 6, "chars_per_file": 1200},
+    "rag": {"max_files": 4, "chars_per_file": 900},
+    "master": {"max_files": 3, "chars_per_file": 700},
+}
+
+CODE_EXTS = {".cs", ".md", ".json", ".ps1", ".py", ".ts", ".js", ".yaml", ".yml"}
+SKIP_PARTS = {"bin", "obj", ".git", "node_modules", ".venv", "dist", "build", "__pycache__"}
+
+
+def select_files(repo_root: Path, max_files: int) -> List[Path]:
+    candidates: List[Path] = []
+    for p in repo_root.rglob("*"):
+        if not p.is_file():
+            continue
+        if p.suffix.lower() not in CODE_EXTS:
+            continue
+        if any(part in SKIP_PARTS for part in p.parts):
+            continue
+        candidates.append(p)
+        if len(candidates) >= max_files:
+            break
+    return candidates
+
+
+def build_context(files: List[Path], repo_root: Path, chars_per_file: int) -> str:
+    chunks: List[str] = []
+    for f in files:
+        try:
+            text = f.read_text(encoding="utf-8", errors="ignore")
+        except Exception:
+            continue
+        rel = f.relative_to(repo_root).as_posix()
+        snippet = text[:chars_per_file]
+        chunks.append(f"FILE: {rel}\n{snippet}")
+    return "\n\n".join(chunks)
+
+
+def to_float(value: str | None, default: float) -> float:
+    if not value:
+        return default
+    try:
+        return float(value)
+    except Exception:
+        return default
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--profile", choices=sorted(PROFILES.keys()), required=True)
+    parser.add_argument("--repo-root", required=True)
+    parser.add_argument("--question", required=True)
+    args = parser.parse_args()
+
+    repo_root = Path(args.repo_root)
+    if not repo_root.exists():
+        print("prompt_tokens=0")
+        print("completion_tokens=0")
+        print("total_tokens=0")
+        print("total_cost_usd=0")
+        print("probe_status=repo_root_missing")
+        return 0
+
+    cfg = PROFILES[args.profile]
+    files = select_files(repo_root, cfg["max_files"])
+    context = build_context(files, repo_root, cfg["chars_per_file"])
+
+    api_key = os.getenv("AZURE_OPENAI_API_KEY")
+    endpoint = os.getenv("AZURE_OPENAI_ENDPOINT")
+    api_version = os.getenv("AZURE_OPENAI_API_VERSION", "2024-08-01-preview")
+    model = os.getenv("OPENAI_CHAT_MODEL", "gpt-4o-mini")
+
+    if AzureOpenAI is None or not api_key or not endpoint:
+        print("prompt_tokens=0")
+        print("completion_tokens=0")
+        print("total_tokens=0")
+        print("total_cost_usd=0")
+        print("probe_status=missing_credentials_or_openai_pkg")
+        return 0
+
+    client = AzureOpenAI(api_key=api_key, api_version=api_version, azure_endpoint=endpoint)
+
+    system_prompt = (
+        "Eres un asistente tecnico. Responde de forma breve y con foco en evidencia "
+        "sobre validacion de sesion, JWT y permisos."
+    )
+    user_prompt = (
+        f"Arquitectura: {args.profile}\n"
+        f"Pregunta: {args.question}\n\n"
+        f"Contexto:\n{context}\n\n"
+        "Devuelve un resumen tecnico corto con riesgos y evidencia."
+    )
+
+    try:
+        response = client.chat.completions.create(
+            model=model,
+            messages=[
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_prompt},
+            ],
+            temperature=0,
+            max_completion_tokens=400,
+        )
+
+        usage = response.usage
+        prompt_tokens = int(getattr(usage, "prompt_tokens", 0) or 0)
+        completion_tokens = int(getattr(usage, "completion_tokens", 0) or 0)
+        total_tokens = int(getattr(usage, "total_tokens", prompt_tokens + completion_tokens) or 0)
+
+        price_in = to_float(os.getenv("LLM_PRICE_INPUT_USD_PER_1K"), 0.0)
+        price_out = to_float(os.getenv("LLM_PRICE_OUTPUT_USD_PER_1K"), 0.0)
+        if price_in > 0 or price_out > 0:
+            cost = (prompt_tokens / 1000.0) * price_in + (completion_tokens / 1000.0) * price_out
+        else:
+            # fallback simple when split pricing is unknown
+            flat = to_float(os.getenv("LLM_PRICE_USD_PER_1K_TOKENS"), 0.0)
+            cost = (total_tokens / 1000.0) * flat if flat > 0 else 0.0
+
+        print(f"prompt_tokens={prompt_tokens}")
+        print(f"completion_tokens={completion_tokens}")
+        print(f"total_tokens={total_tokens}")
+        print(f"total_cost_usd={cost:.6f}")
+        print("probe_status=ok")
+        return 0
+
+    except Exception as ex:
+        print("prompt_tokens=0")
+        print("completion_tokens=0")
+        print("total_tokens=0")
+        print("total_cost_usd=0")
+        print(f"probe_status=error:{str(ex).replace(os.linesep, ' ')[:200]}")
+        return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/ops/run-all-demos.ps1

@@ -0,0 +1,39 @@
+param(
+    [string]$ConfigPath = ".\projects\TSS2026\analysis_mcpee\demo-session.config.json",
+    [switch]$StopOnError,
+    [switch]$DryRun
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+$demo1Exit = 0
+$demo2Exit = 0
+$demo3Exit = 0
+$demo4Exit = 0
+
+& (Join-Path $PSScriptRoot 'demo1.ps1') -ConfigPath $ConfigPath -StopOnError:$StopOnError -DryRun:$DryRun
+$demo1Exit = $LASTEXITCODE
+
+& (Join-Path $PSScriptRoot 'demo2.ps1') -ConfigPath $ConfigPath -StopOnError:$StopOnError -DryRun:$DryRun
+$demo2Exit = $LASTEXITCODE
+
+& (Join-Path $PSScriptRoot 'demo3.ps1') -ConfigPath $ConfigPath -StopOnError:$StopOnError -DryRun:$DryRun
+$demo3Exit = $LASTEXITCODE
+
+& (Join-Path $PSScriptRoot 'demo4.ps1') -ConfigPath $ConfigPath -StopOnError:$StopOnError -DryRun:$DryRun
+$demo4Exit = $LASTEXITCODE
+
+& (Join-Path $PSScriptRoot 'compare-demo-results.ps1') -ShowConsole
+
+if ($demo1Exit -ne 0 -or $demo2Exit -ne 0 -or $demo3Exit -ne 0 -or $demo4Exit -ne 0) {
+    Write-Host ''
+    Write-Host '=== FIN SESION DEMOS: TERMINADO CON INCIDENCIAS ===' -ForegroundColor Red
+    Write-Host ("Codigos: demo1={0}, demo2={1}, demo3={2}, demo4={3}" -f $demo1Exit, $demo2Exit, $demo3Exit, $demo4Exit)
+    exit 1
+}
+
+Write-Host ''
+Write-Host '=== FIN SESION DEMOS: TERMINADO OK ===' -ForegroundColor Green
+Write-Host 'Las 4 demos finalizaron correctamente y la comparativa fue generada.'
+exit 0

package/scripts/ops/run-demo-block.ps1

@@ -0,0 +1,85 @@
+param(
+    [ValidateSet('demo1', 'demo2', 'demo3', 'demo4', 'all')]
+    [string]$Demo,
+    [string]$ConfigPath = ".\projects\TSS2026\analysis_mcpee\demo-session.config.json",
+    [switch]$StopOnError,
+    [switch]$DryRun
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+function Resolve-AbsolutePath {
+    param(
+        [string]$Path,
+        [string]$BasePath
+    )
+
+    if ([System.IO.Path]::IsPathRooted($Path)) {
+        return [System.IO.Path]::GetFullPath($Path)
+    }
+
+    return [System.IO.Path]::GetFullPath((Join-Path $BasePath $Path))
+}
+
+$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path
+$configAbsPath = Resolve-AbsolutePath -Path $ConfigPath -BasePath $repoRoot
+
+if (-not (Test-Path $configAbsPath)) {
+    $fallbackConfig = Resolve-AbsolutePath -Path ".\projects\TSS2026\analysis_mcpee\demo-session.config.example.json" -BasePath $repoRoot
+    if (Test-Path $fallbackConfig) {
+        Write-Host ("Config no encontrado, usando ejemplo: {0}" -f $fallbackConfig) -ForegroundColor Yellow
+        $configAbsPath = $fallbackConfig
+    }
+    else {
+        throw "Config file not found: $configAbsPath"
+    }
+}
+
+$config = Get-Content $configAbsPath -Raw | ConvertFrom-Json -Depth 20
+if ($null -eq $config.steps -or @($config.steps).Count -eq 0) {
+    throw "Config has no steps: $configAbsPath"
+}
+
+$pattern = switch ($Demo) {
+    'demo1' { '^Demo 1' }
+    'demo2' { '^Demo 2' }
+    'demo3' { '^Demo 3' }
+    'demo4' { '^Demo 4|^Cierre' }
+    default { '.*' }
+}
+
+$selectedSteps = @($config.steps | Where-Object { [string]$_.name -match $pattern })
+if ($selectedSteps.Count -eq 0) {
+    throw "No steps match block '$Demo' in config: $configAbsPath"
+}
+
+$tmpRoot = Join-Path $repoRoot "projects\TSS2026\analysis_mcpee\tmp"
+New-Item -ItemType Directory -Path $tmpRoot -Force | Out-Null
+$tmpConfigPath = Join-Path $tmpRoot ("demo-block-{0}-{1}.json" -f $Demo, (Get-Date -Format 'yyyyMMdd-HHmmss-fff'))
+
+$sessionNameBase = if ([string]::IsNullOrWhiteSpace([string]$config.session_name)) {
+    'demo-session'
+}
+else {
+    [string]$config.session_name
+}
+
+$blockConfig = [ordered]@{
+    session_name = "{0}-{1}" -f $sessionNameBase, $Demo
+    output_root = [string]$config.output_root
+    stop_on_error = [bool]($StopOnError -or ($config.stop_on_error -eq $true))
+    steps = @($selectedSteps)
+}
+
+$blockConfig | ConvertTo-Json -Depth 20 | Set-Content -Path $tmpConfigPath -Encoding UTF8
+
+try {
+    & (Join-Path $PSScriptRoot 'run-demo-session.ps1') -ConfigPath $tmpConfigPath -StopOnError:$StopOnError -DryRun:$DryRun
+    exit $LASTEXITCODE
+}
+finally {
+    if (Test-Path $tmpConfigPath) {
+        Remove-Item -Path $tmpConfigPath -Force -ErrorAction SilentlyContinue
+    }
+}