mcp-efficiency-engine 0.1.0 → 0.1.1

package/.github/workflows/autodocs-sync.yml

@@ -3,22 +3,8 @@ name: AutoDocs Sync
 on:
   push:
     branches: [ main, develop ]
-    paths:
-      - 'repo-intake/generated/**'
-      - 'autodocs/**'
-      - 'scripts/wiki/**'
-      - 'orchestrator/wiki/**'
-      - '.github/skills/wiki-tools.json'
-      - '.github/agents/wiki-agent.agent.md'
   pull_request:
     branches: [ main, develop ]
-    paths:
-      - 'repo-intake/generated/**'
-      - 'autodocs/**'
-      - 'scripts/wiki/**'
-      - 'orchestrator/wiki/**'
-      - '.github/skills/wiki-tools.json'
-      - '.github/agents/wiki-agent.agent.md'
   workflow_dispatch:
 
 permissions:
@@ -68,7 +54,7 @@ jobs:
           echo "- Required artifacts: autodocs/generated + autodocs/site" >> $env:GITHUB_STEP_SUMMARY
 
   autodocs-sync:
-    if: github.event_name != 'pull_request' && github.actor != 'github-actions[bot]'
+    if: github.actor != 'github-actions[bot]'
     permissions:
       contents: write
     runs-on: windows-latest
@@ -89,6 +75,7 @@ jobs:
         shell: pwsh
 
       - name: Commit projected docs and unified graph
+        if: github.event_name != 'pull_request'
         shell: pwsh
         run: |
           git config user.name "github-actions[bot]"

package/.github/workflows/release-npm.yml

@@ -0,0 +1,111 @@
+name: Release & Publish NPM
+
+on:
+  push:
+    branches: [ main ]
+  workflow_dispatch:
+    inputs:
+      npm_version:
+        description: "Version especifica a publicar (ej. 0.2.0). Si se omite, se autoajusta."
+        required: false
+        type: string
+
+permissions:
+  contents: write
+
+jobs:
+  release-and-publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          registry-url: https://registry.npmjs.org
+
+      - name: Resolve publish version
+        id: version
+        run: |
+          PACKAGE_NAME=$(node -p "require('./package.json').name")
+          BASE_VERSION=$(node -p "require('./package.json').version")
+          REQUESTED_VERSION="${{ github.event.inputs.npm_version || '' }}"
+
+          version_exists() {
+            npm view "$PACKAGE_NAME@$1" version >/dev/null 2>&1
+          }
+
+          if [ -n "$REQUESTED_VERSION" ]; then
+            TARGET_VERSION="$REQUESTED_VERSION"
+            MODE="explicit"
+          else
+            TARGET_VERSION="$BASE_VERSION"
+            MODE="auto"
+          fi
+
+          if [ "$MODE" = "explicit" ]; then
+            if version_exists "$TARGET_VERSION"; then
+              echo "Requested version $TARGET_VERSION is already published in npm."
+              exit 1
+            fi
+          else
+            CANDIDATE="$TARGET_VERSION"
+            while version_exists "$CANDIDATE"; do
+              CANDIDATE=$(node -e "const parts = process.argv[1].split('.').map(Number); if (parts.length !== 3 || parts.some(Number.isNaN)) { process.exit(2); } parts[2] += 1; process.stdout.write(parts.join('.'));" "$CANDIDATE") || {
+                echo "Cannot auto-bump non x.y.z version: $CANDIDATE"
+                exit 1
+              }
+            done
+            TARGET_VERSION="$CANDIDATE"
+          fi
+
+          CHANGED=false
+          if [ "$TARGET_VERSION" != "$BASE_VERSION" ]; then
+            npm version "$TARGET_VERSION" --no-git-tag-version
+            CHANGED=true
+          fi
+
+          echo "version=$TARGET_VERSION" >> "$GITHUB_OUTPUT"
+          echo "tag=v$TARGET_VERSION" >> "$GITHUB_OUTPUT"
+          echo "changed=$CHANGED" >> "$GITHUB_OUTPUT"
+          echo "mode=$MODE" >> "$GITHUB_OUTPUT"
+
+      - name: Ensure NPM token is configured
+        run: |
+          if [ -z "${NODE_AUTH_TOKEN}" ]; then
+            echo "Missing NODE_AUTH_TOKEN/NPM_TOKEN secret. Configure repository secret NPM_TOKEN."
+            exit 1
+          fi
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      - name: Remove Python cache artifacts from workspace
+        run: |
+          find . -type d -name "__pycache__" -prune -exec rm -rf {} +
+          find . -type f -name "*.pyc" -delete
+
+      - name: Publish to npm
+        run: npm publish --access public
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      - name: Sync package.json version to repository
+        if: steps.version.outputs.changed == 'true'
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add package.json
+          git commit -m "chore(release): sync npm version to ${{ steps.version.outputs.tag }} [skip ci]"
+          git push
+
+      - name: Create GitHub release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ steps.version.outputs.tag }}
+          name: ${{ steps.version.outputs.tag }}
+          generate_release_notes: true

package/FINAL_USAGE_GUIDE.md

@@ -85,6 +85,62 @@ Esto significa:
 - las fuentes se conservan cuando sean obligatorias,
 - el sistema solo aumenta detalle si el usuario o el caso lo requiere.
 
+## 7. Telemetry Engine
+
+El sistema incluye un engine de telemetría desacoplado con collector único.
+
+Flujo:
+
+```txt
+Componente -> TelemetryCollector -> Pipeline -> Exporters
+```
+
+Exporters soportados:
+
+- `console` (siempre disponible)
+- `json` (persistencia local en `.telemetry/`)
+- `langsmith` (opcional, solo con configuración válida)
+
+Si un exporter falla:
+
+- se registra warning,
+- los demás exporters continúan,
+- la ejecución principal no se detiene.
+
+### Configuración rápida
+
+Archivo `telemetry/config.json`:
+
+```json
+{
+  "telemetry": {
+    "enabled": true,
+    "batch_size": 25,
+    "telemetry_dir": ".telemetry",
+    "exporters": ["console", "json", "langsmith"]
+  },
+  "langsmith": {
+    "enabled": false,
+    "api_key": "",
+    "project": "",
+    "endpoint": ""
+  }
+}
+```
+
+### Cómo activar LangSmith
+
+1. Configurar `LANGSMITH_ENABLED=true`.
+2. Definir `LANGSMITH_API_KEY` y `LANGSMITH_PROJECT`.
+3. Mantener `langsmith` en `telemetry.exporters`.
+
+Si falta configuración, se omite automáticamente.
+
+### Cómo deshabilitar exporters
+
+- Desactivar todo: `TELEMETRY_ENABLED=false`
+- Solo algunos: `TELEMETRY_EXPORTERS=console,json`
+
 ## Diagrama Visual De Uso Diario
 
 ```mermaid

package/README.md

@@ -245,6 +245,139 @@ Artefactos canónicos:
 
 ## Observabilidad
 
+### Telemetry Engine (desacoplado y extensible)
+
+La observabilidad ahora se soporta mediante un engine propio en `telemetry/`.
+
+Principios:
+
+- Telemetría siempre activa a nivel de modelo de datos.
+- Exporters opcionales (`console`, `json`, `langsmith`).
+- Ningún flujo de negocio depende de LangSmith.
+- Si un exporter falla, la ejecución principal continua.
+
+Arquitectura:
+
+```mermaid
+flowchart TD
+  T[Tool/Flow] --> C[TelemetryCollector]
+  C --> P[Telemetry Pipeline]
+  P --> EX1[Console Exporter]
+  P --> EX2[JSON Exporter]
+  P --> EX3[LangSmith Exporter]
+  P --> EXN[Future Exporters]
+```
+
+Trazas jerárquicas:
+
+- Cada ejecución genera `execution_id`, `trace_id`, `span_id`, `parent_span_id`.
+- Se propaga contexto con `contextvars` (sin variables globales).
+- Spans soportan `events`, `status`, `duration_ms` y error asociado.
+
+Configuración base (`telemetry/config.json`):
+
+```json
+{
+  "telemetry": {
+    "enabled": true,
+    "batch_size": 100,
+    "telemetry_dir": ".telemetry",
+    "exporters": ["console", "json"]
+  },
+  "langsmith": {
+    "enabled": false,
+    "api_key": "",
+    "project": "",
+    "endpoint": "",
+    "high_signal_only": true,
+    "min_span_duration_ms": 100,
+    "emit_execution_summary": true
+  }
+}
+```
+
+Conexión segura a LangSmith (sin subir token al repo/npm):
+
+1. No guardes el token en `telemetry/config.json`.
+2. Define variables de entorno locales (usuario o sesión).
+3. Activa exporter por entorno con `TELEMETRY_EXPORTERS=console,json,langsmith`.
+4. Verifica que `.env` y variantes están ignorados por Git y npm.
+
+Ejemplo (PowerShell, solo sesión actual):
+
+```powershell
+$env:LANGSMITH_ENABLED='true'
+$env:LANGSMITH_API_KEY='tu_token'
+$env:LANGSMITH_PROJECT='mcpee-local'
+$env:LANGSMITH_ENDPOINT='https://api.smith.langchain.com'
+$env:LANGSMITH_HIGH_SIGNAL_ONLY='true'
+$env:LANGSMITH_MIN_SPAN_DURATION_MS='100'
+$env:LANGSMITH_EMIT_EXECUTION_SUMMARY='true'
+$env:TELEMETRY_EXPORTERS='console,json,langsmith'
+```
+
+Persistente para tu usuario Windows:
+
+```powershell
+[System.Environment]::SetEnvironmentVariable('LANGSMITH_ENABLED','true','User')
+[System.Environment]::SetEnvironmentVariable('LANGSMITH_API_KEY','tu_token','User')
+[System.Environment]::SetEnvironmentVariable('LANGSMITH_PROJECT','mcpee-local','User')
+[System.Environment]::SetEnvironmentVariable('LANGSMITH_ENDPOINT','https://api.smith.langchain.com','User')
+[System.Environment]::SetEnvironmentVariable('LANGSMITH_HIGH_SIGNAL_ONLY','true','User')
+[System.Environment]::SetEnvironmentVariable('LANGSMITH_MIN_SPAN_DURATION_MS','100','User')
+[System.Environment]::SetEnvironmentVariable('LANGSMITH_EMIT_EXECUTION_SUMMARY','true','User')
+[System.Environment]::SetEnvironmentVariable('TELEMETRY_EXPORTERS','console,json,langsmith','User')
+
+Modo high-signal recomendado en LangSmith:
+
+- `LANGSMITH_HIGH_SIGNAL_ONLY=true` (default): prioriza trazas útiles y reduce ruido.
+- `LANGSMITH_MIN_SPAN_DURATION_MS=100` (default): solo mantiene spans rápidos cuando fallan; los de éxito deben superar el umbral.
+- `LANGSMITH_EMIT_EXECUTION_SUMMARY=true` (default): añade un resumen consolidado por ejecución (duración, estado, warnings/errors, tokens/coste).
+- Mantiene eventos clave (`ExecutionStarted`, `ExecutionFinished`, `RoutingResolved`, warnings/errores) y resumen `UsageSummary` con modelo/tokens/coste.
+- Omite eventos de bajo valor para la UI de LangSmith, pero conserva debug detallado en exporters locales (`console`, `json`).
+```
+
+Para volver al modo local sin LangSmith:
+
+```powershell
+$env:LANGSMITH_ENABLED='false'
+$env:TELEMETRY_EXPORTERS='console,json'
+```
+
+Variables de entorno soportadas:
+
+- `TELEMETRY_ENABLED`
+- `TELEMETRY_EXPORTERS`
+- `TELEMETRY_BATCH_SIZE`
+- `TELEMETRY_DIR`
+- `LANGSMITH_ENABLED`
+- `LANGSMITH_API_KEY`
+- `LANGSMITH_PROJECT`
+- `LANGSMITH_ENDPOINT`
+- `LANGSMITH_WORKSPACE_ID` (opcional, recomendado para cuentas con multiples workspaces)
+
+Nota: el engine carga automáticamente `.env` en la raíz del repo. Si también existe variable en el entorno del sistema/proceso, esa tiene prioridad.
+
+Dependencia runtime: `langsmith` está incluida en `requirements.txt` para que `scripts/setup/setup-prerequisites.ps1` la instale automáticamente cuando prepares el entorno Python.
+
+Cómo crear un exporter nuevo:
+
+1. Implementar contrato `export/flush/shutdown` en `telemetry/exporters/<nuevo>/exporter.py`.
+2. Registrar el exporter en `telemetry/bootstrap.py`.
+3. Añadirlo en `telemetry/config.json` o `TELEMETRY_EXPORTERS`.
+
+Si LangSmith no está configurado correctamente, el exporter se omite y el engine sigue funcionando con `console/json`.
+
+Benchmark de overhead on/off:
+
+```powershell
+py -3 .\scripts\ops\telemetry-benchmark.py --iterations 10
+```
+
+Salida:
+
+- `observability/evals/telemetry-overhead-benchmark.json`
+
 Registros principales:
 
 - `observability/logs/routing-decisions.jsonl`

package/autodocs/README.md

@@ -14,6 +14,12 @@ AutoDocs es la wiki interna nativa de `mcp-efficiency-engine`.
 py -3 -m scripts.wiki.wiki_compiler
 ```
 
+## Sincronizacion automatica
+
+- En CI, el workflow `autodocs-sync` valida en PR y sincroniza en `main`/`develop`.
+- La ejecucion ya no depende de filtros de path: cualquier cambio potencialmente relevante recompila AutoDocs.
+- Si hay drift, CI exige regenerar y commitear `autodocs/generated` y `autodocs/site`.
+
 ## Artefactos clave
 
 - `autodocs/generated/unified-graph.json`

package/observability/metrics.md

@@ -28,6 +28,39 @@ Métricas: routing accuracy, tool usage, tool misuse, tool switching, grounding
 - token_efficiency: tendencia estable o mejorando
 - caveman_effectiveness: `>= 85%`
 
+## Telemetry Engine native metrics
+
+El engine expone métricas propias y no depende de proveedores externos:
+
+- execution_time_ms
+- tool_duration_ms
+- average_duration_ms
+- parallel_tasks
+- sequential_tasks
+- retry_count
+- error_count
+- warning_count
+- llm_calls
+- provider_calls
+- mcp_calls
+- tool_invocations
+- tokens_input
+- tokens_output
+- total_tokens
+- estimated_cost
+- cache_hits
+- cache_misses
+- compression_ratio
+- parallel_efficiency
+- prompt_reduction
+- context_reduction
+- efficiency_score (0-100)
+
+Los cálculos viven en:
+
+- `telemetry/metrics/aggregator.py`
+- `telemetry/scoring/efficiency.py`
+
 ## Semaforo
 
 - `OK`: todos los umbrales en rango.

package/observability/scoring-rubric.md

@@ -1,5 +1,7 @@
 # Scoring Rubric
 
+> El score de eficiencia es nativo del Telemetry Engine y no depende de LangSmith.
+
 ## Componentes y pesos
 
 - Routing score: 40%
@@ -36,3 +38,21 @@
 - `80-89`: Aceptable
 - `70-79`: Mejorable
 - `< 70`: No apto
+
+## Efficiency Score (Telemetry Engine)
+
+Salida: `0..100`.
+
+Factores por defecto:
+
+- time
+- cost
+- tokens
+- cache
+- parallelism
+- reuse
+- calls
+- compression
+
+Implementación base en `telemetry/scoring/efficiency.py`.
+Los pesos son configurables y el algoritmo se puede ajustar sin tocar exporters.

package/orchestrator/wiki/graph_consolidator.py

@@ -11,12 +11,21 @@ SECTIONS = section_catalog()
 
 
 class GraphConsolidator:
-    def __init__(self, output_graph_path: Path, markdown_output_dir: Path) -> None:
+    def __init__(self, output_graph_path: Path, markdown_output_dir: Path, telemetry_collector: Any | None = None) -> None:
         self.output_graph_path = output_graph_path
         self.markdown_output_dir = markdown_output_dir
         self.generated_output_dir = output_graph_path.parent
+        self._telemetry = telemetry_collector
 
     def consolidate(self, provider_contracts: List[Dict[str, Any]]) -> Dict[str, Any]:
+        if self._telemetry is not None:
+            self._telemetry.record_event(
+                "KnowledgeGenerated",
+                {
+                    "operation": "graph_consolidate_start",
+                    "provider_contracts": len(provider_contracts),
+                },
+            )
         raw_nodes: Dict[str, Dict[str, Any]] = {}
         wiki_nodes: Dict[str, Dict[str, Any]] = {}
         for contract in provider_contracts:
@@ -54,6 +63,17 @@ class GraphConsolidator:
         relations_index = self._build_relations_index(wiki_nodes)
         section_manifest = self._build_section_manifest(wiki_nodes)
 
+        if self._telemetry is not None:
+            self._telemetry.record_metric("tool_invocations", float(len(provider_contracts)), unit="count")
+            self._telemetry.record_metric("cache_misses", 0.0, unit="count")
+            self._telemetry.record_event(
+                "KnowledgeGenerated",
+                {
+                    "operation": "graph_consolidate_finish",
+                    "nodes": len(wiki_nodes),
+                },
+            )
+
         return {
             "last_updated": self._deterministic_marker(wiki_nodes),
             "nodes": wiki_nodes,
@@ -71,11 +91,23 @@ class GraphConsolidator:
         return f"sha256:{digest}"
 
     def persist_graph(self, unified_graph: Dict[str, Any]) -> None:
+        if self._telemetry is not None:
+            self._telemetry.record_event("ToolStarted", {"tool": "persist_graph"})
         self.output_graph_path.parent.mkdir(parents=True, exist_ok=True)
         with self.output_graph_path.open("w", encoding="utf-8") as handle:
             json.dump(unified_graph, handle, indent=2, ensure_ascii=True)
+        if self._telemetry is not None:
+            self._telemetry.record_event(
+                "ToolFinished",
+                {
+                    "tool": "persist_graph",
+                    "path": str(self.output_graph_path),
+                },
+            )
 
     def persist_manifests(self, unified_graph: Dict[str, Any]) -> Dict[str, int]:
+        if self._telemetry is not None:
+            self._telemetry.record_event("ToolStarted", {"tool": "persist_manifests"})
         self.generated_output_dir.mkdir(parents=True, exist_ok=True)
         artifacts = {
             "search-index.json": unified_graph.get("search_index", []),
@@ -87,6 +119,14 @@ class GraphConsolidator:
             output_path = self.generated_output_dir / file_name
             with output_path.open("w", encoding="utf-8") as handle:
                 json.dump(payload, handle, indent=2, ensure_ascii=True)
+        if self._telemetry is not None:
+            self._telemetry.record_event(
+                "ToolFinished",
+                {
+                    "tool": "persist_manifests",
+                    "manifests": len(artifacts),
+                },
+            )
         return {"manifests": len(artifacts)}
 
     def project_dirty_nodes(
@@ -96,6 +136,15 @@ class GraphConsolidator:
         all_nodes: Dict[str, Dict[str, Any]],
         cached_nodes: Dict[str, Dict[str, Any]],
     ) -> Dict[str, int]:
+        if self._telemetry is not None:
+            self._telemetry.record_event(
+                "ToolStarted",
+                {
+                    "tool": "project_dirty_nodes",
+                    "dirty_nodes": len(dirty_nodes),
+                    "deleted_nodes": len(deleted_nodes),
+                },
+            )
         self.markdown_output_dir.mkdir(parents=True, exist_ok=True)
 
         rendered = 0
@@ -124,6 +173,16 @@ class GraphConsolidator:
                 deleted += 1
 
         indexes = self._render_index_pages(all_nodes)
+        if self._telemetry is not None:
+            self._telemetry.record_event(
+                "ToolFinished",
+                {
+                    "tool": "project_dirty_nodes",
+                    "rendered": rendered,
+                    "deleted": deleted,
+                    "indexes": indexes,
+                },
+            )
         return {
             "rendered": rendered,
             "deleted": deleted,

package/orchestrator/wiki/incremental_engine.py

@@ -4,8 +4,9 @@ from typing import Any, Dict
 
 
 class IncrementalEngine:
-    def __init__(self, cache_path: Path) -> None:
+    def __init__(self, cache_path: Path, telemetry_collector: Any | None = None) -> None:
         self.cache_path = cache_path
+        self._telemetry = telemetry_collector
 
     def _extract_nodes(self, loaded: Dict[str, Any]) -> Dict[str, Dict[str, Any]]:
         preferred = loaded.get("wiki_nodes")
@@ -37,6 +38,14 @@ class IncrementalEngine:
             return {"last_updated": "", "nodes": {}}
 
     def diff(self, incoming_nodes: Dict[str, Dict[str, Any]]) -> Dict[str, Any]:
+        if self._telemetry is not None:
+            self._telemetry.record_event(
+                "ToolStarted",
+                {
+                    "tool": "incremental_diff",
+                    "incoming_nodes": len(incoming_nodes),
+                },
+            )
         cached = self._load_cached_graph()
         cached_nodes = cached.get("nodes", {})
 
@@ -50,6 +59,19 @@ class IncrementalEngine:
                 dirty_nodes[node_key] = incoming
 
         deleted_nodes = [key for key in cached_nodes.keys() if key not in incoming_nodes]
+        if self._telemetry is not None:
+            cache_hits = max(0, len(incoming_nodes) - len(dirty_nodes))
+            cache_misses = len(dirty_nodes)
+            self._telemetry.record_event(
+                "ToolFinished",
+                {
+                    "tool": "incremental_diff",
+                    "dirty_nodes": len(dirty_nodes),
+                    "deleted_nodes": len(deleted_nodes),
+                },
+            )
+            self._telemetry.record_metric("cache_hits", float(cache_hits), unit="count")
+            self._telemetry.record_metric("cache_misses", float(cache_misses), unit="count")
         return {
             "dirty_nodes": dirty_nodes,
             "deleted_nodes": deleted_nodes,