@miller-tech/uap 1.109.1 → 1.111.0

package/tools/agents/scripts/anthropic_proxy.py

@@ -401,6 +401,22 @@ except Exception:  # pragma: no cover - optional middleware
     _TOOLCALL_NORMALIZER_OK = False
 
 
+def _record_project_telemetry(body, model, usage) -> None:
+    """Fail-open per-project routing/cost telemetry: derive the request's project
+    dir and append a task_outcomes row to its analytics DB so per-project
+    dashboards see the model calls this shared proxy makes. See
+    tools/agents/scripts/project_telemetry.py. Never raises."""
+    try:
+        import sys as _s
+        _d = os.path.dirname(os.path.abspath(__file__))
+        if _d not in _s.path:
+            _s.path.insert(0, _d)
+        import project_telemetry as _pt
+        _pt.record_from_request(body, model, usage)
+    except Exception:
+        pass
+
+
 def _toolcall_workdir_hint(messages: list, limit: int = 8000) -> str:
     """Recent text/tool_result content (capped) so derive_workdir can recover the
     real workdir echoed in command outputs even when the model's own tool-call
@@ -9364,6 +9380,7 @@ async def messages(request: Request):
             upstream_input = anthropic_resp.get("usage", {}).get("input_tokens", 0)
             if upstream_input > 0:
                 monitor.last_input_tokens = upstream_input
+            _record_project_telemetry(body, model, anthropic_resp.get("usage", {}))
             if PROXY_FORCE_NON_STREAM:
                 logger.info(
                     "FORCED NON-STREAM: served stream response via guarded non-stream path"
@@ -9803,6 +9820,7 @@ async def messages(request: Request):
         if upstream_input > 0:
             monitor.last_input_tokens = upstream_input
 
+        _record_project_telemetry(body, model, anthropic_resp.get("usage", {}))
         return anthropic_resp
 
 

package/tools/agents/scripts/project_telemetry.py

@@ -0,0 +1,183 @@
+"""Per-project routing/cost telemetry for the shared UAP anthropic proxy.
+
+The proxy is a shared singleton that serves *every* project, so per-project
+dashboards (which read ``<project>/agents/data/memory/model_analytics.db`` ->
+``task_outcomes``) never see the model calls the proxy actually makes. This
+module closes that gap: after each completed response the proxy calls
+``record_from_request`` with the request body + model + usage; we derive the
+project directory from absolute paths echoed in the request and append one
+``task_outcomes`` row to that project's analytics DB.
+
+Design constraints (it runs in the hot path of a live proxy):
+  * fully fail-open — never raise into the caller;
+  * bounded work — scans a capped slice of the request, does a few fs stat calls;
+  * schema-compatible with src/models/analytics.ts so the existing dashboard
+    reads it with no change. Local models are zero-cost, so ``cost=0`` and the
+    dashboard computes the counterfactual-vs-frontier saving itself.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import sqlite3
+from datetime import datetime, timezone
+
+# A UAP project root is identifiable by one of these child markers.
+_PROJECT_MARKERS = ("agents/data/memory", ".uap", ".git")
+_ABS_PATH_RE = re.compile(r"/(?:[\w.\-]+/){2,}[\w.\-]*")
+# Bounds so a huge request can never stall the proxy hot path.
+_MAX_SCAN_CHARS = 24_000
+_MAX_CANDIDATE_PATHS = 40
+_MAX_WALK_UP = 8
+
+_CREATE_SQL = (
+    "CREATE TABLE IF NOT EXISTS task_outcomes ("
+    "modelId TEXT, taskType TEXT, complexity TEXT, success INTEGER, "
+    "durationMs INTEGER, tokensIn INTEGER, tokensOut INTEGER, cost REAL, "
+    "taskId TEXT, timestamp TEXT)"
+)
+
+
+def _iter_text(body: dict):
+    """Yield string fragments from an Anthropic request body, capped in total."""
+    budget = _MAX_SCAN_CHARS
+    msgs = body.get("messages") or []
+    # Newest messages first — the current workdir shows up in recent tool output.
+    for msg in reversed(msgs):
+        content = msg.get("content") if isinstance(msg, dict) else None
+        if isinstance(content, str):
+            frag = content[:budget]
+            budget -= len(frag)
+            yield frag
+        elif isinstance(content, list):
+            for block in content:
+                if not isinstance(block, dict):
+                    continue
+                for key in ("text", "content", "input"):
+                    val = block.get(key)
+                    if isinstance(val, str):
+                        frag = val[:budget]
+                        budget -= len(frag)
+                        yield frag
+                    elif isinstance(val, (dict, list)):
+                        frag = str(val)[:budget]
+                        budget -= len(frag)
+                        yield frag
+                if budget <= 0:
+                    return
+        if budget <= 0:
+            return
+
+
+def _project_root_for(path: str) -> str | None:
+    """Walk up from ``path`` to the nearest dir carrying a UAP project marker."""
+    d = path if os.path.isdir(path) else os.path.dirname(path)
+    for _ in range(_MAX_WALK_UP):
+        if not d or d == "/":
+            break
+        for marker in _PROJECT_MARKERS:
+            if os.path.exists(os.path.join(d, marker)):
+                return d
+        d = os.path.dirname(d)
+    return None
+
+
+def derive_project_dir(body: dict) -> str | None:
+    """Best-effort project directory for a request, or None. Never raises."""
+    try:
+        counts: dict[str, int] = {}
+        seen = 0
+        for frag in _iter_text(body):
+            for m in _ABS_PATH_RE.finditer(frag):
+                if seen >= _MAX_CANDIDATE_PATHS:
+                    break
+                seen += 1
+                root = _project_root_for(m.group(0))
+                if root:
+                    counts[root] = counts.get(root, 0) + 1
+            if seen >= _MAX_CANDIDATE_PATHS:
+                break
+        if not counts:
+            return None
+        # Most-referenced project root wins; tie-break to the deepest path.
+        return max(counts.items(), key=lambda kv: (kv[1], len(kv[0])))[0]
+    except Exception:
+        return None
+
+
+def _analytics_db_path(project_dir: str) -> str:
+    return os.path.join(project_dir, "agents", "data", "memory", "model_analytics.db")
+
+
+def record_task_outcome(
+    project_dir: str,
+    model: str,
+    tokens_in: int,
+    tokens_out: int,
+    *,
+    duration_ms: int = 0,
+    cost: float = 0.0,
+    task_id: str = "",
+    task_type: str = "proxy",
+    complexity: str = "medium",
+    success: bool = True,
+) -> bool:
+    """Append one task_outcomes row to a project's analytics DB. Never raises.
+
+    Returns True on write, False if skipped (no dir, no tokens, or error).
+    """
+    try:
+        if not project_dir:
+            return False
+        db_dir = os.path.join(project_dir, "agents", "data", "memory")
+        if not os.path.isdir(db_dir):
+            # Only record where the project already has a memory dir — do not
+            # scaffold analytics into arbitrary derived directories.
+            return False
+        if (tokens_in or 0) <= 0 and (tokens_out or 0) <= 0:
+            return False
+        conn = sqlite3.connect(_analytics_db_path(project_dir), timeout=1.0)
+        try:
+            conn.execute(_CREATE_SQL)
+            conn.execute(
+                "INSERT INTO task_outcomes (modelId, taskType, complexity, success, "
+                "durationMs, tokensIn, tokensOut, cost, taskId, timestamp) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                (
+                    str(model or "unknown"),
+                    task_type,
+                    complexity,
+                    1 if success else 0,
+                    int(duration_ms or 0),
+                    int(tokens_in or 0),
+                    int(tokens_out or 0),
+                    float(cost or 0.0),
+                    str(task_id or ""),
+                    datetime.now(timezone.utc).isoformat(),
+                ),
+            )
+            conn.commit()
+        finally:
+            conn.close()
+        return True
+    except Exception:
+        return False
+
+
+def record_from_request(body: dict, model: str, usage: dict, *, task_id: str = "") -> bool:
+    """Proxy hot-path entry point: derive project + record. Never raises."""
+    try:
+        project_dir = derive_project_dir(body or {})
+        if not project_dir:
+            return False
+        u = usage or {}
+        return record_task_outcome(
+            project_dir,
+            model,
+            int(u.get("input_tokens") or 0),
+            int(u.get("output_tokens") or 0),
+            task_id=task_id,
+        )
+    except Exception:
+        return False

package/tools/agents/tests/test_project_telemetry.py

@@ -0,0 +1,107 @@
+"""Tests for project_telemetry: per-project proxy routing/cost telemetry."""
+import importlib.util
+import os
+import sqlite3
+import tempfile
+import unittest
+from pathlib import Path
+
+
+def _load():
+    p = Path(__file__).resolve().parents[1] / "scripts" / "project_telemetry.py"
+    spec = importlib.util.spec_from_file_location("project_telemetry", p)
+    m = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(m)
+    return m
+
+
+pt = _load()
+
+
+def _make_project(root: str) -> str:
+    os.makedirs(os.path.join(root, "agents", "data", "memory"), exist_ok=True)
+    return root
+
+
+class DeriveProjectDir(unittest.TestCase):
+    def test_finds_project_root_from_paths_in_messages(self):
+        with tempfile.TemporaryDirectory() as d:
+            proj = _make_project(os.path.join(d, "project-x"))
+            body = {"messages": [
+                {"role": "user", "content": "run the build"},
+                {"role": "assistant", "content": [
+                    {"type": "tool_use", "input": {"file_path": f"{proj}/src/app.ts"}},
+                ]},
+                {"role": "user", "content": [
+                    {"type": "tool_result", "content": f"wrote {proj}/src/app.ts and {proj}/README.md"},
+                ]},
+            ]}
+            self.assertEqual(pt.derive_project_dir(body), proj)
+
+    def test_returns_none_when_no_project_paths(self):
+        self.assertIsNone(pt.derive_project_dir({"messages": [
+            {"role": "user", "content": "hello, no paths here"}]}))
+
+    def test_fail_open_on_garbage(self):
+        for bad in (None, {}, {"messages": None}, {"messages": [1, 2, 3]}, "nope"):
+            self.assertIsNone(pt.derive_project_dir(bad if isinstance(bad, dict) else {"messages": bad}))
+
+
+class RecordTaskOutcome(unittest.TestCase):
+    def _count_and_sums(self, proj):
+        db = os.path.join(proj, "agents", "data", "memory", "model_analytics.db")
+        c = sqlite3.connect(db)
+        row = c.execute("SELECT COUNT(*), SUM(tokensIn), SUM(tokensOut), SUM(cost) FROM task_outcomes").fetchone()
+        c.close()
+        return row
+
+    def test_writes_dashboard_compatible_row(self):
+        with tempfile.TemporaryDirectory() as d:
+            proj = _make_project(os.path.join(d, "p"))
+            ok = pt.record_task_outcome(proj, "qwen36-a3b", 1000, 500)
+            self.assertTrue(ok)
+            n, ti, to, cost = self._count_and_sums(proj)
+            self.assertEqual(n, 1)
+            self.assertEqual(ti, 1000)
+            self.assertEqual(to, 500)
+            self.assertEqual(cost, 0.0)  # local model is zero-cost; dashboard computes counterfactual
+
+    def test_skips_when_no_memory_dir(self):
+        with tempfile.TemporaryDirectory() as d:
+            # no agents/data/memory created
+            self.assertFalse(pt.record_task_outcome(os.path.join(d, "bare"), "m", 100, 50))
+
+    def test_skips_when_zero_tokens(self):
+        with tempfile.TemporaryDirectory() as d:
+            proj = _make_project(os.path.join(d, "p"))
+            self.assertFalse(pt.record_task_outcome(proj, "m", 0, 0))
+
+    def test_appends_multiple_rows(self):
+        with tempfile.TemporaryDirectory() as d:
+            proj = _make_project(os.path.join(d, "p"))
+            pt.record_task_outcome(proj, "m", 10, 5)
+            pt.record_task_outcome(proj, "m", 20, 7)
+            n, ti, to, _ = self._count_and_sums(proj)
+            self.assertEqual((n, ti, to), (2, 30, 12))
+
+
+class RecordFromRequest(unittest.TestCase):
+    def test_end_to_end_derive_and_write(self):
+        with tempfile.TemporaryDirectory() as d:
+            proj = _make_project(os.path.join(d, "proj"))
+            body = {"messages": [{"role": "user", "content": [
+                {"type": "tool_result", "content": f"ls {proj}/src returned app.ts"}]}]}
+            ok = pt.record_from_request(body, "qwen36", {"input_tokens": 1234, "output_tokens": 88})
+            self.assertTrue(ok)
+            c = sqlite3.connect(os.path.join(proj, "agents", "data", "memory", "model_analytics.db"))
+            n = c.execute("SELECT COUNT(*) FROM task_outcomes WHERE modelId='qwen36'").fetchone()[0]
+            c.close()
+            self.assertEqual(n, 1)
+
+    def test_fail_open_returns_false_never_raises(self):
+        self.assertFalse(pt.record_from_request(None, "m", None))
+        self.assertFalse(pt.record_from_request({"messages": []}, "m", {"input_tokens": 5}))
+
+
+if __name__ == "__main__":
+    unittest.main()

package/web/dashboard.html

@@ -685,6 +685,16 @@
               renderLiveEvents();
             } catch {}
           };
+          // Full live snapshot (cross-process) — used when the WebSocket is down so
+          // SSE alone keeps the dashboard live rather than falling to slow polling. (C)
+          eventSource.addEventListener('snapshot', (e) => {
+            if (ws && ws.readyState === WebSocket.OPEN) return;
+            try {
+              render(JSON.parse(e.data));
+              document.getElementById('ws-status').className = 'status-dot connected';
+              document.getElementById('refresh-info').textContent = 'Live (SSE) - ' + new Date().toLocaleTimeString();
+            } catch {}
+          });
           eventSource.onerror = () => { eventSource.close(); setTimeout(connectSSE, 5000); };
         } catch {}
       }