@miller-tech/uap 1.129.3 → 1.129.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@miller-tech/uap",
-  "version": "1.129.3",
+  "version": "1.129.5",
   "description": "Autonomous AI agent memory system with CLAUDE.md protocol enforcement",
   "type": "module",
   "main": "dist/index.js",
@@ -21,7 +21,7 @@
     "start": "node dist/bin/cli.js",
     "test": "vitest",
     "test:ci": "vitest run",
-    "test:enforcers": "python3 -m unittest tools.agents.tests.test_enforcement_self_protect tools.agents.tests.test_schema_diff_gate tools.agents.tests.test_stream_telemetry tools.agents.tests.test_workdir_scope_enforcer tools.agents.tests.test_path_containment tools.agents.tests.test_path_normalizer_hardened tools.agents.tests.test_sandbox_tool_strip tools.agents.tests.test_proxy_env_loader tools.agents.tests.test_stream_required_tool tools.agents.tests.test_tool_call_wrapper_profiles tools.agents.tests.test_tool_convert_cache",
+    "test:enforcers": "python3 -m unittest tools.agents.tests.test_enforcement_self_protect tools.agents.tests.test_schema_diff_gate tools.agents.tests.test_stream_telemetry tools.agents.tests.test_project_telemetry_events tools.agents.tests.test_workdir_scope_enforcer tools.agents.tests.test_path_containment tools.agents.tests.test_path_normalizer_hardened tools.agents.tests.test_sandbox_tool_strip tools.agents.tests.test_proxy_env_loader tools.agents.tests.test_stream_required_tool tools.agents.tests.test_tool_call_wrapper_profiles tools.agents.tests.test_tool_convert_cache",
     "test:coverage": "vitest --coverage",
     "bench": "vitest --config vitest.bench.config.ts",
     "lint": "eslint src --ext .ts",

package/tools/agents/scripts/anthropic_proxy.py

@@ -536,18 +536,19 @@ except Exception:  # pragma: no cover - optional middleware
     _TOOLCALL_NORMALIZER_OK = False
 
 
-def _record_project_telemetry(body, model, usage) -> None:
+def _record_project_telemetry(body, model, usage, duration_ms: float = 0.0) -> None:
     """Fail-open per-project routing/cost telemetry: derive the request's project
-    dir and append a task_outcomes row to its analytics DB so per-project
-    dashboards see the model calls this shared proxy makes. See
-    tools/agents/scripts/project_telemetry.py. Never raises."""
+    dir and append a task_outcomes row (plus a dashboard live-feed event) to its
+    analytics/telemetry DBs so per-project dashboards see the model calls this
+    shared proxy makes. See tools/agents/scripts/project_telemetry.py. Never
+    raises."""
     try:
         import sys as _s
         _d = os.path.dirname(os.path.abspath(__file__))
         if _d not in _s.path:
             _s.path.insert(0, _d)
         import project_telemetry as _pt
-        _pt.record_from_request(body, model, usage)
+        _pt.record_from_request(body, model, usage, duration_ms=duration_ms)
     except Exception:
         pass
 
@@ -8930,6 +8931,7 @@ async def stream_anthropic_response(
     - Proper upstream response closure on client disconnect
     """
     msg_id = f"msg_{uuid.uuid4().hex[:24]}"
+    stream_started = time.monotonic()
 
     # message_start
     yield (
@@ -9257,16 +9259,40 @@ async def stream_anthropic_response(
     upstream_input = int(upstream_usage.get("prompt_tokens") or 0)
     if upstream_input > 0:
         monitor.last_input_tokens = upstream_input
+    final_output = int(upstream_usage.get("completion_tokens") or 0)
+    if final_output <= 0:
+        # Upstream sent no usable usage (older servers ignore stream_options):
+        # estimate from everything actually generated — text, tool-call
+        # arguments, and reasoning — at ~4 chars/token, never below the
+        # per-delta chunk count.
+        generated_chars = (
+            sum(len(c) for c in text_chunks)
+            # isinstance guard: the XML-recovery path stores a bare sentinel
+            # (tool_calls_by_index["_xml_recovered"] = True) in this map.
+            + sum(
+                len(tc.get("arguments", ""))
+                for tc in tool_calls_by_index.values()
+                if isinstance(tc, dict)
+            )
+            + sum(len(c) for c in reasoning_chunks)
+        )
+        final_output = max(output_tokens, (generated_chars + 3) // 4)
     usage_final = {
         "input_tokens": upstream_input or getattr(monitor, "last_input_tokens", 0) or 0,
-        "output_tokens": int(upstream_usage.get("completion_tokens") or 0) or output_tokens,
+        "output_tokens": final_output,
     }
     # After message_stop (never delays the client's final frame; a disconnect
     # parked exactly on that yield skips recording, matching the non-stream
     # "completed responses only" semantics) and off the event loop (a locked
     # analytics DB must not stall token delivery for concurrent sessions).
     try:
-        await asyncio.to_thread(_record_project_telemetry, anthropic_body, model, usage_final)
+        await asyncio.to_thread(
+            _record_project_telemetry,
+            anthropic_body,
+            model,
+            usage_final,
+            (time.monotonic() - stream_started) * 1000.0,
+        )
     except Exception:
         pass
 
@@ -9836,7 +9862,12 @@ async def messages(request: Request):
             upstream_input = anthropic_resp.get("usage", {}).get("input_tokens", 0)
             if upstream_input > 0:
                 monitor.last_input_tokens = upstream_input
-            _record_project_telemetry(body, model, anthropic_resp.get("usage", {}))
+            # Off the event loop: the recorder now writes TWO DBs (analytics +
+            # dashboard telemetry) with 2s busy timeouts — a lock stall here
+            # would block every concurrent session's turn.
+            await asyncio.to_thread(
+                _record_project_telemetry, body, model, anthropic_resp.get("usage", {})
+            )
             if PROXY_FORCE_NON_STREAM:
                 logger.info(
                     "FORCED NON-STREAM: served stream response via guarded non-stream path"
@@ -9877,6 +9908,10 @@ async def messages(request: Request):
 
     if is_stream:
         openai_body["stream"] = True
+        # Without this, OpenAI-compat upstreams (llama-server included) omit
+        # completion_tokens from the final stream chunk — the reason streamed
+        # tool-use turns recorded tokensOut=0 in per-project telemetry.
+        openai_body["stream_options"] = {"include_usage": True}
 
         # Retry upstream connection with backoff to handle
         # llama-server restarts gracefully instead of 500-ing to the client.
@@ -10276,7 +10311,10 @@ async def messages(request: Request):
         if upstream_input > 0:
             monitor.last_input_tokens = upstream_input
 
-        _record_project_telemetry(body, model, anthropic_resp.get("usage", {}))
+        # Off the event loop — see the guarded-non-stream call site.
+        await asyncio.to_thread(
+            _record_project_telemetry, body, model, anthropic_resp.get("usage", {})
+        )
         return anthropic_resp
 
 

package/tools/agents/scripts/project_telemetry.py

@@ -18,6 +18,7 @@ Design constraints (it runs in the hot path of a live proxy):
 
 from __future__ import annotations
 
+import json
 import os
 import re
 import sqlite3
@@ -171,19 +172,135 @@ def record_task_outcome(
         return False
 
 
-def record_from_request(body: dict, model: str, usage: dict, *, task_id: str = "") -> bool:
+# ── dashboard live-feed (telemetry.db) ─────────────────────────────────────
+# The Live Events / Performance panels read <project>/agents/data/memory/
+# telemetry.db, which is otherwise written only by the TS producers
+# (mcp-router/executor). Plain claude+proxy sessions never run those, so the
+# panels stayed empty forever. The proxy is the one process that sees every
+# turn — mirror the turn into the same store. Compression stats are NOT
+# fabricated here: the proxy does no compression, that panel is honestly
+# empty for plain sessions.
+#
+# DDL matches src/utils/telemetry-store.ts exactly (CREATE IF NOT EXISTS
+# no-ops when the TS side created the tables first; the shape only matters
+# when the proxy creates them for a fresh project).
+_TELEMETRY_CREATE_SQL = """
+CREATE TABLE IF NOT EXISTS perf_samples (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  metric TEXT NOT NULL,
+  duration_ms REAL NOT NULL,
+  ts TEXT NOT NULL DEFAULT (datetime('now'))
+);
+CREATE INDEX IF NOT EXISTS idx_perf_samples_metric ON perf_samples(metric);
+CREATE TABLE IF NOT EXISTS dashboard_events (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  ts TEXT NOT NULL DEFAULT (datetime('now')),
+  category TEXT NOT NULL,
+  type TEXT NOT NULL,
+  severity TEXT NOT NULL DEFAULT 'info',
+  title TEXT NOT NULL,
+  detail TEXT,
+  metadata TEXT
+);
+"""
+
+# NOTE: the TS side trims events harder (MAX_PERSISTED_EVENTS = 500); this
+# looser cap only matters for proxy-only projects where no TS producer runs.
+_EVENTS_KEEP = 5000
+_PERF_KEEP_PER_METRIC = 1000  # mirrors telemetry-store.ts PERF_SAMPLES_PER_METRIC
+# Best-effort trim trigger: global across projects and not thread-safe, so a
+# given DB trims roughly every 50*N-projects inserts — overshoot is bounded
+# by the DELETEs themselves, precision is not needed here.
+_trim_counter = 0
+
+
+def record_dashboard_turn(
+    project_dir: str,
+    model: str,
+    tokens_in: int,
+    tokens_out: int,
+    duration_ms: float = 0.0,
+) -> bool:
+    """Mirror one completed proxy turn into the project's telemetry.db so the
+    dashboard's Live Events and Performance panels move for plain
+    claude+proxy sessions. Never raises; a dropped event is strictly better
+    than a stalled turn."""
+    global _trim_counter
+    try:
+        db_dir = os.path.join(project_dir, "agents", "data", "memory")
+        if not os.path.isdir(db_dir):
+            return False
+        conn = sqlite3.connect(os.path.join(db_dir, "telemetry.db"), timeout=1.0)
+        try:
+            # Match the TS producers' settings so cross-process writes don't
+            # silently drop under lock contention (busy_timeout 2000: telemetry
+            # must never stall the hot path).
+            conn.execute("PRAGMA journal_mode = WAL")
+            conn.execute("PRAGMA synchronous = NORMAL")
+            conn.execute("PRAGMA busy_timeout = 2000")
+            conn.executescript(_TELEMETRY_CREATE_SQL)
+            conn.execute(
+                "INSERT INTO dashboard_events (category, type, severity, title, metadata) "
+                "VALUES ('model', 'turn', 'info', ?, ?)",
+                (
+                    f"{model}: {int(tokens_out or 0)} tok out / {int(tokens_in or 0)} in",
+                    json.dumps(
+                        {
+                            "model": str(model or "unknown"),
+                            "tokensIn": int(tokens_in or 0),
+                            "tokensOut": int(tokens_out or 0),
+                            "durationMs": round(float(duration_ms or 0), 1),
+                        }
+                    ),
+                ),
+            )
+            if duration_ms and duration_ms > 0:
+                conn.execute(
+                    "INSERT INTO perf_samples (metric, duration_ms) VALUES ('proxy_turn_ms', ?)",
+                    (float(duration_ms),),
+                )
+            _trim_counter += 1
+            if _trim_counter % 50 == 0:
+                conn.execute(
+                    "DELETE FROM dashboard_events WHERE id NOT IN "
+                    "(SELECT id FROM dashboard_events ORDER BY id DESC LIMIT ?)",
+                    (_EVENTS_KEEP,),
+                )
+                conn.execute(
+                    "DELETE FROM perf_samples WHERE metric = 'proxy_turn_ms' AND id NOT IN "
+                    "(SELECT id FROM perf_samples WHERE metric = 'proxy_turn_ms' "
+                    "ORDER BY id DESC LIMIT ?)",
+                    (_PERF_KEEP_PER_METRIC,),
+                )
+            conn.commit()
+        finally:
+            conn.close()
+        return True
+    except Exception:
+        return False
+
+
+def record_from_request(
+    body: dict, model: str, usage: dict, *, task_id: str = "", duration_ms: float = 0.0
+) -> bool:
     """Proxy hot-path entry point: derive project + record. Never raises."""
     try:
         project_dir = derive_project_dir(body or {})
         if not project_dir:
             return False
         u = usage or {}
-        return record_task_outcome(
+        tokens_in = int(u.get("input_tokens") or 0)
+        tokens_out = int(u.get("output_tokens") or 0)
+        wrote = record_task_outcome(
             project_dir,
             model,
-            int(u.get("input_tokens") or 0),
-            int(u.get("output_tokens") or 0),
+            tokens_in,
+            tokens_out,
+            duration_ms=int(duration_ms or 0),
             task_id=task_id,
         )
+        if wrote:
+            record_dashboard_turn(project_dir, model, tokens_in, tokens_out, duration_ms)
+        return wrote
     except Exception:
         return False

package/tools/agents/tests/test_project_telemetry_events.py

@@ -0,0 +1,95 @@
+"""record_from_request must feed the dashboard live panels, not just analytics.
+
+Live Events / Performance read <project>/agents/data/memory/telemetry.db,
+which only the TS producers wrote — plain claude+proxy sessions left those
+panels empty forever. The proxy now mirrors each completed turn there.
+"""
+import importlib.util
+import json
+import sqlite3
+import tempfile
+import unittest
+from pathlib import Path
+
+pt_path = Path(__file__).resolve().parents[1] / "scripts" / "project_telemetry.py"
+spec = importlib.util.spec_from_file_location("project_telemetry", pt_path)
+pt = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(pt)
+
+
+class ProjectTelemetryEventsTest(unittest.TestCase):
+    def setUp(self):
+        self._tmp = tempfile.TemporaryDirectory(prefix="pt-events-")
+        self.root = Path(self._tmp.name)
+        # A recordable UAP project: marker + existing memory dir (the writer
+        # must never scaffold into arbitrary derived directories).
+        (self.root / ".git").mkdir()
+        (self.root / "agents" / "data" / "memory").mkdir(parents=True)
+
+    def tearDown(self):
+        self._tmp.cleanup()
+
+    def _body(self):
+        # derive_project_dir walks absolute paths echoed in the request.
+        return {"messages": [{"role": "user", "content": f"edit {self.root}/src/main.rs"}]}
+
+    def test_turn_writes_analytics_event_and_perf_sample(self):
+        ok = pt.record_from_request(
+            self._body(),
+            "qwen-test",
+            {"input_tokens": 1000, "output_tokens": 50},
+            duration_ms=1234.5,
+        )
+        self.assertTrue(ok)
+        mem = self.root / "agents" / "data" / "memory"
+
+        con = sqlite3.connect(mem / "model_analytics.db")
+        row = con.execute(
+            "SELECT modelId, tokensIn, tokensOut, durationMs FROM task_outcomes"
+        ).fetchone()
+        con.close()
+        self.assertEqual(row, ("qwen-test", 1000, 50, 1234))
+
+        con = sqlite3.connect(mem / "telemetry.db")
+        ev = con.execute(
+            "SELECT category, type, severity, title, metadata FROM dashboard_events"
+        ).fetchone()
+        perf = con.execute("SELECT metric, duration_ms FROM perf_samples").fetchone()
+        con.close()
+        self.assertEqual(ev[0:3], ("model", "turn", "info"))
+        self.assertIn("qwen-test", ev[3])
+        meta = json.loads(ev[4])
+        self.assertEqual(meta["tokensIn"], 1000)
+        self.assertEqual(meta["tokensOut"], 50)
+        self.assertEqual(perf, ("proxy_turn_ms", 1234.5))
+
+    def test_schema_matches_ts_producer_shape(self):
+        # The TS side (telemetry-store.ts) CREATE IF NOT EXISTS must no-op on a
+        # DB the proxy created first — the column shape has to match exactly.
+        pt.record_from_request(self._body(), "m", {"input_tokens": 1, "output_tokens": 1})
+        con = sqlite3.connect(self.root / "agents" / "data" / "memory" / "telemetry.db")
+        cols = [r[1] for r in con.execute("PRAGMA table_info(dashboard_events)")]
+        perf_cols = [r[1] for r in con.execute("PRAGMA table_info(perf_samples)")]
+        con.close()
+        self.assertEqual(cols, ["id", "ts", "category", "type", "severity", "title", "detail", "metadata"])
+        self.assertEqual(perf_cols, ["id", "metric", "duration_ms", "ts"])
+
+    def test_never_scaffolds_memory_dir(self):
+        bare = Path(self._tmp.name) / "bare"
+        (bare / ".git").mkdir(parents=True)
+        body = {"messages": [{"role": "user", "content": f"see {bare}/x.txt"}]}
+        self.assertFalse(pt.record_from_request(body, "m", {"input_tokens": 1, "output_tokens": 1}))
+        self.assertFalse((bare / "agents").exists())
+
+    def test_zero_duration_skips_perf_sample_but_keeps_event(self):
+        pt.record_from_request(self._body(), "m", {"input_tokens": 5, "output_tokens": 5})
+        con = sqlite3.connect(self.root / "agents" / "data" / "memory" / "telemetry.db")
+        events = con.execute("SELECT count(*) FROM dashboard_events").fetchone()[0]
+        perf = con.execute("SELECT count(*) FROM perf_samples").fetchone()[0]
+        con.close()
+        self.assertEqual(events, 1)
+        self.assertEqual(perf, 0)
+
+
+if __name__ == "__main__":
+    unittest.main()

package/tools/agents/tests/test_stream_telemetry.py

@@ -4,8 +4,11 @@ Regression for the frozen-dashboard incident (2026-07-08): the only
 _record_project_telemetry call sites were on the non-stream / guarded
 non-stream paths, so streaming sessions (the client default) wrote no
 task_outcomes rows and per-project dashboards showed static data forever.
+Follow-up coverage: real output tokens on tool-use turns (upstream usage via
+stream_options.include_usage, char-estimate fallback) and turn duration.
 """
 import asyncio
+import json
 import importlib.util
 import unittest
 from pathlib import Path
@@ -30,24 +33,32 @@ class FakeOpenAIStream:
         pass
 
 
-OPENAI_LINES = [
+TEXT_TURN_LINES = [
     'data: {"choices": [{"delta": {"content": "hello"}, "finish_reason": null}]}',
     'data: {"choices": [{"delta": {}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 456, "completion_tokens": 5}}',
     "data: [DONE]",
 ]
 
+# A pure tool-call turn whose upstream sends NO usage (older servers ignore
+# stream_options.include_usage): output must be estimated from the generated
+# tool-call arguments, not recorded as 0.
+TOOL_TURN_NO_USAGE_LINES = [
+    'data: {"choices": [{"delta": {"tool_calls": [{"index": 0, "id": "call_1", "function": {"name": "Bash", "arguments": ""}}]}, "finish_reason": null}]}',
+    'data: {"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": "{\\"command\\": \\"echo hello world this is a long command string\\"}"}}]}, "finish_reason": null}]}',
+    'data: {"choices": [{"delta": {}, "finish_reason": "tool_calls"}]}',
+    "data: [DONE]",
+]
+
 
 class StreamTelemetryTest(unittest.TestCase):
-    def _drive(self, body):
+    def _drive(self, body, lines=TEXT_TURN_LINES):
         monitor = ap.SessionMonitor()
         monitor.last_input_tokens = 123
         recorded = []
         saved = ap._record_project_telemetry
-        ap._record_project_telemetry = lambda b, m, u: recorded.append((b, m, u))
+        ap._record_project_telemetry = lambda b, m, u, d=0.0: recorded.append((b, m, u, d))
         try:
-            gen = ap.stream_anthropic_response(
-                FakeOpenAIStream(OPENAI_LINES), "test-model", monitor, body
-            )
+            gen = ap.stream_anthropic_response(FakeOpenAIStream(lines), "test-model", monitor, body)
 
             async def consume():
                 return [frame async for frame in gen]
@@ -62,13 +73,37 @@ class StreamTelemetryTest(unittest.TestCase):
         frames, recorded = self._drive(body)
         self.assertTrue(any("message_stop" in f for f in frames))
         self.assertEqual(len(recorded), 1)
-        recorded_body, model, usage = recorded[0]
+        recorded_body, model, usage, duration_ms = recorded[0]
         self.assertIs(recorded_body, body)
         self.assertEqual(model, "test-model")
         # Upstream's final usage chunk wins over the request-side estimate and
         # the per-delta chunk counter (which misses tool-call turns).
         self.assertEqual(usage.get("input_tokens"), 456)
         self.assertEqual(usage.get("output_tokens"), 5)
+        self.assertGreaterEqual(duration_ms, 0)
+
+    def test_tool_turn_without_upstream_usage_estimates_output_tokens(self):
+        frames, recorded = self._drive({"messages": []}, TOOL_TURN_NO_USAGE_LINES)
+        self.assertEqual(len(recorded), 1)
+        _, _, usage, _ = recorded[0]
+        # ~60 chars of tool arguments -> well above zero at ~4 chars/token.
+        self.assertGreater(usage.get("output_tokens"), 5)
+
+    def test_xml_recovered_tool_turn_without_usage_still_records(self):
+        # The XML-recovery path plants a bare sentinel in tool_calls_by_index
+        # (_xml_recovered = True); the char-estimate fallback must not choke on
+        # it (regression: AttributeError after message_stop, telemetry lost).
+        xml = '<tool_call>{"name": "Bash", "arguments": {"command": "ls -la"}}</tool_call>'
+        lines = [
+            f'data: {json.dumps({"choices": [{"delta": {"content": xml}, "finish_reason": None}]})}',
+            'data: {"choices": [{"delta": {}, "finish_reason": "stop"}]}',
+            "data: [DONE]",
+        ]
+        frames, recorded = self._drive({"messages": []}, lines)
+        self.assertTrue(any("tool_use" in f for f in frames))  # recovery actually fired
+        self.assertEqual(len(recorded), 1)
+        _, _, usage, _ = recorded[0]
+        self.assertGreater(usage.get("output_tokens"), 0)
 
     def test_telemetry_recorded_after_client_visible_stream_end(self):
         # The recorder must not delay or reorder client frames: message_stop is