@miller-tech/uap 1.129.2 → 1.129.4

package/tools/agents/tests/test_project_telemetry_events.py

@@ -0,0 +1,95 @@
+"""record_from_request must feed the dashboard live panels, not just analytics.
+
+Live Events / Performance read <project>/agents/data/memory/telemetry.db,
+which only the TS producers wrote — plain claude+proxy sessions left those
+panels empty forever. The proxy now mirrors each completed turn there.
+"""
+import importlib.util
+import json
+import sqlite3
+import tempfile
+import unittest
+from pathlib import Path
+
+pt_path = Path(__file__).resolve().parents[1] / "scripts" / "project_telemetry.py"
+spec = importlib.util.spec_from_file_location("project_telemetry", pt_path)
+pt = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(pt)
+
+
+class ProjectTelemetryEventsTest(unittest.TestCase):
+    def setUp(self):
+        self._tmp = tempfile.TemporaryDirectory(prefix="pt-events-")
+        self.root = Path(self._tmp.name)
+        # A recordable UAP project: marker + existing memory dir (the writer
+        # must never scaffold into arbitrary derived directories).
+        (self.root / ".git").mkdir()
+        (self.root / "agents" / "data" / "memory").mkdir(parents=True)
+
+    def tearDown(self):
+        self._tmp.cleanup()
+
+    def _body(self):
+        # derive_project_dir walks absolute paths echoed in the request.
+        return {"messages": [{"role": "user", "content": f"edit {self.root}/src/main.rs"}]}
+
+    def test_turn_writes_analytics_event_and_perf_sample(self):
+        ok = pt.record_from_request(
+            self._body(),
+            "qwen-test",
+            {"input_tokens": 1000, "output_tokens": 50},
+            duration_ms=1234.5,
+        )
+        self.assertTrue(ok)
+        mem = self.root / "agents" / "data" / "memory"
+
+        con = sqlite3.connect(mem / "model_analytics.db")
+        row = con.execute(
+            "SELECT modelId, tokensIn, tokensOut, durationMs FROM task_outcomes"
+        ).fetchone()
+        con.close()
+        self.assertEqual(row, ("qwen-test", 1000, 50, 1234))
+
+        con = sqlite3.connect(mem / "telemetry.db")
+        ev = con.execute(
+            "SELECT category, type, severity, title, metadata FROM dashboard_events"
+        ).fetchone()
+        perf = con.execute("SELECT metric, duration_ms FROM perf_samples").fetchone()
+        con.close()
+        self.assertEqual(ev[0:3], ("model", "turn", "info"))
+        self.assertIn("qwen-test", ev[3])
+        meta = json.loads(ev[4])
+        self.assertEqual(meta["tokensIn"], 1000)
+        self.assertEqual(meta["tokensOut"], 50)
+        self.assertEqual(perf, ("proxy_turn_ms", 1234.5))
+
+    def test_schema_matches_ts_producer_shape(self):
+        # The TS side (telemetry-store.ts) CREATE IF NOT EXISTS must no-op on a
+        # DB the proxy created first — the column shape has to match exactly.
+        pt.record_from_request(self._body(), "m", {"input_tokens": 1, "output_tokens": 1})
+        con = sqlite3.connect(self.root / "agents" / "data" / "memory" / "telemetry.db")
+        cols = [r[1] for r in con.execute("PRAGMA table_info(dashboard_events)")]
+        perf_cols = [r[1] for r in con.execute("PRAGMA table_info(perf_samples)")]
+        con.close()
+        self.assertEqual(cols, ["id", "ts", "category", "type", "severity", "title", "detail", "metadata"])
+        self.assertEqual(perf_cols, ["id", "metric", "duration_ms", "ts"])
+
+    def test_never_scaffolds_memory_dir(self):
+        bare = Path(self._tmp.name) / "bare"
+        (bare / ".git").mkdir(parents=True)
+        body = {"messages": [{"role": "user", "content": f"see {bare}/x.txt"}]}
+        self.assertFalse(pt.record_from_request(body, "m", {"input_tokens": 1, "output_tokens": 1}))
+        self.assertFalse((bare / "agents").exists())
+
+    def test_zero_duration_skips_perf_sample_but_keeps_event(self):
+        pt.record_from_request(self._body(), "m", {"input_tokens": 5, "output_tokens": 5})
+        con = sqlite3.connect(self.root / "agents" / "data" / "memory" / "telemetry.db")
+        events = con.execute("SELECT count(*) FROM dashboard_events").fetchone()[0]
+        perf = con.execute("SELECT count(*) FROM perf_samples").fetchone()[0]
+        con.close()
+        self.assertEqual(events, 1)
+        self.assertEqual(perf, 0)
+
+
+if __name__ == "__main__":
+    unittest.main()

package/tools/agents/tests/test_stream_telemetry.py

@@ -4,8 +4,11 @@ Regression for the frozen-dashboard incident (2026-07-08): the only
 _record_project_telemetry call sites were on the non-stream / guarded
 non-stream paths, so streaming sessions (the client default) wrote no
 task_outcomes rows and per-project dashboards showed static data forever.
+Follow-up coverage: real output tokens on tool-use turns (upstream usage via
+stream_options.include_usage, char-estimate fallback) and turn duration.
 """
 import asyncio
+import json
 import importlib.util
 import unittest
 from pathlib import Path
@@ -30,24 +33,32 @@ class FakeOpenAIStream:
         pass
 
 
-OPENAI_LINES = [
+TEXT_TURN_LINES = [
     'data: {"choices": [{"delta": {"content": "hello"}, "finish_reason": null}]}',
     'data: {"choices": [{"delta": {}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 456, "completion_tokens": 5}}',
     "data: [DONE]",
 ]
 
+# A pure tool-call turn whose upstream sends NO usage (older servers ignore
+# stream_options.include_usage): output must be estimated from the generated
+# tool-call arguments, not recorded as 0.
+TOOL_TURN_NO_USAGE_LINES = [
+    'data: {"choices": [{"delta": {"tool_calls": [{"index": 0, "id": "call_1", "function": {"name": "Bash", "arguments": ""}}]}, "finish_reason": null}]}',
+    'data: {"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": "{\\"command\\": \\"echo hello world this is a long command string\\"}"}}]}, "finish_reason": null}]}',
+    'data: {"choices": [{"delta": {}, "finish_reason": "tool_calls"}]}',
+    "data: [DONE]",
+]
+
 
 class StreamTelemetryTest(unittest.TestCase):
-    def _drive(self, body):
+    def _drive(self, body, lines=TEXT_TURN_LINES):
         monitor = ap.SessionMonitor()
         monitor.last_input_tokens = 123
         recorded = []
         saved = ap._record_project_telemetry
-        ap._record_project_telemetry = lambda b, m, u: recorded.append((b, m, u))
+        ap._record_project_telemetry = lambda b, m, u, d=0.0: recorded.append((b, m, u, d))
         try:
-            gen = ap.stream_anthropic_response(
-                FakeOpenAIStream(OPENAI_LINES), "test-model", monitor, body
-            )
+            gen = ap.stream_anthropic_response(FakeOpenAIStream(lines), "test-model", monitor, body)
 
             async def consume():
                 return [frame async for frame in gen]
@@ -62,13 +73,37 @@ class StreamTelemetryTest(unittest.TestCase):
         frames, recorded = self._drive(body)
         self.assertTrue(any("message_stop" in f for f in frames))
         self.assertEqual(len(recorded), 1)
-        recorded_body, model, usage = recorded[0]
+        recorded_body, model, usage, duration_ms = recorded[0]
         self.assertIs(recorded_body, body)
         self.assertEqual(model, "test-model")
         # Upstream's final usage chunk wins over the request-side estimate and
         # the per-delta chunk counter (which misses tool-call turns).
         self.assertEqual(usage.get("input_tokens"), 456)
         self.assertEqual(usage.get("output_tokens"), 5)
+        self.assertGreaterEqual(duration_ms, 0)
+
+    def test_tool_turn_without_upstream_usage_estimates_output_tokens(self):
+        frames, recorded = self._drive({"messages": []}, TOOL_TURN_NO_USAGE_LINES)
+        self.assertEqual(len(recorded), 1)
+        _, _, usage, _ = recorded[0]
+        # ~60 chars of tool arguments -> well above zero at ~4 chars/token.
+        self.assertGreater(usage.get("output_tokens"), 5)
+
+    def test_xml_recovered_tool_turn_without_usage_still_records(self):
+        # The XML-recovery path plants a bare sentinel in tool_calls_by_index
+        # (_xml_recovered = True); the char-estimate fallback must not choke on
+        # it (regression: AttributeError after message_stop, telemetry lost).
+        xml = '<tool_call>{"name": "Bash", "arguments": {"command": "ls -la"}}</tool_call>'
+        lines = [
+            f'data: {json.dumps({"choices": [{"delta": {"content": xml}, "finish_reason": None}]})}',
+            'data: {"choices": [{"delta": {}, "finish_reason": "stop"}]}',
+            "data: [DONE]",
+        ]
+        frames, recorded = self._drive({"messages": []}, lines)
+        self.assertTrue(any("tool_use" in f for f in frames))  # recovery actually fired
+        self.assertEqual(len(recorded), 1)
+        _, _, usage, _ = recorded[0]
+        self.assertGreater(usage.get("output_tokens"), 0)
 
     def test_telemetry_recorded_after_client_visible_stream_end(self):
         # The recorder must not delay or reorder client frames: message_stop is