memindex 1.4.5 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/.qoder-plugin/plugin.json +1 -1
- package/README.md +8 -2
- package/dist/bug-report/index.js +94 -83
- package/dist/npx-cli/index.js +300 -299
- package/dist/opencode-plugin/index.js +40 -40
- package/dist/pi-extension/TYPEBOX-PROVENANCE.json +4 -4
- package/dist/pi-extension/index.js +9 -9
- package/integrations/dsh/lib/index.js +19 -19
- package/integrations/dsh/package.json +1 -1
- package/integrations/openclaw/install.sh +16 -10
- package/integrations/openclaw/openclaw.plugin.json +1 -5
- package/integrations/openclaw/test-install.sh +20 -13
- package/integrations/pi/TYPEBOX-PROVENANCE.json +4 -4
- package/package.json +5 -5
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/.codex-plugin/plugin.json +1 -1
- package/plugin/.qoder-plugin/plugin.json +1 -1
- package/plugin/hooks/codex-hooks.json +12 -0
- package/plugin/package.json +1 -1
- package/plugin/scripts/context-generator.cjs +65 -63
- package/plugin/scripts/context-generator.cjs.map +3 -3
- package/plugin/scripts/mcp-server.cjs +71 -70
- package/plugin/scripts/mcp-server.cjs.map +3 -3
- package/plugin/scripts/statusline.cjs +1 -1
- package/plugin/scripts/transcript-watcher.cjs +36 -35
- package/plugin/scripts/transcript-watcher.cjs.map +3 -3
- package/plugin/scripts/worker-service.cjs +714 -674
- package/plugin/scripts/worker-service.cjs.map +3 -3
- package/plugin/skills/cloud-sync/SKILL.md +5 -5
- package/plugin/skills/distill/SKILL.md +22 -0
- package/plugin/skills/how-it-works/SKILL.md +1 -1
- package/plugin/skills/how-it-works/onboarding-explainer.md +1 -1
- package/plugin/skills/import-chats/SKILL.md +22 -0
- package/plugin/skills/import-jira/SKILL.md +22 -0
- package/plugin/skills/import-reviews/SKILL.md +22 -0
- package/plugin/skills/mode-creator/SKILL.md +7 -1
- package/plugin/skills/pin/SKILL.md +26 -0
- package/plugin/skills/savings/SKILL.md +22 -0
- package/plugin/skills/timeline-report/SKILL.md +8 -8
- package/plugin/skills/weekly-digests/SKILL.md +5 -5
- package/plugin/skills/worker-request.mjs +45 -0
- package/plugin/sqlite/SessionStore.js +109 -109
- package/plugin/sqlite/SessionStore.js.map +3 -3
- package/plugin/sqlite/observations/files.js +12 -9
- package/plugin/sqlite/observations/files.js.map +3 -3
- package/plugin/ui/viewer-bundle.js +18 -18
- package/integrations/openclaw/skills/agent-cost-report/CHECKSUMS.txt +0 -50
- package/integrations/openclaw/skills/agent-cost-report/SKILL.md +0 -241
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/__init__.py +0 -1
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/behavior.py +0 -303
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/classify.py +0 -83
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/costs.py +0 -119
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/devices.py +0 -88
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/evidence.py +0 -160
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/labels.py +0 -115
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/measure.py +0 -89
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/mistakes.py +0 -310
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/patterns.py +0 -391
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/period.py +0 -98
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/prices.py +0 -95
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/render.py +0 -586
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/rollup.py +0 -355
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/rules.json +0 -4
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/rules.py +0 -76
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/sample.py +0 -45
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/snapshot.py +0 -53
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/sync.py +0 -79
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/transcripts.py +0 -126
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr/wins.py +0 -341
- package/integrations/openclaw/skills/agent-cost-report/scripts/acr.py +0 -219
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/_db.py +0 -49
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/_paths.py +0 -9
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/codex/sessions/2026/09/18/rollout-codex-1.jsonl +0 -4
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/make_report_fixtures.py +0 -46
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/prices_sample.json +0 -6
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/projects/-workspace-proj-a/sess-aaaa.jsonl +0 -8
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/projects/-workspace-proj-b/sess-bbbb.jsonl +0 -1
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/report-1day.json +0 -823
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/report-30day.json +0 -2213
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/report-7day.json +0 -1252
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/report-empty.json +0 -536
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/fixtures/report-session.json +0 -662
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_behavior.py +0 -299
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_devices.py +0 -99
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_labels.py +0 -96
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_measure.py +0 -120
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_pdf_windows.py +0 -54
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_period.py +0 -137
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_prices.py +0 -126
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_render.py +0 -238
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_render_encoding.py +0 -70
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_rollup.py +0 -263
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_snapshot.py +0 -63
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_sync.py +0 -52
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_transcript_encoding.py +0 -109
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_transcripts.py +0 -151
- package/integrations/openclaw/skills/agent-cost-report/scripts/tests/test_wins.py +0 -222
- package/plugin/skills/agent-cost-report/CHECKSUMS.txt +0 -50
- package/plugin/skills/agent-cost-report/SKILL.md +0 -241
- package/plugin/skills/agent-cost-report/scripts/acr/__init__.py +0 -1
- package/plugin/skills/agent-cost-report/scripts/acr/behavior.py +0 -303
- package/plugin/skills/agent-cost-report/scripts/acr/classify.py +0 -83
- package/plugin/skills/agent-cost-report/scripts/acr/costs.py +0 -119
- package/plugin/skills/agent-cost-report/scripts/acr/devices.py +0 -88
- package/plugin/skills/agent-cost-report/scripts/acr/evidence.py +0 -160
- package/plugin/skills/agent-cost-report/scripts/acr/labels.py +0 -115
- package/plugin/skills/agent-cost-report/scripts/acr/measure.py +0 -89
- package/plugin/skills/agent-cost-report/scripts/acr/mistakes.py +0 -310
- package/plugin/skills/agent-cost-report/scripts/acr/patterns.py +0 -391
- package/plugin/skills/agent-cost-report/scripts/acr/period.py +0 -98
- package/plugin/skills/agent-cost-report/scripts/acr/prices.py +0 -95
- package/plugin/skills/agent-cost-report/scripts/acr/render.py +0 -586
- package/plugin/skills/agent-cost-report/scripts/acr/rollup.py +0 -355
- package/plugin/skills/agent-cost-report/scripts/acr/rules.json +0 -4
- package/plugin/skills/agent-cost-report/scripts/acr/rules.py +0 -76
- package/plugin/skills/agent-cost-report/scripts/acr/sample.py +0 -45
- package/plugin/skills/agent-cost-report/scripts/acr/snapshot.py +0 -53
- package/plugin/skills/agent-cost-report/scripts/acr/sync.py +0 -79
- package/plugin/skills/agent-cost-report/scripts/acr/transcripts.py +0 -126
- package/plugin/skills/agent-cost-report/scripts/acr/wins.py +0 -341
- package/plugin/skills/agent-cost-report/scripts/acr.py +0 -219
- package/plugin/skills/agent-cost-report/scripts/tests/_db.py +0 -49
- package/plugin/skills/agent-cost-report/scripts/tests/_paths.py +0 -9
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/codex/sessions/2026/09/18/rollout-codex-1.jsonl +0 -4
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/make_report_fixtures.py +0 -46
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/prices_sample.json +0 -6
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/projects/-workspace-proj-a/sess-aaaa.jsonl +0 -8
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/projects/-workspace-proj-b/sess-bbbb.jsonl +0 -1
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/report-1day.json +0 -823
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/report-30day.json +0 -2213
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/report-7day.json +0 -1252
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/report-empty.json +0 -536
- package/plugin/skills/agent-cost-report/scripts/tests/fixtures/report-session.json +0 -662
- package/plugin/skills/agent-cost-report/scripts/tests/test_behavior.py +0 -299
- package/plugin/skills/agent-cost-report/scripts/tests/test_devices.py +0 -99
- package/plugin/skills/agent-cost-report/scripts/tests/test_labels.py +0 -96
- package/plugin/skills/agent-cost-report/scripts/tests/test_measure.py +0 -120
- package/plugin/skills/agent-cost-report/scripts/tests/test_pdf_windows.py +0 -54
- package/plugin/skills/agent-cost-report/scripts/tests/test_period.py +0 -137
- package/plugin/skills/agent-cost-report/scripts/tests/test_prices.py +0 -126
- package/plugin/skills/agent-cost-report/scripts/tests/test_render.py +0 -238
- package/plugin/skills/agent-cost-report/scripts/tests/test_render_encoding.py +0 -70
- package/plugin/skills/agent-cost-report/scripts/tests/test_rollup.py +0 -263
- package/plugin/skills/agent-cost-report/scripts/tests/test_snapshot.py +0 -63
- package/plugin/skills/agent-cost-report/scripts/tests/test_sync.py +0 -52
- package/plugin/skills/agent-cost-report/scripts/tests/test_transcript_encoding.py +0 -109
- package/plugin/skills/agent-cost-report/scripts/tests/test_transcripts.py +0 -151
- package/plugin/skills/agent-cost-report/scripts/tests/test_wins.py +0 -222
- package/plugin/skills/babysit/SKILL.md +0 -87
- package/plugin/skills/design-is/SKILL.md +0 -312
- package/plugin/skills/do/SKILL.md +0 -45
- package/plugin/skills/learn-codebase/SKILL.md +0 -21
- package/plugin/skills/make-plan/SKILL.md +0 -67
- package/plugin/skills/oh-my-issues/SKILL.md +0 -226
- package/plugin/skills/pathfinder/SKILL.md +0 -111
- package/plugin/skills/screenpipe/SKILL.md +0 -78
- package/plugin/skills/screenpipe/screenpipe.mjs +0 -196
- package/plugin/skills/smart-explore/SKILL.md +0 -193
- package/plugin/skills/standup/SKILL.md +0 -142
- package/plugin/skills/standup/agent-brief.md +0 -47
- package/plugin/skills/standup/standup.mjs +0 -586
- package/plugin/skills/what-the/SKILL.md +0 -6
- package/plugin/skills/wowerpoint/SKILL.md +0 -205
|
@@ -1,299 +0,0 @@
|
|
|
1
|
-
"""Phase 2B tests (plan 2B.8): tiny hand-made transcript lines with the field names verified in 2B.7."""
|
|
2
|
-
import json
|
|
3
|
-
import os
|
|
4
|
-
import shutil
|
|
5
|
-
import tempfile
|
|
6
|
-
import unittest
|
|
7
|
-
from unittest import mock
|
|
8
|
-
|
|
9
|
-
import _paths # noqa: F401
|
|
10
|
-
import _db
|
|
11
|
-
from acr import behavior, classify, costs, evidence, mistakes, patterns, period, prices, rules
|
|
12
|
-
|
|
13
|
-
PRICES = json.load(open(os.path.join(_paths.FIXTURES, "prices_sample.json")))
|
|
14
|
-
W = period.resolve_window("2026-09-18", "2026-09-26")
|
|
15
|
-
S_MS, E_MS = W.start_epoch_ms, W.end_epoch_ms
|
|
16
|
-
BASE = S_MS + 2 * 86_400_000 + 10 * 3600_000
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
def iso(ms):
|
|
20
|
-
import datetime as dt
|
|
21
|
-
return dt.datetime.fromtimestamp(ms / 1000, dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
USAGE = dict(input_tokens=1000, output_tokens=100, cache_creation_input_tokens=0, cache_read_input_tokens=0,
|
|
25
|
-
cache_creation=dict(ephemeral_5m_input_tokens=0, ephemeral_1h_input_tokens=0))
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
class Tx:
|
|
29
|
-
"""Builds one session's jsonl lines."""
|
|
30
|
-
def __init__(self, sid="s1", entrypoint="cli"):
|
|
31
|
-
self.sid, self.entry, self.lines, self.n = sid, entrypoint, [], 0
|
|
32
|
-
|
|
33
|
-
def _line(self, typ, ms, message, **kw):
|
|
34
|
-
self.n += 1
|
|
35
|
-
d = dict(type=typ, timestamp=iso(ms), sessionId=self.sid, cwd="/w/p", entrypoint=self.entry, isSidechain=False, uuid=f"u{self.n}", message=message, **kw)
|
|
36
|
-
self.lines.append(json.dumps(d)); return d
|
|
37
|
-
|
|
38
|
-
def user(self, ms, text, **kw):
|
|
39
|
-
return self._line("user", ms, dict(role="user", content=text), **kw)
|
|
40
|
-
|
|
41
|
-
def assistant(self, ms, text=None, tools=(), model="claude-fable-5-1", usage=USAGE, mid=None):
|
|
42
|
-
content = ([dict(type="text", text=text)] if text else []) + [dict(type="tool_use", id=t[0], name=t[1], input=t[2]) for t in tools]
|
|
43
|
-
return self._line("assistant", ms, dict(id=mid or f"m{self.n + 1}", model=model, usage=usage, content=content), requestId=f"r{self.n + 1}")
|
|
44
|
-
|
|
45
|
-
def result(self, ms, tool_use_id, content, is_error=False):
|
|
46
|
-
return self._line("user", ms, dict(role="user", content=[dict(type="tool_result", tool_use_id=tool_use_id, is_error=is_error, content=content)]))
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
class Base(unittest.TestCase):
|
|
50
|
-
def setUp(self):
|
|
51
|
-
self.tmp = tempfile.mkdtemp(); self.pat = os.path.join(self.tmp, "projects", "**", "*.jsonl")
|
|
52
|
-
os.makedirs(os.path.join(self.tmp, "projects", "-w-p"))
|
|
53
|
-
self.dbp = os.path.join(self.tmp, "snap.db"); self.db = _db.make_db(self.dbp)
|
|
54
|
-
self.pricer = costs.Pricer(PRICES)
|
|
55
|
-
|
|
56
|
-
def tearDown(self):
|
|
57
|
-
self.db.close(); shutil.rmtree(self.tmp)
|
|
58
|
-
|
|
59
|
-
def write(self, *txs):
|
|
60
|
-
for t in txs:
|
|
61
|
-
with open(os.path.join(self.tmp, "projects", "-w-p", f"{t.sid}.jsonl"), "w") as fh: fh.write("\n".join(t.lines) + "\n")
|
|
62
|
-
|
|
63
|
-
def scan(self):
|
|
64
|
-
sessions, _ = behavior.scan(S_MS, E_MS, pattern=self.pat)
|
|
65
|
-
for S in sessions.values(): behavior.tag_user_turns(S)
|
|
66
|
-
behavior.price_turns(sessions, self.pricer)
|
|
67
|
-
return sessions
|
|
68
|
-
|
|
69
|
-
def run_all(self, items=None, projects=None, tagger=True, classify=None):
|
|
70
|
-
scope = evidence.Scope("period", S=S_MS, E=E_MS)
|
|
71
|
-
items = items if items is not None else []
|
|
72
|
-
return mistakes.run(self.db, scope, W.block(), self.pricer, projects or {}, items, glob_pattern=self.pat, tagger=tagger, classify=classify)
|
|
73
|
-
|
|
74
|
-
def item(self, cs, usd=10.0):
|
|
75
|
-
return dict(content_session_id=cs, attributed_usd=usd, failure_signals=[], failure_type="", status="completed", trivial=False)
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
class Tagger(Base):
|
|
79
|
-
def test_every_agent_marker_tags_bot(self):
|
|
80
|
-
for marker in ("STEER from Sam: stop", "CORRECTION from lead: x", "[Cross-session idle notice] hi", "<system-reminder>x", "You are a planner", "/do plan.md", "/make-plan x", "<command-message>do</command-message>"):
|
|
81
|
-
self.assertEqual(behavior.tag_prompt_row(marker), "bot", marker)
|
|
82
|
-
self.assertEqual(behavior.tag_prompt_row("why is this broken again"), "human")
|
|
83
|
-
|
|
84
|
-
def test_headless_and_relayed_sessions_are_bot(self):
|
|
85
|
-
a = Tx("a", "sdk-cli"); a.user(BASE, "fix the bug"); a.assistant(BASE + 1000, "ok")
|
|
86
|
-
b = Tx("b", "cli"); b.user(BASE, "You are one lane of a plan"); b.user(BASE + 5000, "STOP DOING THAT RIGHT NOW"); b.assistant(BASE + 6000, "ok")
|
|
87
|
-
c = Tx("c", "cli"); c.user(BASE, "please fix the build"); c.assistant(BASE + 1000, "ok")
|
|
88
|
-
self.write(a, b, c); s = self.scan()
|
|
89
|
-
self.assertEqual([s["a"]["tag"], s["b"]["tag"], s["c"]["tag"]], ["agent_relayed", "agent_relayed", "user_direct"])
|
|
90
|
-
self.assertEqual({u["author"] for u in s["b"]["users"]}, {"bot"})
|
|
91
|
-
self.assertEqual(s["c"]["users"][0]["author"], "human")
|
|
92
|
-
|
|
93
|
-
def test_one_marker_mid_session_does_not_hide_the_user(self):
|
|
94
|
-
a = Tx("a", "cli"); a.user(BASE, "fix the build"); a.user(BASE + 5000, "<command-name>/clear</command-name>"); a.user(BASE + 9000, "why is this broken again"); a.assistant(BASE + 10000, "ok")
|
|
95
|
-
self.write(a); s = self.scan()
|
|
96
|
-
self.assertEqual(s["a"]["tag"], "user_direct"); self.assertEqual([u["author"] for u in s["a"]["users"]], ["human", "bot", "human"])
|
|
97
|
-
block, spend, _ = self.run_all(items=[self.item("a")])
|
|
98
|
-
self.assertEqual(spend["mistakes_episodes_n"], 1)
|
|
99
|
-
b = Tx("b", "cli"); b.user(BASE, "<system-reminder>x"); b.user(BASE + 5000, "[Cross-session idle notice] y"); b.assistant(BASE + 6000, "ok")
|
|
100
|
-
self.write(b); s = self.scan(); self.assertEqual(s["b"]["tag"], "agent_relayed")
|
|
101
|
-
|
|
102
|
-
def test_scrub_redacts_key_value_credentials(self):
|
|
103
|
-
t = behavior.scrub("log in with password=hunter2 then Token: abc123 and api_key=\"q9\" ok", 300)
|
|
104
|
-
self.assertNotIn("hunter2", t); self.assertNotIn("abc123", t); self.assertNotIn("q9", t)
|
|
105
|
-
self.assertEqual(t, "log in with password=[redacted] then Token: [redacted] and api_key=[redacted] ok")
|
|
106
|
-
self.assertEqual(behavior.scrub("the token limit was hit", 100), "the token limit was hit")
|
|
107
|
-
|
|
108
|
-
def test_tagger_off_renders_unavailable(self):
|
|
109
|
-
a = Tx("a"); a.user(BASE, "this is broken again"); a.assistant(BASE + 1000, "ok")
|
|
110
|
-
self.write(a); block, spend, _ = self.run_all(tagger=False)
|
|
111
|
-
self.assertEqual(block["author_tags"], mistakes.UNTAGGED); self.assertEqual(block["episodes"], []); self.assertFalse(block["tagger_ran"])
|
|
112
|
-
self.assertEqual(block["human_message_counts"], mistakes.UNTAGGED)
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
class Episodes(Base):
|
|
116
|
-
def test_bot_tagged_anger_never_starts_an_episode(self):
|
|
117
|
-
a = Tx("a", "sdk-cli"); a.user(BASE, "WHAT IS THIS MESS"); a.assistant(BASE + 1000, "ok")
|
|
118
|
-
self.write(a); block, spend, _ = self.run_all()
|
|
119
|
-
self.assertEqual(block["episodes"], []); self.assertEqual(spend["mistakes_episodes_n"], 0)
|
|
120
|
-
|
|
121
|
-
def test_gap_45_minutes_groups(self):
|
|
122
|
-
msgs = {"s": [dict(ts_ms=BASE, text="stop", author="human", source="t"), dict(ts_ms=BASE + 40 * 60_000, text="stop", author="human", source="t")]}
|
|
123
|
-
self.assertEqual(len(behavior.find_episodes(msgs)), 1)
|
|
124
|
-
msgs["s"][1]["ts_ms"] = BASE + 50 * 60_000
|
|
125
|
-
self.assertEqual(len(behavior.find_episodes(msgs)), 2)
|
|
126
|
-
|
|
127
|
-
def test_wasted_window_caps_at_6_hours_and_stays_in_session(self):
|
|
128
|
-
a = Tx("a"); a.user(BASE - 8 * 3600_000, "do the thing")
|
|
129
|
-
for i in range(8): a.assistant(BASE - (7 - i) * 3600_000, "working", tools=[(f"t{i}", "Bash", dict(command="ls"))])
|
|
130
|
-
a.user(BASE, "why did you break it again")
|
|
131
|
-
b = Tx("b"); b.user(BASE - 3600_000, "x"); b.assistant(BASE - 1800_000, "y")
|
|
132
|
-
self.write(a, b); block, spend, _ = self.run_all(items=[self.item("a"), self.item("b")])
|
|
133
|
-
ep = block["episodes"][0]
|
|
134
|
-
self.assertEqual(ep["cost_status"], "measured_tokens")
|
|
135
|
-
s = self.scan(); wasted, redo, _ = behavior.episode_windows(dict(session="a", first_ts=BASE, last_ts=BASE, messages=[{}]), s["a"], {"a": [dict(ts_ms=BASE - 8 * 3600_000, author="human")]})
|
|
136
|
-
self.assertEqual(len(wasted), 6)
|
|
137
|
-
self.assertTrue(all(T["key"] in s["a"]["turns"] for T in wasted))
|
|
138
|
-
self.assertGreaterEqual(block["union_high_usd"], block["union_low_usd"])
|
|
139
|
-
|
|
140
|
-
def test_unmeasured_episode_has_null_dollars(self):
|
|
141
|
-
_db.add_prompt(self.db, "mac1", 1, "WHAT IS A CODEBASE MAP", BASE, device="dev-uuid"); self.db.commit()
|
|
142
|
-
block, spend, _ = self.run_all(projects={"mac1": "p"})
|
|
143
|
-
ep = block["episodes"][0]
|
|
144
|
-
self.assertEqual((ep["cost_status"], ep["low_usd"], ep["high_usd"]), ("unmeasured", None, None))
|
|
145
|
-
self.assertEqual(spend["mistakes_unmeasured_n"], 1)
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
def test_measured_episode_records_observed_model_and_day_example_carries_episode_id(self):
|
|
149
|
-
a = Tx("a"); a.user(BASE - 3600_000, "do the thing"); a.assistant(BASE - 1800_000, "working", tools=[("t1", "Bash", dict(command="ls"))]); a.user(BASE, "why did you break it again")
|
|
150
|
-
self.write(a); block, _, by_day = self.run_all(items=[self.item("a")])
|
|
151
|
-
ep = block["episodes"][0]
|
|
152
|
-
self.assertEqual((ep["model_status"], ep["model"]), ("observed", prices.norm("claude-fable-5-1")))
|
|
153
|
-
ex = next(d for d in by_day if d["day_pt"] == ep["day_pt"])["top_examples"][0]
|
|
154
|
-
self.assertEqual(ex["episode_id"], ep["episode_id"]); self.assertTrue(ex["mistake_id"].startswith("MK-"))
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
class Detectors(Base):
|
|
158
|
-
def test_model_tile_prices_difference_and_assumed_not_counted(self):
|
|
159
|
-
a = Tx("a"); a.user(BASE, "x"); a.assistant(BASE + 1000, "y", model="gpt-5-codex"); a.assistant(BASE + 2000, "z", model=None)
|
|
160
|
-
self.write(a); block, spend, _ = self.run_all(items=[self.item("a")])
|
|
161
|
-
p3 = next(p for p in block["patterns"] if p["key"] == "P3_wrong_model")
|
|
162
|
-
self.assertEqual(p3["count"], 1); self.assertEqual(p3["model_not_logged"], 1)
|
|
163
|
-
self.assertEqual(p3["delta_usd"], 0.0)
|
|
164
|
-
pr = costs.Pricer(dict(models={"anthropic/claude-fable-5.1": PRICES["models"]["anthropic/claude-fable-5.1"], "openai/gpt-5-codex": dict(input=20.0, output=100.0, cache_read=2.0, cache_write=25.0, cache_write_1h=40.0)}))
|
|
165
|
-
s = self.scan(); behavior.price_turns(s, pr); f, _ = patterns.detect_p3(s["a"], "a", "anthropic/claude-fable-5.1", pr)
|
|
166
|
-
row = behavior._usage_row(USAGE)
|
|
167
|
-
self.assertEqual(f[0]["delta_x1e6"], costs.api_equiv_x1e6(row, pr.rate("gpt-5-codex")) - costs.api_equiv_x1e6(row, pr.rate("anthropic/claude-fable-5.1")))
|
|
168
|
-
|
|
169
|
-
def test_send_without_approval_is_incident_times_recipients_no_dollars(self):
|
|
170
|
-
a = Tx("a"); a.user(BASE, "draft the email"); a.assistant(BASE + 1000, "sending", tools=[("t1", "mcp__claude_ai_Gmail__send_message", dict(to=["a@x.com", "b@x.com"], body="hi"))])
|
|
171
|
-
self.write(a); block, spend, _ = self.run_all(items=[self.item("a")])
|
|
172
|
-
self.assertEqual(block["outbound"], dict(incidents=1, recipients=2, user_minutes=None))
|
|
173
|
-
p9 = next(p for p in block["patterns"] if p["key"] == "P9_bad_outbound"); self.assertIsNone(p9["low_usd"])
|
|
174
|
-
self.assertEqual(spend["mistakes_estimated_usd"], 0.0)
|
|
175
|
-
|
|
176
|
-
def test_fixed_with_passing_pytest_not_flagged_without_is(self):
|
|
177
|
-
a = Tx("a"); a.user(BASE, "fix it"); a.assistant(BASE + 1000, None, tools=[("t1", "Edit", dict(file_path="/w/p/x.py"))]); a.result(BASE + 2000, "t1", "ok")
|
|
178
|
-
a.assistant(BASE + 3000, None, tools=[("t2", "Bash", dict(command="pytest -q"))]); a.result(BASE + 4000, "t2", "3 passed in 0.1s")
|
|
179
|
-
a.assistant(BASE + 5000, "Fixed, all tests pass."); a.user(BASE + 6000, "thanks")
|
|
180
|
-
b = Tx("b"); b.user(BASE, "fix it"); b.assistant(BASE + 1000, None, tools=[("t1", "Edit", dict(file_path="/w/p/x.py"))]); b.result(BASE + 2000, "t1", "ok")
|
|
181
|
-
b.assistant(BASE + 3000, "Fixed, all tests pass."); b.user(BASE + 6000, "thanks")
|
|
182
|
-
self.write(a, b); s = self.scan()
|
|
183
|
-
self.assertEqual(patterns.detect_p7(s["a"], "a"), [])
|
|
184
|
-
self.assertEqual(len(patterns.detect_p7(s["b"], "b")), 1)
|
|
185
|
-
|
|
186
|
-
def test_claimed_skill_without_skill_call_is_p6(self):
|
|
187
|
-
a = Tx("a"); a.user(BASE, "prime"); a.assistant(BASE + 1000, "I ran /learn-codebase and built the codebase map."); a.user(BASE + 2000, "ok")
|
|
188
|
-
self.write(a); s = self.scan()
|
|
189
|
-
f = patterns.detect_p6(s["a"], "a"); self.assertEqual(len(f), 1); self.assertTrue(f[0]["trust_flag"])
|
|
190
|
-
|
|
191
|
-
def test_s1_error_then_similar_retry_that_succeeds(self):
|
|
192
|
-
a = Tx("a"); a.user(BASE, "run"); a.assistant(BASE + 1000, None, tools=[("t1", "Bash", dict(command="npm test"))]); a.result(BASE + 2000, "t1", "Error: x", is_error=True)
|
|
193
|
-
a.assistant(BASE + 3000, None, tools=[("t2", "Bash", dict(command="npm test "))]); a.result(BASE + 4000, "t2", "passed")
|
|
194
|
-
self.write(a); s = self.scan(); f, denials, st = patterns.detect_s1(s["a"], "a")
|
|
195
|
-
self.assertEqual((len(f), st["error_chains"], len(f[0]["recovery_keys"]), f[0]["wasted_extra"]), (1, 1, 1, []))
|
|
196
|
-
block, spend, _ = self.run_all(items=[self.item("a")])
|
|
197
|
-
self.assertLess(block["union_low_usd"], block["union_high_usd"])
|
|
198
|
-
|
|
199
|
-
def test_permission_denial_not_priced(self):
|
|
200
|
-
a = Tx("a"); a.user(BASE, "run"); a.assistant(BASE + 1000, None, tools=[("t1", "Bash", dict(command="curl x"))])
|
|
201
|
-
a.result(BASE + 2000, "t1", "Permission to use Bash with command curl x has been denied.", is_error=True)
|
|
202
|
-
self.write(a); block, spend, _ = self.run_all(items=[self.item("a")])
|
|
203
|
-
self.assertEqual(block["permission_denials"]["count"], 1); self.assertEqual(spend["mistakes_estimated_usd"], 0.0)
|
|
204
|
-
|
|
205
|
-
def test_report_labels_are_not_hedges(self):
|
|
206
|
-
a = Tx("a"); a.user(BASE, "x"); a.assistant(BASE + 1000, "Spend is $12.00 ESTIMATED at list prices, low confidence; measured spend unavailable."); a.user(BASE + 2000, "ok")
|
|
207
|
-
self.write(a); s = self.scan(); f, unknown = patterns.detect_s2(s["a"], "a")
|
|
208
|
-
self.assertEqual((f, unknown), ([], 0))
|
|
209
|
-
|
|
210
|
-
def test_hedge_with_answer_available_counts(self):
|
|
211
|
-
a = Tx("a"); a.user(BASE, "x"); a.assistant(BASE + 1000, None, tools=[("t1", "Bash", dict(command="cat /w/p/x.py"))]); a.result(BASE + 2000, "t1", "/w/p/x.py: def f(): return 1")
|
|
212
|
-
a.assistant(BASE + 3000, "It should be fixed in /w/p/x.py, I think."); a.user(BASE + 4000, "is it?")
|
|
213
|
-
self.write(a); s = self.scan(); f, _ = patterns.detect_s2(s["a"], "a")
|
|
214
|
-
self.assertEqual(len(f), 1); self.assertTrue(f[0]["caused_followup"])
|
|
215
|
-
|
|
216
|
-
def test_union_never_exceeds_agent_cost_and_line_items_share_it(self):
|
|
217
|
-
a = Tx("a"); a.user(BASE, "x")
|
|
218
|
-
for i in range(3): a.assistant(BASE + 1000 * (i + 1), "Done and verified." if i == 2 else None, tools=[(f"t{i}", "Bash", dict(command="ls"))] if i < 2 else ())
|
|
219
|
-
a.user(BASE + 9000, "why did you say done, it's broken again")
|
|
220
|
-
self.write(a); it = self.item("a", usd=99.0); block, spend, by_day = self.run_all(items=[it])
|
|
221
|
-
total = costs.usd(sum(T["x1e6"] for S in self.scan().values() for T in S["turns"].values()))
|
|
222
|
-
self.assertLessEqual(block["union_low_usd"], total); self.assertEqual(it["wasted_cost"], spend["mistakes_estimated_usd"])
|
|
223
|
-
self.assertEqual(round(sum(d["usd"] for d in by_day), 2), spend["mistakes_estimated_usd"]); self.assertEqual(len(by_day), 8)
|
|
224
|
-
|
|
225
|
-
def test_secret_leaves_no_trace(self):
|
|
226
|
-
a = Tx("a"); a.user(BASE, "here is sk-or-abcdefghijklmnop1234 use it"); a.assistant(BASE + 1000, "Blocked on the user: paste the key sk-or-abcdefghijklmnop1234"); a.user(BASE + 2000, "I already gave you the key")
|
|
227
|
-
self.write(a); block, spend, _ = self.run_all(items=[self.item("a")])
|
|
228
|
-
self.assertNotIn("sk-or-abc", json.dumps(block, default=str))
|
|
229
|
-
p1 = next(p for p in block["patterns"] if p["key"] == "P1_invented_gates"); self.assertGreaterEqual(p1["count"], 1)
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
class Classifier(unittest.TestCase):
|
|
233
|
-
def test_stops_at_cap(self):
|
|
234
|
-
rate = dict(input_usd_per_mtok=1000.0, output_usd_per_mtok=5000.0)
|
|
235
|
-
calls = []
|
|
236
|
-
def fake(model, prompt, key): calls.append(prompt); return dict(label="yes", pattern="P1", reason="r"), 1000, 100
|
|
237
|
-
cands = [dict(id=str(i), kind="P1_invented_gates", text="x") for i in range(50)]
|
|
238
|
-
labels, summary = classify.run(cands, rate, cap_usd=2.0, key="k", call=fake)
|
|
239
|
-
self.assertLessEqual(summary["spend_usd"], 2.0 + 1.6); self.assertLess(len(labels), 50); self.assertTrue(summary["sampled_n"] > 0)
|
|
240
|
-
with self.assertRaises(classify.ClassifierError): classify.run(cands, rate, key="", call=fake)
|
|
241
|
-
|
|
242
|
-
def test_cap_is_a_hard_limit_not_rounded_to_cents(self):
|
|
243
|
-
rate = dict(input_usd_per_mtok=1.0, output_usd_per_mtok=10.0)
|
|
244
|
-
def fake(model, prompt, key): return dict(label="yes", pattern="P1", reason="r"), 1000, 100
|
|
245
|
-
cands = [dict(id=str(i), kind="P1_invented_gates", text="x") for i in range(20)]
|
|
246
|
-
labels, summary = classify.run(cands, rate, cap_usd=0.005, key="k", call=fake)
|
|
247
|
-
self.assertEqual(len(labels), 2); self.assertTrue(summary["stopped_at_cap"]); self.assertLessEqual(summary["spend_usd"], 0.005)
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
class OwnerIdentity(unittest.TestCase):
|
|
251
|
-
def test_only_sender_fields_count_as_sent_as_the_user(self):
|
|
252
|
-
with mock.patch.object(patterns, "OWNER_EMAIL", "me@example.com"):
|
|
253
|
-
self.assertTrue(patterns.sent_as_owner({"from": "Me <ME@example.com>", "to": ["a@example.org"]}))
|
|
254
|
-
self.assertFalse(patterns.sent_as_owner({"from": "team-me@example.com"}))
|
|
255
|
-
self.assertFalse(patterns.sent_as_owner({"sender": "me@example.com.evil.test"}))
|
|
256
|
-
self.assertFalse(patterns.sent_as_owner({"to": ["me@example.com"], "body": "reply to me@example.com"}))
|
|
257
|
-
self.assertFalse(patterns.sent_as_owner("from: me@example.com"))
|
|
258
|
-
self.assertFalse(patterns.sent_as_owner({"from": "me@example.com"}))
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
class ShippedRules(unittest.TestCase):
|
|
262
|
-
def test_shipped_seed_is_empty_and_any_owner_dates_a_header(self):
|
|
263
|
-
self.assertEqual(rules.seed_rules(), [])
|
|
264
|
-
for header in ("# Ask first (HARD 2026-09-18)", "# Ask first (HARD — Sam 2026-09-18)"):
|
|
265
|
-
m = rules.HARD_HEADER.search(header); self.assertIsNotNone(m, header); self.assertEqual(m.group("date"), "2026-09-18")
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
class Rules(unittest.TestCase):
|
|
269
|
-
SEED = [dict(rule_key="no_invented_gates", name="No invented gates", landed_pt="2026-09-10", pattern="invented_gates_asking")]
|
|
270
|
-
|
|
271
|
-
def setUp(self):
|
|
272
|
-
patcher = mock.patch.object(rules, "seed_rules", return_value=self.SEED); patcher.start(); self.addCleanup(patcher.stop)
|
|
273
|
-
|
|
274
|
-
def _msgs(self, n_before, n_after, landed, author="human"):
|
|
275
|
-
return {"s": [dict(ts_ms=landed - 3 * 86_400_000 + i * 60_000, author=author, text="x") for i in range(n_before)]
|
|
276
|
-
+ [dict(ts_ms=landed + 3 * 86_400_000 + i * 60_000, author=author, text="x") for i in range(n_after)]}
|
|
277
|
-
|
|
278
|
-
def _eps(self, n_before, n_after, landed, pat):
|
|
279
|
-
import datetime as dt
|
|
280
|
-
mk = lambda ms: dt.datetime.fromtimestamp(ms / 1000, period.PT).isoformat(timespec="minutes")
|
|
281
|
-
return [dict(pattern=pat, ts_pt=mk(landed - 2 * 86_400_000 + i * 3600_000), episode_id=f"b{i}") for i in range(n_before)] + \
|
|
282
|
-
[dict(pattern=pat, ts_pt=mk(landed + 2 * 86_400_000 + i * 3600_000), episode_id=f"a{i}") for i in range(n_after)]
|
|
283
|
-
|
|
284
|
-
def test_not_stopped_and_not_enough_data(self):
|
|
285
|
-
landed = rules._day_ms("2026-09-10"); s = landed - 5 * 86_400_000; e = landed + 8 * 86_400_000
|
|
286
|
-
rows = rules.effectiveness(self._eps(3, 8, landed, "P1_invented_gates"), self._msgs(211, 211, landed), s, e)
|
|
287
|
-
r = next(x for x in rows if x["rule_key"] == "no_invented_gates")
|
|
288
|
-
self.assertEqual((r["verdict"], r["card"], r["before"]["per_100"], r["after"]["per_100"], len(r["episode_ids_after"])), ("not_stopped", True, 1.4, 3.8, 8))
|
|
289
|
-
rows = rules.effectiveness(self._eps(3, 8, landed, "P1_invented_gates"), self._msgs(18, 18, landed), s, e)
|
|
290
|
-
r = next(x for x in rows if x["rule_key"] == "no_invented_gates"); self.assertEqual((r["verdict"], r["card"]), ("not_enough_data", False))
|
|
291
|
-
|
|
292
|
-
def test_bot_prompts_not_in_denominator(self):
|
|
293
|
-
landed = rules._day_ms("2026-09-10"); s = landed - 5 * 86_400_000; e = landed + 8 * 86_400_000
|
|
294
|
-
rows = rules.effectiveness(self._eps(3, 8, landed, "P1_invented_gates"), self._msgs(211, 211, landed, author="bot"), s, e)
|
|
295
|
-
r = next(x for x in rows if x["rule_key"] == "no_invented_gates"); self.assertEqual((r["before"]["human_prompts"], r["verdict"]), (0, "not_enough_data"))
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
if __name__ == "__main__":
|
|
299
|
-
unittest.main()
|
|
@@ -1,99 +0,0 @@
|
|
|
1
|
-
"""Phase 5 tests (plan 5.3): device export carries no text, and merging a second device's export flips
|
|
2
|
-
replica sessions from extrapolated to measured while the extrapolation is recomputed for the rest."""
|
|
3
|
-
import copy
|
|
4
|
-
import json
|
|
5
|
-
import os
|
|
6
|
-
import shutil
|
|
7
|
-
import sqlite3
|
|
8
|
-
import tempfile
|
|
9
|
-
import unittest
|
|
10
|
-
|
|
11
|
-
import _paths # noqa: F401
|
|
12
|
-
import test_rollup as tr
|
|
13
|
-
from acr import devices, evidence, rollup, transcripts
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
class Export(unittest.TestCase):
|
|
17
|
-
def setUp(self):
|
|
18
|
-
self.fx = tr.Fixture("setUp"); self.fx.setUp(); self.tmp = tempfile.mkdtemp()
|
|
19
|
-
os.makedirs(os.path.join(self.tmp, "projects", "-w-p"))
|
|
20
|
-
line = dict(type="assistant", timestamp="2026-09-18T17:00:00.000Z", sessionId="cs1", requestId="r1", cwd="/w/p", isSidechain=False,
|
|
21
|
-
message=dict(id="m1", model="claude-fable-5-1", usage=dict(input_tokens=10, output_tokens=5, cache_creation_input_tokens=0, cache_read_input_tokens=0), content=[dict(type="text", text="secret prompt text")]))
|
|
22
|
-
with open(os.path.join(self.tmp, "projects", "-w-p", "cs1.jsonl"), "w") as fh: fh.write(json.dumps(line) + "\n")
|
|
23
|
-
|
|
24
|
-
def tearDown(self):
|
|
25
|
-
self.fx.tearDown(); shutil.rmtree(self.tmp)
|
|
26
|
-
|
|
27
|
-
def test_export_has_ids_tokens_models_and_no_text(self):
|
|
28
|
-
doc = transcripts.collect(tr.W, claude_glob=os.path.join(self.tmp, "projects", "**", "*.jsonl"), codex_glob=os.path.join(self.tmp, "none", "*.jsonl"), window_block=tr.W.block())
|
|
29
|
-
path, out = devices.export("mac", doc, self.tmp, tr.W.block(), live_db=self.fx.dbp)
|
|
30
|
-
self.assertTrue(path.endswith("device-usage-mac.json")); self.assertFalse(os.path.exists(os.path.join(self.tmp, "snapshot.db")))
|
|
31
|
-
d = devices.load(path)
|
|
32
|
-
self.assertEqual((d["device"], len(d["rows"]), d["rows"][0]["device"], d["rows"][0]["session"]), ("mac", 1, "mac", "cs1"))
|
|
33
|
-
self.assertIn("m1", d["sessions"]); self.assertEqual(d["sessions"]["m1"]["content_session_id"], "cs1")
|
|
34
|
-
txt = open(path).read(); self.assertNotIn("secret prompt", txt); self.assertNotIn("/w/p", txt); self.assertNotIn(".jsonl", txt)
|
|
35
|
-
self.assertFalse(any(k in r for r in d["rows"] for k in devices.FORBIDDEN))
|
|
36
|
-
|
|
37
|
-
def test_load_rejects_text_fields(self):
|
|
38
|
-
p = os.path.join(self.tmp, "bad.json")
|
|
39
|
-
with open(p, "w") as fh: json.dump(dict(device="x", window={}, rows=[dict(session="a", prompt="hi")], sessions={}), fh)
|
|
40
|
-
with self.assertRaises(ValueError): devices.load(p)
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
class OutsideWindow(unittest.TestCase):
|
|
44
|
-
def test_rows_outside_the_report_window_are_never_appended(self):
|
|
45
|
-
rows = []; export = dict(device="mac", window=dict(start_pt="2026-09-21", end_exclusive_pt="2026-09-22"), sessions={},
|
|
46
|
-
rows=[tr.row("x", tr.ms(0, 9)), tr.row("x", tr.ms(3, 1)), tr.row("x", tr.ms(-1, 23))])
|
|
47
|
-
out = devices.merge(rows, [export], {}, tr.W.block())
|
|
48
|
-
self.assertEqual(len(rows), 1); self.assertEqual((out[0]["outside_window"], out[0]["unmatched"], out[0]["window_matches_report"]), (2, 1, False))
|
|
49
|
-
rows = []; devices.merge(rows, [export], {}, dict(start_epoch_ms=None, end_epoch_ms=None))
|
|
50
|
-
self.assertEqual(len(rows), 3)
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
class Merge(unittest.TestCase):
|
|
54
|
-
"""cs2 is a replica session on the box (remote observations, no transcript). A Mac export whose local
|
|
55
|
-
session map names its memory id m2 must join it."""
|
|
56
|
-
def setUp(self):
|
|
57
|
-
self.fx = tr.Fixture("setUp"); self.fx.setUp()
|
|
58
|
-
|
|
59
|
-
def tearDown(self):
|
|
60
|
-
self.fx.tearDown()
|
|
61
|
-
|
|
62
|
-
def build(self, exports=()):
|
|
63
|
-
db = sqlite3.connect(self.fx.dbp); db.row_factory = sqlite3.Row
|
|
64
|
-
try:
|
|
65
|
-
usage = dict(window=tr.W.block(), rows=copy.deepcopy(self.fx.rows))
|
|
66
|
-
report, _, _ = rollup.build(usage, tr.PRICES, db, evidence.Scope("period", S=tr.W.start_epoch_ms, E=tr.W.end_epoch_ms), tr.W.block(), now=tr.NOW, use_gh=False,
|
|
67
|
-
remote=lambda cwd: None, behavior=False, device_exports=list(exports))
|
|
68
|
-
return report
|
|
69
|
-
finally:
|
|
70
|
-
db.close()
|
|
71
|
-
|
|
72
|
-
def mac_export(self, cs="mac-local-cs2", mid="m2", n=1, inp=50000):
|
|
73
|
-
rows = [dict(src="claude-code", session=cs, sidechain=False, model="claude-fable-5-1", ts=tr.ts(tr.ms(1, 8.3 + i * 0.1)), input=inp, output=1000, cache_write=0, cache_write_1h=0, cache_write_5m=0, cache_read=0, cost_usd_reported=None, device="mac") for i in range(n)]
|
|
74
|
-
return dict(device="mac", host_label="my-laptop", exported_at_pt="x", window=tr.W.block(), rows=rows, sessions={mid: dict(content_session_id=cs, project="memindex/wt", started_at_epoch=tr.ms(1, 8))})
|
|
75
|
-
|
|
76
|
-
def test_merge_flips_extrapolated_to_measured(self):
|
|
77
|
-
before = self.build(); li0 = {li["session_ids"][0]: li for li in before["line_items"]}
|
|
78
|
-
self.assertEqual(li0["m2"]["cost_basis"], "extrapolated"); self.assertGreater(before["spend"]["extrapolated_unmeasured_usd"], 0)
|
|
79
|
-
after = self.build([self.mac_export()]); li1 = {li["session_ids"][0]: li for li in after["line_items"]}
|
|
80
|
-
self.assertEqual((li1["m2"]["cost_basis"], li1["m2"]["has_transcript"], li1["m2"]["device"]), ("estimated_usage", True, ["mac", "remote-1"]))
|
|
81
|
-
self.assertGreater(after["spend"]["agent_estimated_usd"], before["spend"]["agent_estimated_usd"])
|
|
82
|
-
self.assertEqual(after["spend"]["extrapolated_unmeasured_usd"], 0.0)
|
|
83
|
-
self.assertIn("from 1 sessions with no transcript", after["spend"]["extrapolation_basis"])
|
|
84
|
-
for li in after["line_items"]: li["has_transcript"] = True
|
|
85
|
-
from acr import costs; self.assertIn("all sessions measured", costs.extrapolation(after["line_items"])[1])
|
|
86
|
-
self.assertIn("device export: mac", li1["m2"]["notes"])
|
|
87
|
-
de = after["device_exports"][0]; self.assertEqual((de["device"], de["joined"], de["unmatched"], de["duplicates_skipped"], de["window_matches_report"]), ("mac", 1, 0, 0, True))
|
|
88
|
-
self.assertEqual(after["unmatched_transcripts"]["count"], before["unmatched_transcripts"]["count"])
|
|
89
|
-
|
|
90
|
-
def test_unknown_session_is_unmatched_and_duplicates_skipped(self):
|
|
91
|
-
ex = self.mac_export(cs="nobody-knows", mid="m-none")
|
|
92
|
-
after = self.build([ex, ex])
|
|
93
|
-
de = after["device_exports"]; self.assertEqual((de[0]["unmatched"], de[1]["duplicates_skipped"]), (1, 1))
|
|
94
|
-
self.assertEqual(after["unmatched_transcripts"]["count"], 2)
|
|
95
|
-
self.assertIn("device:mac", after["unmatched_transcripts"]["by_dir"])
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
if __name__ == "__main__":
|
|
99
|
-
unittest.main()
|
|
@@ -1,96 +0,0 @@
|
|
|
1
|
-
import unittest
|
|
2
|
-
|
|
3
|
-
import _paths # noqa: F401
|
|
4
|
-
from acr import labels
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
class Classify(unittest.TestCase):
|
|
8
|
-
def ev(self, prompts=(), text=()):
|
|
9
|
-
return dict(prompts=list(prompts), text=list(text))
|
|
10
|
-
|
|
11
|
-
def test_category_head_then_tail_then_default(self):
|
|
12
|
-
d = labels.classify(self.ev(["please fix the broken build"]), dict(project="p", user_prompt=None))
|
|
13
|
-
self.assertEqual((d["category"], d["matched"]["category_where"]), ("Bug fix", "head"))
|
|
14
|
-
d = labels.classify(self.ev([], ["benchmark results"]), dict(project="p", user_prompt=None))
|
|
15
|
-
self.assertEqual((d["category"], d["matched"]["category_where"]), ("Experiment", "tail"))
|
|
16
|
-
d = labels.classify(self.ev(), dict(project="p", user_prompt=None))
|
|
17
|
-
self.assertEqual((d["category"], d["matched"]["category_where"]), ("Investigation", "default"))
|
|
18
|
-
|
|
19
|
-
def test_failures_closed_set_and_rework_only_as_failure(self):
|
|
20
|
-
d = labels.classify(self.ev(["redo this, verifier found gaps", "x"] + ["same prompt prefix"] * 3), dict(project="p", user_prompt=None))
|
|
21
|
-
self.assertEqual(d["failure_signals"], ["Rework", "Looping"])
|
|
22
|
-
self.assertTrue(all(f in labels.FAILURE_TYPES for f in d["failure_signals"]))
|
|
23
|
-
self.assertNotIn("Rework", labels.CATEGORIES); self.assertNotIn("Rework", [c for c, _ in labels.CAT_RULES])
|
|
24
|
-
self.assertEqual(len(labels.FAILURE_TYPES), 16)
|
|
25
|
-
|
|
26
|
-
def test_active_minutes(self):
|
|
27
|
-
m = 60000
|
|
28
|
-
self.assertEqual(labels.active_minutes([0, 5 * m, 10 * m, 60 * m, 61 * m]), 11)
|
|
29
|
-
self.assertEqual(labels.active_minutes([]), 0)
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
class Review(unittest.TestCase):
|
|
33
|
-
def items(self):
|
|
34
|
-
return [dict(work_item_id="WI-1", category="Investigation", work_category="Investigation", failure_type="", failure_signals=[], label_source="keyword", status="completed"),
|
|
35
|
-
dict(work_item_id="WI-2", category="Investigation", work_category="Investigation", failure_type="", failure_signals=[], label_source="keyword", status="completed")]
|
|
36
|
-
|
|
37
|
-
def test_apply_sets_source_and_leaves_unreviewed(self):
|
|
38
|
-
li = self.items()
|
|
39
|
-
n = labels.apply_review(li, [dict(work_item_id="WI-1", category="Feature", failure_signals=["Looping", "Rework"], reviewed_by="user", status="blocked")], "t")
|
|
40
|
-
self.assertEqual(n, 1); self.assertEqual((li[0]["label_source"], li[0]["failure_type"], li[0]["status"], li[0]["reviewed_at_pt"]), ("human", "Looping", "blocked", "t"))
|
|
41
|
-
self.assertEqual(li[1]["label_source"], "keyword")
|
|
42
|
-
labels.apply_review(li, [dict(work_item_id="WI-2", category="Feature", reviewed_by="model-x")], "t")
|
|
43
|
-
self.assertEqual(li[1]["label_source"], "llm")
|
|
44
|
-
|
|
45
|
-
def test_file_level_reviewed_by_default(self):
|
|
46
|
-
li = self.items()
|
|
47
|
-
n = labels.apply_review(li, dict(reviewed_by="user", items=[dict(work_item_id="WI-1", category="Feature")]), "t")
|
|
48
|
-
self.assertEqual((n, li[0]["label_source"], li[0]["reviewed_by"]), (1, "human", "user"))
|
|
49
|
-
self.assertEqual(labels.apply_review(li, dict(items=[dict(work_item_id="WI-2", category="Feature")]), "t"), 0)
|
|
50
|
-
|
|
51
|
-
def test_generated_review_template_inherits_file_level_reviewer(self):
|
|
52
|
-
li = self.items()
|
|
53
|
-
draft = labels.classify(dict(prompts=["build the feature"], text=[]), dict(project="p", user_prompt=None))
|
|
54
|
-
template = labels.review_entry(dict(li[0], session_ids=["session-1"], title="Feature work"), draft)
|
|
55
|
-
self.assertIsNone(template["reviewed_by"])
|
|
56
|
-
template["category"] = "Feature"
|
|
57
|
-
reviewed = dict(reviewed_by="user", items=[template])
|
|
58
|
-
self.assertEqual(labels.apply_review(li, reviewed, "t"), 1)
|
|
59
|
-
self.assertEqual((li[0]["reviewed_by"], li[0]["label_source"]), ("user", "human"))
|
|
60
|
-
self.assertIsNone(template["reviewed_by"])
|
|
61
|
-
|
|
62
|
-
def test_explicit_entry_reviewer_overrides_file_default(self):
|
|
63
|
-
li = self.items()
|
|
64
|
-
reviewed = dict(reviewed_by="user", items=[dict(work_item_id="WI-1", category="Feature", reviewed_by="model-x")])
|
|
65
|
-
self.assertEqual(labels.apply_review(li, reviewed, "t"), 1)
|
|
66
|
-
self.assertEqual((li[0]["reviewed_by"], li[0]["label_source"]), ("model-x", "llm"))
|
|
67
|
-
|
|
68
|
-
def test_empty_entry_reviewer_and_unconfirmed_category_remain_unreviewed(self):
|
|
69
|
-
li = self.items()
|
|
70
|
-
reviewed = dict(reviewed_by="user", items=[
|
|
71
|
-
dict(work_item_id="WI-1", category="Feature", reviewed_by=""),
|
|
72
|
-
dict(work_item_id="WI-2", category=None, reviewed_by=None),
|
|
73
|
-
])
|
|
74
|
-
self.assertEqual(labels.apply_review(li, reviewed, "t"), 0)
|
|
75
|
-
self.assertEqual([item["label_source"] for item in li], ["keyword", "keyword"])
|
|
76
|
-
|
|
77
|
-
def test_confirming_a_category_keeps_the_draft_failure_signals(self):
|
|
78
|
-
li = self.items(); li[0]["failure_signals"] = ["Recovery after miss"]; li[0]["failure_type"] = "Recovery after miss"
|
|
79
|
-
labels.apply_review(li, [dict(work_item_id="WI-1", category="Investigation", reviewed_by="user", failure_type=None, failure_signals=None)], "t")
|
|
80
|
-
self.assertEqual((li[0]["failure_signals"], li[0]["failure_type"], li[0]["label_source"]), (["Recovery after miss"], "Recovery after miss", "human"))
|
|
81
|
-
labels.apply_review(li, [dict(work_item_id="WI-1", category="Investigation", reviewed_by="user", failure_signals=[])], "t")
|
|
82
|
-
self.assertEqual((li[0]["failure_signals"], li[0]["failure_type"]), ([], ""))
|
|
83
|
-
|
|
84
|
-
def test_reviewed_title_replaces_the_draft_title(self):
|
|
85
|
-
li = self.items(); li[0]["title"] = li[1]["title"] = "first prompt text"
|
|
86
|
-
labels.apply_review(li, [dict(work_item_id="WI-1", category="Feature", reviewed_by="user", title="Rollup shipped"), dict(work_item_id="WI-2", category="Feature", reviewed_by="user", title="")], "t")
|
|
87
|
-
self.assertEqual((li[0]["title"], li[1]["title"]), ("Rollup shipped", "first prompt text"))
|
|
88
|
-
|
|
89
|
-
def test_rejects_unknown_values(self):
|
|
90
|
-
for bad in (dict(category="Rework"), dict(category="Feature", failure_type="Made up"), dict(category="Feature", label_source="magic"), dict(category="Feature", status="done")):
|
|
91
|
-
with self.assertRaises(labels.ReviewError):
|
|
92
|
-
labels.apply_review(self.items(), [dict(work_item_id="WI-1", reviewed_by="user", **bad)], "t")
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
if __name__ == "__main__":
|
|
96
|
-
unittest.main()
|
|
@@ -1,120 +0,0 @@
|
|
|
1
|
-
"""Phase 4 tests (plan 4.3): mocked urlopen for the unavailable / ok / error paths, the bucket-vs-window
|
|
2
|
-
rule, and the no-key-material guard."""
|
|
3
|
-
import datetime as dt
|
|
4
|
-
import io
|
|
5
|
-
import json
|
|
6
|
-
import os
|
|
7
|
-
import shutil
|
|
8
|
-
import subprocess
|
|
9
|
-
import sys
|
|
10
|
-
import tempfile
|
|
11
|
-
import unittest
|
|
12
|
-
import urllib.error
|
|
13
|
-
|
|
14
|
-
import _paths # noqa: F401
|
|
15
|
-
from acr import measure, period, render
|
|
16
|
-
|
|
17
|
-
KEY = "sk-or-v1-testkeytestkeytestkeytestkey0000"
|
|
18
|
-
UTC = dt.timezone.utc
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
class FakeResp(io.BytesIO):
|
|
22
|
-
def __enter__(self): return self
|
|
23
|
-
def __exit__(self, *a): return False
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
def ok_opener(req, timeout=0):
|
|
27
|
-
assert req.get_header("Authorization") == f"Bearer {KEY}"
|
|
28
|
-
return FakeResp(json.dumps(dict(data=dict(usage=123.45, usage_daily=1.5, usage_weekly=42.25, usage_monthly=99.0, limit=None, limit_remaining=None, is_free_tier=False))).encode())
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
def http_error(req, timeout=0):
|
|
32
|
-
raise urllib.error.HTTPError(req.full_url, 401, "unauthorized", {}, None)
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
class Measure(unittest.TestCase):
|
|
36
|
-
def setUp(self):
|
|
37
|
-
self.tmp = tempfile.mkdtemp()
|
|
38
|
-
|
|
39
|
-
def tearDown(self):
|
|
40
|
-
shutil.rmtree(self.tmp)
|
|
41
|
-
|
|
42
|
-
def read(self):
|
|
43
|
-
with open(os.path.join(self.tmp, "measured.json")) as fh: return fh.read()
|
|
44
|
-
|
|
45
|
-
def test_unavailable_without_key(self):
|
|
46
|
-
out = measure.measure(self.tmp, env={}, opener=lambda *a, **k: self.fail("must not call the endpoint"))
|
|
47
|
-
self.assertEqual(out, dict(status="unavailable", reason="OPENROUTER_API_KEY not provided")); self.assertIn('"unavailable"', self.read())
|
|
48
|
-
|
|
49
|
-
def test_ok_writes_buckets_and_no_key_material(self):
|
|
50
|
-
now = dt.datetime(2026, 9, 26, 12, 0, tzinfo=UTC)
|
|
51
|
-
out = measure.measure(self.tmp, env={"OPENROUTER_API_KEY": KEY}, opener=ok_opener, now=now)
|
|
52
|
-
self.assertEqual((out["status"], out["usage_weekly"], out["usage_monthly"], out["usage_lifetime"], out["key_hint"]), ("ok", 42.25, 99.0, 123.45, None))
|
|
53
|
-
txt = self.read(); self.assertNotIn("sk-", txt); self.assertNotIn(KEY[-8:], txt); self.assertNotIn("Bearer", txt)
|
|
54
|
-
|
|
55
|
-
def test_error_keeps_http_status_only(self):
|
|
56
|
-
out = measure.measure(self.tmp, env={"OPENROUTER_API_KEY": KEY}, opener=http_error)
|
|
57
|
-
self.assertEqual(out, dict(status="error", http_status=401)); self.assertNotIn("sk-", self.read())
|
|
58
|
-
|
|
59
|
-
def test_cli_unavailable_exit_zero(self):
|
|
60
|
-
env = {k: v for k, v in os.environ.items() if k != "OPENROUTER_API_KEY"}
|
|
61
|
-
r = subprocess.run([sys.executable, _paths.ACR_PY, "measure-openrouter", "--out", self.tmp], capture_output=True, text=True, env=env)
|
|
62
|
-
self.assertEqual(r.returncode, 0, r.stderr); self.assertIn("status=unavailable", r.stdout)
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
class Bucket(unittest.TestCase):
|
|
66
|
-
def measured(self, fetched):
|
|
67
|
-
return dict(status="ok", fetched_at_utc=fetched.isoformat(timespec="seconds"), usage_weekly=42.25, usage_monthly=99.0)
|
|
68
|
-
|
|
69
|
-
def window(self, start, end):
|
|
70
|
-
return period.resolve_window(start, end).block()
|
|
71
|
-
|
|
72
|
-
def test_weekly_bucket_covers_window(self):
|
|
73
|
-
w = self.window("2026-09-22", "2026-09-24")
|
|
74
|
-
b = measure.bucket_rule(self.measured(dt.datetime(2026, 9, 25, 12, 0, tzinfo=UTC)), w)
|
|
75
|
-
self.assertEqual((b["bucket"], b["usd"], b["covers_window"]), ("usage_weekly", 42.25, True)); self.assertIn("week Mon 21 Sep", b["label"]); self.assertIsNone(b["note"])
|
|
76
|
-
|
|
77
|
-
def test_window_not_finished_is_reference_only(self):
|
|
78
|
-
w = self.window("2026-09-22", "2026-09-27")
|
|
79
|
-
b = measure.bucket_rule(self.measured(dt.datetime(2026, 9, 25, 12, 0, tzinfo=UTC)), w)
|
|
80
|
-
self.assertFalse(b["covers_window"]); self.assertIn("shown for reference", b["note"])
|
|
81
|
-
|
|
82
|
-
def test_window_before_week_uses_month_or_reference(self):
|
|
83
|
-
w = self.window("2026-09-18", "2026-09-26")
|
|
84
|
-
b = measure.bucket_rule(self.measured(dt.datetime(2026, 9, 27, 12, 0, tzinfo=UTC)), w)
|
|
85
|
-
self.assertEqual((b["bucket"], b["usd"], b["covers_window"]), ("usage_monthly", 99.0, True))
|
|
86
|
-
self.assertEqual(b["label"], "OpenRouter measured, current UTC month Tue 1 Sep – now")
|
|
87
|
-
w2 = self.window("2026-08-28", "2026-09-02")
|
|
88
|
-
b2 = measure.bucket_rule(self.measured(dt.datetime(2026, 9, 27, 12, 0, tzinfo=UTC)), w2)
|
|
89
|
-
self.assertFalse(b2["covers_window"])
|
|
90
|
-
|
|
91
|
-
def test_apply_and_render_measured(self):
|
|
92
|
-
fx = os.path.join(_paths.FIXTURES, "report-7day.json")
|
|
93
|
-
with open(fx) as fh: d = json.load(fh)
|
|
94
|
-
measure.apply(d, dict(status="unavailable")); self.assertEqual((d["spend"]["agent_measured_usd"], d["spend"]["measured_status"]), (None, "unavailable"))
|
|
95
|
-
measure.apply(d, dict(status="error", http_status=500)); self.assertIsNone(d["spend"]["agent_measured_usd"]); self.assertIn("error", d["spend"]["measured_status"])
|
|
96
|
-
h = render.page(d); self.assertIn("Measured provider spend: unavailable", h); self.assertNotIn("$0.00<span class=\"est meas\"", h)
|
|
97
|
-
fetched = dt.datetime(2026, 10, 2, 12, 0, tzinfo=UTC)
|
|
98
|
-
measure.apply(d, self.measured(fetched), now=fetched)
|
|
99
|
-
self.assertIsNone(d["spend"]["agent_measured_usd"]); self.assertIn("reference", d["spend"]["measured_status"])
|
|
100
|
-
h = render.page(d); self.assertIn("shown for reference", h); self.assertIn("MEASURED", h)
|
|
101
|
-
for li in d["line_items"]: self.assertEqual(li["cost_basis"] in ("estimated_usage", "extrapolated"), True)
|
|
102
|
-
fetched = dt.datetime(2026, 9, 27, 12, 0, tzinfo=UTC)
|
|
103
|
-
measure.apply(d, self.measured(fetched), now=fetched)
|
|
104
|
-
self.assertEqual((d["spend"]["agent_measured_usd"], d["spend"]["measured_status"]), (99.0, "ok (period bucket)"))
|
|
105
|
-
h = render.page(d); self.assertGreaterEqual(h.count("MEASURED"), 1); self.assertIn("note-taker calls", h); self.assertIn("$99.00", h)
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
class SessionScope(unittest.TestCase):
|
|
109
|
-
def test_session_window_without_bounds_is_reference_only_not_a_crash(self):
|
|
110
|
-
ok = dict(status="ok", fetched_at_utc="2026-09-25T19:00:00+00:00", usage_weekly=12.5, usage_monthly=40.0)
|
|
111
|
-
b = measure.bucket_rule(ok, period.session_block("cs1"))
|
|
112
|
-
self.assertEqual((b["covers_window"], b["usd"], b["bucket"]), (False, 12.5, "usage_weekly")); self.assertIn("reference only", b["note"])
|
|
113
|
-
self.assertEqual(b["label"], "OpenRouter measured, current UTC week Mon 21 Sep – now")
|
|
114
|
-
report = dict(window=period.session_block("cs1"), spend=dict(agent_estimated_usd=1.0))
|
|
115
|
-
s = measure.apply(report, ok)
|
|
116
|
-
self.assertIsNone(s["agent_measured_usd"]); self.assertTrue(s["measured_status"].startswith("reference only"))
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
if __name__ == "__main__":
|
|
120
|
-
unittest.main()
|