claude-multiacc 2.0.58 → 2.0.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +125 -4
- package/bin/claude +389 -94
- package/bin/claude-accounts +276 -42
- package/bin/codex +69 -5
- package/bin/codex-accounts +33 -4
- package/docs/ACCOUNT_OPERATIONS.md +62 -2
- package/docs/AUTORESUME-HANDOFF.md +28 -3
- package/docs/AUTORESUME.md +275 -18
- package/docs/CODEX.md +1 -0
- package/docs/INSTALLATION.md +7 -1
- package/docs/VERIFICATION.md +4 -0
- package/lib/__pycache__/audit.cpython-312.pyc +0 -0
- package/lib/__pycache__/autoresume.cpython-312.pyc +0 -0
- package/lib/__pycache__/autoresume_cross.cpython-312.pyc +0 -0
- package/lib/__pycache__/autoresume_cross_dispatch.cpython-312.pyc +0 -0
- package/lib/__pycache__/autoresume_cross_receipt.cpython-312.pyc +0 -0
- package/lib/__pycache__/autoresume_handoff.cpython-312.pyc +0 -0
- package/lib/__pycache__/autoresume_probe.cpython-312.pyc +0 -0
- package/lib/__pycache__/autoresume_receipt.cpython-312.pyc +0 -0
- package/lib/__pycache__/claude_client_error.cpython-312.pyc +0 -0
- package/lib/__pycache__/claude_reset.cpython-312.pyc +0 -0
- package/lib/__pycache__/claude_token_identity.cpython-312.pyc +0 -0
- package/lib/__pycache__/codex_autoresume_mode.cpython-312.pyc +0 -0
- package/lib/__pycache__/codex_config_edit.cpython-312.pyc +0 -0
- package/lib/__pycache__/codex_credit_controls.cpython-312.pyc +0 -0
- package/lib/__pycache__/codex_credit_local_scope.cpython-312.pyc +0 -0
- package/lib/__pycache__/codex_credit_selection.cpython-312.pyc +0 -0
- package/lib/__pycache__/codex_credits.cpython-312.pyc +0 -0
- package/lib/__pycache__/codex_limits_lock.cpython-312.pyc +0 -0
- package/lib/__pycache__/codex_python.cpython-312.pyc +0 -0
- package/lib/__pycache__/codex_trust.cpython-312.pyc +0 -0
- package/lib/__pycache__/codex_usage_fetch.cpython-312.pyc +0 -0
- package/lib/__pycache__/keychain.cpython-312.pyc +0 -0
- package/lib/__pycache__/mcp_oauth.cpython-312.pyc +0 -0
- package/lib/__pycache__/mcp_registry.cpython-312.pyc +0 -0
- package/lib/__pycache__/model_catalog.cpython-312.pyc +0 -0
- package/lib/__pycache__/model_match.cpython-312.pyc +0 -0
- package/lib/__pycache__/report.cpython-312.pyc +0 -0
- package/lib/__pycache__/selector_credit_scope.cpython-312.pyc +0 -0
- package/lib/__pycache__/selector_credits.cpython-312.pyc +0 -0
- package/lib/__pycache__/selector_funding.cpython-312.pyc +0 -0
- package/lib/__pycache__/selector_policy.cpython-312.pyc +0 -0
- package/lib/__pycache__/selector_primitives.cpython-312.pyc +0 -0
- package/lib/__pycache__/session_settings.cpython-312.pyc +0 -0
- package/lib/__pycache__/shim_path.cpython-312.pyc +0 -0
- package/lib/audit.py +88 -22
- package/lib/autoresume.py +1197 -82
- package/lib/autoresume_cross.py +3 -1
- package/lib/autoresume_handoff.py +81 -9
- package/lib/autoresume_probe.py +2 -2
- package/lib/claude_client_error.py +9 -4
- package/lib/codex_trust.py +263 -0
- package/lib/common.sh +146 -3
- package/lib/credential.py +5 -2
- package/lib/install_actions.sh +19 -0
- package/lib/keychain.py +180 -36
- package/lib/mcp_oauth.py +1748 -0
- package/lib/mcp_registry.py +230 -8
- package/lib/model_catalog.json +404 -0
- package/lib/model_catalog.py +201 -0
- package/lib/model_match.py +104 -0
- package/lib/report.py +8 -5
- package/lib/session_settings.py +123 -0
- package/package.json +2 -2
- package/tests/__pycache__/autoresume_cross_receipt_support.cpython-312.pyc +0 -0
- package/tests/__pycache__/autoresume_cross_tmux_environment.cpython-312.pyc +0 -0
- package/tests/__pycache__/autoresume_cross_tmux_support.cpython-312.pyc +0 -0
- package/tests/__pycache__/autoresume_cross_unit_support.cpython-312.pyc +0 -0
- package/tests/__pycache__/autoresume_tmux_support.cpython-312.pyc +0 -0
- package/tests/__pycache__/packaged_command_support.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_catalog.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_catalog_cli.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_codex_records.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_codex_trust.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_cross_claude_holders.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_cross_guards.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_cross_holders.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_cross_origin.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_cross_policy.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_cross_receipt.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_cross_tmux.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_cross_tmux_races.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_cross_tokens.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_cursor.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_fallback.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_handoff_argv.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_handoff_history.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_handoff_origins.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_memory.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_memory_tmux.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_mode.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_probe.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_receipt.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_relaunch_settings.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_settings.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_sidechain.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_sidechain_tmux.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_tmux.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_tmux_modes.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume_trust_tmux.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_claude_reset.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_claude_token_identity.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_codex_reset.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_codex_reset_credits.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_codex_reset_credits_polling.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_codex_reset_credits_report.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_codex_reset_fetch.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_codex_reset_polling.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_codex_reset_reporting.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_codex_reset_windows.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_selector.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_selector_cached_credits.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_selector_credit_scope.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_selector_credit_shim.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_selector_credits.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_selector_spread.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_token_identity_ceremony.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_token_identity_integration.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_token_identity_retry.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_token_identity_sync.cpython-312.pyc +0 -0
- package/tests/fake_codex_tui.py +25 -0
- package/tests/fake_security.sh +24 -2
- package/tests/fixtures/cross_provider_cli.py +109 -0
- package/tests/run-tests.sh +1171 -23
- package/tests/test_autoresume.py +215 -8
- package/tests/test_autoresume_catalog.py +226 -0
- package/tests/test_autoresume_catalog_cli.py +185 -0
- package/tests/test_autoresume_codex_records.py +16 -0
- package/tests/test_autoresume_codex_trust.py +247 -0
- package/tests/test_autoresume_cross_guards.py +28 -0
- package/tests/test_autoresume_cross_tmux.py +27 -2
- package/tests/test_autoresume_cross_tokens.py +32 -2
- package/tests/test_autoresume_handoff_argv.py +120 -16
- package/tests/test_autoresume_handoff_origins.py +14 -0
- package/tests/test_autoresume_memory.py +1141 -0
- package/tests/test_autoresume_memory_tmux.py +94 -0
- package/tests/test_autoresume_probe.py +2 -1
- package/tests/test_autoresume_relaunch_settings.py +317 -0
- package/tests/test_autoresume_settings.py +135 -0
- package/tests/test_autoresume_sidechain.py +1518 -0
- package/tests/test_autoresume_sidechain_tmux.py +79 -0
- package/tests/test_autoresume_trust_tmux.py +63 -0
- package/tests/test_claude_client_error.py +10 -7
- package/tests/test_claude_org_block.py +1 -0
- package/tests/test_claude_pool_prepare.py +4 -1
- package/tests/test_claude_reset.py +3 -0
- package/tests/test_keychain_nologin.py +340 -0
- package/tests/test_mcp_oauth.py +1414 -0
- package/tests/test_mcp_registry.py +234 -3
- package/tests/test_packaged_commands.py +41 -0
|
@@ -0,0 +1,1518 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Account-level rejections that only a claude session's subagent and workflow transcripts
|
|
3
|
+
record. 2026-10-19: two sessions sat idle behind paused workflows whose agents alone had
|
|
4
|
+
met the five-hour limit, and the watcher, tailing the main transcript only, never moved
|
|
5
|
+
them. Disposable pools, the fake runner and its fake clock only: no real signal or tmux."""
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import signal
|
|
10
|
+
import unittest
|
|
11
|
+
from unittest import mock
|
|
12
|
+
|
|
13
|
+
import test_autoresume as fixtures
|
|
14
|
+
from test_autoresume import (SID, SID2, ar, cc_api_error, cc_concurrents, cc_quota, cc_reply,
|
|
15
|
+
cc_slash, cc_user, iso)
|
|
16
|
+
|
|
17
|
+
AGENT = "a33659a7131e55067"
|
|
18
|
+
AGENT2 = "ab12cd34ef56a7890"
|
|
19
|
+
AGENT3 = "a0f1e2d3c4b5a6978"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def side(rec, agent=AGENT):
|
|
23
|
+
"""A record as a subagent transcript holds it (claude 2.1.303): the session's
|
|
24
|
+
sidechain, named by its agent."""
|
|
25
|
+
return dict(rec, isSidechain=True, agentId=agent)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def side_quota(ts, resets, agent=AGENT, rtype="five_hour"):
|
|
29
|
+
rec = cc_quota(ts, rtype, resets=resets, apiError="usage_limit_reached")
|
|
30
|
+
rec["message"]["content"][0]["text"] = ("You've hit your session limit · resets 12:40am "
|
|
31
|
+
"(Europe/Istanbul)")
|
|
32
|
+
return side(rec, agent)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def side_reply(ts, agent=AGENT, text="Checked the build.", ref=None):
|
|
36
|
+
rec = side(cc_reply(ts, text), agent)
|
|
37
|
+
if ref:
|
|
38
|
+
rec["requestRef"] = ref
|
|
39
|
+
return rec
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def final_answer(ts, agent, ref):
|
|
43
|
+
"""An agent's last reply (end_turn): nothing of that agent follows it."""
|
|
44
|
+
rec = side_reply(ts, agent, "All done: the section is written.", ref)
|
|
45
|
+
rec["message"]["stop_reason"] = "end_turn"
|
|
46
|
+
return rec
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def api_request(ts, rid, agent=None):
|
|
50
|
+
"""What claude 2.1.303 writes right before it sends a request, in an agent's transcript
|
|
51
|
+
and the main one alike; every reply to that request names it by requestRef."""
|
|
52
|
+
rec = {"type": "api-request", "sessionId": SID, "id": rid, "timestamp": iso(ts),
|
|
53
|
+
"version": "2.1.303", "querySource": "repl_main_thread", "shapeHash": "9f2c41",
|
|
54
|
+
"params": {"model": "claude-opus-5-5", "max_tokens": 64000}, "keep": 0, "tail": 1,
|
|
55
|
+
"cacheControl": "5m", "requestDigest": "4b1d0c"}
|
|
56
|
+
if agent:
|
|
57
|
+
rec.update(agentId=agent, querySource="agent:builtin:workflow-subagent")
|
|
58
|
+
return rec
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def main_reply(ts, ref, text="Done."):
|
|
62
|
+
return dict(cc_reply(ts, text), requestRef=ref)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def queue_op(ts):
|
|
66
|
+
"""All an idle main transcript waiting on its workflows still gets."""
|
|
67
|
+
return {"type": "queue-operation", "operation": "enqueue", "timestamp": iso(ts),
|
|
68
|
+
"sessionId": SID, "content": "<task-notification>paused</task-notification>"}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def request_blob():
|
|
72
|
+
"""A request body the client keeps among the records (claude 2.1.303): no timestamp of
|
|
73
|
+
its own, and it quotes the conversation's "assistant" turns."""
|
|
74
|
+
return {"type": "api-request-blob", "hash": "5d41402abc4b2a76b9719d911017c592",
|
|
75
|
+
"sessionId": SID, "message": {"role": "assistant", "content": [
|
|
76
|
+
{"type": "text", "text": "Started it; waiting for the workflow."}]}}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def tool_result(ts):
|
|
80
|
+
return cc_user(ts, [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "ok"}],
|
|
81
|
+
toolUseResult={"stdout": "ok"})
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def synthetic(ts, text="No response requested."):
|
|
85
|
+
"""The client's own words in the main thread, no API error and no reply of the API's."""
|
|
86
|
+
rec = cc_reply(ts, text)
|
|
87
|
+
rec["message"]["model"] = "<synthetic>"
|
|
88
|
+
return rec
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def compact(records):
|
|
92
|
+
"""JSONL as the client writes it: compact JSON, a record a line."""
|
|
93
|
+
return "".join(json.dumps(rec, separators=(",", ":")) + "\n" for rec in records)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def write(path, records):
|
|
97
|
+
with open(path, "a") as fh:
|
|
98
|
+
fh.write(compact(records))
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def main_lines(records):
|
|
102
|
+
"""Main transcript lines as the client writes them (compact JSON)."""
|
|
103
|
+
return [line.encode() for line in compact(records).splitlines()]
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class SidechainCase(fixtures.PoolCase):
|
|
107
|
+
def agent_file(self, records, agent=AGENT, workflow="wf_7f3a9c", sid=SID, mtime=None):
|
|
108
|
+
"""<proj>/<sid>/subagents/workflows/<wf>/agent-<id>.jsonl (workflow=None: an Agent
|
|
109
|
+
tool subagent, <sid>/subagents/agent-<id>.jsonl)."""
|
|
110
|
+
folder = self.proj / sid / "subagents"
|
|
111
|
+
if workflow:
|
|
112
|
+
folder = folder / "workflows" / workflow
|
|
113
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
114
|
+
path = folder / ("agent-%s.jsonl" % agent)
|
|
115
|
+
write(path, records)
|
|
116
|
+
if mtime is not None:
|
|
117
|
+
os.utime(str(path), (mtime, mtime))
|
|
118
|
+
return path
|
|
119
|
+
|
|
120
|
+
def transcript(self, records, sid=SID):
|
|
121
|
+
path = self.proj / (sid + ".jsonl")
|
|
122
|
+
write(path, records)
|
|
123
|
+
return path
|
|
124
|
+
|
|
125
|
+
def start(self, argv=None, main=None, env=None):
|
|
126
|
+
"""A session whose main thread started workflows and has idled on them since."""
|
|
127
|
+
state = self.write_state(["--dangerously-skip-permissions"] if argv is None else argv)
|
|
128
|
+
self.registry()
|
|
129
|
+
self.transcript([cc_user(self.launched + 1, "run the release workflow"),
|
|
130
|
+
cc_reply(self.launched + 2, "Started it; waiting for the workflow."),
|
|
131
|
+
queue_op(self.t0 - 1)] if main is None else main)
|
|
132
|
+
return self.watcher(state, env)
|
|
133
|
+
|
|
134
|
+
def stop_at(self, runner, when):
|
|
135
|
+
"""The operator ends the session (/exit: its registry goes)."""
|
|
136
|
+
reg = self.root / "acct-01" / "sessions" / ("%d.json" % self.PID)
|
|
137
|
+
runner.at(when, lambda: (reg.unlink() if reg.exists() else None,
|
|
138
|
+
runner.exit(self.PID)))
|
|
139
|
+
|
|
140
|
+
def tail_of(self, watcher, name):
|
|
141
|
+
found = [tail for path, tail in watcher.side.items() if os.path.basename(path) == name]
|
|
142
|
+
self.assertEqual(len(found), 1, sorted(watcher.side))
|
|
143
|
+
return found[0]
|
|
144
|
+
|
|
145
|
+
def assert_untouched(self, runner):
|
|
146
|
+
self.assertEqual((runner.probes, runner.signals, runner.sends, self.relaunch_files()),
|
|
147
|
+
([], [], [], []))
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
# =====================================================================================
|
|
151
|
+
# classification and the tracker
|
|
152
|
+
# =====================================================================================
|
|
153
|
+
|
|
154
|
+
class SidechainClassifierTests(unittest.TestCase):
|
|
155
|
+
T = 1792440000
|
|
156
|
+
|
|
157
|
+
def test_only_the_accounts_own_verdicts_count(self):
|
|
158
|
+
verdicts = {
|
|
159
|
+
"quota": side_quota(self.T, self.T + 6000),
|
|
160
|
+
"auth": side(cc_api_error(self.T, "authentication_failed", "Not logged in")),
|
|
161
|
+
"blocked": side(cc_api_error(self.T, "oauth_org_not_allowed")),
|
|
162
|
+
}
|
|
163
|
+
for cls, rec in verdicts.items():
|
|
164
|
+
kind, verdict = ar.classify_claude_sidechain(rec)
|
|
165
|
+
self.assertEqual((kind, verdict["class"]), ("error", cls))
|
|
166
|
+
_, quota = ar.classify_claude_sidechain(verdicts["quota"])
|
|
167
|
+
self.assertEqual((quota["reset"], quota["rtype"], quota["marked"]),
|
|
168
|
+
(self.T + 6000, "five_hour", ar.utc_iso(self.T)))
|
|
169
|
+
_, org = ar.classify_claude_sidechain(verdicts["blocked"])
|
|
170
|
+
self.assertEqual(org["rtype"], "oauth_org_not_allowed")
|
|
171
|
+
# the subagent's own business: its model's bucket, a transient error, the rest
|
|
172
|
+
for rec in (side_quota(self.T, self.T + 60, rtype="seven_day_opus"),
|
|
173
|
+
side(cc_api_error(self.T, "rate_limit", "You've reached your Fable limit.")),
|
|
174
|
+
side(cc_api_error(self.T, "rate_limit", "nope",
|
|
175
|
+
apiError="model_requires_usage_credits")),
|
|
176
|
+
side(cc_api_error(self.T, "rate_limit", "temporary capacity issue")),
|
|
177
|
+
side(cc_api_error(self.T, "overloaded")),
|
|
178
|
+
side(cc_api_error(self.T, "invalid_request")),
|
|
179
|
+
side(cc_api_error(self.T, None))):
|
|
180
|
+
self.assertIsNone(ar.classify_claude_sidechain(rec), rec.get("error"))
|
|
181
|
+
|
|
182
|
+
def test_a_real_reply_proves_service_and_nothing_else_does(self):
|
|
183
|
+
self.assertEqual(ar.classify_claude_sidechain(side_reply(self.T)), ("cancel", "sidechain"))
|
|
184
|
+
synthetic = side_reply(self.T, text="No response requested.")
|
|
185
|
+
synthetic["message"]["model"] = "<synthetic>"
|
|
186
|
+
for rec in (synthetic, side(cc_user(self.T, "please continue")),
|
|
187
|
+
side(cc_user(self.T, [{"type": "tool_result", "tool_use_id": "toolu_1",
|
|
188
|
+
"content": "ok"}])),
|
|
189
|
+
queue_op(self.T), {"type": "system", "timestamp": iso(self.T)}, "x", None):
|
|
190
|
+
self.assertIsNone(ar.classify_claude_sidechain(rec), rec)
|
|
191
|
+
# in the MAIN transcript a sidechain record stays ignored, as before
|
|
192
|
+
self.assertIsNone(ar.classify_claude(side_quota(self.T, self.T + 60)))
|
|
193
|
+
|
|
194
|
+
def test_agent_names_and_the_newest_reply(self):
|
|
195
|
+
path = "/p/agent-%s.jsonl" % AGENT2
|
|
196
|
+
self.assertEqual(ar.sidechain_agent(side_reply(self.T), path), AGENT)
|
|
197
|
+
self.assertEqual(ar.sidechain_agent({"agentId": "a b\n"}, path), AGENT2)
|
|
198
|
+
self.assertEqual(ar.sidechain_agent({}, "/p/agent-../../x.jsonl"), "-")
|
|
199
|
+
lines = main_lines([side_reply(self.T + 1, text="first"),
|
|
200
|
+
side_quota(self.T + 2, self.T + 60),
|
|
201
|
+
side_reply(self.T + 3, text="last"),
|
|
202
|
+
side(cc_user(self.T + 4, "tool output"))])
|
|
203
|
+
at, rec = ar.newest_answer(lines, ar.side_answer_at)
|
|
204
|
+
self.assertEqual((at, rec["timestamp"]), (self.T + 3.009, iso(self.T + 3)))
|
|
205
|
+
self.assertIsNone(ar.newest_answer(lines[1:2], ar.side_answer_at))
|
|
206
|
+
# found behind any run of lines that are none (a real main transcript had 193 in a
|
|
207
|
+
# row), and a line without the mark every reply holds is never even parsed
|
|
208
|
+
blobs = main_lines([request_blob()]) * 200
|
|
209
|
+
quiet = main_lines([side(tool_result(self.T + 5))]) * 50
|
|
210
|
+
with mock.patch.object(ar, "read_json_line", wraps=ar.read_json_line) as parse:
|
|
211
|
+
at, rec = ar.newest_answer(lines[:1] + blobs + quiet, ar.side_answer_at)
|
|
212
|
+
self.assertEqual((rec["timestamp"], parse.call_count), (iso(self.T + 1), 201))
|
|
213
|
+
|
|
214
|
+
def test_a_reply_answers_from_when_its_request_was_sent(self):
|
|
215
|
+
sent = {}
|
|
216
|
+
for rec in (api_request(self.T + 10, "req_1", AGENT),
|
|
217
|
+
api_request(self.T + 30, "req_2", AGENT)):
|
|
218
|
+
ar.note_request(rec, sent)
|
|
219
|
+
self.assertEqual(sent, {"req_1": self.T + 10.009, "req_2": self.T + 30.009})
|
|
220
|
+
self.assertEqual(ar.claude_served_at(side_reply(self.T + 25, ref="req_1"), sent),
|
|
221
|
+
self.T + 10.009)
|
|
222
|
+
# a request not on record (an older client names none) leaves the reply's own time
|
|
223
|
+
for ref in (None, "req_9", ["req_1"]):
|
|
224
|
+
rec = side_reply(self.T + 25)
|
|
225
|
+
rec["requestRef"] = ref
|
|
226
|
+
self.assertEqual(ar.claude_served_at(rec, sent), self.T + 25.009, ref)
|
|
227
|
+
self.assertEqual(ar.claude_served_at(side_reply(self.T + 25, ref="req_1")), self.T + 25.009)
|
|
228
|
+
self.assertIsNone(ar.side_answer_at(side_quota(self.T + 25, self.T + 60), sent))
|
|
229
|
+
# the newest ones are kept
|
|
230
|
+
with mock.patch.object(ar, "REQUESTS_KEPT", 2):
|
|
231
|
+
ar.note_request(api_request(self.T + 40, "req_3"), sent)
|
|
232
|
+
ar.note_request({"type": "api-request", "id": "req_4"}, sent) # undated: nothing
|
|
233
|
+
self.assertEqual(sorted(sent), ["req_2", "req_3"])
|
|
234
|
+
|
|
235
|
+
def test_the_latest_answer_is_not_always_the_last_line(self):
|
|
236
|
+
# a prompt typed while a stream ran: the reply, written last, answers from its request
|
|
237
|
+
sent = {"req_1": self.T + 10.009}
|
|
238
|
+
tr = ar.ClaudeTracker(self.T - 100)
|
|
239
|
+
tr.requests = sent
|
|
240
|
+
lines = main_lines([cc_user(self.T + 15, "also check the docs"),
|
|
241
|
+
main_reply(self.T + 40, "req_1")])
|
|
242
|
+
at, rec = ar.newest_answer(lines, lambda r: tr.answer_at(ar.classify_claude(r), r),
|
|
243
|
+
ar.ANSWER_MARKS)
|
|
244
|
+
self.assertEqual((at, rec["type"]), (self.T + 15.009, "user"))
|
|
245
|
+
# and the scan ends at the first record written before the best moment found
|
|
246
|
+
earlier = main_lines([side_reply(self.T - 100 + i) for i in range(100)])
|
|
247
|
+
turn = main_lines([api_request(self.T + 200, "req_x", AGENT)]
|
|
248
|
+
+ [side_reply(self.T + 201 + i, ref="req_x") for i in range(3)])
|
|
249
|
+
with mock.patch.object(ar, "read_json_line", wraps=ar.read_json_line) as parse:
|
|
250
|
+
at, _rec = ar.newest_answer(earlier + turn,
|
|
251
|
+
lambda r: ar.side_answer_at(r, {"req_x": self.T + 200.009}))
|
|
252
|
+
self.assertEqual((at, parse.call_count), (self.T + 200.009, 4))
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
class SidechainTrackerTests(unittest.TestCase):
|
|
256
|
+
LAUNCHED = 1792440000
|
|
257
|
+
|
|
258
|
+
def tracker(self):
|
|
259
|
+
return ar.ClaudeTracker(self.LAUNCHED - 2)
|
|
260
|
+
|
|
261
|
+
def test_a_rejection_must_prove_it_is_from_this_launch(self):
|
|
262
|
+
tr = self.tracker()
|
|
263
|
+
self.assertIsNone(tr.feed_side(side_quota(self.LAUNCHED - 3, self.LAUNCHED + 99), 1))
|
|
264
|
+
undated = side_quota(self.LAUNCHED + 5, self.LAUNCHED + 99)
|
|
265
|
+
del undated["timestamp"]
|
|
266
|
+
self.assertIsNone(tr.feed_side(undated, 1))
|
|
267
|
+
self.assertIsNone(tr.pending)
|
|
268
|
+
ev = tr.feed_side(side_quota(self.LAUNCHED + 5, self.LAUNCHED + 99), 7, AGENT)
|
|
269
|
+
self.assertEqual(ev[0], "error")
|
|
270
|
+
self.assertEqual((tr.pending["class"], tr.pending["src"], tr.pending["agent"],
|
|
271
|
+
tr.pending_since), ("quota", "sidechain", AGENT, 7))
|
|
272
|
+
# an old reply proves nothing new
|
|
273
|
+
self.assertIsNone(tr.feed_side(side_reply(self.LAUNCHED - 60), 8))
|
|
274
|
+
self.assertIsNotNone(tr.pending)
|
|
275
|
+
|
|
276
|
+
def test_the_replaced_processs_agents_never_reject_for_this_launch(self):
|
|
277
|
+
# A relaunch replaces a process whose workflow agents met the limit until it was gone,
|
|
278
|
+
# inside the second the shim's floored `date +%s` names as this launch (since is
|
|
279
|
+
# launched - 2): only a rejection from a second past the launch is this launch's own.
|
|
280
|
+
for offset in (-2, -1, 0):
|
|
281
|
+
tr = self.tracker()
|
|
282
|
+
self.assertIsNone(tr.feed_side(side_quota(self.LAUNCHED + offset, self.LAUNCHED + 99),
|
|
283
|
+
1), offset)
|
|
284
|
+
self.assertIsNone(tr.pending)
|
|
285
|
+
tr = self.tracker()
|
|
286
|
+
self.assertEqual(tr.feed_side(side_quota(self.LAUNCHED + 1, self.LAUNCHED + 99), 1)[0],
|
|
287
|
+
"error")
|
|
288
|
+
# its replies still answer, and the main thread keeps its two seconds of slack
|
|
289
|
+
tr = self.tracker()
|
|
290
|
+
tr.feed_side(side_reply(self.LAUNCHED - 1), 1)
|
|
291
|
+
self.assertEqual(tr.cleared, self.LAUNCHED - 0.991)
|
|
292
|
+
self.assertEqual(tr.feed(cc_quota(self.LAUNCHED - 1, resets=self.LAUNCHED + 99), 2)[0],
|
|
293
|
+
"error")
|
|
294
|
+
|
|
295
|
+
def test_a_reply_to_a_request_sent_before_the_rejection_answers_nothing(self):
|
|
296
|
+
launched = self.LAUNCHED
|
|
297
|
+
sent = {}
|
|
298
|
+
for rec in (api_request(launched + 10, "req_1", AGENT2),
|
|
299
|
+
api_request(launched + 30, "req_2", AGENT2)):
|
|
300
|
+
ar.note_request(rec, sent)
|
|
301
|
+
rejection = side_quota(launched + 20, launched + 9000)
|
|
302
|
+
# a stream the account accepted before the limit, ending after it
|
|
303
|
+
streamed = final_answer(launched + 25, AGENT2, "req_1")
|
|
304
|
+
tr = self.tracker()
|
|
305
|
+
tr.feed_side(rejection, 1)
|
|
306
|
+
self.assertIsNone(tr.feed_side(streamed, 2, AGENT2, sent))
|
|
307
|
+
self.assertEqual((tr.pending["class"], tr.cleared), ("quota", launched + 10.009))
|
|
308
|
+
tr = self.tracker() # read first, it answers nothing either
|
|
309
|
+
self.assertIsNone(tr.feed_side(streamed, 1, AGENT2, sent))
|
|
310
|
+
self.assertEqual(tr.feed_side(rejection, 2)[0], "error")
|
|
311
|
+
# a request sent after the limit, and served, is the answer
|
|
312
|
+
tr = self.tracker()
|
|
313
|
+
tr.feed_side(rejection, 1)
|
|
314
|
+
self.assertEqual(tr.feed_side(side_reply(launched + 40, AGENT2, ref="req_2"), 2, AGENT2,
|
|
315
|
+
sent), ("cancel", "sidechain"))
|
|
316
|
+
# with no request on record the reply's own time counts, as with older clients
|
|
317
|
+
tr = self.tracker()
|
|
318
|
+
tr.feed_side(rejection, 1)
|
|
319
|
+
self.assertEqual(tr.feed_side(streamed, 2, AGENT2), ("cancel", "sidechain"))
|
|
320
|
+
|
|
321
|
+
def test_the_main_threads_reply_counts_from_its_request_too(self):
|
|
322
|
+
launched = self.LAUNCHED
|
|
323
|
+
tr = self.tracker()
|
|
324
|
+
self.assertIsNone(tr.feed(api_request(launched + 10, "req_m1"), 1))
|
|
325
|
+
tr.feed_side(side_quota(launched + 20, launched + 9000), 1)
|
|
326
|
+
self.assertIsNone(tr.feed(main_reply(launched + 25, "req_m1"), 2))
|
|
327
|
+
self.assertEqual((tr.pending["class"], tr.cleared), ("quota", launched + 10.009))
|
|
328
|
+
tr.feed(api_request(launched + 30, "req_m2"), 3)
|
|
329
|
+
self.assertEqual(tr.feed(main_reply(launched + 40, "req_m2"), 4), ("cancel", "assistant"))
|
|
330
|
+
# what the prefilter skips the same way: request records are read, nothing else
|
|
331
|
+
tr, passed = self.tracker(), []
|
|
332
|
+
for raw in main_lines([cc_user(launched + 5, "write the section"),
|
|
333
|
+
api_request(launched + 10, "req_m1"),
|
|
334
|
+
main_reply(launched + 25, "req_m1"), tool_result(launched + 26)]):
|
|
335
|
+
if tr.wants(raw):
|
|
336
|
+
self.assertIsNone(tr.feed(json.loads(raw), 1))
|
|
337
|
+
else:
|
|
338
|
+
passed.append(raw)
|
|
339
|
+
self.assertEqual((len(passed), sorted(tr.requests)), (3, ["req_m1"]))
|
|
340
|
+
tr.passed_over(passed)
|
|
341
|
+
self.assertEqual(tr.cleared, launched + 10.009)
|
|
342
|
+
self.assertEqual(tr.feed_side(side_quota(launched + 20, launched + 9000), 5)[0], "error")
|
|
343
|
+
# a new conversation starts with no requests on record
|
|
344
|
+
tr.reset()
|
|
345
|
+
self.assertEqual((tr.requests, tr.cleared), ({}, 0.0))
|
|
346
|
+
|
|
347
|
+
def test_only_a_newer_reply_cancels_and_order_of_reading_does_not_matter(self):
|
|
348
|
+
rej, ok = side_quota(self.LAUNCHED + 10, self.LAUNCHED + 99), side_reply(self.LAUNCHED + 11)
|
|
349
|
+
tr = self.tracker() # the rejection read first, then the newer reply
|
|
350
|
+
tr.feed_side(rej, 1)
|
|
351
|
+
self.assertEqual(tr.feed_side(side_reply(self.LAUNCHED + 10), 2), None) # not newer
|
|
352
|
+
self.assertEqual(tr.feed_side(ok, 2), ("cancel", "sidechain"))
|
|
353
|
+
self.assertIsNone(tr.pending)
|
|
354
|
+
tr = self.tracker() # the newer reply read first: the rejection is answered already
|
|
355
|
+
self.assertIsNone(tr.feed_side(ok, 1))
|
|
356
|
+
self.assertIsNone(tr.feed_side(rej, 2))
|
|
357
|
+
self.assertIsNone(tr.pending)
|
|
358
|
+
# a rejection after that reply stands again
|
|
359
|
+
self.assertEqual(tr.feed_side(side_quota(self.LAUNCHED + 12, self.LAUNCHED + 99), 3)[0],
|
|
360
|
+
"error")
|
|
361
|
+
|
|
362
|
+
def test_the_newest_subagent_verdict_wins_and_a_repeat_changes_nothing(self):
|
|
363
|
+
tr = self.tracker()
|
|
364
|
+
first = side_quota(self.LAUNCHED + 10, self.LAUNCHED + 99)
|
|
365
|
+
tr.feed_side(first, 1, AGENT)
|
|
366
|
+
pending = tr.pending
|
|
367
|
+
# another agent meets the same limit later: the same verdict, still standing
|
|
368
|
+
self.assertIsNone(tr.feed_side(side_quota(self.LAUNCHED + 20, self.LAUNCHED + 99,
|
|
369
|
+
AGENT2), 5, AGENT2))
|
|
370
|
+
self.assertIs(tr.pending, pending)
|
|
371
|
+
self.assertEqual((tr.pending["at"], tr.pending["agent"], tr.pending_since),
|
|
372
|
+
(self.LAUNCHED + 20.009, AGENT, 1))
|
|
373
|
+
# ...so a reply between the two rejections answers only the first
|
|
374
|
+
self.assertIsNone(tr.feed_side(side_reply(self.LAUNCHED + 15), 6))
|
|
375
|
+
self.assertIs(tr.pending, pending)
|
|
376
|
+
# an older verdict read late never replaces a newer one; a newer class does
|
|
377
|
+
auth = side(cc_api_error(self.LAUNCHED + 5, "authentication_failed"))
|
|
378
|
+
self.assertIsNone(tr.feed_side(auth, 7))
|
|
379
|
+
auth = side(cc_api_error(self.LAUNCHED + 30, "authentication_failed"), AGENT3)
|
|
380
|
+
self.assertEqual(tr.feed_side(auth, 8, AGENT3)[0], "error")
|
|
381
|
+
self.assertEqual((tr.pending["class"], tr.pending["agent"], tr.pending_since),
|
|
382
|
+
("auth", AGENT3, 8))
|
|
383
|
+
|
|
384
|
+
def test_the_main_threads_own_account_verdict_stands(self):
|
|
385
|
+
tr = self.tracker()
|
|
386
|
+
tr.feed(cc_api_error(self.LAUNCHED + 10, "authentication_failed"), 1)
|
|
387
|
+
main = tr.pending
|
|
388
|
+
self.assertIsNone(tr.feed_side(side_quota(self.LAUNCHED + 20, self.LAUNCHED + 99), 2))
|
|
389
|
+
self.assertIsNone(tr.feed_side(side_reply(self.LAUNCHED + 30), 3)) # nor cancels it
|
|
390
|
+
self.assertIs(tr.pending, main)
|
|
391
|
+
self.assertIsNone(tr.fallback)
|
|
392
|
+
# and a main-thread verdict replaces a subagent's
|
|
393
|
+
tr = self.tracker()
|
|
394
|
+
tr.feed_side(side_quota(self.LAUNCHED + 20, self.LAUNCHED + 99), 1)
|
|
395
|
+
tr.feed(cc_api_error(self.LAUNCHED + 21, "authentication_failed"), 2)
|
|
396
|
+
self.assertEqual((tr.pending["class"], tr.pending.get("src")), ("auth", None))
|
|
397
|
+
self.assertIsNone(tr.feed_side(side_reply(self.LAUNCHED + 40), 3))
|
|
398
|
+
self.assertEqual(tr.pending["class"], "auth")
|
|
399
|
+
|
|
400
|
+
def test_main_thread_cancels_only_what_they_postdate(self):
|
|
401
|
+
for rec in (cc_reply(self.LAUNCHED + 30), cc_user(self.LAUNCHED + 30, "stop, I'm on it")):
|
|
402
|
+
tr = self.tracker()
|
|
403
|
+
tr.feed_side(side_quota(self.LAUNCHED + 20, self.LAUNCHED + 99), 1)
|
|
404
|
+
older = dict(rec, timestamp=iso(self.LAUNCHED + 19))
|
|
405
|
+
self.assertIsNone(tr.feed(older, 2))
|
|
406
|
+
self.assertIsNotNone(tr.pending)
|
|
407
|
+
self.assertEqual(tr.feed(rec, 3)[0], "cancel")
|
|
408
|
+
self.assertIsNone(tr.pending)
|
|
409
|
+
# the main thread's answer, read first, answers a rejection read later
|
|
410
|
+
self.assertIsNone(tr.feed_side(side_quota(self.LAUNCHED + 25, self.LAUNCHED + 99), 4))
|
|
411
|
+
self.assertIsNone(tr.pending)
|
|
412
|
+
# the auto-resume prompt is no human
|
|
413
|
+
tr = self.tracker()
|
|
414
|
+
tr.feed_side(side_quota(self.LAUNCHED + 20, self.LAUNCHED + 99), 1)
|
|
415
|
+
self.assertIsNone(tr.feed(cc_user(self.LAUNCHED + 30, ar.resume_prompt("quota")), 2))
|
|
416
|
+
self.assertIsNotNone(tr.pending)
|
|
417
|
+
|
|
418
|
+
def test_a_new_conversation_forgets_what_answered_the_old_one(self):
|
|
419
|
+
tr = self.tracker()
|
|
420
|
+
tr.feed_side(side_reply(self.LAUNCHED + 50), 1)
|
|
421
|
+
tr.reset()
|
|
422
|
+
self.assertEqual(tr.feed_side(side_quota(self.LAUNCHED + 40, self.LAUNCHED + 99), 2)[0],
|
|
423
|
+
"error")
|
|
424
|
+
|
|
425
|
+
def test_what_the_prefilter_passed_over_still_answers(self):
|
|
426
|
+
launched = self.LAUNCHED
|
|
427
|
+
tr = self.tracker()
|
|
428
|
+
lines = main_lines([cc_user(launched + 10, "run the workflow"),
|
|
429
|
+
cc_reply(launched + 20, "Started it."),
|
|
430
|
+
cc_user(launched + 590, "forget the workflow, do X"),
|
|
431
|
+
cc_reply(launched + 600, "Doing X.")]
|
|
432
|
+
+ [request_blob()] * 200 + [tool_result(launched + 601),
|
|
433
|
+
queue_op(launched + 602)])
|
|
434
|
+
self.assertFalse(any(tr.wants(raw) for raw in lines)) # nothing pending: all skipped
|
|
435
|
+
tr.passed_over(lines)
|
|
436
|
+
self.assertEqual(tr.cleared, launched + 600.009)
|
|
437
|
+
# an agent's rejection before that reply is answered, whenever it is read...
|
|
438
|
+
self.assertIsNone(tr.feed_side(side_quota(launched + 300, launched + 9000), 1))
|
|
439
|
+
self.assertIsNone(tr.pending)
|
|
440
|
+
# ...and one after it stands
|
|
441
|
+
self.assertEqual(tr.feed_side(side_quota(launched + 700, launched + 9000), 2)[0], "error")
|
|
442
|
+
|
|
443
|
+
def test_only_a_real_reply_or_the_user_typing_answers_there(self):
|
|
444
|
+
launched = self.LAUNCHED
|
|
445
|
+
later = (cc_slash(launched + 60) + cc_slash(launched + 61, "usage")
|
|
446
|
+
+ [synthetic(launched + 62), cc_user(launched + 63, ar.resume_prompt("quota")),
|
|
447
|
+
tool_result(launched + 64), queue_op(launched + 65)])
|
|
448
|
+
for answer in (cc_reply(launched + 50), cc_user(launched + 50, "never mind the workflow")):
|
|
449
|
+
with self.subTest(answer=answer["type"]):
|
|
450
|
+
tr = self.tracker()
|
|
451
|
+
tr.passed_over(main_lines([answer] + later))
|
|
452
|
+
self.assertEqual(tr.cleared, launched + 50.009)
|
|
453
|
+
# an undated answer dates nothing, one from before the launch is not this launch's
|
|
454
|
+
undated = cc_reply(launched + 70)
|
|
455
|
+
del undated["timestamp"]
|
|
456
|
+
tr = self.tracker()
|
|
457
|
+
tr.passed_over(main_lines([cc_reply(launched - 60), undated] + later))
|
|
458
|
+
self.assertEqual(tr.cleared, 0.0)
|
|
459
|
+
|
|
460
|
+
def test_what_the_prefilter_passed_over_cancels_nothing(self):
|
|
461
|
+
# read in one go: a reply the prefilter skipped, then the main thread's own rejection
|
|
462
|
+
tr = self.tracker()
|
|
463
|
+
passed = []
|
|
464
|
+
for raw in main_lines([cc_reply(self.LAUNCHED + 10),
|
|
465
|
+
cc_quota(self.LAUNCHED + 20, resets=self.LAUNCHED + 9000)]):
|
|
466
|
+
if tr.wants(raw):
|
|
467
|
+
tr.feed(json.loads(raw), 1)
|
|
468
|
+
else:
|
|
469
|
+
passed.append(raw)
|
|
470
|
+
tr.passed_over(passed)
|
|
471
|
+
self.assertEqual((tr.pending["class"], tr.cleared), ("quota", self.LAUNCHED + 10.009))
|
|
472
|
+
|
|
473
|
+
def test_a_slash_command_never_answers_the_accounts_rejection(self):
|
|
474
|
+
launched = self.LAUNCHED
|
|
475
|
+
tr = self.tracker()
|
|
476
|
+
tr.feed_side(side_quota(launched + 20, launched + 9000), 1)
|
|
477
|
+
pending = tr.pending
|
|
478
|
+
for rec in cc_slash(launched + 30) + [synthetic(launched + 31)]:
|
|
479
|
+
self.assertIsNone(tr.feed(rec, 2))
|
|
480
|
+
self.assertEqual((tr.pending, tr.cleared), (pending, 0.0))
|
|
481
|
+
self.assertEqual(tr.feed(cc_user(launched + 32, "stop, I'm on it"), 3), ("cancel", "user"))
|
|
482
|
+
# read before an older rejection, skipped or parsed, it answers that one neither
|
|
483
|
+
for parsed in (False, True):
|
|
484
|
+
tr = self.tracker()
|
|
485
|
+
if parsed:
|
|
486
|
+
for rec in cc_slash(launched + 30):
|
|
487
|
+
tr.feed(rec, 1)
|
|
488
|
+
else:
|
|
489
|
+
tr.passed_over(main_lines(cc_slash(launched + 30)))
|
|
490
|
+
self.assertEqual(tr.feed_side(side_quota(launched + 20, launched + 9000), 2)[0],
|
|
491
|
+
"error")
|
|
492
|
+
|
|
493
|
+
def test_a_command_that_can_fix_the_account_answers_its_rejection(self):
|
|
494
|
+
# /login after "Not logged in", /limit-reset after the limit: the user fixed the
|
|
495
|
+
# account, like a typed prompt; so does any command not known to leave it as it was
|
|
496
|
+
launched = self.LAUNCHED
|
|
497
|
+
for err, name in ((side(cc_api_error(launched + 20, "authentication_failed",
|
|
498
|
+
"Not logged in · Please run /login")), "login"),
|
|
499
|
+
(side_quota(launched + 20, launched + 9000), "limit-reset"),
|
|
500
|
+
(side_quota(launched + 20, launched + 9000), "extra-usage"),
|
|
501
|
+
(side_quota(launched + 20, launched + 9000), "brand-new-command")):
|
|
502
|
+
with self.subTest(command=name):
|
|
503
|
+
tr = self.tracker()
|
|
504
|
+
tr.feed_side(err, 1)
|
|
505
|
+
for rec in cc_slash(launched + 30, name):
|
|
506
|
+
tr.feed(rec, 2)
|
|
507
|
+
self.assertEqual((tr.pending, tr.cleared), (None, launched + 30.009))
|
|
508
|
+
tr = self.tracker() # skipped by the prefilter, read before the rejection
|
|
509
|
+
tr.passed_over(main_lines(cc_slash(launched + 30, name)))
|
|
510
|
+
self.assertIsNone(tr.feed_side(err, 3))
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
# A model's limit, as the main thread records it (claude 2.1.303): /model is how one handles it.
|
|
514
|
+
def model_limit(ts):
|
|
515
|
+
return cc_api_error(ts, "rate_limit",
|
|
516
|
+
"You've reached your Fable limit. Switch to another model to continue.")
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
class LiftedFallbackLatestResetTests(unittest.TestCase):
|
|
520
|
+
"""A quota verdict whose reset has passed is dropped; an agent's account verdict
|
|
521
|
+
outranks the main thread's model or transient one, which waits behind it until the main
|
|
522
|
+
thread itself moves on; several rejections of one class are marked with the latest
|
|
523
|
+
reset."""
|
|
524
|
+
L = 1792440000
|
|
525
|
+
|
|
526
|
+
def tracker(self):
|
|
527
|
+
return ar.ClaudeTracker(self.L - 2)
|
|
528
|
+
|
|
529
|
+
def test_a_quota_verdict_whose_reset_has_passed_is_dropped(self):
|
|
530
|
+
L, now = self.L, self.L + 600
|
|
531
|
+
tr = self.tracker()
|
|
532
|
+
self.assertIsNone(tr.feed(cc_quota(L + 10, resets=L + 300), now)) # main thread
|
|
533
|
+
self.assertIsNone(tr.pending)
|
|
534
|
+
self.assertEqual(tr.feed(cc_quota(L + 11), now)[0], "error") # no reset: stands
|
|
535
|
+
self.assertIsNone(tr.pending["reset"])
|
|
536
|
+
tr = self.tracker()
|
|
537
|
+
self.assertEqual(tr.feed(cc_quota(L + 12, resets=L + 9000), now)[0], "error")
|
|
538
|
+
tr = self.tracker()
|
|
539
|
+
self.assertIsNone(tr.feed_side(side_quota(L + 20, L + 300), now)) # an agent's
|
|
540
|
+
self.assertIsNone(tr.pending)
|
|
541
|
+
self.assertEqual(tr.feed_side(side_quota(L + 21, L + 9000), now)[0], "error")
|
|
542
|
+
# one whose reset passes while it waits is dropped then (Watcher.decide)
|
|
543
|
+
self.assertEqual(tr.expire(L + 8999), (False, None))
|
|
544
|
+
self.assertEqual(tr.expire(L + 9000), (True, None))
|
|
545
|
+
self.assertIsNone(tr.pending)
|
|
546
|
+
# nothing but a quota verdict has a reset to pass
|
|
547
|
+
tr = self.tracker()
|
|
548
|
+
tr.feed(cc_api_error(L + 30, "authentication_failed"), now)
|
|
549
|
+
self.assertEqual(tr.expire(L + 10 ** 6), (False, None))
|
|
550
|
+
self.assertEqual(tr.pending["class"], "auth")
|
|
551
|
+
|
|
552
|
+
def test_an_agents_account_verdict_outranks_the_main_threads_model_or_transient_one(self):
|
|
553
|
+
L = self.L
|
|
554
|
+
mains = {"model": model_limit, "concurrents": cc_concurrents,
|
|
555
|
+
"overloaded": lambda ts: cc_api_error(ts, "overloaded")}
|
|
556
|
+
for name, main in sorted(mains.items()):
|
|
557
|
+
for agent_first in (False, True):
|
|
558
|
+
with self.subTest(main=name, agent_first=agent_first):
|
|
559
|
+
tr = self.tracker()
|
|
560
|
+
if agent_first:
|
|
561
|
+
self.assertEqual(tr.feed_side(side_quota(L + 5, L + 9000), 1, AGENT)[0],
|
|
562
|
+
"error")
|
|
563
|
+
self.assertIsNone(tr.feed(main(L + 10), 2))
|
|
564
|
+
else:
|
|
565
|
+
self.assertEqual(tr.feed(main(L + 10), 1)[0], "error")
|
|
566
|
+
self.assertEqual(tr.feed_side(side_quota(L + 20, L + 9000), 2, AGENT)[0],
|
|
567
|
+
"error")
|
|
568
|
+
self.assertEqual((tr.pending["class"], tr.pending["src"], tr.pending["agent"],
|
|
569
|
+
tr.pending["reset"], tr.pending_since),
|
|
570
|
+
("quota", "sidechain", AGENT, L + 9000, 1 + (not agent_first)))
|
|
571
|
+
self.assertIn(tr.fallback["class"], ("model", "transient"))
|
|
572
|
+
self.assertIsNone(tr.fallback.get("src"))
|
|
573
|
+
# /model ends the model's limit or the failed turn, never the account's
|
|
574
|
+
self.assertIsNone(tr.feed(cc_slash(L + 30)[1], 3))
|
|
575
|
+
self.assertEqual((tr.pending["class"], tr.fallback), ("quota", None))
|
|
576
|
+
# an agent's failed login just the same
|
|
577
|
+
tr = self.tracker()
|
|
578
|
+
tr.feed(cc_api_error(L + 10, "overloaded"), 1)
|
|
579
|
+
tr.feed_side(side(cc_api_error(L + 20, "authentication_failed")), 2, AGENT2)
|
|
580
|
+
self.assertEqual((tr.pending["class"], tr.pending["agent"], tr.fallback["class"]),
|
|
581
|
+
("auth", AGENT2, "transient"))
|
|
582
|
+
# nothing waits behind a main-thread account verdict an agent's cannot outrank...
|
|
583
|
+
tr = self.tracker()
|
|
584
|
+
tr.feed(cc_api_error(L + 10, "authentication_failed"), 1)
|
|
585
|
+
tr.feed_side(side_quota(L + 20, L + 9000), 2)
|
|
586
|
+
self.assertEqual((tr.pending["class"], tr.fallback), ("auth", None))
|
|
587
|
+
# ...which outranks the main thread's own later throttle too
|
|
588
|
+
tr = self.tracker()
|
|
589
|
+
tr.feed(cc_quota(L + 10, resets=L + 9000), 1)
|
|
590
|
+
self.assertIsNone(tr.feed(cc_concurrents(L + 20), 2))
|
|
591
|
+
self.assertEqual((tr.pending["class"], tr.pending_since, tr.fallback["class"]),
|
|
592
|
+
("quota", 1, "transient"))
|
|
593
|
+
# a main-thread account verdict of the same class takes the agent's in
|
|
594
|
+
tr = self.tracker()
|
|
595
|
+
tr.feed(model_limit(L + 10), 1)
|
|
596
|
+
tr.feed_side(side_quota(L + 20, L + 9000, rtype="seven_day"), 2)
|
|
597
|
+
tr.feed(cc_quota(L + 25, resets=L + 3600), 3)
|
|
598
|
+
self.assertEqual((tr.pending["reset"], tr.pending["rtype"], tr.pending.get("src"),
|
|
599
|
+
tr.fallback), (L + 9000, "seven_day", None, None))
|
|
600
|
+
|
|
601
|
+
def test_what_it_outranks_ends_with_the_main_threads_own_next_word(self):
|
|
602
|
+
L = self.L
|
|
603
|
+
# the user typing after both: neither stands
|
|
604
|
+
tr = self.tracker()
|
|
605
|
+
tr.feed(model_limit(L + 10), 1)
|
|
606
|
+
tr.feed_side(side_quota(L + 20, L + 9000), 2)
|
|
607
|
+
self.assertEqual(tr.feed(cc_user(L + 30, "use sonnet then"), 3), ("cancel", "user"))
|
|
608
|
+
self.assertEqual((tr.pending, tr.fallback), (None, None))
|
|
609
|
+
# an agent's reply to a request sent after the account's verdict answers the account
|
|
610
|
+
# alone: the main thread, still stopped, has its own verdict back
|
|
611
|
+
tr = self.tracker()
|
|
612
|
+
tr.feed(cc_concurrents(L + 10), 1)
|
|
613
|
+
main = tr.pending
|
|
614
|
+
tr.feed_side(side_quota(L + 20, L + 9000), 2)
|
|
615
|
+
self.assertEqual(tr.feed_side(side_reply(L + 25, AGENT2), 3, AGENT2), ("error", main))
|
|
616
|
+
self.assertEqual((tr.pending, tr.pending_since, tr.fallback), (main, 3, None))
|
|
617
|
+
self.assertEqual(tr.feed(cc_slash(L + 30)[1], 4), ("cancel", "command"))
|
|
618
|
+
self.assertIsNone(tr.pending)
|
|
619
|
+
# a main-thread reply to a request sent before the account's verdict ends the main
|
|
620
|
+
# thread's own verdict only
|
|
621
|
+
tr = self.tracker()
|
|
622
|
+
tr.feed(api_request(L + 5, "req_m1"), 1)
|
|
623
|
+
tr.feed(model_limit(L + 10), 1)
|
|
624
|
+
tr.feed_side(side_quota(L + 20, L + 9000), 2)
|
|
625
|
+
self.assertIsNone(tr.feed(main_reply(L + 25, "req_m1"), 3))
|
|
626
|
+
self.assertEqual((tr.pending["class"], tr.pending["src"], tr.fallback),
|
|
627
|
+
("quota", "sidechain", None))
|
|
628
|
+
# a quota verdict that lapses lets the one behind it stand...
|
|
629
|
+
tr = self.tracker()
|
|
630
|
+
tr.feed(cc_concurrents(L + 10), 1)
|
|
631
|
+
tr.feed_side(side_quota(L + 20, L + 900), 2, AGENT3)
|
|
632
|
+
self.assertEqual((tr.pending["class"], tr.fallback["class"]), ("quota", "transient"))
|
|
633
|
+
dropped, promoted = tr.expire(L + 900)
|
|
634
|
+
self.assertTrue(dropped)
|
|
635
|
+
self.assertIs(tr.pending, promoted)
|
|
636
|
+
self.assertEqual((promoted["class"], promoted.get("src"), tr.pending_since),
|
|
637
|
+
("transient", None, L + 900))
|
|
638
|
+
# ...unless the main thread moved on meanwhile
|
|
639
|
+
tr = self.tracker()
|
|
640
|
+
tr.feed(cc_concurrents(L + 10), 1)
|
|
641
|
+
tr.feed_side(side_quota(L + 20, L + 900), 2)
|
|
642
|
+
tr.feed(cc_slash(L + 30)[1], 3)
|
|
643
|
+
self.assertEqual(tr.expire(L + 900), (True, None))
|
|
644
|
+
self.assertIsNone(tr.pending)
|
|
645
|
+
# a new conversation forgets it
|
|
646
|
+
tr = self.tracker()
|
|
647
|
+
tr.feed(model_limit(L + 10), 1)
|
|
648
|
+
tr.feed_side(side_quota(L + 20, L + 9000), 2)
|
|
649
|
+
tr.reset()
|
|
650
|
+
self.assertIsNone(tr.fallback)
|
|
651
|
+
|
|
652
|
+
def test_one_classes_rejections_are_marked_with_the_latest_reset(self):
|
|
653
|
+
L = self.L
|
|
654
|
+
five, week = (L + 3600, "five_hour"), (L + 86400, "seven_day")
|
|
655
|
+
for first, second in ((five, week), (week, five)):
|
|
656
|
+
with self.subTest(first=first[1]):
|
|
657
|
+
# the main thread: the newest record stands, with the latest reset
|
|
658
|
+
tr = self.tracker()
|
|
659
|
+
tr.feed(cc_quota(L + 10, first[1], resets=first[0]), 1)
|
|
660
|
+
tr.feed(cc_quota(L + 20, second[1], resets=second[0]), 2)
|
|
661
|
+
self.assertEqual((tr.pending["reset"], tr.pending["rtype"], tr.pending["at"],
|
|
662
|
+
tr.pending_since), (L + 86400, "seven_day", L + 20.009, 1))
|
|
663
|
+
# agents, in either order of reading
|
|
664
|
+
for stamps in ((L + 10, L + 20), (L + 20, L + 10)):
|
|
665
|
+
tr = self.tracker()
|
|
666
|
+
tr.feed_side(side_quota(stamps[0], first[0], rtype=first[1]), 1, AGENT)
|
|
667
|
+
self.assertIsNone(tr.feed_side(side_quota(stamps[1], second[0], AGENT2,
|
|
668
|
+
rtype=second[1]), 2, AGENT2))
|
|
669
|
+
self.assertEqual((tr.pending["reset"], tr.pending["rtype"], tr.pending["at"],
|
|
670
|
+
tr.pending["agent"]),
|
|
671
|
+
(L + 86400, "seven_day", L + 20.009, AGENT))
|
|
672
|
+
# an agent's later reset reaches the main thread's verdict too
|
|
673
|
+
tr = self.tracker()
|
|
674
|
+
tr.feed(cc_quota(L + 10, first[1], resets=first[0]), 1)
|
|
675
|
+
tr.feed_side(side_quota(L + 20, second[0], rtype=second[1]), 2)
|
|
676
|
+
self.assertEqual((tr.pending["reset"], tr.pending["rtype"], tr.pending.get("src")),
|
|
677
|
+
(L + 86400, "seven_day", None))
|
|
678
|
+
# a rejection with no reset takes nothing away
|
|
679
|
+
tr = self.tracker()
|
|
680
|
+
tr.feed(cc_quota(L + 10, resets=L + 3600), 1)
|
|
681
|
+
tr.feed(cc_quota(L + 20), 2)
|
|
682
|
+
self.assertEqual(tr.pending["reset"], L + 3600)
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
# =====================================================================================
|
|
686
|
+
# the watcher
|
|
687
|
+
# =====================================================================================
|
|
688
|
+
|
|
689
|
+
class SidechainWatcherTests(SidechainCase):
|
|
690
|
+
def test_a_workflow_agents_five_hour_limit_moves_the_idle_session(self):
|
|
691
|
+
reset = int(self.t0) + 7200
|
|
692
|
+
w, r = self.start(["--dangerously-skip-permissions", "fix the flaky test"])
|
|
693
|
+
self.agent_file([side_reply(self.launched + 3), side_quota(self.t0 - 1, reset)])
|
|
694
|
+
# a workflow's journal and an agent's metadata are no transcripts
|
|
695
|
+
wf = self.proj / SID / "subagents" / "workflows" / "wf_7f3a9c"
|
|
696
|
+
(wf / "journal.jsonl").write_text(compact([side_quota(self.t0 - 1, reset, AGENT2)]))
|
|
697
|
+
(wf / ("agent-%s.meta.json" % AGENT2)).write_text(compact([side_quota(self.t0 - 1, reset)]))
|
|
698
|
+
self.assertEqual(w.run(), 0)
|
|
699
|
+
prompt = ar.resume_prompt("quota")
|
|
700
|
+
self.assertEqual(len(r.probes), 1)
|
|
701
|
+
probe = r.probes[0]
|
|
702
|
+
self.assertGreaterEqual(probe["t"] - self.t0, 5.0) # GRACE, as for the main thread
|
|
703
|
+
self.assertEqual(probe["argv"], [self.self_path, "--dangerously-skip-permissions",
|
|
704
|
+
"--resume", SID, prompt])
|
|
705
|
+
self.assertEqual(probe["env"]["CLAUDE_MULTIACC_AR_AVOID"], "acct-01:%d" % reset)
|
|
706
|
+
self.assertEqual(r.signals, [(self.PID, signal.SIGTERM)])
|
|
707
|
+
self.assert_sequence(r)
|
|
708
|
+
kv, argv = self.relaunch_of(r)
|
|
709
|
+
self.assertEqual(argv, ["--dangerously-skip-permissions", "--resume", SID, prompt])
|
|
710
|
+
self.assertEqual({k: kv[k] for k in ("class", "acct", "reset", "rtype", "marked", "sid",
|
|
711
|
+
"depth", "avoid")},
|
|
712
|
+
{"class": "quota", "acct": "acct-01", "reset": str(reset),
|
|
713
|
+
"rtype": "five_hour", "marked": ar.utc_iso(self.t0 - 1), "sid": SID,
|
|
714
|
+
"depth": "1", "avoid": "acct-01:%d" % reset})
|
|
715
|
+
self.assertEqual(self.events(), ["watch", "detect", "switch"])
|
|
716
|
+
self.assertIn(" autoresume detect provider=claude chain=%s class=quota code=rate_limit "
|
|
717
|
+
"sid=%s acct=acct-01 reset=%d src=sidechain agent=%s"
|
|
718
|
+
% (kv["chain"], SID, reset, AGENT), self.log_lines()[1])
|
|
719
|
+
for line in self.log_lines():
|
|
720
|
+
self.assertEqual(line.split()[1], "autoresume")
|
|
721
|
+
self.assertNotIn("session limit", line)
|
|
722
|
+
self.assertEqual(sorted(os.path.basename(p) for p in w.side),
|
|
723
|
+
["agent-%s.jsonl" % AGENT])
|
|
724
|
+
|
|
725
|
+
def test_a_subagents_failed_login_moves_it_too(self):
|
|
726
|
+
w, r = self.start(["-c"])
|
|
727
|
+
self.agent_file([side(cc_api_error(self.t0, "authentication_failed",
|
|
728
|
+
"Not logged in · Please run /login"))], workflow=None)
|
|
729
|
+
w.run()
|
|
730
|
+
until = int(r.probes[0]["t"]) + 3600
|
|
731
|
+
self.assertEqual(r.probes[0]["env"]["CLAUDE_MULTIACC_AR_AVOID"], "acct-01:%d" % until)
|
|
732
|
+
kv, argv = self.relaunch_of(r)
|
|
733
|
+
self.assertEqual(argv, ["--resume", SID, ar.resume_prompt("auth")])
|
|
734
|
+
self.assertEqual((kv["class"], kv["reset"], kv["rtype"]), ("auth", "", ""))
|
|
735
|
+
self.assertIn("class=auth code=authentication_failed", self.log_lines()[1])
|
|
736
|
+
self.assertIn("src=sidechain agent=%s" % AGENT, self.log_lines()[1])
|
|
737
|
+
|
|
738
|
+
def test_an_organization_refusal_in_a_subagent_parks_like_the_main_threads(self):
|
|
739
|
+
w, r = self.start([])
|
|
740
|
+
self.agent_file([side(cc_api_error(self.t0, "oauth_org_not_allowed"))])
|
|
741
|
+
w.run()
|
|
742
|
+
kv, _ = self.relaunch_of(r)
|
|
743
|
+
self.assertEqual((kv["class"], kv["rtype"]), ("blocked", "oauth_org_not_allowed"))
|
|
744
|
+
|
|
745
|
+
def test_a_subagents_own_model_transient_or_other_errors_never_move_it(self):
|
|
746
|
+
w, r = self.start([])
|
|
747
|
+
self.agent_file([side_quota(self.t0, int(self.t0) + 999, rtype="seven_day_opus"),
|
|
748
|
+
side(cc_api_error(self.t0, "rate_limit",
|
|
749
|
+
"You've reached your Fable limit. Switch models")),
|
|
750
|
+
side(cc_api_error(self.t0, "overloaded")),
|
|
751
|
+
side(cc_api_error(self.t0, "invalid_request", "bad tool schema")),
|
|
752
|
+
side(cc_user(self.t0 + 1, "carry on"))])
|
|
753
|
+
self.stop_at(r, self.t0 + 30)
|
|
754
|
+
self.assertEqual(w.run(), 0)
|
|
755
|
+
self.assert_untouched(r)
|
|
756
|
+
self.assertEqual(self.events(), ["watch"]) # not even a `never` line
|
|
757
|
+
|
|
758
|
+
def test_a_rejection_from_before_the_launch_never_fires(self):
|
|
759
|
+
# a resumed conversation's agents met a limit in an earlier launch
|
|
760
|
+
w, r = self.start(["--resume", SID])
|
|
761
|
+
self.agent_file([side_quota(self.launched - 3, int(self.t0) + 999)])
|
|
762
|
+
self.stop_at(r, self.t0 + 20)
|
|
763
|
+
self.assertEqual(w.run(), 0)
|
|
764
|
+
self.assert_untouched(r)
|
|
765
|
+
self.assertEqual(self.events(), ["watch"])
|
|
766
|
+
|
|
767
|
+
def test_a_transcript_last_written_before_the_launch_is_followed_from_its_end(self):
|
|
768
|
+
reset = int(self.t0) + 999
|
|
769
|
+
# mis-dated on purpose: were this file read from its start, the rejection would count
|
|
770
|
+
path = self.agent_file([side_quota(self.t0, reset)], mtime=self.launched - 60)
|
|
771
|
+
w, r = self.start([])
|
|
772
|
+
size = path.stat().st_size
|
|
773
|
+
r.at(self.t0 + 10, lambda: fixtures.append(path, [side_quota(self.t0 + 10, reset)]))
|
|
774
|
+
self.assertEqual(w.run(), 0)
|
|
775
|
+
self.assertGreaterEqual(r.probes[0]["t"], self.t0 + 15)
|
|
776
|
+
self.assertGreater(self.tail_of(w, path.name).off, size)
|
|
777
|
+
kv, _ = self.relaunch_of(r)
|
|
778
|
+
self.assertEqual((kv["class"], kv["marked"]), ("quota", ar.utc_iso(self.t0 + 10)))
|
|
779
|
+
|
|
780
|
+
def test_a_later_reply_of_that_agent_or_another_cancels(self):
|
|
781
|
+
for later in (AGENT, AGENT2):
|
|
782
|
+
with self.subTest(agent=later):
|
|
783
|
+
log = self.root / "selection.log"
|
|
784
|
+
if log.exists():
|
|
785
|
+
log.unlink()
|
|
786
|
+
w, r = self.start([])
|
|
787
|
+
path = self.agent_file([side_quota(self.t0, int(self.t0) + 999)],
|
|
788
|
+
workflow="wf_" + later)
|
|
789
|
+
other = self.agent_file([side_reply(self.launched + 1, AGENT2)],
|
|
790
|
+
agent=AGENT2, workflow="wf_" + later + "x")
|
|
791
|
+
target = path if later == AGENT else other
|
|
792
|
+
r.at(self.t0 + 2, lambda target=target, later=later: fixtures.append(
|
|
793
|
+
target, [side_reply(self.t0 + 2, later)]))
|
|
794
|
+
self.stop_at(r, self.t0 + 20)
|
|
795
|
+
self.assertEqual(w.run(), 0)
|
|
796
|
+
self.assert_untouched(r)
|
|
797
|
+
self.assertEqual(self.events(), ["watch", "detect"])
|
|
798
|
+
for p in (path, other):
|
|
799
|
+
p.unlink()
|
|
800
|
+
|
|
801
|
+
def test_a_later_main_thread_reply_or_the_user_typing_cancels(self):
|
|
802
|
+
for rec in (cc_reply(self.t0 + 2, "The workflow finished; all green."),
|
|
803
|
+
cc_user(self.t0 + 2, "never mind the workflow, look at this")):
|
|
804
|
+
with self.subTest(type=rec["type"]):
|
|
805
|
+
w, r = self.start([])
|
|
806
|
+
tpath = self.proj / (SID + ".jsonl")
|
|
807
|
+
agent = self.agent_file([side_quota(self.t0, int(self.t0) + 999)])
|
|
808
|
+
r.at(self.t0 + 2, lambda rec=rec: fixtures.append(tpath, [rec]))
|
|
809
|
+
self.stop_at(r, self.t0 + 20)
|
|
810
|
+
self.assertEqual(w.run(), 0)
|
|
811
|
+
self.assert_untouched(r)
|
|
812
|
+
for path in (tpath, agent):
|
|
813
|
+
path.unlink()
|
|
814
|
+
|
|
815
|
+
def test_main_thread_words_older_than_the_rejection_do_not_cancel(self):
|
|
816
|
+
# written (or read) late, a reply from before the rejection answers nothing
|
|
817
|
+
w, r = self.start([])
|
|
818
|
+
tpath = self.proj / (SID + ".jsonl")
|
|
819
|
+
self.agent_file([side_quota(self.t0, int(self.t0) + 999)])
|
|
820
|
+
r.at(self.t0 + 2, lambda: fixtures.append(tpath, [cc_reply(self.t0 - 1, "earlier"),
|
|
821
|
+
cc_user(self.t0 - 1, "earlier")]))
|
|
822
|
+
self.assertEqual(w.run(), 0)
|
|
823
|
+
self.assertEqual(len(r.probes), 1)
|
|
824
|
+
self.assertEqual(self.events(), ["watch", "detect", "switch"])
|
|
825
|
+
|
|
826
|
+
def test_an_answered_rejection_never_fires_whichever_file_is_read_first(self):
|
|
827
|
+
reset = int(self.t0) + 999
|
|
828
|
+
w, r = self.start([])
|
|
829
|
+
# one agent was rejected; another got a reply after that (the limit had reset)
|
|
830
|
+
self.agent_file([side_quota(self.t0 - 2, reset)], workflow="wf_b")
|
|
831
|
+
self.agent_file([side_reply(self.t0 - 1, AGENT2)], agent=AGENT2, workflow="wf_a")
|
|
832
|
+
self.stop_at(r, self.t0 + 20)
|
|
833
|
+
self.assertEqual(w.run(), 0)
|
|
834
|
+
self.assert_untouched(r)
|
|
835
|
+
self.assertEqual(self.events(), ["watch"]) # never even shown as pending
|
|
836
|
+
|
|
837
|
+
def test_many_agents_meeting_the_same_limit_probe_once(self):
|
|
838
|
+
reset = int(self.t0) + 999
|
|
839
|
+
w, r = self.start([])
|
|
840
|
+
path = self.agent_file([side_quota(self.t0 - 1, reset)])
|
|
841
|
+
self.agent_file([side_quota(self.t0 - 0.5, reset, AGENT2)], agent=AGENT2,
|
|
842
|
+
workflow="wf_2")
|
|
843
|
+
real_run = r.run
|
|
844
|
+
|
|
845
|
+
def run(argv, timeout, env=None, cwd=None):
|
|
846
|
+
if argv[0] == self.self_path: # one more agent is rejected during the probe
|
|
847
|
+
fixtures.append(path, [side_quota(self.t0 + 5, reset)])
|
|
848
|
+
return real_run(argv, timeout, env, cwd)
|
|
849
|
+
r.run = run
|
|
850
|
+
self.assertEqual(w.run(), 0)
|
|
851
|
+
self.assertEqual(len(r.probes), 1)
|
|
852
|
+
self.assertEqual(self.events(), ["watch", "detect", "switch"])
|
|
853
|
+
kv, _ = self.relaunch_of(r)
|
|
854
|
+
self.assertEqual(kv["marked"], ar.utc_iso(self.t0 - 1))
|
|
855
|
+
|
|
856
|
+
def test_a_reply_during_the_probe_wins(self):
|
|
857
|
+
w, r = self.start([])
|
|
858
|
+
path = self.agent_file([side_quota(self.t0, int(self.t0) + 999)])
|
|
859
|
+
real_run = r.run
|
|
860
|
+
|
|
861
|
+
def run(argv, timeout, env=None, cwd=None):
|
|
862
|
+
if argv[0] == self.self_path and not r.probes:
|
|
863
|
+
fixtures.append(path, [side_reply(self.t0 + 5)])
|
|
864
|
+
return real_run(argv, timeout, env, cwd)
|
|
865
|
+
r.run = run
|
|
866
|
+
self.stop_at(r, self.t0 + 20)
|
|
867
|
+
w.run()
|
|
868
|
+
self.assertEqual(len(r.probes), 1)
|
|
869
|
+
self.assertEqual((r.signals, r.sends, self.relaunch_files()), ([], [], []))
|
|
870
|
+
|
|
871
|
+
def test_no_room_holds_like_the_main_thread(self):
|
|
872
|
+
w, r = self.start([], env={"CLAUDE_MULTIACC_AR_HOLD_REPROBE": "5"})
|
|
873
|
+
self.agent_file([side_quota(self.t0, int(self.t0) + 999)])
|
|
874
|
+
r.probe_out = "pick=acct-01 tier=soft\n"
|
|
875
|
+
self.stop_at(r, self.t0 + 16.5)
|
|
876
|
+
self.assertEqual(w.run(), 0)
|
|
877
|
+
self.assertEqual(len(r.probes), 3)
|
|
878
|
+
self.assertEqual((r.signals, r.sends, self.relaunch_files()), ([], [], []))
|
|
879
|
+
self.assertEqual(self.events(), ["watch", "detect", "hold"])
|
|
880
|
+
|
|
881
|
+
def test_the_replaced_processs_last_rejections_never_move_the_relaunch(self):
|
|
882
|
+
# The watcher of a RELAUNCHED session (on acct-01, the account it moved to). The process
|
|
883
|
+
# it replaced ran a workflow whose agents kept meeting the old account's limit, in a
|
|
884
|
+
# burst, until it was gone: inside the second `date +%s` floored to this launch.
|
|
885
|
+
reset = int(self.t0) + 9000
|
|
886
|
+
for later in (False, True):
|
|
887
|
+
with self.subTest(own_agent_rejected_later=later):
|
|
888
|
+
log = self.root / "selection.log"
|
|
889
|
+
if log.exists():
|
|
890
|
+
log.unlink()
|
|
891
|
+
launched = self.launched = int(self.t0) - 1
|
|
892
|
+
state = self.write_state(["--dangerously-skip-permissions", "--resume", SID],
|
|
893
|
+
depth=1, avoid="acct-02:%d" % reset)
|
|
894
|
+
self.registry()
|
|
895
|
+
tpath = self.transcript([cc_user(launched - 3600, "run the release workflow"),
|
|
896
|
+
cc_reply(launched - 3590, "Started it; waiting for it."),
|
|
897
|
+
cc_user(launched + 1, ar.resume_prompt("quota"))])
|
|
898
|
+
old = [self.agent_file([side_reply(launched - 400),
|
|
899
|
+
side_quota(launched - 2, reset)], mtime=launched - 2),
|
|
900
|
+
self.agent_file([side_reply(launched - 60, AGENT2),
|
|
901
|
+
side_quota(launched - 1, reset, AGENT2)],
|
|
902
|
+
agent=AGENT2, workflow="wf_2", mtime=launched - 1),
|
|
903
|
+
self.agent_file([side_quota(launched, reset, AGENT3)], agent=AGENT3,
|
|
904
|
+
workflow="wf_3", mtime=launched)]
|
|
905
|
+
w, r = self.watcher(state)
|
|
906
|
+
r.probe_out = "pick=acct-03 tier=eligible\n"
|
|
907
|
+
# the resumed conversation's first reply is slow (a cold cache on this account)
|
|
908
|
+
r.at(self.t0 + 12, lambda tpath=tpath: fixtures.append(
|
|
909
|
+
tpath, [cc_reply(self.t0 + 12, "Picking up where it left off.")]))
|
|
910
|
+
if later: # this launch's own agent meets a limit: that one counts
|
|
911
|
+
r.at(self.t0 + 20, lambda: write(old[2], [side_quota(self.t0 + 20, reset,
|
|
912
|
+
AGENT3)]))
|
|
913
|
+
self.stop_at(r, self.t0 + 40)
|
|
914
|
+
self.assertEqual(w.run(), 0)
|
|
915
|
+
if later:
|
|
916
|
+
self.assertEqual(len(r.probes), 1)
|
|
917
|
+
self.assertEqual(self.events(), ["watch", "detect", "switch"])
|
|
918
|
+
self.assertEqual(self.relaunch_of(r)[0]["marked"], ar.utc_iso(self.t0 + 20))
|
|
919
|
+
else:
|
|
920
|
+
self.assert_untouched(r)
|
|
921
|
+
self.assertEqual(self.events(), ["watch"])
|
|
922
|
+
for path in old + [tpath]:
|
|
923
|
+
path.unlink()
|
|
924
|
+
|
|
925
|
+
def test_a_reply_the_account_accepted_before_the_limit_answers_nothing(self):
|
|
926
|
+
# One agent's stream ran on past another agent's rejection (by minutes in session
|
|
927
|
+
# 5526ddd6, 2026-10-19). Here it is that agent's final answer: the workflow then waits
|
|
928
|
+
# for the rejected one, and nothing more is written.
|
|
929
|
+
reset = int(self.t0) + 9000
|
|
930
|
+
w, r = self.start([])
|
|
931
|
+
self.agent_file([side_reply(self.launched + 3), side_quota(self.t0 - 1, reset)])
|
|
932
|
+
other = self.agent_file([api_request(self.launched + 3, "req_b1", AGENT2),
|
|
933
|
+
side_reply(self.launched + 4, AGENT2, ref="req_b1"),
|
|
934
|
+
api_request(self.t0 - 30, "req_b2", AGENT2)],
|
|
935
|
+
agent=AGENT2, workflow="wf_b")
|
|
936
|
+
r.at(self.t0 + 2, lambda: write(other, [final_answer(self.t0 + 2, AGENT2, "req_b2")]))
|
|
937
|
+
self.assertEqual(w.run(), 0)
|
|
938
|
+
self.assertEqual(len(r.probes), 1)
|
|
939
|
+
self.assertEqual(self.events(), ["watch", "detect", "switch"])
|
|
940
|
+
self.assertEqual(self.relaunch_of(r)[0]["marked"], ar.utc_iso(self.t0 - 1))
|
|
941
|
+
|
|
942
|
+
def test_a_reply_to_a_request_sent_after_the_limit_still_cancels(self):
|
|
943
|
+
w, r = self.start([])
|
|
944
|
+
self.agent_file([side_quota(self.t0 - 1, int(self.t0) + 9000)])
|
|
945
|
+
other = self.agent_file([api_request(self.t0 - 30, "req_b1", AGENT2)], agent=AGENT2,
|
|
946
|
+
workflow="wf_b")
|
|
947
|
+
# the limit is lifted (a reset), the agent's next request is served
|
|
948
|
+
r.at(self.t0 + 1, lambda: write(other, [final_answer(self.t0 + 1, AGENT2, "req_b1"),
|
|
949
|
+
api_request(self.t0 + 1, "req_b2", AGENT2)]))
|
|
950
|
+
r.at(self.t0 + 3, lambda: write(other, [side_reply(self.t0 + 3, AGENT2, ref="req_b2")]))
|
|
951
|
+
self.stop_at(r, self.t0 + 30)
|
|
952
|
+
self.assertEqual(w.run(), 0)
|
|
953
|
+
self.assert_untouched(r)
|
|
954
|
+
self.assertEqual(self.events(), ["watch", "detect"])
|
|
955
|
+
|
|
956
|
+
def test_a_command_that_fixes_the_account_keeps_the_session(self):
|
|
957
|
+
# /login after the agents' "Not logged in"; /limit-reset while the watcher holds the
|
|
958
|
+
# agents' limit for want of room, before room appears
|
|
959
|
+
for err, name, hold in ((side(cc_api_error(self.t0, "authentication_failed",
|
|
960
|
+
"Not logged in · Please run /login")),
|
|
961
|
+
"login", False),
|
|
962
|
+
(side_quota(self.t0, int(self.t0) + 9000), "limit-reset", True)):
|
|
963
|
+
with self.subTest(command=name):
|
|
964
|
+
log = self.root / "selection.log"
|
|
965
|
+
if log.exists():
|
|
966
|
+
log.unlink()
|
|
967
|
+
w, r = self.start([], env={"CLAUDE_MULTIACC_AR_HOLD_REPROBE": "10"})
|
|
968
|
+
tpath = self.proj / (SID + ".jsonl")
|
|
969
|
+
agent = self.agent_file([err])
|
|
970
|
+
at = self.t0 + (8 if hold else 2)
|
|
971
|
+
if hold:
|
|
972
|
+
r.probe_out = "pick=acct-01 tier=soft\n"
|
|
973
|
+
r.at(self.t0 + 9, lambda r=r: setattr(r, "probe_out",
|
|
974
|
+
"pick=acct-02 tier=eligible\n"))
|
|
975
|
+
r.at(at, lambda at=at, name=name: fixtures.append(tpath, cc_slash(at, name)))
|
|
976
|
+
self.stop_at(r, self.t0 + 40)
|
|
977
|
+
self.assertEqual(w.run(), 0)
|
|
978
|
+
self.assertEqual((len(r.probes), r.signals, r.sends, self.relaunch_files()),
|
|
979
|
+
(int(hold), [], [], []))
|
|
980
|
+
self.assertEqual(self.events(), ["watch", "detect"] + ["hold"] * hold)
|
|
981
|
+
for path in (tpath, agent):
|
|
982
|
+
path.unlink()
|
|
983
|
+
|
|
984
|
+
def test_a_session_change_resets_the_subagent_tails(self):
|
|
985
|
+
reset = int(self.t0) + 999
|
|
986
|
+
w, r = self.start([])
|
|
987
|
+
old = self.agent_file([side_quota(self.t0, reset)])
|
|
988
|
+
self.transcript([cc_user(self.t0 + 1, "a new task")], sid=SID2)
|
|
989
|
+
# /clear before GRACE: the old conversation's agents are no longer this session's
|
|
990
|
+
r.at(self.t0 + 1.5, lambda: self.registry(sid=SID2))
|
|
991
|
+
r.at(self.t0 + 3, lambda: fixtures.append(old, [side_quota(self.t0 + 3, reset)]))
|
|
992
|
+
r.at(self.t0 + 20, lambda: self.agent_file([side_quota(self.t0 + 20, reset)], sid=SID2))
|
|
993
|
+
self.assertEqual(w.run(), 0)
|
|
994
|
+
self.assertEqual(len(r.probes), 1)
|
|
995
|
+
self.assertGreaterEqual(r.probes[0]["t"], self.t0 + 25)
|
|
996
|
+
kv, argv = self.relaunch_of(r)
|
|
997
|
+
self.assertEqual((kv["sid"], argv[-2]), (SID2, SID2))
|
|
998
|
+
self.assertEqual(self.events(), ["watch", "detect", "detect", "switch"])
|
|
999
|
+
self.assertTrue(w.side and all(SID2 in path and SID not in path for path in w.side))
|
|
1000
|
+
|
|
1001
|
+
|
|
1002
|
+
class LiftedFallbackLatestResetWatcherTests(SidechainCase):
|
|
1003
|
+
def fresh(self):
|
|
1004
|
+
for path in [self.root / "selection.log", self.proj / (SID + ".jsonl")] \
|
|
1005
|
+
+ list((self.proj / SID).glob("subagents/**/agent-*.jsonl")):
|
|
1006
|
+
if path.exists():
|
|
1007
|
+
path.unlink()
|
|
1008
|
+
|
|
1009
|
+
def test_a_limit_that_has_reset_already_moves_nothing(self):
|
|
1010
|
+
# a watcher that reads a rejection only once its reset has passed (restarted onto
|
|
1011
|
+
# the session, or a slow replay): the account serves again
|
|
1012
|
+
for where in ("main", "agent"):
|
|
1013
|
+
with self.subTest(where=where):
|
|
1014
|
+
self.fresh()
|
|
1015
|
+
main = [cc_user(self.launched + 1, "go"), cc_reply(self.launched + 2, "ok")]
|
|
1016
|
+
if where == "main":
|
|
1017
|
+
main.append(cc_quota(self.t0 - 2, resets=int(self.t0) - 1))
|
|
1018
|
+
w, r = self.start([], main=main)
|
|
1019
|
+
if where == "agent":
|
|
1020
|
+
self.agent_file([side_quota(self.t0 - 2, int(self.t0) - 1)])
|
|
1021
|
+
self.stop_at(r, self.t0 + 20)
|
|
1022
|
+
self.assertEqual(w.run(), 0)
|
|
1023
|
+
self.assert_untouched(r)
|
|
1024
|
+
self.assertEqual(self.events(), ["watch"])
|
|
1025
|
+
|
|
1026
|
+
def test_a_verdict_whose_reset_passes_in_a_hold_is_dropped(self):
|
|
1027
|
+
reset = int(self.t0) + 9
|
|
1028
|
+
for where in ("main", "agent"):
|
|
1029
|
+
with self.subTest(where=where):
|
|
1030
|
+
self.fresh()
|
|
1031
|
+
main = [cc_user(self.launched + 1, "go"), cc_reply(self.launched + 2, "ok")]
|
|
1032
|
+
if where == "main":
|
|
1033
|
+
main.append(cc_quota(self.t0, resets=reset))
|
|
1034
|
+
w, r = self.start([], main=main, env={"CLAUDE_MULTIACC_AR_HOLD_REPROBE": "2"})
|
|
1035
|
+
if where == "agent":
|
|
1036
|
+
self.agent_file([side_quota(self.t0, reset)])
|
|
1037
|
+
r.probe_out = "pick=acct-01 tier=soft\n" # nowhere to go: hold
|
|
1038
|
+
r.at(self.t0 + 12, lambda r=r: setattr(r, "probe_out",
|
|
1039
|
+
"pick=acct-02 tier=eligible\n"))
|
|
1040
|
+
self.stop_at(r, self.t0 + 30)
|
|
1041
|
+
self.assertEqual(w.run(), 0)
|
|
1042
|
+
self.assertTrue(r.probes)
|
|
1043
|
+
self.assertTrue(all(p["t"] < reset for p in r.probes), r.probes)
|
|
1044
|
+
self.assertEqual((r.signals, r.sends, self.relaunch_files()), ([], [], []))
|
|
1045
|
+
self.assertEqual(self.events(), ["watch", "detect", "hold"])
|
|
1046
|
+
|
|
1047
|
+
def test_model_ends_the_main_threads_model_verdict_but_not_the_agents_limit(self):
|
|
1048
|
+
reset = int(self.t0) + 7200
|
|
1049
|
+
w, r = self.start([], main=[cc_user(self.launched + 1, "run the release workflow"),
|
|
1050
|
+
cc_reply(self.launched + 2, "Started it."),
|
|
1051
|
+
model_limit(self.t0)])
|
|
1052
|
+
self.agent_file([side_quota(self.t0, reset)])
|
|
1053
|
+
tpath = self.proj / (SID + ".jsonl")
|
|
1054
|
+
r.at(self.t0 + 2, lambda: fixtures.append(tpath, cc_slash(self.t0 + 2)))
|
|
1055
|
+
self.stop_at(r, self.t0 + 40)
|
|
1056
|
+
self.assertEqual(w.run(), 0)
|
|
1057
|
+
self.assertEqual(len(r.probes), 1)
|
|
1058
|
+
# the agent's verdict outranked the model's from the start: its grace is its own
|
|
1059
|
+
self.assertGreaterEqual(r.probes[0]["t"], self.t0 + 5)
|
|
1060
|
+
self.assertLess(r.probes[0]["t"], self.t0 + 7)
|
|
1061
|
+
kv, _ = self.relaunch_of(r)
|
|
1062
|
+
self.assertEqual((kv["class"], kv["reset"], kv["rtype"]),
|
|
1063
|
+
("quota", str(reset), "five_hour"))
|
|
1064
|
+
self.assertEqual(self.events(), ["watch", "detect", "detect", "switch"])
|
|
1065
|
+
detects = [line for line in self.log_lines() if line.split()[2] == "detect"]
|
|
1066
|
+
self.assertIn("class=model", detects[0])
|
|
1067
|
+
self.assertIn("class=quota", detects[1])
|
|
1068
|
+
self.assertIn("src=sidechain agent=%s" % AGENT, detects[1])
|
|
1069
|
+
|
|
1070
|
+
def test_agents_meeting_the_limit_move_a_session_its_throttled_main_thread_held(self):
|
|
1071
|
+
# 2026-10-20, session 6: the main thread's "concurrents" rejection (read as a model's
|
|
1072
|
+
# limit) waited on its busy agents while, from three minutes later, they were all
|
|
1073
|
+
# meeting the five-hour limit; the session moved six minutes after that rejection
|
|
1074
|
+
reset = int(self.t0) + 9000
|
|
1075
|
+
w, r = self.start([], main=[cc_user(self.launched + 1, "run the i18n workflows"),
|
|
1076
|
+
cc_reply(self.launched + 2, "Started them."),
|
|
1077
|
+
cc_concurrents(self.t0)])
|
|
1078
|
+
self.agent_file([side_reply(self.launched + 3, AGENT2)], agent=AGENT2,
|
|
1079
|
+
workflow="wf_2", mtime=self.t0 + 900) # writing all along
|
|
1080
|
+
path = self.agent_file([side_reply(self.launched + 3)])
|
|
1081
|
+
r.at(self.t0 + 180, lambda: fixtures.append(path, [side_quota(self.t0 + 180, reset)]))
|
|
1082
|
+
self.assertEqual(w.run(), 0)
|
|
1083
|
+
self.assertEqual(len(r.probes), 1)
|
|
1084
|
+
self.assertGreaterEqual(r.probes[0]["t"], self.t0 + 185)
|
|
1085
|
+
self.assertLess(r.probes[0]["t"], self.t0 + 195) # its grace, not the agents' quiet
|
|
1086
|
+
kv, argv = self.relaunch_of(r)
|
|
1087
|
+
self.assertEqual((kv["class"], kv["reset"], kv["rtype"]),
|
|
1088
|
+
("quota", str(reset), "five_hour"))
|
|
1089
|
+
self.assertEqual(argv, ["--resume", SID, ar.resume_prompt("quota")])
|
|
1090
|
+
self.assertEqual(self.events(), ["watch", "detect", "detect", "switch"])
|
|
1091
|
+
detects = [line for line in self.log_lines() if line.split()[2] == "detect"]
|
|
1092
|
+
self.assertIn("class=transient code=rate_limit", detects[0])
|
|
1093
|
+
self.assertIn("class=quota", detects[1])
|
|
1094
|
+
self.assertIn("src=sidechain agent=%s" % AGENT, detects[1])
|
|
1095
|
+
|
|
1096
|
+
def test_a_throttled_main_thread_waits_for_its_working_agents_and_keeps_the_account(self):
|
|
1097
|
+
w, r = self.start([], main=[cc_user(self.launched + 1, "run the i18n workflows"),
|
|
1098
|
+
cc_reply(self.launched + 2, "Started them."),
|
|
1099
|
+
cc_concurrents(self.t0)])
|
|
1100
|
+
self.agent_file([side_reply(self.launched + 3)], mtime=self.t0 + 100) # until ~t0+100
|
|
1101
|
+
self.assertEqual(w.run(), 0)
|
|
1102
|
+
self.assertEqual(len(r.probes), 1)
|
|
1103
|
+
self.assertGreaterEqual(r.probes[0]["t"], self.t0 + 120) # 20 s of agent quiet
|
|
1104
|
+
self.assertEqual(r.probes[0]["env"]["CLAUDE_MULTIACC_AR_AVOID"], "") # nothing avoided
|
|
1105
|
+
kv, argv = self.relaunch_of(r)
|
|
1106
|
+
self.assertEqual((kv["class"], kv["avoid"]), ("transient", ""))
|
|
1107
|
+
self.assertEqual(argv, ["--resume", SID, ar.resume_prompt("transient")])
|
|
1108
|
+
|
|
1109
|
+
def test_the_latest_reset_of_one_limit_is_the_one_marked(self):
|
|
1110
|
+
five, week = int(self.t0) + 3600, int(self.t0) + 86400
|
|
1111
|
+
w, r = self.start([])
|
|
1112
|
+
# the weekly limit, met first, sits in the transcript read second
|
|
1113
|
+
self.agent_file([side_quota(self.t0 - 1, week, rtype="seven_day")], mtime=self.t0 - 2)
|
|
1114
|
+
self.agent_file([side_quota(self.t0 - 0.5, five, AGENT2)], agent=AGENT2,
|
|
1115
|
+
workflow="wf_2", mtime=self.t0 - 1)
|
|
1116
|
+
self.assertEqual(w.run(), 0)
|
|
1117
|
+
self.assertEqual(len(r.probes), 1)
|
|
1118
|
+
kv, _ = self.relaunch_of(r)
|
|
1119
|
+
self.assertEqual((kv["class"], kv["reset"], kv["rtype"]),
|
|
1120
|
+
("quota", str(week), "seven_day"))
|
|
1121
|
+
|
|
1122
|
+
|
|
1123
|
+
class SidechainReplayTests(SidechainCase):
|
|
1124
|
+
"""A watcher (re)started onto a live session reads everything written since its launch,
|
|
1125
|
+
hours of it: an agent's old rejection must never stop a session served since."""
|
|
1126
|
+
|
|
1127
|
+
def setUp(self):
|
|
1128
|
+
super().setUp()
|
|
1129
|
+
self.launched = int(self.t0) - 3600 # launched an hour ago; the watcher starts now
|
|
1130
|
+
|
|
1131
|
+
def restarted(self, main, agent=None):
|
|
1132
|
+
state = self.write_state(["--dangerously-skip-permissions"])
|
|
1133
|
+
self.registry()
|
|
1134
|
+
self.transcript(main)
|
|
1135
|
+
if agent:
|
|
1136
|
+
self.agent_file(agent)
|
|
1137
|
+
return self.watcher(state)
|
|
1138
|
+
|
|
1139
|
+
def replay(self, last_reply):
|
|
1140
|
+
"""Read 8 KB a tick: the rejected agent, written last (its re-run request is in
|
|
1141
|
+
flight), comes first; a busy agent whose newest reply is ``last_reply`` is reached
|
|
1142
|
+
only well after GRACE."""
|
|
1143
|
+
w, r = self.restarted([cc_user(self.launched + 10, "run the workflow"),
|
|
1144
|
+
cc_reply(self.launched + 20, "Started it.")])
|
|
1145
|
+
self.agent_file([side_quota(self.t0 - 1800, int(self.t0) + 9000), request_blob()],
|
|
1146
|
+
mtime=self.t0 - 60)
|
|
1147
|
+
busy = [side_reply(self.launched + 100 + i, AGENT2, "x" * 900) for i in range(100)]
|
|
1148
|
+
path = self.agent_file(busy + [side_reply(last_reply, AGENT2)], agent=AGENT2,
|
|
1149
|
+
workflow="wf_busy", mtime=self.t0 - 590)
|
|
1150
|
+
self.assertGreater(path.stat().st_size, 10 * 8192)
|
|
1151
|
+
return w, r
|
|
1152
|
+
|
|
1153
|
+
def test_a_rejection_waits_for_the_rest_of_the_replay(self):
|
|
1154
|
+
w, r = self.replay(self.t0 - 600) # served since the rejection
|
|
1155
|
+
self.stop_at(r, self.t0 + 30)
|
|
1156
|
+
with mock.patch.object(ar, "SIDE_BYTES", 8192):
|
|
1157
|
+
self.assertEqual(w.run(), 0)
|
|
1158
|
+
self.assert_untouched(r)
|
|
1159
|
+
self.assertEqual(self.events(), ["watch", "detect"]) # pending, then answered
|
|
1160
|
+
|
|
1161
|
+
def test_a_rejection_nothing_answers_still_moves_once_all_is_read(self):
|
|
1162
|
+
w, r = self.replay(self.launched + 300) # the busy agent's last reply came before it
|
|
1163
|
+
real_run, unread = r.run, []
|
|
1164
|
+
|
|
1165
|
+
def run(argv, timeout, env=None, cwd=None):
|
|
1166
|
+
if argv[0] == self.self_path:
|
|
1167
|
+
unread.extend(p for p, t in w.side.items() if t.off != os.path.getsize(p))
|
|
1168
|
+
return real_run(argv, timeout, env, cwd)
|
|
1169
|
+
r.run = run
|
|
1170
|
+
with mock.patch.object(ar, "SIDE_BYTES", 8192):
|
|
1171
|
+
self.assertEqual(w.run(), 0)
|
|
1172
|
+
self.assertEqual(len(r.probes), 1)
|
|
1173
|
+
self.assertEqual(unread, []) # probed only once the whole replay was read
|
|
1174
|
+
self.assertGreaterEqual(r.probes[0]["t"], self.t0 + 10)
|
|
1175
|
+
self.assertEqual(self.events(), ["watch", "detect", "switch"])
|
|
1176
|
+
|
|
1177
|
+
def test_a_main_thread_served_since_the_rejection_is_left_alone(self):
|
|
1178
|
+
# 2026-10-19: a workflow agent rejected 30 min ago, the main thread replying since
|
|
1179
|
+
w, r = self.restarted(
|
|
1180
|
+
[cc_user(self.launched + 10, "run the workflow"),
|
|
1181
|
+
api_request(self.launched + 11, "req_m1"),
|
|
1182
|
+
main_reply(self.launched + 20, "req_m1", "Started it."),
|
|
1183
|
+
cc_user(self.t0 - 600, "forget the workflow, do X"),
|
|
1184
|
+
api_request(self.t0 - 599, "req_m2"),
|
|
1185
|
+
main_reply(self.t0 - 590, "req_m2", "Doing X.")]
|
|
1186
|
+
+ [request_blob()] * 200 + [tool_result(self.t0 - 589)] + cc_slash(self.t0 - 60),
|
|
1187
|
+
[side_reply(self.launched + 30), side_quota(self.t0 - 1800, int(self.t0) + 9000)])
|
|
1188
|
+
self.stop_at(r, self.t0 + 30)
|
|
1189
|
+
self.assertEqual(w.run(), 0)
|
|
1190
|
+
self.assert_untouched(r)
|
|
1191
|
+
self.assertEqual(self.events(), ["watch"]) # never even pending
|
|
1192
|
+
|
|
1193
|
+
def test_a_main_thread_stream_that_spanned_the_rejection_still_moves(self):
|
|
1194
|
+
# the main thread's last reply ended after the agents' limit, but the account had
|
|
1195
|
+
# accepted its request before it; idle since
|
|
1196
|
+
reset = int(self.t0) + 9000
|
|
1197
|
+
w, r = self.restarted(
|
|
1198
|
+
[cc_user(self.launched + 10, "run the workflow, then write the report"),
|
|
1199
|
+
api_request(self.t0 - 1900, "req_m1"),
|
|
1200
|
+
main_reply(self.t0 - 1700, "req_m1", "Report written; waiting for the workflow."),
|
|
1201
|
+
queue_op(self.t0 - 1650)],
|
|
1202
|
+
[side_reply(self.launched + 30), side_quota(self.t0 - 1800, reset)])
|
|
1203
|
+
self.assertEqual(w.run(), 0)
|
|
1204
|
+
self.assertEqual(len(r.probes), 1)
|
|
1205
|
+
self.assertEqual(self.events(), ["watch", "detect", "switch"])
|
|
1206
|
+
|
|
1207
|
+
def test_an_agents_stream_accepted_before_the_rejection_answers_nothing_on_replay(self):
|
|
1208
|
+
reset = int(self.t0) + 9000
|
|
1209
|
+
w, r = self.restarted([cc_user(self.launched + 10, "run the workflow"),
|
|
1210
|
+
cc_reply(self.launched + 20, "Started it."),
|
|
1211
|
+
queue_op(self.t0 - 1700)],
|
|
1212
|
+
[side_reply(self.launched + 30), side_quota(self.t0 - 1800, reset)])
|
|
1213
|
+
self.agent_file([api_request(self.t0 - 1830, "req_b1", AGENT2),
|
|
1214
|
+
final_answer(self.t0 - 1500, AGENT2, "req_b1")], agent=AGENT2,
|
|
1215
|
+
workflow="wf_b")
|
|
1216
|
+
self.assertEqual(w.run(), 0)
|
|
1217
|
+
self.assertEqual(len(r.probes), 1)
|
|
1218
|
+
self.assertEqual(self.events(), ["watch", "detect", "switch"])
|
|
1219
|
+
|
|
1220
|
+
def test_a_prompt_typed_since_counts_while_its_reply_is_on_its_way(self):
|
|
1221
|
+
w, r = self.restarted(
|
|
1222
|
+
[cc_user(self.launched + 10, "run the workflow"),
|
|
1223
|
+
cc_reply(self.launched + 20, "Started it."),
|
|
1224
|
+
cc_user(self.t0 - 2, "the limit has reset, carry on")],
|
|
1225
|
+
[side_quota(self.t0 - 1800, int(self.t0) + 9000)])
|
|
1226
|
+
self.stop_at(r, self.t0 + 30)
|
|
1227
|
+
self.assertEqual(w.run(), 0)
|
|
1228
|
+
self.assert_untouched(r)
|
|
1229
|
+
self.assertEqual(self.events(), ["watch"])
|
|
1230
|
+
|
|
1231
|
+
def test_a_session_stuck_since_the_rejection_still_moves(self):
|
|
1232
|
+
# sessions 5 and 8 as a restarted watcher reads them: the main thread's last reply
|
|
1233
|
+
# came before the agents' limit; since then only the paused workflow's notices,
|
|
1234
|
+
# /model, and words of the client's own
|
|
1235
|
+
reset = int(self.t0) + 9000
|
|
1236
|
+
w, r = self.restarted(
|
|
1237
|
+
[cc_user(self.launched + 10, "run the workflow"),
|
|
1238
|
+
cc_reply(self.launched + 20, "Started it; waiting for the workflow."),
|
|
1239
|
+
queue_op(self.t0 - 1700)] + cc_slash(self.t0 - 600) + [synthetic(self.t0 - 599)],
|
|
1240
|
+
[side_reply(self.launched + 30), side_quota(self.t0 - 1800, reset)])
|
|
1241
|
+
self.assertEqual(w.run(), 0)
|
|
1242
|
+
self.assertEqual(len(r.probes), 1)
|
|
1243
|
+
self.assertEqual(self.events(), ["watch", "detect", "switch"])
|
|
1244
|
+
kv, _ = self.relaunch_of(r)
|
|
1245
|
+
self.assertEqual((kv["class"], kv["reset"], kv["marked"]),
|
|
1246
|
+
("quota", str(reset), ar.utc_iso(self.t0 - 1800)))
|
|
1247
|
+
|
|
1248
|
+
def test_an_agent_transcript_found_after_the_main_threads_answer_is_answered(self):
|
|
1249
|
+
# whatever order things are read in: the reply was read (and skipped) first
|
|
1250
|
+
w, r = self.start([], main=[cc_user(self.launched + 1, "run the release workflow"),
|
|
1251
|
+
cc_reply(self.t0 - 2, "On it.")])
|
|
1252
|
+
r.at(self.t0 + 6, lambda: self.agent_file([side_quota(self.t0 - 3, int(self.t0) + 999)]))
|
|
1253
|
+
self.stop_at(r, self.t0 + 30)
|
|
1254
|
+
self.assertEqual(w.run(), 0)
|
|
1255
|
+
self.assert_untouched(r)
|
|
1256
|
+
self.assertEqual(self.events(), ["watch"])
|
|
1257
|
+
|
|
1258
|
+
|
|
1259
|
+
class SidechainDiscoveryTests(SidechainCase):
|
|
1260
|
+
def test_new_agents_are_looked_for_every_subagent_scan_seconds(self):
|
|
1261
|
+
w, r = self.start([], env={"CLAUDE_MULTIACC_AR_SUBAGENT_SCAN": "3"})
|
|
1262
|
+
self.assertEqual(w.cfg.subagent_scan, 3.0)
|
|
1263
|
+
self.assertEqual(ar.Config("claude", {}).subagent_scan, 5.0)
|
|
1264
|
+
self.agent_file([side_reply(self.launched + 1)])
|
|
1265
|
+
w.read()
|
|
1266
|
+
self.assertEqual(len(w.side), 1)
|
|
1267
|
+
later = self.agent_file([side_quota(self.t0, int(self.t0) + 999, AGENT2)],
|
|
1268
|
+
agent=AGENT2, workflow=None)
|
|
1269
|
+
r.t += 2
|
|
1270
|
+
w.read()
|
|
1271
|
+
self.assertEqual((len(w.side), w.tracker.pending), (1, None))
|
|
1272
|
+
r.t += 1
|
|
1273
|
+
w.read()
|
|
1274
|
+
self.assertEqual(self.tail_of(w, later.name).off, later.stat().st_size)
|
|
1275
|
+
self.assertEqual(w.tracker.pending["agent"], AGENT2)
|
|
1276
|
+
|
|
1277
|
+
def test_only_the_most_recently_written_are_followed(self):
|
|
1278
|
+
reset = int(self.t0) + 999
|
|
1279
|
+
w, r = self.start([])
|
|
1280
|
+
oldest = self.agent_file([side_quota(self.t0, reset)], workflow="wf_1",
|
|
1281
|
+
mtime=self.launched + 1)
|
|
1282
|
+
self.agent_file([side_reply(self.launched + 1, AGENT2)], agent=AGENT2,
|
|
1283
|
+
workflow="wf_2", mtime=self.launched + 2)
|
|
1284
|
+
newest = self.agent_file([side_reply(self.launched + 1, AGENT3)], agent=AGENT3,
|
|
1285
|
+
workflow="wf_3", mtime=self.launched + 3)
|
|
1286
|
+
with mock.patch.object(ar, "SIDE_FILES", 2):
|
|
1287
|
+
w.read()
|
|
1288
|
+
self.assertEqual([os.path.basename(p) for p in w.side_order],
|
|
1289
|
+
[newest.name, "agent-%s.jsonl" % AGENT2])
|
|
1290
|
+
self.assertIsNone(w.tracker.pending)
|
|
1291
|
+
# the rejected agent writes again: it is among the newest, read from its start
|
|
1292
|
+
fixtures.append(oldest, [side_quota(self.t0 + 1, reset)])
|
|
1293
|
+
r.t += 5
|
|
1294
|
+
w.read()
|
|
1295
|
+
self.assertEqual([os.path.basename(p) for p in w.side_order],
|
|
1296
|
+
[oldest.name, newest.name])
|
|
1297
|
+
self.assertEqual((w.tracker.pending["class"], w.tracker.pending["at"]),
|
|
1298
|
+
("quota", self.t0 + 1.009))
|
|
1299
|
+
# a file no longer followed keeps its place while it is there
|
|
1300
|
+
self.assertEqual(self.tail_of(w, "agent-%s.jsonl" % AGENT2).off,
|
|
1301
|
+
(self.proj / SID / "subagents" / "workflows" / "wf_2" /
|
|
1302
|
+
("agent-%s.jsonl" % AGENT2)).stat().st_size)
|
|
1303
|
+
|
|
1304
|
+
def test_the_walk_is_bounded(self):
|
|
1305
|
+
w, r = self.start([])
|
|
1306
|
+
for i in range(6):
|
|
1307
|
+
self.agent_file([side_reply(self.launched + 1, "a%d" % i)], agent="a%d" % i,
|
|
1308
|
+
workflow="wf_%d" % i)
|
|
1309
|
+
with mock.patch.object(ar, "SIDE_SCAN", 3):
|
|
1310
|
+
w.read()
|
|
1311
|
+
self.assertEqual(len(w.side), 3)
|
|
1312
|
+
|
|
1313
|
+
def test_the_bytes_read_per_tick_are_bounded(self):
|
|
1314
|
+
reset = int(self.t0) + 999
|
|
1315
|
+
w, r = self.start([])
|
|
1316
|
+
filler = [side_reply(self.launched + 1 + i * 0.01, text="x" * 900) for i in range(60)]
|
|
1317
|
+
big = self.agent_file(filler + [side_quota(self.t0, reset)])
|
|
1318
|
+
small = self.agent_file([side_reply(self.launched + 1, AGENT2)], agent=AGENT2,
|
|
1319
|
+
workflow="wf_small")
|
|
1320
|
+
reads = 0
|
|
1321
|
+
with mock.patch.object(ar, "SIDE_BYTES", 8192):
|
|
1322
|
+
while w.tracker.pending is None and reads < 100:
|
|
1323
|
+
w.read()
|
|
1324
|
+
reads += 1
|
|
1325
|
+
self.assertLessEqual(sum(t.got for t in w.side.values()), 8192)
|
|
1326
|
+
self.assertGreater(reads, big.stat().st_size // 8192)
|
|
1327
|
+
self.assertEqual(w.tracker.pending["class"], "quota")
|
|
1328
|
+
self.assertEqual(self.tail_of(w, small.name).off, small.stat().st_size)
|
|
1329
|
+
|
|
1330
|
+
def test_subagents_wait_for_the_main_transcript_to_be_caught_up(self):
|
|
1331
|
+
main = [cc_reply(self.launched + 1, "y" * 2000) for _ in range(8)]
|
|
1332
|
+
w, r = self.start([], main=main)
|
|
1333
|
+
self.agent_file([side_quota(self.t0, int(self.t0) + 999)])
|
|
1334
|
+
with mock.patch.object(ar.JsonlTail, "MAX_READ", 4096):
|
|
1335
|
+
w.read()
|
|
1336
|
+
self.assertGreater(w.tail.behind, 0)
|
|
1337
|
+
self.assertEqual((w.side, w.tracker.pending), ({}, None))
|
|
1338
|
+
for _ in range(10):
|
|
1339
|
+
w.read()
|
|
1340
|
+
self.assertEqual(w.tail.behind, 0)
|
|
1341
|
+
self.assertEqual(w.tracker.pending["src"], "sidechain")
|
|
1342
|
+
|
|
1343
|
+
|
|
1344
|
+
# =====================================================================================
|
|
1345
|
+
# the main thread's own work in flight
|
|
1346
|
+
# =====================================================================================
|
|
1347
|
+
|
|
1348
|
+
def block(ts, ref, content, stop, mid="msg_01Main"):
|
|
1349
|
+
"""One block of a main-thread reply as claude 2.1.304 writes it: a record per content
|
|
1350
|
+
block, each naming its request; "stop_reason": null while the reply still streams."""
|
|
1351
|
+
rec = main_reply(ts, ref)
|
|
1352
|
+
rec["message"].update(id=mid, content=[content], stop_reason=stop)
|
|
1353
|
+
return rec
|
|
1354
|
+
|
|
1355
|
+
|
|
1356
|
+
def tool_use(tid="toolu_1", command="./migrate.sh"):
|
|
1357
|
+
return {"type": "tool_use", "id": tid, "name": "Bash", "input": {"command": command}}
|
|
1358
|
+
|
|
1359
|
+
|
|
1360
|
+
def result_of(ts, tid="toolu_1"):
|
|
1361
|
+
return cc_user(ts, [{"type": "tool_result", "tool_use_id": tid, "content": "ok"}],
|
|
1362
|
+
toolUseResult={"stdout": "ok"})
|
|
1363
|
+
|
|
1364
|
+
|
|
1365
|
+
THINKING = {"type": "thinking", "thinking": "", "signature": "c2ln"}
|
|
1366
|
+
TEXT = {"type": "text", "text": "Done."}
|
|
1367
|
+
|
|
1368
|
+
|
|
1369
|
+
class MainThreadBusyTests(unittest.TestCase):
|
|
1370
|
+
T = 1792440000
|
|
1371
|
+
|
|
1372
|
+
def busy(self, records):
|
|
1373
|
+
return ar.main_thread_busy(main_lines(records))
|
|
1374
|
+
|
|
1375
|
+
def test_what_the_main_thread_is_in_the_middle_of(self):
|
|
1376
|
+
t, sent = self.T, api_request(self.T, "req_m1")
|
|
1377
|
+
cases = {
|
|
1378
|
+
"request": [cc_user(t - 1, "migrate the db"), sent],
|
|
1379
|
+
"stream": [sent, block(t + 20, "req_m1", THINKING, None)],
|
|
1380
|
+
"tool": [sent, block(t + 20, "req_m1", THINKING, "tool_use"),
|
|
1381
|
+
block(t + 21, "req_m1", tool_use(), "tool_use")],
|
|
1382
|
+
}
|
|
1383
|
+
for want, records in cases.items():
|
|
1384
|
+
self.assertEqual(self.busy(records), want, want)
|
|
1385
|
+
# a reply whose tool_use block is still to come (its thinking says tool_use already)
|
|
1386
|
+
self.assertEqual(self.busy([sent, block(t + 20, "req_m1", THINKING, "tool_use")]),
|
|
1387
|
+
"stream")
|
|
1388
|
+
# two tools at once: one result is not both
|
|
1389
|
+
two = [sent, block(t + 20, "req_m1", tool_use("toolu_1"), "tool_use"),
|
|
1390
|
+
block(t + 20, "req_m1", tool_use("toolu_2", "make test"), "tool_use"),
|
|
1391
|
+
result_of(t + 30, "toolu_1")]
|
|
1392
|
+
self.assertEqual(self.busy(two), "tool")
|
|
1393
|
+
self.assertIsNone(self.busy(two + [result_of(t + 90, "toolu_2")]))
|
|
1394
|
+
|
|
1395
|
+
def test_by_request_whatever_the_order_of_the_file(self):
|
|
1396
|
+
t = self.T
|
|
1397
|
+
# 2.1.304 writes a request as it sends it: before the prompt that led to it...
|
|
1398
|
+
self.assertEqual(self.busy([api_request(t, "req_m1"), cc_user(t, "migrate the db")]),
|
|
1399
|
+
"request")
|
|
1400
|
+
# ...and before the previous reply's last blocks and tool results
|
|
1401
|
+
self.assertEqual(self.busy([api_request(t, "req_m1"), api_request(t + 5, "req_m2"),
|
|
1402
|
+
block(t + 4, "req_m1", tool_use(), "tool_use"),
|
|
1403
|
+
result_of(t + 5)]), "request")
|
|
1404
|
+
self.assertIsNone(self.busy([api_request(t, "req_m1"), api_request(t + 5, "req_m2"),
|
|
1405
|
+
block(t + 4, "req_m1", tool_use(), "tool_use"),
|
|
1406
|
+
result_of(t + 5),
|
|
1407
|
+
block(t + 9, "req_m2", TEXT, "end_turn")]))
|
|
1408
|
+
|
|
1409
|
+
def test_idle_or_untold(self):
|
|
1410
|
+
t, sent = self.T, api_request(self.T, "req_m1")
|
|
1411
|
+
compact_boundary = {"type": "system", "subtype": "compact_boundary", "timestamp": iso(t + 60)}
|
|
1412
|
+
for name, records in {
|
|
1413
|
+
"a finished reply": [sent, block(t + 5, "req_m1", TEXT, "end_turn")],
|
|
1414
|
+
"its tools all answered": [sent, block(t + 5, "req_m1", tool_use(), "tool_use"),
|
|
1415
|
+
result_of(t + 9)],
|
|
1416
|
+
"an API error": [sent, cc_api_error(t + 1, "overloaded")],
|
|
1417
|
+
"interrupted mid-stream": [sent, block(t + 5, "req_m1", THINKING, None),
|
|
1418
|
+
cc_user(t + 6, "[Request interrupted by user]")],
|
|
1419
|
+
"interrupted before a word": [sent, cc_user(t + 1, "migrate the db"),
|
|
1420
|
+
cc_user(t + 2, "[Request interrupted by user]")],
|
|
1421
|
+
"a tool interrupted": [sent, block(t + 5, "req_m1", tool_use(), "tool_use"),
|
|
1422
|
+
cc_user(t + 6, [{"type": "text", "text":
|
|
1423
|
+
"[Request interrupted by user for tool use]"}])],
|
|
1424
|
+
"a command after the reply": [sent, block(t + 5, "req_m1", TEXT, "end_turn")]
|
|
1425
|
+
+ cc_slash(t + 6),
|
|
1426
|
+
"a reply without the key": [sent, main_reply(t + 5, "req_m1", "Started it.")],
|
|
1427
|
+
"a compaction after the turn": [sent, block(t + 5, "req_m1", TEXT, "end_turn"),
|
|
1428
|
+
compact_boundary,
|
|
1429
|
+
cc_user(t + 60, "summary", isCompactSummary=True)],
|
|
1430
|
+
"notices of paused workflows": [sent, block(t + 5, "req_m1", TEXT, "end_turn"),
|
|
1431
|
+
queue_op(t + 9), request_blob(),
|
|
1432
|
+
cc_user(t + 10, "<task-notification>paused"
|
|
1433
|
+
"</task-notification>")],
|
|
1434
|
+
"an agent's request": [sent, block(t + 5, "req_m1", TEXT, "end_turn"),
|
|
1435
|
+
side(api_request(t + 6, "req_a1", AGENT))],
|
|
1436
|
+
"no request on record (an older client)": [cc_user(t, "go"), cc_reply(t + 1),
|
|
1437
|
+
synthetic(t + 2)],
|
|
1438
|
+
"nothing": [],
|
|
1439
|
+
}.items():
|
|
1440
|
+
self.assertIsNone(self.busy(records), name)
|
|
1441
|
+
|
|
1442
|
+
|
|
1443
|
+
class MainThreadInFlightWatcherTests(SidechainCase):
|
|
1444
|
+
"""An agent's verdict waits for the main thread's own request, stream or tool: the
|
|
1445
|
+
account accepted it before the limit, and stopping the client would cut it off (the
|
|
1446
|
+
tool's processes die with it). Fake runner and clock only."""
|
|
1447
|
+
|
|
1448
|
+
def run_until_probe(self, main, later, env=None, waits=None):
|
|
1449
|
+
"""The first probe's time after t0; ``waits``: the main-thread work the one `hold`
|
|
1450
|
+
line names (reason=main-<waits>), or None when nothing waited."""
|
|
1451
|
+
reset = int(self.t0) + 9000
|
|
1452
|
+
w, r = self.start([], main=[cc_user(self.launched + 1, "migrate the db")] + main,
|
|
1453
|
+
env=env)
|
|
1454
|
+
self.agent_file([side_reply(self.launched + 3), side_quota(self.t0 - 1, reset)])
|
|
1455
|
+
for when, records in later:
|
|
1456
|
+
r.at(when, lambda records=records: self.transcript(records))
|
|
1457
|
+
self.assertEqual(w.run(), 0)
|
|
1458
|
+
self.assertEqual(len(r.probes), 1)
|
|
1459
|
+
self.assertEqual(r.signals[0], (self.PID, signal.SIGTERM))
|
|
1460
|
+
if waits is None:
|
|
1461
|
+
self.assertEqual(self.events(), ["watch", "detect", "switch"])
|
|
1462
|
+
else:
|
|
1463
|
+
self.assertEqual(self.events(), ["watch", "detect", "hold", "switch"])
|
|
1464
|
+
self.assertIn(" class=quota sid=%s acct=acct-01 reason=main-%s" % (SID, waits),
|
|
1465
|
+
self.log_lines()[2])
|
|
1466
|
+
return r.probes[0]["t"] - self.t0, r
|
|
1467
|
+
|
|
1468
|
+
def test_a_tool_the_main_thread_runs_finishes_first(self):
|
|
1469
|
+
t, _ = self.run_until_probe(
|
|
1470
|
+
[api_request(self.t0 - 60, "req_m1"),
|
|
1471
|
+
block(self.t0 - 55, "req_m1", tool_use(), "tool_use")],
|
|
1472
|
+
[(self.t0 + 240, [result_of(self.t0 + 240)])], waits="tool")
|
|
1473
|
+
self.assertGreaterEqual(t, 240)
|
|
1474
|
+
self.assertLess(t, 245)
|
|
1475
|
+
|
|
1476
|
+
def test_a_stream_the_account_accepted_finishes_first(self):
|
|
1477
|
+
t, _ = self.run_until_probe(
|
|
1478
|
+
[api_request(self.t0 - 40, "req_m1")],
|
|
1479
|
+
[(self.t0 + 100, [block(self.t0 + 100, "req_m1", THINKING, None)]),
|
|
1480
|
+
(self.t0 + 300, [block(self.t0 + 300, "req_m1", TEXT, "end_turn")])],
|
|
1481
|
+
waits="request")
|
|
1482
|
+
self.assertGreaterEqual(t, 300)
|
|
1483
|
+
self.assertLess(t, 305)
|
|
1484
|
+
|
|
1485
|
+
def test_the_main_threads_own_rejection_then_stands(self):
|
|
1486
|
+
# its tool ends, its next request meets the limit: that verdict moves the session
|
|
1487
|
+
t, r = self.run_until_probe(
|
|
1488
|
+
[api_request(self.t0 - 60, "req_m1"),
|
|
1489
|
+
block(self.t0 - 55, "req_m1", tool_use(), "tool_use")],
|
|
1490
|
+
[(self.t0 + 30, [result_of(self.t0 + 30), api_request(self.t0 + 30, "req_m2"),
|
|
1491
|
+
cc_quota(self.t0 + 31, resets=int(self.t0) + 9000)])],
|
|
1492
|
+
waits="tool")
|
|
1493
|
+
self.assertGreaterEqual(t, 30)
|
|
1494
|
+
self.assertLess(t, 40)
|
|
1495
|
+
kv, _ = self.relaunch_of(r)
|
|
1496
|
+
self.assertEqual((kv["class"], kv["reset"]), ("quota", str(int(self.t0) + 9000)))
|
|
1497
|
+
|
|
1498
|
+
def test_the_wait_is_capped(self):
|
|
1499
|
+
# a tool that never ends (or a prompt for permission nobody answers) waits SUBAGENT_CAP
|
|
1500
|
+
t, _ = self.run_until_probe(
|
|
1501
|
+
[api_request(self.t0 - 60, "req_m1"),
|
|
1502
|
+
block(self.t0 - 55, "req_m1", tool_use(), "tool_use")], [],
|
|
1503
|
+
env={"CLAUDE_MULTIACC_AR_SUBAGENT_CAP": "120"}, waits="tool")
|
|
1504
|
+
self.assertGreaterEqual(t, 120)
|
|
1505
|
+
self.assertLess(t, 125)
|
|
1506
|
+
|
|
1507
|
+
def test_an_idle_main_thread_moves_at_grace_as_before(self):
|
|
1508
|
+
t, _ = self.run_until_probe(
|
|
1509
|
+
[api_request(self.t0 - 60, "req_m1"),
|
|
1510
|
+
block(self.t0 - 55, "req_m1", tool_use(), "tool_use"), result_of(self.t0 - 50),
|
|
1511
|
+
api_request(self.t0 - 50, "req_m2"),
|
|
1512
|
+
block(self.t0 - 45, "req_m2", TEXT, "end_turn"), queue_op(self.t0 - 2)], [])
|
|
1513
|
+
self.assertGreaterEqual(t, 5)
|
|
1514
|
+
self.assertLess(t, 10)
|
|
1515
|
+
|
|
1516
|
+
|
|
1517
|
+
if __name__ == "__main__":
|
|
1518
|
+
unittest.main()
|