claude-multiacc 2.0.57 → 2.0.59

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/README.md +111 -3
  2. package/bin/claude +323 -84
  3. package/bin/claude-accounts +273 -42
  4. package/bin/codex +15 -0
  5. package/bin/codex-accounts +31 -4
  6. package/docs/ACCOUNT_OPERATIONS.md +62 -2
  7. package/docs/AUTORESUME-HANDOFF.md +30 -3
  8. package/docs/AUTORESUME.md +275 -18
  9. package/docs/CODEX.md +1 -0
  10. package/docs/INSTALLATION.md +7 -1
  11. package/docs/VERIFICATION.md +4 -0
  12. package/lib/__pycache__/audit.cpython-312.pyc +0 -0
  13. package/lib/__pycache__/autoresume.cpython-312.pyc +0 -0
  14. package/lib/__pycache__/autoresume_cross.cpython-312.pyc +0 -0
  15. package/lib/__pycache__/autoresume_cross_dispatch.cpython-312.pyc +0 -0
  16. package/lib/__pycache__/autoresume_cross_receipt.cpython-312.pyc +0 -0
  17. package/lib/__pycache__/autoresume_handoff.cpython-312.pyc +0 -0
  18. package/lib/__pycache__/autoresume_probe.cpython-312.pyc +0 -0
  19. package/lib/__pycache__/autoresume_receipt.cpython-312.pyc +0 -0
  20. package/lib/__pycache__/claude_client_error.cpython-312.pyc +0 -0
  21. package/lib/__pycache__/claude_reset.cpython-312.pyc +0 -0
  22. package/lib/__pycache__/claude_token_identity.cpython-312.pyc +0 -0
  23. package/lib/__pycache__/codex_autoresume_mode.cpython-312.pyc +0 -0
  24. package/lib/__pycache__/codex_config_edit.cpython-312.pyc +0 -0
  25. package/lib/__pycache__/codex_credit_controls.cpython-312.pyc +0 -0
  26. package/lib/__pycache__/codex_credit_local_scope.cpython-312.pyc +0 -0
  27. package/lib/__pycache__/codex_credit_selection.cpython-312.pyc +0 -0
  28. package/lib/__pycache__/codex_credits.cpython-312.pyc +0 -0
  29. package/lib/__pycache__/codex_limits_lock.cpython-312.pyc +0 -0
  30. package/lib/__pycache__/codex_python.cpython-312.pyc +0 -0
  31. package/lib/__pycache__/codex_trust.cpython-312.pyc +0 -0
  32. package/lib/__pycache__/codex_usage_fetch.cpython-312.pyc +0 -0
  33. package/lib/__pycache__/keychain.cpython-312.pyc +0 -0
  34. package/lib/__pycache__/mcp_oauth.cpython-312.pyc +0 -0
  35. package/lib/__pycache__/mcp_registry.cpython-312.pyc +0 -0
  36. package/lib/__pycache__/model_catalog.cpython-312.pyc +0 -0
  37. package/lib/__pycache__/model_match.cpython-312.pyc +0 -0
  38. package/lib/__pycache__/report.cpython-312.pyc +0 -0
  39. package/lib/__pycache__/selector_credit_scope.cpython-312.pyc +0 -0
  40. package/lib/__pycache__/selector_credits.cpython-312.pyc +0 -0
  41. package/lib/__pycache__/selector_funding.cpython-312.pyc +0 -0
  42. package/lib/__pycache__/selector_policy.cpython-312.pyc +0 -0
  43. package/lib/__pycache__/selector_primitives.cpython-312.pyc +0 -0
  44. package/lib/__pycache__/session_settings.cpython-312.pyc +0 -0
  45. package/lib/__pycache__/shim_path.cpython-312.pyc +0 -0
  46. package/lib/audit.py +88 -22
  47. package/lib/autoresume.py +1197 -82
  48. package/lib/autoresume_cross.py +3 -1
  49. package/lib/autoresume_handoff.py +82 -10
  50. package/lib/autoresume_probe.py +2 -2
  51. package/lib/claude_client_error.py +9 -4
  52. package/lib/codex_trust.py +263 -0
  53. package/lib/common.sh +131 -3
  54. package/lib/credential.py +5 -2
  55. package/lib/install_actions.sh +19 -0
  56. package/lib/keychain.py +180 -36
  57. package/lib/mcp_oauth.py +1748 -0
  58. package/lib/model_catalog.json +404 -0
  59. package/lib/model_catalog.py +201 -0
  60. package/lib/model_match.py +104 -0
  61. package/lib/report.py +8 -5
  62. package/lib/session_settings.py +123 -0
  63. package/package.json +2 -2
  64. package/tests/__pycache__/autoresume_cross_receipt_support.cpython-312.pyc +0 -0
  65. package/tests/__pycache__/autoresume_cross_tmux_environment.cpython-312.pyc +0 -0
  66. package/tests/__pycache__/autoresume_cross_tmux_support.cpython-312.pyc +0 -0
  67. package/tests/__pycache__/autoresume_cross_unit_support.cpython-312.pyc +0 -0
  68. package/tests/__pycache__/autoresume_tmux_support.cpython-312.pyc +0 -0
  69. package/tests/__pycache__/packaged_command_support.cpython-312.pyc +0 -0
  70. package/tests/__pycache__/test_autoresume.cpython-312.pyc +0 -0
  71. package/tests/__pycache__/test_autoresume_catalog.cpython-312.pyc +0 -0
  72. package/tests/__pycache__/test_autoresume_catalog_cli.cpython-312.pyc +0 -0
  73. package/tests/__pycache__/test_autoresume_codex_records.cpython-312.pyc +0 -0
  74. package/tests/__pycache__/test_autoresume_codex_trust.cpython-312.pyc +0 -0
  75. package/tests/__pycache__/test_autoresume_cross_claude_holders.cpython-312.pyc +0 -0
  76. package/tests/__pycache__/test_autoresume_cross_guards.cpython-312.pyc +0 -0
  77. package/tests/__pycache__/test_autoresume_cross_holders.cpython-312.pyc +0 -0
  78. package/tests/__pycache__/test_autoresume_cross_origin.cpython-312.pyc +0 -0
  79. package/tests/__pycache__/test_autoresume_cross_policy.cpython-312.pyc +0 -0
  80. package/tests/__pycache__/test_autoresume_cross_receipt.cpython-312.pyc +0 -0
  81. package/tests/__pycache__/test_autoresume_cross_tmux.cpython-312.pyc +0 -0
  82. package/tests/__pycache__/test_autoresume_cross_tmux_races.cpython-312.pyc +0 -0
  83. package/tests/__pycache__/test_autoresume_cross_tokens.cpython-312.pyc +0 -0
  84. package/tests/__pycache__/test_autoresume_cursor.cpython-312.pyc +0 -0
  85. package/tests/__pycache__/test_autoresume_fallback.cpython-312.pyc +0 -0
  86. package/tests/__pycache__/test_autoresume_handoff_argv.cpython-312.pyc +0 -0
  87. package/tests/__pycache__/test_autoresume_handoff_history.cpython-312.pyc +0 -0
  88. package/tests/__pycache__/test_autoresume_handoff_origins.cpython-312.pyc +0 -0
  89. package/tests/__pycache__/test_autoresume_memory.cpython-312.pyc +0 -0
  90. package/tests/__pycache__/test_autoresume_memory_tmux.cpython-312.pyc +0 -0
  91. package/tests/__pycache__/test_autoresume_mode.cpython-312.pyc +0 -0
  92. package/tests/__pycache__/test_autoresume_probe.cpython-312.pyc +0 -0
  93. package/tests/__pycache__/test_autoresume_receipt.cpython-312.pyc +0 -0
  94. package/tests/__pycache__/test_autoresume_relaunch_settings.cpython-312.pyc +0 -0
  95. package/tests/__pycache__/test_autoresume_settings.cpython-312.pyc +0 -0
  96. package/tests/__pycache__/test_autoresume_sidechain.cpython-312.pyc +0 -0
  97. package/tests/__pycache__/test_autoresume_sidechain_tmux.cpython-312.pyc +0 -0
  98. package/tests/__pycache__/test_autoresume_tmux.cpython-312.pyc +0 -0
  99. package/tests/__pycache__/test_autoresume_tmux_modes.cpython-312.pyc +0 -0
  100. package/tests/__pycache__/test_autoresume_trust_tmux.cpython-312.pyc +0 -0
  101. package/tests/__pycache__/test_claude_reset.cpython-312.pyc +0 -0
  102. package/tests/__pycache__/test_claude_token_identity.cpython-312.pyc +0 -0
  103. package/tests/__pycache__/test_codex_reset.cpython-312.pyc +0 -0
  104. package/tests/__pycache__/test_codex_reset_credits.cpython-312.pyc +0 -0
  105. package/tests/__pycache__/test_codex_reset_credits_polling.cpython-312.pyc +0 -0
  106. package/tests/__pycache__/test_codex_reset_credits_report.cpython-312.pyc +0 -0
  107. package/tests/__pycache__/test_codex_reset_fetch.cpython-312.pyc +0 -0
  108. package/tests/__pycache__/test_codex_reset_polling.cpython-312.pyc +0 -0
  109. package/tests/__pycache__/test_codex_reset_reporting.cpython-312.pyc +0 -0
  110. package/tests/__pycache__/test_codex_reset_windows.cpython-312.pyc +0 -0
  111. package/tests/__pycache__/test_selector.cpython-312.pyc +0 -0
  112. package/tests/__pycache__/test_selector_cached_credits.cpython-312.pyc +0 -0
  113. package/tests/__pycache__/test_selector_credit_scope.cpython-312.pyc +0 -0
  114. package/tests/__pycache__/test_selector_credit_shim.cpython-312.pyc +0 -0
  115. package/tests/__pycache__/test_selector_credits.cpython-312.pyc +0 -0
  116. package/tests/__pycache__/test_selector_spread.cpython-312.pyc +0 -0
  117. package/tests/__pycache__/test_token_identity_ceremony.cpython-312.pyc +0 -0
  118. package/tests/__pycache__/test_token_identity_integration.cpython-312.pyc +0 -0
  119. package/tests/__pycache__/test_token_identity_retry.cpython-312.pyc +0 -0
  120. package/tests/__pycache__/test_token_identity_sync.cpython-312.pyc +0 -0
  121. package/tests/fake_codex_tui.py +25 -0
  122. package/tests/fake_security.sh +24 -2
  123. package/tests/fixtures/cross_provider_cli.py +111 -1
  124. package/tests/run-tests.sh +1028 -23
  125. package/tests/test_autoresume.py +215 -8
  126. package/tests/test_autoresume_catalog.py +226 -0
  127. package/tests/test_autoresume_catalog_cli.py +185 -0
  128. package/tests/test_autoresume_codex_records.py +16 -0
  129. package/tests/test_autoresume_codex_trust.py +247 -0
  130. package/tests/test_autoresume_cross_guards.py +28 -0
  131. package/tests/test_autoresume_cross_origin.py +70 -0
  132. package/tests/test_autoresume_cross_tmux.py +27 -2
  133. package/tests/test_autoresume_cross_tokens.py +32 -2
  134. package/tests/test_autoresume_handoff_argv.py +120 -16
  135. package/tests/test_autoresume_handoff_origins.py +67 -0
  136. package/tests/test_autoresume_memory.py +1141 -0
  137. package/tests/test_autoresume_memory_tmux.py +94 -0
  138. package/tests/test_autoresume_probe.py +2 -1
  139. package/tests/test_autoresume_relaunch_settings.py +317 -0
  140. package/tests/test_autoresume_settings.py +135 -0
  141. package/tests/test_autoresume_sidechain.py +1518 -0
  142. package/tests/test_autoresume_sidechain_tmux.py +79 -0
  143. package/tests/test_autoresume_trust_tmux.py +63 -0
  144. package/tests/test_claude_client_error.py +10 -7
  145. package/tests/test_claude_org_block.py +1 -0
  146. package/tests/test_claude_pool_prepare.py +4 -1
  147. package/tests/test_claude_reset.py +3 -0
  148. package/tests/test_keychain_nologin.py +340 -0
  149. package/tests/test_mcp_oauth.py +1414 -0
  150. package/tests/test_packaged_commands.py +41 -0
@@ -0,0 +1,1518 @@
1
+ #!/usr/bin/env python3
2
+ """Account-level rejections that only a claude session's subagent and workflow transcripts
3
+ record. 2026-10-19: two sessions sat idle behind paused workflows whose agents alone had
4
+ met the five-hour limit, and the watcher, tailing the main transcript only, never moved
5
+ them. Disposable pools, the fake runner and its fake clock only: no real signal or tmux."""
6
+
7
+ import json
8
+ import os
9
+ import signal
10
+ import unittest
11
+ from unittest import mock
12
+
13
+ import test_autoresume as fixtures
14
+ from test_autoresume import (SID, SID2, ar, cc_api_error, cc_concurrents, cc_quota, cc_reply,
15
+ cc_slash, cc_user, iso)
16
+
17
+ AGENT = "a33659a7131e55067"
18
+ AGENT2 = "ab12cd34ef56a7890"
19
+ AGENT3 = "a0f1e2d3c4b5a6978"
20
+
21
+
22
+ def side(rec, agent=AGENT):
23
+ """A record as a subagent transcript holds it (claude 2.1.303): the session's
24
+ sidechain, named by its agent."""
25
+ return dict(rec, isSidechain=True, agentId=agent)
26
+
27
+
28
+ def side_quota(ts, resets, agent=AGENT, rtype="five_hour"):
29
+ rec = cc_quota(ts, rtype, resets=resets, apiError="usage_limit_reached")
30
+ rec["message"]["content"][0]["text"] = ("You've hit your session limit · resets 12:40am "
31
+ "(Europe/Istanbul)")
32
+ return side(rec, agent)
33
+
34
+
35
+ def side_reply(ts, agent=AGENT, text="Checked the build.", ref=None):
36
+ rec = side(cc_reply(ts, text), agent)
37
+ if ref:
38
+ rec["requestRef"] = ref
39
+ return rec
40
+
41
+
42
+ def final_answer(ts, agent, ref):
43
+ """An agent's last reply (end_turn): nothing of that agent follows it."""
44
+ rec = side_reply(ts, agent, "All done: the section is written.", ref)
45
+ rec["message"]["stop_reason"] = "end_turn"
46
+ return rec
47
+
48
+
49
+ def api_request(ts, rid, agent=None):
50
+ """What claude 2.1.303 writes right before it sends a request, in an agent's transcript
51
+ and the main one alike; every reply to that request names it by requestRef."""
52
+ rec = {"type": "api-request", "sessionId": SID, "id": rid, "timestamp": iso(ts),
53
+ "version": "2.1.303", "querySource": "repl_main_thread", "shapeHash": "9f2c41",
54
+ "params": {"model": "claude-opus-5-5", "max_tokens": 64000}, "keep": 0, "tail": 1,
55
+ "cacheControl": "5m", "requestDigest": "4b1d0c"}
56
+ if agent:
57
+ rec.update(agentId=agent, querySource="agent:builtin:workflow-subagent")
58
+ return rec
59
+
60
+
61
+ def main_reply(ts, ref, text="Done."):
62
+ return dict(cc_reply(ts, text), requestRef=ref)
63
+
64
+
65
+ def queue_op(ts):
66
+ """All an idle main transcript waiting on its workflows still gets."""
67
+ return {"type": "queue-operation", "operation": "enqueue", "timestamp": iso(ts),
68
+ "sessionId": SID, "content": "<task-notification>paused</task-notification>"}
69
+
70
+
71
+ def request_blob():
72
+ """A request body the client keeps among the records (claude 2.1.303): no timestamp of
73
+ its own, and it quotes the conversation's "assistant" turns."""
74
+ return {"type": "api-request-blob", "hash": "5d41402abc4b2a76b9719d911017c592",
75
+ "sessionId": SID, "message": {"role": "assistant", "content": [
76
+ {"type": "text", "text": "Started it; waiting for the workflow."}]}}
77
+
78
+
79
+ def tool_result(ts):
80
+ return cc_user(ts, [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "ok"}],
81
+ toolUseResult={"stdout": "ok"})
82
+
83
+
84
+ def synthetic(ts, text="No response requested."):
85
+ """The client's own words in the main thread, no API error and no reply of the API's."""
86
+ rec = cc_reply(ts, text)
87
+ rec["message"]["model"] = "<synthetic>"
88
+ return rec
89
+
90
+
91
+ def compact(records):
92
+ """JSONL as the client writes it: compact JSON, a record a line."""
93
+ return "".join(json.dumps(rec, separators=(",", ":")) + "\n" for rec in records)
94
+
95
+
96
+ def write(path, records):
97
+ with open(path, "a") as fh:
98
+ fh.write(compact(records))
99
+
100
+
101
+ def main_lines(records):
102
+ """Main transcript lines as the client writes them (compact JSON)."""
103
+ return [line.encode() for line in compact(records).splitlines()]
104
+
105
+
106
+ class SidechainCase(fixtures.PoolCase):
107
+ def agent_file(self, records, agent=AGENT, workflow="wf_7f3a9c", sid=SID, mtime=None):
108
+ """<proj>/<sid>/subagents/workflows/<wf>/agent-<id>.jsonl (workflow=None: an Agent
109
+ tool subagent, <sid>/subagents/agent-<id>.jsonl)."""
110
+ folder = self.proj / sid / "subagents"
111
+ if workflow:
112
+ folder = folder / "workflows" / workflow
113
+ folder.mkdir(parents=True, exist_ok=True)
114
+ path = folder / ("agent-%s.jsonl" % agent)
115
+ write(path, records)
116
+ if mtime is not None:
117
+ os.utime(str(path), (mtime, mtime))
118
+ return path
119
+
120
+ def transcript(self, records, sid=SID):
121
+ path = self.proj / (sid + ".jsonl")
122
+ write(path, records)
123
+ return path
124
+
125
+ def start(self, argv=None, main=None, env=None):
126
+ """A session whose main thread started workflows and has idled on them since."""
127
+ state = self.write_state(["--dangerously-skip-permissions"] if argv is None else argv)
128
+ self.registry()
129
+ self.transcript([cc_user(self.launched + 1, "run the release workflow"),
130
+ cc_reply(self.launched + 2, "Started it; waiting for the workflow."),
131
+ queue_op(self.t0 - 1)] if main is None else main)
132
+ return self.watcher(state, env)
133
+
134
+ def stop_at(self, runner, when):
135
+ """The operator ends the session (/exit: its registry goes)."""
136
+ reg = self.root / "acct-01" / "sessions" / ("%d.json" % self.PID)
137
+ runner.at(when, lambda: (reg.unlink() if reg.exists() else None,
138
+ runner.exit(self.PID)))
139
+
140
+ def tail_of(self, watcher, name):
141
+ found = [tail for path, tail in watcher.side.items() if os.path.basename(path) == name]
142
+ self.assertEqual(len(found), 1, sorted(watcher.side))
143
+ return found[0]
144
+
145
+ def assert_untouched(self, runner):
146
+ self.assertEqual((runner.probes, runner.signals, runner.sends, self.relaunch_files()),
147
+ ([], [], [], []))
148
+
149
+
150
+ # =====================================================================================
151
+ # classification and the tracker
152
+ # =====================================================================================
153
+
154
+ class SidechainClassifierTests(unittest.TestCase):
155
+ T = 1792440000
156
+
157
+ def test_only_the_accounts_own_verdicts_count(self):
158
+ verdicts = {
159
+ "quota": side_quota(self.T, self.T + 6000),
160
+ "auth": side(cc_api_error(self.T, "authentication_failed", "Not logged in")),
161
+ "blocked": side(cc_api_error(self.T, "oauth_org_not_allowed")),
162
+ }
163
+ for cls, rec in verdicts.items():
164
+ kind, verdict = ar.classify_claude_sidechain(rec)
165
+ self.assertEqual((kind, verdict["class"]), ("error", cls))
166
+ _, quota = ar.classify_claude_sidechain(verdicts["quota"])
167
+ self.assertEqual((quota["reset"], quota["rtype"], quota["marked"]),
168
+ (self.T + 6000, "five_hour", ar.utc_iso(self.T)))
169
+ _, org = ar.classify_claude_sidechain(verdicts["blocked"])
170
+ self.assertEqual(org["rtype"], "oauth_org_not_allowed")
171
+ # the subagent's own business: its model's bucket, a transient error, the rest
172
+ for rec in (side_quota(self.T, self.T + 60, rtype="seven_day_opus"),
173
+ side(cc_api_error(self.T, "rate_limit", "You've reached your Fable limit.")),
174
+ side(cc_api_error(self.T, "rate_limit", "nope",
175
+ apiError="model_requires_usage_credits")),
176
+ side(cc_api_error(self.T, "rate_limit", "temporary capacity issue")),
177
+ side(cc_api_error(self.T, "overloaded")),
178
+ side(cc_api_error(self.T, "invalid_request")),
179
+ side(cc_api_error(self.T, None))):
180
+ self.assertIsNone(ar.classify_claude_sidechain(rec), rec.get("error"))
181
+
182
+ def test_a_real_reply_proves_service_and_nothing_else_does(self):
183
+ self.assertEqual(ar.classify_claude_sidechain(side_reply(self.T)), ("cancel", "sidechain"))
184
+ synthetic = side_reply(self.T, text="No response requested.")
185
+ synthetic["message"]["model"] = "<synthetic>"
186
+ for rec in (synthetic, side(cc_user(self.T, "please continue")),
187
+ side(cc_user(self.T, [{"type": "tool_result", "tool_use_id": "toolu_1",
188
+ "content": "ok"}])),
189
+ queue_op(self.T), {"type": "system", "timestamp": iso(self.T)}, "x", None):
190
+ self.assertIsNone(ar.classify_claude_sidechain(rec), rec)
191
+ # in the MAIN transcript a sidechain record stays ignored, as before
192
+ self.assertIsNone(ar.classify_claude(side_quota(self.T, self.T + 60)))
193
+
194
+ def test_agent_names_and_the_newest_reply(self):
195
+ path = "/p/agent-%s.jsonl" % AGENT2
196
+ self.assertEqual(ar.sidechain_agent(side_reply(self.T), path), AGENT)
197
+ self.assertEqual(ar.sidechain_agent({"agentId": "a b\n"}, path), AGENT2)
198
+ self.assertEqual(ar.sidechain_agent({}, "/p/agent-../../x.jsonl"), "-")
199
+ lines = main_lines([side_reply(self.T + 1, text="first"),
200
+ side_quota(self.T + 2, self.T + 60),
201
+ side_reply(self.T + 3, text="last"),
202
+ side(cc_user(self.T + 4, "tool output"))])
203
+ at, rec = ar.newest_answer(lines, ar.side_answer_at)
204
+ self.assertEqual((at, rec["timestamp"]), (self.T + 3.009, iso(self.T + 3)))
205
+ self.assertIsNone(ar.newest_answer(lines[1:2], ar.side_answer_at))
206
+ # found behind any run of lines that are none (a real main transcript had 193 in a
207
+ # row), and a line without the mark every reply holds is never even parsed
208
+ blobs = main_lines([request_blob()]) * 200
209
+ quiet = main_lines([side(tool_result(self.T + 5))]) * 50
210
+ with mock.patch.object(ar, "read_json_line", wraps=ar.read_json_line) as parse:
211
+ at, rec = ar.newest_answer(lines[:1] + blobs + quiet, ar.side_answer_at)
212
+ self.assertEqual((rec["timestamp"], parse.call_count), (iso(self.T + 1), 201))
213
+
214
+ def test_a_reply_answers_from_when_its_request_was_sent(self):
215
+ sent = {}
216
+ for rec in (api_request(self.T + 10, "req_1", AGENT),
217
+ api_request(self.T + 30, "req_2", AGENT)):
218
+ ar.note_request(rec, sent)
219
+ self.assertEqual(sent, {"req_1": self.T + 10.009, "req_2": self.T + 30.009})
220
+ self.assertEqual(ar.claude_served_at(side_reply(self.T + 25, ref="req_1"), sent),
221
+ self.T + 10.009)
222
+ # a request not on record (an older client names none) leaves the reply's own time
223
+ for ref in (None, "req_9", ["req_1"]):
224
+ rec = side_reply(self.T + 25)
225
+ rec["requestRef"] = ref
226
+ self.assertEqual(ar.claude_served_at(rec, sent), self.T + 25.009, ref)
227
+ self.assertEqual(ar.claude_served_at(side_reply(self.T + 25, ref="req_1")), self.T + 25.009)
228
+ self.assertIsNone(ar.side_answer_at(side_quota(self.T + 25, self.T + 60), sent))
229
+ # the newest ones are kept
230
+ with mock.patch.object(ar, "REQUESTS_KEPT", 2):
231
+ ar.note_request(api_request(self.T + 40, "req_3"), sent)
232
+ ar.note_request({"type": "api-request", "id": "req_4"}, sent) # undated: nothing
233
+ self.assertEqual(sorted(sent), ["req_2", "req_3"])
234
+
235
+ def test_the_latest_answer_is_not_always_the_last_line(self):
236
+ # a prompt typed while a stream ran: the reply, written last, answers from its request
237
+ sent = {"req_1": self.T + 10.009}
238
+ tr = ar.ClaudeTracker(self.T - 100)
239
+ tr.requests = sent
240
+ lines = main_lines([cc_user(self.T + 15, "also check the docs"),
241
+ main_reply(self.T + 40, "req_1")])
242
+ at, rec = ar.newest_answer(lines, lambda r: tr.answer_at(ar.classify_claude(r), r),
243
+ ar.ANSWER_MARKS)
244
+ self.assertEqual((at, rec["type"]), (self.T + 15.009, "user"))
245
+ # and the scan ends at the first record written before the best moment found
246
+ earlier = main_lines([side_reply(self.T - 100 + i) for i in range(100)])
247
+ turn = main_lines([api_request(self.T + 200, "req_x", AGENT)]
248
+ + [side_reply(self.T + 201 + i, ref="req_x") for i in range(3)])
249
+ with mock.patch.object(ar, "read_json_line", wraps=ar.read_json_line) as parse:
250
+ at, _rec = ar.newest_answer(earlier + turn,
251
+ lambda r: ar.side_answer_at(r, {"req_x": self.T + 200.009}))
252
+ self.assertEqual((at, parse.call_count), (self.T + 200.009, 4))
253
+
254
+
255
+ class SidechainTrackerTests(unittest.TestCase):
256
+ LAUNCHED = 1792440000
257
+
258
+ def tracker(self):
259
+ return ar.ClaudeTracker(self.LAUNCHED - 2)
260
+
261
+ def test_a_rejection_must_prove_it_is_from_this_launch(self):
262
+ tr = self.tracker()
263
+ self.assertIsNone(tr.feed_side(side_quota(self.LAUNCHED - 3, self.LAUNCHED + 99), 1))
264
+ undated = side_quota(self.LAUNCHED + 5, self.LAUNCHED + 99)
265
+ del undated["timestamp"]
266
+ self.assertIsNone(tr.feed_side(undated, 1))
267
+ self.assertIsNone(tr.pending)
268
+ ev = tr.feed_side(side_quota(self.LAUNCHED + 5, self.LAUNCHED + 99), 7, AGENT)
269
+ self.assertEqual(ev[0], "error")
270
+ self.assertEqual((tr.pending["class"], tr.pending["src"], tr.pending["agent"],
271
+ tr.pending_since), ("quota", "sidechain", AGENT, 7))
272
+ # an old reply proves nothing new
273
+ self.assertIsNone(tr.feed_side(side_reply(self.LAUNCHED - 60), 8))
274
+ self.assertIsNotNone(tr.pending)
275
+
276
+ def test_the_replaced_processs_agents_never_reject_for_this_launch(self):
277
+ # A relaunch replaces a process whose workflow agents met the limit until it was gone,
278
+ # inside the second the shim's floored `date +%s` names as this launch (since is
279
+ # launched - 2): only a rejection from a second past the launch is this launch's own.
280
+ for offset in (-2, -1, 0):
281
+ tr = self.tracker()
282
+ self.assertIsNone(tr.feed_side(side_quota(self.LAUNCHED + offset, self.LAUNCHED + 99),
283
+ 1), offset)
284
+ self.assertIsNone(tr.pending)
285
+ tr = self.tracker()
286
+ self.assertEqual(tr.feed_side(side_quota(self.LAUNCHED + 1, self.LAUNCHED + 99), 1)[0],
287
+ "error")
288
+ # its replies still answer, and the main thread keeps its two seconds of slack
289
+ tr = self.tracker()
290
+ tr.feed_side(side_reply(self.LAUNCHED - 1), 1)
291
+ self.assertEqual(tr.cleared, self.LAUNCHED - 0.991)
292
+ self.assertEqual(tr.feed(cc_quota(self.LAUNCHED - 1, resets=self.LAUNCHED + 99), 2)[0],
293
+ "error")
294
+
295
+ def test_a_reply_to_a_request_sent_before_the_rejection_answers_nothing(self):
296
+ launched = self.LAUNCHED
297
+ sent = {}
298
+ for rec in (api_request(launched + 10, "req_1", AGENT2),
299
+ api_request(launched + 30, "req_2", AGENT2)):
300
+ ar.note_request(rec, sent)
301
+ rejection = side_quota(launched + 20, launched + 9000)
302
+ # a stream the account accepted before the limit, ending after it
303
+ streamed = final_answer(launched + 25, AGENT2, "req_1")
304
+ tr = self.tracker()
305
+ tr.feed_side(rejection, 1)
306
+ self.assertIsNone(tr.feed_side(streamed, 2, AGENT2, sent))
307
+ self.assertEqual((tr.pending["class"], tr.cleared), ("quota", launched + 10.009))
308
+ tr = self.tracker() # read first, it answers nothing either
309
+ self.assertIsNone(tr.feed_side(streamed, 1, AGENT2, sent))
310
+ self.assertEqual(tr.feed_side(rejection, 2)[0], "error")
311
+ # a request sent after the limit, and served, is the answer
312
+ tr = self.tracker()
313
+ tr.feed_side(rejection, 1)
314
+ self.assertEqual(tr.feed_side(side_reply(launched + 40, AGENT2, ref="req_2"), 2, AGENT2,
315
+ sent), ("cancel", "sidechain"))
316
+ # with no request on record the reply's own time counts, as with older clients
317
+ tr = self.tracker()
318
+ tr.feed_side(rejection, 1)
319
+ self.assertEqual(tr.feed_side(streamed, 2, AGENT2), ("cancel", "sidechain"))
320
+
321
+ def test_the_main_threads_reply_counts_from_its_request_too(self):
322
+ launched = self.LAUNCHED
323
+ tr = self.tracker()
324
+ self.assertIsNone(tr.feed(api_request(launched + 10, "req_m1"), 1))
325
+ tr.feed_side(side_quota(launched + 20, launched + 9000), 1)
326
+ self.assertIsNone(tr.feed(main_reply(launched + 25, "req_m1"), 2))
327
+ self.assertEqual((tr.pending["class"], tr.cleared), ("quota", launched + 10.009))
328
+ tr.feed(api_request(launched + 30, "req_m2"), 3)
329
+ self.assertEqual(tr.feed(main_reply(launched + 40, "req_m2"), 4), ("cancel", "assistant"))
330
+ # what the prefilter skips the same way: request records are read, nothing else
331
+ tr, passed = self.tracker(), []
332
+ for raw in main_lines([cc_user(launched + 5, "write the section"),
333
+ api_request(launched + 10, "req_m1"),
334
+ main_reply(launched + 25, "req_m1"), tool_result(launched + 26)]):
335
+ if tr.wants(raw):
336
+ self.assertIsNone(tr.feed(json.loads(raw), 1))
337
+ else:
338
+ passed.append(raw)
339
+ self.assertEqual((len(passed), sorted(tr.requests)), (3, ["req_m1"]))
340
+ tr.passed_over(passed)
341
+ self.assertEqual(tr.cleared, launched + 10.009)
342
+ self.assertEqual(tr.feed_side(side_quota(launched + 20, launched + 9000), 5)[0], "error")
343
+ # a new conversation starts with no requests on record
344
+ tr.reset()
345
+ self.assertEqual((tr.requests, tr.cleared), ({}, 0.0))
346
+
347
+ def test_only_a_newer_reply_cancels_and_order_of_reading_does_not_matter(self):
348
+ rej, ok = side_quota(self.LAUNCHED + 10, self.LAUNCHED + 99), side_reply(self.LAUNCHED + 11)
349
+ tr = self.tracker() # the rejection read first, then the newer reply
350
+ tr.feed_side(rej, 1)
351
+ self.assertEqual(tr.feed_side(side_reply(self.LAUNCHED + 10), 2), None) # not newer
352
+ self.assertEqual(tr.feed_side(ok, 2), ("cancel", "sidechain"))
353
+ self.assertIsNone(tr.pending)
354
+ tr = self.tracker() # the newer reply read first: the rejection is answered already
355
+ self.assertIsNone(tr.feed_side(ok, 1))
356
+ self.assertIsNone(tr.feed_side(rej, 2))
357
+ self.assertIsNone(tr.pending)
358
+ # a rejection after that reply stands again
359
+ self.assertEqual(tr.feed_side(side_quota(self.LAUNCHED + 12, self.LAUNCHED + 99), 3)[0],
360
+ "error")
361
+
362
+ def test_the_newest_subagent_verdict_wins_and_a_repeat_changes_nothing(self):
363
+ tr = self.tracker()
364
+ first = side_quota(self.LAUNCHED + 10, self.LAUNCHED + 99)
365
+ tr.feed_side(first, 1, AGENT)
366
+ pending = tr.pending
367
+ # another agent meets the same limit later: the same verdict, still standing
368
+ self.assertIsNone(tr.feed_side(side_quota(self.LAUNCHED + 20, self.LAUNCHED + 99,
369
+ AGENT2), 5, AGENT2))
370
+ self.assertIs(tr.pending, pending)
371
+ self.assertEqual((tr.pending["at"], tr.pending["agent"], tr.pending_since),
372
+ (self.LAUNCHED + 20.009, AGENT, 1))
373
+ # ...so a reply between the two rejections answers only the first
374
+ self.assertIsNone(tr.feed_side(side_reply(self.LAUNCHED + 15), 6))
375
+ self.assertIs(tr.pending, pending)
376
+ # an older verdict read late never replaces a newer one; a newer class does
377
+ auth = side(cc_api_error(self.LAUNCHED + 5, "authentication_failed"))
378
+ self.assertIsNone(tr.feed_side(auth, 7))
379
+ auth = side(cc_api_error(self.LAUNCHED + 30, "authentication_failed"), AGENT3)
380
+ self.assertEqual(tr.feed_side(auth, 8, AGENT3)[0], "error")
381
+ self.assertEqual((tr.pending["class"], tr.pending["agent"], tr.pending_since),
382
+ ("auth", AGENT3, 8))
383
+
384
+ def test_the_main_threads_own_account_verdict_stands(self):
385
+ tr = self.tracker()
386
+ tr.feed(cc_api_error(self.LAUNCHED + 10, "authentication_failed"), 1)
387
+ main = tr.pending
388
+ self.assertIsNone(tr.feed_side(side_quota(self.LAUNCHED + 20, self.LAUNCHED + 99), 2))
389
+ self.assertIsNone(tr.feed_side(side_reply(self.LAUNCHED + 30), 3)) # nor cancels it
390
+ self.assertIs(tr.pending, main)
391
+ self.assertIsNone(tr.fallback)
392
+ # and a main-thread verdict replaces a subagent's
393
+ tr = self.tracker()
394
+ tr.feed_side(side_quota(self.LAUNCHED + 20, self.LAUNCHED + 99), 1)
395
+ tr.feed(cc_api_error(self.LAUNCHED + 21, "authentication_failed"), 2)
396
+ self.assertEqual((tr.pending["class"], tr.pending.get("src")), ("auth", None))
397
+ self.assertIsNone(tr.feed_side(side_reply(self.LAUNCHED + 40), 3))
398
+ self.assertEqual(tr.pending["class"], "auth")
399
+
400
+ def test_main_thread_cancels_only_what_they_postdate(self):
401
+ for rec in (cc_reply(self.LAUNCHED + 30), cc_user(self.LAUNCHED + 30, "stop, I'm on it")):
402
+ tr = self.tracker()
403
+ tr.feed_side(side_quota(self.LAUNCHED + 20, self.LAUNCHED + 99), 1)
404
+ older = dict(rec, timestamp=iso(self.LAUNCHED + 19))
405
+ self.assertIsNone(tr.feed(older, 2))
406
+ self.assertIsNotNone(tr.pending)
407
+ self.assertEqual(tr.feed(rec, 3)[0], "cancel")
408
+ self.assertIsNone(tr.pending)
409
+ # the main thread's answer, read first, answers a rejection read later
410
+ self.assertIsNone(tr.feed_side(side_quota(self.LAUNCHED + 25, self.LAUNCHED + 99), 4))
411
+ self.assertIsNone(tr.pending)
412
+ # the auto-resume prompt is no human
413
+ tr = self.tracker()
414
+ tr.feed_side(side_quota(self.LAUNCHED + 20, self.LAUNCHED + 99), 1)
415
+ self.assertIsNone(tr.feed(cc_user(self.LAUNCHED + 30, ar.resume_prompt("quota")), 2))
416
+ self.assertIsNotNone(tr.pending)
417
+
418
+ def test_a_new_conversation_forgets_what_answered_the_old_one(self):
419
+ tr = self.tracker()
420
+ tr.feed_side(side_reply(self.LAUNCHED + 50), 1)
421
+ tr.reset()
422
+ self.assertEqual(tr.feed_side(side_quota(self.LAUNCHED + 40, self.LAUNCHED + 99), 2)[0],
423
+ "error")
424
+
425
+ def test_what_the_prefilter_passed_over_still_answers(self):
426
+ launched = self.LAUNCHED
427
+ tr = self.tracker()
428
+ lines = main_lines([cc_user(launched + 10, "run the workflow"),
429
+ cc_reply(launched + 20, "Started it."),
430
+ cc_user(launched + 590, "forget the workflow, do X"),
431
+ cc_reply(launched + 600, "Doing X.")]
432
+ + [request_blob()] * 200 + [tool_result(launched + 601),
433
+ queue_op(launched + 602)])
434
+ self.assertFalse(any(tr.wants(raw) for raw in lines)) # nothing pending: all skipped
435
+ tr.passed_over(lines)
436
+ self.assertEqual(tr.cleared, launched + 600.009)
437
+ # an agent's rejection before that reply is answered, whenever it is read...
438
+ self.assertIsNone(tr.feed_side(side_quota(launched + 300, launched + 9000), 1))
439
+ self.assertIsNone(tr.pending)
440
+ # ...and one after it stands
441
+ self.assertEqual(tr.feed_side(side_quota(launched + 700, launched + 9000), 2)[0], "error")
442
+
443
+ def test_only_a_real_reply_or_the_user_typing_answers_there(self):
444
+ launched = self.LAUNCHED
445
+ later = (cc_slash(launched + 60) + cc_slash(launched + 61, "usage")
446
+ + [synthetic(launched + 62), cc_user(launched + 63, ar.resume_prompt("quota")),
447
+ tool_result(launched + 64), queue_op(launched + 65)])
448
+ for answer in (cc_reply(launched + 50), cc_user(launched + 50, "never mind the workflow")):
449
+ with self.subTest(answer=answer["type"]):
450
+ tr = self.tracker()
451
+ tr.passed_over(main_lines([answer] + later))
452
+ self.assertEqual(tr.cleared, launched + 50.009)
453
+ # an undated answer dates nothing, one from before the launch is not this launch's
454
+ undated = cc_reply(launched + 70)
455
+ del undated["timestamp"]
456
+ tr = self.tracker()
457
+ tr.passed_over(main_lines([cc_reply(launched - 60), undated] + later))
458
+ self.assertEqual(tr.cleared, 0.0)
459
+
460
+ def test_what_the_prefilter_passed_over_cancels_nothing(self):
461
+ # read in one go: a reply the prefilter skipped, then the main thread's own rejection
462
+ tr = self.tracker()
463
+ passed = []
464
+ for raw in main_lines([cc_reply(self.LAUNCHED + 10),
465
+ cc_quota(self.LAUNCHED + 20, resets=self.LAUNCHED + 9000)]):
466
+ if tr.wants(raw):
467
+ tr.feed(json.loads(raw), 1)
468
+ else:
469
+ passed.append(raw)
470
+ tr.passed_over(passed)
471
+ self.assertEqual((tr.pending["class"], tr.cleared), ("quota", self.LAUNCHED + 10.009))
472
+
473
+ def test_a_slash_command_never_answers_the_accounts_rejection(self):
474
+ launched = self.LAUNCHED
475
+ tr = self.tracker()
476
+ tr.feed_side(side_quota(launched + 20, launched + 9000), 1)
477
+ pending = tr.pending
478
+ for rec in cc_slash(launched + 30) + [synthetic(launched + 31)]:
479
+ self.assertIsNone(tr.feed(rec, 2))
480
+ self.assertEqual((tr.pending, tr.cleared), (pending, 0.0))
481
+ self.assertEqual(tr.feed(cc_user(launched + 32, "stop, I'm on it"), 3), ("cancel", "user"))
482
+ # read before an older rejection, skipped or parsed, it answers that one neither
483
+ for parsed in (False, True):
484
+ tr = self.tracker()
485
+ if parsed:
486
+ for rec in cc_slash(launched + 30):
487
+ tr.feed(rec, 1)
488
+ else:
489
+ tr.passed_over(main_lines(cc_slash(launched + 30)))
490
+ self.assertEqual(tr.feed_side(side_quota(launched + 20, launched + 9000), 2)[0],
491
+ "error")
492
+
493
+ def test_a_command_that_can_fix_the_account_answers_its_rejection(self):
494
+ # /login after "Not logged in", /limit-reset after the limit: the user fixed the
495
+ # account, like a typed prompt; so does any command not known to leave it as it was
496
+ launched = self.LAUNCHED
497
+ for err, name in ((side(cc_api_error(launched + 20, "authentication_failed",
498
+ "Not logged in · Please run /login")), "login"),
499
+ (side_quota(launched + 20, launched + 9000), "limit-reset"),
500
+ (side_quota(launched + 20, launched + 9000), "extra-usage"),
501
+ (side_quota(launched + 20, launched + 9000), "brand-new-command")):
502
+ with self.subTest(command=name):
503
+ tr = self.tracker()
504
+ tr.feed_side(err, 1)
505
+ for rec in cc_slash(launched + 30, name):
506
+ tr.feed(rec, 2)
507
+ self.assertEqual((tr.pending, tr.cleared), (None, launched + 30.009))
508
+ tr = self.tracker() # skipped by the prefilter, read before the rejection
509
+ tr.passed_over(main_lines(cc_slash(launched + 30, name)))
510
+ self.assertIsNone(tr.feed_side(err, 3))
511
+
512
+
513
+ # A model's limit, as the main thread records it (claude 2.1.303): /model is how one handles it.
514
+ def model_limit(ts):
515
+ return cc_api_error(ts, "rate_limit",
516
+ "You've reached your Fable limit. Switch to another model to continue.")
517
+
518
+
519
+ class LiftedFallbackLatestResetTests(unittest.TestCase):
520
+ """A quota verdict whose reset has passed is dropped; an agent's account verdict
521
+ outranks the main thread's model or transient one, which waits behind it until the main
522
+ thread itself moves on; several rejections of one class are marked with the latest
523
+ reset."""
524
+ L = 1792440000
525
+
526
+ def tracker(self):
527
+ return ar.ClaudeTracker(self.L - 2)
528
+
529
+ def test_a_quota_verdict_whose_reset_has_passed_is_dropped(self):
530
+ L, now = self.L, self.L + 600
531
+ tr = self.tracker()
532
+ self.assertIsNone(tr.feed(cc_quota(L + 10, resets=L + 300), now)) # main thread
533
+ self.assertIsNone(tr.pending)
534
+ self.assertEqual(tr.feed(cc_quota(L + 11), now)[0], "error") # no reset: stands
535
+ self.assertIsNone(tr.pending["reset"])
536
+ tr = self.tracker()
537
+ self.assertEqual(tr.feed(cc_quota(L + 12, resets=L + 9000), now)[0], "error")
538
+ tr = self.tracker()
539
+ self.assertIsNone(tr.feed_side(side_quota(L + 20, L + 300), now)) # an agent's
540
+ self.assertIsNone(tr.pending)
541
+ self.assertEqual(tr.feed_side(side_quota(L + 21, L + 9000), now)[0], "error")
542
+ # one whose reset passes while it waits is dropped then (Watcher.decide)
543
+ self.assertEqual(tr.expire(L + 8999), (False, None))
544
+ self.assertEqual(tr.expire(L + 9000), (True, None))
545
+ self.assertIsNone(tr.pending)
546
+ # nothing but a quota verdict has a reset to pass
547
+ tr = self.tracker()
548
+ tr.feed(cc_api_error(L + 30, "authentication_failed"), now)
549
+ self.assertEqual(tr.expire(L + 10 ** 6), (False, None))
550
+ self.assertEqual(tr.pending["class"], "auth")
551
+
552
+ def test_an_agents_account_verdict_outranks_the_main_threads_model_or_transient_one(self):
553
+ L = self.L
554
+ mains = {"model": model_limit, "concurrents": cc_concurrents,
555
+ "overloaded": lambda ts: cc_api_error(ts, "overloaded")}
556
+ for name, main in sorted(mains.items()):
557
+ for agent_first in (False, True):
558
+ with self.subTest(main=name, agent_first=agent_first):
559
+ tr = self.tracker()
560
+ if agent_first:
561
+ self.assertEqual(tr.feed_side(side_quota(L + 5, L + 9000), 1, AGENT)[0],
562
+ "error")
563
+ self.assertIsNone(tr.feed(main(L + 10), 2))
564
+ else:
565
+ self.assertEqual(tr.feed(main(L + 10), 1)[0], "error")
566
+ self.assertEqual(tr.feed_side(side_quota(L + 20, L + 9000), 2, AGENT)[0],
567
+ "error")
568
+ self.assertEqual((tr.pending["class"], tr.pending["src"], tr.pending["agent"],
569
+ tr.pending["reset"], tr.pending_since),
570
+ ("quota", "sidechain", AGENT, L + 9000, 1 + (not agent_first)))
571
+ self.assertIn(tr.fallback["class"], ("model", "transient"))
572
+ self.assertIsNone(tr.fallback.get("src"))
573
+ # /model ends the model's limit or the failed turn, never the account's
574
+ self.assertIsNone(tr.feed(cc_slash(L + 30)[1], 3))
575
+ self.assertEqual((tr.pending["class"], tr.fallback), ("quota", None))
576
+ # an agent's failed login just the same
577
+ tr = self.tracker()
578
+ tr.feed(cc_api_error(L + 10, "overloaded"), 1)
579
+ tr.feed_side(side(cc_api_error(L + 20, "authentication_failed")), 2, AGENT2)
580
+ self.assertEqual((tr.pending["class"], tr.pending["agent"], tr.fallback["class"]),
581
+ ("auth", AGENT2, "transient"))
582
+ # nothing waits behind a main-thread account verdict an agent's cannot outrank...
583
+ tr = self.tracker()
584
+ tr.feed(cc_api_error(L + 10, "authentication_failed"), 1)
585
+ tr.feed_side(side_quota(L + 20, L + 9000), 2)
586
+ self.assertEqual((tr.pending["class"], tr.fallback), ("auth", None))
587
+ # ...which outranks the main thread's own later throttle too
588
+ tr = self.tracker()
589
+ tr.feed(cc_quota(L + 10, resets=L + 9000), 1)
590
+ self.assertIsNone(tr.feed(cc_concurrents(L + 20), 2))
591
+ self.assertEqual((tr.pending["class"], tr.pending_since, tr.fallback["class"]),
592
+ ("quota", 1, "transient"))
593
+ # a main-thread account verdict of the same class takes the agent's in
594
+ tr = self.tracker()
595
+ tr.feed(model_limit(L + 10), 1)
596
+ tr.feed_side(side_quota(L + 20, L + 9000, rtype="seven_day"), 2)
597
+ tr.feed(cc_quota(L + 25, resets=L + 3600), 3)
598
+ self.assertEqual((tr.pending["reset"], tr.pending["rtype"], tr.pending.get("src"),
599
+ tr.fallback), (L + 9000, "seven_day", None, None))
600
+
601
+ def test_what_it_outranks_ends_with_the_main_threads_own_next_word(self):
602
+ L = self.L
603
+ # the user typing after both: neither stands
604
+ tr = self.tracker()
605
+ tr.feed(model_limit(L + 10), 1)
606
+ tr.feed_side(side_quota(L + 20, L + 9000), 2)
607
+ self.assertEqual(tr.feed(cc_user(L + 30, "use sonnet then"), 3), ("cancel", "user"))
608
+ self.assertEqual((tr.pending, tr.fallback), (None, None))
609
+ # an agent's reply to a request sent after the account's verdict answers the account
610
+ # alone: the main thread, still stopped, has its own verdict back
611
+ tr = self.tracker()
612
+ tr.feed(cc_concurrents(L + 10), 1)
613
+ main = tr.pending
614
+ tr.feed_side(side_quota(L + 20, L + 9000), 2)
615
+ self.assertEqual(tr.feed_side(side_reply(L + 25, AGENT2), 3, AGENT2), ("error", main))
616
+ self.assertEqual((tr.pending, tr.pending_since, tr.fallback), (main, 3, None))
617
+ self.assertEqual(tr.feed(cc_slash(L + 30)[1], 4), ("cancel", "command"))
618
+ self.assertIsNone(tr.pending)
619
+ # a main-thread reply to a request sent before the account's verdict ends the main
620
+ # thread's own verdict only
621
+ tr = self.tracker()
622
+ tr.feed(api_request(L + 5, "req_m1"), 1)
623
+ tr.feed(model_limit(L + 10), 1)
624
+ tr.feed_side(side_quota(L + 20, L + 9000), 2)
625
+ self.assertIsNone(tr.feed(main_reply(L + 25, "req_m1"), 3))
626
+ self.assertEqual((tr.pending["class"], tr.pending["src"], tr.fallback),
627
+ ("quota", "sidechain", None))
628
+ # a quota verdict that lapses lets the one behind it stand...
629
+ tr = self.tracker()
630
+ tr.feed(cc_concurrents(L + 10), 1)
631
+ tr.feed_side(side_quota(L + 20, L + 900), 2, AGENT3)
632
+ self.assertEqual((tr.pending["class"], tr.fallback["class"]), ("quota", "transient"))
633
+ dropped, promoted = tr.expire(L + 900)
634
+ self.assertTrue(dropped)
635
+ self.assertIs(tr.pending, promoted)
636
+ self.assertEqual((promoted["class"], promoted.get("src"), tr.pending_since),
637
+ ("transient", None, L + 900))
638
+ # ...unless the main thread moved on meanwhile
639
+ tr = self.tracker()
640
+ tr.feed(cc_concurrents(L + 10), 1)
641
+ tr.feed_side(side_quota(L + 20, L + 900), 2)
642
+ tr.feed(cc_slash(L + 30)[1], 3)
643
+ self.assertEqual(tr.expire(L + 900), (True, None))
644
+ self.assertIsNone(tr.pending)
645
+ # a new conversation forgets it
646
+ tr = self.tracker()
647
+ tr.feed(model_limit(L + 10), 1)
648
+ tr.feed_side(side_quota(L + 20, L + 9000), 2)
649
+ tr.reset()
650
+ self.assertIsNone(tr.fallback)
651
+
652
+ def test_one_classes_rejections_are_marked_with_the_latest_reset(self):
653
+ L = self.L
654
+ five, week = (L + 3600, "five_hour"), (L + 86400, "seven_day")
655
+ for first, second in ((five, week), (week, five)):
656
+ with self.subTest(first=first[1]):
657
+ # the main thread: the newest record stands, with the latest reset
658
+ tr = self.tracker()
659
+ tr.feed(cc_quota(L + 10, first[1], resets=first[0]), 1)
660
+ tr.feed(cc_quota(L + 20, second[1], resets=second[0]), 2)
661
+ self.assertEqual((tr.pending["reset"], tr.pending["rtype"], tr.pending["at"],
662
+ tr.pending_since), (L + 86400, "seven_day", L + 20.009, 1))
663
+ # agents, in either order of reading
664
+ for stamps in ((L + 10, L + 20), (L + 20, L + 10)):
665
+ tr = self.tracker()
666
+ tr.feed_side(side_quota(stamps[0], first[0], rtype=first[1]), 1, AGENT)
667
+ self.assertIsNone(tr.feed_side(side_quota(stamps[1], second[0], AGENT2,
668
+ rtype=second[1]), 2, AGENT2))
669
+ self.assertEqual((tr.pending["reset"], tr.pending["rtype"], tr.pending["at"],
670
+ tr.pending["agent"]),
671
+ (L + 86400, "seven_day", L + 20.009, AGENT))
672
+ # an agent's later reset reaches the main thread's verdict too
673
+ tr = self.tracker()
674
+ tr.feed(cc_quota(L + 10, first[1], resets=first[0]), 1)
675
+ tr.feed_side(side_quota(L + 20, second[0], rtype=second[1]), 2)
676
+ self.assertEqual((tr.pending["reset"], tr.pending["rtype"], tr.pending.get("src")),
677
+ (L + 86400, "seven_day", None))
678
+ # a rejection with no reset takes nothing away
679
+ tr = self.tracker()
680
+ tr.feed(cc_quota(L + 10, resets=L + 3600), 1)
681
+ tr.feed(cc_quota(L + 20), 2)
682
+ self.assertEqual(tr.pending["reset"], L + 3600)
683
+
684
+
685
+ # =====================================================================================
686
+ # the watcher
687
+ # =====================================================================================
688
+
689
+ class SidechainWatcherTests(SidechainCase):
690
+ def test_a_workflow_agents_five_hour_limit_moves_the_idle_session(self):
691
+ reset = int(self.t0) + 7200
692
+ w, r = self.start(["--dangerously-skip-permissions", "fix the flaky test"])
693
+ self.agent_file([side_reply(self.launched + 3), side_quota(self.t0 - 1, reset)])
694
+ # a workflow's journal and an agent's metadata are no transcripts
695
+ wf = self.proj / SID / "subagents" / "workflows" / "wf_7f3a9c"
696
+ (wf / "journal.jsonl").write_text(compact([side_quota(self.t0 - 1, reset, AGENT2)]))
697
+ (wf / ("agent-%s.meta.json" % AGENT2)).write_text(compact([side_quota(self.t0 - 1, reset)]))
698
+ self.assertEqual(w.run(), 0)
699
+ prompt = ar.resume_prompt("quota")
700
+ self.assertEqual(len(r.probes), 1)
701
+ probe = r.probes[0]
702
+ self.assertGreaterEqual(probe["t"] - self.t0, 5.0) # GRACE, as for the main thread
703
+ self.assertEqual(probe["argv"], [self.self_path, "--dangerously-skip-permissions",
704
+ "--resume", SID, prompt])
705
+ self.assertEqual(probe["env"]["CLAUDE_MULTIACC_AR_AVOID"], "acct-01:%d" % reset)
706
+ self.assertEqual(r.signals, [(self.PID, signal.SIGTERM)])
707
+ self.assert_sequence(r)
708
+ kv, argv = self.relaunch_of(r)
709
+ self.assertEqual(argv, ["--dangerously-skip-permissions", "--resume", SID, prompt])
710
+ self.assertEqual({k: kv[k] for k in ("class", "acct", "reset", "rtype", "marked", "sid",
711
+ "depth", "avoid")},
712
+ {"class": "quota", "acct": "acct-01", "reset": str(reset),
713
+ "rtype": "five_hour", "marked": ar.utc_iso(self.t0 - 1), "sid": SID,
714
+ "depth": "1", "avoid": "acct-01:%d" % reset})
715
+ self.assertEqual(self.events(), ["watch", "detect", "switch"])
716
+ self.assertIn(" autoresume detect provider=claude chain=%s class=quota code=rate_limit "
717
+ "sid=%s acct=acct-01 reset=%d src=sidechain agent=%s"
718
+ % (kv["chain"], SID, reset, AGENT), self.log_lines()[1])
719
+ for line in self.log_lines():
720
+ self.assertEqual(line.split()[1], "autoresume")
721
+ self.assertNotIn("session limit", line)
722
+ self.assertEqual(sorted(os.path.basename(p) for p in w.side),
723
+ ["agent-%s.jsonl" % AGENT])
724
+
725
+ def test_a_subagents_failed_login_moves_it_too(self):
726
+ w, r = self.start(["-c"])
727
+ self.agent_file([side(cc_api_error(self.t0, "authentication_failed",
728
+ "Not logged in · Please run /login"))], workflow=None)
729
+ w.run()
730
+ until = int(r.probes[0]["t"]) + 3600
731
+ self.assertEqual(r.probes[0]["env"]["CLAUDE_MULTIACC_AR_AVOID"], "acct-01:%d" % until)
732
+ kv, argv = self.relaunch_of(r)
733
+ self.assertEqual(argv, ["--resume", SID, ar.resume_prompt("auth")])
734
+ self.assertEqual((kv["class"], kv["reset"], kv["rtype"]), ("auth", "", ""))
735
+ self.assertIn("class=auth code=authentication_failed", self.log_lines()[1])
736
+ self.assertIn("src=sidechain agent=%s" % AGENT, self.log_lines()[1])
737
+
738
+ def test_an_organization_refusal_in_a_subagent_parks_like_the_main_threads(self):
739
+ w, r = self.start([])
740
+ self.agent_file([side(cc_api_error(self.t0, "oauth_org_not_allowed"))])
741
+ w.run()
742
+ kv, _ = self.relaunch_of(r)
743
+ self.assertEqual((kv["class"], kv["rtype"]), ("blocked", "oauth_org_not_allowed"))
744
+
745
+ def test_a_subagents_own_model_transient_or_other_errors_never_move_it(self):
746
+ w, r = self.start([])
747
+ self.agent_file([side_quota(self.t0, int(self.t0) + 999, rtype="seven_day_opus"),
748
+ side(cc_api_error(self.t0, "rate_limit",
749
+ "You've reached your Fable limit. Switch models")),
750
+ side(cc_api_error(self.t0, "overloaded")),
751
+ side(cc_api_error(self.t0, "invalid_request", "bad tool schema")),
752
+ side(cc_user(self.t0 + 1, "carry on"))])
753
+ self.stop_at(r, self.t0 + 30)
754
+ self.assertEqual(w.run(), 0)
755
+ self.assert_untouched(r)
756
+ self.assertEqual(self.events(), ["watch"]) # not even a `never` line
757
+
758
+ def test_a_rejection_from_before_the_launch_never_fires(self):
759
+ # a resumed conversation's agents met a limit in an earlier launch
760
+ w, r = self.start(["--resume", SID])
761
+ self.agent_file([side_quota(self.launched - 3, int(self.t0) + 999)])
762
+ self.stop_at(r, self.t0 + 20)
763
+ self.assertEqual(w.run(), 0)
764
+ self.assert_untouched(r)
765
+ self.assertEqual(self.events(), ["watch"])
766
+
767
+ def test_a_transcript_last_written_before_the_launch_is_followed_from_its_end(self):
768
+ reset = int(self.t0) + 999
769
+ # mis-dated on purpose: were this file read from its start, the rejection would count
770
+ path = self.agent_file([side_quota(self.t0, reset)], mtime=self.launched - 60)
771
+ w, r = self.start([])
772
+ size = path.stat().st_size
773
+ r.at(self.t0 + 10, lambda: fixtures.append(path, [side_quota(self.t0 + 10, reset)]))
774
+ self.assertEqual(w.run(), 0)
775
+ self.assertGreaterEqual(r.probes[0]["t"], self.t0 + 15)
776
+ self.assertGreater(self.tail_of(w, path.name).off, size)
777
+ kv, _ = self.relaunch_of(r)
778
+ self.assertEqual((kv["class"], kv["marked"]), ("quota", ar.utc_iso(self.t0 + 10)))
779
+
780
+ def test_a_later_reply_of_that_agent_or_another_cancels(self):
781
+ for later in (AGENT, AGENT2):
782
+ with self.subTest(agent=later):
783
+ log = self.root / "selection.log"
784
+ if log.exists():
785
+ log.unlink()
786
+ w, r = self.start([])
787
+ path = self.agent_file([side_quota(self.t0, int(self.t0) + 999)],
788
+ workflow="wf_" + later)
789
+ other = self.agent_file([side_reply(self.launched + 1, AGENT2)],
790
+ agent=AGENT2, workflow="wf_" + later + "x")
791
+ target = path if later == AGENT else other
792
+ r.at(self.t0 + 2, lambda target=target, later=later: fixtures.append(
793
+ target, [side_reply(self.t0 + 2, later)]))
794
+ self.stop_at(r, self.t0 + 20)
795
+ self.assertEqual(w.run(), 0)
796
+ self.assert_untouched(r)
797
+ self.assertEqual(self.events(), ["watch", "detect"])
798
+ for p in (path, other):
799
+ p.unlink()
800
+
801
+ def test_a_later_main_thread_reply_or_the_user_typing_cancels(self):
802
+ for rec in (cc_reply(self.t0 + 2, "The workflow finished; all green."),
803
+ cc_user(self.t0 + 2, "never mind the workflow, look at this")):
804
+ with self.subTest(type=rec["type"]):
805
+ w, r = self.start([])
806
+ tpath = self.proj / (SID + ".jsonl")
807
+ agent = self.agent_file([side_quota(self.t0, int(self.t0) + 999)])
808
+ r.at(self.t0 + 2, lambda rec=rec: fixtures.append(tpath, [rec]))
809
+ self.stop_at(r, self.t0 + 20)
810
+ self.assertEqual(w.run(), 0)
811
+ self.assert_untouched(r)
812
+ for path in (tpath, agent):
813
+ path.unlink()
814
+
815
+ def test_main_thread_words_older_than_the_rejection_do_not_cancel(self):
816
+ # written (or read) late, a reply from before the rejection answers nothing
817
+ w, r = self.start([])
818
+ tpath = self.proj / (SID + ".jsonl")
819
+ self.agent_file([side_quota(self.t0, int(self.t0) + 999)])
820
+ r.at(self.t0 + 2, lambda: fixtures.append(tpath, [cc_reply(self.t0 - 1, "earlier"),
821
+ cc_user(self.t0 - 1, "earlier")]))
822
+ self.assertEqual(w.run(), 0)
823
+ self.assertEqual(len(r.probes), 1)
824
+ self.assertEqual(self.events(), ["watch", "detect", "switch"])
825
+
826
+ def test_an_answered_rejection_never_fires_whichever_file_is_read_first(self):
827
+ reset = int(self.t0) + 999
828
+ w, r = self.start([])
829
+ # one agent was rejected; another got a reply after that (the limit had reset)
830
+ self.agent_file([side_quota(self.t0 - 2, reset)], workflow="wf_b")
831
+ self.agent_file([side_reply(self.t0 - 1, AGENT2)], agent=AGENT2, workflow="wf_a")
832
+ self.stop_at(r, self.t0 + 20)
833
+ self.assertEqual(w.run(), 0)
834
+ self.assert_untouched(r)
835
+ self.assertEqual(self.events(), ["watch"]) # never even shown as pending
836
+
837
+ def test_many_agents_meeting_the_same_limit_probe_once(self):
838
+ reset = int(self.t0) + 999
839
+ w, r = self.start([])
840
+ path = self.agent_file([side_quota(self.t0 - 1, reset)])
841
+ self.agent_file([side_quota(self.t0 - 0.5, reset, AGENT2)], agent=AGENT2,
842
+ workflow="wf_2")
843
+ real_run = r.run
844
+
845
+ def run(argv, timeout, env=None, cwd=None):
846
+ if argv[0] == self.self_path: # one more agent is rejected during the probe
847
+ fixtures.append(path, [side_quota(self.t0 + 5, reset)])
848
+ return real_run(argv, timeout, env, cwd)
849
+ r.run = run
850
+ self.assertEqual(w.run(), 0)
851
+ self.assertEqual(len(r.probes), 1)
852
+ self.assertEqual(self.events(), ["watch", "detect", "switch"])
853
+ kv, _ = self.relaunch_of(r)
854
+ self.assertEqual(kv["marked"], ar.utc_iso(self.t0 - 1))
855
+
856
+ def test_a_reply_during_the_probe_wins(self):
857
+ w, r = self.start([])
858
+ path = self.agent_file([side_quota(self.t0, int(self.t0) + 999)])
859
+ real_run = r.run
860
+
861
+ def run(argv, timeout, env=None, cwd=None):
862
+ if argv[0] == self.self_path and not r.probes:
863
+ fixtures.append(path, [side_reply(self.t0 + 5)])
864
+ return real_run(argv, timeout, env, cwd)
865
+ r.run = run
866
+ self.stop_at(r, self.t0 + 20)
867
+ w.run()
868
+ self.assertEqual(len(r.probes), 1)
869
+ self.assertEqual((r.signals, r.sends, self.relaunch_files()), ([], [], []))
870
+
871
+ def test_no_room_holds_like_the_main_thread(self):
872
+ w, r = self.start([], env={"CLAUDE_MULTIACC_AR_HOLD_REPROBE": "5"})
873
+ self.agent_file([side_quota(self.t0, int(self.t0) + 999)])
874
+ r.probe_out = "pick=acct-01 tier=soft\n"
875
+ self.stop_at(r, self.t0 + 16.5)
876
+ self.assertEqual(w.run(), 0)
877
+ self.assertEqual(len(r.probes), 3)
878
+ self.assertEqual((r.signals, r.sends, self.relaunch_files()), ([], [], []))
879
+ self.assertEqual(self.events(), ["watch", "detect", "hold"])
880
+
881
+ def test_the_replaced_processs_last_rejections_never_move_the_relaunch(self):
882
+ # The watcher of a RELAUNCHED session (on acct-01, the account it moved to). The process
883
+ # it replaced ran a workflow whose agents kept meeting the old account's limit, in a
884
+ # burst, until it was gone: inside the second `date +%s` floored to this launch.
885
+ reset = int(self.t0) + 9000
886
+ for later in (False, True):
887
+ with self.subTest(own_agent_rejected_later=later):
888
+ log = self.root / "selection.log"
889
+ if log.exists():
890
+ log.unlink()
891
+ launched = self.launched = int(self.t0) - 1
892
+ state = self.write_state(["--dangerously-skip-permissions", "--resume", SID],
893
+ depth=1, avoid="acct-02:%d" % reset)
894
+ self.registry()
895
+ tpath = self.transcript([cc_user(launched - 3600, "run the release workflow"),
896
+ cc_reply(launched - 3590, "Started it; waiting for it."),
897
+ cc_user(launched + 1, ar.resume_prompt("quota"))])
898
+ old = [self.agent_file([side_reply(launched - 400),
899
+ side_quota(launched - 2, reset)], mtime=launched - 2),
900
+ self.agent_file([side_reply(launched - 60, AGENT2),
901
+ side_quota(launched - 1, reset, AGENT2)],
902
+ agent=AGENT2, workflow="wf_2", mtime=launched - 1),
903
+ self.agent_file([side_quota(launched, reset, AGENT3)], agent=AGENT3,
904
+ workflow="wf_3", mtime=launched)]
905
+ w, r = self.watcher(state)
906
+ r.probe_out = "pick=acct-03 tier=eligible\n"
907
+ # the resumed conversation's first reply is slow (a cold cache on this account)
908
+ r.at(self.t0 + 12, lambda tpath=tpath: fixtures.append(
909
+ tpath, [cc_reply(self.t0 + 12, "Picking up where it left off.")]))
910
+ if later: # this launch's own agent meets a limit: that one counts
911
+ r.at(self.t0 + 20, lambda: write(old[2], [side_quota(self.t0 + 20, reset,
912
+ AGENT3)]))
913
+ self.stop_at(r, self.t0 + 40)
914
+ self.assertEqual(w.run(), 0)
915
+ if later:
916
+ self.assertEqual(len(r.probes), 1)
917
+ self.assertEqual(self.events(), ["watch", "detect", "switch"])
918
+ self.assertEqual(self.relaunch_of(r)[0]["marked"], ar.utc_iso(self.t0 + 20))
919
+ else:
920
+ self.assert_untouched(r)
921
+ self.assertEqual(self.events(), ["watch"])
922
+ for path in old + [tpath]:
923
+ path.unlink()
924
+
925
+ def test_a_reply_the_account_accepted_before_the_limit_answers_nothing(self):
926
+ # One agent's stream ran on past another agent's rejection (by minutes in session
927
+ # 5526ddd6, 2026-10-19). Here it is that agent's final answer: the workflow then waits
928
+ # for the rejected one, and nothing more is written.
929
+ reset = int(self.t0) + 9000
930
+ w, r = self.start([])
931
+ self.agent_file([side_reply(self.launched + 3), side_quota(self.t0 - 1, reset)])
932
+ other = self.agent_file([api_request(self.launched + 3, "req_b1", AGENT2),
933
+ side_reply(self.launched + 4, AGENT2, ref="req_b1"),
934
+ api_request(self.t0 - 30, "req_b2", AGENT2)],
935
+ agent=AGENT2, workflow="wf_b")
936
+ r.at(self.t0 + 2, lambda: write(other, [final_answer(self.t0 + 2, AGENT2, "req_b2")]))
937
+ self.assertEqual(w.run(), 0)
938
+ self.assertEqual(len(r.probes), 1)
939
+ self.assertEqual(self.events(), ["watch", "detect", "switch"])
940
+ self.assertEqual(self.relaunch_of(r)[0]["marked"], ar.utc_iso(self.t0 - 1))
941
+
942
+ def test_a_reply_to_a_request_sent_after_the_limit_still_cancels(self):
943
+ w, r = self.start([])
944
+ self.agent_file([side_quota(self.t0 - 1, int(self.t0) + 9000)])
945
+ other = self.agent_file([api_request(self.t0 - 30, "req_b1", AGENT2)], agent=AGENT2,
946
+ workflow="wf_b")
947
+ # the limit is lifted (a reset), the agent's next request is served
948
+ r.at(self.t0 + 1, lambda: write(other, [final_answer(self.t0 + 1, AGENT2, "req_b1"),
949
+ api_request(self.t0 + 1, "req_b2", AGENT2)]))
950
+ r.at(self.t0 + 3, lambda: write(other, [side_reply(self.t0 + 3, AGENT2, ref="req_b2")]))
951
+ self.stop_at(r, self.t0 + 30)
952
+ self.assertEqual(w.run(), 0)
953
+ self.assert_untouched(r)
954
+ self.assertEqual(self.events(), ["watch", "detect"])
955
+
956
+ def test_a_command_that_fixes_the_account_keeps_the_session(self):
957
+ # /login after the agents' "Not logged in"; /limit-reset while the watcher holds the
958
+ # agents' limit for want of room, before room appears
959
+ for err, name, hold in ((side(cc_api_error(self.t0, "authentication_failed",
960
+ "Not logged in · Please run /login")),
961
+ "login", False),
962
+ (side_quota(self.t0, int(self.t0) + 9000), "limit-reset", True)):
963
+ with self.subTest(command=name):
964
+ log = self.root / "selection.log"
965
+ if log.exists():
966
+ log.unlink()
967
+ w, r = self.start([], env={"CLAUDE_MULTIACC_AR_HOLD_REPROBE": "10"})
968
+ tpath = self.proj / (SID + ".jsonl")
969
+ agent = self.agent_file([err])
970
+ at = self.t0 + (8 if hold else 2)
971
+ if hold:
972
+ r.probe_out = "pick=acct-01 tier=soft\n"
973
+ r.at(self.t0 + 9, lambda r=r: setattr(r, "probe_out",
974
+ "pick=acct-02 tier=eligible\n"))
975
+ r.at(at, lambda at=at, name=name: fixtures.append(tpath, cc_slash(at, name)))
976
+ self.stop_at(r, self.t0 + 40)
977
+ self.assertEqual(w.run(), 0)
978
+ self.assertEqual((len(r.probes), r.signals, r.sends, self.relaunch_files()),
979
+ (int(hold), [], [], []))
980
+ self.assertEqual(self.events(), ["watch", "detect"] + ["hold"] * hold)
981
+ for path in (tpath, agent):
982
+ path.unlink()
983
+
984
+ def test_a_session_change_resets_the_subagent_tails(self):
985
+ reset = int(self.t0) + 999
986
+ w, r = self.start([])
987
+ old = self.agent_file([side_quota(self.t0, reset)])
988
+ self.transcript([cc_user(self.t0 + 1, "a new task")], sid=SID2)
989
+ # /clear before GRACE: the old conversation's agents are no longer this session's
990
+ r.at(self.t0 + 1.5, lambda: self.registry(sid=SID2))
991
+ r.at(self.t0 + 3, lambda: fixtures.append(old, [side_quota(self.t0 + 3, reset)]))
992
+ r.at(self.t0 + 20, lambda: self.agent_file([side_quota(self.t0 + 20, reset)], sid=SID2))
993
+ self.assertEqual(w.run(), 0)
994
+ self.assertEqual(len(r.probes), 1)
995
+ self.assertGreaterEqual(r.probes[0]["t"], self.t0 + 25)
996
+ kv, argv = self.relaunch_of(r)
997
+ self.assertEqual((kv["sid"], argv[-2]), (SID2, SID2))
998
+ self.assertEqual(self.events(), ["watch", "detect", "detect", "switch"])
999
+ self.assertTrue(w.side and all(SID2 in path and SID not in path for path in w.side))
1000
+
1001
+
1002
+ class LiftedFallbackLatestResetWatcherTests(SidechainCase):
1003
+ def fresh(self):
1004
+ for path in [self.root / "selection.log", self.proj / (SID + ".jsonl")] \
1005
+ + list((self.proj / SID).glob("subagents/**/agent-*.jsonl")):
1006
+ if path.exists():
1007
+ path.unlink()
1008
+
1009
+ def test_a_limit_that_has_reset_already_moves_nothing(self):
1010
+ # a watcher that reads a rejection only once its reset has passed (restarted onto
1011
+ # the session, or a slow replay): the account serves again
1012
+ for where in ("main", "agent"):
1013
+ with self.subTest(where=where):
1014
+ self.fresh()
1015
+ main = [cc_user(self.launched + 1, "go"), cc_reply(self.launched + 2, "ok")]
1016
+ if where == "main":
1017
+ main.append(cc_quota(self.t0 - 2, resets=int(self.t0) - 1))
1018
+ w, r = self.start([], main=main)
1019
+ if where == "agent":
1020
+ self.agent_file([side_quota(self.t0 - 2, int(self.t0) - 1)])
1021
+ self.stop_at(r, self.t0 + 20)
1022
+ self.assertEqual(w.run(), 0)
1023
+ self.assert_untouched(r)
1024
+ self.assertEqual(self.events(), ["watch"])
1025
+
1026
+ def test_a_verdict_whose_reset_passes_in_a_hold_is_dropped(self):
1027
+ reset = int(self.t0) + 9
1028
+ for where in ("main", "agent"):
1029
+ with self.subTest(where=where):
1030
+ self.fresh()
1031
+ main = [cc_user(self.launched + 1, "go"), cc_reply(self.launched + 2, "ok")]
1032
+ if where == "main":
1033
+ main.append(cc_quota(self.t0, resets=reset))
1034
+ w, r = self.start([], main=main, env={"CLAUDE_MULTIACC_AR_HOLD_REPROBE": "2"})
1035
+ if where == "agent":
1036
+ self.agent_file([side_quota(self.t0, reset)])
1037
+ r.probe_out = "pick=acct-01 tier=soft\n" # nowhere to go: hold
1038
+ r.at(self.t0 + 12, lambda r=r: setattr(r, "probe_out",
1039
+ "pick=acct-02 tier=eligible\n"))
1040
+ self.stop_at(r, self.t0 + 30)
1041
+ self.assertEqual(w.run(), 0)
1042
+ self.assertTrue(r.probes)
1043
+ self.assertTrue(all(p["t"] < reset for p in r.probes), r.probes)
1044
+ self.assertEqual((r.signals, r.sends, self.relaunch_files()), ([], [], []))
1045
+ self.assertEqual(self.events(), ["watch", "detect", "hold"])
1046
+
1047
+ def test_model_ends_the_main_threads_model_verdict_but_not_the_agents_limit(self):
1048
+ reset = int(self.t0) + 7200
1049
+ w, r = self.start([], main=[cc_user(self.launched + 1, "run the release workflow"),
1050
+ cc_reply(self.launched + 2, "Started it."),
1051
+ model_limit(self.t0)])
1052
+ self.agent_file([side_quota(self.t0, reset)])
1053
+ tpath = self.proj / (SID + ".jsonl")
1054
+ r.at(self.t0 + 2, lambda: fixtures.append(tpath, cc_slash(self.t0 + 2)))
1055
+ self.stop_at(r, self.t0 + 40)
1056
+ self.assertEqual(w.run(), 0)
1057
+ self.assertEqual(len(r.probes), 1)
1058
+ # the agent's verdict outranked the model's from the start: its grace is its own
1059
+ self.assertGreaterEqual(r.probes[0]["t"], self.t0 + 5)
1060
+ self.assertLess(r.probes[0]["t"], self.t0 + 7)
1061
+ kv, _ = self.relaunch_of(r)
1062
+ self.assertEqual((kv["class"], kv["reset"], kv["rtype"]),
1063
+ ("quota", str(reset), "five_hour"))
1064
+ self.assertEqual(self.events(), ["watch", "detect", "detect", "switch"])
1065
+ detects = [line for line in self.log_lines() if line.split()[2] == "detect"]
1066
+ self.assertIn("class=model", detects[0])
1067
+ self.assertIn("class=quota", detects[1])
1068
+ self.assertIn("src=sidechain agent=%s" % AGENT, detects[1])
1069
+
1070
+ def test_agents_meeting_the_limit_move_a_session_its_throttled_main_thread_held(self):
1071
+ # 2026-10-20, session 6: the main thread's "concurrents" rejection (read as a model's
1072
+ # limit) waited on its busy agents while, from three minutes later, they were all
1073
+ # meeting the five-hour limit; the session moved six minutes after that rejection
1074
+ reset = int(self.t0) + 9000
1075
+ w, r = self.start([], main=[cc_user(self.launched + 1, "run the i18n workflows"),
1076
+ cc_reply(self.launched + 2, "Started them."),
1077
+ cc_concurrents(self.t0)])
1078
+ self.agent_file([side_reply(self.launched + 3, AGENT2)], agent=AGENT2,
1079
+ workflow="wf_2", mtime=self.t0 + 900) # writing all along
1080
+ path = self.agent_file([side_reply(self.launched + 3)])
1081
+ r.at(self.t0 + 180, lambda: fixtures.append(path, [side_quota(self.t0 + 180, reset)]))
1082
+ self.assertEqual(w.run(), 0)
1083
+ self.assertEqual(len(r.probes), 1)
1084
+ self.assertGreaterEqual(r.probes[0]["t"], self.t0 + 185)
1085
+ self.assertLess(r.probes[0]["t"], self.t0 + 195) # its grace, not the agents' quiet
1086
+ kv, argv = self.relaunch_of(r)
1087
+ self.assertEqual((kv["class"], kv["reset"], kv["rtype"]),
1088
+ ("quota", str(reset), "five_hour"))
1089
+ self.assertEqual(argv, ["--resume", SID, ar.resume_prompt("quota")])
1090
+ self.assertEqual(self.events(), ["watch", "detect", "detect", "switch"])
1091
+ detects = [line for line in self.log_lines() if line.split()[2] == "detect"]
1092
+ self.assertIn("class=transient code=rate_limit", detects[0])
1093
+ self.assertIn("class=quota", detects[1])
1094
+ self.assertIn("src=sidechain agent=%s" % AGENT, detects[1])
1095
+
1096
+ def test_a_throttled_main_thread_waits_for_its_working_agents_and_keeps_the_account(self):
1097
+ w, r = self.start([], main=[cc_user(self.launched + 1, "run the i18n workflows"),
1098
+ cc_reply(self.launched + 2, "Started them."),
1099
+ cc_concurrents(self.t0)])
1100
+ self.agent_file([side_reply(self.launched + 3)], mtime=self.t0 + 100) # until ~t0+100
1101
+ self.assertEqual(w.run(), 0)
1102
+ self.assertEqual(len(r.probes), 1)
1103
+ self.assertGreaterEqual(r.probes[0]["t"], self.t0 + 120) # 20 s of agent quiet
1104
+ self.assertEqual(r.probes[0]["env"]["CLAUDE_MULTIACC_AR_AVOID"], "") # nothing avoided
1105
+ kv, argv = self.relaunch_of(r)
1106
+ self.assertEqual((kv["class"], kv["avoid"]), ("transient", ""))
1107
+ self.assertEqual(argv, ["--resume", SID, ar.resume_prompt("transient")])
1108
+
1109
+ def test_the_latest_reset_of_one_limit_is_the_one_marked(self):
1110
+ five, week = int(self.t0) + 3600, int(self.t0) + 86400
1111
+ w, r = self.start([])
1112
+ # the weekly limit, met first, sits in the transcript read second
1113
+ self.agent_file([side_quota(self.t0 - 1, week, rtype="seven_day")], mtime=self.t0 - 2)
1114
+ self.agent_file([side_quota(self.t0 - 0.5, five, AGENT2)], agent=AGENT2,
1115
+ workflow="wf_2", mtime=self.t0 - 1)
1116
+ self.assertEqual(w.run(), 0)
1117
+ self.assertEqual(len(r.probes), 1)
1118
+ kv, _ = self.relaunch_of(r)
1119
+ self.assertEqual((kv["class"], kv["reset"], kv["rtype"]),
1120
+ ("quota", str(week), "seven_day"))
1121
+
1122
+
1123
+ class SidechainReplayTests(SidechainCase):
1124
+ """A watcher (re)started onto a live session reads everything written since its launch,
1125
+ hours of it: an agent's old rejection must never stop a session served since."""
1126
+
1127
+ def setUp(self):
1128
+ super().setUp()
1129
+ self.launched = int(self.t0) - 3600 # launched an hour ago; the watcher starts now
1130
+
1131
+ def restarted(self, main, agent=None):
1132
+ state = self.write_state(["--dangerously-skip-permissions"])
1133
+ self.registry()
1134
+ self.transcript(main)
1135
+ if agent:
1136
+ self.agent_file(agent)
1137
+ return self.watcher(state)
1138
+
1139
+ def replay(self, last_reply):
1140
+ """Read 8 KB a tick: the rejected agent, written last (its re-run request is in
1141
+ flight), comes first; a busy agent whose newest reply is ``last_reply`` is reached
1142
+ only well after GRACE."""
1143
+ w, r = self.restarted([cc_user(self.launched + 10, "run the workflow"),
1144
+ cc_reply(self.launched + 20, "Started it.")])
1145
+ self.agent_file([side_quota(self.t0 - 1800, int(self.t0) + 9000), request_blob()],
1146
+ mtime=self.t0 - 60)
1147
+ busy = [side_reply(self.launched + 100 + i, AGENT2, "x" * 900) for i in range(100)]
1148
+ path = self.agent_file(busy + [side_reply(last_reply, AGENT2)], agent=AGENT2,
1149
+ workflow="wf_busy", mtime=self.t0 - 590)
1150
+ self.assertGreater(path.stat().st_size, 10 * 8192)
1151
+ return w, r
1152
+
1153
+ def test_a_rejection_waits_for_the_rest_of_the_replay(self):
1154
+ w, r = self.replay(self.t0 - 600) # served since the rejection
1155
+ self.stop_at(r, self.t0 + 30)
1156
+ with mock.patch.object(ar, "SIDE_BYTES", 8192):
1157
+ self.assertEqual(w.run(), 0)
1158
+ self.assert_untouched(r)
1159
+ self.assertEqual(self.events(), ["watch", "detect"]) # pending, then answered
1160
+
1161
+ def test_a_rejection_nothing_answers_still_moves_once_all_is_read(self):
1162
+ w, r = self.replay(self.launched + 300) # the busy agent's last reply came before it
1163
+ real_run, unread = r.run, []
1164
+
1165
+ def run(argv, timeout, env=None, cwd=None):
1166
+ if argv[0] == self.self_path:
1167
+ unread.extend(p for p, t in w.side.items() if t.off != os.path.getsize(p))
1168
+ return real_run(argv, timeout, env, cwd)
1169
+ r.run = run
1170
+ with mock.patch.object(ar, "SIDE_BYTES", 8192):
1171
+ self.assertEqual(w.run(), 0)
1172
+ self.assertEqual(len(r.probes), 1)
1173
+ self.assertEqual(unread, []) # probed only once the whole replay was read
1174
+ self.assertGreaterEqual(r.probes[0]["t"], self.t0 + 10)
1175
+ self.assertEqual(self.events(), ["watch", "detect", "switch"])
1176
+
1177
+ def test_a_main_thread_served_since_the_rejection_is_left_alone(self):
1178
+ # 2026-10-19: a workflow agent rejected 30 min ago, the main thread replying since
1179
+ w, r = self.restarted(
1180
+ [cc_user(self.launched + 10, "run the workflow"),
1181
+ api_request(self.launched + 11, "req_m1"),
1182
+ main_reply(self.launched + 20, "req_m1", "Started it."),
1183
+ cc_user(self.t0 - 600, "forget the workflow, do X"),
1184
+ api_request(self.t0 - 599, "req_m2"),
1185
+ main_reply(self.t0 - 590, "req_m2", "Doing X.")]
1186
+ + [request_blob()] * 200 + [tool_result(self.t0 - 589)] + cc_slash(self.t0 - 60),
1187
+ [side_reply(self.launched + 30), side_quota(self.t0 - 1800, int(self.t0) + 9000)])
1188
+ self.stop_at(r, self.t0 + 30)
1189
+ self.assertEqual(w.run(), 0)
1190
+ self.assert_untouched(r)
1191
+ self.assertEqual(self.events(), ["watch"]) # never even pending
1192
+
1193
+ def test_a_main_thread_stream_that_spanned_the_rejection_still_moves(self):
1194
+ # the main thread's last reply ended after the agents' limit, but the account had
1195
+ # accepted its request before it; idle since
1196
+ reset = int(self.t0) + 9000
1197
+ w, r = self.restarted(
1198
+ [cc_user(self.launched + 10, "run the workflow, then write the report"),
1199
+ api_request(self.t0 - 1900, "req_m1"),
1200
+ main_reply(self.t0 - 1700, "req_m1", "Report written; waiting for the workflow."),
1201
+ queue_op(self.t0 - 1650)],
1202
+ [side_reply(self.launched + 30), side_quota(self.t0 - 1800, reset)])
1203
+ self.assertEqual(w.run(), 0)
1204
+ self.assertEqual(len(r.probes), 1)
1205
+ self.assertEqual(self.events(), ["watch", "detect", "switch"])
1206
+
1207
+ def test_an_agents_stream_accepted_before_the_rejection_answers_nothing_on_replay(self):
1208
+ reset = int(self.t0) + 9000
1209
+ w, r = self.restarted([cc_user(self.launched + 10, "run the workflow"),
1210
+ cc_reply(self.launched + 20, "Started it."),
1211
+ queue_op(self.t0 - 1700)],
1212
+ [side_reply(self.launched + 30), side_quota(self.t0 - 1800, reset)])
1213
+ self.agent_file([api_request(self.t0 - 1830, "req_b1", AGENT2),
1214
+ final_answer(self.t0 - 1500, AGENT2, "req_b1")], agent=AGENT2,
1215
+ workflow="wf_b")
1216
+ self.assertEqual(w.run(), 0)
1217
+ self.assertEqual(len(r.probes), 1)
1218
+ self.assertEqual(self.events(), ["watch", "detect", "switch"])
1219
+
1220
+ def test_a_prompt_typed_since_counts_while_its_reply_is_on_its_way(self):
1221
+ w, r = self.restarted(
1222
+ [cc_user(self.launched + 10, "run the workflow"),
1223
+ cc_reply(self.launched + 20, "Started it."),
1224
+ cc_user(self.t0 - 2, "the limit has reset, carry on")],
1225
+ [side_quota(self.t0 - 1800, int(self.t0) + 9000)])
1226
+ self.stop_at(r, self.t0 + 30)
1227
+ self.assertEqual(w.run(), 0)
1228
+ self.assert_untouched(r)
1229
+ self.assertEqual(self.events(), ["watch"])
1230
+
1231
+ def test_a_session_stuck_since_the_rejection_still_moves(self):
1232
+ # sessions 5 and 8 as a restarted watcher reads them: the main thread's last reply
1233
+ # came before the agents' limit; since then only the paused workflow's notices,
1234
+ # /model, and words of the client's own
1235
+ reset = int(self.t0) + 9000
1236
+ w, r = self.restarted(
1237
+ [cc_user(self.launched + 10, "run the workflow"),
1238
+ cc_reply(self.launched + 20, "Started it; waiting for the workflow."),
1239
+ queue_op(self.t0 - 1700)] + cc_slash(self.t0 - 600) + [synthetic(self.t0 - 599)],
1240
+ [side_reply(self.launched + 30), side_quota(self.t0 - 1800, reset)])
1241
+ self.assertEqual(w.run(), 0)
1242
+ self.assertEqual(len(r.probes), 1)
1243
+ self.assertEqual(self.events(), ["watch", "detect", "switch"])
1244
+ kv, _ = self.relaunch_of(r)
1245
+ self.assertEqual((kv["class"], kv["reset"], kv["marked"]),
1246
+ ("quota", str(reset), ar.utc_iso(self.t0 - 1800)))
1247
+
1248
+ def test_an_agent_transcript_found_after_the_main_threads_answer_is_answered(self):
1249
+ # whatever order things are read in: the reply was read (and skipped) first
1250
+ w, r = self.start([], main=[cc_user(self.launched + 1, "run the release workflow"),
1251
+ cc_reply(self.t0 - 2, "On it.")])
1252
+ r.at(self.t0 + 6, lambda: self.agent_file([side_quota(self.t0 - 3, int(self.t0) + 999)]))
1253
+ self.stop_at(r, self.t0 + 30)
1254
+ self.assertEqual(w.run(), 0)
1255
+ self.assert_untouched(r)
1256
+ self.assertEqual(self.events(), ["watch"])
1257
+
1258
+
1259
+ class SidechainDiscoveryTests(SidechainCase):
1260
+ def test_new_agents_are_looked_for_every_subagent_scan_seconds(self):
1261
+ w, r = self.start([], env={"CLAUDE_MULTIACC_AR_SUBAGENT_SCAN": "3"})
1262
+ self.assertEqual(w.cfg.subagent_scan, 3.0)
1263
+ self.assertEqual(ar.Config("claude", {}).subagent_scan, 5.0)
1264
+ self.agent_file([side_reply(self.launched + 1)])
1265
+ w.read()
1266
+ self.assertEqual(len(w.side), 1)
1267
+ later = self.agent_file([side_quota(self.t0, int(self.t0) + 999, AGENT2)],
1268
+ agent=AGENT2, workflow=None)
1269
+ r.t += 2
1270
+ w.read()
1271
+ self.assertEqual((len(w.side), w.tracker.pending), (1, None))
1272
+ r.t += 1
1273
+ w.read()
1274
+ self.assertEqual(self.tail_of(w, later.name).off, later.stat().st_size)
1275
+ self.assertEqual(w.tracker.pending["agent"], AGENT2)
1276
+
1277
+ def test_only_the_most_recently_written_are_followed(self):
1278
+ reset = int(self.t0) + 999
1279
+ w, r = self.start([])
1280
+ oldest = self.agent_file([side_quota(self.t0, reset)], workflow="wf_1",
1281
+ mtime=self.launched + 1)
1282
+ self.agent_file([side_reply(self.launched + 1, AGENT2)], agent=AGENT2,
1283
+ workflow="wf_2", mtime=self.launched + 2)
1284
+ newest = self.agent_file([side_reply(self.launched + 1, AGENT3)], agent=AGENT3,
1285
+ workflow="wf_3", mtime=self.launched + 3)
1286
+ with mock.patch.object(ar, "SIDE_FILES", 2):
1287
+ w.read()
1288
+ self.assertEqual([os.path.basename(p) for p in w.side_order],
1289
+ [newest.name, "agent-%s.jsonl" % AGENT2])
1290
+ self.assertIsNone(w.tracker.pending)
1291
+ # the rejected agent writes again: it is among the newest, read from its start
1292
+ fixtures.append(oldest, [side_quota(self.t0 + 1, reset)])
1293
+ r.t += 5
1294
+ w.read()
1295
+ self.assertEqual([os.path.basename(p) for p in w.side_order],
1296
+ [oldest.name, newest.name])
1297
+ self.assertEqual((w.tracker.pending["class"], w.tracker.pending["at"]),
1298
+ ("quota", self.t0 + 1.009))
1299
+ # a file no longer followed keeps its place while it is there
1300
+ self.assertEqual(self.tail_of(w, "agent-%s.jsonl" % AGENT2).off,
1301
+ (self.proj / SID / "subagents" / "workflows" / "wf_2" /
1302
+ ("agent-%s.jsonl" % AGENT2)).stat().st_size)
1303
+
1304
+ def test_the_walk_is_bounded(self):
1305
+ w, r = self.start([])
1306
+ for i in range(6):
1307
+ self.agent_file([side_reply(self.launched + 1, "a%d" % i)], agent="a%d" % i,
1308
+ workflow="wf_%d" % i)
1309
+ with mock.patch.object(ar, "SIDE_SCAN", 3):
1310
+ w.read()
1311
+ self.assertEqual(len(w.side), 3)
1312
+
1313
+ def test_the_bytes_read_per_tick_are_bounded(self):
1314
+ reset = int(self.t0) + 999
1315
+ w, r = self.start([])
1316
+ filler = [side_reply(self.launched + 1 + i * 0.01, text="x" * 900) for i in range(60)]
1317
+ big = self.agent_file(filler + [side_quota(self.t0, reset)])
1318
+ small = self.agent_file([side_reply(self.launched + 1, AGENT2)], agent=AGENT2,
1319
+ workflow="wf_small")
1320
+ reads = 0
1321
+ with mock.patch.object(ar, "SIDE_BYTES", 8192):
1322
+ while w.tracker.pending is None and reads < 100:
1323
+ w.read()
1324
+ reads += 1
1325
+ self.assertLessEqual(sum(t.got for t in w.side.values()), 8192)
1326
+ self.assertGreater(reads, big.stat().st_size // 8192)
1327
+ self.assertEqual(w.tracker.pending["class"], "quota")
1328
+ self.assertEqual(self.tail_of(w, small.name).off, small.stat().st_size)
1329
+
1330
+ def test_subagents_wait_for_the_main_transcript_to_be_caught_up(self):
1331
+ main = [cc_reply(self.launched + 1, "y" * 2000) for _ in range(8)]
1332
+ w, r = self.start([], main=main)
1333
+ self.agent_file([side_quota(self.t0, int(self.t0) + 999)])
1334
+ with mock.patch.object(ar.JsonlTail, "MAX_READ", 4096):
1335
+ w.read()
1336
+ self.assertGreater(w.tail.behind, 0)
1337
+ self.assertEqual((w.side, w.tracker.pending), ({}, None))
1338
+ for _ in range(10):
1339
+ w.read()
1340
+ self.assertEqual(w.tail.behind, 0)
1341
+ self.assertEqual(w.tracker.pending["src"], "sidechain")
1342
+
1343
+
1344
+ # =====================================================================================
1345
+ # the main thread's own work in flight
1346
+ # =====================================================================================
1347
+
1348
+ def block(ts, ref, content, stop, mid="msg_01Main"):
1349
+ """One block of a main-thread reply as claude 2.1.304 writes it: a record per content
1350
+ block, each naming its request; "stop_reason": null while the reply still streams."""
1351
+ rec = main_reply(ts, ref)
1352
+ rec["message"].update(id=mid, content=[content], stop_reason=stop)
1353
+ return rec
1354
+
1355
+
1356
+ def tool_use(tid="toolu_1", command="./migrate.sh"):
1357
+ return {"type": "tool_use", "id": tid, "name": "Bash", "input": {"command": command}}
1358
+
1359
+
1360
+ def result_of(ts, tid="toolu_1"):
1361
+ return cc_user(ts, [{"type": "tool_result", "tool_use_id": tid, "content": "ok"}],
1362
+ toolUseResult={"stdout": "ok"})
1363
+
1364
+
1365
+ THINKING = {"type": "thinking", "thinking": "", "signature": "c2ln"}
1366
+ TEXT = {"type": "text", "text": "Done."}
1367
+
1368
+
1369
+ class MainThreadBusyTests(unittest.TestCase):
1370
+ T = 1792440000
1371
+
1372
+ def busy(self, records):
1373
+ return ar.main_thread_busy(main_lines(records))
1374
+
1375
+ def test_what_the_main_thread_is_in_the_middle_of(self):
1376
+ t, sent = self.T, api_request(self.T, "req_m1")
1377
+ cases = {
1378
+ "request": [cc_user(t - 1, "migrate the db"), sent],
1379
+ "stream": [sent, block(t + 20, "req_m1", THINKING, None)],
1380
+ "tool": [sent, block(t + 20, "req_m1", THINKING, "tool_use"),
1381
+ block(t + 21, "req_m1", tool_use(), "tool_use")],
1382
+ }
1383
+ for want, records in cases.items():
1384
+ self.assertEqual(self.busy(records), want, want)
1385
+ # a reply whose tool_use block is still to come (its thinking says tool_use already)
1386
+ self.assertEqual(self.busy([sent, block(t + 20, "req_m1", THINKING, "tool_use")]),
1387
+ "stream")
1388
+ # two tools at once: one result is not both
1389
+ two = [sent, block(t + 20, "req_m1", tool_use("toolu_1"), "tool_use"),
1390
+ block(t + 20, "req_m1", tool_use("toolu_2", "make test"), "tool_use"),
1391
+ result_of(t + 30, "toolu_1")]
1392
+ self.assertEqual(self.busy(two), "tool")
1393
+ self.assertIsNone(self.busy(two + [result_of(t + 90, "toolu_2")]))
1394
+
1395
+ def test_by_request_whatever_the_order_of_the_file(self):
1396
+ t = self.T
1397
+ # 2.1.304 writes a request as it sends it: before the prompt that led to it...
1398
+ self.assertEqual(self.busy([api_request(t, "req_m1"), cc_user(t, "migrate the db")]),
1399
+ "request")
1400
+ # ...and before the previous reply's last blocks and tool results
1401
+ self.assertEqual(self.busy([api_request(t, "req_m1"), api_request(t + 5, "req_m2"),
1402
+ block(t + 4, "req_m1", tool_use(), "tool_use"),
1403
+ result_of(t + 5)]), "request")
1404
+ self.assertIsNone(self.busy([api_request(t, "req_m1"), api_request(t + 5, "req_m2"),
1405
+ block(t + 4, "req_m1", tool_use(), "tool_use"),
1406
+ result_of(t + 5),
1407
+ block(t + 9, "req_m2", TEXT, "end_turn")]))
1408
+
1409
+ def test_idle_or_untold(self):
1410
+ t, sent = self.T, api_request(self.T, "req_m1")
1411
+ compact_boundary = {"type": "system", "subtype": "compact_boundary", "timestamp": iso(t + 60)}
1412
+ for name, records in {
1413
+ "a finished reply": [sent, block(t + 5, "req_m1", TEXT, "end_turn")],
1414
+ "its tools all answered": [sent, block(t + 5, "req_m1", tool_use(), "tool_use"),
1415
+ result_of(t + 9)],
1416
+ "an API error": [sent, cc_api_error(t + 1, "overloaded")],
1417
+ "interrupted mid-stream": [sent, block(t + 5, "req_m1", THINKING, None),
1418
+ cc_user(t + 6, "[Request interrupted by user]")],
1419
+ "interrupted before a word": [sent, cc_user(t + 1, "migrate the db"),
1420
+ cc_user(t + 2, "[Request interrupted by user]")],
1421
+ "a tool interrupted": [sent, block(t + 5, "req_m1", tool_use(), "tool_use"),
1422
+ cc_user(t + 6, [{"type": "text", "text":
1423
+ "[Request interrupted by user for tool use]"}])],
1424
+ "a command after the reply": [sent, block(t + 5, "req_m1", TEXT, "end_turn")]
1425
+ + cc_slash(t + 6),
1426
+ "a reply without the key": [sent, main_reply(t + 5, "req_m1", "Started it.")],
1427
+ "a compaction after the turn": [sent, block(t + 5, "req_m1", TEXT, "end_turn"),
1428
+ compact_boundary,
1429
+ cc_user(t + 60, "summary", isCompactSummary=True)],
1430
+ "notices of paused workflows": [sent, block(t + 5, "req_m1", TEXT, "end_turn"),
1431
+ queue_op(t + 9), request_blob(),
1432
+ cc_user(t + 10, "<task-notification>paused"
1433
+ "</task-notification>")],
1434
+ "an agent's request": [sent, block(t + 5, "req_m1", TEXT, "end_turn"),
1435
+ side(api_request(t + 6, "req_a1", AGENT))],
1436
+ "no request on record (an older client)": [cc_user(t, "go"), cc_reply(t + 1),
1437
+ synthetic(t + 2)],
1438
+ "nothing": [],
1439
+ }.items():
1440
+ self.assertIsNone(self.busy(records), name)
1441
+
1442
+
1443
+ class MainThreadInFlightWatcherTests(SidechainCase):
1444
+ """An agent's verdict waits for the main thread's own request, stream or tool: the
1445
+ account accepted it before the limit, and stopping the client would cut it off (the
1446
+ tool's processes die with it). Fake runner and clock only."""
1447
+
1448
+ def run_until_probe(self, main, later, env=None, waits=None):
1449
+ """The first probe's time after t0; ``waits``: the main-thread work the one `hold`
1450
+ line names (reason=main-<waits>), or None when nothing waited."""
1451
+ reset = int(self.t0) + 9000
1452
+ w, r = self.start([], main=[cc_user(self.launched + 1, "migrate the db")] + main,
1453
+ env=env)
1454
+ self.agent_file([side_reply(self.launched + 3), side_quota(self.t0 - 1, reset)])
1455
+ for when, records in later:
1456
+ r.at(when, lambda records=records: self.transcript(records))
1457
+ self.assertEqual(w.run(), 0)
1458
+ self.assertEqual(len(r.probes), 1)
1459
+ self.assertEqual(r.signals[0], (self.PID, signal.SIGTERM))
1460
+ if waits is None:
1461
+ self.assertEqual(self.events(), ["watch", "detect", "switch"])
1462
+ else:
1463
+ self.assertEqual(self.events(), ["watch", "detect", "hold", "switch"])
1464
+ self.assertIn(" class=quota sid=%s acct=acct-01 reason=main-%s" % (SID, waits),
1465
+ self.log_lines()[2])
1466
+ return r.probes[0]["t"] - self.t0, r
1467
+
1468
+ def test_a_tool_the_main_thread_runs_finishes_first(self):
1469
+ t, _ = self.run_until_probe(
1470
+ [api_request(self.t0 - 60, "req_m1"),
1471
+ block(self.t0 - 55, "req_m1", tool_use(), "tool_use")],
1472
+ [(self.t0 + 240, [result_of(self.t0 + 240)])], waits="tool")
1473
+ self.assertGreaterEqual(t, 240)
1474
+ self.assertLess(t, 245)
1475
+
1476
+ def test_a_stream_the_account_accepted_finishes_first(self):
1477
+ t, _ = self.run_until_probe(
1478
+ [api_request(self.t0 - 40, "req_m1")],
1479
+ [(self.t0 + 100, [block(self.t0 + 100, "req_m1", THINKING, None)]),
1480
+ (self.t0 + 300, [block(self.t0 + 300, "req_m1", TEXT, "end_turn")])],
1481
+ waits="request")
1482
+ self.assertGreaterEqual(t, 300)
1483
+ self.assertLess(t, 305)
1484
+
1485
+ def test_the_main_threads_own_rejection_then_stands(self):
1486
+ # its tool ends, its next request meets the limit: that verdict moves the session
1487
+ t, r = self.run_until_probe(
1488
+ [api_request(self.t0 - 60, "req_m1"),
1489
+ block(self.t0 - 55, "req_m1", tool_use(), "tool_use")],
1490
+ [(self.t0 + 30, [result_of(self.t0 + 30), api_request(self.t0 + 30, "req_m2"),
1491
+ cc_quota(self.t0 + 31, resets=int(self.t0) + 9000)])],
1492
+ waits="tool")
1493
+ self.assertGreaterEqual(t, 30)
1494
+ self.assertLess(t, 40)
1495
+ kv, _ = self.relaunch_of(r)
1496
+ self.assertEqual((kv["class"], kv["reset"]), ("quota", str(int(self.t0) + 9000)))
1497
+
1498
+ def test_the_wait_is_capped(self):
1499
+ # a tool that never ends (or a prompt for permission nobody answers) waits SUBAGENT_CAP
1500
+ t, _ = self.run_until_probe(
1501
+ [api_request(self.t0 - 60, "req_m1"),
1502
+ block(self.t0 - 55, "req_m1", tool_use(), "tool_use")], [],
1503
+ env={"CLAUDE_MULTIACC_AR_SUBAGENT_CAP": "120"}, waits="tool")
1504
+ self.assertGreaterEqual(t, 120)
1505
+ self.assertLess(t, 125)
1506
+
1507
+ def test_an_idle_main_thread_moves_at_grace_as_before(self):
1508
+ t, _ = self.run_until_probe(
1509
+ [api_request(self.t0 - 60, "req_m1"),
1510
+ block(self.t0 - 55, "req_m1", tool_use(), "tool_use"), result_of(self.t0 - 50),
1511
+ api_request(self.t0 - 50, "req_m2"),
1512
+ block(self.t0 - 45, "req_m2", TEXT, "end_turn"), queue_op(self.t0 - 2)], [])
1513
+ self.assertGreaterEqual(t, 5)
1514
+ self.assertLess(t, 10)
1515
+
1516
+
1517
+ if __name__ == "__main__":
1518
+ unittest.main()