claude-multiacc 2.0.40 → 2.0.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -2
- package/bin/claude +394 -2
- package/bin/codex +406 -4
- package/docs/ACCOUNT_OPERATIONS.md +2 -1
- package/docs/AUTORESUME.md +321 -0
- package/docs/CODEX.md +12 -1
- package/docs/VERIFICATION.md +9 -1
- package/lib/__pycache__/audit.cpython-312.pyc +0 -0
- package/lib/__pycache__/autoresume.cpython-312.pyc +0 -0
- package/lib/__pycache__/claude_reset.cpython-312.pyc +0 -0
- package/lib/__pycache__/codex_config_edit.cpython-312.pyc +0 -0
- package/lib/__pycache__/codex_python.cpython-312.pyc +0 -0
- package/lib/__pycache__/keychain.cpython-312.pyc +0 -0
- package/lib/__pycache__/mcp_registry.cpython-312.pyc +0 -0
- package/lib/__pycache__/selector_policy.cpython-312.pyc +0 -0
- package/lib/__pycache__/selector_primitives.cpython-312.pyc +0 -0
- package/lib/__pycache__/shim_path.cpython-312.pyc +0 -0
- package/lib/autoresume.py +2271 -0
- package/lib/report.py +16 -1
- package/package.json +1 -1
- package/tests/__pycache__/packaged_command_support.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_claude_reset.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_codex_reset.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_codex_reset_polling.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_codex_reset_reporting.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_codex_reset_windows.cpython-312.pyc +0 -0
- package/tests/run-tests.sh +935 -3
- package/tests/test_autoresume.py +2565 -0
- package/tests/test_claude_reset.py +30 -0
|
@@ -0,0 +1,2565 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Unit tests for lib/autoresume.py — the auto-resume watcher (docs/AUTORESUME.md).
|
|
3
|
+
|
|
4
|
+
Sandboxed: no real claude/codex, no network, no quota, never the operator's pools. Every
|
|
5
|
+
pool is a temp dir; fixture records are inline, in the shapes the live clients write
|
|
6
|
+
(claude 2.1.280 transcripts, codex 0.156.0 rollouts, 2026-09-22); every process side
|
|
7
|
+
effect of the watcher goes through a fake Runner with a fake clock. One test drives the
|
|
8
|
+
real ``watch`` CLI end to end against a disposable ``sleep`` process, a fake ``tmux`` on
|
|
9
|
+
PATH and a fake probe script. Runs under python3.9+.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import contextlib
|
|
15
|
+
import importlib.util
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
import re
|
|
20
|
+
import shutil
|
|
21
|
+
import signal
|
|
22
|
+
import stat
|
|
23
|
+
import subprocess
|
|
24
|
+
import sys
|
|
25
|
+
import tempfile
|
|
26
|
+
import time
|
|
27
|
+
import unittest
|
|
28
|
+
from unittest import mock
|
|
29
|
+
|
|
30
|
+
REPO = Path(__file__).resolve().parents[1]
|
|
31
|
+
LIB = REPO / "lib" / "autoresume.py"
|
|
32
|
+
|
|
33
|
+
_spec = importlib.util.spec_from_file_location("autoresume", str(LIB))
|
|
34
|
+
ar = importlib.util.module_from_spec(_spec)
|
|
35
|
+
_spec.loader.exec_module(ar)
|
|
36
|
+
|
|
37
|
+
SID = "55f5e7a8-955c-4ab0-a533-08852c296213"
|
|
38
|
+
SID2 = "30316873-ce79-4132-aa64-30702aaee95b"
|
|
39
|
+
CX_SID = "01a0cb0f-a719-7461-bba6-a59bdf2412cc"
|
|
40
|
+
CX_SID2 = "01a0cb0e-685c-79c3-a64f-618c4ada3bca"
|
|
41
|
+
LSTART = "Tue Sep 22 21:52:12 2026"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def iso(epoch):
|
|
45
|
+
"""The transcripts' own timestamp shape: UTC, milliseconds, Z."""
|
|
46
|
+
return time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(epoch)) + ".009Z"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# ---- claude transcript records (claude 2.1.280 shapes) ------------------------------
|
|
50
|
+
|
|
51
|
+
def cc_base(ts, typ, **extra):
|
|
52
|
+
rec = {"parentUuid": "06c84ec0-a9db-4387-b695-a6e0bdf91685", "isSidechain": False,
|
|
53
|
+
"type": typ, "uuid": "3a416522-6c97-44cf-adfe-00b44d2dcad5", "timestamp": iso(ts),
|
|
54
|
+
"userType": "external", "entrypoint": "cli", "cwd": "/w", "sessionId": SID,
|
|
55
|
+
"version": "2.1.280", "gitBranch": "HEAD"}
|
|
56
|
+
rec.update(extra)
|
|
57
|
+
return rec
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def cc_api_error(ts, code, text="API Error: 529 Overloaded.", **extra):
|
|
61
|
+
msg = {"id": "71cbd922-b089-4a6a-a628-acb45a61dc63", "model": "<synthetic>",
|
|
62
|
+
"role": "assistant", "stop_reason": "stop_sequence", "type": "message",
|
|
63
|
+
"usage": {"input_tokens": 0, "output_tokens": 0},
|
|
64
|
+
"content": [{"type": "text", "text": text}]}
|
|
65
|
+
rec = cc_base(ts, "assistant", message=msg, isApiErrorMessage=True, apiErrorStatus=500,
|
|
66
|
+
perTurnEffort=None, session_id=SID)
|
|
67
|
+
if code is not None:
|
|
68
|
+
rec["error"] = code
|
|
69
|
+
rec.update(extra)
|
|
70
|
+
return rec
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def cc_quota(ts, rtype="five_hour", resets=None, **extra):
|
|
74
|
+
rec = cc_api_error(ts, "rate_limit",
|
|
75
|
+
"You've hit your limit · resets 3:36am (Europe/Istanbul)",
|
|
76
|
+
apiErrorStatus=429,
|
|
77
|
+
quotaLimits={"status": "rejected", "resetsAt": resets,
|
|
78
|
+
"unifiedRateLimitFallbackAvailable": False,
|
|
79
|
+
"rateLimitType": rtype, "overageStatus": "rejected",
|
|
80
|
+
"isUsingOverage": False})
|
|
81
|
+
rec.update(extra)
|
|
82
|
+
return rec
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def cc_system(ts):
|
|
86
|
+
return cc_base(ts, "system", subtype="informational", level="notice", isMeta=False,
|
|
87
|
+
content="Usage limit reached · continuing automatically at 3:36am "
|
|
88
|
+
"· esc or type to cancel")
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def cc_user(ts, content, **extra):
|
|
92
|
+
rec = cc_base(ts, "user", promptId="07b69536-f70e-474e-b2bb-666b0f9bb539",
|
|
93
|
+
message={"role": "user", "content": content})
|
|
94
|
+
rec.update(extra)
|
|
95
|
+
return rec
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def cc_reply(ts, text="Done.", **extra):
|
|
99
|
+
rec = cc_base(ts, "assistant", message={"model": "claude-opus-5-5", "role": "assistant",
|
|
100
|
+
"type": "message",
|
|
101
|
+
"content": [{"type": "text", "text": text}]})
|
|
102
|
+
rec.update(extra)
|
|
103
|
+
return rec
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
# ---- codex rollout records (codex 0.156.0 shapes) ------------------------------------
|
|
107
|
+
|
|
108
|
+
def cx_meta(ts, sid=CX_SID, session_id=None, cwd="/w", originator="codex-tui"):
|
|
109
|
+
return {"timestamp": iso(ts), "type": "session_meta",
|
|
110
|
+
"payload": {"session_id": sid if session_id is None else session_id, "id": sid,
|
|
111
|
+
"timestamp": iso(ts), "cwd": cwd, "runtime_workspace_roots": [cwd],
|
|
112
|
+
"originator": originator, "cli_version": "0.156.0", "source": "cli",
|
|
113
|
+
"thread_source": "user", "model_provider": "openai",
|
|
114
|
+
"history_mode": "paginated"}}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def cx_event(ts, payload):
|
|
118
|
+
return {"timestamp": iso(ts), "ordinal": 9, "type": "event_msg", "payload": payload}
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def cx_error(ts, info, message="boom", completed_at=None):
|
|
122
|
+
done = int(ts) if completed_at is None else completed_at
|
|
123
|
+
return cx_event(ts, {"type": "task_complete", "turn_id": "01a0cb0f-a763-7310-92e6",
|
|
124
|
+
"last_agent_message": None,
|
|
125
|
+
"error": {"message": message, "codex_error_info": info},
|
|
126
|
+
"started_at": done, "completed_at": done, "duration_ms": 32})
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
LIMIT_MSG = ("You’ve hit your usage limit. Visit https://chatgpt.com/codex/settings/usage "
|
|
130
|
+
"to purchase more credits or try again at Sep 26th, 2026 12:39 AM.")
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def cx_tokens(ts, primary=None, secondary=None, limit_id="codex", info=None):
|
|
134
|
+
return cx_event(ts, {"type": "token_count", "info": info,
|
|
135
|
+
"rate_limits": {"limit_id": limit_id, "limit_name": None,
|
|
136
|
+
"primary": primary, "secondary": secondary,
|
|
137
|
+
"credits": None, "individual_limit": None,
|
|
138
|
+
"spend_control_reached": None, "plan_type": None,
|
|
139
|
+
"rate_limit_reached_type": None}})
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def jl(records):
|
|
143
|
+
return "".join(json.dumps(r) + "\n" for r in records)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def append(path, records):
|
|
147
|
+
with open(path, "a") as fh:
|
|
148
|
+
fh.write(jl(records))
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
@contextlib.contextmanager
|
|
152
|
+
def local_tz(name):
|
|
153
|
+
old = os.environ.get("TZ")
|
|
154
|
+
os.environ["TZ"] = name
|
|
155
|
+
time.tzset()
|
|
156
|
+
try:
|
|
157
|
+
yield
|
|
158
|
+
finally:
|
|
159
|
+
if old is None:
|
|
160
|
+
os.environ.pop("TZ", None)
|
|
161
|
+
else:
|
|
162
|
+
os.environ["TZ"] = old
|
|
163
|
+
time.tzset()
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def cfg(provider="claude", **env):
|
|
167
|
+
return ar.Config(provider, {"%s_MULTIACC_AR_%s" % (provider.upper(), k): str(v)
|
|
168
|
+
for k, v in env.items()})
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
# =====================================================================================
|
|
172
|
+
# classifiers
|
|
173
|
+
# =====================================================================================
|
|
174
|
+
|
|
175
|
+
class ClaudeClassifierTests(unittest.TestCase):
|
|
176
|
+
T = 1790113000
|
|
177
|
+
|
|
178
|
+
def verdict(self, rec):
|
|
179
|
+
ev = ar.classify_claude(rec)
|
|
180
|
+
self.assertIsNotNone(ev)
|
|
181
|
+
self.assertEqual(ev[0], "error", ev)
|
|
182
|
+
return ev[1]
|
|
183
|
+
|
|
184
|
+
def test_session_and_weekly_rejections_are_quota(self):
|
|
185
|
+
for rtype in ("five_hour", "seven_day"):
|
|
186
|
+
v = self.verdict(cc_quota(self.T, rtype, resets=self.T + 9000))
|
|
187
|
+
self.assertEqual(v["class"], "quota")
|
|
188
|
+
self.assertEqual(v["rtype"], rtype)
|
|
189
|
+
self.assertEqual(v["reset"], self.T + 9000)
|
|
190
|
+
self.assertEqual(v["marked"], ar.utc_iso(self.T))
|
|
191
|
+
self.assertEqual(v["code"], "rate_limit")
|
|
192
|
+
|
|
193
|
+
def test_other_rejected_bucket_is_model_scoped(self):
|
|
194
|
+
for rtype in ("seven_day_opus", "seven_day_overage_included", None):
|
|
195
|
+
self.assertEqual(self.verdict(cc_quota(self.T, rtype, resets=self.T + 60))["class"],
|
|
196
|
+
"model", rtype)
|
|
197
|
+
|
|
198
|
+
def test_model_limit_without_quota_limits(self):
|
|
199
|
+
for text in ("You've reached your Fable limit. Run /usage-credits to keep going.",
|
|
200
|
+
"You've reached your Fable limit. Switch to another model to continue."):
|
|
201
|
+
self.assertEqual(self.verdict(cc_api_error(self.T, "rate_limit", text))["class"],
|
|
202
|
+
"model")
|
|
203
|
+
rec = cc_api_error(self.T, "rate_limit", "nope", apiError="model_requires_usage_credits")
|
|
204
|
+
self.assertEqual(self.verdict(rec)["class"], "model")
|
|
205
|
+
|
|
206
|
+
def test_other_rate_limit_is_transient(self):
|
|
207
|
+
rec = cc_api_error(self.T, "rate_limit", "We're experiencing a temporary capacity issue.")
|
|
208
|
+
self.assertEqual(self.verdict(rec)["class"], "transient")
|
|
209
|
+
allowed = cc_quota(self.T)
|
|
210
|
+
allowed["quotaLimits"]["status"] = "allowed_warning"
|
|
211
|
+
self.assertEqual(self.verdict(allowed)["class"], "transient")
|
|
212
|
+
|
|
213
|
+
def test_server_side_errors_are_transient(self):
|
|
214
|
+
for code in ("overloaded", "server_error", "unknown"):
|
|
215
|
+
self.assertEqual(self.verdict(cc_api_error(self.T, code))["class"], "transient")
|
|
216
|
+
|
|
217
|
+
def test_auth_and_blocked(self):
|
|
218
|
+
self.assertEqual(self.verdict(cc_api_error(
|
|
219
|
+
self.T, "authentication_failed", "Not logged in · Please run /login"))["class"],
|
|
220
|
+
"auth")
|
|
221
|
+
for code in ("oauth_org_not_allowed", "account_on_hold", "billing_error",
|
|
222
|
+
"verification_required"):
|
|
223
|
+
self.assertEqual(self.verdict(cc_api_error(self.T, code))["class"], "blocked", code)
|
|
224
|
+
|
|
225
|
+
def test_everything_else_is_never(self):
|
|
226
|
+
for code in ("invalid_request", "max_output_tokens", "model_not_found",
|
|
227
|
+
"cloud_credential_error", "brand_new_error"):
|
|
228
|
+
self.assertEqual(ar.classify_claude(cc_api_error(self.T, code)), ("never", code))
|
|
229
|
+
self.assertEqual(ar.classify_claude(cc_api_error(self.T, None)), ("never", "?"))
|
|
230
|
+
|
|
231
|
+
def test_system_records_never_count(self):
|
|
232
|
+
self.assertIsNone(ar.classify_claude(cc_system(self.T)))
|
|
233
|
+
|
|
234
|
+
def test_sidechain_records_are_ignored(self):
|
|
235
|
+
self.assertIsNone(ar.classify_claude(cc_quota(self.T, isSidechain=True)))
|
|
236
|
+
self.assertIsNone(ar.classify_claude(cc_user(self.T, "go on then", isSidechain=True)))
|
|
237
|
+
self.assertIsNone(ar.classify_claude(cc_reply(self.T, isSidechain=True)))
|
|
238
|
+
|
|
239
|
+
def test_what_counts_as_the_user_typing(self):
|
|
240
|
+
cancel = ("cancel", "user")
|
|
241
|
+
self.assertEqual(ar.classify_claude(cc_user(self.T, "please continue")), cancel)
|
|
242
|
+
self.assertEqual(ar.classify_claude(cc_user(
|
|
243
|
+
self.T, "<command-name>/model</command-name>\n<command-args></command-args>")),
|
|
244
|
+
cancel)
|
|
245
|
+
self.assertEqual(ar.classify_claude(cc_user(
|
|
246
|
+
self.T, [{"type": "text", "text": "look at this"},
|
|
247
|
+
{"type": "image", "source": {"type": "base64", "data": "AA=="}}])), cancel)
|
|
248
|
+
self.assertEqual(ar.classify_claude(cc_user(
|
|
249
|
+
self.T, [{"type": "image", "source": {"type": "base64", "data": "AA=="}}])), cancel)
|
|
250
|
+
not_human = [
|
|
251
|
+
cc_user(self.T, "<local-command-caveat>Caveat</local-command-caveat>", isMeta=True),
|
|
252
|
+
cc_user(self.T, [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "ok"}]),
|
|
253
|
+
cc_user(self.T, "result", toolUseResult={"stdout": ""}),
|
|
254
|
+
cc_user(self.T, ar.resume_prompt("quota")),
|
|
255
|
+
cc_user(self.T, "<task-notification><task-id>b1</task-id></task-notification>"),
|
|
256
|
+
cc_user(self.T, "<local-command-stdout></local-command-stdout>"),
|
|
257
|
+
cc_user(self.T, "This session is being continued from a previous conversation",
|
|
258
|
+
isCompactSummary=True),
|
|
259
|
+
cc_user(self.T, []),
|
|
260
|
+
]
|
|
261
|
+
for rec in not_human:
|
|
262
|
+
self.assertIsNone(ar.classify_claude(rec), rec["message"])
|
|
263
|
+
|
|
264
|
+
def test_a_real_reply_cancels(self):
|
|
265
|
+
self.assertEqual(ar.classify_claude(cc_reply(self.T)), ("cancel", "assistant"))
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
class ClaudeTrackerTests(unittest.TestCase):
|
|
269
|
+
LAUNCHED = 1790113000
|
|
270
|
+
|
|
271
|
+
def tracker(self):
|
|
272
|
+
return ar.ClaudeTracker(self.LAUNCHED - 2)
|
|
273
|
+
|
|
274
|
+
def test_old_rejection_in_a_resumed_transcript_never_fires(self):
|
|
275
|
+
tr = self.tracker()
|
|
276
|
+
self.assertIsNone(tr.feed(cc_quota(self.LAUNCHED - 3, resets=self.LAUNCHED + 99), 1))
|
|
277
|
+
self.assertIsNone(tr.feed(cc_quota(self.LAUNCHED - 86400), 1))
|
|
278
|
+
self.assertIsNone(tr.pending)
|
|
279
|
+
# Two seconds of slack for the shim's clock read.
|
|
280
|
+
ev = tr.feed(cc_quota(self.LAUNCHED - 1, resets=self.LAUNCHED + 99), 5)
|
|
281
|
+
self.assertEqual(ev[0], "error")
|
|
282
|
+
self.assertEqual(tr.pending["class"], "quota")
|
|
283
|
+
self.assertEqual(tr.pending_since, 5)
|
|
284
|
+
|
|
285
|
+
def test_error_without_a_timestamp_cannot_prove_it_is_new(self):
|
|
286
|
+
rec = cc_quota(self.LAUNCHED + 5)
|
|
287
|
+
del rec["timestamp"]
|
|
288
|
+
tr = self.tracker()
|
|
289
|
+
self.assertIsNone(tr.feed(rec, 1))
|
|
290
|
+
self.assertIsNone(tr.pending)
|
|
291
|
+
|
|
292
|
+
def test_cancel_and_rearm(self):
|
|
293
|
+
tr = self.tracker()
|
|
294
|
+
self.assertIsNone(tr.feed(cc_user(self.LAUNCHED + 1, "hi"), 1)) # nothing to cancel
|
|
295
|
+
tr.feed(cc_quota(self.LAUNCHED + 2), 2)
|
|
296
|
+
self.assertIsNone(tr.feed(cc_system(self.LAUNCHED + 2), 3))
|
|
297
|
+
self.assertIsNotNone(tr.pending)
|
|
298
|
+
self.assertIsNone(tr.feed(cc_user(self.LAUNCHED + 3, ar.resume_prompt("quota")), 3))
|
|
299
|
+
self.assertIsNotNone(tr.pending)
|
|
300
|
+
self.assertEqual(tr.feed(cc_user(self.LAUNCHED + 4, "wait, stop"), 4), ("cancel", "user"))
|
|
301
|
+
self.assertIsNone(tr.pending)
|
|
302
|
+
tr.feed(cc_api_error(self.LAUNCHED + 5, "overloaded"), 5)
|
|
303
|
+
self.assertEqual(tr.pending["class"], "transient")
|
|
304
|
+
self.assertEqual(tr.feed(cc_reply(self.LAUNCHED + 6), 6), ("cancel", "assistant"))
|
|
305
|
+
self.assertIsNone(tr.pending)
|
|
306
|
+
|
|
307
|
+
def test_a_cancel_older_than_the_launch_does_not_count(self):
|
|
308
|
+
tr = self.tracker()
|
|
309
|
+
tr.feed(cc_quota(self.LAUNCHED + 2), 2)
|
|
310
|
+
self.assertIsNone(tr.feed(cc_user(self.LAUNCHED - 60, "old"), 3))
|
|
311
|
+
self.assertIsNotNone(tr.pending)
|
|
312
|
+
|
|
313
|
+
def test_pending_since_restarts_only_on_a_class_change(self):
|
|
314
|
+
tr = self.tracker()
|
|
315
|
+
tr.feed(cc_quota(self.LAUNCHED + 1), 10)
|
|
316
|
+
tr.feed(cc_quota(self.LAUNCHED + 2), 20)
|
|
317
|
+
self.assertEqual(tr.pending_since, 10)
|
|
318
|
+
tr.feed(cc_api_error(self.LAUNCHED + 3, "authentication_failed"), 30)
|
|
319
|
+
self.assertEqual((tr.pending["class"], tr.pending_since), ("auth", 30))
|
|
320
|
+
|
|
321
|
+
def test_never_leaves_the_pending_verdict_alone(self):
|
|
322
|
+
tr = self.tracker()
|
|
323
|
+
tr.feed(cc_quota(self.LAUNCHED + 1), 10)
|
|
324
|
+
self.assertEqual(tr.feed(cc_api_error(self.LAUNCHED + 2, "invalid_request"), 11),
|
|
325
|
+
("never", "invalid_request"))
|
|
326
|
+
self.assertEqual(tr.pending["class"], "quota")
|
|
327
|
+
|
|
328
|
+
def test_prefilter(self):
|
|
329
|
+
tr = self.tracker()
|
|
330
|
+
err = json.dumps(cc_quota(self.LAUNCHED + 1), separators=(",", ":")).encode()
|
|
331
|
+
user = json.dumps(cc_user(self.LAUNCHED + 1, "hi")).encode()
|
|
332
|
+
self.assertTrue(tr.wants(err))
|
|
333
|
+
self.assertFalse(tr.wants(user))
|
|
334
|
+
tr.feed(json.loads(err), 1)
|
|
335
|
+
self.assertTrue(tr.wants(user))
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
class CodexClassifierTests(unittest.TestCase):
|
|
339
|
+
LAUNCHED = 1790113100
|
|
340
|
+
|
|
341
|
+
def tracker(self):
|
|
342
|
+
return ar.CodexTracker(self.LAUNCHED - 2)
|
|
343
|
+
|
|
344
|
+
def test_error_info_string_or_one_key_object(self):
|
|
345
|
+
cases = {
|
|
346
|
+
"usage_limit_exceeded": "quota", "rate_limit_exceeded": "quota",
|
|
347
|
+
"unauthorized": "auth", "server_overloaded": "transient",
|
|
348
|
+
"internal_server_error": "transient", "http_connection_failed": "transient",
|
|
349
|
+
"response_stream_connection_failed": "transient",
|
|
350
|
+
"response_stream_disconnected": "transient",
|
|
351
|
+
"response_too_many_failed_attempts": "transient",
|
|
352
|
+
}
|
|
353
|
+
for code, cls in cases.items():
|
|
354
|
+
for info in (code, {code: {"http_status_code": None}}):
|
|
355
|
+
ev = ar.classify_codex(cx_error(self.LAUNCHED, info))
|
|
356
|
+
self.assertEqual((ev[0], ev[1]["class"], ev[1]["code"]), ("error", cls, code))
|
|
357
|
+
for code in ("cyber_policy", "context_window_exceeded", "bad_request", "other"):
|
|
358
|
+
self.assertEqual(ar.classify_codex(cx_error(self.LAUNCHED, code)), ("never", code))
|
|
359
|
+
self.assertEqual(ar.classify_codex(cx_error(self.LAUNCHED, {code: None})),
|
|
360
|
+
("never", code))
|
|
361
|
+
self.assertEqual(ar.classify_codex(cx_error(self.LAUNCHED, None)), ("never", "?"))
|
|
362
|
+
|
|
363
|
+
def test_a_clean_turn_is_nothing(self):
|
|
364
|
+
ok = cx_event(self.LAUNCHED, {"type": "task_complete", "last_agent_message": "hi",
|
|
365
|
+
"completed_at": self.LAUNCHED})
|
|
366
|
+
self.assertIsNone(ar.classify_codex(ok))
|
|
367
|
+
self.assertEqual(ar.classify_codex(cx_meta(self.LAUNCHED))[0], "meta")
|
|
368
|
+
self.assertIsNone(ar.classify_codex({"type": "response_item", "payload": {}}))
|
|
369
|
+
|
|
370
|
+
def test_cancel_events(self):
|
|
371
|
+
tr = self.tracker()
|
|
372
|
+
for ptype in ("user_message", "task_started", "turn_aborted"):
|
|
373
|
+
tr.feed(cx_error(self.LAUNCHED + 1, "unauthorized"), 1)
|
|
374
|
+
self.assertEqual(tr.feed(cx_event(self.LAUNCHED + 2, {"type": ptype}), 2),
|
|
375
|
+
("cancel", ptype))
|
|
376
|
+
self.assertIsNone(tr.pending)
|
|
377
|
+
|
|
378
|
+
def test_time_filter_uses_completed_at(self):
|
|
379
|
+
tr = self.tracker()
|
|
380
|
+
# A resumed rollout: the record is re-read now, but the turn finished long ago.
|
|
381
|
+
self.assertIsNone(tr.feed(cx_error(self.LAUNCHED + 50, "usage_limit_exceeded",
|
|
382
|
+
completed_at=self.LAUNCHED - 3600), 1))
|
|
383
|
+
self.assertIsNone(tr.pending)
|
|
384
|
+
tr.feed(cx_error(self.LAUNCHED + 50, "usage_limit_exceeded"), 1)
|
|
385
|
+
self.assertEqual(tr.pending["at"], self.LAUNCHED + 50)
|
|
386
|
+
|
|
387
|
+
def test_premium_null_line_is_skipped_and_the_message_date_is_used(self):
|
|
388
|
+
with local_tz("UTC"):
|
|
389
|
+
tr = self.tracker()
|
|
390
|
+
tr.feed(cx_tokens(self.LAUNCHED + 1, limit_id="premium"), 1)
|
|
391
|
+
tr.feed(cx_tokens(self.LAUNCHED + 1), 1) # codex, both windows null
|
|
392
|
+
self.assertIsNone(tr.snapshot)
|
|
393
|
+
tr.feed(cx_error(self.LAUNCHED + 1, "usage_limit_exceeded", LIMIT_MSG), 1)
|
|
394
|
+
self.assertEqual(tr.pending["reset"], 1790383140) # 2026-09-26 00:39 UTC
|
|
395
|
+
self.assertNotIn("_message", ar.verdict_public(tr.pending))
|
|
396
|
+
|
|
397
|
+
def test_codex_window_snapshot_wins(self):
|
|
398
|
+
tr = self.tracker()
|
|
399
|
+
now = self.LAUNCHED + 5
|
|
400
|
+
tr.feed(cx_tokens(self.LAUNCHED + 1,
|
|
401
|
+
primary={"used_percent": 42.0, "window_minutes": 300,
|
|
402
|
+
"resets_at": now + 600},
|
|
403
|
+
secondary={"used_percent": 100.0, "window_minutes": 10080,
|
|
404
|
+
"resets_at": now + 86400}), now)
|
|
405
|
+
tr.feed(cx_tokens(self.LAUNCHED + 2, limit_id="premium"), now) # must not erase it
|
|
406
|
+
tr.feed(cx_error(self.LAUNCHED + 3, "usage_limit_exceeded", LIMIT_MSG), now)
|
|
407
|
+
self.assertEqual(tr.pending["reset"], now + 86400)
|
|
408
|
+
self.assertEqual(tr.pending["rtype_window"], "7d")
|
|
409
|
+
|
|
410
|
+
def test_stale_snapshot_falls_back(self):
|
|
411
|
+
now = self.LAUNCHED + 5
|
|
412
|
+
# older than the launch: ignored
|
|
413
|
+
tr = self.tracker()
|
|
414
|
+
tr.feed(cx_tokens(self.LAUNCHED - 600, primary={"used_percent": 99.0,
|
|
415
|
+
"window_minutes": 300,
|
|
416
|
+
"resets_at": now + 50}), now)
|
|
417
|
+
self.assertIsNone(tr.snapshot)
|
|
418
|
+
# a reset already behind us, and no date in the message: an hour
|
|
419
|
+
tr.feed(cx_tokens(self.LAUNCHED + 1, primary={"used_percent": 99.0,
|
|
420
|
+
"window_minutes": 300,
|
|
421
|
+
"resets_at": now - 5}), now)
|
|
422
|
+
tr.feed(cx_error(self.LAUNCHED + 2, "rate_limit_exceeded", "slow down"), now)
|
|
423
|
+
self.assertEqual(tr.pending["reset"], now + 3600)
|
|
424
|
+
|
|
425
|
+
def test_try_again_at_is_local_time(self):
|
|
426
|
+
with local_tz("UTC"):
|
|
427
|
+
self.assertEqual(ar.parse_try_again("try again at Sep 26th, 2026 11:22 AM."),
|
|
428
|
+
1790421720)
|
|
429
|
+
self.assertEqual(ar.parse_try_again("try again at Sep 26th, 2026 12:39 AM."),
|
|
430
|
+
1790383140)
|
|
431
|
+
self.assertEqual(ar.parse_try_again("try again at Oct 1st, 2026 12:05 PM"),
|
|
432
|
+
1790856300)
|
|
433
|
+
self.assertEqual(ar.parse_try_again("try again at Nov 2nd, 2026 3:00 pm."),
|
|
434
|
+
1793631600)
|
|
435
|
+
self.assertEqual(ar.parse_try_again("try again at December 3rd, 2026 9:15 AM"),
|
|
436
|
+
1796289300)
|
|
437
|
+
if os.path.exists("/usr/share/zoneinfo/America/New_York"):
|
|
438
|
+
with local_tz("America/New_York"): # EDT, UTC-4
|
|
439
|
+
self.assertEqual(ar.parse_try_again(LIMIT_MSG), 1790383140 + 4 * 3600)
|
|
440
|
+
for bad in (None, "", "try again later", "try again at Foo 3rd, 2026 1:00 AM",
|
|
441
|
+
"try again at Sep 26th, 2026 13:00 PM", "try again at Sep 40th, 2026 1:00 AM"):
|
|
442
|
+
self.assertIsNone(ar.parse_try_again(bad), bad)
|
|
443
|
+
|
|
444
|
+
def test_prefilter(self):
|
|
445
|
+
tr = self.tracker()
|
|
446
|
+
self.assertTrue(tr.wants(json.dumps(cx_tokens(1)).encode()))
|
|
447
|
+
self.assertTrue(tr.wants(json.dumps(cx_error(1, "unauthorized")).encode()))
|
|
448
|
+
start = json.dumps(cx_event(1, {"type": "task_started"})).encode()
|
|
449
|
+
self.assertFalse(tr.wants(start))
|
|
450
|
+
self.assertFalse(tr.wants(b'{"type":"response_item","payload":{"type":"message"}}'))
|
|
451
|
+
tr.feed(cx_error(self.LAUNCHED, "unauthorized"), 1)
|
|
452
|
+
self.assertTrue(tr.wants(start))
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
# =====================================================================================
|
|
456
|
+
# argv
|
|
457
|
+
# =====================================================================================
|
|
458
|
+
|
|
459
|
+
class ArgvTests(unittest.TestCase):
|
|
460
|
+
def rl(self, provider, argv, cls="quota", prompt=None, sid=None):
|
|
461
|
+
sid = sid or (SID if provider == "claude" else CX_SID)
|
|
462
|
+
return ar.relaunch_argv(provider, argv, sid, cls, prompt)
|
|
463
|
+
|
|
464
|
+
def test_claude_relaunch_drops_resume_continue_and_the_prompt(self):
|
|
465
|
+
p = ar.resume_prompt("quota")
|
|
466
|
+
self.assertEqual(self.rl("claude", ["--dangerously-skip-permissions", "fix the tests"]),
|
|
467
|
+
["--dangerously-skip-permissions", "--resume", SID, p])
|
|
468
|
+
self.assertEqual(self.rl("claude", ["-c"]), ["--resume", SID, p])
|
|
469
|
+
self.assertEqual(self.rl("claude", ["--continue", "--effort", "high"]),
|
|
470
|
+
["--effort", "high", "--resume", SID, p])
|
|
471
|
+
self.assertEqual(self.rl("claude", ["-r", SID2, "--permission-mode", "plan"]),
|
|
472
|
+
["--permission-mode", "plan", "--resume", SID, p])
|
|
473
|
+
self.assertEqual(self.rl("claude", ["--resume=" + SID2,
|
|
474
|
+
"--allow-dangerously-skip-permissions"]),
|
|
475
|
+
["--allow-dangerously-skip-permissions", "--resume", SID, p])
|
|
476
|
+
self.assertEqual(self.rl("claude", []), ["--resume", SID, p])
|
|
477
|
+
|
|
478
|
+
def test_claude_model_is_only_the_launchs_own(self):
|
|
479
|
+
# Claude restores the session's model on --resume ([1m] included); an added
|
|
480
|
+
# --model would switch that off. An explicit one from the launch is kept as is.
|
|
481
|
+
p = ar.resume_prompt("model")
|
|
482
|
+
self.assertEqual(self.rl("claude", ["hello there"], "model"), ["--resume", SID, p])
|
|
483
|
+
self.assertEqual(self.rl("claude", ["--model", "sonnet"], "model"),
|
|
484
|
+
["--model", "sonnet", "--resume", SID, p])
|
|
485
|
+
self.assertEqual(self.rl("claude", ["--model=sonnet"], "model"),
|
|
486
|
+
["--model=sonnet", "--resume", SID, p])
|
|
487
|
+
|
|
488
|
+
def test_claude_allowlist_rejects(self):
|
|
489
|
+
for argv in (["-p", "hi there"], ["--print"], ["doctor"], ["mcp", "list"],
|
|
490
|
+
["one prompt", "two prompts"], ["-r"], ["-r", "not-hex-zz"],
|
|
491
|
+
["-r", "abcdef"], ["--resume=-x"], ["--model"], ["--model", "-p"],
|
|
492
|
+
["--model="], ["--verbose"], ["--", "hi there"], ["-x"]):
|
|
493
|
+
self.assertIsNone(ar.claude_argv_parse(argv), argv)
|
|
494
|
+
self.assertIsNone(self.rl("claude", argv), argv)
|
|
495
|
+
|
|
496
|
+
def test_codex_relaunch(self):
|
|
497
|
+
p = ar.resume_prompt("quota")
|
|
498
|
+
self.assertEqual(self.rl("codex", ["--dangerously-bypass-approvals-and-sandbox"]),
|
|
499
|
+
["resume", "--dangerously-bypass-approvals-and-sandbox", CX_SID, p])
|
|
500
|
+
self.assertEqual(self.rl("codex", ["--yolo", "resume", CX_SID2, "keep going now"]),
|
|
501
|
+
["resume", "--yolo", CX_SID, p])
|
|
502
|
+
self.assertEqual(self.rl("codex", ["resume", "--last", "-m", "gpt-5.5"]),
|
|
503
|
+
["resume", "-m", "gpt-5.5", CX_SID, p])
|
|
504
|
+
self.assertEqual(self.rl("codex", ["--model=gpt-5", "fix the build"]),
|
|
505
|
+
["resume", "--model=gpt-5", CX_SID, p])
|
|
506
|
+
self.assertEqual(ar.codex_resume_id(["--yolo", "resume", CX_SID2]), CX_SID2)
|
|
507
|
+
self.assertIsNone(ar.codex_resume_id(["resume", "--last"]))
|
|
508
|
+
self.assertIsNone(ar.codex_resume_id(["do the thing"]))
|
|
509
|
+
|
|
510
|
+
def test_codex_allowlist_rejects(self):
|
|
511
|
+
for argv in (["exec", "hi there"], ["e"], ["-p", "work"], ["--profile", "x"],
|
|
512
|
+
["resume"], ["--last"], ["resume", "--last", CX_SID],
|
|
513
|
+
["resume", "notauuid"], ["hi there", "resume", CX_SID],
|
|
514
|
+
["resume", CX_SID, "a b", "c d"], ["-c", "model=x"], ["-m"],
|
|
515
|
+
["-m", "-x"], ["resume", "resume"], ["login"], ["app-server"]):
|
|
516
|
+
self.assertIsNone(ar.codex_argv_parse(argv), argv)
|
|
517
|
+
self.assertIsNone(self.rl("codex", argv), argv)
|
|
518
|
+
|
|
519
|
+
def test_relaunch_refuses_a_bad_session_id_or_class(self):
|
|
520
|
+
for sid in ("", "-rf", "abcdef", "../x", SID + "zz", "a-" * 40):
|
|
521
|
+
self.assertIsNone(ar.relaunch_argv("claude", [], sid, "quota"), sid)
|
|
522
|
+
self.assertIsNone(ar.relaunch_argv("claude", [], SID, "nope"))
|
|
523
|
+
|
|
524
|
+
def test_prompt(self):
|
|
525
|
+
q = ar.resume_prompt("quota")
|
|
526
|
+
self.assertTrue(q.startswith(ar.PROMPT_PREFIX + " This session was restarted "
|
|
527
|
+
"automatically on another account because the previous "
|
|
528
|
+
"account hit its usage limit. Continue the task"))
|
|
529
|
+
self.assertIn("the user has not sent a new message", q)
|
|
530
|
+
self.assertIn("do not repeat work that is already done.", q)
|
|
531
|
+
self.assertIn("this model's usage limit", ar.resume_prompt("model"))
|
|
532
|
+
self.assertIn("login failed", ar.resume_prompt("auth"))
|
|
533
|
+
self.assertIn("was refused", ar.resume_prompt("blocked"))
|
|
534
|
+
for cls in ("crash", "transient"):
|
|
535
|
+
text = ar.resume_prompt(cls)
|
|
536
|
+
self.assertNotIn("on another account", text)
|
|
537
|
+
self.assertIn("was restarted automatically because", text)
|
|
538
|
+
self.assertIn("exited unexpectedly", ar.resume_prompt("crash"))
|
|
539
|
+
self.assertIn("an API error ended the last turn", ar.resume_prompt("transient"))
|
|
540
|
+
self.assertEqual(ar.resume_prompt("auth", "Resume: {reason} ok"),
|
|
541
|
+
"Resume: the previous account's login failed ok")
|
|
542
|
+
for bad in ("-p boom", "oneword", "two\nlines", " "):
|
|
543
|
+
self.assertEqual(ar.resume_prompt("quota", bad), q, bad)
|
|
544
|
+
self.assertEqual(ar.Config("claude", {"CLAUDE_MULTIACC_AUTORESUME_PROMPT": "go on now"})
|
|
545
|
+
.prompt, "go on now")
|
|
546
|
+
self.assertIsNone(ar.Config("codex", {"CODEX_MULTIACC_AUTORESUME_PROMPT": "-x y"}).prompt)
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
# =====================================================================================
|
|
550
|
+
# relaunch file, state file, budgets, tmux, tokens
|
|
551
|
+
# =====================================================================================
|
|
552
|
+
|
|
553
|
+
class RelaunchFileTests(unittest.TestCase):
|
|
554
|
+
def fields(self, **over):
|
|
555
|
+
verdict = {"class": "quota", "reset": 1790123817, "rtype": "five_hour",
|
|
556
|
+
"marked": "2026-09-22T21:36:58Z"}
|
|
557
|
+
verdict.update(over.pop("verdict", {}))
|
|
558
|
+
args = dict(provider="claude", verdict=verdict, acct="acct-01", sid=SID, cwd="/w d",
|
|
559
|
+
depth=2, avoid="acct-01:1790123817", hist="quota:1790100000,quota:17901",
|
|
560
|
+
chain="Ab3dEf9h")
|
|
561
|
+
args.update(over)
|
|
562
|
+
return ar.relaunch_fields(**args)
|
|
563
|
+
|
|
564
|
+
def test_format(self):
|
|
565
|
+
text = ar.format_relaunch(self.fields())
|
|
566
|
+
self.assertEqual(text.split("\n"), [
|
|
567
|
+
"v=1", "provider=claude", "class=quota", "acct=acct-01", "reset=1790123817",
|
|
568
|
+
"rtype=five_hour", "marked=2026-09-22T21:36:58Z", "sid=" + SID, "cwd=/w d",
|
|
569
|
+
"depth=3", "avoid=acct-01:1790123817", "hist=quota:1790100000,quota:17901",
|
|
570
|
+
"chain=Ab3dEf9h", ""])
|
|
571
|
+
|
|
572
|
+
def test_reset_and_rtype_belong_to_a_claude_quota_only(self):
|
|
573
|
+
kv = ar.parse_kv(ar.format_relaunch(self.fields(verdict={"class": "auth"})),
|
|
574
|
+
ar.RELAUNCH_KEYS)
|
|
575
|
+
self.assertEqual((kv["class"], kv["reset"], kv["rtype"]), ("auth", "", ""))
|
|
576
|
+
kv = ar.parse_kv(ar.format_relaunch(self.fields(provider="codex")), ar.RELAUNCH_KEYS)
|
|
577
|
+
self.assertEqual((kv["provider"], kv["reset"], kv["rtype"]),
|
|
578
|
+
("codex", "1790123817", ""))
|
|
579
|
+
|
|
580
|
+
def test_write_is_atomic_private_and_bash_readable(self):
|
|
581
|
+
argv = ["--dangerously-skip-permissions", "--resume", SID,
|
|
582
|
+
"multi\nline prompt with 'quotes' and é"]
|
|
583
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
584
|
+
d = os.path.join(tmp, "tmp", "autoresume")
|
|
585
|
+
rel, av = ar.write_relaunch(d, "Tok3nTok3nTok3n0", self.fields(), argv)
|
|
586
|
+
self.assertEqual(rel, os.path.join(d, "r-Tok3nTok3nTok3n0.relaunch"))
|
|
587
|
+
self.assertEqual(av, os.path.join(d, "r-Tok3nTok3nTok3n0.argv"))
|
|
588
|
+
self.assertEqual(sorted(os.listdir(d)),
|
|
589
|
+
["r-Tok3nTok3nTok3n0.argv", "r-Tok3nTok3nTok3n0.relaunch"])
|
|
590
|
+
for path in (rel, av):
|
|
591
|
+
self.assertEqual(stat.S_IMODE(os.stat(path).st_mode), 0o600)
|
|
592
|
+
self.assertEqual(ar.decode_argv(Path(av).read_bytes()), argv)
|
|
593
|
+
# The shim's own reader: while IFS= read -r -d '' a
|
|
594
|
+
out = subprocess.run(
|
|
595
|
+
["bash", "-c", "n=0; while IFS= read -r -d '' a; do n=$((n+1)); "
|
|
596
|
+
"printf '%s\\0' \"$a\"; done < \"$1\"", "_", av],
|
|
597
|
+
stdout=subprocess.PIPE, check=True, timeout=10).stdout
|
|
598
|
+
self.assertEqual(ar.decode_argv(out), argv)
|
|
599
|
+
|
|
600
|
+
def test_unwritable_values_leave_nothing(self):
|
|
601
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
602
|
+
with self.assertRaises(ValueError):
|
|
603
|
+
ar.write_relaunch(tmp, "Tok3nTok3nTok3n0", self.fields(cwd="/a\nb"), ["x"])
|
|
604
|
+
with self.assertRaises(ValueError):
|
|
605
|
+
ar.write_relaunch(tmp, "bad/token", self.fields(), ["x"])
|
|
606
|
+
with self.assertRaises(ValueError):
|
|
607
|
+
ar.write_relaunch(tmp, "Tok3nTok3nTok3n0", self.fields(), ["a\0b"])
|
|
608
|
+
self.assertEqual(os.listdir(tmp), [])
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
class StateTests(unittest.TestCase):
|
|
612
|
+
def good(self):
|
|
613
|
+
return {"v": "1", "provider": "claude", "pid": "4242", "ppid": "777",
|
|
614
|
+
"acct": "acct-01", "acct_dir": "/p/acct-01", "cwd": "/w", "launched": "1790113000",
|
|
615
|
+
"tmux": "/private/tmp/tmux-501/default,12345,0", "pane": "%3",
|
|
616
|
+
"self": "/p/bin/claude", "acc_root": "/p", "depth": "0", "avoid": "",
|
|
617
|
+
"hist": "", "chain": ""}
|
|
618
|
+
|
|
619
|
+
def load(self, kv, extra=""):
|
|
620
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
621
|
+
path = os.path.join(tmp, "4242.state")
|
|
622
|
+
Path(path).write_text("".join("%s=%s\n" % i for i in kv.items()) + extra)
|
|
623
|
+
return ar.load_state(path)
|
|
624
|
+
|
|
625
|
+
def test_valid_state(self):
|
|
626
|
+
st = self.load(self.good(), "future_key=whatever\ngarbage line\n")
|
|
627
|
+
self.assertEqual((st["pid"], st["ppid"], st["depth"], st["launched"], st["pane"]),
|
|
628
|
+
(4242, 777, 0, 1790113000, "%3"))
|
|
629
|
+
self.assertEqual(ar.tmux_socket(st["tmux"]), "/private/tmp/tmux-501/default")
|
|
630
|
+
|
|
631
|
+
def test_malformed_state_is_refused(self):
|
|
632
|
+
for key, val in (("v", "2"), ("provider", "gemini"), ("pid", "x"), ("pid", "1"),
|
|
633
|
+
("ppid", ""), ("acct", "../etc"), ("acct_dir", "/p/acct-02"),
|
|
634
|
+
("acct_dir", ""), ("cwd", "rel"), ("launched", "-1"),
|
|
635
|
+
("tmux", "default,1,0"), ("pane", "3"), ("self", "claude"),
|
|
636
|
+
("acc_root", ""), ("depth", "x")):
|
|
637
|
+
kv = self.good()
|
|
638
|
+
kv[key] = val
|
|
639
|
+
self.assertIsNone(self.load(kv), (key, val))
|
|
640
|
+
kv = self.good()
|
|
641
|
+
del kv["pid"]
|
|
642
|
+
self.assertIsNone(self.load(kv))
|
|
643
|
+
self.assertIsNone(ar.load_state("/nonexistent/4242.state"))
|
|
644
|
+
|
|
645
|
+
def test_relative_pool_paths_resolve_against_the_cwd(self):
|
|
646
|
+
kv = self.good()
|
|
647
|
+
kv.update(acct_dir="pool/acct-01", acc_root="pool")
|
|
648
|
+
st = self.load(kv)
|
|
649
|
+
self.assertEqual((st["acct_dir"], st["acc_root"]),
|
|
650
|
+
(os.path.join(os.getcwd(), "pool", "acct-01"),
|
|
651
|
+
os.path.join(os.getcwd(), "pool")))
|
|
652
|
+
|
|
653
|
+
def test_bad_chain_is_dropped_not_fatal(self):
|
|
654
|
+
kv = self.good()
|
|
655
|
+
kv["chain"] = "a b"
|
|
656
|
+
self.assertEqual(self.load(kv)["chain"], "")
|
|
657
|
+
|
|
658
|
+
def test_probe_env_and_answer(self):
|
|
659
|
+
env = {"PATH": "/bin", "CLAUDE_CONFIG_DIR": "/p/acct-01", "CLAUDE_CODE_OAUTH_TOKEN": "t",
|
|
660
|
+
"CLAUDE_SHIM_ACTIVE": "1", "CLAUDE_ACCOUNT": "acct-01", "CLAUDE_MULTIACC_AR": "x",
|
|
661
|
+
"CLAUDE_MULTIACC_AR_AVOID": "stale"}
|
|
662
|
+
out = ar.probe_env(env, "claude", "acct-01:99")
|
|
663
|
+
self.assertEqual(out, {"PATH": "/bin", "CLAUDE_MULTIACC_AR_PROBE": "1",
|
|
664
|
+
"CLAUDE_MULTIACC_AR_AVOID": "acct-01:99"})
|
|
665
|
+
cx = ar.probe_env({"CODEX_HOME": "/h", "CODEX_SHIM_ACTIVE": "1", "CODEX_ACCOUNT": "a",
|
|
666
|
+
"CODEX_MULTIACC_AR": "t", "HOME": "/u"}, "codex", "")
|
|
667
|
+
self.assertEqual(cx, {"HOME": "/u", "CODEX_MULTIACC_AR_PROBE": "1",
|
|
668
|
+
"CODEX_MULTIACC_AR_AVOID": ""})
|
|
669
|
+
self.assertEqual(ar.parse_probe("noise\npick=acct-02 tier=eligible\n"),
|
|
670
|
+
("acct-02", "eligible"))
|
|
671
|
+
self.assertEqual(ar.parse_probe("pick= tier=none\n"), ("", "none"))
|
|
672
|
+
for bad in ("", "pick=acct-02", "pick=../x tier=eligible", "pick=acct-02 tier=great"):
|
|
673
|
+
self.assertEqual(ar.parse_probe(bad), ("", ""), bad)
|
|
674
|
+
self.assertTrue(ar.probe_allows("quota", "acct-02", "eligible", "acct-01"))
|
|
675
|
+
self.assertFalse(ar.probe_allows("quota", "acct-01", "eligible", "acct-01"))
|
|
676
|
+
self.assertFalse(ar.probe_allows("auth", "acct-02", "soft", "acct-01"))
|
|
677
|
+
self.assertTrue(ar.probe_allows("transient", "acct-01", "soft", "acct-01"))
|
|
678
|
+
self.assertFalse(ar.probe_allows("transient", "acct-01", "hard", "acct-01"))
|
|
679
|
+
self.assertFalse(ar.probe_allows("transient", "", "none", "acct-01"))
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
class BudgetTests(unittest.TestCase):
|
|
683
|
+
NOW = 1790113000
|
|
684
|
+
|
|
685
|
+
def test_depth_is_final(self):
|
|
686
|
+
c = cfg()
|
|
687
|
+
self.assertEqual(ar.budget_check("quota", 20, "", self.NOW, c), (False, "depth"))
|
|
688
|
+
self.assertEqual(ar.budget_check("crash", 19, "", self.NOW, c), (True, ""))
|
|
689
|
+
self.assertEqual(ar.budget_check("quota", 3, "", self.NOW, cfg(MAX_DEPTH=3)),
|
|
690
|
+
(False, "depth"))
|
|
691
|
+
|
|
692
|
+
def test_hourly_windows(self):
|
|
693
|
+
c = cfg()
|
|
694
|
+
recent = ",".join("%s:%d" % (k, self.NOW - 60 * i) for i, k in
|
|
695
|
+
enumerate(["quota", "auth", "model", "blocked"] * 2))
|
|
696
|
+
self.assertEqual(ar.budget_check("quota", 0, recent, self.NOW, c),
|
|
697
|
+
(False, "rotate-budget"))
|
|
698
|
+
old = ",".join("quota:%d" % (self.NOW - 3600 - i) for i in range(8))
|
|
699
|
+
self.assertEqual(ar.budget_check("auth", 0, old, self.NOW, c), (True, ""))
|
|
700
|
+
# rotations do not spend the transient budget and vice versa
|
|
701
|
+
self.assertEqual(ar.budget_check("transient", 0, recent, self.NOW, c), (True, ""))
|
|
702
|
+
tr = "transient:%d,transient:%d,transient:%d" % (self.NOW - 10, self.NOW - 20,
|
|
703
|
+
self.NOW - 30)
|
|
704
|
+
self.assertEqual(ar.budget_check("transient", 0, tr, self.NOW, c),
|
|
705
|
+
(False, "transient-budget"))
|
|
706
|
+
self.assertEqual(ar.budget_check("quota", 0, tr, self.NOW, c), (True, ""))
|
|
707
|
+
crash = "crash:%d,crash:%d" % (self.NOW - 100, self.NOW - 500)
|
|
708
|
+
self.assertEqual(ar.budget_check("crash", 0, crash, self.NOW, c),
|
|
709
|
+
(False, "crash-budget"))
|
|
710
|
+
self.assertEqual(ar.budget_check("crash", 0, "crash:%d,crash:%d" % (
|
|
711
|
+
self.NOW - 100, self.NOW - 700), self.NOW, c), (True, ""))
|
|
712
|
+
|
|
713
|
+
def test_transient_ladder(self):
|
|
714
|
+
ladder = cfg().transient_ladder
|
|
715
|
+
self.assertEqual(ladder, (30.0, 60.0, 120.0))
|
|
716
|
+
self.assertEqual(ar.ladder_wait("", self.NOW, ladder), 30.0)
|
|
717
|
+
self.assertEqual(ar.ladder_wait("transient:%d" % (self.NOW - 5), self.NOW, ladder), 60.0)
|
|
718
|
+
many = ",".join("transient:%d" % (self.NOW - i) for i in range(5))
|
|
719
|
+
self.assertEqual(ar.ladder_wait(many, self.NOW, ladder), 120.0)
|
|
720
|
+
self.assertEqual(ar.ladder_wait("transient:%d" % (self.NOW - 4000), self.NOW, ladder),
|
|
721
|
+
30.0)
|
|
722
|
+
self.assertEqual(cfg(TRANSIENT_LADDER="1,2").transient_ladder, (1.0, 2.0))
|
|
723
|
+
self.assertEqual(cfg(TRANSIENT_LADDER="1,x").transient_ladder, (30.0, 60.0, 120.0))
|
|
724
|
+
|
|
725
|
+
def test_knobs(self):
|
|
726
|
+
c = cfg(POLL="0.5", GRACE="bad", MAX_DEPTH="7", TERM_GRACE="-3")
|
|
727
|
+
self.assertEqual((c.poll, c.grace, c.max_depth, c.term_grace), (0.5, 5.0, 7, 10.0))
|
|
728
|
+
self.assertEqual(ar.Config("claude", {}).grace, 5.0)
|
|
729
|
+
self.assertEqual(cfg(POLL="0").poll, 0.02)
|
|
730
|
+
d = ar.Config("codex", {})
|
|
731
|
+
self.assertEqual((d.hold_reprobe, d.rotate_per_hour, d.transient_per_hour,
|
|
732
|
+
d.crash_per_10min, d.crash_min_runtime, d.pane_wait,
|
|
733
|
+
d.probe_timeout, d.discover_timeout),
|
|
734
|
+
(60.0, 8, 3, 2, 60.0, 15.0, 30.0, 300.0))
|
|
735
|
+
self.assertFalse(d.anypane)
|
|
736
|
+
self.assertTrue(ar.Config("codex", {"CODEX_MULTIACC_AR_TEST_TMUX_ANYPANE": "1"}).anypane)
|
|
737
|
+
|
|
738
|
+
def test_avoid_and_hist(self):
|
|
739
|
+
now = self.NOW
|
|
740
|
+
raw = "acct-02:%d,acct-01:%d,acct-01:%d,bogus,acct-03:%d,../x:%d" % (
|
|
741
|
+
now + 50, now + 10, now + 99, now - 1, now + 5)
|
|
742
|
+
self.assertEqual(ar.avoid_merge(raw, now), "acct-01:%d,acct-02:%d" % (now + 99, now + 50))
|
|
743
|
+
self.assertEqual(ar.avoid_merge(raw, now, "acct-01", now + 20),
|
|
744
|
+
"acct-01:%d,acct-02:%d" % (now + 99, now + 50))
|
|
745
|
+
self.assertEqual(ar.avoid_merge("", now, "acct-04", now + 7), "acct-04:%d" % (now + 7))
|
|
746
|
+
self.assertEqual(ar.avoid_until({"class": "quota", "reset": now + 500}, now), now + 500)
|
|
747
|
+
self.assertEqual(ar.avoid_until({"class": "quota", "reset": now - 5}, now), now + 3600)
|
|
748
|
+
self.assertEqual(ar.avoid_until({"class": "quota", "reset": None}, now), now + 3600)
|
|
749
|
+
self.assertEqual(ar.avoid_until({"class": "auth"}, now), now + 3600)
|
|
750
|
+
self.assertEqual(ar.avoid_until({"class": "blocked"}, now), now + 6 * 3600)
|
|
751
|
+
self.assertEqual(ar.avoid_until({"class": "model"}, now), now + 5 * 3600)
|
|
752
|
+
hist = ",".join("quota:%d" % i for i in range(40))
|
|
753
|
+
out = ar.hist_append(hist + ",BAD:1,x", "auth", now)
|
|
754
|
+
self.assertEqual(len(out.split(",")), 32)
|
|
755
|
+
self.assertTrue(out.endswith(",auth:%d" % now))
|
|
756
|
+
self.assertTrue(out.startswith("quota:9,"))
|
|
757
|
+
|
|
758
|
+
|
|
759
|
+
class TmuxAndTokenTests(unittest.TestCase):
|
|
760
|
+
SOCK = "/private/tmp/tmux-501/default"
|
|
761
|
+
|
|
762
|
+
def test_socket(self):
|
|
763
|
+
self.assertEqual(ar.tmux_socket(self.SOCK + ",4211,0"), self.SOCK)
|
|
764
|
+
self.assertEqual(ar.tmux_socket(self.SOCK), self.SOCK)
|
|
765
|
+
for bad in ("", ",1,0", "default,1,0", None):
|
|
766
|
+
self.assertIsNone(ar.tmux_socket(bad), bad)
|
|
767
|
+
|
|
768
|
+
def test_pane_query(self):
|
|
769
|
+
self.assertEqual(ar.pane_query_argv(self.SOCK, "%7"),
|
|
770
|
+
["tmux", "-S", self.SOCK, "display-message", "-p", "-t", "%7",
|
|
771
|
+
"#{pane_pid}|#{pane_in_mode}|#{synchronize-panes}"
|
|
772
|
+
"|#{pane_current_command}"])
|
|
773
|
+
# (pid, command, a tmux mode owns the keys, keys go to every pane)
|
|
774
|
+
self.assertEqual(ar.parse_pane_reply("8123|0|0|zsh\n"), (8123, "zsh", False, False))
|
|
775
|
+
self.assertEqual(ar.parse_pane_reply("8123|0|0|2.1.280\n"),
|
|
776
|
+
(8123, "2.1.280", False, False))
|
|
777
|
+
self.assertEqual(ar.parse_pane_reply("8123|1|0|claude"), (8123, "claude", True, False))
|
|
778
|
+
# modes stack: tree + clock + copy reads 3
|
|
779
|
+
self.assertEqual(ar.parse_pane_reply("8123|3|1|zsh"), (8123, "zsh", True, True))
|
|
780
|
+
# an option/format an old tmux lacks expands to nothing: off
|
|
781
|
+
self.assertEqual(ar.parse_pane_reply("8123|||zsh"), (8123, "zsh", False, False))
|
|
782
|
+
self.assertEqual(ar.parse_pane_reply("8123|0|0|a|b c"), (8123, "a|b c", False, False))
|
|
783
|
+
for bad in ("", "no server running on /x\n", "zsh 8123", "8123 zsh", "x|0|0|zsh"):
|
|
784
|
+
self.assertIsNone(ar.parse_pane_reply(bad), bad)
|
|
785
|
+
for sh in ("zsh", "bash", "sh", "dash", "ksh", "-zsh", "/bin/bash", "-/bin/zsh"):
|
|
786
|
+
self.assertTrue(ar.is_shell(sh), sh)
|
|
787
|
+
# fish < 3.1 cannot run `VAR=x cmd`; the rest never can
|
|
788
|
+
for other in ("claude", "node", "codex", "2.1.280", "", "-", "vim", "nu", "tcsh",
|
|
789
|
+
"-tcsh", "pwsh", "xonsh", "fish", "-fish", "elvish", "mksh"):
|
|
790
|
+
self.assertFalse(ar.is_shell(other), other)
|
|
791
|
+
# a mode is only ever left by a mode command, never a key
|
|
792
|
+
self.assertEqual(ar.mode_cancel_argv(self.SOCK, "%7"),
|
|
793
|
+
["tmux", "-S", self.SOCK, "send-keys", "-X", "-t", "%7", "cancel"])
|
|
794
|
+
|
|
795
|
+
def test_foreground(self):
|
|
796
|
+
self.assertTrue(ar.ps_foreground(("Ss+", "ttys003", "-zsh")))
|
|
797
|
+
self.assertFalse(ar.ps_foreground(("Ss", "ttys003", "-zsh")))
|
|
798
|
+
self.assertFalse(ar.ps_foreground(("T", "pts/3", "claude")))
|
|
799
|
+
for no_answer in (None, ("S", "??", "bash"), ("S", "?", "bash"), ("S", "-", "sh")):
|
|
800
|
+
self.assertIsNone(ar.ps_foreground(no_answer), no_answer)
|
|
801
|
+
|
|
802
|
+
def test_typeable_paths(self):
|
|
803
|
+
for good in ("/Users/gas/.claude-accounts", "/opt/x_y/v1.2+b-c/bin/claude", "/"):
|
|
804
|
+
self.assertTrue(ar.typeable(good), good)
|
|
805
|
+
for bad in ("", None, "rel/path", "~/.claude-accounts", "/Users/John Doe/pool",
|
|
806
|
+
"/a/$(rm -rf x)", "/a/it's", "/a\\b", "/a\nb", "/a*b", "/a;b", "/a=b",
|
|
807
|
+
"/a,b", "/a:b", "/a@b", "/a%b", "/é"):
|
|
808
|
+
self.assertFalse(ar.typeable(bad), bad)
|
|
809
|
+
|
|
810
|
+
def test_send_keys(self):
|
|
811
|
+
tok = "AbCdEfGh12345678"
|
|
812
|
+
shim = "/opt/cm/lib/node_modules/claude-multiacc/bin/claude"
|
|
813
|
+
self.assertEqual(ar.send_keys_argvs(self.SOCK, "%7", "claude", tok, SID, shim), [
|
|
814
|
+
["tmux", "-S", self.SOCK, "send-keys", "-R", "-t", "%7"],
|
|
815
|
+
["tmux", "-S", self.SOCK, "send-keys", "-t", "%7", "C-u"],
|
|
816
|
+
["tmux", "-S", self.SOCK, "send-keys", "-t", "%7", "-l",
|
|
817
|
+
" CLAUDE_MULTIACC_AR=AbCdEfGh12345678:%s %s" % (SID, shim)],
|
|
818
|
+
["tmux", "-S", self.SOCK, "send-keys", "-t", "%7", "Enter"]])
|
|
819
|
+
# the pool is named only when the relaunch has to (codex reads CODEX_ACCOUNTS_ROOT)
|
|
820
|
+
self.assertEqual(ar.relaunch_command("codex", tok, CX_SID, "/x/bin/codex", "/srv/p-1"),
|
|
821
|
+
" CODEX_MULTIACC_AR=%s:%s CODEX_ACCOUNTS_ROOT=/srv/p-1 /x/bin/codex"
|
|
822
|
+
% (tok, CX_SID))
|
|
823
|
+
self.assertEqual(ar.relaunch_command("claude", tok, SID, "/x/claude", "/p"),
|
|
824
|
+
" CLAUDE_MULTIACC_AR=%s:%s CLAUDE_ACCOUNTS_ROOT=/p /x/claude"
|
|
825
|
+
% (tok, SID))
|
|
826
|
+
for args in (("claude", "short", SID, shim), ("claude", "a;rm -rf ~ #xxxxx", SID, shim),
|
|
827
|
+
("gemini", tok, SID, shim), ("claude", tok, "abcdef", shim),
|
|
828
|
+
("claude", tok, SID, "claude"), ("claude", tok, SID, "/a b/claude"),
|
|
829
|
+
("claude", tok, SID, shim, "/p q"), ("claude", tok, SID, shim, "")):
|
|
830
|
+
with self.assertRaises(ValueError):
|
|
831
|
+
ar.relaunch_command(*args)
|
|
832
|
+
|
|
833
|
+
def test_notice(self):
|
|
834
|
+
# -d 0: the message stays until a key is pressed (tmux >= 3.2)...
|
|
835
|
+
self.assertEqual(ar.notice_argv(self.SOCK, "%7", "claude", SID), [
|
|
836
|
+
"tmux", "-S", self.SOCK, "display-message", "-d", "0", "-t", "%7",
|
|
837
|
+
"claude-multiacc: auto-resume failed — run: claude --resume " + SID])
|
|
838
|
+
# ...and the plain form is the fallback for an older tmux
|
|
839
|
+
self.assertEqual(ar.notice_argv(self.SOCK, "%7", "claude", SID, sticky=False), [
|
|
840
|
+
"tmux", "-S", self.SOCK, "display-message", "-t", "%7",
|
|
841
|
+
"claude-multiacc: auto-resume failed — run: claude --resume " + SID])
|
|
842
|
+
cx = ar.notice_argv(self.SOCK, "%7", "codex", CX_SID)
|
|
843
|
+
self.assertEqual(cx[-1], "claude-multiacc: auto-resume failed — run: codex resume "
|
|
844
|
+
+ CX_SID)
|
|
845
|
+
self.assertNotIn("-p", cx) # the status line, never the pane's input
|
|
846
|
+
|
|
847
|
+
def test_tokens(self):
|
|
848
|
+
seen = set()
|
|
849
|
+
for _ in range(500):
|
|
850
|
+
tok = ar.make_token()
|
|
851
|
+
self.assertRegex(tok, r"^[A-Za-z0-9]{16}$")
|
|
852
|
+
seen.add(tok)
|
|
853
|
+
self.assertEqual(len(seen), 500)
|
|
854
|
+
self.assertRegex(ar.make_token(8), r"^[A-Za-z0-9]{8}$")
|
|
855
|
+
|
|
856
|
+
def test_misc_parsers(self):
|
|
857
|
+
self.assertEqual(ar.iso_to_epoch("2026-09-22T21:36:58.009Z"), 1790113018.009)
|
|
858
|
+
self.assertEqual(ar.iso_to_epoch("2026-09-22T21:36:58+00:00"), 1790113018)
|
|
859
|
+
self.assertEqual(ar.iso_to_epoch("2026-09-23T00:36:58+03:00"), 1790113018)
|
|
860
|
+
for bad in (None, 5, "", "yesterday", "2026-13-40T99:00:00Z"):
|
|
861
|
+
self.assertIsNone(ar.iso_to_epoch(bad), bad)
|
|
862
|
+
self.assertEqual(ar.utc_iso(1790113018), "2026-09-22T21:36:58Z")
|
|
863
|
+
self.assertEqual(ar.epoch_of(1790123817), 1790123817)
|
|
864
|
+
self.assertEqual(ar.epoch_of("1790123817"), 1790123817)
|
|
865
|
+
for bad in (True, None, -1, 0, "x", float("nan"), 10 ** 12):
|
|
866
|
+
self.assertIsNone(ar.epoch_of(bad), bad)
|
|
867
|
+
self.assertEqual(ar.descendants([(10, 1, ""), (11, 10, ""), (12, 11, ""), (13, 1, ""),
|
|
868
|
+
(14, 10, "")], 10), [11, 14, 12])
|
|
869
|
+
self.assertTrue(ar.uuid_ok(SID))
|
|
870
|
+
self.assertFalse(ar.uuid_ok("abcdef"))
|
|
871
|
+
|
|
872
|
+
|
|
873
|
+
# =====================================================================================
|
|
874
|
+
# the watcher, with a fake runner
|
|
875
|
+
# =====================================================================================
|
|
876
|
+
|
|
877
|
+
class FakeRunner:
|
|
878
|
+
"""Processes, clock and tmux, simulated. ``hooks`` fire as the fake clock passes
|
|
879
|
+
their time, so a scenario can make the client exit, the user type, etc."""
|
|
880
|
+
|
|
881
|
+
def __init__(self, t0, pid, ppid, self_path):
|
|
882
|
+
self.t = float(t0)
|
|
883
|
+
self.pid = pid
|
|
884
|
+
self.live = {pid}
|
|
885
|
+
self.starts = {pid: LSTART}
|
|
886
|
+
self.ppid = ppid
|
|
887
|
+
self.pane_pid = ppid
|
|
888
|
+
self.pane_cmd = "claude"
|
|
889
|
+
self.pane_mode = 0 # #{pane_in_mode}: >0 while a tmux mode owns the keys
|
|
890
|
+
self.mode_sticky = False # True: `send-keys -X cancel` cannot end the mode
|
|
891
|
+
self.pane_sync = 0 # #{synchronize-panes}
|
|
892
|
+
self.shell_comm = "-zsh" # what ps says the launching shell runs
|
|
893
|
+
self.client_stat = "S+" # the client's ps stat (T = stopped with Ctrl-Z)
|
|
894
|
+
self.tty = "ttys003"
|
|
895
|
+
self.open_files_ok = True # /proc or lsof available
|
|
896
|
+
self.lookup_fails = False # the lookup is there but every call fails
|
|
897
|
+
self.pgids = {} # pid -> process group; default its own (a job leader)
|
|
898
|
+
self.stuck = set() # pids no signal ends (uninterruptible sleep)
|
|
899
|
+
self.old_tmux = False # True: display-message refuses -d (tmux < 3.2)
|
|
900
|
+
self.ar_dir = None # the pool's tmp/autoresume (PoolCase.watcher sets it)
|
|
901
|
+
self.consume = True # False: the typed line never runs (token left behind)
|
|
902
|
+
self.consumed = {} # token -> (relaunch text, argv bytes) the shim took
|
|
903
|
+
self.line = "" # the last -l text typed
|
|
904
|
+
self.self_path = self_path
|
|
905
|
+
self.probe_out = "pick=acct-02 tier=eligible\n"
|
|
906
|
+
self.term_kills = True
|
|
907
|
+
self.table = []
|
|
908
|
+
self.files = {}
|
|
909
|
+
self.calls = []
|
|
910
|
+
self.signals = []
|
|
911
|
+
self.hooks = []
|
|
912
|
+
self.limit = t0 + 4000
|
|
913
|
+
|
|
914
|
+
def at(self, when, fn):
|
|
915
|
+
self.hooks.append((when, fn))
|
|
916
|
+
|
|
917
|
+
def now(self):
|
|
918
|
+
return self.t
|
|
919
|
+
|
|
920
|
+
def sleep(self, seconds):
|
|
921
|
+
self.t += max(seconds, 0.001)
|
|
922
|
+
if self.t > self.limit:
|
|
923
|
+
raise RuntimeError("fake clock ran away")
|
|
924
|
+
for hook in sorted(self.hooks, key=lambda h: h[0]):
|
|
925
|
+
if self.t >= hook[0]:
|
|
926
|
+
self.hooks.remove(hook)
|
|
927
|
+
hook[1]()
|
|
928
|
+
|
|
929
|
+
def alive(self, pid):
|
|
930
|
+
return pid in self.live
|
|
931
|
+
|
|
932
|
+
def gone(self, pid):
|
|
933
|
+
return pid not in self.live
|
|
934
|
+
|
|
935
|
+
def exit(self, pid):
|
|
936
|
+
self.live.discard(pid)
|
|
937
|
+
if pid == self.pid:
|
|
938
|
+
self.pane_cmd = "zsh"
|
|
939
|
+
|
|
940
|
+
def signal(self, pid, sig):
|
|
941
|
+
self.signals.append((pid, sig))
|
|
942
|
+
self.calls.append({"argv": ["<signal>", pid, sig], "env": None, "cwd": None,
|
|
943
|
+
"t": self.t})
|
|
944
|
+
if pid not in self.live:
|
|
945
|
+
return False
|
|
946
|
+
if pid not in self.stuck and (
|
|
947
|
+
sig == signal.SIGKILL or (sig == signal.SIGTERM and self.term_kills)):
|
|
948
|
+
self.exit(pid)
|
|
949
|
+
return True
|
|
950
|
+
|
|
951
|
+
def pgid(self, pid):
|
|
952
|
+
return self.pgids.get(pid, pid) if pid in self.live else None
|
|
953
|
+
|
|
954
|
+
def lstart(self, pid):
|
|
955
|
+
return self.starts.get(pid) if pid in self.live else None
|
|
956
|
+
|
|
957
|
+
def ps_table(self):
|
|
958
|
+
return [row for row in self.table if row[0] in self.live]
|
|
959
|
+
|
|
960
|
+
def open_files(self, pid):
|
|
961
|
+
return None if self.lookup_fails else list(self.files.get(pid, ()))
|
|
962
|
+
|
|
963
|
+
def can_list_open_files(self):
|
|
964
|
+
return self.open_files_ok
|
|
965
|
+
|
|
966
|
+
def proc_status(self, pid):
|
|
967
|
+
if pid == self.ppid and self.shell_comm is not None:
|
|
968
|
+
# the shell holds the terminal again once its foreground job is gone
|
|
969
|
+
return ("Ss" if self.pid in self.live else "Ss+", self.tty, self.shell_comm)
|
|
970
|
+
if pid == self.pid and pid in self.live:
|
|
971
|
+
return (self.client_stat, self.tty, "claude")
|
|
972
|
+
return None
|
|
973
|
+
|
|
974
|
+
def consume_token(self):
|
|
975
|
+
"""Enter at the shell: the relaunched shim reads its relaunch files and removes
|
|
976
|
+
them (single use), first thing. Kept here for the assertions."""
|
|
977
|
+
if not self.consume or not self.ar_dir or "=" not in self.line:
|
|
978
|
+
return
|
|
979
|
+
tok = self.line.split("=", 1)[1].split()[0].split(":")[0]
|
|
980
|
+
base = os.path.join(str(self.ar_dir), "r-" + tok)
|
|
981
|
+
try:
|
|
982
|
+
with open(base + ".relaunch") as fh:
|
|
983
|
+
text = fh.read()
|
|
984
|
+
with open(base + ".argv", "rb") as fh:
|
|
985
|
+
data = fh.read()
|
|
986
|
+
except OSError:
|
|
987
|
+
return
|
|
988
|
+
self.consumed[tok] = (text, data)
|
|
989
|
+
ar.remove_quietly(base + ".relaunch", base + ".argv")
|
|
990
|
+
|
|
991
|
+
def run(self, argv, timeout, env=None, cwd=None):
|
|
992
|
+
self.calls.append({"argv": list(argv), "env": env, "cwd": cwd, "t": self.t})
|
|
993
|
+
if argv[0] == "tmux":
|
|
994
|
+
if "display-message" in argv and "-p" in argv:
|
|
995
|
+
return 0, "%d|%d|%d|%s\n" % (self.pane_pid, self.pane_mode, self.pane_sync,
|
|
996
|
+
self.pane_cmd)
|
|
997
|
+
if "send-keys" in argv and "-X" in argv:
|
|
998
|
+
if argv[-1] == "cancel" and not self.mode_sticky:
|
|
999
|
+
self.pane_mode = 0
|
|
1000
|
+
return 0, ""
|
|
1001
|
+
if "send-keys" in argv and "-l" in argv:
|
|
1002
|
+
self.line = argv[-1]
|
|
1003
|
+
elif "send-keys" in argv and argv[-1] == "Enter":
|
|
1004
|
+
self.consume_token()
|
|
1005
|
+
if "display-message" in argv and "-d" in argv and self.old_tmux:
|
|
1006
|
+
return 1, ""
|
|
1007
|
+
return 0, ""
|
|
1008
|
+
if argv[0] == self.self_path:
|
|
1009
|
+
return (3 if "tier=none" in self.probe_out else 0), self.probe_out
|
|
1010
|
+
return 1, ""
|
|
1011
|
+
|
|
1012
|
+
@property
|
|
1013
|
+
def probes(self):
|
|
1014
|
+
return [c for c in self.calls if c["argv"][0] == self.self_path]
|
|
1015
|
+
|
|
1016
|
+
@property
|
|
1017
|
+
def sends(self):
|
|
1018
|
+
"""What was keyed into the pane (a -X mode command is not a key)."""
|
|
1019
|
+
return [c["argv"] for c in self.calls if c["argv"][0] == "tmux"
|
|
1020
|
+
and "send-keys" in c["argv"] and "-X" not in c["argv"]]
|
|
1021
|
+
|
|
1022
|
+
@property
|
|
1023
|
+
def notices(self):
|
|
1024
|
+
return [c["argv"] for c in self.calls if c["argv"][0] == "tmux"
|
|
1025
|
+
and "display-message" in c["argv"] and "-p" not in c["argv"]]
|
|
1026
|
+
|
|
1027
|
+
def index(self, pred):
|
|
1028
|
+
return next(i for i, c in enumerate(self.calls) if pred(c["argv"]))
|
|
1029
|
+
|
|
1030
|
+
|
|
1031
|
+
class PoolCase(unittest.TestCase):
|
|
1032
|
+
PID = 4242
|
|
1033
|
+
PPID = 777
|
|
1034
|
+
|
|
1035
|
+
def setUp(self):
|
|
1036
|
+
self._tmp = tempfile.TemporaryDirectory()
|
|
1037
|
+
self.tmp = Path(self._tmp.name).resolve()
|
|
1038
|
+
self.root = self.tmp / "pool"
|
|
1039
|
+
self.ar_dir = self.root / "tmp" / "autoresume"
|
|
1040
|
+
self.ar_dir.mkdir(parents=True)
|
|
1041
|
+
self.work = self.tmp / "work dir"
|
|
1042
|
+
self.work.mkdir()
|
|
1043
|
+
self.shared = self.tmp / "shared"
|
|
1044
|
+
for acct in ("acct-01", "acct-02"):
|
|
1045
|
+
d = self.root / acct
|
|
1046
|
+
(d / "sessions").mkdir(parents=True)
|
|
1047
|
+
self.proj = self.shared / "projects" / "-work-dir"
|
|
1048
|
+
self.proj.mkdir(parents=True)
|
|
1049
|
+
for acct in ("acct-01", "acct-02"):
|
|
1050
|
+
os.symlink(str(self.shared / "projects"), str(self.root / acct / "projects"))
|
|
1051
|
+
self.self_path = str(self.tmp / "bin" / "claude")
|
|
1052
|
+
self.t0 = float(int(time.time()))
|
|
1053
|
+
self.launched = int(self.t0) - 5
|
|
1054
|
+
|
|
1055
|
+
def tearDown(self):
|
|
1056
|
+
self._tmp.cleanup()
|
|
1057
|
+
|
|
1058
|
+
# -- fixtures --
|
|
1059
|
+
|
|
1060
|
+
def write_state(self, argv, provider="claude", depth=0, avoid="", hist="", chain="",
|
|
1061
|
+
launched=None, pid=None, acc_root=None):
|
|
1062
|
+
pid = pid or self.PID
|
|
1063
|
+
kv = {"v": "1", "provider": provider, "pid": pid, "ppid": self.PPID,
|
|
1064
|
+
"acct": "acct-01", "acct_dir": str(self.root / "acct-01"), "cwd": str(self.work),
|
|
1065
|
+
"launched": self.launched if launched is None else launched,
|
|
1066
|
+
"tmux": "/private/tmp/tmux-501/default,4211,0", "pane": "%7",
|
|
1067
|
+
"self": self.self_path, "acc_root": str(acc_root or self.root), "depth": depth,
|
|
1068
|
+
"avoid": avoid, "hist": hist, "chain": chain}
|
|
1069
|
+
path = self.ar_dir / ("%d.state" % pid)
|
|
1070
|
+
path.write_text("".join("%s=%s\n" % i for i in kv.items()))
|
|
1071
|
+
(self.ar_dir / ("%d.argv" % pid)).write_bytes(ar.encode_argv(argv))
|
|
1072
|
+
return str(path)
|
|
1073
|
+
|
|
1074
|
+
def registry(self, sid=SID, pid=None):
|
|
1075
|
+
pid = pid or self.PID
|
|
1076
|
+
doc = {"pid": pid, "sessionId": sid, "cwd": str(self.work),
|
|
1077
|
+
"startedAt": int(self.launched * 1000 + 700), "version": "2.1.280",
|
|
1078
|
+
"kind": "interactive", "entrypoint": "cli", "status": "idle"}
|
|
1079
|
+
path = self.root / "acct-01" / "sessions" / ("%d.json" % pid)
|
|
1080
|
+
path.write_text(json.dumps(doc))
|
|
1081
|
+
return path
|
|
1082
|
+
|
|
1083
|
+
def transcript(self, records, sid=SID):
|
|
1084
|
+
path = self.proj / (sid + ".jsonl")
|
|
1085
|
+
with open(path, "a") as fh:
|
|
1086
|
+
fh.write(jl(records))
|
|
1087
|
+
return path
|
|
1088
|
+
|
|
1089
|
+
def watcher(self, state, env=None, runner=None):
|
|
1090
|
+
environ = {"PATH": "/usr/bin:/bin", "CLAUDE_CONFIG_DIR": str(self.root / "acct-01"),
|
|
1091
|
+
"CLAUDE_SHIM_ACTIVE": "1", "CODEX_HOME": str(self.root / "acct-01")}
|
|
1092
|
+
environ.update(env or {})
|
|
1093
|
+
runner = runner or FakeRunner(self.t0, self.PID, self.PPID, self.self_path)
|
|
1094
|
+
runner.ar_dir = self.ar_dir
|
|
1095
|
+
w = ar.Watcher(state, ar.load_state(state), ar.load_argv(ar.argv_path_for(state)),
|
|
1096
|
+
runner=runner, environ=environ)
|
|
1097
|
+
return w, runner
|
|
1098
|
+
|
|
1099
|
+
def log_lines(self):
|
|
1100
|
+
path = self.root / "selection.log"
|
|
1101
|
+
return path.read_text().splitlines() if path.exists() else []
|
|
1102
|
+
|
|
1103
|
+
def events(self):
|
|
1104
|
+
return [line.split()[2] for line in self.log_lines()]
|
|
1105
|
+
|
|
1106
|
+
def relaunch_files(self):
|
|
1107
|
+
return sorted(p.name for p in self.ar_dir.glob("r-*"))
|
|
1108
|
+
|
|
1109
|
+
def token_of(self, runner):
|
|
1110
|
+
typed = [a[-1] for a in runner.sends if "-l" in a]
|
|
1111
|
+
self.assertEqual(len(typed), 1)
|
|
1112
|
+
return typed[0].split("=", 1)[1].split()[0].split(":")[0]
|
|
1113
|
+
|
|
1114
|
+
def relaunch_of(self, runner):
|
|
1115
|
+
"""The relaunch files as the relaunched shim consumed them (the fake shell runs
|
|
1116
|
+
the typed line at Enter, and the token is single use)."""
|
|
1117
|
+
tok = self.token_of(runner)
|
|
1118
|
+
self.assertEqual(self.relaunch_files(), [])
|
|
1119
|
+
text, data = runner.consumed[tok]
|
|
1120
|
+
return ar.parse_kv(text, ar.RELAUNCH_KEYS), ar.decode_argv(data)
|
|
1121
|
+
|
|
1122
|
+
def assert_sequence(self, runner, provider="claude", sid=None, root="pool"):
|
|
1123
|
+
"""The four send-keys calls. The suites' pool is not $HOME/.<p>-accounts, so the
|
|
1124
|
+
typed line names it (root=None: a default pool, nothing named)."""
|
|
1125
|
+
tok = self.token_of(runner)
|
|
1126
|
+
sid = sid or (SID if provider == "claude" else CX_SID)
|
|
1127
|
+
self.assertEqual(runner.sends, ar.send_keys_argvs(
|
|
1128
|
+
"/private/tmp/tmux-501/default", "%7", provider, tok, sid, self.self_path,
|
|
1129
|
+
str(self.root) if root == "pool" else root))
|
|
1130
|
+
|
|
1131
|
+
|
|
1132
|
+
class ClaudeWatcherTests(PoolCase):
|
|
1133
|
+
def test_quota_rotates_to_another_account(self):
|
|
1134
|
+
reset = int(self.t0) + 7200
|
|
1135
|
+
state = self.write_state(["--dangerously-skip-permissions", "fix the flaky test"])
|
|
1136
|
+
self.registry()
|
|
1137
|
+
self.transcript([cc_user(self.launched - 4000, "old prompt"),
|
|
1138
|
+
cc_quota(self.launched - 3600, resets=self.launched - 100),
|
|
1139
|
+
cc_quota(self.t0 - 1, resets=reset), cc_system(self.t0 - 1)])
|
|
1140
|
+
w, r = self.watcher(state)
|
|
1141
|
+
self.assertEqual(w.run(), 0)
|
|
1142
|
+
prompt = ar.resume_prompt("quota")
|
|
1143
|
+
# probe: selection from scratch, the old account avoided until its reset
|
|
1144
|
+
self.assertEqual(len(r.probes), 1)
|
|
1145
|
+
probe = r.probes[0]
|
|
1146
|
+
self.assertGreaterEqual(probe["t"] - self.t0, 5.0) # GRACE
|
|
1147
|
+
self.assertEqual(probe["argv"], [self.self_path, "--dangerously-skip-permissions",
|
|
1148
|
+
"--resume", SID, prompt])
|
|
1149
|
+
self.assertEqual(probe["cwd"], str(self.work))
|
|
1150
|
+
self.assertEqual(probe["env"]["CLAUDE_MULTIACC_AR_PROBE"], "1")
|
|
1151
|
+
self.assertEqual(probe["env"]["CLAUDE_MULTIACC_AR_AVOID"], "acct-01:%d" % reset)
|
|
1152
|
+
self.assertNotIn("CLAUDE_CONFIG_DIR", probe["env"])
|
|
1153
|
+
self.assertNotIn("CLAUDE_SHIM_ACTIVE", probe["env"])
|
|
1154
|
+
# stop, then type into the shell: token + session id, the pool, the shim itself
|
|
1155
|
+
self.assertEqual(r.signals, [(self.PID, signal.SIGTERM)])
|
|
1156
|
+
self.assert_sequence(r)
|
|
1157
|
+
self.assertEqual([a[-1] for a in r.sends if "-l" in a],
|
|
1158
|
+
[" CLAUDE_MULTIACC_AR=%s:%s CLAUDE_ACCOUNTS_ROOT=%s %s"
|
|
1159
|
+
% (self.token_of(r), SID, self.root, self.self_path)])
|
|
1160
|
+
kv, argv = self.relaunch_of(r)
|
|
1161
|
+
self.assertEqual(argv, ["--dangerously-skip-permissions", "--resume", SID, prompt])
|
|
1162
|
+
self.assertEqual({k: kv[k] for k in ("v", "provider", "class", "acct", "reset", "rtype",
|
|
1163
|
+
"marked", "sid", "cwd", "depth", "avoid")},
|
|
1164
|
+
{"v": "1", "provider": "claude", "class": "quota", "acct": "acct-01",
|
|
1165
|
+
"reset": str(reset), "rtype": "five_hour",
|
|
1166
|
+
"marked": ar.utc_iso(self.t0 - 1), "sid": SID, "cwd": str(self.work),
|
|
1167
|
+
"depth": "1", "avoid": "acct-01:%d" % reset})
|
|
1168
|
+
self.assertRegex(kv["hist"], r"^quota:\d+$")
|
|
1169
|
+
self.assertRegex(kv["chain"], r"^[A-Za-z0-9]{8}$")
|
|
1170
|
+
# log grammar: field 2 is always the word autoresume
|
|
1171
|
+
self.assertEqual(self.events(), ["watch", "detect", "switch"])
|
|
1172
|
+
for line in self.log_lines():
|
|
1173
|
+
self.assertEqual(line.split()[1], "autoresume")
|
|
1174
|
+
self.assertNotIn("hit your limit", line)
|
|
1175
|
+
self.assertIn("from=acct-01 class=quota sid=%s depth=1 probe=acct-02" % SID,
|
|
1176
|
+
self.log_lines()[-1])
|
|
1177
|
+
|
|
1178
|
+
def test_chain_state_is_carried(self):
|
|
1179
|
+
far = int(self.t0) + 50000
|
|
1180
|
+
state = self.write_state(["-c"], depth=4, avoid="acct-05:%d,acct-06:1" % far,
|
|
1181
|
+
hist="quota:%d" % (self.t0 - 7200), chain="ChainId1")
|
|
1182
|
+
self.registry()
|
|
1183
|
+
self.transcript([cc_api_error(self.t0, "authentication_failed",
|
|
1184
|
+
"Not logged in · Please run /login")])
|
|
1185
|
+
w, r = self.watcher(state)
|
|
1186
|
+
w.run()
|
|
1187
|
+
until = int(r.probes[0]["t"]) + 3600
|
|
1188
|
+
self.assertEqual(r.probes[0]["env"]["CLAUDE_MULTIACC_AR_AVOID"],
|
|
1189
|
+
"acct-01:%d,acct-05:%d" % (until, far))
|
|
1190
|
+
kv, argv = self.relaunch_of(r)
|
|
1191
|
+
self.assertEqual(argv, ["--resume", SID, ar.resume_prompt("auth")])
|
|
1192
|
+
self.assertEqual((kv["class"], kv["reset"], kv["rtype"], kv["depth"], kv["chain"]),
|
|
1193
|
+
("auth", "", "", "5", "ChainId1"))
|
|
1194
|
+
self.assertTrue(kv["hist"].startswith("quota:%d,auth:" % (self.t0 - 7200)))
|
|
1195
|
+
self.assertIn("chain=ChainId1", self.log_lines()[0])
|
|
1196
|
+
|
|
1197
|
+
def test_old_rejection_in_a_resumed_transcript_does_not_fire(self):
|
|
1198
|
+
state = self.write_state(["--resume", SID])
|
|
1199
|
+
self.registry()
|
|
1200
|
+
self.transcript([cc_quota(self.launched - 3, resets=int(self.t0) + 999),
|
|
1201
|
+
cc_system(self.launched - 3)])
|
|
1202
|
+
w, r = self.watcher(state)
|
|
1203
|
+
reg = self.root / "acct-01" / "sessions" / ("%d.json" % self.PID)
|
|
1204
|
+
r.at(self.t0 + 20, lambda: (reg.unlink(), r.exit(self.PID)))
|
|
1205
|
+
self.assertEqual(w.run(), 0)
|
|
1206
|
+
self.assertEqual((r.probes, r.signals, r.sends), ([], [], []))
|
|
1207
|
+
self.assertEqual(self.events(), ["watch"])
|
|
1208
|
+
|
|
1209
|
+
def test_no_alternative_holds_and_reprobes(self):
|
|
1210
|
+
state = self.write_state([])
|
|
1211
|
+
self.registry()
|
|
1212
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1213
|
+
w, r = self.watcher(state, {"CLAUDE_MULTIACC_AR_HOLD_REPROBE": "5"})
|
|
1214
|
+
r.probe_out = "pick=acct-01 tier=soft\n"
|
|
1215
|
+
reg = self.root / "acct-01" / "sessions" / ("%d.json" % self.PID)
|
|
1216
|
+
r.at(self.t0 + 16.5, lambda: (reg.unlink(), r.exit(self.PID)))
|
|
1217
|
+
self.assertEqual(w.run(), 0)
|
|
1218
|
+
self.assertEqual(len(r.probes), 3) # ~5 s, ~10 s, ~15 s
|
|
1219
|
+
self.assertEqual((r.signals, r.sends, self.relaunch_files()), ([], [], []))
|
|
1220
|
+
self.assertEqual(self.events(), ["watch", "detect", "hold"])
|
|
1221
|
+
self.assertIn("reason=no-room pick=acct-01 tier=soft", self.log_lines()[-1])
|
|
1222
|
+
|
|
1223
|
+
def test_nobody_left_holds(self):
|
|
1224
|
+
state = self.write_state([])
|
|
1225
|
+
self.registry()
|
|
1226
|
+
self.transcript([cc_api_error(self.t0, "account_on_hold")])
|
|
1227
|
+
w, r = self.watcher(state)
|
|
1228
|
+
r.probe_out = "pick= tier=none\n"
|
|
1229
|
+
r.at(self.t0 + 10, lambda: r.exit(self.PID))
|
|
1230
|
+
w.st["launched"] = w.launched = int(self.t0) - 5 # too young to be a crash
|
|
1231
|
+
self.assertEqual(w.run(), 0)
|
|
1232
|
+
self.assertEqual(r.signals, [])
|
|
1233
|
+
self.assertIn("hold", self.events())
|
|
1234
|
+
|
|
1235
|
+
def test_a_cancel_ends_a_hold_and_nothing_is_stopped(self):
|
|
1236
|
+
state = self.write_state([])
|
|
1237
|
+
self.registry()
|
|
1238
|
+
tpath = self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1239
|
+
w, r = self.watcher(state)
|
|
1240
|
+
# the user types before GRACE runs out
|
|
1241
|
+
r.at(self.t0 + 1.5, lambda: append(tpath, [cc_user(self.t0 + 1, "never mind")]))
|
|
1242
|
+
r.at(self.t0 + 9, lambda: r.exit(self.PID))
|
|
1243
|
+
w.run()
|
|
1244
|
+
self.assertEqual((r.probes, r.signals), ([], []))
|
|
1245
|
+
|
|
1246
|
+
def test_input_that_arrives_during_the_probe_wins(self):
|
|
1247
|
+
state = self.write_state([])
|
|
1248
|
+
self.registry()
|
|
1249
|
+
tpath = self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1250
|
+
w, r = self.watcher(state)
|
|
1251
|
+
real_run = r.run
|
|
1252
|
+
|
|
1253
|
+
def run(argv, timeout, env=None, cwd=None):
|
|
1254
|
+
if argv[0] == self.self_path and not r.probes:
|
|
1255
|
+
append(tpath, [cc_user(self.t0 + 3, "actually, let me handle this")])
|
|
1256
|
+
return real_run(argv, timeout, env, cwd)
|
|
1257
|
+
r.run = run
|
|
1258
|
+
r.at(self.t0 + 20, lambda: r.exit(self.PID))
|
|
1259
|
+
w.run()
|
|
1260
|
+
self.assertEqual(len(r.probes), 1)
|
|
1261
|
+
self.assertEqual((r.signals, r.sends, self.relaunch_files()), ([], [], []))
|
|
1262
|
+
|
|
1263
|
+
def test_kill_switch_file_reaches_a_running_watcher(self):
|
|
1264
|
+
state = self.write_state([])
|
|
1265
|
+
self.registry()
|
|
1266
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1267
|
+
w, r = self.watcher(state)
|
|
1268
|
+
r.at(self.t0 + 1, lambda: (self.root / "autoresume.off").write_text(""))
|
|
1269
|
+
self.assertEqual(w.run(), 0)
|
|
1270
|
+
self.assertEqual((r.probes, r.signals, r.sends), ([], [], []))
|
|
1271
|
+
|
|
1272
|
+
def test_kill_switch_file_before_start(self):
|
|
1273
|
+
(self.root / "autoresume.off").write_text("")
|
|
1274
|
+
state = self.write_state([])
|
|
1275
|
+
self.registry()
|
|
1276
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1277
|
+
w, r = self.watcher(state)
|
|
1278
|
+
self.assertEqual(w.run(), 0)
|
|
1279
|
+
self.assertEqual((r.probes, r.signals), ([], []))
|
|
1280
|
+
|
|
1281
|
+
def test_invalid_request_is_logged_once_and_left_alone(self):
|
|
1282
|
+
state = self.write_state([])
|
|
1283
|
+
self.registry()
|
|
1284
|
+
self.transcript([cc_api_error(self.t0, "invalid_request", "Autocompact is thrashing"),
|
|
1285
|
+
cc_api_error(self.t0, "invalid_request", "again")])
|
|
1286
|
+
w, r = self.watcher(state)
|
|
1287
|
+
r.at(self.t0 + 8, lambda: r.exit(self.PID))
|
|
1288
|
+
w.run()
|
|
1289
|
+
self.assertEqual((r.probes, r.signals), ([], []))
|
|
1290
|
+
self.assertEqual(self.events(), ["watch", "never"])
|
|
1291
|
+
self.assertIn("code=invalid_request", self.log_lines()[1])
|
|
1292
|
+
|
|
1293
|
+
def test_crash_relaunches_through_normal_selection(self):
|
|
1294
|
+
state = self.write_state(["--dangerously-skip-permissions"],
|
|
1295
|
+
avoid="acct-05:%d" % (self.t0 + 900))
|
|
1296
|
+
self.registry()
|
|
1297
|
+
self.transcript([cc_reply(self.t0, "working on it")])
|
|
1298
|
+
w, r = self.watcher(state)
|
|
1299
|
+
r.at(self.t0 + 70, lambda: r.exit(self.PID)) # registry left behind: SIGKILL/crash
|
|
1300
|
+
self.assertEqual(w.run(), 0)
|
|
1301
|
+
self.assertEqual((r.probes, r.signals), ([], []))
|
|
1302
|
+
kv, argv = self.relaunch_of(r)
|
|
1303
|
+
self.assertEqual(argv, ["--dangerously-skip-permissions", "--resume", SID,
|
|
1304
|
+
ar.resume_prompt("crash")])
|
|
1305
|
+
self.assertEqual((kv["class"], kv["acct"], kv["avoid"], kv["depth"]),
|
|
1306
|
+
("crash", "acct-01", "acct-05:%d" % (self.t0 + 900), "1"))
|
|
1307
|
+
self.assert_sequence(r)
|
|
1308
|
+
self.assertEqual(self.events(), ["watch", "crash", "switch"])
|
|
1309
|
+
|
|
1310
|
+
def test_a_clean_exit_or_an_early_death_is_final(self):
|
|
1311
|
+
state = self.write_state([])
|
|
1312
|
+
reg = self.registry()
|
|
1313
|
+
w, r = self.watcher(state)
|
|
1314
|
+
r.at(self.t0 + 70, lambda: (reg.unlink(), r.exit(self.PID))) # /exit: registry gone
|
|
1315
|
+
w.run()
|
|
1316
|
+
self.assertEqual((r.sends, self.relaunch_files()), ([], []))
|
|
1317
|
+
state = self.write_state([])
|
|
1318
|
+
self.registry()
|
|
1319
|
+
w, r = self.watcher(state)
|
|
1320
|
+
r.at(self.t0 + 30, lambda: r.exit(self.PID)) # died after 35 s: not a crash yet
|
|
1321
|
+
w.run()
|
|
1322
|
+
self.assertEqual(r.sends, [])
|
|
1323
|
+
|
|
1324
|
+
def test_crash_budget(self):
|
|
1325
|
+
hist = "crash:%d,crash:%d" % (self.t0 - 60, self.t0 - 120)
|
|
1326
|
+
state = self.write_state([], hist=hist)
|
|
1327
|
+
self.registry()
|
|
1328
|
+
w, r = self.watcher(state)
|
|
1329
|
+
r.at(self.t0 + 70, lambda: r.exit(self.PID))
|
|
1330
|
+
w.run()
|
|
1331
|
+
self.assertEqual(r.sends, [])
|
|
1332
|
+
self.assertIn("reason=crash-budget", self.log_lines()[-1])
|
|
1333
|
+
|
|
1334
|
+
def test_pane_must_be_the_launching_shell(self):
|
|
1335
|
+
state = self.write_state([])
|
|
1336
|
+
self.registry()
|
|
1337
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1338
|
+
w, r = self.watcher(state)
|
|
1339
|
+
r.pane_pid = 31337
|
|
1340
|
+
self.assertEqual(w.run(), 0)
|
|
1341
|
+
self.assertEqual(len(r.probes), 1)
|
|
1342
|
+
self.assertEqual((r.signals, r.sends, self.relaunch_files()), ([], [], []))
|
|
1343
|
+
self.assertIn("reason=pane", self.log_lines()[-1])
|
|
1344
|
+
# the test knob skips only the pid equality
|
|
1345
|
+
state = self.write_state([])
|
|
1346
|
+
w, r = self.watcher(state, {"CLAUDE_MULTIACC_AR_TEST_TMUX_ANYPANE": "1"})
|
|
1347
|
+
r.pane_pid = 31337
|
|
1348
|
+
w.run()
|
|
1349
|
+
self.assertEqual(r.signals, [(self.PID, signal.SIGTERM)])
|
|
1350
|
+
self.assertEqual(len(r.sends), 4)
|
|
1351
|
+
|
|
1352
|
+
def test_depth_cap_gives_up_without_probing(self):
|
|
1353
|
+
state = self.write_state([], depth=20)
|
|
1354
|
+
self.registry()
|
|
1355
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1356
|
+
w, r = self.watcher(state)
|
|
1357
|
+
self.assertEqual(w.run(), 0)
|
|
1358
|
+
self.assertEqual((r.probes, r.signals), ([], []))
|
|
1359
|
+
self.assertIn("reason=depth", self.log_lines()[-1])
|
|
1360
|
+
|
|
1361
|
+
def test_rotate_budget_holds_without_probing(self):
|
|
1362
|
+
hist = ",".join("quota:%d" % (self.t0 - 60 * i) for i in range(1, 9))
|
|
1363
|
+
state = self.write_state([], hist=hist)
|
|
1364
|
+
self.registry()
|
|
1365
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1366
|
+
w, r = self.watcher(state)
|
|
1367
|
+
r.at(self.t0 + 10, lambda: r.exit(self.PID))
|
|
1368
|
+
w.run()
|
|
1369
|
+
self.assertEqual((r.probes, r.signals), ([], []))
|
|
1370
|
+
self.assertIn("reason=rotate-budget", self.log_lines()[-1])
|
|
1371
|
+
|
|
1372
|
+
def test_transient_waits_its_ladder_step_and_may_stay_put(self):
|
|
1373
|
+
far = int(self.t0) + 9000
|
|
1374
|
+
state = self.write_state(["--model", "opus"], avoid="acct-09:%d" % far,
|
|
1375
|
+
hist="transient:%d" % (self.t0 - 100))
|
|
1376
|
+
self.registry()
|
|
1377
|
+
self.transcript([cc_api_error(self.t0, "server_error",
|
|
1378
|
+
"API Error: Connection lost mid-response.")])
|
|
1379
|
+
w, r = self.watcher(state)
|
|
1380
|
+
r.probe_out = "pick=acct-01 tier=soft\n"
|
|
1381
|
+
w.run()
|
|
1382
|
+
self.assertGreaterEqual(r.probes[0]["t"] - self.t0, 60.0) # second ladder step
|
|
1383
|
+
self.assertEqual(r.probes[0]["env"]["CLAUDE_MULTIACC_AR_AVOID"], "acct-09:%d" % far)
|
|
1384
|
+
kv, argv = self.relaunch_of(r)
|
|
1385
|
+
self.assertEqual(argv, ["--model", "opus", "--resume", SID,
|
|
1386
|
+
ar.resume_prompt("transient")])
|
|
1387
|
+
self.assertEqual((kv["class"], kv["avoid"]), ("transient", "acct-09:%d" % far))
|
|
1388
|
+
|
|
1389
|
+
def test_model_limit_waits_for_busy_subagents_and_adds_no_model(self):
|
|
1390
|
+
# settings.json names a model, but --resume restores the session's own
|
|
1391
|
+
(self.root / "acct-01" / "settings.json").write_text(json.dumps({"model": "opus[1m]"}))
|
|
1392
|
+
sub = self.proj / SID / "subagents" / "workflows" / "w1"
|
|
1393
|
+
sub.mkdir(parents=True)
|
|
1394
|
+
busy = sub / "agent-1.jsonl"
|
|
1395
|
+
busy.write_text("{}\n")
|
|
1396
|
+
os.utime(str(busy), (self.t0 + 100, self.t0 + 100)) # still writing until ~t0+100
|
|
1397
|
+
state = self.write_state(["open the report please"])
|
|
1398
|
+
self.registry()
|
|
1399
|
+
self.transcript([cc_api_error(self.t0, "rate_limit",
|
|
1400
|
+
"You've reached your Fable limit. Switch to another model")])
|
|
1401
|
+
w, r = self.watcher(state)
|
|
1402
|
+
w.run()
|
|
1403
|
+
self.assertGreaterEqual(r.probes[0]["t"], self.t0 + 120) # 20 s of subagent quiet
|
|
1404
|
+
until = int(r.probes[0]["t"]) + 5 * 3600
|
|
1405
|
+
self.assertEqual(r.probes[0]["env"]["CLAUDE_MULTIACC_AR_AVOID"], "acct-01:%d" % until)
|
|
1406
|
+
kv, argv = self.relaunch_of(r)
|
|
1407
|
+
self.assertEqual(argv, ["--resume", SID, ar.resume_prompt("model")])
|
|
1408
|
+
self.assertEqual(kv["class"], "model")
|
|
1409
|
+
|
|
1410
|
+
def test_session_switch_drops_the_pending_verdict(self):
|
|
1411
|
+
state = self.write_state([])
|
|
1412
|
+
reg = self.registry()
|
|
1413
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1414
|
+
w, r = self.watcher(state)
|
|
1415
|
+
# /clear before GRACE: a new conversation, nothing wrong with it
|
|
1416
|
+
r.at(self.t0 + 1.5, lambda: self.registry(sid=SID2))
|
|
1417
|
+
r.at(self.t0 + 10, lambda: (reg.unlink(), r.exit(self.PID)))
|
|
1418
|
+
w.run()
|
|
1419
|
+
self.assertEqual((r.probes, r.signals), ([], []))
|
|
1420
|
+
self.assertEqual(w.sid, SID2)
|
|
1421
|
+
|
|
1422
|
+
def test_stale_registry_of_a_reused_pid_is_ignored(self):
|
|
1423
|
+
state = self.write_state([])
|
|
1424
|
+
path = self.registry()
|
|
1425
|
+
doc = json.loads(path.read_text())
|
|
1426
|
+
doc["startedAt"] = (self.launched - 86400) * 1000
|
|
1427
|
+
path.write_text(json.dumps(doc))
|
|
1428
|
+
w, _ = self.watcher(state)
|
|
1429
|
+
self.assertIsNone(w.registry())
|
|
1430
|
+
|
|
1431
|
+
REFRESH = 5002 # the shim's detached `limits --quiet`: same group, reparented to init
|
|
1432
|
+
|
|
1433
|
+
def job(self, r):
|
|
1434
|
+
"""The client's job as an interactive shell leaves it: the client leads the group;
|
|
1435
|
+
5001 is its child (a tool or MCP process); REFRESH shares the group but is not the
|
|
1436
|
+
client's — the shim started it in the background before its exec."""
|
|
1437
|
+
r.table = [(self.PID, self.PPID, LSTART), (5001, self.PID, "d"), (self.REFRESH, 1, "x")]
|
|
1438
|
+
r.live |= {5001, self.REFRESH}
|
|
1439
|
+
r.starts.update({5001: "d", self.REFRESH: "x"})
|
|
1440
|
+
r.pgids.update({5001: self.PID, self.REFRESH: self.PID})
|
|
1441
|
+
|
|
1442
|
+
def test_a_tree_read_naming_foreign_processes_never_signals_them(self):
|
|
1443
|
+
# 2026-09-23: a deliberately broken copy of descendants() under test named every
|
|
1444
|
+
# process on the machine as the client's child, and the stop SIGKILLed them. The
|
|
1445
|
+
# stop itself must refuse anything outside the client's own process group, and
|
|
1446
|
+
# never touch the launching shell, whatever the tree read says.
|
|
1447
|
+
state = self.write_state([])
|
|
1448
|
+
self.registry()
|
|
1449
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1450
|
+
w, r = self.watcher(state, {"CLAUDE_MULTIACC_AR_TERM_GRACE": "2"})
|
|
1451
|
+
self.job(r)
|
|
1452
|
+
foreign = [6001, 6002, 6003]
|
|
1453
|
+
r.table += [(p, self.PID, "f%d" % p) for p in foreign] + [(self.PPID, self.PID, "sh")]
|
|
1454
|
+
r.live |= set(foreign) | {self.PPID}
|
|
1455
|
+
r.starts.update({p: "f%d" % p for p in foreign})
|
|
1456
|
+
r.starts[self.PPID] = "sh"
|
|
1457
|
+
r.pgids.update({p: p for p in foreign})
|
|
1458
|
+
r.pgids[self.PPID] = self.PID # even sharing the group, the shell is never a target
|
|
1459
|
+
r.term_kills = False
|
|
1460
|
+
w.run()
|
|
1461
|
+
hit = {pid for pid, _sig in r.signals}
|
|
1462
|
+
self.assertEqual(hit, {self.PID, 5001})
|
|
1463
|
+
self.assertTrue(set(foreign) | {self.PPID} <= r.live)
|
|
1464
|
+
|
|
1465
|
+
def test_an_implausibly_large_tree_is_not_trusted(self):
|
|
1466
|
+
state = self.write_state([])
|
|
1467
|
+
self.registry()
|
|
1468
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1469
|
+
w, r = self.watcher(state, {"CLAUDE_MULTIACC_AR_TERM_GRACE": "2"})
|
|
1470
|
+
self.job(r)
|
|
1471
|
+
many = list(range(7000, 7000 + 70))
|
|
1472
|
+
r.table += [(p, self.PID, "m") for p in many]
|
|
1473
|
+
r.live |= set(many)
|
|
1474
|
+
r.starts.update({p: "m" for p in many})
|
|
1475
|
+
r.pgids.update({p: self.PID for p in many})
|
|
1476
|
+
r.term_kills = False
|
|
1477
|
+
w.run()
|
|
1478
|
+
self.assertEqual([pid for pid, _sig in r.signals], [self.PID, self.PID])
|
|
1479
|
+
self.assertTrue(set(many) <= r.live)
|
|
1480
|
+
|
|
1481
|
+
def test_term_refused_kills_the_client_and_its_descendants_never_its_group(self):
|
|
1482
|
+
# Review 2026-09-23: the group-wide wait and killpg took the shim's background
|
|
1483
|
+
# limits refresh along (leaving its lock behind), and the refresh made every
|
|
1484
|
+
# relaunch wait out the whole TERM_GRACE.
|
|
1485
|
+
state = self.write_state([])
|
|
1486
|
+
self.registry()
|
|
1487
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1488
|
+
w, r = self.watcher(state, {"CLAUDE_MULTIACC_AR_TERM_GRACE": "2"})
|
|
1489
|
+
self.job(r)
|
|
1490
|
+
r.term_kills = False
|
|
1491
|
+
w.run()
|
|
1492
|
+
self.assertEqual(r.signals, [(self.PID, signal.SIGTERM), (self.PID, signal.SIGKILL),
|
|
1493
|
+
(5001, signal.SIGKILL)])
|
|
1494
|
+
self.assertEqual(r.live, {self.REFRESH})
|
|
1495
|
+
self.assertEqual(len(r.sends), 4)
|
|
1496
|
+
|
|
1497
|
+
def test_a_process_of_the_group_that_is_not_the_clients_is_never_waited_for(self):
|
|
1498
|
+
state = self.write_state([])
|
|
1499
|
+
self.registry()
|
|
1500
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1501
|
+
w, r = self.watcher(state) # TERM_GRACE 10 s
|
|
1502
|
+
self.job(r)
|
|
1503
|
+
real_exit = r.exit
|
|
1504
|
+
|
|
1505
|
+
def exit_with_its_child(pid):
|
|
1506
|
+
real_exit(pid)
|
|
1507
|
+
if pid == self.PID:
|
|
1508
|
+
r.live.discard(5001)
|
|
1509
|
+
r.exit = exit_with_its_child
|
|
1510
|
+
w.run()
|
|
1511
|
+
self.assertEqual(r.signals, [(self.PID, signal.SIGTERM)])
|
|
1512
|
+
self.assertIn(self.REFRESH, r.live)
|
|
1513
|
+
term = r.index(lambda a: a[0] == "<signal>")
|
|
1514
|
+
typed = r.index(lambda a: a[0] == "tmux" and "-l" in a)
|
|
1515
|
+
self.assertLess(r.calls[typed]["t"] - r.calls[term]["t"], 2)
|
|
1516
|
+
self.assert_sequence(r)
|
|
1517
|
+
|
|
1518
|
+
def test_a_descendant_outliving_the_client_is_killed_after_the_grace(self):
|
|
1519
|
+
state = self.write_state([])
|
|
1520
|
+
self.registry()
|
|
1521
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1522
|
+
w, r = self.watcher(state, {"CLAUDE_MULTIACC_AR_TERM_GRACE": "2"})
|
|
1523
|
+
self.job(r) # 5001 ignores the client's exit
|
|
1524
|
+
w.run()
|
|
1525
|
+
self.assertEqual(r.signals, [(self.PID, signal.SIGTERM), (5001, signal.SIGKILL)])
|
|
1526
|
+
term = r.index(lambda a: a[0] == "<signal>")
|
|
1527
|
+
kill = r.index(lambda a: a[:2] == ["<signal>", 5001])
|
|
1528
|
+
self.assertGreaterEqual(r.calls[kill]["t"], r.calls[term]["t"] + 2) # TERM_GRACE
|
|
1529
|
+
self.assertEqual(r.live, {self.REFRESH})
|
|
1530
|
+
self.assert_sequence(r)
|
|
1531
|
+
|
|
1532
|
+
def test_a_recorded_descendant_whose_pid_was_reused_is_left_alone(self):
|
|
1533
|
+
state = self.write_state([])
|
|
1534
|
+
self.registry()
|
|
1535
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1536
|
+
w, r = self.watcher(state, {"CLAUDE_MULTIACC_AR_TERM_GRACE": "2"})
|
|
1537
|
+
self.job(r)
|
|
1538
|
+
real_signal = r.signal
|
|
1539
|
+
|
|
1540
|
+
def signal_and_reuse(pid, sig):
|
|
1541
|
+
out = real_signal(pid, sig)
|
|
1542
|
+
if sig == signal.SIGTERM:
|
|
1543
|
+
r.starts[5001] = "someone else's" # 5001 died; its pid names another
|
|
1544
|
+
return out
|
|
1545
|
+
r.signal = signal_and_reuse
|
|
1546
|
+
w.run()
|
|
1547
|
+
self.assertEqual(r.signals, [(self.PID, signal.SIGTERM)])
|
|
1548
|
+
self.assertIn(5001, r.live)
|
|
1549
|
+
term = r.index(lambda a: a[0] == "<signal>")
|
|
1550
|
+
typed = r.index(lambda a: a[0] == "tmux" and "-l" in a)
|
|
1551
|
+
self.assertLess(r.calls[typed]["t"] - r.calls[term]["t"], 2)
|
|
1552
|
+
|
|
1553
|
+
def test_nothing_is_typed_while_the_stopped_tree_lives(self):
|
|
1554
|
+
state = self.write_state([])
|
|
1555
|
+
self.registry()
|
|
1556
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1557
|
+
w, r = self.watcher(state, {"CLAUDE_MULTIACC_AR_TERM_GRACE": "1",
|
|
1558
|
+
"CLAUDE_MULTIACC_AR_PANE_WAIT": "2"})
|
|
1559
|
+
self.job(r)
|
|
1560
|
+
r.stuck.add(5001) # not even SIGKILL ends it (uninterruptible sleep)
|
|
1561
|
+
w.run()
|
|
1562
|
+
self.assertEqual([a for a in r.sends if "-l" in a], [])
|
|
1563
|
+
self.assertEqual(self.relaunch_files(), [])
|
|
1564
|
+
line = self.log_lines()[-1]
|
|
1565
|
+
self.assertIn("reason=tree", line)
|
|
1566
|
+
self.assertIn('stopped=1 resume="claude --resume %s"' % SID, line)
|
|
1567
|
+
# the notice stays up until a key is pressed
|
|
1568
|
+
self.assertEqual(r.notices, [ar.notice_argv("/private/tmp/tmux-501/default", "%7",
|
|
1569
|
+
"claude", SID)])
|
|
1570
|
+
self.assertIn("-d", r.notices[0])
|
|
1571
|
+
|
|
1572
|
+
def test_a_client_that_does_not_lead_its_group_is_never_stopped(self):
|
|
1573
|
+
# Review 2026-09-23: a client that is not a job of a job-control shell (started
|
|
1574
|
+
# by a script, `sh -c`, a pipeline) gets no prompt back when it ends
|
|
1575
|
+
state = self.write_state([])
|
|
1576
|
+
self.registry()
|
|
1577
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1578
|
+
w, r = self.watcher(state)
|
|
1579
|
+
r.pane_mode = 1 # not even the mode is cancelled: nothing is touched
|
|
1580
|
+
r.live.add(self.PPID)
|
|
1581
|
+
r.pgids[self.PID] = self.PPID
|
|
1582
|
+
self.assertEqual(w.run(), 0)
|
|
1583
|
+
self.assertEqual((r.signals, r.sends, self.relaunch_files(), r.notices),
|
|
1584
|
+
([], [], [], []))
|
|
1585
|
+
self.assertEqual([c for c in r.calls if "-X" in c["argv"]], [])
|
|
1586
|
+
self.assertIn("reason=pgrp", self.log_lines()[-1])
|
|
1587
|
+
self.assertNotIn("stopped=", self.log_lines()[-1])
|
|
1588
|
+
# the suites' no-terminal knob (their harness has no job control) waives it; the
|
|
1589
|
+
# client is then stopped alone, never the group it shares with the shell
|
|
1590
|
+
state = self.write_state([])
|
|
1591
|
+
w, r = self.watcher(state, {"CLAUDE_MULTIACC_AR_TEST_TTY": "1",
|
|
1592
|
+
"CLAUDE_MULTIACC_AR_TERM_GRACE": "2"})
|
|
1593
|
+
r.term_kills = False
|
|
1594
|
+
r.live.add(self.PPID)
|
|
1595
|
+
r.pgids[self.PID] = self.PPID
|
|
1596
|
+
w.run()
|
|
1597
|
+
self.assertEqual(r.signals, [(self.PID, signal.SIGTERM), (self.PID, signal.SIGKILL)])
|
|
1598
|
+
self.assertIn(self.PPID, r.live)
|
|
1599
|
+
self.assertEqual(len(r.sends), 4)
|
|
1600
|
+
|
|
1601
|
+
def test_a_relaunch_that_never_takes_is_announced(self):
|
|
1602
|
+
# Review 2026-09-23: Enter was typed, but the line never ran (the relaunch file
|
|
1603
|
+
# is still there): withdraw the token and say how to resume, never log a switch
|
|
1604
|
+
state = self.write_state([])
|
|
1605
|
+
self.registry()
|
|
1606
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1607
|
+
w, r = self.watcher(state)
|
|
1608
|
+
r.consume = False
|
|
1609
|
+
self.assertEqual(w.run(), 0)
|
|
1610
|
+
self.assertEqual(r.signals, [(self.PID, signal.SIGTERM)])
|
|
1611
|
+
self.assertEqual(len(r.sends), 4)
|
|
1612
|
+
self.assertEqual(self.relaunch_files(), [])
|
|
1613
|
+
self.assertNotIn("switch", self.events())
|
|
1614
|
+
line = self.log_lines()[-1]
|
|
1615
|
+
self.assertIn("autoresume giveup", line)
|
|
1616
|
+
self.assertIn("reason=relaunch", line)
|
|
1617
|
+
self.assertIn('stopped=1 resume="claude --resume %s"' % SID, line)
|
|
1618
|
+
self.assertEqual(r.notices, [ar.notice_argv("/private/tmp/tmux-501/default", "%7",
|
|
1619
|
+
"claude", SID)])
|
|
1620
|
+
enter = r.index(lambda a: a[0] == "tmux" and a[-1] == "Enter")
|
|
1621
|
+
notice = r.index(lambda a: a[0] == "tmux" and "-d" in a)
|
|
1622
|
+
self.assertGreaterEqual(r.calls[notice]["t"] - r.calls[enter]["t"], 10) # ~10 s
|
|
1623
|
+
# a relaunch that takes within the wait is a switch
|
|
1624
|
+
state = self.write_state([])
|
|
1625
|
+
w, r = self.watcher(state)
|
|
1626
|
+
r.consume = False
|
|
1627
|
+
real_run = r.run
|
|
1628
|
+
|
|
1629
|
+
def slow_shell(argv, timeout, env=None, cwd=None):
|
|
1630
|
+
out = real_run(argv, timeout, env, cwd)
|
|
1631
|
+
if argv[0] == "tmux" and argv[-1] == "Enter":
|
|
1632
|
+
r.consume = True
|
|
1633
|
+
r.at(r.t + 3, r.consume_token)
|
|
1634
|
+
return out
|
|
1635
|
+
r.run = slow_shell
|
|
1636
|
+
self.assertEqual(w.run(), 0)
|
|
1637
|
+
self.assertEqual(self.events()[-1], "switch")
|
|
1638
|
+
self.assertEqual(r.notices, [])
|
|
1639
|
+
|
|
1640
|
+
def test_process_replaced_before_the_signal(self):
|
|
1641
|
+
state = self.write_state([])
|
|
1642
|
+
self.registry()
|
|
1643
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1644
|
+
w, r = self.watcher(state)
|
|
1645
|
+
real_run = r.run
|
|
1646
|
+
|
|
1647
|
+
def run(argv, timeout, env=None, cwd=None):
|
|
1648
|
+
out = real_run(argv, timeout, env, cwd)
|
|
1649
|
+
if argv[0] == self.self_path:
|
|
1650
|
+
r.starts[self.PID] = "Wed Sep 23 09:00:00 2026" # pid reused
|
|
1651
|
+
return out
|
|
1652
|
+
r.run = run
|
|
1653
|
+
w.run()
|
|
1654
|
+
self.assertEqual((r.signals, r.sends, self.relaunch_files()), ([], [], []))
|
|
1655
|
+
self.assertIn("reason=gone", self.log_lines()[-1])
|
|
1656
|
+
|
|
1657
|
+
def test_pane_never_returns_to_a_shell(self):
|
|
1658
|
+
state = self.write_state([])
|
|
1659
|
+
self.registry()
|
|
1660
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1661
|
+
w, r = self.watcher(state, {"CLAUDE_MULTIACC_AR_PANE_WAIT": "3"})
|
|
1662
|
+
r.exit = lambda pid: r.live.discard(pid) # pane keeps showing another program
|
|
1663
|
+
r.old_tmux = True
|
|
1664
|
+
w.run()
|
|
1665
|
+
self.assertEqual(r.signals, [(self.PID, signal.SIGTERM)])
|
|
1666
|
+
self.assertEqual((r.sends, self.relaunch_files()), ([], []))
|
|
1667
|
+
# the session is stopped and stays stopped: the log line and the pane's status
|
|
1668
|
+
# line say so, with the command that brings it back (tmux < 3.2 refuses -d 0:
|
|
1669
|
+
# the plain notice follows)
|
|
1670
|
+
self.assertIn("reason=shell", self.log_lines()[-1])
|
|
1671
|
+
self.assertIn('stopped=1 resume="claude --resume %s"' % SID, self.log_lines()[-1])
|
|
1672
|
+
self.assertEqual(r.notices, [
|
|
1673
|
+
ar.notice_argv("/private/tmp/tmux-501/default", "%7", "claude", SID),
|
|
1674
|
+
ar.notice_argv("/private/tmp/tmux-501/default", "%7", "claude", SID,
|
|
1675
|
+
sticky=False)])
|
|
1676
|
+
|
|
1677
|
+
def test_nothing_is_announced_for_a_session_that_was_never_stopped(self):
|
|
1678
|
+
state = self.write_state([])
|
|
1679
|
+
self.registry()
|
|
1680
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1681
|
+
w, r = self.watcher(state)
|
|
1682
|
+
r.pane_pid = 31337
|
|
1683
|
+
w.run()
|
|
1684
|
+
self.assertIn("reason=pane", self.log_lines()[-1])
|
|
1685
|
+
self.assertNotIn("stopped=", self.log_lines()[-1])
|
|
1686
|
+
self.assertNotIn("resume=", self.log_lines()[-1])
|
|
1687
|
+
self.assertEqual(r.notices, [])
|
|
1688
|
+
|
|
1689
|
+
def test_a_shell_loop_restarting_the_client_is_never_typed_into(self):
|
|
1690
|
+
# `while :; do claude; done`: one reading may catch the shell between two runs;
|
|
1691
|
+
# it must hold for SHELL_SETTLE before anything is typed.
|
|
1692
|
+
state = self.write_state([])
|
|
1693
|
+
self.registry()
|
|
1694
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1695
|
+
w, r = self.watcher(state, {"CLAUDE_MULTIACC_AR_PANE_WAIT": "3"})
|
|
1696
|
+
real_exit = r.exit
|
|
1697
|
+
|
|
1698
|
+
def exit_into_the_next_run(pid):
|
|
1699
|
+
real_exit(pid)
|
|
1700
|
+
if pid == self.PID:
|
|
1701
|
+
r.at(r.t + 0.3, lambda: setattr(r, "pane_cmd", "claude"))
|
|
1702
|
+
r.exit = exit_into_the_next_run
|
|
1703
|
+
w.run()
|
|
1704
|
+
self.assertEqual(r.signals, [(self.PID, signal.SIGTERM)])
|
|
1705
|
+
self.assertEqual([a for a in r.sends if "-l" in a], [])
|
|
1706
|
+
self.assertIn("reason=shell", self.log_lines()[-1])
|
|
1707
|
+
self.assertIn("stopped=1", self.log_lines()[-1])
|
|
1708
|
+
|
|
1709
|
+
def test_typing_waits_for_the_shell_to_hold_the_terminal(self):
|
|
1710
|
+
# the pane names the shell, but a job it started still owns the terminal
|
|
1711
|
+
state = self.write_state([])
|
|
1712
|
+
self.registry()
|
|
1713
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1714
|
+
w, r = self.watcher(state)
|
|
1715
|
+
real_status = r.proc_status
|
|
1716
|
+
ready_at = []
|
|
1717
|
+
|
|
1718
|
+
def status(pid):
|
|
1719
|
+
out = real_status(pid)
|
|
1720
|
+
if pid == self.PPID and out and not ready_at:
|
|
1721
|
+
ready_at.append(r.t + 2)
|
|
1722
|
+
if pid == self.PPID and out and r.t < (ready_at or [0])[0] and self.PID not in r.live:
|
|
1723
|
+
return ("Ss", out[1], out[2])
|
|
1724
|
+
return out
|
|
1725
|
+
r.proc_status = status
|
|
1726
|
+
w.run()
|
|
1727
|
+
typed = r.index(lambda a: a[0] == "tmux" and "-l" in a)
|
|
1728
|
+
self.assertGreaterEqual(r.calls[typed]["t"], ready_at[0] + ar.SHELL_SETTLE)
|
|
1729
|
+
self.assert_sequence(r)
|
|
1730
|
+
|
|
1731
|
+
def test_a_conversation_picked_with_resume_brings_no_old_errors(self):
|
|
1732
|
+
# Review 2026-09-23 (blocker): another session, on another account, hit ITS limit
|
|
1733
|
+
# after this launch; the operator /resumes that conversation here. Its rejection
|
|
1734
|
+
# is not this client's: nothing is stopped and this account is never parked.
|
|
1735
|
+
state = self.write_state([])
|
|
1736
|
+
self.registry()
|
|
1737
|
+
self.transcript([cc_reply(self.t0, "hello")])
|
|
1738
|
+
other = self.transcript([cc_user(self.t0 - 2, "go"),
|
|
1739
|
+
cc_quota(self.t0 + 1, resets=int(self.t0) + 999)], sid=SID2)
|
|
1740
|
+
w, r = self.watcher(state)
|
|
1741
|
+
r.at(self.t0 + 20, lambda: self.registry(sid=SID2))
|
|
1742
|
+
# ...while a limit THIS client meets in that conversation still counts
|
|
1743
|
+
r.at(self.t0 + 60, lambda: append(other, [cc_quota(self.t0 + 60,
|
|
1744
|
+
resets=int(self.t0) + 999)]))
|
|
1745
|
+
self.assertEqual(w.run(), 0)
|
|
1746
|
+
self.assertEqual(len(r.probes), 1)
|
|
1747
|
+
self.assertGreaterEqual(r.probes[0]["t"], self.t0 + 60)
|
|
1748
|
+
kv, argv = self.relaunch_of(r)
|
|
1749
|
+
self.assertEqual((kv["sid"], argv[-2]), (SID2, SID2))
|
|
1750
|
+
self.assertEqual(self.events(), ["watch", "detect", "switch"])
|
|
1751
|
+
|
|
1752
|
+
def test_copy_mode_is_left_by_command_before_the_kill(self):
|
|
1753
|
+
# a pane scrolled back (copy mode) swallows keys: the mode is cancelled with a
|
|
1754
|
+
# tmux command first — never a keystroke — and only then is anything stopped
|
|
1755
|
+
state = self.write_state([])
|
|
1756
|
+
self.registry()
|
|
1757
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1758
|
+
w, r = self.watcher(state)
|
|
1759
|
+
r.pane_mode = 1
|
|
1760
|
+
w.run()
|
|
1761
|
+
cancel = r.index(lambda a: "-X" in a)
|
|
1762
|
+
self.assertEqual(r.calls[cancel]["argv"],
|
|
1763
|
+
ar.mode_cancel_argv("/private/tmp/tmux-501/default", "%7"))
|
|
1764
|
+
self.assertLess(cancel, r.index(lambda a: a[0] == "<signal>"))
|
|
1765
|
+
self.assertEqual(r.signals, [(self.PID, signal.SIGTERM)])
|
|
1766
|
+
self.assert_sequence(r)
|
|
1767
|
+
|
|
1768
|
+
def test_a_mode_that_will_not_end_gives_up_before_the_kill(self):
|
|
1769
|
+
state = self.write_state([])
|
|
1770
|
+
self.registry()
|
|
1771
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1772
|
+
w, r = self.watcher(state)
|
|
1773
|
+
r.pane_mode, r.mode_sticky = 2, True
|
|
1774
|
+
self.assertEqual(w.run(), 0)
|
|
1775
|
+
self.assertEqual((r.signals, r.sends, self.relaunch_files()), ([], [], []))
|
|
1776
|
+
self.assertIn("reason=mode", self.log_lines()[-1])
|
|
1777
|
+
self.assertEqual(self.events()[-1], "giveup")
|
|
1778
|
+
self.assertEqual(r.notices, [])
|
|
1779
|
+
|
|
1780
|
+
def test_synchronized_panes_give_up_without_touching_anything(self):
|
|
1781
|
+
# with synchronize-panes on, typed keys reach EVERY pane of the window
|
|
1782
|
+
state = self.write_state([])
|
|
1783
|
+
self.registry()
|
|
1784
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1785
|
+
w, r = self.watcher(state)
|
|
1786
|
+
r.pane_sync, r.pane_mode = 1, 1
|
|
1787
|
+
self.assertEqual(w.run(), 0)
|
|
1788
|
+
self.assertEqual((r.signals, r.sends, self.relaunch_files()), ([], [], []))
|
|
1789
|
+
self.assertEqual([c for c in r.calls if "copy-mode" in c["argv"]
|
|
1790
|
+
or "-X" in c["argv"]], [])
|
|
1791
|
+
self.assertIn("reason=sync", self.log_lines()[-1])
|
|
1792
|
+
|
|
1793
|
+
def test_a_mode_entered_after_the_kill_is_cancelled_before_typing(self):
|
|
1794
|
+
state = self.write_state([])
|
|
1795
|
+
self.registry()
|
|
1796
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1797
|
+
w, r = self.watcher(state)
|
|
1798
|
+
real_exit = r.exit
|
|
1799
|
+
|
|
1800
|
+
def scroll_back_on_exit(pid):
|
|
1801
|
+
real_exit(pid)
|
|
1802
|
+
r.pane_mode = 1 # the operator scrolls the pane while the shell comes back
|
|
1803
|
+
r.exit = scroll_back_on_exit
|
|
1804
|
+
w.run()
|
|
1805
|
+
cancel = r.index(lambda a: "-X" in a)
|
|
1806
|
+
self.assertEqual(r.calls[cancel]["argv"],
|
|
1807
|
+
ar.mode_cancel_argv("/private/tmp/tmux-501/default", "%7"))
|
|
1808
|
+
self.assertLess(r.index(lambda a: a[0] == "<signal>"), cancel)
|
|
1809
|
+
self.assertLess(cancel, r.index(lambda a: a[0] == "tmux" and "-l" in a))
|
|
1810
|
+
self.assert_sequence(r)
|
|
1811
|
+
|
|
1812
|
+
def test_a_mode_or_sync_that_appears_after_the_kill_is_announced_not_typed_into(self):
|
|
1813
|
+
for attr, sticky, reason in (("pane_mode", True, "mode"), ("pane_sync", False, "sync")):
|
|
1814
|
+
(self.root / "selection.log").unlink() if (self.root / "selection.log").exists() \
|
|
1815
|
+
else None
|
|
1816
|
+
state = self.write_state([])
|
|
1817
|
+
self.registry()
|
|
1818
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1819
|
+
w, r = self.watcher(state)
|
|
1820
|
+
r.mode_sticky = sticky
|
|
1821
|
+
real_exit = r.exit
|
|
1822
|
+
|
|
1823
|
+
def on_exit(pid, r=r, real_exit=real_exit, attr=attr):
|
|
1824
|
+
real_exit(pid)
|
|
1825
|
+
setattr(r, attr, 1)
|
|
1826
|
+
r.exit = on_exit
|
|
1827
|
+
w.run()
|
|
1828
|
+
self.assertEqual([a for a in r.sends if "-l" in a], [], reason)
|
|
1829
|
+
self.assertEqual(self.relaunch_files(), [], reason)
|
|
1830
|
+
self.assertIn("reason=%s" % reason, self.log_lines()[-1])
|
|
1831
|
+
self.assertIn('resume="claude --resume %s"' % SID, self.log_lines()[-1])
|
|
1832
|
+
self.assertEqual(len(r.notices), 1, reason)
|
|
1833
|
+
|
|
1834
|
+
def test_a_pane_shell_the_relaunch_cannot_be_typed_into_is_never_stopped(self):
|
|
1835
|
+
for comm in ("nu", "/usr/local/bin/tcsh", "-tcsh", "pwsh", "xonsh", "fish", "-fish"):
|
|
1836
|
+
(self.root / "selection.log").unlink() if (self.root / "selection.log").exists() \
|
|
1837
|
+
else None
|
|
1838
|
+
state = self.write_state([])
|
|
1839
|
+
self.registry()
|
|
1840
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1841
|
+
w, r = self.watcher(state)
|
|
1842
|
+
r.shell_comm = comm
|
|
1843
|
+
self.assertEqual(w.run(), 0)
|
|
1844
|
+
self.assertEqual((r.signals, r.sends, self.relaunch_files()), ([], [], []), comm)
|
|
1845
|
+
self.assertIn("reason=pane-shell", self.log_lines()[-1], comm)
|
|
1846
|
+
|
|
1847
|
+
def test_a_suspended_or_background_client_is_never_stopped(self):
|
|
1848
|
+
# Ctrl-Z'd; not in its terminal's foreground; no terminal at all
|
|
1849
|
+
for flags, tty in (("T", "ttys003"), ("T+", "ttys003"), ("S", "ttys003"),
|
|
1850
|
+
("S", "??")):
|
|
1851
|
+
(self.root / "selection.log").unlink() if (self.root / "selection.log").exists() \
|
|
1852
|
+
else None
|
|
1853
|
+
state = self.write_state([])
|
|
1854
|
+
self.registry()
|
|
1855
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1856
|
+
w, r = self.watcher(state)
|
|
1857
|
+
r.client_stat, r.tty = flags, tty
|
|
1858
|
+
self.assertEqual(w.run(), 0)
|
|
1859
|
+
self.assertEqual((r.signals, r.sends, self.relaunch_files()), ([], [], []), flags)
|
|
1860
|
+
self.assertIn("reason=background", self.log_lines()[-1], flags)
|
|
1861
|
+
# the suites' no-terminal knob waives the foreground (never the stop) check
|
|
1862
|
+
for flags, stopped in (("S", False), ("T", True)):
|
|
1863
|
+
state = self.write_state([])
|
|
1864
|
+
self.registry()
|
|
1865
|
+
w, r = self.watcher(state, {"CLAUDE_MULTIACC_AR_TEST_TTY": "1"})
|
|
1866
|
+
r.client_stat, r.tty = flags, "??"
|
|
1867
|
+
w.run()
|
|
1868
|
+
self.assertEqual(r.signals, [] if stopped else [(self.PID, signal.SIGTERM)], flags)
|
|
1869
|
+
|
|
1870
|
+
def test_another_live_holder_of_the_session_is_never_raced(self):
|
|
1871
|
+
# the same conversation open in a second client (any account of the pool)
|
|
1872
|
+
state = self.write_state([])
|
|
1873
|
+
self.registry()
|
|
1874
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1875
|
+
other = self.root / "acct-02" / "sessions" / "6060.json"
|
|
1876
|
+
other.write_text(json.dumps({"pid": 6060, "sessionId": SID, "cwd": str(self.work)}))
|
|
1877
|
+
w, r = self.watcher(state)
|
|
1878
|
+
r.live.add(6060)
|
|
1879
|
+
self.assertEqual(w.run(), 0)
|
|
1880
|
+
self.assertEqual((r.signals, r.sends, self.relaunch_files()), ([], [], []))
|
|
1881
|
+
self.assertIn("reason=holder", self.log_lines()[-1])
|
|
1882
|
+
self.assertIn("holder=6060", self.log_lines()[-1])
|
|
1883
|
+
# a registry left behind by a dead client is no holder
|
|
1884
|
+
state = self.write_state([])
|
|
1885
|
+
w, r = self.watcher(state)
|
|
1886
|
+
w.run()
|
|
1887
|
+
self.assertEqual(r.signals, [(self.PID, signal.SIGTERM)])
|
|
1888
|
+
self.assertEqual(len(r.sends), 4)
|
|
1889
|
+
|
|
1890
|
+
def test_the_default_pool_is_not_named(self):
|
|
1891
|
+
# $HOME/.claude-accounts is what the relaunched shim finds on its own; a pool
|
|
1892
|
+
# anywhere else (the suites' pool, `CLAUDE_ACCOUNTS_ROOT=/other claude`) is named
|
|
1893
|
+
self.registry()
|
|
1894
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1895
|
+
home = self.tmp / "home"
|
|
1896
|
+
home.mkdir()
|
|
1897
|
+
os.symlink(str(self.root), str(home / ".claude-accounts"))
|
|
1898
|
+
state = self.write_state([], acc_root=str(home / ".claude-accounts"))
|
|
1899
|
+
w, r = self.watcher(state, {"HOME": str(home)})
|
|
1900
|
+
w.run()
|
|
1901
|
+
self.assertEqual([a[-1] for a in r.sends if "-l" in a],
|
|
1902
|
+
[" CLAUDE_MULTIACC_AR=%s:%s %s" % (self.token_of(r), SID,
|
|
1903
|
+
self.self_path)])
|
|
1904
|
+
self.assert_sequence(r, root=None)
|
|
1905
|
+
|
|
1906
|
+
def test_a_path_the_relaunch_cannot_type_is_never_supervised(self):
|
|
1907
|
+
odd = self.tmp / "my pool"
|
|
1908
|
+
odd.mkdir()
|
|
1909
|
+
state = self.write_state([], acc_root=odd)
|
|
1910
|
+
self.registry()
|
|
1911
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1912
|
+
w, r = self.watcher(state)
|
|
1913
|
+
self.assertEqual(w.run(), 0)
|
|
1914
|
+
self.assertEqual((r.probes, r.signals), ([], []))
|
|
1915
|
+
log = (odd / "selection.log").read_text()
|
|
1916
|
+
self.assertIn("autoresume giveup", log)
|
|
1917
|
+
self.assertIn("reason=path", log)
|
|
1918
|
+
# the shim's own path, likewise
|
|
1919
|
+
self.self_path = str(self.tmp / "b in" / "claude")
|
|
1920
|
+
state = self.write_state([])
|
|
1921
|
+
w, r = self.watcher(state)
|
|
1922
|
+
self.assertEqual(w.run(), 0)
|
|
1923
|
+
self.assertEqual((r.probes, r.signals), ([], []))
|
|
1924
|
+
self.assertIn("reason=path", self.log_lines()[-1])
|
|
1925
|
+
|
|
1926
|
+
def test_unsupported_argv_never_watches(self):
|
|
1927
|
+
state = self.write_state(["-p", "summarize this"])
|
|
1928
|
+
self.registry()
|
|
1929
|
+
self.transcript([cc_quota(self.t0, resets=int(self.t0) + 999)])
|
|
1930
|
+
w, r = self.watcher(state)
|
|
1931
|
+
self.assertEqual(w.run(), 0)
|
|
1932
|
+
self.assertEqual((r.probes, r.signals), ([], []))
|
|
1933
|
+
self.assertEqual(self.events(), ["giveup"])
|
|
1934
|
+
|
|
1935
|
+
def test_a_dead_pid_or_unknown_identity_exits_at_once(self):
|
|
1936
|
+
state = self.write_state([])
|
|
1937
|
+
w, r = self.watcher(state)
|
|
1938
|
+
r.live.clear()
|
|
1939
|
+
self.assertEqual(w.run(), 0)
|
|
1940
|
+
w, r = self.watcher(state)
|
|
1941
|
+
r.starts.clear()
|
|
1942
|
+
self.assertEqual(w.run(), 0)
|
|
1943
|
+
self.assertEqual(self.log_lines(), [])
|
|
1944
|
+
|
|
1945
|
+
|
|
1946
|
+
class CodexWatcherTests(PoolCase):
|
|
1947
|
+
CHILD = 4300
|
|
1948
|
+
GRAND = 4301
|
|
1949
|
+
|
|
1950
|
+
def rollout(self, records, sid=CX_SID, day="2026/09/23", name=None):
|
|
1951
|
+
d = self.shared / "sessions" / day
|
|
1952
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
1953
|
+
path = d / (name or "rollout-2026-09-23T00-39-55-%s.jsonl" % sid)
|
|
1954
|
+
with open(path, "a") as fh:
|
|
1955
|
+
fh.write(jl(records))
|
|
1956
|
+
return path
|
|
1957
|
+
|
|
1958
|
+
def setUp(self):
|
|
1959
|
+
super().setUp()
|
|
1960
|
+
self.self_path = str(self.tmp / "bin" / "codex")
|
|
1961
|
+
for acct in ("acct-01", "acct-02"):
|
|
1962
|
+
os.rmdir(str(self.root / acct / "sessions"))
|
|
1963
|
+
os.symlink(str(self.shared / "sessions"), str(self.root / acct / "sessions"))
|
|
1964
|
+
|
|
1965
|
+
def run_codex(self, argv, records, env=None, setup=None):
|
|
1966
|
+
state = self.write_state(argv, provider="codex")
|
|
1967
|
+
path = self.rollout(records)
|
|
1968
|
+
w, r = self.watcher(state, env)
|
|
1969
|
+
r.pane_cmd = "node"
|
|
1970
|
+
if setup:
|
|
1971
|
+
setup(w, r, path)
|
|
1972
|
+
rc = w.run()
|
|
1973
|
+
return rc, w, r, path
|
|
1974
|
+
|
|
1975
|
+
def test_quota_rotates_with_resume_and_the_message_date(self):
|
|
1976
|
+
with local_tz("UTC"):
|
|
1977
|
+
rc, w, r, _ = self.run_codex(
|
|
1978
|
+
["--yolo", "resume", CX_SID], [
|
|
1979
|
+
cx_meta(self.launched - 7200, cwd=str(self.work)),
|
|
1980
|
+
cx_error(self.launched - 7000, "usage_limit_exceeded", LIMIT_MSG),
|
|
1981
|
+
cx_event(self.t0 - 2, {"type": "task_started"}),
|
|
1982
|
+
cx_tokens(self.t0 - 1, limit_id="premium"),
|
|
1983
|
+
cx_error(self.t0 - 1, "usage_limit_exceeded", LIMIT_MSG)])
|
|
1984
|
+
self.assertEqual(rc, 0)
|
|
1985
|
+
reset = 1790383140 # Sep 26th, 2026 12:39 AM, read in the local (UTC) zone
|
|
1986
|
+
prompt = ar.resume_prompt("quota")
|
|
1987
|
+
probe = r.probes[0]
|
|
1988
|
+
self.assertEqual(probe["argv"], [self.self_path, "resume", "--yolo", CX_SID, prompt])
|
|
1989
|
+
self.assertEqual(probe["env"]["CODEX_MULTIACC_AR_PROBE"], "1")
|
|
1990
|
+
self.assertEqual(probe["env"]["CODEX_MULTIACC_AR_AVOID"], "acct-01:%d" % reset)
|
|
1991
|
+
self.assertNotIn("CODEX_HOME", probe["env"])
|
|
1992
|
+
self.assertEqual(r.signals, [(self.PID, signal.SIGTERM)])
|
|
1993
|
+
self.assert_sequence(r, "codex")
|
|
1994
|
+
kv, argv = self.relaunch_of(r)
|
|
1995
|
+
self.assertEqual(argv, ["resume", "--yolo", CX_SID, prompt])
|
|
1996
|
+
self.assertEqual((kv["provider"], kv["class"], kv["reset"], kv["rtype"], kv["sid"],
|
|
1997
|
+
kv["cwd"]),
|
|
1998
|
+
("codex", "quota", str(reset), "", CX_SID, str(self.work)))
|
|
1999
|
+
self.assertEqual(self.events(), ["watch", "detect", "switch"])
|
|
2000
|
+
self.assertEqual(r.sends[2][-1], " CODEX_MULTIACC_AR=%s:%s CODEX_ACCOUNTS_ROOT=%s %s"
|
|
2001
|
+
% (self.token_of(r), CX_SID, self.root, self.self_path))
|
|
2002
|
+
|
|
2003
|
+
def tree(self, r, pgid):
|
|
2004
|
+
r.term_kills = False
|
|
2005
|
+
r.table = [(self.PID, self.PPID, LSTART), (self.CHILD, self.PID, "c"),
|
|
2006
|
+
(self.GRAND, self.CHILD, "g"), (9999, 1, "x")]
|
|
2007
|
+
r.live |= {self.CHILD, self.GRAND, 9999}
|
|
2008
|
+
r.starts.update({self.CHILD: "c", self.GRAND: "g", 9999: "x"})
|
|
2009
|
+
r.pgids.update({self.PID: pgid, self.CHILD: pgid, self.GRAND: pgid, 9999: 9999})
|
|
2010
|
+
if pgid != self.PID:
|
|
2011
|
+
r.live.add(pgid)
|
|
2012
|
+
|
|
2013
|
+
def test_term_refused_kills_the_recorded_tree(self):
|
|
2014
|
+
# node wrapper + native binary: the client and the tree it spawned go, each
|
|
2015
|
+
# re-verified by start time; a process of the same group that is not the
|
|
2016
|
+
# client's (9999: the shim's detached limits refresh) stays
|
|
2017
|
+
def setup(w, r, p):
|
|
2018
|
+
self.tree(r, self.PID)
|
|
2019
|
+
r.pgids[9999] = self.PID
|
|
2020
|
+
rc, w, r, _ = self.run_codex(
|
|
2021
|
+
["resume", CX_SID], [cx_meta(self.launched, cwd=str(self.work)),
|
|
2022
|
+
cx_error(self.t0, "unauthorized", "401")],
|
|
2023
|
+
{"CODEX_MULTIACC_AR_TERM_GRACE": "2"}, setup)
|
|
2024
|
+
self.assertEqual(r.signals, [(self.PID, signal.SIGTERM), (self.PID, signal.SIGKILL),
|
|
2025
|
+
(self.CHILD, signal.SIGKILL), (self.GRAND, signal.SIGKILL)])
|
|
2026
|
+
self.assertEqual(r.live, {9999})
|
|
2027
|
+
kv, _ = self.relaunch_of(r)
|
|
2028
|
+
self.assertEqual(kv["class"], "auth")
|
|
2029
|
+
|
|
2030
|
+
def test_a_client_outside_its_own_group_is_never_stopped(self):
|
|
2031
|
+
rc, w, r, _ = self.run_codex(
|
|
2032
|
+
["resume", CX_SID], [cx_meta(self.launched, cwd=str(self.work)),
|
|
2033
|
+
cx_error(self.t0, "unauthorized", "401")],
|
|
2034
|
+
{"CODEX_MULTIACC_AR_TERM_GRACE": "2"}, lambda w, r, p: self.tree(r, 7000))
|
|
2035
|
+
self.assertEqual((rc, r.signals, r.sends, self.relaunch_files()), (0, [], [], []))
|
|
2036
|
+
self.assertIn("reason=pgrp", self.log_lines()[-1])
|
|
2037
|
+
# under the suites' knob the recorded tree is stopped, never the group
|
|
2038
|
+
rc, w, r, _ = self.run_codex(
|
|
2039
|
+
["resume", CX_SID], [], {"CODEX_MULTIACC_AR_TERM_GRACE": "2",
|
|
2040
|
+
"CODEX_MULTIACC_AR_TEST_TTY": "1"},
|
|
2041
|
+
lambda w, r, p: self.tree(r, 7000))
|
|
2042
|
+
self.assertEqual(r.signals, [(self.PID, signal.SIGTERM), (self.PID, signal.SIGKILL),
|
|
2043
|
+
(self.CHILD, signal.SIGKILL), (self.GRAND, signal.SIGKILL)])
|
|
2044
|
+
self.assertEqual(r.live, {9999, 7000})
|
|
2045
|
+
self.assertEqual(len(r.sends), 4)
|
|
2046
|
+
|
|
2047
|
+
def test_discovery_by_the_native_childs_open_file(self):
|
|
2048
|
+
def setup(w, r, path):
|
|
2049
|
+
r.table = [(self.PID, self.PPID, LSTART), (self.CHILD, self.PID, "c")]
|
|
2050
|
+
r.live.add(self.CHILD)
|
|
2051
|
+
r.files[self.CHILD] = ["/dev/ttys003", str(path)]
|
|
2052
|
+
# a decoy: another new rollout in the same cwd — (c) alone would refuse
|
|
2053
|
+
self.rollout([cx_meta(self.t0, sid=CX_SID2, cwd=str(self.work))], sid=CX_SID2)
|
|
2054
|
+
rc, w, r, path = self.run_codex(
|
|
2055
|
+
["--dangerously-bypass-approvals-and-sandbox"],
|
|
2056
|
+
[cx_meta(self.t0 - 1, cwd=str(self.work)),
|
|
2057
|
+
cx_error(self.t0, "server_overloaded")],
|
|
2058
|
+
{"CODEX_MULTIACC_AR_TRANSIENT_LADDER": "0"}, setup)
|
|
2059
|
+
self.assertEqual(w.sid, CX_SID)
|
|
2060
|
+
kv, argv = self.relaunch_of(r)
|
|
2061
|
+
self.assertEqual(argv, ["resume", "--dangerously-bypass-approvals-and-sandbox", CX_SID,
|
|
2062
|
+
ar.resume_prompt("transient")])
|
|
2063
|
+
self.assertEqual(kv["class"], "transient")
|
|
2064
|
+
|
|
2065
|
+
def no_open_files(self, w, r, path):
|
|
2066
|
+
r.open_files_ok = False # no /proc, no lsof: the cwd fallback is all there is
|
|
2067
|
+
|
|
2068
|
+
def test_discovery_fallback_needs_exactly_one_candidate(self):
|
|
2069
|
+
# one new rollout in this cwd: taken
|
|
2070
|
+
rc, w, r, _ = self.run_codex(["fix the build now"],
|
|
2071
|
+
[cx_meta(self.t0 - 1, cwd=str(self.work)),
|
|
2072
|
+
cx_error(self.t0, "rate_limit_exceeded", "slow")],
|
|
2073
|
+
None, self.no_open_files)
|
|
2074
|
+
self.assertEqual(w.sid, CX_SID)
|
|
2075
|
+
self.assertEqual(len(r.sends), 4)
|
|
2076
|
+
|
|
2077
|
+
def test_fallback_never_guesses_where_open_files_can_be_read(self):
|
|
2078
|
+
# Review 2026-09-23: a fresh, idle codex has no rollout yet (codex writes it at the
|
|
2079
|
+
# first message), and a NEIGHBOUR session in the same cwd — unsupervised, or
|
|
2080
|
+
# supervised past the 2-day prune — wrote one since this launch. The fallback
|
|
2081
|
+
# adopted it; a limit there stopped THIS session and typed the neighbour's
|
|
2082
|
+
# `codex resume` into its pane. Where the open-file lookup works it is the answer.
|
|
2083
|
+
def setup(w, r, path):
|
|
2084
|
+
r.at(self.t0 + 400, lambda: r.exit(self.PID))
|
|
2085
|
+
rc, w, r, _ = self.run_codex(["fix the build now"],
|
|
2086
|
+
[cx_meta(self.t0 - 1, cwd=str(self.work)),
|
|
2087
|
+
cx_error(self.t0, "usage_limit_exceeded", LIMIT_MSG)],
|
|
2088
|
+
{"CODEX_MULTIACC_AR_DISCOVER_TIMEOUT": "20"}, setup)
|
|
2089
|
+
self.assertEqual(rc, 0)
|
|
2090
|
+
self.assertIsNone(w.tail)
|
|
2091
|
+
self.assertEqual((r.probes, r.signals, r.sends), ([], [], []))
|
|
2092
|
+
# ...and it never stopped looking: past DISCOVER_TIMEOUT only more slowly
|
|
2093
|
+
self.assertNotIn("giveup", self.events())
|
|
2094
|
+
|
|
2095
|
+
def test_session_first_used_after_the_discovery_window_is_still_found(self):
|
|
2096
|
+
# codex creates the rollout at the first message: a TUI opened now and used
|
|
2097
|
+
# ten minutes later is found by the (slowed) open-file lookup then.
|
|
2098
|
+
seen = []
|
|
2099
|
+
|
|
2100
|
+
def setup(w, r, path):
|
|
2101
|
+
r.table = [(self.PID, self.PPID, LSTART), (self.CHILD, self.PID, "c")]
|
|
2102
|
+
r.live.add(self.CHILD)
|
|
2103
|
+
real = r.ps_table
|
|
2104
|
+
r.ps_table = lambda: (seen.append(r.t), real())[1]
|
|
2105
|
+
r.at(self.t0 + 600, lambda: r.files.__setitem__(self.CHILD, [str(path)]))
|
|
2106
|
+
rc, w, r, path = self.run_codex(
|
|
2107
|
+
["--yolo"], [cx_meta(self.t0 + 599, cwd=str(self.work)),
|
|
2108
|
+
cx_error(self.t0 + 599, "usage_limit_exceeded", LIMIT_MSG)],
|
|
2109
|
+
{"CODEX_MULTIACC_AR_DISCOVER_TIMEOUT": "20"}, setup)
|
|
2110
|
+
self.assertEqual(w.sid, CX_SID)
|
|
2111
|
+
self.assertEqual(len(r.sends), 4)
|
|
2112
|
+
late = [t for t in seen if self.t0 + 30 < t <= self.t0 + 601] # discovery only
|
|
2113
|
+
self.assertTrue(late)
|
|
2114
|
+
gaps = [b - a for a, b in zip(late, late[1:])]
|
|
2115
|
+
self.assertTrue(all(g >= ar.DISCOVER_SLOW - 0.5 for g in gaps), gaps)
|
|
2116
|
+
|
|
2117
|
+
def test_fallback_needs_a_session_created_since_the_launch(self):
|
|
2118
|
+
# a rollout WRITTEN since the launch but created before it is someone else's
|
|
2119
|
+
def setup(w, r, path):
|
|
2120
|
+
r.open_files_ok = False
|
|
2121
|
+
r.at(self.t0 + 30, lambda: r.exit(self.PID))
|
|
2122
|
+
rc, w, r, _ = self.run_codex(["fix the build now"],
|
|
2123
|
+
[cx_meta(self.launched - 600, cwd=str(self.work)),
|
|
2124
|
+
cx_error(self.t0, "usage_limit_exceeded", LIMIT_MSG)],
|
|
2125
|
+
{"CODEX_MULTIACC_AR_DISCOVER_TIMEOUT": "20"}, setup)
|
|
2126
|
+
self.assertIsNone(w.tail)
|
|
2127
|
+
self.assertEqual((r.probes, r.signals), ([], []))
|
|
2128
|
+
self.assertIn("reason=discover", self.log_lines()[-1])
|
|
2129
|
+
|
|
2130
|
+
def test_fallback_once_the_open_file_lookup_keeps_failing(self):
|
|
2131
|
+
# lsof is there but every call fails: after LOOKUP_FAILED s it counts as absent
|
|
2132
|
+
seen = []
|
|
2133
|
+
|
|
2134
|
+
def setup(w, r, path):
|
|
2135
|
+
r.lookup_fails = True
|
|
2136
|
+
real = r.run
|
|
2137
|
+
r.run = lambda argv, *a, **k: (seen.append(r.t) if argv[0] == self.self_path
|
|
2138
|
+
else None, real(argv, *a, **k))[1]
|
|
2139
|
+
rc, w, r, _ = self.run_codex(["fix the build now"],
|
|
2140
|
+
[cx_meta(self.t0 - 1, cwd=str(self.work)),
|
|
2141
|
+
cx_error(self.t0, "usage_limit_exceeded", LIMIT_MSG)],
|
|
2142
|
+
None, setup)
|
|
2143
|
+
self.assertEqual(w.sid, CX_SID)
|
|
2144
|
+
self.assertGreaterEqual(seen[0], self.t0 + ar.LOOKUP_FAILED)
|
|
2145
|
+
self.assertEqual(len(r.sends), 4)
|
|
2146
|
+
|
|
2147
|
+
def test_discovery_gives_up_when_ambiguous(self):
|
|
2148
|
+
def setup(w, r, path):
|
|
2149
|
+
r.open_files_ok = False
|
|
2150
|
+
self.rollout([cx_meta(self.t0, sid=CX_SID2, cwd=str(self.work))], sid=CX_SID2)
|
|
2151
|
+
rc, w, r, _ = self.run_codex(["fix the build now"],
|
|
2152
|
+
[cx_meta(self.t0 - 1, cwd=str(self.work)),
|
|
2153
|
+
cx_error(self.t0, "usage_limit_exceeded", LIMIT_MSG)],
|
|
2154
|
+
{"CODEX_MULTIACC_AR_DISCOVER_TIMEOUT": "20"}, setup)
|
|
2155
|
+
self.assertEqual(rc, 0)
|
|
2156
|
+
self.assertIsNone(w.tail)
|
|
2157
|
+
self.assertEqual((r.probes, r.signals), ([], []))
|
|
2158
|
+
self.assertIn("reason=discover", self.log_lines()[-1])
|
|
2159
|
+
|
|
2160
|
+
def test_fallback_skips_rollouts_other_watchers_follow_and_foreign_ones(self):
|
|
2161
|
+
def setup(w, r, path):
|
|
2162
|
+
r.open_files_ok = False
|
|
2163
|
+
other = self.rollout([cx_meta(self.t0, sid=CX_SID2, cwd=str(self.work))],
|
|
2164
|
+
sid=CX_SID2)
|
|
2165
|
+
(self.ar_dir / "5555.rollout").write_text(str(other))
|
|
2166
|
+
r.live.add(5555)
|
|
2167
|
+
# neither a subagent file, another cwd, nor an exec session is a candidate
|
|
2168
|
+
self.rollout([cx_meta(self.t0, sid="01a0cb0f-0000-7000-8000-000000000001",
|
|
2169
|
+
session_id=CX_SID, cwd=str(self.work))],
|
|
2170
|
+
name="rollout-sub.jsonl")
|
|
2171
|
+
self.rollout([cx_meta(self.t0, sid="01a0cb0f-0000-7000-8000-000000000002",
|
|
2172
|
+
cwd="/elsewhere")], name="rollout-cwd.jsonl")
|
|
2173
|
+
self.rollout([cx_meta(self.t0, sid="01a0cb0f-0000-7000-8000-000000000003",
|
|
2174
|
+
cwd=str(self.work), originator="codex_exec")],
|
|
2175
|
+
name="rollout-exec.jsonl")
|
|
2176
|
+
rc, w, r, _ = self.run_codex(["fix the build now"],
|
|
2177
|
+
[cx_meta(self.t0 - 1, cwd=str(self.work)),
|
|
2178
|
+
cx_error(self.t0, "usage_limit_exceeded", LIMIT_MSG)],
|
|
2179
|
+
None, setup)
|
|
2180
|
+
self.assertEqual(w.sid, CX_SID)
|
|
2181
|
+
self.assertEqual(len(r.sends), 4)
|
|
2182
|
+
|
|
2183
|
+
def test_codex_never_classes_and_cancel(self):
|
|
2184
|
+
def setup(w, r, path):
|
|
2185
|
+
r.at(self.t0 + 1.5, lambda: append(path, [
|
|
2186
|
+
cx_event(self.t0 + 1, {"type": "user_message", "message": "wait"})]))
|
|
2187
|
+
r.at(self.t0 + 9, lambda: r.exit(self.PID))
|
|
2188
|
+
rc, w, r, _ = self.run_codex(["resume", CX_SID],
|
|
2189
|
+
[cx_meta(self.launched, cwd=str(self.work)),
|
|
2190
|
+
cx_error(self.t0 - 2, "cyber_policy"),
|
|
2191
|
+
cx_error(self.t0, "usage_limit_exceeded", LIMIT_MSG)],
|
|
2192
|
+
None, setup)
|
|
2193
|
+
self.assertEqual((r.probes, r.signals), ([], []))
|
|
2194
|
+
self.assertEqual(self.events(), ["watch", "never", "detect"])
|
|
2195
|
+
|
|
2196
|
+
def test_codex_exit_is_never_a_crash(self):
|
|
2197
|
+
def setup(w, r, path):
|
|
2198
|
+
r.at(self.t0 + 90, lambda: r.exit(self.PID))
|
|
2199
|
+
rc, w, r, _ = self.run_codex(["resume", CX_SID],
|
|
2200
|
+
[cx_meta(self.launched, cwd=str(self.work))], None, setup)
|
|
2201
|
+
self.assertEqual((rc, r.sends, self.relaunch_files()), (0, [], []))
|
|
2202
|
+
|
|
2203
|
+
|
|
2204
|
+
# =====================================================================================
|
|
2205
|
+
# the CLI, and the real watch process end to end
|
|
2206
|
+
# =====================================================================================
|
|
2207
|
+
|
|
2208
|
+
class CliTests(PoolCase):
|
|
2209
|
+
def cli(self, *args, env=None, timeout=30):
|
|
2210
|
+
full = dict(os.environ)
|
|
2211
|
+
full.update(env or {})
|
|
2212
|
+
for key in [k for k, v in full.items() if v is None]:
|
|
2213
|
+
del full[key] # None: this variable must not reach the child at all
|
|
2214
|
+
return subprocess.run([sys.executable, "-I", str(LIB)] + list(args),
|
|
2215
|
+
stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=full,
|
|
2216
|
+
timeout=timeout, universal_newlines=True)
|
|
2217
|
+
|
|
2218
|
+
def test_classify(self):
|
|
2219
|
+
path = self.transcript([cc_quota(self.t0 - 10000, resets=1790123817),
|
|
2220
|
+
cc_api_error(self.t0, "invalid_request"),
|
|
2221
|
+
cc_quota(self.t0, "seven_day", resets=1790372787),
|
|
2222
|
+
cc_user(self.t0 + 1, "go"),
|
|
2223
|
+
cc_api_error(self.t0 + 2, "authentication_failed")])
|
|
2224
|
+
out = self.cli("classify", "--provider", "claude", str(path),
|
|
2225
|
+
"--since", str(int(self.t0) - 5))
|
|
2226
|
+
self.assertEqual(out.returncode, 0, out.stderr)
|
|
2227
|
+
doc = json.loads(out.stdout)
|
|
2228
|
+
self.assertEqual([e["kind"] for e in doc["events"]],
|
|
2229
|
+
["never", "verdict", "cancel", "verdict"])
|
|
2230
|
+
self.assertEqual((doc["events"][1]["class"], doc["events"][1]["rtype"],
|
|
2231
|
+
doc["events"][1]["reset"]), ("quota", "seven_day", 1790372787))
|
|
2232
|
+
self.assertEqual(doc["pending"]["class"], "auth")
|
|
2233
|
+
doc = json.loads(self.cli("classify", "--provider", "claude", str(path)).stdout)
|
|
2234
|
+
self.assertEqual(doc["events"][0]["kind"], "verdict") # no --since: everything counts
|
|
2235
|
+
|
|
2236
|
+
def test_classify_codex(self):
|
|
2237
|
+
path = self.tmp / "r.jsonl"
|
|
2238
|
+
path.write_text(jl([cx_meta(self.t0), cx_tokens(self.t0, limit_id="premium"),
|
|
2239
|
+
cx_error(self.t0, {"usage_limit_exceeded": None}, LIMIT_MSG)]))
|
|
2240
|
+
doc = json.loads(self.cli("classify", "--provider", "codex", str(path),
|
|
2241
|
+
env={"TZ": "UTC"}).stdout)
|
|
2242
|
+
self.assertEqual(doc["pending"]["class"], "quota")
|
|
2243
|
+
self.assertEqual(doc["pending"]["reset"], 1790383140)
|
|
2244
|
+
self.assertNotIn("_message", doc["pending"])
|
|
2245
|
+
self.assertNotIn("usage limit", json.dumps(doc))
|
|
2246
|
+
|
|
2247
|
+
def test_relaunch_argv(self):
|
|
2248
|
+
(self.root / "acct-01" / "settings.json").write_text(json.dumps({"model": "opus"}))
|
|
2249
|
+
state = self.write_state(["--dangerously-skip-permissions", "-c"])
|
|
2250
|
+
out = self.cli("relaunch-argv", "--provider", "claude", "--state", state, "--sid", SID,
|
|
2251
|
+
"--class", "quota")
|
|
2252
|
+
self.assertEqual(out.returncode, 0, out.stderr)
|
|
2253
|
+
self.assertEqual(json.loads(out.stdout), ["--dangerously-skip-permissions", "--resume",
|
|
2254
|
+
SID, ar.resume_prompt("quota")])
|
|
2255
|
+
out = self.cli("relaunch-argv", "--provider", "claude", "--state", state, "--sid", SID,
|
|
2256
|
+
"--class", "crash",
|
|
2257
|
+
env={"CLAUDE_MULTIACC_AUTORESUME_PROMPT": "carry on: {reason}"})
|
|
2258
|
+
self.assertEqual(json.loads(out.stdout)[-1],
|
|
2259
|
+
"carry on: the previous process exited unexpectedly")
|
|
2260
|
+
bad = self.write_state(["mcp"])
|
|
2261
|
+
out = self.cli("relaunch-argv", "--provider", "claude", "--state", bad, "--sid", SID,
|
|
2262
|
+
"--class", "quota")
|
|
2263
|
+
self.assertEqual((out.returncode, out.stdout.strip()), (1, "null"))
|
|
2264
|
+
cx = self.write_state(["resume", "--last"], provider="codex")
|
|
2265
|
+
out = self.cli("relaunch-argv", "--provider", "codex", "--state", cx, "--sid", CX_SID,
|
|
2266
|
+
"--class", "auth")
|
|
2267
|
+
self.assertEqual(json.loads(out.stdout),
|
|
2268
|
+
["resume", CX_SID, ar.resume_prompt("auth")])
|
|
2269
|
+
|
|
2270
|
+
def test_usage_errors(self):
|
|
2271
|
+
self.assertEqual(self.cli().returncode, 2)
|
|
2272
|
+
self.assertEqual(self.cli("bogus").returncode, 2)
|
|
2273
|
+
self.assertEqual(self.cli("watch").returncode, 2)
|
|
2274
|
+
self.assertEqual(self.cli("classify", "--provider", "gemini", "x").returncode, 2)
|
|
2275
|
+
missing = self.cli("classify", "--provider", "claude", str(self.tmp / "none.jsonl"))
|
|
2276
|
+
self.assertEqual(missing.returncode, 2)
|
|
2277
|
+
self.assertNotIn("Traceback", missing.stderr)
|
|
2278
|
+
path = self.transcript([])
|
|
2279
|
+
self.assertEqual(self.cli("classify", "--provider", "claude", str(path),
|
|
2280
|
+
"--since", "soon").returncode, 2)
|
|
2281
|
+
|
|
2282
|
+
def trust(self, *args):
|
|
2283
|
+
out = self.cli("trust", *args)
|
|
2284
|
+
self.assertEqual((out.returncode, out.stdout, out.stderr), (0, "", ""), args)
|
|
2285
|
+
|
|
2286
|
+
def test_trust_marks_the_cwd_and_keeps_everything_else(self):
|
|
2287
|
+
acct = self.root / "acct-02"
|
|
2288
|
+
conf = acct / ".claude.json"
|
|
2289
|
+
doc = {"numStartups": 3, "mcpServers": {"x": {"command": "y"}},
|
|
2290
|
+
"projects": {"/elsewhere": {"hasTrustDialogAccepted": True},
|
|
2291
|
+
str(self.work): {"allowedTools": ["Bash"],
|
|
2292
|
+
"hasTrustDialogAccepted": False}},
|
|
2293
|
+
"oauthAccount": {"emailAddress": "a@b.c"}}
|
|
2294
|
+
conf.write_text(json.dumps(doc))
|
|
2295
|
+
conf.chmod(0o640)
|
|
2296
|
+
self.trust("--acct-dir", str(acct), "--cwd", str(self.work))
|
|
2297
|
+
got = json.loads(conf.read_text())
|
|
2298
|
+
doc["projects"][str(self.work)]["hasTrustDialogAccepted"] = True
|
|
2299
|
+
self.assertEqual(got, doc)
|
|
2300
|
+
self.assertEqual(list(got), list(doc)) # key order kept
|
|
2301
|
+
self.assertEqual(stat.S_IMODE(conf.stat().st_mode), 0o640)
|
|
2302
|
+
# a directory with no entry gets one holding just that key
|
|
2303
|
+
self.trust("--acct-dir", str(acct), "--cwd", "/new/dir")
|
|
2304
|
+
got = json.loads(conf.read_text())
|
|
2305
|
+
self.assertEqual(got["projects"]["/new/dir"], {"hasTrustDialogAccepted": True})
|
|
2306
|
+
self.assertEqual(got["projects"][str(self.work)]["allowedTools"], ["Bash"])
|
|
2307
|
+
# already trusted: the file is not rewritten
|
|
2308
|
+
before = conf.stat()
|
|
2309
|
+
os.utime(str(conf), (before.st_atime, before.st_mtime - 100))
|
|
2310
|
+
mtime = conf.stat().st_mtime
|
|
2311
|
+
self.trust("--acct-dir", str(acct), "--cwd", "/new/dir")
|
|
2312
|
+
self.assertEqual(conf.stat().st_mtime, mtime)
|
|
2313
|
+
self.assertEqual(sorted(p.name for p in acct.iterdir()),
|
|
2314
|
+
[".claude.json", "projects", "sessions"]) # no temp file left
|
|
2315
|
+
|
|
2316
|
+
def test_trust_without_projects_and_fail_open_cases(self):
|
|
2317
|
+
acct = self.root / "acct-02"
|
|
2318
|
+
conf = acct / ".claude.json"
|
|
2319
|
+
conf.write_text(json.dumps({"numStartups": 1}))
|
|
2320
|
+
self.trust("--acct-dir", str(acct), "--cwd", str(self.work))
|
|
2321
|
+
self.assertEqual(json.loads(conf.read_text()),
|
|
2322
|
+
{"numStartups": 1,
|
|
2323
|
+
"projects": {str(self.work): {"hasTrustDialogAccepted": True}}})
|
|
2324
|
+
# nothing to change, nothing written, always 0 and silent
|
|
2325
|
+
self.trust("--acct-dir", str(self.root / "acct-01"), "--cwd", str(self.work))
|
|
2326
|
+
self.assertFalse((self.root / "acct-01" / ".claude.json").exists())
|
|
2327
|
+
for text in ("{not json", "[1, 2]", json.dumps({"projects": []}),
|
|
2328
|
+
json.dumps({"projects": {str(self.work): "odd"}})):
|
|
2329
|
+
conf.write_text(text)
|
|
2330
|
+
self.trust("--acct-dir", str(acct), "--cwd", str(self.work))
|
|
2331
|
+
self.assertEqual(conf.read_text(), text)
|
|
2332
|
+
conf.write_text("{}")
|
|
2333
|
+
self.trust("--acct-dir", str(acct), "--cwd", "relative/dir")
|
|
2334
|
+
self.trust("--acct-dir", str(acct))
|
|
2335
|
+
self.trust("--cwd", str(self.work))
|
|
2336
|
+
self.trust("--acct-dir")
|
|
2337
|
+
self.trust()
|
|
2338
|
+
self.assertEqual(conf.read_text(), "{}")
|
|
2339
|
+
|
|
2340
|
+
def test_trust_marks_the_logical_and_the_physical_path(self):
|
|
2341
|
+
# the shim passes the logical $PWD; Claude may key projects by the physical one
|
|
2342
|
+
acct = self.root / "acct-02"
|
|
2343
|
+
conf = acct / ".claude.json"
|
|
2344
|
+
conf.write_text(json.dumps({"projects": {}}))
|
|
2345
|
+
link = self.tmp / "link to work"
|
|
2346
|
+
os.symlink(str(self.work), str(link))
|
|
2347
|
+
self.trust("--acct-dir", str(acct), "--cwd", str(link))
|
|
2348
|
+
projects = json.loads(conf.read_text())["projects"]
|
|
2349
|
+
self.assertEqual(projects, {str(link): {"hasTrustDialogAccepted": True},
|
|
2350
|
+
str(self.work): {"hasTrustDialogAccepted": True}})
|
|
2351
|
+
# one of the two already trusted: the other is added, the first left as it was
|
|
2352
|
+
conf.write_text(json.dumps({"projects": {str(self.work): {
|
|
2353
|
+
"hasTrustDialogAccepted": True, "allowedTools": []}}}))
|
|
2354
|
+
self.trust("--acct-dir", str(acct), "--cwd", str(link))
|
|
2355
|
+
projects = json.loads(conf.read_text())["projects"]
|
|
2356
|
+
self.assertEqual(projects, {str(self.work): {"hasTrustDialogAccepted": True,
|
|
2357
|
+
"allowedTools": []},
|
|
2358
|
+
str(link): {"hasTrustDialogAccepted": True}})
|
|
2359
|
+
# an entry of an unexpected shape under either name: nothing is written
|
|
2360
|
+
text = json.dumps({"projects": {str(self.work): "odd"}})
|
|
2361
|
+
conf.write_text(text)
|
|
2362
|
+
self.trust("--acct-dir", str(acct), "--cwd", str(link))
|
|
2363
|
+
self.assertEqual(conf.read_text(), text)
|
|
2364
|
+
|
|
2365
|
+
def test_trust_never_overwrites_a_file_rewritten_meanwhile(self):
|
|
2366
|
+
# Review 2026-09-23: a running Claude rewrites .claude.json whenever it likes; a
|
|
2367
|
+
# write between our read and our replace must survive, not be lost to a stale copy
|
|
2368
|
+
acct = self.root / "acct-02"
|
|
2369
|
+
conf = acct / ".claude.json"
|
|
2370
|
+
real_mkstemp = tempfile.mkstemp
|
|
2371
|
+
for concurrent in ('{"numStartups": 99, "projects": {}}', # size differs
|
|
2372
|
+
'{"numStartups": 2}'): # same size, newer mtime
|
|
2373
|
+
conf.write_text('{"numStartups": 1}')
|
|
2374
|
+
os.utime(str(conf), (self.t0 - 100, self.t0 - 100))
|
|
2375
|
+
|
|
2376
|
+
def racing_mkstemp(*args, concurrent=concurrent, **kwargs):
|
|
2377
|
+
conf.write_text(concurrent) # Claude writes right after our read
|
|
2378
|
+
return real_mkstemp(*args, **kwargs)
|
|
2379
|
+
with mock.patch.object(ar.tempfile, "mkstemp", racing_mkstemp):
|
|
2380
|
+
self.assertIsNone(ar.cmd_trust(str(acct), str(self.work)))
|
|
2381
|
+
self.assertEqual(conf.read_text(), concurrent)
|
|
2382
|
+
self.assertEqual(sorted(p.name for p in acct.iterdir()),
|
|
2383
|
+
[".claude.json", "projects", "sessions"]) # no temp file left
|
|
2384
|
+
# untouched meanwhile: written
|
|
2385
|
+
conf.write_text('{"numStartups": 1}')
|
|
2386
|
+
ar.cmd_trust(str(acct), str(self.work))
|
|
2387
|
+
self.assertEqual(json.loads(conf.read_text())["projects"],
|
|
2388
|
+
{str(self.work): {"hasTrustDialogAccepted": True}})
|
|
2389
|
+
|
|
2390
|
+
def test_watch_on_a_malformed_state_exits_quietly(self):
|
|
2391
|
+
bad = self.ar_dir / "1234.state"
|
|
2392
|
+
bad.write_text("v=1\nprovider=claude\n")
|
|
2393
|
+
out = self.cli("watch", "--state", str(bad), timeout=15)
|
|
2394
|
+
self.assertEqual((out.returncode, out.stdout, out.stderr), (0, "", ""))
|
|
2395
|
+
|
|
2396
|
+
def test_watch_prunes_leftovers(self):
|
|
2397
|
+
old = self.ar_dir / "r-OldTokenOldToken.relaunch"
|
|
2398
|
+
old.write_text("v=1\n")
|
|
2399
|
+
os.utime(str(old), (time.time() - 3 * 86400,) * 2)
|
|
2400
|
+
fresh = self.ar_dir / "r-NewTokenNewToken.relaunch"
|
|
2401
|
+
fresh.write_text("v=1\n")
|
|
2402
|
+
# an old claim of a session that still runs stays (a neighbour's watcher reads
|
|
2403
|
+
# it); a dead one's goes
|
|
2404
|
+
live = self.ar_dir / ("%d.rollout" % os.getpid())
|
|
2405
|
+
dead = self.ar_dir / ("%d.rollout" % (2 ** 22 + 12346))
|
|
2406
|
+
for path in (live, dead):
|
|
2407
|
+
path.write_text("/x/rollout.jsonl")
|
|
2408
|
+
os.utime(str(path), (time.time() - 3 * 86400,) * 2)
|
|
2409
|
+
state = self.write_state([], pid=2 ** 22 + 12345) # no such process
|
|
2410
|
+
self.assertEqual(self.cli("watch", "--state", state, timeout=15).returncode, 0)
|
|
2411
|
+
self.assertFalse(old.exists())
|
|
2412
|
+
self.assertTrue(fresh.exists())
|
|
2413
|
+
self.assertTrue(live.exists())
|
|
2414
|
+
self.assertFalse(dead.exists())
|
|
2415
|
+
|
|
2416
|
+
@unittest.skipUnless(shutil.which("ps") and shutil.which("sleep"), "needs ps and sleep")
|
|
2417
|
+
def test_watch_end_to_end_with_real_processes(self):
|
|
2418
|
+
"""The real Runner: ps/lstart, kill, a probe subprocess and tmux on PATH."""
|
|
2419
|
+
bindir = self.tmp / "fakebin"
|
|
2420
|
+
bindir.mkdir()
|
|
2421
|
+
tmux_log = self.tmp / "tmux.log"
|
|
2422
|
+
probe_log = self.tmp / "probe.log"
|
|
2423
|
+
# Enter stands in for the relaunched shim, which consumes its token first thing
|
|
2424
|
+
(bindir / "tmux").write_text(
|
|
2425
|
+
"#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$FAKE_TMUX_LOG\"\n"
|
|
2426
|
+
"case \"$*\" in *display-message\\ -p*) echo \"$FAKE_PANE_PID|0|0|zsh\" ;;\n"
|
|
2427
|
+
" *send-keys*Enter) for f in \"$FAKE_AR_DIR\"/r-*.relaunch; do\n"
|
|
2428
|
+
" [ -f \"$f\" ] && mv \"$f\" \"$f.used\"; done ;; esac\n"
|
|
2429
|
+
"exit 0\n")
|
|
2430
|
+
os.makedirs(os.path.dirname(self.self_path), exist_ok=True)
|
|
2431
|
+
Path(self.self_path).write_text(
|
|
2432
|
+
"#!/bin/sh\nprintf 'avoid=%s probe=%s cfg=%s args=%s\\n' "
|
|
2433
|
+
"\"$CLAUDE_MULTIACC_AR_AVOID\" \"$CLAUDE_MULTIACC_AR_PROBE\" "
|
|
2434
|
+
"\"${CLAUDE_CONFIG_DIR:-unset}\" \"$*\" >> \"$FAKE_PROBE_LOG\"\n"
|
|
2435
|
+
"echo 'pick=acct-02 tier=eligible'\n")
|
|
2436
|
+
for path in (bindir / "tmux", Path(self.self_path)):
|
|
2437
|
+
path.chmod(0o755)
|
|
2438
|
+
# A disposable stand-in for the TUI, reparented to init so its exit is reaped, and
|
|
2439
|
+
# one for the pane's shell (ps must name a shell there). Both in sessions of their
|
|
2440
|
+
# own: no controlling terminal, whatever terminal (or none) runs this suite.
|
|
2441
|
+
out = subprocess.run(["sh", "-c", "sleep 120 </dev/null >/dev/null 2>&1 & echo $!"],
|
|
2442
|
+
stdout=subprocess.PIPE, universal_newlines=True, timeout=10,
|
|
2443
|
+
start_new_session=True)
|
|
2444
|
+
pid = int(out.stdout.strip())
|
|
2445
|
+
self.addCleanup(lambda: _kill_quietly(pid))
|
|
2446
|
+
shell = subprocess.Popen(["sh", "-c", "sleep 120; :"], stdin=subprocess.DEVNULL,
|
|
2447
|
+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
2448
|
+
start_new_session=True)
|
|
2449
|
+
self.addCleanup(lambda: (_kill_group_quietly(shell.pid), shell.wait()))
|
|
2450
|
+
self.PPID = shell.pid
|
|
2451
|
+
self.launched = int(time.time()) - 1
|
|
2452
|
+
state = self.write_state(["--dangerously-skip-permissions", "do the thing"], pid=pid)
|
|
2453
|
+
self.registry(pid=pid)
|
|
2454
|
+
self.transcript([cc_quota(time.time(), resets=int(time.time()) + 3600)])
|
|
2455
|
+
env = {"PATH": "%s:%s" % (bindir, os.environ.get("PATH", "/usr/bin:/bin")),
|
|
2456
|
+
"FAKE_TMUX_LOG": str(tmux_log), "FAKE_PROBE_LOG": str(probe_log),
|
|
2457
|
+
"FAKE_PANE_PID": str(self.PPID), "FAKE_AR_DIR": str(self.ar_dir),
|
|
2458
|
+
"CLAUDE_CONFIG_DIR": str(self.root / "acct-01"),
|
|
2459
|
+
"CLAUDE_ACCOUNTS_ROOT": str(self.root), "CLAUDE_MULTIACC_AR_TEST_TTY": "1",
|
|
2460
|
+
"CLAUDE_MULTIACC_AR_POLL": "0.1", "CLAUDE_MULTIACC_AR_GRACE": "0.3",
|
|
2461
|
+
"CLAUDE_MULTIACC_AR_TERM_GRACE": "3", "CLAUDE_MULTIACC_AR_PANE_WAIT": "5",
|
|
2462
|
+
"CLAUDE_MULTIACC_AUTORESUME_DEBUG": "1"}
|
|
2463
|
+
started = time.time()
|
|
2464
|
+
res = self.cli("watch", "--state", state, env=env, timeout=60)
|
|
2465
|
+
self.assertEqual(res.returncode, 0, res.stderr)
|
|
2466
|
+
self.assertLess(time.time() - started, 30)
|
|
2467
|
+
# The watcher returns once the pid is gone OR a zombie; reaping the orphan is
|
|
2468
|
+
# init's (or a CI subreaper's) job, and it may take a moment.
|
|
2469
|
+
deadline = time.time() + 10
|
|
2470
|
+
while time.time() < deadline:
|
|
2471
|
+
try:
|
|
2472
|
+
os.kill(pid, 0)
|
|
2473
|
+
except ProcessLookupError:
|
|
2474
|
+
break
|
|
2475
|
+
time.sleep(0.05)
|
|
2476
|
+
with self.assertRaises(ProcessLookupError):
|
|
2477
|
+
os.kill(pid, 0)
|
|
2478
|
+
probe = probe_log.read_text()
|
|
2479
|
+
self.assertIn("probe=1 cfg=unset args=--dangerously-skip-permissions --resume %s "
|
|
2480
|
+
"(claude-multiacc auto-resume)" % SID, probe)
|
|
2481
|
+
self.assertIn("avoid=acct-01:", probe)
|
|
2482
|
+
lines = [l for l in tmux_log.read_text().splitlines() if "send-keys" in l]
|
|
2483
|
+
sock = "-S /private/tmp/tmux-501/default"
|
|
2484
|
+
self.assertEqual(len(lines), 4, lines)
|
|
2485
|
+
self.assertEqual(lines[0], "%s send-keys -R -t %%7" % sock)
|
|
2486
|
+
self.assertEqual(lines[1], "%s send-keys -t %%7 C-u" % sock)
|
|
2487
|
+
# typed through the shim that launched the session, naming the pool it used
|
|
2488
|
+
self.assertRegex(lines[2], r"^%s send-keys -t %%7 -l CLAUDE_MULTIACC_AR="
|
|
2489
|
+
r"[A-Za-z0-9]{16}:%s CLAUDE_ACCOUNTS_ROOT=%s %s$"
|
|
2490
|
+
% (sock, SID, re.escape(str(self.root)), re.escape(self.self_path)))
|
|
2491
|
+
self.assertEqual(lines[3], "%s send-keys -t %%7 Enter" % sock)
|
|
2492
|
+
tok = lines[2].split("CLAUDE_MULTIACC_AR=", 1)[1].split(":")[0]
|
|
2493
|
+
self.assertTrue((self.ar_dir / ("r-%s.relaunch.used" % tok)).exists())
|
|
2494
|
+
# the watcher's own state/argv files are gone; its debug log stays
|
|
2495
|
+
self.assertFalse(Path(state).exists())
|
|
2496
|
+
self.assertFalse(Path(ar.argv_path_for(state)).exists())
|
|
2497
|
+
self.assertTrue((self.ar_dir / ("%d.log" % pid)).exists())
|
|
2498
|
+
self.assertEqual(self.events(), ["watch", "detect", "switch"])
|
|
2499
|
+
|
|
2500
|
+
|
|
2501
|
+
class RunnerTests(unittest.TestCase):
|
|
2502
|
+
"""The real side-effect layer, on this machine's own ps/lsof or /proc."""
|
|
2503
|
+
|
|
2504
|
+
def test_signal_refuses_init_groups_and_itself(self):
|
|
2505
|
+
# Signal 0 only: a regression here must never be able to hurt this machine.
|
|
2506
|
+
r = ar.Runner()
|
|
2507
|
+
for pid in (1, 0, -1, os.getpid(), "123", None):
|
|
2508
|
+
self.assertFalse(r.signal(pid, 0), pid)
|
|
2509
|
+
|
|
2510
|
+
def test_run_is_bounded(self):
|
|
2511
|
+
r = ar.Runner()
|
|
2512
|
+
started = time.time()
|
|
2513
|
+
# a grandchild left in the background must not hold the call open
|
|
2514
|
+
rc, out = r.run(["sh", "-c", "sleep 3 >/dev/null 2>&1 & echo hi"], 10)
|
|
2515
|
+
self.assertEqual((rc, out), (0, "hi\n"))
|
|
2516
|
+
self.assertLess(time.time() - started, 2.5)
|
|
2517
|
+
started = time.time()
|
|
2518
|
+
rc, _ = r.run(["sh", "-c", "sleep 30"], 0.5)
|
|
2519
|
+
self.assertIsNone(rc)
|
|
2520
|
+
self.assertLess(time.time() - started, 5)
|
|
2521
|
+
self.assertEqual(r.run(["/nonexistent/binary"], 5), (None, ""))
|
|
2522
|
+
|
|
2523
|
+
@unittest.skipUnless(shutil.which("ps"), "needs ps")
|
|
2524
|
+
def test_identity_and_zombies(self):
|
|
2525
|
+
r = ar.Runner()
|
|
2526
|
+
self.assertTrue(r.lstart(os.getpid()))
|
|
2527
|
+
self.assertIn((os.getpid(), os.getppid()),
|
|
2528
|
+
[(p, pp) for p, pp, _ in r.ps_table()])
|
|
2529
|
+
child = subprocess.Popen(["sh", "-c", "exit 0"])
|
|
2530
|
+
try:
|
|
2531
|
+
deadline = time.time() + 10
|
|
2532
|
+
while not r.gone(child.pid) and time.time() < deadline:
|
|
2533
|
+
time.sleep(0.05)
|
|
2534
|
+
# not reaped yet: kill -0 still succeeds, but it is gone
|
|
2535
|
+
self.assertTrue(r.alive(child.pid))
|
|
2536
|
+
self.assertTrue(r.gone(child.pid))
|
|
2537
|
+
finally:
|
|
2538
|
+
child.wait()
|
|
2539
|
+
self.assertFalse(r.alive(child.pid))
|
|
2540
|
+
self.assertIsNone(r.lstart(child.pid))
|
|
2541
|
+
|
|
2542
|
+
@unittest.skipUnless(os.path.isdir("/proc/self/fd") or shutil.which("lsof")
|
|
2543
|
+
or os.path.exists("/usr/sbin/lsof"), "needs /proc or lsof")
|
|
2544
|
+
def test_open_files(self):
|
|
2545
|
+
with tempfile.NamedTemporaryFile(suffix=".jsonl") as fh:
|
|
2546
|
+
names = {os.path.realpath(p) for p in ar.Runner().open_files(os.getpid())}
|
|
2547
|
+
self.assertIn(os.path.realpath(fh.name), names)
|
|
2548
|
+
|
|
2549
|
+
|
|
2550
|
+
def _kill_quietly(pid):
|
|
2551
|
+
try:
|
|
2552
|
+
os.kill(pid, signal.SIGKILL)
|
|
2553
|
+
except OSError:
|
|
2554
|
+
pass
|
|
2555
|
+
|
|
2556
|
+
|
|
2557
|
+
def _kill_group_quietly(pgid):
|
|
2558
|
+
try:
|
|
2559
|
+
os.killpg(pgid, signal.SIGKILL)
|
|
2560
|
+
except OSError:
|
|
2561
|
+
pass
|
|
2562
|
+
|
|
2563
|
+
|
|
2564
|
+
if __name__ == "__main__":
|
|
2565
|
+
unittest.main()
|