claude-multiacc 2.0.41 → 2.0.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -2
- package/bin/claude +414 -5
- package/bin/claude-accounts +36 -11
- package/bin/codex +406 -4
- package/docs/ACCOUNT_OPERATIONS.md +7 -1
- package/docs/AUTORESUME.md +321 -0
- package/docs/CODEX.md +12 -1
- package/docs/VERIFICATION.md +9 -1
- package/lib/__pycache__/audit.cpython-312.pyc +0 -0
- package/lib/__pycache__/autoresume.cpython-312.pyc +0 -0
- package/lib/__pycache__/claude_reset.cpython-312.pyc +0 -0
- package/lib/__pycache__/codex_config_edit.cpython-312.pyc +0 -0
- package/lib/__pycache__/codex_python.cpython-312.pyc +0 -0
- package/lib/__pycache__/keychain.cpython-312.pyc +0 -0
- package/lib/__pycache__/mcp_registry.cpython-312.pyc +0 -0
- package/lib/__pycache__/selector_policy.cpython-312.pyc +0 -0
- package/lib/__pycache__/selector_primitives.cpython-312.pyc +0 -0
- package/lib/__pycache__/shim_path.cpython-312.pyc +0 -0
- package/lib/autoresume.py +2271 -0
- package/lib/common.sh +24 -1
- package/lib/keychain.py +274 -59
- package/package.json +1 -1
- package/tests/__pycache__/packaged_command_support.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_autoresume.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_claude_reset.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_codex_reset.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_codex_reset_polling.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_codex_reset_reporting.cpython-312.pyc +0 -0
- package/tests/__pycache__/test_codex_reset_windows.cpython-312.pyc +0 -0
- package/tests/fake_security.sh +90 -0
- package/tests/run-tests.sh +1178 -67
- package/tests/test_autoresume.py +2565 -0
- package/tests/test_keychain.py +323 -50
|
@@ -0,0 +1,2271 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Auto-resume watcher: move a stuck interactive session to another pooled account.
|
|
3
|
+
|
|
4
|
+
An interactive ``claude`` or ``codex`` that runs into a usage limit does NOT exit. Claude
|
|
5
|
+
idles on "continuing automatically at <reset>" (which can be days away) and codex sits on
|
|
6
|
+
"You've hit your usage limit ... try again at <date>", both waiting for a human. The shim
|
|
7
|
+
(bin/claude, bin/codex) spawns this watcher, detached, right before its byte-identical
|
|
8
|
+
``exec`` of an interactive TUI inside tmux. The watcher tails that session's own
|
|
9
|
+
transcript (claude) or rollout (codex) and, on a verdict it can act on:
|
|
10
|
+
|
|
11
|
+
1. asks the SHIM which account a relaunch would get (probe mode: the real candidate loop,
|
|
12
|
+
``pick=<acct> tier=<t>``, never an exec) — so it only stops a session there is
|
|
13
|
+
somewhere to move to;
|
|
14
|
+
2. writes a single-use relaunch file (verdict + argv + chain state);
|
|
15
|
+
3. stops the client with SIGTERM (SIGKILL after a grace);
|
|
16
|
+
4. waits for the pane to be back at its SHELL and types
|
|
17
|
+
`` CLAUDE_MULTIACC_AR=<token>:<sid> [CLAUDE_ACCOUNTS_ROOT=<root>] <shim path>`` there.
|
|
18
|
+
|
|
19
|
+
It never types into a TUI: Claude's limit menu can "Add funds" or spend the one-shot
|
|
20
|
+
``/limit-reset`` on a stray Enter. The relaunch goes back through the shim, which parks
|
|
21
|
+
the old account with the existing writers and runs ordinary selection.
|
|
22
|
+
|
|
23
|
+
Everything fails OPEN: an unreadable state file, a missing transcript, an ambiguous
|
|
24
|
+
rollout, a pane that is not the launching shell — the watcher simply exits and the
|
|
25
|
+
session stays exactly as it would be without it. docs/AUTORESUME.md is the reference.
|
|
26
|
+
|
|
27
|
+
CLI (``python3 -I lib/autoresume.py ...``)::
|
|
28
|
+
|
|
29
|
+
watch --state FILE the daemon (spawned by the shim)
|
|
30
|
+
classify --provider claude|codex FILE [--since EPOCH] JSON verdicts (debug, tests)
|
|
31
|
+
relaunch-argv --provider P --state FILE --sid ID --class C JSON argv (tests)
|
|
32
|
+
trust --acct-dir DIR --cwd PATH mark PATH trusted in DIR/.claude.json
|
|
33
|
+
|
|
34
|
+
Python 3.9 compatible, stdlib only. The pure parts (classifiers, argv allowlist and
|
|
35
|
+
sanitizer, relaunch-file writer, budget math, tmux command builders, the codex "try again
|
|
36
|
+
at" parser) are plain functions; every process side effect (ps, lsof, kill, tmux, the
|
|
37
|
+
probe) goes through ``Runner`` so the tests can inject a fake one.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
import calendar
|
|
43
|
+
import glob
|
|
44
|
+
import json
|
|
45
|
+
import os
|
|
46
|
+
import re
|
|
47
|
+
import secrets
|
|
48
|
+
import shutil
|
|
49
|
+
import signal
|
|
50
|
+
import stat
|
|
51
|
+
import string
|
|
52
|
+
import subprocess
|
|
53
|
+
import sys
|
|
54
|
+
import tempfile
|
|
55
|
+
import time
|
|
56
|
+
import traceback
|
|
57
|
+
|
|
58
|
+
# ---- vocabulary ---------------------------------------------------------------------
|
|
59
|
+
|
|
60
|
+
PROVIDERS = ("claude", "codex")
|
|
61
|
+
|
|
62
|
+
# The fixed prefix of the auto-resume prompt. A user record that starts with it is the
|
|
63
|
+
# watcher's own relaunch, never a human typing, so it never cancels a verdict.
|
|
64
|
+
PROMPT_PREFIX = "(claude-multiacc auto-resume)"
|
|
65
|
+
|
|
66
|
+
PROMPT_TEMPLATE = (
|
|
67
|
+
PROMPT_PREFIX + " This session was restarted automatically{where} because {reason}. "
|
|
68
|
+
"Continue the task from where you left off; the user has not sent a new message. "
|
|
69
|
+
"Anything that was running in the background before the restart was stopped, so "
|
|
70
|
+
"re-check it before relying on it, and do not repeat work that is already done."
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
REASONS = {
|
|
74
|
+
"quota": "the previous account hit its usage limit",
|
|
75
|
+
"model": "the previous account hit this model's usage limit",
|
|
76
|
+
"auth": "the previous account's login failed",
|
|
77
|
+
"blocked": "the previous account was refused",
|
|
78
|
+
"transient": "an API error ended the last turn",
|
|
79
|
+
"crash": "the previous process exited unexpectedly",
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
# Classes that MOVE the session to a different account (the probe must offer an eligible
|
|
83
|
+
# one that is not the current account). transient/crash may land on the same account.
|
|
84
|
+
ROTATE = ("quota", "auth", "blocked", "model")
|
|
85
|
+
|
|
86
|
+
# How long the chain keeps away from the account it is leaving. quota uses the server's
|
|
87
|
+
# own reset instead (QUOTA_FALLBACK when it named none that is still ahead).
|
|
88
|
+
AVOID_SECONDS = {"auth": 3600, "blocked": 6 * 3600, "model": 5 * 3600}
|
|
89
|
+
QUOTA_FALLBACK = 3600
|
|
90
|
+
|
|
91
|
+
CLAUDE_QUOTA_TYPES = ("five_hour", "seven_day")
|
|
92
|
+
CLAUDE_TRANSIENT = ("overloaded", "server_error", "unknown")
|
|
93
|
+
CLAUDE_BLOCKED = ("oauth_org_not_allowed", "account_on_hold", "billing_error",
|
|
94
|
+
"verification_required")
|
|
95
|
+
# "You've reached your Fable limit. Switch to another model ..." — a MODEL-scoped limit:
|
|
96
|
+
# the account still serves other models, so it is avoided for a while, never parked.
|
|
97
|
+
MODEL_TEXT = re.compile(r"reached your .{0,40} limit|switch to another model", re.I)
|
|
98
|
+
# User records that are machinery, not a human at the keyboard.
|
|
99
|
+
NON_HUMAN_PREFIXES = ("<task-notification", "<local-command", "<system-reminder")
|
|
100
|
+
|
|
101
|
+
CODEX_QUOTA = ("usage_limit_exceeded", "rate_limit_exceeded")
|
|
102
|
+
CODEX_AUTH = ("unauthorized",)
|
|
103
|
+
CODEX_TRANSIENT = ("server_overloaded", "internal_server_error", "http_connection_failed",
|
|
104
|
+
"response_stream_connection_failed", "response_stream_disconnected",
|
|
105
|
+
"response_too_many_failed_attempts")
|
|
106
|
+
CODEX_CANCEL = ("user_message", "task_started", "turn_aborted")
|
|
107
|
+
|
|
108
|
+
# Shells the typed ``VAR=x /path/cmd`` line works in. fish only learned that syntax in
|
|
109
|
+
# 3.1; nu, tcsh, xonsh, pwsh... never: a pane running one of them is never stopped.
|
|
110
|
+
SHELLS = ("zsh", "bash", "sh", "dash", "ksh")
|
|
111
|
+
|
|
112
|
+
TOKEN_ALPHABET = string.ascii_letters + string.digits
|
|
113
|
+
ACCT_RE = re.compile(r"^acct-[A-Za-z0-9._-]{1,64}$")
|
|
114
|
+
PANE_RE = re.compile(r"^%[0-9]{1,9}$")
|
|
115
|
+
CHAIN_RE = re.compile(r"^[A-Za-z0-9]{1,64}$")
|
|
116
|
+
TOKEN_RE = re.compile(r"^[A-Za-z0-9]{8,64}$")
|
|
117
|
+
CLASS_RE = re.compile(r"^[a-z_]{1,16}$")
|
|
118
|
+
# What the relaunch line may carry unquoted (the shims' gate refuses anything else).
|
|
119
|
+
TYPEABLE_RE = re.compile(r"^/[A-Za-z0-9/._+-]*$")
|
|
120
|
+
ISO_RE = re.compile(r"^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?"
|
|
121
|
+
r"(Z|[+-]\d{2}:?\d{2})?$")
|
|
122
|
+
TRY_AGAIN_RE = re.compile(
|
|
123
|
+
r"try again at\s+([A-Za-z]{3})[A-Za-z]*\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(\d{4})"
|
|
124
|
+
r"\s+(\d{1,2}):(\d{2})\s*([AaPp])\.?[Mm]\.?")
|
|
125
|
+
MONTHS = {m: i for i, m in enumerate(
|
|
126
|
+
("jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"), 1)}
|
|
127
|
+
|
|
128
|
+
STATE_KEYS = ("v", "provider", "pid", "ppid", "acct", "acct_dir", "cwd", "launched", "tmux",
|
|
129
|
+
"pane", "self", "acc_root", "depth", "avoid", "hist", "chain")
|
|
130
|
+
RELAUNCH_KEYS = ("v", "provider", "class", "acct", "reset", "rtype", "marked", "sid", "cwd",
|
|
131
|
+
"depth", "avoid", "hist", "chain")
|
|
132
|
+
HIST_KEEP = 32
|
|
133
|
+
|
|
134
|
+
# ---- knobs --------------------------------------------------------------------------
|
|
135
|
+
# Every timing constant is overridable as <P>_MULTIACC_AR_<NAME> (P = CLAUDE or CODEX) so
|
|
136
|
+
# the suites can run the whole stop/relaunch sequence in a second. The last five are not
|
|
137
|
+
# in the spec's list; they bound waits the spec fixes (probe 30 s, rollout discovery
|
|
138
|
+
# 5 min, subagent quiet 20 s capped at 15 min, the typed relaunch taking effect 10 s) and
|
|
139
|
+
# exist for the same reason.
|
|
140
|
+
KNOBS = {
|
|
141
|
+
"POLL": 1.0,
|
|
142
|
+
"GRACE": 5.0,
|
|
143
|
+
"HOLD_REPROBE": 60.0,
|
|
144
|
+
"MAX_DEPTH": 20,
|
|
145
|
+
"ROTATE_PER_HOUR": 8,
|
|
146
|
+
"TRANSIENT_PER_HOUR": 3,
|
|
147
|
+
"CRASH_PER_10MIN": 2,
|
|
148
|
+
"CRASH_MIN_RUNTIME": 60.0,
|
|
149
|
+
"TERM_GRACE": 10.0,
|
|
150
|
+
"PANE_WAIT": 15.0,
|
|
151
|
+
"PROBE_TIMEOUT": 30.0,
|
|
152
|
+
"DISCOVER_TIMEOUT": 300.0,
|
|
153
|
+
"SUBAGENT_QUIET": 20.0,
|
|
154
|
+
"SUBAGENT_CAP": 900.0,
|
|
155
|
+
"RELAUNCH_WAIT": 10.0,
|
|
156
|
+
}
|
|
157
|
+
LADDER_DEFAULT = (30.0, 60.0, 120.0)
|
|
158
|
+
# After DISCOVER_TIMEOUT a codex watcher still looks for its rollout (a session first
|
|
159
|
+
# used an hour after it was opened is still a session), just this rarely.
|
|
160
|
+
DISCOVER_SLOW = 15.0
|
|
161
|
+
# The pane counts as back at its prompt only when two readings this far apart agree: one
|
|
162
|
+
# reading can land in the instant between two programs a shell loop runs.
|
|
163
|
+
SHELL_SETTLE = 0.5
|
|
164
|
+
MAX_TREE = 64 # more "descendants" than this is not one TUI's tree: kill none of them
|
|
165
|
+
# The cwd fallback is for a machine that cannot list open files; a lookup that is there
|
|
166
|
+
# but has failed this long counts as not there.
|
|
167
|
+
LOOKUP_FAILED = 30.0
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _knob(raw, default):
|
|
171
|
+
try:
|
|
172
|
+
val = float(raw)
|
|
173
|
+
except (TypeError, ValueError):
|
|
174
|
+
return default
|
|
175
|
+
if val != val or val < 0 or val > 10 ** 7: # NaN, negative, absurd
|
|
176
|
+
return default
|
|
177
|
+
return int(val) if isinstance(default, int) else val
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _ladder(raw):
|
|
181
|
+
if not raw:
|
|
182
|
+
return LADDER_DEFAULT
|
|
183
|
+
steps = []
|
|
184
|
+
for part in str(raw).split(","):
|
|
185
|
+
val = _knob(part.strip(), -1.0)
|
|
186
|
+
if val < 0:
|
|
187
|
+
return LADDER_DEFAULT
|
|
188
|
+
steps.append(float(val))
|
|
189
|
+
return tuple(steps) or LADDER_DEFAULT
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
class Config:
|
|
193
|
+
"""The watcher's constants for one provider, read from the environment."""
|
|
194
|
+
|
|
195
|
+
def __init__(self, provider, environ=None):
|
|
196
|
+
env = os.environ if environ is None else environ
|
|
197
|
+
up = provider.upper()
|
|
198
|
+
pfx = up + "_MULTIACC_AR_"
|
|
199
|
+
for name, default in KNOBS.items():
|
|
200
|
+
setattr(self, name.lower(), _knob(env.get(pfx + name), default))
|
|
201
|
+
# A zero poll would spin a core; 20 ms is already far below anything useful.
|
|
202
|
+
self.poll = max(self.poll, 0.02)
|
|
203
|
+
self.transient_ladder = _ladder(env.get(pfx + "TRANSIENT_LADDER"))
|
|
204
|
+
self.anypane = env.get(pfx + "TEST_TMUX_ANYPANE") == "1"
|
|
205
|
+
# The shim's own no-terminal test knob: a client with no terminal cannot be in its
|
|
206
|
+
# foreground, so only then is the foreground requirement waived.
|
|
207
|
+
self.test_tty = env.get(pfx + "TEST_TTY") == "1"
|
|
208
|
+
self.debug = env.get(up + "_MULTIACC_AUTORESUME_DEBUG") == "1"
|
|
209
|
+
override = env.get(up + "_MULTIACC_AUTORESUME_PROMPT")
|
|
210
|
+
self.prompt = override if prompt_override_ok(override) else None
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
# ---- small parsers ------------------------------------------------------------------
|
|
214
|
+
|
|
215
|
+
def num_ok(val):
|
|
216
|
+
"""The shims' num_ok: digits only, at most 18 of them."""
|
|
217
|
+
return isinstance(val, str) and val.isdigit() and val.isascii() and len(val) <= 18
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def sess_id_ok(val):
|
|
221
|
+
"""The shims' sess_id_ok: hex and dashes, never a leading dash."""
|
|
222
|
+
return (isinstance(val, str) and val != "" and not val.startswith("-")
|
|
223
|
+
and all(c in "0123456789abcdefABCDEF-" for c in val))
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def uuid_ok(val):
|
|
227
|
+
"""A session id a relaunch may name (the shims' ar_uuid_ok): dashed, bounded."""
|
|
228
|
+
return sess_id_ok(val) and "-" in val and len(val) <= 64
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def epoch_of(val):
|
|
232
|
+
"""A positive epoch (seconds) from a JSON number or digit string; None otherwise."""
|
|
233
|
+
if isinstance(val, bool):
|
|
234
|
+
return None
|
|
235
|
+
if isinstance(val, (int, float)):
|
|
236
|
+
if val != val or val <= 0 or val >= 10 ** 11:
|
|
237
|
+
return None
|
|
238
|
+
return int(val)
|
|
239
|
+
if isinstance(val, str) and num_ok(val) and 0 < int(val) < 10 ** 11:
|
|
240
|
+
return int(val)
|
|
241
|
+
return None
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def iso_to_epoch(val):
|
|
245
|
+
"""``2026-09-22T21:36:58.009Z`` -> float epoch; None when it is not that shape."""
|
|
246
|
+
if not isinstance(val, str):
|
|
247
|
+
return None
|
|
248
|
+
m = ISO_RE.match(val.strip())
|
|
249
|
+
if not m:
|
|
250
|
+
return None
|
|
251
|
+
try:
|
|
252
|
+
base = calendar.timegm((int(m.group(1)), int(m.group(2)), int(m.group(3)),
|
|
253
|
+
int(m.group(4)), int(m.group(5)), int(m.group(6)), 0, 0, 0))
|
|
254
|
+
except (ValueError, OverflowError):
|
|
255
|
+
return None
|
|
256
|
+
frac = float(m.group(7)) if m.group(7) else 0.0
|
|
257
|
+
tz = m.group(8)
|
|
258
|
+
off = 0
|
|
259
|
+
if tz and tz != "Z":
|
|
260
|
+
sign = -1 if tz[0] == "-" else 1
|
|
261
|
+
digits = tz[1:].replace(":", "")
|
|
262
|
+
off = sign * (int(digits[:2]) * 3600 + int(digits[2:]) * 60)
|
|
263
|
+
return base + frac - off
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def utc_iso(epoch):
|
|
267
|
+
"""Epoch -> ``YYYY-MM-DDTHH:MM:SSZ`` (the shims' marked_at / sel_log shape)."""
|
|
268
|
+
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(int(epoch)))
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def parse_kv(text, keys):
|
|
272
|
+
"""``key=value`` lines -> dict of the known keys (last one wins, unknown ignored)."""
|
|
273
|
+
out = {}
|
|
274
|
+
for line in text.split("\n"):
|
|
275
|
+
if "=" not in line:
|
|
276
|
+
continue
|
|
277
|
+
key, val = line.split("=", 1)
|
|
278
|
+
if key in keys:
|
|
279
|
+
out[key] = val
|
|
280
|
+
return out
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def avoid_parse(raw, now):
|
|
284
|
+
"""``acct-NN:until,...`` -> {acct: until} with only the entries still in force."""
|
|
285
|
+
out = {}
|
|
286
|
+
for entry in (raw or "").split(",")[:256]:
|
|
287
|
+
acct, _, until = entry.partition(":")
|
|
288
|
+
if not ACCT_RE.match(acct) or not num_ok(until) or int(until) <= now:
|
|
289
|
+
continue
|
|
290
|
+
out[acct] = max(out.get(acct, 0), int(until))
|
|
291
|
+
return out
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def avoid_format(entries):
|
|
295
|
+
return ",".join("%s:%d" % (a, entries[a]) for a in sorted(entries))
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def avoid_merge(raw, now, acct=None, until=None):
|
|
299
|
+
"""The chain's AVOID list with expired entries dropped, plus ``acct`` until ``until``
|
|
300
|
+
(never shortening an entry already there)."""
|
|
301
|
+
entries = avoid_parse(raw, now)
|
|
302
|
+
if acct and until and int(until) > now:
|
|
303
|
+
entries[acct] = max(entries.get(acct, 0), int(until))
|
|
304
|
+
return avoid_format(entries)
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def hist_parse(raw):
|
|
308
|
+
"""``class:epoch,...`` -> [(class, epoch)] — malformed entries dropped."""
|
|
309
|
+
out = []
|
|
310
|
+
for entry in (raw or "").split(",")[:1024]:
|
|
311
|
+
cls, _, when = entry.partition(":")
|
|
312
|
+
if CLASS_RE.match(cls) and num_ok(when):
|
|
313
|
+
out.append((cls, int(when)))
|
|
314
|
+
return out
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def hist_append(raw, cls, now):
|
|
318
|
+
"""The chain history with this verdict appended; the newest HIST_KEEP entries."""
|
|
319
|
+
entries = hist_parse(raw) + [(cls, int(now))]
|
|
320
|
+
return ",".join("%s:%d" % e for e in entries[-HIST_KEEP:])
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def make_token(length=16):
|
|
324
|
+
return "".join(secrets.choice(TOKEN_ALPHABET) for _ in range(length))
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
# ---- budgets ------------------------------------------------------------------------
|
|
328
|
+
|
|
329
|
+
def budget_check(cls, depth, hist, now, cfg):
|
|
330
|
+
"""(ok, reason). A chain stops moving at MAX_DEPTH relaunches for good; the hourly
|
|
331
|
+
windows only pause it (they slide, so a held verdict may proceed later)."""
|
|
332
|
+
if depth >= cfg.max_depth:
|
|
333
|
+
return False, "depth"
|
|
334
|
+
entries = hist_parse(hist) if isinstance(hist, str) else list(hist)
|
|
335
|
+
if cls in ROTATE:
|
|
336
|
+
used = sum(1 for c, t in entries if c in ROTATE and now - t < 3600)
|
|
337
|
+
if used >= cfg.rotate_per_hour:
|
|
338
|
+
return False, "rotate-budget"
|
|
339
|
+
elif cls == "transient":
|
|
340
|
+
used = sum(1 for c, t in entries if c == "transient" and now - t < 3600)
|
|
341
|
+
if used >= cfg.transient_per_hour:
|
|
342
|
+
return False, "transient-budget"
|
|
343
|
+
elif cls == "crash":
|
|
344
|
+
used = sum(1 for c, t in entries if c == "crash" and now - t < 600)
|
|
345
|
+
if used >= cfg.crash_per_10min:
|
|
346
|
+
return False, "crash-budget"
|
|
347
|
+
return True, ""
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def ladder_wait(hist, now, ladder):
|
|
351
|
+
"""How long a transient verdict must stand before the watcher acts: the next step of
|
|
352
|
+
the ladder after this chain's transient relaunches in the last hour."""
|
|
353
|
+
entries = hist_parse(hist) if isinstance(hist, str) else list(hist)
|
|
354
|
+
used = sum(1 for c, t in entries if c == "transient" and now - t < 3600)
|
|
355
|
+
return ladder[min(used, len(ladder) - 1)] if ladder else 0.0
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def avoid_until(verdict, now):
|
|
359
|
+
"""Until when the chain keeps away from the account a verdict is leaving."""
|
|
360
|
+
cls = verdict["class"]
|
|
361
|
+
if cls == "quota":
|
|
362
|
+
reset = verdict.get("reset")
|
|
363
|
+
return int(reset) if reset and reset > now else int(now) + QUOTA_FALLBACK
|
|
364
|
+
return int(now) + AVOID_SECONDS.get(cls, QUOTA_FALLBACK)
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
# ---- claude classification ----------------------------------------------------------
|
|
368
|
+
|
|
369
|
+
def _message_text(rec):
|
|
370
|
+
msg = rec.get("message")
|
|
371
|
+
content = msg.get("content") if isinstance(msg, dict) else None
|
|
372
|
+
if isinstance(content, str):
|
|
373
|
+
return content
|
|
374
|
+
if isinstance(content, list):
|
|
375
|
+
return " ".join(b.get("text") for b in content
|
|
376
|
+
if isinstance(b, dict) and isinstance(b.get("text"), str))
|
|
377
|
+
return ""
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def claude_human_input(rec, prefixes=(PROMPT_PREFIX,)):
|
|
381
|
+
"""True for a main-thread user record a HUMAN produced (typed text, a slash command,
|
|
382
|
+
a pasted image) — never a tool result, meta/compaction record, a harness
|
|
383
|
+
notification, or the watcher's own auto-resume prompt."""
|
|
384
|
+
if rec.get("isMeta") is True or rec.get("isCompactSummary") is True \
|
|
385
|
+
or rec.get("isVisibleInTranscriptOnly") is True or "toolUseResult" in rec:
|
|
386
|
+
return False
|
|
387
|
+
msg = rec.get("message")
|
|
388
|
+
content = msg.get("content") if isinstance(msg, dict) else None
|
|
389
|
+
if isinstance(content, list):
|
|
390
|
+
blocks = [b for b in content if isinstance(b, dict)]
|
|
391
|
+
if not blocks or any(b.get("type") == "tool_result" for b in blocks):
|
|
392
|
+
return False
|
|
393
|
+
text = " ".join(b.get("text") for b in blocks
|
|
394
|
+
if b.get("type") == "text" and isinstance(b.get("text"), str))
|
|
395
|
+
elif isinstance(content, str):
|
|
396
|
+
text = content
|
|
397
|
+
else:
|
|
398
|
+
return False
|
|
399
|
+
head = text.lstrip()
|
|
400
|
+
if head.startswith(NON_HUMAN_PREFIXES):
|
|
401
|
+
return False
|
|
402
|
+
return not any(p and head.startswith(p) for p in prefixes)
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def _verdict(cls, code, at, reset=None, rtype=""):
|
|
406
|
+
return {"class": cls, "code": code or "?", "at": at, "reset": reset, "rtype": rtype,
|
|
407
|
+
"marked": utc_iso(at) if at else ""}
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def classify_claude(rec, prefixes=(PROMPT_PREFIX,)):
|
|
411
|
+
"""One transcript record -> None | ("error", verdict) | ("never", code) |
|
|
412
|
+
("cancel", why). No time filtering here — the tracker applies it."""
|
|
413
|
+
if not isinstance(rec, dict) or rec.get("isSidechain") is True:
|
|
414
|
+
return None
|
|
415
|
+
typ = rec.get("type")
|
|
416
|
+
if typ == "user":
|
|
417
|
+
return ("cancel", "user") if claude_human_input(rec, prefixes) else None
|
|
418
|
+
if typ != "assistant":
|
|
419
|
+
return None # system records ("continuing automatically at ...") never count
|
|
420
|
+
if rec.get("isApiErrorMessage") is not True:
|
|
421
|
+
return ("cancel", "assistant") # a real reply: the user or auto-continue moved on
|
|
422
|
+
code = rec.get("error") if isinstance(rec.get("error"), str) else ""
|
|
423
|
+
at = iso_to_epoch(rec.get("timestamp"))
|
|
424
|
+
if rec.get("apiError") == "model_requires_usage_credits":
|
|
425
|
+
return ("error", _verdict("model", code or "model_requires_usage_credits", at))
|
|
426
|
+
if code == "rate_limit":
|
|
427
|
+
quota = rec.get("quotaLimits")
|
|
428
|
+
if isinstance(quota, dict) and quota.get("status") == "rejected":
|
|
429
|
+
rtype = quota.get("rateLimitType")
|
|
430
|
+
if rtype in CLAUDE_QUOTA_TYPES:
|
|
431
|
+
return ("error", _verdict("quota", code, at, epoch_of(quota.get("resetsAt")),
|
|
432
|
+
rtype))
|
|
433
|
+
# seven_day_opus, seven_day_overage_included, ...: one model's bucket.
|
|
434
|
+
return ("error", _verdict("model", code, at))
|
|
435
|
+
if MODEL_TEXT.search(_message_text(rec)):
|
|
436
|
+
return ("error", _verdict("model", code, at))
|
|
437
|
+
return ("error", _verdict("transient", code, at))
|
|
438
|
+
if code in CLAUDE_TRANSIENT:
|
|
439
|
+
return ("error", _verdict("transient", code, at))
|
|
440
|
+
if code == "authentication_failed":
|
|
441
|
+
return ("error", _verdict("auth", code, at))
|
|
442
|
+
if code in CLAUDE_BLOCKED:
|
|
443
|
+
return ("error", _verdict("blocked", code, at))
|
|
444
|
+
# invalid_request, max_output_tokens, model_not_found, ...: another account would
|
|
445
|
+
# fail the same way, so it is logged once and left alone.
|
|
446
|
+
return ("never", code or "?")
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
class ClaudeTracker:
|
|
450
|
+
"""Folds a claude transcript's records into at most one pending verdict."""
|
|
451
|
+
|
|
452
|
+
def __init__(self, since, prefixes=(PROMPT_PREFIX,)):
|
|
453
|
+
self.since = since
|
|
454
|
+
self.prefixes = prefixes
|
|
455
|
+
self.pending = None
|
|
456
|
+
self.pending_since = None
|
|
457
|
+
|
|
458
|
+
def clear(self):
|
|
459
|
+
self.pending = None
|
|
460
|
+
self.pending_since = None
|
|
461
|
+
|
|
462
|
+
def wants(self, raw):
|
|
463
|
+
"""Cheap byte prefilter: with nothing pending only an API error can matter (a
|
|
464
|
+
cancel needs something to cancel), so a resumed 100 MB transcript is scanned for
|
|
465
|
+
one substring instead of parsed line by line."""
|
|
466
|
+
return self.pending is not None or b'"isApiErrorMessage"' in raw
|
|
467
|
+
|
|
468
|
+
def feed(self, rec, now):
|
|
469
|
+
ev = classify_claude(rec, self.prefixes)
|
|
470
|
+
if ev is None:
|
|
471
|
+
return None
|
|
472
|
+
kind, val = ev
|
|
473
|
+
stamp = iso_to_epoch(rec.get("timestamp"))
|
|
474
|
+
if kind == "cancel":
|
|
475
|
+
# A cancel only ever stops an action, so one without a timestamp still counts.
|
|
476
|
+
if self.pending is None or (stamp is not None and stamp < self.since):
|
|
477
|
+
return None
|
|
478
|
+
self.clear()
|
|
479
|
+
return ev
|
|
480
|
+
# An error must PROVE it is from this launch: a resumed transcript carries every
|
|
481
|
+
# rejection the session ever met, and an old one must never fire.
|
|
482
|
+
if stamp is None or stamp < self.since:
|
|
483
|
+
return None
|
|
484
|
+
if kind == "never":
|
|
485
|
+
return ev
|
|
486
|
+
if self.pending is None or self.pending["class"] != val["class"]:
|
|
487
|
+
self.pending_since = now
|
|
488
|
+
self.pending = val
|
|
489
|
+
return ev
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
# ---- codex classification -----------------------------------------------------------
|
|
493
|
+
|
|
494
|
+
def codex_error_code(info):
|
|
495
|
+
"""``codex_error_info`` is a plain string or a one-key object (a Rust enum variant
|
|
496
|
+
with data, e.g. {"http_connection_failed": {"http_status_code": null}})."""
|
|
497
|
+
if isinstance(info, str):
|
|
498
|
+
return info
|
|
499
|
+
if isinstance(info, dict) and info:
|
|
500
|
+
key = next(iter(info))
|
|
501
|
+
return key if isinstance(key, str) else ""
|
|
502
|
+
return ""
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def codex_rate_snapshot(rate_limits):
|
|
506
|
+
"""The usable window of a token_count's ``rate_limits``: only ``limit_id == "codex"``
|
|
507
|
+
with a non-null window counts (the all-null ``premium`` line that precedes most
|
|
508
|
+
errors says nothing), and of its windows the most-used one."""
|
|
509
|
+
if not isinstance(rate_limits, dict) or rate_limits.get("limit_id") != "codex":
|
|
510
|
+
return None
|
|
511
|
+
best = None
|
|
512
|
+
for key in ("primary", "secondary"):
|
|
513
|
+
win = rate_limits.get(key)
|
|
514
|
+
if not isinstance(win, dict):
|
|
515
|
+
continue
|
|
516
|
+
used = win.get("used_percent")
|
|
517
|
+
used = float(used) if isinstance(used, (int, float)) and not isinstance(used, bool) \
|
|
518
|
+
else -1.0
|
|
519
|
+
cand = {"used_percent": used, "resets_at": epoch_of(win.get("resets_at")),
|
|
520
|
+
"window_minutes": win.get("window_minutes")}
|
|
521
|
+
if best is None or cand["used_percent"] > best["used_percent"]:
|
|
522
|
+
best = cand
|
|
523
|
+
return best
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
def parse_try_again(message):
|
|
527
|
+
""""... try again at Sep 26th, 2026 11:22 AM." -> epoch, read as LOCAL time (that is
|
|
528
|
+
how the codex CLI renders it); None when the message carries no such date."""
|
|
529
|
+
if not isinstance(message, str):
|
|
530
|
+
return None
|
|
531
|
+
m = TRY_AGAIN_RE.search(message)
|
|
532
|
+
if not m:
|
|
533
|
+
return None
|
|
534
|
+
month = MONTHS.get(m.group(1).lower())
|
|
535
|
+
day, year, hour, minute = int(m.group(2)), int(m.group(3)), int(m.group(4)), \
|
|
536
|
+
int(m.group(5))
|
|
537
|
+
if not month or not 1 <= day <= 31 or not 1 <= hour <= 12 or minute > 59:
|
|
538
|
+
return None
|
|
539
|
+
hour = hour % 12 + (12 if m.group(6).lower() == "p" else 0)
|
|
540
|
+
try:
|
|
541
|
+
return int(time.mktime((year, month, day, hour, minute, 0, 0, 0, -1)))
|
|
542
|
+
except (OverflowError, ValueError):
|
|
543
|
+
return None
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def codex_quota_reset(snapshot, message, now):
|
|
547
|
+
"""(reset epoch, window label) for a codex quota verdict — used for AVOID only (the
|
|
548
|
+
shim writes a 10-minute error-cooldown, never a sticky weekly marker): the last codex
|
|
549
|
+
token_count's most-used window, else the message's "try again at", else an hour."""
|
|
550
|
+
if snapshot and snapshot.get("resets_at") and snapshot["resets_at"] > now:
|
|
551
|
+
mins = snapshot.get("window_minutes")
|
|
552
|
+
label = ""
|
|
553
|
+
if isinstance(mins, (int, float)) and not isinstance(mins, bool):
|
|
554
|
+
label = "7d" if mins >= 1440 else "5h"
|
|
555
|
+
return int(snapshot["resets_at"]), label
|
|
556
|
+
when = parse_try_again(message)
|
|
557
|
+
if when and when > now:
|
|
558
|
+
return when, ""
|
|
559
|
+
return int(now) + QUOTA_FALLBACK, ""
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def classify_codex(rec):
|
|
563
|
+
"""One rollout record -> None | ("meta", payload) | ("rate", snapshot) |
|
|
564
|
+
("error", verdict) | ("never", code) | ("cancel", type). Untimed."""
|
|
565
|
+
if not isinstance(rec, dict):
|
|
566
|
+
return None
|
|
567
|
+
payload = rec.get("payload")
|
|
568
|
+
if not isinstance(payload, dict):
|
|
569
|
+
return None
|
|
570
|
+
if rec.get("type") == "session_meta":
|
|
571
|
+
return ("meta", payload)
|
|
572
|
+
if rec.get("type") != "event_msg":
|
|
573
|
+
return None
|
|
574
|
+
ptype = payload.get("type")
|
|
575
|
+
if ptype == "token_count":
|
|
576
|
+
snap = codex_rate_snapshot(payload.get("rate_limits"))
|
|
577
|
+
return ("rate", snap) if snap else None
|
|
578
|
+
if ptype in CODEX_CANCEL:
|
|
579
|
+
return ("cancel", ptype)
|
|
580
|
+
if ptype != "task_complete" or not isinstance(payload.get("error"), dict):
|
|
581
|
+
return None
|
|
582
|
+
err = payload["error"]
|
|
583
|
+
code = codex_error_code(err.get("codex_error_info"))
|
|
584
|
+
at = epoch_of(payload.get("completed_at"))
|
|
585
|
+
if at is None:
|
|
586
|
+
at = iso_to_epoch(rec.get("timestamp"))
|
|
587
|
+
if code in CODEX_QUOTA:
|
|
588
|
+
cls = "quota"
|
|
589
|
+
elif code in CODEX_AUTH:
|
|
590
|
+
cls = "auth"
|
|
591
|
+
elif code in CODEX_TRANSIENT:
|
|
592
|
+
cls = "transient"
|
|
593
|
+
else:
|
|
594
|
+
return ("never", code or "?")
|
|
595
|
+
verdict = _verdict(cls, code, at)
|
|
596
|
+
# Kept in memory for the "try again at" parse only; never logged or printed.
|
|
597
|
+
msg = err.get("message")
|
|
598
|
+
verdict["_message"] = msg if isinstance(msg, str) else ""
|
|
599
|
+
return ("error", verdict)
|
|
600
|
+
|
|
601
|
+
|
|
602
|
+
class CodexTracker:
|
|
603
|
+
"""Folds a codex rollout's records into at most one pending verdict."""
|
|
604
|
+
|
|
605
|
+
def __init__(self, since):
|
|
606
|
+
self.since = since
|
|
607
|
+
self.pending = None
|
|
608
|
+
self.pending_since = None
|
|
609
|
+
self.snapshot = None
|
|
610
|
+
|
|
611
|
+
def clear(self):
|
|
612
|
+
self.pending = None
|
|
613
|
+
self.pending_since = None
|
|
614
|
+
|
|
615
|
+
def wants(self, raw):
|
|
616
|
+
if b'"token_count"' in raw or b'"task_complete"' in raw:
|
|
617
|
+
return True
|
|
618
|
+
return self.pending is not None and (
|
|
619
|
+
b'"user_message"' in raw or b'"task_started"' in raw or b'"turn_aborted"' in raw)
|
|
620
|
+
|
|
621
|
+
def feed(self, rec, now):
|
|
622
|
+
ev = classify_codex(rec)
|
|
623
|
+
if ev is None or ev[0] == "meta":
|
|
624
|
+
return None
|
|
625
|
+
kind, val = ev
|
|
626
|
+
if kind == "error":
|
|
627
|
+
stamp = val["at"]
|
|
628
|
+
else:
|
|
629
|
+
stamp = iso_to_epoch(rec.get("timestamp"))
|
|
630
|
+
if kind == "rate":
|
|
631
|
+
if stamp is not None and stamp >= self.since:
|
|
632
|
+
self.snapshot = val
|
|
633
|
+
return None
|
|
634
|
+
if kind == "cancel":
|
|
635
|
+
if self.pending is None or (stamp is not None and stamp < self.since):
|
|
636
|
+
return None
|
|
637
|
+
self.clear()
|
|
638
|
+
return ev
|
|
639
|
+
if stamp is None or stamp < self.since:
|
|
640
|
+
return None
|
|
641
|
+
if kind == "never":
|
|
642
|
+
return ev
|
|
643
|
+
if val["class"] == "quota":
|
|
644
|
+
val["reset"], val["rtype_window"] = codex_quota_reset(
|
|
645
|
+
self.snapshot, val.get("_message"), now)
|
|
646
|
+
if self.pending is None or self.pending["class"] != val["class"]:
|
|
647
|
+
self.pending_since = now
|
|
648
|
+
self.pending = val
|
|
649
|
+
return ev
|
|
650
|
+
|
|
651
|
+
|
|
652
|
+
def make_tracker(provider, since, prefixes=(PROMPT_PREFIX,)):
|
|
653
|
+
return ClaudeTracker(since, prefixes) if provider == "claude" else CodexTracker(since)
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
def verdict_public(verdict):
|
|
657
|
+
"""A verdict without its in-memory-only fields (the codex error message)."""
|
|
658
|
+
if verdict is None:
|
|
659
|
+
return None
|
|
660
|
+
return {k: v for k, v in verdict.items() if not k.startswith("_")}
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
# ---- argv: allowlist and relaunch ---------------------------------------------------
|
|
664
|
+
|
|
665
|
+
def _value_ok(val):
|
|
666
|
+
return isinstance(val, str) and val != "" and not val.startswith("-")
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
def claude_argv_parse(argv):
|
|
670
|
+
"""The argv shapes a claude relaunch can rebuild (mirrors bin/claude's ar_argv_ok) ->
|
|
671
|
+
the kept options, with -c/--continue/--resume and the prompt dropped, or None for
|
|
672
|
+
anything else."""
|
|
673
|
+
kept, prompt, i = [], False, 0
|
|
674
|
+
while i < len(argv):
|
|
675
|
+
arg = argv[i]
|
|
676
|
+
if arg in ("--dangerously-skip-permissions", "--allow-dangerously-skip-permissions"):
|
|
677
|
+
kept.append(arg)
|
|
678
|
+
elif arg in ("-c", "--continue"):
|
|
679
|
+
pass
|
|
680
|
+
elif arg in ("-r", "--resume"):
|
|
681
|
+
if i + 1 >= len(argv) or not uuid_ok(argv[i + 1]):
|
|
682
|
+
return None
|
|
683
|
+
i += 1
|
|
684
|
+
elif arg.startswith("--resume="):
|
|
685
|
+
if not uuid_ok(arg[len("--resume="):]):
|
|
686
|
+
return None
|
|
687
|
+
elif arg in ("--model", "--effort", "--permission-mode"):
|
|
688
|
+
if i + 1 >= len(argv) or not _value_ok(argv[i + 1]):
|
|
689
|
+
return None
|
|
690
|
+
kept += [arg, argv[i + 1]]
|
|
691
|
+
i += 1
|
|
692
|
+
elif arg.startswith("--model=") and len(arg) > len("--model="):
|
|
693
|
+
kept.append(arg)
|
|
694
|
+
elif arg.startswith("-"):
|
|
695
|
+
return None
|
|
696
|
+
else:
|
|
697
|
+
# One positional, and it must look like a prompt: a single word may be a
|
|
698
|
+
# subcommand (`claude doctor`), which is nothing to resume.
|
|
699
|
+
if prompt or " " not in arg:
|
|
700
|
+
return None
|
|
701
|
+
prompt = True
|
|
702
|
+
i += 1
|
|
703
|
+
return kept
|
|
704
|
+
|
|
705
|
+
|
|
706
|
+
def codex_argv_parse(argv):
|
|
707
|
+
"""The argv shapes a codex relaunch can rebuild (mirrors bin/codex's ar_argv_ok) ->
|
|
708
|
+
(kept options, resume target or None), or None for anything else."""
|
|
709
|
+
kept, resume, target, prompt, i = [], False, None, False, 0
|
|
710
|
+
while i < len(argv):
|
|
711
|
+
arg = argv[i]
|
|
712
|
+
if arg in ("--dangerously-bypass-approvals-and-sandbox", "--yolo"):
|
|
713
|
+
kept.append(arg)
|
|
714
|
+
elif arg in ("-m", "--model"):
|
|
715
|
+
if i + 1 >= len(argv) or not _value_ok(argv[i + 1]):
|
|
716
|
+
return None
|
|
717
|
+
kept += [arg, argv[i + 1]]
|
|
718
|
+
i += 1
|
|
719
|
+
elif arg.startswith("--model=") and len(arg) > len("--model="):
|
|
720
|
+
kept.append(arg)
|
|
721
|
+
elif arg == "--last":
|
|
722
|
+
if not resume or target is not None:
|
|
723
|
+
return None
|
|
724
|
+
target = "--last"
|
|
725
|
+
elif arg == "resume" and not resume and not prompt:
|
|
726
|
+
resume = True
|
|
727
|
+
elif arg.startswith("-"):
|
|
728
|
+
return None
|
|
729
|
+
elif resume and target is None:
|
|
730
|
+
if not uuid_ok(arg):
|
|
731
|
+
return None
|
|
732
|
+
target = arg
|
|
733
|
+
else:
|
|
734
|
+
if prompt or " " not in arg:
|
|
735
|
+
return None
|
|
736
|
+
prompt = True
|
|
737
|
+
i += 1
|
|
738
|
+
if resume and target is None:
|
|
739
|
+
return None # a bare `resume` is the picker: no session to follow
|
|
740
|
+
return kept, target
|
|
741
|
+
|
|
742
|
+
|
|
743
|
+
def argv_ok(provider, argv):
|
|
744
|
+
parse = claude_argv_parse if provider == "claude" else codex_argv_parse
|
|
745
|
+
return parse(argv) is not None
|
|
746
|
+
|
|
747
|
+
|
|
748
|
+
def codex_resume_id(argv):
|
|
749
|
+
"""The session id a codex launch resumes by name (None for --last / a new session)."""
|
|
750
|
+
parsed = codex_argv_parse(argv)
|
|
751
|
+
if parsed and parsed[1] and parsed[1] != "--last":
|
|
752
|
+
return parsed[1]
|
|
753
|
+
return None
|
|
754
|
+
|
|
755
|
+
|
|
756
|
+
def prompt_override_ok(text):
|
|
757
|
+
"""An operator prompt must survive as ONE positional the next shim still accepts: not
|
|
758
|
+
empty, not option-shaped, containing a space, and without NUL/newlines."""
|
|
759
|
+
return (isinstance(text, str) and text.strip() != "" and not text.startswith("-")
|
|
760
|
+
and " " in text and not any(c in text for c in "\0\n\r"))
|
|
761
|
+
|
|
762
|
+
|
|
763
|
+
def resume_prompt(cls, override=None):
|
|
764
|
+
reason = REASONS[cls]
|
|
765
|
+
if override and prompt_override_ok(override):
|
|
766
|
+
return override.replace("{reason}", reason)
|
|
767
|
+
where = "" if cls in ("crash", "transient") else " on another account"
|
|
768
|
+
return PROMPT_TEMPLATE.format(where=where, reason=reason)
|
|
769
|
+
|
|
770
|
+
|
|
771
|
+
def prompt_prefixes(override=None):
|
|
772
|
+
out = [PROMPT_PREFIX]
|
|
773
|
+
if override and prompt_override_ok(override):
|
|
774
|
+
out.append(override.strip()[:40])
|
|
775
|
+
return tuple(out)
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
def relaunch_argv(provider, argv, sid, cls, prompt=None):
|
|
779
|
+
"""The argv the relaunched shim runs, built from the ORIGINAL argv.
|
|
780
|
+
|
|
781
|
+
claude: the kept options + ``--resume <sid> <prompt>`` — no --model added: Claude
|
|
782
|
+
restores the session's own model on --resume ([1m] included), and an explicit
|
|
783
|
+
--model would switch that restore off.
|
|
784
|
+
codex: ``resume <kept options> <sid> <prompt>``.
|
|
785
|
+
None when the argv is outside the allowlist or the id is not a session id."""
|
|
786
|
+
if not uuid_ok(sid) or cls not in REASONS:
|
|
787
|
+
return None
|
|
788
|
+
text = resume_prompt(cls, prompt)
|
|
789
|
+
if provider == "claude":
|
|
790
|
+
kept = claude_argv_parse(argv)
|
|
791
|
+
if kept is None:
|
|
792
|
+
return None
|
|
793
|
+
return kept + ["--resume", sid, text]
|
|
794
|
+
if provider == "codex":
|
|
795
|
+
parsed = codex_argv_parse(argv)
|
|
796
|
+
if parsed is None:
|
|
797
|
+
return None
|
|
798
|
+
return ["resume"] + parsed[0] + [sid, text]
|
|
799
|
+
return None
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
# ---- relaunch / state files ---------------------------------------------------------
|
|
803
|
+
|
|
804
|
+
def format_relaunch(fields):
|
|
805
|
+
"""Relaunch file text in RELAUNCH_KEYS order. The shim reads it line by line, so a
|
|
806
|
+
value carrying a line break (or NUL) cannot be written faithfully: ValueError."""
|
|
807
|
+
lines = []
|
|
808
|
+
for key in RELAUNCH_KEYS:
|
|
809
|
+
val = "" if fields.get(key) is None else str(fields[key])
|
|
810
|
+
if any(c in val for c in "\0\n\r"):
|
|
811
|
+
raise ValueError("unwritable value for " + key)
|
|
812
|
+
lines.append("%s=%s" % (key, val))
|
|
813
|
+
return "\n".join(lines) + "\n"
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
def encode_argv(argv):
|
|
817
|
+
"""NUL-separated, NUL-terminated — what `while IFS= read -r -d ''` reads back."""
|
|
818
|
+
out = b""
|
|
819
|
+
for arg in argv:
|
|
820
|
+
raw = arg.encode("utf-8", "surrogateescape")
|
|
821
|
+
if b"\0" in raw:
|
|
822
|
+
raise ValueError("NUL in argv")
|
|
823
|
+
out += raw + b"\0"
|
|
824
|
+
return out
|
|
825
|
+
|
|
826
|
+
|
|
827
|
+
def decode_argv(data):
|
|
828
|
+
parts = data.split(b"\0")
|
|
829
|
+
if parts and parts[-1] == b"":
|
|
830
|
+
parts.pop()
|
|
831
|
+
return [p.decode("utf-8", "surrogateescape") for p in parts]
|
|
832
|
+
|
|
833
|
+
|
|
834
|
+
def _atomic_write(path, data):
|
|
835
|
+
tmp = "%s.tmp.%d" % (path, os.getpid())
|
|
836
|
+
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
837
|
+
try:
|
|
838
|
+
with os.fdopen(fd, "wb") as fh:
|
|
839
|
+
fh.write(data)
|
|
840
|
+
os.replace(tmp, path)
|
|
841
|
+
except BaseException:
|
|
842
|
+
try:
|
|
843
|
+
os.unlink(tmp)
|
|
844
|
+
except OSError:
|
|
845
|
+
pass
|
|
846
|
+
raise
|
|
847
|
+
|
|
848
|
+
|
|
849
|
+
def write_relaunch(dirpath, token, fields, argv):
|
|
850
|
+
"""Write ``r-<token>.argv`` then ``r-<token>.relaunch`` (the file the shim tests for
|
|
851
|
+
first), each atomically. Returns both paths; raises OSError/ValueError."""
|
|
852
|
+
if not TOKEN_RE.match(token or ""):
|
|
853
|
+
raise ValueError("bad token")
|
|
854
|
+
text = format_relaunch(fields).encode("utf-8", "surrogateescape")
|
|
855
|
+
data = encode_argv(argv)
|
|
856
|
+
os.makedirs(dirpath, mode=0o700, exist_ok=True)
|
|
857
|
+
base = os.path.join(dirpath, "r-" + token)
|
|
858
|
+
_atomic_write(base + ".argv", data)
|
|
859
|
+
try:
|
|
860
|
+
_atomic_write(base + ".relaunch", text)
|
|
861
|
+
except BaseException:
|
|
862
|
+
remove_quietly(base + ".argv")
|
|
863
|
+
raise
|
|
864
|
+
return base + ".relaunch", base + ".argv"
|
|
865
|
+
|
|
866
|
+
|
|
867
|
+
def remove_quietly(*paths):
|
|
868
|
+
for path in paths:
|
|
869
|
+
try:
|
|
870
|
+
os.unlink(path)
|
|
871
|
+
except OSError:
|
|
872
|
+
pass
|
|
873
|
+
|
|
874
|
+
|
|
875
|
+
def relaunch_fields(provider, verdict, acct, sid, cwd, depth, avoid, hist, chain):
|
|
876
|
+
"""The relaunch file's fields for one verdict. reset/rtype/marked only mean something
|
|
877
|
+
to the shim for a quota verdict (claude marks ``client:<rtype>`` until ``reset``;
|
|
878
|
+
codex writes its fixed cooldown)."""
|
|
879
|
+
cls = verdict["class"]
|
|
880
|
+
quota = cls == "quota"
|
|
881
|
+
return {
|
|
882
|
+
"v": "1", "provider": provider, "class": cls, "acct": acct,
|
|
883
|
+
"reset": str(int(verdict["reset"])) if quota and verdict.get("reset") else "",
|
|
884
|
+
"rtype": verdict.get("rtype", "") if quota and provider == "claude" else "",
|
|
885
|
+
"marked": verdict.get("marked") or "",
|
|
886
|
+
"sid": sid, "cwd": cwd, "depth": str(int(depth) + 1), "avoid": avoid,
|
|
887
|
+
"hist": hist, "chain": chain,
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
|
|
891
|
+
def argv_path_for(state_path):
|
|
892
|
+
return state_path[:-len(".state")] + ".argv" if state_path.endswith(".state") \
|
|
893
|
+
else state_path + ".argv"
|
|
894
|
+
|
|
895
|
+
|
|
896
|
+
def load_state(path):
|
|
897
|
+
"""The shim's ``<pid>.state`` -> typed dict, or None on anything malformed (the
|
|
898
|
+
watcher then exits silently: an unsupervised session is today's behaviour)."""
|
|
899
|
+
try:
|
|
900
|
+
with open(path, "rb") as fh:
|
|
901
|
+
text = fh.read(65536).decode("utf-8", "surrogateescape")
|
|
902
|
+
except OSError:
|
|
903
|
+
return None
|
|
904
|
+
kv = parse_kv(text, STATE_KEYS)
|
|
905
|
+
try:
|
|
906
|
+
if kv.get("v") != "1" or kv.get("provider") not in PROVIDERS:
|
|
907
|
+
return None
|
|
908
|
+
for key in ("pid", "ppid", "launched"):
|
|
909
|
+
if not num_ok(kv.get(key, "")):
|
|
910
|
+
return None
|
|
911
|
+
pid, ppid = int(kv["pid"]), int(kv["ppid"])
|
|
912
|
+
if pid <= 1 or ppid <= 1:
|
|
913
|
+
return None
|
|
914
|
+
# A relative pool root (bin/claude writes $ACC_ROOT as given) resolves against
|
|
915
|
+
# the watcher's cwd, which is the shim's own $PWD at spawn time.
|
|
916
|
+
acct = kv.get("acct", "")
|
|
917
|
+
acct_dir = os.path.abspath(kv["acct_dir"]) if kv.get("acct_dir") else ""
|
|
918
|
+
acc_root = os.path.abspath(kv["acc_root"]) if kv.get("acc_root") else ""
|
|
919
|
+
if not ACCT_RE.match(acct) or not acct_dir or not acc_root \
|
|
920
|
+
or os.path.basename(acct_dir) != acct:
|
|
921
|
+
return None
|
|
922
|
+
for key in ("cwd", "self"):
|
|
923
|
+
if not os.path.isabs(kv.get(key, "")):
|
|
924
|
+
return None
|
|
925
|
+
if tmux_socket(kv.get("tmux", "")) is None or not PANE_RE.match(kv.get("pane", "")):
|
|
926
|
+
return None
|
|
927
|
+
depth = kv.get("depth", "") or "0"
|
|
928
|
+
if not num_ok(depth):
|
|
929
|
+
return None
|
|
930
|
+
chain = kv.get("chain", "")
|
|
931
|
+
if chain and not CHAIN_RE.match(chain):
|
|
932
|
+
chain = ""
|
|
933
|
+
except (TypeError, ValueError):
|
|
934
|
+
return None
|
|
935
|
+
return {
|
|
936
|
+
"provider": kv["provider"], "pid": pid, "ppid": ppid, "acct": acct,
|
|
937
|
+
"acct_dir": acct_dir, "cwd": kv["cwd"], "launched": int(kv["launched"]),
|
|
938
|
+
"tmux": kv["tmux"], "pane": kv["pane"], "self": kv["self"],
|
|
939
|
+
"acc_root": acc_root, "depth": int(depth),
|
|
940
|
+
"avoid": kv.get("avoid", ""), "hist": kv.get("hist", ""), "chain": chain,
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
|
|
944
|
+
def load_argv(path):
|
|
945
|
+
try:
|
|
946
|
+
with open(path, "rb") as fh:
|
|
947
|
+
data = fh.read(1 << 20)
|
|
948
|
+
except OSError:
|
|
949
|
+
return None
|
|
950
|
+
return decode_argv(data)
|
|
951
|
+
|
|
952
|
+
|
|
953
|
+
PID_FILE = re.compile(r"^(\d{1,9})\.(state|argv|rollout|log)$")
|
|
954
|
+
|
|
955
|
+
|
|
956
|
+
def _pid_alive(pid):
|
|
957
|
+
try:
|
|
958
|
+
os.kill(pid, 0)
|
|
959
|
+
except ProcessLookupError:
|
|
960
|
+
return False
|
|
961
|
+
except OSError:
|
|
962
|
+
return True
|
|
963
|
+
return True
|
|
964
|
+
|
|
965
|
+
|
|
966
|
+
def prune(dirpath, now, keep=(), age=2 * 86400):
|
|
967
|
+
"""Remove state/relaunch/log leftovers older than two days (crashed watchers, tokens
|
|
968
|
+
nobody typed). A per-pid file whose client still runs is a session that simply lasts
|
|
969
|
+
longer than that — its ``.rollout`` claim is what keeps a neighbour's watcher off its
|
|
970
|
+
rollout — so it stays."""
|
|
971
|
+
try:
|
|
972
|
+
entries = list(os.scandir(dirpath))
|
|
973
|
+
except OSError:
|
|
974
|
+
return
|
|
975
|
+
for ent in entries:
|
|
976
|
+
try:
|
|
977
|
+
if ent.path in keep or not ent.is_file(follow_symlinks=False):
|
|
978
|
+
continue
|
|
979
|
+
if now - ent.stat(follow_symlinks=False).st_mtime > age:
|
|
980
|
+
m = PID_FILE.match(ent.name)
|
|
981
|
+
if m and int(m.group(1)) > 1 and _pid_alive(int(m.group(1))):
|
|
982
|
+
continue
|
|
983
|
+
os.unlink(ent.path)
|
|
984
|
+
except OSError:
|
|
985
|
+
continue
|
|
986
|
+
|
|
987
|
+
|
|
988
|
+
# ---- tmux ---------------------------------------------------------------------------
|
|
989
|
+
|
|
990
|
+
def tmux_socket(tmux_env):
|
|
991
|
+
"""$TMUX is ``<socket>,<server pid>,<session>``: the socket is everything before the
|
|
992
|
+
first comma, and it is always an absolute path."""
|
|
993
|
+
if not isinstance(tmux_env, str):
|
|
994
|
+
return None
|
|
995
|
+
sock = tmux_env.split(",", 1)[0]
|
|
996
|
+
return sock if sock.startswith("/") else None
|
|
997
|
+
|
|
998
|
+
|
|
999
|
+
def tmux_cmd(sock, *args):
|
|
1000
|
+
return ["tmux", "-S", sock] + list(args)
|
|
1001
|
+
|
|
1002
|
+
|
|
1003
|
+
# '|' separated, the command LAST (it is the only field that may hold anything): the
|
|
1004
|
+
# pane's root pid, whether a tmux mode (copy/view/tree/clock...) owns its keys, whether
|
|
1005
|
+
# synchronize-panes would copy typed keys into every pane of the window, and the
|
|
1006
|
+
# foreground program. An option or format an old tmux lacks expands to "" — read as off.
|
|
1007
|
+
PANE_FORMAT = "#{pane_pid}|#{pane_in_mode}|#{synchronize-panes}|#{pane_current_command}"
|
|
1008
|
+
|
|
1009
|
+
|
|
1010
|
+
def pane_query_argv(sock, pane):
|
|
1011
|
+
return tmux_cmd(sock, "display-message", "-p", "-t", pane, PANE_FORMAT)
|
|
1012
|
+
|
|
1013
|
+
|
|
1014
|
+
def parse_pane_reply(out):
|
|
1015
|
+
"""PANE_FORMAT's reply -> (pid, command, in_mode, synchronized) or None."""
|
|
1016
|
+
line = (out or "").strip().split("\n", 1)[0].strip()
|
|
1017
|
+
parts = line.split("|", 3)
|
|
1018
|
+
if len(parts) != 4 or not parts[0].isdigit():
|
|
1019
|
+
return None
|
|
1020
|
+
mode, sync = parts[1].strip(), parts[2].strip()
|
|
1021
|
+
return (int(parts[0]), parts[3].strip(), mode not in ("", "0"),
|
|
1022
|
+
sync not in ("", "0", "off"))
|
|
1023
|
+
|
|
1024
|
+
|
|
1025
|
+
def mode_cancel_argv(sock, pane):
|
|
1026
|
+
"""Leaves copy/view mode by a mode COMMAND, never a keystroke: a key sent to a pane in
|
|
1027
|
+
a mode is eaten by the mode, and one sent after the mode ended would reach whatever
|
|
1028
|
+
runs there. A mode this does not end is read again and the watcher gives up."""
|
|
1029
|
+
return tmux_cmd(sock, "send-keys", "-X", "-t", pane, "cancel")
|
|
1030
|
+
|
|
1031
|
+
|
|
1032
|
+
def is_shell(command):
|
|
1033
|
+
"""A shell from SHELLS, as ps or tmux name it (``-zsh`` for a login shell, or a
|
|
1034
|
+
path)."""
|
|
1035
|
+
return os.path.basename((command or "").strip().lstrip("-")) in SHELLS
|
|
1036
|
+
|
|
1037
|
+
|
|
1038
|
+
NO_TTY = ("", "?", "??", "-")
|
|
1039
|
+
|
|
1040
|
+
|
|
1041
|
+
def ps_foreground(status):
|
|
1042
|
+
"""From Runner.proc_status: is the process in its terminal's foreground process group
|
|
1043
|
+
(ps marks it '+')? None when that cannot be told — no ps answer, or no controlling
|
|
1044
|
+
terminal to be in front of."""
|
|
1045
|
+
if not status or status[1] in NO_TTY:
|
|
1046
|
+
return None
|
|
1047
|
+
return "+" in status[0]
|
|
1048
|
+
|
|
1049
|
+
|
|
1050
|
+
def typeable(path):
|
|
1051
|
+
"""An absolute path the relaunch line can carry as one bare word in every shell of
|
|
1052
|
+
SHELLS: nothing to quote, expand or glob."""
|
|
1053
|
+
return isinstance(path, str) and bool(TYPEABLE_RE.match(path))
|
|
1054
|
+
|
|
1055
|
+
|
|
1056
|
+
def relaunch_command(provider, token, sid, target, root=None):
|
|
1057
|
+
"""What is typed into the pane's shell:
|
|
1058
|
+
`` <P>_MULTIACC_AR=<token>:<sid> [<P>_ACCOUNTS_ROOT=<root>] <target>``.
|
|
1059
|
+
The leading space keeps it out of history where HIST_IGNORE_SPACE is set. ``target``
|
|
1060
|
+
is the shim that launched the session (its own path, so neither an alias nor the
|
|
1061
|
+
shell's PATH can send it elsewhere); ``root`` is the pool, named only when it is not
|
|
1062
|
+
the default (the pane's shell does not have a one-shot `CLAUDE_ACCOUNTS_ROOT=/x
|
|
1063
|
+
claude`'s variable). The session id rides along so a token the shim cannot honour
|
|
1064
|
+
still names what to resume."""
|
|
1065
|
+
if provider not in PROVIDERS or not TOKEN_RE.match(token or "") or not uuid_ok(sid) \
|
|
1066
|
+
or not typeable(target) or (root is not None and not typeable(root)):
|
|
1067
|
+
raise ValueError("bad relaunch command")
|
|
1068
|
+
up = provider.upper()
|
|
1069
|
+
line = " %s_MULTIACC_AR=%s:%s" % (up, token, sid)
|
|
1070
|
+
if root is not None:
|
|
1071
|
+
line += " %s_ACCOUNTS_ROOT=%s" % (up, root)
|
|
1072
|
+
return line + " " + target
|
|
1073
|
+
|
|
1074
|
+
|
|
1075
|
+
def send_keys_argvs(sock, pane, provider, token, sid, target, root=None):
|
|
1076
|
+
"""-R resets the pane's terminal modes first (codex dies on SIGTERM without restoring
|
|
1077
|
+
them), C-u clears anything typed at the prompt meanwhile, then the command, then
|
|
1078
|
+
Enter."""
|
|
1079
|
+
return [
|
|
1080
|
+
tmux_cmd(sock, "send-keys", "-R", "-t", pane),
|
|
1081
|
+
tmux_cmd(sock, "send-keys", "-t", pane, "C-u"),
|
|
1082
|
+
tmux_cmd(sock, "send-keys", "-t", pane, "-l",
|
|
1083
|
+
relaunch_command(provider, token, sid, target, root)),
|
|
1084
|
+
tmux_cmd(sock, "send-keys", "-t", pane, "Enter"),
|
|
1085
|
+
]
|
|
1086
|
+
|
|
1087
|
+
|
|
1088
|
+
def resume_hint(provider, sid):
|
|
1089
|
+
return ("claude --resume %s" if provider == "claude" else "codex resume %s") % sid
|
|
1090
|
+
|
|
1091
|
+
|
|
1092
|
+
def notice_argv(sock, pane, provider, sid, sticky=True):
|
|
1093
|
+
"""A status-line message (display-message without -p): shown to whoever watches the
|
|
1094
|
+
pane, never keyed into it. For a session the watcher stopped but could not relaunch.
|
|
1095
|
+
``sticky``: ``-d 0`` keeps it up until a key is pressed (tmux >= 3.2; an older tmux
|
|
1096
|
+
refuses the flag, and the caller retries without it)."""
|
|
1097
|
+
return tmux_cmd(sock, "display-message", *(("-d", "0") if sticky else ()), "-t", pane,
|
|
1098
|
+
"claude-multiacc: auto-resume failed — run: " + resume_hint(provider, sid))
|
|
1099
|
+
|
|
1100
|
+
|
|
1101
|
+
# ---- probe --------------------------------------------------------------------------
|
|
1102
|
+
|
|
1103
|
+
PROBE_DROP = {
|
|
1104
|
+
"claude": ("CLAUDE_CONFIG_DIR", "CLAUDE_CODE_OAUTH_TOKEN", "CLAUDE_SHIM_ACTIVE",
|
|
1105
|
+
"CLAUDE_ACCOUNT", "CLAUDE_MULTIACC_AR"),
|
|
1106
|
+
"codex": ("CODEX_HOME", "CODEX_SHIM_ACTIVE", "CODEX_ACCOUNT", "CODEX_MULTIACC_AR"),
|
|
1107
|
+
}
|
|
1108
|
+
|
|
1109
|
+
|
|
1110
|
+
def probe_env(environ, provider, avoid):
|
|
1111
|
+
"""The watcher inherited the picked account's exports; the probe must select from
|
|
1112
|
+
scratch, so they go, and probe mode + the AVOID list come in."""
|
|
1113
|
+
env = {k: v for k, v in environ.items() if k not in PROBE_DROP[provider]}
|
|
1114
|
+
up = provider.upper()
|
|
1115
|
+
env[up + "_MULTIACC_AR_PROBE"] = "1"
|
|
1116
|
+
env[up + "_MULTIACC_AR_AVOID"] = avoid or ""
|
|
1117
|
+
return env
|
|
1118
|
+
|
|
1119
|
+
|
|
1120
|
+
def parse_probe(out):
|
|
1121
|
+
"""The probe's ``pick=<acct> tier=<eligible|soft|hard|none>`` -> (pick, tier)."""
|
|
1122
|
+
pick, tier = "", ""
|
|
1123
|
+
for line in (out or "").split("\n"):
|
|
1124
|
+
m = re.match(r"^pick=(\S*) tier=(\S+)\s*$", line.strip())
|
|
1125
|
+
if m:
|
|
1126
|
+
pick, tier = m.group(1), m.group(2)
|
|
1127
|
+
if pick and not ACCT_RE.match(pick):
|
|
1128
|
+
return "", ""
|
|
1129
|
+
if tier not in ("eligible", "soft", "hard", "none"):
|
|
1130
|
+
return "", ""
|
|
1131
|
+
return pick, tier
|
|
1132
|
+
|
|
1133
|
+
|
|
1134
|
+
def probe_allows(cls, pick, tier, current):
|
|
1135
|
+
"""Rotate classes need an ELIGIBLE account that is not the one being left (holding
|
|
1136
|
+
beats landing on a soft/hard fallback); transient/crash may retry anywhere usable."""
|
|
1137
|
+
if cls in ROTATE:
|
|
1138
|
+
return tier == "eligible" and bool(pick) and pick != current
|
|
1139
|
+
return tier in ("eligible", "soft") and bool(pick)
|
|
1140
|
+
|
|
1141
|
+
|
|
1142
|
+
# ---- side effects -------------------------------------------------------------------
|
|
1143
|
+
|
|
1144
|
+
def descendants(table, pid):
|
|
1145
|
+
"""Every descendant pid of ``pid`` in a ps table [(pid, ppid, lstart)]."""
|
|
1146
|
+
kids = {}
|
|
1147
|
+
for p, pp, _ in table:
|
|
1148
|
+
kids.setdefault(pp, []).append(p)
|
|
1149
|
+
out, todo, seen = [], [pid], {pid}
|
|
1150
|
+
while todo:
|
|
1151
|
+
for child in kids.get(todo.pop(), ()):
|
|
1152
|
+
if child not in seen:
|
|
1153
|
+
seen.add(child)
|
|
1154
|
+
out.append(child)
|
|
1155
|
+
todo.append(child)
|
|
1156
|
+
return out
|
|
1157
|
+
|
|
1158
|
+
|
|
1159
|
+
class Runner:
|
|
1160
|
+
"""Every process side effect the watcher has. Tests substitute a fake."""
|
|
1161
|
+
|
|
1162
|
+
def now(self):
|
|
1163
|
+
return time.time()
|
|
1164
|
+
|
|
1165
|
+
def sleep(self, seconds):
|
|
1166
|
+
time.sleep(max(0.0, seconds))
|
|
1167
|
+
|
|
1168
|
+
def alive(self, pid):
|
|
1169
|
+
try:
|
|
1170
|
+
os.kill(pid, 0)
|
|
1171
|
+
except ProcessLookupError:
|
|
1172
|
+
return False
|
|
1173
|
+
except PermissionError:
|
|
1174
|
+
return True
|
|
1175
|
+
except OSError:
|
|
1176
|
+
return False
|
|
1177
|
+
return True
|
|
1178
|
+
|
|
1179
|
+
def gone(self, pid):
|
|
1180
|
+
"""Dead, or a zombie nobody has reaped yet: it still owns the pid, so kill -0
|
|
1181
|
+
succeeds, but it runs nothing. Only the stop/wait sequence asks this — the pane's
|
|
1182
|
+
shell reaps its foreground job at once, so the per-tick check stays fork-free."""
|
|
1183
|
+
if not self.alive(pid):
|
|
1184
|
+
return True
|
|
1185
|
+
try:
|
|
1186
|
+
with open("/proc/%d/stat" % pid) as fh:
|
|
1187
|
+
return fh.read().rsplit(")", 1)[1].split()[0] == "Z"
|
|
1188
|
+
except (OSError, IndexError):
|
|
1189
|
+
pass
|
|
1190
|
+
rc, out = self.run(["ps", "-o", "stat=", "-p", str(pid)], 5)
|
|
1191
|
+
if rc != 0:
|
|
1192
|
+
return not self.alive(pid)
|
|
1193
|
+
return out.strip().startswith("Z")
|
|
1194
|
+
|
|
1195
|
+
def signal(self, pid, sig):
|
|
1196
|
+
# Never init, never a process group (pid <= 0), never this watcher itself.
|
|
1197
|
+
if not isinstance(pid, int) or pid <= 1 or pid == os.getpid():
|
|
1198
|
+
return False
|
|
1199
|
+
try:
|
|
1200
|
+
os.kill(pid, sig)
|
|
1201
|
+
except OSError:
|
|
1202
|
+
return False
|
|
1203
|
+
return True
|
|
1204
|
+
|
|
1205
|
+
def pgid(self, pid):
|
|
1206
|
+
try:
|
|
1207
|
+
return os.getpgid(pid)
|
|
1208
|
+
except OSError:
|
|
1209
|
+
return None
|
|
1210
|
+
|
|
1211
|
+
def run(self, argv, timeout, env=None, cwd=None):
|
|
1212
|
+
"""(returncode or None, stdout). stdout goes to a temp FILE, not a pipe: a
|
|
1213
|
+
grandchild left in the background would hold a pipe open and turn a bounded call
|
|
1214
|
+
into an unbounded one. The child gets its own process group so a timeout takes
|
|
1215
|
+
everything it started with it."""
|
|
1216
|
+
try:
|
|
1217
|
+
with tempfile.TemporaryFile() as out:
|
|
1218
|
+
proc = subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=out,
|
|
1219
|
+
stderr=subprocess.DEVNULL, env=env, cwd=cwd,
|
|
1220
|
+
start_new_session=True, close_fds=True)
|
|
1221
|
+
try:
|
|
1222
|
+
rc = proc.wait(timeout=timeout)
|
|
1223
|
+
except subprocess.TimeoutExpired:
|
|
1224
|
+
try:
|
|
1225
|
+
os.killpg(proc.pid, signal.SIGKILL)
|
|
1226
|
+
except OSError:
|
|
1227
|
+
pass
|
|
1228
|
+
proc.wait()
|
|
1229
|
+
rc = None
|
|
1230
|
+
out.seek(0)
|
|
1231
|
+
data = out.read(1 << 20)
|
|
1232
|
+
except (OSError, ValueError, subprocess.SubprocessError):
|
|
1233
|
+
return None, ""
|
|
1234
|
+
return rc, data.decode("utf-8", "replace")
|
|
1235
|
+
|
|
1236
|
+
def proc_status(self, pid):
|
|
1237
|
+
"""(stat, tty, comm) as ps prints them — the command last, it may hold spaces —
|
|
1238
|
+
or None when ps has nothing for the pid."""
|
|
1239
|
+
rc, out = self.run(["ps", "-o", "stat=,tty=,comm=", "-p", str(pid)], 10)
|
|
1240
|
+
parts = out.strip().split("\n", 1)[0].split(None, 2) if rc == 0 else []
|
|
1241
|
+
return tuple(parts) if len(parts) == 3 else None
|
|
1242
|
+
|
|
1243
|
+
def can_list_open_files(self):
|
|
1244
|
+
return os.path.isdir("/proc/self/fd") or bool(
|
|
1245
|
+
shutil.which("lsof") or os.path.exists("/usr/sbin/lsof"))
|
|
1246
|
+
|
|
1247
|
+
def lstart(self, pid):
|
|
1248
|
+
"""The process's start time as ps prints it: with the pid, an identity that pid
|
|
1249
|
+
reuse cannot fake."""
|
|
1250
|
+
rc, out = self.run(["ps", "-o", "lstart=", "-p", str(pid)], 10)
|
|
1251
|
+
text = " ".join(out.split())
|
|
1252
|
+
return text if rc == 0 and text else None
|
|
1253
|
+
|
|
1254
|
+
def ps_table(self):
|
|
1255
|
+
rc, out = self.run(["ps", "-A", "-o", "pid=,ppid=,lstart="], 10)
|
|
1256
|
+
rows = []
|
|
1257
|
+
for line in out.split("\n"):
|
|
1258
|
+
parts = line.split(None, 2)
|
|
1259
|
+
if len(parts) >= 2 and parts[0].isdigit() and parts[1].isdigit():
|
|
1260
|
+
rows.append((int(parts[0]), int(parts[1]),
|
|
1261
|
+
" ".join(parts[2].split()) if len(parts) > 2 else ""))
|
|
1262
|
+
return rows
|
|
1263
|
+
|
|
1264
|
+
def open_files(self, pid):
|
|
1265
|
+
"""The paths ``pid`` holds open; None when the lookup itself failed."""
|
|
1266
|
+
if os.path.isdir("/proc/self/fd"):
|
|
1267
|
+
fd_dir = "/proc/%d/fd" % pid
|
|
1268
|
+
out = []
|
|
1269
|
+
try:
|
|
1270
|
+
names = os.listdir(fd_dir)
|
|
1271
|
+
except OSError:
|
|
1272
|
+
return None
|
|
1273
|
+
for name in names:
|
|
1274
|
+
try:
|
|
1275
|
+
out.append(os.readlink(os.path.join(fd_dir, name)))
|
|
1276
|
+
except OSError:
|
|
1277
|
+
continue
|
|
1278
|
+
return out
|
|
1279
|
+
lsof = shutil.which("lsof") or ("/usr/sbin/lsof" if os.path.exists("/usr/sbin/lsof")
|
|
1280
|
+
else None)
|
|
1281
|
+
if not lsof:
|
|
1282
|
+
return None
|
|
1283
|
+
rc, out = self.run([lsof, "-n", "-P", "-p", str(pid), "-Fn"], 10)
|
|
1284
|
+
if rc != 0:
|
|
1285
|
+
return None
|
|
1286
|
+
return [line[1:] for line in out.split("\n") if line.startswith("n")]
|
|
1287
|
+
|
|
1288
|
+
|
|
1289
|
+
# ---- transcript / rollout tailing ---------------------------------------------------
|
|
1290
|
+
|
|
1291
|
+
class JsonlTail:
|
|
1292
|
+
"""Appended complete lines of a JSONL file, as bytes. A partial trailing line waits
|
|
1293
|
+
for its newline; a truncated or replaced file starts over; one absurdly long line is
|
|
1294
|
+
skipped rather than buffered forever."""
|
|
1295
|
+
|
|
1296
|
+
MAX_READ = 32 << 20
|
|
1297
|
+
MAX_LINE = 64 << 20
|
|
1298
|
+
|
|
1299
|
+
def __init__(self, path):
|
|
1300
|
+
self.path = path
|
|
1301
|
+
self.off = 0
|
|
1302
|
+
self.buf = b""
|
|
1303
|
+
self.skip = False
|
|
1304
|
+
self.ino = None
|
|
1305
|
+
|
|
1306
|
+
def read_lines(self):
|
|
1307
|
+
try:
|
|
1308
|
+
fh = open(self.path, "rb")
|
|
1309
|
+
except OSError:
|
|
1310
|
+
return []
|
|
1311
|
+
with fh:
|
|
1312
|
+
try:
|
|
1313
|
+
st = os.fstat(fh.fileno())
|
|
1314
|
+
except OSError:
|
|
1315
|
+
return []
|
|
1316
|
+
if (self.ino is not None and st.st_ino != self.ino) or st.st_size < self.off:
|
|
1317
|
+
self.off, self.buf, self.skip = 0, b"", False
|
|
1318
|
+
self.ino = st.st_ino
|
|
1319
|
+
if st.st_size == self.off:
|
|
1320
|
+
return []
|
|
1321
|
+
fh.seek(self.off)
|
|
1322
|
+
data = fh.read(self.MAX_READ)
|
|
1323
|
+
self.off += len(data)
|
|
1324
|
+
if self.skip:
|
|
1325
|
+
cut = data.find(b"\n")
|
|
1326
|
+
if cut < 0:
|
|
1327
|
+
return []
|
|
1328
|
+
data, self.skip = data[cut + 1:], False
|
|
1329
|
+
parts = (self.buf + data).split(b"\n")
|
|
1330
|
+
self.buf = parts.pop()
|
|
1331
|
+
if len(self.buf) > self.MAX_LINE:
|
|
1332
|
+
self.buf, self.skip = b"", True
|
|
1333
|
+
return [p for p in parts if p.strip()]
|
|
1334
|
+
|
|
1335
|
+
|
|
1336
|
+
def read_json_line(raw):
|
|
1337
|
+
try:
|
|
1338
|
+
rec = json.loads(raw.decode("utf-8", "replace"))
|
|
1339
|
+
except ValueError:
|
|
1340
|
+
return None
|
|
1341
|
+
return rec if isinstance(rec, dict) else None
|
|
1342
|
+
|
|
1343
|
+
|
|
1344
|
+
def read_first_line(path, limit=4 << 20):
|
|
1345
|
+
"""Line 1 of a rollout (its session_meta), bounded: base instructions ride in it."""
|
|
1346
|
+
try:
|
|
1347
|
+
with open(path, "rb") as fh:
|
|
1348
|
+
line = fh.readline(limit)
|
|
1349
|
+
except OSError:
|
|
1350
|
+
return None
|
|
1351
|
+
return read_json_line(line.rstrip(b"\n"))
|
|
1352
|
+
|
|
1353
|
+
|
|
1354
|
+
def rollout_meta(path):
|
|
1355
|
+
"""The session_meta payload of a MAIN rollout (id == session_id), else None —
|
|
1356
|
+
subagent and fork files carry their parent's session_id under their own id."""
|
|
1357
|
+
rec = read_first_line(path)
|
|
1358
|
+
if not rec or rec.get("type") != "session_meta" or not isinstance(rec.get("payload"),
|
|
1359
|
+
dict):
|
|
1360
|
+
return None
|
|
1361
|
+
meta = rec["payload"]
|
|
1362
|
+
sid = meta.get("id")
|
|
1363
|
+
if not uuid_ok(sid) or meta.get("session_id") != sid:
|
|
1364
|
+
return None
|
|
1365
|
+
return meta
|
|
1366
|
+
|
|
1367
|
+
|
|
1368
|
+
ROLLOUT_NAME = re.compile(r"^rollout-.*\.jsonl$")
|
|
1369
|
+
|
|
1370
|
+
|
|
1371
|
+
def codex_fallback_candidates(sessions_dir, launched, cwd, exclude=()):
|
|
1372
|
+
"""Rollouts under the two newest day dirs that could only be this launch's: a
|
|
1373
|
+
codex-tui main session in its cwd, CREATED since the launch (session_meta's own
|
|
1374
|
+
timestamp — a file's mtime only says someone wrote to it, and a neighbour session
|
|
1375
|
+
writes all day). The caller needs EXACTLY one."""
|
|
1376
|
+
days = sorted(glob.glob(os.path.join(sessions_dir, "[0-9]*", "[0-9]*", "[0-9]*")))[-2:]
|
|
1377
|
+
want = os.path.realpath(cwd)
|
|
1378
|
+
out = []
|
|
1379
|
+
for day in days:
|
|
1380
|
+
for path in glob.glob(os.path.join(day, "rollout-*.jsonl")):
|
|
1381
|
+
try:
|
|
1382
|
+
if os.stat(path).st_mtime < launched:
|
|
1383
|
+
continue
|
|
1384
|
+
except OSError:
|
|
1385
|
+
continue
|
|
1386
|
+
if os.path.realpath(path) in exclude:
|
|
1387
|
+
continue
|
|
1388
|
+
meta = rollout_meta(path)
|
|
1389
|
+
if not meta or meta.get("originator") != "codex-tui":
|
|
1390
|
+
continue
|
|
1391
|
+
created = iso_to_epoch(meta.get("timestamp"))
|
|
1392
|
+
if created is None or created < launched - 2:
|
|
1393
|
+
continue
|
|
1394
|
+
mcwd = meta.get("cwd")
|
|
1395
|
+
if not isinstance(mcwd, str) or os.path.realpath(mcwd) != want:
|
|
1396
|
+
continue
|
|
1397
|
+
out.append(path)
|
|
1398
|
+
return out
|
|
1399
|
+
|
|
1400
|
+
|
|
1401
|
+
# ---- the watcher --------------------------------------------------------------------
|
|
1402
|
+
|
|
1403
|
+
class GiveUp(Exception):
|
|
1404
|
+
pass
|
|
1405
|
+
|
|
1406
|
+
|
|
1407
|
+
class Quoted(str):
|
|
1408
|
+
"""A log value written as ``key="value"`` with its spaces kept (the resume command)."""
|
|
1409
|
+
|
|
1410
|
+
|
|
1411
|
+
class Watcher:
|
|
1412
|
+
"""One supervised session: tail, classify, decide, stop, relaunch — or just exit."""
|
|
1413
|
+
|
|
1414
|
+
PLACEHOLDER_SID = "00000000-0000-0000-0000-000000000000"
|
|
1415
|
+
|
|
1416
|
+
def __init__(self, statefile, st, argv, runner=None, environ=None):
|
|
1417
|
+
self.statefile = os.path.abspath(statefile)
|
|
1418
|
+
self.st = st
|
|
1419
|
+
self.provider = st["provider"]
|
|
1420
|
+
self.env = dict(os.environ if environ is None else environ)
|
|
1421
|
+
self.cfg = Config(self.provider, self.env)
|
|
1422
|
+
self.r = runner or Runner()
|
|
1423
|
+
self.argv = list(argv)
|
|
1424
|
+
self.pid = st["pid"]
|
|
1425
|
+
self.acct = st["acct"]
|
|
1426
|
+
self.acct_dir = st["acct_dir"]
|
|
1427
|
+
self.launched = st["launched"]
|
|
1428
|
+
self.stdir = os.path.dirname(self.statefile)
|
|
1429
|
+
self.ar_dir = os.path.join(st["acc_root"], "tmp", "autoresume")
|
|
1430
|
+
self.chain = st["chain"] or make_token(8)
|
|
1431
|
+
self.tracker = make_tracker(self.provider, self.launched - 2,
|
|
1432
|
+
prompt_prefixes(self.cfg.prompt))
|
|
1433
|
+
self.sid = None
|
|
1434
|
+
self.cwd = None
|
|
1435
|
+
self.tail = None
|
|
1436
|
+
self.transcript = None
|
|
1437
|
+
self.lstart = None
|
|
1438
|
+
self.started = None
|
|
1439
|
+
self.next_find = 0.0
|
|
1440
|
+
self.next_lsof = 0.0
|
|
1441
|
+
self.next_fallback = 0.0
|
|
1442
|
+
self.never_seen = set()
|
|
1443
|
+
self.hold_logged = False
|
|
1444
|
+
self.last_probe = None
|
|
1445
|
+
self.claim = os.path.join(self.stdir, "%d.rollout" % self.pid)
|
|
1446
|
+
self.root = None # the pool, when the relaunch has to name it
|
|
1447
|
+
self.stopped = False # True once this watcher has signalled the client
|
|
1448
|
+
self.tree = [] # [(pid, lstart)] of the client's descendants at the stop
|
|
1449
|
+
self._open_files_ok = None
|
|
1450
|
+
self.lookup_failed_at = None
|
|
1451
|
+
|
|
1452
|
+
# -- logging --
|
|
1453
|
+
|
|
1454
|
+
def log(self, event, **fields):
|
|
1455
|
+
"""selection.log grammar: ``<UTC ISO> autoresume <event> k=v ...``. Field 2 is
|
|
1456
|
+
the word autoresume, never an account id — lib/report.py and the *-accounts tools
|
|
1457
|
+
read an acct-NN there as a pick. No transcript text, tokens or secrets."""
|
|
1458
|
+
parts = ["autoresume", event, "provider=" + self.provider, "chain=" + self.chain]
|
|
1459
|
+
for key, val in fields.items():
|
|
1460
|
+
if isinstance(val, Quoted):
|
|
1461
|
+
text = "".join(c for c in val if c.isprintable() and c != '"')[:120]
|
|
1462
|
+
parts.append('%s="%s"' % (key, text))
|
|
1463
|
+
continue
|
|
1464
|
+
text = re.sub(r"\s+", "_", str(val if val is not None and val != "" else "-"))
|
|
1465
|
+
parts.append("%s=%s" % (key, "".join(c for c in text if c.isprintable())[:120]))
|
|
1466
|
+
line = "%s %s\n" % (utc_iso(self.r.now()), " ".join(parts))
|
|
1467
|
+
try:
|
|
1468
|
+
with open(os.path.join(self.st["acc_root"], "selection.log"), "a") as fh:
|
|
1469
|
+
fh.write(line)
|
|
1470
|
+
except OSError:
|
|
1471
|
+
pass
|
|
1472
|
+
self.dbg(line.strip())
|
|
1473
|
+
|
|
1474
|
+
def dbg(self, msg):
|
|
1475
|
+
if not self.cfg.debug:
|
|
1476
|
+
return
|
|
1477
|
+
try:
|
|
1478
|
+
with open(os.path.join(self.stdir, "%d.log" % self.pid), "a") as fh:
|
|
1479
|
+
fh.write("%s %s\n" % (utc_iso(time.time()), msg))
|
|
1480
|
+
except OSError:
|
|
1481
|
+
pass
|
|
1482
|
+
|
|
1483
|
+
# -- the loop --
|
|
1484
|
+
|
|
1485
|
+
def switched_off(self):
|
|
1486
|
+
return os.path.exists(os.path.join(self.st["acc_root"], "autoresume.off"))
|
|
1487
|
+
|
|
1488
|
+
def run(self):
|
|
1489
|
+
if not self.r.alive(self.pid):
|
|
1490
|
+
return 0
|
|
1491
|
+
self.lstart = self.r.lstart(self.pid)
|
|
1492
|
+
if not self.lstart:
|
|
1493
|
+
return 0 # no identity to re-check before a signal: stay passive
|
|
1494
|
+
self.started = self.r.now()
|
|
1495
|
+
if relaunch_argv(self.provider, self.argv, self.PLACEHOLDER_SID, "quota") is None:
|
|
1496
|
+
self.log("giveup", reason="argv", pid=self.pid)
|
|
1497
|
+
return 0
|
|
1498
|
+
if not typeable(self.st["self"]) or not typeable(self.st["acc_root"]):
|
|
1499
|
+
# The relaunch line carries both bare (the shims' gate refuses the same
|
|
1500
|
+
# paths): never stop a session whose way back cannot be typed.
|
|
1501
|
+
self.log("giveup", reason="path", pid=self.pid)
|
|
1502
|
+
return 0
|
|
1503
|
+
default = os.path.join(self.env.get("HOME") or "/nonexistent",
|
|
1504
|
+
".%s-accounts" % self.provider)
|
|
1505
|
+
if os.path.normpath(self.st["acc_root"]) != os.path.normpath(default):
|
|
1506
|
+
self.root = self.st["acc_root"]
|
|
1507
|
+
self.log("watch", pid=self.pid, acct=self.acct, depth=self.st["depth"])
|
|
1508
|
+
try:
|
|
1509
|
+
while True:
|
|
1510
|
+
if self.switched_off():
|
|
1511
|
+
self.dbg("autoresume.off: exiting")
|
|
1512
|
+
return 0
|
|
1513
|
+
if not self.r.alive(self.pid):
|
|
1514
|
+
return self.on_exit()
|
|
1515
|
+
self.read()
|
|
1516
|
+
if self.tracker.pending is not None:
|
|
1517
|
+
rc = self.decide()
|
|
1518
|
+
if rc is not None:
|
|
1519
|
+
return rc
|
|
1520
|
+
self.r.sleep(self.cfg.poll)
|
|
1521
|
+
except GiveUp:
|
|
1522
|
+
return 0
|
|
1523
|
+
|
|
1524
|
+
def giveup(self, reason, **fields):
|
|
1525
|
+
if self.stopped:
|
|
1526
|
+
# This watcher stopped the session and cannot bring it back: say so where
|
|
1527
|
+
# the operator looks — the log line carries the command that resumes it,
|
|
1528
|
+
# and so does the pane's status line (display-message, never a keystroke).
|
|
1529
|
+
fields["stopped"] = 1
|
|
1530
|
+
if self.sid:
|
|
1531
|
+
fields["resume"] = Quoted(resume_hint(self.provider, self.sid))
|
|
1532
|
+
self.notify_stopped()
|
|
1533
|
+
self.log("giveup", reason=reason, **fields)
|
|
1534
|
+
raise GiveUp(reason)
|
|
1535
|
+
|
|
1536
|
+
def notify_stopped(self):
|
|
1537
|
+
sock = tmux_socket(self.st["tmux"])
|
|
1538
|
+
if sock and self.sid:
|
|
1539
|
+
rc, _ = self.r.run(notice_argv(sock, self.st["pane"], self.provider, self.sid), 10)
|
|
1540
|
+
if rc != 0: # tmux < 3.2 has no -d: the plain notice, gone after display-time
|
|
1541
|
+
self.r.run(notice_argv(sock, self.st["pane"], self.provider, self.sid,
|
|
1542
|
+
sticky=False), 10)
|
|
1543
|
+
|
|
1544
|
+
# -- readers --
|
|
1545
|
+
|
|
1546
|
+
def read(self):
|
|
1547
|
+
if self.provider == "claude":
|
|
1548
|
+
self.read_claude()
|
|
1549
|
+
else:
|
|
1550
|
+
self.read_codex()
|
|
1551
|
+
|
|
1552
|
+
def registry(self):
|
|
1553
|
+
"""``<acct>/sessions/<pid>.json`` -> (sessionId, cwd) of THIS run, or None. A file
|
|
1554
|
+
older than the launch belongs to a crashed run whose pid was reused."""
|
|
1555
|
+
path = os.path.join(self.acct_dir, "sessions", "%d.json" % self.pid)
|
|
1556
|
+
try:
|
|
1557
|
+
with open(path, "rb") as fh:
|
|
1558
|
+
doc = json.loads(fh.read(1 << 20).decode("utf-8", "replace"))
|
|
1559
|
+
except (OSError, ValueError):
|
|
1560
|
+
return None
|
|
1561
|
+
if not isinstance(doc, dict) or not uuid_ok(doc.get("sessionId")):
|
|
1562
|
+
return None
|
|
1563
|
+
if isinstance(doc.get("pid"), int) and doc["pid"] != self.pid:
|
|
1564
|
+
return None
|
|
1565
|
+
started = doc.get("startedAt")
|
|
1566
|
+
if isinstance(started, (int, float)) and not isinstance(started, bool) \
|
|
1567
|
+
and started / 1000.0 < self.launched - 30:
|
|
1568
|
+
return None
|
|
1569
|
+
cwd = doc.get("cwd")
|
|
1570
|
+
return doc["sessionId"], (cwd if isinstance(cwd, str) and os.path.isabs(cwd) else None)
|
|
1571
|
+
|
|
1572
|
+
def read_claude(self):
|
|
1573
|
+
now = self.r.now()
|
|
1574
|
+
reg = self.registry()
|
|
1575
|
+
if reg:
|
|
1576
|
+
sid, cwd = reg
|
|
1577
|
+
if sid != self.sid:
|
|
1578
|
+
# /clear, /resume: a different conversation — whatever was pending
|
|
1579
|
+
# belonged to the old one.
|
|
1580
|
+
if self.sid is not None:
|
|
1581
|
+
self.dbg("session changed %s -> %s" % (self.sid, sid))
|
|
1582
|
+
# A conversation picked with /resume carries whatever happened to it
|
|
1583
|
+
# elsewhere since this launch — another process's limit error
|
|
1584
|
+
# included, which is not this client's and not this account's.
|
|
1585
|
+
# Only what is written from now on is ours.
|
|
1586
|
+
self.tracker.since = max(self.tracker.since, now - 2)
|
|
1587
|
+
self.tracker.clear()
|
|
1588
|
+
self.sid, self.tail, self.transcript = sid, None, None
|
|
1589
|
+
if cwd:
|
|
1590
|
+
self.cwd = cwd
|
|
1591
|
+
if self.sid and self.tail is None and now >= self.next_find:
|
|
1592
|
+
self.next_find = now + 2.0
|
|
1593
|
+
found = sorted(glob.glob(os.path.join(self.acct_dir, "projects", "*",
|
|
1594
|
+
self.sid + ".jsonl")))
|
|
1595
|
+
if found:
|
|
1596
|
+
self.transcript = found[0]
|
|
1597
|
+
self.tail = JsonlTail(found[0])
|
|
1598
|
+
self.dbg("transcript %s" % found[0])
|
|
1599
|
+
if self.tail is not None:
|
|
1600
|
+
self.pump()
|
|
1601
|
+
|
|
1602
|
+
def read_codex(self):
|
|
1603
|
+
if self.tail is None and not self.discover():
|
|
1604
|
+
return
|
|
1605
|
+
self.pump()
|
|
1606
|
+
|
|
1607
|
+
def pump(self):
|
|
1608
|
+
now = self.r.now()
|
|
1609
|
+
for raw in self.tail.read_lines():
|
|
1610
|
+
if not self.tracker.wants(raw):
|
|
1611
|
+
continue
|
|
1612
|
+
rec = read_json_line(raw)
|
|
1613
|
+
if rec is None:
|
|
1614
|
+
continue
|
|
1615
|
+
before = self.tracker.pending
|
|
1616
|
+
ev = self.tracker.feed(rec, now)
|
|
1617
|
+
if ev is not None:
|
|
1618
|
+
self.on_event(ev, before)
|
|
1619
|
+
|
|
1620
|
+
def on_event(self, ev, before):
|
|
1621
|
+
kind, val = ev
|
|
1622
|
+
if kind == "never":
|
|
1623
|
+
if val not in self.never_seen:
|
|
1624
|
+
self.never_seen.add(val)
|
|
1625
|
+
self.log("never", code=val, sid=self.sid, acct=self.acct)
|
|
1626
|
+
return
|
|
1627
|
+
# Any change of state re-arms the hold log and probes immediately.
|
|
1628
|
+
self.hold_logged = False
|
|
1629
|
+
self.last_probe = None
|
|
1630
|
+
if kind == "cancel":
|
|
1631
|
+
self.dbg("verdict cancelled by %s" % val)
|
|
1632
|
+
return
|
|
1633
|
+
if before is None or before["class"] != val["class"]:
|
|
1634
|
+
self.log("detect", **{"class": val["class"], "code": val["code"],
|
|
1635
|
+
"sid": self.sid, "acct": self.acct,
|
|
1636
|
+
"reset": val.get("reset") or "-"})
|
|
1637
|
+
|
|
1638
|
+
# -- codex rollout discovery --
|
|
1639
|
+
|
|
1640
|
+
def claimed_rollouts(self):
|
|
1641
|
+
"""Rollouts another LIVE watcher already follows, so the cwd fallback cannot hand
|
|
1642
|
+
this session a neighbour's rollout that happens to be the only candidate."""
|
|
1643
|
+
out = set()
|
|
1644
|
+
for path in glob.glob(os.path.join(self.stdir, "*.rollout")):
|
|
1645
|
+
if path == self.claim:
|
|
1646
|
+
continue
|
|
1647
|
+
name = os.path.basename(path)[:-len(".rollout")]
|
|
1648
|
+
if not name.isdigit() or not self.r.alive(int(name)):
|
|
1649
|
+
continue
|
|
1650
|
+
try:
|
|
1651
|
+
with open(path) as fh:
|
|
1652
|
+
out.add(os.path.realpath(fh.read(4096).strip()))
|
|
1653
|
+
except OSError:
|
|
1654
|
+
continue
|
|
1655
|
+
return out
|
|
1656
|
+
|
|
1657
|
+
def attach(self, path, meta, how):
|
|
1658
|
+
self.tail = JsonlTail(path)
|
|
1659
|
+
self.transcript = path
|
|
1660
|
+
self.sid = meta["id"]
|
|
1661
|
+
mcwd = meta.get("cwd")
|
|
1662
|
+
self.cwd = mcwd if isinstance(mcwd, str) and os.path.isabs(mcwd) else None
|
|
1663
|
+
try:
|
|
1664
|
+
_atomic_write(self.claim, os.path.realpath(path).encode("utf-8", "surrogateescape"))
|
|
1665
|
+
except OSError:
|
|
1666
|
+
pass
|
|
1667
|
+
self.dbg("rollout %s (%s)" % (path, how))
|
|
1668
|
+
return True
|
|
1669
|
+
|
|
1670
|
+
def open_files_work(self):
|
|
1671
|
+
if self._open_files_ok is None:
|
|
1672
|
+
self._open_files_ok = bool(self.r.can_list_open_files())
|
|
1673
|
+
return self._open_files_ok
|
|
1674
|
+
|
|
1675
|
+
def discover(self):
|
|
1676
|
+
"""Find this session's rollout. codex writes it only once the session has a first
|
|
1677
|
+
message, which can be long after the launch, so the search never simply ends
|
|
1678
|
+
while the open-file lookup works: after DISCOVER_TIMEOUT only that lookup runs,
|
|
1679
|
+
every DISCOVER_SLOW s, for as long as the client lives."""
|
|
1680
|
+
sessions = os.path.join(self.acct_dir, "sessions")
|
|
1681
|
+
now = self.r.now()
|
|
1682
|
+
early = now - self.started <= self.cfg.discover_timeout
|
|
1683
|
+
if not early:
|
|
1684
|
+
if not self.open_files_work():
|
|
1685
|
+
self.giveup("discover", pid=self.pid)
|
|
1686
|
+
if now < self.next_lsof:
|
|
1687
|
+
return False
|
|
1688
|
+
# (a) the id the launch resumes by name.
|
|
1689
|
+
rid = codex_resume_id(self.argv) if early else None
|
|
1690
|
+
if rid:
|
|
1691
|
+
for path in sorted(glob.glob(os.path.join(sessions, "*", "*", "*",
|
|
1692
|
+
"rollout-*-%s.jsonl" % rid))):
|
|
1693
|
+
meta = rollout_meta(path)
|
|
1694
|
+
if meta and meta["id"] == rid:
|
|
1695
|
+
return self.attach(path, meta, "resume-id")
|
|
1696
|
+
# (b) the file the native child actually holds open (lsof is slow: every 2 s,
|
|
1697
|
+
# every DISCOVER_SLOW s once the launch is old).
|
|
1698
|
+
if now >= self.next_lsof:
|
|
1699
|
+
self.next_lsof = now + (2.0 if early else DISCOVER_SLOW)
|
|
1700
|
+
table = self.r.ps_table()
|
|
1701
|
+
for proc in [self.pid] + descendants(table, self.pid):
|
|
1702
|
+
files = self.r.open_files(proc)
|
|
1703
|
+
if proc == self.pid:
|
|
1704
|
+
if files is not None:
|
|
1705
|
+
self.lookup_failed_at = None
|
|
1706
|
+
elif self.lookup_failed_at is None:
|
|
1707
|
+
self.lookup_failed_at = now
|
|
1708
|
+
for path in files or ():
|
|
1709
|
+
if ROLLOUT_NAME.match(os.path.basename(path)):
|
|
1710
|
+
meta = rollout_meta(path)
|
|
1711
|
+
if meta:
|
|
1712
|
+
return self.attach(path, meta, "open-file")
|
|
1713
|
+
# (c) only where (b) cannot see open files (no /proc, no lsof, or a lookup that
|
|
1714
|
+
# has failed for LOOKUP_FAILED s): exactly one plausible new rollout for this
|
|
1715
|
+
# cwd. Where (b) works it is the whole answer — a guess would adopt a
|
|
1716
|
+
# neighbour session's rollout while this one has none yet, and a limit there
|
|
1717
|
+
# would stop THIS session and resume the other one here.
|
|
1718
|
+
failing = self.lookup_failed_at is not None \
|
|
1719
|
+
and now - self.lookup_failed_at >= LOOKUP_FAILED
|
|
1720
|
+
if early and now >= self.next_fallback and (not self.open_files_work() or failing):
|
|
1721
|
+
self.next_fallback = now + 2.0
|
|
1722
|
+
cands = codex_fallback_candidates(sessions, self.launched, self.st["cwd"],
|
|
1723
|
+
self.claimed_rollouts())
|
|
1724
|
+
if len(cands) == 1:
|
|
1725
|
+
meta = rollout_meta(cands[0])
|
|
1726
|
+
if meta:
|
|
1727
|
+
return self.attach(cands[0], meta, "cwd-fallback")
|
|
1728
|
+
return False
|
|
1729
|
+
|
|
1730
|
+
# -- deciding --
|
|
1731
|
+
|
|
1732
|
+
def subagents_busy(self, now):
|
|
1733
|
+
"""A model/transient error on the main thread while subagents are still writing
|
|
1734
|
+
is not a dead session yet — stopping it would kill their work."""
|
|
1735
|
+
if self.provider != "claude" or not self.sid:
|
|
1736
|
+
return False
|
|
1737
|
+
newest, budget = 0.0, 5000 # stat at most this many files per check
|
|
1738
|
+
for base in glob.glob(os.path.join(self.acct_dir, "projects", "*", self.sid,
|
|
1739
|
+
"subagents")):
|
|
1740
|
+
for root, _dirs, files in os.walk(base):
|
|
1741
|
+
for name in files[:budget]:
|
|
1742
|
+
try:
|
|
1743
|
+
newest = max(newest, os.stat(os.path.join(root, name)).st_mtime)
|
|
1744
|
+
except OSError:
|
|
1745
|
+
pass
|
|
1746
|
+
budget -= min(len(files), budget)
|
|
1747
|
+
if budget <= 0:
|
|
1748
|
+
return newest > 0 and now - newest < self.cfg.subagent_quiet
|
|
1749
|
+
return newest > 0 and now - newest < self.cfg.subagent_quiet
|
|
1750
|
+
|
|
1751
|
+
def session_cwd(self):
|
|
1752
|
+
"""Where the probe and the relaunch run. claude: the registry's cwd first
|
|
1753
|
+
(``--resume`` finds the transcript by it). codex: the launch directory first, so
|
|
1754
|
+
the relaunch repeats the user's own invocation; the rollout's cwd after it."""
|
|
1755
|
+
order = (self.cwd, self.st["cwd"]) if self.provider == "claude" \
|
|
1756
|
+
else (self.st["cwd"], self.cwd)
|
|
1757
|
+
for cand in order:
|
|
1758
|
+
if cand and os.path.isdir(cand):
|
|
1759
|
+
return cand
|
|
1760
|
+
return None
|
|
1761
|
+
|
|
1762
|
+
def build_relaunch_argv(self, cls):
|
|
1763
|
+
return relaunch_argv(self.provider, self.argv, self.sid, cls, self.cfg.prompt)
|
|
1764
|
+
|
|
1765
|
+
def hold(self, verdict, **fields):
|
|
1766
|
+
if not self.hold_logged:
|
|
1767
|
+
self.hold_logged = True
|
|
1768
|
+
self.log("hold", **dict({"class": verdict["class"], "sid": self.sid,
|
|
1769
|
+
"acct": self.acct}, **fields))
|
|
1770
|
+
|
|
1771
|
+
def decide(self):
|
|
1772
|
+
"""Act on the pending verdict once it has stood long enough; None = keep going."""
|
|
1773
|
+
verdict = self.tracker.pending
|
|
1774
|
+
now = self.r.now()
|
|
1775
|
+
cls = verdict["class"]
|
|
1776
|
+
age = now - self.tracker.pending_since
|
|
1777
|
+
need = self.cfg.grace
|
|
1778
|
+
if cls == "transient":
|
|
1779
|
+
need = max(need, ladder_wait(self.st["hist"], now, self.cfg.transient_ladder))
|
|
1780
|
+
if age < need:
|
|
1781
|
+
return None
|
|
1782
|
+
ok, why = budget_check(cls, self.st["depth"], self.st["hist"], now, self.cfg)
|
|
1783
|
+
if not ok:
|
|
1784
|
+
if why == "depth":
|
|
1785
|
+
self.giveup("depth", depth=self.st["depth"], sid=self.sid)
|
|
1786
|
+
self.hold(verdict, reason=why)
|
|
1787
|
+
return None
|
|
1788
|
+
if cls in ("model", "transient") and age < self.cfg.subagent_cap \
|
|
1789
|
+
and self.subagents_busy(now):
|
|
1790
|
+
return None
|
|
1791
|
+
if self.last_probe is not None and now - self.last_probe < self.cfg.hold_reprobe:
|
|
1792
|
+
return None
|
|
1793
|
+
self.last_probe = now
|
|
1794
|
+
argv = self.build_relaunch_argv(cls)
|
|
1795
|
+
if argv is None:
|
|
1796
|
+
self.giveup("argv", sid=self.sid)
|
|
1797
|
+
if cls in ROTATE:
|
|
1798
|
+
avoid = avoid_merge(self.st["avoid"], now, self.acct, avoid_until(verdict, now))
|
|
1799
|
+
else:
|
|
1800
|
+
avoid = avoid_merge(self.st["avoid"], now)
|
|
1801
|
+
cwd = self.session_cwd()
|
|
1802
|
+
if cwd is None:
|
|
1803
|
+
self.giveup("cwd", sid=self.sid)
|
|
1804
|
+
rc, out = self.r.run([self.st["self"]] + argv, self.cfg.probe_timeout,
|
|
1805
|
+
env=probe_env(self.env, self.provider, avoid), cwd=cwd)
|
|
1806
|
+
pick, tier = parse_probe(out)
|
|
1807
|
+
self.dbg("probe rc=%s pick=%s tier=%s" % (rc, pick, tier))
|
|
1808
|
+
if not probe_allows(cls, pick, tier, self.acct):
|
|
1809
|
+
# Claude keeps its own auto-continue meanwhile; a cancel ends the hold.
|
|
1810
|
+
self.hold(verdict, reason="no-room", pick=pick or "-", tier=tier or "-")
|
|
1811
|
+
return None
|
|
1812
|
+
# The probe can take seconds: whatever the session wrote meanwhile (the user
|
|
1813
|
+
# typing, auto-continue getting through) is read before anything is stopped.
|
|
1814
|
+
self.read()
|
|
1815
|
+
if self.tracker.pending is not verdict or not self.r.alive(self.pid):
|
|
1816
|
+
return None
|
|
1817
|
+
return self.stop_and_relaunch(verdict, argv, avoid, probe_pick=pick)
|
|
1818
|
+
|
|
1819
|
+
# -- stopping and relaunching --
|
|
1820
|
+
|
|
1821
|
+
def pane_info(self, sock):
|
|
1822
|
+
rc, out = self.r.run(pane_query_argv(sock, self.st["pane"]), 10)
|
|
1823
|
+
return parse_pane_reply(out) if rc == 0 else None
|
|
1824
|
+
|
|
1825
|
+
def pane_is_ours(self, info):
|
|
1826
|
+
return info is not None and (self.cfg.anypane or info[0] == self.st["ppid"])
|
|
1827
|
+
|
|
1828
|
+
def pane_problem(self, sock, info):
|
|
1829
|
+
"""None when the pane can take the relaunch; else why not. It must still be the
|
|
1830
|
+
launching shell's pane; with synchronize-panes on, typed keys would reach every
|
|
1831
|
+
pane of the window — other TUIs included; and a tmux mode (a pane scrolled back
|
|
1832
|
+
in copy mode stays there after the operator switches away) would eat the keys,
|
|
1833
|
+
so it is cancelled by command and read again."""
|
|
1834
|
+
if not self.pane_is_ours(info):
|
|
1835
|
+
return "pane"
|
|
1836
|
+
if info[3]:
|
|
1837
|
+
return "sync"
|
|
1838
|
+
if info[2]:
|
|
1839
|
+
self.r.run(mode_cancel_argv(sock, self.st["pane"]), 10)
|
|
1840
|
+
info = self.pane_info(sock)
|
|
1841
|
+
if not self.pane_is_ours(info):
|
|
1842
|
+
return "pane"
|
|
1843
|
+
if info[3]:
|
|
1844
|
+
return "sync"
|
|
1845
|
+
if info[2]:
|
|
1846
|
+
return "mode"
|
|
1847
|
+
return None
|
|
1848
|
+
|
|
1849
|
+
def other_holder(self):
|
|
1850
|
+
"""Another live claude whose registry (any account of this pool) names this
|
|
1851
|
+
session: stopping this one would hand the conversation to two clients."""
|
|
1852
|
+
if self.provider != "claude" or not self.sid:
|
|
1853
|
+
return None
|
|
1854
|
+
for path in glob.glob(os.path.join(self.st["acc_root"], "acct-*", "sessions",
|
|
1855
|
+
"*.json")):
|
|
1856
|
+
name = os.path.basename(path)[:-len(".json")]
|
|
1857
|
+
if not name.isdigit() or int(name) in (self.pid, 0, 1):
|
|
1858
|
+
continue
|
|
1859
|
+
try:
|
|
1860
|
+
with open(path, "rb") as fh:
|
|
1861
|
+
doc = json.loads(fh.read(1 << 20).decode("utf-8", "replace"))
|
|
1862
|
+
except (OSError, ValueError):
|
|
1863
|
+
continue
|
|
1864
|
+
if isinstance(doc, dict) and doc.get("sessionId") == self.sid \
|
|
1865
|
+
and self.r.alive(int(name)):
|
|
1866
|
+
return int(name)
|
|
1867
|
+
return None
|
|
1868
|
+
|
|
1869
|
+
def ours_alive(self, pid, started):
|
|
1870
|
+
"""``pid`` still runs (a zombie does not) and is still the process recorded with
|
|
1871
|
+
start time ``started`` — a pid reused since names someone else's process."""
|
|
1872
|
+
return not self.r.gone(pid) and self.r.lstart(pid) == started
|
|
1873
|
+
|
|
1874
|
+
def job_gone(self):
|
|
1875
|
+
"""The stopped client and the descendants recorded right before the signal: none
|
|
1876
|
+
of them left. Only that tree — never the client's whole process group, which also
|
|
1877
|
+
holds what the shim started in the background before its exec (the detached
|
|
1878
|
+
`limits --quiet` refresh), reparented away and none of this session's business."""
|
|
1879
|
+
return not any(self.ours_alive(p, s) for p, s in [(self.pid, self.lstart)] + self.tree)
|
|
1880
|
+
|
|
1881
|
+
def shell_ready(self, sock):
|
|
1882
|
+
"""One reading: the client's job gone, and the pane back at the launching shell's
|
|
1883
|
+
prompt — its program in front, and the shell itself holding the terminal (a job
|
|
1884
|
+
it runs next would take it away)."""
|
|
1885
|
+
if not self.job_gone():
|
|
1886
|
+
return False
|
|
1887
|
+
info = self.pane_info(sock)
|
|
1888
|
+
if not self.pane_is_ours(info) or not is_shell(info[1]):
|
|
1889
|
+
return False
|
|
1890
|
+
shell = self.r.proc_status(self.st["ppid"])
|
|
1891
|
+
return shell is not None and ps_foreground(shell) is not False
|
|
1892
|
+
|
|
1893
|
+
def stop_process(self):
|
|
1894
|
+
"""SIGTERM to the client, then after TERM_GRACE SIGKILL for whatever is left of it
|
|
1895
|
+
and of the descendants recorded before the TERM (codex's node wrapper and native
|
|
1896
|
+
binary, claude's tool and MCP children) — each re-verified by its start time
|
|
1897
|
+
right before the signal. False when the process was already gone (or replaced)
|
|
1898
|
+
before the first one."""
|
|
1899
|
+
if self.r.lstart(self.pid) != self.lstart:
|
|
1900
|
+
return False
|
|
1901
|
+
table = self.r.ps_table()
|
|
1902
|
+
starts = {p: ls for p, _pp, ls in table}
|
|
1903
|
+
# Only ever the client's own job. A process-tree read that goes wrong must not be
|
|
1904
|
+
# able to turn a session stop into a machine-wide SIGKILL (a broken copy of this
|
|
1905
|
+
# code under test once did exactly that): a descendant is kept only while it
|
|
1906
|
+
# shares the client's process group, it is never pid 1, this watcher, its parent
|
|
1907
|
+
# or the launching shell, and a "tree" too large to be one TUI's is not trusted at
|
|
1908
|
+
# all. A descendant without a start time could never be re-verified: left alone.
|
|
1909
|
+
job = self.r.pgid(self.pid)
|
|
1910
|
+
never = {1, os.getpid(), os.getppid(), self.st.get("ppid")}
|
|
1911
|
+
tree = [(p, starts[p]) for p in descendants(table, self.pid)
|
|
1912
|
+
if starts.get(p) and p > 1 and p not in never
|
|
1913
|
+
and job is not None and self.r.pgid(p) == job]
|
|
1914
|
+
self.tree = tree if len(tree) <= MAX_TREE else []
|
|
1915
|
+
if not self.r.signal(self.pid, signal.SIGTERM):
|
|
1916
|
+
return False
|
|
1917
|
+
self.stopped = True
|
|
1918
|
+
deadline = self.r.now() + self.cfg.term_grace
|
|
1919
|
+
while not self.job_gone() and self.r.now() < deadline:
|
|
1920
|
+
self.r.sleep(min(0.2, self.cfg.poll))
|
|
1921
|
+
for pid, started in [(self.pid, self.lstart)] + self.tree:
|
|
1922
|
+
if pid != self.pid and self.r.pgid(pid) != job:
|
|
1923
|
+
continue
|
|
1924
|
+
if self.ours_alive(pid, started):
|
|
1925
|
+
self.r.signal(pid, signal.SIGKILL)
|
|
1926
|
+
return True
|
|
1927
|
+
|
|
1928
|
+
def wait_shell(self, sock):
|
|
1929
|
+
"""Until shell_ready holds on two readings SHELL_SETTLE apart (one reading can
|
|
1930
|
+
fall between two programs of a `while :; do claude; done`), or PANE_WAIT ends."""
|
|
1931
|
+
deadline = self.r.now() + self.cfg.pane_wait
|
|
1932
|
+
ready_since = None
|
|
1933
|
+
while True:
|
|
1934
|
+
now = self.r.now()
|
|
1935
|
+
if self.shell_ready(sock):
|
|
1936
|
+
if ready_since is None:
|
|
1937
|
+
ready_since = now
|
|
1938
|
+
elif now - ready_since >= SHELL_SETTLE:
|
|
1939
|
+
return True
|
|
1940
|
+
else:
|
|
1941
|
+
ready_since = None
|
|
1942
|
+
if now >= deadline:
|
|
1943
|
+
return False
|
|
1944
|
+
self.r.sleep(min(0.2, self.cfg.poll))
|
|
1945
|
+
|
|
1946
|
+
def stop_and_relaunch(self, verdict, argv, avoid, probe_pick="", kill=True):
|
|
1947
|
+
"""0 once the typed relaunch has consumed its token; GiveUp otherwise. Every check
|
|
1948
|
+
that can refuse runs BEFORE the kill: a stopped session must always have its
|
|
1949
|
+
relaunch."""
|
|
1950
|
+
cls = verdict["class"]
|
|
1951
|
+
sock = tmux_socket(self.st["tmux"])
|
|
1952
|
+
# 1. The pane must still be the shell that launched the shim — otherwise the
|
|
1953
|
+
# typed command would land somewhere else (a subshell, another program) — and
|
|
1954
|
+
# that shell must be one the typed syntax works in: stopping a session whose
|
|
1955
|
+
# pane runs nu or tcsh would leave it stopped with nowhere to relaunch.
|
|
1956
|
+
info = self.pane_info(sock) if sock else None
|
|
1957
|
+
if not self.pane_is_ours(info):
|
|
1958
|
+
self.giveup("pane", sid=self.sid, **{"class": cls})
|
|
1959
|
+
shell = self.r.proc_status(self.st["ppid"])
|
|
1960
|
+
if shell is None or not is_shell(shell[2]):
|
|
1961
|
+
self.giveup("pane-shell", sid=self.sid, **{"class": cls})
|
|
1962
|
+
# Only a client in the foreground of its terminal, and not stopped, is killed:
|
|
1963
|
+
# one suspended with Ctrl-Z leaves another program in front, which the relaunch
|
|
1964
|
+
# must not be typed into.
|
|
1965
|
+
if kill:
|
|
1966
|
+
client = self.r.proc_status(self.pid)
|
|
1967
|
+
flags = client[0] if client else ""
|
|
1968
|
+
if not client or "T" in flags or flags.startswith("t") \
|
|
1969
|
+
or ("+" not in flags and not self.cfg.test_tty):
|
|
1970
|
+
self.giveup("background", sid=self.sid, **{"class": cls})
|
|
1971
|
+
# ...and only one that leads its own process group: a job of a job-control
|
|
1972
|
+
# shell. Anything else (a client started by a script, `sh -c`, a pipeline
|
|
1973
|
+
# stage) is not the prompt's job, and no prompt comes back when it ends.
|
|
1974
|
+
# The suites' no-terminal knob waives this too: their harness has no job
|
|
1975
|
+
# control to give the client a group of its own.
|
|
1976
|
+
if not self.cfg.test_tty and self.r.pgid(self.pid) != self.pid:
|
|
1977
|
+
self.giveup("pgrp", sid=self.sid, **{"class": cls})
|
|
1978
|
+
# One conversation, one client: another live process holding this session
|
|
1979
|
+
# means stopping this one would not free it.
|
|
1980
|
+
holder = self.other_holder()
|
|
1981
|
+
if holder:
|
|
1982
|
+
self.giveup("holder", sid=self.sid, holder=holder, **{"class": cls})
|
|
1983
|
+
cwd = self.session_cwd()
|
|
1984
|
+
if cwd is None:
|
|
1985
|
+
self.giveup("cwd", sid=self.sid, **{"class": cls})
|
|
1986
|
+
# Last, as it is the one check that touches the pane: synchronized panes, or a
|
|
1987
|
+
# tmux mode that a cancel does not end, refuse.
|
|
1988
|
+
why = self.pane_problem(sock, info)
|
|
1989
|
+
if why:
|
|
1990
|
+
self.giveup(why, sid=self.sid, **{"class": cls})
|
|
1991
|
+
if self.switched_off():
|
|
1992
|
+
raise GiveUp("off")
|
|
1993
|
+
# 2. The relaunch file exists BEFORE the kill: a kill always has its relaunch.
|
|
1994
|
+
now = self.r.now()
|
|
1995
|
+
token = make_token()
|
|
1996
|
+
fields = relaunch_fields(self.provider, verdict, self.acct, self.sid, cwd,
|
|
1997
|
+
self.st["depth"], avoid,
|
|
1998
|
+
hist_append(self.st["hist"], cls, now), self.chain)
|
|
1999
|
+
try:
|
|
2000
|
+
keys = send_keys_argvs(sock, self.st["pane"], self.provider, token, self.sid,
|
|
2001
|
+
self.st["self"], self.root)
|
|
2002
|
+
paths = write_relaunch(self.ar_dir, token, fields, argv)
|
|
2003
|
+
except (OSError, ValueError):
|
|
2004
|
+
self.giveup("write", sid=self.sid, **{"class": cls})
|
|
2005
|
+
# 3. Stop the client (a crash needs no stopping).
|
|
2006
|
+
if kill and not self.stop_process():
|
|
2007
|
+
remove_quietly(*paths)
|
|
2008
|
+
self.giveup("gone", sid=self.sid, **{"class": cls})
|
|
2009
|
+
# 4. Only a shell prompt may receive the command, once nothing of the stopped job
|
|
2010
|
+
# is left — and the pane is read once more right before typing.
|
|
2011
|
+
if not self.wait_shell(sock):
|
|
2012
|
+
remove_quietly(*paths)
|
|
2013
|
+
self.giveup("shell" if self.job_gone() else "tree", sid=self.sid,
|
|
2014
|
+
**{"class": cls})
|
|
2015
|
+
why = self.pane_problem(sock, self.pane_info(sock))
|
|
2016
|
+
if why:
|
|
2017
|
+
remove_quietly(*paths)
|
|
2018
|
+
self.giveup(why, sid=self.sid, **{"class": cls})
|
|
2019
|
+
# 5. Type the relaunch into the shell: through the shim that launched the session
|
|
2020
|
+
# (and the pool it used), not whatever the shell's PATH says today.
|
|
2021
|
+
for i, cmd in enumerate(keys):
|
|
2022
|
+
rc, _ = self.r.run(cmd, 10)
|
|
2023
|
+
if rc != 0:
|
|
2024
|
+
# Before the command text is in, the token is useless; after it, the
|
|
2025
|
+
# line sits at the prompt and Enter would still work — the file expires
|
|
2026
|
+
# on its own (600 s) and the watcher's prune removes it.
|
|
2027
|
+
if i < 2:
|
|
2028
|
+
remove_quietly(*paths)
|
|
2029
|
+
self.giveup("send-keys", sid=self.sid, **{"class": cls})
|
|
2030
|
+
# 6. The relaunched shim consumes the token first thing: a relaunch file still
|
|
2031
|
+
# there after RELAUNCH_WAIT means the line never ran (a shell that ignored it, a
|
|
2032
|
+
# prompt that was not one). The token is withdrawn — a line run later gets the
|
|
2033
|
+
# shim's resume hint — and the operator is told how to resume.
|
|
2034
|
+
deadline = self.r.now() + self.cfg.relaunch_wait
|
|
2035
|
+
while os.path.exists(paths[0]):
|
|
2036
|
+
if self.r.now() >= deadline:
|
|
2037
|
+
remove_quietly(*paths)
|
|
2038
|
+
self.giveup("relaunch", sid=self.sid, **{"class": cls})
|
|
2039
|
+
self.r.sleep(min(0.2, self.cfg.poll))
|
|
2040
|
+
# 7.
|
|
2041
|
+
self.log("switch", **{"from": self.acct, "class": cls, "sid": self.sid,
|
|
2042
|
+
"depth": int(self.st["depth"]) + 1, "probe": probe_pick or "-"})
|
|
2043
|
+
return 0
|
|
2044
|
+
|
|
2045
|
+
def on_exit(self):
|
|
2046
|
+
"""The client is gone and not by our hand. A clean exit (/exit, SIGHUP) deletes
|
|
2047
|
+
claude's registry; a crash or SIGKILL leaves it behind — that is the only crash
|
|
2048
|
+
signal, and it is claude-only."""
|
|
2049
|
+
if self.provider != "claude":
|
|
2050
|
+
return 0
|
|
2051
|
+
try:
|
|
2052
|
+
self.read()
|
|
2053
|
+
except GiveUp:
|
|
2054
|
+
return 0
|
|
2055
|
+
if self.tracker.pending is not None:
|
|
2056
|
+
return 0
|
|
2057
|
+
reg = self.registry()
|
|
2058
|
+
if reg is None:
|
|
2059
|
+
return 0
|
|
2060
|
+
now = self.r.now()
|
|
2061
|
+
if now - self.launched < self.cfg.crash_min_runtime:
|
|
2062
|
+
self.dbg("exit after %ds: too early to call a crash" % (now - self.launched))
|
|
2063
|
+
return 0
|
|
2064
|
+
self.sid = reg[0]
|
|
2065
|
+
if reg[1]:
|
|
2066
|
+
self.cwd = reg[1]
|
|
2067
|
+
ok, why = budget_check("crash", self.st["depth"], self.st["hist"], now, self.cfg)
|
|
2068
|
+
if not ok:
|
|
2069
|
+
self.log("giveup", reason=why, sid=self.sid, **{"class": "crash"})
|
|
2070
|
+
return 0
|
|
2071
|
+
verdict = _verdict("crash", "crash", now)
|
|
2072
|
+
argv = self.build_relaunch_argv("crash")
|
|
2073
|
+
if argv is None:
|
|
2074
|
+
self.log("giveup", reason="argv", sid=self.sid, **{"class": "crash"})
|
|
2075
|
+
return 0
|
|
2076
|
+
self.log("crash", sid=self.sid, acct=self.acct, runtime=int(now - self.launched))
|
|
2077
|
+
try:
|
|
2078
|
+
return self.stop_and_relaunch(verdict, argv, avoid_merge(self.st["avoid"], now),
|
|
2079
|
+
kill=False)
|
|
2080
|
+
except GiveUp:
|
|
2081
|
+
return 0
|
|
2082
|
+
|
|
2083
|
+
def cleanup(self):
|
|
2084
|
+
remove_quietly(self.statefile, argv_path_for(self.statefile), self.claim)
|
|
2085
|
+
|
|
2086
|
+
|
|
2087
|
+
# ---- CLI ----------------------------------------------------------------------------
|
|
2088
|
+
|
|
2089
|
+
def daemonize():
|
|
2090
|
+
"""Own session, deaf to the terminal's signals: closing the pane or Ctrl-C in it
|
|
2091
|
+
must not take the watcher along before it can decide anything."""
|
|
2092
|
+
try:
|
|
2093
|
+
os.setsid()
|
|
2094
|
+
except OSError:
|
|
2095
|
+
pass
|
|
2096
|
+
for sig in (signal.SIGINT, signal.SIGHUP, signal.SIGTSTP):
|
|
2097
|
+
try:
|
|
2098
|
+
signal.signal(sig, signal.SIG_IGN)
|
|
2099
|
+
except (OSError, ValueError):
|
|
2100
|
+
pass
|
|
2101
|
+
|
|
2102
|
+
|
|
2103
|
+
def cmd_watch(state_path, runner=None, environ=None):
|
|
2104
|
+
st = load_state(state_path)
|
|
2105
|
+
if st is None:
|
|
2106
|
+
return 0
|
|
2107
|
+
argv = load_argv(argv_path_for(state_path))
|
|
2108
|
+
if argv is None:
|
|
2109
|
+
return 0
|
|
2110
|
+
stdir = os.path.dirname(os.path.abspath(state_path))
|
|
2111
|
+
prune(stdir, time.time(),
|
|
2112
|
+
keep=(os.path.abspath(state_path), os.path.abspath(argv_path_for(state_path))))
|
|
2113
|
+
watcher = Watcher(state_path, st, argv, runner, environ)
|
|
2114
|
+
try:
|
|
2115
|
+
return watcher.run()
|
|
2116
|
+
except Exception: # noqa: BLE001 — fail open: the session simply goes unsupervised
|
|
2117
|
+
watcher.dbg("watcher error: " + traceback.format_exc().replace("\n", " | "))
|
|
2118
|
+
return 0
|
|
2119
|
+
finally:
|
|
2120
|
+
watcher.cleanup()
|
|
2121
|
+
|
|
2122
|
+
|
|
2123
|
+
def cmd_classify(provider, path, since):
|
|
2124
|
+
tracker = make_tracker(provider, since)
|
|
2125
|
+
events = []
|
|
2126
|
+
now = time.time()
|
|
2127
|
+
with open(path, "rb") as fh:
|
|
2128
|
+
for raw in fh:
|
|
2129
|
+
rec = read_json_line(raw.rstrip(b"\n"))
|
|
2130
|
+
if rec is None:
|
|
2131
|
+
continue
|
|
2132
|
+
ev = tracker.feed(rec, now)
|
|
2133
|
+
if ev is None:
|
|
2134
|
+
continue
|
|
2135
|
+
kind, val = ev
|
|
2136
|
+
if kind == "error":
|
|
2137
|
+
events.append(dict(verdict_public(val), kind="verdict"))
|
|
2138
|
+
elif kind == "never":
|
|
2139
|
+
events.append({"kind": "never", "code": val})
|
|
2140
|
+
else:
|
|
2141
|
+
events.append({"kind": "cancel", "by": val})
|
|
2142
|
+
print(json.dumps({"events": events, "pending": verdict_public(tracker.pending)},
|
|
2143
|
+
sort_keys=True))
|
|
2144
|
+
return 0
|
|
2145
|
+
|
|
2146
|
+
|
|
2147
|
+
def cmd_relaunch_argv(provider, state_path, sid, cls):
|
|
2148
|
+
st = load_state(state_path)
|
|
2149
|
+
argv = load_argv(argv_path_for(state_path))
|
|
2150
|
+
if st is None or argv is None or st["provider"] != provider:
|
|
2151
|
+
print("null")
|
|
2152
|
+
return 1
|
|
2153
|
+
cfg = Config(provider)
|
|
2154
|
+
out = relaunch_argv(provider, argv, sid, cls, cfg.prompt)
|
|
2155
|
+
print(json.dumps(out))
|
|
2156
|
+
return 0 if out is not None else 1
|
|
2157
|
+
|
|
2158
|
+
|
|
2159
|
+
def cmd_trust(acct_dir, cwd):
|
|
2160
|
+
"""Mark ``cwd`` trusted in ``<acct_dir>/.claude.json`` (projects[cwd].
|
|
2161
|
+
hasTrustDialogAccepted), for a relaunch landing on an account that never opened that
|
|
2162
|
+
directory: Claude asks "Is this a project you trust?" there even under
|
|
2163
|
+
--dangerously-skip-permissions, and nobody is watching to answer. The shim passes the
|
|
2164
|
+
logical $PWD, and Claude may key ``projects`` by the physical path, so both are marked
|
|
2165
|
+
when they differ. Every other key is kept; the write is atomic and keeps the file's
|
|
2166
|
+
mode. A missing or unreadable file, anything unexpected, or a file some other process
|
|
2167
|
+
(a running Claude) rewrote meanwhile, changes nothing."""
|
|
2168
|
+
path = os.path.join(acct_dir, ".claude.json")
|
|
2169
|
+
if not os.path.isabs(cwd or ""):
|
|
2170
|
+
return
|
|
2171
|
+
try:
|
|
2172
|
+
with open(path, "rb") as fh:
|
|
2173
|
+
st = os.fstat(fh.fileno())
|
|
2174
|
+
doc = json.loads(fh.read().decode("utf-8"))
|
|
2175
|
+
except (OSError, ValueError):
|
|
2176
|
+
return
|
|
2177
|
+
if not isinstance(doc, dict):
|
|
2178
|
+
return
|
|
2179
|
+
projects = doc.setdefault("projects", {})
|
|
2180
|
+
if not isinstance(projects, dict):
|
|
2181
|
+
return
|
|
2182
|
+
keys = [cwd]
|
|
2183
|
+
real = os.path.realpath(cwd)
|
|
2184
|
+
if real != cwd:
|
|
2185
|
+
keys.append(real)
|
|
2186
|
+
entries = [projects.setdefault(key, {}) for key in keys]
|
|
2187
|
+
if not all(isinstance(entry, dict) for entry in entries):
|
|
2188
|
+
return
|
|
2189
|
+
todo = [entry for entry in entries if entry.get("hasTrustDialogAccepted") is not True]
|
|
2190
|
+
if not todo:
|
|
2191
|
+
return
|
|
2192
|
+
for entry in todo:
|
|
2193
|
+
entry["hasTrustDialogAccepted"] = True
|
|
2194
|
+
handle, scratch = tempfile.mkstemp(prefix=".claude.json.", dir=os.path.dirname(path))
|
|
2195
|
+
try:
|
|
2196
|
+
with os.fdopen(handle, "w", encoding="utf-8") as fh:
|
|
2197
|
+
os.fchmod(fh.fileno(), stat.S_IMODE(st.st_mode))
|
|
2198
|
+
fh.write(json.dumps(doc, indent=2) + "\n")
|
|
2199
|
+
# Claude rewrites this file whenever it likes: one written since it was read
|
|
2200
|
+
# would lose that write to a stale copy, so it is left as it is.
|
|
2201
|
+
now = os.stat(path)
|
|
2202
|
+
if (now.st_mtime_ns, now.st_size) != (st.st_mtime_ns, st.st_size):
|
|
2203
|
+
remove_quietly(scratch)
|
|
2204
|
+
return
|
|
2205
|
+
os.replace(scratch, path)
|
|
2206
|
+
except BaseException:
|
|
2207
|
+
remove_quietly(scratch)
|
|
2208
|
+
raise
|
|
2209
|
+
|
|
2210
|
+
|
|
2211
|
+
def _opts(args, names):
|
|
2212
|
+
"""Minimal ``--name value`` parser (argparse would print usage to a daemon's
|
|
2213
|
+
/dev/null and exit 2 on anything odd; this just returns None)."""
|
|
2214
|
+
out, rest, i = {}, [], 0
|
|
2215
|
+
while i < len(args):
|
|
2216
|
+
if args[i] in names:
|
|
2217
|
+
if i + 1 >= len(args):
|
|
2218
|
+
return None, None
|
|
2219
|
+
out[args[i][2:]] = args[i + 1]
|
|
2220
|
+
i += 2
|
|
2221
|
+
else:
|
|
2222
|
+
rest.append(args[i])
|
|
2223
|
+
i += 1
|
|
2224
|
+
return out, rest
|
|
2225
|
+
|
|
2226
|
+
|
|
2227
|
+
def main(argv=None):
|
|
2228
|
+
args = list(sys.argv[1:] if argv is None else argv)
|
|
2229
|
+
if not args:
|
|
2230
|
+
sys.stderr.write(__doc__.split("CLI", 1)[1].split("Python 3.9", 1)[0])
|
|
2231
|
+
return 2
|
|
2232
|
+
cmd, args = args[0], args[1:]
|
|
2233
|
+
if cmd == "watch":
|
|
2234
|
+
opts, _ = _opts(args, ("--state",))
|
|
2235
|
+
if not opts or "state" not in opts:
|
|
2236
|
+
return 2
|
|
2237
|
+
daemonize()
|
|
2238
|
+
try:
|
|
2239
|
+
return cmd_watch(opts["state"])
|
|
2240
|
+
except Exception: # noqa: BLE001 — a watcher never surfaces a traceback
|
|
2241
|
+
return 0
|
|
2242
|
+
if cmd == "classify":
|
|
2243
|
+
opts, rest = _opts(args, ("--provider", "--since"))
|
|
2244
|
+
if opts is None or opts.get("provider") not in PROVIDERS or len(rest) != 1:
|
|
2245
|
+
return 2
|
|
2246
|
+
try:
|
|
2247
|
+
since = float(opts["since"]) if opts.get("since") else 0.0
|
|
2248
|
+
return cmd_classify(opts["provider"], rest[0], since)
|
|
2249
|
+
except (OSError, ValueError) as exc:
|
|
2250
|
+
sys.stderr.write("autoresume classify: %s\n" % exc)
|
|
2251
|
+
return 2
|
|
2252
|
+
if cmd == "relaunch-argv":
|
|
2253
|
+
opts, _ = _opts(args, ("--provider", "--state", "--sid", "--class"))
|
|
2254
|
+
if opts is None or opts.get("provider") not in PROVIDERS \
|
|
2255
|
+
or not all(k in opts for k in ("state", "sid", "class")):
|
|
2256
|
+
return 2
|
|
2257
|
+
return cmd_relaunch_argv(opts["provider"], opts["state"], opts["sid"], opts["class"])
|
|
2258
|
+
if cmd == "trust":
|
|
2259
|
+
# Called by the shim right before its exec: always 0, never a word of output.
|
|
2260
|
+
try:
|
|
2261
|
+
opts, _ = _opts(args, ("--acct-dir", "--cwd"))
|
|
2262
|
+
if opts and opts.get("acct-dir") and opts.get("cwd"):
|
|
2263
|
+
cmd_trust(opts["acct-dir"], opts["cwd"])
|
|
2264
|
+
except Exception: # noqa: BLE001 — fail open
|
|
2265
|
+
pass
|
|
2266
|
+
return 0
|
|
2267
|
+
return 2
|
|
2268
|
+
|
|
2269
|
+
|
|
2270
|
+
if __name__ == "__main__":
|
|
2271
|
+
sys.exit(main())
|