mlai-code 0.3.8 → 0.3.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,22 +1,56 @@
1
- """Supervision for test and build processes the agent starts (PROC-001).
1
+ """Supervision for test, build and verification processes the agent starts (PROC-001, CLI r7).
2
2
 
3
- A test runner that spins at full CPU with no output for minutes burns time and tokens until the run limit.
4
- This watchdog looks only at descendants of the engine process (the processes this run owns), finds test/build
5
- runner trees, and stops a tree when it has been silent for IDLE seconds or alive for WALL seconds. The agent is
6
- told what was stopped, why, and how to rerun it in a bounded way.
3
+ A test runner that spins at full CPU with no output for minutes burns time and tokens until the run limit, and a
4
+ verification script whose headless browser never closes (`node e2e.js` with puppeteer driving Edge) keeps the command
5
+ running forever. This watchdog looks only at descendants of the engine process (the processes this run owns) and stops:
7
6
 
8
- Output progress is the bytes the runner tree has written (/proc/<pid>/io wchar), so it works for pipes and files
9
- alike: a process that neither writes nor finishes is silent. Linux /proc only; elsewhere the watchdog is a no-op.
7
+ - test/build runner trees (vitest, jest, pytest, ...) and the agent's own verification scripts (node/python/deno/bun
8
+ running a file such as e2e.js, .e2e-tmp/e2e.mjs, scripts/check.mjs, smoke_test.py) when they have been silent for
9
+ IDLE seconds or alive for WALL seconds;
10
+ - any foreground command (a Bash call without run_in_background) still running after COMMAND_WALL seconds.
11
+
12
+ The whole tree goes, including a browser the script left behind (msedge/chrome children, also ones whose parent already
13
+ exited), so the pipes close and the engine's command returns. The agent is told what was stopped, why, and how to
14
+ rerun it in a bounded way.
15
+
16
+ Platforms:
17
+ - Linux: /proc. Output progress is the bytes the tree has written (/proc/<pid>/io wchar), so pipes and files count.
18
+ - Windows: ctypes only (Toolhelp snapshot for parents, GetProcessTimes, GetProcessIoCounters for written bytes,
19
+ NtQueryInformationProcess for command lines, TerminateProcess to stop). No wmic or PowerShell: they take seconds.
20
+ - macOS: one `ps -axo pid=,ppid=,etime=,time=,command=` per tick. There are no I/O counters, so a tree is silent when
21
+ its CPU time stops moving; the wall limits apply as everywhere.
10
22
  """
11
- import os, re, signal, time
23
+ import os, re, signal, subprocess, sys, threading, time
12
24
  from pathlib import Path
13
25
 
14
26
  IDLE = int(os.getenv('MINI_CHECK_IDLE_SECONDS', '300')) # no new output for five minutes
15
27
  WALL = int(os.getenv('MINI_CHECK_WALL_SECONDS', '1200')) # twenty minutes for any single test or build command
28
+ COMMAND_WALL = int(os.getenv('MINI_COMMAND_WALL_SECONDS', str(WALL))) # any foreground command
16
29
  RUNNER = re.compile(r'(^|/|\s)(vitest|jest|playwright|pytest|mocha|ava|tap|cypress|karma|tsc|eslint|next build|vite build|webpack|cargo test|go test|node --test|python3? -m (pytest|unittest))(\s|$)', re.I)
30
+ # The agent's own verification scripts: an interpreter as the program, then a script whose file or folder name says
31
+ # what it is (e2e.js, .e2e-tmp/e2e.mjs, scripts/check.mjs, tests/smoke.py, verify-ui.ts, puppeteer-run.cjs).
32
+ SCRIPT = re.compile(r'''^\s*(?:"[^"]*[/\\]|'[^']*[/\\]|[^\s"']*[/\\])?(node|nodejs|python[\d.]*|py|deno|bun|tsx|ts-node)(?:\.exe)?["']?((?:\s+(?:-[-\w=.:]*|run|test))*)\s+["']?([^\s"']+)''', re.I)
33
+ SCRIPT_NAME = re.compile(r'(^|[/\\._-])(e2e|tests?|checks?|verify|verification|smoke|harness|puppeteer|playwright)(?=$|[^a-z])', re.I)
34
+ INLINE = re.compile(r'(^|\s)(-e|-c|-p|--eval|--print)(\s|$)')
35
+ BROWSER = re.compile(r'(^|[/\\\s"])(msedge|chrome|chromium|chromium-browser|google-chrome|headless_shell|chrome-headless-shell|firefox|Microsoft Edge|Google Chrome)(\.exe)?(["\s]|$)', re.I)
36
+ SHELL = re.compile(r'^\s*(?:"[^"]*[/\\]|[^\s"]*[/\\])?(bash|sh|zsh|dash|busybox|cmd|powershell|pwsh)(\.exe)?"?(\s|$)', re.I)
37
+ HERE = str(Path(__file__).resolve().parent)
17
38
  HZ = os.sysconf('SC_CLK_TCK') if hasattr(os, 'sysconf') else 100
18
39
  PROC = Path('/proc')
19
40
 
41
+ # Foreground Bash calls the engine is running: tool_call_id -> (command, started). Filled from the bridge's stream.
42
+ FOREGROUND, _FG_LOCK = {}, threading.Lock()
43
+
44
+
45
+ def foreground(call_id, command, now=None):
46
+ with _FG_LOCK: FOREGROUND[str(call_id)] = (str(command or ''), now or time.time())
47
+
48
+
49
+ def done(call_id):
50
+ with _FG_LOCK: FOREGROUND.pop(str(call_id), None)
51
+
52
+
53
+ # ---------- Linux (/proc) ----------
20
54
 
21
55
  def _read(pid, name):
22
56
  try: return (PROC / str(pid) / name).read_bytes()
@@ -41,21 +75,6 @@ def uptime():
41
75
  except (OSError, ValueError, IndexError): return 0.0
42
76
 
43
77
 
44
- def descendants(root):
45
- """pid -> (ppid, cpu, start) for every live descendant of root."""
46
- table = {}
47
- for entry in PROC.iterdir() if PROC.is_dir() else []:
48
- if entry.name.isdigit():
49
- st = stat(int(entry.name))
50
- if st: table[int(entry.name)] = st
51
- kids, out, todo = {}, {}, [root]
52
- for pid, st in table.items(): kids.setdefault(st[0], []).append(pid)
53
- while todo:
54
- for child in kids.get(todo.pop(), []):
55
- if child not in out: out[child] = table[child]; todo.append(child)
56
- return out
57
-
58
-
59
78
  def written(pid):
60
79
  """Bytes this process has written through write calls (pipes, files, sockets). Silent loops do not move it."""
61
80
  for line in _read(pid, 'io').decode('ascii', 'replace').splitlines():
@@ -65,60 +84,379 @@ def written(pid):
65
84
  return 0
66
85
 
67
86
 
87
+ class Linux:
88
+ name = 'linux'
89
+ has_io = True
90
+
91
+ def table(self):
92
+ """pid -> (ppid, cpu, start_epoch) for every process."""
93
+ boot, out = time.time() - uptime(), {}
94
+ for entry in PROC.iterdir() if PROC.is_dir() else []:
95
+ if entry.name.isdigit():
96
+ st = stat(int(entry.name))
97
+ if st: out[int(entry.name)] = (st[0], st[1], boot + st[2])
98
+ return out
99
+
100
+ def command(self, pid): return cmdline(pid)
101
+ def written(self, pid): return written(pid)
102
+
103
+ def kill(self, pids):
104
+ for sig in (signal.SIGTERM, signal.SIGKILL):
105
+ for p in pids:
106
+ try: os.kill(p, sig)
107
+ except (ProcessLookupError, PermissionError, OSError): pass
108
+ if sig == signal.SIGTERM and pids: time.sleep(2)
109
+
110
+
111
+ # ---------- macOS (ps) ----------
112
+
113
+ def duration(text):
114
+ """ps etime/time ([[dd-]hh:]mm:ss[.cc]) in seconds."""
115
+ text = text.strip()
116
+ days = 0
117
+ if '-' in text:
118
+ d, text = text.split('-', 1); days = int(d)
119
+ parts = [float(p) for p in text.split(':')]
120
+ secs = 0.0
121
+ for p in parts: secs = secs * 60 + p
122
+ return days * 86400 + secs
123
+
124
+
125
+ def parse_ps(output, now=None):
126
+ """`ps -axo pid=,ppid=,etime=,time=,command=` -> {pid: (ppid, cpu, start_epoch)}, {pid: command}."""
127
+ now = now or time.time()
128
+ table, commands = {}, {}
129
+ for line in output.splitlines():
130
+ bits = line.split(None, 4)
131
+ if len(bits) < 4: continue
132
+ try: pid, ppid, age, cpu = int(bits[0]), int(bits[1]), duration(bits[2]), duration(bits[3])
133
+ except ValueError: continue
134
+ table[pid] = (ppid, cpu, now - age)
135
+ commands[pid] = bits[4].strip() if len(bits) > 4 else ''
136
+ return table, commands
137
+
138
+
139
+ class Mac:
140
+ name = 'darwin'
141
+ has_io = False
142
+
143
+ def __init__(self): self.commands = {}
144
+
145
+ def table(self):
146
+ try: out = subprocess.run(['ps', '-axo', 'pid=,ppid=,etime=,time=,command='], capture_output=True, text=True, timeout=10).stdout
147
+ except (OSError, subprocess.SubprocessError): return {}
148
+ table, self.commands = parse_ps(out)
149
+ return table
150
+
151
+ def command(self, pid): return self.commands.get(pid, '')
152
+ def written(self, pid): return None
153
+ kill = Linux.kill
154
+
155
+
156
+ # ---------- Windows (ctypes) ----------
157
+
158
+ EPOCH_AS_FILETIME = 116444736000000000
159
+ PROCESS_TERMINATE, PROCESS_VM_READ, PROCESS_QUERY_LIMITED_INFORMATION = 0x0001, 0x0010, 0x1000
160
+
161
+
162
+ class Windows:
163
+ name = 'win32'
164
+ has_io = True
165
+
166
+ def __init__(self, api=None):
167
+ self.api = api or WinApi()
168
+ self.exe = {}
169
+
170
+ def table(self):
171
+ """Parents come from one Toolhelp snapshot; times only for processes that are asked about (see times)."""
172
+ snap = self.api.snapshot() # [(pid, ppid, exe)]
173
+ self.exe = {pid: exe for pid, _, exe in snap}
174
+ self._times = {}
175
+ return LazyTable({pid: ppid for pid, ppid, _ in snap}, self.times)
176
+
177
+ def times(self, pid):
178
+ if pid not in self._times: self._times[pid] = self.api.times(pid) # (cpu, start_epoch) or None
179
+ return self._times[pid]
180
+
181
+ def command(self, pid): return self.api.command_line(pid) or self.exe.get(pid, '')
182
+ def written(self, pid): return self.api.written(pid) or 0
183
+
184
+ def kill(self, pids):
185
+ for p in pids: self.api.terminate(p)
186
+
187
+
188
+ class LazyTable(dict):
189
+ """pid -> (ppid, cpu, start) whose times are read on first access. Windows reuses PIDs, so a child must have
190
+ started after the parent it names; iterating yields every pid in the snapshot."""
191
+
192
+ def __init__(self, parents, times):
193
+ super().__init__(); self.parents, self.times_of = parents, times
194
+
195
+ def __iter__(self): return iter(self.parents)
196
+ def __len__(self): return len(self.parents)
197
+ def __contains__(self, pid): return pid in self.parents and self.times_of(pid) is not None
198
+ def keys(self): return list(self.parents)
199
+ def items(self): return [(p, self[p]) for p in list(self.parents) if p in self]
200
+
201
+ def __getitem__(self, pid):
202
+ t = self.times_of(pid) if pid in self.parents else None
203
+ if t is None: raise KeyError(pid)
204
+ return (self.parents[pid], t[0], t[1])
205
+
206
+ def get(self, pid, default=None):
207
+ try: return self[pid]
208
+ except KeyError: return default
209
+
210
+
211
+ class WinApi:
212
+ """The few kernel32/ntdll calls the watchdog needs."""
213
+
214
+ def __init__(self):
215
+ import ctypes
216
+ from ctypes import wintypes
217
+ self.c, self.w = ctypes, wintypes
218
+ self.k = ctypes.WinDLL('kernel32', use_last_error=True)
219
+ self.k.OpenProcess.restype = wintypes.HANDLE
220
+ self.k.CreateToolhelp32Snapshot.restype = wintypes.HANDLE
221
+ try: self.nt = ctypes.WinDLL('ntdll')
222
+ except OSError: self.nt = None
223
+
224
+ class PROCESSENTRY32W(ctypes.Structure):
225
+ _fields_ = [('dwSize', wintypes.DWORD), ('cntUsage', wintypes.DWORD), ('th32ProcessID', wintypes.DWORD),
226
+ ('th32DefaultHeapID', ctypes.c_size_t), ('th32ModuleID', wintypes.DWORD), ('cntThreads', wintypes.DWORD),
227
+ ('th32ParentProcessID', wintypes.DWORD), ('pcPriClassBase', wintypes.LONG), ('dwFlags', wintypes.DWORD),
228
+ ('szExeFile', wintypes.WCHAR * 260)]
229
+
230
+ class IO_COUNTERS(ctypes.Structure):
231
+ _fields_ = [(n, ctypes.c_ulonglong) for n in ('ReadOperationCount', 'WriteOperationCount', 'OtherOperationCount',
232
+ 'ReadTransferCount', 'WriteTransferCount', 'OtherTransferCount')]
233
+ self.Entry, self.IO = PROCESSENTRY32W, IO_COUNTERS
234
+
235
+ def snapshot(self):
236
+ c, k = self.c, self.k
237
+ h = k.CreateToolhelp32Snapshot(0x2, 0) # TH32CS_SNAPPROCESS
238
+ if not h or h == self.w.HANDLE(-1).value: return []
239
+ out = []
240
+ try:
241
+ e = self.Entry(); e.dwSize = c.sizeof(e)
242
+ ok = k.Process32FirstW(h, c.byref(e))
243
+ while ok:
244
+ out.append((int(e.th32ProcessID), int(e.th32ParentProcessID), e.szExeFile))
245
+ ok = k.Process32NextW(h, c.byref(e))
246
+ finally: k.CloseHandle(h)
247
+ return out
248
+
249
+ def _open(self, pid, access=PROCESS_QUERY_LIMITED_INFORMATION):
250
+ return self.k.OpenProcess(access, False, int(pid)) or None
251
+
252
+ def times(self, pid):
253
+ h = self._open(pid)
254
+ if not h: return None
255
+ try:
256
+ ft = [self.w.FILETIME() for _ in range(4)]
257
+ if not self.k.GetProcessTimes(h, *[self.c.byref(t) for t in ft]): return None
258
+ v = [(t.dwHighDateTime << 32) | t.dwLowDateTime for t in ft]
259
+ if v[1]: return None # has an exit time: gone, only a handle keeps it
260
+ return (v[2] + v[3]) / 1e7, (v[0] - EPOCH_AS_FILETIME) / 1e7
261
+ finally: self.k.CloseHandle(h)
262
+
263
+ def written(self, pid):
264
+ h = self._open(pid)
265
+ if not h: return 0
266
+ try:
267
+ io = self.IO()
268
+ return int(io.WriteTransferCount + io.OtherTransferCount) if self.k.GetProcessIoCounters(h, self.c.byref(io)) else 0
269
+ finally: self.k.CloseHandle(h)
270
+
271
+ def command_line(self, pid):
272
+ """ProcessCommandLineInformation (60, Windows 8.1+): a UNICODE_STRING followed by its text."""
273
+ if not self.nt: return ''
274
+ c = self.c
275
+ h = self._open(pid)
276
+ if not h: return ''
277
+ try:
278
+ size = c.c_ulong(0)
279
+ self.nt.NtQueryInformationProcess(h, 60, None, 0, c.byref(size))
280
+ if not size.value or size.value > 1 << 20: return ''
281
+ buf = c.create_string_buffer(size.value)
282
+ if self.nt.NtQueryInformationProcess(h, 60, buf, size, c.byref(size)) != 0: return ''
283
+
284
+ class US(c.Structure):
285
+ _fields_ = [('Length', c.c_ushort), ('MaximumLength', c.c_ushort), ('Buffer', c.c_void_p)]
286
+ us = US.from_buffer(buf)
287
+ return c.wstring_at(us.Buffer, us.Length // 2) if us.Buffer and us.Length else ''
288
+ finally: self.k.CloseHandle(h)
289
+
290
+ def terminate(self, pid):
291
+ h = self._open(pid, PROCESS_TERMINATE | PROCESS_QUERY_LIMITED_INFORMATION)
292
+ if not h: return False
293
+ try: return bool(self.k.TerminateProcess(h, 1))
294
+ finally: self.k.CloseHandle(h)
295
+
296
+
297
+ def backend(platform=None):
298
+ platform = platform or sys.platform
299
+ if platform == 'win32':
300
+ try: return Windows()
301
+ except Exception: return None
302
+ if platform == 'darwin': return Mac()
303
+ return Linux() if PROC.is_dir() else None
304
+
305
+
306
+ # ---------- trees ----------
307
+
308
+ def descendants(root, table=None, family=None):
309
+ """pid -> (ppid, cpu, start) for every live descendant of root. A child must not predate its parent (Windows reuses
310
+ PIDs). family {pid: start} adds processes whose parent was one of ours and already exited: on Windows they keep the
311
+ dead parent's PID (Edge left behind by a killed node), on Linux they were reparented and are found by their own PID."""
312
+ if table is None: table = Linux().table()
313
+ kids, out, todo = {}, {}, [root]
314
+ for pid in list(table): kids.setdefault(_ppid(table, pid), []).append(pid)
315
+ for pid, start in (family or {}).items():
316
+ if pid != root and pid not in todo: todo.append(pid)
317
+ while todo:
318
+ parent = todo.pop()
319
+ pst = table.get(parent)
320
+ if parent in (family or {}) and pst and abs(pst[2] - family[parent]) >= 2: continue # PID reused by another process
321
+ if parent != root and parent in (family or {}) and pst and abs(pst[2] - family[parent]) < 2 and parent not in out:
322
+ out[parent] = pst # a remembered member still alive
323
+ for child in kids.get(parent, []):
324
+ if child in out or child == root: continue
325
+ st = table.get(child)
326
+ if not st: continue
327
+ born = pst[2] if pst else (family or {}).get(parent)
328
+ if born is not None and st[2] + 1 < born: continue # PID reused: older than its "parent"
329
+ out[child] = st; todo.append(child)
330
+ return out
331
+
332
+
333
+ def _ppid(table, pid):
334
+ parents = getattr(table, 'parents', None)
335
+ if parents is not None: return parents.get(pid)
336
+ return table[pid][0]
337
+
338
+
339
+ def script(command):
340
+ """True for the agent's own verification script: `node e2e.js`, `python scripts/check.py`, `bun run smoke.ts`."""
341
+ m = SCRIPT.match(command or '')
342
+ if not m or INLINE.search(m.group(2) or ''): return False
343
+ if HERE in command: return False # the harness's own helpers
344
+ return bool(SCRIPT_NAME.search(m.group(3)))
345
+
346
+
347
+ def supervised_kind(command):
348
+ if RUNNER.search(command or '') and HERE not in (command or ''): return 'runner'
349
+ if script(command): return 'script'
350
+ return None
351
+
352
+
68
353
  def span(seconds):
69
354
  seconds = int(seconds)
70
355
  return '%d seconds' % seconds if seconds < 120 else '%d minutes' % (seconds // 60)
71
356
 
72
357
 
358
+ def _norm(s): return re.sub(r'\s+', ' ', s or '').strip()
359
+
360
+
73
361
  class Watch:
74
- def __init__(self, root_pid, notify, idle=IDLE, wall=WALL):
75
- self.root, self.notify, self.idle, self.wall = root_pid, notify, idle, wall
76
- self.seen = {} # pid -> {'out': size, 'cpu': cpu, 'progress': t}
362
+ def __init__(self, root_pid, notify, idle=IDLE, wall=WALL, command_wall=COMMAND_WALL, os_=None):
363
+ self.root, self.notify, self.idle, self.wall, self.command_wall = root_pid, notify, idle, wall, command_wall
364
+ self.os = os_ if os_ is not None else backend()
365
+ self.seen = {} # pid -> {'out': size, 'cpu': cpu, 'progress': t, 'start': t, 'family': {pid: start}}
77
366
  self.stopped = []
367
+ with _FG_LOCK: FOREGROUND.clear() # a new engine process: calls of an earlier one are gone with it
78
368
 
79
369
  def runners(self, procs):
80
- """Topmost runner process of each tree: its parent is not itself a runner."""
81
- found = {pid: cmdline(pid) for pid in procs}
82
- found = {pid: c for pid, c in found.items() if RUNNER.search(c)}
370
+ """Topmost supervised process of each tree: its parent is not itself supervised."""
371
+ found = {pid: self.os.command(pid) for pid in procs}
372
+ found = {pid: c for pid, c in found.items() if supervised_kind(c)}
83
373
  return {pid: c for pid, c in found.items() if procs[pid][0] not in found}
84
374
 
85
375
  def tick(self, now=None):
376
+ if self.os is None: return
86
377
  now = now or time.time()
87
- procs = descendants(self.root)
88
- boot = now - uptime()
378
+ table = self.os.table()
379
+ procs = descendants(self.root, table)
89
380
  tops = self.runners(procs)
90
381
  for pid in list(self.seen):
91
382
  if pid not in tops: self.seen.pop(pid)
92
383
  for pid, command in tops.items():
93
384
  ppid, cpu, start = procs[pid]
94
- tree = [pid] + [p for p in descendants(pid)]
95
- size = sum(written(p) for p in tree)
96
- cpu = sum(procs.get(p, (0, 0, 0))[1] for p in tree)
97
385
  rec = self.seen.get(pid)
386
+ family = dict(rec['family']) if rec else {}
387
+ sub = descendants(pid, table, family)
388
+ tree = [pid] + list(sub)
389
+ for p, st in sub.items(): family[p] = st[2]
390
+ size = sum(self.os.written(p) or 0 for p in tree) if self.os.has_io else None
391
+ cpu = sum(procs.get(p, sub.get(p, (0, 0, 0)))[1] for p in tree)
98
392
  if rec is None:
99
- self.seen[pid] = {'out': size, 'cpu': cpu, 'cpu0': cpu, 'progress': now, 'start': boot + start}; continue
100
- if size != rec['out']: rec['progress'] = now; rec['cpu0'] = cpu
393
+ self.seen[pid] = {'out': size, 'cpu': cpu, 'cpu0': cpu, 'progress': now, 'start': start, 'family': family}; continue
394
+ rec['family'] = family
395
+ moved = size != rec['out'] if size is not None else cpu - rec['cpu'] > 0.05
396
+ if moved: rec['progress'] = now; rec['cpu0'] = cpu
101
397
  rec['out'], rec['cpu'] = size, cpu
102
398
  age, silent = now - rec['start'], now - rec['progress']
103
399
  reason = None
104
400
  if age > self.wall: reason = 'ran for %s, past the %s limit for one command' % (span(age), span(self.wall))
105
- elif silent > self.idle: reason = 'wrote no output for %s while using %s CPU' % (span(silent), 'high' if cpu - rec.get('cpu0', cpu) > silent * 0.5 else 'little')
106
- if reason: self.stop(pid, tree, command, reason)
401
+ elif silent > self.idle and size is not None: reason = 'wrote no output for %s while using %s CPU' % (span(silent), 'high' if cpu - rec.get('cpu0', cpu) > silent * 0.5 else 'little')
402
+ elif silent > self.idle: reason = 'made no progress for %s (no CPU use, no exit)' % span(silent)
403
+ if reason: self.stop(pid, tree, command, reason, table)
404
+ self.foreground(now, table, procs)
107
405
 
108
- def stop(self, pid, tree, command, reason):
109
- for sig in (signal.SIGTERM, signal.SIGKILL):
110
- for p in reversed(tree):
111
- try: os.kill(p, sig)
112
- except (ProcessLookupError, PermissionError): pass
113
- if sig == signal.SIGTERM: time.sleep(2)
114
- self.seen.pop(pid, None)
115
- record = {'pid': pid, 'command': command[:300], 'reason': reason, 'time': time.time()}
406
+ def foreground(self, now, table, procs):
407
+ """Foreground Bash calls past the command wall: stop the shell running each one and everything under it."""
408
+ with _FG_LOCK: late = [(k, c, t) for k, (c, t) in FOREGROUND.items() if now - t > self.command_wall]
409
+ for call_id, command, started in late:
410
+ want = _norm(command)[:120]
411
+ hits = {p: st for p, st in procs.items() if want and want in _norm(self.os.command(p))}
412
+ if not hits: # the engine may wrap the command; take the shells it started for this call
413
+ hits = {p: st for p, st in procs.items() if started - 2 <= st[2] <= started + 15 and SHELL.match(self.os.command(p) or '')}
414
+ tops = [p for p, st in hits.items() if st[0] not in hits]
415
+ if not tops: continue
416
+ done(call_id)
417
+ for pid in tops:
418
+ tree = [pid] + list(descendants(pid, table, (self.seen.get(pid) or {}).get('family')))
419
+ self.stop(pid, tree, command, 'was still running after %s, past the %s limit for a foreground command' % (span(now - started), span(self.command_wall)), table)
420
+
421
+ def stop(self, pid, tree, command, reason, table=None):
422
+ rec = self.seen.pop(pid, None) or {}
423
+ # Remembered members (a browser whose parent exited) go too, when they are still the same process.
424
+ for p, start in (rec.get('family') or {}).items():
425
+ st = table.get(p) if table is not None else None
426
+ if p not in tree and st and abs(st[2] - start) < 2: tree.append(p)
427
+ browsers = sorted({_exe(self.os.command(p)) for p in tree if BROWSER.search(self.os.command(p) or '')} - {''})
428
+ self.os.kill(list(reversed(tree)))
429
+ record = {'pid': pid, 'command': command[:300], 'reason': reason, 'time': time.time(), 'processes': len(tree)}
430
+ if browsers: record['browsers'] = browsers
116
431
  self.stopped.append(record)
117
432
  self.notify(record)
118
433
 
434
+ def kill_all(self):
435
+ """Stop/time limit on Windows: no process groups there, so end every process this run owns, including the
436
+ ones remembered from earlier ticks whose parent already exited (they would hold the engine's pipes open)."""
437
+ if self.os is None: return 0
438
+ table = self.os.table()
439
+ family = {}
440
+ for rec in self.seen.values(): family.update(rec.get('family') or {})
441
+ tree = list(descendants(self.root, table, family))
442
+ self.os.kill(list(reversed(tree)))
443
+ return len(tree)
444
+
445
+
446
+ def kill_all(root_pid, watch=None):
447
+ """Ends every descendant of root_pid (Windows stop path). Never raises."""
448
+ try: return (watch or Watch(root_pid, lambda r: None)).kill_all()
449
+ except Exception: return 0
450
+
451
+
452
+ def _exe(command):
453
+ m = BROWSER.search(command or '')
454
+ return m.group(2).lower() if m else ''
455
+
119
456
 
120
457
  def guidance(record):
121
- return ('The platform stopped a test or build process that ' + record['reason'] + ': `' + record['command'][:200] + '`. '
122
- 'It was the only process stopped. Treat this as a hang, not a pass. Find why it hangs (watch mode, an open handle, '
123
- 'a server that never closes, an infinite loop), then rerun it bounded, for example with --run or CI=1, a per-test '
124
- 'timeout, and `timeout 600` in front of the command.')
458
+ browsers = (' Its browser processes (' + ', '.join(record['browsers']) + ') were stopped with it.') if record.get('browsers') else ''
459
+ return ('The platform stopped a test, build or verification process that ' + record['reason'] + ': `' + record['command'][:200] + '`. '
460
+ 'Only that command and the processes it started were stopped.' + browsers + ' Treat this as a hang, not a pass. Find why it hangs (watch mode, an open handle, '
461
+ 'a browser or server that never closes, an infinite loop), then rerun it bounded, for example with --run or CI=1, a per-test '
462
+ 'timeout, browser.close() in a finally block, and `timeout 600` in front of the command.')
package/worker/runner.py CHANGED
@@ -479,6 +479,47 @@ def start_base(repo):
479
479
  except (OSError,subprocess.SubprocessError):pass
480
480
  return EMPTY_TREE
481
481
 
482
+ # Opus 5.5R: when this many steps in a row end in a failed command or tool, the next step runs on the backup model.
483
+ ESCALATE_AFTER=int(os.getenv('MINI_ESCALATE_AFTER','2') or 2)
484
+ STEP_FAILED=re.compile(r'returncode\W+[1-9]|Command failed with exit code: [1-9]|^(?:Error|error):|<system>ERROR|Tool execution failed',re.M)
485
+ def _content_text(content):
486
+ if isinstance(content,str):return content
487
+ if isinstance(content,list):return ' '.join(str(p.get('text') or '') for p in content if isinstance(p,dict))
488
+ return ''
489
+ EDIT_TOOL_NAMES={'edit_file','write_file','Edit','Write','MultiEdit','StrReplaceFile','WriteFile','apply_patch'}
490
+ SHELL_EDIT=re.compile(r"\bsed\s+-i|\bpatch\b|\bgit\s+apply\b|\btee\b|(?<![0-9&>])>>?\s*[\w./-]+\.\w+|open\([^)]*['\"][wa]")
491
+ def _is_edit(call):
492
+ fn=(call or {}).get('function') or {}
493
+ name=fn.get('name') or '';args=str(fn.get('arguments') or '')
494
+ if name in EDIT_TOOL_NAMES:return True
495
+ if name=='workspace_tool' and re.search(r'"name"\s*:\s*"(edit_file|write_file)"',args):return True
496
+ return bool(SHELL_EDIT.search(args.replace('2>&1','').replace('>/dev/null','').replace('> /dev/null','')))
497
+ def first_edit_index(messages):
498
+ for i,m in enumerate(messages or []):
499
+ if not isinstance(m,dict) or m.get('role')!='assistant':continue
500
+ calls=m.get('tool_calls') or []
501
+ if any(_is_edit(c) for c in calls) or (not calls and SHELL_EDIT.search(_content_text(m.get('content')).replace('2>&1','').replace('/dev/null',''))):return i
502
+ return None
503
+ def failing_streak(messages,rx=STEP_FAILED):
504
+ """Steps in a row, newest first, whose results (tool or observation messages after an assistant turn) show a failure.
505
+ Only steps after the model's first file change count: reproducing a known failure before any fix is expected, not a failed attempt."""
506
+ at=first_edit_index(messages)
507
+ if at is None:return 0
508
+ messages=messages[at:]
509
+ streak=0;group=[]
510
+ for m in reversed(messages or []):
511
+ role=m.get('role') if isinstance(m,dict) else None
512
+ if role in ('tool','user'):group.append(_content_text(m.get('content'))[-6000:]);continue
513
+ if role=='assistant':
514
+ if group and any(rx.search(t) for t in group):streak+=1;group=[];continue
515
+ break
516
+ if role=='system':continue
517
+ break
518
+ return streak
519
+
520
+ class StepsFailing(Exception):
521
+ """Opus 5.5R: Pixis failed ESCALATE_AFTER steps in a row, so this step goes to the backup."""
522
+
482
523
  class RunClosed(RuntimeError):
483
524
  """The server closed this run (409 run_finished); str() is the plain message, without "HTTP Error 409"."""
484
525
  RUN_CLOSED_MESSAGE='This run was already closed on the server, so the agent stopped. Press Resume to carry on in a fresh run; your files are kept.'
@@ -705,7 +746,34 @@ class Model:
705
746
  last=next((m for m in reversed(clean) if m.get('role') in ('user','tool')),None)
706
747
  text=str((last or {}).get('content') or '')[-3000:]
707
748
  return bool(self.FAILURE_SIGNS.search(text))
749
+ FALLBACK_ERRORS=(urllib.error.URLError,TimeoutError,ConnectionError,json.JSONDecodeError,KeyError,IndexError)
708
750
  def request(self,clean,max_tokens=16384,retried=False,allow_tools=True,recovered=False):
751
+ """Opus 5.5R: a Pixis call that still fails after its own retries is sent once to the backup model
752
+ (config.fallback, key FALLBACK_MODEL_API_KEY), billed like any Pixis call. Other models call _request_once only."""
753
+ fb=self.config.get('fallback')
754
+ if not isinstance(fb,dict) or not fb.get('endpoint') or not fb.get('model') or self.config.get('_key_env') or not os.environ.get('FALLBACK_MODEL_API_KEY'):
755
+ return self._request_once(clean,max_tokens,retried,allow_tools,recovered)
756
+ try:
757
+ if allow_tools and failing_streak(clean)>=ESCALATE_AFTER:raise StepsFailing()
758
+ return self._request_once(clean,max_tokens,retried,allow_tools,recovered)
759
+ except RunClosed:raise
760
+ except (StepsFailing,*self.FALLBACK_ERRORS) as exc:
761
+ try:cap=max(0,int(fb.get('maxCalls',40)))
762
+ except (TypeError,ValueError):cap=40
763
+ used=getattr(self,'fallback_calls',0)
764
+ if (ROOT/'cancel').exists() or used>=cap:raise
765
+ self.fallback_calls=used+1
766
+ keys=('provider','model','endpoint','_key_env','_effective_reasoning_effort')
767
+ saved={k:self.config[k] for k in keys if k in self.config}
768
+ self.config.update(provider='openai-compatible',model=str(fb['model']),endpoint=str(fb['endpoint']).rstrip('/'),_key_env='FALLBACK_MODEL_API_KEY')
769
+ reason='steps_failing' if isinstance(exc,StepsFailing) else 'HTTP '+str(exc.code) if isinstance(exc,urllib.error.HTTPError) else type(exc).__name__
770
+ label=str(fb.get('label') or fb['model'])
771
+ event('model_fallback',content=('Pixis could not get the last steps working, so '+label+' took this step.') if reason=='steps_failing' else 'Pixis did not answer this step, so it ran on '+label+'.',model=str(fb['model']),reason=reason,calls=self.fallback_calls)
772
+ try:return self._request_once(clean,max_tokens,retried,allow_tools,recovered)
773
+ finally:
774
+ for k in keys:self.config.pop(k,None)
775
+ self.config.update(saved)
776
+ def _request_once(self,clean,max_tokens=16384,retried=False,allow_tools=True,recovered=False):
709
777
  if self.config.get('focused_tools') and not self.config.get('subagent') and allow_tools:
710
778
  clean=[*clean,{'role':'system','content':'Available tool references (permissions unchanged):\n'+tool_reference(self.config)}]
711
779
  payload = {'model': self.config['model'], 'messages': clean, 'max_tokens': max_tokens}
@@ -1793,7 +1861,7 @@ class Environment(LocalEnvironment):
1793
1861
  why=step_intent('cmd',None,command)
1794
1862
  event('command', command=command,**({'why':why} if why else {}))
1795
1863
  # Never pass model credentials into generated shell commands.
1796
- env = {k:v for k,v in os.environ.items() if k not in ('MODEL_API_KEY','SIDE_MODEL_API_KEY','VERCEL_OIDC_TOKEN','EXA_API_KEY','FIRECRAWL_API_KEY','OPENROUTER_API_KEY','PIXABAY_API_KEY','BRANDFETCH_API_KEY','MINI_IMAGE_URL','MINI_IMAGE_TOKEN','MINI_RESEARCH_TOKEN')}
1864
+ env = {k:v for k,v in os.environ.items() if k not in ('MODEL_API_KEY','SIDE_MODEL_API_KEY','FALLBACK_MODEL_API_KEY','VERCEL_OIDC_TOKEN','EXA_API_KEY','FIRECRAWL_API_KEY','OPENROUTER_API_KEY','PIXABAY_API_KEY','BRANDFETCH_API_KEY','MINI_IMAGE_URL','MINI_IMAGE_TOKEN','MINI_RESEARCH_TOKEN')}
1797
1865
  env.update(VAULT)
1798
1866
  # The preview iframe reaches dev servers through the sandbox's public *.vercel.run host; Vite rejects unknown hosts otherwise.
1799
1867
  env.setdefault('__VITE_ADDITIONAL_SERVER_ALLOWED_HOSTS','.vercel.run')
@@ -2051,6 +2119,9 @@ def main():
2051
2119
  try:
2052
2120
  if mlai_hooks.active():system+='\n'+mlai_hooks.guidance()
2053
2121
  if (config.get('laptop') or os.getenv('MINI_LAPTOP')=='1') and not config.get('subagent'):system+='\n'+mlai_hooks.LAPTOP_RULES # BUG-0523/0529/0531
2122
+ if not config.get('subagent'):
2123
+ from conversation import terminal_deck_rule
2124
+ if terminal_deck_rule(config.get('task','')):system+='\n'+terminal_deck_rule(config.get('task','')) # BUG-0691
2054
2125
  try:
2055
2126
  import mcp_local
2056
2127
  mcp_note=mcp_local.guidance(config,readonly=config.get('mode')=='plan' or conversation_readonly(config))