mlai-code 0.3.8 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +1 -1
- package/src/home.mjs +1 -1
- package/src/render.mjs +1 -0
- package/src/tui/app.mjs +1 -0
- package/src/tui/transcript.mjs +13 -0
- package/worker/browser_tool.py +42 -4
- package/worker/design_director.py +2 -1
- package/worker/kimi_bridge.py +15 -7
- package/worker/kimi_gates.py +13 -1
- package/worker/kimi_hook.py +3 -1
- package/worker/mlai_mcp.py +16 -1
- package/worker/preview.py +352 -29
- package/worker/proc_watch.py +388 -50
- package/worker/verification.py +69 -3
package/worker/proc_watch.py
CHANGED
|
@@ -1,22 +1,56 @@
|
|
|
1
|
-
"""Supervision for test and
|
|
1
|
+
"""Supervision for test, build and verification processes the agent starts (PROC-001, CLI r7).
|
|
2
2
|
|
|
3
|
-
A test runner that spins at full CPU with no output for minutes burns time and tokens until the run limit
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
told what was stopped, why, and how to rerun it in a bounded way.
|
|
3
|
+
A test runner that spins at full CPU with no output for minutes burns time and tokens until the run limit, and a
|
|
4
|
+
verification script whose headless browser never closes (`node e2e.js` with puppeteer driving Edge) keeps the command
|
|
5
|
+
running forever. This watchdog looks only at descendants of the engine process (the processes this run owns) and stops:
|
|
7
6
|
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
- test/build runner trees (vitest, jest, pytest, ...) and the agent's own verification scripts (node/python/deno/bun
|
|
8
|
+
running a file such as e2e.js, .e2e-tmp/e2e.mjs, scripts/check.mjs, smoke_test.py) when they have been silent for
|
|
9
|
+
IDLE seconds or alive for WALL seconds;
|
|
10
|
+
- any foreground command (a Bash call without run_in_background) still running after COMMAND_WALL seconds.
|
|
11
|
+
|
|
12
|
+
The whole tree goes, including a browser the script left behind (msedge/chrome children, also ones whose parent already
|
|
13
|
+
exited), so the pipes close and the engine's command returns. The agent is told what was stopped, why, and how to
|
|
14
|
+
rerun it in a bounded way.
|
|
15
|
+
|
|
16
|
+
Platforms:
|
|
17
|
+
- Linux: /proc. Output progress is the bytes the tree has written (/proc/<pid>/io wchar), so pipes and files count.
|
|
18
|
+
- Windows: ctypes only (Toolhelp snapshot for parents, GetProcessTimes, GetProcessIoCounters for written bytes,
|
|
19
|
+
NtQueryInformationProcess for command lines, TerminateProcess to stop). No wmic or PowerShell: they take seconds.
|
|
20
|
+
- macOS: one `ps -axo pid=,ppid=,etime=,time=,command=` per tick. There are no I/O counters, so a tree is silent when
|
|
21
|
+
its CPU time stops moving; the wall limits apply as everywhere.
|
|
10
22
|
"""
|
|
11
|
-
import os, re, signal, time
|
|
23
|
+
import os, re, signal, subprocess, sys, threading, time
|
|
12
24
|
from pathlib import Path
|
|
13
25
|
|
|
14
26
|
IDLE = int(os.getenv('MINI_CHECK_IDLE_SECONDS', '300')) # no new output for five minutes
|
|
15
27
|
WALL = int(os.getenv('MINI_CHECK_WALL_SECONDS', '1200')) # twenty minutes for any single test or build command
|
|
28
|
+
COMMAND_WALL = int(os.getenv('MINI_COMMAND_WALL_SECONDS', str(WALL))) # any foreground command
|
|
16
29
|
RUNNER = re.compile(r'(^|/|\s)(vitest|jest|playwright|pytest|mocha|ava|tap|cypress|karma|tsc|eslint|next build|vite build|webpack|cargo test|go test|node --test|python3? -m (pytest|unittest))(\s|$)', re.I)
|
|
30
|
+
# The agent's own verification scripts: an interpreter as the program, then a script whose file or folder name says
|
|
31
|
+
# what it is (e2e.js, .e2e-tmp/e2e.mjs, scripts/check.mjs, tests/smoke.py, verify-ui.ts, puppeteer-run.cjs).
|
|
32
|
+
SCRIPT = re.compile(r'''^\s*(?:"[^"]*[/\\]|'[^']*[/\\]|[^\s"']*[/\\])?(node|nodejs|python[\d.]*|py|deno|bun|tsx|ts-node)(?:\.exe)?["']?((?:\s+(?:-[-\w=.:]*|run|test))*)\s+["']?([^\s"']+)''', re.I)
|
|
33
|
+
SCRIPT_NAME = re.compile(r'(^|[/\\._-])(e2e|tests?|checks?|verify|verification|smoke|harness|puppeteer|playwright)(?=$|[^a-z])', re.I)
|
|
34
|
+
INLINE = re.compile(r'(^|\s)(-e|-c|-p|--eval|--print)(\s|$)')
|
|
35
|
+
BROWSER = re.compile(r'(^|[/\\\s"])(msedge|chrome|chromium|chromium-browser|google-chrome|headless_shell|chrome-headless-shell|firefox|Microsoft Edge|Google Chrome)(\.exe)?(["\s]|$)', re.I)
|
|
36
|
+
SHELL = re.compile(r'^\s*(?:"[^"]*[/\\]|[^\s"]*[/\\])?(bash|sh|zsh|dash|busybox|cmd|powershell|pwsh)(\.exe)?"?(\s|$)', re.I)
|
|
37
|
+
HERE = str(Path(__file__).resolve().parent)
|
|
17
38
|
HZ = os.sysconf('SC_CLK_TCK') if hasattr(os, 'sysconf') else 100
|
|
18
39
|
PROC = Path('/proc')
|
|
19
40
|
|
|
41
|
+
# Foreground Bash calls the engine is running: tool_call_id -> (command, started). Filled from the bridge's stream.
|
|
42
|
+
FOREGROUND, _FG_LOCK = {}, threading.Lock()
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def foreground(call_id, command, now=None):
|
|
46
|
+
with _FG_LOCK: FOREGROUND[str(call_id)] = (str(command or ''), now or time.time())
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def done(call_id):
|
|
50
|
+
with _FG_LOCK: FOREGROUND.pop(str(call_id), None)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# ---------- Linux (/proc) ----------
|
|
20
54
|
|
|
21
55
|
def _read(pid, name):
|
|
22
56
|
try: return (PROC / str(pid) / name).read_bytes()
|
|
@@ -41,21 +75,6 @@ def uptime():
|
|
|
41
75
|
except (OSError, ValueError, IndexError): return 0.0
|
|
42
76
|
|
|
43
77
|
|
|
44
|
-
def descendants(root):
|
|
45
|
-
"""pid -> (ppid, cpu, start) for every live descendant of root."""
|
|
46
|
-
table = {}
|
|
47
|
-
for entry in PROC.iterdir() if PROC.is_dir() else []:
|
|
48
|
-
if entry.name.isdigit():
|
|
49
|
-
st = stat(int(entry.name))
|
|
50
|
-
if st: table[int(entry.name)] = st
|
|
51
|
-
kids, out, todo = {}, {}, [root]
|
|
52
|
-
for pid, st in table.items(): kids.setdefault(st[0], []).append(pid)
|
|
53
|
-
while todo:
|
|
54
|
-
for child in kids.get(todo.pop(), []):
|
|
55
|
-
if child not in out: out[child] = table[child]; todo.append(child)
|
|
56
|
-
return out
|
|
57
|
-
|
|
58
|
-
|
|
59
78
|
def written(pid):
|
|
60
79
|
"""Bytes this process has written through write calls (pipes, files, sockets). Silent loops do not move it."""
|
|
61
80
|
for line in _read(pid, 'io').decode('ascii', 'replace').splitlines():
|
|
@@ -65,60 +84,379 @@ def written(pid):
|
|
|
65
84
|
return 0
|
|
66
85
|
|
|
67
86
|
|
|
87
|
+
class Linux:
|
|
88
|
+
name = 'linux'
|
|
89
|
+
has_io = True
|
|
90
|
+
|
|
91
|
+
def table(self):
|
|
92
|
+
"""pid -> (ppid, cpu, start_epoch) for every process."""
|
|
93
|
+
boot, out = time.time() - uptime(), {}
|
|
94
|
+
for entry in PROC.iterdir() if PROC.is_dir() else []:
|
|
95
|
+
if entry.name.isdigit():
|
|
96
|
+
st = stat(int(entry.name))
|
|
97
|
+
if st: out[int(entry.name)] = (st[0], st[1], boot + st[2])
|
|
98
|
+
return out
|
|
99
|
+
|
|
100
|
+
def command(self, pid): return cmdline(pid)
|
|
101
|
+
def written(self, pid): return written(pid)
|
|
102
|
+
|
|
103
|
+
def kill(self, pids):
|
|
104
|
+
for sig in (signal.SIGTERM, signal.SIGKILL):
|
|
105
|
+
for p in pids:
|
|
106
|
+
try: os.kill(p, sig)
|
|
107
|
+
except (ProcessLookupError, PermissionError, OSError): pass
|
|
108
|
+
if sig == signal.SIGTERM and pids: time.sleep(2)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
# ---------- macOS (ps) ----------
|
|
112
|
+
|
|
113
|
+
def duration(text):
|
|
114
|
+
"""ps etime/time ([[dd-]hh:]mm:ss[.cc]) in seconds."""
|
|
115
|
+
text = text.strip()
|
|
116
|
+
days = 0
|
|
117
|
+
if '-' in text:
|
|
118
|
+
d, text = text.split('-', 1); days = int(d)
|
|
119
|
+
parts = [float(p) for p in text.split(':')]
|
|
120
|
+
secs = 0.0
|
|
121
|
+
for p in parts: secs = secs * 60 + p
|
|
122
|
+
return days * 86400 + secs
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def parse_ps(output, now=None):
|
|
126
|
+
"""`ps -axo pid=,ppid=,etime=,time=,command=` -> {pid: (ppid, cpu, start_epoch)}, {pid: command}."""
|
|
127
|
+
now = now or time.time()
|
|
128
|
+
table, commands = {}, {}
|
|
129
|
+
for line in output.splitlines():
|
|
130
|
+
bits = line.split(None, 4)
|
|
131
|
+
if len(bits) < 4: continue
|
|
132
|
+
try: pid, ppid, age, cpu = int(bits[0]), int(bits[1]), duration(bits[2]), duration(bits[3])
|
|
133
|
+
except ValueError: continue
|
|
134
|
+
table[pid] = (ppid, cpu, now - age)
|
|
135
|
+
commands[pid] = bits[4].strip() if len(bits) > 4 else ''
|
|
136
|
+
return table, commands
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class Mac:
|
|
140
|
+
name = 'darwin'
|
|
141
|
+
has_io = False
|
|
142
|
+
|
|
143
|
+
def __init__(self): self.commands = {}
|
|
144
|
+
|
|
145
|
+
def table(self):
|
|
146
|
+
try: out = subprocess.run(['ps', '-axo', 'pid=,ppid=,etime=,time=,command='], capture_output=True, text=True, timeout=10).stdout
|
|
147
|
+
except (OSError, subprocess.SubprocessError): return {}
|
|
148
|
+
table, self.commands = parse_ps(out)
|
|
149
|
+
return table
|
|
150
|
+
|
|
151
|
+
def command(self, pid): return self.commands.get(pid, '')
|
|
152
|
+
def written(self, pid): return None
|
|
153
|
+
kill = Linux.kill
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
# ---------- Windows (ctypes) ----------
|
|
157
|
+
|
|
158
|
+
EPOCH_AS_FILETIME = 116444736000000000
|
|
159
|
+
PROCESS_TERMINATE, PROCESS_VM_READ, PROCESS_QUERY_LIMITED_INFORMATION = 0x0001, 0x0010, 0x1000
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
class Windows:
|
|
163
|
+
name = 'win32'
|
|
164
|
+
has_io = True
|
|
165
|
+
|
|
166
|
+
def __init__(self, api=None):
|
|
167
|
+
self.api = api or WinApi()
|
|
168
|
+
self.exe = {}
|
|
169
|
+
|
|
170
|
+
def table(self):
|
|
171
|
+
"""Parents come from one Toolhelp snapshot; times only for processes that are asked about (see times)."""
|
|
172
|
+
snap = self.api.snapshot() # [(pid, ppid, exe)]
|
|
173
|
+
self.exe = {pid: exe for pid, _, exe in snap}
|
|
174
|
+
self._times = {}
|
|
175
|
+
return LazyTable({pid: ppid for pid, ppid, _ in snap}, self.times)
|
|
176
|
+
|
|
177
|
+
def times(self, pid):
|
|
178
|
+
if pid not in self._times: self._times[pid] = self.api.times(pid) # (cpu, start_epoch) or None
|
|
179
|
+
return self._times[pid]
|
|
180
|
+
|
|
181
|
+
def command(self, pid): return self.api.command_line(pid) or self.exe.get(pid, '')
|
|
182
|
+
def written(self, pid): return self.api.written(pid) or 0
|
|
183
|
+
|
|
184
|
+
def kill(self, pids):
|
|
185
|
+
for p in pids: self.api.terminate(p)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
class LazyTable(dict):
|
|
189
|
+
"""pid -> (ppid, cpu, start) whose times are read on first access. Windows reuses PIDs, so a child must have
|
|
190
|
+
started after the parent it names; iterating yields every pid in the snapshot."""
|
|
191
|
+
|
|
192
|
+
def __init__(self, parents, times):
|
|
193
|
+
super().__init__(); self.parents, self.times_of = parents, times
|
|
194
|
+
|
|
195
|
+
def __iter__(self): return iter(self.parents)
|
|
196
|
+
def __len__(self): return len(self.parents)
|
|
197
|
+
def __contains__(self, pid): return pid in self.parents and self.times_of(pid) is not None
|
|
198
|
+
def keys(self): return list(self.parents)
|
|
199
|
+
def items(self): return [(p, self[p]) for p in list(self.parents) if p in self]
|
|
200
|
+
|
|
201
|
+
def __getitem__(self, pid):
|
|
202
|
+
t = self.times_of(pid) if pid in self.parents else None
|
|
203
|
+
if t is None: raise KeyError(pid)
|
|
204
|
+
return (self.parents[pid], t[0], t[1])
|
|
205
|
+
|
|
206
|
+
def get(self, pid, default=None):
|
|
207
|
+
try: return self[pid]
|
|
208
|
+
except KeyError: return default
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
class WinApi:
|
|
212
|
+
"""The few kernel32/ntdll calls the watchdog needs."""
|
|
213
|
+
|
|
214
|
+
def __init__(self):
|
|
215
|
+
import ctypes
|
|
216
|
+
from ctypes import wintypes
|
|
217
|
+
self.c, self.w = ctypes, wintypes
|
|
218
|
+
self.k = ctypes.WinDLL('kernel32', use_last_error=True)
|
|
219
|
+
self.k.OpenProcess.restype = wintypes.HANDLE
|
|
220
|
+
self.k.CreateToolhelp32Snapshot.restype = wintypes.HANDLE
|
|
221
|
+
try: self.nt = ctypes.WinDLL('ntdll')
|
|
222
|
+
except OSError: self.nt = None
|
|
223
|
+
|
|
224
|
+
class PROCESSENTRY32W(ctypes.Structure):
|
|
225
|
+
_fields_ = [('dwSize', wintypes.DWORD), ('cntUsage', wintypes.DWORD), ('th32ProcessID', wintypes.DWORD),
|
|
226
|
+
('th32DefaultHeapID', ctypes.c_size_t), ('th32ModuleID', wintypes.DWORD), ('cntThreads', wintypes.DWORD),
|
|
227
|
+
('th32ParentProcessID', wintypes.DWORD), ('pcPriClassBase', wintypes.LONG), ('dwFlags', wintypes.DWORD),
|
|
228
|
+
('szExeFile', wintypes.WCHAR * 260)]
|
|
229
|
+
|
|
230
|
+
class IO_COUNTERS(ctypes.Structure):
|
|
231
|
+
_fields_ = [(n, ctypes.c_ulonglong) for n in ('ReadOperationCount', 'WriteOperationCount', 'OtherOperationCount',
|
|
232
|
+
'ReadTransferCount', 'WriteTransferCount', 'OtherTransferCount')]
|
|
233
|
+
self.Entry, self.IO = PROCESSENTRY32W, IO_COUNTERS
|
|
234
|
+
|
|
235
|
+
def snapshot(self):
|
|
236
|
+
c, k = self.c, self.k
|
|
237
|
+
h = k.CreateToolhelp32Snapshot(0x2, 0) # TH32CS_SNAPPROCESS
|
|
238
|
+
if not h or h == self.w.HANDLE(-1).value: return []
|
|
239
|
+
out = []
|
|
240
|
+
try:
|
|
241
|
+
e = self.Entry(); e.dwSize = c.sizeof(e)
|
|
242
|
+
ok = k.Process32FirstW(h, c.byref(e))
|
|
243
|
+
while ok:
|
|
244
|
+
out.append((int(e.th32ProcessID), int(e.th32ParentProcessID), e.szExeFile))
|
|
245
|
+
ok = k.Process32NextW(h, c.byref(e))
|
|
246
|
+
finally: k.CloseHandle(h)
|
|
247
|
+
return out
|
|
248
|
+
|
|
249
|
+
def _open(self, pid, access=PROCESS_QUERY_LIMITED_INFORMATION):
|
|
250
|
+
return self.k.OpenProcess(access, False, int(pid)) or None
|
|
251
|
+
|
|
252
|
+
def times(self, pid):
|
|
253
|
+
h = self._open(pid)
|
|
254
|
+
if not h: return None
|
|
255
|
+
try:
|
|
256
|
+
ft = [self.w.FILETIME() for _ in range(4)]
|
|
257
|
+
if not self.k.GetProcessTimes(h, *[self.c.byref(t) for t in ft]): return None
|
|
258
|
+
v = [(t.dwHighDateTime << 32) | t.dwLowDateTime for t in ft]
|
|
259
|
+
if v[1]: return None # has an exit time: gone, only a handle keeps it
|
|
260
|
+
return (v[2] + v[3]) / 1e7, (v[0] - EPOCH_AS_FILETIME) / 1e7
|
|
261
|
+
finally: self.k.CloseHandle(h)
|
|
262
|
+
|
|
263
|
+
def written(self, pid):
|
|
264
|
+
h = self._open(pid)
|
|
265
|
+
if not h: return 0
|
|
266
|
+
try:
|
|
267
|
+
io = self.IO()
|
|
268
|
+
return int(io.WriteTransferCount + io.OtherTransferCount) if self.k.GetProcessIoCounters(h, self.c.byref(io)) else 0
|
|
269
|
+
finally: self.k.CloseHandle(h)
|
|
270
|
+
|
|
271
|
+
def command_line(self, pid):
|
|
272
|
+
"""ProcessCommandLineInformation (60, Windows 8.1+): a UNICODE_STRING followed by its text."""
|
|
273
|
+
if not self.nt: return ''
|
|
274
|
+
c = self.c
|
|
275
|
+
h = self._open(pid)
|
|
276
|
+
if not h: return ''
|
|
277
|
+
try:
|
|
278
|
+
size = c.c_ulong(0)
|
|
279
|
+
self.nt.NtQueryInformationProcess(h, 60, None, 0, c.byref(size))
|
|
280
|
+
if not size.value or size.value > 1 << 20: return ''
|
|
281
|
+
buf = c.create_string_buffer(size.value)
|
|
282
|
+
if self.nt.NtQueryInformationProcess(h, 60, buf, size, c.byref(size)) != 0: return ''
|
|
283
|
+
|
|
284
|
+
class US(c.Structure):
|
|
285
|
+
_fields_ = [('Length', c.c_ushort), ('MaximumLength', c.c_ushort), ('Buffer', c.c_void_p)]
|
|
286
|
+
us = US.from_buffer(buf)
|
|
287
|
+
return c.wstring_at(us.Buffer, us.Length // 2) if us.Buffer and us.Length else ''
|
|
288
|
+
finally: self.k.CloseHandle(h)
|
|
289
|
+
|
|
290
|
+
def terminate(self, pid):
|
|
291
|
+
h = self._open(pid, PROCESS_TERMINATE | PROCESS_QUERY_LIMITED_INFORMATION)
|
|
292
|
+
if not h: return False
|
|
293
|
+
try: return bool(self.k.TerminateProcess(h, 1))
|
|
294
|
+
finally: self.k.CloseHandle(h)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def backend(platform=None):
|
|
298
|
+
platform = platform or sys.platform
|
|
299
|
+
if platform == 'win32':
|
|
300
|
+
try: return Windows()
|
|
301
|
+
except Exception: return None
|
|
302
|
+
if platform == 'darwin': return Mac()
|
|
303
|
+
return Linux() if PROC.is_dir() else None
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
# ---------- trees ----------
|
|
307
|
+
|
|
308
|
+
def descendants(root, table=None, family=None):
|
|
309
|
+
"""pid -> (ppid, cpu, start) for every live descendant of root. A child must not predate its parent (Windows reuses
|
|
310
|
+
PIDs). family {pid: start} adds processes whose parent was one of ours and already exited: on Windows they keep the
|
|
311
|
+
dead parent's PID (Edge left behind by a killed node), on Linux they were reparented and are found by their own PID."""
|
|
312
|
+
if table is None: table = Linux().table()
|
|
313
|
+
kids, out, todo = {}, {}, [root]
|
|
314
|
+
for pid in list(table): kids.setdefault(_ppid(table, pid), []).append(pid)
|
|
315
|
+
for pid, start in (family or {}).items():
|
|
316
|
+
if pid != root and pid not in todo: todo.append(pid)
|
|
317
|
+
while todo:
|
|
318
|
+
parent = todo.pop()
|
|
319
|
+
pst = table.get(parent)
|
|
320
|
+
if parent in (family or {}) and pst and abs(pst[2] - family[parent]) >= 2: continue # PID reused by another process
|
|
321
|
+
if parent != root and parent in (family or {}) and pst and abs(pst[2] - family[parent]) < 2 and parent not in out:
|
|
322
|
+
out[parent] = pst # a remembered member still alive
|
|
323
|
+
for child in kids.get(parent, []):
|
|
324
|
+
if child in out or child == root: continue
|
|
325
|
+
st = table.get(child)
|
|
326
|
+
if not st: continue
|
|
327
|
+
born = pst[2] if pst else (family or {}).get(parent)
|
|
328
|
+
if born is not None and st[2] + 1 < born: continue # PID reused: older than its "parent"
|
|
329
|
+
out[child] = st; todo.append(child)
|
|
330
|
+
return out
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _ppid(table, pid):
|
|
334
|
+
parents = getattr(table, 'parents', None)
|
|
335
|
+
if parents is not None: return parents.get(pid)
|
|
336
|
+
return table[pid][0]
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def script(command):
|
|
340
|
+
"""True for the agent's own verification script: `node e2e.js`, `python scripts/check.py`, `bun run smoke.ts`."""
|
|
341
|
+
m = SCRIPT.match(command or '')
|
|
342
|
+
if not m or INLINE.search(m.group(2) or ''): return False
|
|
343
|
+
if HERE in command: return False # the harness's own helpers
|
|
344
|
+
return bool(SCRIPT_NAME.search(m.group(3)))
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def supervised_kind(command):
|
|
348
|
+
if RUNNER.search(command or '') and HERE not in (command or ''): return 'runner'
|
|
349
|
+
if script(command): return 'script'
|
|
350
|
+
return None
|
|
351
|
+
|
|
352
|
+
|
|
68
353
|
def span(seconds):
|
|
69
354
|
seconds = int(seconds)
|
|
70
355
|
return '%d seconds' % seconds if seconds < 120 else '%d minutes' % (seconds // 60)
|
|
71
356
|
|
|
72
357
|
|
|
358
|
+
def _norm(s): return re.sub(r'\s+', ' ', s or '').strip()
|
|
359
|
+
|
|
360
|
+
|
|
73
361
|
class Watch:
|
|
74
|
-
def __init__(self, root_pid, notify, idle=IDLE, wall=WALL):
|
|
75
|
-
self.root, self.notify, self.idle, self.wall = root_pid, notify, idle, wall
|
|
76
|
-
self.
|
|
362
|
+
def __init__(self, root_pid, notify, idle=IDLE, wall=WALL, command_wall=COMMAND_WALL, os_=None):
|
|
363
|
+
self.root, self.notify, self.idle, self.wall, self.command_wall = root_pid, notify, idle, wall, command_wall
|
|
364
|
+
self.os = os_ if os_ is not None else backend()
|
|
365
|
+
self.seen = {} # pid -> {'out': size, 'cpu': cpu, 'progress': t, 'start': t, 'family': {pid: start}}
|
|
77
366
|
self.stopped = []
|
|
367
|
+
with _FG_LOCK: FOREGROUND.clear() # a new engine process: calls of an earlier one are gone with it
|
|
78
368
|
|
|
79
369
|
def runners(self, procs):
|
|
80
|
-
"""Topmost
|
|
81
|
-
found = {pid:
|
|
82
|
-
found = {pid: c for pid, c in found.items() if
|
|
370
|
+
"""Topmost supervised process of each tree: its parent is not itself supervised."""
|
|
371
|
+
found = {pid: self.os.command(pid) for pid in procs}
|
|
372
|
+
found = {pid: c for pid, c in found.items() if supervised_kind(c)}
|
|
83
373
|
return {pid: c for pid, c in found.items() if procs[pid][0] not in found}
|
|
84
374
|
|
|
85
375
|
def tick(self, now=None):
|
|
376
|
+
if self.os is None: return
|
|
86
377
|
now = now or time.time()
|
|
87
|
-
|
|
88
|
-
|
|
378
|
+
table = self.os.table()
|
|
379
|
+
procs = descendants(self.root, table)
|
|
89
380
|
tops = self.runners(procs)
|
|
90
381
|
for pid in list(self.seen):
|
|
91
382
|
if pid not in tops: self.seen.pop(pid)
|
|
92
383
|
for pid, command in tops.items():
|
|
93
384
|
ppid, cpu, start = procs[pid]
|
|
94
|
-
tree = [pid] + [p for p in descendants(pid)]
|
|
95
|
-
size = sum(written(p) for p in tree)
|
|
96
|
-
cpu = sum(procs.get(p, (0, 0, 0))[1] for p in tree)
|
|
97
385
|
rec = self.seen.get(pid)
|
|
386
|
+
family = dict(rec['family']) if rec else {}
|
|
387
|
+
sub = descendants(pid, table, family)
|
|
388
|
+
tree = [pid] + list(sub)
|
|
389
|
+
for p, st in sub.items(): family[p] = st[2]
|
|
390
|
+
size = sum(self.os.written(p) or 0 for p in tree) if self.os.has_io else None
|
|
391
|
+
cpu = sum(procs.get(p, sub.get(p, (0, 0, 0)))[1] for p in tree)
|
|
98
392
|
if rec is None:
|
|
99
|
-
self.seen[pid] = {'out': size, 'cpu': cpu, 'cpu0': cpu, 'progress': now, 'start':
|
|
100
|
-
|
|
393
|
+
self.seen[pid] = {'out': size, 'cpu': cpu, 'cpu0': cpu, 'progress': now, 'start': start, 'family': family}; continue
|
|
394
|
+
rec['family'] = family
|
|
395
|
+
moved = size != rec['out'] if size is not None else cpu - rec['cpu'] > 0.05
|
|
396
|
+
if moved: rec['progress'] = now; rec['cpu0'] = cpu
|
|
101
397
|
rec['out'], rec['cpu'] = size, cpu
|
|
102
398
|
age, silent = now - rec['start'], now - rec['progress']
|
|
103
399
|
reason = None
|
|
104
400
|
if age > self.wall: reason = 'ran for %s, past the %s limit for one command' % (span(age), span(self.wall))
|
|
105
|
-
elif silent > self.idle: reason = 'wrote no output for %s while using %s CPU' % (span(silent), 'high' if cpu - rec.get('cpu0', cpu) > silent * 0.5 else 'little')
|
|
106
|
-
|
|
401
|
+
elif silent > self.idle and size is not None: reason = 'wrote no output for %s while using %s CPU' % (span(silent), 'high' if cpu - rec.get('cpu0', cpu) > silent * 0.5 else 'little')
|
|
402
|
+
elif silent > self.idle: reason = 'made no progress for %s (no CPU use, no exit)' % span(silent)
|
|
403
|
+
if reason: self.stop(pid, tree, command, reason, table)
|
|
404
|
+
self.foreground(now, table, procs)
|
|
107
405
|
|
|
108
|
-
def
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
406
|
+
def foreground(self, now, table, procs):
|
|
407
|
+
"""Foreground Bash calls past the command wall: stop the shell running each one and everything under it."""
|
|
408
|
+
with _FG_LOCK: late = [(k, c, t) for k, (c, t) in FOREGROUND.items() if now - t > self.command_wall]
|
|
409
|
+
for call_id, command, started in late:
|
|
410
|
+
want = _norm(command)[:120]
|
|
411
|
+
hits = {p: st for p, st in procs.items() if want and want in _norm(self.os.command(p))}
|
|
412
|
+
if not hits: # the engine may wrap the command; take the shells it started for this call
|
|
413
|
+
hits = {p: st for p, st in procs.items() if started - 2 <= st[2] <= started + 15 and SHELL.match(self.os.command(p) or '')}
|
|
414
|
+
tops = [p for p, st in hits.items() if st[0] not in hits]
|
|
415
|
+
if not tops: continue
|
|
416
|
+
done(call_id)
|
|
417
|
+
for pid in tops:
|
|
418
|
+
tree = [pid] + list(descendants(pid, table, (self.seen.get(pid) or {}).get('family')))
|
|
419
|
+
self.stop(pid, tree, command, 'was still running after %s, past the %s limit for a foreground command' % (span(now - started), span(self.command_wall)), table)
|
|
420
|
+
|
|
421
|
+
def stop(self, pid, tree, command, reason, table=None):
|
|
422
|
+
rec = self.seen.pop(pid, None) or {}
|
|
423
|
+
# Remembered members (a browser whose parent exited) go too, when they are still the same process.
|
|
424
|
+
for p, start in (rec.get('family') or {}).items():
|
|
425
|
+
st = table.get(p) if table is not None else None
|
|
426
|
+
if p not in tree and st and abs(st[2] - start) < 2: tree.append(p)
|
|
427
|
+
browsers = sorted({_exe(self.os.command(p)) for p in tree if BROWSER.search(self.os.command(p) or '')} - {''})
|
|
428
|
+
self.os.kill(list(reversed(tree)))
|
|
429
|
+
record = {'pid': pid, 'command': command[:300], 'reason': reason, 'time': time.time(), 'processes': len(tree)}
|
|
430
|
+
if browsers: record['browsers'] = browsers
|
|
116
431
|
self.stopped.append(record)
|
|
117
432
|
self.notify(record)
|
|
118
433
|
|
|
434
|
+
def kill_all(self):
|
|
435
|
+
"""Stop/time limit on Windows: no process groups there, so end every process this run owns, including the
|
|
436
|
+
ones remembered from earlier ticks whose parent already exited (they would hold the engine's pipes open)."""
|
|
437
|
+
if self.os is None: return 0
|
|
438
|
+
table = self.os.table()
|
|
439
|
+
family = {}
|
|
440
|
+
for rec in self.seen.values(): family.update(rec.get('family') or {})
|
|
441
|
+
tree = list(descendants(self.root, table, family))
|
|
442
|
+
self.os.kill(list(reversed(tree)))
|
|
443
|
+
return len(tree)
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def kill_all(root_pid, watch=None):
|
|
447
|
+
"""Ends every descendant of root_pid (Windows stop path). Never raises."""
|
|
448
|
+
try: return (watch or Watch(root_pid, lambda r: None)).kill_all()
|
|
449
|
+
except Exception: return 0
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def _exe(command):
|
|
453
|
+
m = BROWSER.search(command or '')
|
|
454
|
+
return m.group(2).lower() if m else ''
|
|
455
|
+
|
|
119
456
|
|
|
120
457
|
def guidance(record):
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
'
|
|
124
|
-
'
|
|
458
|
+
browsers = (' Its browser processes (' + ', '.join(record['browsers']) + ') were stopped with it.') if record.get('browsers') else ''
|
|
459
|
+
return ('The platform stopped a test, build or verification process that ' + record['reason'] + ': `' + record['command'][:200] + '`. '
|
|
460
|
+
'Only that command and the processes it started were stopped.' + browsers + ' Treat this as a hang, not a pass. Find why it hangs (watch mode, an open handle, '
|
|
461
|
+
'a browser or server that never closes, an infinite loop), then rerun it bounded, for example with --run or CI=1, a per-test '
|
|
462
|
+
'timeout, browser.close() in a finally block, and `timeout 600` in front of the command.')
|
package/worker/verification.py
CHANGED
|
@@ -9,7 +9,7 @@ pre_existing(repo, base, command) -> True when the same command also fails on t
|
|
|
9
9
|
Findings are dicts {'kind', 'message', 'severity': 'block'|'note'}; the hook and bridge turn `block` findings into
|
|
10
10
|
repair rounds and every finding into a `verification_report` event for the activity feed.
|
|
11
11
|
"""
|
|
12
|
-
import os, re, subprocess, tempfile, shutil
|
|
12
|
+
import os, re, signal, subprocess, sys, tempfile, time, shutil
|
|
13
13
|
from pathlib import Path
|
|
14
14
|
from runtime_control import latest_checks, check_status, clip, EXTERNAL_SCOPES
|
|
15
15
|
|
|
@@ -255,8 +255,10 @@ def pre_existing(repo, base, command, timeout=300):
|
|
|
255
255
|
try: os.symlink(src, Path(tmp) / link)
|
|
256
256
|
except OSError: pass
|
|
257
257
|
env = {k: v for k, v in os.environ.items() if k not in ('MODEL_API_KEY', 'MINI_KIMI_PROXY_TOKEN', 'VERCEL_OIDC_TOKEN')}
|
|
258
|
-
|
|
259
|
-
|
|
258
|
+
code, _out, why = bounded_run( # bounded on Windows too: a leftover child holding the output pipe cannot block it
|
|
259
|
+
['bash', '-lc', command], cwd=tmp, env=env, timeout=timeout)
|
|
260
|
+
if why: return None
|
|
261
|
+
return code != 0
|
|
260
262
|
except (OSError, subprocess.SubprocessError): return None
|
|
261
263
|
finally:
|
|
262
264
|
subprocess.run(['git', 'worktree', 'remove', '--force', tmp], cwd=repo, capture_output=True, timeout=60)
|
|
@@ -979,3 +981,67 @@ def program_run_check(command, repo):
|
|
|
979
981
|
if target.is_file() and Path(repo).resolve() in target.parents: return m.group(1)
|
|
980
982
|
except OSError: pass
|
|
981
983
|
return ''
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
# Bounded shell commands for checks the harness replays on its own (CLI r7).
|
|
987
|
+
# Bounded shell commands for the harness (checks it replays on its own).
|
|
988
|
+
#
|
|
989
|
+
# subprocess.run(..., capture_output=True, timeout=N) is not bounded on Windows: after the timeout it kills only the shell
|
|
990
|
+
# and then reads the pipes with no timeout, so a child that is still alive (a headless browser left by an e2e script) keeps
|
|
991
|
+
# the pipe open and the harness waits forever. One tester's run sat at "Working" for 45 minutes after its final answer.
|
|
992
|
+
#
|
|
993
|
+
# run() sends output to a temporary file instead of a pipe, starts the command in its own process group, and stops the
|
|
994
|
+
# whole tree on timeout or when stop() returns True. It always returns within about the timeout.
|
|
995
|
+
WINDOWS = os.name == 'nt'
|
|
996
|
+
|
|
997
|
+
|
|
998
|
+
def _kill_tree(proc):
|
|
999
|
+
"""Stops a started command and everything it launched. Never raises."""
|
|
1000
|
+
try:
|
|
1001
|
+
if proc.poll() is not None and not WINDOWS:
|
|
1002
|
+
# The shell is gone; its children may still be in the group.
|
|
1003
|
+
os.killpg(proc.pid, signal.SIGKILL); return
|
|
1004
|
+
except (OSError, ProcessLookupError): return
|
|
1005
|
+
try:
|
|
1006
|
+
if WINDOWS:
|
|
1007
|
+
import winproc
|
|
1008
|
+
winproc.kill_tree(proc.pid)
|
|
1009
|
+
else:
|
|
1010
|
+
os.killpg(proc.pid, signal.SIGKILL)
|
|
1011
|
+
except (OSError, ProcessLookupError, subprocess.SubprocessError):
|
|
1012
|
+
try: proc.kill()
|
|
1013
|
+
except OSError: pass
|
|
1014
|
+
|
|
1015
|
+
|
|
1016
|
+
def bounded_run(argv, cwd=None, env=None, timeout=150, stop=None, tail=4000, poll=0.25):
|
|
1017
|
+
"""Returns (returncode, output, reason). returncode is None when the command was stopped; reason is '' when it
|
|
1018
|
+
finished, 'timeout' or 'stopped' otherwise. output is the last `tail` characters of stdout and stderr together."""
|
|
1019
|
+
out = tempfile.TemporaryFile()
|
|
1020
|
+
kwargs = {'cwd': cwd, 'env': env, 'stdin': subprocess.DEVNULL, 'stdout': out, 'stderr': subprocess.STDOUT}
|
|
1021
|
+
if WINDOWS: kwargs['creationflags'] = getattr(subprocess, 'CREATE_NEW_PROCESS_GROUP', 0) | getattr(subprocess, 'CREATE_NO_WINDOW', 0)
|
|
1022
|
+
else: kwargs['start_new_session'] = True
|
|
1023
|
+
try:
|
|
1024
|
+
try: proc = subprocess.Popen(argv, **kwargs)
|
|
1025
|
+
except OSError as exc: return None, str(exc)[-tail:], 'error'
|
|
1026
|
+
deadline = time.time() + max(1, timeout)
|
|
1027
|
+
reason = ''
|
|
1028
|
+
while proc.poll() is None:
|
|
1029
|
+
if stop and stop(): reason = 'stopped'; break
|
|
1030
|
+
if time.time() >= deadline: reason = 'timeout'; break
|
|
1031
|
+
time.sleep(poll)
|
|
1032
|
+
code = proc.returncode
|
|
1033
|
+
if reason:
|
|
1034
|
+
_kill_tree(proc)
|
|
1035
|
+
try: proc.wait(timeout=10)
|
|
1036
|
+
except subprocess.TimeoutExpired: pass
|
|
1037
|
+
code = None
|
|
1038
|
+
elif not WINDOWS:
|
|
1039
|
+
# A background child left by the command must not outlive the check.
|
|
1040
|
+
try: os.killpg(proc.pid, signal.SIGKILL)
|
|
1041
|
+
except (OSError, ProcessLookupError): pass
|
|
1042
|
+
out.seek(0)
|
|
1043
|
+
text = out.read()[-tail * 4:].decode('utf-8', 'replace')[-tail:]
|
|
1044
|
+
if reason == 'timeout': text = (text + '\n' if text else '') + 'Stopped after %ds (timeout).' % timeout
|
|
1045
|
+
return code, text, reason
|
|
1046
|
+
finally:
|
|
1047
|
+
out.close()
|