@giamme/forge 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/SKILL.md +19 -1
- package/VERSION +1 -1
- package/docs/artifacts.md +17 -2
- package/docs/commands.md +50 -0
- package/docs/dataset.md +9 -0
- package/docs/decomposed-runs.md +42 -0
- package/docs/forge-ui.md +42 -6
- package/docs/reviews.md +14 -0
- package/forge +15 -0
- package/package.json +1 -1
- package/references/dataset.md +18 -1
- package/references/decompose.md +19 -1
- package/references/ui.md +27 -5
- package/scripts/forge-config.py +347 -0
- package/scripts/forge-control.py +645 -0
- package/scripts/forge-issues.py +806 -0
- package/scripts/forge-lib-task.sh +234 -35
- package/scripts/forge-parallel-options.sh +3 -1
- package/scripts/forge-parallel.sh +3 -2
- package/scripts/forge-route.py +184 -27
- package/scripts/forge-runs-rm.py +840 -0
- package/scripts/forge-runs.py +19 -1
- package/scripts/forge-schedule.py +227 -11
- package/scripts/forge-ui.py +85 -19
- package/scripts/forge_dataset/__init__.py +77 -6
- package/scripts/forge_dataset/capture.py +5 -2
- package/scripts/forge_dataset/harvest.py +22 -0
- package/scripts/forge_dataset/importer.py +419 -17
- package/scripts/forge_registry.py +168 -16
- package/ui/README.md +37 -4
- package/ui/docs/api.md +184 -6
- package/ui/forge_ui/__main__.py +10 -2
- package/ui/forge_ui/control.py +575 -0
- package/ui/forge_ui/export.py +17 -3
- package/ui/forge_ui/reader.py +26 -1
- package/ui/forge_ui/server.py +125 -4
- package/ui/forge_ui/static/app.js +80 -8
- package/ui/forge_ui/static/controls.js +619 -0
- package/ui/forge_ui/static/findingsview.js +412 -0
- package/ui/forge_ui/static/index.html +7 -1
- package/ui/forge_ui/static/panel.js +24 -4
- package/ui/forge_ui/static/rundelete.js +388 -0
- package/ui/forge_ui/static/settings.js +399 -0
- package/ui/forge_ui/static/style.css +177 -2
- package/ui/forge_ui/static/uikit.js +161 -0
|
@@ -0,0 +1,645 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""forge control <command> <plan-dir> ...: show and change a plan's live controls.
|
|
3
|
+
|
|
4
|
+
show <plan> [--json] every control, per-task overrides, pins and the change log
|
|
5
|
+
set <plan> <key> <value> [--task ID] qa-threshold, timeout, escalate-after, verify-retries,
|
|
6
|
+
retry-budget, infra-retries
|
|
7
|
+
pin <plan> <task> <spec> the model the task's next FULL implementer retry uses
|
|
8
|
+
unpin <plan> <task>
|
|
9
|
+
|
|
10
|
+
`show` reports escalate-after and verify-retries at their defaults (3 and 1) when unset, and the two
|
|
11
|
+
budgets only while a scheduler is live.
|
|
12
|
+
|
|
13
|
+
Every change is validated, written atomically under the plan's control lock (`control.lock`),
|
|
14
|
+
appended to `control.log` and recorded as a dataset `control` event. This is the only writer of
|
|
15
|
+
human changes to a running plan; forge-ui calls it and never writes run files itself. Some
|
|
16
|
+
controls can only change while a scheduler is live (`scheduler.live` names a live pid):
|
|
17
|
+
retry-budget and infra-retries. A task-level change or a pin on a MERGED task, and a pin on a
|
|
18
|
+
Fractal plan, are refused.
|
|
19
|
+
|
|
20
|
+
A pin is checked with `forge-dispatch.sh doctor --spec <spec> --role dwarf`, which spends no
|
|
21
|
+
quota, before the lock is taken. FORGE_CONTROL_DOCTOR replaces `bash forge-dispatch.sh doctor`
|
|
22
|
+
with another command line, which then gets `--spec <spec> --role dwarf`; tests use it.
|
|
23
|
+
|
|
24
|
+
Exit codes: 0 ok, 2 usage or invalid input (including an unsafe path), 3 refused (not live, MERGED,
|
|
25
|
+
Fractal, a spec that fails doctor, the control lock busy for over 5 s).
|
|
26
|
+
"""
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import argparse
|
|
30
|
+
import contextlib
|
|
31
|
+
import errno
|
|
32
|
+
import fcntl
|
|
33
|
+
import json
|
|
34
|
+
import os
|
|
35
|
+
import re
|
|
36
|
+
import shlex
|
|
37
|
+
import stat
|
|
38
|
+
import subprocess
|
|
39
|
+
import sys
|
|
40
|
+
import time
|
|
41
|
+
|
|
42
|
+
SKILL = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
|
43
|
+
LOCK_WAIT = 5.0
|
|
44
|
+
LOG_SHOWN = 20
|
|
45
|
+
LOG_TAIL = 256 * 1024
|
|
46
|
+
READ_MAX = 8 * 1024 * 1024
|
|
47
|
+
DEFAULT_ESCALATE_AFTER = 3 # forge-route.py
|
|
48
|
+
DEFAULT_VERIFY_RETRIES = 1 # forge-parallel.sh
|
|
49
|
+
LEVELS = ('P0', 'P1', 'P2', 'P3')
|
|
50
|
+
STOP_MARKERS = ('escalation.stop', 'split.suggested', 'escalation.next')
|
|
51
|
+
DIR_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
|
|
52
|
+
|
|
53
|
+
NEXT_ATTEMPT = "at the task's next attempt"
|
|
54
|
+
NEXT_DECISION = 'at the next retry decision'
|
|
55
|
+
NEXT_RETRY = 'at the next full retry (not QA-only resumes)'
|
|
56
|
+
NEXT_VERIFY = 'at the next verification'
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class Invalid(Exception):
|
|
60
|
+
"""Usage or invalid input, an unsafe path included: exit 2."""
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class Refused(Exception):
|
|
64
|
+
"""A well-formed change the plan's state does not allow: exit 3."""
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
# ------------------------------------------------------------------------------------ path safety
|
|
68
|
+
# Every access walks down from the canonical plan dir one component at a time with O_NOFOLLOW, so
|
|
69
|
+
# the parent is a real directory inside the plan by construction, and the leaf is checked to be a
|
|
70
|
+
# regular file (or absent). Writes go to a temp file in the same dir and replace the target.
|
|
71
|
+
|
|
72
|
+
def valid_task_id(name):
|
|
73
|
+
"""The scheduler's rule (forge-schedule.py), applied whether or not the id is in tasks.tsv."""
|
|
74
|
+
return bool(name) and name not in ('.', '..') and '/' not in name and '\0' not in name
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _enter(fd, part, create, rel):
|
|
78
|
+
try:
|
|
79
|
+
return os.open(part, DIR_FLAGS, dir_fd=fd)
|
|
80
|
+
except FileNotFoundError:
|
|
81
|
+
if not create:
|
|
82
|
+
return None
|
|
83
|
+
except OSError as error:
|
|
84
|
+
if error.errno in (errno.ELOOP, errno.ENOTDIR):
|
|
85
|
+
raise Invalid(f'{rel}: {part} is not a real directory')
|
|
86
|
+
raise
|
|
87
|
+
try:
|
|
88
|
+
os.mkdir(part, dir_fd=fd)
|
|
89
|
+
except FileExistsError:
|
|
90
|
+
pass
|
|
91
|
+
try:
|
|
92
|
+
return os.open(part, DIR_FLAGS, dir_fd=fd)
|
|
93
|
+
except OSError as error:
|
|
94
|
+
if error.errno in (errno.ELOOP, errno.ENOTDIR):
|
|
95
|
+
raise Invalid(f'{rel}: {part} is not a real directory')
|
|
96
|
+
raise
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _open_parent(plan, rel, create=False):
|
|
100
|
+
"""(fd of rel's parent dir, leaf name); (None, leaf) when a parent does not exist and create is off."""
|
|
101
|
+
*dirs, leaf = rel.split('/')
|
|
102
|
+
fd = os.open(plan, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC)
|
|
103
|
+
try:
|
|
104
|
+
for part in dirs:
|
|
105
|
+
nxt = _enter(fd, part, create, rel)
|
|
106
|
+
os.close(fd)
|
|
107
|
+
fd = nxt
|
|
108
|
+
if fd is None:
|
|
109
|
+
return None, leaf
|
|
110
|
+
return fd, leaf
|
|
111
|
+
except BaseException:
|
|
112
|
+
if fd is not None:
|
|
113
|
+
os.close(fd)
|
|
114
|
+
raise
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _regular(fd, leaf, rel):
|
|
118
|
+
"""The lstat of an existing regular file, None when absent; anything else is unsafe."""
|
|
119
|
+
try:
|
|
120
|
+
st = os.stat(leaf, dir_fd=fd, follow_symlinks=False)
|
|
121
|
+
except FileNotFoundError:
|
|
122
|
+
return None
|
|
123
|
+
if not stat.S_ISREG(st.st_mode):
|
|
124
|
+
raise Invalid(f'{rel} is not a regular file')
|
|
125
|
+
return st
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def probe(plan, rel):
|
|
129
|
+
"""Whether plan/rel exists as a safe regular file; Invalid when its path or type is unsafe."""
|
|
130
|
+
fd, leaf = _open_parent(plan, rel)
|
|
131
|
+
if fd is None:
|
|
132
|
+
return False
|
|
133
|
+
try:
|
|
134
|
+
return _regular(fd, leaf, rel) is not None
|
|
135
|
+
finally:
|
|
136
|
+
os.close(fd)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _read(plan, rel, tail):
|
|
140
|
+
fd, leaf = _open_parent(plan, rel)
|
|
141
|
+
if fd is None:
|
|
142
|
+
return None
|
|
143
|
+
try:
|
|
144
|
+
try:
|
|
145
|
+
rfd = os.open(leaf, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC, dir_fd=fd)
|
|
146
|
+
except FileNotFoundError:
|
|
147
|
+
return None
|
|
148
|
+
except OSError as error:
|
|
149
|
+
if error.errno == errno.ELOOP:
|
|
150
|
+
raise Invalid(f'{rel} is not a regular file')
|
|
151
|
+
raise
|
|
152
|
+
try:
|
|
153
|
+
st = os.fstat(rfd)
|
|
154
|
+
if not stat.S_ISREG(st.st_mode):
|
|
155
|
+
raise Invalid(f'{rel} is not a regular file')
|
|
156
|
+
cut = bool(tail) and st.st_size > tail
|
|
157
|
+
if cut:
|
|
158
|
+
os.lseek(rfd, st.st_size - tail, os.SEEK_SET)
|
|
159
|
+
data = os.read(rfd, tail if cut else READ_MAX)
|
|
160
|
+
finally:
|
|
161
|
+
os.close(rfd)
|
|
162
|
+
finally:
|
|
163
|
+
os.close(fd)
|
|
164
|
+
if cut:
|
|
165
|
+
data = data.partition(b'\n')[2] # the first line is cut in half
|
|
166
|
+
return data.decode('utf-8', 'replace')
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def read_file(plan, rel, strict=False, tail=0):
|
|
170
|
+
"""The file's text, None when absent. An unsafe path or type is Invalid when strict, else None."""
|
|
171
|
+
try:
|
|
172
|
+
return _read(plan, rel, tail)
|
|
173
|
+
except Invalid:
|
|
174
|
+
if strict:
|
|
175
|
+
raise
|
|
176
|
+
return None
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def text_of(plan, rel, strict=False):
|
|
180
|
+
"""The file's stripped text; empty counts as unset."""
|
|
181
|
+
return (read_file(plan, rel, strict) or '').strip() or None
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def has(plan, rel):
|
|
185
|
+
try:
|
|
186
|
+
return probe(plan, rel)
|
|
187
|
+
except Invalid:
|
|
188
|
+
return False
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def write_atomic(plan, rel, text):
|
|
192
|
+
fd, leaf = _open_parent(plan, rel, create=True)
|
|
193
|
+
tmp = f'.{leaf}.{os.getpid()}.{time.time_ns()}.tmp'
|
|
194
|
+
try:
|
|
195
|
+
_regular(fd, leaf, rel)
|
|
196
|
+
tfd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o644, dir_fd=fd)
|
|
197
|
+
try:
|
|
198
|
+
os.write(tfd, text.encode('utf-8'))
|
|
199
|
+
finally:
|
|
200
|
+
os.close(tfd)
|
|
201
|
+
os.replace(tmp, leaf, src_dir_fd=fd, dst_dir_fd=fd)
|
|
202
|
+
except BaseException:
|
|
203
|
+
with contextlib.suppress(OSError):
|
|
204
|
+
os.unlink(tmp, dir_fd=fd)
|
|
205
|
+
raise
|
|
206
|
+
finally:
|
|
207
|
+
os.close(fd)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def remove_safe(plan, rel):
|
|
211
|
+
"""Delete plan/rel if it is a safe regular file; whether anything was deleted."""
|
|
212
|
+
fd, leaf = _open_parent(plan, rel)
|
|
213
|
+
if fd is None:
|
|
214
|
+
return False
|
|
215
|
+
try:
|
|
216
|
+
if _regular(fd, leaf, rel) is None:
|
|
217
|
+
return False
|
|
218
|
+
os.unlink(leaf, dir_fd=fd)
|
|
219
|
+
return True
|
|
220
|
+
except FileNotFoundError:
|
|
221
|
+
return False
|
|
222
|
+
finally:
|
|
223
|
+
os.close(fd)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def append_line(plan, rel, line):
|
|
227
|
+
fd, leaf = _open_parent(plan, rel)
|
|
228
|
+
try:
|
|
229
|
+
_regular(fd, leaf, rel)
|
|
230
|
+
afd = os.open(leaf, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o644, dir_fd=fd)
|
|
231
|
+
try:
|
|
232
|
+
if not stat.S_ISREG(os.fstat(afd).st_mode):
|
|
233
|
+
raise Invalid(f'{rel} is not a regular file')
|
|
234
|
+
os.write(afd, line.encode('utf-8'))
|
|
235
|
+
finally:
|
|
236
|
+
os.close(afd)
|
|
237
|
+
finally:
|
|
238
|
+
os.close(fd)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
@contextlib.contextmanager
|
|
242
|
+
def control_lock(plan):
|
|
243
|
+
"""flock on control.lock, retried for LOCK_WAIT seconds. Never touches schedule.lock."""
|
|
244
|
+
fd, leaf = _open_parent(plan, 'control.lock')
|
|
245
|
+
try:
|
|
246
|
+
_regular(fd, leaf, 'control.lock') # a FIFO would block the open
|
|
247
|
+
try:
|
|
248
|
+
lock = os.open(leaf, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o644, dir_fd=fd)
|
|
249
|
+
except OSError as error:
|
|
250
|
+
if error.errno == errno.ELOOP:
|
|
251
|
+
raise Invalid('control.lock is not a regular file')
|
|
252
|
+
raise
|
|
253
|
+
finally:
|
|
254
|
+
os.close(fd)
|
|
255
|
+
try:
|
|
256
|
+
deadline = time.monotonic() + LOCK_WAIT
|
|
257
|
+
while True:
|
|
258
|
+
try:
|
|
259
|
+
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
260
|
+
break
|
|
261
|
+
except OSError as error:
|
|
262
|
+
if error.errno not in (errno.EAGAIN, errno.EACCES, errno.EWOULDBLOCK):
|
|
263
|
+
raise
|
|
264
|
+
if time.monotonic() >= deadline:
|
|
265
|
+
raise Refused('another change holds the control lock (busy); try again')
|
|
266
|
+
time.sleep(0.05)
|
|
267
|
+
try:
|
|
268
|
+
yield
|
|
269
|
+
finally:
|
|
270
|
+
fcntl.flock(lock, fcntl.LOCK_UN)
|
|
271
|
+
finally:
|
|
272
|
+
os.close(lock)
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
# ------------------------------------------------------------------------------------ plan state
|
|
276
|
+
|
|
277
|
+
def load(plan_arg):
|
|
278
|
+
"""(canonical plan dir, tasks.tsv rows); Invalid when this is not a plan dir."""
|
|
279
|
+
plan = os.path.realpath(plan_arg)
|
|
280
|
+
text = read_file(plan, 'tasks.tsv', strict=True) if os.path.isdir(plan) else None
|
|
281
|
+
if text is None:
|
|
282
|
+
raise Invalid(f'{plan_arg}: not a forge plan (no tasks.tsv)')
|
|
283
|
+
rows = [line.split('\t') for line in text.splitlines() if line.strip() and not line.startswith('#')]
|
|
284
|
+
return plan, rows
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def require_task(rows, task):
|
|
288
|
+
if not valid_task_id(task):
|
|
289
|
+
raise Invalid(f'invalid task id: {task!r}')
|
|
290
|
+
if not any(row[0] == task for row in rows):
|
|
291
|
+
raise Invalid(f'task {task!r} is not in this plan')
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def is_live(plan):
|
|
295
|
+
"""scheduler.live names a live pid. Nothing probes schedule.lock: a probe could fail a starting run."""
|
|
296
|
+
try:
|
|
297
|
+
pid = json.loads(read_file(plan, 'scheduler.live') or '')['pid']
|
|
298
|
+
except (ValueError, KeyError, TypeError):
|
|
299
|
+
return False
|
|
300
|
+
if type(pid) is not int or pid <= 0:
|
|
301
|
+
return False
|
|
302
|
+
try:
|
|
303
|
+
os.kill(pid, 0)
|
|
304
|
+
except ProcessLookupError:
|
|
305
|
+
return False
|
|
306
|
+
except PermissionError:
|
|
307
|
+
return True
|
|
308
|
+
except (OSError, OverflowError):
|
|
309
|
+
return False
|
|
310
|
+
return True
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def is_fractal(plan):
|
|
314
|
+
"""fractal-selection.json present and not explicitly disabled (unreadable counts: fail closed)."""
|
|
315
|
+
raw = read_file(plan, 'fractal-selection.json')
|
|
316
|
+
if raw is None or not raw.strip():
|
|
317
|
+
return False
|
|
318
|
+
try:
|
|
319
|
+
return bool(json.loads(raw).get('enabled'))
|
|
320
|
+
except (ValueError, AttributeError):
|
|
321
|
+
return True
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def task_status(plan, task, strict=False):
|
|
325
|
+
try:
|
|
326
|
+
if probe(plan, f'tasks/{task}/merged'):
|
|
327
|
+
return 'MERGED'
|
|
328
|
+
words = (read_file(plan, f'tasks/{task}/status', strict) or '').split()
|
|
329
|
+
except Invalid:
|
|
330
|
+
if strict:
|
|
331
|
+
raise
|
|
332
|
+
return 'PENDING'
|
|
333
|
+
return words[0] if words else 'PENDING'
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def refuse_task(plan, task, pin=False):
|
|
337
|
+
if task_status(plan, task, strict=True) == 'MERGED':
|
|
338
|
+
raise Refused(f'task {task} is MERGED: nothing left to change')
|
|
339
|
+
if pin and is_fractal(plan):
|
|
340
|
+
raise Refused('this plan runs under Fractal, which never applies pins')
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def number(text):
|
|
344
|
+
return int(text) if text and re.fullmatch(r'[0-9]{1,9}', text) else None
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def level(text, task_level=False):
|
|
348
|
+
text = text.splitlines()[0].replace(' ', '') if text else ''
|
|
349
|
+
return text if text in LEVELS or (task_level and text == 'none') else None
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def nullable(value):
|
|
353
|
+
return None if value in (None, '', '-', 'UNASSIGNED') else value
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def show(plan, rows):
|
|
357
|
+
live = is_live(plan)
|
|
358
|
+
|
|
359
|
+
def get(rel):
|
|
360
|
+
return text_of(plan, rel)
|
|
361
|
+
|
|
362
|
+
def budget(rel):
|
|
363
|
+
return number(get(rel)) if live else None
|
|
364
|
+
|
|
365
|
+
run = dict(qa_threshold=level(get('qa_threshold')), timeout=number(get('timeout')),
|
|
366
|
+
escalate_after=number(get('escalate_after')), verify_retries=number(get('verify_retries')),
|
|
367
|
+
retry_budget=budget('retry_budget'), infra_budget=budget('infra_budget'))
|
|
368
|
+
if run['escalate_after'] is None:
|
|
369
|
+
run['escalate_after'] = DEFAULT_ESCALATE_AFTER
|
|
370
|
+
if run['verify_retries'] is None:
|
|
371
|
+
run['verify_retries'] = DEFAULT_VERIFY_RETRIES
|
|
372
|
+
tasks = {}
|
|
373
|
+
for row in rows:
|
|
374
|
+
task = row[0]
|
|
375
|
+
if not valid_task_id(task):
|
|
376
|
+
continue
|
|
377
|
+
base = f'tasks/{task}/'
|
|
378
|
+
tasks[task] = dict(status=task_status(plan, task), dwarf=nullable(row[4] if len(row) > 4 else None),
|
|
379
|
+
qa=nullable(row[5] if len(row) > 5 else None),
|
|
380
|
+
qa_threshold=level(get(base + 'qa_threshold'), task_level=True),
|
|
381
|
+
retry_budget=number(get(base + 'retry_budget')),
|
|
382
|
+
retries_used=number(get(base + 'retries.used')) or 0,
|
|
383
|
+
pin=get(base + 'escalation.pin'), escalation_stop=has(plan, base + 'escalation.stop'),
|
|
384
|
+
escalation_next=get(base + 'escalation.next'))
|
|
385
|
+
log = []
|
|
386
|
+
for line in reversed((read_file(plan, 'control.log', tail=LOG_TAIL) or '').splitlines()):
|
|
387
|
+
try:
|
|
388
|
+
entry = json.loads(line)
|
|
389
|
+
except ValueError:
|
|
390
|
+
continue
|
|
391
|
+
if isinstance(entry, dict):
|
|
392
|
+
log.append(entry)
|
|
393
|
+
if len(log) == LOG_SHOWN:
|
|
394
|
+
break
|
|
395
|
+
return dict(plan=plan, run_id=get('run_id') or os.path.basename(plan).removeprefix('forge-'),
|
|
396
|
+
live=live, fractal=is_fractal(plan), run=run, tasks=tasks, log=log)
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
# ------------------------------------------------------------------------------------ changes
|
|
400
|
+
|
|
401
|
+
def _qa(value, task):
|
|
402
|
+
if value in LEVELS or value == 'none' or (task and value == 'inherit'):
|
|
403
|
+
return value
|
|
404
|
+
raise Invalid('qa-threshold needs P0, P1, P2, P3 or none' + (' (or inherit for a task)' if task else ''))
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _int(value, task):
|
|
408
|
+
if re.fullmatch(r'[0-9]{1,9}', value):
|
|
409
|
+
return str(int(value))
|
|
410
|
+
raise Invalid('needs a non-negative integer (up to 9 digits)')
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def _budget(value, task):
|
|
414
|
+
return value if task and value == 'inherit' else _int(value, task)
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
# key -> (plan only or both, validator, file, live only, when it applies)
|
|
418
|
+
KEYS = {
|
|
419
|
+
'qa-threshold': ('both', _qa, 'qa_threshold', False, NEXT_ATTEMPT),
|
|
420
|
+
'timeout': ('plan', _int, 'timeout', False, NEXT_ATTEMPT),
|
|
421
|
+
'escalate-after': ('plan', _int, 'escalate_after', False, NEXT_DECISION),
|
|
422
|
+
'verify-retries': ('plan', _int, 'verify_retries', False, NEXT_VERIFY),
|
|
423
|
+
'retry-budget': ('both', _budget, 'retry_budget', True, NEXT_DECISION),
|
|
424
|
+
'infra-retries': ('plan', _int, 'infra_budget', True, NEXT_DECISION),
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def emit(plan, entry):
|
|
429
|
+
"""The dataset `control` event: the log line's fields, the same `at`. Best effort."""
|
|
430
|
+
if os.environ.get('FORGE_DATASET', 'on').lower() in ('off', '0', 'no', 'false'):
|
|
431
|
+
return
|
|
432
|
+
cmd = [sys.executable, os.path.join(SKILL, 'scripts', 'forge-dataset.py'), 'event', 'control', '--dir', plan]
|
|
433
|
+
if entry['task']:
|
|
434
|
+
cmd += ['--task', entry['task']]
|
|
435
|
+
for field in ('at', 'actor', 'action', 'key', 'value', 'previous'):
|
|
436
|
+
value = entry[field]
|
|
437
|
+
cmd += ['--set', f"{field}={'' if value is None else value}"]
|
|
438
|
+
try:
|
|
439
|
+
subprocess.run(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=10)
|
|
440
|
+
except (OSError, subprocess.SubprocessError):
|
|
441
|
+
pass
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def audit(plan, actor, action, task, key, value, previous):
|
|
445
|
+
entry = dict(at=time.time(), actor=actor, action=action, task=task, key=key, value=value, previous=previous)
|
|
446
|
+
append_line(plan, 'control.log', json.dumps(entry) + '\n')
|
|
447
|
+
emit(plan, entry)
|
|
448
|
+
return entry
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def change(args, plan, rows, action, task, key, work, applies, summary, live_only=False, pin_guard=False):
|
|
452
|
+
"""Run work() -> (value, previous, changed) under the control lock, audit a change, report the result."""
|
|
453
|
+
with control_lock(plan):
|
|
454
|
+
probe(plan, 'control.log') # an unsafe log refuses the change instead of leaving it unaudited
|
|
455
|
+
if live_only and not is_live(plan):
|
|
456
|
+
raise Refused(f'{key} can only change while a run is live (no live scheduler for this plan)')
|
|
457
|
+
if task:
|
|
458
|
+
refuse_task(plan, task, pin_guard)
|
|
459
|
+
value, previous, changed = work()
|
|
460
|
+
if changed: # setting a value it already has, or unpinning nothing, is not a change
|
|
461
|
+
audit(plan, args.actor, action, task, key, value, previous)
|
|
462
|
+
controls = show(plan, rows)
|
|
463
|
+
if args.json:
|
|
464
|
+
record = dict(action=action, task=task, key=key, value=value, previous=previous)
|
|
465
|
+
json.dump(dict(ok=True, changed=record, applies=applies, controls=controls), sys.stdout, indent=2)
|
|
466
|
+
sys.stdout.write('\n')
|
|
467
|
+
else:
|
|
468
|
+
print(f"{summary(value)} (was {'unset' if previous is None else previous}); applies {applies}"
|
|
469
|
+
+ ('' if changed else ' (nothing changed, not logged)'))
|
|
470
|
+
return 0
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def cmd_set(args):
|
|
474
|
+
plan, rows = load(args.plan)
|
|
475
|
+
scope, check, name, live_only, applies = KEYS[args.key]
|
|
476
|
+
task = args.task
|
|
477
|
+
if task is not None:
|
|
478
|
+
require_task(rows, task)
|
|
479
|
+
if scope != 'both':
|
|
480
|
+
raise Invalid(f'{args.key} applies to the whole plan, not to a task')
|
|
481
|
+
try:
|
|
482
|
+
value = check(args.value, task)
|
|
483
|
+
except Invalid as error:
|
|
484
|
+
raise Invalid(f'{args.key}: {error}')
|
|
485
|
+
rel = f'tasks/{task}/{name}' if task else name
|
|
486
|
+
drop = value == 'inherit' or (value == 'none' and not task)
|
|
487
|
+
wide = args.key == 'escalate-after' and value == '0'
|
|
488
|
+
markers = [f'tasks/{row[0]}/{m}' for row in rows if valid_task_id(row[0]) for m in STOP_MARKERS] if wide else []
|
|
489
|
+
|
|
490
|
+
def work():
|
|
491
|
+
previous = text_of(plan, rel, strict=True)
|
|
492
|
+
for target in markers: # all checked before the first change: nothing is half done
|
|
493
|
+
probe(plan, target)
|
|
494
|
+
if drop:
|
|
495
|
+
changed = remove_safe(plan, rel)
|
|
496
|
+
else:
|
|
497
|
+
write_atomic(plan, rel, value + '\n')
|
|
498
|
+
changed = previous != value
|
|
499
|
+
swept = [remove_safe(plan, target) for target in markers] # as `run --escalate-after 0`: off means off
|
|
500
|
+
return value, previous, changed or any(swept)
|
|
501
|
+
|
|
502
|
+
where = f'task {task}' if task else 'the plan'
|
|
503
|
+
return change(args, plan, rows, 'set', task, args.key, work, applies,
|
|
504
|
+
lambda v: f'set {args.key} {v} on {where}', live_only=live_only)
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def doctor(spec):
|
|
508
|
+
"""Offline capability check of a dwarf spec: Refused when it fails, no quota spent."""
|
|
509
|
+
override = os.environ.get('FORGE_CONTROL_DOCTOR')
|
|
510
|
+
base = shlex.split(override) if override else ['bash', os.path.join(SKILL, 'scripts', 'forge-dispatch.sh'), 'doctor']
|
|
511
|
+
try:
|
|
512
|
+
done = subprocess.run(base + ['--spec', spec, '--role', 'dwarf'], capture_output=True, text=True,
|
|
513
|
+
stdin=subprocess.DEVNULL, timeout=60)
|
|
514
|
+
except (OSError, subprocess.SubprocessError, ValueError) as error:
|
|
515
|
+
raise Refused(f'could not check the spec: {error}')
|
|
516
|
+
if done.returncode != 0:
|
|
517
|
+
lines = [x.strip() for x in (done.stderr + done.stdout).splitlines() if x.strip()]
|
|
518
|
+
raise Refused(f'spec {spec!r} does not pass the doctor check' + (f': {lines[0]}' if lines else ''))
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def cmd_pin(args):
|
|
522
|
+
plan, rows = load(args.plan)
|
|
523
|
+
require_task(rows, args.task)
|
|
524
|
+
spec = args.spec
|
|
525
|
+
if not spec or re.search(r'[\s\x00-\x1f\x7f]', spec):
|
|
526
|
+
raise Invalid('a pin is one dwarf spec such as sol:high:codex, without whitespace')
|
|
527
|
+
refuse_task(plan, args.task, pin=True)
|
|
528
|
+
doctor(spec) # before the lock: it takes a moment, and the lock is for check-and-write only
|
|
529
|
+
rel = f'tasks/{args.task}/escalation.pin'
|
|
530
|
+
|
|
531
|
+
def work():
|
|
532
|
+
previous = text_of(plan, rel, strict=True)
|
|
533
|
+
write_atomic(plan, rel, spec + '\n')
|
|
534
|
+
return spec, previous, previous != spec
|
|
535
|
+
|
|
536
|
+
return change(args, plan, rows, 'pin', args.task, 'pin', work, NEXT_RETRY,
|
|
537
|
+
lambda v: f'pinned task {args.task} to {v}', pin_guard=True)
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def cmd_unpin(args):
|
|
541
|
+
plan, rows = load(args.plan)
|
|
542
|
+
require_task(rows, args.task)
|
|
543
|
+
rel = f'tasks/{args.task}/escalation.pin'
|
|
544
|
+
|
|
545
|
+
def work():
|
|
546
|
+
previous = text_of(plan, rel, strict=True)
|
|
547
|
+
remove_safe(plan, rel)
|
|
548
|
+
return None, previous, previous is not None
|
|
549
|
+
|
|
550
|
+
return change(args, plan, rows, 'unpin', args.task, 'pin', work, NEXT_RETRY,
|
|
551
|
+
lambda v: f'unpinned task {args.task}')
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
# ------------------------------------------------------------------------------------ show
|
|
555
|
+
|
|
556
|
+
def stamp(epoch):
|
|
557
|
+
try:
|
|
558
|
+
return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(epoch))
|
|
559
|
+
except (TypeError, ValueError, OverflowError, OSError):
|
|
560
|
+
return '?'
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
def cmd_show(args):
|
|
564
|
+
plan, rows = load(args.plan)
|
|
565
|
+
controls = show(plan, rows)
|
|
566
|
+
if args.json:
|
|
567
|
+
json.dump(controls, sys.stdout, indent=2)
|
|
568
|
+
sys.stdout.write('\n')
|
|
569
|
+
return 0
|
|
570
|
+
cell = lambda v: '-' if v is None else str(v) # noqa: E731
|
|
571
|
+
run = controls['run']
|
|
572
|
+
print(f"{plan} run {cell(controls['run_id'])} live {'yes' if controls['live'] else 'no'}"
|
|
573
|
+
f" fractal {'yes' if controls['fractal'] else 'no'}")
|
|
574
|
+
print(' ' + ' '.join(f"{k.replace('_', '-')} {cell(v)}" for k, v in run.items()))
|
|
575
|
+
columns = ('task', 'status', 'dwarf', 'qa', 'threshold', 'budget', 'used', 'pin', 'stop', 'next')
|
|
576
|
+
table = [columns]
|
|
577
|
+
for task, t in controls['tasks'].items():
|
|
578
|
+
table.append((task, t['status'], cell(t['dwarf']), cell(t['qa']), cell(t['qa_threshold']), cell(t['retry_budget']),
|
|
579
|
+
cell(t['retries_used']), cell(t['pin']), 'yes' if t['escalation_stop'] else '-',
|
|
580
|
+
cell(t['escalation_next'])))
|
|
581
|
+
widths = [max(len(r[i]) for r in table) for i in range(len(columns))]
|
|
582
|
+
for r in table:
|
|
583
|
+
print(' ' + ' '.join(c.ljust(w) for c, w in zip(r, widths)).rstrip())
|
|
584
|
+
if controls['log']:
|
|
585
|
+
print('changes, newest first:')
|
|
586
|
+
for e in controls['log']:
|
|
587
|
+
scope = f" {e['task']}" if e.get('task') else ''
|
|
588
|
+
was = '' if e.get('previous') is None else f" (was {e['previous']})"
|
|
589
|
+
print(f" {stamp(e.get('at'))} {e.get('actor')} {e.get('action')} {e.get('key')}{scope}"
|
|
590
|
+
f" = {cell(e.get('value'))}{was}")
|
|
591
|
+
return 0
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
# ------------------------------------------------------------------------------------ entry point
|
|
595
|
+
|
|
596
|
+
def parser():
|
|
597
|
+
top = argparse.ArgumentParser(prog='forge control', description="Show and change a plan's live controls.")
|
|
598
|
+
sub = top.add_subparsers(dest='command', required=True, metavar='{show,set,pin,unpin}')
|
|
599
|
+
|
|
600
|
+
def common(p, mutator=True):
|
|
601
|
+
p.add_argument('plan', metavar='PLAN_DIR', help="the plan dir (the one with tasks.tsv)")
|
|
602
|
+
p.add_argument('--json', action='store_true', help='print JSON, for scripts and forge-ui')
|
|
603
|
+
if mutator:
|
|
604
|
+
p.add_argument('--actor', choices=('ui', 'cli'), default='cli', help='who is asking, for control.log (default cli)')
|
|
605
|
+
|
|
606
|
+
p = sub.add_parser('show', help='every control, per-task overrides, pins and the change log')
|
|
607
|
+
common(p, mutator=False)
|
|
608
|
+
p.set_defaults(func=cmd_show)
|
|
609
|
+
p = sub.add_parser('set', help='change a control', description='Keys: ' + ', '.join(KEYS) + '. '
|
|
610
|
+
'retry-budget and infra-retries change only while a run is live. '
|
|
611
|
+
'qa-threshold and retry-budget also take --task (value inherit drops the override).')
|
|
612
|
+
common(p)
|
|
613
|
+
p.add_argument('key', choices=list(KEYS))
|
|
614
|
+
p.add_argument('value')
|
|
615
|
+
p.add_argument('--task', metavar='ID', help='change this task only (qa-threshold, retry-budget)')
|
|
616
|
+
p.set_defaults(func=cmd_set)
|
|
617
|
+
p = sub.add_parser('pin', help="set the model a task's next full retry uses")
|
|
618
|
+
common(p)
|
|
619
|
+
p.add_argument('task')
|
|
620
|
+
p.add_argument('spec', help='a dwarf spec, e.g. sol:high:codex')
|
|
621
|
+
p.set_defaults(func=cmd_pin)
|
|
622
|
+
p = sub.add_parser('unpin', help='drop a task pin')
|
|
623
|
+
common(p)
|
|
624
|
+
p.add_argument('task')
|
|
625
|
+
p.set_defaults(func=cmd_unpin)
|
|
626
|
+
return top
|
|
627
|
+
|
|
628
|
+
|
|
629
|
+
def main(argv=None) -> int:
|
|
630
|
+
args = parser().parse_args(argv)
|
|
631
|
+
try:
|
|
632
|
+
return args.func(args)
|
|
633
|
+
except Invalid as error:
|
|
634
|
+
print(f'forge control: {error}', file=sys.stderr)
|
|
635
|
+
return 2
|
|
636
|
+
except Refused as error:
|
|
637
|
+
print(f'forge control: {error}', file=sys.stderr)
|
|
638
|
+
return 3
|
|
639
|
+
except OSError as error:
|
|
640
|
+
print(f'forge control: {error}', file=sys.stderr)
|
|
641
|
+
return 1
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
if __name__ == '__main__':
|
|
645
|
+
sys.exit(main())
|