@giamme/forge 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/SKILL.md +4 -2
- package/VERSION +1 -1
- package/docs/artifacts.md +46 -12
- package/docs/commands.md +48 -9
- package/docs/decomposed-runs.md +43 -2
- package/docs/forge-ui.md +27 -6
- package/docs/reviews.md +31 -6
- package/package.json +1 -1
- package/references/dataset.md +3 -1
- package/references/decompose.md +159 -17
- package/references/harnesses.md +13 -1
- package/references/ui.md +14 -4
- package/scripts/forge-control.py +268 -25
- package/scripts/forge-dispatch.sh +80 -3
- package/scripts/forge-guard.py +156 -15
- package/scripts/forge-issues.py +1 -1
- package/scripts/forge-lib-plans.sh +9 -3
- package/scripts/forge-lib-review.sh +22 -3
- package/scripts/forge-lib-task.sh +51 -4
- package/scripts/forge-lib-verify.sh +137 -6
- package/scripts/forge-parallel-options.sh +59 -0
- package/scripts/forge-parallel.sh +212 -28
- package/scripts/forge-ripwire.py +1 -1
- package/scripts/forge-route.py +157 -5
- package/scripts/forge-schedule.py +687 -50
- package/scripts/forge-ui.py +8 -4
- package/scripts/forge-verify.py +878 -0
- package/scripts/forge_dataset/forecast.py +29 -4
- package/scripts/forge_dataset/harvest.py +10 -4
- package/scripts/forge_verify_reports.py +225 -0
- package/ui/README.md +19 -4
- package/ui/docs/api.md +139 -25
- package/ui/forge_ui/control.py +26 -7
- package/ui/forge_ui/export.py +20 -7
- package/ui/forge_ui/reader.py +307 -11
- package/ui/forge_ui/server.py +16 -0
- package/ui/forge_ui/static/app.js +2 -1
- package/ui/forge_ui/static/controls.js +217 -33
- package/ui/forge_ui/static/findingsview.js +37 -12
- package/ui/forge_ui/static/gantt.js +14 -9
- package/ui/forge_ui/static/index.html +1 -0
- package/ui/forge_ui/static/panel.js +2 -1
- package/ui/forge_ui/static/runmodel.js +92 -11
- package/ui/forge_ui/static/scene3d.js +9 -5
- package/ui/forge_ui/static/settings.js +48 -12
- package/ui/forge_ui/static/style.css +60 -0
- package/ui/forge_ui/static/verifyview.js +324 -0
package/scripts/forge-control.py
CHANGED
|
@@ -2,24 +2,41 @@
|
|
|
2
2
|
"""forge control <command> <plan-dir> ...: show and change a plan's live controls.
|
|
3
3
|
|
|
4
4
|
show <plan> [--json] every control, per-task overrides, pins and the change log
|
|
5
|
-
set <plan> <key> <value> [--task ID] qa-threshold, timeout, escalate-after,
|
|
6
|
-
retry-budget, infra-retries
|
|
5
|
+
set <plan> <key> <value> [--task ID] qa-threshold, timeout, escalate-after, escalate-effort,
|
|
6
|
+
verify-retries, retry-budget, infra-retries, infra-backoff
|
|
7
|
+
(none drops it), max-parallel (1-64)
|
|
7
8
|
pin <plan> <task> <spec> the model the task's next FULL implementer retry uses
|
|
8
9
|
unpin <plan> <task>
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
retry <plan> <task> [--dwarf SPEC] queue a retry of a stopped or failed task into the live run
|
|
11
|
+
add <plan> <task> queue a row added to tasks.tsv (with its prompt.md) into the live run
|
|
12
|
+
drain <plan> start nothing new; running tasks finish and the scheduler waits
|
|
13
|
+
while tasks remain (with none left to start, the run ends as usual)
|
|
14
|
+
resume <plan> end a drain
|
|
15
|
+
stop <plan> [--now] start nothing new and end the run once running tasks are done
|
|
16
|
+
(exit 130, no verification; when every task has merged by then,
|
|
17
|
+
the run ends as usual: it verifies and exits 0); --now interrupts
|
|
18
|
+
them like SIGTERM
|
|
19
|
+
|
|
20
|
+
`show` reports escalate-after and verify-retries at their defaults (3 and 1) when unset,
|
|
21
|
+
escalate-effort as on or off (off when the file is absent), and the two budgets only while a
|
|
22
|
+
scheduler is live.
|
|
12
23
|
|
|
13
24
|
Every change is validated, written atomically under the plan's control lock (`control.lock`),
|
|
14
25
|
appended to `control.log` and recorded as a dataset `control` event. This is the only writer of
|
|
15
26
|
human changes to a running plan; forge-ui calls it and never writes run files itself. Some
|
|
16
27
|
controls can only change while a scheduler is live (`scheduler.live` names a live pid):
|
|
17
|
-
retry-budget
|
|
28
|
+
retry-budget, infra-retries, max-parallel, drain, resume, stop, retry and add.
|
|
29
|
+
|
|
30
|
+
retry and add only queue a request in control.requests/ (one JSON file each): the scheduler
|
|
31
|
+
applies or refuses it at its next pass and logs the outcome in control.log, with the same
|
|
32
|
+
`request` id, as actor `scheduler`. A request never outlives the scheduler that was live
|
|
33
|
+
when it was queued. A task-level change or a pin on a MERGED task, and a pin on a
|
|
18
34
|
Fractal plan, are refused.
|
|
19
35
|
|
|
20
|
-
A pin
|
|
21
|
-
quota, before the lock is taken.
|
|
22
|
-
with another command line, which
|
|
36
|
+
A pin, a retry's --dwarf and an added row's specs are checked with `forge-dispatch.sh doctor
|
|
37
|
+
--spec <spec> --role <role>`, which spends no quota, before the lock is taken.
|
|
38
|
+
FORGE_CONTROL_DOCTOR replaces `bash forge-dispatch.sh doctor` with another command line, which
|
|
39
|
+
then gets `--spec <spec> --role <role>`; tests use it.
|
|
23
40
|
|
|
24
41
|
Exit codes: 0 ok, 2 usage or invalid input (including an unsafe path), 3 refused (not live, MERGED,
|
|
25
42
|
Fractal, a spec that fails doctor, the control lock busy for over 5 s).
|
|
@@ -54,6 +71,8 @@ NEXT_ATTEMPT = "at the task's next attempt"
|
|
|
54
71
|
NEXT_DECISION = 'at the next retry decision'
|
|
55
72
|
NEXT_RETRY = 'at the next full retry (not QA-only resumes)'
|
|
56
73
|
NEXT_VERIFY = 'at the next verification'
|
|
74
|
+
NEXT_INFRA_WAIT = 'at the next infrastructure wait'
|
|
75
|
+
NEXT_DISPATCH = "at the scheduler's next dispatch decision"
|
|
57
76
|
|
|
58
77
|
|
|
59
78
|
class Invalid(Exception):
|
|
@@ -357,6 +376,20 @@ def nullable(value):
|
|
|
357
376
|
return None if value in (None, '', '-', 'UNASSIGNED') else value
|
|
358
377
|
|
|
359
378
|
|
|
379
|
+
PAUSE_KEYS = ('scope', 'class', 'harness', 'model', 'tasks', 'cycle', 'until', 'exhausted')
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def paused_scopes(text):
|
|
383
|
+
"""The scheduler's infra.paused scopes; [] when nothing is paused or the file is unreadable."""
|
|
384
|
+
try:
|
|
385
|
+
scopes = json.loads(text or '')['scopes']
|
|
386
|
+
except (ValueError, KeyError, TypeError):
|
|
387
|
+
return []
|
|
388
|
+
if not isinstance(scopes, list):
|
|
389
|
+
return []
|
|
390
|
+
return [{key: scope.get(key) for key in PAUSE_KEYS} for scope in scopes if isinstance(scope, dict)]
|
|
391
|
+
|
|
392
|
+
|
|
360
393
|
def show(plan, rows):
|
|
361
394
|
live = is_live(plan)
|
|
362
395
|
|
|
@@ -367,8 +400,17 @@ def show(plan, rows):
|
|
|
367
400
|
return number(get(rel)) if live else None
|
|
368
401
|
|
|
369
402
|
run = dict(qa_threshold=level(get('qa_threshold')), timeout=number(get('timeout')),
|
|
370
|
-
escalate_after=number(get('escalate_after')),
|
|
403
|
+
escalate_after=number(get('escalate_after')),
|
|
404
|
+
escalate_effort='on' if get('escalate_effort') == 'on' else 'off',
|
|
405
|
+
verify_retries=number(get('verify_retries')),
|
|
371
406
|
retry_budget=budget('retry_budget'), infra_budget=budget('infra_budget'))
|
|
407
|
+
run['infra_backoff'] = get('infra_backoff') or default_backoff()
|
|
408
|
+
parallel = number(get('max_parallel'))
|
|
409
|
+
run['max_parallel'] = parallel if parallel and parallel <= 64 else None
|
|
410
|
+
stopping = get('stop') if live else None
|
|
411
|
+
run['drained'] = live and has(plan, 'drain')
|
|
412
|
+
run['stopping'] = stopping if stopping in ('graceful', 'now') else None
|
|
413
|
+
run['infra_paused'] = paused_scopes(get('infra.paused')) if live else None
|
|
372
414
|
if run['escalate_after'] is None:
|
|
373
415
|
run['escalate_after'] = DEFAULT_ESCALATE_AFTER
|
|
374
416
|
if run['verify_retries'] is None:
|
|
@@ -414,18 +456,61 @@ def _int(value, task):
|
|
|
414
456
|
raise Invalid('needs a non-negative integer (up to 9 digits)')
|
|
415
457
|
|
|
416
458
|
|
|
459
|
+
_SCHEDULE = None
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def schedule_module():
|
|
463
|
+
"""forge-schedule.py, loaded by path: the one owner of the backoff shape grammar."""
|
|
464
|
+
global _SCHEDULE
|
|
465
|
+
if _SCHEDULE is None:
|
|
466
|
+
import importlib.util
|
|
467
|
+
spec = importlib.util.spec_from_file_location('forge_schedule', os.path.join(SKILL, 'scripts', 'forge-schedule.py'))
|
|
468
|
+
_SCHEDULE = importlib.util.module_from_spec(spec)
|
|
469
|
+
spec.loader.exec_module(_SCHEDULE)
|
|
470
|
+
return _SCHEDULE
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def _backoff(value, task):
|
|
474
|
+
if value == 'none':
|
|
475
|
+
return value
|
|
476
|
+
try:
|
|
477
|
+
schedule_module().parse_backoff(value)
|
|
478
|
+
except ValueError as error:
|
|
479
|
+
raise Invalid(str(error).replace('backoff ', '', 1))
|
|
480
|
+
return value.strip()
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def default_backoff():
|
|
484
|
+
return schedule_module().default_backoff()[1]
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def _parallel(value, task):
|
|
488
|
+
if re.fullmatch(r'[0-9]{1,2}', value) and 1 <= int(value) <= 64:
|
|
489
|
+
return str(int(value))
|
|
490
|
+
raise Invalid('needs an integer from 1 to 64')
|
|
491
|
+
|
|
492
|
+
|
|
417
493
|
def _budget(value, task):
|
|
418
494
|
return value if task and value == 'inherit' else _int(value, task)
|
|
419
495
|
|
|
420
496
|
|
|
497
|
+
def _onoff(value, _task):
|
|
498
|
+
if value in ('on', 'off'):
|
|
499
|
+
return value
|
|
500
|
+
raise Invalid('needs on or off')
|
|
501
|
+
|
|
502
|
+
|
|
421
503
|
# key -> (plan only or both, validator, file, live only, when it applies)
|
|
422
504
|
KEYS = {
|
|
423
505
|
'qa-threshold': ('both', _qa, 'qa_threshold', False, NEXT_ATTEMPT),
|
|
424
506
|
'timeout': ('plan', _int, 'timeout', False, NEXT_ATTEMPT),
|
|
425
507
|
'escalate-after': ('plan', _int, 'escalate_after', False, NEXT_DECISION),
|
|
508
|
+
'escalate-effort': ('plan', _onoff, 'escalate_effort', False, NEXT_DECISION),
|
|
426
509
|
'verify-retries': ('plan', _int, 'verify_retries', False, NEXT_VERIFY),
|
|
427
510
|
'retry-budget': ('both', _budget, 'retry_budget', True, NEXT_DECISION),
|
|
428
511
|
'infra-retries': ('plan', _int, 'infra_budget', True, NEXT_DECISION),
|
|
512
|
+
'infra-backoff': ('plan', _backoff, 'infra_backoff', False, NEXT_INFRA_WAIT),
|
|
513
|
+
'max-parallel': ('plan', _parallel, 'max_parallel', True, NEXT_DISPATCH),
|
|
429
514
|
}
|
|
430
515
|
|
|
431
516
|
|
|
@@ -436,7 +521,7 @@ def emit(plan, entry):
|
|
|
436
521
|
cmd = [sys.executable, os.path.join(SKILL, 'scripts', 'forge-dataset.py'), 'event', 'control', '--dir', plan]
|
|
437
522
|
if entry['task']:
|
|
438
523
|
cmd += ['--task', entry['task']]
|
|
439
|
-
for field in ('at', 'actor', 'action', 'key', 'value', 'previous'):
|
|
524
|
+
for field in ('at', 'actor', 'action', 'key', 'value', 'previous') + tuple(k for k in ('request',) if k in entry):
|
|
440
525
|
value = entry[field]
|
|
441
526
|
cmd += ['--set', f"{field}={'' if value is None else value}"]
|
|
442
527
|
try:
|
|
@@ -445,8 +530,9 @@ def emit(plan, entry):
|
|
|
445
530
|
pass
|
|
446
531
|
|
|
447
532
|
|
|
448
|
-
def audit(plan, actor, action, task, key, value, previous):
|
|
449
|
-
entry = dict(at=time.time(), actor=actor, action=action, task=task, key=key, value=value, previous=previous
|
|
533
|
+
def audit(plan, actor, action, task, key, value, previous, **extra):
|
|
534
|
+
entry = dict(at=time.time(), actor=actor, action=action, task=task, key=key, value=value, previous=previous,
|
|
535
|
+
**extra)
|
|
450
536
|
append_line(plan, 'control.log', json.dumps(entry) + '\n')
|
|
451
537
|
emit(plan, entry)
|
|
452
538
|
return entry
|
|
@@ -457,7 +543,7 @@ def change(args, plan, rows, action, task, key, work, applies, summary, live_onl
|
|
|
457
543
|
with control_lock(plan):
|
|
458
544
|
probe(plan, 'control.log') # an unsafe log refuses the change instead of leaving it unaudited
|
|
459
545
|
if live_only and not is_live(plan):
|
|
460
|
-
raise Refused(f'{key} can only change while a run is live (no live scheduler for this plan)')
|
|
546
|
+
raise Refused(f'{key or action} can only change while a run is live (no live scheduler for this plan)')
|
|
461
547
|
if task:
|
|
462
548
|
refuse_task(plan, task, pin_guard)
|
|
463
549
|
value, previous, changed = work()
|
|
@@ -469,8 +555,8 @@ def change(args, plan, rows, action, task, key, work, applies, summary, live_onl
|
|
|
469
555
|
json.dump(dict(ok=True, changed=record, applies=applies, controls=controls), sys.stdout, indent=2)
|
|
470
556
|
sys.stdout.write('\n')
|
|
471
557
|
else:
|
|
472
|
-
|
|
473
|
-
|
|
558
|
+
was = '' if key is None else f" (was {'unset' if previous is None else previous})"
|
|
559
|
+
print(f"{summary(value)}{was}; applies {applies}" + ('' if changed else ' (nothing changed, not logged)'))
|
|
474
560
|
return 0
|
|
475
561
|
|
|
476
562
|
|
|
@@ -487,8 +573,10 @@ def cmd_set(args):
|
|
|
487
573
|
except Invalid as error:
|
|
488
574
|
raise Invalid(f'{args.key}: {error}')
|
|
489
575
|
rel = f'tasks/{task}/{name}' if task else name
|
|
490
|
-
drop = value == 'inherit' or (value == 'none' and not task)
|
|
491
|
-
wide
|
|
576
|
+
drop = value == 'inherit' or (value == 'none' and not task) or (args.key == 'escalate-effort' and value == 'off')
|
|
577
|
+
# Plan-wide stop markers go when the policy that wrote them changes: the policy turned off
|
|
578
|
+
# (escalate-after 0), or the effort step turned on, which may yet have a move for them.
|
|
579
|
+
wide = (args.key == 'escalate-after' and value == '0') or (args.key == 'escalate-effort' and value == 'on')
|
|
492
580
|
markers = [f'tasks/{row[0]}/{m}' for row in rows if valid_task_id(row[0]) for m in STOP_MARKERS] if wide else []
|
|
493
581
|
|
|
494
582
|
def work():
|
|
@@ -500,7 +588,7 @@ def cmd_set(args):
|
|
|
500
588
|
else:
|
|
501
589
|
write_atomic(plan, rel, value + '\n')
|
|
502
590
|
changed = previous != value
|
|
503
|
-
swept = [remove_safe(plan, target) for target in markers] # as `run --escalate-after 0
|
|
591
|
+
swept = [remove_safe(plan, target) for target in markers] # as `run --escalate-after 0` and `--escalate-effort`
|
|
504
592
|
return value, previous, changed or any(swept)
|
|
505
593
|
|
|
506
594
|
where = f'task {task}' if task else 'the plan'
|
|
@@ -508,12 +596,12 @@ def cmd_set(args):
|
|
|
508
596
|
lambda v: f'set {args.key} {v} on {where}', live_only=live_only)
|
|
509
597
|
|
|
510
598
|
|
|
511
|
-
def doctor(spec):
|
|
512
|
-
"""Offline capability check of a
|
|
599
|
+
def doctor(spec, role='dwarf'):
|
|
600
|
+
"""Offline capability check of a spec: Refused when it fails, no quota spent."""
|
|
513
601
|
override = os.environ.get('FORGE_CONTROL_DOCTOR')
|
|
514
602
|
base = shlex.split(override) if override else ['bash', os.path.join(SKILL, 'scripts', 'forge-dispatch.sh'), 'doctor']
|
|
515
603
|
try:
|
|
516
|
-
done = subprocess.run(base + ['--spec', spec, '--role',
|
|
604
|
+
done = subprocess.run(base + ['--spec', spec, '--role', role], capture_output=True, text=True,
|
|
517
605
|
stdin=subprocess.DEVNULL, timeout=60)
|
|
518
606
|
except (OSError, subprocess.SubprocessError, ValueError) as error:
|
|
519
607
|
raise Refused(f'could not check the spec: {error}')
|
|
@@ -555,6 +643,132 @@ def cmd_unpin(args):
|
|
|
555
643
|
lambda v: f'unpinned task {args.task}')
|
|
556
644
|
|
|
557
645
|
|
|
646
|
+
def run_command(args, action, rel, value, summary):
|
|
647
|
+
"""drain, resume and stop: a live-only plan file the scheduler reads at every pass.
|
|
648
|
+
value None removes the file; logged with the action and no key."""
|
|
649
|
+
plan, rows = load(args.plan)
|
|
650
|
+
|
|
651
|
+
def work():
|
|
652
|
+
previous = text_of(plan, rel, strict=True)
|
|
653
|
+
present = previous is not None or has(plan, rel)
|
|
654
|
+
if value is None:
|
|
655
|
+
return None, previous, remove_safe(plan, rel)
|
|
656
|
+
write_atomic(plan, rel, value + ('\n' if value else ''))
|
|
657
|
+
return value or None, previous, not present or (previous or '') != value
|
|
658
|
+
|
|
659
|
+
return change(args, plan, rows, action, None, None, work, NEXT_DISPATCH, summary, live_only=True)
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
def cmd_drain(args):
|
|
663
|
+
return run_command(args, 'drain', 'drain', '', lambda v: 'drained the plan: nothing new starts')
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def cmd_resume(args):
|
|
667
|
+
return run_command(args, 'resume', 'drain', None, lambda v: 'resumed the plan')
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def cmd_stop(args):
|
|
671
|
+
mode = 'now' if args.now else 'graceful'
|
|
672
|
+
return run_command(args, 'stop', 'stop', mode, lambda v: 'stopping the run now' if args.now else
|
|
673
|
+
'stopping the run once running tasks are done')
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
# ------------------------------------------------------------------------------------ requests
|
|
677
|
+
|
|
678
|
+
REQUESTS = 'control.requests'
|
|
679
|
+
DIFFICULTIES = ('low', 'medium', 'high')
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
def clean_spec(spec, what):
|
|
683
|
+
if not spec or re.search(r'[\s\x00-\x1f\x7f,]', spec) or spec in ('-', 'UNASSIGNED') or spec.startswith('-'):
|
|
684
|
+
raise Invalid(f'{what} must be one explicit spec such as sol:high:codex')
|
|
685
|
+
return spec
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
def check_row(plan, rows, task):
|
|
689
|
+
"""The checks `plan` would need before a row may join a live run; Invalid otherwise."""
|
|
690
|
+
if not valid_task_id(task):
|
|
691
|
+
raise Invalid(f'invalid task id: {task!r}')
|
|
692
|
+
found = [row for row in rows if row[0] == task]
|
|
693
|
+
if not found:
|
|
694
|
+
raise Invalid(f'task {task!r} is not in tasks.tsv')
|
|
695
|
+
if len(found) > 1:
|
|
696
|
+
raise Invalid(f'task {task!r} appears more than once in tasks.tsv')
|
|
697
|
+
row = found[0]
|
|
698
|
+
if len(row) != 7:
|
|
699
|
+
raise Invalid(f'task {task!r}: tasks.tsv rows need 7 tab-separated columns, this one has {len(row)}')
|
|
700
|
+
_, deps, difficulty, files, dwarf, qa, _ = row
|
|
701
|
+
if difficulty not in DIFFICULTIES:
|
|
702
|
+
raise Invalid(f'task {task!r}: difficulty must be low, medium or high')
|
|
703
|
+
ids = {r[0] for r in rows}
|
|
704
|
+
wanted = [d for d in deps.split(',') if d and d != '-']
|
|
705
|
+
missing = [d for d in wanted if d not in ids]
|
|
706
|
+
if missing:
|
|
707
|
+
raise Invalid(f'task {task!r} depends on unknown task(s): {", ".join(missing)}')
|
|
708
|
+
graph = {r[0]: [d for d in (r[1].split(',') if len(r) > 1 else []) if d and d != '-'] for r in rows}
|
|
709
|
+
state = {}
|
|
710
|
+
|
|
711
|
+
def cyclic(node):
|
|
712
|
+
state[node] = 1
|
|
713
|
+
for dep in graph.get(node, []):
|
|
714
|
+
if state.get(dep) == 1 or (dep not in state and dep in graph and cyclic(dep)):
|
|
715
|
+
return True
|
|
716
|
+
state[node] = 2
|
|
717
|
+
return False
|
|
718
|
+
|
|
719
|
+
if cyclic(task):
|
|
720
|
+
raise Invalid(f'task {task!r}: its dependencies form a cycle')
|
|
721
|
+
if not files.strip() or files.strip() == '-':
|
|
722
|
+
raise Invalid(f'task {task!r}: the files column is empty')
|
|
723
|
+
clean_spec(dwarf, f'task {task!r}: the dwarf column')
|
|
724
|
+
clean_spec(qa, f'task {task!r}: the qa column')
|
|
725
|
+
prompt = read_file(plan, f'tasks/{task}/prompt.md', strict=True)
|
|
726
|
+
if not (prompt or '').strip():
|
|
727
|
+
raise Invalid(f'task {task!r}: tasks/{task}/prompt.md is missing or empty')
|
|
728
|
+
return dwarf, qa
|
|
729
|
+
|
|
730
|
+
|
|
731
|
+
def queue(args, plan, verb, task, dwarf):
|
|
732
|
+
"""Write the request under the control lock, live only; log it with its id."""
|
|
733
|
+
with control_lock(plan):
|
|
734
|
+
probe(plan, 'control.log')
|
|
735
|
+
if not is_live(plan):
|
|
736
|
+
raise Refused(f'{verb} needs a live run (no live scheduler for this plan)')
|
|
737
|
+
if verb == 'retry':
|
|
738
|
+
refuse_task(plan, task, pin=True)
|
|
739
|
+
name = f'{time.time_ns()}-{os.getpid()}.json'
|
|
740
|
+
request = dict(verb=verb, task=task, dwarf=dwarf, at=time.time(), actor=args.actor)
|
|
741
|
+
write_atomic(plan, f'{REQUESTS}/{name}', json.dumps(request) + '\n')
|
|
742
|
+
audit(plan, args.actor, verb, task, None, dwarf, None, request=name)
|
|
743
|
+
if args.json:
|
|
744
|
+
json.dump(dict(queued=True, request=name), sys.stdout)
|
|
745
|
+
sys.stdout.write('\n')
|
|
746
|
+
else:
|
|
747
|
+
print(f'queued {verb} of task {task}' + (f' on {dwarf}' if dwarf else '') + f' (request {name}); '
|
|
748
|
+
'the scheduler applies or refuses it at its next pass and logs the outcome in control.log')
|
|
749
|
+
return 0
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
def cmd_retry(args):
|
|
753
|
+
plan, rows = load(args.plan)
|
|
754
|
+
require_task(rows, args.task)
|
|
755
|
+
dwarf = None
|
|
756
|
+
if args.dwarf is not None:
|
|
757
|
+
dwarf = clean_spec(args.dwarf, '--dwarf')
|
|
758
|
+
refuse_task(plan, args.task, pin=True) # before the doctor: MERGED and Fractal need no model check
|
|
759
|
+
if dwarf:
|
|
760
|
+
doctor(dwarf)
|
|
761
|
+
return queue(args, plan, 'retry', args.task, dwarf)
|
|
762
|
+
|
|
763
|
+
|
|
764
|
+
def cmd_add(args):
|
|
765
|
+
plan, rows = load(args.plan)
|
|
766
|
+
dwarf, qa = check_row(plan, rows, args.task)
|
|
767
|
+
doctor(dwarf, 'dwarf')
|
|
768
|
+
doctor(qa, 'qa')
|
|
769
|
+
return queue(args, plan, 'add', args.task, None)
|
|
770
|
+
|
|
771
|
+
|
|
558
772
|
# ------------------------------------------------------------------------------------ show
|
|
559
773
|
|
|
560
774
|
def stamp(epoch):
|
|
@@ -575,7 +789,16 @@ def cmd_show(args):
|
|
|
575
789
|
run = controls['run']
|
|
576
790
|
print(f"{plan} run {cell(controls['run_id'])} live {'yes' if controls['live'] else 'no'}"
|
|
577
791
|
f" fractal {'yes' if controls['fractal'] else 'no'}")
|
|
578
|
-
print(' ' + ' '.join(f"{k.replace('_', '-')} {cell(v)}" for k, v in run.items()))
|
|
792
|
+
print(' ' + ' '.join(f"{k.replace('_', '-')} {cell(v)}" for k, v in run.items() if k != 'infra_paused'))
|
|
793
|
+
for p in run['infra_paused'] or []:
|
|
794
|
+
tasks = ','.join(t for t in (p['tasks'] or []) if isinstance(t, str)) or '-'
|
|
795
|
+
if p['exhausted']:
|
|
796
|
+
when = 'waits used up'
|
|
797
|
+
elif isinstance(p['until'], (int, float)):
|
|
798
|
+
when = f"wait {cell(p['cycle'])} until {stamp(p['until'])}"
|
|
799
|
+
else:
|
|
800
|
+
when = 'waiting for running tasks to finish'
|
|
801
|
+
print(f" paused {p['scope']} ({cell(p['class'])}) tasks {tasks} {when}")
|
|
579
802
|
columns = ('task', 'status', 'dwarf', 'qa', 'threshold', 'budget', 'used', 'pin', 'stop', 'next')
|
|
580
803
|
table = [columns]
|
|
581
804
|
for task, t in controls['tasks'].items():
|
|
@@ -599,7 +822,8 @@ def cmd_show(args):
|
|
|
599
822
|
|
|
600
823
|
def parser():
|
|
601
824
|
top = argparse.ArgumentParser(prog='forge control', description="Show and change a plan's live controls.")
|
|
602
|
-
sub = top.add_subparsers(dest='command', required=True,
|
|
825
|
+
sub = top.add_subparsers(dest='command', required=True,
|
|
826
|
+
metavar='{show,set,pin,unpin,retry,add,drain,resume,stop}')
|
|
603
827
|
|
|
604
828
|
def common(p, mutator=True):
|
|
605
829
|
p.add_argument('plan', metavar='PLAN_DIR', help="the plan dir (the one with tasks.tsv)")
|
|
@@ -611,7 +835,7 @@ def parser():
|
|
|
611
835
|
common(p, mutator=False)
|
|
612
836
|
p.set_defaults(func=cmd_show)
|
|
613
837
|
p = sub.add_parser('set', help='change a control', description='Keys: ' + ', '.join(KEYS) + '. '
|
|
614
|
-
'retry-budget
|
|
838
|
+
'retry-budget, infra-retries and max-parallel change only while a run is live. '
|
|
615
839
|
'qa-threshold and retry-budget also take --task (value inherit drops the override).')
|
|
616
840
|
common(p)
|
|
617
841
|
p.add_argument('key', choices=list(KEYS))
|
|
@@ -627,6 +851,25 @@ def parser():
|
|
|
627
851
|
common(p)
|
|
628
852
|
p.add_argument('task')
|
|
629
853
|
p.set_defaults(func=cmd_unpin)
|
|
854
|
+
p = sub.add_parser('retry', help='queue a retry of a stopped or failed task into the live run')
|
|
855
|
+
common(p)
|
|
856
|
+
p.add_argument('task')
|
|
857
|
+
p.add_argument('--dwarf', metavar='SPEC', help='the implementer for this retry (an explicit escalation)')
|
|
858
|
+
p.set_defaults(func=cmd_retry)
|
|
859
|
+
p = sub.add_parser('add', help='queue a row added to tasks.tsv into the live run')
|
|
860
|
+
common(p)
|
|
861
|
+
p.add_argument('task')
|
|
862
|
+
p.set_defaults(func=cmd_add)
|
|
863
|
+
p = sub.add_parser('drain', help='start nothing new; running tasks finish and the scheduler waits while tasks remain (live only)')
|
|
864
|
+
common(p)
|
|
865
|
+
p.set_defaults(func=cmd_drain)
|
|
866
|
+
p = sub.add_parser('resume', help='end a drain (live only)')
|
|
867
|
+
common(p)
|
|
868
|
+
p.set_defaults(func=cmd_resume)
|
|
869
|
+
p = sub.add_parser('stop', help='end the run once running tasks finish, exit 130 without verifying unless every task merged (live only)')
|
|
870
|
+
common(p)
|
|
871
|
+
p.add_argument('--now', action='store_true', help='interrupt running tasks, as SIGTERM does')
|
|
872
|
+
p.set_defaults(func=cmd_stop)
|
|
630
873
|
return top
|
|
631
874
|
|
|
632
875
|
|
|
@@ -13,6 +13,14 @@
|
|
|
13
13
|
# (claude, openclaude, codex; custom prompt only)
|
|
14
14
|
# any role: [--no-ripwire] [--ripwire-query-file <requirements>]
|
|
15
15
|
# any role: [--timeout <seconds>] 0 disables the watchdog
|
|
16
|
+
# forge-dispatch.sh efforts --spec <spec>
|
|
17
|
+
# the efforts this pairing accepts, weakest first, one per
|
|
18
|
+
# line. Prints nothing when the pairing has no ladder or no
|
|
19
|
+
# effort dial. A literal model id (an alias with no row for the
|
|
20
|
+
# named harness) takes its harness's whole ladder, as on a
|
|
21
|
+
# dispatch. Exit 2 for an alias the registry does not know when
|
|
22
|
+
# no harness is named. Never dispatches, never needs the
|
|
23
|
+
# harness installed, and never writes a file.
|
|
16
24
|
#
|
|
17
25
|
# Timeout precedence: --timeout, then FORGE_TIMEOUT (both used as given, 0 included),
|
|
18
26
|
# then the registry row's optional 5th `timeout` column, then the built-in 2700 (45m).
|
|
@@ -103,6 +111,61 @@ known_harness() {
|
|
|
103
111
|
case "$1" in codex|claude|openclaude|opencode|antigravity|grok) return 0 ;; *) return 1 ;; esac
|
|
104
112
|
}
|
|
105
113
|
|
|
114
|
+
# Read-only listing of the efforts a pairing accepts. Resolves alias, default harness and the
|
|
115
|
+
# effort column the same way a dispatch does, then prints. No harness binary, no run dir.
|
|
116
|
+
efforts_cmd() { # efforts_cmd --spec <spec>
|
|
117
|
+
if [ "${1:-}" != --spec ] || [ $# -ne 2 ]; then
|
|
118
|
+
die "usage: forge-dispatch.sh efforts --spec <spec>"
|
|
119
|
+
fi
|
|
120
|
+
local SPEC="$2" ALIAS="" REST="" H="" HARNESS="" ROW="" MODEL="" CEILING="" timeout=""
|
|
121
|
+
local LADDER="" ALLOWED="" top="" w="" pad=""
|
|
122
|
+
ALIAS="${SPEC%%:*}"
|
|
123
|
+
REST="${SPEC#*:}"
|
|
124
|
+
if [ "$REST" = "$SPEC" ]; then HARNESS=""
|
|
125
|
+
else
|
|
126
|
+
H="${REST#*:}"
|
|
127
|
+
if [ "$H" = "$REST" ]; then HARNESS=""; else HARNESS="$H"; fi
|
|
128
|
+
fi
|
|
129
|
+
[ -n "$ALIAS" ] || die "empty alias in spec '$SPEC'"
|
|
130
|
+
if [ -n "$HARNESS" ] && ! known_harness "$HARNESS"; then
|
|
131
|
+
die "unknown harness '$HARNESS' (expected codex, claude, openclaude, opencode, antigravity or grok)"
|
|
132
|
+
fi
|
|
133
|
+
ROW="$(lookup "$ALIAS" "$HARNESS")"
|
|
134
|
+
if [ -z "$ROW" ]; then
|
|
135
|
+
# As on a dispatch: with a harness named, an alias with no row for it is a literal model
|
|
136
|
+
# id with no ceiling; with none named, an alias the registry does not know is an error.
|
|
137
|
+
[ -n "$HARNESS" ] || die "'$ALIAS' is not a known alias"
|
|
138
|
+
CEILING="-"
|
|
139
|
+
else
|
|
140
|
+
IFS=$'\t' read -r HARNESS MODEL CEILING timeout <<< "$ROW"
|
|
141
|
+
fi
|
|
142
|
+
LADDER="$(ladder_for "$HARNESS")"
|
|
143
|
+
[ -n "$LADDER" ] || exit 0 # opencode and anything else with no effort dial
|
|
144
|
+
case "$CEILING" in
|
|
145
|
+
none) exit 0 ;;
|
|
146
|
+
-) # no ceiling: a dispatch takes any word of the harness's ladder
|
|
147
|
+
for w in $LADDER; do printf '%s\n' "$w"; done
|
|
148
|
+
;;
|
|
149
|
+
"<="*)
|
|
150
|
+
top="${CEILING#<=}"
|
|
151
|
+
for w in $LADDER; do
|
|
152
|
+
printf '%s\n' "$w"
|
|
153
|
+
[ "$w" = "$top" ] && break
|
|
154
|
+
done
|
|
155
|
+
;;
|
|
156
|
+
*)
|
|
157
|
+
# The list form, weakest first in ladder order. A listed word the ladder does not know
|
|
158
|
+
# is not printed: a dispatch at it stops with "invalid effort".
|
|
159
|
+
ALLOWED="$(printf '%s' "$CEILING" | tr ',' ' ')"
|
|
160
|
+
pad=" $ALLOWED "
|
|
161
|
+
for w in $LADDER; do
|
|
162
|
+
case "$pad" in *" $w "*) printf '%s\n' "$w" ;; esac
|
|
163
|
+
done
|
|
164
|
+
;;
|
|
165
|
+
esac
|
|
166
|
+
exit 0
|
|
167
|
+
}
|
|
168
|
+
|
|
106
169
|
# --- doctor -------------------------------------------------------------------
|
|
107
170
|
doctor() {
|
|
108
171
|
local rc=0 h bin
|
|
@@ -127,7 +190,8 @@ EOT
|
|
|
127
190
|
return $rc
|
|
128
191
|
}
|
|
129
192
|
|
|
130
|
-
[ $# -ge 1 ] || die "usage: forge-dispatch.sh <doctor|dwarf|qa|planner> [spec] [options]"
|
|
193
|
+
[ $# -ge 1 ] || die "usage: forge-dispatch.sh <doctor|dwarf|qa|planner|efforts> [spec] [options]"
|
|
194
|
+
if [ "$1" = efforts ]; then shift; efforts_cmd "$@"; exit $?; fi
|
|
131
195
|
ROLE="$1"; shift
|
|
132
196
|
if [ "$ROLE" = report ]; then python3 "$RUNTIME" report "${1:?report needs a run directory}"; exit $?; fi
|
|
133
197
|
PREFLIGHT=0
|
|
@@ -317,7 +381,7 @@ write_continue() { :; } # redefined below once there is an attempt to archive
|
|
|
317
381
|
ATTEMPT=""
|
|
318
382
|
if [ "$PREFLIGHT" != 1 ] && [ "$DRY" != 1 ]; then
|
|
319
383
|
ATTEMPT="$(python3 "$RUNTIME" begin "$RUN_DIR" "$ROLE")" || exit 3
|
|
320
|
-
rm -f "$RUN_DIR/$ROLE.last" "$RUN_DIR/$ROLE.log" "$RUN_DIR/$ROLE.resolved" "$RUN_DIR/$ROLE.cmd" "$RUN_DIR/$ROLE.timedout" "$RUN_DIR/$ROLE.infra" "$RUN_DIR/$ROLE.orphans" "$RUN_DIR/$ROLE.session" "$RUN_DIR/$ROLE.continue"
|
|
384
|
+
rm -f "$RUN_DIR/$ROLE.last" "$RUN_DIR/$ROLE.log" "$RUN_DIR/$ROLE.resolved" "$RUN_DIR/$ROLE.cmd" "$RUN_DIR/$ROLE.timedout" "$RUN_DIR/$ROLE.infra" "$RUN_DIR/$ROLE.orphans" "$RUN_DIR/$ROLE.session" "$RUN_DIR/$ROLE.continue" "$RUN_DIR/$ROLE.harness"
|
|
321
385
|
trap 'exit 143' TERM
|
|
322
386
|
trap 'exit 130' INT
|
|
323
387
|
# GUARD_ON is 1 only when the harness was started in its own process group (see the
|
|
@@ -334,6 +398,7 @@ if [ "$PREFLIGHT" != 1 ] && [ "$DRY" != 1 ]; then
|
|
|
334
398
|
python3 "$GUARD" reap "$hpid" "$RUN_DIR/$ROLE.orphans" >/dev/null 2>&1
|
|
335
399
|
fi
|
|
336
400
|
if [ -n "${wpid:-}" ]; then pkill -P "$wpid" 2>/dev/null; kill "$wpid" 2>/dev/null; fi
|
|
401
|
+
rm -f "$RUN_DIR/$ROLE.harness"
|
|
337
402
|
}
|
|
338
403
|
# What the EXIT trap's `finish` archives. Pessimistic until an outcome is known, so every
|
|
339
404
|
# path out of a continue request (a signal, a usage error after this point) leaves a record.
|
|
@@ -661,7 +726,7 @@ fi
|
|
|
661
726
|
# when a stale $LAST is already there. That is harmless when each run gets a fresh
|
|
662
727
|
# run-dir, and silently wrong for `forge-parallel.sh retry`, which reuses the task
|
|
663
728
|
# directory — it would read the first attempt's result and never notice.
|
|
664
|
-
rm -f "$LAST" "$LOG" "$RUN_DIR/$ROLE.timedout" "$RUN_DIR/$ROLE.infra" "$RUN_DIR/$ROLE.orphans"
|
|
729
|
+
rm -f "$LAST" "$LOG" "$RUN_DIR/$ROLE.timedout" "$RUN_DIR/$ROLE.infra" "$RUN_DIR/$ROLE.orphans" "$RUN_DIR/$ROLE.harness"
|
|
665
730
|
python3 "$RUNTIME" phase "$ATTEMPT" model
|
|
666
731
|
t0="$(date +%s)"
|
|
667
732
|
# Process-group guard. A dwarf that starts `npm test &` and ends its turn leaves that
|
|
@@ -683,6 +748,17 @@ else
|
|
|
683
748
|
( cd "$REPO" && exec "${RUNCMD[@]}" </dev/null ) > "$LOG" 2>&1 &
|
|
684
749
|
fi
|
|
685
750
|
hpid=$!
|
|
751
|
+
# The harness leads its own session, so a scheduler's signal to this dispatch's group never
|
|
752
|
+
# reaches it; only the EXIT trap below does. Record it for the scheduler's backstop
|
|
753
|
+
# (forge-guard.py stop) in case this dispatch dies before the trap finishes. The start time
|
|
754
|
+
# tells the pid apart from a later reuse, and survives the exec chain $! still goes through.
|
|
755
|
+
if [ "$GUARD_ON" = 1 ]; then
|
|
756
|
+
hident="$(python3 "$GUARD" identity "$hpid" 2>/dev/null)" || hident=""
|
|
757
|
+
if [ -n "$hident" ]; then
|
|
758
|
+
printf '%s\t%s\n' "$hpid" "$hident" > "$RUN_DIR/.$ROLE.harness.tmp" \
|
|
759
|
+
&& mv -f "$RUN_DIR/.$ROLE.harness.tmp" "$RUN_DIR/$ROLE.harness"
|
|
760
|
+
fi
|
|
761
|
+
fi
|
|
686
762
|
|
|
687
763
|
# Watchdog. Only antigravity has a time bound of its own, and `timeout` is not
|
|
688
764
|
# present on stock macOS, so without this a hung backend blocks forever — and under
|
|
@@ -725,6 +801,7 @@ if [ "$GUARD_ON" = 1 ]; then
|
|
|
725
801
|
case "$orphans_found" in ''|*[!0-9]*|0) ;; *) note "$HARNESS left $orphans_found process(es) running after it exited; killed — see $RUN_DIR/$ROLE.orphans" ;; esac
|
|
726
802
|
fi
|
|
727
803
|
reaped=1
|
|
804
|
+
rm -f "$RUN_DIR/$ROLE.harness"
|
|
728
805
|
python3 "$RUNTIME" phase "$ATTEMPT" finalization
|
|
729
806
|
DURATION=$(( $(date +%s) - t0 ))
|
|
730
807
|
echo "duration_s=$DURATION" >> "$RUN_DIR/$ROLE.resolved"
|