@giamme/forge 0.9.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/CHANGELOG.md +56 -0
  2. package/SKILL.md +22 -3
  3. package/VERSION +1 -1
  4. package/docs/artifacts.md +28 -2
  5. package/docs/commands.md +55 -0
  6. package/docs/dataset.md +22 -0
  7. package/docs/decomposed-runs.md +54 -2
  8. package/docs/forge-ui.md +48 -7
  9. package/docs/memory.md +4 -2
  10. package/docs/reviews.md +87 -2
  11. package/forge +31 -1
  12. package/package.json +1 -1
  13. package/references/dataset.md +103 -4
  14. package/references/decompose.md +41 -5
  15. package/references/harnesses.md +43 -1
  16. package/references/ui.md +31 -6
  17. package/scripts/forge-config.py +350 -0
  18. package/scripts/forge-control.py +649 -0
  19. package/scripts/forge-dispatch.sh +235 -121
  20. package/scripts/forge-issues.py +1134 -0
  21. package/scripts/forge-lib-plans.sh +6 -2
  22. package/scripts/forge-lib-review.sh +158 -3
  23. package/scripts/forge-lib-task.sh +358 -41
  24. package/scripts/forge-parallel-options.sh +3 -1
  25. package/scripts/forge-parallel.sh +99 -13
  26. package/scripts/forge-prompt.py +61 -4
  27. package/scripts/forge-route.py +245 -28
  28. package/scripts/forge-runs-rm.py +840 -0
  29. package/scripts/forge-runs.py +19 -1
  30. package/scripts/forge-runtime.py +47 -1
  31. package/scripts/forge-schedule.py +227 -11
  32. package/scripts/forge-ui.py +85 -19
  33. package/scripts/forge_dataset/__init__.py +84 -8
  34. package/scripts/forge_dataset/capture.py +5 -2
  35. package/scripts/forge_dataset/cli.py +44 -1
  36. package/scripts/forge_dataset/export.py +36 -3
  37. package/scripts/forge_dataset/forecast.py +355 -0
  38. package/scripts/forge_dataset/harvest.py +45 -2
  39. package/scripts/forge_dataset/importer.py +431 -17
  40. package/scripts/forge_dataset/reviewers.py +145 -0
  41. package/scripts/forge_dataset/routing.py +1 -0
  42. package/scripts/forge_jev/__init__.py +7 -3
  43. package/scripts/forge_registry.py +183 -26
  44. package/ui/README.md +41 -7
  45. package/ui/docs/api.md +259 -21
  46. package/ui/forge_ui/__main__.py +10 -2
  47. package/ui/forge_ui/control.py +587 -0
  48. package/ui/forge_ui/export.py +17 -3
  49. package/ui/forge_ui/reader.py +210 -12
  50. package/ui/forge_ui/server.py +125 -4
  51. package/ui/forge_ui/static/app.js +104 -18
  52. package/ui/forge_ui/static/cloud3d.js +4 -1
  53. package/ui/forge_ui/static/cloudmodel.js +1 -1
  54. package/ui/forge_ui/static/cloudui.js +13 -4
  55. package/ui/forge_ui/static/controls.js +619 -0
  56. package/ui/forge_ui/static/findingsview.js +438 -0
  57. package/ui/forge_ui/static/graph.js +6 -5
  58. package/ui/forge_ui/static/index.html +7 -1
  59. package/ui/forge_ui/static/leaderboard.js +18 -0
  60. package/ui/forge_ui/static/modelstats.js +19 -1
  61. package/ui/forge_ui/static/panel.js +76 -14
  62. package/ui/forge_ui/static/rundelete.js +388 -0
  63. package/ui/forge_ui/static/runmodel.js +128 -13
  64. package/ui/forge_ui/static/scene3d.js +32 -15
  65. package/ui/forge_ui/static/settings.js +399 -0
  66. package/ui/forge_ui/static/style.css +184 -5
  67. package/ui/forge_ui/static/uikit.js +161 -0
@@ -42,6 +42,21 @@ def _kind(path):
42
42
  return None
43
43
 
44
44
 
45
+ def _default_registry_dir():
46
+ """forge's run registry, resolved the way forge writes it (scripts/forge_registry.registry_dir).
47
+
48
+ ``XDG_CONFIG_HOME`` (``~`` expanded) when it is an absolute path, otherwise ``~/.config``: an
49
+ empty or relative value counts as unset (XDG spec), as it does when forge registers a run.
50
+ None only when even that is not absolute (a relative HOME): forge never registers a run
51
+ there, so reading it would only pick up whatever the current directory happens to hold.
52
+ """
53
+ value = os.environ.get('XDG_CONFIG_HOME') or ''
54
+ base = Path(value).expanduser() if value else None
55
+ if base is None or not base.is_absolute():
56
+ base = Path.home() / '.config'
57
+ return base / 'forge' / 'runs' if base.is_absolute() else None
58
+
59
+
45
60
  def discover(plans=(), roots=(), registry=True, registry_dir=None):
46
61
  found, seen = [], set()
47
62
 
@@ -76,10 +91,9 @@ def discover(plans=(), roots=(), registry=True, registry_dir=None):
76
91
  for path in roots:
77
92
  scan(Path(path))
78
93
  if registry:
79
- base = Path(registry_dir) if registry_dir is not None else Path(
80
- os.environ.get('XDG_CONFIG_HOME') or Path.home() / '.config') / 'forge' / 'runs'
94
+ base = Path(registry_dir) if registry_dir is not None else _default_registry_dir()
81
95
  try:
82
- entries = sorted(base.glob('*.txt'))
96
+ entries = sorted(base.glob('*.txt')) if base is not None else []
83
97
  except OSError:
84
98
  entries = []
85
99
  for entry in entries:
@@ -310,10 +324,14 @@ def _tasks(reader):
310
324
  reader.warn(label, 'invalid task id')
311
325
  continue
312
326
  prefix = f'tasks/{ident}'
313
- status = 'MERGED' if reader.exists(prefix + '/merged') else reader.word(prefix + '/status') or 'PENDING'
327
+ # One stat: the run list needs the mtime, and detail reads the same marker again.
328
+ merged = reader.stat(prefix + '/merged')
329
+ status = 'MERGED' if merged is not None else reader.word(prefix + '/status') or 'PENDING'
314
330
  tasks.append(dict(id=ident, title='\t'.join(row[6:]), deps=_list(deps),
315
331
  difficulty=difficulty, files=_list(files), dwarf=_nullable(dwarf),
316
- qa=_nullable(qa), status=status))
332
+ qa=_nullable(qa), status=status,
333
+ merged_at=float(merged.st_mtime) if merged is not None else None,
334
+ accepted=_nonempty(reader, prefix + '/accepted.by')))
317
335
  return tasks
318
336
 
319
337
 
@@ -353,12 +371,15 @@ def _summary(reader, source):
353
371
  final_review = reader.word('final-review.verdict')
354
372
  dwarf_models = sorted({task['dwarf'] for task in tasks if task['dwarf']})
355
373
  qa_models = sorted({task['qa'] for task in tasks if task['qa']})
374
+ # Before snapshotting `updated`: these reads are files the summary read, so a newer
375
+ # accepted.integration (or the published verdict / review.meta) moves the activity time.
376
+ final_review_outdated = _published_outdated(reader, tasks)
356
377
  result.update(updated=reader.updated, created=_created(reader.path, _PLAN_START_FILES),
357
378
  task_count=len(tasks), counts=counts,
358
379
  task_statuses=[dict(id=task['id'], status=task['status'], difficulty=task['difficulty'] or None,
359
- title=_brief(task['title'])) for task in tasks],
380
+ title=_brief(task['title']), accepted=task['accepted']) for task in tasks],
360
381
  models=sorted(set(dwarf_models) | set(qa_models)), dwarf_models=dwarf_models, qa_models=qa_models,
361
- final_review=final_review)
382
+ final_review=final_review, final_review_outdated=final_review_outdated)
362
383
  return result, tasks
363
384
 
364
385
 
@@ -389,9 +410,10 @@ def _solo_summary(reader, source):
389
410
  decompose_level=None, verification_status=None, updated=reader.updated,
390
411
  created=_created(reader.path, _SOLO_START_FILES),
391
412
  task_count=1, counts={status: 1},
392
- task_statuses=[dict(id='solo', status=status, difficulty=None, title=_brief(goal or 'Solo run'))],
413
+ task_statuses=[dict(id='solo', status=status, difficulty=None, title=_brief(goal or 'Solo run'),
414
+ accepted=False)],
393
415
  models=sorted({dwarf, qa} - {None}), dwarf_models=[dwarf] if dwarf else [],
394
- qa_models=[qa] if qa else [], final_review=None)
416
+ qa_models=[qa] if qa else [], final_review=None, final_review_outdated=None)
395
417
 
396
418
 
397
419
  def summarize(path, source='plan') -> dict:
@@ -413,7 +435,9 @@ def _history(reader, name, escalation=False):
413
435
  continue
414
436
  entry = dict(at=at, attempt=attempt)
415
437
  if escalation:
416
- entry.update({'from': row[2], 'to': row[3]})
438
+ # the 5th column says who decided (policy, a human's retry --dwarf, a pin); older rows have 4
439
+ source = row[4].strip() if len(row) > 4 else ''
440
+ entry.update({'from': row[2], 'to': row[3], 'source': source or 'policy'})
417
441
  else:
418
442
  entry.update(status=row[2], infra=_nullable(row[3]) if len(row) > 3 else None)
419
443
  result.append(entry)
@@ -500,6 +524,26 @@ def _shadows(reader, ident, task_execs):
500
524
  return result
501
525
 
502
526
 
527
+ def _accepted_fields(reader, name, text):
528
+ """(attempt, over) from an accepted.by body. A non-integer attempt= is null and warned.
529
+
530
+ status= FAIL or UNKNOWN is what was accepted over; any other or missing value is null
531
+ (a PASS record is an acceptance with nothing to say beyond "accepted").
532
+ """
533
+ if not text:
534
+ return None, None
535
+ fields = executions.parse_resolved(text)
536
+ attempt = None
537
+ raw = fields.get('attempt')
538
+ if raw:
539
+ try:
540
+ attempt = int(raw)
541
+ except (ValueError, OverflowError):
542
+ reader.warn(name, 'attempt is not an integer')
543
+ status = fields.get('status')
544
+ return attempt, status if status in ('FAIL', 'UNKNOWN') else None
545
+
546
+
503
547
  def _task_detail(reader, task, run_id, waves, branches):
504
548
  """Fill in a plan task; returns every execution it read (its own and its shadows')."""
505
549
  ident = task['id']
@@ -518,6 +562,8 @@ def _task_detail(reader, task, run_id, waves, branches):
518
562
  task['qa_tail'] = reader.cap(qa, prefix + '/qa_tail', 4000, True)
519
563
  task['dwarf_tail'] = reader.cap(reader.text(prefix + '/dwarf.last'), prefix + '/dwarf_tail', 4000, True)
520
564
  task['accepted_by'] = reader.text(prefix + '/accepted.by')
565
+ task['accepted_attempt'], task['accepted_over'] = _accepted_fields(
566
+ reader, prefix + '/accepted.by', task['accepted_by'])
521
567
  for field, filename in [('prompt', 'prompt.md'), ('approach', 'approach.md'), ('retry_findings', 'retry_findings.md')]:
522
568
  task[field] = reader.cap(reader.text(prefix + '/' + filename), prefix + '/' + field)
523
569
  task['route'] = []
@@ -548,9 +594,20 @@ def _task_detail(reader, task, run_id, waves, branches):
548
594
  task['fail_first'] = _fail_first(reader, prefix + '/failfirst.json')
549
595
  shadows = _shadows(reader, ident, execs)
550
596
  task['shadow'] = [entry for entry, _, _ in shadows]
597
+ # What `forge control show` reads of the task. A change there (a write, or a delete: the dir's mtime) moves `updated`,
598
+ # so the page redraws and the Controls panel asks again.
599
+ reader.stat(prefix)
600
+ for name in _TASK_CONTROLS:
601
+ reader.stat(f'{prefix}/{name}')
551
602
  return execs + [item for _, items, _ in shadows for item in items]
552
603
 
553
604
 
605
+ # The control files (CONTRACT.md) nothing else here reads. `forge control` appends control.log and writes the task's
606
+ # files without touching the run dir, so they are stat'd for `updated` only.
607
+ _RUN_CONTROLS = ('control.log', 'retry_budget', 'infra_budget')
608
+ _TASK_CONTROLS = ('qa_threshold', 'retry_budget', 'retries.used', 'escalation.pin', 'escalation.stop', 'escalation.next')
609
+
610
+
554
611
  _FAIL_FIRST_RESULTS = ('proven', 'passes_without_change', 'inconclusive', 'error')
555
612
 
556
613
 
@@ -593,6 +650,120 @@ def _published(reader):
593
650
  return real if executions.inside(reader.path, real) else None
594
651
 
595
652
 
653
+ def _review_clock(reader, name):
654
+ """(source, at) of a `final-review-<e>-<p>` or `integrate-<e>-<p>` dir name, else None.
655
+
656
+ `at` is the whole-second epoch in the name. The same clock `executions.review_dirs` uses,
657
+ so a summary does not have to scan the plan to know when the published review started.
658
+ """
659
+ match = executions._KIND.fullmatch(name)
660
+ if not match:
661
+ return None
662
+ kind, epoch, _pid = match.groups()
663
+ if len(epoch) > 15:
664
+ reader.warn(name, 'invalid epoch')
665
+ return None
666
+ try:
667
+ at = float(epoch)
668
+ if not math.isfinite(at):
669
+ raise ValueError()
670
+ except (ValueError, OverflowError):
671
+ reader.warn(name, 'invalid epoch')
672
+ return None
673
+ return ('review' if kind == 'final-review' else 'integrate'), at
674
+
675
+
676
+ def _published_identity(reader):
677
+ """(source, at) of the dir `final-review` links to, when the link stays inside the plan."""
678
+ real = _published(reader)
679
+ if real is None:
680
+ return None
681
+ path = Path(real)
682
+ # An integrate review is `<integrate-dir>/final-review`; a final-review dir is named itself.
683
+ if path.name == 'final-review':
684
+ found = _review_clock(reader, path.parent.name)
685
+ return found if found is not None and found[0] == 'integrate' else None
686
+ found = _review_clock(reader, path.name)
687
+ return found if found is not None and found[0] == 'review' else None
688
+
689
+
690
+ def _merge_pairs(tasks):
691
+ """(id, merged_at) of every merged task, oldest merge first, then id."""
692
+ pairs = [(task['id'], task['merged_at']) for task in tasks if task.get('merged_at') is not None]
693
+ pairs.sort(key=lambda item: (item[1], item[0]))
694
+ return pairs
695
+
696
+
697
+ # What `git rev-parse` writes for a commit: sha1 (40) or sha256 (64), lowercase hex.
698
+ _COMMIT = re.compile(r'(?:[0-9a-f]{40}|[0-9a-f]{64})')
699
+
700
+
701
+ def _meta_integration(reader, name, meta):
702
+ """The `integration=` commit of a review.meta, or None.
703
+
704
+ Missing or empty is absent (older runs record no integration commit). Any other value
705
+ that is not a commit object name is malformed: null, and a warning, so freshness uses
706
+ the next rule instead of treating the text as the commit the review judged.
707
+ """
708
+ value = meta.get('integration') or None
709
+ if not value:
710
+ return None
711
+ if _COMMIT.fullmatch(value):
712
+ return value
713
+ reader.warn(name, 'integration is not a commit')
714
+ return None
715
+
716
+
717
+ def _review_freshness(accepted, integration, tip, source, at, merges, verdict):
718
+ """(outdated, merged_since) for one final review.
719
+
720
+ Null when the review has no verdict or there is no `accepted.integration` to compare with.
721
+ Otherwise the first rule that applies: a known `integration` commit, else `tip` when
722
+ `source` is `review`, else (an older integrate review) any task merged after `at`.
723
+ `integration` is a commit object name or None (a malformed value is None). A merge counts
724
+ when its marker mtime is `>= at + 1`. `merged_since` is those ids, oldest first, and is
725
+ empty unless the review is outdated.
726
+ """
727
+ later = []
728
+ if at is not None:
729
+ later = [item for item in merges if item[1] is not None and item[1] >= at + 1]
730
+ later.sort(key=lambda item: (item[1], item[0]))
731
+ ids = [task_id for task_id, _merged_at in later]
732
+ if not verdict or not accepted:
733
+ return None, []
734
+ if integration:
735
+ moved = integration != accepted
736
+ elif source == 'review':
737
+ moved = tip != accepted
738
+ else:
739
+ moved = bool(ids)
740
+ return (True, ids) if moved else (False, [])
741
+
742
+
743
+ def _stamp_review(summary, accepted, merges):
744
+ summary['outdated'], summary['merged_since'] = _review_freshness(
745
+ accepted, summary['integration'], summary['tip'], summary['source'], summary['at'], merges,
746
+ summary['verdict'])
747
+ return summary
748
+
749
+
750
+ def _published_outdated(reader, tasks):
751
+ """final_review_outdated for the published review only.
752
+
753
+ Reads `accepted.integration` and, through the `final-review` link, that dir's `verdict`
754
+ and `review.meta`. Does not scan other review dirs or open a review's attempts.
755
+ """
756
+ found = _published_identity(reader)
757
+ if found is None:
758
+ return None
759
+ source, at = found
760
+ verdict = reader.word('final-review/verdict')
761
+ meta = executions.parse_resolved(reader.text('final-review/review.meta'))
762
+ return _review_freshness(reader.line('accepted.integration'),
763
+ _meta_integration(reader, 'final-review/review.meta', meta),
764
+ meta.get('tip') or None, source, at, _merge_pairs(tasks), verdict)[0]
765
+
766
+
596
767
  def _final_review(reader, item, newest, published):
597
768
  """(FinalReviewSummary, its executions, review.meta) of one review dir."""
598
769
  name, source, at, rel = item
@@ -617,6 +788,8 @@ def _final_review(reader, item, newest, published):
617
788
  verdict=verdict, state=state, spec=meta.get('spec') or None,
618
789
  threshold=None if threshold == 'none' else threshold,
619
790
  base=meta.get('base') or None, tip=meta.get('tip') or None,
791
+ integration=_meta_integration(reader, rel + '/review.meta', meta),
792
+ outdated=None, merged_since=[],
620
793
  gate=_gate(reader, rel + '/qa.gate'), findings=executions.qa_result(reader, rel + '/qa.last')[1],
621
794
  execution=run['id'] if run else None, started=run['started'] if run else None,
622
795
  ended=run['ended'] if run else None, duration_s=run['duration_s'] if run else None)
@@ -688,6 +861,7 @@ def _solo_detail(reader, source):
688
861
  dwarf=_role_spec(reader, 'dwarf'), qa=_role_spec(reader, 'qa'), wave=1, status=status,
689
862
  branch=None, attempt=max((item['attempt'] for item in attempts), default=None),
690
863
  attempts=attempts, escalations=[], verdict=_verdict(qa), accepted_by=None,
864
+ accepted=False, accepted_attempt=None, accepted_over=None,
691
865
  explored=False, route=[], clef={'first_pass': None, 'retry_pass': {}}, shadow=[],
692
866
  prompt=reader.cap(reader.text('prompt.md'), 'tasks/solo/prompt'), approach=None,
693
867
  retry_findings=None, qa_tail=reader.cap(qa, 'tasks/solo/qa_tail', 4000, True),
@@ -741,6 +915,8 @@ def detail(path, source='plan') -> dict:
741
915
  pool = []
742
916
  for task in tasks:
743
917
  pool += _task_detail(reader, task, result['run_id'], wave_map, branches)
918
+ for name in _RUN_CONTROLS:
919
+ reader.stat(name)
744
920
  run_execs = _run_execs(reader)
745
921
  items, integrations = executions.review_dirs(reader)
746
922
  published = _published(reader)
@@ -752,6 +928,10 @@ def detail(path, source='plan') -> dict:
752
928
  result['timeline'] = _timeline(pool + run_execs, [item for item in run_execs if item['role'] == 'parallel'],
753
929
  [item for item in run_execs if item['role'] == 'verification'])
754
930
  result['integration'] = _integration(reader, tasks, integrations, reviews, published)
931
+ accepted = result['integration']['accepted'] if result['integration'] is not None else None
932
+ merges = [(item['id'], item['at']) for item in result['integration']['merged']] if result['integration'] else []
933
+ for summary in reviews:
934
+ _stamp_review(summary, accepted, merges)
755
935
  result['final_review'] = reviews[0] if reviews else None
756
936
  result['final_reviews'] = reviews
757
937
  result['deferrals'] = [line.strip() for line in (reader.text('deferrals.txt') or '').splitlines() if line.strip()]
@@ -928,6 +1108,7 @@ def final_review_detail(path, name, source='plan'):
928
1108
  if index is None:
929
1109
  return None
930
1110
  summary, _, meta = _final_review(reader, items[index], index == 0, _published(reader))
1111
+ _stamp_review(summary, reader.line('accepted.integration'), _merge_pairs(_tasks(reader)))
931
1112
  rel = items[index][3]
932
1113
  qa_last = reader.text(rel + '/qa.last')
933
1114
  result = dict(summary)
@@ -1200,8 +1381,13 @@ def run_findings(path, source='plan'):
1200
1381
  if kind is None:
1201
1382
  return None
1202
1383
  keys = ('kind', 'execution', 'attempt', 'spec', 'verdict')
1384
+ if kind == 'solo':
1385
+ task_ids, merges, accepted = ['solo'], [], None
1386
+ else:
1387
+ rows = _tasks(reader)
1388
+ task_ids, merges, accepted = [task['id'] for task in rows], _merge_pairs(rows), reader.line('accepted.integration')
1203
1389
  tasks = []
1204
- for task_id in ['solo'] if kind == 'solo' else [task['id'] for task in _tasks(reader)]:
1390
+ for task_id in task_ids:
1205
1391
  ctx = _diff_task(reader, task_id)
1206
1392
  reviews = [dict({key: review[key] for key in keys}, findings=found)
1207
1393
  for review in (ctx['reviews'] if ctx else []) for found in [_findings_of(reader, review)] if found]
@@ -1212,13 +1398,15 @@ def run_findings(path, source='plan'):
1212
1398
  items, published = executions.review_dirs(reader)[0], _published(reader)
1213
1399
  for index, item in enumerate(items[:_MAX_REVIEWS]):
1214
1400
  summary = _final_review(reader, item, index == 0, published)[0]
1401
+ _stamp_review(summary, accepted, merges)
1215
1402
  # the same review final_review_diff pins, so a Finding's id opens it there
1216
1403
  review = _diff_review('final', None, None, summary['spec'], summary['verdict'], item[3] + '/qa.last', [],
1217
1404
  summary['execution'])
1218
1405
  found = _findings_of(reader, review)
1219
1406
  if found:
1220
1407
  finals.append(dict(name=item[0], execution=summary['execution'], spec=summary['spec'],
1221
- verdict=summary['verdict'], findings=found))
1408
+ verdict=summary['verdict'], outdated=summary['outdated'],
1409
+ merged_since=summary['merged_since'], findings=found))
1222
1410
  return dict(run=run_key(reader.path), available=findings.available(), tasks=tasks, final_reviews=finals,
1223
1411
  truncated=reader.truncated, warnings=reader.warnings)
1224
1412
 
@@ -1380,6 +1568,16 @@ class Catalog:
1380
1568
  return result if _kind(path) else None
1381
1569
  return None
1382
1570
 
1571
+ def locate(self, key):
1572
+ """(path, kind) of the run with this key, kind 'plan' or 'solo'; None for an unknown key.
1573
+
1574
+ The only way a write finds a run dir: the client names a key, never a path."""
1575
+ for path, _source in discover(**self.options):
1576
+ if run_key(path) == key:
1577
+ kind = _kind(path)
1578
+ return (str(path), kind) if kind else None
1579
+ return None
1580
+
1383
1581
  def get(self, key):
1384
1582
  return self._find(key, detail)
1385
1583
 
@@ -1,26 +1,38 @@
1
- """Local HTTP server: the JSON API of docs/api.md and the static page. Read-only."""
1
+ """Local HTTP server: the JSON API of docs/api.md and the static page.
2
+
3
+ Reads are open to the local page. Writes (POST) pass the checks of `_write_response` and then run one
4
+ forge CLI call through control.py; the server itself never writes a run dir, a repo or config.
5
+ """
2
6
  from __future__ import annotations
3
7
 
4
8
  import base64
5
9
  from contextlib import contextmanager
6
10
  import hashlib
11
+ import hmac
7
12
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
8
13
  import json
9
14
  import os
10
15
  from pathlib import Path
11
16
  import re
17
+ import secrets
12
18
  import sys
13
19
  import threading
14
20
  import time
15
21
  import urllib.parse
16
22
 
17
- from forge_ui import __version__, build_id
23
+ from forge_ui import __version__, build_id, control
18
24
 
19
25
 
20
26
  STATIC_DIR = Path(__file__).resolve().parent / 'static'
21
27
  # The build of the code this process loaded. Files edited later must not change it: the launcher
22
28
  # compares /api/health with its own build_id() to find a server still running old code.
23
29
  BUILD = build_id()
30
+ # The forge command every write calls. A test points it at a fake that records its argv.
31
+ forge_cli = Path(__file__).resolve().parents[2] / 'forge'
32
+ MAX_BODY = 64 * 1024
33
+ DRAIN_WAIT = 1.0
34
+ DRAIN_MAX = 1024 * 1024 # an unread request body is read (to here) before a refusal, so the client sees the answer
35
+ TOKEN_HEADER = 'X-Forge-UI-Token'
24
36
 
25
37
  _RUN_PATH = re.compile(r'/api/runs/([0-9a-f]{12})')
26
38
  _TASK_PATH = re.compile(r'/api/runs/([0-9a-f]{12})/tasks/([^/]+)')
@@ -50,6 +62,8 @@ _CSP = ("default-src 'self'; style-src 'self'; script-src 'self'; img-src 'self'
50
62
  _PAGE_CSP = ("default-src 'self'; style-src 'self'; script-src 'self' https://cdn.jsdelivr.net/npm/three@0.160.0/{}; "
51
63
  "img-src 'self' data:; connect-src 'self'; frame-ancestors 'none'; base-uri 'none'; object-src 'none'")
52
64
  _IMPORTMAP = re.compile(r'<script type="importmap">(.*?)</script>', re.DOTALL | re.IGNORECASE)
65
+ _HEAD_TAG = re.compile(rb'<head(?:\s[^>]*)?>', re.IGNORECASE)
66
+ _DOCTYPE = re.compile(rb'<!doctype[^>]*>', re.IGNORECASE)
53
67
 
54
68
 
55
69
  def _log(message):
@@ -65,6 +79,15 @@ def page_csp(body):
65
79
  return _PAGE_CSP.format(hashes)
66
80
 
67
81
 
82
+ def with_token(body, token):
83
+ """index.html with `<meta name="forge-ui-token">` as the first thing in its <head>."""
84
+ meta = b'<meta name="forge-ui-token" content="%s">' % token.encode('ascii')
85
+ # Without a <head> tag the meta goes after the doctype (before it the page would render in quirks mode).
86
+ found = _HEAD_TAG.search(body) or _DOCTYPE.search(body)
87
+ end = found.end() if found else 0
88
+ return body[:end] + meta + body[end:]
89
+
90
+
68
91
  def _task_id(segment):
69
92
  """A task id from its URL segment, or None when it cannot name a task dir."""
70
93
  try:
@@ -140,6 +163,14 @@ class Handler(BaseHTTPRequestHandler):
140
163
  def do_HEAD(self):
141
164
  self._serve(head=True)
142
165
 
166
+ def do_POST(self):
167
+ with self._active():
168
+ self._send(*self._write_response())
169
+
170
+ def do_OPTIONS(self):
171
+ # No CORS: a preflight is never answered, so no other origin can learn that a write is allowed.
172
+ self._method_not_allowed()
173
+
143
174
  def __getattr__(self, name):
144
175
  # Every other method (POST, PUT, a made-up verb, ...) gets a 405, not the 501 of the base class.
145
176
  if name.startswith('do_'):
@@ -189,7 +220,10 @@ class Handler(BaseHTTPRequestHandler):
189
220
  if not self._host_allowed():
190
221
  self._send(*self._json(403, {'error': 'forbidden'}))
191
222
  return
192
- self._send(*self._json(405, {'error': 'method not allowed'}, {'Allow': 'GET, HEAD'}))
223
+ allow = 'GET, HEAD'
224
+ if not self.server.read_only and control.has_route('POST', self.path.partition('?')[0]):
225
+ allow += ', POST'
226
+ self._send(*self._json(405, {'error': 'method not allowed'}, {'Allow': allow}))
193
227
 
194
228
  @contextmanager
195
229
  def _active(self):
@@ -231,6 +265,7 @@ class Handler(BaseHTTPRequestHandler):
231
265
  'version': __version__,
232
266
  'build': BUILD,
233
267
  'pid': os.getpid(),
268
+ 'controls': not self.server.read_only,
234
269
  })
235
270
  catalog = self.server.catalog
236
271
  match = _EXPORT_PATH.fullmatch(path)
@@ -239,6 +274,9 @@ class Handler(BaseHTTPRequestHandler):
239
274
  value = api_get(catalog, path)
240
275
  if value is not NO_ROUTE:
241
276
  return self._found(value)
277
+ routed = control.dispatch(self.server, forge_cli, 'GET', path, None, query=self.query)
278
+ if routed:
279
+ return self._json(routed[0], routed[1])
242
280
  if path == '/':
243
281
  return self._static('index.html')
244
282
  match = _STATIC_PATH.fullmatch(path)
@@ -260,8 +298,88 @@ class Handler(BaseHTTPRequestHandler):
260
298
  if body is None:
261
299
  return self._json(404, {'error': 'not found'})
262
300
  kind = name.rsplit('.', 1)[1]
301
+ if name == 'index.html' and not self.server.read_only:
302
+ body = with_token(body, self.server.token)
263
303
  return 200, _TYPES[kind], body, {'Content-Security-Policy': page_csp(body)} if kind == 'html' else {}
264
304
 
305
+ # ------------------------------------------------------------------------------ writes
306
+
307
+ def _drain(self):
308
+ """Read (and drop) the request body that is already on its way, up to DRAIN_MAX, so a refusal is not
309
+ lost to a connection reset. A client that claims more than it sends costs DRAIN_WAIT, not the timeout."""
310
+ self.close_connection = True
311
+ try:
312
+ left = min(int(self.headers.get('Content-Length') or 0), DRAIN_MAX)
313
+ self.connection.settimeout(DRAIN_WAIT)
314
+ while left > 0:
315
+ chunk = self.rfile.read1(min(left, 65536))
316
+ if not chunk:
317
+ break
318
+ left -= len(chunk)
319
+ except (OSError, ValueError):
320
+ pass
321
+
322
+ def _refuse(self, status, error):
323
+ self._drain()
324
+ return self._json(status, {'error': error})
325
+
326
+ def _body(self):
327
+ """(object, None) for a valid JSON object body, else (None, refusal response)."""
328
+ if self.headers.get('Transfer-Encoding') is not None:
329
+ return None, self._refuse(400, 'bad request')
330
+ length = (self.headers.get('Content-Length') or '').strip()
331
+ if not length.isascii() or not length.isdigit():
332
+ return None, self._refuse(400, 'bad request')
333
+ if int(length) > MAX_BODY:
334
+ return None, self._refuse(413, 'payload too large')
335
+ try:
336
+ raw = self.rfile.read(int(length))
337
+ if len(raw) != int(length):
338
+ raise ValueError('short body')
339
+ body = json.loads(raw.decode('utf-8'))
340
+ except (OSError, ValueError, RecursionError): # not UTF-8, not JSON, or the client hung up
341
+ self.close_connection = True
342
+ return None, self._json(400, {'error': 'bad request'})
343
+ if not isinstance(body, dict):
344
+ return None, self._json(400, {'error': 'bad request'})
345
+ return body, None
346
+
347
+ def _write_response(self):
348
+ """The answer to a POST. A request passes every check, in this order, before any route sees it:
349
+ the Host, read-only mode (405), the token, the content type, the Origin and the body."""
350
+ server = self.server
351
+ path = self.path.partition('?')[0]
352
+ if not self._host_allowed():
353
+ return self._refuse(403, 'forbidden')
354
+ if server.read_only:
355
+ self._drain()
356
+ return self._json(405, {'error': 'method not allowed'}, {'Allow': 'GET, HEAD'})
357
+ # compare_digest takes bytes here: a non-ASCII header value must not raise.
358
+ sent = (self.headers.get(TOKEN_HEADER) or '').encode('utf-8', 'surrogateescape')
359
+ if not hmac.compare_digest(sent, server.token.encode('ascii')):
360
+ return self._refuse(403, 'forbidden')
361
+ kind = (self.headers.get('Content-Type') or '').partition(';')[0].strip().lower()
362
+ if kind != 'application/json':
363
+ return self._refuse(400, 'content type must be application/json')
364
+ origin = self.headers.get('Origin')
365
+ if origin is not None and origin != 'http://' + (self.headers.get('Host') or ''):
366
+ return self._refuse(403, 'forbidden')
367
+ body, refusal = self._body()
368
+ if refusal:
369
+ return refusal
370
+ try:
371
+ routed = control.dispatch(server, forge_cli, 'POST', path, body)
372
+ except Exception as exc: # one bad request must not kill the server
373
+ detail = ' '.join(str(exc).split())
374
+ _log(f'internal error on POST {path}: {type(exc).__name__}: {detail}')
375
+ return self._json(500, {'error': 'internal error'})
376
+ if routed is None:
377
+ return self._json(405, {'error': 'method not allowed'}, {'Allow': 'GET, HEAD'})
378
+ status, obj, route, key = routed
379
+ if not self._quiet():
380
+ _log(f'control {route} {key} -> {status}')
381
+ return self._json(status, obj)
382
+
265
383
  def _export(self, catalog, key):
266
384
  # One run as one HTML file, to download: never rendered from this origin (sandbox, no sources).
267
385
  from forge_ui import export
@@ -301,9 +419,12 @@ class Handler(BaseHTTPRequestHandler):
301
419
  pass # the client hung up (BrokenPipeError, ConnectionResetError, ...): nothing to tell it
302
420
 
303
421
 
304
- def make_server(catalog, host='127.0.0.1', port=0, static_dir=None, quiet=False):
422
+ def make_server(catalog, host='127.0.0.1', port=0, static_dir=None, quiet=False, read_only=False):
305
423
  server = ThreadingHTTPServer((host, port), Handler)
306
424
  server.catalog = catalog
425
+ server.read_only = read_only
426
+ server.token = secrets.token_urlsafe(32) # per process; only the page this process serves carries it
427
+ server.run_locks = control.RunLocks()
307
428
  server.static_dir = STATIC_DIR if static_dir is None else static_dir
308
429
  server.quiet = quiet
309
430
  server.last_request = time.monotonic() # stamped when a request arrives and when it ends