@giamme/forge 0.8.0 → 0.10.0

package/ui/forge_ui/fractal.py

@@ -47,8 +47,7 @@ def supported():
 
 
 def state_root():
-    """forge_fractal.state_root(), except that an empty XDG_STATE_HOME counts as unset (forge
-    resolves '' to its working directory)."""
+    """forge_fractal.state_root(): an empty XDG_STATE_HOME counts as unset."""
     base = os.environ.get('XDG_STATE_HOME') or os.path.join(os.path.expanduser('~'), '.local', 'state')
     return os.path.join(os.path.realpath(os.path.expanduser(base)), 'forge', 'fractal')
 

package/ui/forge_ui/reader.py

@@ -11,7 +11,7 @@ import re
 import stat
 import urllib.parse
 
-from forge_ui import diffs, executions, findings, fractal
+from forge_ui import diffs, executions, findings, fractal, transcripts
 from forge_ui.findings import verdict as _verdict
 
 
@@ -413,7 +413,9 @@ def _history(reader, name, escalation=False):
             continue
         entry = dict(at=at, attempt=attempt)
         if escalation:
-            entry.update({'from': row[2], 'to': row[3]})
+            # the 5th column says who decided (policy, a human's retry --dwarf, a pin); older rows have 4
+            source = row[4].strip() if len(row) > 4 else ''
+            entry.update({'from': row[2], 'to': row[3], 'source': source or 'policy'})
         else:
             entry.update(status=row[2], infra=_nullable(row[3]) if len(row) > 3 else None)
         result.append(entry)
@@ -548,9 +550,20 @@ def _task_detail(reader, task, run_id, waves, branches):
     task['fail_first'] = _fail_first(reader, prefix + '/failfirst.json')
     shadows = _shadows(reader, ident, execs)
     task['shadow'] = [entry for entry, _, _ in shadows]
+    # What `forge control show` reads of the task. A change there (a write, or a delete: the dir's mtime) moves `updated`,
+    # so the page redraws and the Controls panel asks again.
+    reader.stat(prefix)
+    for name in _TASK_CONTROLS:
+        reader.stat(f'{prefix}/{name}')
     return execs + [item for _, items, _ in shadows for item in items]
 
 
+# The control files (CONTRACT.md) nothing else here reads. `forge control` appends control.log and writes the task's
+# files without touching the run dir, so they are stat'd for `updated` only.
+_RUN_CONTROLS = ('control.log', 'retry_budget', 'infra_budget')
+_TASK_CONTROLS = ('qa_threshold', 'retry_budget', 'retries.used', 'escalation.pin', 'escalation.stop', 'escalation.next')
+
+
 _FAIL_FIRST_RESULTS = ('proven', 'passes_without_change', 'inconclusive', 'error')
 
 
@@ -741,6 +754,8 @@ def detail(path, source='plan') -> dict:
     pool = []
     for task in tasks:
         pool += _task_detail(reader, task, result['run_id'], wave_map, branches)
+    for name in _RUN_CONTROLS:
+        reader.stat(name)
     run_execs = _run_execs(reader)
     items, integrations = executions.review_dirs(reader)
     published = _published(reader)
@@ -824,9 +839,8 @@ def task_detail(path, task_id, source='plan'):
                 reviews=reviews, truncated=reader.truncated, warnings=reader.warnings)
 
 
-def execution_detail(path, name, source='plan'):
-    """ExecutionDetail of one attempts/<role>-<uuid>/ dir of the run, or None."""
-    reader = _Read(path)
+def _execution(reader, name):
+    """(Execution, owner, owner dir) of the execution `name` of the run, or None."""
     kind = _kind(reader.path)
     if kind is None or not executions.NAME.fullmatch(name or ''):
         return None
@@ -856,8 +870,16 @@ def execution_detail(path, name, source='plan'):
         execs = _final_review(reader, items[index], index == 0, _published(reader))[1]
         owner, owner_dir = dict(kind='final_review', id=parts[0]), items[index][3]
     item = next((candidate for candidate in execs if candidate['id'] == name), None)
-    if item is None:
+    return None if item is None else (item, owner, owner_dir)
+
+
+def execution_detail(path, name, source='plan'):
+    """ExecutionDetail of one attempts/<role>-<uuid>/ dir of the run, or None."""
+    reader = _Read(path)
+    found = _execution(reader, name)
+    if found is None:
         return None
+    item, owner, owner_dir = found
     role, running = item['role'], item['state'] == 'running'
     resolved = executions.resolved_of(reader, f'{item.rel}/{role}.resolved')
     prompt = last = listed = None
@@ -880,6 +902,37 @@ def execution_detail(path, name, source='plan'):
     return result
 
 
+def execution_transcript(path, name, source='plan'):
+    """ExecutionTranscript of a dwarf or qa execution: its agent's conversation, read from its `.log`; None
+    when there is no such execution or it is of another role."""
+    reader = _Read(path)
+    found = _execution(reader, name)
+    if found is None or found[0]['role'] not in ('dwarf', 'qa'):
+        return None
+    item, owner, owner_dir = found
+    role, running = item['role'], item['state'] == 'running'
+    # A finished execution's log is its copy; the owner dir's is the one being written, or the next one's.
+    rel = posixpath.join(owner_dir, role + '.log') if running else f'{item.rel}/{role}.log'
+    log = None
+    try:
+        real = Path(os.path.realpath(reader.path / rel))
+        if executions.inside(reader.path, str(real)) and stat.S_ISREG(real.stat().st_mode):
+            log = real
+        elif real.exists():
+            reader.warn(rel, 'outside the run dir or not a regular file')
+    except (OSError, ValueError):
+        pass
+    resolved = executions.resolved_of(reader, f'{item.rel}/{role}.resolved')
+    if resolved is None and running:
+        resolved = executions.parse_resolved(reader.text(posixpath.join(owner_dir, role + '.resolved'))) or None
+    model = item['model'] or {}
+    result = transcripts.transcript(log, model.get('harness'), running)
+    result.update(run=run_key(reader.path), execution=name, role=role, state=item['state'], attempt=item['attempt'],
+                  spec=model.get('spec'), harness=model.get('harness'), repo=(resolved or {}).get('repo') or None,
+                  owner=owner, warnings=reader.warnings + result['warnings'])
+    return result
+
+
 def final_review_detail(path, name, source='plan'):
     """FinalReviewDetail of final-review-<e>-<p> or integrate-<e>-<p>, or None."""
     reader = _Read(path)
@@ -1028,8 +1081,8 @@ def _diff_detail(reader, key, origin, sources, loaded, reason, reviews):
     found = []
     for review in reviews:
         text = reader.text(review['last']) if review['last'] else None
-        for item in findings.listing(text) or []:
-            found.append(dict(review={key: review[key] for key in ('kind', 'execution', 'attempt', 'spec')},
+        for index, item in enumerate(findings.listing(text) or []):
+            found.append(dict(id=_finding_id(review, index), review={key: review[key] for key in ('kind', 'execution', 'attempt', 'spec')},
                               severity=item['severity'], status=item['status'], location=item['location'],
                               summary=item['summary'], text=item['text']))
     return dict(run=key, source=origin, sources=sources, available=loaded is not None,
@@ -1141,6 +1194,50 @@ def final_review_diff(path, name, source='plan'):
                         _NOT_INLINED if prompted else 'no review prompt yet', [review])
 
 
+# --- findings: every review's findings of a run, for the stage's marks ---------------------
+
+def _finding_id(review, index):
+    """A Finding's id: its review's execution (its kind when it has none) and its place in that reply."""
+    return f"{review['execution'] or review['kind']}:{index}"
+
+
+def _findings_of(reader, review):
+    listed = findings.listing(reader.text(review['last'])) if review['last'] else None
+    return [dict(id=_finding_id(review, index), severity=item['severity'], status=item['status'],
+                 location=item['location'], summary=item['summary'])
+            for index, item in enumerate(listed or [])]
+
+
+def run_findings(path, source='plan'):
+    """RunFindings: each review's Findings, without their text, by task and by final review; None if no run."""
+    reader = _Read(path)
+    kind = _kind(reader.path)
+    if kind is None:
+        return None
+    keys = ('kind', 'execution', 'attempt', 'spec', 'verdict')
+    tasks = []
+    for task_id in ['solo'] if kind == 'solo' else [task['id'] for task in _tasks(reader)]:
+        ctx = _diff_task(reader, task_id)
+        reviews = [dict({key: review[key] for key in keys}, findings=found)
+                   for review in (ctx['reviews'] if ctx else []) for found in [_findings_of(reader, review)] if found]
+        if reviews:
+            tasks.append(dict(id=task_id, reviews=reviews))
+    finals = []
+    if kind == 'plan':
+        items, published = executions.review_dirs(reader)[0], _published(reader)
+        for index, item in enumerate(items[:_MAX_REVIEWS]):
+            summary = _final_review(reader, item, index == 0, published)[0]
+            # the same review final_review_diff pins, so a Finding's id opens it there
+            review = _diff_review('final', None, None, summary['spec'], summary['verdict'], item[3] + '/qa.last', [],
+                                  summary['execution'])
+            found = _findings_of(reader, review)
+            if found:
+                finals.append(dict(name=item[0], execution=summary['execution'], spec=summary['spec'],
+                                   verdict=summary['verdict'], findings=found))
+    return dict(run=run_key(reader.path), available=findings.available(), tasks=tasks, final_reviews=finals,
+                truncated=reader.truncated, warnings=reader.warnings)
+
+
 # --- files: who planned, changed, drifted into or conflicted on each file --------------------
 
 _MAX_FILES, _MAX_TASKS, _MAX_OVERLAPS = 2000, 300, 5000
@@ -1298,6 +1395,16 @@ class Catalog:
                 return result if _kind(path) else None
         return None
 
+    def locate(self, key):
+        """(path, kind) of the run with this key, kind 'plan' or 'solo'; None for an unknown key.
+
+        The only way a write finds a run dir: the client names a key, never a path."""
+        for path, _source in discover(**self.options):
+            if run_key(path) == key:
+                kind = _kind(path)
+                return (str(path), kind) if kind else None
+        return None
+
     def get(self, key):
         return self._find(key, detail)
 
@@ -1316,12 +1423,18 @@ class Catalog:
     def execution_diff(self, key, name):
         return self._find(key, execution_diff, name)
 
+    def transcript(self, key, name):
+        return self._find(key, execution_transcript, name)
+
     def final_review_diff(self, key, name):
         return self._find(key, final_review_diff, name)
 
     def files(self, key):
         return self._find(key, files_detail)
 
+    def findings(self, key):
+        return self._find(key, run_findings)
+
     def fractal(self, key, task_id):
         return self._find(key, fractal_detail, task_id)
 

package/ui/forge_ui/server.py

@@ -1,26 +1,38 @@
-"""Local HTTP server: the JSON API of docs/api.md and the static page. Read-only."""
+"""Local HTTP server: the JSON API of docs/api.md and the static page.
+
+Reads are open to the local page. Writes (POST) pass the checks of `_write_response` and then run one
+forge CLI call through control.py; the server itself never writes a run dir, a repo or config.
+"""
 from __future__ import annotations
 
 import base64
 from contextlib import contextmanager
 import hashlib
+import hmac
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 import json
 import os
 from pathlib import Path
 import re
+import secrets
 import sys
 import threading
 import time
 import urllib.parse
 
-from forge_ui import __version__, build_id
+from forge_ui import __version__, build_id, control
 
 
 STATIC_DIR = Path(__file__).resolve().parent / 'static'
 # The build of the code this process loaded. Files edited later must not change it: the launcher
 # compares /api/health with its own build_id() to find a server still running old code.
 BUILD = build_id()
+# The forge command every write calls. A test points it at a fake that records its argv.
+forge_cli = Path(__file__).resolve().parents[2] / 'forge'
+MAX_BODY = 64 * 1024
+DRAIN_WAIT = 1.0
+DRAIN_MAX = 1024 * 1024  # an unread request body is read (to here) before a refusal, so the client sees the answer
+TOKEN_HEADER = 'X-Forge-UI-Token'
 
 _RUN_PATH = re.compile(r'/api/runs/([0-9a-f]{12})')
 _TASK_PATH = re.compile(r'/api/runs/([0-9a-f]{12})/tasks/([^/]+)')
@@ -28,16 +40,19 @@ _EXEC_PATH = re.compile(r'/api/runs/([0-9a-f]{12})/executions/((?:task|dwarf|qa|
 _REVIEW_PATH = re.compile(r'/api/runs/([0-9a-f]{12})/final-reviews/((?:final-review|integrate)-[0-9]+-[0-9]+)')
 _TASK_DIFF_PATH = re.compile(_TASK_PATH.pattern + r'/diff(?:/(current|prev|committed|existing))?')
 _EXEC_DIFF_PATH = re.compile(_EXEC_PATH.pattern + '/diff')
+_TRANSCRIPT_PATH = re.compile(_EXEC_PATH.pattern + '/transcript')
 _REVIEW_DIFF_PATH = re.compile(_REVIEW_PATH.pattern + '/diff')
 _FILES_PATH = re.compile(r'/api/runs/([0-9a-f]{12})/files')
+_FINDINGS_PATH = re.compile(r'/api/runs/([0-9a-f]{12})/findings')
 _EXPORT_PATH = re.compile(r'/api/runs/([0-9a-f]{12})/export')
 _FRACTAL_PATH = re.compile(_TASK_PATH.pattern + r'/fractal(?:/nodes/([A-Za-z0-9][A-Za-z0-9_-]{0,100})(/diff)?)?')
-_STATIC_PATH = re.compile(r'/([A-Za-z0-9_.-]+\.(?:html|js|css|svg))')
+_STATIC_PATH = re.compile(r'/([A-Za-z0-9_.-]+\.(?:html|js|css|svg|woff2))')
 _TYPES = {
     'html': 'text/html; charset=utf-8',
     'js': 'text/javascript; charset=utf-8',
     'css': 'text/css; charset=utf-8',
     'svg': 'image/svg+xml',
+    'woff2': 'font/woff2',
 }
 _JSON = 'application/json; charset=utf-8'
 _CSP = ("default-src 'self'; style-src 'self'; script-src 'self'; img-src 'self' data:; "
@@ -47,6 +62,8 @@ _CSP = ("default-src 'self'; style-src 'self'; script-src 'self'; img-src 'self'
 _PAGE_CSP = ("default-src 'self'; style-src 'self'; script-src 'self' https://cdn.jsdelivr.net/npm/three@0.160.0/{}; "
              "img-src 'self' data:; connect-src 'self'; frame-ancestors 'none'; base-uri 'none'; object-src 'none'")
 _IMPORTMAP = re.compile(r'<script type="importmap">(.*?)</script>', re.DOTALL | re.IGNORECASE)
+_HEAD_TAG = re.compile(rb'<head(?:\s[^>]*)?>', re.IGNORECASE)
+_DOCTYPE = re.compile(rb'<!doctype[^>]*>', re.IGNORECASE)
 
 
 def _log(message):
@@ -62,6 +79,15 @@ def page_csp(body):
     return _PAGE_CSP.format(hashes)
 
 
+def with_token(body, token):
+    """index.html with `<meta name="forge-ui-token">` as the first thing in its <head>."""
+    meta = b'<meta name="forge-ui-token" content="%s">' % token.encode('ascii')
+    # Without a <head> tag the meta goes after the doctype (before it the page would render in quirks mode).
+    found = _HEAD_TAG.search(body) or _DOCTYPE.search(body)
+    end = found.end() if found else 0
+    return body[:end] + meta + body[end:]
+
+
 def _task_id(segment):
     """A task id from its URL segment, or None when it cannot name a task dir."""
     try:
@@ -106,9 +132,15 @@ def api_get(catalog, path):
     match = _REVIEW_DIFF_PATH.fullmatch(path)
     if match:
         return catalog.final_review_diff(*match.groups())
+    match = _TRANSCRIPT_PATH.fullmatch(path)
+    if match:
+        return catalog.transcript(*match.groups())
     match = _FILES_PATH.fullmatch(path)
     if match:
         return catalog.files(match.group(1))
+    match = _FINDINGS_PATH.fullmatch(path)
+    if match:
+        return catalog.findings(match.group(1))
     match = _FRACTAL_PATH.fullmatch(path)
     if match:
         key, task_id, node, diff = match.group(1), _task_id(match.group(2)), match.group(3), match.group(4)
@@ -131,6 +163,14 @@ class Handler(BaseHTTPRequestHandler):
     def do_HEAD(self):
         self._serve(head=True)
 
+    def do_POST(self):
+        with self._active():
+            self._send(*self._write_response())
+
+    def do_OPTIONS(self):
+        # No CORS: a preflight is never answered, so no other origin can learn that a write is allowed.
+        self._method_not_allowed()
+
     def __getattr__(self, name):
         # Every other method (POST, PUT, a made-up verb, ...) gets a 405, not the 501 of the base class.
         if name.startswith('do_'):
@@ -180,7 +220,10 @@ class Handler(BaseHTTPRequestHandler):
         if not self._host_allowed():
             self._send(*self._json(403, {'error': 'forbidden'}))
             return
-        self._send(*self._json(405, {'error': 'method not allowed'}, {'Allow': 'GET, HEAD'}))
+        allow = 'GET, HEAD'
+        if not self.server.read_only and control.has_route('POST', self.path.partition('?')[0]):
+            allow += ', POST'
+        self._send(*self._json(405, {'error': 'method not allowed'}, {'Allow': allow}))
 
     @contextmanager
     def _active(self):
@@ -222,6 +265,7 @@ class Handler(BaseHTTPRequestHandler):
                 'version': __version__,
                 'build': BUILD,
                 'pid': os.getpid(),
+                'controls': not self.server.read_only,
             })
         catalog = self.server.catalog
         match = _EXPORT_PATH.fullmatch(path)
@@ -230,6 +274,9 @@ class Handler(BaseHTTPRequestHandler):
         value = api_get(catalog, path)
         if value is not NO_ROUTE:
             return self._found(value)
+        routed = control.dispatch(self.server, forge_cli, 'GET', path, None, query=self.query)
+        if routed:
+            return self._json(routed[0], routed[1])
         if path == '/':
             return self._static('index.html')
         match = _STATIC_PATH.fullmatch(path)
@@ -251,8 +298,88 @@ class Handler(BaseHTTPRequestHandler):
         if body is None:
             return self._json(404, {'error': 'not found'})
         kind = name.rsplit('.', 1)[1]
+        if name == 'index.html' and not self.server.read_only:
+            body = with_token(body, self.server.token)
         return 200, _TYPES[kind], body, {'Content-Security-Policy': page_csp(body)} if kind == 'html' else {}
 
+    # ------------------------------------------------------------------------------ writes
+
+    def _drain(self):
+        """Read (and drop) the request body that is already on its way, up to DRAIN_MAX, so a refusal is not
+        lost to a connection reset. A client that claims more than it sends costs DRAIN_WAIT, not the timeout."""
+        self.close_connection = True
+        try:
+            left = min(int(self.headers.get('Content-Length') or 0), DRAIN_MAX)
+            self.connection.settimeout(DRAIN_WAIT)
+            while left > 0:
+                chunk = self.rfile.read1(min(left, 65536))
+                if not chunk:
+                    break
+                left -= len(chunk)
+        except (OSError, ValueError):
+            pass
+
+    def _refuse(self, status, error):
+        self._drain()
+        return self._json(status, {'error': error})
+
+    def _body(self):
+        """(object, None) for a valid JSON object body, else (None, refusal response)."""
+        if self.headers.get('Transfer-Encoding') is not None:
+            return None, self._refuse(400, 'bad request')
+        length = (self.headers.get('Content-Length') or '').strip()
+        if not length.isascii() or not length.isdigit():
+            return None, self._refuse(400, 'bad request')
+        if int(length) > MAX_BODY:
+            return None, self._refuse(413, 'payload too large')
+        try:
+            raw = self.rfile.read(int(length))
+            if len(raw) != int(length):
+                raise ValueError('short body')
+            body = json.loads(raw.decode('utf-8'))
+        except (OSError, ValueError, RecursionError):  # not UTF-8, not JSON, or the client hung up
+            self.close_connection = True
+            return None, self._json(400, {'error': 'bad request'})
+        if not isinstance(body, dict):
+            return None, self._json(400, {'error': 'bad request'})
+        return body, None
+
+    def _write_response(self):
+        """The answer to a POST. A request passes every check, in this order, before any route sees it:
+        the Host, read-only mode (405), the token, the content type, the Origin and the body."""
+        server = self.server
+        path = self.path.partition('?')[0]
+        if not self._host_allowed():
+            return self._refuse(403, 'forbidden')
+        if server.read_only:
+            self._drain()
+            return self._json(405, {'error': 'method not allowed'}, {'Allow': 'GET, HEAD'})
+        # compare_digest takes bytes here: a non-ASCII header value must not raise.
+        sent = (self.headers.get(TOKEN_HEADER) or '').encode('utf-8', 'surrogateescape')
+        if not hmac.compare_digest(sent, server.token.encode('ascii')):
+            return self._refuse(403, 'forbidden')
+        kind = (self.headers.get('Content-Type') or '').partition(';')[0].strip().lower()
+        if kind != 'application/json':
+            return self._refuse(400, 'content type must be application/json')
+        origin = self.headers.get('Origin')
+        if origin is not None and origin != 'http://' + (self.headers.get('Host') or ''):
+            return self._refuse(403, 'forbidden')
+        body, refusal = self._body()
+        if refusal:
+            return refusal
+        try:
+            routed = control.dispatch(server, forge_cli, 'POST', path, body)
+        except Exception as exc:  # one bad request must not kill the server
+            detail = ' '.join(str(exc).split())
+            _log(f'internal error on POST {path}: {type(exc).__name__}: {detail}')
+            return self._json(500, {'error': 'internal error'})
+        if routed is None:
+            return self._json(405, {'error': 'method not allowed'}, {'Allow': 'GET, HEAD'})
+        status, obj, route, key = routed
+        if not self._quiet():
+            _log(f'control {route} {key} -> {status}')
+        return self._json(status, obj)
+
     def _export(self, catalog, key):
         # One run as one HTML file, to download: never rendered from this origin (sandbox, no sources).
         from forge_ui import export
@@ -292,9 +419,12 @@ class Handler(BaseHTTPRequestHandler):
             pass  # the client hung up (BrokenPipeError, ConnectionResetError, ...): nothing to tell it
 
 
-def make_server(catalog, host='127.0.0.1', port=0, static_dir=None, quiet=False):
+def make_server(catalog, host='127.0.0.1', port=0, static_dir=None, quiet=False, read_only=False):
     server = ThreadingHTTPServer((host, port), Handler)
     server.catalog = catalog
+    server.read_only = read_only
+    server.token = secrets.token_urlsafe(32)  # per process; only the page this process serves carries it
+    server.run_locks = control.RunLocks()
     server.static_dir = STATIC_DIR if static_dir is None else static_dir
     server.quiet = quiet
     server.last_request = time.monotonic()  # stamped when a request arrives and when it ends