@giamme/forge 0.0.0-stage → 0.8.0

package/scripts/forge_fractal/inspection.py

@@ -0,0 +1,261 @@
+"""Read-only managed artifacts and pinned SQLite ledger projections."""
+from __future__ import annotations
+
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+import contextlib
+import json
+from pathlib import Path
+import secrets
+import sqlite3
+import time
+from urllib.parse import parse_qs, urlparse
+import webbrowser
+
+from . import SCRIPTS, identifier, managed_run, read_json, safe_file, state_root
+
+
+def runs(repository: str | None = None) -> dict:
+    result, diagnostics = [], []
+    root = state_root() / 'runs'
+    if root.is_dir():
+        for path in sorted(root.iterdir())[:1000]:
+            try:
+                run = managed_run(path.name)
+                config = read_json(run / 'run.json')
+                if repository and config['repository'] != str(Path(repository).resolve()):
+                    continue
+                result.append(config)
+            except (ValueError, OSError, KeyError) as error:
+                diagnostics.append(dict(run=path.name, error=str(error)))
+    return dict(runs=result[:200], diagnostics=diagnostics)
+
+
+def read_text(root: Path, name: str, maximum: int | None = 262144) -> dict:
+    try:
+        path = safe_file(root, name)
+        if maximum is None:
+            content = path.read_bytes()
+        else:
+            with path.open('rb') as stream:
+                content = stream.read(maximum)
+        return dict(text=content.decode(errors='replace'), truncated=maximum is not None and path.stat().st_size > len(content))
+    except (ValueError, OSError) as error:
+        return dict(missing=str(error))
+
+
+def ledger(task: Path, table: str, offset: int, limit: int) -> dict:
+    if table not in ('nodes', 'runs', 'iters', 'steps', 'events', 'messages'):
+        raise ValueError('Unsupported ledger projection')
+    try:
+        initialization = read_json(safe_file(task, 'initialization.json'))
+        relative = str(Path(initialization['ledger']).relative_to(task))
+        path = safe_file(task, relative)
+        # No Node construction: several upstream inspection verbs reconcile state.
+        with contextlib.closing(sqlite3.connect(path.as_uri() + '?mode=ro', uri=True, timeout=.2)) as connection:
+            connection.execute('PRAGMA query_only=ON')
+            connection.row_factory = sqlite3.Row
+            rows = connection.execute(f'SELECT * FROM {table} LIMIT ? OFFSET ?', (limit, offset)).fetchall()
+        return dict(rows=[dict(row) for row in rows], offset=offset, limit=limit)
+    except (OSError, ValueError, KeyError, sqlite3.Error) as error:
+        return dict(missing=str(error))
+
+
+def _step_logs(task: Path, directory: Path, maximum: int | None) -> dict:
+    result = {}
+    for step in sorted(directory.glob('steps/*')):
+        request = step / 'request.json'
+        role = read_json(safe_file(task, str(request.relative_to(task)))).get('role', 'dwarf') if request.exists() else 'dwarf'
+        if role not in ('dwarf', 'planner'):
+            raise ValueError('Invalid step role')
+        for suffix in (role + '.log', role + '.resolved', role + '.last', 'dispatch.out', 'events.jsonl'):
+            result[str((step / suffix).relative_to(directory))] = read_text(task, str((step / suffix).relative_to(task)), maximum)
+    return result
+
+
+def capture(run: Path, *, task_id: str | None = None, node_id: str | None = None,
+            logs: bool = False, offset: int = 0, limit: int = 100, portable: bool = False) -> dict:
+    data = dict(config=read_json(run / 'run.json'), captured_at=time.time(), tasks=[], missing=[])
+    task_root = run / 'tasks'
+    for task in sorted(task_root.iterdir()) if task_root.exists() else []:
+        if task_id and task.name != task_id:
+            continue
+        safe_file(run, str(task.relative_to(run)))
+        state = read_json(safe_file(task, 'state.json'))
+        request = read_json(safe_file(task, 'request.json'))
+        item = dict(id=task.name, state=state, request=request, nodes=[], acceptance='pending', verification='unverified')
+        # Only explicitly captured Forge outcome files are read from runner directories.
+        outcome = task / 'outcome.json'
+        if outcome.exists():
+            item.update(read_json(safe_file(task, 'outcome.json')))
+            if logs:
+                item['qa'] = {name: read_text(task, name, None if portable else 262144) for name in ('qa.last', 'qa.log', 'changes.diff')}
+        for path in sorted((task / 'nodes').glob('*/node.json')):
+            if node_id and path.parent.name != node_id:
+                continue
+            node = read_json(safe_file(task, str(path.relative_to(task))))
+            if logs:
+                node['logs'] = _step_logs(task, path.parent, None if portable else 262144)
+                node['candidate_changes'] = read_text(task, str((path.parent / 'candidate.diff').relative_to(task)), None if portable else 262144)
+            item['nodes'].append(node)
+        item['activity'] = ledger(task, 'events', offset, limit)
+        item['coordinator_events'] = read_text(task, 'events.jsonl', None if portable else 262144)
+        item['steps'] = ledger(task, 'steps', offset, limit)
+        item['messages'] = ledger(task, 'messages', offset, limit)
+        if portable:
+            for field, table in (('activity', 'events'), ('steps', 'steps'), ('messages', 'messages')):
+                records = []
+                page_offset = 0
+                while True:
+                    page = ledger(task, table, page_offset, 1000)
+                    if 'missing' in page:
+                        item[field] = page
+                        break
+                    records.extend(page['rows'])
+                    if len(page['rows']) < 1000:
+                        item[field] = dict(rows=records, complete=True)
+                        break
+                    page_offset += 1000
+        if logs:
+            item['coordinator_log'] = read_text(task, 'coordinator.log', None if portable else 262144)
+        data['tasks'].append(item)
+    if task_id and not data['tasks']:
+        raise ValueError('Task selector did not match a managed task')
+    if node_id and not any(task['nodes'] for task in data['tasks']):
+        raise ValueError('Node selector did not match a managed node')
+    return data
+
+
+def progress(run: Path) -> dict:
+    """Small, read-only projection for polling; no ledger or text artifacts."""
+    data = dict(config=read_json(safe_file(run, 'run.json')), captured_at=time.time(), tasks=[])
+    task_root = run / 'tasks'
+    for task in sorted(task_root.iterdir()) if task_root.exists() else []:
+        safe_file(run, str(task.relative_to(run)))
+        request = read_json(safe_file(task, 'request.json'))
+        item = dict(id=task.name, goal=str(request.get('prompt', ''))[:240],
+                    model=request.get('model'), state=read_json(safe_file(task, 'state.json')),
+                    nodes=[], acceptance='pending', verification='unverified')
+        if (task / 'outcome.json').exists():
+            item.update(read_json(safe_file(task, 'outcome.json')))
+        for path in sorted((task / 'nodes').glob('*/node.json')):
+            node = read_json(safe_file(task, str(path.relative_to(task))))
+            item['nodes'].append({key: node.get(key) for key in (
+                'id', 'parent', 'status', 'model', 'iteration', 'cost', 'unknown_cost_steps',
+                'difficulty', 'paths', 'deps', 'phase')}
+                | dict(goal=str(node.get('goal', ''))[:240], error=str(node.get('error', ''))[:240]))
+        data['tasks'].append(item)
+    return data
+
+
+def scoped(run: Path, task_id: str, *, node_id: str | None = None,
+           artifact: str = 'task', table: str = 'events', offset: int = 0) -> dict:
+    """Read only the selected task or node's requested heavy material."""
+    task = safe_file(run, 'tasks/' + identifier(task_id))
+    if not task.is_dir():
+        raise ValueError('Task selector did not match a managed task')
+    if artifact == 'task':
+        return dict(request=read_json(safe_file(task, 'request.json')),
+                    outcome=read_json(safe_file(task, 'outcome.json')) if (task / 'outcome.json').exists() else {})
+    if artifact == 'history':
+        if table not in ('events', 'steps', 'messages'):
+            raise ValueError('Unsupported history table')
+        return ledger(task, table, offset, 100)
+    if artifact == 'qa':
+        return {name: read_text(task, name) for name in ('qa.last', 'qa.log', 'changes.diff', 'coordinator.log')}
+    if artifact == 'node' and node_id:
+        node = safe_file(task, 'nodes/' + identifier(node_id))
+        if not (node / 'node.json').is_file():
+            raise ValueError('Node selector did not match a managed node')
+        return read_json(safe_file(task, 'nodes/' + node_id + '/node.json'))
+    if artifact not in ('logs', 'changes') or not node_id:
+        raise ValueError('Unsupported scoped artifact')
+    node = safe_file(task, 'nodes/' + identifier(node_id))
+    if not (node / 'node.json').is_file():
+        raise ValueError('Node selector did not match a managed node')
+    if artifact == 'logs':
+        return _step_logs(task, node, 262144)
+    return read_text(task, str((node / 'candidate.diff').relative_to(task)))
+
+
+def html(data: dict, live: bool = False) -> str:
+    template = (SCRIPTS / 'forge_fractal/dashboard.html').read_text()
+    css = (SCRIPTS / 'forge_fractal/dashboard.css').read_text()
+    script = (SCRIPTS / 'forge_fractal/dashboard.js').read_text()
+    embedded = json.dumps(data).replace('<', '\\u003c').replace('>', '\\u003e').replace('&', '\\u0026')
+    return (template.replace('/*__STYLE__*/', css).replace('/*__SCRIPT__*/', script)
+            .replace('__LIVE__', 'true' if live else 'false').replace('__CAPTURE__', embedded))
+
+
+def _http_data(selected: str | None, query: dict) -> dict:
+    if query.get('scope') == ['runs'] or not selected:
+        return runs()
+    run = managed_run(selected)
+    if query.get('scope') == ['progress']:
+        return progress(run)
+    if query.get('scope') == ['artifact']:
+        offset = int(query.get('offset', ['0'])[0])
+        if offset < 0:
+            raise ValueError('History offset must be nonnegative')
+        return scoped(run, query.get('task', [''])[0], node_id=query.get('node', [None])[0],
+                      artifact=query.get('artifact', ['task'])[0],
+                      table=query.get('table', ['events'])[0], offset=offset)
+    return capture(run, logs=query.get('logs') == ['1'],
+                   offset=max(0, int(query.get('offset', ['0'])[0])), limit=100)
+
+
+class DashboardHandler(BaseHTTPRequestHandler):
+    def log_message(self, *args):
+        pass  # never log the access token
+
+    def do_GET(self):
+        target = urlparse(self.path)
+        prefix = self.server.dashboard_prefix
+        if self.headers.get('Host') != f'127.0.0.1:{self.server.server_port}' or not target.path.startswith(prefix):
+            self.send_error(403)
+            return
+        route = target.path[len(prefix):]
+        if route not in ('', 'data'):
+            self.send_error(404)
+            return
+        query = parse_qs(target.query, keep_blank_values=True)
+        selected = query.get('run', [self.server.dashboard_run_id])[0]
+        try:
+            if route == 'data':
+                body = json.dumps(_http_data(selected, query)).encode()
+                content_type = 'application/json'
+            else:
+                data = progress(managed_run(selected)) if selected else runs()
+                body = html(data, live=True).encode()
+                content_type = 'text/html; charset=utf-8'
+        except (OSError, ValueError, KeyError) as error:
+            body = json.dumps(dict(error=str(error))).encode()
+            content_type = 'application/json'
+            self.send_response(400)
+        else:
+            self.send_response(200)
+        self.send_header('Content-Type', content_type)
+        self.send_header('Cache-Control', 'no-store')
+        self.send_header('Referrer-Policy', 'no-referrer')
+        self.send_header('X-Content-Type-Options', 'nosniff')
+        self.send_header('Content-Length', str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+
+def make_server(run_id: str | None, port: int = 0) -> tuple[ThreadingHTTPServer, str]:
+    prefix = '/' + secrets.token_urlsafe(32) + '/'
+    server = ThreadingHTTPServer(('127.0.0.1', port), DashboardHandler)
+    server.dashboard_prefix = prefix
+    server.dashboard_run_id = run_id
+    return server, f'http://127.0.0.1:{server.server_port}{prefix}'
+
+
+def serve(run_id: str | None, port: int = 0) -> None:
+    server, url = make_server(run_id, port)
+    print('Read-only dashboard: ' + url, flush=True)
+    print('Ctrl-C closes the dashboard; execution continues.', flush=True)
+    webbrowser.open(url)
+    try:
+        server.serve_forever()
+    finally:
+        server.server_close()

package/scripts/forge_fractal/install.py

@@ -0,0 +1,158 @@
+"""Versioned, isolated installation; no shell profile or activation preference."""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from pathlib import Path
+import platform
+import shutil
+import sys
+import tarfile
+import time
+import urllib.request
+
+from . import ARCHIVE_SHA256, REVISION, VERSION, SCRIPTS, command, lock, read_json, runtime_python, state_root, write_json
+
+UV_VERSION = '0.8.22'
+UV_HASHES = {
+    'aarch64-apple-darwin': '3f61099e261e449527141dbf125629fab33ad696468c8c90cebbac40185a306c',
+    'x86_64-apple-darwin': '76638fdcfa91357858771551a1c88de1f7c3b270b33ab1866f8a0618d9e442d8',
+    'aarch64-unknown-linux-gnu': '726b72a137fda33565143325f7d31c42cd30ff9ccdf067e00d124d37b4081cb2',
+    'x86_64-unknown-linux-gnu': '741ff1f5742c5a4a25d2f829e8395355e43f7a5ae2ebc6368e9ae2df0efb69cf',
+}
+
+
+def compatible_python() -> str | None:
+    for name in (sys.executable, 'python3.14', 'python3.13', 'python3.12'):
+        binary = shutil.which(name)
+        if binary:
+            try:
+                version = json.loads(command([binary, '-c', 'import sys,json; print(json.dumps(list(sys.version_info[:2])))']))
+                if (3, 12) <= tuple(version) < (3, 15):
+                    return binary
+            except (RuntimeError, ValueError):
+                pass
+    return None
+
+
+def tmux_install() -> list[str] | None:
+    if platform.system() == 'Darwin' and shutil.which('brew'):
+        return [shutil.which('brew'), 'install', 'tmux']
+    release = Path('/etc/os-release')
+    if release.exists() and any(v in release.read_text().lower() for v in ('debian', 'ubuntu')) and shutil.which('apt-get'):
+        return ([] if os.geteuid() == 0 else ['sudo', '-n']) + ['apt-get', 'install', '-y', 'tmux']
+    return None
+
+
+def doctor() -> dict:
+    python = runtime_python()
+    data = dict(version=VERSION, revision=REVISION, python=str(python), tmux=shutil.which('tmux'), ready=False)
+    provenance = state_root() / 'provenance.json'
+    try:
+        recorded = read_json(provenance)
+        if recorded['revision'] != REVISION or recorded['archive_sha256'] != ARCHIVE_SHA256:
+            raise ValueError('Managed provenance does not match the pinned revision')
+        code = (f'import sys; sys.path.insert(0, {str(SCRIPTS)!r}); '
+                'from forge_fractal.bridge import ForgeAgent; assert not ForgeAgent.enforces_budget; '
+                'import importlib.metadata as m; from fractal.core.agent import Agent, Invocation, StreamParser, resolve; '
+                'from fractal.core.node import Node; '
+                'assert m.version("plasma-fractal")=="1.2.0"; print(m.version("plasma-wiki"))')
+        data['wiki_version'] = command([python, '-c', code], timeout=20).decode().strip()
+        if not (python.parent / 'wiki').is_file():
+            raise ValueError('Managed wiki executable is missing')
+        data['ready'] = bool(data['tmux'])
+        if not data['ready']:
+            data['error'] = 'tmux is missing'
+    except (OSError, RuntimeError, ValueError, KeyError) as error:
+        data['error'] = str(error)
+    data['remedy'] = 'forge fractal install --yes --with-prerequisites'
+    return data
+
+
+def download(url: str, path: Path, digest: str) -> None:
+    with urllib.request.urlopen(url, timeout=45) as response, path.open('wb') as output:
+        shutil.copyfileobj(response, output)
+    if hashlib.sha256(path.read_bytes()).hexdigest() != digest:
+        raise RuntimeError('Checksum mismatch: ' + url + '; retained ' + str(path))
+
+
+def install(args) -> int:
+    root = state_root()
+    uv = shutil.which('uv') or (str(root / 'bin/uv') if (root / 'bin/uv').is_file() else None)
+    python = compatible_python()
+    tmux = shutil.which('tmux')
+    prereq = tmux_install()
+    target = ('aarch64' if platform.machine() in ('arm64', 'aarch64') else platform.machine()) + (
+        '-apple-darwin' if platform.system() == 'Darwin' else '-unknown-linux-gnu')
+    preview = dict(destination=str(root / 'runtime'), version=VERSION, revision=REVISION,
+                   downloads=[f'https://codeload.github.com/plasma-ai/fractal/tar.gz/{REVISION}',
+                              'Fractal dependencies from PyPI, including plasma-wiki'],
+                   uv=uv or f'Official uv {UV_VERSION} checksum-verified release for {target}',
+                   python=python or 'uv python install 3.13',
+                   tmux=tmux or prereq or 'Install tmux with your system package manager; on macOS install Homebrew first',
+                   activation='off; each new run makes a separate choice')
+    print(json.dumps(preview, indent=2))
+    if args.dry_run:
+        return 0
+    if doctor()['ready']:
+        print('Pinned managed runtime already installed; no changes.')
+        return 0
+    missing = not uv or not python or not tmux
+    if not args.yes:
+        if not sys.stdin.isatty():
+            raise ValueError('Unattended installation requires --yes; prerequisites also require --with-prerequisites')
+        if input('Install the managed Fractal runtime? [y/N] ').lower() != 'y':
+            return 0
+    if missing and not args.with_prerequisites:
+        raise ValueError('Missing prerequisites shown above. Install them, or explicitly use --yes --with-prerequisites')
+    root.mkdir(parents=True, exist_ok=True, mode=0o700)
+    with lock(root / 'install.lock'):
+        attempt = root / 'installs' / str(time.time_ns())
+        attempt.mkdir(parents=True)
+        write_json(attempt / 'preview.json', preview)
+        try:
+            if not uv:
+                if target not in UV_HASHES:
+                    raise ValueError('Unsupported uv bootstrap platform; install uv >=0.8 manually')
+                archive = attempt / 'uv.tar.gz'
+                download(f'https://github.com/astral-sh/uv/releases/download/{UV_VERSION}/uv-{target}.tar.gz', archive, UV_HASHES[target])
+                with tarfile.open(archive) as tar:
+                    member = next(m for m in tar.getmembers() if m.name.endswith('/uv') and m.isfile())
+                    (root / 'bin').mkdir(exist_ok=True)
+                    destination = root / 'bin/uv'
+                    with tar.extractfile(member) as src, destination.open('wb') as dst:
+                        shutil.copyfileobj(src, dst)
+                    destination.chmod(0o755)
+                    uv = str(destination)
+            if not python:
+                command([uv, 'python', 'install', '3.13'], timeout=600)
+                python = command([uv, 'python', 'find', '3.13']).decode().strip()
+            if not tmux:
+                if prereq is None:
+                    raise ValueError(str(preview['tmux']))
+                try:
+                    command(prereq, timeout=600)
+                except RuntimeError as error:
+                    raise RuntimeError(f'tmux installation failed. Run {prereq!r} in a terminal with privileges, then retry. {error}') from error
+            archive = attempt / 'fractal.tar.gz'
+            download(preview['downloads'][0], archive, ARCHIVE_SHA256)
+            # A failed runtime is retained and never overlays unrelated environments.
+            destination = root / 'runtime'
+            if destination.exists():
+                destination.rename(attempt / 'previous-runtime')
+            command([uv, 'venv', destination, '--python', python], timeout=120)
+            output = command([uv, 'pip', 'install', '--python', destination / 'bin/python', archive], timeout=600)
+            (attempt / 'install.log').write_bytes(output)
+            versions = command([uv, 'pip', 'freeze', '--python', destination / 'bin/python']).decode()
+            write_json(root / 'provenance.json', dict(version=VERSION, revision=REVISION,
+                       archive_sha256=ARCHIVE_SHA256, source=preview['downloads'][0], versions=versions,
+                       uv=command([uv, '--version']).decode().strip(), installed_at=time.time()))
+            result = doctor()
+            if not result['ready']:
+                raise RuntimeError(str(result))
+            print('Installed. Fractal remains opt-in for each run.')
+        except Exception as error:
+            (attempt / 'error.txt').write_text(str(error))
+            raise
+    return 0

package/scripts/forge_fractal/invocation.py

@@ -0,0 +1,94 @@
+"""Provider invocation lifecycle shared by Fractal planning and implementation."""
+from __future__ import annotations
+
+from pathlib import Path
+import threading
+import time
+import uuid
+
+from . import read_json, write_json
+
+
+def _dataset_step(task, node: dict, directory: Path, **values) -> None:
+    """Record the step in the run dataset (scripts/forge_dataset). Never raises."""
+    try:
+        from forge_dataset.capture import fractal_step
+        fractal_step(task, node, directory, **values)
+    except Exception:
+        pass
+
+
+def run_step(tree, node: dict, workspace: Path, prompt: str, *, phase: str = 'implementation') -> tuple[int, str]:
+    from .execution import Halt, event, process_identity, slot, terminate
+    from fractal.core.node import Node
+    from fractal.core.agent import resolve
+    directory = tree.task / 'nodes' / node['id'] / 'steps' / uuid.uuid4().hex
+    directory.mkdir(parents=True)
+    planning = phase == 'planning'
+    role = 'planner' if planning else 'dwarf'
+    spec = tree.request['planner'] if planning else node['model']
+    resolutions = tree.config['decomposition']['resolved'] if planning else tree.config['resolved']
+    resolved = next(item['canonical'] for item in resolutions if item['spec'] == spec)
+    iteration = node['decomposition']['attempts'] if planning else node['iteration']
+    with slot(tree.run, lambda: tree.check(node)):
+        tree.pause_gate(node)
+        fractal = Node(node['control'])
+        agent = resolve('forge-bridge', root=Node(tree.task / 'control').db.path.parent)(fractal)
+        agent.forge_step, agent.forge_workspace = directory, workspace
+        remaining = max(1, tree.limits['deadline'] - tree.elapsed)
+        if tree.request.get('step_timeout', 0) > 0:
+            remaining = min(remaining, tree.request['step_timeout'])
+        write_json(directory / 'request.json', dict(workspace=str(workspace),
+                   yolo=False if planning else tree.config['yolo_dwarf'], role=role, remaining=remaining))
+        tree.update(node, **{('resolved_planner' if planning else 'resolved_model'): resolved})
+        invocation = agent.invocation(prompt, model=resolved)
+        record = fractal.record
+        run_id = record.run_start()
+        iter_id = record.iter_start(run_id=run_id, iter=iteration)
+        step_id = record.step_start(run_id=run_id, iter_id=iter_id, step=1,
+                                    step_name='FORGE_PLAN' if planning else 'FORGE_WORK')
+        tree.update(node, status='active', phase=phase, started_at=time.time(),
+                    **{('planning_step' if planning else 'step'): str(directory.relative_to(tree.task))})
+        output = directory / 'events.jsonl'
+        rc = 1
+        cost = None
+        try:
+            with output.open('w') as stream:
+                process = agent.spawn(invocation, start_new_session=True)
+                write_json(directory / 'process.json', dict(pid=process.pid, identity=process_identity(process.pid)))
+                def drain():
+                    for line in process.stdout:
+                        stream.write(line)
+                        stream.flush()
+                reader = threading.Thread(target=drain, daemon=True)
+                reader.start()
+                try:
+                    while process.poll() is None:
+                        tree.check(node)
+                        if tree.control(node) == 'pause':
+                            raise Halt('paused')
+                        time.sleep(.1)
+                    rc = process.returncode
+                finally:
+                    terminate(process)
+                    reader.join(timeout=3)
+            if rc == 0:
+                result = agent.stream(output.read_text(errors='replace').splitlines(), step_id=step_id, model=resolved, detached=True)
+                cost = result.cost
+        finally:
+            status = 'completed' if rc == 0 else 'exited'
+            record.step_end(step_id=step_id, status=status, exit_code=rc)
+            record.iter_end(iter_id=iter_id, status=status, exit_code=rc)
+            record.run_end(run_id=run_id, status=status, exit_code=rc)
+        last = directory / (role + '.last')
+        text = last.read_text(errors='replace') if last.exists() else ''
+        attempts = sorted(directory.glob('attempts/' + role + '-*/metrics.json'))
+        if attempts:
+            metrics = read_json(attempts[-1])
+            tree.update(node, **{('planning_tokens' if planning else 'tokens'):
+                        {key: metrics.get(key) for key in ('input_tokens', 'cached_input_tokens', 'output_tokens')}})
+        tree.update(node, cost=(node.get('cost') or 0) + cost if cost is not None else node.get('cost'),
+                    unknown_cost_steps=node.get('unknown_cost_steps', 0) + int(cost is None))
+        event(tree.task, 'step_finished', node=node['id'], phase=phase, iteration=iteration, exit_code=rc, cost=cost)
+        _dataset_step(tree.task, node, directory, phase=phase, iteration=iteration, rc=rc, cost=cost, resolved=resolved)
+        return rc, text

package/scripts/forge_fractal/pipeline.py

@@ -0,0 +1,124 @@
+"""Durable Forge pipeline receipts and explicit retry preparation."""
+from __future__ import annotations
+
+from pathlib import Path
+import shutil
+import shlex
+import subprocess
+import time
+
+from . import artifact, lock, read_json, write_json
+
+
+def remember(run: Path, argv: list[str]) -> None:
+    with lock(run / 'pipeline-record.lock'):
+        config = read_json(run / 'run.json')
+        if 'pipeline_argv' not in config:
+            config.update(pipeline_argv=['/bin/bash', *argv], pipeline_cwd=str(Path.cwd()))
+            write_json(run / 'run.json', config)
+
+
+def resume_pipeline(run: Path) -> bool:
+    config = read_json(run / 'run.json')
+    if 'pipeline_argv' not in config:
+        return False
+    runner = Path(config['runner'])
+    path = runner / ('pipeline.lock' if config['mode'] == 'solo' else 'schedule.lock')
+    try:
+        with lock(path, blocking=False):
+            pass
+    except BlockingIOError:
+        return False  # Original runner is still waiting and will continue itself.
+    if config['mode'] == 'parallel':
+        for task in (run / 'tasks').iterdir():
+            request = read_json(task / 'request.json')
+            status = Path(request['output']) / 'status'
+            if status.exists() and status.read_text().strip() in ('RUNNING', 'INTERRUPTED', 'ERROR', 'TIMEOUT', 'INFRA'):
+                status.write_text('PENDING\n')
+    script = run / 'resume-pipeline.sh'
+    script.write_text('#!/bin/sh\ncd ' + shlex.quote(config['pipeline_cwd']) + '\nexec ' +
+                      shlex.join(config['pipeline_argv']) + ' >>' + shlex.quote(str(run / 'pipeline.log')) + ' 2>&1\n')
+    name = 'forge-pipeline-' + config['id']
+    result = subprocess.run(['tmux', '-L', name, '-f', '/dev/null', 'new-session', '-d', '-s', name,
+                             '/bin/sh ' + shlex.quote(str(script))], capture_output=True)
+    if result.returncode and b'duplicate session' not in result.stderr:
+        raise RuntimeError(result.stderr.decode())
+    return True
+
+
+def outcome(run: Path, runner: Path, rc: int) -> None:
+    config = read_json(run / 'run.json')
+    for task in (run / 'tasks').iterdir() if (run / 'tasks').exists() else []:
+        request = read_json(task / 'request.json')
+        output = Path(request['output'])
+        if output != runner and str(runner) != config['runner']:
+            continue
+        data = dict(acceptance='pending', verification='unverified', pipeline_exit_code=rc, captured_at=time.time())
+        for name, field in (('verdict', 'acceptance'), ('status', 'task_status'), ('verification.status', 'verification')):
+            path = output / name
+            if path.is_file():
+                data[field] = path.read_text().strip()
+        if config['mode'] == 'parallel':
+            data['acceptance'] = data.get('task_status', 'pending')
+            verification = Path(config['runner']) / 'verification.status'
+            if verification.exists():
+                data['verification'] = verification.read_text().strip()
+        for name in ('qa.last', 'qa.log', 'changes.diff'):
+            path = output / name
+            if path.is_file():
+                shutil.copyfile(path, task / name)
+        if Path(request['repo']).exists():
+            data['fingerprint'] = artifact('forge_fingerprint', request['repo'])
+        write_json(task / 'outcome.json', data)
+        try:
+            from forge_dataset.capture import fractal_outcome
+            fractal_outcome(task)
+        except Exception:
+            pass
+
+
+def checkpoint(run: Path, source: Path) -> bool:
+    config = read_json(run / 'run.json')
+    if config['mode'] != 'solo':
+        return False
+    for task in (run / 'tasks').iterdir() if (run / 'tasks').exists() else []:
+        path = task / 'outcome.json'
+        if path.exists():
+            data = read_json(path)
+            if data['acceptance'] == 'PASS' and data['pipeline_exit_code'] == 0:
+                if artifact('forge_fingerprint', source) != data['fingerprint']:
+                    raise ValueError('Source changed after accepted QA; cannot reuse this checkpoint')
+                print('Forge QA checkpoint retained; no model invocation repeated.')
+                return True
+    return False
+
+
+def retry(run: Path, task: Path, prompt: str, model: str) -> None:
+    from .decomposition import retry_decision
+    config = read_json(run / 'run.json')
+    if model not in config['eligible']:
+        raise ValueError('Retry model is outside the frozen routing pool; create a new run to change the pool')
+    with lock(task / 'owner.lock', blocking=False):
+        previous = task / 'attempts' / str(time.time_ns())
+        previous.mkdir(parents=True)
+        for name in ('state.json', 'request.json', 'outcome.json'):
+            if (task / name).exists():
+                shutil.copyfile(task / name, previous / name)
+        request = read_json(task / 'request.json')
+        request.update(prompt=prompt, model=model)
+        write_json(task / 'request.json', request)
+        for path in (task / 'nodes').glob('*/node.json'):
+            node = read_json(path)
+            shutil.copyfile(path, previous / (node['id'] + '.json'))
+            retry_decision(node, prompt)
+            if node['id'] == 'implementation':
+                node.update(goal=prompt, model=model, status='pending', iteration=0,
+                            fingerprint=artifact('forge_fingerprint', request['repo']),
+                            base=artifact('forge_tree', request['repo']))
+                intent = path.parent / 'candidate.import.json'
+                if intent.exists():
+                    intent.rename(previous / 'implementation-import.json')
+            elif node['status'] != 'completed':
+                node.update(status='pending', iteration=0)
+            write_json(path, node)
+        write_json(task / 'state.json', dict(status='pending', acceptance='pending', elapsed=0))