okstra 0.76.0 → 0.78.0

package/runtime/python/okstra_ctl/dispatch_core.py

@@ -0,0 +1,897 @@
+"""Backend-neutral worker dispatch core."""
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import time
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Mapping, Sequence
+
+from . import tmux
+from .lead_events import LeadEvent, append_lead_event
+from .wrapper_status import read_wrapper_status, status_path_for_prompt
+
+
+BACKEND_CLI_WRAPPER = "cli-wrapper"
+BACKEND_TMUX_PANE = "tmux-pane"
+BACKEND_MIXED = "mixed"
+MAX_WORKER_ATTEMPTS = 2
+REPORT_WRITER_WORKER_ID = "report-writer"
+TERMINAL_DISPATCH_STATUSES = {"completed", "timeout", "error", "not-run"}
+
+
+class DispatchError(Exception):
+    """Raised when a worker dispatch request cannot be executed."""
+
+
+@dataclass(frozen=True)
+class WorkerJob:
+    worker_id: str
+    provider: str
+    backend: str
+    project_root: Path
+    model_execution_value: str
+    wrapper_path: Path
+    prompt_path: Path
+    result_path: Path
+    worker_result_path: Path
+    completion_paths: tuple[Path, ...]
+    worktree_path: str
+    role: str
+    idle_timeout_seconds: int
+    dispatch_kind: str
+
+    @property
+    def command(self) -> list[str]:
+        return [
+            str(self.wrapper_path),
+            str(self.project_root),
+            self.model_execution_value,
+            str(self.prompt_path),
+            self.worktree_path,
+            self.role,
+            str(self.idle_timeout_seconds),
+        ]
+
+    def to_payload(self) -> dict[str, Any]:
+        return {
+            "workerId": self.worker_id,
+            "provider": self.provider,
+            "backend": self.backend,
+            "modelExecutionValue": self.model_execution_value,
+            "wrapperPath": str(self.wrapper_path),
+            "promptPath": str(self.prompt_path),
+            "resultPath": str(self.result_path),
+            "workerResultPath": str(self.worker_result_path),
+            "completionPaths": [str(path) for path in self.completion_paths],
+            "worktreePath": self.worktree_path,
+            "role": self.role,
+            "dispatchKind": self.dispatch_kind,
+            "command": self.command,
+        }
+
+
+@dataclass(frozen=True)
+class WorkerHandle:
+    job: WorkerJob
+    pane_id: str
+    completed_process: subprocess.CompletedProcess[str] | None
+    status_sidecar_path: Path | None
+    degraded_from: str
+
+
+@dataclass(frozen=True)
+class WorkerOutcome:
+    returncode: int
+    missing_completion_paths: tuple[Path, ...]
+    pane_id: str
+    status_sidecar_path: Path | None
+    timeout: bool
+    terminal_stage: str
+    degraded_from: str
+
+
+@dataclass(frozen=True)
+class DispatchPlan:
+    project_root: Path
+    workspace_root: Path
+    manifest_path: Path
+    team_state_path: Path
+    lead_events_path: Path
+    manifest: Mapping[str, Any]
+    jobs: tuple[WorkerJob, ...]
+    default_backend: str
+
+    def to_payload(self, dry_run: bool) -> dict[str, Any]:
+        payload = {
+            "ok": True,
+            "dryRun": dry_run,
+            "dispatchMode": _dispatch_mode(self.jobs),
+            "workerBackends": {job.worker_id: job.backend for job in self.jobs},
+            "runManifestPath": str(self.manifest_path),
+            "teamStatePath": str(self.team_state_path),
+            "leadEventsPath": str(self.lead_events_path),
+            "workspaceRoot": str(self.workspace_root),
+            "workers": [job.to_payload() for job in self.jobs],
+        }
+        if self.jobs:
+            payload["workerDispatches"] = [
+                _dispatch_record(job, 1, "not-run", "", "") for job in self.jobs
+            ]
+        return payload
+
+
+@dataclass(frozen=True)
+class _BuildOptions:
+    okstra_bin: Path | None
+    idle_timeout_seconds: int
+    default_backend: str
+    supported_worker_wrappers: Mapping[str, str]
+    unsupported_worker_label: str
+    dispatch_kind: str
+
+
+def build_dispatch_plan(
+    *,
+    project_root: Path,
+    run_manifest_path: Path,
+    workspace_root: Path,
+    okstra_bin: Path | None = None,
+    requested_workers: Sequence[str] = (),
+    idle_timeout_seconds: int = 600,
+    enable_report_writer: bool = False,
+    report_writer_model: str = "",
+    required_lead_runtime: str | None = None,
+    default_backend: str = BACKEND_CLI_WRAPPER,
+    supported_worker_wrappers: Mapping[str, str],
+    unsupported_worker_label: str,
+    dispatch_kind: str = "initial",
+    jobs_file: Path | None = None,
+) -> DispatchPlan:
+    project_root = project_root.resolve()
+    manifest_path = _resolve_project_path(project_root, str(run_manifest_path))
+    manifest = _load_json_object(manifest_path, "run manifest")
+    _validate_manifest(manifest, manifest_path, required_lead_runtime)
+    team_state_path = _resolve_required_path(project_root, manifest, "teamStatePath")
+    team_state = _load_json_object(team_state_path, "team-state")
+    active_context = _load_optional_json(project_root, manifest.get("activeRunContextPath"))
+    options = _BuildOptions(
+        okstra_bin=okstra_bin,
+        idle_timeout_seconds=idle_timeout_seconds,
+        default_backend=default_backend,
+        supported_worker_wrappers=supported_worker_wrappers,
+        unsupported_worker_label=unsupported_worker_label,
+        dispatch_kind=dispatch_kind,
+    )
+    jobs = _jobs_from_file(project_root, workspace_root, jobs_file, options) if jobs_file else _jobs_from_roster(
+        project_root,
+        workspace_root,
+        manifest,
+        team_state,
+        active_context,
+        requested_workers,
+        enable_report_writer,
+        report_writer_model,
+        options,
+    )
+    return DispatchPlan(
+        project_root=project_root,
+        workspace_root=workspace_root.resolve(),
+        manifest_path=manifest_path,
+        team_state_path=team_state_path,
+        lead_events_path=_resolve_required_path(project_root, manifest, "leadEventsPath"),
+        manifest=manifest,
+        jobs=tuple(jobs),
+        default_backend=default_backend,
+    )
+
+
+def dispatch_plan(plan: DispatchPlan, *, wait: bool = True) -> int:
+    if wait:
+        _set_dispatch_mode(plan.team_state_path, _dispatch_mode(plan.jobs))
+        for job in plan.jobs:
+            result = _dispatch_job_with_retry(plan, job)
+            if result != 0:
+                return result
+        return 0
+    handles = [_spawn_job(plan, job, 1) for job in plan.jobs]
+    _set_dispatch_mode(plan.team_state_path, _mode_from_handles(handles))
+    return 0
+
+
+def await_dispatches(
+    plan: DispatchPlan,
+    *,
+    poll_interval_seconds: int = 5,
+    timeout_seconds: int | None = None,
+    heartbeat_seconds: int = 30,
+) -> int:
+    deadline = time.monotonic() + timeout_seconds if timeout_seconds is not None else None
+    last_heartbeat = 0.0
+    while True:
+        running = _running_dispatches(plan.team_state_path)
+        if not running:
+            return 0
+        _advance_running_dispatches(plan, running)
+        running = _running_dispatches(plan.team_state_path)
+        if not running:
+            return 0
+        if deadline is not None and time.monotonic() >= deadline:
+            return 1
+        now = time.monotonic()
+        if heartbeat_seconds > 0 and now - last_heartbeat >= heartbeat_seconds:
+            print(f"WAITING {len(running)} worker dispatch(es)")
+            last_heartbeat = now
+        time.sleep(max(poll_interval_seconds, 0))
+
+
+def _jobs_from_roster(
+    project_root: Path,
+    workspace_root: Path,
+    manifest: Mapping[str, Any],
+    team_state: Mapping[str, Any],
+    active_context: Mapping[str, Any],
+    requested_workers: Sequence[str],
+    enable_report_writer: bool,
+    report_writer_model: str,
+    options: _BuildOptions,
+) -> list[WorkerJob]:
+    selected = _select_workers(manifest, requested_workers, enable_report_writer, options)
+    _mark_roster_skips(project_root, manifest, team_state, selected, requested_workers, options)
+    return [
+        _job_from_roster_worker(
+            project_root, workspace_root, manifest, team_state, active_context,
+            worker_id, report_writer_model, options,
+        )
+        for worker_id in selected
+    ]
+
+
+def _select_workers(
+    manifest: Mapping[str, Any],
+    requested_workers: Sequence[str],
+    enable_report_writer: bool,
+    options: _BuildOptions,
+) -> list[str]:
+    recommended = _string_list(manifest.get("recommendedWorkers"))
+    if not recommended:
+        raise DispatchError("run manifest has no recommendedWorkers entries")
+    supported = set(options.supported_worker_wrappers)
+    if not enable_report_writer and not requested_workers:
+        supported.discard(REPORT_WRITER_WORKER_ID)
+    selected = list(requested_workers) if requested_workers else [w for w in recommended if w in supported]
+    unknown = [worker for worker in selected if worker not in recommended]
+    if unknown:
+        raise DispatchError("requested worker(s) are not in this run roster: " + ", ".join(unknown))
+    unsupported = [worker for worker in selected if worker not in supported]
+    if unsupported:
+        allowed = ", ".join(sorted(supported))
+        raise DispatchError(
+            f"unsupported {options.unsupported_worker_label} worker(s): "
+            + ", ".join(unsupported)
+            + f" (dispatcher supports: {allowed})"
+        )
+    return selected
+
+
+def _job_from_roster_worker(
+    project_root: Path,
+    workspace_root: Path,
+    manifest: Mapping[str, Any],
+    team_state: Mapping[str, Any],
+    active_context: Mapping[str, Any],
+    worker_id: str,
+    report_writer_model: str,
+    options: _BuildOptions,
+) -> WorkerJob:
+    state = _worker_state(team_state, worker_id)
+    provider = _provider_for_worker(worker_id)
+    model = _model_for_worker(worker_id, state, report_writer_model)
+    prompt_path = _resolve_project_path(project_root, _worker_prompt_path(manifest, worker_id))
+    result_path = _resolve_project_path(project_root, _require_string(state, "resultPath"))
+    _materialize_prompt_if_missing(prompt_path, worker_id, result_path, project_root)
+    return WorkerJob(
+        worker_id=worker_id,
+        provider=provider,
+        backend=options.default_backend,
+        project_root=project_root,
+        model_execution_value=model,
+        wrapper_path=_resolve_wrapper(provider, workspace_root, options),
+        prompt_path=prompt_path,
+        result_path=_result_path_for_worker(worker_id, result_path, manifest, project_root),
+        worker_result_path=result_path,
+        completion_paths=_completion_paths(worker_id, result_path, manifest, project_root),
+        worktree_path=_worktree_path(active_context),
+        role=_string_value(state.get("role")) or "worker",
+        idle_timeout_seconds=options.idle_timeout_seconds,
+        dispatch_kind=options.dispatch_kind,
+    )
+
+
+def _jobs_from_file(
+    project_root: Path,
+    workspace_root: Path,
+    jobs_file: Path | None,
+    options: _BuildOptions,
+) -> list[WorkerJob]:
+    if jobs_file is None:
+        return []
+    payload = _load_json_object(_resolve_project_path(project_root, str(jobs_file)), "jobs file")
+    dispatch_kind = _string_value(payload.get("dispatchKind")) or options.dispatch_kind
+    workers = payload.get("workers")
+    if not isinstance(workers, list):
+        raise DispatchError("jobs file workers must be an array")
+    return [
+        _job_from_file_worker(project_root, workspace_root, item, options, dispatch_kind)
+        for item in workers
+        if isinstance(item, dict)
+    ]
+
+
+def _job_from_file_worker(
+    project_root: Path,
+    workspace_root: Path,
+    item: Mapping[str, Any],
+    options: _BuildOptions,
+    dispatch_kind: str,
+) -> WorkerJob:
+    worker_id = _require_string(item, "workerId")
+    provider = _provider_for_job(item, worker_id)
+    prompt_path = _resolve_project_path(project_root, _require_string(item, "promptPath"))
+    result_path = _resolve_project_path(project_root, _require_string(item, "resultPath"))
+    worker_result_path = _resolve_project_path(project_root, _require_string(item, "workerResultPath"))
+    completion_paths = tuple(
+        _resolve_project_path(project_root, path)
+        for path in _string_list(item.get("completionPaths"))
+    ) or (result_path,)
+    return WorkerJob(
+        worker_id=worker_id,
+        provider=provider,
+        backend=options.default_backend,
+        project_root=project_root,
+        model_execution_value=_require_string(item, "modelExecutionValue"),
+        wrapper_path=_resolve_wrapper(provider, workspace_root, options),
+        prompt_path=prompt_path,
+        result_path=result_path,
+        worker_result_path=worker_result_path,
+        completion_paths=completion_paths,
+        worktree_path=_string_value(item.get("worktreePath")),
+        role=_require_string(item, "role"),
+        idle_timeout_seconds=options.idle_timeout_seconds,
+        dispatch_kind=dispatch_kind,
+    )
+
+
+def _spawn_job(plan: DispatchPlan, job: WorkerJob, attempt: int) -> WorkerHandle:
+    _set_worker_status(plan.team_state_path, job.worker_id, "running", "", job.model_execution_value)
+    handle = _start_job(plan, job)
+    _record_dispatch(plan.team_state_path, handle, attempt, "running", "")
+    _append_event(plan, "worker-dispatched", _attempt_details(job, attempt, handle))
+    if handle.completed_process is not None:
+        outcome = _outcome_from_completed(handle)
+        _finish_attempt(plan, job, attempt, outcome)
+    return handle
+
+
+def _dispatch_job_with_retry(plan: DispatchPlan, job: WorkerJob) -> int:
+    for attempt in range(1, MAX_WORKER_ATTEMPTS + 1):
+        handle = _spawn_job(plan, job, attempt)
+        if handle.completed_process is None:
+            return await_dispatches(plan, timeout_seconds=None)
+        outcome = _outcome_from_completed(handle)
+        if outcome.returncode == 0 and not outcome.missing_completion_paths:
+            return 0
+        if _should_retry(outcome, attempt):
+            _append_event(plan, "worker-retry-scheduled", _retry_details(job, attempt, outcome))
+            continue
+        return outcome.returncode or 1
+    return 1
+
+
+def _start_job(plan: DispatchPlan, job: WorkerJob) -> WorkerHandle:
+    if job.backend == BACKEND_CLI_WRAPPER:
+        return _run_cli_wrapper(plan, job, "")
+    if job.backend == BACKEND_TMUX_PANE:
+        return _start_tmux_or_degrade(plan, job)
+    raise DispatchError(f"unsupported worker backend: {job.backend}")
+
+
+def _start_tmux_or_degrade(plan: DispatchPlan, job: WorkerJob) -> WorkerHandle:
+    lead_pane = tmux.resolve_caller_pane()
+    if not lead_pane:
+        return _run_cli_wrapper(plan, job, BACKEND_TMUX_PANE)
+    try:
+        pane_id = tmux.split_worker_pane(
+            target_pane=lead_pane,
+            cwd=plan.project_root,
+            command=job.command,
+            title=f"{job.worker_id}-worker",
+            run_dir=_run_dir(plan),
+        )
+    except RuntimeError:
+        return _run_cli_wrapper(plan, job, BACKEND_TMUX_PANE)
+    return WorkerHandle(job, pane_id, None, status_path_for_prompt(job.prompt_path), "")
+
+
+def _run_cli_wrapper(plan: DispatchPlan, job: WorkerJob, degraded_from: str) -> WorkerHandle:
+    env = {
+        **os.environ,
+        "OKSTRA_WORKER_ID": job.worker_id,
+        "OKSTRA_WORKER_RESULT_PATH": str(job.result_path),
+        "OKSTRA_WORKER_AUDIT_PATH": str(job.worker_result_path),
+        "OKSTRA_RUN_MANIFEST_PATH": str(plan.manifest_path),
+    }
+    if job.worker_id == REPORT_WRITER_WORKER_ID:
+        env["OKSTRA_REPORT_WRITER_MARKDOWN_PATH"] = str(_final_report_markdown_path(job.result_path))
+    completed = subprocess.run(job.command, cwd=plan.project_root, env=env, text=True)
+    return WorkerHandle(job, "", completed, status_path_for_prompt(job.prompt_path), degraded_from)
+
+
+def _outcome_from_completed(handle: WorkerHandle) -> WorkerOutcome:
+    completed = handle.completed_process
+    if completed is None:
+        raise DispatchError("completed process missing")
+    return WorkerOutcome(
+        returncode=completed.returncode,
+        missing_completion_paths=_missing_completion_paths(handle.job),
+        pane_id=handle.pane_id,
+        status_sidecar_path=handle.status_sidecar_path,
+        timeout=False,
+        terminal_stage="exited",
+        degraded_from=handle.degraded_from,
+    )
+
+
+def _advance_running_dispatches(plan: DispatchPlan, records: Sequence[Mapping[str, Any]]) -> None:
+    for record in records:
+        status_path = _optional_path(record.get("statusSidecarPath"))
+        status = read_wrapper_status(status_path) if status_path else None
+        if status is None or not status.is_terminal:
+            continue
+        outcome = _outcome_from_status(record, status)
+        if _should_retry(outcome, int(record.get("attempt", 1))):
+            _retry_from_record(plan, record, outcome)
+            continue
+        _finish_record(plan, record, outcome)
+
+
+def _retry_from_record(
+    plan: DispatchPlan, record: Mapping[str, Any], outcome: WorkerOutcome
+) -> None:
+    worker_id = _require_string(record, "workerId")
+    attempt = int(record.get("attempt", 1))
+    _update_dispatch_status(plan.team_state_path, worker_id, _require_string(record, "kind"), attempt, "error", "required worker artifact was not produced")
+    _append_event(plan, "worker-retry-scheduled", {"workerId": worker_id, "attempt": attempt})
+    job = _job_from_record(plan.project_root, record)
+    _spawn_job(plan, job, attempt + 1)
+
+
+def _finish_attempt(plan: DispatchPlan, job: WorkerJob, attempt: int, outcome: WorkerOutcome) -> None:
+    if outcome.returncode == 0 and not outcome.missing_completion_paths and not outcome.timeout:
+        _set_worker_status(plan.team_state_path, job.worker_id, "completed", "")
+        _update_dispatch_status(plan.team_state_path, job.worker_id, job.dispatch_kind, attempt, "completed", "")
+        _append_event(plan, "worker-result-collected", _result_details(job, attempt, outcome))
+        return
+    reason = _failure_reason(outcome)
+    status = "timeout" if outcome.timeout else "error"
+    _set_worker_status(plan.team_state_path, job.worker_id, status, reason)
+    _update_dispatch_status(plan.team_state_path, job.worker_id, job.dispatch_kind, attempt, status, reason)
+    _append_event(plan, "worker-failed", _failure_details(job, attempt, outcome, reason))
+
+
+def _finish_record(plan: DispatchPlan, record: Mapping[str, Any], outcome: WorkerOutcome) -> None:
+    job = _job_from_record(plan.project_root, record)
+    _finish_attempt(plan, job, int(record.get("attempt", 1)), outcome)
+
+
+def _record_dispatch(
+    team_state_path: Path, handle: WorkerHandle, attempt: int, status: str, reason: str
+) -> None:
+    payload = _load_json_object(team_state_path, "team-state")
+    dispatches = payload.setdefault("workerDispatches", [])
+    if not isinstance(dispatches, list):
+        raise DispatchError(f"team-state workerDispatches must be an array: {team_state_path}")
+    dispatches.append(_dispatch_record(handle.job, attempt, status, handle.pane_id, handle.degraded_from, reason))
+    _write_json(team_state_path, payload)
+
+
+def _dispatch_record(
+    job: WorkerJob, attempt: int, status: str, pane_id: str, degraded_from: str, reason: str = ""
+) -> dict[str, Any]:
+    return {
+        "workerId": job.worker_id,
+        "role": job.role,
+        "kind": job.dispatch_kind,
+        "attempt": attempt,
+        "backendType": BACKEND_CLI_WRAPPER if degraded_from else job.backend,
+        "provider": job.provider,
+        "status": status,
+        "paneId": pane_id,
+        "promptPath": str(job.prompt_path),
+        "resultPath": str(job.result_path),
+        "workerResultPath": str(job.worker_result_path),
+        "statusSidecarPath": str(status_path_for_prompt(job.prompt_path)),
+        "degradedFrom": degraded_from,
+        "reason": reason,
+        "modelExecutionValue": job.model_execution_value,
+        "wrapperPath": str(job.wrapper_path),
+        "completionPaths": [str(path) for path in job.completion_paths],
+        "worktreePath": job.worktree_path,
+    }
+
+
+def _update_dispatch_status(
+    team_state_path: Path, worker_id: str, kind: str, attempt: int, status: str, reason: str
+) -> None:
+    payload = _load_json_object(team_state_path, "team-state")
+    for record in payload.get("workerDispatches", []):
+        if _record_matches(record, worker_id, kind, attempt):
+            record["status"] = status
+            record["reason"] = reason
+            _write_json(team_state_path, payload)
+            return
+
+
+def _record_matches(record: object, worker_id: str, kind: str, attempt: int) -> bool:
+    return isinstance(record, dict) and record.get("workerId") == worker_id and record.get("kind") == kind and record.get("attempt") == attempt
+
+
+def _set_worker_status(
+    team_state_path: Path, worker_id: str, status: str, reason: str, model: str = ""
+) -> None:
+    payload = _load_json_object(team_state_path, "team-state")
+    workers = payload.get("workers")
+    if not isinstance(workers, list):
+        raise DispatchError(f"team-state workers must be an array: {team_state_path}")
+    for worker in workers:
+        if isinstance(worker, dict) and worker.get("workerId") == worker_id:
+            worker["status"] = status
+            worker["reason"] = reason
+            if model:
+                worker["model"] = model
+                worker["modelExecutionValue"] = model
+            _write_json(team_state_path, payload)
+            return
+
+
+def _set_dispatch_mode(team_state_path: Path, dispatch_mode: str) -> None:
+    payload = _load_json_object(team_state_path, "team-state")
+    payload["dispatchMode"] = dispatch_mode
+    _write_json(team_state_path, payload)
+
+
+def _running_dispatches(team_state_path: Path) -> list[Mapping[str, Any]]:
+    payload = _load_json_object(team_state_path, "team-state")
+    dispatches = payload.get("workerDispatches")
+    if not isinstance(dispatches, list):
+        return []
+    return [record for record in dispatches if isinstance(record, dict) and record.get("status") == "running"]
+
+
+def _mark_roster_skips(
+    project_root: Path,
+    manifest: Mapping[str, Any],
+    team_state: Mapping[str, Any],
+    selected: Sequence[str],
+    requested: Sequence[str],
+    options: _BuildOptions,
+) -> None:
+    reasons = _skip_reasons(manifest, selected, requested, options)
+    if not reasons:
+        return
+    team_state_path = _resolve_required_path(project_root, manifest, "teamStatePath")
+    for worker_id, reason in reasons.items():
+        _set_worker_status(team_state_path, worker_id, "not-run", reason)
+
+
+def _skip_reasons(
+    manifest: Mapping[str, Any], selected: Sequence[str], requested: Sequence[str], options: _BuildOptions
+) -> dict[str, str]:
+    if requested:
+        return {w: "skipped by worker dispatch: worker was not requested in this invocation" for w in _string_list(manifest.get("recommendedWorkers")) if w not in set(selected)}
+    supported = set(options.supported_worker_wrappers)
+    return {w: "skipped by worker dispatch default: worker is not supported by this dispatcher" for w in _string_list(manifest.get("recommendedWorkers")) if w not in set(selected) and w not in supported}
+
+
+def _append_event(plan: DispatchPlan, event_type: str, details: Mapping[str, Any]) -> None:
+    append_lead_event(
+        plan.lead_events_path,
+        LeadEvent(
+            event_type=event_type,
+            lead_runtime=_require_string(plan.manifest, "leadRuntime"),
+            task_key=_require_string(plan.manifest, "taskKey"),
+            task_type=_require_string(plan.manifest, "taskType"),
+            run_seq=_run_seq(plan.manifest),
+            timestamp=_utc_now(),
+            details=details,
+        ),
+    )
+
+
+def _attempt_details(job: WorkerJob, attempt: int, handle: WorkerHandle) -> dict[str, Any]:
+    return {**_event_details(job, handle), "attempt": attempt, "maxAttempts": MAX_WORKER_ATTEMPTS}
+
+
+def _event_details(job: WorkerJob, handle: WorkerHandle) -> dict[str, Any]:
+    return {
+        "dispatchMode": BACKEND_CLI_WRAPPER if handle.degraded_from else job.backend,
+        "workerId": job.worker_id,
+        "promptPath": str(job.prompt_path),
+        "resultPath": str(job.result_path),
+        "workerResultPath": str(job.worker_result_path),
+        "wrapperPath": str(job.wrapper_path),
+        "role": job.role,
+        "paneId": handle.pane_id,
+        "degradedFrom": handle.degraded_from,
+    }
+
+
+def _result_details(job: WorkerJob, attempt: int, outcome: WorkerOutcome) -> dict[str, Any]:
+    return {
+        "workerId": job.worker_id,
+        "attempt": attempt,
+        "dispatchMode": BACKEND_CLI_WRAPPER if outcome.degraded_from else job.backend,
+        "missingCompletionPaths": [],
+    }
+
+
+def _failure_details(job: WorkerJob, attempt: int, outcome: WorkerOutcome, reason: str) -> dict[str, Any]:
+    return {
+        "workerId": job.worker_id,
+        "attempt": attempt,
+        "dispatchMode": BACKEND_CLI_WRAPPER if outcome.degraded_from else job.backend,
+        "exitCode": outcome.returncode,
+        "missingCompletionPaths": [str(path) for path in outcome.missing_completion_paths],
+        "reason": reason,
+    }
+
+
+def _retry_details(job: WorkerJob, attempt: int, outcome: WorkerOutcome) -> dict[str, Any]:
+    return {
+        "workerId": job.worker_id,
+        "attempt": attempt,
+        "missingCompletionPaths": [str(path) for path in outcome.missing_completion_paths],
+        "reason": "required worker artifact was not produced",
+    }
+
+
+def _outcome_from_status(record: Mapping[str, Any], status) -> WorkerOutcome:
+    return WorkerOutcome(
+        returncode=status.exit_code or 0,
+        missing_completion_paths=tuple(path for path in _record_completion_paths(record) if not path.is_file()),
+        pane_id=_string_value(record.get("paneId")),
+        status_sidecar_path=status.path,
+        timeout=status.timeout,
+        terminal_stage=status.stage,
+        degraded_from=_string_value(record.get("degradedFrom")),
+    )
+
+
+def _job_from_record(project_root: Path, record: Mapping[str, Any]) -> WorkerJob:
+    return WorkerJob(
+        worker_id=_require_string(record, "workerId"),
+        provider=_require_string(record, "provider"),
+        backend=BACKEND_CLI_WRAPPER,
+        project_root=project_root,
+        model_execution_value=_require_string(record, "modelExecutionValue"),
+        wrapper_path=Path(_require_string(record, "wrapperPath")),
+        prompt_path=Path(_require_string(record, "promptPath")),
+        result_path=Path(_require_string(record, "resultPath")),
+        worker_result_path=Path(_require_string(record, "workerResultPath")),
+        completion_paths=tuple(_record_completion_paths(record)),
+        worktree_path=_string_value(record.get("worktreePath")),
+        role=_require_string(record, "role"),
+        idle_timeout_seconds=600,
+        dispatch_kind=_require_string(record, "kind"),
+    )
+
+
+def _record_completion_paths(record: Mapping[str, Any]) -> list[Path]:
+    return [Path(path) for path in _string_list(record.get("completionPaths"))]
+
+
+def _should_retry(outcome: WorkerOutcome, attempt: int) -> bool:
+    return outcome.returncode == 0 and bool(outcome.missing_completion_paths) and attempt < MAX_WORKER_ATTEMPTS
+
+
+def _failure_reason(outcome: WorkerOutcome) -> str:
+    if outcome.timeout:
+        return "worker wrapper timed out"
+    if outcome.returncode != 0:
+        return f"wrapper exited with code {outcome.returncode}"
+    if outcome.missing_completion_paths:
+        missing = ", ".join(str(path) for path in outcome.missing_completion_paths)
+        return f"required worker artifact was not produced: {missing}"
+    return "worker dispatch failed"
+
+
+def _missing_completion_paths(job: WorkerJob) -> tuple[Path, ...]:
+    return tuple(path for path in job.completion_paths if not path.is_file())
+
+
+def _mode_from_handles(handles: Sequence[WorkerHandle]) -> str:
+    modes = {BACKEND_CLI_WRAPPER if h.degraded_from else h.job.backend for h in handles}
+    if len(modes) == 1:
+        return next(iter(modes))
+    return BACKEND_MIXED
+
+
+def _dispatch_mode(jobs: Sequence[WorkerJob]) -> str:
+    backends = {job.backend for job in jobs}
+    if len(backends) == 1:
+        return next(iter(backends))
+    return BACKEND_MIXED
+
+
+def _validate_manifest(manifest: Mapping[str, Any], path: Path, required: str | None) -> None:
+    if required is not None and manifest.get("leadRuntime") != required:
+        raise DispatchError(f"run manifest is not a {required} lead run: {path}")
+    if not isinstance(manifest.get("leadEventsPath"), str):
+        raise DispatchError(f"run manifest has no leadEventsPath: {path}")
+
+
+def _resolve_wrapper(provider: str, workspace_root: Path, options: _BuildOptions) -> Path:
+    script = options.supported_worker_wrappers.get(provider)
+    if not script:
+        raise DispatchError(f"unsupported worker provider: {provider}")
+    candidates = []
+    if options.okstra_bin is not None:
+        candidates.append(options.okstra_bin / script)
+    candidates.extend([workspace_root / "bin" / script, workspace_root / "scripts" / script])
+    for candidate in candidates:
+        if candidate.is_file():
+            return candidate.resolve()
+    raise DispatchError(f"{script} not found (searched: {', '.join(str(c) for c in candidates)})")
+
+
+def _materialize_prompt_if_missing(prompt_path: Path, worker_id: str, result_path: Path, project_root: Path) -> None:
+    if prompt_path.is_file():
+        return
+    prompt_path.parent.mkdir(parents=True, exist_ok=True)
+    prompt_path.write_text(
+        "\n".join([
+            f"# {worker_id} worker prompt",
+            f"**Project Root:** {project_root}",
+            f"**Result Path:** {result_path}",
+            "",
+        ]),
+        encoding="utf-8",
+    )
+
+
+def _provider_for_worker(worker_id: str) -> str:
+    if worker_id == REPORT_WRITER_WORKER_ID:
+        return "claude"
+    return worker_id
+
+
+def _provider_for_job(item: Mapping[str, Any], worker_id: str) -> str:
+    provider = _string_value(item.get("provider"))
+    if provider:
+        return provider
+    return _provider_for_worker(worker_id)
+
+
+def _model_for_worker(worker_id: str, state: Mapping[str, Any], report_writer_model: str) -> str:
+    if worker_id == REPORT_WRITER_WORKER_ID and report_writer_model.strip():
+        return report_writer_model.strip()
+    return _require_string(state, "modelExecutionValue")
+
+
+def _result_path_for_worker(worker_id: str, result_path: Path, manifest: Mapping[str, Any], project_root: Path) -> Path:
+    if worker_id != REPORT_WRITER_WORKER_ID:
+        return result_path
+    return _final_report_data_path(_resolve_required_path(project_root, manifest, "expectedReportPath"))
+
+
+def _completion_paths(worker_id: str, result_path: Path, manifest: Mapping[str, Any], project_root: Path) -> tuple[Path, ...]:
+    if worker_id != REPORT_WRITER_WORKER_ID:
+        return (result_path,)
+    data_json = _final_report_data_path(_resolve_required_path(project_root, manifest, "expectedReportPath"))
+    return (data_json, _final_report_markdown_path(data_json), result_path)
+
+
+def _final_report_data_path(markdown_path: Path) -> Path:
+    if markdown_path.name.endswith(".md"):
+        return markdown_path.with_name(markdown_path.name[:-3] + ".data.json")
+    return markdown_path.with_suffix(".data.json")
+
+
+def _final_report_markdown_path(data_json_path: Path) -> Path:
+    if data_json_path.name.endswith(".data.json"):
+        return data_json_path.with_name(data_json_path.name[:-10] + ".md")
+    return data_json_path.with_suffix(".md")
+
+
+def _run_dir(plan: DispatchPlan) -> Path:
+    value = _string_value(plan.manifest.get("runDirectoryPath"))
+    return _resolve_project_path(plan.project_root, value) if value else plan.team_state_path.parent.parent
+
+
+def _worktree_path(active_context: Mapping[str, Any]) -> str:
+    worktree = active_context.get("executorWorktree")
+    if isinstance(worktree, dict):
+        return _string_value(worktree.get("path"))
+    return ""
+
+
+def _worker_state(team_state: Mapping[str, Any], worker_id: str) -> Mapping[str, Any]:
+    workers = team_state.get("workers")
+    if isinstance(workers, list):
+        for worker in workers:
+            if isinstance(worker, dict) and worker.get("workerId") == worker_id:
+                return worker
+    raise DispatchError(f"team-state has no workerId={worker_id}")
+
+
+def _worker_prompt_path(manifest: Mapping[str, Any], worker_id: str) -> str:
+    paths = manifest.get("workerPromptPathByWorkerId")
+    if not isinstance(paths, Mapping):
+        raise DispatchError("run manifest has no workerPromptPathByWorkerId")
+    return _require_string(paths, worker_id)
+
+
+def _load_json_object(path: Path, label: str) -> dict[str, Any]:
+    if not path.is_file():
+        raise DispatchError(f"{label} not found: {path}")
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(payload, dict):
+        raise DispatchError(f"{label} must be a JSON object: {path}")
+    return payload
+
+
+def _load_optional_json(project_root: Path, value: Any) -> dict[str, Any]:
+    if not isinstance(value, str) or not value.strip():
+        return {}
+    path = _resolve_project_path(project_root, value)
+    return _load_json_object(path, "active-run-context") if path.is_file() else {}
+
+
+def _resolve_required_path(project_root: Path, manifest: Mapping[str, Any], key: str) -> Path:
+    return _resolve_project_path(project_root, _require_string(manifest, key))
+
+
+def _resolve_project_path(project_root: Path, value: str) -> Path:
+    path = Path(value)
+    return path if path.is_absolute() else project_root / path
+
+
+def _optional_path(value: object) -> Path | None:
+    if not isinstance(value, str) or not value.strip():
+        return None
+    return Path(value)
+
+
+def _require_string(payload: Mapping[str, Any], key: str) -> str:
+    value = payload.get(key)
+    if not isinstance(value, str) or not value.strip():
+        raise DispatchError(f"required string missing: {key}")
+    return value.strip()
+
+
+def _string_value(value: Any) -> str:
+    return value.strip() if isinstance(value, str) else ""
+
+
+def _string_list(value: Any) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    return [str(item).strip() for item in value if str(item).strip()]
+
+
+def _run_seq(manifest: Mapping[str, Any]) -> str:
+    seqs = manifest.get("runSequencesByCategory")
+    if isinstance(seqs, Mapping):
+        return _string_value(seqs.get("manifests")) or "001"
+    return "001"
+
+
+def _utc_now() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _write_json(path: Path, payload: Mapping[str, Any]) -> None:
+    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")