okstra 0.169.0 → 0.170.0

package/runtime/python/okstra_ctl/cmux.py

@@ -199,17 +199,23 @@ def plan_worker_placement(
     return _extend_the_shortest_column(columns)
 
 
-def lead_shrink_points(lead: PaneGeometry, *, target_columns: int) -> int:
-    """How far to push the lead's right border, in the points `pane.resize` takes.
+def lead_resize_points(lead: PaneGeometry, *, target_columns: int) -> int:
+    """How far to move the lead's right border, in the points `pane.resize` takes.
+
+    Signed: positive when the lead is too wide and the border comes in, negative
+    when it is too narrow and the border goes out.
+
+    Both directions are needed. A split halves whatever pane it lands on, and
+    the first worker of every round lands on the lead — so a rule that only ever
+    shrinks leaves that half permanent, and the round after it takes half of
+    what is left. Measured on this display: 215 columns becomes 80, then 40,
+    then 20, until neither the lead nor its workers can be read.
 
     The API's `amount` is points, not cells — measured at this pane's own
-    `cell_width_points`, so passing a column count shrinks by an eighth of the
-    intent on a typical display.
+    `cell_width_points`, so passing a column count moves the border by an eighth
+    of the intent on a typical display.
     """
-    surplus = lead.columns - target_columns
-    if surplus <= 0:
-        return 0
-    return surplus * lead.cell_width_points
+    return (lead.columns - target_columns) * lead.cell_width_points
 
 
 def _holds_an_okstra_surface(
@@ -326,7 +332,7 @@ def spawn_worker_surface(
     surface_uuid = _open_worker_surface(workspace, placement, target)
     run_cmux(["rename-tab", "--surface", surface_uuid, "--title", title])
     _exec_worker(surface_uuid, cwd=cwd, command=command)
-    _shrink_lead_pane(workspace)
+    _size_lead_pane(workspace)
     return surface_uuid
 
 
@@ -530,28 +536,39 @@ def _exec_worker(surface_uuid: str, *, cwd: Path, command: Sequence[str]) -> Non
         raise RuntimeError(started.stderr.strip() or "cmux could not start the worker")
 
 
-def _shrink_lead_pane(workspace: str) -> None:
-    """Give the lead's width to the workers by pushing its right border left.
+def _size_lead_pane(workspace: str) -> None:
+    """Put the lead back on its target width, leaving the rest to the workers.
+
+    Which pane carries the request follows from what `pane.resize` does: it
+    moves the named pane's own border in the direction given. The lead can push
+    its right border out — that is `right` on the lead itself — but it cannot
+    pull that border in, because `left` on the leftmost pane finds no adjacent
+    border to move. Narrowing is therefore the right-hand neighbour's request,
+    and widening is the lead's.
 
-    The lead cannot shrink itself: `pane.resize` moves the named pane's border,
-    so asking the leftmost pane to move `left` fails with no adjacent border and
-    `right` widens it. The neighbour on its right carries the request instead.
+    Run after every worker opens rather than once per round: the split that just
+    happened is what knocked the lead off its width, and no other event does.
     """
     panes = list_panes(workspace)
     lead = _lead_pane(panes)
-    amount = lead_shrink_points(lead, target_columns=LEAD_TARGET_COLUMNS)
-    if amount <= 0:
+    offset = lead_resize_points(lead, target_columns=LEAD_TARGET_COLUMNS)
+    if offset == 0:
         return
     neighbours = [pane for pane in panes if pane.x > lead.x]
     if not neighbours:
         return
+    narrowing = offset > 0
     rpc(
         "pane.resize",
         {
             "workspace_id": workspace,
-            "pane_id": min(neighbours, key=lambda pane: pane.x).pane_id,
-            "direction": "left",
-            "amount": amount,
+            "pane_id": (
+                min(neighbours, key=lambda pane: pane.x).pane_id
+                if narrowing
+                else lead.pane_id
+            ),
+            "direction": "left" if narrowing else "right",
+            "amount": abs(offset),
         },
     )
 

package/runtime/python/okstra_ctl/dispatch_core.py

@@ -19,7 +19,9 @@ from .dispatch_state import (
     append_worker_dispatch as _append_worker_dispatch,
     DispatchError,
     WorkerJob,
+    dispatch_completion_paths as _completion_paths,
     dispatch_mode as _dispatch_mode,
+    dispatch_result_path as _result_path_for_worker,
     LIVENESS_AUDIT_HEARTBEAT,
     LIVENESS_WRAPPER_STATUS,
     load_json_object as _load_json_object,
@@ -41,10 +43,7 @@ from .dispatch_state import (
     worker_state as _worker_state,
     worktree_path as _worktree_path,
 )
-from .final_report_paths import (
-    final_report_data_path as _final_report_data_path,
-    final_report_markdown_path as _final_report_markdown_path,
-)
+from .error_log_write import append_observed
 from .lead_events import LeadEvent, append_lead_event
 from .initial_prompt_materialization import (
     InitialPromptMaterializationError,
@@ -54,19 +53,35 @@ from .initial_prompt_materialization import (
     materialize_initial_prompts,
 )
 from .path_hints import hydrate_active_run_context
+from .schema_excerpt import bundle_excerpt_path, excerpt_version_skew
+from .seeding import installed_version
 from .report_finalize import (
     STEP_VALIDATE_RUN,
     FinalizeContext,
     FinalizeError,
     run_finalize,
 )
+from .worker_audit_ledger import (
+    check_worker_results_audit,
+    parse_worker_result_name,
+)
 from .worker_prompt_body import REPORT_WRITER_WORKER_ID
+from .worker_prompt_headers import (
+    WorkerPromptHeaderError,
+    resolve_errors_log_path,
+)
 from .worker_artifact_paths import audit_sidecar_rel
 from .wrapper_status import read_wrapper_status, status_path_for_prompt
 
 
 MAX_WORKER_ATTEMPTS = 2
 TERMINAL_DISPATCH_STATUSES = {"completed", "timeout", "error", "not-run"}
+# What the error log records for a wrapper the dispatcher timed out, matching
+# the value `team-contract` prescribes for a polling-cap termination.
+_WRAPPER_TIMEOUT_EXIT_CODE = 124
+# The excerpt shares one atomic PIPE_BUF append with the rest of the record, so
+# it is capped far below the writer's own 2048-byte stderr limit.
+_WRAPPER_LOG_TAIL_BYTES = 800
 
 
 @dataclass(frozen=True)
@@ -176,7 +191,7 @@ def build_dispatch_plan(
         default_provider_by_worker_id=dict(default_provider_by_worker_id or {}),
     )
     if jobs_file:
-        jobs = _jobs_from_file(project_root, workspace_root, jobs_file, options)
+        jobs = _jobs_from_file(project_root, workspace_root, jobs_file, manifest, options)
     else:
         jobs = _jobs_from_roster(
             project_root,
@@ -189,6 +204,7 @@ def build_dispatch_plan(
             options,
         )
     _validate_dispatch_prompts(manifest, active_context, jobs)
+    _reject_stale_schema_excerpt(project_root, manifest, jobs)
     return DispatchPlan(
         project_root=project_root,
         workspace_root=workspace_root.resolve(),
@@ -690,6 +706,7 @@ def _jobs_from_file(
     project_root: Path,
     workspace_root: Path,
     jobs_file: Path | None,
+    manifest: Mapping[str, Any],
     options: _BuildOptions,
 ) -> list[WorkerJob]:
     if jobs_file is None:
@@ -697,6 +714,7 @@ def _jobs_from_file(
     return _worker_jobs_from_file(
         project_root,
         jobs_file,
+        manifest=manifest,
         backend=options.default_backend,
         idle_timeout_seconds=options.idle_timeout_seconds,
         default_dispatch_kind=options.dispatch_kind,
@@ -927,20 +945,23 @@ def _retry_from_record(
     worker_id = _require_string(record, "workerId")
     attempt = int(record.get("attempt", 1))
     job = _job_from_record(plan.project_root, record)
-    _update_dispatch_status(plan.team_state_path, job, attempt, "error", "required worker artifact was not produced")
-    _append_event(plan, "worker-retry-scheduled", {"workerId": worker_id, "attempt": attempt})
+    reason = "required worker artifact was not produced"
+    _update_dispatch_status(plan.team_state_path, job, attempt, "error", reason)
+    # `team-contract` counts the first attempt's failure as a recorded
+    # `cli-failure`; a retry that succeeds settles `completed` and would
+    # otherwise leave no trace that anything had to be re-run.
+    details: dict[str, Any] = {"workerId": worker_id, "attempt": attempt}
+    details["errorLogAppend"] = _record_wrapper_failure(
+        plan, job, attempt, outcome, reason
+    )
+    _append_event(plan, "worker-retry-scheduled", details)
     _spawn_job(plan, job, attempt + 1)
 
 
 def _finish_attempt(plan: DispatchPlan, job: WorkerJob, attempt: int, outcome: WorkerOutcome) -> None:
-    if outcome.returncode == 0 and not outcome.missing_completion_paths and not outcome.timeout:
-        if job.invocation_id:
-            _link_agent_dispatch_result(
-                project_root=plan.project_root,
-                run_manifest_path=plan.manifest_path,
-                dispatch_id=f"{job.invocation_id}:attempt-{attempt}",
-                result_path=job.worker_result_path,
-            )
+    settlement = _settle(plan, job, attempt, outcome)
+    if settlement.completed:
+        result_link = _link_result(plan, job, attempt)
         post_process = _post_process_report_writer_result(plan, job)
         if not post_process["ok"]:
             reason = _require_string(post_process, "reason")
@@ -965,16 +986,65 @@ def _finish_attempt(plan: DispatchPlan, job: WorkerJob, attempt: int, outcome: W
         _transition_worker_status(
             plan.team_state_path, job.worker_id, "completed", ""
         )
-        _update_dispatch_status(plan.team_state_path, job, attempt, "completed", "")
+        _update_dispatch_status(
+            plan.team_state_path,
+            job,
+            attempt,
+            "completed",
+            "; ".join(part for part in (settlement.note, result_link["reason"]) if part),
+        )
         details = _result_details(job, attempt, outcome)
         details["postProcessing"] = post_process["steps"]
+        details["resultLink"] = result_link
+        if settlement.error_log_append is not None:
+            details["errorLogAppend"] = settlement.error_log_append
         _append_event(plan, "worker-result-collected", details)
         return
-    reason = _failure_reason(outcome)
+    reason = settlement.reason
     status = "timeout" if outcome.timeout else "error"
     _transition_worker_status(plan.team_state_path, job.worker_id, status, reason)
     _update_dispatch_status(plan.team_state_path, job, attempt, status, reason)
-    _append_event(plan, "worker-failed", _failure_details(job, attempt, outcome, reason))
+    details = _failure_details(job, attempt, outcome, reason)
+    if settlement.error_log_append is not None:
+        details["errorLogAppend"] = settlement.error_log_append
+    _append_event(plan, "worker-failed", details)
+
+
+def _link_result(
+    plan: DispatchPlan, job: WorkerJob, attempt: int
+) -> dict[str, Any]:
+    """Bind this result to its verified dispatch, reporting rather than raising.
+
+    Linking is bookkeeping around a dispatch that has already settled, and it
+    used to run before any status was written. A refused link — the live case is
+    a corrective re-dispatch claiming a path the first attempt still owns — threw
+    out of `_finish_attempt`, so nothing transitioned, the row stayed `running`,
+    and the exception reached the caller as exit 2. The next `await` re-read the
+    same terminal sidecar, re-settled the same way, and threw at the same line:
+    a worker with complete artifacts wedged the run permanently, and no amount of
+    waiting could clear it.
+
+    So the settle is written either way and the refusal travels back as data. It
+    is not swallowed: the reason lands in the dispatch row and in the lead event,
+    and the post-hoc validators still require every accepted result to carry a
+    link, so an unlinked result fails where an audit failure belongs rather than
+    by stopping the run mid-phase. `agent-prompt reject-result` is the remedy the
+    refusal names.
+    """
+    link: dict[str, Any] = {"ok": True, "reason": ""}
+    if not job.invocation_id:
+        return link
+    try:
+        _link_agent_dispatch_result(
+            project_root=plan.project_root,
+            run_manifest_path=plan.manifest_path,
+            dispatch_id=f"{job.invocation_id}:attempt-{attempt}",
+            result_path=job.worker_result_path,
+        )
+    except (DispatchError, OSError) as exc:
+        link["ok"] = False
+        link["reason"] = f"result link refused: {exc}"
+    return link
 
 
 def _finish_record(plan: DispatchPlan, record: Mapping[str, Any], outcome: WorkerOutcome) -> None:
@@ -982,6 +1052,191 @@ def _finish_record(plan: DispatchPlan, record: Mapping[str, Any], outcome: Worke
     _finish_attempt(plan, job, int(record.get("attempt", 1)), outcome)
 
 
+@dataclass(frozen=True)
+class _Settlement:
+    """How one attempt's terminal status was decided."""
+
+    completed: bool
+    reason: str
+    note: str
+    error_log_append: dict[str, Any] | None
+
+
+def _settle(
+    plan: DispatchPlan, job: WorkerJob, attempt: int, outcome: WorkerOutcome
+) -> _Settlement:
+    """Judge an attempt by its artifacts, not by the wrapper's exit code alone.
+
+    A wrapper can die after its worker has already written everything — an
+    observed case is a connection dropped at session teardown, long after the
+    result file and its audit sidecar were on disk. Settling that as `error`
+    discards a complete analysis, and not figuratively: `convergence_engine`
+    admits only dispatches that settled `completed`, so the worker's findings
+    never reach re-verification. The lead's own re-dispatch triggers agree —
+    they name a missing, unparseable, or audit-failing result, never an exit
+    code — but the only signal `team await` gave was the status.
+
+    So a non-zero exit with every completion path present is re-judged by the
+    audit-sidecar contract, the same rules `okstra worker-audit-check` runs. It
+    passes and the dispatch settles `completed`; it fails and the dispatch stays
+    `error` exactly as before. Either way the wrapper's failure is written to the
+    run error log, so a `completed` here is never a swallowed failure.
+    """
+    if outcome.returncode == 0 and not outcome.missing_completion_paths and not outcome.timeout:
+        return _Settlement(True, "", "", None)
+    reason = _failure_reason(outcome)
+    completed = False
+    note = ""
+    if not outcome.timeout and not outcome.missing_completion_paths:
+        audit_failures = _audit_sidecar_failures(job)
+        if audit_failures:
+            reason = (
+                f"{reason}; worker artifacts failed the audit-sidecar "
+                f"contract: {audit_failures[0]}"
+            )
+        else:
+            completed = True
+            note = (
+                f"{reason}, but every completion artifact was written and "
+                f"passed the audit-sidecar contract"
+            )
+    append = _record_wrapper_failure(plan, job, attempt, outcome, note or reason)
+    return _Settlement(completed, "" if completed else reason, note, append)
+
+
+def _audit_sidecar_failures(job: WorkerJob) -> tuple[str, ...]:
+    """This worker's audit-sidecar contract failures, if the check can run.
+
+    The check's arguments come from the result filename rather than the manifest
+    so the scan cannot widen past the file this job produced: `worker-results/`
+    accumulates every run's artifacts, and the `worker=` filter matches the
+    `-worker`-suffixed role, not the bare provider id. A non-canonical name
+    leaves nothing to enforce, and an unverifiable artifact must not be promoted
+    to `completed`, so that reports one failure rather than an empty tuple.
+    """
+    parsed = parse_worker_result_name(job.worker_result_path.name)
+    if parsed is None:
+        return (
+            f"worker result `{job.worker_result_path.name}` is not a canonical "
+            f"`<role>-worker-<task-type>-<seq>.md` name, so the audit-sidecar "
+            f"contract could not be checked",
+        )
+    return tuple(
+        check_worker_results_audit(
+            job.worker_result_path.parent.parent,
+            parsed.task_type,
+            parsed.seq,
+            worker=parsed.worker_role,
+        )
+    )
+
+
+def _record_wrapper_failure(
+    plan: DispatchPlan,
+    job: WorkerJob,
+    attempt: int,
+    outcome: WorkerOutcome,
+    message: str,
+) -> dict[str, Any]:
+    """Write the wrapper's own failure to the run-level error log.
+
+    `okstra-lead-contract` tells Lead the deterministic dispatcher records this
+    and that Lead does not need to re-record it. Nothing did: no code path
+    anywhere called the error-log writer, so every wrapper failure vanished, and
+    with it the `instruction-set/prior-run-errors.md` digest the next run reads
+    and the `/okstra-inspect errors` report. The dispatcher is the only component
+    that holds the exit code, so it is the one that writes.
+
+    Never raises. Logging is bookkeeping around a dispatch that has already
+    settled; letting a rejected or unwritable record throw here would turn a
+    recorded outcome into an unrecorded crash. What went wrong travels back in
+    the lead event instead.
+    """
+    result: dict[str, Any] = {"ok": False, "reason": "", "path": ""}
+    try:
+        out_path = resolve_errors_log_path(
+            plan.project_root,
+            plan.manifest,
+            _load_optional_json(
+                plan.project_root, plan.manifest.get("activeRunContextPath")
+            ),
+        )
+        result["path"] = str(out_path)
+        append_observed(
+            out_path=out_path,
+            task_key=_string_value(plan.manifest.get("taskKey")),
+            phase=_workflow_phase(plan.manifest),
+            agent=_error_log_agent(job.worker_id),
+            agent_role=(
+                "report-writer"
+                if job.worker_id == REPORT_WRITER_WORKER_ID
+                else "worker"
+            ),
+            model=job.model_execution_value,
+            error_type="cli-failure",
+            command=" ".join(job.command),
+            command_kind="wrapper",
+            exit_code=_WRAPPER_TIMEOUT_EXIT_CODE if outcome.timeout else outcome.returncode,
+            duration_ms=_wrapper_duration_ms(outcome),
+            message=f"attempt {attempt}: {message}",
+            stderr_excerpt=_wrapper_log_tail(job),
+            context=None,
+        )
+    except (OSError, ValueError, TypeError, WorkerPromptHeaderError) as exc:
+        result["reason"] = f"{type(exc).__name__}: {exc}"
+        return result
+    result["ok"] = True
+    return result
+
+
+def _error_log_agent(worker_id: str) -> str:
+    """The error log's `--agent` enum value for a worker id.
+
+    The log's own allow-list is the authority on what it accepts; a worker whose
+    name is outside it is reported as such by `append_observed` rather than
+    silently rewritten into some other agent's records.
+    """
+    if worker_id == REPORT_WRITER_WORKER_ID:
+        return REPORT_WRITER_WORKER_ID
+    return f"{worker_id}-worker"
+
+
+def _workflow_phase(manifest: Mapping[str, Any]) -> str:
+    workflow = manifest.get("workflow")
+    if isinstance(workflow, Mapping):
+        return _string_value(workflow.get("currentPhase"))
+    return ""
+
+
+def _wrapper_duration_ms(outcome: WorkerOutcome) -> int | None:
+    if outcome.status_sidecar_path is None:
+        return None
+    status = read_wrapper_status(outcome.status_sidecar_path)
+    if status is None:
+        return None
+    value = status.raw.get("duration_ms")
+    return value if isinstance(value, int) and not isinstance(value, bool) else None
+
+
+def _wrapper_log_tail(job: WorkerJob) -> str | None:
+    """The tail of the wrapper transcript, which usually names the real failure.
+
+    The observed case put `API Error: Connection lost mid-response.` in the last
+    two lines and nothing anywhere else; without it the record says only that
+    some process exited 1. Capped well under the writer's own excerpt limit
+    because a whole record must stay inside one atomic `PIPE_BUF` append.
+    """
+    log_path = job.prompt_path.with_suffix(job.prompt_path.suffix + ".log")
+    try:
+        with log_path.open("rb") as handle:
+            handle.seek(0, 2)
+            handle.seek(max(0, handle.tell() - _WRAPPER_LOG_TAIL_BYTES))
+            tail = handle.read()
+    except OSError:
+        return None
+    return tail.decode("utf-8", errors="replace").strip() or None
+
+
 def _post_process_report_writer_result(
     plan: DispatchPlan,
     job: WorkerJob,
@@ -1262,6 +1517,10 @@ def _result_details(job: WorkerJob, attempt: int, outcome: WorkerOutcome) -> dic
         "attempt": attempt,
         "dispatchMode": BACKEND_CLI_WRAPPER if outcome.degraded_from else job.backend,
         "missingCompletionPaths": [],
+        # A collected result can still come from a wrapper that exited non-zero
+        # (`_settle`). The status says the artifacts are good; this says what the
+        # process did, so the trace never loses one fact to the other.
+        "wrapperExitCode": outcome.returncode,
     }
 
 
@@ -1406,17 +1665,48 @@ def _provider_for_worker(
     return worker_id
 
 
-def _result_path_for_worker(worker_id: str, result_path: Path, manifest: Mapping[str, Any], project_root: Path) -> Path:
-    if worker_id != REPORT_WRITER_WORKER_ID:
-        return result_path
-    return _final_report_data_path(_resolve_required_path(project_root, manifest, "expectedReportPath"))
 
 
-def _completion_paths(worker_id: str, result_path: Path, manifest: Mapping[str, Any], project_root: Path) -> tuple[Path, ...]:
-    if worker_id != REPORT_WRITER_WORKER_ID:
-        return (result_path,)
-    data_json = _final_report_data_path(_resolve_required_path(project_root, manifest, "expectedReportPath"))
-    return (data_json, _final_report_markdown_path(data_json), result_path)
+def _reject_stale_schema_excerpt(
+    project_root: Path,
+    manifest: Mapping[str, Any],
+    jobs: Sequence[WorkerJob],
+) -> None:
+    """Refuse to send the report writer at a schema excerpt from another runtime.
+
+    The bundle's `instruction-set/final-report-schema.json` is cut at prep time
+    and never moves again, while validation always runs against the installed
+    schema. A run long enough to straddle a runtime upgrade therefore has the
+    author writing to one contract and the validator reading another — and the
+    only thing that noticed was the renderer, in Phase 6, after the worker had
+    authored the whole report. The two versions are comparable the moment the
+    dispatch is built, and the remedy is the same either way, so it belongs here.
+
+    Only the report writer is stopped: it is the only worker that authors against
+    the excerpt. Re-running bundle prep re-cuts it from the installed schema.
+    """
+    writer = next(
+        (job for job in jobs if job.worker_id == REPORT_WRITER_WORKER_ID), None
+    )
+    if writer is None:
+        return
+    expected = _string_value(manifest.get("expectedReportPath"))
+    if not expected:
+        return
+    excerpt_path = bundle_excerpt_path(_resolve_project_path(project_root, expected))
+    if excerpt_path is None:
+        return
+    installed = installed_version()
+    cut_from = excerpt_version_skew(excerpt_path, installed)
+    if not cut_from:
+        return
+    raise DispatchError(
+        f"the bundle's schema excerpt ({excerpt_path}) was cut from okstra "
+        f"{cut_from} but this runtime is {installed}. The report writer authors "
+        f"against that excerpt and validation runs against the installed schema, "
+        f"so dispatching now spends a full authoring pass on the wrong contract. "
+        f"Re-prepare the task bundle to re-cut the excerpt, then dispatch again."
+    )
 
 
 def _run_dir(plan: DispatchPlan) -> Path: