okstra 0.186.7 → 0.188.0

package/runtime/python/okstra_ctl/dispatch_core.py

@@ -7,10 +7,12 @@ import subprocess
 import time
 from dataclasses import dataclass, field, replace
 from datetime import datetime, timezone
+import os
 from pathlib import Path
 from typing import Any, Mapping, Sequence
 
 from .dispatch_state import (
+    CompletedWithoutResultError,
     BACKEND_CLI_WRAPPER,
     BACKEND_CMUX_PANE,
     BACKEND_MIXED,
@@ -27,6 +29,7 @@ from .dispatch_state import (
     load_json_object as _load_json_object,
     link_agent_dispatch_result as _link_agent_dispatch_result,
     missing_completion_paths as _missing_completion_paths,
+    _plan_verify_result_aliases,
     mutate_team_state as _mutate_team_state,
     require_string as _require_string,
     resolve_project_path as _resolve_project_path,
@@ -54,7 +57,7 @@ from .assignment_resolver import (
 from .application.open_worker import open_worker
 from .domain.host import HostSessionContext
 from .domain.provider import HostModelBinding, ServedModelAttestation
-from .domain.worker_exec import SERVED_MODEL_MISMATCH_EXIT_CODE
+from .domain.worker_exec import NO_RESULT_EXIT_CODE, SERVED_MODEL_MISMATCH_EXIT_CODE
 from .domain.worker_runtime import (
     SURFACE_CMUX_PANE,
     EnvironmentBlocked,
@@ -90,7 +93,7 @@ from .initial_prompt_materialization import (
     PromptDeliveryMode,
     materialize_initial_prompts,
 )
-from .agent_invocation import AgentInvocationError, materialize_retry_invocation
+from .agent.invocation import AgentInvocationError, materialize_retry_invocation
 from .path_hints import hydrate_active_run_context
 from .schema_excerpt import (
     bundle_excerpt_path,
@@ -102,6 +105,7 @@ from .worker_audit_ledger import (
     check_worker_results_audit,
     parse_worker_result_name,
 )
+from .worker_artifacts import worker_provider_id
 from .worker_prompt_body import REPORT_WRITER_WORKER_ID
 from .worker_prompt_headers import (
     WorkerPromptHeaderError,
@@ -121,6 +125,7 @@ from .write_policy import (
     WritePolicy,
     build_invocation_write_contract,
     planned_paths_from_run_manifest,
+    task_root_from_run_manifest,
     write_enforcement_from_payload,
     write_policy_from_payload,
 )
@@ -914,24 +919,32 @@ def _select_workers(
             options,
         )
     supported = set(options.supported_worker_wrappers)
+    state = team_state or {}
+
+    def _wrapper_supported(worker_id: str) -> bool:
+        return _worker_provider(state, worker_id, options) in supported
+
     selected = (
         list(requested_workers)
         if requested_workers
         else [
             worker
             for worker in recommended
-            if worker in supported and worker != REPORT_WRITER_WORKER_ID
+            if _wrapper_supported(worker) and worker != REPORT_WRITER_WORKER_ID
         ]
     )
     unknown = [worker for worker in selected if worker not in recommended]
     if unknown:
         raise DispatchError("requested worker(s) are not in this run roster: " + ", ".join(unknown))
-    unsupported = [worker for worker in selected if worker not in supported]
+    unsupported = [worker for worker in selected if not _wrapper_supported(worker)]
     if unsupported:
         allowed = ", ".join(sorted(supported))
         raise DispatchError(
             f"unsupported {options.unsupported_worker_label} worker(s): "
-            + ", ".join(unsupported)
+            + ", ".join(
+                f"{worker} (provider {_worker_provider(state, worker, options)})"
+                for worker in unsupported
+            )
             + f" (dispatcher supports: {allowed})"
         )
     return selected
@@ -1006,28 +1019,20 @@ def _translator_job_from_reservation(
         for row in collection
         if isinstance(row, Mapping)
     }
-    candidates: list[Mapping[str, Any]] = []
-    for path in sorted(reservation_root.glob("*.json")):
-        reservation = _load_json_object(path, "agent invocation reservation")
-        if (
-            set(reservation) == _TRANSLATOR_RESERVATION_KEYS
-            and reservation.get("schemaVersion") == "2.0"
-            and reservation.get("executionIdentityVersion") == 2
-            and reservation.get("assignmentRef") == "translator"
-            and reservation.get("audience") == "translator"
-            and reservation.get("dispatchKind") == "translator"
-            and reservation.get("dutyId") == "translator"
-            and reservation.get("participantRef") == execution.get("participantRef")
-            and reservation.get("roleExecutionRef") == execution.get("roleExecutionRef")
-            and _build_dispatch_id(
-                str(reservation.get("invocationId") or ""),
-                int(reservation.get("attempt") or 0),
-            ) not in dispatched_ids
-        ):
-            candidates.append(reservation)
+    run_manifest_rel = _string_value(manifest.get("runManifestPath"))
+    candidates, seen = _translator_reservation_candidates(
+        project_root,
+        reservation_root,
+        execution=execution,
+        dispatched_ids=dispatched_ids,
+        run_manifest_rel=run_manifest_rel,
+    )
     if len(candidates) != 1:
+        run_label = run_manifest_rel or "run manifest path unknown"
         raise DispatchError(
-            "translator dispatch requires exactly one canonical invocation reservation"
+            "translator dispatch requires exactly one canonical invocation "
+            f"reservation for this run ({run_label}); found {len(candidates)}. "
+            "Translator reservations seen: " + ("; ".join(seen) if seen else "none")
         )
     reservation = candidates[0]
     prompt_path = _resolve_project_path(
@@ -1087,6 +1092,86 @@ def _translator_job_from_reservation(
     )
 
 
+def _translator_reservation_candidates(
+    project_root: Path,
+    reservation_root: Path,
+    *,
+    execution: Mapping[str, Any],
+    dispatched_ids: set[str],
+    run_manifest_rel: str,
+) -> tuple[list[Mapping[str, Any]], list[str]]:
+    """이 run 이 디스패치할 수 있는 translator 예약과, 본 예약 전부의 요약.
+
+    예약 디렉터리(`prompts/.agent-invocations/`)는 run 이 아니라 task type 단위라
+    같은 task 의 이전 run 이 남긴 translator 예약이 함께 놓인다. translator 는
+    `assignmentRef` 가 접미 없는 `translator` 하나이고, 이전 run 의 디스패치는
+    이번 run 의 team-state 에 없으므로 "아직 디스패치 안 됨" 으로 읽혀 후보가
+    둘이 됐다 — 두 번째 run 부터 번역이 영영 막혔다(2026-09-03 실측,
+    fontsninja-v3-site dev-10626-1 r02·r04). 예약 자체는 run 을 모르지만 그 옆의
+    `.meta.json` 은 `contractSource.runManifestPath` 로 자기 run 을 안다. 그
+    값을 이번 run 의 `runManifestPath` 와 대조한다. 매니페스트가 그 경로를 안
+    가진 레거시 run 은 종전대로 run 을 가리지 않는다.
+    """
+    candidates: list[Mapping[str, Any]] = []
+    seen: list[str] = []
+    for path in sorted(reservation_root.glob("*.json")):
+        reservation = _load_json_object(path, "agent invocation reservation")
+        if not (
+            set(reservation) == _TRANSLATOR_RESERVATION_KEYS
+            and reservation.get("schemaVersion") == "2.0"
+            and reservation.get("executionIdentityVersion") == 2
+            and reservation.get("assignmentRef") == "translator"
+            and reservation.get("audience") == "translator"
+            and reservation.get("dispatchKind") == "translator"
+            and reservation.get("dutyId") == "translator"
+        ):
+            continue
+        invocation_id = str(reservation.get("invocationId") or "")
+        reservation_run = _reservation_run_manifest_rel(project_root, reservation)
+        dispatched = _build_dispatch_id(
+            invocation_id, int(reservation.get("attempt") or 0),
+        ) in dispatched_ids
+        seen.append(
+            f"{invocation_id} (run {reservation_run or 'unknown'}"
+            + (", dispatched" if dispatched else "")
+            + ")"
+        )
+        if dispatched:
+            continue
+        if (
+            reservation.get("participantRef") != execution.get("participantRef")
+            or reservation.get("roleExecutionRef") != execution.get("roleExecutionRef")
+        ):
+            continue
+        if run_manifest_rel and (
+            reservation_run is None
+            or os.path.normpath(reservation_run) != os.path.normpath(run_manifest_rel)
+        ):
+            continue
+        candidates.append(reservation)
+    return candidates, seen
+
+
+def _reservation_run_manifest_rel(
+    project_root: Path, reservation: Mapping[str, Any],
+) -> str | None:
+    """예약 옆 `.meta.json` 의 `contractSource.runManifestPath`. 못 읽으면 None."""
+    metadata_value = _string_value(reservation.get("metadataPath"))
+    if not metadata_value:
+        return None
+    try:
+        metadata = _load_json_object(
+            _resolve_project_path(project_root, metadata_value),
+            "agent invocation metadata",
+        )
+    except DispatchError:
+        return None
+    source = metadata.get("contractSource")
+    if not isinstance(source, Mapping):
+        return None
+    return _string_value(source.get("runManifestPath")) or None
+
+
 def _translator_prompt_outputs(
     project_root: Path, prompt_path: Path,
 ) -> tuple[Path, Path]:
@@ -1162,13 +1247,11 @@ def _is_cli_wrapper_assignment(
     options: _BuildOptions,
 ) -> bool:
     state = _worker_state(team_state, worker_id)
-    provider = _string_value(state.get("provider")) or _provider_for_worker(
-        worker_id,
-        options,
-    )
     return (
         _string_value(state.get("runner")) or BACKEND_CLI_WRAPPER
-    ) == BACKEND_CLI_WRAPPER and provider in options.supported_worker_wrappers
+    ) == BACKEND_CLI_WRAPPER and _worker_provider(
+        team_state, worker_id, options
+    ) in options.supported_worker_wrappers
 
 
 def _runner_for_worker(team_state: Mapping[str, Any], worker_id: str) -> str:
@@ -1203,6 +1286,7 @@ def _job_from_roster_worker(
         completion_paths=_completion_paths(
             fact.worker_id, result_path, manifest, project_root
         ),
+        result_aliases=_plan_verify_result_aliases(result_path, manifest),
         worktree_path=_worktree_path(manifest, active_context),
         role=fact.role,
         idle_timeout_seconds=options.idle_timeout_seconds,
@@ -1484,26 +1568,28 @@ def _persisted_write_contract(
     if not job.has_execution_identity:
         return None
     manifest = read_execution_manifest(plan.manifest_path)
-    invocation = next(
+    # 계약은 attempt 소유다 — 재시도는 attempt 마다 다른 프롬프트 파생 파일을
+    # 쓰므로, invocation 으로 찾으면 1회차의 허용 경로를 2회차에 물린다.
+    attempt = next(
         (
-            row for row in manifest.invocations
-            if row.invocation_ref == job.invocation_ref
+            row for row in manifest.attempts
+            if row.invocation_ref == job.invocation_ref and row.attempt == job.attempt
         ),
         None,
     )
-    if invocation is None:
-        raise DispatchError("worker invocation write policy was not persisted")
+    if attempt is None:
+        raise DispatchError("worker attempt write policy was not persisted")
     try:
         enforcement = write_enforcement_from_payload(
-            invocation.write_enforcement
+            attempt.write_enforcement
         )
         policy = replace(
-            write_policy_from_payload(invocation.write_policy),
+            write_policy_from_payload(attempt.write_policy),
             maximum_boundary_precision=enforcement.boundary_precision,
         )
     except (TypeError, ValueError) as exc:
         raise DispatchError(f"persisted worker write policy is invalid: {exc}") from exc
-    if policy.digest != invocation.write_policy_digest:
+    if policy.digest != attempt.write_policy_digest:
         raise DispatchError("persisted worker write policy digest changed")
     return policy, enforcement
 
@@ -1555,9 +1641,6 @@ def _record_execution_attempt(plan: DispatchPlan, job: WorkerJob) -> None:
             dispatch_kind=job.dispatch_kind,
             round=_dispatch_round(job.dispatch_kind),
             input_digest=job.prompt_digest,
-            write_policy=write_policy.to_payload(),
-            write_policy_digest=write_policy.digest,
-            write_enforcement=write_enforcement.to_payload(),
         ),
         Attempt(
             invocation_ref=job.invocation_ref,
@@ -1569,6 +1652,9 @@ def _record_execution_attempt(plan: DispatchPlan, job: WorkerJob) -> None:
             error_path=None,
             change_summary={},
             evidence_seal_ref=None,
+            write_policy=write_policy.to_payload(),
+            write_policy_digest=write_policy.digest,
+            write_enforcement=write_enforcement.to_payload(),
         ),
         task_key=task_key,
     )
@@ -1603,6 +1689,9 @@ def _canonical_write_contract(
             project_root=plan.project_root,
             worktree=worktree,
             artifact_paths=artifacts,
+            task_root=task_root_from_run_manifest(
+                plan.project_root, plan.manifest
+            ),
             maximum_precision=capability.max_boundary_precision,
             planned_paths=planned[0],
             planned_paths_declared=planned[1],
@@ -1913,10 +2002,10 @@ def _finish_attempt(
         "mutation-present-unresolved",
     }:
         reason = _mutation_failure_reason(mutation)
-        _transition_job_status(plan, job, "error", reason)
         _update_dispatch_status(
             plan.team_state_path, job, attempt, "error", reason
         )
+        _transition_job_status(plan, job, "error", reason)
         details = _failure_details(job, attempt, outcome, reason)
         details["mutationAudit"] = mutation.change_summary()
         _append_event(plan, "worker-failed", details)
@@ -1925,13 +2014,20 @@ def _finish_attempt(
     settlement = _settle(plan, job, attempt, outcome)
     if settlement.completed:
         result_link = _link_result(plan, job, attempt)
-        _transition_job_status(plan, job, "completed", "")
+        # 디스패치 행을 먼저 종결한다. 명부 전이가 거절될 수 있고, 그 거절이
+        # 행보다 앞서면 행은 `running` 으로 남아 다음 await 가 같은 sidecar 를
+        # 다시 읽고 같은 자리에서 죽는다(`_link_result` 와 같은 결함 형태).
+        roster_refusal = _transition_roster_completed(plan, job, attempt)
         _update_dispatch_status(
             plan.team_state_path,
             job,
             attempt,
             "completed",
-            "; ".join(part for part in (settlement.note, result_link["reason"]) if part),
+            "; ".join(
+                part
+                for part in (settlement.note, result_link["reason"], roster_refusal)
+                if part
+            ),
         )
         details = _result_details(job, attempt, outcome)
         details["resultLink"] = result_link
@@ -1946,8 +2042,8 @@ def _finish_attempt(
         return False
     reason = settlement.reason
     status = "timeout" if outcome.timeout else "error"
-    _transition_job_status(plan, job, status, reason)
     _update_dispatch_status(plan.team_state_path, job, attempt, status, reason)
+    _transition_job_status(plan, job, status, reason)
     details = _failure_details(job, attempt, outcome, reason)
     if settlement.error_log_append is not None:
         details["errorLogAppend"] = settlement.error_log_append
@@ -1958,6 +2054,35 @@ def _finish_attempt(
     return mutation.retry_allowed if mutation is not None else True
 
 
+def _transition_roster_completed(
+    plan: DispatchPlan, job: WorkerJob, attempt: int,
+) -> str:
+    """명부 행을 completed 로 올리되, 거절은 이벤트와 사유로 돌려세운다.
+
+    명부는 `resultPath` 파일이 있어야 completed 를 받는다
+    (`_reject_completed_without_result`). 디스패치의 산출물은 전부 있는데 그
+    파일이 다른 경우 — 리드가 명부와 다른 포인터 경로로 재디스패치한 뒤 —
+    그 거절이 예외로 `await` 를 죽였고, 행이 `running` 으로 남아 다음 await 도
+    같은 자리에서 죽었다. 실측(2026-09-02~03, fontsninja-v3-site dev-10626-1
+    r04): report-writer a2·a3 행이 그렇게 남아, 정본 경로로 보낸 a4 가 도는
+    동안에도 `await` 가 즉시 exit 2 였다.
+
+    명부는 손대지 않는다 — 그 불변식(completed ⇒ 파일 있음)은 그대로다. 정본
+    경로를 쓰는 다음 디스패치가 그 파일을 쓰면 그때 completed 가 된다.
+    """
+    try:
+        _transition_job_status(plan, job, "completed", "")
+    except CompletedWithoutResultError as exc:
+        reason = f"roster not updated: {exc}"
+        _append_event(plan, "worker-roster-not-updated", {
+            **_event_execution_identity(job, attempt),
+            "dispatchWorkerResultPath": str(job.worker_result_path),
+            "reason": reason,
+        })
+        return reason
+    return ""
+
+
 def _audit_attempt(
     job: WorkerJob,
     outcome: WorkerOutcome,
@@ -2573,6 +2698,7 @@ def _dispatch_record(
         "modelExecutionValue": job.model_execution_value,
         "wrapperPath": str(job.wrapper_path),
         "completionPaths": [str(path) for path in job.completion_paths],
+        "resultAliases": [str(path) for path in job.result_aliases],
         "worktreePath": job.worktree_path,
     }
     if job.invocation_id:
@@ -2713,12 +2839,14 @@ def _skip_reasons(
             if worker_id not in set(selected)
         }
     supported = set(options.supported_worker_wrappers)
+    state = team_state or {}
     return {
         worker_id: (
             "skipped by worker dispatch default: worker is not supported by this dispatcher"
         )
         for worker_id in recommended
-        if worker_id not in set(selected) and worker_id not in supported
+        if worker_id not in set(selected)
+        and _worker_provider(state, worker_id, options) not in supported
     }
 
 
@@ -2857,7 +2985,7 @@ def _retry_details(job: WorkerJob, attempt: int, outcome: WorkerOutcome) -> dict
 def _outcome_from_status(record: Mapping[str, Any], status) -> WorkerOutcome:
     return WorkerOutcome(
         returncode=status.exit_code if status.exit_code is not None else 1,
-        missing_completion_paths=tuple(path for path in _record_completion_paths(record) if not path.is_file()),
+        missing_completion_paths=_record_missing_completion_paths(record),
         pane_id=_string_value(record.get("paneId")),
         status_sidecar_path=status.path,
         timeout=status.timeout,
@@ -2925,8 +3053,43 @@ def _record_completion_paths(record: Mapping[str, Any]) -> list[Path]:
     return [Path(path) for path in _string_list(record.get("completionPaths"))]
 
 
+def _record_missing_completion_paths(record: Mapping[str, Any]) -> tuple[Path, ...]:
+    aliases = {Path(path) for path in _string_list(record.get("resultAliases"))}
+    result_path = Path(_string_value(record.get("resultPath")))
+    worker_result = Path(_string_value(record.get("workerResultPath")))
+    missing: list[Path] = []
+    for path in _record_completion_paths(record):
+        if path.is_file():
+            continue
+        if path in {result_path, worker_result} and any(alias.is_file() for alias in aliases):
+            continue
+        missing.append(path)
+    return tuple(missing)
+
+
 def _should_retry(outcome: WorkerOutcome, attempt: int) -> bool:
-    return outcome.returncode == 0 and bool(outcome.missing_completion_paths) and attempt < MAX_WORKER_ATTEMPTS
+    """결과가 없으면 재시도한다 — 기준은 종료 코드가 아니라 산출물이다.
+
+    종전에는 `returncode == 0` 을 요구했다. 그래서 시작하자마자 죽은 CLI 는
+    재시도를 **한 번도** 못 받았고, 150초를 태우고 0 으로 끝난 CLI 만 받았다.
+    비용 순서가 거꾸로였다.
+
+    실측(fontsninja-v3-site `dev-10341`, grok 워커 13회 디스패치): 세션 생성이
+    거부돼 1.6~4.7초 만에 코드 1 로 죽은 경우가 4회다. 그중
+    `requirements-discovery-002` 는 그것이 1차 시도였는데 재시도가 없어 분석자
+    한 명이 그대로 사라졌고, 바로 다음 디스패치(`-003`)는 222초를 정상 수행했다.
+    같은 명령을 지금 돌리면 동시 3회까지 전부 성공한다 — 조건은 분 단위로
+    풀리는 일시적인 것이고, 재시도 비용은 2초다.
+
+    산출물이 이미 나왔으면 종료 코드가 무엇이든 재시도하지 않는다. 일이 끝났기
+    때문이다. 타임아웃도 재시도하지 않는다 — 그것은 예산을 다 쓴 실패라
+    재시도가 같은 예산을 한 번 더 쓴다.
+    """
+    if outcome.timeout:
+        return False
+    if attempt >= MAX_WORKER_ATTEMPTS:
+        return False
+    return bool(outcome.missing_completion_paths)
 
 
 def _job_for_next_attempt(job: WorkerJob) -> WorkerJob:
@@ -2946,6 +3109,8 @@ def _job_for_next_attempt(job: WorkerJob) -> WorkerJob:
 def _failure_reason(outcome: WorkerOutcome) -> str:
     if outcome.timeout:
         return "worker wrapper timed out"
+    if outcome.returncode == NO_RESULT_EXIT_CODE:
+        return "CLI ended cleanly without emitting a result event"
     if outcome.returncode != 0:
         return f"wrapper exited with code {outcome.returncode}"
     if outcome.missing_completion_paths:
@@ -2982,6 +3147,28 @@ def _resolve_wrapper(provider: str, workspace_root: Path, options: _BuildOptions
     raise DispatchError(f"{script} not found (searched: {', '.join(str(c) for c in candidates)})")
 
 
+def _worker_provider(
+    team_state: Mapping[str, Any],
+    worker_id: str,
+    options: _BuildOptions | None = None,
+) -> str:
+    """이 슬롯이 실제로 실행할 provider.
+
+    로스터의 worker_id 는 provider 이름이 아니다. 한 provider 가 두 역할을 맡으면
+    `grok-implementer` · `grok-verifier` 처럼 역할 접미사가 붙기 때문이다. 지원 여부는
+    항상 이 해석을 거친 provider 로 판정한다 — id 를 provider 집합과 직접 비교하면
+    역할 접미사가 붙은 슬롯이 전부 미지원으로 떨어진다.
+    """
+    workers = team_state.get("workers")
+    recorded = ""
+    if isinstance(workers, list):
+        for worker in workers:
+            if isinstance(worker, Mapping) and worker.get("workerId") == worker_id:
+                recorded = _string_value(worker.get("provider"))
+                break
+    return recorded or _provider_for_worker(worker_id, options)
+
+
 def _provider_for_worker(
     worker_id: str,
     options: _BuildOptions | None = None,
@@ -2992,7 +3179,7 @@ def _provider_for_worker(
             return configured
     if worker_id == REPORT_WRITER_WORKER_ID:
         return "claude"
-    return worker_id
+    return worker_provider_id(worker_id)