@matt82198/aesop 0.3.2 → 0.4.0

package/driver/wave_loop.py

@@ -109,15 +109,32 @@ def _quote_arg(s: str) -> str:
         str: properly quoted argument safe for shell execution on this OS
     """
     if os.name == 'nt':
-        # Windows (cmd.exe): use subprocess.list2cmdline semantics.
-        # Double quotes are the safe quoting mechanism; embed quotes are escaped
-        # with backslash, and backslashes before quotes are escaped.
-        # For safety, we wrap in double quotes and escape embedded quotes/backslashes.
+        # Windows: forced-quote variant of the MS C runtime argv rules
+        # (subprocess.list2cmdline semantics). RS3-W N6: only backslashes
+        # that precede a double quote (or the closing quote) are doubled --
+        # doubling EVERY backslash turned `src\util.py` into `src\\util.py`
+        # and broke `git add` pathspecs for Windows-separator paths.
         if not s:
             return '""'
-        # Escape backslashes before quotes, then escape quotes
-        escaped = s.replace('\\', '\\\\').replace('"', '\\"')
-        return f'"{escaped}"'
+        out = ['"']
+        pending_backslashes = 0
+        for ch in s:
+            if ch == '\\':
+                pending_backslashes += 1
+            elif ch == '"':
+                # Backslashes before a quote are doubled, then the quote
+                # itself is escaped.
+                out.append('\\' * (pending_backslashes * 2 + 1))
+                out.append('"')
+                pending_backslashes = 0
+            else:
+                out.append('\\' * pending_backslashes)
+                out.append(ch)
+                pending_backslashes = 0
+        # Backslashes before the CLOSING quote must be doubled.
+        out.append('\\' * (pending_backslashes * 2))
+        out.append('"')
+        return ''.join(out)
     else:
         # POSIX: shlex.quote handles all cases safely
         return shlex.quote(s)
@@ -277,6 +294,32 @@ def _journal_key_for_item(item: Dict[str, Any]) -> str:
 # Wave Recovery: Journal and Resume Support
 # ========================================================================
 
+def _item_fingerprint(item: Dict[str, Any]) -> str:
+    """Stable identity fingerprint of a manifest item (RS3-W N10).
+
+    Journal entries persist across waves in state_dir. Without an identity
+    check, a NEW tracker item that reuses a prior wave's slug would inherit
+    that stale entry's verified=True and be silently skipped from build.
+    The fingerprint binds a journal entry to the item CONTENT (slug, prompt,
+    ownsFiles, testCmd), so resume-skip only applies to the same work.
+
+    Args:
+        item: manifest item dict.
+
+    Returns:
+        str: hex sha1 over the canonical identity fields.
+    """
+    basis = {
+        "slug": item.get("slug"),
+        "prompt": item.get("prompt"),
+        "ownsFiles": sorted(str(f) for f in (item.get("ownsFiles") or [])),
+        "testCmd": item.get("testCmd"),
+    }
+    return hashlib.sha1(
+        json.dumps(basis, sort_keys=True, default=str).encode("utf-8")
+    ).hexdigest()
+
+
 def _write_journal_entry(state_dir: str, slug: str, phase: str, data: Dict[str, Any], repo: str = None) -> None:
     """Write a journal entry for an item's progress.
 
@@ -313,11 +356,22 @@ def _write_journal_entry(state_dir: str, slug: str, phase: str, data: Dict[str,
         **data,
     }
 
+    tmp_file = journal_file.with_name(journal_file.name + ".tmp")
     try:
-        journal_file.write_text(json.dumps(entry, default=str) + "\n")
+        # ATOMIC write (RS3-W N10): serialize first, write to a sibling temp
+        # file, then os.replace into place. A crash mid-write can no longer
+        # leave a torn/malformed entry at the journal path (a torn entry is
+        # silently skipped on load, losing the item's recovery state).
+        payload = json.dumps(entry, default=str) + "\n"
+        tmp_file.write_text(payload)
+        os.replace(str(tmp_file), str(journal_file))
     except Exception:
         # Fail-closed: if journal write fails, continue without journaling.
-        pass
+        try:
+            if tmp_file.exists():
+                tmp_file.unlink()
+        except Exception:
+            pass
 
 
 def _load_journal_state(state_dir: str) -> Dict[str, Dict[str, Any]]:
@@ -402,11 +456,17 @@ def _release_stale_leases(state_dir: str, journal_state: Dict[str, Dict[str, Any
 
         event_store = store.EventStore(str(db_path))
 
-        for slug, entry in journal_state.items():
+        # RS3-W N4: journal_state KEYS are (repo, slug) tuples, but claims
+        # are made with resource=<slug string> (see build_item's try_claim).
+        # Releasing with the tuple never matched a claim, so a crashed
+        # instance's lease persisted forever. Release with the SAME key
+        # shape claims use: the entry's slug string.
+        for entry in journal_state.values():
             old_instance_id = entry.get("instance_id")
-            if old_instance_id:
+            resource = entry.get("slug")
+            if old_instance_id and resource:
                 try:
-                    coordination.release(event_store, resource=slug, instance_id=old_instance_id)
+                    coordination.release(event_store, resource=resource, instance_id=old_instance_id)
                 except Exception:
                     # Ignore release errors; fail-closed.
                     pass
@@ -415,6 +475,448 @@ def _release_stale_leases(state_dir: str, journal_state: Dict[str, Dict[str, Any
         pass
 
 
+# ========================================================================
+# RS5: claim-gate lifecycle (ttl sized to the work, fence before repair/
+# ship, hold across the full lifecycle, exactly-once release)
+# ========================================================================
+
+# F1a: the claim TTL must comfortably cover the WORK it protects (build +
+# bounded repair + ship), not coordination's 300s default -- a single real
+# build outlives 300s, letting a second instance reclaim a LIVE slug
+# mid-build (double-dispatch). Sized from the driver's command timeout with
+# a generous multiple and a sane floor; the fence (F1b) closes the window
+# even when a build outruns this ttl.
+_CLAIM_TTL_FLOOR_S = 3600.0          # never below 1h, even for fast drivers
+_CLAIM_TTL_TIMEOUT_MULTIPLE = 10.0   # covers build + repair rounds + ship
+
+
+def _claim_ttl_for_driver(driver: Any) -> float:
+    """Derive the per-item claim TTL from the driver's command timeout.
+
+    Checks the public and private timeout knobs used by the concrete
+    drivers (CodexDriver: command_timeout_s/_command_timeout_s;
+    ClaudeCodeDriver: _timeout_s). Unusable values fall back to the floor.
+    """
+    best = 0.0
+    for attr in (
+        "command_timeout_s", "_command_timeout_s", "timeout_s", "_timeout_s"
+    ):
+        try:
+            value = getattr(driver, attr, None)
+            if value is None:
+                continue
+            value = float(value)
+        except (TypeError, ValueError):
+            continue
+        if value > best:
+            best = value
+    return max(_CLAIM_TTL_FLOOR_S, best * _CLAIM_TTL_TIMEOUT_MULTIPLE)
+
+
+class _ClaimContext:
+    """Per-wave claim registry: acquire in build, fence before repair/ship,
+    release exactly once at the true end of the lifecycle (RS5 F1/F3).
+
+    - acquire(slug): coordination.try_claim with the wave-level instance id
+      and the work-sized ttl. Exceptions propagate so build_item keeps its
+      fail-closed SKIP (RS3-W N5).
+    - fence_ok(slug): True unless WE claimed the slug and can no longer
+      prove we hold it (ttl lapsed / reclaimed / read error). Items this
+      wave never claimed (journal-resumed, no state_dir, no coordination
+      module) always pass: single-instance behavior is unchanged.
+    - release_all(): pops each held slug exactly once and best-effort
+      releases it; called from run_wave's finally on EVERY exit path
+      (success, abort, exception) -- never from build_item, so the claim
+      survives Phase 5 repair and Phase 7 ship.
+    """
+
+    def __init__(self, state_dir: Optional[str], instance_id: str, ttl: float):
+        self.state_dir = state_dir
+        self.instance_id = instance_id
+        self.ttl = ttl
+        self._held: Dict[str, bool] = {}
+        self._lock = threading.Lock()
+
+    def _enabled(self) -> bool:
+        return coordination is not None and self.state_dir is not None
+
+    def _event_store(self):
+        from state_store import store
+        return store.EventStore(str(Path(self.state_dir) / "state.db"))
+
+    def acquire(self, slug: str) -> bool:
+        """Claim slug for this wave; True only if the claim is held."""
+        if not self._enabled():
+            return False
+        won = coordination.try_claim(
+            self._event_store(),
+            resource=slug,
+            instance_id=self.instance_id,
+            ttl=self.ttl,
+        )
+        if won:
+            with self._lock:
+                self._held[slug] = True
+        return bool(won)
+
+    def holds(self, slug: str) -> bool:
+        with self._lock:
+            return slug in self._held
+
+    def fence_ok(self, slug: str) -> bool:
+        """FENCE (F1b): may this wave still ship/re-dispatch slug?
+
+        Fail-closed: if we claimed the slug but cannot prove we still hold
+        it, the answer is False -- never double-ship on a lapsed claim.
+        """
+        if not self._enabled() or not self.holds(slug):
+            return True
+        try:
+            return (
+                coordination.current_holder(self._event_store(), slug)
+                == self.instance_id
+            )
+        except Exception:
+            return False
+
+    def release_all(self) -> None:
+        """Release every held claim exactly once (idempotent by pop)."""
+        with self._lock:
+            slugs = list(self._held)
+            self._held.clear()
+        if coordination is None or self.state_dir is None:
+            return
+        for slug in slugs:
+            try:
+                coordination.release(
+                    self._event_store(),
+                    resource=slug,
+                    instance_id=self.instance_id,
+                )
+            except Exception:
+                # Best-effort: TTL expiry is the backstop for a failed
+                # release (never raise out of run_wave's finally).
+                pass
+
+
+def _is_live_orchestrator_backend(backend: Any) -> bool:
+    """True when a passed orchestrator backend is a REAL swapped seat.
+
+    None and the null HarnessOrchestratorBackend both mean "the live harness
+    is the orchestrator" (the no-op default): the wave engine must behave
+    byte-identically to pre-HS-2 in that case.
+    """
+    if backend is None:
+        return False
+    try:
+        from orchestrator_backend import HarnessOrchestratorBackend
+    except ImportError:
+        # Cannot classify; an explicitly passed backend is treated as live
+        # (fail loud downstream rather than silently ignoring the config).
+        return True
+    return not isinstance(backend, HarnessOrchestratorBackend)
+
+
+def _seat_tokens_spent(backend: Any) -> int:
+    """Best-effort token spend of an orchestrator seat backend (0 if the
+    backend does not meter). Never raises; never fabricates."""
+    getter = getattr(backend, "get_tokens_spent", None)
+    if not callable(getter):
+        return 0
+    try:
+        return max(0, int(getter()))
+    except Exception:
+        return 0
+
+
+def _quarantine_blocked_files(
+    driver: Any, item: Dict[str, Any], item_result: Dict[str, Any]
+) -> None:
+    """Restore a BLOCKED item's written files to their pre-build state.
+
+    Without this, Phase 4's writes stay in the working tree after a block and
+    a later `git add -A` (outside this loop) would ship the refused code.
+
+    Semantics (conservative, only the blocked item's own filesWritten):
+      - tracked file  -> `git checkout -- <file>` (restore index version)
+      - untracked file -> did not exist pre-build; deleted (ONLY on a clean
+        exit-1 "untracked" determination; any other ls-files failure is
+        AMBIGUOUS -> error record, never delete: fail-safe, not fail-delete)
+      - not inside a git worktree -> SKIP with an honest record (we cannot
+        know pre-build state without git; never guess-delete)
+
+    PATHSPEC GUARD (re-attack-proven destructive defect): quarantine acts on
+    actual FILE paths ONLY. Empty strings, "."/"..", and directory entries
+    are REJECTED with an error record -- `git checkout -- subdir` (or ".")
+    would revert OTHER items' uncommitted verified work under that tree, not
+    just the blocked item's files. On Windows, backslash separators are
+    normalized to "/" first so a tracked file is never misclassified
+    untracked (and deleted) because git could not match the pathspec.
+
+    Records the outcome in item_result["quarantine"]:
+      {"attempted", "restored", "deleted", "errors", "skipped_reason"}.
+    Windows + POSIX safe: git args go through _quote_arg; deletion is
+    pathlib. Errors are per-file and never raise out of the gate.
+    """
+    files = list(item_result.get("filesWritten") or [])
+    outcome = {
+        "attempted": bool(files),
+        "restored": [],
+        "deleted": [],
+        "errors": [],
+        "skipped_reason": None,
+    }
+    item_result["quarantine"] = outcome
+    if not files:
+        return
+    if driver is None:
+        outcome["skipped_reason"] = "no_driver"
+        return
+
+    root = item.get("repo") or item.get("workDir") or "."
+    try:
+        probe = driver.run_command(
+            "git rev-parse --is-inside-work-tree", cwd=root
+        )
+        inside = probe.exit_code == 0 and "true" in (probe.stdout or "").lower()
+    except Exception:
+        inside = False
+    if not inside:
+        outcome["skipped_reason"] = "not_a_git_worktree"
+        return
+
+    for f in files:
+        raw = f if isinstance(f, str) else str(f)
+        # Windows: normalize separators so the git pathspec matches the
+        # index entry (backslash pathspecs silently match nothing -> a
+        # tracked file would be misclassified untracked and DELETED).
+        # POSIX: backslash is a legal filename character; leave it alone.
+        norm = raw.replace("\\", "/") if os.name == "nt" else raw
+        # PATHSPEC GUARD: reject empty/dot specs outright -- "." or ""
+        # as a checkout pathspec reverts the WHOLE repo, destroying other
+        # items' uncommitted verified work. Record, never act.
+        stripped = norm.strip().rstrip("/")
+        if stripped in ("", ".", ".."):
+            outcome["errors"].append(
+                {"file": raw, "error": "rejected pathspec: empty or dot "
+                 "entry (would revert beyond the blocked item's own files)"}
+            )
+            continue
+        # Only touch paths that stay inside the item's root (never escape).
+        try:
+            _validate_file_path(norm, root)
+        except ValueError as exc:
+            outcome["errors"].append({"file": raw, "error": f"path validation: {exc}"})
+            continue
+        try:
+            target = Path(root) / norm
+            # PATHSPEC GUARD: directories are never quarantined -- a
+            # directory checkout reverts EVERY file under it, including
+            # other items' uncommitted verified work. Files only.
+            if target.is_dir():
+                outcome["errors"].append(
+                    {"file": raw, "error": "rejected pathspec: resolves to "
+                     "a directory (quarantine operates on file paths only; "
+                     "a directory spec would revert other items' files "
+                     "under it)"}
+                )
+                continue
+            # :(literal) pathspec magic: glob characters (*, ?, [) in an
+            # entry must never expand -- "*" would match (and checkout
+            # would revert) EVERY tracked file, not the blocked item's.
+            spec = _quote_arg(":(literal)" + norm)
+            tracked = driver.run_command(
+                "git ls-files --error-unmatch -- " + spec, cwd=root
+            )
+            if tracked.exit_code == 0:
+                restore = driver.run_command(
+                    "git checkout -- " + spec, cwd=root
+                )
+                if restore.exit_code == 0:
+                    outcome["restored"].append(f)
+                else:
+                    outcome["errors"].append(
+                        {"file": raw, "error": "git checkout failed: "
+                         + (restore.stderr or "")[:200]}
+                    )
+            elif tracked.exit_code == 1:
+                # Clean "untracked" determination (--error-unmatch exits 1
+                # on no-match): the file did not exist pre-build; remove it.
+                if target.exists() or target.is_symlink():
+                    target.unlink()
+                outcome["deleted"].append(f)
+            else:
+                # AMBIGUOUS classification (exit 128, index lock, ...):
+                # fail-SAFE, never fail-delete. Tracked content is index-
+                # recoverable; an uncertain file must not be destroyed.
+                outcome["errors"].append(
+                    {"file": raw, "error": "untracked classification "
+                     "ambiguous (git ls-files exit "
+                     f"{tracked.exit_code}): refusing to delete"}
+                )
+        except Exception as exc:
+            outcome["errors"].append({"file": raw, "error": str(exc)})
+
+
+def _orchestrator_final_catch(
+    backend: Any,
+    items: List[Dict[str, Any]],
+    result: Dict[str, Any],
+    state_dir: Optional[str] = None,
+    driver: Any = None,
+) -> None:
+    """HS-2: route the pre-ship final-catch decision through a configured
+    orchestrator seat (Phase 6, replacing 'deferred' ONLY when a seat is
+    configured).
+
+    Semantics (conservative, incumbent-safe):
+      - Only test-VERIFIED items are reviewed (failed items already do not
+        ship; they consume no seat decisions).
+      - verdict 'merge'  -> approved; item ships as today.
+      - verdict 'block'  -> verified flipped False; item does NOT ship;
+        journal updated so a resume cannot skip-and-ship it.
+      - 'escalate' / 'undetermined' / DECISION_FAILED -> degrade to today's
+        behavior (ship to branch; merge stays manual downstream) with an
+        honest per-item record. A seat outage NEVER fabricates a verdict
+        and NEVER blocks a test-proven item (crash-only degradation).
+
+    Mutates result in place: per-item 'final_catch' + 'adversarial_review'
+    (+ 'quarantine' on block), plus a wave-level 'orchestrator_review'
+    summary block with verdict counts, blocked detail, seat token spend,
+    and a gate_status flag ("active" | "degraded" when every decision
+    failed | "no_decisions" when nothing was verified to review).
+    """
+    from context_pack import ContextPack
+    from orchestrator_driver import OrchestratorDriver
+
+    orch = OrchestratorDriver(backend, schema_dir=str(DRIVER_DIR))
+    review = {
+        "seat": type(backend).__name__,
+        "model": getattr(backend, "model", None),
+        "decisions": 0,
+        "blocked": [],
+        "blocked_detail": [],
+        "decision_failed": [],
+        "verdict_counts": {
+            "merge": 0,
+            "block": 0,
+            "escalate": 0,
+            "undetermined": 0,
+            "decision_failed": 0,
+        },
+    }
+    slug_to_item = {
+        item.get("slug", f"item-{i}"): item for i, item in enumerate(items)
+    }
+    result["adversarial_review"] = "orchestrator_final_catch"
+
+    for item_result in result["built"]:
+        if not item_result.get("verified", False):
+            item_result["adversarial_review"] = "skipped_not_verified"
+            continue
+
+        slug = item_result.get("slug", "unknown")
+        item = slug_to_item.get(slug, {})
+        evidence = {
+            "item": json.dumps(
+                {
+                    "slug": slug,
+                    "prompt_excerpt": str(item.get("prompt", ""))[:1000],
+                    "ownsFiles": list(item.get("ownsFiles", [])),
+                    "filesWritten": item_result.get("filesWritten", []),
+                    "repairs": item_result.get("repairs", 0),
+                },
+                sort_keys=True,
+            ),
+            # NOTE (F6): this final_catch evidence is currently LOW-SIGNAL --
+            # test_passed is hardcoded True (only verified items reach this
+            # point) and no secret-scan/CI/branch-protection results are fed
+            # in yet. Future work should enrich it with the real gate outputs
+            # so the seat has something substantive to judge.
+            "verification_results": json.dumps(
+                {
+                    "test_passed": True,
+                    "test_exit_code": item_result.get("testExit"),
+                    "spot_check_failed": bool(
+                        item_result.get("spot_check_failed", False)
+                    ),
+                },
+                sort_keys=True,
+            ),
+        }
+        pack = ContextPack(
+            decision_type="final_catch",
+            sources_requested=(),
+            evidence=evidence,
+        )
+        try:
+            decision = orch.decide("final_catch", pack)
+        except Exception as exc:
+            # decide() never raises by contract; belt and braces anyway.
+            decision = {
+                "verdict": "DECISION_FAILED",
+                "evidence": [f"decide() raised: {exc}"],
+            }
+        review["decisions"] += 1
+        verdict = str(decision.get("verdict", "DECISION_FAILED"))
+        item_result["final_catch"] = verdict
+
+        if verdict == "block":
+            review["verdict_counts"]["block"] += 1
+            item_result["verified"] = False
+            item_result["adversarial_review"] = "blocked_by_orchestrator"
+            item_result["error"] = "orchestrator final_catch verdict: block"
+            review["blocked"].append(slug)
+            # Persist WHY (hold_reason, else first evidence citation) so the
+            # Report's blocked lane is actionable, not just a slug.
+            reason = decision.get("hold_reason")
+            if not reason:
+                evidence_list = decision.get("evidence")
+                if isinstance(evidence_list, list) and evidence_list:
+                    reason = str(evidence_list[0])
+            review["blocked_detail"].append(
+                {"slug": slug, "reason": reason or "final_catch verdict: block"}
+            )
+            if state_dir:
+                _write_journal_entry(
+                    state_dir,
+                    slug,
+                    "final_catch_blocked",
+                    {
+                        "verified": False,
+                        "testExit": item_result.get("testExit"),
+                        "final_catch": "block",
+                    },
+                    repo=item.get("repo"),
+                )
+            # QUARANTINE: refused code must not linger in the working tree.
+            _quarantine_blocked_files(driver, item, item_result)
+        elif verdict == "merge":
+            review["verdict_counts"]["merge"] += 1
+            item_result["adversarial_review"] = "approved_by_orchestrator"
+        elif verdict in ("escalate", "undetermined"):
+            review["verdict_counts"][verdict] += 1
+            item_result["adversarial_review"] = verdict
+        else:  # DECISION_FAILED (or anything unrecognized -> fail-safe path)
+            review["verdict_counts"]["decision_failed"] += 1
+            item_result["adversarial_review"] = "decision_failed_deferred"
+            review["decision_failed"].append(slug)
+
+    # Gate visibility: a 100%-failing seat must NOT look like an approving
+    # one. decisions>0 with every one DECISION_FAILED = the gate made zero
+    # successful decisions -> "degraded" (crash-only ship semantics are
+    # unchanged; this only makes the outage VISIBLE).
+    if review["decisions"] == 0:
+        review["gate_status"] = "no_decisions"
+    elif review["verdict_counts"]["decision_failed"] == review["decisions"]:
+        review["gate_status"] = "degraded"
+    else:
+        review["gate_status"] = "active"
+    review["seat_tokens_spent"] = _seat_tokens_spent(backend)
+
+    result["orchestrator_review"] = review
+
+
 def run_wave(
     driver: AgentDriver,
     manifest: Dict[str, Any],
@@ -422,6 +924,49 @@ def run_wave(
     state_dir: Optional[str] = None,
     git: Optional[Dict[str, str]] = None,
     resume_journal: bool = False,
+    orchestrator_backend: Any = None,
+) -> Dict[str, Any]:
+    """Run a full multi-item wave through an AgentDriver backend.
+
+    Public entry point: owns the wave's claim lifecycle (RS5). A single
+    wave-level instance id claims each dispatched item with a ttl sized to
+    the driver's command timeout (_claim_ttl_for_driver), HOLDS the claim
+    across build -> repair -> ship, fences repair/ship against lost claims,
+    and releases every claim exactly once here -- on every exit path,
+    including exceptions. All other semantics are documented on
+    _run_wave_inner (same signature plus claim_ctx).
+    """
+    claim_ctx = _ClaimContext(
+        state_dir=state_dir,
+        instance_id="wave-%s" % uuid.uuid4(),
+        ttl=_claim_ttl_for_driver(driver),
+    )
+    try:
+        return _run_wave_inner(
+            driver,
+            manifest,
+            state_dir=state_dir,
+            git=git,
+            resume_journal=resume_journal,
+            orchestrator_backend=orchestrator_backend,
+            claim_ctx=claim_ctx,
+        )
+    finally:
+        # RS5 F3: exactly-once release at the TRUE end of the lifecycle
+        # (build -> repair -> ship), success or terminal failure -- never
+        # in build_item's finally, which left Phase 5/7 claim-less.
+        claim_ctx.release_all()
+
+
+def _run_wave_inner(
+    driver: AgentDriver,
+    manifest: Dict[str, Any],
+    *,
+    state_dir: Optional[str] = None,
+    git: Optional[Dict[str, str]] = None,
+    resume_journal: bool = False,
+    orchestrator_backend: Any = None,
+    claim_ctx: "_ClaimContext",
 ) -> Dict[str, Any]:
     """Run a full multi-item wave through an AgentDriver backend.
 
@@ -443,6 +988,15 @@ def run_wave(
              ship phase is skipped.
         resume_journal: if True and state_dir exists, load journal and skip items
                        marked as verified (but re-run their tests for trust-but-verify).
+        orchestrator_backend: optional OrchestratorBackend for the DECISION seat
+                       (HS-2). None or the null HarnessOrchestratorBackend means
+                       the live harness stays the orchestrator: Phase 6 remains
+                       'deferred', byte-identical to pre-HS-2 (no key required,
+                       no backend called). A live backend (e.g. from
+                       build_orchestrator_backend(load_backend_config()) with a
+                       seats.orchestrator block) routes a final_catch decision
+                       per verified item through OrchestratorDriver.decide();
+                       verdict 'block' stops that item from shipping.
 
     Returns:
         dict with structure:
@@ -503,9 +1057,46 @@ def run_wave(
         "resume_stats": None,
     }
 
+    # Ensure the state directory exists up-front: claims (EventStore) and
+    # the journal both live under it. Without this, a fresh state_dir made
+    # EventStore construction raise inside the claim block -- which is a
+    # fail-closed SKIP (RS3-W N5), not a license to dispatch claim-less.
+    if state_dir:
+        try:
+            Path(state_dir).mkdir(parents=True, exist_ok=True)
+        except Exception:
+            # Downstream consumers (claims, journal, ceiling) each fail
+            # closed on their own if the directory is truly unusable.
+            pass
+
     # Extract items from manifest.
     items = manifest.get("items", [])
 
+    # SLUG-UNIQUENESS PREFLIGHT (RS3-W N7): slugs are the identity key for
+    # journal entries, coordination claims, and slug->item lookups. Duplicate
+    # slugs silently collide in all three (last-writer-wins journal, claim
+    # starvation, wrong item shipped). Reject loudly, before any dispatch.
+    seen_slugs = {}
+    duplicate_slugs = []
+    for item in items:
+        slug = item.get("slug")
+        if slug is None:
+            continue
+        if slug in seen_slugs:
+            duplicate_slugs.append(slug)
+        else:
+            seen_slugs[slug] = True
+    if duplicate_slugs:
+        result["aborted"] = True
+        result["abort_reason"] = "duplicate_slugs"
+        result["duplicate_slugs"] = sorted(set(duplicate_slugs))
+        result["error"] = (
+            "duplicate item slugs in manifest (slugs key the journal, "
+            "coordination claims, and ship lookups): "
+            + ", ".join(sorted(set(duplicate_slugs)))
+        )
+        return result
+
     # ========================================================================
     # PHASE 0 (optional): Resume - Load journal state and release stale leases
     # ========================================================================
@@ -692,11 +1283,27 @@ def run_wave(
         journal_key = (repo, slug)
         journal_entry = journal_state.get(journal_key)
         skipped_from_journal = False
-        if journal_entry and _should_skip_from_journal(journal_entry):
+        # RS3-W N10: resume-skip requires the journal entry to MATCH the
+        # current item's content fingerprint. Entries without a fingerprint
+        # (older format) or with a different one (a NEW item reusing a prior
+        # wave's slug) are rebuilt -- never silently inherited (fail-closed).
+        if journal_entry and _should_skip_from_journal(journal_entry) and (
+            journal_entry.get("fingerprint") == _item_fingerprint(item)
+        ):
             skipped_from_journal = True
             with resume_stats_lock:
                 resume_stats["skipped_from_journal"] += 1
 
+            # RS3-W N3: restore the journaled filesWritten so a resumed
+            # verified item can still SHIP (Phase 7) or be quarantined on a
+            # block. Without this the resumed item verified green but never
+            # produced a shipped record -> tracker stayed todo -> the item
+            # was re-selected and re-verified every wave, forever.
+            journal_files_written = [
+                str(f) for f in (journal_entry.get("filesWritten") or [])
+                if isinstance(f, str) and f
+            ]
+
             # Trust-but-verify: re-run the test for the journaled item
             test_cmd = item.get("testCmd", "")
             if test_cmd:
@@ -713,7 +1320,7 @@ def run_wave(
                                 "testExit": 0,
                                 "repairs": 0,
                                 "error": None,
-                                "filesWritten": [],
+                                "filesWritten": journal_files_written,
                                 "skipped_from_journal": True,
                                 "trust_verified": True,
                             },
@@ -778,19 +1385,14 @@ def run_wave(
             resume_stats["rebuilt"] += 1
 
         # Try to claim the item if state_dir is given (fail-closed on claim failure).
-        instance_id = f"wave-{uuid.uuid4()}"
-        claim_held = False
+        # RS5 F1/F3: the claim uses the WAVE-level instance id and a ttl
+        # sized to the driver's command timeout (not the 300s default), and
+        # is HELD across the item's full lifecycle (build -> repair -> ship).
+        # Release happens exactly once in run_wave's finally, never here.
+        instance_id = claim_ctx.instance_id
         if coordination is not None and state_dir is not None:
             try:
-                from state_store import store
-                db_path = Path(state_dir) / "state.db"
-                event_store = store.EventStore(str(db_path))
-                claim_held = coordination.try_claim(
-                    event_store,
-                    resource=slug,
-                    instance_id=instance_id,
-                )
-                if not claim_held:
+                if not claim_ctx.acquire(slug):
                     # Item is claimed by another instance; skip it.
                     return (
                         item_index,
@@ -802,11 +1404,28 @@ def run_wave(
                             "repairs": 0,
                             "error": "resource claimed by another instance",
                             "filesWritten": [],
+                            "claim_skipped": True,
                         },
                     )
-            except Exception:
-                # On exception, fail-closed: skip the item.
-                claim_held = False
+            except Exception as claim_exc:
+                # RS3-W N5: fail-CLOSED means SKIP, not dispatch. Falling
+                # through here dispatched the item WITHOUT holding a claim
+                # (two racing instances both hit the SQLite lock exception
+                # and both dispatched -> double-dispatch). Skip the item and
+                # record why; another instance (or the next wave) retries.
+                return (
+                    item_index,
+                    {
+                        "slug": slug,
+                        "dispatched": False,
+                        "verified": False,
+                        "testExit": None,
+                        "repairs": 0,
+                        "error": f"claim check failed (fail-closed skip): {claim_exc}",
+                        "filesWritten": [],
+                        "claim_skipped": True,
+                    },
+                )
 
         try:
             # Build the manifest item with policy.
@@ -826,12 +1445,17 @@ def run_wave(
                 "workerId": dispatch_result.get("workerId"),
             }
 
-            # Write journal entry for this item's outcome.
+            # Write journal entry for this item's outcome. filesWritten is
+            # persisted so a journal-resumed item can still ship/quarantine
+            # (RS3-W N3); fingerprint binds the entry to this item's content
+            # so a new item reusing the slug is never skipped (RS3-W N10).
             if state_dir:
                 _write_journal_entry(state_dir, slug, "dispatched", {
                     "verified": item_result["verified"],
                     "testExit": item_result["testExit"],
                     "instance_id": instance_id,
+                    "filesWritten": item_result["filesWritten"],
+                    "fingerprint": _item_fingerprint(item),
                 }, repo=repo)
 
             return (item_index, item_result)
@@ -858,35 +1482,43 @@ def run_wave(
                     "filesWritten": [],
                 },
             )
-        finally:
-            # Release the claim if held.
-            if claim_held and coordination is not None and state_dir is not None:
-                try:
-                    from state_store import store
-                    db_path = Path(state_dir) / "state.db"
-                    event_store = store.EventStore(str(db_path))
-                    coordination.release(event_store, resource=slug, instance_id=instance_id)
-                except Exception:
-                    pass
+        # NOTE (RS5 F3): no finally-release here. The claim protects the
+        # WHOLE lifecycle -- Phase 5 repair and Phase 7 ship run under it --
+        # and run_wave's finally releases it exactly once at the true end.
 
     # Run build in parallel.
     max_workers = min(8, len(items)) if items else 1
     with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
-        futures = [
-            executor.submit(build_item, i, item)
+        future_to_item = {
+            executor.submit(build_item, i, item): (i, item)
             for i, item in enumerate(items)
-        ]
-        for future in concurrent.futures.as_completed(futures):
+        }
+        for future in concurrent.futures.as_completed(future_to_item):
+            submitted_index, submitted_item = future_to_item[future]
             try:
                 item_index, item_result = future.result()
-                built_items.append((item_index, items[item_index], item_result))
+            except Exception as exc:
+                # RS3-W N7: build_item catches internally, but if the future
+                # STILL raises the item must NEVER silently vanish from both
+                # built and failed_items (it made green vacuously true).
+                # Record an honest failed result for the submitted item.
+                item_index = submitted_index
+                item_result = {
+                    "slug": submitted_item.get("slug", f"item-{submitted_index}"),
+                    "dispatched": False,
+                    "verified": False,
+                    "testExit": None,
+                    "repairs": 0,
+                    "error": f"executor exception: {exc}",
+                    "filesWritten": [],
+                }
+            built_items.append((item_index, items[item_index], item_result))
 
-                # Track failed items for repair.
-                if not item_result["verified"]:
-                    failed_items.append((item_index, items[item_index], item_result))
-            except Exception:
-                # Should not happen (build_item catches internally), but just in case.
-                pass
+            # Track failed items for repair. Claim-skipped items are NOT
+            # repair candidates: repair would dispatch them WITHOUT a claim
+            # (the double-dispatch the skip exists to prevent).
+            if not item_result["verified"] and not item_result.get("claim_skipped"):
+                failed_items.append((item_index, items[item_index], item_result))
 
     # Sort built_items by index to preserve order.
     built_items.sort(key=lambda x: x[0])
@@ -918,6 +1550,25 @@ def run_wave(
             workdir = item.get("workDir", ".")
             test_cmd = item.get("testCmd", "")
 
+            # RS5 F1b FENCE: never repair-dispatch an item whose claim we no
+            # longer hold (ttl lapsed and another instance may have reclaimed
+            # it and be dispatching the same files right now). Terminal
+            # honest abort -- recorded, never silently retried.
+            if not claim_ctx.fence_ok(slug):
+                item_result["error"] = (
+                    "claim lost before repair (fenced): not re-dispatched"
+                )
+                item_result["claim_lost"] = True
+                if state_dir:
+                    _write_journal_entry(state_dir, slug, "claim_lost", {
+                        "verified": False,
+                        "testExit": item_result.get("testExit"),
+                        "repairs": item_result.get("repairs", 0),
+                        "instance_id": claim_ctx.instance_id,
+                        "error": "claim lost before repair",
+                    }, repo=item.get("repo"))
+                continue
+
             # Build repair prompt: append test output to original prompt.
             original_prompt = item.get("prompt", "")
             test_output = f"\n\nTest failed with exit code {item_result['testExit']}.\n"
@@ -943,13 +1594,16 @@ def run_wave(
                 item_result["filesWritten"] = dispatch_result.get("filesWritten", [])
                 item_result["repairs"] += 1
 
-                # Write journal entry for repair outcome.
+                # Write journal entry for repair outcome (filesWritten +
+                # fingerprint: same resume contract as the dispatch entry).
                 if state_dir:
                     repo = item.get("repo")
                     _write_journal_entry(state_dir, slug, "repaired", {
                         "verified": item_result["verified"],
                         "testExit": item_result["testExit"],
                         "repairs": item_result["repairs"],
+                        "filesWritten": item_result["filesWritten"],
+                        "fingerprint": _item_fingerprint(item),
                     }, repo=repo)
 
                 # If still failed, mark for next round.
@@ -1012,13 +1666,50 @@ def run_wave(
                     item_result["spot_check_failed"] = True
 
     # ========================================================================
-    # PHASE 6: Adversarial review (deferred, not yet enforced)
+    # PHASE 6: Adversarial review / orchestrator final catch (HS-2)
     # ========================================================================
-    # Adversarial review is not yet implemented; mark all as deferred.
-    # (TODO in a later increment: real adversarial review dispatch via driver)
-    result["adversarial_review"] = "deferred"
-    for item_result in result["built"]:
-        item_result["adversarial_review"] = "deferred"
+    if _is_live_orchestrator_backend(orchestrator_backend):
+        # A configured orchestrator seat is LIVE: route a final_catch
+        # decision per verified item through the swapped backend.
+        _orchestrator_final_catch(
+            orchestrator_backend, items, result, state_dir=state_dir,
+            driver=driver,
+        )
+        # HS-2 hardening: the seat's own spend (up to 3 calls/item) counts
+        # against the ceiling too. Re-check AFTER decisions, BEFORE ship,
+        # including metered seat tokens. Runs ONLY on the live-seat path
+        # (no-op default keeps the pre-HS-2 check pattern byte-identical).
+        if cost_ceiling is not None and state_dir is not None:
+            # RS3-W N1: driver.get_tokens_spent() may be None BY CONTRACT
+            # (ClaudeCodeDriver always; CodexDriver at zero spend). Adding
+            # None + int crashed the flagship "Claude worker + swapped
+            # orchestrator seat" wave AFTER seat decisions, BEFORE ship --
+            # verified work never shipped and the item looped. With an
+            # unmetered driver: count the metered seat spend when there is
+            # any; otherwise pass None so cost_ceiling keeps its windowed
+            # ledger fallback (same contract as the Phase 3 check).
+            driver_spent = driver.get_tokens_spent()
+            seat_spent = _seat_tokens_spent(orchestrator_backend)
+            if driver_spent is None:
+                spent_arg = seat_spent if seat_spent > 0 else None
+            else:
+                spent_arg = driver_spent + seat_spent
+            ceiling_result = cost_ceiling.check(
+                spent=spent_arg,
+                trip=True,
+                state_dir=state_dir,
+            )
+            result["ceiling"] = ceiling_result
+            if ceiling_result.get("exceeded", False):
+                result["aborted"] = True
+                result["abort_reason"] = "cost_ceiling_exceeded_after_decisions"
+                return result
+    else:
+        # No configured seat: the live harness IS the orchestrator; review
+        # stays deferred to it. Byte-identical to pre-HS-2 behavior.
+        result["adversarial_review"] = "deferred"
+        for item_result in result["built"]:
+            item_result["adversarial_review"] = "deferred"
 
     # ========================================================================
     # PHASE 7: Per-repo ship (git operations, if configured)
@@ -1041,6 +1732,17 @@ def run_wave(
             if item_result.get("verified", False):
                 slug = item_result.get("slug")
                 if slug in slug_to_item:
+                    # RS5 F1b FENCE: an item whose claim lapsed and was
+                    # reclaimed must NOT ship -- the reclaiming instance may
+                    # ship its own build of the same slug (double-ship).
+                    # Items this wave never claimed (journal-resumed,
+                    # claim-gate disabled) always pass the fence.
+                    if not claim_ctx.fence_ok(slug):
+                        item_result["ship_error"] = (
+                            "claim lost before ship (fenced): not shipped"
+                        )
+                        item_result["claim_lost"] = True
+                        continue
                     item_index, original_item = slug_to_item[slug]
                     verified_items.append((item_index, original_item, item_result))
 
@@ -1166,6 +1868,25 @@ def run_wave(
                     commit_msg = f"Wave: {len(repo_items)} items verified"
                     commit_cmd = f"git commit -m {_quote_arg(commit_msg)}"
                     commit_result = driver.run_command(commit_cmd, cwd=repo_path)
+                    if commit_result.exit_code != 0 and (
+                        "nothing to commit"
+                        in ((commit_result.stdout or "") + (commit_result.stderr or "")).lower()
+                    ):
+                        # RS3-W N3: the staged content is ALREADY in HEAD
+                        # (e.g. a prior run committed then crashed before the
+                        # tracker was marked). The item is verified and its
+                        # work is committed: emit a terminal shipped record
+                        # instead of failing forever on re-commit.
+                        repo_ship_results.append({
+                            "repo": repo_path,
+                            "committed": False,
+                            "no_changes": True,
+                            "files_count": len(repo_items),
+                        })
+                        for _, _, item_result in repo_items:
+                            item_result["ship_no_changes"] = True
+                            shipped_items.append(item_result["slug"])
+                        continue
                     if commit_result.exit_code != 0:
                         # UNSTAGE P3: Commit failed; run git reset to unstage the files.
                         # This prevents staged-files residue on partial failure.
@@ -1220,6 +1941,22 @@ def run_wave(
                     # Mark items from this repo as shipped.
                     for _, _, item_result in repo_items:
                         shipped_items.append(item_result["slug"])
+                else:
+                    # RS3-W N3: a VERIFIED item with no files to add must
+                    # still reach a TERMINAL shipped record -- otherwise the
+                    # scheduler never marks the tracker and the item is
+                    # re-selected, re-verified, and "succeeds" every wave,
+                    # forever (recovery livelock). Nothing to commit is an
+                    # honest no-op ship, not a silent drop.
+                    repo_ship_results.append({
+                        "repo": repo_path,
+                        "committed": False,
+                        "no_changes": True,
+                        "files_count": len(repo_items),
+                    })
+                    for _, _, item_result in repo_items:
+                        item_result["ship_no_changes"] = True
+                        shipped_items.append(item_result["slug"])
 
             # Record shipped items and per-repo results.
             if shipped_items:
@@ -1251,8 +1988,13 @@ def result_to_report(wave_result: Dict[str, Any]) -> Dict[str, Any]:
     repairs_used = sum(item.get("repairs", 0) for item in built_items)
 
     # Determine if wave was fully green (all items verified and not aborted).
-    green = not wave_result.get("aborted", False) and all(
-        item.get("verified", False) for item in built_items
+    # RS3-W N7: green must be False when ZERO items ran -- all() over an
+    # empty list is vacuously True, which turned a wave where every item
+    # silently vanished into a green report.
+    green = (
+        not wave_result.get("aborted", False)
+        and len(built_items) > 0
+        and all(item.get("verified", False) for item in built_items)
     )
 
     report = {