@matt82198/aesop 0.3.2 → 0.4.0

package/state_store/coordination.py

@@ -18,7 +18,38 @@ from __future__ import annotations
 import time
 
 
-def fold_claims(events: list) -> dict[str, str]:
+def _read_claim_events(store) -> list:
+    """Read the claims stream from a StateAPI (.get) OR an EventStore (.read).
+
+    RS3-W: wave_loop passes a raw EventStore, which exposes read() but not
+    get(). try_claim called store.get() unconditionally, so EVERY claim
+    attempt raised AttributeError and fail-closed to False -- the claim gate
+    was dead code and items fell through to claim-less dispatch paths.
+    """
+    getter = getattr(store, "get", None)
+    if callable(getter):
+        return getter("claims")
+    return store.read("claims")
+
+
+def _claim_expired(ev: dict, payload: dict, now: float) -> bool:
+    """True when a claim_requested event is past its TTL (RS3-W N4).
+
+    A claim whose event timestamp plus payload ttl is in the past is treated
+    as released: a crashed holder's claim must not persist forever. Events
+    without a ttl or timestamp (legacy) never expire (backward compatible).
+    """
+    ttl = payload.get("ttl")
+    ts = ev.get("ts")
+    if ttl is None or ts is None:
+        return False
+    try:
+        return float(now) > float(ts) + float(ttl)
+    except (TypeError, ValueError):
+        return False
+
+
+def fold_claims(events: list, now: float | None = None) -> dict[str, str]:
     """Fold a claims stream into the current state of who holds each resource.
 
     Processes claim_requested and claim_released events to determine the winner
@@ -28,14 +59,23 @@ def fold_claims(events: list) -> dict[str, str]:
     subsequent claim_released event. If an instance releases and then re-claims,
     the re-claim is the new active claim for that instance.
 
+    TTL enforcement (RS3-W N4): a claim_requested event whose ts + payload
+    ttl is earlier than ``now`` is EXPIRED and ignored -- a crashed instance's
+    claim becomes claimable again after its TTL instead of persisting forever.
+    Legacy events without ts/ttl never expire.
+
     Args:
         events: list of event dicts from the claims stream
                 (as returned by EventStore.read('claims'))
+        now: reference time for TTL expiry (default: time.time())
 
     Returns:
         dict mapping resource_id -> holding_instance_id for all currently
         held resources. Empty dict if no claims exist or all have been released.
     """
+    if now is None:
+        now = time.time()
+
     # Track all claims by resource and instance (as list, to preserve order)
     claims_by_resource = {}  # resource -> {instance_id: [versions...]}
     # Track all releases by resource and instance (as sorted list)
@@ -48,6 +88,9 @@ def fold_claims(events: list) -> dict[str, str]:
         version = ev.get("version", 0)
 
         if etype == "claim_requested":
+            if _claim_expired(ev, payload, now):
+                # Expired claim: releasable/ignored (never a live holder).
+                continue
             resource = payload.get("resource")
             instance_id = payload.get("instance_id")
             if resource is not None and instance_id is not None:
@@ -119,7 +162,8 @@ def try_claim(store, resource: str, instance_id: str, ttl: float = 300.0) -> boo
     in the stream could later become the winner if the true holder releases.
 
     Args:
-        store: StateAPI or EventStore instance (must have append() and get() methods)
+        store: StateAPI or EventStore instance (must have append() plus
+               get() or read())
         resource: the resource identifier to claim (e.g., "wave_123", "lane_0", ...)
         instance_id: the instance identifier requesting the claim
         ttl: time-to-live in seconds (default 300s = 5min); embedded in the payload
@@ -140,7 +184,7 @@ def try_claim(store, resource: str, instance_id: str, ttl: float = 300.0) -> boo
         )
 
         # Re-read claims stream and fold to see current state
-        events = store.get("claims")
+        events = _read_claim_events(store)
         claims = fold_claims(events)
 
         # Check if we won
@@ -162,7 +206,23 @@ def try_claim(store, resource: str, instance_id: str, ttl: float = 300.0) -> boo
 
         return False
     except Exception:
-        # Fail-closed: any exception means we don't hold the claim
+        # Fail-closed: any exception means we don't hold the claim.
+        # RS5 F2: our claim_requested may already be in the stream (append
+        # succeeded, then the re-read/fold raised -- e.g. SQLite lock retries
+        # exhausted). Left un-retracted it becomes a PHANTOM holder: it wins
+        # as soon as the true holder releases, blocking the resource for a
+        # full TTL while we believe we failed. Best-effort retract our own
+        # request before returning False; if the retract itself fails there
+        # is nothing more we can do (TTL expiry remains the backstop).
+        try:
+            store.append(
+                "claims",
+                "claim_released",
+                {"resource": resource, "instance_id": instance_id},
+                actor=instance_id,
+            )
+        except Exception:
+            pass
         return False
 
 
@@ -201,7 +261,7 @@ def current_holder(store, resource: str) -> str | None:
         The instance_id of the current holder, or None if unclaimed.
     """
     try:
-        events = store.get("claims")
+        events = _read_claim_events(store)
         claims = fold_claims(events)
         return claims.get(resource)
     except Exception:

package/tools/ci_merge_wait.py

@@ -44,6 +44,7 @@ def run_gh_command(args):
     """
     Run gh CLI command; return parsed JSON or None if gh missing/error.
     Raises subprocess.CalledProcessError on non-zero exit.
+    N9 FIX: Raises TimeoutExpired on timeout (caller handles transient errors).
     """
     try:
         result = subprocess.run(
@@ -59,8 +60,8 @@ def run_gh_command(args):
             result.check_returncode()
         return json.loads(result.stdout) if result.stdout.strip() else None
     except subprocess.TimeoutExpired:
-        print("ERROR: gh command timed out")
-        sys.exit(1)
+        # N9 FIX: Re-raise timeout so caller (poll loop) can retry
+        raise
     except FileNotFoundError:
         print("ERROR: gh CLI not found on PATH")
         sys.exit(1)
@@ -69,12 +70,13 @@ def run_gh_command(args):
 def get_pr_status(pr_number):
     """
     Fetch PR status via gh pr view.
-    Returns dict with 'mergeable' and 'statusCheckRollup' keys.
+    Returns dict with 'mergeable', 'statusCheckRollup', and 'headRefOid' keys.
     Returns None if gh is missing.
+    F5 FIX: headRefOid is fetched to enable SHA-pinned merge via --match-head-commit.
     """
     data = run_gh_command([
         "gh", "pr", "view", str(pr_number),
-        "--json", "mergeable,statusCheckRollup"
+        "--json", "mergeable,statusCheckRollup,headRefOid"
     ])
     return data
 
@@ -119,8 +121,13 @@ def check_ci_status(status_rollup, allow_no_checks=False, expected_checks=None):
     Returns: ("pending", None), ("success", None), or ("failure", check_name)
     """
     # Fail-closed: empty rollup defaults to PENDING unless explicitly allowed
+    # F2 FIX: expected_checks takes PRECEDENCE over allow_no_checks
+    # If expected_checks is non-empty and rollup is empty, return pending (missing required checks)
     if not status_rollup:
-        if allow_no_checks:
+        if expected_checks:
+            # Expected checks are required but rollup is empty (they're missing)
+            return ("pending", None)
+        elif allow_no_checks:
             return ("success", None)
         else:
             # Empty rollup: fail-closed to PENDING (window where checks vanished/haven't registered yet)
@@ -148,12 +155,13 @@ def check_ci_status(status_rollup, allow_no_checks=False, expected_checks=None):
                 elif not conclusion or conclusion == "":
                     # COMPLETED with null/empty conclusion = fail-closed to PENDING (API anomaly)
                     check_status = "pending"
-                elif conclusion.upper() in ("NEUTRAL", "SKIPPED"):
-                    # Non-blocking advisory or skipped checks
+                elif conclusion.upper() in ("SUCCESS", "NEUTRAL", "SKIPPED"):
+                    # SUCCESS, NEUTRAL, SKIPPED are all passing/non-blocking
                     check_status = "success"
                 else:
-                    # Other conclusion values (unknown) = success (GitHub default)
-                    check_status = "success"
+                    # F6 FIX: Unknown conclusion values must fail-closed to PENDING (not success)
+                    # Future GitHub API conclusion values should not auto-succeed
+                    check_status = "pending"
             elif status in ("QUEUED", "IN_PROGRESS"):
                 check_status = "pending"
             else:
@@ -202,20 +210,16 @@ def check_ci_status(status_rollup, allow_no_checks=False, expected_checks=None):
                 # Expected check still pending
                 return ("pending", None)
 
-        # All expected checks passed - warn about any non-expected check failures but don't block
-        # Non-expected checks are informational/flaky third-party checks; they should not gate merge
-        non_expected_failures = []
-        for check_name, check_status in found_checks.items():
-            if check_name not in expected_checks and check_status == "failure":
-                non_expected_failures.append(check_name)
-
-        if non_expected_failures:
-            # Log the failures loudly but don't block merge
-            print(f"WARNING: Non-expected check(s) failed (informational, does not block merge): {', '.join(non_expected_failures)}")
-
-        # All expected checks passed - merge is allowed even if non-expected checks are failing
-        # (pending non-expected checks are also acceptable)
-        return ("success", None)
+        # All expected checks passed. Now check if ANY check failed (expected or not).
+        # CRITICAL (BL4 P2 fix): Any check failure blocks merge, not just expected checks.
+        # The expected_checks parameter only REQUIRES certain checks to be present and pass,
+        # but doesn't exempt other checks from the failure rule (fail-closed semantics).
+        if failed_checks:
+            return ("failure", failed_checks[0])
+        elif pending_checks:
+            return ("pending", None)
+        else:
+            return ("success", None)
 
     # Determine overall status (when expected_checks is not specified)
     if failed_checks:
@@ -226,19 +230,32 @@ def check_ci_status(status_rollup, allow_no_checks=False, expected_checks=None):
         return ("success", None)
 
 
-def merge_pr(pr_number, merge_method, dry_run=False):
+def merge_pr(pr_number, merge_method, dry_run=False, head_ref_oid=None):
     """
-    Merge the PR using gh pr merge.
+    Merge the PR using gh pr merge with SHA-pinned commit.
     This call is STRUCTURALLY UNREACHABLE unless CI is SUCCESS.
+    F5 FIX: Uses --match-head-commit <sha> to ensure merge targets the exact commit
+    whose CI checks passed, preventing TOCTOU race where a push between CI check and
+    merge could sneak in an untested commit.
+    RS-B FIX: Requires headRefOid to be present (fail-closed). Does NOT merge unpinned.
     If dry_run is True, report what would be done without actually merging.
     Returns True on success, False on error.
     """
+    # RS-B FIX: Fail-closed if headRefOid is missing - require the SHA pin or abort
+    if not head_ref_oid:
+        print("ERROR: Cannot merge without SHA pin (headRefOid missing). Aborting merge to prevent TOCTOU race.")
+        return False
+
     if dry_run:
-        print(f"[DRY-RUN] Would merge PR #{pr_number} with --{merge_method}")
+        print(f"[DRY-RUN] Would merge PR #{pr_number} with --{merge_method} --match-head-commit {head_ref_oid}")
         return True
 
+    merge_cmd = ["gh", "pr", "merge", str(pr_number), f"--{merge_method}"]
+    # F5 FIX: Add --match-head-commit to pin merge to the SHA that passed CI
+    merge_cmd.extend(["--match-head-commit", head_ref_oid])
+
     result = subprocess.run(
-        ["gh", "pr", "merge", str(pr_number), f"--{merge_method}"],
+        merge_cmd,
         capture_output=True,
         text=True,
         timeout=30,
@@ -467,10 +484,10 @@ def run_self_test():
         return False
     print("[OK] Expected check failed: returns FAILURE")
 
-    # Test 20b: Expected checks all pass, but non-expected check FAILED (P2 audit bug fix)
-    # FIXED: When --expect-checks is given, non-expected failures should NOT block merge.
-    # Non-expected checks are informational/flaky third-party checks; only expected checks gate merge.
-    # The failure is logged loudly as a warning, but merge is allowed.
+    # Test 20b: Expected checks all pass, but non-expected check FAILED (P2 audit bug fix - BL4)
+    # FIXED: When --expect-checks is given, ANY check failure (expected or not) must block merge.
+    # The --expect-checks parameter only REQUIRES certain checks to be present and pass,
+    # but doesn't exempt other checks from the failure rule (fail-closed semantics).
     rollup_expected_pass_noncxpected_fail = [
         {"name": "unit-tests", "status": "COMPLETED", "conclusion": "SUCCESS"},
         {"name": "integration-tests", "status": "COMPLETED", "conclusion": "SUCCESS"},
@@ -480,12 +497,12 @@ def run_self_test():
         rollup_expected_pass_noncxpected_fail,
         expected_checks={"unit-tests", "integration-tests"}
     )
-    if ci_status != "success":
-        print(f"FAIL: Expected 'success' with non-expected check failed (audit P2 fix), got '{ci_status}'")
+    if ci_status != "failure" or failed_check != "lint":
+        print(f"FAIL: Expected 'failure' with non-expected check failed (BL4 P2 fix), got '{ci_status}' / '{failed_check}'")
         return False
-    print("[OK] Non-expected check failed while expected pass: returns SUCCESS with warning (P2 audit fix)")
+    print("[OK] Non-expected check failed while expected pass: returns FAILURE (BL4 P2 audit fix)")
 
-    # Test 20c: Expected checks all pass, non-expected pending is OK (does not block)
+    # Test 20c: Expected checks all pass, but non-expected pending (must wait for completion)
     rollup_expected_pass_noncexpected_pending = [
         {"name": "unit-tests", "status": "COMPLETED", "conclusion": "SUCCESS"},
         {"name": "integration-tests", "status": "COMPLETED", "conclusion": "SUCCESS"},
@@ -495,10 +512,10 @@ def run_self_test():
         rollup_expected_pass_noncexpected_pending,
         expected_checks={"unit-tests", "integration-tests"}
     )
-    if ci_status != "success":
-        print(f"FAIL: Expected 'success' when all expected pass + non-expected pending, got '{ci_status}'")
+    if ci_status != "pending":
+        print(f"FAIL: Expected 'pending' when all expected pass + non-expected pending, got '{ci_status}'")
         return False
-    print("[OK] Expected all pass, non-expected pending: returns SUCCESS (pending non-expected is OK)")
+    print("[OK] Expected all pass, non-expected pending: returns PENDING (wait for all checks)")
 
     # Test 22: CheckRun STALE conclusion (invalidated check, counts as failure)
     stale_check = [
@@ -659,7 +676,16 @@ def main():
             print(f"TIMEOUT: CI did not conclude within {args.timeout}s")
             sys.exit(3)
 
-        status = get_pr_status(args.pr_number)
+        # N9 FIX: Catch transient gh errors (CalledProcessError, TimeoutExpired) and retry
+        try:
+            status = get_pr_status(args.pr_number)
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
+            print(f"TRANSIENT ERROR: gh command failed ({type(e).__name__}), retrying...")
+            # Treat as "status unknown this tick", continue polling
+            print(f"CI PENDING ({elapsed:.0f}s elapsed)... waiting {args.poll}s")
+            time.sleep(args.poll)
+            continue
+
         if status is None:
             print("ERROR: gh CLI check failed")
             sys.exit(1)
@@ -677,7 +703,16 @@ def main():
         elif ci_status == "success":
             # SUCCESS: CI is green, re-check immediately before proceeding
             print(f"CI GREEN: All checks passed. Re-checking status before merge...")
-            final_check = get_pr_status(args.pr_number)
+            # N9 FIX: Also catch transient errors on final check
+            try:
+                final_check = get_pr_status(args.pr_number)
+            except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
+                print(f"TRANSIENT ERROR: final gh check failed ({type(e).__name__}), retrying...")
+                # Treat as status change, go back to polling
+                print(f"CI PENDING ({elapsed:.0f}s elapsed)... waiting {args.poll}s")
+                time.sleep(args.poll)
+                continue
+
             if final_check is None:
                 print("ERROR: final status check failed")
                 sys.exit(1)
@@ -695,19 +730,30 @@ def main():
 
             # SUCCESS: CI is still green, proceed to merge
             # This merge call is STRUCTURALLY UNREACHABLE unless ci_status == "success"
+            # F5 FIX: Capture headRefOid for SHA-pinned merge (prevents TOCTOU)
+            head_ref_oid = final_check.get("headRefOid")
             if args.dry_run:
                 print(f"[DRY-RUN] PR #{args.pr_number} CI is green, would merge...")
             else:
                 print(f"CI CONFIRMED GREEN. Merging PR #{args.pr_number}...")
 
-            if merge_pr(args.pr_number, args.merge_method, dry_run=args.dry_run):
+            # N9 FIX: Catch TimeoutExpired from merge subprocess and treat as transient error
+            try:
+                merge_result = merge_pr(args.pr_number, args.merge_method, dry_run=args.dry_run, head_ref_oid=head_ref_oid)
+            except subprocess.TimeoutExpired:
+                print(f"TRANSIENT ERROR: merge command timed out, retrying...")
+                print(f"CI PENDING ({elapsed:.0f}s elapsed)... waiting {args.poll}s")
+                time.sleep(args.poll)
+                continue
+
+            if merge_result:
                 if args.dry_run:
                     print(f"[DRY-RUN] PR #{args.pr_number} merge command would succeed")
                 else:
                     print(f"MERGED: PR #{args.pr_number} merged successfully")
                 sys.exit(0)
             else:
-                print("ERROR: merge failed (PR state changed?)")
+                print("ERROR: merge failed (PR state changed? or missing SHA pin?)")
                 sys.exit(1)
 
         # Still pending: wait and retry

package/tools/ci_shard_runner.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+"""
+Shard-aware Python test runner for CI.
+
+Distributes tracked test files across N shards using round-robin assignment.
+Shard ID and total shard count come from environment variables or CLI args.
+
+Solves the multiprocessing spawn-loop problem on Windows: inline heredocs set
+__main__ = "<string>", and spawn-mode children recursively re-import and re-execute
+the script. A guarded __main__ block prevents re-execution in child processes.
+
+Usage:
+  python tools/ci_shard_runner.py                   # Uses SHARD_ID, TOTAL_SHARDS env vars
+  python tools/ci_shard_runner.py 0 4               # Or pass shard_id and total_shards as args
+
+Exit codes:
+  0: All tests passed
+  1: Test failures, import failures, distribution errors, or no tests collected
+"""
+import os
+import subprocess
+import sys
+import unittest
+from pathlib import Path
+
+
+def distribute_shards(test_files, shard_id, total_shards):
+    """Distribute test files across shards using round-robin.
+
+    Args:
+        test_files: sorted list of test module names (stems, e.g., 'test_foo')
+        shard_id: integer 0..total_shards-1
+        total_shards: total number of shards
+
+    Returns:
+        list of test module names assigned to this shard
+    """
+    return [test_files[i] for i in range(len(test_files)) if i % total_shards == shard_id]
+
+
+def main():
+    """Run Python tests for the assigned shard."""
+    # Ensure repo root is in sys.path for test imports
+    script_dir = Path(__file__).resolve().parent
+    root = script_dir.parent
+    if str(root) not in sys.path:
+        sys.path.insert(0, str(root))
+
+    # Parse shard ID and total shards from env or CLI args
+    if len(sys.argv) >= 3:
+        try:
+            shard_id = int(sys.argv[1])
+            total_shards = int(sys.argv[2])
+        except (ValueError, IndexError):
+            print("ERROR: Usage: python ci_shard_runner.py [shard_id total_shards]")
+            sys.exit(1)
+    else:
+        try:
+            shard_id = int(os.environ.get("SHARD_ID", os.environ.get("MATRIX_PYTHON_SHARD", "0")))
+            total_shards = int(os.environ.get("TOTAL_SHARDS", "4"))
+        except ValueError:
+            print("ERROR: SHARD_ID and TOTAL_SHARDS must be integers")
+            sys.exit(1)
+
+    # Get tracked test files only (exclude WIP untracked files)
+    try:
+        result = subprocess.run(
+            ["git", "ls-files", "tests/test_*.py"],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        tracked_files = result.stdout.strip().split("\n") if result.stdout.strip() else []
+        # Extract just the stem for test discovery
+        test_files = sorted(set(Path(f).stem for f in tracked_files if f))
+    except Exception as e:
+        print(f"ERROR: Failed to get tracked test files: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    # Distribute files across shards (round-robin for balance)
+    shard_files = distribute_shards(test_files, shard_id, total_shards)
+
+    if not shard_files:
+        print(
+            f"ERROR: No tests assigned to shard {shard_id} (total test files: {len(test_files)})",
+            file=sys.stderr,
+        )
+        print(f"This indicates a configuration error in the shard distribution.", file=sys.stderr)
+        sys.exit(1)
+
+    print(f"Shard {shard_id}: running {len(shard_files)} tests")
+    loader = unittest.TestLoader()
+    suite = unittest.TestSuite()
+    failed_imports = []
+
+    for test_name in shard_files:
+        try:
+            module = __import__(f"tests.{test_name}", fromlist=[test_name])
+            suite.addTests(loader.loadTestsFromModule(module))
+        except Exception as e:
+            failed_imports.append((test_name, str(e)))
+            print(f"ERROR: Failed to load {test_name}: {e}", file=sys.stderr)
+
+    # If there were import failures, exit immediately
+    if failed_imports:
+        print(
+            f"\n{len(failed_imports)} test module(s) failed to import:",
+            file=sys.stderr,
+        )
+        for name, error in failed_imports:
+            print(f"  - {name}: {error}", file=sys.stderr)
+        sys.exit(1)
+
+    # If no tests were collected, this is also an error
+    if suite.countTestCases() == 0:
+        print(
+            f"ERROR: No tests were collected for shard {shard_id}",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    runner = unittest.TextTestRunner(verbosity=2)
+    result = runner.run(suite)
+    sys.exit(0 if result.wasSuccessful() else 1)
+
+
+if __name__ == "__main__":
+    main()