@hunter-harness/workflow-harness 0.2.42 → 0.2.44

package/harness/bundles/general/claude-code/scripts/harness_test_runner.py

@@ -29,6 +29,7 @@ if str(SCRIPT_DIR) not in sys.path:
     sys.path.insert(0, str(SCRIPT_DIR))
 
 import harness_environment as henv
+import harness_process as hprocess
 
 DEFAULT_MAX_WORKERS = 2
 DEFAULT_TIMEOUT_SECONDS = 300.0
@@ -37,7 +38,10 @@ DEFAULT_RESOURCE_MODULES = (
     "test_harness_integration.py",
     "test_harness_service.py",
 )
-DEFAULT_DETACHED_SERVICE_MODULES = ("test_harness_service.py",)
+DEFAULT_DETACHED_SERVICE_MODULES = (
+    "test_harness_service.py",
+    "test_harness_runtime.py",
+)
 TRUTHY_VALUES = frozenset({"1", "true", "yes", "on"})
 
 
@@ -136,35 +140,30 @@ def build_unittest_plan(
 
 
 def _pid_is_running(pid: int) -> bool:
-    if pid <= 0:
-        return False
-    if pid == os.getpid():
-        return True
-    if os.name == "nt":
-        process_query_limited_information = 0x1000
-        kernel32 = ctypes.windll.kernel32
-        kernel32.OpenProcess.restype = ctypes.c_void_p
-        handle = kernel32.OpenProcess(
-            process_query_limited_information,
-            False,
-            pid,
-        )
-        if not handle:
-            return False
-        try:
-            exit_code = ctypes.c_uint32()
-            if not kernel32.GetExitCodeProcess(handle, ctypes.byref(exit_code)):
-                return False
-            return int(exit_code.value) == 259
-        finally:
-            kernel32.CloseHandle(ctypes.c_void_p(handle))
+    return hprocess.is_pid_alive(pid)
+
+
+def _verify_recorded_owner(owner: Mapping[str, object]) -> bool | None:
+    """Verify a legacy lock owner through the canonical process provider."""
+
     try:
-        os.kill(pid, 0)
-    except ProcessLookupError:
-        return False
-    except PermissionError:
+        pid = int(owner.get("pid") or 0)
+    except (TypeError, ValueError):
+        return None
+    observed = hprocess.observe_process_identity(pid)
+    decision = hprocess.verify_process_identity(
+        {
+            "pid": pid,
+            "createdAt": owner.get("startedAt"),
+            "executable": owner.get("executable"),
+        },
+        observed,
+    )
+    if decision.get("ok") is True:
         return True
-    return True
+    if decision.get("reasonCode") == "IDENTITY_UNVERIFIABLE":
+        return None
+    return False
 
 
 def classify_test_lock(path: Path) -> dict[str, object]:
@@ -189,16 +188,17 @@ def classify_test_lock(path: Path) -> dict[str, object]:
             "path": str(path),
             "token": payload.get("token"),
         }
-    if time.time() < float(expires):
-        return {
-            "classification": "ACTIVE",
-            "path": str(path),
-            "token": payload.get("token"),
-            "owner": owner,
-        }
+    unexpired = time.time() < float(expires)
     if not isinstance(owner, dict) or any(
         not owner.get(field) for field in ("pid", "executable", "startedAt")
     ):
+        if unexpired:
+            return {
+                "classification": "ACTIVE",
+                "path": str(path),
+                "token": payload.get("token"),
+                "owner": owner,
+            }
         return {
             "classification": "OWNER_IDENTITY_INCOMPLETE",
             "path": str(path),
@@ -207,6 +207,13 @@ def classify_test_lock(path: Path) -> dict[str, object]:
     try:
         owner_pid = int(owner["pid"])
     except (TypeError, ValueError):
+        if unexpired:
+            return {
+                "classification": "ACTIVE",
+                "path": str(path),
+                "token": payload.get("token"),
+                "owner": owner,
+            }
         return {
             "classification": "OWNER_IDENTITY_INCOMPLETE",
             "path": str(path),
@@ -219,13 +226,14 @@ def classify_test_lock(path: Path) -> dict[str, object]:
             "token": payload.get("token"),
             "owner": owner,
         }
-    verified = henv.hservice.verify_process_identity(
-        {
-            "pid": owner_pid,
-            "startedAt": owner["startedAt"],
-            "processIdentity": {"executable": owner["executable"]},
+    if unexpired:
+        return {
+            "classification": "ACTIVE",
+            "path": str(path),
+            "token": payload.get("token"),
+            "owner": owner,
         }
-    )
+    verified = _verify_recorded_owner(owner)
     classification = (
         "OWNER_ACTIVE"
         if verified is True
@@ -244,7 +252,7 @@ def classify_test_lock(path: Path) -> dict[str, object]:
 
 
 def reap_test_lock(path: Path) -> dict[str, object]:
-    """Reap only an expired lock whose exact recorded owner is absent."""
+    """Reap only a lock whose exact recorded owner is definitely absent."""
     classification = classify_test_lock(path)
     reaped = classification.get("classification") == "RECLAIMABLE"
     if reaped:
@@ -330,9 +338,9 @@ class TestRunLock:
                 )
             else:
                 now = time.time()
-                created = henv.hservice.get_process_create_time(os.getpid())
+                created = hprocess.get_process_create_time(os.getpid())
                 executable = (
-                    henv.hservice.get_process_executable(os.getpid())
+                    hprocess.get_process_executable(os.getpid())
                     or str(Path(sys.executable).resolve())
                 )
                 payload = {
@@ -484,6 +492,20 @@ class _WindowsBasicAccountingInformation(ctypes.Structure):
     ]
 
 
+_WINDOWS_JOB_PROCESS_ID_CAPACITY = 4096
+
+
+class _WindowsJobProcessIdList(ctypes.Structure):
+    _fields_ = [
+        ("NumberOfAssignedProcesses", ctypes.c_uint32),
+        ("NumberOfProcessIdsInList", ctypes.c_uint32),
+        (
+            "ProcessIdList",
+            ctypes.c_size_t * _WINDOWS_JOB_PROCESS_ID_CAPACITY,
+        ),
+    ]
+
+
 class _WindowsKillOnCloseJob:
     """Best-effort Windows Job Object that kills descendants when closed."""
 
@@ -491,9 +513,11 @@ class _WindowsKillOnCloseJob:
     _JOB_OBJECT_LIMIT_BREAKAWAY_OK = 0x00000800
     _JOB_OBJECT_EXTENDED_LIMIT_INFORMATION_CLASS = 9
     _JOB_OBJECT_BASIC_ACCOUNTING_INFORMATION_CLASS = 1
+    _JOB_OBJECT_BASIC_PROCESS_ID_LIST_CLASS = 3
 
-    def __init__(self) -> None:
+    def __init__(self, *, allow_breakaway: bool = False) -> None:
         self.handle: int | None = None
+        self.allow_breakaway = allow_breakaway
 
     def assign(self, process: subprocess.Popen[object]) -> bool:
         if os.name != "nt":
@@ -506,7 +530,11 @@ class _WindowsKillOnCloseJob:
         info = _WindowsExtendedLimitInformation()
         info.BasicLimitInformation.LimitFlags = (
             self._JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
-            | self._JOB_OBJECT_LIMIT_BREAKAWAY_OK
+            | (
+                self._JOB_OBJECT_LIMIT_BREAKAWAY_OK
+                if self.allow_breakaway
+                else 0
+            )
         )
         configured = kernel32.SetInformationJobObject(
             ctypes.c_void_p(handle),
@@ -535,26 +563,73 @@ class _WindowsKillOnCloseJob:
         ctypes.windll.kernel32.CloseHandle(ctypes.c_void_p(self.handle))
         self.handle = None
 
-    def terminate_and_wait(self, timeout_seconds: float = 5.0) -> None:
+    def terminate_and_wait(self, timeout_seconds: float = 5.0) -> bool:
         if self.handle is None:
-            return
+            return True
         kernel32 = ctypes.windll.kernel32
         handle = ctypes.c_void_p(self.handle)
-        kernel32.TerminateJobObject(handle, 1)
         deadline = time.monotonic() + max(0.0, timeout_seconds)
-        while time.monotonic() < deadline:
-            accounting = _WindowsBasicAccountingInformation()
-            queried = kernel32.QueryInformationJobObject(
-                handle,
-                self._JOB_OBJECT_BASIC_ACCOUNTING_INFORMATION_CLASS,
-                ctypes.byref(accounting),
-                ctypes.sizeof(accounting),
-                None,
-            )
-            if not queried or accounting.ActiveProcesses == 0:
-                break
-            time.sleep(0.02)
-        self.close()
+        process_ids = _WindowsJobProcessIdList()
+        process_list_queried = kernel32.QueryInformationJobObject(
+            handle,
+            self._JOB_OBJECT_BASIC_PROCESS_ID_LIST_CLASS,
+            ctypes.byref(process_ids),
+            ctypes.sizeof(process_ids),
+            None,
+        )
+        process_list_complete = bool(
+            process_list_queried
+            and process_ids.NumberOfAssignedProcesses
+            == process_ids.NumberOfProcessIdsInList
+        )
+        kernel32.OpenProcess.restype = ctypes.c_void_p
+        process_handles: list[int] = []
+        if process_list_complete:
+            for index in range(int(process_ids.NumberOfProcessIdsInList)):
+                process_handle = kernel32.OpenProcess(
+                    0x00100000,
+                    False,
+                    int(process_ids.ProcessIdList[index]),
+                )
+                if process_handle:
+                    process_handles.append(int(process_handle))
+                else:
+                    process_list_complete = False
+        terminated = bool(kernel32.TerminateJobObject(handle, 1))
+        handles_signaled = process_list_complete
+        try:
+            for process_handle in process_handles:
+                remaining_ms = max(
+                    0,
+                    int((deadline - time.monotonic()) * 1000),
+                )
+                if (
+                    kernel32.WaitForSingleObject(
+                        ctypes.c_void_p(process_handle),
+                        remaining_ms,
+                    )
+                    != 0
+                ):
+                    handles_signaled = False
+            accounting_drained = False
+            while time.monotonic() < deadline:
+                accounting = _WindowsBasicAccountingInformation()
+                queried = kernel32.QueryInformationJobObject(
+                    handle,
+                    self._JOB_OBJECT_BASIC_ACCOUNTING_INFORMATION_CLASS,
+                    ctypes.byref(accounting),
+                    ctypes.sizeof(accounting),
+                    None,
+                )
+                if queried and accounting.ActiveProcesses == 0:
+                    accounting_drained = True
+                    break
+                time.sleep(0.02)
+        finally:
+            for process_handle in process_handles:
+                kernel32.CloseHandle(ctypes.c_void_p(process_handle))
+            self.close()
+        return terminated and handles_signaled and accounting_drained
 
 
 class _WindowsProcessEntry(ctypes.Structure):
@@ -709,50 +784,22 @@ def _terminate_process_tree(
     *,
     windows_job: _WindowsKillOnCloseJob | None,
 ) -> None:
+    """Legacy seam retained for callers; ownership must be provider-backed."""
+
     if windows_job is not None and windows_job.handle is not None:
         windows_job.terminate_and_wait()
-    elif os.name == "nt":
-        subprocess.run(
-            ["taskkill", "/PID", str(process.pid), "/T", "/F"],
-            stdout=subprocess.DEVNULL,
-            stderr=subprocess.DEVNULL,
-            check=False,
-            timeout=10,
-            creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
-        )
     else:
-        try:
-            os.killpg(process.pid, signal.SIGTERM)
-        except ProcessLookupError:
-            pass
-        try:
-            process.wait(timeout=2)
-        except subprocess.TimeoutExpired:
-            try:
-                os.killpg(process.pid, signal.SIGKILL)
-            except ProcessLookupError:
-                pass
-    if process.poll() is None:
-        process.kill()
-    try:
-        process.wait(timeout=5)
-    except subprocess.TimeoutExpired:
-        pass
+        raise RuntimeError("PROCESS_OWNERSHIP_REQUIRED")
 
 
 def _cleanup_descendants_after_success(
     process: subprocess.Popen[object],
     *,
     windows_job: _WindowsKillOnCloseJob | None,
-) -> None:
+) -> bool:
     if windows_job is not None and windows_job.handle is not None:
-        windows_job.terminate_and_wait()
-        return
-    if os.name != "nt":
-        try:
-            os.killpg(process.pid, signal.SIGTERM)
-        except ProcessLookupError:
-            pass
+        return windows_job.terminate_and_wait()
+    return False
 
 
 def run_managed_command(
@@ -774,74 +821,84 @@ def run_managed_command(
     child_env.setdefault("PYTHONUNBUFFERED", "1")
     lower_process_priority()
 
-    popen_kwargs: dict[str, object] = {
-        "cwd": str(Path(cwd).resolve()),
-        "env": child_env,
-        "shell": False,
-    }
     output_stream = tempfile.TemporaryFile(mode="w+b") if capture_output else None
-    if output_stream is not None:
-        popen_kwargs["stdout"] = output_stream
-        popen_kwargs["stderr"] = subprocess.STDOUT
-    if os.name == "nt":
-        popen_kwargs["creationflags"] = getattr(
-            subprocess,
-            "CREATE_NEW_PROCESS_GROUP",
-            0,
-        )
-    else:
-        popen_kwargs["start_new_session"] = True
-
     started = time.monotonic()
+    spawned = None
+    process: subprocess.Popen[object] | None = None
+    cleanup_complete = False
     try:
-        process = subprocess.Popen(list(argv), **popen_kwargs)
-        windows_job = (
-            _WindowsKillOnCloseJob()
-            if os.name == "nt" and not allow_detached_processes
-            else None
+        spawned = hprocess.spawn_structured_argv(
+            list(argv),
+            cwd=Path(cwd).resolve(),
+            environment=child_env,
+            owner_token=f"test-runner:{uuid.uuid4().hex}",
+            stdout=output_stream,
+            stderr=(subprocess.STDOUT if output_stream is not None else None),
         )
-        isolated = bool(windows_job and windows_job.assign(process))
-        known_windows_descendants: set[int] = set()
-        windows_snapshot_ok = True
-        if os.name != "nt":
-            isolated = True
+        process = spawned.process
+        isolated = bool(
+            isinstance(spawned.ownershipProof, dict)
+            and spawned.ownershipProof.get("membersComplete") is True
+        )
+
+        def provider_cleanup() -> bool:
+            hprocess.capture_owned_members(spawned)
+            result = hprocess.terminate_owned_tree(
+                spawned.attestation,
+                spawned.ownershipProof,
+                {"graceSeconds": 3.0},
+            )
+            return bool(result.get("cleanupComplete"))
+
+        def wait_owned_members_gone() -> bool:
+            proof = spawned.ownershipProof
+            if not isinstance(proof, dict):
+                return False
+            members = proof.get("members")
+            if not isinstance(members, list):
+                return False
+            pids = {
+                int(item["pid"])
+                for item in members
+                if isinstance(item, dict) and isinstance(item.get("pid"), int)
+            }
+            deadline = time.monotonic() + 5.0
+            while pids and time.monotonic() < deadline:
+                pids = {pid for pid in pids if hprocess.is_pid_alive(pid)}
+                if pids:
+                    time.sleep(0.05)
+            return not pids
+
         timed_out = False
         try:
             deadline = time.monotonic() + max(0.01, float(timeout_seconds))
             while True:
-                if os.name == "nt":
-                    windows_snapshot_ok = (
-                        _update_windows_descendants(
-                            process.pid,
-                            known_windows_descendants,
-                        )
-                        and windows_snapshot_ok
-                    )
                 polled = process.poll()
                 if polled is not None:
                     returncode = polled
                     break
                 if time.monotonic() >= deadline:
                     timed_out = True
-                    _terminate_process_tree(process, windows_job=windows_job)
+                    cleanup_complete = provider_cleanup()
                     returncode = 124
                     break
                 time.sleep(0.05)
         except KeyboardInterrupt:
-            _terminate_process_tree(process, windows_job=windows_job)
-            if os.name == "nt":
-                _terminate_tracked_windows_descendants(known_windows_descendants)
+            cleanup_complete = provider_cleanup()
             raise
         if not timed_out:
-            _cleanup_descendants_after_success(process, windows_job=windows_job)
-        if os.name == "nt":
-            _update_windows_descendants(process.pid, known_windows_descendants)
-            _terminate_tracked_windows_descendants(known_windows_descendants)
-            isolated = (
-                windows_snapshot_ok
-                if allow_detached_processes
-                else isolated and windows_snapshot_ok
-            )
+            cleanup_complete = provider_cleanup()
+            # A named Windows Job remains an owned capability after the leader
+            # exits. Closing that provider handle drains any attested children;
+            # PID-only or POSIX group fallbacks stay fail-closed.
+            if not cleanup_complete and os.name == "nt":
+                spawned.close()
+                cleanup_complete = wait_owned_members_gone()
+        if process.poll() is None:
+            try:
+                process.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                cleanup_complete = False
         duration = time.monotonic() - started
         output_tail = ""
         if output_stream is not None:
@@ -854,10 +911,28 @@ def run_managed_command(
             returncode=returncode,
             timed_out=timed_out,
             duration_seconds=duration,
-            process_tree_isolated=isolated,
+            process_tree_isolated=isolated and cleanup_complete,
             output_tail=output_tail,
         )
     finally:
+        if (
+            not cleanup_complete
+            and process is not None
+            and process.poll() is None
+            and spawned is not None
+        ):
+            try:
+                hprocess.capture_owned_members(spawned)
+                cleanup = hprocess.terminate_owned_tree(
+                    spawned.attestation,
+                    spawned.ownershipProof,
+                    {"graceSeconds": 3.0},
+                )
+                cleanup_complete = bool(cleanup.get("cleanupComplete"))
+            except Exception:
+                cleanup_complete = False
+        if spawned is not None:
+            spawned.close()
         if output_stream is not None:
             output_stream.close()
 
@@ -1320,7 +1395,10 @@ def build_parser() -> argparse.ArgumentParser:
     exec_parser.add_argument(
         "--allow-detached-processes",
         action="store_true",
-        help="Use exact Windows PID-lineage tracking instead of a Job Object.",
+        help=(
+            "Permit explicit Windows Job breakaway while retaining ordinary "
+            "descendants and tracking escaped process lineages."
+        ),
     )
     exec_parser.set_defaults(handler=_run_exec)