agent-control-plane 0.1.9 → 0.1.12

package/tools/bin/agent-project-run-codex-resilient

@@ -18,6 +18,7 @@ Options:
   --max-resume-attempts <count>           Maximum resume attempts after interruption
   --auth-refresh-timeout-seconds <secs>   How long to wait for refreshed auth before failing
   --auth-refresh-poll-seconds <secs>      Poll interval while waiting for refreshed auth
+  --stall-seconds <secs>                  Fail if Codex stops producing output for too long
   --help                                  Show this help
 EOF
 }
@@ -35,6 +36,8 @@ max_resume_attempts="${ACP_CODEX_MAX_RESUME_ATTEMPTS:-${F_LOSNING_CODEX_MAX_RESU
 auth_refresh_timeout_seconds="${ACP_CODEX_AUTH_REFRESH_TIMEOUT_SECONDS:-${F_LOSNING_CODEX_AUTH_REFRESH_TIMEOUT_SECONDS:-900}}"
 auth_refresh_poll_seconds="${ACP_CODEX_AUTH_REFRESH_POLL_SECONDS:-${F_LOSNING_CODEX_AUTH_REFRESH_POLL_SECONDS:-10}}"
 max_quota_autoswitch_attempts="${ACP_CODEX_MAX_AUTOSWITCH_ATTEMPTS:-${F_LOSNING_CODEX_MAX_AUTOSWITCH_ATTEMPTS:-1}}"
+codex_progress_heartbeat_seconds="${ACP_CODEX_PROGRESS_HEARTBEAT_SECONDS:-${F_LOSNING_CODEX_PROGRESS_HEARTBEAT_SECONDS:-30}}"
+codex_stall_seconds="${ACP_CODEX_STALL_SECONDS:-${F_LOSNING_CODEX_STALL_SECONDS:-300}}"
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
@@ -50,6 +53,7 @@ while [[ $# -gt 0 ]]; do
     --max-resume-attempts) max_resume_attempts="${2:-}"; shift 2 ;;
     --auth-refresh-timeout-seconds) auth_refresh_timeout_seconds="${2:-}"; shift 2 ;;
     --auth-refresh-poll-seconds) auth_refresh_poll_seconds="${2:-}"; shift 2 ;;
+    --stall-seconds) codex_stall_seconds="${2:-}"; shift 2 ;;
     --help|-h) usage; exit 0 ;;
     *) echo "Unknown argument: $1" >&2; usage >&2; exit 1 ;;
   esac
@@ -80,6 +84,13 @@ esac
 case "$max_quota_autoswitch_attempts" in
   ''|*[!0-9]*) echo "ACP_CODEX_MAX_AUTOSWITCH_ATTEMPTS must be numeric" >&2; exit 1 ;;
 esac
+case "$codex_progress_heartbeat_seconds" in
+  ''|*[!0-9]*) echo "ACP_CODEX_PROGRESS_HEARTBEAT_SECONDS must be numeric" >&2; exit 1 ;;
+  0) echo "ACP_CODEX_PROGRESS_HEARTBEAT_SECONDS must be greater than zero" >&2; exit 1 ;;
+esac
+case "$codex_stall_seconds" in
+  ''|*[!0-9]*) echo "ACP_CODEX_STALL_SECONDS must be numeric" >&2; exit 1 ;;
+esac
 
 FLOW_SKILL_DIR="$(resolve_flow_skill_dir "${BASH_SOURCE[0]}")"
 state_file="${host_run_dir}/runner.env"
@@ -116,6 +127,7 @@ last_quota_switch_status=""
 last_quota_next_retry_at=""
 last_quota_selected_label=""
 quota_autoswitch_attempt_count=0
+last_attempt_started_epoch=0
 
 mkdir -p "$host_run_dir"
 touch "$output_file"
@@ -148,6 +160,19 @@ write_state() {
   mv "$tmp_file" "$state_file"
 }
 
+run_codex_command() {
+  # Nested workers must not inherit a parent thread id; the wrapper persists the child thread explicitly.
+  env -u CODEX_THREAD_ID "$codex_bin" "$@"
+}
+
+codex_recovery_target() {
+  if [[ -n "$thread_id" ]]; then
+    printf 'thread %s' "$thread_id"
+    return 0
+  fi
+  printf 'initial Codex exec'
+}
+
 run_with_timeout() {
   local timeout_seconds="${1:?timeout seconds required}"
   shift
@@ -398,13 +423,43 @@ persist_thread_id_from_line() {
   fi
 }
 
+terminate_codex_producer_tree() {
+  local pid="${1:?pid required}"
+  local deadline=""
+
+  if ! kill -0 "$pid" 2>/dev/null; then
+    return 0
+  fi
+
+  pkill -TERM -P "$pid" 2>/dev/null || true
+  kill "$pid" 2>/dev/null || true
+
+  deadline=$(( $(date +%s) + 2 ))
+  while kill -0 "$pid" 2>/dev/null; do
+    if (( $(date +%s) >= deadline )); then
+      break
+    fi
+    sleep 0.1
+  done
+
+  if kill -0 "$pid" 2>/dev/null; then
+    pkill -KILL -P "$pid" 2>/dev/null || true
+    kill -9 "$pid" 2>/dev/null || true
+  fi
+}
+
 stream_codex_exec() {
   local phase="${1:?phase required}"
   local stream_fifo=""
   local producer_pid=""
+  local heartbeat_pid=""
+  local progress_file=""
   local line=""
 
   last_attempt_start_size="$(stat -f %z "$output_file" 2>/dev/null || printf '0')"
+  last_attempt_started_epoch="$(date +%s)"
+  progress_file="${host_run_dir}/.codex-progress.$$"
+  rm -f "$progress_file"
   stream_fifo="$(mktemp -u "${TMPDIR:-/tmp}/codex-stream.XXXXXX")"
   mkfifo "$stream_fifo"
 
@@ -413,10 +468,10 @@ stream_codex_exec() {
       (
         case "$mode" in
           safe)
-            "$codex_bin" exec --json --profile "$safe_profile" --full-auto <"$prompt_file"
+            run_codex_command exec --json --profile "$safe_profile" --full-auto <"$prompt_file"
             ;;
           bypass)
-            "$codex_bin" exec --json --profile "$bypass_profile" --dangerously-bypass-approvals-and-sandbox <"$prompt_file"
+            run_codex_command exec --json --profile "$bypass_profile" --dangerously-bypass-approvals-and-sandbox <"$prompt_file"
             ;;
         esac
       ) >"$stream_fifo" 2>&1 &
@@ -425,10 +480,10 @@ stream_codex_exec() {
       (
         case "$mode" in
           safe)
-            resume_prompt | "$codex_bin" exec resume --json --full-auto "$thread_id" -
+            resume_prompt | run_codex_command exec resume --json --full-auto "$thread_id" -
             ;;
           bypass)
-            resume_prompt | "$codex_bin" exec resume --json --dangerously-bypass-approvals-and-sandbox "$thread_id" -
+            resume_prompt | run_codex_command exec resume --json --dangerously-bypass-approvals-and-sandbox "$thread_id" -
             ;;
         esac
       ) >"$stream_fifo" 2>&1 &
@@ -441,12 +496,55 @@ stream_codex_exec() {
   esac
 
   producer_pid="$!"
+  (
+    local now elapsed last_progress_epoch idle_for
+    while kill -0 "$producer_pid" 2>/dev/null; do
+      sleep "$codex_progress_heartbeat_seconds"
+      if ! kill -0 "$producer_pid" 2>/dev/null; then
+        break
+      fi
+      now="$(date +%s)"
+      elapsed=$((now - last_attempt_started_epoch))
+      if (( codex_stall_seconds > 0 )); then
+        if [[ ! -f "$progress_file" ]]; then
+          if (( elapsed >= codex_stall_seconds )); then
+            write_state "running" ""
+            log_runner "stale-run no-codex-output-before-stall-threshold elapsed=${elapsed}s"
+            terminate_codex_producer_tree "$producer_pid"
+            break
+          fi
+        else
+          last_progress_epoch="$(stat -f %m "$progress_file" 2>/dev/null || printf '0')"
+          if [[ -n "$last_progress_epoch" && "$last_progress_epoch" != "0" ]]; then
+            idle_for=$((now - last_progress_epoch))
+            if (( idle_for >= codex_stall_seconds )); then
+              write_state "running" ""
+              log_runner "stale-run no-codex-progress-before-stall-threshold elapsed=${elapsed}s idle=${idle_for}s"
+              terminate_codex_producer_tree "$producer_pid"
+              break
+            fi
+          fi
+        fi
+      fi
+      write_state "running" ""
+      log_runner "heartbeat waiting-for-codex-output elapsed=${elapsed}s"
+    done
+  ) &
+  heartbeat_pid="$!"
+
   while IFS= read -r line || [[ -n "$line" ]]; do
     printf '%s\n' "$line" | tee -a "$output_file"
+    touch "$progress_file" 2>/dev/null || true
     persist_thread_id_from_line "$line"
   done <"$stream_fifo"
 
+  if [[ -n "$heartbeat_pid" ]] && kill -0 "$heartbeat_pid" 2>/dev/null; then
+    kill "$heartbeat_pid" 2>/dev/null || true
+    wait "$heartbeat_pid" 2>/dev/null || true
+  fi
+
   rm -f "$stream_fifo"
+  rm -f "$progress_file"
 
   if wait "$producer_pid"; then
     last_exit_code="0"
@@ -485,6 +583,16 @@ classify_failure_reason() {
 
   recent_chunk="$(tail -n 120 <<<"$chunk")"
 
+  if grep -Eiq 'stale-run no-codex-output-before-stall-threshold|no-codex-output-before-stall-threshold' <<<"$recent_chunk"; then
+    printf 'no-codex-output-before-stall-threshold\n'
+    return 0
+  fi
+
+  if grep -Eiq 'stale-run no-codex-progress-before-stall-threshold|no-codex-progress-before-stall-threshold' <<<"$recent_chunk"; then
+    printf 'no-codex-progress-before-stall-threshold\n'
+    return 0
+  fi
+
   if grep -Eiq "You've hit your usage limit|You have reached your Codex usage limits|visit https://chatgpt.com/codex/settings/usage|Upgrade to Pro|rate limit exceeded|quota exceeded|usage cap (reached|exceeded)|usage quota (reached|exceeded)" <<<"$recent_chunk"; then
     printf 'usage-limit\n'
     return 0
@@ -523,7 +631,7 @@ EOF
 }
 
 codex_login_healthy() {
-  "$codex_bin" login status >/dev/null 2>&1
+  run_codex_command login status >/dev/null 2>&1
 }
 
 wait_for_auth_refresh() {
@@ -533,7 +641,9 @@ wait_for_auth_refresh() {
   local baseline_switch_signature="${4:-}"
   local deadline now current_fingerprint current_quota_label current_switch_signature
   local sleep_seconds
+  local recovery_target
 
+  recovery_target="$(codex_recovery_target)"
   auth_wait_started_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
   last_trigger_reason="$trigger_reason"
   write_state "waiting-auth-refresh" "$trigger_reason"
@@ -548,28 +658,28 @@ wait_for_auth_refresh() {
         current_switch_signature="$(quota_switch_signature)"
         if codex_login_healthy; then
           if [[ "$current_fingerprint" != "$baseline_fingerprint" ]]; then
-            log_runner "detected refreshed Codex auth after quota interruption; resuming thread ${thread_id}"
+            log_runner "detected refreshed Codex auth after quota interruption; resuming ${recovery_target}"
             auth_wait_started_at=""
             write_state "running" ""
             return 0
           fi
 
           if [[ -n "$baseline_quota_label" && -n "$current_quota_label" && "$current_quota_label" != "$baseline_quota_label" ]]; then
-            log_runner "detected rotated Codex quota account (${baseline_quota_label} -> ${current_quota_label}); resuming thread ${thread_id}"
+            log_runner "detected rotated Codex quota account (${baseline_quota_label} -> ${current_quota_label}); resuming ${recovery_target}"
             auth_wait_started_at=""
             write_state "running" ""
             return 0
           fi
 
           if [[ -n "$baseline_switch_signature" && -n "$current_switch_signature" && "$current_switch_signature" != "$baseline_switch_signature" ]]; then
-            log_runner "detected quota switch state refresh; resuming thread ${thread_id}"
+            log_runner "detected quota switch state refresh; resuming ${recovery_target}"
             auth_wait_started_at=""
             write_state "running" ""
             return 0
           fi
 
           if [[ "$last_quota_switch_status" == "switched" && -n "$current_quota_label" ]]; then
-            log_runner "quota manager reports healthy Codex account ${current_quota_label}; resuming thread ${thread_id}"
+            log_runner "quota manager reports healthy Codex account ${current_quota_label}; resuming ${recovery_target}"
             auth_wait_started_at=""
             write_state "running" ""
             return 0
@@ -580,9 +690,9 @@ wait_for_auth_refresh() {
       *)
         if codex_login_healthy; then
           if [[ "$current_fingerprint" != "$baseline_fingerprint" ]]; then
-            log_runner "detected refreshed Codex auth; resuming thread ${thread_id}"
+            log_runner "detected refreshed Codex auth; resuming ${recovery_target}"
           else
-            log_runner "Codex auth is healthy again; resuming thread ${thread_id}"
+            log_runner "Codex auth is healthy again; resuming ${recovery_target}"
           fi
           auth_wait_started_at=""
           write_state "running" ""
@@ -643,11 +753,6 @@ attempt_run() {
   reason="$(classify_failure_reason "$(new_output_since "$last_attempt_start_size")")"
   last_failure_reason="${reason:-worker-exit-failed}"
 
-  if [[ -z "$thread_id" ]]; then
-    write_state "failed" "$last_failure_reason"
-    return 1
-  fi
-
   case "$last_failure_reason" in
     usage-limit|auth-failure|auth-401|account-banned)
       if (( resume_count >= max_resume_attempts )); then

package/tools/bin/agent-project-run-codex-session

@@ -24,6 +24,7 @@ EOF
 mode=""
 session=""
 worktree=""
+worktree_realpath=""
 prompt_file=""
 runs_root=""
 adapter_id=""
@@ -105,6 +106,12 @@ if [[ -z "$mode" || -z "$session" || -z "$worktree" || -z "$prompt_file" || -z "
   exit 1
 fi
 
+worktree_realpath="$(cd "$worktree" 2>/dev/null && pwd -P || true)"
+if [[ -z "$worktree_realpath" || ! -d "$worktree_realpath" ]]; then
+  echo "unable to resolve worktree realpath: $worktree" >&2
+  exit 1
+fi
+
 case "$mode" in
   safe|bypass) ;;
   *)
@@ -119,7 +126,7 @@ inner_script="${artifact_dir}/${session}.sh"
 meta_file="${artifact_dir}/run.env"
 result_file="${artifact_dir}/result.env"
 runner_state_file="${artifact_dir}/runner.env"
-sandbox_run_dir="${worktree%/}/${sandbox_subdir}/${session}"
+sandbox_run_dir="${worktree_realpath%/}/${sandbox_subdir}/${session}"
 started_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
 codex_bin="$(resolve_codex_bin || true)"
 runner_bin="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/agent-project-run-codex-resilient"
@@ -137,13 +144,14 @@ if [[ -z "$codex_bin" || ! -x "$codex_bin" ]]; then
   exit 1
 fi
 
-branch_name="$(git -C "$worktree" branch --show-current 2>/dev/null || true)"
+branch_name="$(git -C "$worktree_realpath" branch --show-current 2>/dev/null || true)"
 
 printf -v session_q '%q' "$session"
 printf -v task_kind_q '%q' "$task_kind"
 printf -v task_id_q '%q' "$task_id"
 printf -v mode_q '%q' "$mode"
 printf -v worktree_q '%q' "$worktree"
+printf -v worktree_realpath_q '%q' "$worktree_realpath"
 printf -v prompt_q '%q' "$prompt_file"
 printf -v output_q '%q' "$output_file"
 printf -v artifact_dir_q '%q' "$artifact_dir"
@@ -166,6 +174,7 @@ printf -v bypass_profile_q '%q' "$bypass_profile"
   printf 'SESSION=%s\n' "$session_q"
   printf 'MODE=%s\n' "$mode_q"
   printf 'WORKTREE=%s\n' "$worktree_q"
+  printf 'WORKTREE_REALPATH=%s\n' "$worktree_realpath_q"
   printf 'PROMPT_FILE=%s\n' "$prompt_q"
   printf 'OUTPUT_FILE=%s\n' "$output_q"
   printf 'SCRIPT=%s\n' "$script_q"
@@ -264,19 +273,59 @@ cat >"$inner_script" <<EOF
 #!/usr/bin/env bash
 set -euo pipefail
 ${runtime_exports}
-${context_exports}cd ${worktree_q}
+${context_exports}cd ${worktree_realpath_q}
 reset_sandbox_run_dir() {
   mkdir -p ${sandbox_run_dir_q}
   find ${sandbox_run_dir_q} -mindepth 1 -maxdepth 1 -exec rm -rf {} + 2>/dev/null || true
 }
+find_logged_artifact_path() {
+  local artifact_name="\${1:?artifact name required}"
+  local candidate=""
+
+  while IFS= read -r candidate; do
+    [[ -n "\${candidate}" ]] || continue
+    while [[ "\${candidate}" == *')' || "\${candidate}" == *']' || "\${candidate}" == *',' || "\${candidate}" == *'"' || "\${candidate}" == *"'" ]]; do
+      candidate="\${candidate%?}"
+    done
+    if [[ "\$(basename "\${candidate}")" == "\${artifact_name}" && -f "\${candidate}" ]]; then
+      printf '%s\n' "\${candidate}"
+    fi
+  done < <(grep -oE '/(Users|Volumes|tmp)/[^[:space:])"]+' ${output_q} 2>/dev/null || true)
+}
+recover_logged_artifact() {
+  local artifact_name="\${1:?artifact name required}"
+  local destination="\${2:?destination required}"
+  local source_path=""
+
+  source_path="\$(find_logged_artifact_path "\${artifact_name}" | tail -n 1)"
+  [[ -n "\${source_path}" ]] || return 0
+  mkdir -p "\$(dirname "\${destination}")"
+  if [[ "\${source_path}" != "\${destination}" ]]; then
+    cp "\${source_path}" "\${destination}"
+  fi
+}
+recover_collected_artifact() {
+  local artifact_name="\${1:?artifact name required}"
+  local destination="\${2:?destination required}"
+
+  if [[ -f ${sandbox_run_dir_q}/"\${artifact_name}" ]]; then
+    if [[ ${sandbox_run_dir_q}/"\${artifact_name}" != "\${destination}" ]]; then
+      cp ${sandbox_run_dir_q}/"\${artifact_name}" "\${destination}"
+    fi
+    return 0
+  fi
+
+  recover_logged_artifact "\${artifact_name}" "\${destination}"
+}
 record_final_git_state() {
   local final_head final_branch tmp_file
 
-  final_head="\$(git -C ${worktree_q} rev-parse HEAD 2>/dev/null || true)"
-  final_branch="\$(git -C ${worktree_q} branch --show-current 2>/dev/null || true)"
+  final_head="\$(git -C ${worktree_realpath_q} rev-parse HEAD 2>/dev/null || true)"
+  final_branch="\$(git -C ${worktree_realpath_q} branch --show-current 2>/dev/null || true)"
   tmp_file=${meta_file_q}.tmp.final.$$
-  grep -vE '^(FINAL_HEAD|FINAL_BRANCH)=' ${meta_file_q} >"\${tmp_file}" 2>/dev/null || true
+  grep -vE '^(FINAL_HEAD|FINAL_BRANCH|WORKTREE_REALPATH)=' ${meta_file_q} >"\${tmp_file}" 2>/dev/null || true
   {
+    printf 'WORKTREE_REALPATH=%s\n' ${worktree_realpath_q}
     printf 'FINAL_HEAD=%q\n' "\${final_head}"
     printf 'FINAL_BRANCH=%q\n' "\${final_branch}"
   } >>"\${tmp_file}"
@@ -286,7 +335,7 @@ reset_sandbox_run_dir
 set +e
 bash ${runner_bin_q} \\
   --mode ${mode_q} \\
-  --worktree ${worktree_q} \\
+  --worktree ${worktree_realpath_q} \\
   --prompt-file ${prompt_q} \\
   --output-file ${output_q} \\
   --host-run-dir ${artifact_dir_q} \\
@@ -296,9 +345,10 @@ bash ${runner_bin_q} \\
   --codex-bin ${codex_bin_q}
 status=\$?
 record_final_git_state
-if [[ -f ${sandbox_run_dir_q}/result.env ]]; then
-  cp ${sandbox_run_dir_q}/result.env ${result_q}
-fi
+recover_collected_artifact result.env ${result_q}
+recover_collected_artifact issue-comment.md ${artifact_dir_q}/issue-comment.md
+recover_collected_artifact pr-comment.md ${artifact_dir_q}/pr-comment.md
+recover_collected_artifact verification.jsonl ${artifact_dir_q}/verification.jsonl
 ${collect_copy_snippet}${reconcile_snippet}
 printf '\n__CODEX_EXIT__:%s\n' "\$status" | tee -a ${output_q}
 exit "\$status"

package/tools/bin/agent-project-run-openclaw-session

@@ -918,8 +918,9 @@ heartbeat_seconds = max(float(heartbeat_seconds_raw), 1.0)
 hard_deadline = time.monotonic() + timeout_seconds + 15.0
 started_at = time.monotonic()
 next_heartbeat = time.monotonic() + heartbeat_seconds
-seen_agent_output = False
-last_agent_output_at = started_at
+seen_agent_progress = False
+last_agent_progress_at = started_at
+last_progress_source = "none"
 terminal_patterns = [
     re.compile(r"Config was last written by a newer OpenClaw", re.I),
     re.compile(r"invalid api key|authentication failed|unauthorized|provider api key|login required|please authenticate|api_key_invalid", re.I),
@@ -932,6 +933,21 @@ proc = None
 sel = selectors.DefaultSelector()
 matched_terminal_error = False
 tail = ""
+openclaw_state_dir = os.environ.get("OPENCLAW_STATE_DIR", "")
+sandbox_run_dir = (
+    os.environ.get("ACP_RUN_DIR")
+    or os.environ.get("AGENT_PROJECT_RUN_DIR")
+    or os.environ.get("F_LOSNING_RUN_DIR")
+    or ""
+)
+host_managed_prefixes = tuple(
+    prefix
+    for prefix in (
+        os.path.realpath(runner_state_path) if runner_state_path else "",
+        os.path.realpath(output_path) if output_path else "",
+    )
+    if prefix
+)
 
 def write_running_heartbeat() -> None:
     updated_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
@@ -967,6 +983,47 @@ def terminate_process_group(process: subprocess.Popen) -> None:
     except ProcessLookupError:
         return
 
+def progress_signature() -> tuple[tuple[str, int, int], ...]:
+    entries: list[tuple[str, int, int]] = []
+
+    def add_file(path: str) -> None:
+        real_path = ""
+        if not path:
+            return
+        try:
+            stat_result = os.stat(path)
+        except OSError:
+            return
+        if not os.path.isfile(path):
+            return
+        real_path = os.path.realpath(path)
+        for prefix in host_managed_prefixes:
+            if real_path == prefix or real_path.startswith(f"{prefix}.tmp."):
+                return
+        entries.append((real_path, stat_result.st_mtime_ns, stat_result.st_size))
+
+    if sandbox_run_dir:
+        try:
+            for name in os.listdir(sandbox_run_dir):
+                add_file(os.path.join(sandbox_run_dir, name))
+        except OSError:
+            pass
+
+    if openclaw_state_dir:
+        sessions_dir = os.path.join(openclaw_state_dir, "agents", agent_id, "sessions")
+        add_file(os.path.join(sessions_dir, "sessions.json"))
+        try:
+            for name in os.listdir(sessions_dir):
+                if name.endswith(".jsonl") and not name.endswith(".lock"):
+                    add_file(os.path.join(sessions_dir, name))
+        except OSError:
+            pass
+
+    entries.sort()
+    return tuple(entries)
+
+last_progress_signature = progress_signature()
+
 with open(output_path, "ab", buffering=0) as log_handle:
     proc = subprocess.Popen(
         cmd,
@@ -989,26 +1046,42 @@ with open(output_path, "ab", buffering=0) as log_handle:
           terminate_process_group(proc)
           break
 
+        current_progress_signature = progress_signature()
+        if current_progress_signature != last_progress_signature:
+            last_progress_signature = current_progress_signature
+            seen_agent_progress = True
+            last_agent_progress_at = time.monotonic()
+            last_progress_source = "session-state"
+            next_heartbeat = time.monotonic() + heartbeat_seconds
+
         events = sel.select(timeout=0.2)
         if not events:
-            if proc.poll() is None and not seen_agent_output and stall_seconds > 0 and (time.monotonic() - started_at) >= stall_seconds:
+            if proc.poll() is None and not seen_agent_progress and stall_seconds > 0 and (time.monotonic() - started_at) >= stall_seconds:
                 elapsed = int(time.monotonic() - started_at)
                 write_running_heartbeat()
                 log_handle.write(f"[openclaw] stale-run no-agent-output-before-stall-threshold elapsed={elapsed}s\n".encode("utf-8"))
                 terminate_process_group(proc)
                 break
-            if proc.poll() is None and seen_agent_output and stall_seconds > 0 and (time.monotonic() - last_agent_output_at) >= stall_seconds:
+            if proc.poll() is None and seen_agent_progress and stall_seconds > 0 and (time.monotonic() - last_agent_progress_at) >= stall_seconds:
                 elapsed = int(time.monotonic() - started_at)
-                idle_for = int(time.monotonic() - last_agent_output_at)
+                idle_for = int(time.monotonic() - last_agent_progress_at)
                 write_running_heartbeat()
                 log_handle.write(f"[openclaw] stale-run no-agent-progress-before-stall-threshold elapsed={elapsed}s idle={idle_for}s\n".encode("utf-8"))
                 terminate_process_group(proc)
                 break
-            if proc.poll() is None and time.monotonic() >= next_heartbeat:
+            if proc.poll() is None and not seen_agent_progress and time.monotonic() >= next_heartbeat:
                 elapsed = int(time.monotonic() - started_at)
                 write_running_heartbeat()
                 log_handle.write(f"[openclaw] heartbeat waiting-for-agent-output elapsed={elapsed}s\n".encode("utf-8"))
                 next_heartbeat = time.monotonic() + heartbeat_seconds
+            if proc.poll() is None and seen_agent_progress and time.monotonic() >= next_heartbeat:
+                elapsed = int(time.monotonic() - started_at)
+                idle_for = int(time.monotonic() - last_agent_progress_at)
+                write_running_heartbeat()
+                log_handle.write(
+                    f"[openclaw] heartbeat progress source={last_progress_source} elapsed={elapsed}s idle={idle_for}s\n".encode("utf-8")
+                )
+                next_heartbeat = time.monotonic() + heartbeat_seconds
             if proc.poll() is not None:
                 break
             continue
@@ -1022,8 +1095,9 @@ with open(output_path, "ab", buffering=0) as log_handle:
             text = chunk.decode("utf-8", errors="replace")
             tail = (tail + text)[-8192:]
             next_heartbeat = time.monotonic() + heartbeat_seconds
-            seen_agent_output = True
-            last_agent_output_at = time.monotonic()
+            seen_agent_progress = True
+            last_agent_progress_at = time.monotonic()
+            last_progress_source = "stdout"
 
             if not matched_terminal_error and any(pattern.search(tail) for pattern in terminal_patterns):
                 matched_terminal_error = True

package/tools/bin/cleanup-worktree.sh

@@ -12,6 +12,7 @@ AGENT_REPO_ROOT="$(flow_resolve_agent_repo_root "${CONFIG_YAML}")"
 AGENT_ROOT="$(flow_resolve_agent_root "${CONFIG_YAML}")"
 RUNS_ROOT="$(flow_resolve_runs_root "${CONFIG_YAML}")"
 HISTORY_ROOT="$(flow_resolve_history_root "${CONFIG_YAML}")"
+WORKTREE_ROOT="$(flow_resolve_worktree_root "${CONFIG_YAML}")"
 RETAINED_REPO_ROOT="$(flow_resolve_retained_repo_root "${CONFIG_YAML}")"
 VSCODE_WORKSPACE_FILE="$(flow_resolve_vscode_workspace_file "${CONFIG_YAML}")"
 ISSUE_SESSION_PREFIX="$(flow_resolve_issue_session_prefix "${CONFIG_YAML}")"
@@ -36,7 +37,9 @@ if [[ -n "$SESSION" ]]; then
   ARGS+=(--session "$SESSION")
 fi
 
-bash "${FLOW_TOOLS_DIR}/agent-project-cleanup-session" "${ARGS[@]}" >/dev/null
+AGENT_PROJECT_WORKTREE_ROOT="$WORKTREE_ROOT" \
+F_LOSNING_WORKTREE_ROOT="$WORKTREE_ROOT" \
+  bash "${FLOW_TOOLS_DIR}/agent-project-cleanup-session" "${ARGS[@]}" >/dev/null
 
 F_LOSNING_AGENT_REPO_ROOT="$AGENT_REPO_ROOT" \
 F_LOSNING_RETAINED_REPO_ROOT="$RETAINED_REPO_ROOT" \

package/tools/bin/dashboard-launchd-bootstrap.sh

@@ -4,13 +4,14 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 FLOW_SKILL_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
 HOME_DIR="${ACP_DASHBOARD_HOME_DIR:-${HOME:-}}"
-SOURCE_HOME="${ACP_DASHBOARD_SOURCE_HOME:-$(cd "${FLOW_SKILL_DIR}/../../.." && pwd)}"
+SOURCE_HOME="${ACP_DASHBOARD_SOURCE_HOME:-}"
 RUNTIME_HOME="${ACP_DASHBOARD_RUNTIME_HOME:-${HOME_DIR}/.agent-runtime/runtime-home}"
 PROFILE_REGISTRY_ROOT="${ACP_DASHBOARD_PROFILE_REGISTRY_ROOT:-${ACP_PROFILE_REGISTRY_ROOT:-${HOME_DIR}/.agent-runtime/control-plane/profiles}}"
 HOST="${ACP_DASHBOARD_HOST:-127.0.0.1}"
 PORT="${ACP_DASHBOARD_PORT:-8765}"
 BASE_PATH="${ACP_DASHBOARD_PATH:-/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin}"
 SYNC_SCRIPT="${ACP_DASHBOARD_SYNC_SCRIPT:-${FLOW_SKILL_DIR}/tools/bin/sync-shared-agent-home.sh}"
+ENSURE_SYNC_SCRIPT="${ACP_DASHBOARD_ENSURE_SYNC_SCRIPT:-${FLOW_SKILL_DIR}/tools/bin/ensure-runtime-sync.sh}"
 RUNTIME_SERVE_SCRIPT="${ACP_DASHBOARD_RUNTIME_SERVE_SCRIPT:-${RUNTIME_HOME}/skills/openclaw/agent-control-plane/tools/bin/serve-dashboard.sh}"
 
 if [[ -z "${HOME_DIR}" ]]; then
@@ -23,12 +24,23 @@ export PATH="${BASE_PATH}"
 export ACP_PROFILE_REGISTRY_ROOT="${PROFILE_REGISTRY_ROOT}"
 export PYTHONDONTWRITEBYTECODE=1
 
-if [[ ! -x "${SYNC_SCRIPT}" ]]; then
-  echo "dashboard launchd bootstrap missing sync script: ${SYNC_SCRIPT}" >&2
+if [[ ! -x "${ENSURE_SYNC_SCRIPT}" && ! -x "${SYNC_SCRIPT}" ]]; then
+  echo "dashboard launchd bootstrap missing sync helper: ${ENSURE_SYNC_SCRIPT}" >&2
   exit 65
 fi
 
-bash "${SYNC_SCRIPT}" "${SOURCE_HOME}" "${RUNTIME_HOME}" >/dev/null
+if [[ -x "${ENSURE_SYNC_SCRIPT}" ]]; then
+  ensure_args=(--runtime-home "${RUNTIME_HOME}" --quiet)
+  if [[ -n "${SOURCE_HOME}" ]]; then
+    ensure_args=(--source-home "${SOURCE_HOME}" "${ensure_args[@]}")
+  fi
+  bash "${ENSURE_SYNC_SCRIPT}" "${ensure_args[@]}"
+else
+  if [[ -z "${SOURCE_HOME}" ]]; then
+    SOURCE_HOME="${FLOW_SKILL_DIR}"
+  fi
+  bash "${SYNC_SCRIPT}" "${SOURCE_HOME}" "${RUNTIME_HOME}" >/dev/null
+fi
 
 if [[ ! -x "${RUNTIME_SERVE_SCRIPT}" ]]; then
   echo "dashboard launchd bootstrap missing runtime serve script: ${RUNTIME_SERVE_SCRIPT}" >&2