@jonit-dev/night-watch-cli 1.8.10-beta.0 → 1.8.10-beta.11

package/dist/scripts/night-watch-helpers.sh

@@ -87,6 +87,123 @@ validate_provider() {
   return 1
 }
 
+night_watch_push_uses_no_verify() {
+  [ "${NW_GIT_PUSH_NO_VERIFY:-0}" = "1" ]
+}
+
+git_push_for_project() {
+  local repo_dir="${1:?repo_dir required}"
+  shift
+
+  if night_watch_push_uses_no_verify; then
+    git -C "${repo_dir}" push --no-verify "$@"
+  else
+    git -C "${repo_dir}" push "$@"
+  fi
+}
+
+project_git_push_command() {
+  local branch_name="${1:?branch_name required}"
+  local mode="${2:-normal}"
+  local no_verify=""
+
+  if night_watch_push_uses_no_verify; then
+    no_verify=" --no-verify"
+  fi
+
+  case "${mode}" in
+    force-with-lease)
+      printf 'git push%s --force-with-lease origin %s' "${no_verify}" "${branch_name}"
+      ;;
+    *)
+      printf 'git push%s origin %s' "${no_verify}" "${branch_name}"
+      ;;
+  esac
+}
+
+# ── Executor PR status labels ────────────────────────────────────────────────
+
+NW_EXECUTOR_PARTIAL_LABEL="${NW_EXECUTOR_PARTIAL_LABEL:-nw:partial}"
+NW_EXECUTOR_RESUMABLE_LABEL="${NW_EXECUTOR_RESUMABLE_LABEL:-nw:resumable}"
+NW_EXECUTOR_READY_REVIEW_LABEL="${NW_EXECUTOR_READY_REVIEW_LABEL:-nw:ready-review}"
+
+csv_has_label() {
+  local csv="${1:-}"
+  local label="${2:?label required}"
+
+  printf '%s\n' "${csv}" \
+    | tr ',' '\n' \
+    | sed '/^[[:space:]]*$/d' \
+    | grep -Fxq "${label}"
+}
+
+ensure_github_label() {
+  local label_name="${1:?label_name required}"
+  local description="${2:-}"
+  local color="${3:-1d76db}"
+
+  gh label create "${label_name}" \
+    --description "${description}" \
+    --color "${color}" \
+    --force 2>/dev/null || true
+}
+
+ensure_executor_status_labels() {
+  ensure_github_label \
+    "${NW_EXECUTOR_PARTIAL_LABEL}" \
+    "Executor started this PR and work is intentionally incomplete" \
+    "fbca04"
+  ensure_github_label \
+    "${NW_EXECUTOR_RESUMABLE_LABEL}" \
+    "Executor should resume this unfinished PR before starting new work" \
+    "d93f0b"
+  ensure_github_label \
+    "${NW_EXECUTOR_READY_REVIEW_LABEL}" \
+    "Executor finished implementation and the PR is ready for automated/human review" \
+    "0e8a16"
+}
+
+find_open_pr_for_branch() {
+  local branch_name="${1:?branch_name required}"
+  local pr_list=""
+
+  pr_list=$(gh pr list --state open --limit 200 \
+    --json number,headRefName,url,title,isDraft,labels,createdAt 2>/dev/null || echo "[]")
+
+  printf '%s' "${pr_list}" \
+    | jq -c --arg branch_name "${branch_name}" '
+        .[]
+        | select((.headRefName // "") == $branch_name)
+      ' 2>/dev/null \
+    | head -n 1 || true
+}
+
+find_executor_resume_pr() {
+  local branch_prefix="${1:-night-watch}"
+  local pr_list=""
+
+  pr_list=$(gh pr list --state open --limit 200 \
+    --json number,headRefName,url,title,isDraft,labels,createdAt 2>/dev/null || echo "[]")
+
+  printf '%s' "${pr_list}" \
+    | jq -c \
+        --arg primary_prefix "${branch_prefix}/" \
+        --arg resumable_label "${NW_EXECUTOR_RESUMABLE_LABEL}" '
+        [
+          .[]
+          | select(
+              (.headRefName // "" | startswith($primary_prefix))
+              or
+              (.headRefName // "" | startswith("feat/"))
+            )
+          | .labelNames = ((.labels // []) | map(.name))
+          | select((.labelNames | index($resumable_label)) != null)
+        ]
+        | sort_by(.createdAt // "")
+        | .[0] // empty
+      ' 2>/dev/null || true
+}
+
 # ── Generic Provider Command Builder ──────────────────────────────────────────
 
 # Build a provider command from NW_PROVIDER_* environment variables.
@@ -180,6 +297,13 @@ resolve_night_watch_cli() {
 
   local bundled_bin="${script_dir}/../bin/night-watch.mjs"
   if [ -x "${bundled_bin}" ]; then
+    # Verify dist/ exists — prevents ERR_MODULE_NOT_FOUND crashes
+    local dist_dir
+    dist_dir="$(cd "$(dirname "${bundled_bin}")" && pwd)/../dist"
+    if [ ! -d "${dist_dir}" ]; then
+      echo "ERROR: night-watch dist/ not found at ${dist_dir}; run 'yarn build'" >&2
+      return 1
+    fi
     printf "%s" "${bundled_bin}"
     return 0
   fi
@@ -360,20 +484,58 @@ acquire_lock() {
   local lock_file="${1:?lock_file required}"
 
   if [ -f "${lock_file}" ]; then
-    local lock_pid
-    lock_pid=$(cat "${lock_file}" 2>/dev/null || echo "")
+    local lock_content lock_pid lock_ts
+    lock_content=$(cat "${lock_file}" 2>/dev/null || echo "")
+    lock_pid=$(echo "${lock_content}" | awk '{print $1}')
+    lock_ts=$(echo "${lock_content}" | awk '{print $2}')
+
     if [ -n "${lock_pid}" ] && kill -0 "${lock_pid}" 2>/dev/null; then
-      log "SKIP: Previous run (PID ${lock_pid}) still active"
-      return 1
+      # PID is alive — but guard against PID reuse via /proc start time
+      if _is_lock_holder_alive "${lock_pid}" "${lock_ts}"; then
+        log "SKIP: Previous run (PID ${lock_pid}) still active"
+        return 1
+      fi
+      log "WARN: PID ${lock_pid} reused by another process, lock is stale"
+    else
+      log "WARN: Stale lock file found (PID ${lock_pid}), removing"
     fi
-    log "WARN: Stale lock file found (PID ${lock_pid}), removing"
     rm -f "${lock_file}"
   fi
 
   local quoted_lock_file=""
   printf -v quoted_lock_file '%q' "${lock_file}"
   append_exit_trap "rm -f -- ${quoted_lock_file}"
-  echo $$ > "${lock_file}"
+  echo "$$ $(date +%s)" > "${lock_file}"
+  return 0
+}
+
+# Check if the lock holder process is genuinely the one that wrote the lock.
+# Guards against PID reuse: if the process started after the lock was created,
+# it's a different process that reused the same PID.
+_is_lock_holder_alive() {
+  local pid="${1}" lock_ts="${2:-}"
+
+  # No timestamp in lock file (old format) — fall back to simple PID check
+  if [ -z "${lock_ts}" ]; then
+    return 0
+  fi
+
+  # On Linux, check /proc/<pid>/stat for process start time
+  if [ -f "/proc/${pid}/stat" ]; then
+    local stat_content boot_time_secs start_ticks clk_tck proc_start_secs
+    stat_content=$(cat "/proc/${pid}/stat" 2>/dev/null) || return 0
+    # Field 22 is starttime (in clock ticks since boot)
+    start_ticks=$(echo "${stat_content}" | awk '{print $22}')
+    clk_tck=$(getconf CLK_TCK 2>/dev/null || echo 100)
+    boot_time_secs=$(awk '/^btime/{print $2}' /proc/stat 2>/dev/null) || return 0
+    proc_start_secs=$(( boot_time_secs + start_ticks / clk_tck ))
+
+    # If the process started more than 2 seconds after the lock was written, PID was reused
+    if [ "${proc_start_secs}" -gt "$(( lock_ts + 2 ))" ]; then
+      return 1
+    fi
+  fi
+
   return 0
 }
 
@@ -587,7 +749,10 @@ cleanup_worktrees() {
   local project_dir="${1:?project_dir required}"
   local scope="${2:-}"
   local project_name
+  local project_parent
+  local registered_worktrees
   project_name=$(basename "${project_dir}")
+  project_parent=$(dirname "${project_dir}")
 
   # Clear stale worktree registrations first. This fixes cases where a
   # worktree directory was deleted out-of-band (for example by an agent
@@ -599,6 +764,8 @@ cleanup_worktrees() {
     match_token="${scope}"
   fi
 
+  registered_worktrees=$(git -C "${project_dir}" worktree list --porcelain 2>/dev/null || true)
+
   git -C "${project_dir}" worktree list --porcelain 2>/dev/null \
     | grep '^worktree ' \
     | awk '{print $2}' \
@@ -608,6 +775,27 @@ cleanup_worktrees() {
         git -C "${project_dir}" worktree remove --force "${wt}" 2>/dev/null || true
       done || true
 
+  # Also remove stale worktree directories on disk that Git no longer tracks.
+  # These can accumulate after hard crashes or interrupted prompt-driven runs.
+  find "${project_parent}" -maxdepth 1 -mindepth 1 -type d -name "${match_token}*" -print0 2>/dev/null \
+    | while IFS= read -r -d '' wt; do
+        local wt_basename
+        wt_basename=$(basename "${wt}")
+
+        if [ "${wt}" = "${project_dir}" ]; then
+          continue
+        fi
+        if ! printf '%s\n' "${wt_basename}" | grep -Fq "${match_token}"; then
+          continue
+        fi
+        if printf '%s\n' "${registered_worktrees}" | grep -qF "worktree ${wt}"; then
+          continue
+        fi
+
+        log "CLEANUP: Removing unregistered stale worktree directory ${wt}"
+        rm -rf "${wt}" 2>/dev/null || true
+      done || true
+
   # Prune again after removals so Git drops any admin entries left behind by
   # force-removal or previously broken worktrees outside Night Watch naming.
   git -C "${project_dir}" worktree prune >/dev/null 2>&1 || true
@@ -776,6 +964,19 @@ check_rate_limited() {
   fi
 }
 
+# Detect transient API network errors (e.g. "Network error" in a 400 response).
+# Usage: check_network_error <log_file> [start_line]
+# Returns 0 if a network error was detected, 1 otherwise.
+check_network_error() {
+  local log_file="${1:?log_file required}"
+  local start_line="${2:-0}"
+  if [ "${start_line}" -gt 0 ] 2>/dev/null; then
+    tail -n "+$((start_line + 1))" "${log_file}" 2>/dev/null | grep -qi "Network error"
+  else
+    tail -20 "${log_file}" 2>/dev/null | grep -qi "Network error"
+  fi
+}
+
 # Detect context window exhaustion from Claude API logs.
 # Usage: check_context_exhausted <log_file> [start_line]
 # Returns 0 if context exhausted, 1 otherwise.
@@ -1079,7 +1280,7 @@ claim_or_enqueue() {
   }
 
   local claim_id
-  if claim_id=$("${cli_bin}" queue claim "${script_type}" "${project_dir}" --provider-key "${provider_key}" 2>/dev/null); then
+  if claim_id=$("${cli_bin}" queue claim "${script_type}" "${project_dir}" --provider-key "${provider_key}" --pid $$ 2>/dev/null); then
     NW_QUEUE_ENTRY_ID="${claim_id}"
     export NW_QUEUE_ENTRY_ID
     arm_global_queue_cleanup

package/dist/scripts/night-watch-merger-cron.sh

@@ -187,12 +187,12 @@ fi
   kill -TERM $$ 2>/dev/null || true
 ) &
 WATCHDOG_PID=$!
-trap 'kill ${WATCHDOG_PID} 2>/dev/null || true; rm -f "${LOCK_FILE}"' EXIT
+append_exit_trap "kill ${WATCHDOG_PID} 2>/dev/null || true"
 
 # Discover open PRs sorted by creation date (oldest first = FIFO)
 log "INFO: Scanning open PRs..."
 PR_LIST_JSON=$(gh pr list --state open \
-  --json number,headRefName,createdAt,isDraft \
+  --json number,headRefName,createdAt,isDraft,labels \
   --jq 'sort_by(.createdAt)' \
   2>/dev/null || echo "[]")
 
@@ -211,6 +211,7 @@ while IFS= read -r pr_json; do
   pr_number=$(echo "${pr_json}" | jq -r '.number')
   pr_branch=$(echo "${pr_json}" | jq -r '.headRefName')
   is_draft=$(echo "${pr_json}" | jq -r '.isDraft')
+  pr_labels=$(echo "${pr_json}" | jq -r '[.labels[]?.name] | join(",")')
 
   # Skip drafts
   if [ "${is_draft}" = "true" ]; then
@@ -218,6 +219,11 @@ while IFS= read -r pr_json; do
     continue
   fi
 
+  if csv_has_label "${pr_labels:-}" "${NW_EXECUTOR_PARTIAL_LABEL}"; then
+    log "INFO: PR #${pr_number} (${pr_branch}): Skipping partial executor PR"
+    continue
+  fi
+
   # Check branch pattern filter
   if ! matches_branch_patterns "${pr_branch}"; then
     log "DEBUG: PR #${pr_number} (${pr_branch}): Branch pattern mismatch, skipping"

package/dist/scripts/night-watch-plan-cron.sh

@@ -83,7 +83,7 @@ cleanup_on_exit() {
   rm -f "${LOCK_FILE}"
 }
 
-trap cleanup_on_exit EXIT
+append_exit_trap "cleanup_on_exit"
 
 # Dry-run mode
 if [ "${NW_DRY_RUN:-0}" = "1" ]; then

package/dist/scripts/night-watch-pr-resolver-cron.sh

@@ -167,6 +167,8 @@ process_pr() {
         log "INFO: Invoking AI to resolve conflicts for PR #${pr_number}" "branch=${pr_branch}"
 
         local ai_prompt
+        local force_push_cmd
+        force_push_cmd=$(project_git_push_command "${pr_branch}" "force-with-lease")
         ai_prompt="You are working in a git repository at ${worktree_dir}. \
 Branch '${pr_branch}' has merge conflicts with '${default_branch}'. \
 Please resolve the merge conflicts by: \
@@ -174,7 +176,7 @@ Please resolve the merge conflicts by: \
 2) Resolving any conflict markers in the affected files \
 3) Staging resolved files with: git add <files> \
 4) Continuing the rebase with: git rebase --continue \
-5) Finally pushing with: git push --force-with-lease origin ${pr_branch} \
+5) Finally pushing with: ${force_push_cmd} \
 Work exclusively in the directory: ${worktree_dir}"
 
         local -a cmd_parts
@@ -203,7 +205,7 @@ Work exclusively in the directory: ${worktree_dir}"
         return 1
       fi
       # Push the rebased branch (AI may have already pushed; --force-with-lease is idempotent)
-      git -C "${worktree_dir}" push --force-with-lease origin "${pr_branch}" >> "${LOG_FILE}" 2>&1 || {
+      git_push_for_project "${worktree_dir}" --force-with-lease origin "${pr_branch}" >> "${LOG_FILE}" 2>&1 || {
         log "WARN: Push after rebase failed for PR #${pr_number}" "branch=${pr_branch}"
       }
     fi
@@ -224,13 +226,15 @@ Work exclusively in the directory: ${worktree_dir}"
       fi
 
       local review_prompt
+      local review_push_cmd
+      review_push_cmd=$(project_git_push_command "${pr_branch}")
       review_prompt="You are working in the git repository at ${review_workdir}. \
 PR #${pr_number} on branch '${pr_branch}' has unresolved review comments requesting changes. \
 Please: \
 1) Run 'gh pr view ${pr_number} --comments' to read the review comments \
 2) Implement the requested changes \
 3) Commit the changes with a descriptive message \
-4) Push with: git push origin ${pr_branch} \
+4) Push with: ${review_push_cmd} \
 Work in the directory: ${review_workdir}"
 
       local -a review_cmd_parts

package/dist/scripts/night-watch-pr-reviewer-cron.sh

@@ -172,7 +172,7 @@ Auto-merge failed: ${auto_merge_failed_summary}"
     if [ -n "${final_score}" ]; then
       details="${details}|final_score=${final_score}"
     fi
-    log "TIMEOUT: PR reviewer killed after ${MAX_RUNTIME}s"
+    log "TIMEOUT: PR reviewer timed out (runtime budget ${MAX_RUNTIME}s)"
     if [ "${WORKER_MODE}" != "1" ]; then
       send_telegram_status_message "🔍 Night Watch Reviewer: timeout" "Project: ${PROJECT_NAME}
 Provider (model): ${PROVIDER_MODEL_DISPLAY}
@@ -631,6 +631,11 @@ while IFS=$'\t' read -r pr_number pr_branch pr_labels; do
     continue
   fi
 
+  if csv_has_label "${pr_labels:-}" "${NW_EXECUTOR_PARTIAL_LABEL}"; then
+    log "INFO: PR #${pr_number} (${pr_branch}) is labeled ${NW_EXECUTOR_PARTIAL_LABEL}; waiting for executor to finish"
+    continue
+  fi
+
   # Merge-conflict signal: this PR needs action even if CI and score look fine.
   MERGE_STATE=$(gh pr view "${pr_number}" --json mergeStateStatus --jq '.mergeStateStatus' 2>/dev/null || echo "")
   if [ "${MERGE_STATE}" = "DIRTY" ] || [ "${MERGE_STATE}" = "CONFLICTING" ]; then
@@ -1048,7 +1053,10 @@ for ATTEMPT in $(seq 1 "${TOTAL_ATTEMPTS}"); do
 
   ATTEMPT_TIMEOUT="${MAX_RUNTIME}"
   if [ -n "${TARGET_PR}" ]; then
-    # Calculate timeout from remaining runtime budget.
+    # Give each targeted attempt the full remaining runtime budget.
+    # Retries only happen after a quick return (low score / invalid output / rate limit);
+    # a timed-out provider run is not retried, so pre-splitting the budget would
+    # incorrectly cap a 1h review to ~20m on attempt 1.
     NOW_TS=$(date +%s)
     ELAPSED=$((NOW_TS - RUN_STARTED_AT))
     REMAINING_BUDGET=$((MAX_RUNTIME - ELAPSED))
@@ -1057,12 +1065,7 @@ for ATTEMPT in $(seq 1 "${TOTAL_ATTEMPTS}"); do
       log "RETRY: Runtime budget exhausted before attempt ${ATTEMPT}"
       break
     fi
-
-    REMAINING_ATTEMPTS=$((TOTAL_ATTEMPTS - ATTEMPT + 1))
-    ATTEMPT_TIMEOUT=$((REMAINING_BUDGET / REMAINING_ATTEMPTS))
-    if [ "${ATTEMPT_TIMEOUT}" -lt 1 ]; then
-      ATTEMPT_TIMEOUT=1
-    fi
+    ATTEMPT_TIMEOUT="${REMAINING_BUDGET}"
   fi
 
   # Recreate worktree if it was removed unexpectedly between attempts.

package/dist/scripts/night-watch-qa-cron.sh

@@ -313,7 +313,7 @@ classify_qa_comment_outcome() {
 pr_has_qa_generated_files() {
   local pr_number="${1:?PR number required}"
   gh pr view "${pr_number}" --json files --jq '.files[]?.path' 2>/dev/null \
-    | grep -Eq '^(qa-artifacts/|tests/.*/qa/)'
+    | grep -Eq '^(qa-artifacts/|tests/.*/qa/|(.*/)?(__tests__|tests?|spec|specs|e2e|playwright)/.*\.(test|spec)\.[[:alnum:]]+$|(.*/)?[^/]+\.(test|spec)\.[[:alnum:]]+$)'
 }
 
 provider_output_looks_invalid() {
@@ -341,7 +341,7 @@ validate_qa_evidence() {
   fi
 
   if ! pr_has_qa_generated_files "${pr_number}"; then
-    log "WARN-QA-EVIDENCE: PR #${pr_number} has QA marker comment but no qa-artifacts/ or tests/*/qa/ files"
+    log "WARN-QA-EVIDENCE: PR #${pr_number} has QA marker comment but no qa-artifacts/ or generated test files in standard locations"
     return 2
   fi
 
@@ -426,6 +426,11 @@ while IFS=$'\t' read -r pr_number pr_branch pr_title pr_labels; do
     continue
   fi
 
+  if csv_has_label "${pr_labels:-}" "${NW_EXECUTOR_PARTIAL_LABEL}"; then
+    log "SKIP-QA: PR #${pr_number} (${pr_branch}) is labeled ${NW_EXECUTOR_PARTIAL_LABEL}"
+    continue
+  fi
+
   # Skip PRs with the skip label
   if echo "${pr_labels}" | grep -q "${SKIP_LABEL}"; then
     log "SKIP-QA: PR #${pr_number} (${pr_branch}) has '${SKIP_LABEL}' label"
@@ -617,7 +622,7 @@ for pr_ref in ${PRS_NEEDING_QA}; do
       fi
       if [ ${QA_EVIDENCE_STATUS} -eq 2 ]; then
         WARNING_PRS_CSV=$(append_csv "${WARNING_PRS_CSV}" "#${pr_num}")
-        QA_WARNING_SUMMARY="${QA_WARNING_SUMMARY}${QA_WARNING_SUMMARY:+$'\n'}#${pr_num}: no qa-artifacts/ or tests/*/qa/ files"
+        QA_WARNING_SUMMARY="${QA_WARNING_SUMMARY}${QA_WARNING_SUMMARY:+$'\n'}#${pr_num}: no qa-artifacts/ or generated test files in standard locations"
         log "QA: PR #${pr_num} — provider completed with warning-only QA evidence"
       elif [ ${QA_EVIDENCE_STATUS} -ne 0 ]; then
         FAILED_AUTOMATION_PRS_CSV=$(append_csv "${FAILED_AUTOMATION_PRS_CSV}" "#${pr_num}")

package/dist/scripts/night-watch-slicer-cron.sh

@@ -75,7 +75,7 @@ cleanup_on_exit() {
   rm -f "${LOCK_FILE}"
 }
 
-trap cleanup_on_exit EXIT
+append_exit_trap "cleanup_on_exit"
 
 log "START: Running roadmap slicer for ${PROJECT_DIR}"
 send_telegram_status_message "📋 Night Watch Planner: started" "Project: ${PROJECT_NAME}