bosun 0.41.2 → 0.41.4

package/task/task-cli.mjs

@@ -12,7 +12,7 @@
  *   bosun task update <task-id> <json-patch>
  *   bosun task update <task-id> --status todo --priority high
  *   bosun task delete <task-id>
- *   bosun task stats [--json]
+ *   bosun task stats [--json] [--debug]
  *   bosun task import <json-file>
  *
  * EXPORTS:
@@ -234,6 +234,12 @@ function hasFlag(args, flag) {
   return args.includes(flag);
 }
 
+function isDebugModeEnabled(args = []) {
+  if (hasFlag(args, "--debug")) return true;
+  const envValue = String(process.env.BOSUN_DEBUG || "").trim().toLowerCase();
+  return ["1", "true", "yes", "on"].includes(envValue);
+}
+
 // ── Programmatic API ──────────────────────────────────────────────────────────
 
 /**
@@ -529,6 +535,25 @@ function readRepoAreaLocksFromRuntimeState() {
       parsed?.repoAreaDispatchCycle && typeof parsed.repoAreaDispatchCycle === "object"
         ? parsed.repoAreaDispatchCycle
         : {};
+    const contentionEvents = Array.isArray(parsed?.repoAreaContentionEvents)
+      ? parsed.repoAreaContentionEvents
+          .slice(-60)
+          .map((event) => ({
+            at: event?.at ? String(event.at) : null,
+            taskId: normalizeTaskId(event?.taskId),
+            area: normalizeRepoAreaKey(event?.area),
+            waitMs: Math.max(0, Math.trunc(Number(event?.waitMs || 0))),
+            resolutionReason: normalizeRepoAreaResolutionReason(
+              event?.resolutionReason,
+            ),
+          }))
+          .filter((event) => event.taskId && event.area)
+      : [];
+    const contentionByReason = Object.create(null);
+    for (const event of contentionEvents) {
+      const reason = normalizeRepoAreaResolutionReason(event.resolutionReason);
+      contentionByReason[reason] = (contentionByReason[reason] || 0) + 1;
+    }
     const activeSignals = new Map();
     const activeCounts = new Map();
     for (const [taskId, slot] of Object.entries(runtimeSlots)) {
@@ -696,6 +721,16 @@ function readRepoAreaLocksFromRuntimeState() {
         ),
         waitSamples: areas.reduce((sum, area) => sum + (area.waitSamples || 0), 0),
         waitingTasks: areas.reduce((sum, area) => sum + (area.waitingTasks || 0), 0),
+        contentionEvents: contentionEvents.length,
+      },
+      contention: {
+        events: contentionEvents.length,
+        waitMsTotal: contentionEvents.reduce(
+          (sum, event) => sum + Math.max(0, Number(event?.waitMs || 0)),
+          0,
+        ),
+        byReason: contentionByReason,
+        recent: contentionEvents.slice(-10),
       },
       dispatch: {
         cycles: Math.max(0, Math.trunc(Number(parsed?.repoAreaDispatchCycles || 0))),
@@ -761,6 +796,11 @@ function normalizeRepoAreaKey(value) {
   return String(value || "").trim().toLowerCase();
 }
 
+function normalizeRepoAreaResolutionReason(value, fallback = "resolved") {
+  const normalized = normalizeRepoAreaKey(value);
+  return normalized || fallback;
+}
+
 function normalizeRepoAreas(input) {
   if (!Array.isArray(input)) return [];
   return [...new Set(input.map((value) => normalizeRepoAreaKey(value)).filter(Boolean))];
@@ -1408,6 +1448,7 @@ async function cliDelete(args) {
 
 async function cliStats(args) {
   const stats = await taskStats();
+  const debugMode = isDebugModeEnabled(args);
 
   if (hasFlag(args, "--json")) {
     console.log(JSON.stringify(stats, null, 2));
@@ -1422,13 +1463,15 @@ async function cliStats(args) {
   console.log(`    Done:        ${stats.done || 0}`);
   console.log(`    Blocked:     ${stats.blocked || 0}`);
   console.log(`    Total:       ${stats.total || 0}`);
-  if (stats.repoAreaLocks) {
+  if (debugMode && stats.repoAreaLocks) {
     const lockState = stats.repoAreaLocks;
     const totals = lockState.totals || {};
+    const contention = lockState.contention || {};
     console.log(`\n  Repo Area Locks:`);
     console.log(`    Dispatch Cycles:   ${totals.dispatchCycles || lockState.dispatchCycles || 0}`);
     console.log(`    Conflict Events:   ${totals.conflictEvents || lockState.conflictEvents || 0}`);
     console.log(`    Blocked Tracked:   ${lockState.blockedTasksTracked || lockState.dispatch?.blockedTasksTracked || 0}`);
+    console.log(`    Contention Events: ${totals.contentionEvents || contention.events || 0}`);
     const totalWaitSamples = Number(totals.waitSamples || 0);
     const totalWaitMs = Number(totals.waitMsTotal || 0);
     const globalAvgWaitMs =
@@ -1451,6 +1494,14 @@ async function cliStats(args) {
         `    - ${area.area}: blocked=${area.blockedDispatches || 0}, selected=${area.selectedDispatches || 0}, limit=${area.effectiveLimit || 0}/${area.configuredLimit || lockState.configuredLimit || 0}, avgWaitMs=${Math.round(area.averageWaitMs || 0)}`,
       );
     }
+    const recentEvents = Array.isArray(contention.recent)
+      ? contention.recent.slice(-3)
+      : [];
+    for (const event of recentEvents) {
+      console.log(
+        `    - contention: area=${event.area || "unknown"}, waitMs=${Math.max(0, Math.trunc(Number(event.waitMs || 0)))}, reason=${normalizeRepoAreaResolutionReason(event.resolutionReason)}, task=${String(event.taskId || "").slice(0, 8)}`,
+      );
+    }
   }
   console.log("");
 }
@@ -1600,7 +1651,7 @@ function showTaskHelp() {
     get, show   Show task details           bosun task get --help
     update, edit Update task fields         bosun task update --help
     delete, rm  Delete a task              bosun task delete --help
-    stats       Aggregate statistics        bosun task stats --json
+    stats       Aggregate statistics        bosun task stats --json/--debug
     import      Bulk import from JSON file  bosun task import --help
 
   QUICK REFERENCE

package/task/task-executor.mjs

@@ -98,6 +98,7 @@ import {
   initTaskClaims,
   claimTask,
   renewClaim,
+  getClaim,
   releaseTask as releaseTaskClaim,
 } from "./task-claims.mjs";
 import { initPresence, getPresenceState } from "../infra/presence.mjs";
@@ -120,6 +121,7 @@ const NO_COMMIT_MAX_COOLDOWN_MS = 2 * 60 * 60 * 1000;
 const CLAIM_CONFLICT_COMMENT_COOLDOWN_MS = 30 * 60 * 1000; // 30 minutes
 const REPO_AREA_SLOW_MERGE_LATENCY_MS = 4 * 60 * 60 * 1000;
 const REPO_AREA_VERY_SLOW_MERGE_LATENCY_MS = 8 * 60 * 60 * 1000;
+const REPO_AREA_CONTENTION_EVENT_LIMIT = 60;
 const FATAL_CLAIM_RENEW_ERRORS = new Set([
   "task_claimed_by_different_instance",
   "claim_token_mismatch",
@@ -178,6 +180,24 @@ function transitionInternalTaskStatus(taskId, status, source) {
   return setInternalStatus(taskId, status, source);
 }
 
+function isMatchingLocalClaimProcessAlive(ownerId, claim) {
+  if (!ownerId || !claim) return null;
+  const claimInstanceId = String(claim.instance_id || claim.instanceId || "");
+  if (!claimInstanceId || claimInstanceId !== String(ownerId)) return null;
+  const claimHost = String(claim?.metadata?.host || "").trim();
+  if (!claimHost || claimHost.toLowerCase() !== os.hostname().toLowerCase()) {
+    return null;
+  }
+  const claimPid = Number(claim?.metadata?.pid);
+  if (!Number.isFinite(claimPid) || claimPid <= 0) return null;
+  try {
+    process.kill(Math.floor(claimPid), 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 function parseNumberEnv(name, fallback) {
   const value = Number(process.env[name]);
   return Number.isFinite(value) ? value : fallback;
@@ -313,6 +333,11 @@ function createEmptyRepoAreaTelemetryEntry() {
   return normalizeRepoAreaTelemetryEntry();
 }
 
+function normalizeRepoAreaResolutionReason(value, fallback = "resolved") {
+  const normalized = normalizeRepoAreaKey(value);
+  return normalized || fallback;
+}
+
 function averageNumbers(values = []) {
   if (!Array.isArray(values) || values.length === 0) return 0;
   const total = values.reduce((sum, value) => sum + Number(value || 0), 0);
@@ -2511,6 +2536,8 @@ class TaskExecutor {
     this._repoAreaTaskAreas = new Map();
     /** @type {Map<string, number>} */
     this._repoAreaTaskStartedAt = new Map();
+    /** @type {Array<{ at: string, taskId: string, area: string, waitMs: number, resolutionReason: string }>} */
+    this._repoAreaContentionEvents = [];
     this._repoAreaDispatchCycles = 0;
     this._repoAreaConflictCount = 0;
 
@@ -2852,6 +2879,22 @@ class TaskExecutor {
         if (!key || !startedAt) continue;
         this._repoAreaTaskStartedAt.set(key, startedAt);
       }
+      this._repoAreaContentionEvents = Array.isArray(parsed?.repoAreaContentionEvents)
+        ? parsed.repoAreaContentionEvents
+          .slice(-REPO_AREA_CONTENTION_EVENT_LIMIT)
+          .map((event) => ({
+            at: event?.at
+              ? String(event.at)
+              : new Date().toISOString(),
+            taskId: normalizeTaskIdKey(event?.taskId) || "",
+            area: normalizeRepoAreaKey(event?.area) || "",
+            waitMs: Math.max(0, Math.trunc(Number(event?.waitMs || 0))),
+            resolutionReason: normalizeRepoAreaResolutionReason(
+              event?.resolutionReason,
+            ),
+          }))
+          .filter((event) => event.taskId && event.area)
+        : [];
       this._repoAreaDispatchHistory = Array.isArray(parsed?.repoAreaDispatchHistory)
         ? parsed.repoAreaDispatchHistory
           .slice(-20)
@@ -3059,6 +3102,9 @@ class TaskExecutor {
             repoAreaBlockedTasks: Object.fromEntries(this._repoAreaBlockedTasks),
             repoAreaTaskAreas: Object.fromEntries(this._repoAreaTaskAreas),
             repoAreaTaskStartedAt: Object.fromEntries(this._repoAreaTaskStartedAt),
+            repoAreaContentionEvents: this._repoAreaContentionEvents.slice(
+              -REPO_AREA_CONTENTION_EVENT_LIMIT,
+            ),
             repoAreaDispatchHistory: this._repoAreaDispatchHistory.slice(-20),
             repoAreaLockStatus: this._buildRepoAreaLockStatus(),
             slots,
@@ -3207,6 +3253,11 @@ class TaskExecutor {
       }
     }
 
+    this._clearRepoAreaWaitsForTask(
+      typeof taskOrTaskId === "string" ? { id: taskId, repo_areas: areas } : taskOrTaskId,
+      now,
+      "abandoned",
+    );
     this._clearRepoAreaBlockedTask(taskId);
     if (isFailure || isSuccess) {
       this._repoAreaTaskStartedAt.delete(taskId);
@@ -3899,6 +3950,7 @@ class TaskExecutor {
         continue;
       }
 
+      let hasStaleSharedClaim = false;
       if (SHARED_STATE_ENABLED) {
         try {
           const sharedState = await getSharedState(id, this.repoRoot);
@@ -3909,12 +3961,23 @@ class TaskExecutor {
           // block recovery re-dispatch. Removing the ownerId !== instanceId
           // guard ensures workflow-owned tasks (wf-<uuid> owners) are also
           // protected when action.claim_task IS used.
-          if (
-            ownerId &&
-            !isSharedHeartbeatStale(heartbeat, SHARED_STATE_STALE_THRESHOLD_MS)
-          ) {
-            skippedForActiveClaim++;
-            continue;
+          if (ownerId) {
+            const heartbeatIsFresh = !isSharedHeartbeatStale(
+              heartbeat,
+              SHARED_STATE_STALE_THRESHOLD_MS,
+            );
+            if (heartbeatIsFresh) {
+              const claim = await getClaim(id).catch(() => null);
+              const localClaimAlive = isMatchingLocalClaimProcessAlive(
+                ownerId,
+                claim,
+              );
+              if (localClaimAlive !== false) {
+                skippedForActiveClaim++;
+                continue;
+              }
+            }
+            hasStaleSharedClaim = true;
           }
         } catch {
           /* best effort */
@@ -3996,6 +4059,27 @@ class TaskExecutor {
           skippedForActiveClaim++;
           continue;
         }
+        if (hasStaleSharedClaim) {
+          try {
+            await transitionTaskStatus(id, "todo", {
+              source: "task-executor-recovery-stale-workflow-claim",
+            });
+          } catch {
+            /* best effort */
+          }
+          try {
+            transitionInternalTaskStatus(
+              id,
+              "todo",
+              "task-executor-recovery-stale-workflow-claim",
+            );
+          } catch {
+            /* best effort */
+          }
+          this._removeRuntimeSlot(id);
+          resetToTodo++;
+          continue;
+        }
         if (isFreshEnough) {
           skippedForActiveClaim++;
           continue;
@@ -4392,7 +4476,30 @@ class TaskExecutor {
     });
   }
 
-  _finalizeRepoAreaWait(taskId, area, now = Date.now()) {
+  _recordRepoAreaContentionEvent(taskId, area, waitMs, resolutionReason) {
+    const normalizedTaskId = normalizeTaskIdKey(taskId);
+    const normalizedArea = normalizeRepoAreaKey(area);
+    if (!normalizedTaskId || !normalizedArea) return;
+    this._repoAreaContentionEvents.push({
+      at: new Date().toISOString(),
+      taskId: normalizedTaskId,
+      area: normalizedArea,
+      waitMs: Math.max(0, Math.trunc(Number(waitMs || 0))),
+      resolutionReason: normalizeRepoAreaResolutionReason(resolutionReason),
+    });
+    if (this._repoAreaContentionEvents.length > REPO_AREA_CONTENTION_EVENT_LIMIT) {
+      this._repoAreaContentionEvents = this._repoAreaContentionEvents.slice(
+        -REPO_AREA_CONTENTION_EVENT_LIMIT,
+      );
+    }
+  }
+
+  _finalizeRepoAreaWait(
+    taskId,
+    area,
+    now = Date.now(),
+    resolutionReason = "resolved",
+  ) {
     const key = this._makeRepoAreaWaitKey(taskId, area);
     if (!key) return 0;
     const pending = this._repoAreaPendingWaits.get(key);
@@ -4405,6 +4512,12 @@ class TaskExecutor {
       metric.waitSamples += 1;
       metric.maxWaitMs = Math.max(metric.maxWaitMs, durationMs);
     }
+    this._recordRepoAreaContentionEvent(
+      pending.taskId,
+      pending.area,
+      durationMs,
+      resolutionReason,
+    );
     return durationMs;
   }
 
@@ -4419,16 +4532,16 @@ class TaskExecutor {
         !candidateIds.has(pending.taskId) &&
         !this._activeSlots.has(pending.taskId)
       ) {
-        this._finalizeRepoAreaWait(pending.taskId, pending.area, now);
+        this._finalizeRepoAreaWait(pending.taskId, pending.area, now, "dequeued");
       }
     }
   }
 
-  _clearRepoAreaWaitsForTask(task, now = Date.now()) {
-    const taskId = normalizeTaskIdKey(task?.id || task?.task_id);
+  _clearRepoAreaWaitsForTask(task, now = Date.now(), resolutionReason = "resolved") {
+    const taskId = normalizeTaskIdKey(task?.id || task?.task_id || task?.taskId);
     if (!taskId) return;
     for (const area of this._extractTaskRepoAreas(task)) {
-      this._finalizeRepoAreaWait(taskId, area, now);
+      this._finalizeRepoAreaWait(taskId, area, now, resolutionReason);
     }
   }
 
@@ -4626,6 +4739,14 @@ class TaskExecutor {
           lastSelectedAt: metric.lastSelectedAt,
         };
       });
+    const contentionByReason = Object.create(null);
+    for (const event of this._repoAreaContentionEvents) {
+      const reason = normalizeRepoAreaResolutionReason(event?.resolutionReason);
+      contentionByReason[reason] = (contentionByReason[reason] || 0) + 1;
+    }
+    const contentionEvents = this._repoAreaContentionEvents.slice(
+      -REPO_AREA_CONTENTION_EVENT_LIMIT,
+    );
 
     return {
       enabled: Number(this.repoAreaParallelLimit || 0) > 0,
@@ -4648,6 +4769,16 @@ class TaskExecutor {
         waitMsTotal: items.reduce((sum, item) => sum + item.waitMsTotal, 0),
         waitSamples: items.reduce((sum, item) => sum + item.waitSamples, 0),
         waitingTasks: items.reduce((sum, item) => sum + item.waitingTasks, 0),
+        contentionEvents: contentionEvents.length,
+      },
+      contention: {
+        events: contentionEvents.length,
+        waitMsTotal: contentionEvents.reduce(
+          (sum, event) => sum + Math.max(0, Number(event?.waitMs || 0)),
+          0,
+        ),
+        byReason: contentionByReason,
+        recent: contentionEvents.slice(-10),
       },
       dispatch: {
         cycles: Math.max(0, Math.trunc(Number(this._repoAreaDispatchCycles || 0))),
@@ -4811,7 +4942,7 @@ class TaskExecutor {
         }
         this._rememberTaskRepoAreas(task, now);
         this._clearRepoAreaBlockedTask(task?.id || task?.task_id);
-        this._clearRepoAreaWaitsForTask(task, now);
+        this._clearRepoAreaWaitsForTask(task, now, "selected");
         for (const area of areas) {
           repoAreaCounts.set(area, (repoAreaCounts.get(area) || 0) + 1);
           const metric = this._getRepoAreaLockMetric(area);
@@ -5662,4 +5793,3 @@ export function isExecutorDisabled() {
 
 export { TaskExecutor };
 export default TaskExecutor;
-