agentxchain 2.146.0 → 2.148.0

package/src/lib/stale-turn-watchdog.js

@@ -3,18 +3,18 @@
  *
  * Two-tier lazy idle-threshold detection:
  *
- * 1. **Fast startup watchdog (BUG-51):** if an active turn has been dispatched
- *    for >30 seconds with NO dispatch-progress file, NO staged result, and NO
- *    recent events, it is a "ghost turn" — the subprocess never attached.
- *    Transitions to `failed_start` immediately.
+ * 1. **Fast startup watchdog (BUG-51):** if an active turn has been
+ *    `dispatched`/`starting`/`running` for >30 seconds with NO startup proof
+ *    (no first-byte output recorded on the turn or in dispatch-progress) and
+ *    NO staged result, it is a "ghost turn" — the subprocess never reached a
+ *    healthy running state. Transitions to `failed_start` immediately.
  *
- *    Design note: the watchdog intentionally keys on turn-scoped
- *    dispatch-progress rather than `stdout.log` existence. Dispatch-progress is
- *    a framework-authored signal with a stable per-turn contract across runtime
- *    wiring; `stdout.log` is adapter-authored visibility output and is allowed
- *    to be best-effort. Using dispatch-progress therefore gives us the same
- *    operator-facing "no first byte / no worker heartbeat" detection without
- *    coupling the watchdog to adapter-specific log-attachment details.
+ *    Design note: the watchdog intentionally keys on first-output proof from
+ *    the framework-owned dispatch-progress contract rather than `stdout.log`
+ *    existence. `stdout.log` is adapter-authored visibility output and may be
+ *    absent even when the adapter is wired correctly. First-output timestamps
+ *    and output-line counters are the stable health contract across runtime
+ *    wiring.
  *
  * 2. **Stale turn watchdog (BUG-47):** if an active turn has status "running"
  *    for >N minutes with no event log activity AND no staged result file,
@@ -36,6 +36,7 @@ import { safeWriteJson } from './safe-write.js';
 import { emitRunEvent, readRunEvents } from './run-events.js';
 import { getTurnStagingResultPath } from './turn-paths.js';
 import { getDispatchProgressRelativePath } from './dispatch-progress.js';
+import { hasMeaningfulStagedResult } from './staged-result-proof.js';
 
 const DEFAULT_LOCAL_CLI_THRESHOLD_MS = 10 * 60 * 1000; // 10 minutes
 const DEFAULT_API_PROXY_THRESHOLD_MS = 5 * 60 * 1000;  // 5 minutes
@@ -103,12 +104,11 @@ export function detectStaleTurns(root, state, config) {
 /**
  * BUG-51: Detect ghost-dispatched turns — subprocess never started.
  *
- * A ghost turn is one that has been in "running" or "retrying" status for
- * longer than the startup watchdog threshold (default 30s) AND has:
- *   - no dispatch-progress file (framework-observed proof that no subprocess
- *     output or heartbeat was attached)
+ * A ghost turn is one that has been in `dispatched`, `starting`, `running`, or
+ * `retrying` longer than the startup watchdog threshold (default 30s) AND has:
+ *   - no startup proof (no `first_output_at` on the turn or dispatch-progress,
+ *     and no recorded output line counts)
  *   - no staged result file
- *   - no recent turn-scoped events (beyond the initial turn_dispatched)
  *
  * This is a stricter, faster check than detectStaleTurns (BUG-47).
  * Ghost turns transition to "failed_start" rather than "stalled".
@@ -125,31 +125,22 @@ export function detectGhostTurns(root, state, config) {
   const startupThreshold = resolveStartupThreshold(config);
 
   for (const [turnId, turn] of Object.entries(activeTurns)) {
-    if (turn.status !== 'running' && turn.status !== 'retrying') continue;
-    if (!turn.started_at) continue;
+    if (!['dispatched', 'starting', 'running', 'retrying'].includes(turn.status)) continue;
 
-    const startedAt = new Date(turn.started_at).getTime();
-    if (isNaN(startedAt)) continue;
+    const lifecycleStart = parseGhostLifecycleStart(turn);
+    if (!Number.isFinite(lifecycleStart)) continue;
 
-    const runningMs = now - startedAt;
+    const runningMs = now - lifecycleStart;
     if (runningMs < startupThreshold) continue;
 
-    // Ghost detection: NO dispatch-progress file means subprocess never attached
     const progressPath = join(root, getDispatchProgressRelativePath(turnId));
-    const hasProgress = existsSync(progressPath);
+    const progress = readDispatchProgressSafe(progressPath);
 
-    // If dispatch-progress exists, subprocess started — this is NOT a ghost turn.
-    // The regular stale-turn watchdog (BUG-47) will handle it if it goes silent.
-    if (hasProgress) continue;
-
-    // Also check for staged result (unlikely without progress, but be safe)
     if (hasTurnScopedStagedResult(root, turnId)) continue;
-
-    // Check for any turn-scoped events beyond the initial dispatch event
-    if (hasRecentTurnEventActivity(root, turnId, startedAt, startupThreshold, now)) continue;
+    if (hasStartupProof(turn, progress)) continue;
 
     const runningSeconds = Math.floor(runningMs / 1000);
-    const failureType = 'no_subprocess_output';
+    const failureType = classifyStartupFailureType(turn, progress);
     ghosts.push({
       turn_id: turnId,
       role: turn.assigned_role || 'unknown',
@@ -200,37 +191,11 @@ export function reconcileStaleTurns(root, state, config) {
 
   // Process ghost turns (BUG-51) — transition to failed_start
   for (const entry of ghosts) {
-    const turn = activeTurns[entry.turn_id];
-    if (!turn || (turn.status !== 'running' && turn.status !== 'retrying')) continue;
-
-    activeTurns[entry.turn_id] = {
-      ...turn,
-      status: 'failed_start',
-      failed_start_at: nowIso,
-      failed_start_reason: entry.failure_type,
-      failed_start_previous_status: turn.status,
-      failed_start_threshold_ms: entry.threshold_ms,
-      failed_start_running_ms: entry.running_ms,
-      recovery_command: `agentxchain reissue-turn --turn ${entry.turn_id} --reason ghost`,
-    };
-    changed = true;
-
-    // BUG-51 fix #6: Release budget reservation for ghost turns
-    delete budgetReservations[entry.turn_id];
-
-    emitRunEvent(root, 'turn_start_failed', {
-      run_id: state?.run_id || null,
-      phase: state?.phase || null,
-      status: 'blocked',
-      turn: { turn_id: entry.turn_id, role_id: entry.role },
-      payload: {
-        running_ms: entry.running_ms,
-        threshold_ms: entry.threshold_ms,
-        runtime_id: entry.runtime_id,
-        failure_type: entry.failure_type,
-        recommendation: entry.recommendation,
-      },
-    });
+    const applied = applyStartupFailureToActiveTurn(activeTurns, budgetReservations, entry, nowIso);
+    if (applied) {
+      emitStartupFailureEvent(root, state, entry);
+      changed = true;
+    }
   }
 
   // Process stale turns (BUG-47) — transition to stalled
@@ -271,32 +236,9 @@ export function reconcileStaleTurns(root, state, config) {
     return { stale_turns: stale, ghost_turns: ghosts, state, changed: false };
   }
 
-  const allDetected = [...ghosts, ...stale];
-  const primary = allDetected[0];
+  const nextState = buildBlockedStateFromEntries(state, activeTurns, budgetReservations, ghosts, stale, nowIso);
+  const primary = [...ghosts, ...stale][0];
   const category = ghosts.length > 0 ? 'ghost_turn' : 'stale_turn';
-  const blockedOn = allDetected.length === 1
-    ? `turn:${primary.failure_type ? 'failed_start' : 'stalled'}:${primary.turn_id}`
-    : ghosts.length > 0 ? 'turns:failed_start' : 'turns:stalled';
-
-  const nextState = {
-    ...state,
-    status: 'blocked',
-    active_turns: activeTurns,
-    budget_reservations: budgetReservations,
-    blocked_on: blockedOn,
-    blocked_reason: {
-      category,
-      blocked_at: nowIso,
-      turn_id: primary.turn_id,
-      recovery: {
-        typed_reason: category,
-        owner: 'human',
-        recovery_action: primary.recommendation,
-        turn_retained: true,
-        detail: primary.recommendation,
-      },
-    },
-  };
 
   safeWriteJson(join(root, '.agentxchain', 'state.json'), nextState);
   emitRunEvent(root, 'run_blocked', {
@@ -340,13 +282,63 @@ function resolveStartupThreshold(config) {
   return DEFAULT_STARTUP_WATCHDOG_MS;
 }
 
+export function failTurnStartup(root, state, config, turnId, details = {}) {
+  if (!state || typeof state !== 'object') {
+    return { ok: false, error: 'No governed state found' };
+  }
+
+  const turn = state.active_turns?.[turnId];
+  if (!turn) {
+    return { ok: false, error: `Turn ${turnId} not found in active turns` };
+  }
+
+  const nowIso = new Date().toISOString();
+  const activeTurns = { ...(state.active_turns || {}) };
+  const budgetReservations = { ...(state.budget_reservations || {}) };
+  const entry = {
+    turn_id: turnId,
+    role: turn.assigned_role || 'unknown',
+    runtime_id: turn.runtime_id || 'unknown',
+    running_ms: details.running_ms ?? computeLifecycleAgeMs(turn),
+    threshold_ms: details.threshold_ms ?? resolveStartupThreshold(config),
+    failure_type: classifyStartupFailureType(turn, null, details.failure_type || 'no_subprocess_output'),
+    recommendation: details.recommendation
+      || `Turn ${turnId} failed to start cleanly. Run \`agentxchain reissue-turn --turn ${turnId} --reason ghost\` to recover.`,
+  };
+
+  if (!applyStartupFailureToActiveTurn(activeTurns, budgetReservations, entry, nowIso)) {
+    return { ok: false, error: `Turn ${turnId} is not eligible for startup failure transition` };
+  }
+
+  const nextState = buildBlockedStateFromEntries(state, activeTurns, budgetReservations, [entry], [], nowIso);
+  safeWriteJson(join(root, '.agentxchain', 'state.json'), nextState);
+  emitStartupFailureEvent(root, state, entry);
+  emitRunEvent(root, 'run_blocked', {
+    run_id: nextState.run_id || null,
+    phase: nextState.phase || null,
+    status: 'blocked',
+    turn: { turn_id: entry.turn_id, role_id: entry.role },
+    payload: {
+      category: 'ghost_turn',
+      ghost_turn_ids: [entry.turn_id],
+      stalled_turn_ids: [],
+    },
+  });
+  return { ok: true, state: nextState, turn: nextState.active_turns?.[turnId] || null };
+}
+
 function hasRecentTurnEventActivity(root, turnId, startedAt, threshold, now) {
   try {
     const events = readRunEvents(root, { limit: 200 });
     for (let i = events.length - 1; i >= 0; i--) {
       const event = events[i];
       if (event?.turn?.turn_id !== turnId) continue;
-      if (event.event_type === 'turn_stalled' || event.event_type === 'turn_start_failed') continue;
+      if (
+        event.event_type === 'turn_stalled'
+        || event.event_type === 'turn_start_failed'
+        || event.event_type === 'runtime_spawn_failed'
+        || event.event_type === 'stdout_attach_failed'
+      ) continue;
       const timestamp = Date.parse(event.timestamp || '');
       if (!Number.isFinite(timestamp)) continue;
       if (timestamp < startedAt) continue;
@@ -360,9 +352,145 @@ function hasRecentTurnEventActivity(root, turnId, startedAt, threshold, now) {
   return false;
 }
 
+function applyStartupFailureToActiveTurn(activeTurns, budgetReservations, entry, nowIso) {
+  const turn = activeTurns[entry.turn_id];
+  if (!turn || !['dispatched', 'starting', 'running', 'retrying'].includes(turn.status)) {
+    return false;
+  }
+
+  activeTurns[entry.turn_id] = {
+    ...turn,
+    status: 'failed_start',
+    failed_start_at: nowIso,
+    failed_start_reason: entry.failure_type,
+    failed_start_previous_status: turn.status,
+    failed_start_threshold_ms: entry.threshold_ms,
+    failed_start_running_ms: entry.running_ms,
+    recovery_command: `agentxchain reissue-turn --turn ${entry.turn_id} --reason ghost`,
+  };
+  delete budgetReservations[entry.turn_id];
+  return true;
+}
+
+function emitStartupFailureEvent(root, state, entry) {
+  const payload = {
+    running_ms: entry.running_ms,
+    threshold_ms: entry.threshold_ms,
+    runtime_id: entry.runtime_id,
+    failure_type: entry.failure_type,
+    recommendation: entry.recommendation,
+  };
+  const details = {
+    run_id: state?.run_id || null,
+    phase: state?.phase || null,
+    status: 'blocked',
+    turn: { turn_id: entry.turn_id, role_id: entry.role },
+    payload,
+  };
+  emitRunEvent(root, 'turn_start_failed', details);
+  const failureEventType = mapStartupFailureEventType(entry.failure_type);
+  if (failureEventType) {
+    emitRunEvent(root, failureEventType, details);
+  }
+}
+
+function buildBlockedStateFromEntries(state, activeTurns, budgetReservations, ghosts, stale, nowIso) {
+  const allDetected = [...ghosts, ...stale];
+  const primary = allDetected[0];
+  const category = ghosts.length > 0 ? 'ghost_turn' : 'stale_turn';
+  const blockedOn = allDetected.length === 1
+    ? `turn:${primary.failure_type ? 'failed_start' : 'stalled'}:${primary.turn_id}`
+    : ghosts.length > 0 ? 'turns:failed_start' : 'turns:stalled';
+
+  return {
+    ...state,
+    status: 'blocked',
+    active_turns: activeTurns,
+    budget_reservations: budgetReservations,
+    blocked_on: blockedOn,
+    blocked_reason: {
+      category,
+      blocked_at: nowIso,
+      turn_id: primary.turn_id,
+      recovery: {
+        typed_reason: category,
+        owner: 'human',
+        recovery_action: primary.recommendation,
+        turn_retained: true,
+        detail: primary.recommendation,
+      },
+    },
+  };
+}
+
+function parseGhostLifecycleStart(turn) {
+  if (turn.status === 'dispatched') {
+    return Date.parse(turn.dispatched_at || turn.assigned_at || '');
+  }
+  return Date.parse(turn.started_at || turn.dispatched_at || turn.assigned_at || '');
+}
+
+function computeLifecycleAgeMs(turn) {
+  const start = parseGhostLifecycleStart(turn);
+  if (!Number.isFinite(start)) return 0;
+  return Math.max(0, Date.now() - start);
+}
+
+function readDispatchProgressSafe(progressPath) {
+  if (!existsSync(progressPath)) {
+    return null;
+  }
+  try {
+    return JSON.parse(readFileSync(progressPath, 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+function classifyStartupFailureType(turn, progress, fallback = 'no_subprocess_output') {
+  if (fallback === 'runtime_spawn_failed' || fallback === 'stdout_attach_failed') {
+    return fallback;
+  }
+  if (turn?.status === 'dispatched') {
+    return 'runtime_spawn_failed';
+  }
+  const hasWorkerAttachProof = Boolean(
+    turn?.worker_attached_at
+    || turn?.worker_pid != null
+    || progress?.pid != null,
+  );
+  if (turn?.status === 'starting' || hasWorkerAttachProof) {
+    return 'stdout_attach_failed';
+  }
+  return fallback;
+}
+
+function mapStartupFailureEventType(failureType) {
+  if (failureType === 'runtime_spawn_failed') {
+    return 'runtime_spawn_failed';
+  }
+  if (failureType === 'stdout_attach_failed') {
+    return 'stdout_attach_failed';
+  }
+  return null;
+}
+
+function hasStartupProof(turn, progress) {
+  if (turn.first_output_at) {
+    return true;
+  }
+  if (!progress || typeof progress !== 'object') {
+    return false;
+  }
+  if (progress.first_output_at) {
+    return true;
+  }
+  return Number(progress.output_lines || 0) > 0 || Number(progress.stderr_lines || 0) > 0;
+}
+
 function hasTurnScopedStagedResult(root, turnId) {
   const turnScopedPath = join(root, getTurnStagingResultPath(turnId));
-  if (existsSync(turnScopedPath)) {
+  if (hasMeaningfulStagedResult(turnScopedPath)) {
     return true;
   }
 

package/src/lib/turn-checkpoint.js

@@ -139,6 +139,53 @@ function buildCheckpointCommit(entry) {
   return { subject, body: bodyLines.join('\n') };
 }
 
+function diffMissingDeclaredPaths(declaredFiles, stagedFiles) {
+  const stagedSet = new Set((Array.isArray(stagedFiles) ? stagedFiles : []).map((value) => value.trim()).filter(Boolean));
+  return (Array.isArray(declaredFiles) ? declaredFiles : []).filter((filePath) => !stagedSet.has(filePath));
+}
+
+/**
+ * Partition paths that were missing from the staged-diff into
+ * (a) paths genuinely absent from git (untracked or dirty without staging)
+ * (b) paths already committed upstream (tracked in HEAD, no pending diff)
+ *
+ * BUG-55A completeness must only fail on (a). An actor that committed a
+ * declared file before `checkpoint-turn` ran (see BUG-23 scenario) is
+ * already-checkpointed-upstream; treating that as "missing from checkpoint"
+ * is a false positive from the completeness gate.
+ */
+function partitionDeclaredPathsByUpstreamPresence(root, missingPaths) {
+  const genuinelyMissing = [];
+  const alreadyCommittedUpstream = [];
+  for (const filePath of missingPaths) {
+    let tracked = false;
+    try {
+      git(root, ['ls-files', '--error-unmatch', '--', filePath]);
+      tracked = true;
+    } catch {
+      tracked = false;
+    }
+    if (!tracked) {
+      genuinelyMissing.push(filePath);
+      continue;
+    }
+    let hasDivergence = false;
+    try {
+      const headDiff = git(root, ['diff', 'HEAD', '--', filePath]);
+      const cachedDiff = git(root, ['diff', '--cached', '--', filePath]);
+      hasDivergence = Boolean(headDiff) || Boolean(cachedDiff);
+    } catch {
+      hasDivergence = true;
+    }
+    if (hasDivergence) {
+      genuinelyMissing.push(filePath);
+    } else {
+      alreadyCommittedUpstream.push(filePath);
+    }
+  }
+  return { genuinelyMissing, alreadyCommittedUpstream };
+}
+
 export function detectPendingCheckpoint(root, dirtyFiles = []) {
   const actorDirtyFiles = normalizeFilesChanged(dirtyFiles);
   if (actorDirtyFiles.length === 0) return { required: false };
@@ -233,12 +280,29 @@ export function checkpointAcceptedTurn(root, opts = {}) {
     };
   }
 
+  const rawMissingFromStage = diffMissingDeclaredPaths(filesChanged, staged);
+  const { genuinelyMissing, alreadyCommittedUpstream } =
+    partitionDeclaredPathsByUpstreamPresence(root, rawMissingFromStage);
+  if (genuinelyMissing.length > 0) {
+    return {
+      ok: false,
+      turn: entry,
+      error: `Checkpoint completeness failure: accepted turn ${entry.turn_id} declared ${filesChanged.length} checkpointable file(s), but Git staged only ${staged.length} and ${genuinelyMissing.length} declared path(s) are absent from git. Missing from checkpoint: ${genuinelyMissing.join(', ')}.`,
+      missing_declared_paths: genuinelyMissing,
+      already_committed_upstream: alreadyCommittedUpstream,
+      staged_paths: staged,
+    };
+  }
+
   if (staged.length === 0) {
     return {
       ok: true,
       skipped: true,
       turn: entry,
-      reason: `Accepted turn ${entry.turn_id} has no staged repo changes to checkpoint.`,
+      reason: alreadyCommittedUpstream.length > 0
+        ? `Accepted turn ${entry.turn_id} has no staged repo changes to checkpoint; all ${alreadyCommittedUpstream.length} declared file(s) already present in HEAD.`
+        : `Accepted turn ${entry.turn_id} has no staged repo changes to checkpoint.`,
+      already_committed_upstream: alreadyCommittedUpstream,
     };
   }
 

package/src/lib/turn-result-shape.js

@@ -0,0 +1,38 @@
+/**
+ * Lightweight staged turn-result shape guard.
+ *
+ * This is intentionally weaker than full acceptance validation. It exists for
+ * adapter pre-stage checks so obviously incomplete payloads (`{}`,
+ * `{"turn_id":"t1"}`, etc.) are rejected before they can be written into the
+ * governed staging path and mistaken for meaningful execution output.
+ */
+
+function isNonEmptyString(value) {
+  return typeof value === 'string' && value.trim() !== '';
+}
+
+/**
+ * Returns true when `value` has the minimum governed turn-result envelope:
+ *   - `schema_version`
+ *   - at least one identity field (`run_id` or `turn_id`)
+ *   - at least one lifecycle field (`status`, `role`, or `runtime_id`)
+ *
+ * Full schema validation still happens later via `validateStagedTurnResult`.
+ *
+ * @param {unknown} value
+ * @returns {boolean}
+ */
+export function hasMinimumTurnResultShape(value) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return false;
+  }
+
+  const candidate = /** @type {Record<string, unknown>} */ (value);
+  const hasSchemaVersion = isNonEmptyString(candidate.schema_version);
+  const hasIdentity = isNonEmptyString(candidate.run_id) || isNonEmptyString(candidate.turn_id);
+  const hasLifecycle = isNonEmptyString(candidate.status)
+    || isNonEmptyString(candidate.role)
+    || isNonEmptyString(candidate.runtime_id);
+
+  return hasSchemaVersion && hasIdentity && hasLifecycle;
+}

package/src/lib/turn-result-validator.js

@@ -75,7 +75,10 @@ export function validateStagedTurnResult(root, state, config, opts = {}) {
   const normContext = {};
   if (state) {
     normContext.phase = state.phase;
-    // Support both active_turns (v2+) and legacy current_turn formats
+    // Prefer active_turns (the persisted schema field); fall back to the
+    // current_turn compatibility alias for callers that pass a state shape
+    // built outside loadProjectState() (e.g. raw fixtures). Both surfaces are
+    // live per DEC-CURRENT-TURN-COMPAT-ALIAS-001 — current_turn is not legacy.
     const activeTurn = getActiveTurn(state) || state.current_turn;
     if (activeTurn) {
       const roleKey = activeTurn.assigned_role || activeTurn.role;
@@ -597,6 +600,15 @@ function validateVerification(tr) {
     }
   }
 
+  if (Array.isArray(v.commands)) {
+    for (let i = 0; i < v.commands.length; i++) {
+      const command = v.commands[i];
+      if (typeof command !== 'string' || command.trim().length === 0) {
+        errors.push(`verification.commands[${i}] must be a non-empty string.`);
+      }
+    }
+  }
+
   // machine_evidence exit codes should be consistent with status
   if (Array.isArray(v.machine_evidence)) {
     for (let i = 0; i < v.machine_evidence.length; i++) {
@@ -605,8 +617,8 @@ function validateVerification(tr) {
         errors.push(`verification.machine_evidence[${i}] must be an object.`);
         continue;
       }
-      if (typeof entry.command !== 'string') {
-        errors.push(`verification.machine_evidence[${i}].command must be a string.`);
+      if (typeof entry.command !== 'string' || entry.command.trim().length === 0) {
+        errors.push(`verification.machine_evidence[${i}].command must be a non-empty string.`);
       }
       if (typeof entry.exit_code !== 'number' || !Number.isInteger(entry.exit_code)) {
         errors.push(`verification.machine_evidence[${i}].exit_code must be an integer.`);