openclaw-scheduler 0.2.6 → 0.2.7

package/dispatch/README.md

@@ -291,8 +291,8 @@ No manual token configuration needed on a standard OpenClaw install.
 
 When `--deliver-to` is set, dispatch registers a **scheduler watcher job**
 after dispatching the session. The watcher polls the session result every
-minute until the agent produces a reply, then delivers via the scheduler's
-`handleDelivery` pipeline.
+minute until the agent sends the structured `done` completion signal, then
+delivers via the scheduler's `handleDelivery` pipeline.
 
 ```
 dispatch enqueue --deliver-to <telegram-user-id>
@@ -316,6 +316,20 @@ dispatch enqueue --deliver-to <telegram-user-id>
 Exit 1 with no output = retry on next cron tick (no spam — `announce-always`
 only delivers when `output.trim()` is truthy).
 
+Quiet sessions are treated conservatively. The watcher does not mark a running
+job failed just because `sessions.json` or the JSONL transcript has been quiet
+for 60 seconds. For high/xhigh reasoning work, the first idle result probe waits
+at least 10 minutes, idle auto-resolution waits at least 20 minutes, and the hard
+failure ceiling is longer than the requested task timeout. Missing or ambiguous
+gateway/session liveness fails open to "still monitoring" until the hard timeout
+window or a clear terminal error.
+
+While a label is still `running`, a plain assistant reply is diagnostic only.
+Successful final delivery requires the agent-side `done` signal and its
+structured completion payload. If an older watcher records an error and the
+worker later sends a valid `done`, the later completion is authoritative and the
+stale error is cleared from the label.
+
 ### Progress check-ins from subagent sessions
 
 Subagent sessions run without PATH access to the `openclaw` CLI, so

package/dispatch/index.mjs

@@ -40,6 +40,7 @@ import {
   hasCompletionSignal,
   taskRequiresGitSha,
 } from './completion.mjs';
+import { getDispatchLivenessPolicy } from './liveness.mjs';
 import { onStarted, onFinished, onStuck } from './hooks.mjs';
 import { resolveMessageInput } from './message-input.mjs';
 import { buildDispatchDeliverySurface } from '../scripts/dispatch-cli-utils.mjs';
@@ -211,9 +212,9 @@ function setLabelDone(name, data) {
       ...current[name],
       ...data,
       status: 'done',
+      error: null,
       updatedAt: new Date().toISOString(),
     };
-    delete current[name].error;
   });
   return labels[name];
 }
@@ -1311,16 +1312,17 @@ function cmdStatus(flags) {
       //
       // PING_STALE_MS:   3x the 60s ping interval -- if we haven't heard from the
       //                  watcher in 3 min, it's probably dead; fall through to check.
-      // hardCeilingMs:   job timeout * 1.5 -- absolute max regardless of ping age.
-      //                  Catches zombie watchers (watcher alive but session is stuck).
-      // idleThresholdMs: max(job timeout, 10 min) -- replaces the old hardcoded 10-min
-      //                  threshold so longer jobs aren't killed at exactly 10 min.
-      const PING_STALE_MS  = 3 * 60 * 1000;
-      const idleThresholdMs = Math.max((entry.timeoutSeconds || 600) * 1000, 10 * 60 * 1000);
-      // hardCeilingMs must be >= idleThresholdMs to avoid the ceiling undercutting the
-      // idle floor (e.g. timeoutSeconds=300 -> ceiling=7.5 min < idle=10 min would force
-      // zombie-guard threshold for sessions that should still use idleThresholdMs).
-      const hardCeilingMs  = Math.max((entry.timeoutSeconds || 600) * 1000 * 1.5, idleThresholdMs * 1.5);
+      // hardCeilingMs:   timeout/reasoning-aware hard ceiling. High-thinking
+      //                  work gets a larger quiet window before hard failure.
+      // idleThresholdMs: timeout/reasoning-aware quiet threshold. Ambiguous or
+      //                  missing liveness stays running until these thresholds.
+      const livenessPolicy = getDispatchLivenessPolicy(entry, {
+        startupGraceMs: STARTUP_GRACE_MS,
+        defaultTimeoutSeconds: 600,
+      });
+      const PING_STALE_MS = livenessPolicy.pingStaleMs;
+      const idleThresholdMs = livenessPolicy.idleFailureMs;
+      const hardCeilingMs = livenessPolicy.hardCeilingMs;
 
       let check;
       if (ageMs < STARTUP_GRACE_MS) {
@@ -1333,13 +1335,13 @@ function cmdStatus(flags) {
           check = { shouldResolve: false };
         } else {
           // Ping stale OR past hard ceiling: fall through to session store check
-          const thresh = ageMs >= hardCeilingMs ? 2 * 60 * 1000 : idleThresholdMs;
+          const thresh = ageMs >= hardCeilingMs ? livenessPolicy.hardTimeoutIdleMs : idleThresholdMs;
           check = checkSessionDone(entry.sessionKey, sessionsStore, thresh, true, spawnedAtMs);
         }
       } else {
         // No lastPing -- backward compat (sessions dispatched before heartbeat feature).
         // Use idleThresholdMs (job-aware) instead of the old hardcoded 10 min.
-        const thresh = ageMs >= hardCeilingMs ? 2 * 60 * 1000 : idleThresholdMs;
+        const thresh = ageMs >= hardCeilingMs ? livenessPolicy.hardTimeoutIdleMs : idleThresholdMs;
         check = checkSessionDone(entry.sessionKey, sessionsStore, thresh, true, spawnedAtMs);
       }
 
@@ -1616,10 +1618,13 @@ function cmdSync(flags) {
     // -- Heartbeat-based liveness guard (mirrors cmdStatus logic) ---------
     // Skip auto-resolve when the watcher's lastPing heartbeat is fresh.
     // See cmdStatus for full commentary on PING_STALE_MS / hardCeilingMs.
-    const PING_STALE_MS_SYNC  = 3 * 60 * 1000;
-    const idleThresholdMsSync = Math.max((entry.timeoutSeconds || 600) * 1000, 10 * 60 * 1000);
-    // hardCeilingMsSync must be >= idleThresholdMsSync (mirrors cmdStatus fix).
-    const hardCeilingMsSync   = Math.max((entry.timeoutSeconds || 600) * 1000 * 1.5, idleThresholdMsSync * 1.5);
+    const syncPolicy = getDispatchLivenessPolicy(entry, {
+      startupGraceMs: STARTUP_GRACE_MS_SYNC,
+      defaultTimeoutSeconds: 600,
+    });
+    const PING_STALE_MS_SYNC = syncPolicy.pingStaleMs;
+    const idleThresholdMsSync = syncPolicy.idleFailureMs;
+    const hardCeilingMsSync = syncPolicy.hardCeilingMs;
 
     if (entry.lastPing) {
       const pingAgeMs = Date.now() - new Date(entry.lastPing).getTime();
@@ -1629,7 +1634,7 @@ function cmdSync(flags) {
       }
     }
 
-    const syncThresh = elapsedMs >= hardCeilingMsSync ? 2 * 60 * 1000 : idleThresholdMsSync;
+    const syncThresh = elapsedMs >= hardCeilingMsSync ? syncPolicy.hardTimeoutIdleMs : idleThresholdMsSync;
     const check = checkSessionDone(entry.sessionKey, syncStore, syncThresh, true, spawnedAtMs);
 
     if (check.shouldResolve) {

package/dispatch/liveness.mjs

@@ -0,0 +1,61 @@
+const MINUTE_MS = 60 * 1000;
+
+function numberOrNull(value) {
+  const n = Number(value);
+  return Number.isFinite(n) && n > 0 ? n : null;
+}
+
+export function normalizeThinkingLevel(value) {
+  const text = typeof value === 'string' ? value.trim().toLowerCase() : '';
+  if (text === 'xhigh' || text === 'extra-high' || text === 'extra_high') return 'xhigh';
+  if (text === 'high') return 'high';
+  if (text === 'low') return 'low';
+  if (text === 'off' || text === 'none') return 'off';
+  return null;
+}
+
+export function getDispatchTimeoutSeconds(entry = {}, fallbackSeconds = 300) {
+  return numberOrNull(entry.timeoutSeconds)
+    ?? numberOrNull(entry.timeout)
+    ?? numberOrNull(fallbackSeconds)
+    ?? 300;
+}
+
+export function getDispatchLivenessPolicy(entry = {}, opts = {}) {
+  const now = numberOrNull(opts.now) ?? Date.now();
+  const timeoutSeconds = getDispatchTimeoutSeconds(entry, opts.defaultTimeoutSeconds);
+  const timeoutMs = timeoutSeconds * 1000;
+  const thinking = normalizeThinkingLevel(entry.thinking);
+  const isHighThinking = thinking === 'high' || thinking === 'xhigh';
+
+  const startupGraceMs = numberOrNull(opts.startupGraceMs)
+    ?? (isHighThinking ? 10 * MINUTE_MS : 5 * MINUTE_MS);
+  const pingStaleMs = numberOrNull(opts.pingStaleMs) ?? 3 * MINUTE_MS;
+  const idleProbeFloorMs = isHighThinking ? 10 * MINUTE_MS : 1 * MINUTE_MS;
+  const idleProbeMs = Math.max(
+    idleProbeFloorMs,
+    Math.min(timeoutMs * 0.25, isHighThinking ? 15 * MINUTE_MS : 5 * MINUTE_MS),
+  );
+  const idleFailureFloorMs = isHighThinking ? 20 * MINUTE_MS : 10 * MINUTE_MS;
+  const idleFailureMs = Math.max(timeoutMs, idleFailureFloorMs);
+  const hardCeilingMs = Math.max(timeoutMs * 1.5, idleFailureMs * (isHighThinking ? 2 : 1.5));
+  const hardTimeoutIdleMs = isHighThinking ? 5 * MINUTE_MS : 2 * MINUTE_MS;
+  const spawnedAtMs = entry.spawnedAt ? new Date(entry.spawnedAt).getTime() : 0;
+  const ageMs = spawnedAtMs ? now - spawnedAtMs : Infinity;
+
+  return {
+    thinking,
+    isHighThinking,
+    timeoutSeconds,
+    timeoutMs,
+    startupGraceMs,
+    pingStaleMs,
+    idleProbeMs,
+    idleFailureMs,
+    hardCeilingMs,
+    hardTimeoutIdleMs,
+    spawnedAtMs,
+    ageMs,
+    pastHardCeiling: ageMs >= hardCeilingMs,
+  };
+}

package/dispatch/watcher.mjs

@@ -36,6 +36,7 @@ import {
   hasCompletionSignal,
   resolveCompletionDelivery,
 } from './completion.mjs';
+import { getDispatchLivenessPolicy } from './liveness.mjs';
 import { sendMessage } from '../messages.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -988,8 +989,14 @@ const pollS       = parseInt(flags['poll-interval'] || '20', 10);
 const once        = flags.once === true || flags.once === 'true';
 exitZeroOnTerminal = once;
 
-// How long a session must be idle before we proactively check result
-const IDLE_RESULT_CHECK_MS = 60000;
+function getCurrentLivenessPolicy() {
+  const entry = loadLabels()[label] || { timeoutSeconds: timeoutS };
+  return getDispatchLivenessPolicy(entry, { defaultTimeoutSeconds: timeoutS });
+}
+
+function hasStructuredCompletion(result) {
+  return hasCompletionSignal(result?.completion);
+}
 
 if (!label) {
   process.stderr.write('[watcher] --label is required\n');
@@ -1104,18 +1111,22 @@ function runOnceAndExit() {
     const terminalJsonlReply = sessionId ? getSessionTerminalReply(sessionId, sessionAgent) : null;
     if (sessionId && terminalJsonlReply && isSessionCleanlyFinished(sessionId, sessionAgent)) {
       const result = dispatch('result', ['--label', label]);
-      deliverResult(label, result?.lastReply || terminalJsonlReply, 'completed (stop_reason=end_turn)', result?.completion || null);
+      if (hasStructuredCompletion(result)) {
+        deliverResult(label, result?.lastReply || terminalJsonlReply, 'completed (stop_reason=end_turn)', result?.completion || null);
+      }
+      process.stderr.write(`[watcher] stop_reason=end_turn observed without completion signal -- continuing to monitor\n`);
     }
   }
 
   const ageMs = status.liveness?.ageMs;
-  if (ageMs != null && ageMs >= IDLE_RESULT_CHECK_MS) {
+  const idleResultCheckMs = getCurrentLivenessPolicy().idleProbeMs;
+  if (ageMs != null && ageMs >= idleResultCheckMs) {
     const result = dispatch('result', ['--label', label]);
-    if (result?.lastReply || hasCompletionSignal(result?.completion)) {
+    if (hasStructuredCompletion(result)) {
       deliverResult(label, result?.lastReply || null, null, result?.completion || null);
     }
 
-    const stallReason = getRunningSessionStallReason(status, IDLE_RESULT_CHECK_MS);
+    const stallReason = getRunningSessionStallReason(status, idleResultCheckMs);
     if (stallReason) {
       process.stderr.write(`[watcher] [${label}] ${stallReason}\n`);
       markLabelError(label, stallReason);
@@ -1477,8 +1488,11 @@ while (Date.now() < deadline) {
     if (_sid2a && terminalJsonlReply && isSessionCleanlyFinished(_sid2a, _adir2a)) {
       process.stderr.write(`[watcher] stop_reason=end_turn detected -- delivering early\n`);
       const result = dispatch('result', ['--label', label]);
-      deliverResult(label, result?.lastReply || terminalJsonlReply, 'completed (stop_reason=end_turn)', result?.completion || null);
-      // deliverResult exits
+      if (hasStructuredCompletion(result)) {
+        deliverResult(label, result?.lastReply || terminalJsonlReply, 'completed (stop_reason=end_turn)', result?.completion || null);
+        // deliverResult exits
+      }
+      process.stderr.write(`[watcher] stop_reason=end_turn observed without completion signal -- continuing to monitor\n`);
     }
   }
 
@@ -1489,13 +1503,14 @@ while (Date.now() < deadline) {
   // while this watcher's lastPing heartbeat is fresh (written every 60s);
   // this path handles normal completion before the ping goes stale.
   const ageMs = status.liveness?.ageMs;
-  if (ageMs != null && ageMs >= IDLE_RESULT_CHECK_MS) {
+  const idleResultCheckMs = getCurrentLivenessPolicy().idleProbeMs;
+  if (ageMs != null && ageMs >= idleResultCheckMs) {
     const result = dispatch('result', ['--label', label]);
-    if (result?.lastReply || hasCompletionSignal(result?.completion)) {
+    if (hasStructuredCompletion(result)) {
       deliverResult(label, result?.lastReply || null, null, result?.completion || null);
     }
 
-    const stallReason = getRunningSessionStallReason(status, IDLE_RESULT_CHECK_MS);
+    const stallReason = getRunningSessionStallReason(status, idleResultCheckMs);
     if (stallReason) {
       process.stderr.write(`[watcher] [${label}] ${stallReason}\n`);
       markLabelError(label, stallReason);
@@ -1577,7 +1592,7 @@ if (sessionInternalId) {
 // If the session already completed (gateway pruned it -> null tokens), exit cleanly.
 if (statusAtDeadline?.status === 'done' || baselineTokens === null) {
   const r = dispatch('result', ['--label', label]);
-  if (r?.lastReply || hasCompletionSignal(r?.completion)) {
+  if (hasStructuredCompletion(r)) {
     // deliverResult calls process.exit(0) internally
     deliverResult(label, r?.lastReply || null, statusAtDeadline?.summary || null, r?.completion || null);
   }
@@ -1616,7 +1631,7 @@ while (Date.now() - flatSince < FLAT_WINDOW_MS) {
     deliverResult(label, r?.lastReply || null, st.summary, r?.completion || st?.completion || null);
   }
   const r2 = dispatch('result', ['--label', label]);
-  if (r2?.lastReply || hasCompletionSignal(r2?.completion)) {
+  if (hasStructuredCompletion(r2)) {
     // deliverResult calls process.exit(0) internally
     deliverResult(label, r2?.lastReply || null, null, r2?.completion || null);
   }
@@ -1710,7 +1725,7 @@ if (sessionInternalId) {
         deliverResult(label, rExt?.lastReply || null, stExt.summary, rExt?.completion || stExt?.completion || null);
       }
       const rExt2 = dispatch('result', ['--label', label]);
-      if (rExt2?.lastReply || hasCompletionSignal(rExt2?.completion)) {
+      if (hasStructuredCompletion(rExt2)) {
         // deliverResult calls process.exit(0) internally
         deliverResult(label, rExt2?.lastReply || null, null, rExt2?.completion || null);
       }
@@ -1767,7 +1782,7 @@ for (const round of steerRounds) {
     deliverResult(label, r3?.lastReply || null, st2.summary, r3?.completion || st2?.completion || null);
   }
   const r3 = dispatch('result', ['--label', label]);
-  if (r3?.lastReply || hasCompletionSignal(r3?.completion)) {
+  if (hasStructuredCompletion(r3)) {
     // deliverResult calls process.exit(0) internally
     deliverResult(label, r3?.lastReply || null, null, r3?.completion || null);
   }
@@ -1782,7 +1797,7 @@ for (const round of steerRounds) {
       if (st3?.status === 'done') {
         // Check if a result was captured before marking as error
         const r4 = dispatch('result', ['--label', label]);
-        if (r4?.lastReply || hasCompletionSignal(r4?.completion)) {
+        if (hasStructuredCompletion(r4)) {
           deliverResult(label, r4?.lastReply || null, st3.summary, r4?.completion || st3?.completion || null); // deliverResult calls process.exit(0)
         }
         markLabelError(label, 'timed out -- killed after steer attempts (no result captured)');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openclaw-scheduler",
-  "version": "0.2.6",
+  "version": "0.2.7",
   "description": "SQLite-backed job scheduler and workflow engine for OpenClaw agents",
   "type": "module",
   "main": "./index.js",
@@ -42,6 +42,7 @@
     "dispatch/deliver-watcher.sh",
     "dispatch/hooks.mjs",
     "dispatch/index.mjs",
+    "dispatch/liveness.mjs",
     "dispatch/message-input.mjs",
     "dispatch/README.md",
     "dispatch/watcher.mjs",