openclaw-scheduler 0.2.6 → 0.2.8

package/dispatch/529-recovery.mjs

@@ -23,9 +23,10 @@ import { readFileSync, writeFileSync, renameSync } from 'fs';
 import { execFileSync } from 'child_process';
 import { dirname, join } from 'path';
 import { fileURLToPath } from 'url';
+import { resolveLabelsPath } from './paths.mjs';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
-const LABELS_PATH = process.env.DISPATCH_LABELS_PATH || join(__dirname, 'labels.json');
+const LABELS_PATH = resolveLabelsPath({ legacyCandidates: [join(__dirname, 'labels.json')] });
 const INDEX_PATH  = process.env.DISPATCH_INDEX_PATH  || join(__dirname, 'index.mjs');
 
 const MAX_RETRIES = 3;

package/dispatch/README.md

@@ -23,7 +23,7 @@ No scheduler DB dependency. No dispatcher tick delay. Sessions start instantly.
 | `chilisaus.mjs` | Branded wrapper |
 | `config.example.json` | Example config |
 | `test-done-postoffice.mjs` | Done handler test |
-| `labels.json` | Local label→session ledger (gitignored) |
+| `~/.openclaw/scheduler/dispatch/labels.json` | Durable label→session ledger |
 | `README.md` | This file |
 
 ---
@@ -45,7 +45,7 @@ Orchestrator calls:
   → Patches session with model/thinking/spawnDepth
   → Calls gateway `agent` method with the task
   → Session starts immediately (no scheduler tick delay)
-  → Tracks label→sessionKey in labels.json
+  → Tracks label→sessionKey in the durable labels ledger
   → Agent auto-announces results on completion
   → hooks.mjs fires dispatch.started to Loki
 ```
@@ -117,7 +117,7 @@ node dispatch/index.mjs stuck --threshold-min 15
 Exit 0 = nothing stuck (silent).
 Exit 1 = stuck sessions found (triggers announce delivery).
 
-Checks labels.json for sessions marked `running`, cross-references gateway
+Checks the labels ledger for sessions marked `running`, cross-references gateway
 session store for last activity timestamp.
 
 ### `result` — last assistant reply from a session
@@ -206,7 +206,7 @@ Shows all labels in the ledger, sorted by most recent. Filter by status.
 node dispatch/index.mjs sync
 ```
 
-Reconciles `labels.json` with the gateway sessions store. Sessions that no
+Reconciles the labels ledger with the gateway sessions store. Sessions that no
 longer exist on the gateway are marked stale, and sessions present on the
 gateway but missing from the ledger are imported. Useful after gateway restarts
 or manual session cleanup.
@@ -291,8 +291,8 @@ No manual token configuration needed on a standard OpenClaw install.
 
 When `--deliver-to` is set, dispatch registers a **scheduler watcher job**
 after dispatching the session. The watcher polls the session result every
-minute until the agent produces a reply, then delivers via the scheduler's
-`handleDelivery` pipeline.
+minute until the agent sends the structured `done` completion signal, then
+delivers via the scheduler's `handleDelivery` pipeline.
 
 ```
 dispatch enqueue --deliver-to <telegram-user-id>
@@ -316,6 +316,20 @@ dispatch enqueue --deliver-to <telegram-user-id>
 Exit 1 with no output = retry on next cron tick (no spam — `announce-always`
 only delivers when `output.trim()` is truthy).
 
+Quiet sessions are treated conservatively. The watcher does not mark a running
+job failed just because `sessions.json` or the JSONL transcript has been quiet
+for 60 seconds. For high/xhigh reasoning work, the first idle result probe waits
+at least 10 minutes, idle auto-resolution waits at least 20 minutes, and the hard
+failure ceiling is longer than the requested task timeout. Missing or ambiguous
+gateway/session liveness fails open to "still monitoring" until the hard timeout
+window or a clear terminal error.
+
+While a label is still `running`, a plain assistant reply is diagnostic only.
+Successful final delivery requires the agent-side `done` signal and its
+structured completion payload. If an older watcher records an error and the
+worker later sends a valid `done`, the later completion is authoritative and the
+stale error is cleared from the label.
+
 ### Progress check-ins from subagent sessions
 
 Subagent sessions run without PATH access to the `openclaw` CLI, so

package/dispatch/index.mjs

@@ -40,6 +40,8 @@ import {
   hasCompletionSignal,
   taskRequiresGitSha,
 } from './completion.mjs';
+import { getDispatchLivenessPolicy } from './liveness.mjs';
+import { resolveLabelsPath } from './paths.mjs';
 import { onStarted, onFinished, onStuck } from './hooks.mjs';
 import { resolveMessageInput } from './message-input.mjs';
 import { buildDispatchDeliverySurface } from '../scripts/dispatch-cli-utils.mjs';
@@ -66,7 +68,9 @@ const INVOKE_DIR = (() => {
 
 // -- Config ---------------------------------------------------
 
-const LABELS_PATH = process.env.DISPATCH_LABELS_PATH || join(INVOKE_DIR, 'labels.json');
+const LABELS_PATH = resolveLabelsPath({
+  legacyCandidates: [join(INVOKE_DIR, 'labels.json'), join(__dirname, 'labels.json')],
+});
 
 /** Load dispatch config from config.json.
  *  Resolution order:
@@ -211,9 +215,9 @@ function setLabelDone(name, data) {
       ...current[name],
       ...data,
       status: 'done',
+      error: null,
       updatedAt: new Date().toISOString(),
     };
-    delete current[name].error;
   });
   return labels[name];
 }
@@ -1311,16 +1315,17 @@ function cmdStatus(flags) {
       //
       // PING_STALE_MS:   3x the 60s ping interval -- if we haven't heard from the
       //                  watcher in 3 min, it's probably dead; fall through to check.
-      // hardCeilingMs:   job timeout * 1.5 -- absolute max regardless of ping age.
-      //                  Catches zombie watchers (watcher alive but session is stuck).
-      // idleThresholdMs: max(job timeout, 10 min) -- replaces the old hardcoded 10-min
-      //                  threshold so longer jobs aren't killed at exactly 10 min.
-      const PING_STALE_MS  = 3 * 60 * 1000;
-      const idleThresholdMs = Math.max((entry.timeoutSeconds || 600) * 1000, 10 * 60 * 1000);
-      // hardCeilingMs must be >= idleThresholdMs to avoid the ceiling undercutting the
-      // idle floor (e.g. timeoutSeconds=300 -> ceiling=7.5 min < idle=10 min would force
-      // zombie-guard threshold for sessions that should still use idleThresholdMs).
-      const hardCeilingMs  = Math.max((entry.timeoutSeconds || 600) * 1000 * 1.5, idleThresholdMs * 1.5);
+      // hardCeilingMs:   timeout/reasoning-aware hard ceiling. High-thinking
+      //                  work gets a larger quiet window before hard failure.
+      // idleThresholdMs: timeout/reasoning-aware quiet threshold. Ambiguous or
+      //                  missing liveness stays running until these thresholds.
+      const livenessPolicy = getDispatchLivenessPolicy(entry, {
+        startupGraceMs: STARTUP_GRACE_MS,
+        defaultTimeoutSeconds: 600,
+      });
+      const PING_STALE_MS = livenessPolicy.pingStaleMs;
+      const idleThresholdMs = livenessPolicy.idleFailureMs;
+      const hardCeilingMs = livenessPolicy.hardCeilingMs;
 
       let check;
       if (ageMs < STARTUP_GRACE_MS) {
@@ -1333,13 +1338,13 @@ function cmdStatus(flags) {
           check = { shouldResolve: false };
         } else {
           // Ping stale OR past hard ceiling: fall through to session store check
-          const thresh = ageMs >= hardCeilingMs ? 2 * 60 * 1000 : idleThresholdMs;
+          const thresh = ageMs >= hardCeilingMs ? livenessPolicy.hardTimeoutIdleMs : idleThresholdMs;
           check = checkSessionDone(entry.sessionKey, sessionsStore, thresh, true, spawnedAtMs);
         }
       } else {
         // No lastPing -- backward compat (sessions dispatched before heartbeat feature).
         // Use idleThresholdMs (job-aware) instead of the old hardcoded 10 min.
-        const thresh = ageMs >= hardCeilingMs ? 2 * 60 * 1000 : idleThresholdMs;
+        const thresh = ageMs >= hardCeilingMs ? livenessPolicy.hardTimeoutIdleMs : idleThresholdMs;
         check = checkSessionDone(entry.sessionKey, sessionsStore, thresh, true, spawnedAtMs);
       }
 
@@ -1616,10 +1621,13 @@ function cmdSync(flags) {
     // -- Heartbeat-based liveness guard (mirrors cmdStatus logic) ---------
     // Skip auto-resolve when the watcher's lastPing heartbeat is fresh.
     // See cmdStatus for full commentary on PING_STALE_MS / hardCeilingMs.
-    const PING_STALE_MS_SYNC  = 3 * 60 * 1000;
-    const idleThresholdMsSync = Math.max((entry.timeoutSeconds || 600) * 1000, 10 * 60 * 1000);
-    // hardCeilingMsSync must be >= idleThresholdMsSync (mirrors cmdStatus fix).
-    const hardCeilingMsSync   = Math.max((entry.timeoutSeconds || 600) * 1000 * 1.5, idleThresholdMsSync * 1.5);
+    const syncPolicy = getDispatchLivenessPolicy(entry, {
+      startupGraceMs: STARTUP_GRACE_MS_SYNC,
+      defaultTimeoutSeconds: 600,
+    });
+    const PING_STALE_MS_SYNC = syncPolicy.pingStaleMs;
+    const idleThresholdMsSync = syncPolicy.idleFailureMs;
+    const hardCeilingMsSync = syncPolicy.hardCeilingMs;
 
     if (entry.lastPing) {
       const pingAgeMs = Date.now() - new Date(entry.lastPing).getTime();
@@ -1629,7 +1637,7 @@ function cmdSync(flags) {
       }
     }
 
-    const syncThresh = elapsedMs >= hardCeilingMsSync ? 2 * 60 * 1000 : idleThresholdMsSync;
+    const syncThresh = elapsedMs >= hardCeilingMsSync ? syncPolicy.hardTimeoutIdleMs : idleThresholdMsSync;
     const check = checkSessionDone(entry.sessionKey, syncStore, syncThresh, true, spawnedAtMs);
 
     if (check.shouldResolve) {

package/dispatch/liveness.mjs

@@ -0,0 +1,61 @@
+const MINUTE_MS = 60 * 1000;
+
+function numberOrNull(value) {
+  const n = Number(value);
+  return Number.isFinite(n) && n > 0 ? n : null;
+}
+
+export function normalizeThinkingLevel(value) {
+  const text = typeof value === 'string' ? value.trim().toLowerCase() : '';
+  if (text === 'xhigh' || text === 'extra-high' || text === 'extra_high') return 'xhigh';
+  if (text === 'high') return 'high';
+  if (text === 'low') return 'low';
+  if (text === 'off' || text === 'none') return 'off';
+  return null;
+}
+
+export function getDispatchTimeoutSeconds(entry = {}, fallbackSeconds = 300) {
+  return numberOrNull(entry.timeoutSeconds)
+    ?? numberOrNull(entry.timeout)
+    ?? numberOrNull(fallbackSeconds)
+    ?? 300;
+}
+
+export function getDispatchLivenessPolicy(entry = {}, opts = {}) {
+  const now = numberOrNull(opts.now) ?? Date.now();
+  const timeoutSeconds = getDispatchTimeoutSeconds(entry, opts.defaultTimeoutSeconds);
+  const timeoutMs = timeoutSeconds * 1000;
+  const thinking = normalizeThinkingLevel(entry.thinking);
+  const isHighThinking = thinking === 'high' || thinking === 'xhigh';
+
+  const startupGraceMs = numberOrNull(opts.startupGraceMs)
+    ?? (isHighThinking ? 10 * MINUTE_MS : 5 * MINUTE_MS);
+  const pingStaleMs = numberOrNull(opts.pingStaleMs) ?? 3 * MINUTE_MS;
+  const idleProbeFloorMs = isHighThinking ? 10 * MINUTE_MS : 1 * MINUTE_MS;
+  const idleProbeMs = Math.max(
+    idleProbeFloorMs,
+    Math.min(timeoutMs * 0.25, isHighThinking ? 15 * MINUTE_MS : 5 * MINUTE_MS),
+  );
+  const idleFailureFloorMs = isHighThinking ? 20 * MINUTE_MS : 10 * MINUTE_MS;
+  const idleFailureMs = Math.max(timeoutMs, idleFailureFloorMs);
+  const hardCeilingMs = Math.max(timeoutMs * 1.5, idleFailureMs * (isHighThinking ? 2 : 1.5));
+  const hardTimeoutIdleMs = isHighThinking ? 5 * MINUTE_MS : 2 * MINUTE_MS;
+  const spawnedAtMs = entry.spawnedAt ? new Date(entry.spawnedAt).getTime() : 0;
+  const ageMs = spawnedAtMs ? now - spawnedAtMs : Infinity;
+
+  return {
+    thinking,
+    isHighThinking,
+    timeoutSeconds,
+    timeoutMs,
+    startupGraceMs,
+    pingStaleMs,
+    idleProbeMs,
+    idleFailureMs,
+    hardCeilingMs,
+    hardTimeoutIdleMs,
+    spawnedAtMs,
+    ageMs,
+    pastHardCeiling: ageMs >= hardCeilingMs,
+  };
+}

package/dispatch/watcher.mjs

@@ -36,11 +36,13 @@ import {
   hasCompletionSignal,
   resolveCompletionDelivery,
 } from './completion.mjs';
+import { getDispatchLivenessPolicy } from './liveness.mjs';
+import { resolveLabelsPath } from './paths.mjs';
 import { sendMessage } from '../messages.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const INDEX_PATH = process.env.DISPATCH_INDEX_PATH || join(__dirname, 'index.mjs');
-const LABELS_PATH = process.env.DISPATCH_LABELS_PATH || join(__dirname, 'labels.json');
+const LABELS_PATH = resolveLabelsPath({ legacyCandidates: [join(__dirname, 'labels.json')] });
 const HOME_DIR = process.env.HOME || homedir();
 let labelsCache = null;
 let labelsCacheSignature = null;
@@ -988,8 +990,14 @@ const pollS       = parseInt(flags['poll-interval'] || '20', 10);
 const once        = flags.once === true || flags.once === 'true';
 exitZeroOnTerminal = once;
 
-// How long a session must be idle before we proactively check result
-const IDLE_RESULT_CHECK_MS = 60000;
+function getCurrentLivenessPolicy() {
+  const entry = loadLabels()[label] || { timeoutSeconds: timeoutS };
+  return getDispatchLivenessPolicy(entry, { defaultTimeoutSeconds: timeoutS });
+}
+
+function hasStructuredCompletion(result) {
+  return hasCompletionSignal(result?.completion);
+}
 
 if (!label) {
   process.stderr.write('[watcher] --label is required\n');
@@ -1104,18 +1112,22 @@ function runOnceAndExit() {
     const terminalJsonlReply = sessionId ? getSessionTerminalReply(sessionId, sessionAgent) : null;
     if (sessionId && terminalJsonlReply && isSessionCleanlyFinished(sessionId, sessionAgent)) {
       const result = dispatch('result', ['--label', label]);
-      deliverResult(label, result?.lastReply || terminalJsonlReply, 'completed (stop_reason=end_turn)', result?.completion || null);
+      if (hasStructuredCompletion(result)) {
+        deliverResult(label, result?.lastReply || terminalJsonlReply, 'completed (stop_reason=end_turn)', result?.completion || null);
+      }
+      process.stderr.write(`[watcher] stop_reason=end_turn observed without completion signal -- continuing to monitor\n`);
     }
   }
 
   const ageMs = status.liveness?.ageMs;
-  if (ageMs != null && ageMs >= IDLE_RESULT_CHECK_MS) {
+  const idleResultCheckMs = getCurrentLivenessPolicy().idleProbeMs;
+  if (ageMs != null && ageMs >= idleResultCheckMs) {
     const result = dispatch('result', ['--label', label]);
-    if (result?.lastReply || hasCompletionSignal(result?.completion)) {
+    if (hasStructuredCompletion(result)) {
       deliverResult(label, result?.lastReply || null, null, result?.completion || null);
     }
 
-    const stallReason = getRunningSessionStallReason(status, IDLE_RESULT_CHECK_MS);
+    const stallReason = getRunningSessionStallReason(status, idleResultCheckMs);
     if (stallReason) {
       process.stderr.write(`[watcher] [${label}] ${stallReason}\n`);
       markLabelError(label, stallReason);
@@ -1477,8 +1489,11 @@ while (Date.now() < deadline) {
     if (_sid2a && terminalJsonlReply && isSessionCleanlyFinished(_sid2a, _adir2a)) {
       process.stderr.write(`[watcher] stop_reason=end_turn detected -- delivering early\n`);
       const result = dispatch('result', ['--label', label]);
-      deliverResult(label, result?.lastReply || terminalJsonlReply, 'completed (stop_reason=end_turn)', result?.completion || null);
-      // deliverResult exits
+      if (hasStructuredCompletion(result)) {
+        deliverResult(label, result?.lastReply || terminalJsonlReply, 'completed (stop_reason=end_turn)', result?.completion || null);
+        // deliverResult exits
+      }
+      process.stderr.write(`[watcher] stop_reason=end_turn observed without completion signal -- continuing to monitor\n`);
     }
   }
 
@@ -1489,13 +1504,14 @@ while (Date.now() < deadline) {
   // while this watcher's lastPing heartbeat is fresh (written every 60s);
   // this path handles normal completion before the ping goes stale.
   const ageMs = status.liveness?.ageMs;
-  if (ageMs != null && ageMs >= IDLE_RESULT_CHECK_MS) {
+  const idleResultCheckMs = getCurrentLivenessPolicy().idleProbeMs;
+  if (ageMs != null && ageMs >= idleResultCheckMs) {
     const result = dispatch('result', ['--label', label]);
-    if (result?.lastReply || hasCompletionSignal(result?.completion)) {
+    if (hasStructuredCompletion(result)) {
       deliverResult(label, result?.lastReply || null, null, result?.completion || null);
     }
 
-    const stallReason = getRunningSessionStallReason(status, IDLE_RESULT_CHECK_MS);
+    const stallReason = getRunningSessionStallReason(status, idleResultCheckMs);
     if (stallReason) {
       process.stderr.write(`[watcher] [${label}] ${stallReason}\n`);
       markLabelError(label, stallReason);
@@ -1577,7 +1593,7 @@ if (sessionInternalId) {
 // If the session already completed (gateway pruned it -> null tokens), exit cleanly.
 if (statusAtDeadline?.status === 'done' || baselineTokens === null) {
   const r = dispatch('result', ['--label', label]);
-  if (r?.lastReply || hasCompletionSignal(r?.completion)) {
+  if (hasStructuredCompletion(r)) {
     // deliverResult calls process.exit(0) internally
     deliverResult(label, r?.lastReply || null, statusAtDeadline?.summary || null, r?.completion || null);
   }
@@ -1616,7 +1632,7 @@ while (Date.now() - flatSince < FLAT_WINDOW_MS) {
     deliverResult(label, r?.lastReply || null, st.summary, r?.completion || st?.completion || null);
   }
   const r2 = dispatch('result', ['--label', label]);
-  if (r2?.lastReply || hasCompletionSignal(r2?.completion)) {
+  if (hasStructuredCompletion(r2)) {
     // deliverResult calls process.exit(0) internally
     deliverResult(label, r2?.lastReply || null, null, r2?.completion || null);
   }
@@ -1710,7 +1726,7 @@ if (sessionInternalId) {
         deliverResult(label, rExt?.lastReply || null, stExt.summary, rExt?.completion || stExt?.completion || null);
       }
       const rExt2 = dispatch('result', ['--label', label]);
-      if (rExt2?.lastReply || hasCompletionSignal(rExt2?.completion)) {
+      if (hasStructuredCompletion(rExt2)) {
         // deliverResult calls process.exit(0) internally
         deliverResult(label, rExt2?.lastReply || null, null, rExt2?.completion || null);
       }
@@ -1767,7 +1783,7 @@ for (const round of steerRounds) {
     deliverResult(label, r3?.lastReply || null, st2.summary, r3?.completion || st2?.completion || null);
   }
   const r3 = dispatch('result', ['--label', label]);
-  if (r3?.lastReply || hasCompletionSignal(r3?.completion)) {
+  if (hasStructuredCompletion(r3)) {
     // deliverResult calls process.exit(0) internally
     deliverResult(label, r3?.lastReply || null, null, r3?.completion || null);
   }
@@ -1782,7 +1798,7 @@ for (const round of steerRounds) {
       if (st3?.status === 'done') {
         // Check if a result was captured before marking as error
         const r4 = dispatch('result', ['--label', label]);
-        if (r4?.lastReply || hasCompletionSignal(r4?.completion)) {
+        if (hasStructuredCompletion(r4)) {
           deliverResult(label, r4?.lastReply || null, st3.summary, r4?.completion || st3?.completion || null); // deliverResult calls process.exit(0)
         }
         markLabelError(label, 'timed out -- killed after steer attempts (no result captured)');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openclaw-scheduler",
-  "version": "0.2.6",
+  "version": "0.2.8",
   "description": "SQLite-backed job scheduler and workflow engine for OpenClaw agents",
   "type": "module",
   "main": "./index.js",
@@ -42,6 +42,7 @@
     "dispatch/deliver-watcher.sh",
     "dispatch/hooks.mjs",
     "dispatch/index.mjs",
+    "dispatch/liveness.mjs",
     "dispatch/message-input.mjs",
     "dispatch/README.md",
     "dispatch/watcher.mjs",

package/scripts/stuck-run-detector.mjs

@@ -29,12 +29,13 @@ import { join, dirname } from 'path';
 import { fileURLToPath } from 'url';
 import { tmpdir } from 'os';
 import { resolveDispatchCliPath, resolveDispatchLabel } from './dispatch-cli-utils.mjs';
+import { resolveLabelsPath } from '../dispatch/paths.mjs';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
 // -- Paths ----------------------------------------------------
 
-const LABELS_PATH  = process.env.DISPATCH_LABELS_PATH || join(__dirname, '..', 'dispatch', 'labels.json');
+const LABELS_PATH  = resolveLabelsPath({ legacyCandidates: [join(__dirname, '..', 'dispatch', 'labels.json')] });
 const STATE_PATH   = process.env.STUCK_STATE_PATH || join(tmpdir(), 'stuck-detector-state.json');
 const DISPATCH_CLI = resolveDispatchCliPath(process.env);
 const DISPATCH_IS_BIN = !DISPATCH_CLI.includes('/') && !DISPATCH_CLI.includes('\\');