openclaw-scheduler 0.2.5 → 0.2.7

package/dispatch/index.mjs

@@ -40,6 +40,7 @@ import {
   hasCompletionSignal,
   taskRequiresGitSha,
 } from './completion.mjs';
+import { getDispatchLivenessPolicy } from './liveness.mjs';
 import { onStarted, onFinished, onStuck } from './hooks.mjs';
 import { resolveMessageInput } from './message-input.mjs';
 import { buildDispatchDeliverySurface } from '../scripts/dispatch-cli-utils.mjs';
@@ -205,6 +206,19 @@ function setLabel(name, data) {
   return labels[name];
 }
 
+function setLabelDone(name, data) {
+  const labels = mutateLabels((current) => {
+    current[name] = {
+      ...current[name],
+      ...data,
+      status: 'done',
+      error: null,
+      updatedAt: new Date().toISOString(),
+    };
+  });
+  return labels[name];
+}
+
 // -- Gateway Calls --------------------------------------------
 
 /**
@@ -352,7 +366,17 @@ function getSessionJsonlPath(agent = 'main', sessionId) {
 
 function inspectSessionActivitySignal(sessionKey, sessionsStore) {
   if (!sessionKey || !sessionsStore?.[sessionKey]) {
-    return { found: false, hasActivitySignal: false, messageCount: null, jsonlExists: false, hasTokens: false, updatedAtMs: null };
+    return {
+      found: false,
+      hasStartedSignal: false,
+      hasActivitySignal: false,
+      messageCount: null,
+      jsonlExists: false,
+      hasTokens: false,
+      updatedAtMs: null,
+      sessionStartedAtMs: null,
+      sessionId: null,
+    };
   }
 
   const agent = agentFromSessionKey(sessionKey) || 'main';
@@ -360,6 +384,9 @@ function inspectSessionActivitySignal(sessionKey, sessionsStore) {
   const jsonlPath = getSessionJsonlPath(agent, entry.sessionId);
   const jsonlExists = jsonlPath ? existsSync(jsonlPath) : false;
   const hasTokens = typeof entry.totalTokens === 'number' && entry.totalTokens > 0;
+  const sessionStartedAtMs = toTimestampMs(entry.sessionStartedAt || entry.startedAt);
+  const updatedAtMs = toTimestampMs(entry.updatedAt);
+  const hasStartedSignal = Boolean(entry.sessionId) || sessionStartedAtMs !== null || updatedAtMs !== null;
   let messageCount = null;
 
   try {
@@ -371,11 +398,14 @@ function inspectSessionActivitySignal(sessionKey, sessionsStore) {
 
   return {
     found: true,
+    hasStartedSignal,
     hasActivitySignal: jsonlExists || hasTokens || (typeof messageCount === 'number' && messageCount > 0),
     messageCount,
     jsonlExists,
     hasTokens,
-    updatedAtMs: toTimestampMs(entry.updatedAt),
+    updatedAtMs,
+    sessionStartedAtMs,
+    sessionId: entry.sessionId || null,
   };
 }
 
@@ -385,12 +415,7 @@ function inspectSessionBootstrapFailure(sessionKey, sessionsStore, spawnedAtMs,
   }
 
   const ageMs = spawnedAtMs ? Date.now() - spawnedAtMs : Infinity;
-  if (ageMs < startupGraceMs || ageMs > startupGraceMs * 2) {
-    return { shouldResolve: false, reason: null, errorMsg: null };
-  }
-
-  const signal = inspectSessionActivitySignal(sessionKey, sessionsStore);
-  if (signal.hasActivitySignal) {
+  if (ageMs < startupGraceMs) {
     return { shouldResolve: false, reason: null, errorMsg: null };
   }
 
@@ -403,22 +428,10 @@ function inspectSessionBootstrapFailure(sessionKey, sessionsStore, spawnedAtMs,
     };
   }
 
-  if (signal.messageCount === 0) {
-    return {
-      shouldResolve: true,
-      reason: 'session entered sessions store but never wrote transcript/history',
-      errorMsg: 'spawn-failure: session entered sessions store but never wrote transcript/history',
-    };
-  }
-
-  if (signal.updatedAtMs !== null && spawnedAtMs && signal.updatedAtMs <= spawnedAtMs + 5000) {
-    return {
-      shouldResolve: true,
-      reason: 'session entered sessions store but never showed any activity',
-      errorMsg: 'spawn-failure: session entered sessions store but never showed any activity',
-    };
-  }
-
+  // A Codex session can enter the sessions store before chat.history, JSONL, or
+  // token counters are written. Treat that as "still booting"; the watcher and
+  // job timeout own later failure handling. Only fail fast when the gateway has
+  // recorded an explicit lane error above.
   return { shouldResolve: false, reason: null, errorMsg: null };
 }
 
@@ -683,7 +696,7 @@ function quoteForSingleQuotedShell(value) {
 }
 
 /**
- * Schedule a one-shot delivery watcher shell job for a dispatch label.
+ * Schedule a quick-poll delivery watcher shell job for a dispatch label.
  * Used both for the initial watcher registration and SIGTERM handoffs.
  */
 function scheduleDeliveryWatcherJob({
@@ -704,13 +717,19 @@ function scheduleDeliveryWatcherJob({
   const watcherTimeoutS = Number(timeoutSeconds) + 120;
   const idleThresholdS = Number(idleThresholdSeconds) || 300;
   const sq = quoteForSingleQuotedShell;
-  const watcherCmd = `DISPATCH_LABELS_PATH='${sq(LABELS_PATH)}' '${sq(process.execPath)}' '${sq(watcherPath)}' --label '${sq(label)}' --timeout ${watcherTimeoutS} --poll-interval 20 --idle-threshold ${idleThresholdS}`;
+  const watcherCmd =
+    `DISPATCH_LABELS_PATH='${sq(LABELS_PATH)}' ` +
+    `DISPATCH_INDEX_PATH='${sq(join(__dirname, 'index.mjs'))}' ` +
+    `'${sq(process.execPath)}' '${sq(watcherPath)}' ` +
+    `--label '${sq(label)}' --timeout ${watcherTimeoutS} ` +
+    `--poll-interval 20 --idle-threshold ${idleThresholdS} --once`;
 
   const nowUtc = new Date().toISOString().replace('T', ' ').slice(0, 19);
   const jobSpec = {
     name:                     `${agentBrand}-deliver:${label}${nameSuffix}`,
-    schedule_kind:            'at',
-    schedule_at:              nowUtc,
+    schedule_kind:            'cron',
+    schedule_cron:            config.deliver_watcher_cron || '* * * * *',
+    next_run_at:              nowUtc,
     session_target:           'shell',
     payload_kind:             'shellCommand',
     payload_message:          watcherCmd,
@@ -720,8 +739,7 @@ function scheduleDeliveryWatcherJob({
     delivery_guarantee:       'at-least-once',
     ttl_hours:                config.deliver_watcher_ttl_hours ?? 48,
     overlap_policy:           'skip',
-    run_timeout_ms:           Math.max(watcherTimeoutS, 4 * 3600) * 1000
-                              + 420 * 1000,
+    run_timeout_ms:           120_000,
     delete_after_run:         1,
     origin:                   origin || 'system',
   };
@@ -1088,9 +1106,10 @@ async function cmdEnqueue(flags) {
     }
 
     // -- Register scheduler watcher for delivery ---------------
-    // Creates a one-shot shell job that runs watcher.mjs (blocks until session
-    // completes, outputs result). The scheduler's handleDelivery delivers with
-    // retry, alias resolution, and audit trail in scheduler.db.
+    // Creates a quick-poll shell job that runs watcher.mjs once per tick. Empty
+    // stdout means "still running" and advances the next tick without delivery.
+    // Terminal stdout goes through the scheduler's handleDelivery with retry,
+    // alias resolution, and audit trail in scheduler.db.
     // The watcher is the only final-delivery path for dispatched jobs.
     const sq = s => String(s).replace(/'/g, "'\\''");
     let schedulerWatcherOk = false;
@@ -1204,9 +1223,10 @@ async function cmdEnqueue(flags) {
 
     // -- Post-spawn verification (Fix 3) --------------------------------
     // Canary: poll sessions.json up to 3 times at 10s intervals to confirm the
-    // session appeared in the store. Non-fatal -- output is already written above.
-    // If the session never shows up, stderr gets a loud warning and ledger status
-    // is set to 'spawn-warning'. The watcher provides the definitive error path.
+    // session appeared in the store. A session store entry with sessionId or
+    // startedAt/sessionStartedAt is enough: long first turns may not flush JSONL,
+    // token counts, or chat.history until the model call completes. The delivery
+    // watcher owns later completion/failure handling.
     const SPAWN_POLL_MAX = 3;
     const SPAWN_POLL_DELAY_MS = 10_000;
     let spawnConfirmed = false;
@@ -1214,7 +1234,7 @@ async function cmdEnqueue(flags) {
       await sleep(SPAWN_POLL_DELAY_MS);
       const spawnStore = readSessionsStore(agent);
       const signal = inspectSessionActivitySignal(sessionKey, spawnStore);
-      if (signal.hasActivitySignal) {
+      if (signal.hasStartedSignal || signal.hasActivitySignal) {
         spawnConfirmed = true;
         break;
       }
@@ -1292,16 +1312,17 @@ function cmdStatus(flags) {
       //
       // PING_STALE_MS:   3x the 60s ping interval -- if we haven't heard from the
       //                  watcher in 3 min, it's probably dead; fall through to check.
-      // hardCeilingMs:   job timeout * 1.5 -- absolute max regardless of ping age.
-      //                  Catches zombie watchers (watcher alive but session is stuck).
-      // idleThresholdMs: max(job timeout, 10 min) -- replaces the old hardcoded 10-min
-      //                  threshold so longer jobs aren't killed at exactly 10 min.
-      const PING_STALE_MS  = 3 * 60 * 1000;
-      const idleThresholdMs = Math.max((entry.timeoutSeconds || 600) * 1000, 10 * 60 * 1000);
-      // hardCeilingMs must be >= idleThresholdMs to avoid the ceiling undercutting the
-      // idle floor (e.g. timeoutSeconds=300 -> ceiling=7.5 min < idle=10 min would force
-      // zombie-guard threshold for sessions that should still use idleThresholdMs).
-      const hardCeilingMs  = Math.max((entry.timeoutSeconds || 600) * 1000 * 1.5, idleThresholdMs * 1.5);
+      // hardCeilingMs:   timeout/reasoning-aware hard ceiling. High-thinking
+      //                  work gets a larger quiet window before hard failure.
+      // idleThresholdMs: timeout/reasoning-aware quiet threshold. Ambiguous or
+      //                  missing liveness stays running until these thresholds.
+      const livenessPolicy = getDispatchLivenessPolicy(entry, {
+        startupGraceMs: STARTUP_GRACE_MS,
+        defaultTimeoutSeconds: 600,
+      });
+      const PING_STALE_MS = livenessPolicy.pingStaleMs;
+      const idleThresholdMs = livenessPolicy.idleFailureMs;
+      const hardCeilingMs = livenessPolicy.hardCeilingMs;
 
       let check;
       if (ageMs < STARTUP_GRACE_MS) {
@@ -1314,13 +1335,13 @@ function cmdStatus(flags) {
           check = { shouldResolve: false };
         } else {
           // Ping stale OR past hard ceiling: fall through to session store check
-          const thresh = ageMs >= hardCeilingMs ? 2 * 60 * 1000 : idleThresholdMs;
+          const thresh = ageMs >= hardCeilingMs ? livenessPolicy.hardTimeoutIdleMs : idleThresholdMs;
           check = checkSessionDone(entry.sessionKey, sessionsStore, thresh, true, spawnedAtMs);
         }
       } else {
         // No lastPing -- backward compat (sessions dispatched before heartbeat feature).
         // Use idleThresholdMs (job-aware) instead of the old hardcoded 10 min.
-        const thresh = ageMs >= hardCeilingMs ? 2 * 60 * 1000 : idleThresholdMs;
+        const thresh = ageMs >= hardCeilingMs ? livenessPolicy.hardTimeoutIdleMs : idleThresholdMs;
         check = checkSessionDone(entry.sessionKey, sessionsStore, thresh, true, spawnedAtMs);
       }
 
@@ -1597,10 +1618,13 @@ function cmdSync(flags) {
     // -- Heartbeat-based liveness guard (mirrors cmdStatus logic) ---------
     // Skip auto-resolve when the watcher's lastPing heartbeat is fresh.
     // See cmdStatus for full commentary on PING_STALE_MS / hardCeilingMs.
-    const PING_STALE_MS_SYNC  = 3 * 60 * 1000;
-    const idleThresholdMsSync = Math.max((entry.timeoutSeconds || 600) * 1000, 10 * 60 * 1000);
-    // hardCeilingMsSync must be >= idleThresholdMsSync (mirrors cmdStatus fix).
-    const hardCeilingMsSync   = Math.max((entry.timeoutSeconds || 600) * 1000 * 1.5, idleThresholdMsSync * 1.5);
+    const syncPolicy = getDispatchLivenessPolicy(entry, {
+      startupGraceMs: STARTUP_GRACE_MS_SYNC,
+      defaultTimeoutSeconds: 600,
+    });
+    const PING_STALE_MS_SYNC = syncPolicy.pingStaleMs;
+    const idleThresholdMsSync = syncPolicy.idleFailureMs;
+    const hardCeilingMsSync = syncPolicy.hardCeilingMs;
 
     if (entry.lastPing) {
       const pingAgeMs = Date.now() - new Date(entry.lastPing).getTime();
@@ -1610,7 +1634,7 @@ function cmdSync(flags) {
       }
     }
 
-    const syncThresh = elapsedMs >= hardCeilingMsSync ? 2 * 60 * 1000 : idleThresholdMsSync;
+    const syncThresh = elapsedMs >= hardCeilingMsSync ? syncPolicy.hardTimeoutIdleMs : idleThresholdMsSync;
     const check = checkSessionDone(entry.sessionKey, syncStore, syncThresh, true, spawnedAtMs);
 
     if (check.shouldResolve) {
@@ -1972,7 +1996,7 @@ async function cmdDone(flags) {
     // Label was never registered (e.g. direct subagent spawn, not via enqueue).
     // This is not an error -- the work completed, the label just wasn't tracked.
     process.stderr.write(`[${BRAND}] warn: no session found for label "${label}" -- registering as done\n`);
-    setLabel(label, { status: 'done', summary, completion, ...(sha ? { sha } : {}) });
+    setLabelDone(label, { summary, completion, ...(sha ? { sha } : {}) });
 
     // No watcher is polling for this label, so actively notify via the gateway
     // post office using delivery config from config.json as fallback target.
@@ -2001,8 +2025,7 @@ async function cmdDone(flags) {
     return;
   }
 
-  setLabel(label, {
-    status:  'done',
+  setLabelDone(label, {
     summary,
     completion,
     ...(sha ? { sha } : {}),

package/dispatch/liveness.mjs

@@ -0,0 +1,61 @@
+const MINUTE_MS = 60 * 1000;
+
+function numberOrNull(value) {
+  const n = Number(value);
+  return Number.isFinite(n) && n > 0 ? n : null;
+}
+
+export function normalizeThinkingLevel(value) {
+  const text = typeof value === 'string' ? value.trim().toLowerCase() : '';
+  if (text === 'xhigh' || text === 'extra-high' || text === 'extra_high') return 'xhigh';
+  if (text === 'high') return 'high';
+  if (text === 'low') return 'low';
+  if (text === 'off' || text === 'none') return 'off';
+  return null;
+}
+
+export function getDispatchTimeoutSeconds(entry = {}, fallbackSeconds = 300) {
+  return numberOrNull(entry.timeoutSeconds)
+    ?? numberOrNull(entry.timeout)
+    ?? numberOrNull(fallbackSeconds)
+    ?? 300;
+}
+
+export function getDispatchLivenessPolicy(entry = {}, opts = {}) {
+  const now = numberOrNull(opts.now) ?? Date.now();
+  const timeoutSeconds = getDispatchTimeoutSeconds(entry, opts.defaultTimeoutSeconds);
+  const timeoutMs = timeoutSeconds * 1000;
+  const thinking = normalizeThinkingLevel(entry.thinking);
+  const isHighThinking = thinking === 'high' || thinking === 'xhigh';
+
+  const startupGraceMs = numberOrNull(opts.startupGraceMs)
+    ?? (isHighThinking ? 10 * MINUTE_MS : 5 * MINUTE_MS);
+  const pingStaleMs = numberOrNull(opts.pingStaleMs) ?? 3 * MINUTE_MS;
+  const idleProbeFloorMs = isHighThinking ? 10 * MINUTE_MS : 1 * MINUTE_MS;
+  const idleProbeMs = Math.max(
+    idleProbeFloorMs,
+    Math.min(timeoutMs * 0.25, isHighThinking ? 15 * MINUTE_MS : 5 * MINUTE_MS),
+  );
+  const idleFailureFloorMs = isHighThinking ? 20 * MINUTE_MS : 10 * MINUTE_MS;
+  const idleFailureMs = Math.max(timeoutMs, idleFailureFloorMs);
+  const hardCeilingMs = Math.max(timeoutMs * 1.5, idleFailureMs * (isHighThinking ? 2 : 1.5));
+  const hardTimeoutIdleMs = isHighThinking ? 5 * MINUTE_MS : 2 * MINUTE_MS;
+  const spawnedAtMs = entry.spawnedAt ? new Date(entry.spawnedAt).getTime() : 0;
+  const ageMs = spawnedAtMs ? now - spawnedAtMs : Infinity;
+
+  return {
+    thinking,
+    isHighThinking,
+    timeoutSeconds,
+    timeoutMs,
+    startupGraceMs,
+    pingStaleMs,
+    idleProbeMs,
+    idleFailureMs,
+    hardCeilingMs,
+    hardTimeoutIdleMs,
+    spawnedAtMs,
+    ageMs,
+    pastHardCeiling: ageMs >= hardCeilingMs,
+  };
+}