openclaw-scheduler 0.2.0

package/scripts/inbox-consumer.mjs

@@ -0,0 +1,288 @@
+#!/usr/bin/env node
+/**
+ * inbox-consumer.mjs
+ *
+ * Drains pending scheduler messages for an agent and delivers them to a channel target.
+ * Intended for signal-only queue patterns where scripts enqueue actionable messages.
+ *
+ * Usage:
+ *   node scripts/inbox-consumer.mjs --to <target-id> [--channel telegram] [--agent main] [--limit 50]
+ *   node scripts/inbox-consumer.mjs --to <target-id> --watch
+ *
+ * Env fallbacks:
+ *   INBOX_DELIVERY_TO
+ *   INBOX_DELIVERY_CHANNEL (default: telegram)
+ *   INBOX_AGENT (default: main)
+ *   INBOX_LIMIT (default: 50)
+ */
+
+import { dirname, basename, join, resolve } from 'path';
+import { fileURLToPath } from 'url';
+import { watch } from 'fs';
+import { getDb } from '../db.js';
+import { resolveSchedulerDbPath } from '../paths.js';
+import { deliverMessage } from '../gateway.js';
+import { ackMessage, recordMessageAttempt } from '../messages.js';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+function parseArgs(argv) {
+  const out = { watch: false };
+  for (let i = 0; i < argv.length; i += 1) {
+    const k = argv[i];
+    if (k === '--watch') {
+      out.watch = true;
+      continue;
+    }
+    if (!k.startsWith('--')) continue;
+    const key = k.slice(2);
+    const value = argv[i + 1];
+    if (value && !value.startsWith('--')) {
+      out[key] = value;
+      i += 1;
+    } else {
+      out[key] = true;
+    }
+  }
+  return out;
+}
+
+function parsePositiveInt(input, fallback) {
+  const n = Number.parseInt(String(input ?? ''), 10);
+  return Number.isFinite(n) && n > 0 ? n : fallback;
+}
+
+function timeAgo(dateStr) {
+  if (!dateStr) return 'unknown';
+  const normalized = dateStr.includes('T') ? dateStr : dateStr.replace(' ', 'T');
+  const ts = new Date(normalized.endsWith('Z') ? normalized : normalized + 'Z').getTime();
+  if (isNaN(ts)) return 'unknown';
+  const sec = Math.max(0, Math.floor((Date.now() - ts) / 1000));
+  if (sec < 60) return `${sec}s ago`;
+  const min = Math.floor(sec / 60);
+  if (min < 60) return `${min}m ago`;
+  const hr = Math.floor(min / 60);
+  if (hr < 24) return `${hr}h ago`;
+  return `${Math.floor(hr / 24)}d ago`;
+}
+
+/** Sentinel tokens that should never appear in user-facing delivery. */
+const DELIVERY_SENTINELS = ['HEARTBEAT_OK', 'NO_FLUSH', 'IDEMPOTENT_SKIP'];
+
+/**
+ * Strip common shell output noise from delivery content:
+ * - "stdout:\n" prefix added by the shell strategy
+ * - Timestamped INFO log lines like "[2026-03-31 00:21:03] INFO ..."
+ * Keep lines that look like actual results.
+ */
+function cleanShellOutput(text) {
+  let cleaned = text;
+  // Strip leading "stdout:" or "stderr:" prefix
+  cleaned = cleaned.replace(/^stdout:\s*/i, '').replace(/^stderr:\s*/i, '');
+  // Remove timestamped log lines (keep everything else)
+  const lines = cleaned.split('\n');
+  const meaningful = lines.filter(line => {
+    const trimmed = line.trim();
+    if (!trimmed) return false;
+    // Skip lines like "[2026-03-31 00:21:03] INFO === Auto-Settle starting ==="
+    if (/^\[\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}\]\s+(INFO|DEBUG|WARN)\s/.test(trimmed)) return false;
+    return true;
+  });
+  return meaningful.join('\n').trim();
+}
+
+/**
+ * Format a single message for user-facing delivery.
+ * Strips debug metadata, sentinel tokens, shell noise, and adds a branded header.
+ *
+ * Env config:
+ *   INBOX_BRAND: display name for the header (default: "Scheduler")
+ */
+function formatMessageForDelivery(msg, { brand = 'Scheduler' } = {}) {
+  let body = (msg.body || '').trim();
+
+  // Strip sentinel tokens from the end of the body
+  for (const sentinel of DELIVERY_SENTINELS) {
+    if (body.endsWith(sentinel)) {
+      body = body.slice(0, -sentinel.length).trim();
+    }
+  }
+
+  // Clean shell output noise
+  body = cleanShellOutput(body);
+
+  if (!body) return null;
+
+  // Header: brand + subject + age
+  const age = timeAgo(msg.created_at);
+  const subject = msg.subject || 'Notification';
+  const header = `${brand} | ${subject} | ${age}`;
+
+  return `${header}\n\n${body}`.slice(0, 4000);
+}
+
+/**
+ * Legacy debug format for --verbose mode.
+ */
+function _formatMessagesDebug(msgs, agentId) {
+  const lines = [`Inbox for ${agentId}: ${msgs.length} message(s)`];
+  for (const msg of msgs) {
+    lines.push('');
+    lines.push(`[${msg.kind}] from=${msg.from_agent} age=${timeAgo(msg.created_at)} priority=${msg.priority}`);
+    if (msg.subject) lines.push(`subject: ${msg.subject}`);
+    if (msg.body) lines.push(msg.body.trim().slice(0, 1200));
+    lines.push('---');
+  }
+  return lines.join('\n').trim();
+}
+
+function selectPendingMessages(db, agentId, limit) {
+  return db.prepare(`
+    SELECT id, from_agent, to_agent, subject, body, kind, created_at, priority,
+           delivery_to, channel
+    FROM messages
+    WHERE (to_agent = ? OR to_agent = 'broadcast')
+      AND status IN ('pending', 'delivered')
+    ORDER BY
+      CASE kind
+        WHEN 'constraint' THEN 0
+        WHEN 'decision'   THEN 1
+        WHEN 'fact'       THEN 2
+        WHEN 'task'       THEN 3
+        WHEN 'preference' THEN 4
+        ELSE 5
+      END ASC,
+      priority DESC,
+      created_at ASC
+    LIMIT ?
+  `).all(agentId, limit);
+}
+
+async function drainOnce(db, { to, channel, agentId, limit, brand }) {
+  const msgs = selectPendingMessages(db, agentId, limit);
+  if (msgs.length === 0) {
+    return 0;
+  }
+
+  let delivered = 0;
+  const deliveryErrors = [];
+
+  // Deliver each message individually with clean user-facing formatting.
+  // Messages are sent one at a time so a failure on one doesn't block others.
+  for (const msg of msgs) {
+    const msgTarget = msg.delivery_to || to;
+    const msgChannel = msg.channel || channel;
+    const text = formatMessageForDelivery(msg, { brand });
+
+    if (!text) {
+      // Empty after stripping sentinels -- ack without delivering
+      ackMessage(msg.id, 'inbox-consumer', 'Suppressed (empty after sentinel strip)');
+      delivered += 1;
+      continue;
+    }
+
+    try {
+      // Small delay between deliveries to avoid gateway rate/concurrency issues
+      if (delivered > 0) await new Promise(r => setTimeout(r, 1500));
+      await deliverMessage(msgChannel, msgTarget, text);
+      recordMessageAttempt(msg.id, { ok: true, actor: 'inbox-consumer' });
+      ackMessage(msg.id, 'inbox-consumer', `Delivered to ${msgChannel}:${msgTarget}`);
+      delivered += 1;
+    } catch (err) {
+      recordMessageAttempt(msg.id, {
+        ok: false,
+        actor: 'inbox-consumer',
+        error: err.message || 'delivery failed',
+      });
+      deliveryErrors.push(err);
+    }
+  }
+
+  if (delivered > 0) {
+    process.stdout.write(`[inbox-consumer] delivered ${delivered} message(s)\n`);
+  }
+  if (deliveryErrors.length > 0) {
+    throw new Error(`Delivery failed for ${deliveryErrors.length} message(s): ${deliveryErrors.map(e => e.message).join('; ')}`);
+  }
+  return delivered;
+}
+
+const args = parseArgs(process.argv.slice(2));
+const deliveryTo = args.to || process.env.INBOX_DELIVERY_TO || '';
+const channel = args.channel || process.env.INBOX_DELIVERY_CHANNEL || 'telegram';
+const agentId = args.agent || process.env.INBOX_AGENT || 'main';
+const limit = parsePositiveInt(args.limit || process.env.INBOX_LIMIT, 50);
+// Brand resolution: --brand flag > INBOX_BRAND env > dispatch config brand > "Scheduler"
+let brand = args.brand || process.env.INBOX_BRAND || '';
+if (!brand) {
+  try {
+    const configDir = process.env.DISPATCH_CONFIG_DIR || join(resolve(resolveSchedulerDbPath({ env: process.env }), '..'), 'dispatch');
+    const { readFileSync } = await import('node:fs');
+    const config = JSON.parse(readFileSync(join(configDir, 'config.json'), 'utf8'));
+    brand = config.brand || config.name || '';
+  } catch (_e) { /* no dispatch config -- use default */ }
+}
+if (!brand) brand = 'Scheduler';
+const watchMode = Boolean(args.watch);
+
+if (!deliveryTo) {
+  process.stderr.write('[inbox-consumer] missing delivery target; pass --to or set INBOX_DELIVERY_TO\n');
+  process.exit(1);
+}
+
+const dbPath = resolve(resolveSchedulerDbPath({ env: process.env }));
+const watchDir = dirname(dbPath);
+const walFile = `${basename(dbPath)}-wal`;
+
+try {
+  const db = getDb();
+
+  if (!watchMode) {
+    await drainOnce(db, { to: deliveryTo, channel, agentId, limit, brand });
+    process.exit(0);
+  }
+
+  process.stdout.write(`[inbox-consumer] watching ${join(watchDir, walFile)}\n`);
+  try {
+    await drainOnce(db, { to: deliveryTo, channel, agentId, limit, brand });
+  } catch (err) {
+    process.stderr.write(`[inbox-consumer] initial drain error: ${err.message}\n`);
+  }
+
+  let timer = null;
+  let draining = false;
+
+  const runDebouncedDrain = async () => {
+    if (draining) return;
+    draining = true;
+    try {
+      await drainOnce(db, { to: deliveryTo, channel, agentId, limit, brand });
+    } catch (err) {
+      process.stderr.write(`[inbox-consumer] drain error: ${err.message}\n`);
+    } finally {
+      draining = false;
+    }
+  };
+
+  const watcher = watch(watchDir, (_eventType, filename) => {
+    if (filename !== null && filename !== walFile) return;
+    if (timer) clearTimeout(timer);
+    timer = setTimeout(() => {
+      timer = null;
+      runDebouncedDrain();
+    }, 250);
+  });
+
+  const shutdown = (signal) => {
+    if (timer) clearTimeout(timer);
+    watcher.close();
+    process.stdout.write(`[inbox-consumer] ${signal}; exiting\n`);
+    process.exit(0);
+  };
+
+  process.on('SIGINT', () => shutdown('SIGINT'));
+  process.on('SIGTERM', () => shutdown('SIGTERM'));
+} catch (err) {
+  process.stderr.write(`[inbox-consumer] error: ${err.stack || err.message}\n`);
+  process.exit(1);
+}

package/scripts/stuck-detector.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Wrapper for stuck-run-detector.mjs that ensures proper PATH
+
+# Ensure common Node.js install locations are in PATH
+case "$(uname -s)" in
+  Darwin) export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH" ;;
+  *)      export PATH="/usr/local/bin:$PATH" ;;
+esac
+
+NODE_BIN="${NODE_BIN:-$(command -v node 2>/dev/null)}"
+if [ -z "$NODE_BIN" ]; then
+  echo "[stuck-detector] node not found in PATH" >&2
+  exit 1
+fi
+THRESHOLD_MIN="${STUCK_THRESHOLD_MIN:-30}"
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+
+exec "$NODE_BIN" "$SCRIPT_DIR/stuck-run-detector.mjs" --threshold-min "$THRESHOLD_MIN"

package/scripts/stuck-run-detector.mjs

@@ -0,0 +1,333 @@
+#!/usr/bin/env node
+/**
+ * stuck-run-detector.mjs
+ *
+ * Three-phase stuck detection: liveness gating -> steer -> alert.
+ *
+ * Phase 1 (Liveness gate): For dispatch-tracked runs, check actual session
+ *   activity via `dispatch status`. Skip if tokens recently active or status=done.
+ * Phase 2 (Steer): Send a nudge into the session before alerting. Give it
+ *   one more cycle to respond.
+ * Phase 3 (Alert): If steer was ignored (tokens flat), alert as genuinely stuck.
+ *
+ * Non-dispatch scheduler jobs skip phases 1-2 and alert immediately (old behavior).
+ *
+ * State persisted in /tmp/stuck-detector-state.json (volatile, OK to lose on reboot).
+ *
+ * Usage:
+ *   node scripts/stuck-run-detector.mjs [--threshold-min 45] [--limit 20]
+ *
+ * Exit codes:
+ *   0: no stuck runs (includes steered-but-not-yet-stuck)
+ *   1: genuinely stuck runs found or hard error
+ */
+
+import { getDb } from '../db.js';
+import { readFileSync, writeFileSync } from 'fs';
+import { execFileSync } from 'child_process';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { tmpdir } from 'os';
+import { resolveDispatchCliPath, resolveDispatchLabel } from './dispatch-cli-utils.mjs';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+// -- Paths ----------------------------------------------------
+
+const LABELS_PATH  = process.env.DISPATCH_LABELS_PATH || join(__dirname, '..', 'dispatch', 'labels.json');
+const STATE_PATH   = process.env.STUCK_STATE_PATH || join(tmpdir(), 'stuck-detector-state.json');
+const DISPATCH_CLI = resolveDispatchCliPath(process.env);
+const DISPATCH_IS_BIN = !DISPATCH_CLI.includes('/') && !DISPATCH_CLI.includes('\\');
+
+// -- Constants ------------------------------------------------
+
+const LIVENESS_THRESHOLD_MS = 180_000; // 3 min -- tokens active within this = alive
+
+// -- Arg Parsing ----------------------------------------------
+
+function parseArgs(argv) {
+  const out = {};
+  for (let i = 0; i < argv.length; i += 1) {
+    const k = argv[i];
+    if (!k.startsWith('--')) continue;
+    const key = k.slice(2);
+    const value = argv[i + 1];
+    if (value && !value.startsWith('--')) {
+      out[key] = value;
+      i += 1;
+    } else {
+      out[key] = true;
+    }
+  }
+  return out;
+}
+
+function parsePositiveInt(input, fallback) {
+  const n = Number.parseInt(String(input ?? ''), 10);
+  return Number.isFinite(n) && n > 0 ? n : fallback;
+}
+
+// -- Labels & State -------------------------------------------
+
+function loadLabels() {
+  try {
+    return JSON.parse(readFileSync(LABELS_PATH, 'utf-8'));
+  } catch {
+    return {};
+  }
+}
+
+function loadState() {
+  try {
+    return JSON.parse(readFileSync(STATE_PATH, 'utf-8'));
+  } catch {
+    return {};
+  }
+}
+
+function saveState(state) {
+  writeFileSync(STATE_PATH, JSON.stringify(state, null, 2) + '\n');
+}
+
+// -- Dispatch Integration ------------------------------------
+
+/**
+ * Get liveness info from `dispatch status --label <label>`.
+ * Returns { ageMs, tokens, status } or null on failure.
+ */
+function getDispatchLiveness(label) {
+  try {
+    const execArgs = DISPATCH_IS_BIN
+      ? [DISPATCH_CLI, ['status', '--label', label]]
+      : [process.execPath, [DISPATCH_CLI, 'status', '--label', label]];
+    const result = execFileSync(execArgs[0], execArgs[1], {
+      encoding: 'utf-8',
+      timeout: 15_000,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+    const parsed = JSON.parse(result.trim());
+    return {
+      ageMs:     parsed?.liveness?.ageMs ?? null,
+      tokens:    parsed?.liveness?.tokens ?? null,
+      status:    parsed?.status ?? null,
+      updatedAt: parsed?.liveness?.updatedAt ?? null,
+    };
+  } catch (err) {
+    process.stderr.write(
+      `[stuck-detector] dispatch status for "${label}" failed: ${err.message}\n`
+    );
+    return null;
+  }
+}
+
+/**
+ * Send a steering message into a dispatch session.
+ * Returns true on success, false on failure.
+ */
+function steerSession(label, staleMins) {
+  const msg = [
+    `[Auto-steer] You have been silent for ${staleMins} minutes.`,
+    `Please reply with a brief progress update, then continue your task.`,
+    `If you are done, call the done signal as instructed in your prompt.`,
+  ].join(' ');
+  try {
+    const execArgs = DISPATCH_IS_BIN
+      ? [DISPATCH_CLI, ['send', '--label', label, '--message', msg]]
+      : [process.execPath, [DISPATCH_CLI, 'send', '--label', label, '--message', msg]];
+    execFileSync(execArgs[0], execArgs[1], {
+      encoding: 'utf-8',
+      timeout: 15_000,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+    return true;
+  } catch (err) {
+    process.stderr.write(`[stuck-detector] steer for "${label}" failed: ${err.message}\n`);
+    return false;
+  }
+}
+
+// -- Main -----------------------------------------------------
+
+const args         = parseArgs(process.argv.slice(2));
+const thresholdMin = parsePositiveInt(args['threshold-min'], 45);  // coding tasks regularly take 30m+
+const thresholdS   = thresholdMin * 60;
+const limit        = parsePositiveInt(args.limit, 20);
+
+try {
+  const db   = getDb();
+  const rows = db.prepare(`
+    SELECT
+      r.id,
+      r.job_id,
+      r.started_at,
+      r.last_heartbeat,
+      r.run_timeout_ms,
+      j.name AS job_name,
+      CAST((julianday('now') - julianday(COALESCE(r.last_heartbeat, r.started_at))) * 86400 AS INTEGER) AS stale_s
+    FROM runs r
+    JOIN jobs j ON j.id = r.job_id
+    WHERE r.status = 'running'
+      AND COALESCE(r.last_heartbeat, r.started_at) < datetime('now', '-' || ? || ' seconds')
+    ORDER BY stale_s DESC
+    LIMIT ?
+  `).all(thresholdS, limit);
+
+  if (rows.length === 0) {
+    process.stdout.write(`No stale runs older than ${thresholdMin} minute(s).\n`);
+    process.exit(0);
+  }
+
+  const labels       = loadLabels();
+  const state        = loadState();
+  const alertRuns    = [];   // Phase 3: genuinely stuck
+  const steeredRuns  = [];   // Phase 2: nudged, awaiting response
+  const skippedRuns  = [];   // Phase 1: alive or done
+  let   stateChanged = false;
+
+  // Track which labels appeared as stale this cycle (for state cleanup)
+  const staleLabelsThisCycle = new Set();
+
+  for (const r of rows) {
+    const label = resolveDispatchLabel(r.job_name, labels);
+
+    // -- Non-dispatch job: alert immediately (old behavior) --
+    if (!label) {
+      alertRuns.push(r);
+      continue;
+    }
+
+    staleLabelsThisCycle.add(label);
+
+    // -- Phase 1: Liveness gate ------------------------------
+
+    // Quick check: if labels.json already says done, skip (DB lag)
+    if (labels[label]?.status === 'done') {
+      skippedRuns.push({ ...r, reason: 'labels.json status=done (DB lag)', label });
+      continue;
+    }
+
+    const liveness = getDispatchLiveness(label);
+
+    // dispatch status returned done -> skip
+    if (liveness?.status === 'done') {
+      skippedRuns.push({ ...r, reason: 'dispatch status=done', label });
+      continue;
+    }
+
+    // Tokens recently active -> skip (not stuck, just no heartbeat to DB)
+    if (
+      liveness &&
+      typeof liveness.ageMs === 'number' &&
+      liveness.ageMs < LIVENESS_THRESHOLD_MS
+    ) {
+      skippedRuns.push({
+        ...r,
+        reason: `active (ageMs=${liveness.ageMs}ms < ${LIVENESS_THRESHOLD_MS}ms)`,
+        label,
+      });
+      continue;
+    }
+
+    // -- Phase 2 / Phase 3: Steer or Alert -------------------
+
+    if (state[label]?.alertedAt) {
+      // Already alerted -- don't re-steer or re-alert
+      continue;
+    }
+
+    if (!state[label]) {
+      // Phase 2: First detection -- steer before alerting
+      const staleMins = Math.round(r.stale_s / 60);
+      const ok = steerSession(label, staleMins);
+
+      if (ok) {
+        state[label] = {
+          steeredAt:      new Date().toISOString(),
+          tokensAtSteer:  liveness?.tokens ?? 0,
+          staleS:         r.stale_s,
+        };
+        stateChanged = true;
+        steeredRuns.push({
+          ...r,
+          label,
+          ageMs:  liveness?.ageMs ?? null,
+          tokens: liveness?.tokens ?? null,
+        });
+      } else {
+        // Steer call failed -- still alert, note the failure
+        alertRuns.push({ ...r, steerNote: 'steer=failed' });
+      }
+    } else {
+      // Phase 3: Already steered -- did it help?
+      const prevTokens = state[label].tokensAtSteer;
+      const curTokens  = liveness?.tokens ?? 0;
+
+      if (curTokens > prevTokens) {
+        // Tokens grew since steer -> it worked, clear state, skip alert
+        delete state[label];
+        stateChanged = true;
+        skippedRuns.push({
+          ...r,
+          reason: `steer worked (tokens ${prevTokens}->${curTokens})`,
+          label,
+        });
+      } else {
+        // Tokens flat since steer -> genuinely stuck -> alert
+        alertRuns.push({ ...r, steerNote: 'steer=ignored' });
+        state[label] = { alertedAt: new Date().toISOString() };
+        stateChanged = true;
+      }
+    }
+  }
+
+  // -- Clean state entries for labels no longer appearing as stale --
+  for (const key of Object.keys(state)) {
+    if (!staleLabelsThisCycle.has(key)) {
+      delete state[key];
+      stateChanged = true;
+    }
+  }
+
+  if (stateChanged) {
+    saveState(state);
+  }
+
+  // -- Output -------------------------------------------------
+
+  for (const s of skippedRuns) {
+    process.stdout.write(
+      `Skipped: job="${s.job_name}" label="${s.label}" -- ${s.reason}\n`
+    );
+  }
+
+  for (const s of steeredRuns) {
+    process.stdout.write(
+      `Steered: label=${s.label} ageMs=${s.ageMs ?? '?'} tokens=${s.tokens ?? '?'} (awaiting response)\n`
+    );
+  }
+
+  if (alertRuns.length === 0) {
+    if (steeredRuns.length > 0) {
+      process.stdout.write(
+        `No stuck runs -- ${steeredRuns.length} steered, awaiting response.\n`
+      );
+    } else {
+      process.stdout.write(`No stale runs older than ${thresholdMin} minute(s).\n`);
+    }
+    process.exit(0);
+  }
+
+  process.stdout.write(
+    `Detected ${alertRuns.length} stale run(s) older than ${thresholdMin} minute(s):\n`
+  );
+  for (const r of alertRuns) {
+    const suffix = r.steerNote ? ` ${r.steerNote}` : '';
+    process.stdout.write(
+      `- run=${r.id} job="${r.job_name}" job_id=${r.job_id} started=${r.started_at} last_heartbeat=${r.last_heartbeat} stale_s=${r.stale_s} timeout_ms=${r.run_timeout_ms}${suffix}\n`
+    );
+  }
+  process.exit(1);
+} catch (err) {
+  process.stderr.write(`[stuck-run-detector] error: ${err.stack || err.message}\n`);
+  process.exit(1);
+}