create-walle 0.9.25 → 0.9.26

package/template/wall-e/lib/scheduler.js

@@ -1,5 +1,7 @@
 'use strict';
 const EventEmitter = require('events');
+const runtimeHealth = require('./runtime-health');
+const { runtimeDiagnosticsEnabled, perfLogsEnabled } = require('./diagnostics-flags');
 
 let _instance = null;
 
@@ -24,6 +26,9 @@ const MIN_REFIRE_GAP_MS = 2_000;
  */
 const DEFAULT_MISSED_JOB_STAGGER_MS = 5_000;
 const DEFAULT_MAX_MISSED_JOBS_PER_RESTART = 5;
+const DEFAULT_SLOW_TICK_MS = 250;
+const DEFAULT_SLOW_JOB_MS = 2_000;
+const PERF_LOG_THROTTLE_MS = 30_000;
 
 /** Default cascade check: true if any numeric value in result is > 0 */
 function _defaultCascadeCheck(result) {
@@ -39,6 +44,8 @@ class Scheduler extends EventEmitter {
    * @param {number} [opts.tickMs=5000]
    * @param {Object} [opts.pools] - Resource pool limits: { llm: 2, ollama: 1, ... }
    * @param {number} [opts.shutdownTimeoutMs=5000]
+   * @param {Object} [opts.workerPool] - Optional WorkerThreadPool for off-thread job bodies
+   * @param {number} [opts.defaultWorkerJobTimeoutMs=0] - Default timeout for worker-backed jobs
    */
   constructor(opts = {}) {
     super();
@@ -68,6 +75,12 @@ class Scheduler extends EventEmitter {
     this._persistBrain = opts.persistBrain || null;
     this._missedJobStaggerMs = opts.missedJobStaggerMs ?? DEFAULT_MISSED_JOB_STAGGER_MS;
     this._maxMissedJobsPerRestart = opts.maxMissedJobsPerRestart ?? DEFAULT_MAX_MISSED_JOBS_PER_RESTART;
+    this._workerPool = opts.workerPool || null;
+    this._closeWorkerPoolOnShutdown = opts.closeWorkerPoolOnShutdown === true;
+    this._workerJobNames = new Set(Array.isArray(opts.workerJobs) ? opts.workerJobs : []);
+    this._defaultWorkerJobTimeoutMs = Number.isFinite(Number(opts.defaultWorkerJobTimeoutMs))
+      ? Math.max(0, Math.trunc(Number(opts.defaultWorkerJobTimeoutMs)))
+      : 0;
 
     this._poolLimits = {
       llm: 2,
@@ -85,6 +98,25 @@ class Scheduler extends EventEmitter {
     for (const pool of Object.keys(this._poolLimits)) {
       this._poolActive.set(pool, new Set());
     }
+
+    this._metrics = {
+      startedAt: Date.now(),
+      tickCount: 0,
+      tickErrorCount: 0,
+      dispatchCount: 0,
+      completionCount: 0,
+      errorCount: 0,
+      timeoutCount: 0,
+      poolFullCount: 0,
+      maxTickDurationMs: 0,
+      lastTickAt: null,
+      lastTickDurationMs: null,
+      lastTickSummary: null,
+      lastError: null,
+      ineligibleCounts: {},
+      jobs: new Map(),
+      perfLogLastAt: new Map(),
+    };
   }
 
   static getInstance() { return _instance; }
@@ -107,6 +139,9 @@ class Scheduler extends EventEmitter {
    * @param {Function} [desc.shouldRun]  - () => boolean guard
    * @param {Function} [desc.onResult]   - (result) => void
    * @param {Function} [desc.onError]    - (err) => void
+   * @param {boolean}  [desc.worker]     - run body through opts.workerPool
+   * @param {string}   [desc.workerJobName] - worker-side job name (defaults to desc.name)
+   * @param {boolean}  [desc.workerFallbackLocal=true] - fallback to desc.run if worker unavailable
    * @returns {Scheduler}
    */
   register(desc) {
@@ -114,7 +149,15 @@ class Scheduler extends EventEmitter {
       throw new Error('Job requires name, run, and intervalMs');
     }
     // Clone to avoid mutating caller's object
-    desc = { priority: 5, enabled: true, ...desc };
+    desc = {
+      priority: 5,
+      enabled: true,
+      worker: this._workerJobNames.has(desc.name),
+      ...desc,
+    };
+    if (desc.worker && !desc.timeoutMs && this._defaultWorkerJobTimeoutMs > 0) {
+      desc.timeoutMs = this._defaultWorkerJobTimeoutMs;
+    }
 
     const now = Date.now();
     const initialState = {
@@ -333,6 +376,10 @@ class Scheduler extends EventEmitter {
       pool: desc.pool || null,
       priority: desc.priority,
       intervalMs: desc.intervalMs,
+      timeoutMs: desc.timeoutMs || null,
+      workerTimeoutMs: desc.workerTimeoutMs || null,
+      worker: !!desc.worker,
+      workerJobName: desc.workerJobName || null,
     };
   }
 
@@ -351,13 +398,181 @@ class Scheduler extends EventEmitter {
         queued,
       };
     }
+    if (this._workerPool && typeof this._workerPool.getStatus === 'function') {
+      result.worker_threads = this._workerPool.getStatus();
+    }
     return result;
   }
 
+  getActiveSnapshot() {
+    const jobs = [];
+    for (const [name, job] of this._jobs) {
+      if (job.state.status === 'running' || job.state.status === 'queued') {
+        jobs.push({
+          name,
+          status: job.state.status,
+          pool: job.descriptor.pool || null,
+          worker: !!job.descriptor.worker,
+          workerTimeoutMs: job.descriptor.workerTimeoutMs ?? job.descriptor.timeoutMs ?? null,
+          age_ms: job.state.lastRunAt ? Math.max(0, Date.now() - job.state.lastRunAt) : 0,
+        });
+      }
+    }
+    return {
+      jobs,
+      pools: this.getPoolState(),
+    };
+  }
+
   uptime() {
     return this._startedAt ? Date.now() - this._startedAt : 0;
   }
 
+  _metricsEnabled() {
+    return runtimeDiagnosticsEnabled();
+  }
+
+  _safeMetricName(name) {
+    return String(name || 'job')
+      .replace(/[^a-zA-Z0-9_.:-]/g, '_')
+      .slice(0, 80) || 'job';
+  }
+
+  _maybeLogSlow(kind, name, durationMs, thresholdMs, meta = {}) {
+    if (!perfLogsEnabled() || durationMs < thresholdMs) return;
+    const key = `${kind}:${name}`;
+    const now = Date.now();
+    const last = this._metrics.perfLogLastAt.get(key) || 0;
+    if (now - last < PERF_LOG_THROTTLE_MS) return;
+    this._metrics.perfLogLastAt.set(key, now);
+    const safeMeta = {};
+    for (const [k, v] of Object.entries(meta || {})) {
+      if (typeof v === 'number' || typeof v === 'boolean') safeMeta[k] = v;
+      else if (typeof v === 'string') safeMeta[k] = v.slice(0, 80);
+    }
+    console.warn(`[scheduler-perf] ${kind} ${name} took ${durationMs}ms ${JSON.stringify(safeMeta)}`);
+  }
+
+  _recordTickMetric(summary, durationMs, ok = true, err = null) {
+    if (!this._metricsEnabled()) return;
+    const meta = {
+      jobs: summary?.jobs || 0,
+      candidates: summary?.candidates || 0,
+      dispatched: summary?.dispatched || 0,
+      queued: summary?.queued || 0,
+      poolFull: summary?.poolFull || 0,
+      watchdogReset: summary?.watchdogReset || 0,
+      resetQueued: summary?.resetQueued || 0,
+    };
+    this._metrics.tickCount++;
+    if (!ok) this._metrics.tickErrorCount++;
+    this._metrics.lastTickAt = new Date().toISOString();
+    this._metrics.lastTickDurationMs = durationMs;
+    this._metrics.maxTickDurationMs = Math.max(this._metrics.maxTickDurationMs || 0, durationMs);
+    this._metrics.lastTickSummary = summary || null;
+    this._metrics.lastError = err ? (err.message || String(err)).slice(0, 160) : null;
+    if (summary?.ineligible) {
+      for (const [reason, count] of Object.entries(summary.ineligible)) {
+        this._metrics.ineligibleCounts[reason] = (this._metrics.ineligibleCounts[reason] || 0) + count;
+      }
+    }
+    this._metrics.poolFullCount += summary?.poolFull || 0;
+    runtimeHealth.recordOperation('scheduler.tick', durationMs, {
+      ok,
+      error: err ? (err.code || err.name || 'error') : undefined,
+      meta,
+    });
+    this._maybeLogSlow('tick', 'scheduler.tick', durationMs, DEFAULT_SLOW_TICK_MS, meta);
+  }
+
+  _recordDispatchMetric() {
+    if (!this._metricsEnabled()) return;
+    this._metrics.dispatchCount++;
+  }
+
+  _recordJobMetric(name, durationMs, ok = true, err = null, meta = {}) {
+    if (!this._metricsEnabled()) return;
+    const safeName = this._safeMetricName(name);
+    let bucket = this._metrics.jobs.get(safeName);
+    if (!bucket) {
+      bucket = {
+        name: safeName,
+        count: 0,
+        failed: 0,
+        totalMs: 0,
+        maxMs: 0,
+        lastDurationMs: 0,
+        lastAt: null,
+        lastError: null,
+        lastMeta: {},
+      };
+      this._metrics.jobs.set(safeName, bucket);
+    }
+    bucket.count++;
+    bucket.totalMs += durationMs;
+    bucket.maxMs = Math.max(bucket.maxMs, durationMs);
+    bucket.lastDurationMs = durationMs;
+    bucket.lastAt = new Date().toISOString();
+    bucket.lastMeta = { ...meta };
+    if (!ok) {
+      bucket.failed++;
+      bucket.lastError = err ? (err.message || err.code || err.name || String(err)).slice(0, 160) : 'error';
+      this._metrics.errorCount++;
+    } else {
+      this._metrics.completionCount++;
+      bucket.lastError = null;
+    }
+    this._maybeLogSlow('job', safeName, durationMs, DEFAULT_SLOW_JOB_MS, meta);
+  }
+
+  getMetrics(options = {}) {
+    if (!this._metricsEnabled()) {
+      return {
+        enabled: false,
+        uptimeMs: this.uptime(),
+        jobsRegistered: this._jobs.size,
+      };
+    }
+    const jobLimit = Math.max(1, Math.min(100, Number(options.jobLimit) || 20));
+    const jobs = Array.from(this._metrics.jobs.values())
+      .map((job) => ({
+        name: job.name,
+        count: job.count,
+        failed: job.failed,
+        avgMs: job.count ? Math.round(job.totalMs / job.count) : 0,
+        maxMs: job.maxMs,
+        lastDurationMs: job.lastDurationMs,
+        lastAt: job.lastAt,
+        lastError: job.lastError,
+        lastMeta: job.lastMeta || {},
+      }))
+      .sort((a, b) => b.maxMs - a.maxMs || b.count - a.count)
+      .slice(0, jobLimit);
+    return {
+      enabled: true,
+      uptimeMs: this.uptime(),
+      startedAt: this._startedAt ? new Date(this._startedAt).toISOString() : null,
+      tickCount: this._metrics.tickCount,
+      tickErrorCount: this._metrics.tickErrorCount,
+      dispatchCount: this._metrics.dispatchCount,
+      completionCount: this._metrics.completionCount,
+      errorCount: this._metrics.errorCount,
+      timeoutCount: this._metrics.timeoutCount,
+      poolFullCount: this._metrics.poolFullCount,
+      maxTickDurationMs: this._metrics.maxTickDurationMs,
+      lastTickAt: this._metrics.lastTickAt,
+      lastTickDurationMs: this._metrics.lastTickDurationMs,
+      lastTickSummary: this._metrics.lastTickSummary,
+      lastError: this._metrics.lastError,
+      ineligibleCounts: { ...this._metrics.ineligibleCounts },
+      activeJobs: this.getActiveSnapshot(),
+      jobs,
+      workerPool: this._workerPool && typeof this._workerPool.getStatus === 'function'
+        ? this._workerPool.getStatus()
+        : null,
+    };
+  }
+
   async shutdown() {
     if (this._shuttingDown) return;
     this._shuttingDown = true;
@@ -397,27 +612,50 @@ class Scheduler extends EventEmitter {
     }
 
     this.emit('shutdown');
+    if (this._workerPool && this._closeWorkerPoolOnShutdown && typeof this._workerPool.close === 'function') {
+      try { await this._workerPool.close({ timeoutMs: this._shutdownTimeoutMs }); } catch {}
+    }
   }
 
   _tick() {
     if (this._shuttingDown) return;
 
+    const startedAt = Date.now();
+    let summary = null;
     try {
-      this._tickInner();
+      summary = this._tickInner();
+      this._recordTickMetric(summary, Date.now() - startedAt, true);
+      return summary;
     } catch (err) {
+      this._recordTickMetric(summary, Date.now() - startedAt, false, err);
       // Log and recover — don't let a single bad tick kill the scheduler loop.
       // setInterval keeps firing, so the next tick will retry.
       console.error('[scheduler] Tick error (recovering):', err);
       this.emit('tickError', err);
+      return null;
     }
   }
 
   _tickInner() {
     const now = Date.now();
+    const summary = {
+      jobs: this._jobs.size,
+      resetQueued: 0,
+      watchdogReset: 0,
+      candidates: 0,
+      dispatched: 0,
+      queued: 0,
+      poolFull: 0,
+      ineligible: {},
+      poolFullByPool: {},
+    };
 
     // 0. Reset queued→idle so they're reconsidered this tick
     for (const [, job] of this._jobs) {
-      if (job.state.status === 'queued') job.state.status = 'idle';
+      if (job.state.status === 'queued') {
+        job.state.status = 'idle';
+        summary.resetQueued++;
+      }
     }
 
     // 1. Watchdog: check for stuck jobs
@@ -426,6 +664,7 @@ class Scheduler extends EventEmitter {
         if (now - job.state.lastRunAt > job.descriptor.timeoutMs) {
           console.error(`[scheduler] Job "${name}" stuck for ${now - job.state.lastRunAt}ms, force-resetting`);
           this._forceReset(name);
+          summary.watchdogReset++;
         }
       }
     }
@@ -433,22 +672,44 @@ class Scheduler extends EventEmitter {
     // 2. Collect due candidates (idle, past nextEligibleAt, dependencies met, guard passes)
     const candidates = [];
     for (const [, job] of this._jobs) {
-      if (this._ineligibleReason(job, now)) continue;
+      const reason = this._ineligibleReason(job, now);
+      if (reason) {
+        summary.ineligible[reason] = (summary.ineligible[reason] || 0) + 1;
+        continue;
+      }
       candidates.push(job);
     }
+    summary.candidates = candidates.length;
 
     // 3. Sort by priority (lower = higher priority)
     candidates.sort((a, b) => a.descriptor.priority - b.descriptor.priority);
 
-    // 4. Dispatch respecting pool limits
+    // 4. Dispatch respecting pool limits. Worker-thread requests register
+    // asynchronously inside WorkerThreadSlot.request(), so reserve the slots we
+    // consume during this tick to avoid oversubscribing the pool before the
+    // parent-side pending counters catch up.
+    let workerSlotsRemaining = this._workerPoolCapacityAvailable();
     for (const job of candidates) {
       const pool = job.descriptor.pool;
       if (pool && !this._poolHasCapacity(pool)) {
         job.state.status = 'queued';
+        summary.queued++;
+        summary.poolFull++;
+        summary.poolFullByPool[pool] = (summary.poolFullByPool[pool] || 0) + 1;
+        continue;
+      }
+      if (job.descriptor.worker && this._workerPool && workerSlotsRemaining <= 0) {
+        job.state.status = 'queued';
+        summary.queued++;
+        summary.poolFull++;
+        summary.poolFullByPool.worker_threads = (summary.poolFullByPool.worker_threads || 0) + 1;
         continue;
       }
+      if (job.descriptor.worker && this._workerPool) workerSlotsRemaining--;
       this._dispatch(job);
+      summary.dispatched++;
     }
+    return summary;
   }
 
   _poolHasCapacity(pool) {
@@ -458,6 +719,37 @@ class Scheduler extends EventEmitter {
     return active.size < limit;
   }
 
+  _workerPoolHasCapacity() {
+    return this._workerPoolCapacityAvailable() > 0;
+  }
+
+  _workerPoolCapacityAvailable() {
+    if (!this._workerPool || typeof this._workerPool.getStatus !== 'function') return Number.POSITIVE_INFINITY;
+    try {
+      const status = this._workerPool.getStatus();
+      const size = Number(status?.size) || (Array.isArray(status?.workers) ? status.workers.length : 0);
+      if (size <= 0) return Number.POSITIVE_INFINITY;
+      const queuedOrRunning = Number.isFinite(Number(status?.queuedOrRunning))
+        ? Number(status.queuedOrRunning)
+        : (Array.isArray(status?.workers)
+            ? status.workers.reduce((sum, worker) => {
+                return sum
+                  + (Number(worker.pendingClientRequests) || 0)
+                  + (worker.active ? 1 : 0)
+                  + (Number(worker.pending) || 0);
+              }, 0)
+            : 0);
+      return Math.max(0, size - queuedOrRunning);
+    } catch {
+      return Number.POSITIVE_INFINITY;
+    }
+  }
+
+  _markWorkerPoolQueued(job, skipped) {
+    job.state.status = 'queued';
+    skipped.push({ job, reason: 'worker-pool-full' });
+  }
+
   _dependenciesMet(descriptor) {
     if (!descriptor.dependsOn || descriptor.dependsOn.length === 0) return true;
     return descriptor.dependsOn.every(dep => {
@@ -515,6 +807,32 @@ class Scheduler extends EventEmitter {
     job.state.status = 'running';
     job.state.lastRunAt = Date.now();
     const epoch = job._epoch; // capture epoch to detect force-reset
+    const startedAt = Date.now();
+    const trigger = job._wakeTrigger || 'interval';
+    const safeName = this._safeMetricName(name);
+    const workerTimeoutMs = job.descriptor.worker
+      ? (job.descriptor.workerTimeoutMs ?? job.descriptor.timeoutMs ?? this._defaultWorkerJobTimeoutMs ?? 0)
+      : 0;
+    const op = runtimeHealth.beginOperation(`scheduler.job.${safeName}`, {
+      pool: pool || 'none',
+      trigger,
+      worker_requested: !!job.descriptor.worker,
+      ...(workerTimeoutMs > 0 ? { worker_timeout_ms: workerTimeoutMs } : {}),
+    });
+    let failed = false;
+    let failure = null;
+    let workerMeta = {
+      worker_requested: !!job.descriptor.worker,
+      worker_used: false,
+      worker_fallback: false,
+    };
+    this._recordDispatchMetric();
+    job._activeRun = {
+      op,
+      startedAt,
+      trigger,
+      workerMeta,
+    };
 
     if (pool) this._poolActive.get(pool)?.add(name);
 
@@ -531,10 +849,7 @@ class Scheduler extends EventEmitter {
             if (overrides && overrides[name]) modelOverride = overrides[name];
           }
         } catch {}
-        const result = await run({
-          trigger: job._wakeTrigger || 'interval',
-          modelOverride: modelOverride || null,
-        });
+        const result = await this._runJob(job, { trigger, modelOverride: modelOverride || null }, workerMeta);
         // If force-reset happened while we were running, this promise is stale — bail
         if (job._epoch !== epoch) return;
         job._wakeTrigger = null;
@@ -570,6 +885,8 @@ class Scheduler extends EventEmitter {
         this._checkCascades(name, result);
       } catch (err) {
         if (job._epoch !== epoch) return; // stale after force-reset
+        failed = true;
+        failure = err;
         job.state.lastError = err.message;
         job.state.errorCount++;
 
@@ -580,11 +897,23 @@ class Scheduler extends EventEmitter {
         if (onError) onError(err);
         this.emit('jobError', name, err);
       } finally {
-        if (job._epoch !== epoch) return; // stale — _forceReset already cleaned up
+        const stale = job._epoch !== epoch;
+        const finishedAt = Date.now();
+        const durationMs = finishedAt - startedAt;
+        const metricMeta = { pool: pool || 'none', trigger, stale, ...workerMeta };
+        const phaseMeta = this._workerPhaseMetric(job.state.lastResult);
+        Object.assign(metricMeta, phaseMeta);
+        const metricError = failed ? failure : (stale ? new Error('stale_epoch') : null);
+        if (job._activeRun?.op === op) {
+          op.end({ ok: !failed && !stale, error: metricError, meta: metricMeta });
+          job._activeRun = null;
+          this._recordJobMetric(name, durationMs, !failed && !stale, metricError, metricMeta);
+        }
+        if (stale) return; // stale — _forceReset already cleaned up
         // Job may have been unregistered by self-stop
         if (this._jobs.has(name)) {
           job.state.status = 'idle';
-          job.state.lastFinishedAt = Date.now();
+          job.state.lastFinishedAt = finishedAt;
           job.state.lastDurationMs = job.state.lastFinishedAt - job.state.lastRunAt;
           // Floor next-eligible at lastFinishedAt + minRefireGapMs so a
           // misconfigured interval (0/NaN/negative) cannot spin-loop.
@@ -601,6 +930,58 @@ class Scheduler extends EventEmitter {
     })();
   }
 
+  async _runJob(job, context, workerMeta = {}) {
+    const desc = job.descriptor;
+    if (!desc.worker || !this._workerPool) {
+      return await desc.run(context);
+    }
+    workerMeta.worker_requested = true;
+    const payload = {
+      jobName: desc.workerJobName || desc.name,
+      trigger: context.trigger,
+      modelOverride: context.modelOverride || null,
+    };
+    // Scheduler watchdogs own job timeouts. The worker request uses the same
+    // boundary so a timed-out job retires its occupied worker slot instead of
+    // leaving future jobs queued behind an orphaned operation.
+    const timeoutMs = desc.workerTimeoutMs ?? desc.timeoutMs ?? this._defaultWorkerJobTimeoutMs ?? 0;
+    workerMeta.worker_timeout_ms = timeoutMs || 0;
+    try {
+      const result = await this._workerPool.request('scheduler.runJob', payload, {
+        timeoutMs,
+        heavy: true,
+        terminateOnTimeout: timeoutMs > 0,
+      });
+      workerMeta.worker_used = true;
+      return result;
+    } catch (err) {
+      if (err?.code === 'WALL_E_WORKER_REQUEST_TIMEOUT') {
+        workerMeta.worker_timeout = true;
+        throw err;
+      }
+      const fallbackLocal = desc.workerFallbackLocal !== false;
+      if (!fallbackLocal) throw err;
+      workerMeta.worker_fallback = true;
+      workerMeta.worker_error = err.code || err.message || 'worker_error';
+      console.warn(`[scheduler] Worker job "${desc.name}" failed; falling back to local run: ${err.message}`);
+      return await desc.run(context);
+    }
+  }
+
+  _workerPhaseMetric(result) {
+    const phases = Array.isArray(result?._worker_phases) ? result._worker_phases : [];
+    if (phases.length === 0) return {};
+    let slowest = phases[0];
+    for (const phase of phases) {
+      if ((phase.duration_ms || 0) > (slowest.duration_ms || 0)) slowest = phase;
+    }
+    return {
+      worker_phase_count: phases.length,
+      worker_slowest_phase: String(slowest.name || 'unknown').slice(0, 80),
+      worker_slowest_phase_ms: Number(slowest.duration_ms) || 0,
+    };
+  }
+
   _checkCascades(completedJobName, result) {
     if (!result || typeof result !== 'object') return;
 
@@ -617,6 +998,22 @@ class Scheduler extends EventEmitter {
   _forceReset(name) {
     const job = this._jobs.get(name);
     if (!job) return;
+    if (this._metricsEnabled()) this._metrics.timeoutCount++;
+    const timeoutErr = new Error('watchdog timeout');
+    timeoutErr.code = 'WALL_E_SCHEDULER_WATCHDOG_TIMEOUT';
+    const run = job._activeRun || {};
+    const durationMs = run.startedAt ? Date.now() - run.startedAt : 0;
+    const metricMeta = {
+      pool: job.descriptor.pool || 'none',
+      trigger: run.trigger || job._wakeTrigger || 'unknown',
+      watchdog_reset: true,
+      ...(run.workerMeta || {}),
+    };
+    if (run.op && typeof run.op.end === 'function') {
+      run.op.end({ ok: false, error: timeoutErr, meta: metricMeta });
+    }
+    this._recordJobMetric(name, durationMs, false, timeoutErr, metricMeta);
+    job._activeRun = null;
     job._epoch++; // invalidate the orphaned promise so its finally block no-ops
     job.state.status = 'idle';
     job.state.errorCount++;
@@ -633,7 +1030,7 @@ class Scheduler extends EventEmitter {
     if (job.descriptor.pool) {
       this._poolActive.get(job.descriptor.pool)?.delete(name);
     }
-    this._recordJobError(name, new Error('watchdog timeout'), job.state);
+    this._recordJobError(name, timeoutErr, job.state);
     this._persistState(name, job);
     this.emit('jobTimeout', name);
   }
@@ -654,11 +1051,17 @@ class Scheduler extends EventEmitter {
    * Returns a summary so the caller can log + telemeter.
    */
   async runMissedJobs(opts = {}) {
-    if (!this._persistJobState || !this._persistBrain) {
-      return { ran: 0, deferred: 0, reason: 'persistence-disabled' };
-    }
     const max = opts.max ?? this._maxMissedJobsPerRestart;
     const staggerMs = opts.staggerMs ?? this._missedJobStaggerMs;
+    const op = runtimeHealth.beginOperation('scheduler.runMissedJobs', {
+      max,
+      staggerMs,
+    });
+    if (!this._persistJobState || !this._persistBrain) {
+      const summary = { ran: 0, deferred: 0, reason: 'persistence-disabled' };
+      op.end({ ok: true, meta: summary });
+      return summary;
+    }
     const skipNames = new Set(opts.skipNames || []);
 
     let states;
@@ -666,7 +1069,9 @@ class Scheduler extends EventEmitter {
       states = this._persistBrain.listSchedulerJobStates();
     } catch (e) {
       console.warn(`[scheduler] runMissedJobs read failed: ${e.message}`);
-      return { ran: 0, deferred: 0, reason: 'read-error' };
+      const summary = { ran: 0, deferred: 0, reason: 'read-error' };
+      op.end({ ok: false, error: e, meta: summary });
+      return summary;
     }
 
     const now = Date.now();
@@ -689,6 +1094,10 @@ class Scheduler extends EventEmitter {
         skipped.push({ job, reason: 'pool-full' });
         continue;
       }
+      if (job.descriptor.worker && this._workerPool && !this._workerPoolHasCapacity()) {
+        this._markWorkerPoolQueued(job, skipped);
+        continue;
+      }
       candidates.push({ stored, job });
     }
     // Sort by oldest-due first (most lag = highest priority)
@@ -719,6 +1128,8 @@ class Scheduler extends EventEmitter {
     }
 
     let ran = 0;
+    let workMs = 0;
+    let staggerWaitMs = 0;
     for (let i = 0; i < immediate.length; i++) {
       const { job } = immediate[i];
       // Mark eligible RIGHT NOW so _dispatch picks it up
@@ -728,18 +1139,22 @@ class Scheduler extends EventEmitter {
       // hooks all apply uniformly. Wait for the run to finish before
       // staggering the next one.
       try {
+        const workStartedAt = Date.now();
         this._dispatch(job);
         if (job.runPromise) await job.runPromise;
+        workMs += Date.now() - workStartedAt;
         ran++;
       } catch (e) {
         console.warn(`[scheduler] Missed-job replay for "${job.descriptor.name}" threw: ${e.message}`);
       }
       if (i < immediate.length - 1 && staggerMs > 0) {
+        const waitStartedAt = Date.now();
         await new Promise((r) => setTimeout(r, staggerMs));
+        staggerWaitMs += Date.now() - waitStartedAt;
       }
     }
 
-    return {
+    const summary = {
       ran,
       deferred: deferred.length + skipped.length,
       jobs_ran: immediate.map((c) => c.stored.job_name),
@@ -748,7 +1163,21 @@ class Scheduler extends EventEmitter {
         ...skipped.map((c) => c.job.descriptor.name),
       ],
       jobs_skipped: skipped.map((c) => ({ name: c.job.descriptor.name, reason: c.reason })),
+      work_ms: workMs,
+      stagger_wait_ms: staggerWaitMs,
     };
+    runtimeHealth.recordOperation('scheduler.runMissedJobs.work', workMs, {
+      ok: true,
+      meta: { ran, deferred: summary.deferred },
+    });
+    if (staggerWaitMs > 0) {
+      runtimeHealth.recordOperation('scheduler.runMissedJobs.staggerWait', staggerWaitMs, {
+        ok: true,
+        meta: { ran },
+      });
+    }
+    op.end({ ok: true, meta: { ran: summary.ran, deferred: summary.deferred, work_ms: workMs, stagger_wait_ms: staggerWaitMs } });
+    return summary;
   }
 }