@goodandready/dsh-cron 0.3.1 → 0.3.3

package/README.md

@@ -565,6 +565,7 @@ All endpoints are served by the DSH web server under `/dsh-cron/`. Endpoints are
 | `POST` | `/dsh-cron/tasks` | Create or update a task (`id` present → update). Requires `title`, `schedule`, `prompt` |
 | `GET` | `/dsh-cron/tasks/:id/history` | Run history, `?limit=20` |
 | `POST` | `/dsh-cron/tasks/:id/run` | Trigger an immediate manual run |
+| `GET` | `/dsh-cron/tasks/:id/stream` | Real-time Server-Sent Events (SSE) execution log streaming with ring buffer (#336) |
 | `POST` | `/dsh-cron/tasks/:id/pause` | Pause the schedule |
 | `POST` | `/dsh-cron/tasks/:id/resume` | Resume the schedule |
 | `POST` | `/dsh-cron/tasks/:id/toggle` | Toggle active/paused |
@@ -590,6 +591,28 @@ All endpoints are served by the DSH web server under `/dsh-cron/`. Endpoints are
 
 ---
 
+### ⚡ Adaptive Polling & Exponential Backoff (#329)
+
+Save LLM tokens and API quotas on watcher tasks by dynamically slowing down execution when no changes occur:
+
+```json
+"adaptive": {
+  "enabled": true,
+  "minInterval": "5m",
+  "maxInterval": "2h",
+  "backoffFactor": 1.5,
+  "idleRunsThreshold": 3
+}
+```
+- Multiplies interval by `backoffFactor` up to `maxInterval` after `idleRunsThreshold` silent or unchanged runs.
+- Instantly resets to `minInterval` as soon as real changes or non-silent output are detected.
+
+### 📺 Real-Time Streaming Logs via SSE (#336)
+
+Connect to `GET /dsh-cron/tasks/:id/stream` using Server-Sent Events (SSE) to receive live stdout/stderr chunks and execution completion events with an automatic 500-line ring buffer.
+
+---
+
 ## 🧪 Testing & Preflight Verification
 
 Run the automated test suite covering schedule parsing, the scheduler engine, atomic storage, HTTP helpers, notifications and tool contracts:

package/lib/adaptive.js

@@ -0,0 +1,151 @@
+/**
+ * Adaptive Polling & Exponential Backoff (#329)
+ *
+ * Automatically scales execution intervals up when no changes / idle runs are detected
+ * (e.g. silent skips, unchanged outputs) and instantly resets back to minInterval
+ * when new data is returned.
+ */
+
+import { logger } from './logger.js';
+
+export const DEFAULT_ADAPTIVE_CONFIG = {
+  enabled: false,
+  minInterval: '5m',
+  maxInterval: '2h',
+  backoffFactor: 1.5,
+  idleRunsThreshold: 3,
+  detectUnchangedOutput: true,
+};
+
+/**
+ * Parse human interval string (e.g. "30s", "5m", "2h", "1d") or number into milliseconds.
+ */
+export function parseIntervalMs(val, defaultMs = 300000) {
+  if (typeof val === 'number' && !isNaN(val) && val > 0) {
+    return Math.round(val);
+  }
+  if (!val || typeof val !== 'string') return defaultMs;
+
+  const match = val.trim().match(/^(\d+(?:\.\d+)?)\s*(ms|s|m|h|d)?$/i);
+  if (!match) return defaultMs;
+
+  const num = parseFloat(match[1]);
+  const unit = (match[2] || 'm').toLowerCase();
+
+  switch (unit) {
+    case 'ms': return Math.round(num);
+    case 's': return Math.round(num * 1000);
+    case 'm': return Math.round(num * 60 * 1000);
+    case 'h': return Math.round(num * 3600 * 1000);
+    case 'd': return Math.round(num * 86400 * 1000);
+    default: return defaultMs;
+  }
+}
+
+/**
+ * Format milliseconds into human readable interval (e.g. "5m", "1h 30m", "45s").
+ */
+export function formatInterval(ms) {
+  if (!ms || typeof ms !== 'number' || ms <= 0) return '0s';
+  const sec = Math.round(ms / 1000);
+  if (sec < 60) return `${sec}s`;
+  const min = Math.floor(sec / 60);
+  const remSec = sec % 60;
+  if (min < 60) return remSec > 0 ? `${min}m ${remSec}s` : `${min}m`;
+  const hrs = Math.floor(min / 60);
+  const remMin = min % 60;
+  return remMin > 0 ? `${hrs}h ${remMin}m` : `${hrs}h`;
+}
+
+/**
+ * Sanitize and normalize adaptive configuration from payload.
+ */
+export function sanitizeAdaptiveConfig(adaptive) {
+  if (!adaptive || typeof adaptive !== 'object') {
+    return null;
+  }
+  const enabled = Boolean(adaptive.enabled);
+  const minInterval = typeof adaptive.minInterval === 'string' ? adaptive.minInterval.trim() : (adaptive.minInterval ? String(adaptive.minInterval) : '5m');
+  const maxInterval = typeof adaptive.maxInterval === 'string' ? adaptive.maxInterval.trim() : (adaptive.maxInterval ? String(adaptive.maxInterval) : '2h');
+  const backoffFactor = (typeof adaptive.backoffFactor === 'number' && adaptive.backoffFactor >= 1) ? adaptive.backoffFactor : 1.5;
+  const idleRunsThreshold = (typeof adaptive.idleRunsThreshold === 'number' && adaptive.idleRunsThreshold >= 1) ? Math.floor(adaptive.idleRunsThreshold) : 3;
+  const detectUnchangedOutput = adaptive.detectUnchangedOutput !== false;
+
+  return {
+    enabled,
+    minInterval,
+    maxInterval,
+    backoffFactor,
+    idleRunsThreshold,
+    detectUnchangedOutput,
+  };
+}
+
+/**
+ * Evaluate if a completed run constitutes an "idle / no-change" execution.
+ */
+export function evaluateAdaptiveRun(task, outcome, runInfo, lastRun) {
+  if (outcome?.status === 'silentSkip' || runInfo?.silentSkip === true) {
+    return { isIdle: true, reason: 'silent_rule' };
+  }
+  if (runInfo?.validation && runInfo.validation.noChange === true) {
+    return { isIdle: true, reason: 'validator_no_change' };
+  }
+  if (task.adaptive?.detectUnchangedOutput !== false && lastRun && outcome?.status === 'success') {
+    const currentOut = String(outcome.output || '').trim();
+    const lastOut = String(lastRun.output || '').trim();
+    if (currentOut && lastOut && currentOut === lastOut) {
+      return { isIdle: true, reason: 'unchanged_output' };
+    }
+  }
+  return { isIdle: false, reason: 'active_output' };
+}
+
+/**
+ * Calculate the next adaptive state and schedule interval.
+ */
+export function calculateNextAdaptiveState(task, isIdle, now = Date.now()) {
+  const config = sanitizeAdaptiveConfig(task.adaptive) || DEFAULT_ADAPTIVE_CONFIG;
+  const minMs = parseIntervalMs(config.minInterval, 300000);
+  const maxMs = parseIntervalMs(config.maxInterval, 7200000);
+  const factor = config.backoffFactor || 1.5;
+  const threshold = config.idleRunsThreshold || 3;
+
+  const prevState = task.adaptiveState || {
+    idleCount: 0,
+    currentIntervalMs: minMs,
+    effectiveIntervalStr: formatInterval(minMs),
+    active: false,
+    lastAdjustedAt: now,
+  };
+
+  if (!isIdle) {
+    // Reset back to base minInterval immediately upon real changes
+    return {
+      idleCount: 0,
+      currentIntervalMs: minMs,
+      effectiveIntervalStr: formatInterval(minMs),
+      active: false,
+      lastAdjustedAt: now,
+      backedOff: false,
+    };
+  }
+
+  const idleCount = (prevState.idleCount || 0) + 1;
+  let currentIntervalMs = prevState.currentIntervalMs || minMs;
+  let backedOff = false;
+
+  if (idleCount >= threshold) {
+    backedOff = true;
+    currentIntervalMs = Math.min(Math.round(currentIntervalMs * factor), maxMs);
+  }
+
+  return {
+    idleCount,
+    currentIntervalMs,
+    effectiveIntervalStr: formatInterval(currentIntervalMs),
+    active: backedOff,
+    lastAdjustedAt: now,
+    backedOff,
+  };
+}

package/lib/api.js

@@ -1,3 +1,5 @@
+import { sanitizeAdaptiveConfig } from './adaptive.js';
+import { logStreamer } from './log-streamer.js';
 import { approvalManager } from './approval.js';
 import { sanitizeValidators } from './validators.js';
 /**
@@ -289,6 +291,10 @@ async function handleTaskItem({ store, scheduler, req, res, url, id, action, api
   }
 
   if (req.method === 'GET') {
+    if (action === 'stream') {
+      logStreamer.handleSseRequest(id, req, res);
+      return;
+    }
     if (rejectCrossOrigin(req, res, { apiToken })) return;
     if (action === 'archive') {
       const limit = parseInt(url.searchParams.get('limit') || '50', 10);
@@ -596,6 +602,8 @@ function mergeExecutionFields(current, body, parsed) {
     concurrencyGroup: body.concurrencyGroup !== undefined ? String(body.concurrencyGroup).trim() : (current ? (current.concurrencyGroup || 'default') : 'default'),
     selfHealingCommand: body.selfHealingCommand !== undefined ? String(body.selfHealingCommand).trim() : (current ? (current.selfHealingCommand || '') : ''),
     autoDiagnose: body.autoDiagnose !== undefined ? Boolean(body.autoDiagnose) : (current ? Boolean(current.autoDiagnose) : false),
+    adaptive: body.adaptive !== undefined ? sanitizeAdaptiveConfig(body.adaptive) : (current ? current.adaptive : undefined),
+    adaptiveState: current ? current.adaptiveState : undefined,
   };
 }
 

package/lib/chain-checkpoints.js

@@ -1,11 +1,14 @@
 /**
- * Chain Checkpoint and Step Replay Manager (#319)
+ * Chain Checkpoint, Step Replay, and DAG Barrier Manager (#319, #328)
  */
 
+import { sanitizeDependsOn, sanitizeJoinPolicy } from './dag.js';
+
 export class ChainCheckpointManager {
   constructor(store) {
     this.store = store;
     this.activeRetries = new Set();
+    this.triggeredBarriers = new Set();
   }
 
   /**
@@ -48,6 +51,86 @@ export class ChainCheckpointManager {
     return this.store.chainCheckpoints[chainRunId] || [];
   }
 
+  /**
+   * Checks whether all prerequisites for a task are satisfied (DAG Join Barrier, #328).
+   *
+   * @param {string} chainRunId
+   * @param {Object} task
+   * @returns {{ ready: boolean, pending?: boolean, failed?: boolean, outputs: Record<string, string>, latestOutput: string, waitingFor?: string[], failedDep?: string }}
+   */
+  checkJoinBarrier(chainRunId, task) {
+    if (!task) return { ready: false, outputs: {}, latestOutput: '' };
+    const deps = sanitizeDependsOn(task.dependsOn);
+    if (!deps.length) {
+      return { ready: true, outputs: {}, latestOutput: '' };
+    }
+
+    const snapshots = this.getChainSnapshots(chainRunId);
+    const snapMap = new Map();
+    for (const s of snapshots) {
+      if (s?.stepTaskId) snapMap.set(s.stepTaskId, s);
+    }
+
+    const policy = sanitizeJoinPolicy(task.joinPolicy);
+    const outputs = {};
+    let latestOutput = '';
+    const waitingFor = [];
+    let hasSuccess = false;
+    let failedDep = null;
+
+    for (const depId of deps) {
+      const snap = snapMap.get(depId);
+      if (!snap) {
+        waitingFor.push(depId);
+        continue;
+      }
+      const isOk = snap.status === 'completed' || snap.status === 'success';
+      if (isOk) {
+        hasSuccess = true;
+        outputs[depId] = String(snap.output || '');
+        latestOutput = String(snap.output || '');
+      } else {
+        failedDep = depId;
+      }
+    }
+
+    if (policy === 'any_success') {
+      if (hasSuccess) {
+        return { ready: true, outputs, latestOutput };
+      }
+      if (waitingFor.length === 0) {
+        return { ready: false, failed: true, failedDep: failedDep || deps[0], outputs, latestOutput };
+      }
+      return { ready: false, pending: true, waitingFor, outputs, latestOutput };
+    }
+
+    // Default: 'all_success'
+    if (waitingFor.length > 0) {
+      return { ready: false, pending: true, waitingFor, outputs, latestOutput };
+    }
+    if (failedDep) {
+      return { ready: false, failed: true, failedDep, outputs, latestOutput };
+    }
+
+    return { ready: true, outputs, latestOutput };
+  }
+
+  /**
+   * Barrier idempotency guard: returns true if barrier has not yet fired for this task in this run.
+   */
+  canTriggerBarrier(chainRunId, taskId) {
+    if (!chainRunId || !taskId) return false;
+    return !this.triggeredBarriers.has(`${chainRunId}:${taskId}`);
+  }
+
+  /**
+   * Marks a barrier as triggered for this chain run to prevent duplicate executions.
+   */
+  markBarrierTriggered(chainRunId, taskId) {
+    if (!chainRunId || !taskId) return;
+    this.triggeredBarriers.add(`${chainRunId}:${taskId}`);
+  }
+
   /**
    * Checks whether a step can be retried in a chain run
    */
@@ -78,7 +161,7 @@ export class ChainCheckpointManager {
   }
 
   /**
-   * Retries chain execution from the specified failed step (#319)
+   * Retries chain execution from the specified failed step (#319, #328)
    * Reuses upstream outputs without re-executing successful upstream steps.
    */
   async retryChainFromStep(scheduler, chainRunId, stepTaskId) {
@@ -90,25 +173,40 @@ export class ChainCheckpointManager {
 
     try {
       const snapshots = this.getChainSnapshots(chainRunId);
-      // Find the nearest successful preceding step snapshot
-      const stepIndex = snapshots.findIndex((s) => s.stepTaskId === stepTaskId);
-      let prevSnapshot = null;
-      for (let i = stepIndex - 1; i >= 0; i--) {
-        if (snapshots[i].status === 'success' || snapshots[i].status === 'completed') {
-          prevSnapshot = snapshots[i];
-          break;
+
+      // Collect upstream outputs from checkpoints
+      const prevOutputs = {};
+      let prevOutput = '';
+      let prevTaskId = '';
+
+      for (const s of snapshots) {
+        if (s.stepTaskId !== stepTaskId && (s.status === 'success' || s.status === 'completed')) {
+          prevOutputs[s.stepTaskId] = String(s.output || '');
+          prevOutput = String(s.output || '');
+          prevTaskId = s.stepTaskId;
         }
       }
 
-      const prevOutput = prevSnapshot ? prevSnapshot.output : '';
-      const prevTaskId = prevSnapshot ? prevSnapshot.stepTaskId : '';
+      // Reset barrier locks for stepTaskId and downstream dependents so they can re-trigger
+      this.triggeredBarriers.delete(`${chainRunId}:${stepTaskId}`);
+      if (this.store?.tasks) {
+        for (const t of Object.values(this.store.tasks)) {
+          const deps = sanitizeDependsOn(t?.dependsOn);
+          if (deps.includes(stepTaskId)) {
+            this.triggeredBarriers.delete(`${chainRunId}:${t.id}`);
+          }
+        }
+      }
 
       // Execute only the failed step, passing reconstructed chain context
       const runFn = (scheduler.runTask || scheduler.runNow).bind(scheduler);
       const res = await runFn(stepTaskId, {
         chainRunId,
+        prevOutputs,
         chainPrevOutput: prevOutput,
+        prevOutput,
         chainPrevTaskId: prevTaskId,
+        prevTaskId,
         isRetry: true,
       });