@goodandready/dsh-cron 0.3.2 → 0.3.4

package/README.md

@@ -565,6 +565,7 @@ All endpoints are served by the DSH web server under `/dsh-cron/`. Endpoints are
 | `POST` | `/dsh-cron/tasks` | Create or update a task (`id` present → update). Requires `title`, `schedule`, `prompt` |
 | `GET` | `/dsh-cron/tasks/:id/history` | Run history, `?limit=20` |
 | `POST` | `/dsh-cron/tasks/:id/run` | Trigger an immediate manual run |
+| `GET` | `/dsh-cron/tasks/:id/stream` | Real-time Server-Sent Events (SSE) execution log streaming with ring buffer (#336) |
 | `POST` | `/dsh-cron/tasks/:id/pause` | Pause the schedule |
 | `POST` | `/dsh-cron/tasks/:id/resume` | Resume the schedule |
 | `POST` | `/dsh-cron/tasks/:id/toggle` | Toggle active/paused |
@@ -590,6 +591,28 @@ All endpoints are served by the DSH web server under `/dsh-cron/`. Endpoints are
 
 ---
 
+### ⚡ Adaptive Polling & Exponential Backoff (#329)
+
+Save LLM tokens and API quotas on watcher tasks by dynamically slowing down execution when no changes occur:
+
+```json
+"adaptive": {
+  "enabled": true,
+  "minInterval": "5m",
+  "maxInterval": "2h",
+  "backoffFactor": 1.5,
+  "idleRunsThreshold": 3
+}
+```
+- Multiplies interval by `backoffFactor` up to `maxInterval` after `idleRunsThreshold` silent or unchanged runs.
+- Instantly resets to `minInterval` as soon as real changes or non-silent output are detected.
+
+### 📺 Real-Time Streaming Logs via SSE (#336)
+
+Connect to `GET /dsh-cron/tasks/:id/stream` using Server-Sent Events (SSE) to receive live stdout/stderr chunks and execution completion events with an automatic 500-line ring buffer.
+
+---
+
 ## 🧪 Testing & Preflight Verification
 
 Run the automated test suite covering schedule parsing, the scheduler engine, atomic storage, HTTP helpers, notifications and tool contracts:

package/lib/adaptive.js

@@ -0,0 +1,151 @@
+/**
+ * Adaptive Polling & Exponential Backoff (#329)
+ *
+ * Automatically scales execution intervals up when no changes / idle runs are detected
+ * (e.g. silent skips, unchanged outputs) and instantly resets back to minInterval
+ * when new data is returned.
+ */
+
+import { logger } from './logger.js';
+
+export const DEFAULT_ADAPTIVE_CONFIG = {
+  enabled: false,
+  minInterval: '5m',
+  maxInterval: '2h',
+  backoffFactor: 1.5,
+  idleRunsThreshold: 3,
+  detectUnchangedOutput: true,
+};
+
+/**
+ * Parse human interval string (e.g. "30s", "5m", "2h", "1d") or number into milliseconds.
+ */
+export function parseIntervalMs(val, defaultMs = 300000) {
+  if (typeof val === 'number' && !isNaN(val) && val > 0) {
+    return Math.round(val);
+  }
+  if (!val || typeof val !== 'string') return defaultMs;
+
+  const match = val.trim().match(/^(\d+(?:\.\d+)?)\s*(ms|s|m|h|d)?$/i);
+  if (!match) return defaultMs;
+
+  const num = parseFloat(match[1]);
+  const unit = (match[2] || 'm').toLowerCase();
+
+  switch (unit) {
+    case 'ms': return Math.round(num);
+    case 's': return Math.round(num * 1000);
+    case 'm': return Math.round(num * 60 * 1000);
+    case 'h': return Math.round(num * 3600 * 1000);
+    case 'd': return Math.round(num * 86400 * 1000);
+    default: return defaultMs;
+  }
+}
+
+/**
+ * Format milliseconds into human readable interval (e.g. "5m", "1h 30m", "45s").
+ */
+export function formatInterval(ms) {
+  if (!ms || typeof ms !== 'number' || ms <= 0) return '0s';
+  const sec = Math.round(ms / 1000);
+  if (sec < 60) return `${sec}s`;
+  const min = Math.floor(sec / 60);
+  const remSec = sec % 60;
+  if (min < 60) return remSec > 0 ? `${min}m ${remSec}s` : `${min}m`;
+  const hrs = Math.floor(min / 60);
+  const remMin = min % 60;
+  return remMin > 0 ? `${hrs}h ${remMin}m` : `${hrs}h`;
+}
+
+/**
+ * Sanitize and normalize adaptive configuration from payload.
+ */
+export function sanitizeAdaptiveConfig(adaptive) {
+  if (!adaptive || typeof adaptive !== 'object') {
+    return null;
+  }
+  const enabled = Boolean(adaptive.enabled);
+  const minInterval = typeof adaptive.minInterval === 'string' ? adaptive.minInterval.trim() : (adaptive.minInterval ? String(adaptive.minInterval) : '5m');
+  const maxInterval = typeof adaptive.maxInterval === 'string' ? adaptive.maxInterval.trim() : (adaptive.maxInterval ? String(adaptive.maxInterval) : '2h');
+  const backoffFactor = (typeof adaptive.backoffFactor === 'number' && adaptive.backoffFactor >= 1) ? adaptive.backoffFactor : 1.5;
+  const idleRunsThreshold = (typeof adaptive.idleRunsThreshold === 'number' && adaptive.idleRunsThreshold >= 1) ? Math.floor(adaptive.idleRunsThreshold) : 3;
+  const detectUnchangedOutput = adaptive.detectUnchangedOutput !== false;
+
+  return {
+    enabled,
+    minInterval,
+    maxInterval,
+    backoffFactor,
+    idleRunsThreshold,
+    detectUnchangedOutput,
+  };
+}
+
+/**
+ * Evaluate if a completed run constitutes an "idle / no-change" execution.
+ */
+export function evaluateAdaptiveRun(task, outcome, runInfo, lastRun) {
+  if (outcome?.status === 'silentSkip' || runInfo?.silentSkip === true) {
+    return { isIdle: true, reason: 'silent_rule' };
+  }
+  if (runInfo?.validation && runInfo.validation.noChange === true) {
+    return { isIdle: true, reason: 'validator_no_change' };
+  }
+  if (task.adaptive?.detectUnchangedOutput !== false && lastRun && outcome?.status === 'success') {
+    const currentOut = String(outcome.output || '').trim();
+    const lastOut = String(lastRun.output || '').trim();
+    if (currentOut && lastOut && currentOut === lastOut) {
+      return { isIdle: true, reason: 'unchanged_output' };
+    }
+  }
+  return { isIdle: false, reason: 'active_output' };
+}
+
+/**
+ * Calculate the next adaptive state and schedule interval.
+ */
+export function calculateNextAdaptiveState(task, isIdle, now = Date.now()) {
+  const config = sanitizeAdaptiveConfig(task.adaptive) || DEFAULT_ADAPTIVE_CONFIG;
+  const minMs = parseIntervalMs(config.minInterval, 300000);
+  const maxMs = parseIntervalMs(config.maxInterval, 7200000);
+  const factor = config.backoffFactor || 1.5;
+  const threshold = config.idleRunsThreshold || 3;
+
+  const prevState = task.adaptiveState || {
+    idleCount: 0,
+    currentIntervalMs: minMs,
+    effectiveIntervalStr: formatInterval(minMs),
+    active: false,
+    lastAdjustedAt: now,
+  };
+
+  if (!isIdle) {
+    // Reset back to base minInterval immediately upon real changes
+    return {
+      idleCount: 0,
+      currentIntervalMs: minMs,
+      effectiveIntervalStr: formatInterval(minMs),
+      active: false,
+      lastAdjustedAt: now,
+      backedOff: false,
+    };
+  }
+
+  const idleCount = (prevState.idleCount || 0) + 1;
+  let currentIntervalMs = prevState.currentIntervalMs || minMs;
+  let backedOff = false;
+
+  if (idleCount >= threshold) {
+    backedOff = true;
+    currentIntervalMs = Math.min(Math.round(currentIntervalMs * factor), maxMs);
+  }
+
+  return {
+    idleCount,
+    currentIntervalMs,
+    effectiveIntervalStr: formatInterval(currentIntervalMs),
+    active: backedOff,
+    lastAdjustedAt: now,
+    backedOff,
+  };
+}

package/lib/analytics-digest.js

@@ -0,0 +1,236 @@
+/**
+ * Weekly AI Analytics & Cost Digest Engine (#330)
+ *
+ * Aggregates automation performance, success rates, token and USD spending,
+ * identifies top unstable and top expensive tasks, and formats delivery payloads.
+ */
+
+export function buildAnalyticsDigest({ tasks = [], history = [], costLedger = [], periodDays = 7, now = Date.now() }) {
+  const windowDays = typeof periodDays === 'number' && periodDays > 0 ? periodDays : 7;
+  const windowStart = now - (windowDays * 86400000);
+
+  const taskList = Array.isArray(tasks) ? tasks : [];
+  const historyList = Array.isArray(history) ? history : [];
+  const costList = Array.isArray(costLedger) ? costLedger : [];
+
+  // Filter history runs in window
+  const runsInWindow = historyList.filter((r) => {
+    const t = r.startedAt || r.completedAt || r.timestamp;
+    return typeof t === 'number' && t >= windowStart;
+  });
+
+  let totalRuns = 0;
+  let successfulRuns = 0;
+  let failedRuns = 0;
+  let totalTokens = 0;
+  let totalCostUsd = 0;
+  let healedCount = 0;
+
+  const taskStatsMap = new Map();
+
+  for (const task of taskList) {
+    taskStatsMap.set(task.id, {
+      id: task.id,
+      name: task.name || task.id,
+      runs: 0,
+      failures: 0,
+      tokens: 0,
+      costUsd: 0,
+    });
+  }
+
+  if (runsInWindow.length > 0) {
+    totalRuns = runsInWindow.length;
+    for (const r of runsInWindow) {
+      const isFailed = r.status === 'failure' || r.status === 'failed' || Boolean(r.error);
+      if (isFailed) {
+        failedRuns += 1;
+      } else {
+        successfulRuns += 1;
+      }
+
+      if (r.healed || r.autoHealed || (typeof r.retryCount === 'number' && r.retryCount > 0 && !isFailed)) {
+        healedCount += 1;
+      }
+
+      const tokens = typeof r.tokens === 'number' ? r.tokens : (typeof r.tokensUsed === 'number' ? r.tokensUsed : 0);
+      totalTokens += tokens;
+
+      const cost = typeof r.costUsd === 'number' ? r.costUsd : 0;
+      totalCostUsd += cost;
+
+      if (r.taskId && taskStatsMap.has(r.taskId)) {
+        const ts = taskStatsMap.get(r.taskId);
+        ts.runs += 1;
+        if (isFailed) ts.failures += 1;
+        ts.tokens += tokens;
+        ts.costUsd += cost;
+      }
+    }
+  } else {
+    // Fallback: aggregate from task summary counters
+    for (const task of taskList) {
+      const runs = typeof task.runs === 'number' ? task.runs : 0;
+      const failures = typeof task.failures === 'number' ? task.failures : 0;
+      const successes = Math.max(0, runs - failures);
+      const tokens = typeof task.tokensUsed === 'number' ? task.tokensUsed : 0;
+      const cost = typeof task.costUsd === 'number' ? task.costUsd : 0;
+
+      totalRuns += runs;
+      successfulRuns += successes;
+      failedRuns += failures;
+      totalTokens += tokens;
+      totalCostUsd += cost;
+
+      const ts = taskStatsMap.get(task.id);
+      if (ts) {
+        ts.runs = runs;
+        ts.failures = failures;
+        ts.tokens = tokens;
+        ts.costUsd = cost;
+      }
+    }
+  }
+
+  // Calculate cost from ledger if available and higher
+  const ledgerCostInWindow = costList
+    .filter((c) => typeof c.timestamp === 'number' && c.timestamp >= windowStart)
+    .reduce((sum, c) => sum + (typeof c.costUsd === 'number' ? c.costUsd : 0), 0);
+  if (ledgerCostInWindow > totalCostUsd) {
+    totalCostUsd = ledgerCostInWindow;
+  }
+
+  const successRate = totalRuns > 0 ? Number(((successfulRuns / totalRuns) * 100).toFixed(1)) : 100.0;
+
+  // Identify top unstable tasks
+  const topUnstableTasks = Array.from(taskStatsMap.values())
+    .filter((ts) => ts.failures > 0)
+    .sort((a, b) => b.failures - a.failures || (b.runs - a.runs))
+    .slice(0, 3)
+    .map((ts) => ({
+      id: ts.id,
+      name: ts.name,
+      runs: ts.runs,
+      failures: ts.failures,
+      successRate: ts.runs > 0 ? Number((((ts.runs - ts.failures) / ts.runs) * 100).toFixed(1)) : 0.0,
+    }));
+
+  // Identify top expensive tasks
+  const topExpensiveTasks = Array.from(taskStatsMap.values())
+    .filter((ts) => ts.tokens > 0 || ts.costUsd > 0)
+    .sort((a, b) => b.costUsd - a.costUsd || b.tokens - a.tokens)
+    .slice(0, 3)
+    .map((ts) => ({
+      id: ts.id,
+      name: ts.name,
+      tokens: ts.tokens,
+      costUsd: Number(ts.costUsd.toFixed(4)),
+    }));
+
+  const activeTasksCount = taskList.filter((t) => t.status !== 'paused').length;
+  const pausedTasksCount = taskList.filter((t) => t.status === 'paused').length;
+
+  return {
+    periodDays: windowDays,
+    windowStart,
+    windowEnd: now,
+    totalRuns,
+    successfulRuns,
+    failedRuns,
+    successRate,
+    totalTokens,
+    totalCostUsd: Number(totalCostUsd.toFixed(4)),
+    healedCount,
+    activeTasksCount,
+    pausedTasksCount,
+    totalTasksCount: taskList.length,
+    topUnstableTasks,
+    topExpensiveTasks,
+  };
+}
+
+export function formatDigestMarkdown(digest) {
+  const {
+    periodDays,
+    totalRuns,
+    successfulRuns,
+    failedRuns,
+    successRate,
+    totalTokens,
+    totalCostUsd,
+    healedCount,
+    activeTasksCount,
+    pausedTasksCount,
+    topUnstableTasks,
+    topExpensiveTasks,
+  } = digest;
+
+  const lines = [
+    `📊 **DSH Cron Automation Digest (Last ${periodDays} Days)**`,
+    `• **Total Executions:** ${totalRuns} (✅ ${successfulRuns} succeeded, ❌ ${failedRuns} failed — **${successRate}%** SR)`,
+    `• **Token Consumption:** ${totalTokens.toLocaleString('en-US')} tokens (~$${totalCostUsd.toFixed(4)} USD)`,
+    `• **Task Fleet:** ${activeTasksCount} active, ${pausedTasksCount} paused`,
+  ];
+
+  if (healedCount > 0) {
+    lines.push(`• **Self-Healed Runs:** 🔄 ${healedCount} recovered via retry/failure-inspector`);
+  }
+
+  lines.push('');
+  lines.push('⚠️ **Top Unstable Automations:**');
+  if (topUnstableTasks.length === 0) {
+    lines.push('*(All automations healthy — 0 failures recorded!)*');
+  } else {
+    topUnstableTasks.forEach((t, i) => {
+      lines.push(`${i + 1}. \`${t.name}\` — **${t.successRate}%** success (${t.failures} failed of ${t.runs} runs)`);
+    });
+  }
+
+  lines.push('');
+  lines.push('💰 **Top Resource Consumers:**');
+  if (topExpensiveTasks.length === 0) {
+    lines.push('*(No significant token consumption recorded)*');
+  } else {
+    topExpensiveTasks.forEach((t, i) => {
+      lines.push(`${i + 1}. \`${t.name}\` — ${t.tokens.toLocaleString('en-US')} tokens ($${t.costUsd.toFixed(4)})`);
+    });
+  }
+
+  return lines.join('\n');
+}
+
+export function formatDigestTelegram(digest) {
+  // Return clean plain Markdown compatible with standard bot message formatting
+  return formatDigestMarkdown(digest);
+}
+
+export async function sendAnalyticsDigest({ digest, channels, deliveryConfig }) {
+  if (!channels || !digest) {
+    throw new Error('channels and digest are required to send analytics digest');
+  }
+
+  const message = formatDigestMarkdown(digest);
+  const results = { telegram: null, webhook: null };
+
+  if (deliveryConfig?.telegram && typeof channels.sendTelegram === 'function') {
+    try {
+      results.telegram = await channels.sendTelegram(message, { parseMode: 'Markdown' });
+    } catch (err) {
+      results.telegram = { error: err.message };
+    }
+  }
+
+  if (deliveryConfig?.webhookUrl && typeof channels.sendWebhook === 'function') {
+    try {
+      results.webhook = await channels.sendWebhook(deliveryConfig.webhookUrl, {
+        event: 'analytics.digest',
+        digest,
+        text: message,
+      });
+    } catch (err) {
+      results.webhook = { error: err.message };
+    }
+  }
+
+  return results;
+}

package/lib/api.js

@@ -1,3 +1,6 @@
+import { rollbackPromptVersion, sanitizePromptHistory } from './prompt-evolver.js';
+import { sanitizeAdaptiveConfig } from './adaptive.js';
+import { logStreamer } from './log-streamer.js';
 import { approvalManager } from './approval.js';
 import { sanitizeValidators } from './validators.js';
 /**
@@ -289,6 +292,10 @@ async function handleTaskItem({ store, scheduler, req, res, url, id, action, api
   }
 
   if (req.method === 'GET') {
+    if (action === 'stream') {
+      logStreamer.handleSseRequest(id, req, res);
+      return;
+    }
     if (rejectCrossOrigin(req, res, { apiToken })) return;
     if (action === 'archive') {
       const limit = parseInt(url.searchParams.get('limit') || '50', 10);
@@ -305,6 +312,20 @@ async function handleTaskItem({ store, scheduler, req, res, url, id, action, api
       sendJson(res, 200, { ok: true, taskId: id, stats });
       return;
     }
+    if (action === 'prompt-history') {
+      const task = store.get(id);
+      if (!task) {
+        sendJson(res, 404, NOT_FOUND);
+        return;
+      }
+      sendJson(res, 200, {
+        ok: true,
+        taskId: id,
+        promptVersion: task.promptVersion || 1,
+        promptHistory: task.promptHistory || [],
+      });
+      return;
+    }
     if (action === 'history') {
       const limit = parseInt(url.searchParams.get('limit') || '20', 10);
       sendJson(res, 200, { ok: true, history: store.getHistory(id, limit) });
@@ -344,6 +365,32 @@ async function handleTaskItem({ store, scheduler, req, res, url, id, action, api
 
 /** Task actions that are plain POSTs. Returns true when one matched. */
 async function handleItemPost({ store, scheduler, req, res, url, id, action }) {
+  if (action === 'prompt-rollback') {
+    const task = store.get(id);
+    if (!task) {
+      sendJson(res, 404, NOT_FOUND);
+      return true;
+    }
+    const body = await readBodyJson(req).catch(() => ({}));
+    const targetVersion = Number(body.version || url.searchParams.get('version'));
+    if (!targetVersion || targetVersion < 1) {
+      sendJson(res, 400, { ok: false, error: 'A valid target prompt version is required' });
+      return true;
+    }
+    try {
+      rollbackPromptVersion(task, targetVersion);
+      store.set(task);
+      sendJson(res, 200, {
+        ok: true,
+        task: redactTaskSecrets(task),
+        promptVersion: task.promptVersion,
+        promptHistory: task.promptHistory,
+      });
+    } catch (err) {
+      sendJson(res, 400, { ok: false, error: err.message });
+    }
+    return true;
+  }
   if (action === 'retry-delivery') {
     const task = store.get(id);
     if (!task) {
@@ -596,6 +643,8 @@ function mergeExecutionFields(current, body, parsed) {
     concurrencyGroup: body.concurrencyGroup !== undefined ? String(body.concurrencyGroup).trim() : (current ? (current.concurrencyGroup || 'default') : 'default'),
     selfHealingCommand: body.selfHealingCommand !== undefined ? String(body.selfHealingCommand).trim() : (current ? (current.selfHealingCommand || '') : ''),
     autoDiagnose: body.autoDiagnose !== undefined ? Boolean(body.autoDiagnose) : (current ? Boolean(current.autoDiagnose) : false),
+    adaptive: body.adaptive !== undefined ? sanitizeAdaptiveConfig(body.adaptive) : (current ? current.adaptive : undefined),
+    adaptiveState: current ? current.adaptiveState : undefined,
   };
 }