@goodandready/dsh-cron 0.3.24 → 0.3.26

package/lib/store/artifacts.js

@@ -0,0 +1,200 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { createHash, randomUUID } from 'node:crypto';
+
+export const SAFE_MIME_TYPES = {
+  '.csv': 'text/csv; charset=utf-8',
+  '.json': 'application/json; charset=utf-8',
+  '.txt': 'text/plain; charset=utf-8',
+  '.log': 'text/plain; charset=utf-8',
+  '.md': 'text/markdown; charset=utf-8',
+  '.pdf': 'application/pdf',
+  '.png': 'image/png',
+  '.jpg': 'image/jpeg',
+  '.jpeg': 'image/jpeg',
+  '.webp': 'image/webp',
+};
+
+export class ArtifactManager {
+  constructor(baseDir) {
+    this.baseDir = baseDir || path.join(process.cwd(), 'artifacts');
+    this.maxArtifactBytes = 10 * 1024 * 1024; // 10 MiB per artifact
+    this.defaultRetentionDays = 7;
+    this.maxTotalStorageBytes = 50 * 1024 * 1024; // 50 MiB total quota
+  }
+
+  ensureDir() {
+    if (!fs.existsSync(this.baseDir)) {
+      fs.mkdirSync(this.baseDir, { recursive: true });
+    }
+  }
+
+  /**
+   * Sanitizes a requested filename to prevent path traversal (#321)
+   */
+  sanitizeFilename(name) {
+    if (!name || typeof name !== 'string') return 'artifact.txt';
+    const clean = path.basename(name).replace(/[\0\r\n\\\/]/g, '_');
+    return clean || 'artifact.txt';
+  }
+
+  getFilePath(artifactId, filename) {
+    if (!filename || filename.includes('..') || path.isAbsolute(filename)) {
+      throw new Error('Path traversal attempt detected in artifact storage');
+    }
+    const safeName = this.sanitizeFilename(filename);
+    const target = path.resolve(this.baseDir, `${artifactId}_${safeName}`);
+    // Security check: ensure path stays strictly within baseDir
+    if (!target.startsWith(path.resolve(this.baseDir) + path.sep)) {
+      throw new Error('Path traversal attempt detected in artifact storage');
+    }
+    return target;
+  }
+
+  /**
+   * Resolves safe MIME type for download/preview
+   */
+  resolveMimeType(filename) {
+    const ext = path.extname(filename).toLowerCase();
+    // Neutralize dangerous web formats to prevent XSS / script execution
+    if (ext === '.html' || ext === '.htm' || ext === '.svg' || ext === '.js') {
+      return 'text/plain; charset=utf-8';
+    }
+    return SAFE_MIME_TYPES[ext] || 'application/octet-stream';
+  }
+
+  /**
+   * Saves a new artifact file bound to a run (#321)
+   */
+  saveArtifact(runId, taskId, filename, contentBuffer) {
+    this.ensureDir();
+    const buf = Buffer.isBuffer(contentBuffer) ? contentBuffer : Buffer.from(contentBuffer || '', 'utf8');
+
+    if (buf.length > this.maxArtifactBytes) {
+      throw new Error(`Artifact size (${buf.length} bytes) exceeds limit of ${this.maxArtifactBytes} bytes`);
+    }
+
+    const artifactId = 'art_' + randomUUID().replace(/-/g, '').slice(0, 16);
+    const safeName = this.sanitizeFilename(filename);
+    const filePath = this.getFilePath(artifactId, safeName);
+
+    fs.writeFileSync(filePath, buf);
+
+    const sha256 = createHash('sha256').update(buf).digest('hex');
+    const mimeType = this.resolveMimeType(safeName);
+
+    return {
+      id: artifactId,
+      name: safeName,
+      sizeBytes: buf.length,
+      mimeType,
+      sha256,
+      runId,
+      taskId,
+      createdAt: Date.now(),
+      status: 'available',
+    };
+  }
+
+  /**
+   * Reads an artifact file buffer
+   */
+  readArtifact(artifactId, filename) {
+    const filePath = this.getFilePath(artifactId, filename);
+    if (!fs.existsSync(filePath)) {
+      return null;
+    }
+    return fs.readFileSync(filePath);
+  }
+
+  /**
+   * Generates a safe preview of the artifact content (first 64 KiB)
+   */
+  generatePreview(artifactId, filename) {
+    const buf = this.readArtifact(artifactId, filename);
+    if (!buf) return { ok: false, error: 'Artifact file not found on disk' };
+
+    const ext = path.extname(filename).toLowerCase();
+    const isText = ['.txt', '.csv', '.json', '.log', '.md'].includes(ext);
+
+    if (isText) {
+      const slice = buf.subarray(0, 64 * 1024);
+      return {
+        ok: true,
+        kind: 'text',
+        mimeType: this.resolveMimeType(filename),
+        content: slice.toString('utf8'),
+        truncated: buf.length > slice.length,
+        sizeBytes: buf.length,
+      };
+    }
+
+    return {
+      ok: true,
+      kind: 'binary',
+      mimeType: this.resolveMimeType(filename),
+      sizeBytes: buf.length,
+      preview: `[Binary file: ${filename} (${buf.length} bytes)]`,
+    };
+  }
+
+  /**
+   * Cleans up expired artifacts and enforces size quotas (#321)
+   */
+  cleanupArtifacts(store, { retentionDays = this.defaultRetentionDays, maxTotalBytes = this.maxTotalStorageBytes } = {}) {
+    if (!fs.existsSync(this.baseDir)) return { removedCount: 0, freedBytes: 0 };
+
+    const cutoff = Date.now() - retentionDays * 86400000;
+    let files = fs.readdirSync(this.baseDir);
+    let totalBytes = 0;
+    const fileStats = [];
+
+    for (const f of files) {
+      const p = path.join(this.baseDir, f);
+      try {
+        const stat = fs.statSync(p);
+        totalBytes += stat.size;
+        fileStats.push({ path: p, filename: f, mtime: stat.mtimeMs, size: stat.size });
+      } catch (e) {
+        // file unlinked concurrently
+      }
+    }
+
+    let removedCount = 0;
+    let freedBytes = 0;
+
+    // 1. Remove files older than retentionDays
+    for (const item of fileStats) {
+      if (item.mtime < cutoff) {
+        try {
+          fs.unlinkSync(item.path);
+          removedCount++;
+          freedBytes += item.size;
+          totalBytes -= item.size;
+        } catch (e) {
+          void e;
+        }
+      }
+    }
+
+    // 2. Enforce quota: remove oldest files if totalBytes > maxTotalBytes
+    if (totalBytes > maxTotalBytes) {
+      fileStats.sort((a, b) => a.mtime - b.mtime);
+      for (const item of fileStats) {
+        if (totalBytes <= maxTotalBytes) break;
+        if (fs.existsSync(item.path)) {
+          try {
+            fs.unlinkSync(item.path);
+            removedCount++;
+            freedBytes += item.size;
+            totalBytes -= item.size;
+          } catch (e) {
+          void e;
+        }
+        }
+      }
+    }
+
+    return { removedCount, freedBytes };
+  }
+}

package/lib/store/chain-checkpoints.js

@@ -0,0 +1,223 @@
+/**
+ * Chain Checkpoint, Step Replay, and DAG Barrier Manager (#319, #328)
+ */
+
+import { sanitizeDependsOn, sanitizeJoinPolicy } from '../dag.js';
+
+export class ChainCheckpointManager {
+  constructor(store) {
+    this.store = store;
+    this.activeRetries = new Set();
+    this.triggeredBarriers = new Set();
+  }
+
+  /**
+   * Records a snapshot of a completed or failed chain step
+   */
+  recordStepSnapshot(chainRunId, stepTaskId, snapshot) {
+    if (!chainRunId || !stepTaskId) return;
+    this.store.chainCheckpoints = this.store.chainCheckpoints || {};
+    this.store.chainCheckpoints[chainRunId] = this.store.chainCheckpoints[chainRunId] || [];
+
+    const list = this.store.chainCheckpoints[chainRunId];
+    const existingIndex = list.findIndex((s) => s.stepTaskId === stepTaskId);
+
+    const record = {
+      chainRunId,
+      stepTaskId,
+      status: snapshot.status || 'completed',
+      input: snapshot.input || {},
+      output: snapshot.output || '',
+      at: Date.now(),
+      durationMs: snapshot.durationMs || 0,
+      retryCount: (existingIndex >= 0 ? (list[existingIndex].retryCount || 0) + 1 : 0),
+    };
+
+    if (existingIndex >= 0) {
+      list[existingIndex] = record;
+    } else {
+      list.push(record);
+    }
+
+    this.store.scheduleSave(50);
+    return record;
+  }
+
+  /**
+   * Retrieves all step snapshots for a given chain run
+   */
+  getChainSnapshots(chainRunId) {
+    if (!chainRunId || !this.store.chainCheckpoints) return [];
+    return this.store.chainCheckpoints[chainRunId] || [];
+  }
+
+  /**
+   * Checks whether all prerequisites for a task are satisfied (DAG Join Barrier, #328).
+   *
+   * @param {string} chainRunId
+   * @param {Object} task
+   * @returns {{ ready: boolean, pending?: boolean, failed?: boolean, outputs: Record<string, string>, latestOutput: string, waitingFor?: string[], failedDep?: string }}
+   */
+  checkJoinBarrier(chainRunId, task) {
+    if (!task) return { ready: false, outputs: {}, latestOutput: '' };
+    const deps = sanitizeDependsOn(task.dependsOn);
+    if (!deps.length) {
+      return { ready: true, outputs: {}, latestOutput: '' };
+    }
+
+    const snapshots = this.getChainSnapshots(chainRunId);
+    const snapMap = new Map();
+    for (const s of snapshots) {
+      if (s?.stepTaskId) snapMap.set(s.stepTaskId, s);
+    }
+
+    const policy = sanitizeJoinPolicy(task.joinPolicy);
+    const outputs = {};
+    let latestOutput = '';
+    const waitingFor = [];
+    let hasSuccess = false;
+    let failedDep = null;
+
+    for (const depId of deps) {
+      const snap = snapMap.get(depId);
+      if (!snap) {
+        waitingFor.push(depId);
+        continue;
+      }
+      const isOk = snap.status === 'completed' || snap.status === 'success';
+      if (isOk) {
+        hasSuccess = true;
+        outputs[depId] = String(snap.output || '');
+        latestOutput = String(snap.output || '');
+      } else {
+        failedDep = depId;
+      }
+    }
+
+    if (policy === 'any_success') {
+      if (hasSuccess) {
+        return { ready: true, outputs, latestOutput };
+      }
+      if (waitingFor.length === 0) {
+        return { ready: false, failed: true, failedDep: failedDep || deps[0], outputs, latestOutput };
+      }
+      return { ready: false, pending: true, waitingFor, outputs, latestOutput };
+    }
+
+    // Default: 'all_success'
+    if (waitingFor.length > 0) {
+      return { ready: false, pending: true, waitingFor, outputs, latestOutput };
+    }
+    if (failedDep) {
+      return { ready: false, failed: true, failedDep, outputs, latestOutput };
+    }
+
+    return { ready: true, outputs, latestOutput };
+  }
+
+  /**
+   * Barrier idempotency guard: returns true if barrier has not yet fired for this task in this run.
+   */
+  canTriggerBarrier(chainRunId, taskId) {
+    if (!chainRunId || !taskId) return false;
+    return !this.triggeredBarriers.has(`${chainRunId}:${taskId}`);
+  }
+
+  /**
+   * Marks a barrier as triggered for this chain run to prevent duplicate executions.
+   */
+  markBarrierTriggered(chainRunId, taskId) {
+    if (!chainRunId || !taskId) return;
+    this.triggeredBarriers.add(`${chainRunId}:${taskId}`);
+  }
+
+  /**
+   * Checks whether a step can be retried in a chain run
+   */
+  canRetryFromStep(chainRunId, stepTaskId) {
+    if (!chainRunId || !stepTaskId) return { ok: false, error: 'Missing chainRunId or stepTaskId' };
+
+    const lockKey = `${chainRunId}:${stepTaskId}`;
+    if (this.activeRetries.has(lockKey)) {
+      return { ok: false, error: 'Retry already in progress for this step' };
+    }
+
+    const snapshots = this.getChainSnapshots(chainRunId);
+    const stepSnapshot = snapshots.find((s) => s.stepTaskId === stepTaskId);
+    if (!stepSnapshot) {
+      return { ok: false, error: `No checkpoint found for step "${stepTaskId}" in chain "${chainRunId}"` };
+    }
+
+    if (stepSnapshot.status === 'success' || stepSnapshot.status === 'completed') {
+      return { ok: false, error: 'Step already completed successfully; cannot retry a successful step' };
+    }
+
+    const task = this.store.get(stepTaskId);
+    if (!task) {
+      return { ok: false, error: `Task "${stepTaskId}" was deleted or no longer exists` };
+    }
+
+    return { ok: true, task, stepSnapshot };
+  }
+
+  /**
+   * Retries chain execution from the specified failed step (#319, #328)
+   * Reuses upstream outputs without re-executing successful upstream steps.
+   */
+  async retryChainFromStep(scheduler, chainRunId, stepTaskId) {
+    const check = this.canRetryFromStep(chainRunId, stepTaskId);
+    if (!check.ok) return check;
+
+    const lockKey = `${chainRunId}:${stepTaskId}`;
+    this.activeRetries.add(lockKey);
+
+    try {
+      const snapshots = this.getChainSnapshots(chainRunId);
+
+      // Collect upstream outputs from checkpoints
+      const prevOutputs = {};
+      let prevOutput = '';
+      let prevTaskId = '';
+
+      for (const s of snapshots) {
+        if (s.stepTaskId !== stepTaskId && (s.status === 'success' || s.status === 'completed')) {
+          prevOutputs[s.stepTaskId] = String(s.output || '');
+          prevOutput = String(s.output || '');
+          prevTaskId = s.stepTaskId;
+        }
+      }
+
+      // Reset barrier locks for stepTaskId and downstream dependents so they can re-trigger
+      this.triggeredBarriers.delete(`${chainRunId}:${stepTaskId}`);
+      if (this.store?.tasks) {
+        for (const t of Object.values(this.store.tasks)) {
+          const deps = sanitizeDependsOn(t?.dependsOn);
+          if (deps.includes(stepTaskId)) {
+            this.triggeredBarriers.delete(`${chainRunId}:${t.id}`);
+          }
+        }
+      }
+
+      // Execute only the failed step, passing reconstructed chain context
+      const runFn = (scheduler.runTask || scheduler.runNow).bind(scheduler);
+      const res = await runFn(stepTaskId, {
+        chainRunId,
+        prevOutputs,
+        chainPrevOutput: prevOutput,
+        prevOutput,
+        chainPrevTaskId: prevTaskId,
+        prevTaskId,
+        isRetry: true,
+      });
+
+      return {
+        ok: true,
+        chainRunId,
+        stepTaskId,
+        result: res,
+      };
+    } finally {
+      this.activeRetries.delete(lockKey);
+    }
+  }
+}

package/lib/store/durable-journal.js

@@ -0,0 +1,188 @@
+import { createHash, randomUUID } from 'node:crypto';
+
+/**
+ * States for durable task execution lifecycle (#317)
+ */
+export const JOURNAL_STATES = {
+  QUEUED: 'queued',
+  RUNNING: 'running',
+  COMPLETED: 'completed',
+  FAILED: 'failed',
+  CANCELLED: 'cancelled',
+  UNCERTAIN: 'uncertain',
+};
+
+export class DurableJournal {
+  constructor(store) {
+    this.store = store;
+  }
+
+  /**
+   * Generates a deterministic idempotency key for a task run
+   */
+  generateIdempotencyKey(taskId, scheduleTick) {
+    const raw = `${taskId}:${scheduleTick || Date.now()}`;
+    return 'idem_' + createHash('sha256').update(raw).digest('hex').slice(0, 24);
+  }
+
+  /**
+   * Checks whether a run with this idempotency key was already completed
+   */
+  isIdempotencyKeyExecuted(taskId, idempotencyKey) {
+    if (!idempotencyKey) return null;
+    const history = this.store.getHistory(taskId) || [];
+    return history.find(
+      (r) => r.idempotencyKey === idempotencyKey &&
+             (r.status === JOURNAL_STATES.COMPLETED || r.status === 'success')
+    ) || null;
+  }
+
+  /**
+   * Pre-allocates a durable run entry before execution starts (#317)
+   */
+  createRunRecord(taskId, { idempotencyKey, chainRunId, trigger = 'scheduler', scheduleTick } = {}) {
+    const runId = 'run_' + randomUUID().replace(/-/g, '').slice(0, 16);
+    const key = idempotencyKey || this.generateIdempotencyKey(taskId, scheduleTick);
+
+    const record = {
+      id: runId,
+      taskId,
+      at: Date.now(),
+      status: JOURNAL_STATES.QUEUED,
+      trigger,
+      idempotencyKey: key,
+      chainRunId: chainRunId || null,
+      stages: [
+        { name: 'queue', status: 'success', startedAt: Date.now(), finishedAt: Date.now() },
+        { name: 'preflight', status: 'pending' },
+        { name: 'execution', status: 'pending' },
+        { name: 'validation', status: 'pending' },
+        { name: 'delivery', status: 'pending' },
+      ],
+      stoppedReason: null,
+      createdAt: Date.now(),
+    };
+
+    // Record initial queued entry into history/store
+    this.store.recordRun(taskId, record);
+    return record;
+  }
+
+  /**
+   * Transitions a run record to a new state and updates history entry
+   */
+  transition(taskId, runId, toState, metadata = {}) {
+    const runs = this.store.history.get(taskId) || [];
+    const index = runs.findIndex((r) => r.id === runId);
+    if (index === -1) return null;
+
+    const existing = runs[index];
+    const updated = {
+      ...existing,
+      ...metadata,
+      status: toState,
+      updatedAt: Date.now(),
+    };
+
+    if (
+      toState === JOURNAL_STATES.COMPLETED ||
+      toState === JOURNAL_STATES.FAILED ||
+      toState === JOURNAL_STATES.CANCELLED ||
+      toState === JOURNAL_STATES.UNCERTAIN
+    ) {
+      updated.finishedAt = Date.now();
+      if (updated.startedAt && !updated.durationMs) {
+        updated.durationMs = updated.finishedAt - updated.startedAt;
+      }
+    }
+
+    runs[index] = updated;
+    this.store.history.set(taskId, runs);
+    this.store.scheduleSave(50);
+    return updated;
+  }
+
+  /**
+   * Recovers uncompleted / in-flight runs on daemon restart (#317)
+   */
+  recoverPendingRuns(scheduler) {
+    const tasks = this.store.list();
+    const recovered = { uncertain: [], requeued: [] };
+
+    for (const task of tasks) {
+      const history = this.store.history.get(task.id) || [];
+      let historyChanged = false;
+
+      for (let i = 0; i < history.length; i++) {
+        const run = history[i];
+        if (run.status === JOURNAL_STATES.RUNNING) {
+          // Crash during execution -> transition to UNCERTAIN to prevent blind rerun
+          history[i] = {
+            ...run,
+            status: JOURNAL_STATES.UNCERTAIN,
+            stoppedReason: 'interrupted_by_restart',
+            uncertainAt: Date.now(),
+          };
+          historyChanged = true;
+          recovered.uncertain.push({ taskId: task.id, runId: run.id });
+        } else if (run.status === JOURNAL_STATES.QUEUED) {
+          // Queued run that didn't start yet: if task is still active and recent (<1 hour), re-enqueue
+          const ageMs = Date.now() - (run.createdAt || run.at);
+          if (task.status === 'active' && ageMs < 3600000 && scheduler && typeof scheduler.runTask === 'function') {
+            scheduler.queue = scheduler.queue || [];
+            if (!scheduler.queue.includes(task.id)) {
+              scheduler.queue.push(task.id);
+              recovered.requeued.push({ taskId: task.id, runId: run.id });
+            }
+          } else {
+            // Expired or cancelled queued run
+            history[i] = {
+              ...run,
+              status: JOURNAL_STATES.CANCELLED,
+              stoppedReason: 'Queued run expired or cancelled across restart',
+            };
+            historyChanged = true;
+          }
+        }
+      }
+
+      if (historyChanged) {
+        this.store.scheduleSave(50);
+      }
+    }
+
+    return recovered;
+  }
+
+  /**
+   * Manually resolves an uncertain run record (#317)
+   */
+  resolveUncertain(taskId, runId, { decision, note, resolvedBy = 'user' } = {}) {
+    const runs = this.store.history.get(taskId) || [];
+    const index = runs.findIndex((r) => r.id === runId);
+    if (index === -1) return { ok: false, error: 'Run not found' };
+
+    const run = runs[index];
+    if (run.status !== JOURNAL_STATES.UNCERTAIN) {
+      return { ok: false, error: `Run is in status "${run.status}", not uncertain` };
+    }
+
+    let nextStatus = JOURNAL_STATES.FAILED;
+    if (decision === 'completed' || decision === 'mark_completed') nextStatus = JOURNAL_STATES.COMPLETED;
+    else if (decision === 'failed' || decision === 'mark_failed') nextStatus = JOURNAL_STATES.FAILED;
+    else if (decision === 'cancel' || decision === 'cancelled') nextStatus = JOURNAL_STATES.CANCELLED;
+
+    const resolved = {
+      ...run,
+      status: nextStatus,
+      stoppedReason: `Uncertain run resolved by ${resolvedBy}: ${note || decision}`,
+      resolvedBy,
+      resolvedAt: Date.now(),
+    };
+
+    runs[index] = resolved;
+    this.store.history.set(taskId, runs);
+    this.store.scheduleSave(50);
+    return { ok: true, run: resolved };
+  }
+}

package/lib/store/index.js

@@ -0,0 +1,6 @@
+export * from './store.js';
+export * from './durable-journal.js';
+export * from './artifacts.js';
+export * from './chain-checkpoints.js';
+export * from './task-schema.js';
+export * from './task-transfer.js';