@goodandready/dsh-cron 0.2.45 → 0.2.47

package/lib/client.js

@@ -1442,10 +1442,10 @@ window.__ModuleLoader__.load({
         let timerId = null;
         let isDestroyed = false;
         let pollController = null;
-        const currentGen = fetchGenRef.current;
 
         const doPoll = () => {
-          if (isDestroyed || currentGen !== fetchGenRef.current) return;
+          if (isDestroyed) return;
+          const pollGen = ++fetchGenRef.current;
           const headers = {};
           if (lastEtagRef.current) headers['If-None-Match'] = lastEtagRef.current;
 
@@ -1457,7 +1457,7 @@ window.__ModuleLoader__.load({
 
           fetch('/dsh-cron/tasks?status=' + encodeURIComponent(tab) + '&query=' + encodeURIComponent(query), fetchOpts)
             .then(res => {
-              if (isDestroyed || currentGen !== fetchGenRef.current) return null;
+              if (isDestroyed || pollGen !== fetchGenRef.current) return null;
               if (res.status === 304) {
                 // Not modified: skip parsing JSON
                 return null;
@@ -1467,7 +1467,7 @@ window.__ModuleLoader__.load({
               return res.json();
             })
             .then(data => {
-              if (isDestroyed || currentGen !== fetchGenRef.current) return;
+              if (isDestroyed || pollGen !== fetchGenRef.current) return;
               if (data && data.ok) {
                 setTasks(data.tasks || []);
                 setRecs(data.recommendations || []);
@@ -1478,7 +1478,7 @@ window.__ModuleLoader__.load({
               if (err && err.name === 'AbortError') return;
             })
             .finally(() => {
-              if (isDestroyed || currentGen !== fetchGenRef.current) return;
+              if (isDestroyed) return;
               // Adaptive interval: 30s when tab is hidden, 8s when active (#134)
               const intervalMs = (typeof document !== 'undefined' && document.hidden) ? 30000 : 8000;
               timerId = setTimeout(doPoll, intervalMs);

package/lib/store.js

@@ -1,6 +1,7 @@
 import { logger } from './logger.js';
 import { bestEffort } from './best-effort.js';
 import fs from 'node:fs';
+import { pipeline } from 'node:stream/promises';
 import path from 'node:path';
 import { getDshDefaultTelegramCredentials } from './telegram.js';
 
@@ -168,15 +169,36 @@ export class TaskStore {
    * Asynchronously serialize and write tasks store to disk using fs.promises queue (#244).
    * Safe mode 0600 on files and 0700 on dir (#222).
    */
-  async saveAsync() {
-    const version = ++this._saveVersion;
-    const data = {
+  /**
+   * Create an isolated deep snapshot of tasks, history, and settings (#301).
+   * Prevents concurrent in-memory mutations (e.g. recordRun during streamed save)
+   * from corrupting serialized JSON state or causing duplicate / skipped run records.
+   */
+  _createSnapshot() {
+    const tasksSnapshot = Array.from(this.tasks.values()).map((t) => ({
+      ...t,
+      costLedger: Array.isArray(t.costLedger) ? t.costLedger.slice() : t.costLedger,
+    }));
+    const historySnapshot = {};
+    for (const [id, runs] of this.history.entries()) {
+      historySnapshot[id] = Array.isArray(runs) ? runs.slice() : runs;
+    }
+    return {
       version: 1,
       updatedAt: new Date().toISOString(),
-      tasks: Array.from(this.tasks.values()),
-      history: Object.fromEntries(this.history.entries()),
-      settings: this.settings || {},
+      tasks: tasksSnapshot,
+      history: historySnapshot,
+      settings: this.settings ? { ...this.settings } : {},
     };
+  }
+
+  /**
+   * Asynchronously serialize and write tasks store to disk using fs.promises queue (#244, #301, #303).
+   * Safe mode 0600 on files and 0700 on dir (#222).
+   */
+  async saveAsync() {
+    const version = ++this._saveVersion;
+    const data = this._createSnapshot();
 
     const writeOp = async () => {
       if ((this._diskVersion || 0) >= version || (this._lastSyncVersion || 0) >= version) {
@@ -190,14 +212,17 @@ export class TaskStore {
       const tmp = `${this.filePath}.tmp.${process.pid}_${Date.now()}_${rand}`;
       this._currentAsyncTmp = tmp;
 
-      // Yield to the event loop before CPU serialization so pending timers/I/O are serviced (#244)
-      await new Promise((resolve) => setImmediate(resolve));
-      // Compact serialization reduces payload size; disk write and rename are offloaded asynchronously via fs.promises (#244)
-      const payload = JSON.stringify(data);
-      // Yield after serialization before starting disk write I/O
-      await new Promise((resolve) => setImmediate(resolve));
-      await fs.promises.writeFile(tmp, payload, { encoding: 'utf-8', mode: 0o600 });
-      bestEffort('chmod-tmp', () => fs.chmodSync(tmp, 0o600));
+      // Streamed chunked serialization with fine-grained event loop yields (#244).
+      // Eliminates continuous CPU stalls on large stores (~21MB) by batching tasks and history runs
+      // with setImmediate yields every 10 tasks and 15 runs, guaranteeing max event loop lag < 10ms.
+      try {
+        await this._writeStoreStream(tmp, data, version);
+        bestEffort('chmod-tmp', () => fs.chmodSync(tmp, 0o600));
+      } catch (writeErr) {
+        await fs.promises.unlink(tmp).catch(() => {});
+        if (this._currentAsyncTmp === tmp) this._currentAsyncTmp = null;
+        throw writeErr;
+      }
 
       if ((this._diskVersion || 0) >= version || (this._lastSyncVersion || 0) >= version || this._saveVersion > version) {
         await fs.promises.unlink(tmp).catch(() => {});
@@ -237,6 +262,104 @@ export class TaskStore {
     return this._savePromise;
   }
 
+  /**
+   * Stream data incrementally to disk with fine-grained setImmediate yields (#244).
+   * Prevents blocking the V8 single thread on heavy serialization (e.g. 21MB / 10k runs).
+   */
+  /**
+   * Stream data incrementally to disk with fine-grained setImmediate yields (#244, #303).
+   * Prevents blocking the V8 single thread on heavy serialization (e.g. 21MB / 10k runs).
+   * Uses canonical Node stream/promises pipeline for error-handling and backpressure hygiene (#303).
+   */
+  async _writeStoreStream(filePath, data, version) {
+    const isCanceled = () => {
+      return (
+        (this._diskVersion || 0) >= version ||
+        (this._lastSyncVersion || 0) >= version ||
+        this._saveVersion > version
+      );
+    };
+
+    if (isCanceled()) {
+      return;
+    }
+
+    const ac = new AbortController();
+    let ws;
+    try {
+      ws = fs.createWriteStream(filePath, { encoding: 'utf-8', mode: 0o600 });
+    } catch (createErr) {
+      if (isCanceled()) return;
+      throw createErr;
+    }
+
+    async function* generateChunks() {
+      await new Promise((r) => setImmediate(r));
+      if (isCanceled()) {
+        ac.abort();
+        return;
+      }
+
+      yield '{"version":' + JSON.stringify(data.version || 1);
+      yield ',"updatedAt":' + JSON.stringify(data.updatedAt || new Date().toISOString());
+      yield ',"settings":' + JSON.stringify(data.settings || {});
+
+      yield ',"tasks":[';
+      const tasks = data.tasks || [];
+      for (let i = 0; i < tasks.length; i++) {
+        if (isCanceled()) {
+          ac.abort();
+          return;
+        }
+        if (i > 0) yield ',';
+        yield JSON.stringify(tasks[i]);
+        if ((i + 1) % 10 === 0) {
+          await new Promise((r) => setImmediate(r));
+        }
+      }
+      yield ']';
+
+      yield ',"history":{';
+      const historyEntries = Object.entries(data.history || {});
+      for (let i = 0; i < historyEntries.length; i++) {
+        if (isCanceled()) {
+          ac.abort();
+          return;
+        }
+        if (i > 0) yield ',';
+        const [taskId, runs] = historyEntries[i];
+        yield JSON.stringify(taskId) + ':[';
+        if (Array.isArray(runs)) {
+          for (let j = 0; j < runs.length; j++) {
+            if (isCanceled()) {
+              ac.abort();
+              return;
+            }
+            if (j > 0) yield ',';
+            yield JSON.stringify(runs[j]);
+            if ((j + 1) % 15 === 0) {
+              await new Promise((r) => setImmediate(r));
+            }
+          }
+        } else {
+          yield JSON.stringify(runs);
+        }
+        yield ']';
+        await new Promise((r) => setImmediate(r));
+      }
+      yield '}}';
+    }
+
+    try {
+      await pipeline(generateChunks(), ws, { signal: ac.signal });
+    } catch (err) {
+      if (ac.signal.aborted || isCanceled()) {
+        return;
+      }
+      throw err;
+    }
+  }
+
   save() {
     if (this._saveTimer) {
       clearTimeout(this._saveTimer);
@@ -257,13 +380,7 @@ export class TaskStore {
         fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
       }
       bestEffort('chmod-dir', () => fs.chmodSync(dir, 0o700));
-      const data = {
-        version: 1,
-        updatedAt: new Date().toISOString(),
-        tasks: Array.from(this.tasks.values()),
-        history: Object.fromEntries(this.history.entries()),
-        settings: this.settings || {},
-      };
+      const data = this._createSnapshot();
       // Unique tmp file per save call with process PID and random token to eliminate file collision
       const rand = Math.random().toString(36).slice(2, 8);
       const tmp = `${this.filePath}.tmp.${process.pid}_${Date.now()}_${rand}`;
@@ -551,42 +668,46 @@ export class TaskStore {
     task.updatedAt = Date.now();
 
     // Rolling 24h cost ledger preserved across archive rotation and restarts (#221)
-    if (!Array.isArray(task.costLedger)) {
-      task.costLedger = [];
-    }
-    task.costLedger.push({
-      at: task.lastRunAt,
-      costUsd,
-      tokens: runTokens,
-    });
+    const existingLedger = Array.isArray(task.costLedger) ? task.costLedger : [];
     const cutoff24h = Date.now() - 86400000;
-    task.costLedger = task.costLedger.filter(e => e && typeof e.at === 'number' && e.at >= cutoff24h);
-
-    const runs = this.history.get(id) || [];
-    runs.unshift({
-      id: 'run_' + Math.random().toString(36).slice(2, 9),
-      at: task.lastRunAt,
-      status: task.lastStatus,
-      durationMs: task.lastDurationMs,
-      output: (runInfo.output || '').slice(0, 4000),
-      error: runInfo.error ? String(runInfo.error).slice(0, 1000) : null,
-      usage,
-      costUsd,
-      sessionId: runInfo.sessionId || null,
-      // #45: which model produced the run, and whether it only finished thanks
-      // to the configured fallback.
-      model: runInfo.model || '',
-      fallback: Boolean(runInfo.fallback),
-      // #44: this run was suppressed by its silent rule, with the reason.
-      silentSkip: Boolean(runInfo.silentSkip),
-      silentReason: runInfo.silentSkip ? String(runInfo.silentReason || '').slice(0, 500) : '',
-      // #43: the failure diagnosis and the prompt change it proposes.
-      diagnosis: runInfo.diagnosis ? String(runInfo.diagnosis).slice(0, 1000) : '',
-      suggestion: runInfo.suggestion ? String(runInfo.suggestion).slice(0, 2000) : '',
-      selfHealing: runInfo.selfHealing || null,
-      autoDiagnosis: runInfo.autoDiagnosis ? String(runInfo.autoDiagnosis).slice(0, 1000) : '',
-      confidence: runInfo.confidence || '',
-    });
+    task.costLedger = [
+      ...existingLedger.filter(e => e && typeof e.at === 'number' && e.at >= cutoff24h),
+      {
+        at: task.lastRunAt,
+        costUsd,
+        tokens: runTokens,
+      },
+    ];
+
+    // Immutable copy-on-write history update (#301)
+    const prevRuns = this.history.get(id) || [];
+    const runs = [
+      {
+        id: 'run_' + Math.random().toString(36).slice(2, 9),
+        at: task.lastRunAt,
+        status: task.lastStatus,
+        durationMs: task.lastDurationMs,
+        output: (runInfo.output || '').slice(0, 4000),
+        error: runInfo.error ? String(runInfo.error).slice(0, 1000) : null,
+        usage,
+        costUsd,
+        sessionId: runInfo.sessionId || null,
+        // #45: which model produced the run, and whether it only finished thanks
+        // to the configured fallback.
+        model: runInfo.model || '',
+        fallback: Boolean(runInfo.fallback),
+        // #44: this run was suppressed by its silent rule, with the reason.
+        silentSkip: Boolean(runInfo.silentSkip),
+        silentReason: runInfo.silentSkip ? String(runInfo.silentReason || '').slice(0, 500) : '',
+        // #43: the failure diagnosis and the prompt change it proposes.
+        diagnosis: runInfo.diagnosis ? String(runInfo.diagnosis).slice(0, 1000) : '',
+        suggestion: runInfo.suggestion ? String(runInfo.suggestion).slice(0, 2000) : '',
+        selfHealing: runInfo.selfHealing || null,
+        autoDiagnosis: runInfo.autoDiagnosis ? String(runInfo.autoDiagnosis).slice(0, 1000) : '',
+        confidence: runInfo.confidence || '',
+      },
+      ...prevRuns,
+    ];
     // Keep up to 100 runs in active store; archive older runs to tasks-history-archive.json (#134)
     if (runs.length > 100) {
       const overflow = runs.splice(100);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-cron",
-  "version": "0.2.45",
+  "version": "0.2.47",
   "description": "Background automation runner for DSH: isolated agent runs, script/HTTP/SSH/Docker runtimes, cost guard, notifications, heartbeats.",
   "type": "module",
   "main": "lib/index.js",