@goodandready/dsh-cron 0.2.46 → 0.2.47

package/lib/store.js

@@ -1,6 +1,7 @@
 import { logger } from './logger.js';
 import { bestEffort } from './best-effort.js';
 import fs from 'node:fs';
+import { pipeline } from 'node:stream/promises';
 import path from 'node:path';
 import { getDshDefaultTelegramCredentials } from './telegram.js';
 
@@ -168,15 +169,36 @@ export class TaskStore {
    * Asynchronously serialize and write tasks store to disk using fs.promises queue (#244).
    * Safe mode 0600 on files and 0700 on dir (#222).
    */
-  async saveAsync() {
-    const version = ++this._saveVersion;
-    const data = {
+  /**
+   * Create an isolated deep snapshot of tasks, history, and settings (#301).
+   * Prevents concurrent in-memory mutations (e.g. recordRun during streamed save)
+   * from corrupting serialized JSON state or causing duplicate / skipped run records.
+   */
+  _createSnapshot() {
+    const tasksSnapshot = Array.from(this.tasks.values()).map((t) => ({
+      ...t,
+      costLedger: Array.isArray(t.costLedger) ? t.costLedger.slice() : t.costLedger,
+    }));
+    const historySnapshot = {};
+    for (const [id, runs] of this.history.entries()) {
+      historySnapshot[id] = Array.isArray(runs) ? runs.slice() : runs;
+    }
+    return {
       version: 1,
       updatedAt: new Date().toISOString(),
-      tasks: Array.from(this.tasks.values()),
-      history: Object.fromEntries(this.history.entries()),
-      settings: this.settings || {},
+      tasks: tasksSnapshot,
+      history: historySnapshot,
+      settings: this.settings ? { ...this.settings } : {},
     };
+  }
+
+  /**
+   * Asynchronously serialize and write tasks store to disk using fs.promises queue (#244, #301, #303).
+   * Safe mode 0600 on files and 0700 on dir (#222).
+   */
+  async saveAsync() {
+    const version = ++this._saveVersion;
+    const data = this._createSnapshot();
 
     const writeOp = async () => {
       if ((this._diskVersion || 0) >= version || (this._lastSyncVersion || 0) >= version) {
@@ -244,116 +266,98 @@ export class TaskStore {
    * Stream data incrementally to disk with fine-grained setImmediate yields (#244).
    * Prevents blocking the V8 single thread on heavy serialization (e.g. 21MB / 10k runs).
    */
-  _writeStoreStream(filePath, data, version) {
-    return new Promise((resolve, reject) => {
-      let aborted = false;
-      const ws = fs.createWriteStream(filePath, { encoding: 'utf-8', mode: 0o600 });
-
-      const cleanupListeners = () => {
-        ws.removeAllListeners('error');
-        ws.removeAllListeners('drain');
-      };
-
-      ws.on('error', (err) => {
-        aborted = true;
-        cleanupListeners();
-        reject(err);
-      });
+  /**
+   * Stream data incrementally to disk with fine-grained setImmediate yields (#244, #303).
+   * Prevents blocking the V8 single thread on heavy serialization (e.g. 21MB / 10k runs).
+   * Uses canonical Node stream/promises pipeline for error-handling and backpressure hygiene (#303).
+   */
+  async _writeStoreStream(filePath, data, version) {
+    const isCanceled = () => {
+      return (
+        (this._diskVersion || 0) >= version ||
+        (this._lastSyncVersion || 0) >= version ||
+        this._saveVersion > version
+      );
+    };
+
+    if (isCanceled()) {
+      return;
+    }
+
+    const ac = new AbortController();
+    let ws;
+    try {
+      ws = fs.createWriteStream(filePath, { encoding: 'utf-8', mode: 0o600 });
+    } catch (createErr) {
+      if (isCanceled()) return;
+      throw createErr;
+    }
+
+    async function* generateChunks() {
+      await new Promise((r) => setImmediate(r));
+      if (isCanceled()) {
+        ac.abort();
+        return;
+      }
+
+      yield '{"version":' + JSON.stringify(data.version || 1);
+      yield ',"updatedAt":' + JSON.stringify(data.updatedAt || new Date().toISOString());
+      yield ',"settings":' + JSON.stringify(data.settings || {});
 
-      const isCanceled = () => {
-        if ((this._diskVersion || 0) >= version || (this._lastSyncVersion || 0) >= version || this._saveVersion > version) {
-          aborted = true;
-          cleanupListeners();
-          ws.destroy();
-          return true;
+      yield ',"tasks":[';
+      const tasks = data.tasks || [];
+      for (let i = 0; i < tasks.length; i++) {
+        if (isCanceled()) {
+          ac.abort();
+          return;
         }
-        return false;
-      };
-
-      const writeChunk = (chunk) => {
-        if (aborted) return Promise.reject(new Error('Store write aborted'));
-        if (!ws.write(chunk)) {
-          return new Promise((res, rej) => {
-            const onDrain = () => {
-              ws.removeListener('drain', onDrain);
-              ws.removeListener('error', onError);
-              res();
-            };
-            const onError = (e) => {
-              ws.removeListener('drain', onDrain);
-              ws.removeListener('error', onError);
-              rej(e);
-            };
-            ws.once('drain', onDrain);
-            ws.once('error', onError);
-          });
+        if (i > 0) yield ',';
+        yield JSON.stringify(tasks[i]);
+        if ((i + 1) % 10 === 0) {
+          await new Promise((r) => setImmediate(r));
         }
-        return Promise.resolve();
-      };
+      }
+      yield ']';
 
-      const run = async () => {
-        try {
-          await new Promise((r) => setImmediate(r));
-          if (isCanceled()) return resolve();
-
-          await writeChunk('{"version":' + JSON.stringify(data.version || 1));
-          await writeChunk(',"updatedAt":' + JSON.stringify(data.updatedAt || new Date().toISOString()));
-          await writeChunk(',"settings":' + JSON.stringify(data.settings || {}));
-
-          await writeChunk(',"tasks":[');
-          const tasks = data.tasks || [];
-          for (let i = 0; i < tasks.length; i++) {
-            if (isCanceled()) return resolve();
-            if (i > 0) await writeChunk(',');
-            await writeChunk(JSON.stringify(tasks[i]));
-            if ((i + 1) % 10 === 0) {
-              await new Promise((r) => setImmediate(r));
+      yield ',"history":{';
+      const historyEntries = Object.entries(data.history || {});
+      for (let i = 0; i < historyEntries.length; i++) {
+        if (isCanceled()) {
+          ac.abort();
+          return;
+        }
+        if (i > 0) yield ',';
+        const [taskId, runs] = historyEntries[i];
+        yield JSON.stringify(taskId) + ':[';
+        if (Array.isArray(runs)) {
+          for (let j = 0; j < runs.length; j++) {
+            if (isCanceled()) {
+              ac.abort();
+              return;
             }
-          }
-          await writeChunk(']');
-
-          await writeChunk(',"history":{');
-          const historyEntries = Object.entries(data.history || {});
-          for (let i = 0; i < historyEntries.length; i++) {
-            if (isCanceled()) return resolve();
-            if (i > 0) await writeChunk(',');
-            const [taskId, runs] = historyEntries[i];
-            await writeChunk(JSON.stringify(taskId) + ':[');
-            if (Array.isArray(runs)) {
-              for (let j = 0; j < runs.length; j++) {
-                if (isCanceled()) return resolve();
-                if (j > 0) await writeChunk(',');
-                await writeChunk(JSON.stringify(runs[j]));
-                if ((j + 1) % 15 === 0) {
-                  await new Promise((r) => setImmediate(r));
-                }
-              }
-            } else {
-              await writeChunk(JSON.stringify(runs));
+            if (j > 0) yield ',';
+            yield JSON.stringify(runs[j]);
+            if ((j + 1) % 15 === 0) {
+              await new Promise((r) => setImmediate(r));
             }
-            await writeChunk(']');
-            await new Promise((r) => setImmediate(r));
-          }
-          await writeChunk('}}');
-
-          if (isCanceled()) return resolve();
-
-          ws.end(() => {
-            cleanupListeners();
-            resolve();
-          });
-        } catch (err) {
-          if (!aborted) {
-            aborted = true;
-            cleanupListeners();
-            ws.destroy();
-            reject(err);
           }
+        } else {
+          yield JSON.stringify(runs);
         }
-      };
+        yield ']';
+        await new Promise((r) => setImmediate(r));
+      }
+      yield '}}';
+    }
 
-      run();
-    });
+    try {
+      await pipeline(generateChunks(), ws, { signal: ac.signal });
+    } catch (err) {
+      if (ac.signal.aborted || isCanceled()) {
+        return;
+      }
+      throw err;
+    }
   }
 
   save() {
@@ -376,13 +380,7 @@ export class TaskStore {
         fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
       }
       bestEffort('chmod-dir', () => fs.chmodSync(dir, 0o700));
-      const data = {
-        version: 1,
-        updatedAt: new Date().toISOString(),
-        tasks: Array.from(this.tasks.values()),
-        history: Object.fromEntries(this.history.entries()),
-        settings: this.settings || {},
-      };
+      const data = this._createSnapshot();
       // Unique tmp file per save call with process PID and random token to eliminate file collision
       const rand = Math.random().toString(36).slice(2, 8);
       const tmp = `${this.filePath}.tmp.${process.pid}_${Date.now()}_${rand}`;
@@ -670,42 +668,46 @@ export class TaskStore {
     task.updatedAt = Date.now();
 
     // Rolling 24h cost ledger preserved across archive rotation and restarts (#221)
-    if (!Array.isArray(task.costLedger)) {
-      task.costLedger = [];
-    }
-    task.costLedger.push({
-      at: task.lastRunAt,
-      costUsd,
-      tokens: runTokens,
-    });
+    const existingLedger = Array.isArray(task.costLedger) ? task.costLedger : [];
     const cutoff24h = Date.now() - 86400000;
-    task.costLedger = task.costLedger.filter(e => e && typeof e.at === 'number' && e.at >= cutoff24h);
-
-    const runs = this.history.get(id) || [];
-    runs.unshift({
-      id: 'run_' + Math.random().toString(36).slice(2, 9),
-      at: task.lastRunAt,
-      status: task.lastStatus,
-      durationMs: task.lastDurationMs,
-      output: (runInfo.output || '').slice(0, 4000),
-      error: runInfo.error ? String(runInfo.error).slice(0, 1000) : null,
-      usage,
-      costUsd,
-      sessionId: runInfo.sessionId || null,
-      // #45: which model produced the run, and whether it only finished thanks
-      // to the configured fallback.
-      model: runInfo.model || '',
-      fallback: Boolean(runInfo.fallback),
-      // #44: this run was suppressed by its silent rule, with the reason.
-      silentSkip: Boolean(runInfo.silentSkip),
-      silentReason: runInfo.silentSkip ? String(runInfo.silentReason || '').slice(0, 500) : '',
-      // #43: the failure diagnosis and the prompt change it proposes.
-      diagnosis: runInfo.diagnosis ? String(runInfo.diagnosis).slice(0, 1000) : '',
-      suggestion: runInfo.suggestion ? String(runInfo.suggestion).slice(0, 2000) : '',
-      selfHealing: runInfo.selfHealing || null,
-      autoDiagnosis: runInfo.autoDiagnosis ? String(runInfo.autoDiagnosis).slice(0, 1000) : '',
-      confidence: runInfo.confidence || '',
-    });
+    task.costLedger = [
+      ...existingLedger.filter(e => e && typeof e.at === 'number' && e.at >= cutoff24h),
+      {
+        at: task.lastRunAt,
+        costUsd,
+        tokens: runTokens,
+      },
+    ];
+
+    // Immutable copy-on-write history update (#301)
+    const prevRuns = this.history.get(id) || [];
+    const runs = [
+      {
+        id: 'run_' + Math.random().toString(36).slice(2, 9),
+        at: task.lastRunAt,
+        status: task.lastStatus,
+        durationMs: task.lastDurationMs,
+        output: (runInfo.output || '').slice(0, 4000),
+        error: runInfo.error ? String(runInfo.error).slice(0, 1000) : null,
+        usage,
+        costUsd,
+        sessionId: runInfo.sessionId || null,
+        // #45: which model produced the run, and whether it only finished thanks
+        // to the configured fallback.
+        model: runInfo.model || '',
+        fallback: Boolean(runInfo.fallback),
+        // #44: this run was suppressed by its silent rule, with the reason.
+        silentSkip: Boolean(runInfo.silentSkip),
+        silentReason: runInfo.silentSkip ? String(runInfo.silentReason || '').slice(0, 500) : '',
+        // #43: the failure diagnosis and the prompt change it proposes.
+        diagnosis: runInfo.diagnosis ? String(runInfo.diagnosis).slice(0, 1000) : '',
+        suggestion: runInfo.suggestion ? String(runInfo.suggestion).slice(0, 2000) : '',
+        selfHealing: runInfo.selfHealing || null,
+        autoDiagnosis: runInfo.autoDiagnosis ? String(runInfo.autoDiagnosis).slice(0, 1000) : '',
+        confidence: runInfo.confidence || '',
+      },
+      ...prevRuns,
+    ];
     // Keep up to 100 runs in active store; archive older runs to tasks-history-archive.json (#134)
     if (runs.length > 100) {
       const overflow = runs.splice(100);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-cron",
-  "version": "0.2.46",
+  "version": "0.2.47",
   "description": "Background automation runner for DSH: isolated agent runs, script/HTTP/SSH/Docker runtimes, cost guard, notifications, heartbeats.",
   "type": "module",
   "main": "lib/index.js",