dsh-live-compact 1.1.3 → 1.1.4

package/README.md

@@ -14,13 +14,13 @@
 headless：
 
 ```sh
-dsh plugin --profile headless add dsh-live-compact@1.1.3
+dsh plugin --profile headless add dsh-live-compact@1.1.4
 ```
 
 网页端：
 
 ```sh
-dsh plugin --profile web add dsh-live-compact@1.1.3
+dsh plugin --profile web add dsh-live-compact@1.1.4
 ```
 
 想先装到本机看版本：
@@ -30,6 +30,12 @@ npm install -g dsh-live-compact
 dsh-live-compact --version
 ```
 
+## 1.1.4
+
+- 修复 #34：空闲维护的大段 catch-up 改为分批让出事件循环；Windows 阴影文件短暂忙时按 8 次上限和退避自动重试，永久失败保持 fail-closed，不阻塞轮次结束返回。
+- 游标和 retryable 结果按会话缓存，避免长历史重复扫描；工具结果、raw/meta 字节与 SHA256 在重启恢复后继续校验。
+- 本地门禁 `npm run test:gate` 通过 508/508；Windows 2022 x64 与 Windows 11 ARM 的标准用户长会话验收均通过，含 29 轮、进程重启边界和 15,001 事件/356 个工具结果的双阶段捕获恢复。
+
 ## 1.1.3
 
 - 修复 #34：工具结果事件不再同步执行 shadow/ACL 文件操作，改为事件循环后排队并按 toolCallId 去重；维护开始前会等待已排队捕获，`captureNew` 仍负责补抓和重试。
@@ -307,8 +313,8 @@ GLM 5.3 Flash 用了同一个插件、同一套题。完整跑完的 7 组为 23
 从 npm 装。headless 和网页是两个 profile，要各装一次。桌面端是另一个 profile，普通 `dsh plugin --profile desktop` 会被拒绝。
 
 ```sh
-dsh plugin --profile headless add dsh-live-compact@1.1.3
-dsh plugin --profile web add dsh-live-compact@1.1.3
+dsh plugin --profile headless add dsh-live-compact@1.1.4
+dsh plugin --profile web add dsh-live-compact@1.1.4
 ```
 
 想先装到本机看版本：
@@ -321,9 +327,9 @@ dsh-live-compact --version
 上不了 npm 时的镜像安装命令：
 
 ```sh
-dsh plugin --profile headless add https://download.merkyorlynn.com/downloads/plugins/dsh-live-compact-1.1.3.tgz
-dsh plugin --profile web add https://download.merkyorlynn.com/downloads/plugins/dsh-live-compact-1.1.3.tgz
-npm install -g --force "https://download.merkyorlynn.com/downloads/plugins/dsh-live-compact-1.1.3.tgz"
+dsh plugin --profile headless add https://download.merkyorlynn.com/downloads/plugins/dsh-live-compact-1.1.4.tgz
+dsh plugin --profile web add https://download.merkyorlynn.com/downloads/plugins/dsh-live-compact-1.1.4.tgz
+npm install -g --force "https://download.merkyorlynn.com/downloads/plugins/dsh-live-compact-1.1.4.tgz"
 dsh-live-compact --version
 ```
 

package/index.js

@@ -53,6 +53,10 @@ const URGENT_TOKENS = 48000;
 const MAX_TAIL = 2200;
 const MAX_TRACKED_SESSIONS = 64;
 const DEFAULT_KEY = "__default__";
+const CAPTURE_EVENT_BATCH = 128;
+const CAPTURE_RESULT_BATCH = 1;
+const RECOVER_RAW_BATCH = 1;
+const CAPTURE_RETRY_LIMIT = 8;
 
 function eventContent(session, seq) {
   const event = session.eventAt(seq);
@@ -461,7 +465,14 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
   callIndex = new Map();
   captureCursor = new Map();
   pendingCaptures = new Set();
+  pendingCaptureJobs = new Map();
   pendingCaptureKeys = new Set();
+  pendingCatchups = new Map();
+  pendingCatchupJobs = new Set();
+  pendingCuts = new Set();
+  captureRetryStats = new Map();
+  retryableCaptureSeqs = new Map();
+  retryableCaptureSeeded = new Set();
   pruneGuarded = false;
 
   constructor(ctx, config, resolvedConfig) {
@@ -471,9 +482,8 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
     if (process.env.SPINE_ENGINE_FAIL === "after-super") throw new Error("spine: after super");
     this.liveConfig = config;
     try {
-      const pendingCuts = new Set();
       ctx.effect(() => async () => {
-        await Promise.allSettled([...pendingCuts, ...this.pendingCaptures]);
+        await Promise.allSettled([...this.pendingCuts, ...this.pendingCaptures, ...this.pendingCatchupJobs]);
       });
       ctx.on("session/event", (session, event) => {
         try {
@@ -491,17 +501,22 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
             scheduleImmediate(() => {
               try {
                 const current = this.rawSeen.get(key);
-                if (!current || current.retryable) this.captureOne(session, event);
+                if (!current || current.retryable) {
+                  const record = this.captureOne(session, event);
+                  this.noteCaptureResult(session, event, record);
+                }
               } catch (error) {
                 noteShadowWarning(ctx.logger, error);
               } finally {
                 this.pendingCaptureKeys.delete(key);
                 this.pendingCaptures.delete(job);
+                if (this.pendingCaptureJobs.get(key) === job) this.pendingCaptureJobs.delete(key);
                 resolve();
               }
             });
           });
           this.pendingCaptures.add(job);
+          this.pendingCaptureJobs.set(key, job);
         } catch (error) {
           noteShadowWarning(ctx.logger, error);
         }
@@ -514,13 +529,13 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
         const done = events.some((event) => event.type === "turn/end" && event.data?.reason?.kind === "completed");
         if (!done) return;
         state.warming = true;
-        const job = this.finishTurn(agent, events)
+        const job = this.finishTurn(agent, events, { deferCapture: true })
           .catch((error) => noteShadowWarning(this.ctx?.logger, error))
           .finally(() => {
             state.warming = false;
-            pendingCuts.delete(job);
+            this.pendingCuts.delete(job);
           });
-        pendingCuts.add(job);
+        this.pendingCuts.add(job);
       });
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
@@ -570,13 +585,167 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
     return planMessages(messages);
   }
 
-  async finishTurn(agent, events) {
+  scheduleCaptureCatchup(agent, events, settings) {
+    this.pendingCatchups ||= new Map();
+    this.pendingCatchupJobs ||= new Set();
+    this.pendingCuts ||= new Set();
+    const key = sessionKey(agent);
+    let task = this.pendingCatchups.get(key);
+    if (task) {
+      task.agent = agent;
+      task.session = agent.session;
+      task.events = events;
+      task.settings = settings;
+      return task;
+    }
+    let resolve;
+    const promise = new Promise((done) => { resolve = done; });
+    task = {
+      key,
+      agent,
+      session: agent.session,
+      events,
+      settings,
+      phase: "events",
+      eventIndex: 0,
+      recoverIndex: 0,
+      visibleIds: null,
+      scheduled: false,
+      retryRequested: false,
+      captureRetryAttempts: 0,
+      progressed: false,
+      eventTail: Number.isInteger(events?.at?.(-1)?.seq) ? events.at(-1).seq : -1,
+      surfaceTail: agent.session.surface?.nodes?.at?.(-1) ?? -1,
+      resolve,
+      promise,
+    };
+    this.pendingCatchups.set(key, task);
+    this.pendingCatchupJobs.add(promise);
+    this.queueCaptureCatchup(task);
+    return task;
+  }
+
+  queueCaptureRetry(task) {
+    if (!task || task.finished || task.scheduled) return;
+    task.scheduled = true;
+    const delay = Math.min(250, 10 * (2 ** Math.max(0, task.captureRetryAttempts - 1)));
+    setTimeout(() => {
+      task.scheduled = false;
+      if (!task.finished) this.queueCaptureCatchup(task);
+    }, delay);
+  }
+
+  queueCaptureCatchup(task) {
+    if (!task || task.scheduled) return;
+    task.scheduled = true;
+    scheduleImmediate(() => {
+      task.scheduled = false;
+      try {
+        const latest = task.session.snapshotEvents?.();
+        const latestTail = Number.isInteger(latest?.at?.(-1)?.seq) ? latest.at(-1).seq : -1;
+        const latestSurfaceTail = task.session.surface?.nodes?.at?.(-1) ?? -1;
+        if (Array.isArray(latest) && (latestTail !== task.eventTail || latest.length !== task.events.length)) {
+          task.events = latest;
+          task.eventTail = latestTail;
+          if (task.phase === "visible") {
+            task.phase = "events";
+            task.visibleIds = null;
+            task.recoverIndex = 0;
+          }
+        }
+        if (latestSurfaceTail !== task.surfaceTail) {
+          task.surfaceTail = latestSurfaceTail;
+          if (task.phase === "visible") {
+            task.visibleIds = [...new Set(this.surfaceToolIds(task.session))];
+            task.recoverIndex = 0;
+          }
+        }
+        if (task.phase === "events") {
+          const result = this.captureNewBatch(task.session, task.events, task.settings, {
+            eventLimit: CAPTURE_EVENT_BATCH,
+            resultLimit: CAPTURE_RESULT_BATCH,
+            startIndex: task.eventIndex,
+          });
+          task.eventIndex = result.nextIndex;
+          task.progressed ||= result.progressed;
+          if (result.retryable) {
+            task.captureRetryAttempts += 1;
+            const previousMax = this.captureRetryStats?.get(task.key) || 0;
+            if (task.captureRetryAttempts > previousMax) this.captureRetryStats.set(task.key, task.captureRetryAttempts);
+            if (task.captureRetryAttempts >= CAPTURE_RETRY_LIMIT) {
+              this.finishCaptureCatchup(task);
+              return;
+            }
+            this.queueCaptureRetry(task);
+            return;
+          }
+          if (result.progressed) task.captureRetryAttempts = 0;
+          if (!result.done) {
+            this.queueCaptureCatchup(task);
+            return;
+          }
+          task.phase = "visible";
+          task.recoverIndex = 0;
+          task.visibleIds = [...new Set(this.surfaceToolIds(task.session))];
+        }
+        const recovered = this.recoverVisibleRawBatch(task.session, {}, task.recoverIndex, RECOVER_RAW_BATCH, task.visibleIds);
+        task.progressed ||= recovered.progressed;
+        task.recoverIndex = recovered.nextIndex;
+        if (!recovered.done) {
+          this.queueCaptureCatchup(task);
+          return;
+        }
+        this.finishCaptureCatchup(task);
+      } catch (error) {
+        noteShadowWarning(this.ctx?.logger, error);
+        this.finishCaptureCatchup(task);
+      }
+    });
+  }
+
+  finishCaptureCatchup(task) {
+    if (!task || task.finished) return;
+    task.finished = true;
+    this.pendingCatchups.delete(task.key);
+    this.pendingCatchupJobs.delete(task.promise);
+    task.resolve();
+    if (task.retryRequested && task.progressed) this.scheduleMaintenanceRetry(task.agent);
+  }
+
+  scheduleMaintenanceRetry(agent) {
+    const state = this.stateFor(agent);
+    if (state.retryScheduled) return;
+    state.retryScheduled = true;
+    scheduleImmediate(() => {
+      state.retryScheduled = false;
+      if (state.warming) return;
+      const events = agent.session.snapshotEvents();
+      const done = events.some((event) => event.type === "turn/end" && event.data?.reason?.kind === "completed");
+      if (!done) return;
+      state.warming = true;
+      const job = this.finishTurn(agent, events, { deferCapture: true })
+        .catch((error) => noteShadowWarning(this.ctx?.logger, error))
+        .finally(() => {
+          state.warming = false;
+          this.pendingCuts.delete(job);
+        });
+      this.pendingCuts.add(job);
+    });
+  }
+
+  async finishTurn(agent, events, options = {}) {
     const settings = readSettings(this.liveConfig);
     const state = this.stateFor(agent);
     const signal = AbortSignal.timeout(30000);
     const ev = events || agent.session.snapshotEvents();
-    await this.flushPendingCaptures();
-    this.captureNew(agent.session, ev, settings);
+    const deferCapture = options.deferCapture === true;
+    const catchup = deferCapture
+      ? this.scheduleCaptureCatchup(agent, ev, settings)
+      : { finished: true, progressed: false, retryRequested: false };
+    if (!deferCapture) {
+      await this.flushPendingCaptures();
+      this.captureNew(agent.session, ev, settings);
+    }
     const jobs = collectJobs(ev);
     const calls = indexToolCalls(ev);
     try {
@@ -636,6 +805,7 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
           try {
             state.localOnly = true;
             state.approvedCard = chunk.card;
+            if (deferCapture) await this.flushPendingCaptures(agent.session, this.chunkToolCallIds(agent.session, chunk));
             await this.commitSelectedSpan(agent, chunk, op);
             measurement = this.ctx.tokenMeter.measure(agent.session);
             const after = measurement.totalTokens;
@@ -656,6 +826,10 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
             this.ctx.logger?.info?.(`spine cut ${Date.now() - started}ms ${before} -> ${after} chunk=${chunk.tokens} saved=${saved}`);
           } catch (error) {
             const message = error instanceof Error ? error.message : String(error);
+            if (message.includes("raw backup missing before cut")) {
+              catchup.retryRequested = true;
+              if (catchup.finished && catchup.progressed) this.scheduleMaintenanceRetry(agent);
+            }
             this.ctx.logger?.warn?.(`spine cut failed: ${message}`);
             break;
           }
@@ -678,9 +852,33 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
     }
   }
 
-  async flushPendingCaptures() {
-    if (!this.pendingCaptures?.size) return;
-    await Promise.allSettled([...this.pendingCaptures]);
+  async flushPendingCaptures(session = null, ids = null) {
+    if (!session) {
+      if (!this.pendingCaptures?.size) return;
+      await Promise.allSettled([...this.pendingCaptures]);
+      return;
+    }
+    const key = session.id || DEFAULT_KEY;
+    const pendingIds = ids || this.surfaceToolIds(session);
+    const jobs = [...new Set(pendingIds)]
+      .map((id) => this.pendingCaptureJobs?.get?.(`${key}\0${id}`))
+      .filter(Boolean);
+    if (jobs.length) await Promise.allSettled(jobs);
+  }
+
+  chunkToolCallIds(session, chunk) {
+    const nodes = session?.surface?.nodes || [];
+    const start = nodes.indexOf(chunk?.start);
+    const end = nodes.indexOf(chunk?.end);
+    if (start < 0 || end < start) return [];
+    const ids = [];
+    for (const seq of nodes.slice(start, end + 1)) {
+      const event = session.eventAt?.(seq);
+      if (event?.type !== "tool/result") continue;
+      const id = event.data?.message?.toolCallId;
+      if (id) ids.push(id);
+    }
+    return ids;
   }
 
   async commitSelectedSpan(agent, chunk, signal) {
@@ -716,7 +914,8 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
       const event = session.eventAt(seq);
       if (event?.type === "tool/result") {
         const id = event.data?.message?.toolCallId;
-        if (!id || !verifiedRaw(this.rawSeen.get(this.rawKey(session, id))).ok) {
+        const raw = id ? this.rawSeen.get(this.rawKey(session, id)) : null;
+        if (!id || !verifiedRaw(raw).ok) {
           throw new Error(`spine: raw backup missing before cut: ${id || "unknown"}`);
         }
       }
@@ -972,19 +1171,109 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
     }
   }
 
-  captureNew(session, events, settings = readSettings(this.liveConfig)) {
-    if (!Array.isArray(events)) events = [];
+  noteCaptureResult(session, event, record) {
+    const key = session?.id || DEFAULT_KEY;
+    this.retryableCaptureSeqs ||= new Map();
+    this.retryableCaptureSeeded ||= new Set();
+    let seqs = this.retryableCaptureSeqs.get(key);
+    if (!seqs) {
+      seqs = new Set();
+      this.retryableCaptureSeqs.set(key, seqs);
+    }
+    const seq = Number.isInteger(event?.seq) ? event.seq : null;
+    if (seq === null) return;
+    if (record?.retryable) seqs.add(seq);
+    else seqs.delete(seq);
+  }
+
+  captureCursorStart(session, events, settings) {
     const key = session?.id || DEFAULT_KEY;
     let cursor = this.captureCursor.has(key) ? this.captureCursor.get(key) : this.readCaptureCursor(key);
     if (cursor == null) cursor = restartCaptureCursor(events, settings.keepRounds);
     if (!Number.isSafeInteger(cursor) || cursor < 0 || events.length < cursor) cursor = restartCaptureCursor(events, settings.keepRounds);
-    // A live event callback may have failed before the recent-three restart
-    // boundary. Keep those visible originals eligible without replaying history.
-    for (const seq of session?.surface?.nodes || []) {
-      const event = session.eventAt?.(seq);
-      const id = event?.type === "tool/result" && event.data?.message?.toolCallId;
-      if (id && this.rawSeen.get(this.rawKey(session, id))?.retryable) cursor = Math.min(cursor, seq);
+    this.retryableCaptureSeqs ||= new Map();
+    this.retryableCaptureSeeded ||= new Set();
+    let retryable = this.retryableCaptureSeqs.get(key);
+    if (!retryable) {
+      retryable = new Set();
+      this.retryableCaptureSeqs.set(key, retryable);
+    }
+    if (!this.retryableCaptureSeeded.has(key)) {
+      for (const seq of session?.surface?.nodes || []) {
+        const event = session.eventAt?.(seq);
+        const id = event?.type === "tool/result" && event.data?.message?.toolCallId;
+        if (id && this.rawSeen.get(this.rawKey(session, id))?.retryable) retryable.add(seq);
+      }
+      this.retryableCaptureSeeded.add(key);
+    }
+    if (retryable.size) cursor = Math.min(cursor, Math.min(...retryable));
+    return cursor;
+  }
+
+  captureNewBatch(session, events, settings = readSettings(this.liveConfig), limits = {}) {
+    if (!Array.isArray(events)) events = [];
+    const key = session?.id || DEFAULT_KEY;
+    const cursor = this.captureCursorStart(session, events, settings);
+    let next = cursor;
+    let retry = null;
+    let processed = 0;
+    let captures = 0;
+    let progressed = false;
+    let exhausted = true;
+    const seen = new Set();
+    const eventLimit = Math.max(1, limits.eventLimit || CAPTURE_EVENT_BATCH);
+    const resultLimit = Math.max(1, limits.resultLimit || CAPTURE_RESULT_BATCH);
+    let index = Math.max(0, Number.isInteger(limits.startIndex) ? limits.startIndex : 0);
+    const indexedSeq = Number.isInteger(events[index]?.seq) ? events[index].seq : null;
+    if (index > 0 && indexedSeq !== null && indexedSeq > cursor) {
+      const rewind = events.findIndex((event) => Number.isInteger(event?.seq) && event.seq >= cursor);
+      index = rewind >= 0 ? rewind : 0;
+    }
+    for (; index < events.length; index += 1) {
+      const event = events[index];
+      const seq = Number.isInteger(event?.seq) ? event.seq : next;
+      if (seq < cursor) continue;
+      this.noteEvent(session, event);
+      if (event?.type === "tool/result") {
+        const id = event.data?.message?.toolCallId;
+        if (id && !seen.has(id)) {
+          seen.add(id);
+          const cached = this.rawSeen.get(this.rawKey(session, id));
+          const record = cached && !cached.retryable ? cached : this.captureOne(session, event);
+          if (!cached || cached.retryable) {
+            if (record?.ok) progressed = true;
+            captures += 1;
+          }
+          this.noteCaptureResult(session, event, record);
+          if (record?.retryable && retry === null) retry = seq;
+        }
+      }
+      next = seq + 1;
+      processed += 1;
+      if (retry !== null || processed >= eventLimit || captures >= resultLimit) {
+        exhausted = false;
+        index += 1;
+        break;
+      }
     }
+    next = retry ?? next;
+    this.captureCursor.set(key, next);
+    this.writeCaptureCursor(key, next);
+    const lastEvent = events.at(-1);
+    const lastSeq = Number.isInteger(lastEvent?.seq) ? lastEvent.seq : events.length - 1;
+    return {
+      seen,
+      progressed,
+      retryable: retry !== null,
+      done: exhausted && retry === null && next > lastSeq,
+      nextIndex: index,
+    };
+  }
+
+  captureNew(session, events, settings = readSettings(this.liveConfig)) {
+    if (!Array.isArray(events)) events = [];
+    const key = session?.id || DEFAULT_KEY;
+    const cursor = this.captureCursorStart(session, events, settings);
     let next = cursor;
     let retry = null;
     const seen = new Set();
@@ -998,6 +1287,7 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
           seen.add(id);
           const cached = this.rawSeen.get(this.rawKey(session, id));
           const record = cached && !cached.retryable ? cached : this.captureOne(session, event);
+          this.noteCaptureResult(session, event, record);
           if (record?.retryable && retry === null) retry = seq;
         }
       }
@@ -1075,6 +1365,29 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
     }
   }
 
+  recoverVisibleRawBatch(session, access = {}, startIndex = 0, limit = RECOVER_RAW_BATCH, visibleIds = null) {
+    const ids = visibleIds || [...new Set(this.surfaceToolIds(session))];
+    const dir = sessionDir(shadowDirectory(), session?.id || DEFAULT_KEY);
+    let index = Math.max(0, startIndex);
+    let processed = 0;
+    let progressed = false;
+    while (index < ids.length && processed < Math.max(1, limit)) {
+      const id = ids[index++];
+      const mapKey = this.rawKey(session, id);
+      if (this.rawSeen.has(mapKey)) continue;
+      try {
+        const record = readRawToolResult({ dir, toolCallId: id, access });
+        this.rawSeen.set(mapKey, record);
+        progressed ||= Boolean(record?.ok);
+      } catch (error) {
+        noteShadowWarning(this.ctx?.logger, error);
+        this.rawSeen.set(mapKey, { ok: false, reason: "unwritable", toolCallId: id });
+      }
+      processed += 1;
+    }
+    return { nextIndex: index, done: index >= ids.length, progressed };
+  }
+
   captureSession(session) {
     return this.captureNew(session, session?.snapshotEvents?.() || []);
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-live-compact",
-  "version": "1.1.3",
+  "version": "1.1.4",
   "description": "In-place newest-turn compaction for DeepSeek Harness. Compress while the task runs and keep the prefix.",
   "license": "MIT",
   "author": "MerkyorLynn <268568828+MerkyorLynn@users.noreply.github.com>",
@@ -41,7 +41,7 @@
     "compaction"
   ],
   "scripts": {
-    "test:gate": "node test/p0-gate.mjs && node test/p0-blockers.mjs && node test/excerpt.mjs && node --test test/public-exports.test.mjs test/cooldown.test.mjs test/gate-config.test.mjs test/drain-stopline.mjs test/scan-once.mjs test/issue-19.mjs test/issue-20.mjs test/windows-acl.test.mjs test/issue-27.mjs test/issue-29.mjs test/issue-29-notice.mjs test/client-settings.test.mjs test/settings-integration.test.mjs test/runtime-lifecycle.mjs test/shadow-lifecycle.mjs test/finalize-over-32000.mjs",
+    "test:gate": "node test/p0-gate.mjs && node test/p0-blockers.mjs && node test/excerpt.mjs && node --test test/public-exports.test.mjs test/cooldown.test.mjs test/gate-config.test.mjs test/drain-stopline.mjs test/scan-once.mjs test/issue-19.mjs test/issue-20.mjs test/issue-34.mjs test/windows-acl.test.mjs test/issue-27.mjs test/issue-29.mjs test/issue-29-notice.mjs test/client-settings.test.mjs test/settings-integration.test.mjs test/runtime-lifecycle.mjs test/shadow-lifecycle.mjs test/finalize-over-32000.mjs",
     "test:live-gate": "python3 -B -m unittest discover -s test/live-fill -p test_*.py"
   },
   "engines": {