dsh-live-compact 1.1.3 → 1.1.5

package/index.js

@@ -16,28 +16,26 @@ import { randomBytes, randomUUID } from "node:crypto";
 import { closeSync, existsSync, openSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
 import { createRequire } from "node:module";
 import { setImmediate as scheduleImmediate } from "node:timers";
+import { Worker } from "node:worker_threads";
 import { dirname, join } from "node:path";
 import { pathToFileURL } from "node:url";
 import { loadBase, loadCompaction, loadLlm } from "./adapter.js";
 import { DEFAULTS, engineConfig, readSettings } from "./settings.js";
-import { attachBackupHint, backgroundJobId, backupClaimOk, buildCard, collectJobs, cutLongReadRound, finishedToolRounds, indexToolCalls, jobCutDecision, keptToolRounds, messageText, planMessages, pointerOnlyText, pointerSpanAllowed, RAIL_TOKENS, readResultLabel, reservedCutEnd, resolveEngine, resolveProviderBaseURL, roundHasLongRead, selectCutIndexes, shadowSections, takeLocalCut, withoutFullBackupClaim } from "./plan.js";
-import { captureOwnedToolResult, contentHash, rawExcerptText, readRawToolResult, restoreToolText, runPruneAfterCapture, verifiedRaw } from "./raw.js";
+import { attachBackupHint, backgroundJobId, backupClaimOk, buildCard, collectJobs, cutLongReadRound, finishedToolRounds, indexToolCalls, isToolFailure, jobCutDecision, keptToolRounds, messageText, planMessages, pointerOnlyText, pointerSpanAllowed, RAIL_TOKENS, readResultLabel, reservedCutEnd, resolveEngine, resolveProviderBaseURL, roundHasLongRead, selectCutIndexes, shadowSections, takeLocalCut, withoutFullBackupClaim } from "./plan.js";
+import { captureOwnedToolResult, contentHash, rawExcerptText, rawName, readRawToolResult, restoreToolText, runPruneAfterCapture, verifiedRaw } from "./raw.js";
 import {
-  discardShadowFiles,
   ensureShadowDir,
   flushDrainStats,
   noteShadowWarning,
-  rotateShadow,
+  saveShadowTransaction,
   shadowDirectory,
-  sessionDir,
-  shadowAccessFused,
   shadowFileBody,
   shadowMaxBytes,
+  sessionDir,
+  shadowAccessFused,
+  shadowAccessFuseBuffer,
   tightenCreatedShadow,
   tightenShadow,
-  writeShadowFile,
-  writeShadowOwner,
-  withShadowRoot,
 } from "./shadow.js";
 import { buildSendLess, decideThunder } from "./thunder.js";
 import { inCooldown, skipLiveOfficialCompaction } from "./cooldown.js";
@@ -53,6 +51,104 @@ const URGENT_TOKENS = 48000;
 const MAX_TAIL = 2200;
 const MAX_TRACKED_SESSIONS = 64;
 const DEFAULT_KEY = "__default__";
+const CAPTURE_EVENT_BATCH = 128;
+const CAPTURE_RESULT_BATCH = 1;
+const RECOVER_RAW_BATCH = 1;
+const CAPTURE_RETRY_LIMIT = 8;
+const RECOVERY_ACL_REAUTH_LIMIT = 32;
+
+const shadowWorkers = new Map();
+
+function shadowWorkerFor(root) {
+  const key = String(root);
+  let entry = shadowWorkers.get(key);
+  if (entry) return entry;
+  const worker = new Worker(new URL("./shadow-save-worker.mjs", import.meta.url), {
+    workerData: { spineWorker: 1, accessFuse: shadowAccessFuseBuffer() },
+  });
+  entry = { worker, nextId: 1, pending: new Map(), idleTimer: null, retired: false };
+  const removeEntry = () => {
+    if (shadowWorkers.get(key) === entry) shadowWorkers.delete(key);
+  };
+  const scheduleIdleExit = () => {
+    if (entry.retired || entry.pending.size || entry.idleTimer) return;
+    entry.idleTimer = setTimeout(() => {
+      entry.idleTimer = null;
+      if (entry.pending.size) return;
+      entry.retired = true;
+      removeEntry();
+      worker.terminate().catch(() => {});
+    }, 500);
+    entry.idleTimer.unref?.();
+  };
+  entry.scheduleIdleExit = scheduleIdleExit;
+  const fail = (error) => {
+    entry.retired = true;
+    if (entry.idleTimer) {
+      clearTimeout(entry.idleTimer);
+      entry.idleTimer = null;
+    }
+    for (const { reject } of entry.pending.values()) reject(error);
+    entry.pending.clear();
+    removeEntry();
+    worker.terminate().catch(() => {});
+  };
+  worker.on("message", (message) => {
+    const task = entry.pending.get(message?.id);
+    if (!task) return;
+    entry.pending.delete(message.id);
+    if (message?.ok) {
+      task.resolve(message.result);
+      scheduleIdleExit();
+      return;
+    }
+    const error = new Error(message?.error?.message || "spine: shadow worker failed");
+    if (message?.error?.code) error.code = message.error.code;
+    task.reject(error);
+    scheduleIdleExit();
+  });
+  worker.on("error", (error) => fail(error));
+  worker.on("exit", (code) => {
+    if (entry.pending.size || (!entry.retired && code !== 0)) {
+      fail(Object.assign(new Error(`spine: shadow worker exited ${code}`), { code: "ESHADOWWORKER" }));
+    } else {
+      entry.retired = true;
+      if (entry.idleTimer) clearTimeout(entry.idleTimer);
+      entry.idleTimer = null;
+      removeEntry();
+    }
+  });
+  shadowWorkers.set(key, entry);
+  return entry;
+}
+
+function runShadowWorker(payload) {
+  const entry = shadowWorkerFor(payload.shadow);
+  if (entry.idleTimer) {
+    clearTimeout(entry.idleTimer);
+    entry.idleTimer = null;
+  }
+  const id = entry.nextId++;
+  return new Promise((resolve, reject) => {
+    entry.pending.set(id, { resolve, reject });
+    try {
+      entry.worker.postMessage({ id, payload });
+    } catch (error) {
+      entry.pending.delete(id);
+      reject(error);
+      entry.scheduleIdleExit();
+    }
+  });
+}
+
+function recoveryRetryable(record, dir, toolCallId) {
+  if (record?.retryable === true || record?.reason === "unwritable") return true;
+  // A completely absent backup has never been captured and can be retried.
+  // If its metadata is still present, the body was damaged after capture and
+  // must remain fail-closed rather than being silently overwritten.
+  if (record?.reason === "missing-file") return !existsSync(join(dir, `${rawName(toolCallId)}.meta.json`));
+  return false;
+}
 
 function eventContent(session, seq) {
   const event = session.eventAt(seq);
@@ -180,80 +276,41 @@ export function saveShadowImpl(
   rotationIO = {},
 ) {
   const shadow = shadowDirectory();
-  ensureShadowDir(shadow, access);
-
   const file = shadowFilePath(shadowFile, warnKey);
-  const dir = sessionDir(shadow, warnKey);
-  ensureShadowDir(dir, access);
-
-  return withShadowRoot(shadow, () => {
-    const owner = join(dir, "owner");
-    const ownerExisted = existsSync(owner);
-    let ownerWritten = false;
-
-    try {
-      writeShadowOwner(dir, access, shadow);
-      ownerWritten = true;
-
-      const body = shadowFileBody(warnKey, shadowBody(messages));
-      const incoming = Buffer.byteLength(body);
-      const cap = shadowMaxBytes();
-
-      const rotation = rotateShadow({
-        ...rotationIO,
-        dir: shadow,
-        incomingBytes: incoming,
-        protectPath: file,
-        protectPaths,
-        sessionKey: warnKey,
-        cap,
-      });
-
-      if (!rotation.ok) {
-        if (rotation.reason) {
-          const incomplete = rotation.reason === "scan-incomplete";
-          const error = Object.assign(new Error(incomplete
-            ? "spine: shadow capacity scan is incomplete; backup refused; leaving the span"
-            : "spine: shadow capacity measurement is invalid; backup refused; leaving the span"), {
-            code: incomplete ? "ESHADOWSCAN" : "ESHADOWMETER",
-          });
-          if (required) throw error;
-          this.ctx.logger?.warn?.(error.message);
-          return;
-        }
-        this.warnShadowFull(warnKey, shadow, rotation.used, cap);
-        if (required) {
-          throw new Error(
-            "spine: shadow backup is full; leaving the span",
-          );
-        }
-        return;
-      }
-
-      if (rotation.deleted > 0) {
-        this.ctx.logger?.info?.(
-          `spine: shadow 轮换 删除 ${rotation.deleted} 个文件，` +
-          `腾出 ${rotation.freed} 字节，` +
-          `用量 ${rotation.used + incoming}/${cap}`,
-        );
-      }
-
-      if (rotation.oversize) {
-        this.warnShadowOversize(warnKey, cap);
-      }
-
-      writeShadowFile(file, body, access, { root: shadow });
-
-      if (!existsSync(file)) {
-        throw new Error("spine: shadow backup was not written");
-      }
-    } catch (error) {
-      if (ownerWritten && !ownerExisted) {
-        discardShadowFiles([owner], error);
-      }
-      throw error;
-    }
+  const body = shadowFileBody(warnKey, shadowBody(messages));
+  const cap = shadowMaxBytes();
+  const result = saveShadowTransaction({
+    shadow,
+    file,
+    body,
+    required,
+    protectPaths,
+    sessionKey: warnKey,
+    cap,
+    access,
+    rotationIO,
   });
+  if (!result?.ok) {
+    if (result?.reason === "scan-incomplete" || result?.reason === "invalid-measurement") {
+      const error = result.reason === "scan-incomplete"
+        ? "spine: shadow capacity scan is incomplete; backup refused; leaving the span"
+        : "spine: shadow capacity measurement is invalid; backup refused; leaving the span";
+      this.ctx.logger?.warn?.(error);
+    } else if (result?.rotation) {
+      this.warnShadowFull(warnKey, shadow, result.rotation.used, cap);
+    }
+    return result;
+  }
+  const rotation = result.rotation;
+  if (rotation.deleted > 0) {
+    this.ctx.logger?.info?.(
+      `spine: shadow 轮换 删除 ${rotation.deleted} 个文件，` +
+      `腾出 ${rotation.freed} 字节，` +
+      `用量 ${rotation.used + Buffer.byteLength(body)}/${cap}`,
+    );
+  }
+  if (rotation.oversize) this.warnShadowOversize(warnKey, cap);
+  return result;
 }
 
 export { saveShadowImpl as saveShadow };
@@ -300,9 +357,11 @@ function selectNewestChunk(session, measurement, key = DEFAULT_KEY, context = {}
       type: event?.type === "tool/result" ? "tool" : "other",
       toolCalls,
       longRead: event?.type === "tool/result" && call?.name === "read" && (priced[index].tokens || 0) >= settings.chunkMin,
+      checkpoint: isCheckpoint(session, nodes[index]),
     });
   }
-  const rounds = keptToolRounds(visible, settings.keepRounds);
+  const floorVisible = visible.reduce((found, item, index) => item.checkpoint ? index : found, -1);
+  const rounds = keptToolRounds(visible, settings.keepRounds, floorVisible);
   const newestLong = [...finishedToolRounds(visible)].reverse().find((round) => roundHasLongRead(visible, round));
   const span = rounds.length ? { start: rounds[0].start, end: rounds.at(-1).end } : null;
   const cutEnd = span ? reservedCutEnd(visible[span.start].index, first) : null;
@@ -361,7 +420,6 @@ function selectNewestChunk(session, measurement, key = DEFAULT_KEY, context = {}
     }
     return pieces;
   };
-  const spanText = (spanStart, spanEnd) => spanPieces(spanStart, spanEnd).map((piece) => piece.text).join("\n");
   const liveText = (spanStart, spanEnd) => {
     const bits = [];
     for (let index = 0; index < nodes.length; index += 1) {
@@ -433,9 +491,11 @@ function selectNewestChunk(session, measurement, key = DEFAULT_KEY, context = {}
   const start = picked.start;
   end = picked.end;
   const tokens = items.slice(start, end + 1).reduce((sum, item) => sum + item.tokens, 0);
+  const pieces = spanPieces(start, end);
+  if (picked.pointerOnly && pieces.some((piece) => piece.role === "tool" && isToolFailure(piece.text, piece.isError))) return null;
   let card = picked.pointerOnly
-    ? { ok: true, text: pointerOnlyText(spanPieces(start, end)), hardLines: [], missing: [], needsBackup: false, rawPaths: [] }
-    : buildCard(spanText(start, end), liveText(start, end), spanPieces(start, end), extras);
+    ? { ok: true, text: pointerOnlyText(pieces), hardLines: [], missing: [], needsBackup: false, rawPaths: [] }
+    : buildCard(pieces.map((piece) => piece.text).join("\n"), liveText(start, end), pieces, extras);
   if (card.ok !== true) return null;
   card = attachBackupHint(card, freshShadowFile("cut", key));
   if (!card.text.includes("不是备份文件路径")) card = { ...card, text: `${card.text}\n序号是本会话日志里的事件号，不是备份文件路径。` };
@@ -461,7 +521,15 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
   callIndex = new Map();
   captureCursor = new Map();
   pendingCaptures = new Set();
+  pendingCaptureJobs = new Map();
   pendingCaptureKeys = new Set();
+  pendingCatchups = new Map();
+  pendingCatchupJobs = new Set();
+  pendingCuts = new Set();
+  captureRetryStats = new Map();
+  retryableCaptureSeqs = new Map();
+  retryableCaptureSeeded = new Set();
+  captureRecoveryHolds = new Map();
   pruneGuarded = false;
 
   constructor(ctx, config, resolvedConfig) {
@@ -471,9 +539,8 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
     if (process.env.SPINE_ENGINE_FAIL === "after-super") throw new Error("spine: after super");
     this.liveConfig = config;
     try {
-      const pendingCuts = new Set();
       ctx.effect(() => async () => {
-        await Promise.allSettled([...pendingCuts, ...this.pendingCaptures]);
+        await Promise.allSettled([...this.pendingCuts, ...this.pendingCaptures, ...this.pendingCatchupJobs]);
       });
       ctx.on("session/event", (session, event) => {
         try {
@@ -491,17 +558,22 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
             scheduleImmediate(() => {
               try {
                 const current = this.rawSeen.get(key);
-                if (!current || current.retryable) this.captureOne(session, event);
+                if (!current || current.retryable) {
+                  const record = this.captureOne(session, event);
+                  this.noteCaptureResult(session, event, record);
+                }
               } catch (error) {
                 noteShadowWarning(ctx.logger, error);
               } finally {
                 this.pendingCaptureKeys.delete(key);
                 this.pendingCaptures.delete(job);
+                if (this.pendingCaptureJobs.get(key) === job) this.pendingCaptureJobs.delete(key);
                 resolve();
               }
             });
           });
           this.pendingCaptures.add(job);
+          this.pendingCaptureJobs.set(key, job);
         } catch (error) {
           noteShadowWarning(ctx.logger, error);
         }
@@ -514,13 +586,13 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
         const done = events.some((event) => event.type === "turn/end" && event.data?.reason?.kind === "completed");
         if (!done) return;
         state.warming = true;
-        const job = this.finishTurn(agent, events)
+        const job = this.finishTurn(agent, events, { deferCapture: true })
           .catch((error) => noteShadowWarning(this.ctx?.logger, error))
           .finally(() => {
             state.warming = false;
-            pendingCuts.delete(job);
+            this.pendingCuts.delete(job);
           });
-        pendingCuts.add(job);
+        this.pendingCuts.add(job);
       });
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
@@ -570,13 +642,184 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
     return planMessages(messages);
   }
 
-  async finishTurn(agent, events) {
+  scheduleCaptureCatchup(agent, events, settings, options = {}) {
+    this.pendingCatchups ||= new Map();
+    this.pendingCatchupJobs ||= new Set();
+    this.pendingCuts ||= new Set();
+    const key = sessionKey(agent);
+    let task = this.pendingCatchups.get(key);
+    if (task) {
+      task.agent = agent;
+      task.session = agent.session;
+      task.events = events;
+      task.settings = settings;
+      task.freezeSnapshot ||= options.freezeSnapshot === true;
+      return task;
+    }
+    let resolve;
+    const promise = new Promise((done) => { resolve = done; });
+    task = {
+      key,
+      agent,
+      session: agent.session,
+      events,
+      settings,
+      phase: "previsible",
+      eventIndex: 0,
+      recoverIndex: 0,
+      visibleIds: [...new Set(this.surfaceToolIds(agent.session))],
+      scheduled: false,
+      retryRequested: false,
+      captureRetryAttempts: 0,
+      progressed: false,
+      eventTail: Number.isInteger(events?.at?.(-1)?.seq) ? events.at(-1).seq : -1,
+      surfaceTail: agent.session.surface?.nodes?.at?.(-1) ?? -1,
+      freezeSnapshot: options.freezeSnapshot === true,
+      resolve,
+      promise,
+    };
+    this.pendingCatchups.set(key, task);
+    this.pendingCatchupJobs.add(promise);
+    this.queueCaptureCatchup(task);
+    return task;
+  }
+
+  queueCaptureRetry(task) {
+    if (!task || task.finished || task.scheduled) return;
+    task.scheduled = true;
+    const delay = Math.min(250, 10 * (2 ** Math.max(0, task.captureRetryAttempts - 1)));
+    setTimeout(() => {
+      task.scheduled = false;
+      if (!task.finished) this.queueCaptureCatchup(task);
+    }, delay);
+  }
+
+  queueCaptureCatchup(task) {
+    if (!task || task.scheduled) return;
+    task.scheduled = true;
+    scheduleImmediate(() => {
+      task.scheduled = false;
+      try {
+        const latest = task.freezeSnapshot ? null : task.session.snapshotEvents?.();
+        const latestTail = Number.isInteger(latest?.at?.(-1)?.seq) ? latest.at(-1).seq : -1;
+        const latestSurfaceTail = task.session.surface?.nodes?.at?.(-1) ?? -1;
+        if (Array.isArray(latest) && (latestTail !== task.eventTail || latest.length !== task.events.length)) {
+          task.events = latest;
+          task.eventTail = latestTail;
+          if (task.phase === "previsible" || task.phase === "visible") {
+            task.phase = "events";
+            task.visibleIds = null;
+            task.recoverIndex = 0;
+          }
+        }
+        if (latestSurfaceTail !== task.surfaceTail) {
+          task.surfaceTail = latestSurfaceTail;
+          if (task.phase === "previsible" || task.phase === "visible") {
+            task.visibleIds = [...new Set(this.surfaceToolIds(task.session))];
+            task.recoverIndex = 0;
+          }
+        }
+        if (task.phase === "previsible") {
+          const recovered = this.recoverVisibleRawBatch(task.session, {}, task.recoverIndex, RECOVER_RAW_BATCH, task.visibleIds);
+          task.progressed ||= recovered.progressed;
+          task.recoverIndex = recovered.nextIndex;
+          if (!recovered.done) {
+            this.queueCaptureCatchup(task);
+            return;
+          }
+          task.phase = "events";
+          task.eventIndex = 0;
+        }
+        if (task.phase === "events") {
+          const result = this.captureNewBatch(task.session, task.events, task.settings, {
+            eventLimit: CAPTURE_EVENT_BATCH,
+            resultLimit: CAPTURE_RESULT_BATCH,
+            startIndex: task.eventIndex,
+          });
+          task.eventIndex = result.nextIndex;
+          task.progressed ||= result.progressed;
+          if (result.retryable) {
+            task.captureRetryAttempts += 1;
+            const previousMax = this.captureRetryStats?.get(task.key) || 0;
+            if (task.captureRetryAttempts > previousMax) this.captureRetryStats.set(task.key, task.captureRetryAttempts);
+            if (task.captureRetryAttempts >= CAPTURE_RETRY_LIMIT) {
+              this.finishCaptureCatchup(task);
+              return;
+            }
+            this.queueCaptureRetry(task);
+            return;
+          }
+          if (result.progressed) task.captureRetryAttempts = 0;
+          if (!result.done) {
+            this.queueCaptureCatchup(task);
+            return;
+          }
+          task.phase = "visible";
+          task.recoverIndex = 0;
+          task.visibleIds = [...new Set(this.surfaceToolIds(task.session))];
+        }
+        const recovered = this.recoverVisibleRawBatch(task.session, {}, task.recoverIndex, RECOVER_RAW_BATCH, task.visibleIds);
+        task.progressed ||= recovered.progressed;
+        task.recoverIndex = recovered.nextIndex;
+        if (!recovered.done) {
+          this.queueCaptureCatchup(task);
+          return;
+        }
+        this.finishCaptureCatchup(task);
+      } catch (error) {
+        noteShadowWarning(this.ctx?.logger, error);
+        this.finishCaptureCatchup(task);
+      }
+    });
+  }
+
+  finishCaptureCatchup(task) {
+    if (!task || task.finished) return;
+    task.finished = true;
+    this.pendingCatchups.delete(task.key);
+    this.pendingCatchupJobs.delete(task.promise);
+    task.resolve();
+    if (task.retryRequested && task.progressed) this.scheduleMaintenanceRetry(task.agent);
+  }
+
+  scheduleMaintenanceRetry(agent) {
+    const state = this.stateFor(agent);
+    if (state.retryScheduled) return;
+    state.retryScheduled = true;
+    scheduleImmediate(() => {
+      state.retryScheduled = false;
+      if (state.warming) return;
+      const events = agent.session.snapshotEvents();
+      const done = events.some((event) => event.type === "turn/end" && event.data?.reason?.kind === "completed");
+      if (!done) return;
+      state.warming = true;
+      const job = this.finishTurn(agent, events, { deferCapture: true })
+        .catch((error) => noteShadowWarning(this.ctx?.logger, error))
+        .finally(() => {
+          state.warming = false;
+          this.pendingCuts.delete(job);
+        });
+      this.pendingCuts.add(job);
+    });
+  }
+
+  async finishTurn(agent, events, options = {}) {
     const settings = readSettings(this.liveConfig);
     const state = this.stateFor(agent);
     const signal = AbortSignal.timeout(30000);
     const ev = events || agent.session.snapshotEvents();
-    await this.flushPendingCaptures();
-    this.captureNew(agent.session, ev, settings);
+    const deferCapture = options.deferCapture === true;
+    let catchup = { finished: true, progressed: false, retryRequested: false };
+    if (deferCapture) {
+      catchup = this.scheduleCaptureCatchup(agent, ev, settings);
+    } else {
+      if (ev.length > CAPTURE_EVENT_BATCH) {
+        catchup = this.scheduleCaptureCatchup(agent, ev, settings, { freezeSnapshot: true });
+      } else {
+        await this.flushPendingCaptures();
+        this.captureNew(agent.session, ev, settings);
+      }
+    }
     const jobs = collectJobs(ev);
     const calls = indexToolCalls(ev);
     try {
@@ -636,6 +879,7 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
           try {
             state.localOnly = true;
             state.approvedCard = chunk.card;
+            await this.flushPendingCaptures(agent.session, this.chunkToolCallIds(agent.session, chunk));
             await this.commitSelectedSpan(agent, chunk, op);
             measurement = this.ctx.tokenMeter.measure(agent.session);
             const after = measurement.totalTokens;
@@ -656,6 +900,10 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
             this.ctx.logger?.info?.(`spine cut ${Date.now() - started}ms ${before} -> ${after} chunk=${chunk.tokens} saved=${saved}`);
           } catch (error) {
             const message = error instanceof Error ? error.message : String(error);
+            if (message.includes("raw backup missing before cut")) {
+              catchup.retryRequested = true;
+              if (catchup.finished && catchup.progressed) this.scheduleMaintenanceRetry(agent);
+            }
             this.ctx.logger?.warn?.(`spine cut failed: ${message}`);
             break;
           }
@@ -678,9 +926,33 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
     }
   }
 
-  async flushPendingCaptures() {
-    if (!this.pendingCaptures?.size) return;
-    await Promise.allSettled([...this.pendingCaptures]);
+  async flushPendingCaptures(session = null, ids = null) {
+    if (!session) {
+      if (!this.pendingCaptures?.size) return;
+      await Promise.allSettled([...this.pendingCaptures]);
+      return;
+    }
+    const key = session.id || DEFAULT_KEY;
+    const pendingIds = ids || this.surfaceToolIds(session);
+    const jobs = [...new Set(pendingIds)]
+      .map((id) => this.pendingCaptureJobs?.get?.(`${key}\0${id}`))
+      .filter(Boolean);
+    if (jobs.length) await Promise.allSettled(jobs);
+  }
+
+  chunkToolCallIds(session, chunk) {
+    const nodes = session?.surface?.nodes || [];
+    const start = nodes.indexOf(chunk?.start);
+    const end = nodes.indexOf(chunk?.end);
+    if (start < 0 || end < start) return [];
+    const ids = [];
+    for (const seq of nodes.slice(start, end + 1)) {
+      const event = session.eventAt?.(seq);
+      if (event?.type !== "tool/result") continue;
+      const id = event.data?.message?.toolCallId;
+      if (id) ids.push(id);
+    }
+    return ids;
   }
 
   async commitSelectedSpan(agent, chunk, signal) {
@@ -716,7 +988,29 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
       const event = session.eventAt(seq);
       if (event?.type === "tool/result") {
         const id = event.data?.message?.toolCallId;
-        if (!id || !verifiedRaw(this.rawSeen.get(this.rawKey(session, id))).ok) {
+        let raw = id ? this.rawSeen.get(this.rawKey(session, id)) : null;
+        if (id && raw?.ok && raw.recovered && raw.aclVerified !== true) {
+          try {
+            const verified = readRawToolResult({
+              dir: sessionDir(shadowDirectory(), session.id || DEFAULT_KEY),
+              toolCallId: id,
+              tighten: true,
+            });
+            // ACL success cannot establish a new content identity. Keep the
+            // first verified hash even if an external writer replaces both files.
+            const sameBackup = verified?.ok && verified.path === raw.path &&
+              verified.sha256 === raw.sha256 && verified.bytes === raw.bytes;
+            raw = sameBackup
+              ? { ...verified, recovered: false, aclVerified: true }
+              : { ...raw, ok: false, reason: verified?.reason || "hash-mismatch", aclVerified: false };
+            this.rawSeen.set(this.rawKey(session, id), raw);
+          } catch (error) {
+            noteShadowWarning(this.ctx?.logger, error);
+            raw = { ...raw, ok: false, reason: "unwritable", toolCallId: id, aclVerified: false };
+            this.rawSeen.set(this.rawKey(session, id), raw);
+          }
+        }
+        if (!id || !verifiedRaw(raw).ok) {
           throw new Error(`spine: raw backup missing before cut: ${id || "unknown"}`);
         }
       }
@@ -938,8 +1232,9 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
         ),
         access,
       });
-      this.rawSeen.set(mapKey, record);
-      return record;
+      const cached = record?.ok ? { ...record, aclVerified: true } : record;
+      this.rawSeen.set(mapKey, cached);
+      return cached;
     } catch (error) {
       if (
         error?.code === "ESHADOWACCESS" ||
@@ -972,19 +1267,113 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
     }
   }
 
-  captureNew(session, events, settings = readSettings(this.liveConfig)) {
-    if (!Array.isArray(events)) events = [];
+  noteCaptureResult(session, event, record) {
+    const key = session?.id || DEFAULT_KEY;
+    this.retryableCaptureSeqs ||= new Map();
+    this.retryableCaptureSeeded ||= new Set();
+    let seqs = this.retryableCaptureSeqs.get(key);
+    if (!seqs) {
+      seqs = new Set();
+      this.retryableCaptureSeqs.set(key, seqs);
+    }
+    const seq = Number.isInteger(event?.seq) ? event.seq : null;
+    if (seq === null) return;
+    if (record?.retryable) seqs.add(seq);
+    else seqs.delete(seq);
+  }
+
+  captureCursorStart(session, events, settings) {
     const key = session?.id || DEFAULT_KEY;
     let cursor = this.captureCursor.has(key) ? this.captureCursor.get(key) : this.readCaptureCursor(key);
     if (cursor == null) cursor = restartCaptureCursor(events, settings.keepRounds);
     if (!Number.isSafeInteger(cursor) || cursor < 0 || events.length < cursor) cursor = restartCaptureCursor(events, settings.keepRounds);
-    // A live event callback may have failed before the recent-three restart
-    // boundary. Keep those visible originals eligible without replaying history.
-    for (const seq of session?.surface?.nodes || []) {
-      const event = session.eventAt?.(seq);
-      const id = event?.type === "tool/result" && event.data?.message?.toolCallId;
-      if (id && this.rawSeen.get(this.rawKey(session, id))?.retryable) cursor = Math.min(cursor, seq);
+    this.retryableCaptureSeqs ||= new Map();
+    this.retryableCaptureSeeded ||= new Set();
+    let retryable = this.retryableCaptureSeqs.get(key);
+    if (!retryable) {
+      retryable = new Set();
+      this.retryableCaptureSeqs.set(key, retryable);
+    }
+    if (!this.retryableCaptureSeeded.has(key)) {
+      for (const seq of session?.surface?.nodes || []) {
+        const event = session.eventAt?.(seq);
+        const id = event?.type === "tool/result" && event.data?.message?.toolCallId;
+        if (id && this.rawSeen.get(this.rawKey(session, id))?.retryable) retryable.add(seq);
+      }
+      this.retryableCaptureSeeded.add(key);
+    }
+    if (retryable.size) cursor = Math.min(cursor, Math.min(...retryable));
+    return cursor;
+  }
+
+  captureNewBatch(session, events, settings = readSettings(this.liveConfig), limits = {}) {
+    if (!Array.isArray(events)) events = [];
+    const key = session?.id || DEFAULT_KEY;
+    const cursor = this.captureCursorStart(session, events, settings);
+    let next = cursor;
+    let retry = null;
+    let retryIndex = null;
+    let processed = 0;
+    let captures = 0;
+    let progressed = false;
+    let exhausted = true;
+    const seen = new Set();
+    const eventLimit = Math.max(1, limits.eventLimit || CAPTURE_EVENT_BATCH);
+    const resultLimit = Math.max(1, limits.resultLimit || CAPTURE_RESULT_BATCH);
+    let index = Math.max(0, Number.isInteger(limits.startIndex) ? limits.startIndex : 0);
+    const indexedSeq = Number.isInteger(events[index]?.seq) ? events[index].seq : null;
+    if (index > 0 && indexedSeq !== null && indexedSeq > cursor) {
+      const rewind = events.findIndex((event) => Number.isInteger(event?.seq) && event.seq >= cursor);
+      index = rewind >= 0 ? rewind : 0;
     }
+    for (; index < events.length; index += 1) {
+      const event = events[index];
+      const seq = Number.isInteger(event?.seq) ? event.seq : next;
+      if (seq < cursor) continue;
+      this.noteEvent(session, event);
+      if (event?.type === "tool/result") {
+        const id = event.data?.message?.toolCallId;
+        if (id && !seen.has(id)) {
+          seen.add(id);
+          const cached = this.rawSeen.get(this.rawKey(session, id));
+          const record = cached && !cached.retryable ? cached : this.captureOne(session, event);
+          if (!cached || cached.retryable) {
+            if (record?.ok) progressed = true;
+            captures += 1;
+          }
+          this.noteCaptureResult(session, event, record);
+          if (record?.retryable && retry === null) {
+            retry = seq;
+            retryIndex = index;
+          }
+        }
+      }
+      next = seq + 1;
+      processed += 1;
+      if (retry !== null || processed >= eventLimit || captures >= resultLimit) {
+        exhausted = false;
+        index += 1;
+        break;
+      }
+    }
+    next = retry ?? next;
+    this.captureCursor.set(key, next);
+    this.writeCaptureCursor(key, next);
+    const lastEvent = events.at(-1);
+    const lastSeq = Number.isInteger(lastEvent?.seq) ? lastEvent.seq : events.length - 1;
+    return {
+      seen,
+      progressed,
+      retryable: retry !== null,
+      done: exhausted && retry === null && next > lastSeq,
+      nextIndex: retryIndex ?? index,
+    };
+  }
+
+  captureNew(session, events, settings = readSettings(this.liveConfig)) {
+    if (!Array.isArray(events)) events = [];
+    const key = session?.id || DEFAULT_KEY;
+    const cursor = this.captureCursorStart(session, events, settings);
     let next = cursor;
     let retry = null;
     const seen = new Set();
@@ -998,6 +1387,7 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
           seen.add(id);
           const cached = this.rawSeen.get(this.rawKey(session, id));
           const record = cached && !cached.retryable ? cached : this.captureOne(session, event);
+          this.noteCaptureResult(session, event, record);
           if (record?.retryable && retry === null) retry = seq;
         }
       }
@@ -1063,20 +1453,103 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
 
   recoverVisibleRaw(session, access = {}) {
     const dir = sessionDir(shadowDirectory(), session?.id || DEFAULT_KEY);
-    for (const id of new Set(this.surfaceToolIds(session))) {
+    const ids = [...new Set(this.surfaceToolIds(session))];
+    if (ids.length && existsSync(dir)) ensureShadowDir(dir, access);
+    for (const id of ids) {
       const mapKey = this.rawKey(session, id);
       if (this.rawSeen.has(mapKey)) continue;
       try {
-        this.rawSeen.set(mapKey, readRawToolResult({ dir, toolCallId: id, access }));
+        const record = readRawToolResult({ dir, toolCallId: id, access, tighten: false });
+        const retryable = recoveryRetryable(record, dir, id);
+        const cached = record?.ok
+          ? { ...record, recovered: true, aclVerified: false }
+          : { ...record, ...(retryable ? { retryable: true } : {}), recovered: true, aclVerified: false };
+        this.rawSeen.set(mapKey, cached);
+        if (!cached.ok) {
+          const event = (session?.surface?.nodes || [])
+            .map((seq) => session.eventAt?.(seq))
+            .find((candidate) => candidate?.type === "tool/result" && candidate.data?.message?.toolCallId === id);
+          if (event) this.noteCaptureResult(session, event, cached);
+          if (retryable) this.captureRecoveryHolds.set(session?.id || DEFAULT_KEY, this.captureRecoveryFingerprint(session));
+        }
       } catch (error) {
         noteShadowWarning(this.ctx?.logger, error);
-        this.rawSeen.set(mapKey, { ok: false, reason: "unwritable", toolCallId: id });
+        const failed = { ok: false, reason: "unwritable", toolCallId: id, retryable: true, recovered: true, aclVerified: false };
+        this.rawSeen.set(mapKey, failed);
+        const event = (session?.surface?.nodes || [])
+          .map((seq) => session.eventAt?.(seq))
+          .find((candidate) => candidate?.type === "tool/result" && candidate.data?.message?.toolCallId === id);
+        if (event) this.noteCaptureResult(session, event, failed);
+        this.captureRecoveryHolds.set(session?.id || DEFAULT_KEY, this.captureRecoveryFingerprint(session));
       }
     }
   }
 
+  recoverVisibleRawBatch(session, access = {}, startIndex = 0, limit = RECOVER_RAW_BATCH, visibleIds = null) {
+    const ids = visibleIds || [...new Set(this.surfaceToolIds(session))];
+    const dir = sessionDir(shadowDirectory(), session?.id || DEFAULT_KEY);
+    if (ids.length && startIndex === 0 && existsSync(dir)) {
+      try {
+        ensureShadowDir(dir, access);
+      } catch (error) {
+        noteShadowWarning(this.ctx?.logger, error);
+        return { nextIndex: ids.length, done: true, progressed: false };
+      }
+    }
+    let index = Math.max(0, startIndex);
+    let processed = 0;
+    let progressed = false;
+    while (index < ids.length && processed < Math.max(1, limit)) {
+      const id = ids[index++];
+      const mapKey = this.rawKey(session, id);
+      if (this.rawSeen.has(mapKey)) continue;
+      try {
+        const record = readRawToolResult({ dir, toolCallId: id, access, tighten: false });
+        const retryable = recoveryRetryable(record, dir, id);
+        const cached = record?.ok
+          ? { ...record, recovered: true, aclVerified: false }
+          : { ...record, ...(retryable ? { retryable: true } : {}), recovered: true, aclVerified: false };
+        this.rawSeen.set(mapKey, cached);
+        if (!cached.ok) {
+          const event = session?.eventAt?.(session?.surface?.nodes?.find?.((seq) => {
+            const candidate = session.eventAt?.(seq);
+            return candidate?.type === "tool/result" && candidate.data?.message?.toolCallId === id;
+          }));
+          if (event) this.noteCaptureResult(session, event, cached);
+          if (retryable) this.captureRecoveryHolds.set(session?.id || DEFAULT_KEY, this.captureRecoveryFingerprint(session));
+        }
+        progressed ||= Boolean(record?.ok);
+      } catch (error) {
+        noteShadowWarning(this.ctx?.logger, error);
+        const failed = { ok: false, reason: "unwritable", toolCallId: id, retryable: true, recovered: true, aclVerified: false };
+        this.rawSeen.set(mapKey, failed);
+        const event = session?.eventAt?.(session?.surface?.nodes?.find?.((seq) => {
+          const candidate = session.eventAt?.(seq);
+          return candidate?.type === "tool/result" && candidate.data?.message?.toolCallId === id;
+        }));
+        if (event) this.noteCaptureResult(session, event, failed);
+        this.captureRecoveryHolds.set(session?.id || DEFAULT_KEY, this.captureRecoveryFingerprint(session));
+      }
+      processed += 1;
+    }
+    return { nextIndex: index, done: index >= ids.length, progressed };
+  }
+
   captureSession(session) {
-    return this.captureNew(session, session?.snapshotEvents?.() || []);
+    const key = session?.id || DEFAULT_KEY;
+    const events = session?.snapshotEvents?.() || [];
+    const fingerprint = this.captureRecoveryFingerprint(session, events);
+    if (this.captureRecoveryHolds.get(key) === fingerprint) {
+      this.captureRecoveryHolds.delete(key);
+      return new Set();
+    }
+    return this.captureNew(session, events);
+  }
+
+  captureRecoveryFingerprint(session, events = null) {
+    const snapshot = events || session?.snapshotEvents?.() || [];
+    const nodes = session?.surface?.nodes || [];
+    return `${snapshot.length}:${snapshot.at(-1)?.seq ?? -1}|${nodes.length}:${nodes.at(-1) ?? -1}`;
   }
 
   surfaceToolIds(session) {
@@ -1093,14 +1566,24 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
   rawVerifiedSet(session) {
     const ids = new Set();
     const dir = sessionDir(shadowDirectory(), session?.id || DEFAULT_KEY);
-    for (const id of this.surfaceToolIds(session)) {
+    const visibleIds = this.surfaceToolIds(session);
+    const recoveredNeedsAcl = visibleIds.filter((id) => {
+      const cached = this.rawSeen.get(this.rawKey(session, id));
+      return cached?.ok && cached.recovered === true && cached.aclVerified !== true;
+    });
+    if (recoveredNeedsAcl.length > RECOVERY_ACL_REAUTH_LIMIT) return ids;
+    for (const id of visibleIds) {
       const cached = this.rawSeen.get(this.rawKey(session, id));
-      if (!cached?.ok) continue;
+      if (!cached?.ok || cached.aclVerified !== true && cached.recovered !== true) continue;
       try {
         // Recheck both files immediately before prune, retaining the first
         // trusted hash rather than blessing a replacement raw/meta pair.
-        const stored = readRawToolResult({ dir, toolCallId: id });
-        if (stored.ok && stored.path === cached.path && stored.sha256 === cached.sha256 && stored.bytes === cached.bytes) ids.add(id);
+        const needsAcl = cached.recovered === true && cached.aclVerified !== true;
+        const stored = readRawToolResult({ dir, toolCallId: id, tighten: needsAcl });
+        if (stored.ok && stored.path === cached.path && stored.sha256 === cached.sha256 && stored.bytes === cached.bytes) {
+          ids.add(id);
+          if (needsAcl) this.rawSeen.set(this.rawKey(session, id), { ...stored, recovered: false, aclVerified: true });
+        }
       } catch (error) {
         noteShadowWarning(this.ctx?.logger, error);
       }
@@ -1166,6 +1649,7 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
     warnKey = DEFAULT_KEY,
     protectPaths = [],
     access = {},
+    rotationIO = {},
   ) {
     return saveShadowImpl.call(
       this,
@@ -1175,9 +1659,68 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
       warnKey,
       protectPaths,
       access,
+      rotationIO,
     );
   }
 
+  async saveShadowAsync(
+    messages,
+    required,
+    shadowFile,
+    warnKey = DEFAULT_KEY,
+    protectPaths = [],
+    access = {},
+    rotationIO = {},
+  ) {
+    if (!access.runner && (access.platform ?? process.platform) === "win32" && shadowAccessFused()) {
+      const error = new Error("spine: shadow permission already failed; backup refused");
+      error.code = "ESHADOWACCESS";
+      throw error;
+    }
+    const injected = Object.values(access || {}).some((value) => typeof value === "function") ||
+      Object.values(rotationIO || {}).some((value) => typeof value === "function");
+    // Node's test runner can monkey-patch fs bindings in the parent isolate;
+    // a worker cannot observe those injected failures. Keep test/fault-injection
+    // paths synchronous while production uses the complete off-thread txn.
+    if (injected || process.env.NODE_TEST_CONTEXT) {
+      return this.saveShadow(messages, required, shadowFile, warnKey, protectPaths, access, rotationIO);
+    }
+    const shadow = shadowDirectory();
+    const file = shadowFilePath(shadowFile, warnKey);
+    const body = shadowFileBody(warnKey, shadowBody(messages));
+    const cap = shadowMaxBytes();
+    const result = await runShadowWorker({
+      shadow,
+      file,
+      body,
+      required,
+      protectPaths,
+      sessionKey: warnKey,
+      cap,
+      access,
+      rotationIO,
+    });
+    if (!result?.ok) {
+      if (result?.reason === "scan-incomplete" || result?.reason === "invalid-measurement") {
+        this.ctx.logger?.warn?.(result.reason === "scan-incomplete"
+          ? "spine: shadow capacity scan is incomplete; backup refused; leaving the span"
+          : "spine: shadow capacity measurement is invalid; backup refused; leaving the span");
+      } else if (result?.rotation) {
+        this.warnShadowFull(warnKey, shadow, result.rotation.used, cap);
+      }
+      return result;
+    }
+    if (result.rotation?.deleted > 0) {
+      this.ctx.logger?.info?.(
+        `spine: shadow 轮换 删除 ${result.rotation.deleted} 个文件，` +
+        `腾出 ${result.rotation.freed} 字节，` +
+        `用量 ${result.rotation.used + Buffer.byteLength(body)}/${cap}`,
+      );
+    }
+    if (result.rotation?.oversize) this.warnShadowOversize(warnKey, cap);
+    return result;
+  }
+
   async summarize(input, agent, signal) {
     const decision = takeLocalCut(input?.liveCompact);
     const state = this.stateFor(agent);
@@ -1187,14 +1730,14 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
       const card = input?.liveCompact?.card;
       if (card?.needsBackup && !card.shadowFile) throw new Error("spine: omit refused; backup path missing");
       const restored = this.restoreRawMessages(input.messages, state.key);
-      this.saveShadow(restored, true, card?.shadowFile, state.key, card?.rawPaths || []);
+      await this.saveShadowAsync(restored, true, card?.shadowFile, state.key, card?.rawPaths || []);
       if (card?.needsBackup && !existsSync(card.shadowFile)) throw new Error("spine: shadow backup was not written");
       const text = backupClaimOk(restored) ? decision.text : withoutFullBackupClaim(decision.text, card?.shadowFile);
       const block = { type: "text", text };
       return { summary: [block], provider: "local", model: "spine-extract", rawOutput: [block] };
     }
     try {
-      this.saveShadow(
+      await this.saveShadowAsync(
         this.restoreRawMessages(input.messages, state.key),
         false,
         freshShadowFile("official", state.key),
@@ -1209,7 +1752,7 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
       }
     }
     const joined = input.messages.map(messageText).join("\n\n");
-    if (process.env.SPINE_THUNDER !== "1") return super.summarize(input, agent, signal);
+    if (process.env.SPINE_THUNDER !== "1") return this.summarizeOfficial(input, agent, signal);
     const next = joined;
     const mode = decideThunder({
       previous: state.lastPrefix || "",
@@ -1217,7 +1760,7 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
       modelKind: process.env.SPINE_THUNDER_MODEL === "27b" ? "27b" : "flash",
       prefixTokens: state.lastPrefixTokens || 0,
     });
-    if (mode === "attach") return super.summarize(input, agent, signal);
+    if (mode === "attach") return this.summarizeOfficial(input, agent, signal);
     const users = input.messages.filter((message) => (message?.role ?? message?.message?.role) === "user");
     const taskLine = messageText(users.at(-1) || input.messages.at(-1)).slice(0, 500);
     const trunk = buildCard(joined, "", input.messages.map((message) => ({
@@ -1226,9 +1769,25 @@ export default class SpineCompactionEngine extends BasicCompactionEngine {
     })));
     const facts = [...new Set([...(trunk.hardLines || []), ...(trunk.missing || [])])];
     const slim = buildSendLess({ taskLine, uncovered: trunk.uncovered, facts });
-    return super.summarize({
+    return this.summarizeOfficial({
       ...input,
       messages: [{ role: "user", content: [{ type: "text", text: slim }] }],
     }, agent, signal);
   }
+
+  async summarizeOfficial(input, agent, signal) {
+    const result = await super.summarize(input, agent, signal);
+    const blocks = Array.isArray(result?.summary) ? result.summary : [];
+    const usable = blocks.some((block) => {
+      if (typeof block === "string") return block.trim().length > 0;
+      if (!block || typeof block !== "object") return false;
+      if (typeof block.text === "string" && block.text.trim()) return true;
+      return Array.isArray(block.content) && block.content.some((part) => typeof part?.text === "string" && part.text.trim());
+    });
+    if (!usable) {
+      this.ctx.logger?.warn?.("spine: official summarizer returned no usable summary; leaving the span untouched");
+      throw new Error("spine: official summarizer returned no usable summary");
+    }
+    return result;
+  }
 }