@yeaft/webchat-agent 1.0.347 → 1.0.348

package/yeaft/debug-trace.js

@@ -1,21 +1,20 @@
 /**
  * debug-trace.js — file-backed debug trace for Yeaft
  *
- * Stores bounded request traces on disk without SQLite. Each request owns one
- * compact JSON file under the session's debug folder:
- *   <yeaftDir>/sessions/<sessionId>/debug/requests/<requestKey>/trace.json
+ * Stores bounded request traces on disk without SQLite. New requests use a
+ * small `meta.json` plus append-only `events.jsonl` under:
+ *   <yeaftDir>/sessions/<sessionId>/debug/requests/<requestKey>/
  *
- * The file keeps one base request snapshot plus per-loop deltas. This avoids
- * 100-200 tiny loop files and avoids repeating the whole cumulative message
- * array for every loop. Debug history remains best-effort: failures are logged
- * and dropped, never allowed to stop the agent.
+ * Loop requests remain base-plus-delta records, so cumulative messages are not
+ * repeated. Legacy `trace.json` requests stay readable. Debug history is
+ * best-effort: failures are logged and never allowed to stop the agent.
  */
 
 import { promises as fsp } from 'fs';
 import { basename, dirname, extname, join } from 'path';
-import { randomUUID } from 'crypto';
+import { createHash, randomUUID } from 'crypto';
 
-const TRACE_VERSION = 2;
+const TRACE_VERSION = 3;
 const REQUEST_RETENTION = 10;
 const MAX_HISTORY_LIMIT = 5;
 const MAX_DREAM_EVENTS = 100;
@@ -31,8 +30,8 @@ const MAX_RAW_REQUEST_BYTES = 2 * 1024 * 1024;
 // A 64 KiB prefix is enough to inspect provider envelopes without letting
 // always-on diagnostics dominate runtime latency.
 const MAX_RAW_RESPONSE_BYTES = 64 * 1024;
-const TRACE_FLUSH_INTERVAL_MS = 5_000;
-const TRACE_FLUSH_DIRTY_LOOPS = 10;
+const TRACE_APPEND_BATCH_MS = 100;
+const EVENT_FLUSH_INTERVAL_MS = 30_000;
 const MAX_SEARCH_PATTERN_CHARS = 300;
 
 function isPlainObject(value) {
@@ -45,6 +44,15 @@ function safeDirComponent(value, fallback = 'unknown') {
   return raw.replace(/[^a-zA-Z0-9._-]+/g, '_').slice(0, 120) || fallback;
 }
 
+function storageDirComponent(value, fallback = 'unknown') {
+  const raw = String(value || '').trim();
+  if (!raw) return fallback;
+  if (/^[a-zA-Z0-9._-]+$/.test(raw) && Buffer.byteLength(raw, 'utf8') <= 200) return raw;
+  const prefix = raw.replace(/[^a-zA-Z0-9._-]+/g, '_').slice(0, 80) || fallback;
+  const digest = createHash('sha256').update(raw).digest('hex');
+  return `${prefix}-${digest}`;
+}
+
 function fileTraceRoot(inputPath) {
   const raw = String(inputPath || '').trim();
   if (!raw) return null;
@@ -456,16 +464,125 @@ export function reconstructDebugRawRequest(baseRawRequest, requestDelta) {
 }
 
 function sessionRequestsDir(rootDir, sessionId) {
+  if (sessionId) return join(rootDir, 'sessions', storageDirComponent(sessionId, 'session'), 'debug', 'requests');
+  return join(rootDir, 'debug', 'requests');
+}
+
+function legacySessionRequestsDir(rootDir, sessionId) {
   if (sessionId) return join(rootDir, 'sessions', safeDirComponent(sessionId), 'debug', 'requests');
   return join(rootDir, 'debug', 'requests');
 }
 
+function sessionRequestDirs(rootDir, sessionId) {
+  const current = sessionRequestsDir(rootDir, sessionId);
+  const legacy = legacySessionRequestsDir(rootDir, sessionId);
+  return current === legacy ? [current] : [current, legacy];
+}
+
 function requestFilePath(requestDir) {
   return join(requestDir, 'trace.json');
 }
 
+function requestMetaPath(requestDir) {
+  return join(requestDir, 'meta.json');
+}
+
+function requestEventsPath(requestDir) {
+  return join(requestDir, 'events.jsonl');
+}
+
+function requestDirFor(rootDir, sessionId, requestKey) {
+  return join(sessionRequestsDir(rootDir, sessionId), storageDirComponent(requestKey, 'request'));
+}
+
 function tracePathFor(rootDir, sessionId, requestKey) {
-  return requestFilePath(join(sessionRequestsDir(rootDir, sessionId), safeDirComponent(requestKey, 'request')));
+  return requestFilePath(requestDirFor(rootDir, sessionId, requestKey));
+}
+
+function turnLocatorName(turnId) {
+  const raw = String(turnId || 'turn');
+  const digest = createHash('sha256').update(raw).digest('hex').slice(0, 16);
+  return `${safeDirComponent(raw, 'turn')}-${digest}.json`;
+}
+
+function turnLocatorPath(rootDir, sessionId, turnId) {
+  return join(sessionRequestsDir(rootDir, sessionId), '..', 'turns', turnLocatorName(turnId));
+}
+
+function serializableTraceMeta(trace) {
+  const meta = { ...trace };
+  delete meta._lastSnapshot;
+  delete meta._persistedFormat;
+  delete meta._persistedRequestDir;
+  delete meta.baseRequest;
+  delete meta.loops;
+  delete meta.tools;
+  return meta;
+}
+
+function traceMatchesIdentity(trace, sessionId, turnId) {
+  return !!trace
+    && trace.sessionId === sessionId
+    && (trace.requestId === turnId || trace.traceId === turnId);
+}
+
+async function prepareJsonlAppend(filePath) {
+  let text;
+  try { text = await fsp.readFile(filePath, 'utf8'); }
+  catch { return; }
+  if (!text || text.endsWith('\n')) return;
+  const lastNewline = text.lastIndexOf('\n');
+  const tail = text.slice(lastNewline + 1);
+  try {
+    JSON.parse(tail);
+    await fsp.appendFile(filePath, '\n', 'utf8');
+  } catch {
+    const validPrefix = lastNewline >= 0 ? text.slice(0, lastNewline + 1) : '';
+    await fsp.truncate(filePath, Buffer.byteLength(validPrefix, 'utf8'));
+  }
+}
+
+async function readJsonLines(filePath) {
+  let text;
+  try { text = await fsp.readFile(filePath, 'utf8'); }
+  catch { return []; }
+  const records = [];
+  for (const line of text.split('\n')) {
+    if (!line.trim()) continue;
+    try { records.push(JSON.parse(line)); }
+    catch { /* A process can leave one torn final append; prior records remain valid. */ }
+  }
+  return records;
+}
+
+async function readRequestDir(requestDir) {
+  const meta = await readJson(requestMetaPath(requestDir));
+  if (!meta?.requestId) {
+    const legacy = await readJson(requestFilePath(requestDir));
+    return legacy ? { ...legacy, _persistedFormat: 'legacy', _persistedRequestDir: requestDir } : null;
+  }
+  const trace = { ...meta, _persistedFormat: 'events', _persistedRequestDir: requestDir, baseRequest: meta.legacyBaseRequest || null, loops: [], tools: [] };
+  delete trace.legacyBaseRequest;
+  const loopById = new Map();
+  const toolById = new Map();
+  for (const event of await readJsonLines(requestEventsPath(requestDir))) {
+    const record = event?.record;
+    if (event?.type === 'loop' && record) {
+      const key = record.turnRowId || record.loopInstanceId || `${record.loopNumber || 0}`;
+      loopById.set(key, record);
+    } else if (event?.type === 'tool' && record) {
+      const key = record.id || `${record.turnRowId || ''}:${record.toolCallId || ''}`;
+      toolById.set(key, record);
+    }
+  }
+  trace.loops = Array.from(loopById.values()).sort((a, b) => (a.loopNumber || 0) - (b.loopNumber || 0) || String(a.turnRowId || '').localeCompare(String(b.turnRowId || '')));
+  trace.tools = Array.from(toolById.values()).sort((a, b) => (a.createdAt || 0) - (b.createdAt || 0));
+  if (trace.loops.length > 0) {
+    const last = trace.loops.at(-1);
+    trace.closedAt = meta.closedAt || last.at || null;
+    trace.updatedAt = Math.max(Number(meta.updatedAt || 0), Number(last.at || 0));
+  }
+  return trace;
 }
 
 function summarizeTrace(trace, detailsLoaded = false) {
@@ -595,21 +712,18 @@ async function readdirSafe(dir) {
   catch { return []; }
 }
 
-async function collectTraceFiles(rootDir, sessionId = null) {
-  const files = [];
+async function collectRequestDirs(rootDir, sessionId = null) {
+  const dirs = [];
   const addFromRequestsDir = async (requestsDir) => {
     for (const entry of await readdirSafe(requestsDir)) {
-      if (!entry.isDirectory()) continue;
-      const file = requestFilePath(join(requestsDir, entry.name));
-      try {
-        await fsp.access(file);
-        files.push(file);
-      } catch { /* trace.json not yet written for this request dir */ }
+      if (entry.isDirectory()) dirs.push(join(requestsDir, entry.name));
     }
   };
   if (sessionId) {
-    await addFromRequestsDir(sessionRequestsDir(rootDir, sessionId));
-    return files;
+    for (const requestsDir of sessionRequestDirs(rootDir, sessionId)) {
+      await addFromRequestsDir(requestsDir);
+    }
+    return dirs;
   }
   await addFromRequestsDir(sessionRequestsDir(rootDir, null));
   const sessionsRoot = join(rootDir, 'sessions');
@@ -617,20 +731,71 @@ async function collectTraceFiles(rootDir, sessionId = null) {
     if (!entry.isDirectory()) continue;
     await addFromRequestsDir(join(sessionsRoot, entry.name, 'debug', 'requests'));
   }
-  return files;
+  return dirs;
 }
 
 async function readTraceSummaries(rootDir, sessionId = null) {
   const traces = [];
-  for (const file of await collectTraceFiles(rootDir, sessionId)) {
-    const trace = await readJson(file);
+  for (const requestDir of await collectRequestDirs(rootDir, sessionId)) {
+    const trace = await readRequestDir(requestDir);
     if (!trace || !trace.requestId) continue;
-    traces.push({ trace, file, openedAt: Number(trace.openedAt || 0) });
+    if (sessionId && trace.sessionId !== sessionId) continue;
+    const file = requestFilePath(requestDir);
+    traces.push({ trace, file, requestDir, openedAt: Number(trace.openedAt || 0) });
   }
   traces.sort((a, b) => ((a.openedAt || 0) - (b.openedAt || 0)) || String(a.trace?.requestKey || a.file).localeCompare(String(b.trace?.requestKey || b.file)));
   return traces;
 }
 
+async function readTraceIdentity(requestDir) {
+  const meta = await readJson(requestMetaPath(requestDir));
+  if (meta?.requestId) return meta;
+  return readJson(requestFilePath(requestDir));
+}
+
+function sameTraceIdentity(trace, expected) {
+  return !!trace && !!expected
+    && trace.sessionId === expected.sessionId
+    && trace.requestKey === expected.requestKey
+    && trace.requestId === expected.requestId;
+}
+
+async function removeRequestDirIfIdentityMatches(requestDir, expected) {
+  const identity = await readTraceIdentity(requestDir);
+  if (!sameTraceIdentity(identity, expected)) return false;
+  try {
+    await fsp.rm(requestDir, { recursive: true, force: true });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function readTraceHeaders(rootDir, sessionId) {
+  const traces = [];
+  const requestDirs = sessionId == null
+    ? await (async () => {
+      const dirs = [];
+      for (const entry of await readdirSafe(sessionRequestsDir(rootDir, null))) {
+        if (entry.isDirectory()) dirs.push(join(sessionRequestsDir(rootDir, null), entry.name));
+      }
+      return dirs;
+    })()
+    : await collectRequestDirs(rootDir, sessionId);
+  for (const requestDir of requestDirs) {
+    const meta = await readJson(requestMetaPath(requestDir));
+    const trace = meta?.requestId ? meta : await readJson(requestFilePath(requestDir));
+    if (!trace?.requestId || !trace?.requestKey || trace.sessionId !== sessionId) continue;
+    traces.push({
+      trace,
+      file: requestFilePath(requestDir),
+      requestDir,
+      openedAt: Number(trace.openedAt || 0),
+    });
+  }
+  return traces.sort((a, b) => ((a.openedAt || 0) - (b.openedAt || 0)) || String(a.trace.requestKey).localeCompare(String(b.trace.requestKey)));
+}
+
 async function countDirFiles(rootDir) {
   let files = 0;
   let bytes = 0;
@@ -655,10 +820,16 @@ export class DebugTrace {
   #turnIndex = new Map();
   /** @type {Map<string, object>} */
   #requestCache = new Map();
-  /** @type {Map<string, { trace: object, dirtyLoops: number, firstDirtyAt: number }>} */
-  #pendingWrites = new Map();
+  /** @type {Array<{ trace: object, type: 'loop'|'tool', record: object, initialize: boolean, writeMeta: boolean }>} */
+  #pendingWrites = [];
+  /** @type {Set<string>} */
+  #initializedRequestKeys = new Set();
+  /** @type {Set<string>} */
+  #reconciledRetentionSessions = new Set();
+  /** @type {Map<string, Map<string, { trace: object, requestDirs: Set<string>, openedAt: number }>>} */
+  #retentionIndex = new Map();
   /** @type {NodeJS.Timeout|null} */
-  #flushTimer = null;
+  #appendTimer = null;
   /** @type {number} */
   #sequence = 0;
   /**
@@ -693,6 +864,8 @@ export class DebugTrace {
    * @type {Promise<void>}
    */
   #flushChain = Promise.resolve();
+  /** @type {boolean} */
+  #acceptingWrites = true;
 
   /**
    * @param {string} tracePath — Back-compatible path. If it looks like a DB
@@ -708,6 +881,7 @@ export class DebugTrace {
   }
 
   startTurn({ traceId, messageId = null, mode = null, turnNumber = null, sessionId = null, vpId = null, threadId = null, userPrompt = null } = {}) {
+    if (!this.#acceptingWrites) return 'null';
     const turnRowId = randomUUID();
     const now = Date.now();
     const request = this.#getOrCreateRequest({
@@ -777,7 +951,7 @@ export class DebugTrace {
     trace.updatedAt = loop.at;
     trace.active = info.stopReason ? !['end_turn', 'error', 'aborted'].includes(String(info.stopReason)) : false;
     trace._lastSnapshot = snapshot;
-    this.#markDirty(trace, { dirtyLoops: 1, force: !trace.active });
+    this.#appendTraceRecord(trace, 'loop', loop, { writeMeta: !trace.active });
   }
 
   logTool(turnId, { toolName, toolCallId = null, toolInput = null, toolOutput = null, durationMs = null, isError = false } = {}) {
@@ -787,7 +961,7 @@ export class DebugTrace {
     const trace = this.#loadRequest(ctx.sessionId, ctx.requestKey);
     if (!trace) return id;
     if (!Array.isArray(trace.tools)) trace.tools = [];
-    trace.tools.push({
+    const tool = {
       id,
       turnRowId: turnId,
       loopNumber: ctx.loopNumber || 0,
@@ -798,14 +972,16 @@ export class DebugTrace {
       durationMs: Number(durationMs || 0),
       isError: !!isError,
       createdAt: Date.now(),
-    });
-    trace.updatedAt = Date.now();
-    this.#markDirty(trace, { dirtyLoops: 0, force: !trace.active });
+    };
+    trace.tools.push(tool);
+    trace.updatedAt = tool.createdAt;
+    this.#appendTraceRecord(trace, 'tool', tool, { writeMeta: false });
     return id;
   }
 
   logEvent({ traceId, eventType, eventData = null } = {}) {
     const id = randomUUID();
+    if (!this.#acceptingWrites) return id;
     // In-memory authoritative ring; persisted async (fire-and-forget). The
     // dream/event sink runs on the engine hot path, so it must not block on a
     // synchronous read-modify-write of events.json.
@@ -858,11 +1034,54 @@ export class DebugTrace {
       .flatMap(({ trace }) => traceToLegacyRows(trace));
   }
 
+  async fetchTurnDebug({ sessionId, turnId, dreamLimit = 0 } = {}) {
+    const requestedSessionId = typeof sessionId === 'string' && sessionId ? sessionId : null;
+    const requestedTurnId = typeof turnId === 'string' && turnId ? turnId : null;
+    if (!requestedSessionId || !requestedTurnId) {
+      return { loops: [], turns: [], dreamEvents: [], detailTurnId: requestedTurnId };
+    }
+    await this.#drainWrites();
+
+    let trace = Array.from(this.#requestCache.values()).find(item => (
+      item?.sessionId === requestedSessionId
+      && (item.requestId === requestedTurnId || item.traceId === requestedTurnId)
+    )) || null;
+    if (!trace) {
+      const locator = await readJson(turnLocatorPath(this.#rootDir, requestedSessionId, requestedTurnId));
+      if (locator?.requestKey && locator.sessionId === requestedSessionId && locator.requestId === requestedTurnId) {
+        const located = await readRequestDir(requestDirFor(this.#rootDir, requestedSessionId, locator.requestKey));
+        if (traceMatchesIdentity(located, requestedSessionId, requestedTurnId)) {
+          trace = located;
+          this.#requestCache.set(trace.requestKey, trace);
+        }
+      }
+    }
+    if (!trace) {
+      // Legacy v2 traces have no locator. Scan only as a compatibility fallback;
+      // every v3 trace takes the direct sessionId + turnId path above.
+      for (const item of await readTraceSummaries(this.#rootDir, requestedSessionId)) {
+        if (traceMatchesIdentity(item.trace, requestedSessionId, requestedTurnId)) {
+          trace = item.trace;
+          this.#requestCache.set(trace.requestKey, trace);
+          break;
+        }
+      }
+    }
+    const dreamEvents = this.#readDreamEvents({ sessionId: requestedSessionId, dreamLimit });
+    if (!trace) return { loops: [], turns: [], dreamEvents, detailTurnId: requestedTurnId };
+    const expanded = expandTrace(trace);
+    return { ...expanded, dreamEvents, detailTurnId: requestedTurnId };
+  }
+
   async fetchRecentDebugHistory({ limit = MAX_HISTORY_LIMIT, dreamLimit = 5, sessionId = null, threadId = null, indexOnly = false, detailTurnId = null, search = '' } = {}) {
+    const requestedDetailTurnId = typeof detailTurnId === 'string' && detailTurnId ? detailTurnId : null;
+    if (requestedDetailTurnId && sessionId) {
+      const detail = await this.fetchTurnDebug({ sessionId, turnId: requestedDetailTurnId, dreamLimit });
+      return { ...detail, hasMore: false, limit: detail.loops.length, indexOnly: false };
+    }
     await this.#ensureHydrated();
     await this.#drainWrites();
     const lim = Math.max(1, Math.min(MAX_HISTORY_LIMIT, Number(limit) || MAX_HISTORY_LIMIT));
-    const requestedDetailTurnId = typeof detailTurnId === 'string' && detailTurnId ? detailTurnId : null;
     const searchRegex = requestedDetailTurnId ? null : compileTraceSearchRegex(search);
     const traces = this.#traceSummaries(sessionId)
       .filter(({ trace }) => !threadId || trace.threadId === threadId)
@@ -953,20 +1172,25 @@ export class DebugTrace {
   }
 
   async purge() {
+    this.#acceptingWrites = false;
     await this.#drainWrites();
     try { await fsp.rm(this.#rootDir, { recursive: true, force: true }); }
     catch { /* ignore */ }
     await ensureDir(this.#rootDir);
     this.#turnIndex.clear();
     this.#requestCache.clear();
+    this.#initializedRequestKeys.clear();
+    this.#reconciledRetentionSessions.clear();
+    this.#retentionIndex.clear();
     this.#events = [];
     this.#hydrated = false;
     this.#hydratePromise = null;
     this.#eventsHydrated = false;
+    this.#acceptingWrites = true;
   }
 
   async close() {
-    // #drainWrites() flushes pending writes itself; no separate #flushPending().
+    this.#acceptingWrites = false;
     await this.#drainWrites();
   }
 
@@ -1104,47 +1328,32 @@ export class DebugTrace {
     this.#events = [...older, ...this.#events].slice(-MAX_DREAM_EVENTS);
   }
 
-  #traceWriteKey(trace) {
-    return `${trace.sessionId || ''}::${trace.requestKey}`;
-  }
-
   #traceFile(trace) {
     return tracePathFor(this.#rootDir, trace.sessionId || null, trace.requestKey);
   }
 
-  #serializableTrace(trace) {
-    const toWrite = { ...trace };
-    delete toWrite._lastSnapshot;
-    return toWrite;
-  }
-
-  #markDirty(trace, { dirtyLoops = 0, force = false } = {}) {
-    if (!trace?.requestKey) return;
-    const key = this.#traceWriteKey(trace);
-    const existing = this.#pendingWrites.get(key);
-    const now = Date.now();
-    const item = existing || { trace, dirtyLoops: 0, firstDirtyAt: now };
-    item.trace = trace;
-    item.dirtyLoops += Math.max(0, Number(dirtyLoops) || 0);
-    this.#pendingWrites.set(key, item);
+  #appendTraceRecord(trace, type, record, { writeMeta = false } = {}) {
+    if (!this.#acceptingWrites || !trace?.requestKey || !record) return;
     this.#requestCache.set(trace.requestKey, trace);
-
-    if (force || item.dirtyLoops >= TRACE_FLUSH_DIRTY_LOOPS) {
+    const initialize = !this.#initializedRequestKeys.has(trace.requestKey);
+    if (initialize) this.#initializedRequestKeys.add(trace.requestKey);
+    this.#pendingWrites.push({
+      trace,
+      type,
+      record: cloneJsonValue(record),
+      initialize,
+      writeMeta: !!writeMeta,
+    });
+    if (writeMeta) {
       this.#flushPending();
       return;
     }
-    this.#scheduleFlushTimer(now);
-  }
-
-  #scheduleFlushTimer(now = Date.now()) {
-    if (this.#flushTimer || this.#pendingWrites.size === 0) return;
-    const oldestDirtyAt = Math.min(...Array.from(this.#pendingWrites.values()).map(item => item.firstDirtyAt || now));
-    const dueIn = Math.max(0, TRACE_FLUSH_INTERVAL_MS - (now - oldestDirtyAt));
-    this.#flushTimer = setTimeout(() => {
-      this.#flushTimer = null;
+    if (this.#appendTimer) return;
+    this.#appendTimer = setTimeout(() => {
+      this.#appendTimer = null;
       this.#flushPending();
-    }, dueIn);
-    if (typeof this.#flushTimer.unref === 'function') this.#flushTimer.unref();
+    }, TRACE_APPEND_BATCH_MS);
+    if (typeof this.#appendTimer.unref === 'function') this.#appendTimer.unref();
   }
 
   #scheduleEventFlush() {
@@ -1153,43 +1362,73 @@ export class DebugTrace {
     this.#eventFlushTimer = setTimeout(() => {
       this.#eventFlushTimer = null;
       this.#flushEvents();
-    }, TRACE_FLUSH_INTERVAL_MS);
+    }, EVENT_FLUSH_INTERVAL_MS);
     if (typeof this.#eventFlushTimer.unref === 'function') this.#eventFlushTimer.unref();
   }
 
-  /**
-   * Drain pending trace writes synchronously-from-the-caller's-view: snapshot
-   * each dirty trace to a JSON STRING now (so later in-place mutation of the
-   * live `loops` array can't tear the payload), then chain the async writes
-   * onto #flushChain. Returns nothing; callers that need durability await
-   * #drainWrites().
-   */
+  /** Queue append-only loop/tool records onto the single-writer chain. */
   #flushPending() {
-    if (this.#flushTimer) {
-      clearTimeout(this.#flushTimer);
-      this.#flushTimer = null;
+    if (this.#appendTimer) {
+      clearTimeout(this.#appendTimer);
+      this.#appendTimer = null;
     }
-    const entries = Array.from(this.#pendingWrites.values());
+    const entries = this.#pendingWrites.splice(0);
     if (entries.length === 0) {
       this.#flushEvents();
       return;
     }
-    this.#pendingWrites.clear();
-    // Capture point-in-time payloads synchronously (so later in-place mutation
-    // of the live `loops` array can't tear a payload), but remember each
-    // request key: a prune chained ahead of this write may evict the request
-    // before the write runs, and we must NOT resurrect a pruned trace on disk.
-    const jobs = entries.map(({ trace }) => ({
-      requestKey: trace.requestKey,
-      file: this.#traceFile(trace),
-      text: JSON.stringify(this.#serializableTrace(trace)),
-    }));
     this.#chain(async () => {
-      for (const { requestKey, file, text } of jobs) {
-        // Skip if an earlier chained prune already evicted this request.
-        if (!this.#requestCache.has(requestKey)) continue;
-        try { await atomicWriteText(file, text); }
-        catch (err) { console.warn('[Yeaft] debug trace write failed:', err?.message || err); }
+      const batches = new Map();
+      for (const entry of entries) {
+        if (!this.#requestCache.has(entry.trace.requestKey)) continue;
+        const requestDir = requestDirFor(this.#rootDir, entry.trace.sessionId || null, entry.trace.requestKey);
+        const batch = batches.get(requestDir) || { trace: entry.trace, initialize: false, writeMeta: false, lines: [] };
+        batch.trace = entry.trace;
+        batch.initialize ||= entry.initialize;
+        batch.writeMeta ||= entry.writeMeta;
+        batch.lines.push(`${JSON.stringify({ type: entry.type, record: entry.record })}\n`);
+        batches.set(requestDir, batch);
+      }
+      for (const [requestDir, batch] of batches) {
+        const { trace, initialize, writeMeta } = batch;
+        const lines = [...batch.lines];
+        const legacyRequestDir = trace._persistedFormat === 'legacy'
+          ? trace._persistedRequestDir || null
+          : null;
+        try {
+          const metaPath = requestMetaPath(requestDir);
+          if (initialize) {
+            const meta = serializableTraceMeta(trace);
+            if (legacyRequestDir) {
+              meta.legacyBaseRequest = cloneJsonValue(trace.baseRequest);
+              const legacyLines = [];
+              for (const loop of Array.isArray(trace.loops) ? trace.loops : []) {
+                legacyLines.push(`${JSON.stringify({ type: 'loop', record: loop })}\n`);
+              }
+              for (const tool of Array.isArray(trace.tools) ? trace.tools : []) {
+                legacyLines.push(`${JSON.stringify({ type: 'tool', record: tool })}\n`);
+              }
+              lines.unshift(...legacyLines);
+            }
+            await atomicWriteText(metaPath, JSON.stringify(meta));
+            await atomicWriteText(
+              turnLocatorPath(this.#rootDir, trace.sessionId || null, trace.requestId),
+              JSON.stringify({ requestKey: trace.requestKey, requestId: trace.requestId, sessionId: trace.sessionId || null })
+            );
+          }
+          await ensureDir(requestDir);
+          const eventPath = requestEventsPath(requestDir);
+          if (initialize) await prepareJsonlAppend(eventPath);
+          await fsp.appendFile(eventPath, lines.join(''), 'utf8');
+          if (writeMeta) await atomicWriteText(metaPath, JSON.stringify(serializableTraceMeta(trace)));
+          trace._persistedFormat = 'events';
+          trace._persistedRequestDir = requestDir;
+          if (legacyRequestDir && legacyRequestDir !== requestDir) {
+            await removeRequestDirIfIdentityMatches(legacyRequestDir, trace);
+          }
+        } catch (err) {
+          console.warn('[Yeaft] debug trace append failed:', err?.message || err);
+        }
       }
       try { await this.#pruneAll(REQUEST_RETENTION); }
       catch (err) { console.warn('[Yeaft] debug trace prune failed:', err?.message || err); }
@@ -1281,24 +1520,73 @@ export class DebugTrace {
   }
 
   async #pruneAll(keep) {
-    // Cache-only enumeration; rm the pruned request dirs async. No disk scan.
-    const sessions = new Set([null]);
+    const sessions = new Set();
     for (const trace of this.#requestCache.values()) sessions.add(trace.sessionId || null);
-    for (const sid of sessions) await this.#pruneSession(sid, keep);
+    for (const sessionId of sessions) await this.#pruneSession(sessionId, keep);
   }
 
   async #pruneSession(sessionId, keep = REQUEST_RETENTION) {
-    const traces = this.#traceSummaries(sessionId);
+    const sessionKey = sessionId || '';
+    let index = this.#retentionIndex.get(sessionKey);
+    if (!this.#reconciledRetentionSessions.has(sessionKey)) {
+      index = new Map();
+      for (const item of await readTraceHeaders(this.#rootDir, sessionId)) {
+        const existing = index.get(item.trace.requestKey);
+        if (existing) {
+          existing.requestDirs.add(item.requestDir);
+          const currentDir = requestDirFor(this.#rootDir, sessionId, item.trace.requestKey);
+          if (item.requestDir === currentDir || existing.trace?._persistedFormat === 'legacy') {
+            existing.trace = item.trace;
+          }
+          existing.openedAt = Math.min(existing.openedAt, item.openedAt);
+        } else {
+          index.set(item.trace.requestKey, {
+            trace: item.trace,
+            requestDirs: new Set([item.requestDir]),
+            openedAt: item.openedAt,
+          });
+        }
+      }
+      this.#retentionIndex.set(sessionKey, index);
+      this.#reconciledRetentionSessions.add(sessionKey);
+    }
+    for (const trace of this.#requestCache.values()) {
+      if ((trace.sessionId || null) !== sessionId) continue;
+      const requestDir = requestDirFor(this.#rootDir, sessionId, trace.requestKey);
+      const existing = index.get(trace.requestKey);
+      index.set(trace.requestKey, {
+        trace,
+        requestDirs: new Set([...(existing?.requestDirs || []), requestDir]),
+        openedAt: Math.min(Number(trace.openedAt || 0), existing?.openedAt ?? Infinity),
+      });
+    }
+    const traces = Array.from(index.values()).sort((a, b) => (
+      (a.openedAt || 0) - (b.openedAt || 0)
+      || String(a.trace.requestKey).localeCompare(String(b.trace.requestKey))
+    ));
     const activeCutoff = Date.now() - 6 * 60 * 60 * 1000;
     const protectedItems = traces.filter(item => item.trace?.active && Number(item.trace?.updatedAt || 0) >= activeCutoff);
     const pruneCandidates = traces.filter(item => !protectedItems.includes(item));
     const stale = pruneCandidates.slice(0, Math.max(0, traces.length - protectedItems.length - keep));
     for (const item of stale) {
-      // Drop from cache first so subsequent queries never resurface it, even
-      // if the async rm is still in flight or fails.
       this.#requestCache.delete(item.trace.requestKey);
-      try { await fsp.rm(dirname(item.file), { recursive: true, force: true }); }
-      catch { /* ignore */ }
+      this.#initializedRequestKeys.delete(item.trace.requestKey);
+      index.delete(item.trace.requestKey);
+      for (const requestDir of item.requestDirs) {
+        await removeRequestDirIfIdentityMatches(requestDir, item.trace);
+      }
+      const locatorPaths = new Set(sessionRequestDirs(this.#rootDir, sessionId).map(requestsDir => (
+        join(requestsDir, '..', 'turns', turnLocatorName(item.trace.requestId))
+      )));
+      for (const locatorPath of locatorPaths) {
+        const locator = await readJson(locatorPath);
+        if (locator?.sessionId === sessionId
+          && locator.requestId === item.trace.requestId
+          && locator.requestKey === item.trace.requestKey) {
+          try { await fsp.rm(locatorPath, { force: true }); }
+          catch { /* ignore */ }
+        }
+      }
     }
   }
 
@@ -1331,6 +1619,7 @@ export class NullTrace {
   async purge() {}
   async close() {}
   async flush() {}
+  async fetchTurnDebug() { return { loops: [], turns: [], dreamEvents: [] }; }
   async fetchRecentDebugHistory() { return { loops: [], turns: [], dreamEvents: [] }; }
 }