@zergai/ztc 0.4.0-beta.1 → 0.5.0-beta.1

package/dist/agent/tool_response_store.js

@@ -1,28 +1,52 @@
 /**
  * ToolResponseStore — disk-backed store for tool results.
  *
- * Every tool result is written to
- *   ~/.ztc/sessions/{projectHash}/{sessionId}/tool_responses/{tool_use_id}.json
+ * Non-ephemeral tool results are written to
+ *   ~/.ztc/sessions/{projectHash}/{sessionId}/tool_responses/{bounded source filename}.json
  * The conversation thread keeps a small placeholder for large/old results and
  * inlines the full text only when the request-level inline budget allows it.
  * Mirror SSE events carry the full content from the agent's `tool_end`
  * emission, so UX is untouched — the store only governs what appears in the
- * model's context.
+ * model's context. Explicit host-selected ephemeral tools remain memory-only;
+ * their durable evidence contains a placeholder rather than the private body.
  */
-import { mkdir, readFile, readdir } from 'fs/promises';
+import { lstat, mkdir, unlink } from 'fs/promises';
 import { join } from 'path';
+import { createHash } from 'crypto';
 import { estimateTokens, getContextWindowSize } from '../utils/context.js';
 import { writeFileAtomic } from '../utils/atomic_write.js';
 import { debugLog } from '../debug/logger.js';
+import { environmentEphemeralToolNames, environmentEphemeralToolRules, isEphemeralToolCall, EPHEMERAL_TOOL_PLACEHOLDER, validateEphemeralToolNames, validateEphemeralToolRules } from '../utils/ephemeral_tools.js';
 import { compressToolResult, hashToolResultBody, isReadToolName, } from './tool_compression.js';
 import { parseImageToolResult } from './tool_result_parsing.js';
 import { CORE_DISCOVERY_TOOL_NAMES } from './tools/deferred.js';
+import { readBoundedSource, SourceAdmissionError } from './tool_response_source.js';
+import { collectResponseFilenames, emptyDirectoryAdmission } from './tool_response_directory.js';
+import { EVIDENCE_MAX_SNAPSHOT_BYTES, EvidenceLedger, parseEvidenceSnapshot } from './runtime/evidence.js';
+export const TOOL_RESPONSE_MAX_STORED_RESPONSES = 128;
+export const TOOL_RESPONSE_MAX_STORED_BYTES = 8 * 1024 * 1024;
+export class ToolResponseIntegrityError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'ToolResponseIntegrityError';
+    }
+}
 export const INLINE_SIZE_THRESHOLD_TOKENS_DEFAULT = 2000;
 export const INLINE_AGE_TURNS_DEFAULT = 8;
 export const CONFIGURED_INLINE_BUDGET_TOKENS_DEFAULT = INLINE_SIZE_THRESHOLD_TOKENS_DEFAULT * 8; // 16k
 const PER_MODEL_BUDGET_FRACTION = 0.25;
 const PREVIEW_HEAD_TOKEN_CAP = 150;
 const PREVIEW_TAIL_TOKEN_CAP = 50;
+const sourceDigest = (content) => createHash('sha256').update(content, 'utf8').digest('hex');
+function validToolId(id) {
+    if (!/^[A-Za-z0-9_-]+$/.test(id) || Buffer.byteLength(id) > 256)
+        throw new TypeError('Invalid tool_use_id identity');
+}
+// Leave room for the atomic writer's PID/random suffix on 255-byte filesystems.
+// The reserved dot prefix cannot collide with any accepted logical identity.
+function sourceFilename(id) {
+    return Buffer.byteLength(id) <= 200 ? `${id}.json` : `.source-${sourceDigest(id)}.json`;
+}
 /**
  * Render the canonical offload placeholder string. Centralized so the format
  * stays in sync with `describeOffloadPlaceholderFormat` below (quoted in the
@@ -137,8 +161,11 @@ function computePreview(content) {
     };
 }
 export class ToolResponseStore {
+    evidence = new EvidenceLedger();
     dir = null;
     config = {
+        ephemeralToolNames: environmentEphemeralToolNames(),
+        ephemeralToolRules: environmentEphemeralToolRules(),
         offloadEnabled: false,
         inlineSizeThresholdTokens: INLINE_SIZE_THRESHOLD_TOKENS_DEFAULT,
         inlineAgeTurns: INLINE_AGE_TURNS_DEFAULT,
@@ -152,6 +179,19 @@ export class ToolResponseStore {
     totalCachedTokens = 0;
     turnCounter = 0;
     pendingWrites = new Set();
+    persistenceError = null;
+    failedSourceWrites = new Map();
+    persistenceRecovery = null;
+    loadingEvidence = false;
+    generation = 0;
+    indexDirty = false;
+    indexWrite = null;
+    availability = new Map();
+    invalidSources = new Map();
+    retention;
+    retainedSources = new Map();
+    directoryAdmission = emptyDirectoryAdmission();
+    retentionQueue = Promise.resolve();
     // Ids that the most recent selectInlineSet pass decided to render inline.
     // Read by willInlineNextRender (used by the recall tool to pick ack vs
     // content shape) and by the agent dispatch short-circuit.
@@ -162,8 +202,135 @@ export class ToolResponseStore {
     // later request emits byte-identical text (prompt-cache prefix stability).
     renderMemo = new Map();
     compressionStats = { compressedResults: 0, dedupStubs: 0, tokensSaved: 0 };
+    constructor(retention = {}) {
+        this.retention = { maxStoredResponses: retention.maxStoredResponses ?? TOOL_RESPONSE_MAX_STORED_RESPONSES,
+            maxStoredBytes: retention.maxStoredBytes ?? TOOL_RESPONSE_MAX_STORED_BYTES };
+        for (const [field, maximum] of [['maxStoredResponses', TOOL_RESPONSE_MAX_STORED_RESPONSES], ['maxStoredBytes', TOOL_RESPONSE_MAX_STORED_BYTES]]) {
+            const value = this.retention[field];
+            if (!Number.isSafeInteger(value) || value < 1 || value > maximum)
+                throw new RangeError(`${field} must be an integer between 1 and ${maximum}`);
+        }
+        this.evidence.onChange(() => this.scheduleEvidenceWrite());
+    }
+    retentionVictim(ids) {
+        const snapshot = this.evidence.getSnapshot();
+        const current = new Set(snapshot.currentIds);
+        const referenced = new Set();
+        for (const record of snapshot.records) {
+            if ((record.kind === 'hypothesis' || record.kind === 'conclusion') && current.has(record.id)) {
+                for (const id of record.supportIds)
+                    referenced.add(id);
+            }
+        }
+        const priority = (id) => referenced.has(this.meta.get(id)?.observation_id ?? id) ? 1 : 0;
+        return ids.reduce((oldest, id) => priority(id) < priority(oldest)
+            || (priority(id) === priority(oldest) && (this.meta.get(id)?.turn ?? 0) < (this.meta.get(oldest)?.turn ?? 0)) ? id : oldest);
+    }
+    forgetSource(id) {
+        const observationId = this.meta.get(id)?.observation_id ?? id;
+        const cached = this.contentCache.get(id);
+        if (cached !== undefined)
+            this.totalCachedTokens -= Math.max(1, estimateTokens(cached));
+        this.contentCache.delete(id);
+        this.meta.delete(id);
+        this.renderMemo.delete(id);
+        this.lastRenderInline.delete(id);
+        this.retainedSources.delete(id);
+        this.availability.delete(id);
+        this.invalidSources.delete(id);
+        this.failedSourceWrites.delete(id);
+        this.evidence.setAvailability(observationId, 'unavailable');
+    }
+    overRetentionLimit() {
+        return this.retainedSources.size > this.retention.maxStoredResponses
+            || [...this.retainedSources.values()].reduce((total, source) => total + source.bytes, 0) > this.retention.maxStoredBytes;
+    }
+    async enforceRetention(generation = this.generation) {
+        const directory = this.dir;
+        while (this.overRetentionLimit()) {
+            if (generation !== this.generation)
+                return;
+            const id = this.retentionVictim([...this.retainedSources.keys()]);
+            const original = this.retainedSources.get(id);
+            if (directory && original.dev !== undefined && original.ino !== undefined) {
+                const path = join(directory, original.filename);
+                try {
+                    const info = await lstat(path);
+                    if (!info.isFile() || info.dev !== original.dev || info.ino !== original.ino)
+                        throw new ToolResponseIntegrityError(`Refusing retention cleanup of changed source file: ${id}`);
+                    const source = await this.readSource(path, this.retention.maxStoredBytes);
+                    if (source.info.dev !== original.dev || source.info.ino !== original.ino)
+                        throw new ToolResponseIntegrityError(`Refusing retention cleanup of changed source file: ${id}`);
+                    this.decodeRecord(source.text, id);
+                    if (generation !== this.generation)
+                        return;
+                    await unlink(path);
+                }
+                catch (error) {
+                    if (error.code !== 'ENOENT')
+                        throw new Error(`Tool response retention pruning failed: ${error instanceof Error ? error.message : String(error)}`);
+                }
+            }
+            if (generation !== this.generation)
+                return;
+            this.forgetSource(id);
+        }
+    }
+    boundMemoryCache() {
+        const cacheBytes = () => [...this.contentCache.values()].reduce((total, content) => total + Buffer.byteLength(content), 0);
+        while (this.contentCache.size > this.retention.maxStoredResponses || cacheBytes() > this.retention.maxStoredBytes) {
+            const id = this.retentionVictim([...this.contentCache.keys()]);
+            if (!this.dir)
+                this.forgetSource(id);
+            else {
+                const content = this.contentCache.get(id);
+                this.totalCachedTokens -= Math.max(1, estimateTokens(content));
+                this.contentCache.delete(id);
+            }
+        }
+    }
+    scheduleEvidenceWrite() {
+        if (!this.dir || this.loadingEvidence)
+            return;
+        this.indexDirty = true;
+        if (this.indexWrite)
+            return;
+        const directory = this.dir;
+        const generation = this.generation;
+        const task = Promise.resolve().then(async () => {
+            while (this.indexDirty && generation === this.generation) {
+                this.indexDirty = false;
+                await writeFileAtomic(join(directory, '.evidence.json'), JSON.stringify(this.evidence.getSnapshot()));
+            }
+        }).catch((error) => {
+            if (generation !== this.generation)
+                return;
+            this.persistenceError ??= new Error(`Evidence index persistence failed: ${error instanceof Error ? error.message : String(error)}`);
+            this.loadingEvidence = true;
+            try {
+                for (const [id] of this.meta) {
+                    this.availability.set(id, 'unavailable');
+                    this.evidence.setAvailability(this.meta.get(id)?.observation_id ?? id, 'unavailable');
+                }
+            }
+            finally {
+                this.loadingEvidence = false;
+            }
+            this.indexDirty = false;
+        }).finally(() => {
+            this.pendingWrites.delete(task);
+            if (this.indexWrite === task) {
+                this.indexWrite = null;
+                if (this.indexDirty)
+                    this.scheduleEvidenceWrite();
+            }
+        });
+        this.indexWrite = task;
+        this.pendingWrites.add(task);
+    }
     setConfig(cfg) {
-        this.config = { ...this.config, ...cfg };
+        this.config = { ...this.config, ...cfg, ...(cfg.ephemeralToolNames ? { ephemeralToolNames: validateEphemeralToolNames(cfg.ephemeralToolNames) } : {}),
+            ...(cfg.ephemeralToolRules ? { ephemeralToolRules: validateEphemeralToolRules(cfg.ephemeralToolRules) } : {}) };
     }
     getConfig() {
         return { ...this.config };
@@ -173,9 +340,11 @@ export class ToolResponseStore {
      * fraction of the active model's context window so a small-context model
      * (gpt-4 8k → 2k effective) doesn't get crushed by a Claude-tuned default.
      */
-    effectiveBudget(model) {
+    effectiveBudget(model, windowOverride) {
         const configured = this.config.configuredInlineBudgetTokens;
-        const window = getContextWindowSize(model);
+        const window = getContextWindowSize(model, windowOverride);
+        if (window === 0)
+            return configured;
         return Math.min(configured, Math.floor(window * PER_MODEL_BUDGET_FRACTION));
     }
     cacheBudgetTokens() {
@@ -189,43 +358,134 @@ export class ToolResponseStore {
      */
     async setSessionDir(dir) {
         await this.flush();
+        this.clear();
         this.dir = dir;
-        this.meta.clear();
-        this.contentCache.clear();
-        this.totalCachedTokens = 0;
-        this.turnCounter = 0;
-        this.lastRenderInline.clear();
-        this.renderMemo.clear();
-        this.resetCompressionStats();
-        await mkdir(dir, { recursive: true });
-        let files;
+        this.loadingEvidence = true;
+        let shouldPersist = false;
         try {
-            files = await readdir(dir);
-        }
-        catch {
-            return;
-        }
-        const jsonFiles = files.filter(f => f.endsWith('.json'));
-        const records = await Promise.all(jsonFiles.map(async (f) => {
+            await mkdir(dir, { recursive: true });
+            let hadIndex = false;
             try {
-                const raw = await readFile(join(dir, f), 'utf-8');
-                const rec = JSON.parse(raw);
-                return rec?.meta?.tool_use_id ? rec.meta : null;
+                const rawIndex = await this.readSource(join(dir, '.evidence.json'), EVIDENCE_MAX_SNAPSHOT_BYTES);
+                const index = parseEvidenceSnapshot(JSON.parse(rawIndex.text));
+                if (!index)
+                    throw new ToolResponseIntegrityError('Invalid persisted evidence index');
+                this.evidence.restore(index);
+                hadIndex = true;
             }
-            catch {
-                return null;
+            catch (error) {
+                if (error.code !== 'ENOENT')
+                    throw error;
             }
-        }));
-        for (const m of records) {
-            if (!m)
-                continue;
-            this.meta.set(m.tool_use_id, m);
-            if (m.turn >= this.turnCounter)
-                this.turnCounter = m.turn + 1;
+            const before = JSON.stringify(this.evidence.getSnapshot());
+            const indexedIds = new Set();
+            const indexedNames = new Set();
+            for (const record of this.evidence.getSnapshot().records) {
+                if (record.kind !== 'observation' || !record.source.toolCallId)
+                    continue;
+                const id = record.source.toolCallId;
+                try {
+                    validToolId(id);
+                }
+                catch {
+                    this.evidence.setAvailability(record.id, 'unavailable');
+                    continue;
+                }
+                indexedIds.add(id);
+                indexedNames.add(sourceFilename(id));
+                if (Buffer.byteLength(id) > 200 && Buffer.byteLength(id) <= 250)
+                    indexedNames.add(`${id}.json`);
+            }
+            let files;
+            try {
+                const collected = await collectResponseFilenames(dir, [...indexedNames]);
+                files = collected.files;
+                this.directoryAdmission = collected.admission;
+            }
+            catch (error) {
+                this.directoryAdmission = { ...emptyDirectoryAdmission(), status: 'failed', reasons: ['directory_error'],
+                    errorCode: String(error.code ?? 'UNKNOWN').slice(0, 64) };
+                throw error;
+            }
+            // Sequential reads avoid loading every historical source body concurrently.
+            for (const file of files) {
+                let id = file.slice(0, -5);
+                try {
+                    const info = await lstat(join(dir, file));
+                    if (!info.isFile())
+                        continue;
+                    const source = await this.readSource(join(dir, file), this.retention.maxStoredBytes);
+                    if (file.startsWith('.source-')) {
+                        const indexed = this.evidence.getSnapshot().records.find(record => record.kind === 'observation'
+                            && record.source.toolCallId && sourceFilename(record.source.toolCallId) === file);
+                        const candidate = JSON.parse(source.text);
+                        const identity = indexed?.kind === 'observation' ? indexed.source.toolCallId : candidate?.meta?.tool_use_id;
+                        if (typeof identity !== 'string')
+                            throw new ToolResponseIntegrityError(`Invalid source filename identity: ${file}`);
+                        id = identity;
+                    }
+                    validToolId(id);
+                    if (file !== sourceFilename(id) && file !== `${id}.json`)
+                        throw new ToolResponseIntegrityError(`Invalid source filename identity: ${id}`);
+                    if (this.invalidSources.has(id))
+                        throw this.invalidSources.get(id);
+                    if (this.retainedSources.has(id))
+                        throw new ToolResponseIntegrityError(`Ambiguous duplicate source filenames: ${id}`);
+                    const record = this.decodeRecord(source.text, id);
+                    if (record.meta.ephemeral || this.config.ephemeralToolNames?.includes(record.meta.tool_name))
+                        continue;
+                    // Discovery cannot displace the current index with unindexed historical sources.
+                    if (hadIndex && !indexedIds.has(id) && (this.retainedSources.size >= this.retention.maxStoredResponses
+                        || [...this.retainedSources.values()].reduce((total, entry) => total + entry.bytes, 0) + source.info.size > this.retention.maxStoredBytes)) {
+                        this.directoryAdmission.skippedSources++;
+                        if (!this.directoryAdmission.reasons.includes('source_retention_limit'))
+                            this.directoryAdmission.reasons.push('source_retention_limit');
+                        this.directoryAdmission.status = 'limited';
+                        this.directoryAdmission.complete = false;
+                        continue;
+                    }
+                    this.admitLoadedRecord(record, !hadIndex || (record.schemaVersion === undefined && this.evidence.getSnapshot().omittedRecords === 0));
+                    this.retainedSources.set(id, { bytes: source.info.size, filename: file, dev: source.info.dev, ino: source.info.ino });
+                }
+                catch (error) {
+                    if (error.code === 'ENOENT')
+                        continue;
+                    this.meta.delete(id);
+                    this.retainedSources.delete(id);
+                    this.markInvalid(id, error);
+                    continue;
+                }
+                // A valid owned source may be pruned; an I/O refusal is a failed bind,
+                // not a malformed unrelated file that can be ignored.
+                await this.enforceRetention();
+            }
+            this.directoryAdmission.admittedSources = this.retainedSources.size;
+            for (const record of this.evidence.getSnapshot().records) {
+                if (record.kind === 'observation' && record.source.toolCallId && !this.meta.has(record.source.toolCallId)) {
+                    this.evidence.setAvailability(record.id, 'unavailable');
+                }
+            }
+            shouldPersist = before !== JSON.stringify(this.evidence.getSnapshot());
+        }
+        finally {
+            this.loadingEvidence = false;
         }
+        if (shouldPersist)
+            this.scheduleEvidenceWrite();
     }
     clear() {
+        this.generation++;
         this.dir = null;
+        this.indexDirty = false;
+        this.indexWrite = null;
+        this.persistenceError = null;
+        this.failedSourceWrites.clear();
+        this.persistenceRecovery = null;
+        this.availability.clear();
+        this.invalidSources.clear();
+        this.retainedSources.clear();
+        this.directoryAdmission = emptyDirectoryAdmission();
+        this.retentionQueue = Promise.resolve();
         this.meta.clear();
         this.contentCache.clear();
         this.totalCachedTokens = 0;
@@ -233,10 +493,15 @@ export class ToolResponseStore {
         this.lastRenderInline.clear();
         this.renderMemo.clear();
         this.resetCompressionStats();
+        this.evidence.clear();
     }
     hasSession() {
         return this.dir !== null;
     }
+    /** A bounded discovery result never claims that unseen historical files are absent. */
+    getDirectoryAdmission() {
+        return { ...this.directoryAdmission, reasons: [...this.directoryAdmission.reasons] };
+    }
     /** True when the content for `toolUseId` is currently in the in-memory cache. */
     hasCached(toolUseId) {
         return this.contentCache.has(toolUseId);
@@ -247,7 +512,113 @@ export class ToolResponseStore {
      * also used by tests that need to observe a settled filesystem.
      */
     async flush() {
-        await Promise.all(this.pendingWrites);
+        await this.awaitWrites();
+        if (this.persistenceError && !this.persistenceRecovery)
+            this.persistenceRecovery = this.recoverPersistence();
+        const recovery = this.persistenceRecovery;
+        if (recovery) {
+            try {
+                await recovery;
+            }
+            finally {
+                if (this.persistenceRecovery === recovery)
+                    this.persistenceRecovery = null;
+            }
+        }
+        await this.awaitWrites();
+        if (this.persistenceError)
+            throw this.persistenceError;
+    }
+    async awaitWrites() {
+        while (this.pendingWrites.size)
+            await Promise.all([...this.pendingWrites]);
+    }
+    rememberFailedSource(id, serialized) {
+        if (!this.meta.has(id))
+            return; // Already retired under the raw-source quota.
+        this.failedSourceWrites.set(id, serialized);
+        const bytes = () => [...this.failedSourceWrites.values()].reduce((total, value) => total + Buffer.byteLength(value), 0);
+        while (this.failedSourceWrites.size > this.retention.maxStoredResponses || bytes() > this.retention.maxStoredBytes) {
+            this.forgetSource(this.retentionVictim([...this.failedSourceWrites.keys()]));
+        }
+    }
+    /** Retry stored bytes and the evidence index, never the producing tool. */
+    async recoverPersistence() {
+        const directory = this.dir;
+        const generation = this.generation;
+        if (!directory)
+            return;
+        this.persistenceError = null;
+        for (const [id, serialized] of [...this.failedSourceWrites])
+            this.scheduleSourceWrite(id, serialized);
+        await this.awaitWrites();
+        if (generation !== this.generation || this.persistenceError)
+            return;
+        // Index failure can mark otherwise durable sources unavailable. Re-admit
+        // only their actual unchanged bytes before republishing availability.
+        this.loadingEvidence = true;
+        try {
+            for (const [id, retained] of this.retainedSources) {
+                if (this.availability.get(id) === 'persisted')
+                    continue;
+                const source = await this.readSource(join(directory, retained.filename), this.retention.maxStoredBytes);
+                if (generation !== this.generation)
+                    return;
+                const record = this.decodeRecord(source.text, id);
+                if (source.info.dev !== retained.dev || source.info.ino !== retained.ino
+                    || JSON.stringify(record.meta) !== JSON.stringify(this.meta.get(id))) {
+                    throw new ToolResponseIntegrityError(`Refusing recovery of changed tool source: ${id}`);
+                }
+                this.availability.set(id, 'persisted');
+                this.evidence.setAvailability(record.meta.observation_id, 'persisted');
+            }
+        }
+        catch (error) {
+            if (generation === this.generation)
+                this.persistenceError ??= new Error(`Tool response recovery persistence failed: ${String(error)}`);
+        }
+        finally {
+            if (generation === this.generation)
+                this.loadingEvidence = false;
+        }
+        if (generation === this.generation)
+            this.scheduleEvidenceWrite();
+    }
+    scheduleSourceWrite(toolUseId, serialized) {
+        const directory = this.dir;
+        const generation = this.generation;
+        const meta = this.meta.get(toolUseId);
+        const filename = sourceFilename(toolUseId);
+        const path = join(directory, filename);
+        const write = writeFileAtomic(path, serialized)
+            .then(async () => {
+            if (generation !== this.generation)
+                return;
+            const info = await lstat(path);
+            if (generation !== this.generation)
+                return;
+            if (!info.isFile())
+                throw new ToolResponseIntegrityError(`Published source is not a regular file: ${toolUseId}`);
+            this.retainedSources.set(toolUseId, { bytes: info.size, filename, dev: info.dev, ino: info.ino });
+            this.availability.set(toolUseId, 'persisted');
+            this.evidence.setAvailability(meta.observation_id, 'persisted');
+            this.retentionQueue = this.retentionQueue.catch(() => undefined).then(() => this.enforceRetention(generation));
+            await this.retentionQueue;
+            if (generation === this.generation)
+                this.failedSourceWrites.delete(toolUseId);
+        })
+            .catch((error) => {
+            if (generation !== this.generation)
+                return;
+            const message = error instanceof Error ? error.message : String(error);
+            this.persistenceError ??= new Error(`Tool response persistence failed for ${toolUseId}: ${message}`);
+            this.rememberFailedSource(toolUseId, serialized);
+            this.availability.set(toolUseId, 'unavailable');
+            this.evidence.setAvailability(meta.observation_id, 'unavailable');
+            debugLog(`[ToolResponseStore] atomic write failed for ${toolUseId}: ${message}`);
+        });
+        const task = write.finally(() => this.pendingWrites.delete(task));
+        this.pendingWrites.add(task);
     }
     /**
      * Persist a tool result and return the text to inject into context (full
@@ -255,72 +626,218 @@ export class ToolResponseStore {
      * fire-and-forget.
      */
     put(toolUseId, toolName, content, partial = false, context) {
+        validToolId(toolUseId);
+        if (this.invalidSources.has(toolUseId))
+            throw this.invalidSources.get(toolUseId);
+        const sha256 = sourceDigest(content);
+        const old = this.meta.get(toolUseId);
+        if (old) {
+            if (old.result_sha256 !== sha256 || old.tool_name !== toolName || Boolean(old.partial) !== partial
+                || Boolean(old.is_error) !== Boolean(context?.isError)
+                || (context?.observationId !== undefined && context.observationId !== old.observation_id)) {
+                throw new Error(`Immutable tool result identity conflict: ${toolUseId}`);
+            }
+            return { meta: structuredClone(old), rendered: this.renderForContext(toolUseId) ?? content };
+        }
         const sizeTokens = Math.max(1, estimateTokens(content));
         const withinSize = sizeTokens <= this.config.inlineSizeThresholdTokens;
         const isImage = parseImageToolResult(content) !== null;
         const isDiscovery = CORE_DISCOVERY_TOOL_NAMES.has(toolName);
+        const ephemeral = isEphemeralToolCall(toolName, context?.args, this.config.ephemeralToolNames, this.config.ephemeralToolRules);
         // Only generate a preview when the result will actually be offloaded.
         // Inlined small/image/discovery results show full content, no placeholder.
-        const preview = withinSize || isImage || isDiscovery ? null : computePreview(content);
+        const preview = ephemeral || withinSize || isImage || isDiscovery ? null : computePreview(content);
         const meta = {
+            ...(ephemeral ? { ephemeral: true } : {}),
             tool_name: toolName,
             tool_use_id: toolUseId,
             size_tokens: sizeTokens,
             timestamp: new Date().toISOString(),
             turn: this.turnCounter++,
             partial,
+            result_sha256: sha256,
+            observation_id: context?.observationId ?? toolUseId,
             ...(context?.isError === undefined ? {} : { is_error: context.isError }),
             preview,
         };
-        const sourcePath = context?.sourcePath ?? null;
+        const sourcePath = ephemeral ? null : context?.sourcePath ?? null;
         if (sourcePath)
             meta.source_path = sourcePath;
         if (sourcePath && context?.dedupable && isReadToolName(toolName)) {
             meta.content_hash = hashToolResultBody(content);
         }
+        this.evidence.observe({ id: meta.observation_id, recordedAt: meta.timestamp,
+            source: { origin: 'host_tool', ...(sourcePath ? { path: sourcePath } : {}), ...(ephemeral ? {} : context?.source), toolCallId: toolUseId, toolName },
+            excerpt: ephemeral ? EPHEMERAL_TOOL_PLACEHOLDER : content, resultSha256: sha256, resultBytes: Buffer.byteLength(content), partial,
+            isError: Boolean(context?.isError), availability: 'memory',
+            ...(ephemeral || context?.source?.job?.cursor === undefined ? {} : { cursor: context.source.job.cursor }) });
+        const observed = this.evidence.getSnapshot().records.find(record => record.id === meta.observation_id);
+        const observation = observed?.kind === 'observation' ? { ...observed, availability: 'persisted' } : undefined;
+        const serialized = JSON.stringify({ schemaVersion: 1, meta, content, observation });
+        const serializedBytes = Buffer.byteLength(serialized);
+        if (serializedBytes > this.retention.maxStoredBytes) {
+            this.evidence.setAvailability(meta.observation_id, 'unavailable');
+            // Preserve the caller's last usable full copy instead of offloading to nonexistent bytes.
+            return { meta: structuredClone(meta), rendered: content };
+        }
         this.meta.set(toolUseId, meta);
+        this.availability.set(toolUseId, 'memory');
         this.renderMemo.delete(toolUseId);
         this.cacheContent(toolUseId, content);
-        if (this.dir) {
-            const path = join(this.dir, `${toolUseId}.json`);
-            // Fire-and-forget — put() stays off the await path. Write failures must
-            // still surface: a silently-dropped write becomes a permanent null from
-            // recall_tool_response once the LRU evicts the in-memory copy.
-            const write = writeFileAtomic(path, JSON.stringify({ meta, content }))
-                .catch((err) => {
-                const message = err instanceof Error ? err.message : String(err);
-                debugLog(`[ToolResponseStore] atomic write failed for ${toolUseId}: ${message}`);
-            });
-            const task = write.finally(() => this.pendingWrites.delete(task));
-            this.pendingWrites.add(task);
-        }
-        const shouldOffload = this.config.offloadEnabled && !withinSize && !isImage && !isDiscovery;
+        if (this.dir && !ephemeral) {
+            this.scheduleSourceWrite(toolUseId, serialized);
+        }
+        else {
+            this.retainedSources.set(toolUseId, { bytes: serializedBytes });
+            while (this.overRetentionLimit())
+                this.forgetSource(this.retentionVictim([...this.retainedSources.keys()]));
+        }
+        const shouldOffload = !ephemeral && this.config.offloadEnabled && !withinSize && !isImage && !isDiscovery;
         const rendered = shouldOffload ? formatOffloadPlaceholder(meta) : content;
-        return { meta, rendered };
+        return { meta: structuredClone(meta), rendered };
     }
     /** Load the full content for a tool_use_id. Returns null if missing. */
     async get(toolUseId) {
+        validToolId(toolUseId);
+        if (this.invalidSources.has(toolUseId))
+            throw this.invalidSources.get(toolUseId);
         const cached = this.contentCache.get(toolUseId);
         if (cached !== undefined)
             return cached;
+        if (this.meta.get(toolUseId)?.ephemeral || this.config.ephemeralToolNames?.includes(this.meta.get(toolUseId)?.tool_name ?? ''))
+            return null;
         if (!this.dir)
             return null;
         try {
-            const raw = await readFile(join(this.dir, `${toolUseId}.json`), 'utf-8');
-            const rec = JSON.parse(raw);
-            if (rec && typeof rec.content === 'string') {
-                if (rec.meta && !this.meta.has(toolUseId)) {
-                    this.meta.set(toolUseId, rec.meta);
-                }
-                this.cacheContent(toolUseId, rec.content);
-                return rec.content;
+            let filename = this.retainedSources.get(toolUseId)?.filename ?? sourceFilename(toolUseId);
+            let source;
+            try {
+                source = await this.readSource(join(this.dir, filename), this.retention.maxStoredBytes);
+            }
+            catch (error) {
+                // Longer legacy records can be read in place; never issue an overlong lookup.
+                if (error.code !== 'ENOENT' || filename === `${toolUseId}.json`
+                    || Buffer.byteLength(toolUseId) > 250)
+                    throw error;
+                filename = `${toolUseId}.json`;
+                source = await this.readSource(join(this.dir, filename), this.retention.maxStoredBytes);
             }
+            const rec = this.decodeRecord(source.text, toolUseId);
+            if (rec.meta.ephemeral || this.config.ephemeralToolNames?.includes(rec.meta.tool_name))
+                return null;
+            this.admitLoadedRecord(rec, false);
+            this.retainedSources.set(toolUseId, { bytes: source.info.size, filename, dev: source.info.dev, ino: source.info.ino });
+            await this.enforceRetention();
+            if (!this.meta.has(toolUseId))
+                return null;
+            this.cacheContent(toolUseId, rec.content);
+            return rec.content;
         }
-        catch {
-            // ENOENT or corrupt file — treat as missing.
+        catch (error) {
+            if (error.code !== 'ENOENT') {
+                this.markInvalid(toolUseId, error);
+                throw this.invalidSources.get(toolUseId);
+            }
+            if (this.meta.has(toolUseId))
+                this.availability.set(toolUseId, 'unavailable');
+            this.evidence.setAvailability(this.meta.get(toolUseId)?.observation_id ?? toolUseId, 'unavailable');
         }
         return null;
     }
+    async getChunk(toolUseId, options = {}) {
+        const cursor = options.cursor ?? 0;
+        const maxBytes = options.maxBytes ?? 8192;
+        if (!Number.isSafeInteger(cursor) || cursor < 0)
+            throw new RangeError('cursor must be a nonnegative safe byte offset');
+        if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0 || maxBytes > 32768)
+            throw new RangeError('maxBytes must be an integer between 1 and 32768');
+        const content = await this.get(toolUseId);
+        if (content === null)
+            return null;
+        const body = Buffer.from(content, 'utf8');
+        if (cursor > body.length || (cursor < body.length && (body[cursor] & 0xc0) === 0x80))
+            throw new RangeError('cursor must identify a UTF-8 boundary within the source');
+        let end = Math.min(body.length, cursor + maxBytes);
+        while (end > cursor && end < body.length && (body[end] & 0xc0) === 0x80)
+            end--;
+        if (end === cursor && cursor < body.length)
+            throw new RangeError('maxBytes cannot fit the next UTF-8 character');
+        const meta = this.meta.get(toolUseId);
+        return { toolUseId, observationId: meta.observation_id ?? toolUseId, content: body.subarray(cursor, end).toString('utf8'),
+            cursor, nextCursor: end, totalBytes: body.length, sha256: meta.result_sha256 ?? sourceDigest(content),
+            complete: end === body.length, availability: this.availability.get(toolUseId) ?? 'memory' };
+    }
+    markInvalid(id, error) {
+        const failure = error instanceof ToolResponseIntegrityError ? error
+            : new ToolResponseIntegrityError(`Source integrity failed for ${id}: ${error instanceof Error ? error.message : String(error)}`);
+        this.invalidSources.set(id, failure);
+        this.availability.set(id, 'unavailable');
+        while (this.invalidSources.size > this.retention.maxStoredResponses) {
+            const oldest = this.invalidSources.keys().next().value;
+            this.invalidSources.delete(oldest);
+            if (!this.meta.has(oldest))
+                this.availability.delete(oldest);
+        }
+        const source = this.evidence.getSnapshot().records.find(record => record.kind === 'observation' && record.source.toolCallId === id);
+        this.evidence.setAvailability(source?.id ?? this.meta.get(id)?.observation_id ?? id, 'unavailable');
+    }
+    async readSource(path, maximumBytes) {
+        try {
+            return await readBoundedSource(path, maximumBytes);
+        }
+        catch (error) {
+            if (error instanceof SourceAdmissionError)
+                throw new ToolResponseIntegrityError(error.message);
+            throw error;
+        }
+    }
+    decodeRecord(raw, id) {
+        const record = JSON.parse(raw);
+        if (!record || typeof record.content !== 'string' || !record.meta || record.meta.tool_use_id !== id
+            || typeof record.meta.tool_name !== 'string' || !record.meta.tool_name
+            || !Number.isSafeInteger(record.meta.turn) || record.meta.turn < 0
+            || !Number.isFinite(Date.parse(record.meta.timestamp))
+            || (record.schemaVersion !== undefined && record.schemaVersion !== 1)) {
+            throw new ToolResponseIntegrityError(`Invalid source record identity: ${id}`);
+        }
+        const sha256 = sourceDigest(record.content);
+        const expected = this.evidence.getSnapshot().records.find(entry => entry.kind === 'observation'
+            && (entry.id === (record.meta.observation_id ?? id) || entry.source.toolCallId === id));
+        if ((record.schemaVersion === 1 && (!record.meta.result_sha256 || !record.meta.observation_id))
+            || (record.meta.result_sha256 !== undefined && record.meta.result_sha256 !== sha256)
+            || (expected?.kind === 'observation' && expected.resultSha256 !== sha256)) {
+            throw new ToolResponseIntegrityError(`Source SHA256 integrity mismatch: ${id}`);
+        }
+        if (record.observation) {
+            const parsed = parseEvidenceSnapshot({ schemaVersion: 1, revision: 0, records: [record.observation], currentIds: [record.observation.id], omittedRecords: 0, omittedTextBytes: 0 });
+            if (!parsed || record.observation.kind !== 'observation' || record.observation.resultSha256 !== sha256
+                || record.observation.resultBytes !== Buffer.byteLength(record.content)
+                || record.observation.id !== record.meta.observation_id || record.observation.source.toolCallId !== id) {
+                throw new ToolResponseIntegrityError(`Invalid source observation metadata: ${id}`);
+            }
+        }
+        record.meta.result_sha256 = sha256;
+        record.meta.observation_id ??= id;
+        return record;
+    }
+    admitLoadedRecord(record, recreate) {
+        const id = record.meta.tool_use_id;
+        this.meta.set(id, record.meta);
+        this.availability.set(id, 'persisted');
+        this.turnCounter = Math.max(this.turnCounter, record.meta.turn + 1);
+        const existing = this.evidence.getSnapshot().records.find(entry => entry.id === record.meta.observation_id);
+        if (existing)
+            this.evidence.setAvailability(existing.id, 'persisted');
+        else if (recreate) {
+            const observation = record.observation ?? {
+                id: record.meta.observation_id, kind: 'observation', recordedAt: record.meta.timestamp,
+                source: { origin: 'legacy', toolCallId: id, toolName: record.meta.tool_name, ...(record.meta.source_path ? { path: record.meta.source_path } : {}) },
+                excerpt: record.content, resultSha256: record.meta.result_sha256, resultBytes: Buffer.byteLength(record.content),
+                partial: Boolean(record.meta.partial), isError: Boolean(record.meta.is_error), availability: 'persisted', verification: 'unknown',
+            };
+            this.evidence.observe({ ...observation, availability: 'persisted' });
+        }
+    }
     cacheContent(toolUseId, content) {
         const newSize = this.meta.get(toolUseId)?.size_tokens ?? Math.max(1, estimateTokens(content));
         const existing = this.contentCache.get(toolUseId);
@@ -332,6 +849,7 @@ export class ToolResponseStore {
         }
         this.contentCache.set(toolUseId, content);
         this.totalCachedTokens += newSize;
+        this.boundMemoryCache();
         const budget = this.cacheBudgetTokens();
         while (this.totalCachedTokens > budget && this.contentCache.size > 1) {
             // Prefer evicting non-pinned (non-recalled) entries first, oldest-LRU
@@ -360,7 +878,8 @@ export class ToolResponseStore {
         }
     }
     getMeta(toolUseId) {
-        return this.meta.get(toolUseId);
+        const meta = this.meta.get(toolUseId);
+        return meta ? structuredClone(meta) : undefined;
     }
     /**
      * Mark a tool result as freshly recalled. Stores `recalledAtTurn` so the