@iowarp/clio-coder 0.3.3 → 0.3.6

package/src/tools/result-shaping.ts

@@ -1,8 +1,19 @@
-import { mkdirSync, writeFileSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { mkdirSync, readdirSync, rmdirSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { isAbsolute, join, relative, resolve } from "node:path";
 import { clioStateDir } from "../core/xdg.js";
 import type { ToolInvokeOptions, ToolResult, ToolResultDetails, ToolSpec } from "./registry.js";
-import { DEFAULT_MAX_BYTES } from "./truncate.js";
+import {
+	deterministicToolResultDigest,
+	legacyToolResultDigest,
+	normalizeToolResultDisposition,
+	projectToolResultContext,
+	type ToolResultDigest,
+	type ToolResultDisposition,
+	type ToolResultDispositionFallback,
+	type ToolResultDispositionMetadata,
+} from "./result-disposition.js";
+import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, truncateTail } from "./truncate.js";
 import { byteLength, truncateUtf8 } from "./truncate-utf8.js";
 
 // Backstop for tools without an explicit resultSizePolicy. Sits above the
@@ -12,6 +23,8 @@ import { byteLength, truncateUtf8 } from "./truncate-utf8.js";
 export const DEFAULT_TOOL_RESULT_MAX_BYTES = DEFAULT_MAX_BYTES + 2 * 1024;
 const RESULT_TRUNCATION_MARKER = "\n[tool result truncated]";
 const RESULT_OFFLOAD_MAX_BYTES = 10 * 1024 * 1024;
+const TAIL_NOTICE_RESERVE_BYTES = 512;
+export const TOOL_OFFLOAD_MAX_AGE_MS = 14 * 24 * 60 * 60 * 1000;
 
 type ToolResultShapeContext = Pick<ToolInvokeOptions, "sessionId" | "toolCallId">;
 
@@ -33,6 +46,41 @@ function followUpHint(spec: ToolSpec): string {
 	);
 }
 
+function offloadMaxBytesFor(spec: ToolSpec): number {
+	const configured = spec.metadata?.resultSizePolicy?.offloadMaxBytes;
+	return typeof configured === "number" && Number.isFinite(configured) && configured > 0
+		? Math.floor(configured)
+		: RESULT_OFFLOAD_MAX_BYTES;
+}
+
+/**
+ * Build the task-memory diagnostic from the same normalized disposition that
+ * will shape model context. Legacy tools use the explicit compatibility path.
+ */
+export function toolResultDigestFor(
+	spec: ToolSpec,
+	result: ToolResult,
+	requestedDisposition?: ToolResultDisposition,
+): ToolResultDigest {
+	const text = resultText(result);
+	const disposition = normalizeToolResultDisposition(spec, maxBytesFor(spec), requestedDisposition);
+	if (disposition === null) {
+		const projected = shapeLegacyToolResult(spec, result);
+		return legacyToolResultDigest(resultText(projected), { outcome: result.kind });
+	}
+	const capturedBytes = capturedBytesFor(result, text);
+	return deterministicToolResultDigest({
+		text,
+		kind: result.kind,
+		details: result.details,
+		disposition,
+		capturedBytes,
+		displayedBytes: Math.min(capturedBytes, disposition.presentation.maxBytes),
+		offloadPath: existingOffloadPath(result.details),
+		followUpHint: followUpHint(spec),
+	});
+}
+
 function detailsRecord(details: ToolResultDetails | undefined, key: string): Record<string, unknown> | null {
 	const candidate = details?.[key];
 	return candidate !== null && typeof candidate === "object" && !Array.isArray(candidate)
@@ -41,10 +89,10 @@ function detailsRecord(details: ToolResultDetails | undefined, key: string): Rec
 }
 
 /**
- * A result that already spilled its full text to scratch (bash's tail-biased
- * shaping via `resultSize.offloadPath`, or an OBSERVE tool's envelope via
- * `observation.offloadPath`) is left alone; the tool shaped it deliberately
- * and the backstop would only cut the tool's own continuation notice.
+ * A result that already spilled its full text to scratch (the canonical
+ * tail-biased presentation path via `resultSize.offloadPath`, or an OBSERVE
+ * tool's envelope via `observation.offloadPath`) is left alone; a second pass
+ * would only cut the deliberate continuation notice.
  */
 function existingOffloadPath(details: ToolResultDetails | undefined): string | null {
 	const fromResultSize = detailsRecord(details, "resultSize")?.offloadPath;
@@ -53,6 +101,13 @@ function existingOffloadPath(details: ToolResultDetails | undefined): string | n
 	return typeof fromObservation === "string" && fromObservation.length > 0 ? fromObservation : null;
 }
 
+function existingDisposition(details: ToolResultDetails | undefined): ToolResultDispositionMetadata | null {
+	const disposition = detailsRecord(details, "resultDisposition");
+	return disposition?.version === 1 && disposition.applications === 1
+		? (disposition as unknown as ToolResultDispositionMetadata)
+		: null;
+}
+
 function safePathSegment(value: string): string {
 	const safe = value
 		.trim()
@@ -61,10 +116,6 @@ function safePathSegment(value: string): string {
 	return safe.length > 0 ? safe : "unnamed";
 }
 
-function timestampSegment(): string {
-	return new Date().toISOString().replace(/[^A-Za-z0-9._-]/g, "_");
-}
-
 function offloadBody(text: string, bytes: number, maxBytes: number): string {
 	if (bytes <= maxBytes) return text;
 	const notice = `\n[clio-coder scratch output truncated at ${maxBytes} bytes; original size ${bytes} bytes]`;
@@ -72,24 +123,56 @@ function offloadBody(text: string, bytes: number, maxBytes: number): string {
 	return `${truncateUtf8(text, Math.max(0, prefixBudget), "")}${notice}`;
 }
 
+/** Remove regular offload files older than the boot retention cap. */
+export function sweepExpiredToolOffloads(now = Date.now()): number {
+	const root = join(clioStateDir(), "scratch");
+	const cutoff = now - TOOL_OFFLOAD_MAX_AGE_MS;
+	let removed = 0;
+	try {
+		for (const session of readdirSync(root, { withFileTypes: true })) {
+			if (!session.isDirectory()) continue;
+			const sessionDir = join(root, session.name);
+			for (const entry of readdirSync(sessionDir, { withFileTypes: true })) {
+				if (!entry.isFile()) continue;
+				const path = join(sessionDir, entry.name);
+				try {
+					if (statSync(path).mtimeMs >= cutoff) continue;
+					rmSync(path, { force: true });
+					removed += 1;
+				} catch {}
+			}
+			try {
+				if (readdirSync(sessionDir).length === 0) rmdirSync(sessionDir);
+			} catch {}
+		}
+	} catch {}
+	return removed;
+}
+
 /**
  * Persist the full text of a tool result to a per-session scratch file and
- * return its path (or null if the write fails). Callers that truncate their own
- * display output (for example bash's tail-biased shaping) use this to spill the
- * complete output before truncating, then set `details.resultSize.offloadPath`
- * so the registry re-truncation pass leaves the already-shaped result alone.
+ * return its path (or null if the write fails). Self-shaping callers use this
+ * to spill complete output before truncating and set
+ * `details.resultSize.offloadPath`; canonical disposition shaping also uses it
+ * once when presentation or model context omits captured content.
+ *
+ * The file is named by the sha256 of the captured text, so the path is a pure
+ * function of the bytes: identical captures share one file and the
+ * `retrieve=` header, the eviction marker, and the resume transcript carry the
+ * same bytes every time, which is what keeps the prompt-cache prefix stable
+ * across runs. Pass `digest` when the caller already hashed the text.
  */
 export function writeToolOffload(
 	text: string,
 	context: ToolResultShapeContext | undefined,
 	maxBytes: number = RESULT_OFFLOAD_MAX_BYTES,
+	digest: string = createHash("sha256").update(text).digest("hex"),
 ): string | null {
 	try {
 		const bytes = byteLength(text);
 		const sessionId = safePathSegment(context?.sessionId ?? "no-session");
-		const callId = safePathSegment(context?.toolCallId ?? timestampSegment());
 		const dir = join(clioStateDir(), "scratch", sessionId);
-		const path = join(dir, `${callId}.txt`);
+		const path = join(dir, `${digest}.txt`);
 		mkdirSync(dir, { recursive: true });
 		writeFileSync(path, offloadBody(text, bytes, maxBytes), "utf8");
 		return path;
@@ -145,9 +228,10 @@ function shapeJsonOverflow(
 	bytes: number,
 	maxBytes: number,
 	context: ToolResultShapeContext | undefined,
+	offloadMaxBytes: number,
 ): ToolResult {
 	const observation = detailsRecord(result.details, "observation");
-	const offloadPath = writeToolOffload(text, context);
+	const offloadPath = writeToolOffload(text, context, offloadMaxBytes);
 	const next =
 		typeof observation?.next === "string" && observation.next.length > 0 ? observation.next : followUpHint(spec);
 	const stub = JSON.stringify({
@@ -170,16 +254,64 @@ function shapeJsonOverflow(
 	return { ...result, message: stub, details: mergeDetails(result.details, resultSize) };
 }
 
-export function shapeToolResult(spec: ToolSpec, result: ToolResult, context?: ToolResultShapeContext): ToolResult {
+function shapeTailOverflow(
+	spec: ToolSpec,
+	result: ToolResult,
+	text: string,
+	bytes: number,
+	maxBytes: number,
+	context: ToolResultShapeContext | undefined,
+	offloadMaxBytes: number,
+): ToolResult {
+	const truncation = truncateTail(text, {
+		maxLines: DEFAULT_MAX_LINES,
+		maxBytes: Math.max(1, maxBytes - TAIL_NOTICE_RESERVE_BYTES),
+	});
+	const offloadPath = writeToolOffload(text, context, offloadMaxBytes);
+	const startLine = truncation.totalLines - truncation.outputLines + 1;
+	const scope = truncation.lastLinePartial
+		? `last ${formatSize(truncation.outputBytes)} of line ${truncation.totalLines} (line is large)`
+		: `lines ${startLine}-${truncation.totalLines} of ${truncation.totalLines}`;
+	const location =
+		offloadPath === null
+			? ""
+			: ` Full output saved to ${offloadPath}; ${OFFLOAD_POINTER_NOTE}; read it with offset/limit.`;
+	const note = `[Output tail-truncated: showing ${scope} (${formatSize(maxBytes)} display limit).${location}]`;
+	const resultSize = {
+		bytes,
+		shownBytes: truncation.outputBytes,
+		maxBytes,
+		truncated: true,
+		policy: "tail",
+		followUpHint: followUpHint(spec),
+		...(offloadPath === null ? {} : { offloadPath }),
+	};
+	const displayed = `${truncation.content}\n\n${note}`;
+	if (result.kind === "ok") return { ...result, output: displayed, details: mergeDetails(result.details, resultSize) };
+	return { ...result, message: displayed, details: mergeDetails(result.details, resultSize) };
+}
+
+function shapeLegacyToolResult(
+	spec: ToolSpec,
+	result: ToolResult,
+	context?: ToolResultShapeContext,
+	overflow: "head" | "tail" = "head",
+): ToolResult {
 	if (existingOffloadPath(result.details) !== null) return result;
 	const maxBytes = maxBytesFor(spec);
+	const offloadMaxBytes = offloadMaxBytesFor(spec);
 	const text = result.kind === "ok" ? result.output : result.message;
 	const bytes = byteLength(text);
 	if (bytes <= maxBytes) return result;
 	const observation = detailsRecord(result.details, "observation");
-	if (observation?.format === "json") return shapeJsonOverflow(spec, result, text, bytes, maxBytes, context);
+	if (observation?.format === "json") {
+		return shapeJsonOverflow(spec, result, text, bytes, maxBytes, context, offloadMaxBytes);
+	}
+	if (overflow === "tail") {
+		return shapeTailOverflow(spec, result, text, bytes, maxBytes, context, offloadMaxBytes);
+	}
 	const truncated = truncateUtf8(text, maxBytes, RESULT_TRUNCATION_MARKER);
-	const offloadPath = writeToolOffload(text, context);
+	const offloadPath = writeToolOffload(text, context, offloadMaxBytes);
 	const resultSize = {
 		bytes,
 		shownBytes: byteLength(truncated),
@@ -202,3 +334,172 @@ export function shapeToolResult(spec: ToolSpec, result: ToolResult, context?: To
 		details: mergeDetails(result.details, resultSize),
 	};
 }
+
+function specWithDisplayCap(spec: ToolSpec, maxBytes: number): ToolSpec {
+	const metadata = spec.metadata;
+	if (metadata === undefined) return spec;
+	return {
+		...spec,
+		metadata: {
+			...metadata,
+			resultSizePolicy: { ...metadata.resultSizePolicy, maxBytes },
+		},
+	};
+}
+
+function resultText(result: ToolResult): string {
+	return result.kind === "ok" ? result.output : result.message;
+}
+
+/**
+ * The operator-facing trace of a disposition that was narrowed rather than
+ * requested. Without it a throwing resolver degrades every call of that tool
+ * to metadata-only with nothing on the transcript row to say so.
+ */
+function withFallbackNotice(result: ToolResult, fallback: ToolResultDispositionFallback): ToolResult {
+	const note = `\n\n[result disposition fell back to metadata-only: ${fallback.reason} (${fallback.message}); the model received facts and retrieval only]`;
+	return result.kind === "ok"
+		? { ...result, output: `${result.output}${note}` }
+		: { ...result, message: `${result.message}${note}` };
+}
+
+function numberField(record: Record<string, unknown> | null, key: string): number | null {
+	const value = record?.[key];
+	return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
+}
+
+function capturedBytesFor(result: ToolResult, text: string): number {
+	const declared = numberField(detailsRecord(result.details, "resultSize"), "bytes");
+	return Math.max(byteLength(text), declared ?? 0);
+}
+
+function resultWasTruncated(result: ToolResult): boolean {
+	return detailsRecord(result.details, "resultSize")?.truncated === true;
+}
+
+function withOffloadMetadata(
+	result: ToolResult,
+	offloadPath: string,
+	capturedBytes: number,
+	displayedBytes: number,
+	maxBytes: number,
+	hint: string,
+): ToolResult {
+	const current = detailsRecord(result.details, "resultSize") ?? {};
+	const resultSize = {
+		bytes: numberField(current, "bytes") ?? capturedBytes,
+		shownBytes: numberField(current, "shownBytes") ?? displayedBytes,
+		maxBytes: numberField(current, "maxBytes") ?? maxBytes,
+		truncated: current.truncated === true,
+		policy: typeof current.policy === "string" ? current.policy : "disposition",
+		followUpHint: typeof current.followUpHint === "string" ? current.followUpHint : hint,
+		...current,
+		offloadPath,
+	};
+	return { ...result, details: mergeDetails(result.details, resultSize) };
+}
+
+/**
+ * Apply legacy shaping or one declared canonical disposition. Registry calls
+ * this once, after middleware annotations have produced the terminal result.
+ */
+export function shapeToolResult(
+	spec: ToolSpec,
+	result: ToolResult,
+	context?: ToolResultShapeContext,
+	requestedDisposition?: ToolResultDisposition,
+): ToolResult {
+	if (existingDisposition(result.details) !== null) return result;
+	const hardMaxBytes = maxBytesFor(spec);
+	const disposition = normalizeToolResultDisposition(spec, hardMaxBytes, requestedDisposition);
+	if (disposition === null) return shapeLegacyToolResult(spec, result, context);
+
+	const capturedText = resultText(result);
+	const capturedBytes = capturedBytesFor(result, capturedText);
+	let displayed = shapeLegacyToolResult(
+		specWithDisplayCap(spec, disposition.presentation.maxBytes),
+		result,
+		context,
+		disposition.presentation.overflow,
+	);
+	if (disposition.fallback !== undefined) displayed = withFallbackNotice(displayed, disposition.fallback);
+	let displayedText = resultText(displayed);
+	let displayedBytes = byteLength(displayedText);
+	let offloadPath = existingOffloadPath(displayed.details);
+	const projectionInput = {
+		text: capturedText,
+		kind: result.kind,
+		details: displayed.details,
+		disposition,
+		capturedBytes,
+		displayedBytes,
+		offloadPath,
+		followUpHint: followUpHint(spec),
+	};
+	// Project first, then decide on retrieval. A projection that already carries
+	// every captured byte (a short `summary`, a `full` inside budget) needs no
+	// scratch artifact and must not claim truncation; anything that genuinely
+	// omits content earns one offload and the honest omission facts with it.
+	let projection = projectToolResultContext(projectionInput);
+	if (offloadPath === null && projection.truncated) {
+		offloadPath = writeToolOffload(
+			capturedText,
+			context,
+			offloadMaxBytesFor(spec),
+			projection.summaryProvenance?.sourceSha256,
+		);
+		if (offloadPath !== null) {
+			displayed = withOffloadMetadata(
+				displayed,
+				offloadPath,
+				capturedBytes,
+				displayedBytes,
+				disposition.presentation.maxBytes,
+				followUpHint(spec),
+			);
+			displayedText = resultText(displayed);
+			displayedBytes = byteLength(displayedText);
+			projection = projectToolResultContext({
+				...projectionInput,
+				details: displayed.details,
+				displayedBytes,
+				offloadPath,
+			});
+		}
+	}
+	const metadata: ToolResultDispositionMetadata = {
+		version: 1,
+		applications: 1,
+		presentation: { ...disposition.presentation, content: displayedText },
+		context: {
+			requestedMode: disposition.context.mode,
+			appliedMode: projection.appliedMode,
+			maxBytes: disposition.context.maxBytes,
+			...(disposition.context.mode === "bounded" && disposition.context.excerpt !== undefined
+				? { excerpt: disposition.context.excerpt }
+				: {}),
+			...(disposition.context.mode === "full" && disposition.context.downgradeExcerpt !== undefined
+				? { excerpt: disposition.context.downgradeExcerpt }
+				: {}),
+			...(disposition.context.mode === "summary" && disposition.context.strategy !== undefined
+				? { summaryStrategy: disposition.context.strategy }
+				: {}),
+		},
+		capturedBytes,
+		displayedBytes,
+		contextBytes: byteLength(projection.text),
+		presentationTruncated: resultWasTruncated(displayed),
+		contextTruncated: projection.truncated,
+		...(projection.downgrade === undefined ? {} : { downgrade: projection.downgrade }),
+		...(offloadPath === null ? {} : { offloadPath }),
+		retrieval:
+			offloadPath === null ? followUpHint(spec) : `read ${offloadPath} with offset and limit to retrieve omitted content`,
+		...(projection.summaryProvenance === undefined ? {} : { summaryProvenance: projection.summaryProvenance }),
+		...(disposition.fallback === undefined ? {} : { fallback: disposition.fallback }),
+	};
+	return {
+		...displayed,
+		details: { ...(displayed.details ?? {}), resultDisposition: metadata },
+		modelContext: projection.text,
+	};
+}

package/src/tools/safe-exec.ts

@@ -35,9 +35,11 @@ export function maxOutputArg(args: Record<string, unknown>): number {
 function resultDetails(result: SafeCommandResult): ToolResultDetails {
 	return {
 		command: [result.file, ...result.args].join(" "),
+		argv: [result.file, ...result.args],
 		cwd: result.cwd,
 		exitCode: result.exitCode,
 		durationMs: result.durationMs,
+		aborted: result.aborted,
 		timedOut: result.timedOut,
 		outputCapped: result.outputCapped,
 	};