dsh-context-pruner 0.0.0-stage → 0.1.1

package/lib/compact.js

@@ -0,0 +1,407 @@
+/**
+ * Manual region compaction: compact a CALLER-CHOSEN span while the agent is idle.
+ *
+ * ## Why this exists
+ *
+ * DSH's compaction seam offers two entry points, and neither does what this
+ * plugin needs:
+ *
+ * | Entry | Range source | Turn requirement |
+ * |---|---|---|
+ * | `CompactionEngine#compactRegion(start, end)` | caller-supplied | needs an OPEN TURN (`owner: 'current-turn'`) |
+ * | `CompactionEngine#compactNow()` | internal `selectCompactableRange()` | idle-safe (`owner: null`) |
+ *
+ * So a user asking "drop rounds 3-7, keep everything else" cannot be served:
+ * the ranged call refuses to run from an idle session, and the idle call
+ * chooses its own range.
+ *
+ * This module implements the missing combination: the manual (idle-safe) path
+ * with a caller-supplied span. It reproduces the upstream transaction's shape
+ * faithfully — the log-recorded lock bracket, edge validation, change
+ * detection, shrink validation, and the single surface replacement — because
+ * those are what make a compaction safe and replayable.
+ *
+ * @module dsh-context-pruner/compact
+ */
+
+import { CompactionId, ManualCompactionError, compactCheckpointSource } from "@deepseek-ai/dsh-compaction";
+import { createUserMessage } from "@deepseek-ai/dsh-llm";
+import { randomUUID } from "node:crypto";
+import { isDeepStrictEqual } from "node:util";
+
+/** Framing that marks a replacement as an established checkpoint. */
+const CHECKPOINT_PREAMBLE =
+	"This is a correction checkpoint replacing an earlier span of the conversation to free up context. Treat the capt" +
+	"ured context as established background and build on it without restating it. Continue the task directly from the " +
+	"messages that follow, without acknowledging this checkpoint.";
+
+/** Opening tag of the framed summary body. */
+const SUMMARY_OPEN_TAG = "<compacted-summary>";
+
+/** Closing tag of the framed summary body. */
+const SUMMARY_CLOSE_TAG = "</compacted-summary>";
+
+/** One closed checkpoint in the documented `<compacted-summary>` form. */
+function frameSummary(summary) {
+	return [
+		{ type: "text", text: `${CHECKPOINT_PREAMBLE}\n\n${SUMMARY_OPEN_TAG}` },
+		...summary,
+		{ type: "text", text: SUMMARY_CLOSE_TAG }
+	];
+}
+
+/** Raised when the selected span no longer matches what was validated. */
+class SpanChangedError extends Error {
+	/** @param message - diagnostic text. */
+	constructor(message) {
+		super(message);
+		this.name = "SpanChangedError";
+	}
+}
+
+/**
+ * Inspect the durable entry state: whether a turn is open, whether an
+ * unmatched compaction lock exists, and the newest seed boundary.
+ *
+ * Mirrors the upstream tail scan: an unmatched `compaction/start` AFTER the
+ * newest `session/end-seed` is live and blocks; an older one is stale evidence
+ * from a previous process lifecycle and does not.
+ *
+ * @param session - session whose event tail is scanned.
+ * @returns `{ openTurn, unmatchedCompactionStart, latestEndSeedSeq }`.
+ */
+function inspectEntryState(session) {
+	let openTurn = null;
+	let openTurnStateKnown = false;
+	let unmatchedCompactionStart;
+	let compactionEntryStateKnown = false;
+	let latestEndSeedSeq;
+
+	for (let seq = session.seq - 1; seq >= 0; seq -= 1) {
+		const event = session.eventAt(seq);
+		if (latestEndSeedSeq === undefined && event.type === "session/end-seed") latestEndSeedSeq = event.seq;
+
+		if (!compactionEntryStateKnown) {
+			if (event.type === "compaction/start") {
+				unmatchedCompactionStart = event;
+				compactionEntryStateKnown = true;
+			} else if (event.type === "compaction/end") {
+				compactionEntryStateKnown = true;
+			}
+		}
+		if (!openTurnStateKnown) {
+			if (event.type === "turn/start") {
+				openTurn = event.data.turn;
+				openTurnStateKnown = true;
+			} else if (event.type === "turn/end") {
+				openTurnStateKnown = true;
+			}
+		}
+		if (openTurnStateKnown && compactionEntryStateKnown && latestEndSeedSeq !== undefined) break;
+	}
+
+	return { openTurn, unmatchedCompactionStart, latestEndSeedSeq };
+}
+
+/**
+ * Reject a durable unmatched compaction marker that is still live.
+ *
+ * @param unmatchedCompactionStart - latest unmatched opening marker, if any.
+ * @param latestEndSeedSeq - newest constructor-seed boundary, if any.
+ * @throws {ManualCompactionError} with code `busy` when a lock is live.
+ */
+function assertCompactionInactive(unmatchedCompactionStart, latestEndSeedSeq) {
+	if (
+		unmatchedCompactionStart === undefined ||
+		(latestEndSeedSeq !== undefined && latestEndSeedSeq > unmatchedCompactionStart.seq)
+	) {
+		return;
+	}
+	throw new ManualCompactionError("busy", "compaction already in progress: the session compaction lock is active");
+}
+
+/**
+ * Resolve a positional node range and validate its tool-pairing edges.
+ *
+ * Both edges must be balanced so an assistant tool call is never separated
+ * from its result: splitting a pair corrupts the projected conversation.
+ *
+ * @param session - session supplying the surface.
+ * @param start - inclusive first surface seq.
+ * @param end - inclusive last surface seq.
+ * @returns `{ start, end, startIdx, endIdx, shadowedSeqs }`.
+ * @throws {Error} when a seq is absent, reversed, or an edge is unbalanced.
+ */
+function resolveRange(session, start, end) {
+	const nodes = session.surface.nodes;
+	const startIdx = nodes.indexOf(start);
+	const endIdx = nodes.indexOf(end);
+
+	if (startIdx === -1) throw new Error(`prune: start seq ${start} is not on the surface`);
+	if (endIdx === -1) throw new Error(`prune: end seq ${end} is not on the surface`);
+	if (startIdx > endIdx) {
+		throw new Error(
+			`prune: start seq ${start} (position ${startIdx}) comes after end seq ${end} (position ${endIdx}) on the surface`
+		);
+	}
+
+	return {
+		start,
+		end,
+		startIdx,
+		endIdx,
+		shadowedSeqs: nodes.slice(startIdx, endIdx + 1)
+	};
+}
+
+/**
+ * Check that a chosen span's edges do not split a tool call from its result.
+ *
+ * @param session - session supplying the surface and pairing history.
+ * @param range - the resolved range.
+ * @param balancedBefore - `toolPairingBalancedBefore` from the compaction seam.
+ * @param balancedAfter - `toolPairingBalancedAfter` from the compaction seam.
+ * @returns an error message when unbalanced, otherwise `undefined`.
+ */
+export function checkBalance(session, range, balancedBefore, balancedAfter) {
+	if (!balancedBefore(session, range.start)) {
+		return "the first node starts mid-step: an assistant tool call would be separated from its result";
+	}
+	if (!balancedAfter(session, range.end)) {
+		return "the last node ends mid-step: an assistant tool call would be separated from its result, or the step is still open";
+	}
+	return undefined;
+}
+
+/**
+ * Compact an explicit surface span into one summary, from an idle agent.
+ *
+ * The transaction, in order:
+ *
+ *   1. Resolve and validate the span (present, ordered, balanced edges).
+ *   2. Assert no live compaction lock and no open turn.
+ *   3. Append the log-only `compaction/start` marker — this IS the lock.
+ *   4. Summarize the span through `summarize`.
+ *   5. Re-validate that the span is unchanged (concurrent edits abort).
+ *   6. Append `compaction/summary`, then the replacement `user/message` whose
+ *      `surfaceOp: { op: 'replace' }` shadows the span, then `compaction/end`.
+ *   7. On any failure after the start marker, make exactly one close attempt
+ *      carrying the error, so a crash stays detectable.
+ *
+ * @param options.session - session whose surface is replaced.
+ * @param options.agent - agent used by the summarizer for routing context.
+ * @param options.start - inclusive first surface seq.
+ * @param options.end - inclusive last surface seq.
+ * @param options.summarize - produces `{ summary, provider, model, ... }` for the span.
+ * @param options.meter - token meter used for pricing and shrink validation.
+ * @param options.signal - optional cancellation.
+ * @param options.sourceCommandId - initiating command identity, for correlation.
+ * @param options.balancedBefore - `toolPairingBalancedBefore` from the seam.
+ * @param options.balancedAfter - `toolPairingBalancedAfter` from the seam.
+ * @param options.flush - optional durability checkpoint awaited after success.
+ * @returns the committed `CompactionResult`.
+ * @throws {ManualCompactionError} for expected busy / changed / summary / commit / persistence failures.
+ */
+export async function compactExplicitRegion(options) {
+	const {
+		session,
+		agent,
+		start,
+		end,
+		summarize,
+		meter,
+		signal,
+		sourceCommandId,
+		balancedBefore,
+		balancedAfter,
+		flush
+	} = options;
+
+	// --- 1. Pass 1 validation, before any durable write. -------------------
+	signal?.throwIfAborted();
+	const selection = resolveRange(session, start, end);
+
+	// --- 2. Idle + lock admission. ----------------------------------------
+	const entryState = inspectEntryState(session);
+	assertCompactionInactive(entryState.unmatchedCompactionStart, entryState.latestEndSeedSeq);
+	if (entryState.openTurn !== null) {
+		throw new ManualCompactionError(
+			"busy",
+			"prune needs an idle agent: this session already has an open turn, so compacting a chosen span would race the running turn"
+		);
+	}
+
+	// --- 3. Price the span and capture the exact nodes selected. ----------
+	const measurement = meter.measure(session);
+	const selectedNodes = measurement.nodes.slice(selection.startIdx, selection.endIdx + 1);
+	if (
+		selectedNodes.length !== selection.shadowedSeqs.length ||
+		selectedNodes.some((node, index) => node.seq !== selection.shadowedSeqs[index])
+	) {
+		throw new ManualCompactionError("changed", "prune: the selected surface changed before summarization began");
+	}
+
+	const shadowedTokenCount = selectedNodes.reduce((total, node) => total + node.heuristicTokens, 0);
+	const shadowedRouteTokenCount = selectedNodes.reduce((total, node) => total + node.tokens, 0);
+
+	// Replay exactly the selected nodes, mirroring the shipped input builder.
+	const regionMessages = selection.shadowedSeqs
+		.map((seq) => session.deriveEventMessage(session.eventAt(seq)))
+		.filter((message) => message !== null && message !== undefined);
+	const headSeq = session.surface.nodes[0];
+	const headMessage = headSeq === undefined ? undefined : session.deriveEventMessage(session.eventAt(headSeq));
+	const systemHead = headMessage?.role === "system" ? headMessage : undefined;
+	const input = {
+		...(agent.session.requestHeader()?.tools === undefined
+			? {}
+			: { tools: agent.session.requestHeader().tools }),
+		messages: systemHead === undefined ? regionMessages : [systemHead, ...regionMessages]
+	};
+
+	// --- 4. Acquire the durable lock. -------------------------------------
+	const compactionId = CompactionId(randomUUID());
+	const lifecycle = {
+		compactionId,
+		...(sourceCommandId === undefined ? {} : { sourceCommandId }),
+		turn: null
+	};
+	const startEvent = session.append("compaction/start", lifecycle);
+
+	let failure;
+	let closed = false;
+	let closing = false;
+	let stage = "summary";
+	let committed;
+
+	try {
+		// --- 5. Summarize (yields). ---------------------------------------
+		const summarized = await summarize(input, agent, signal);
+		signal?.throwIfAborted();
+
+		// --- 6. The span must be unchanged after the await. ---------------
+		//
+		// Re-resolving `start`/`end` by seq is NOT sufficient: a concurrent
+		// replacement elsewhere shifts every later position, yet both endpoint
+		// seqs remain findable, so a naive recheck passes while the span now
+		// spans different content. Compare the ABSOLUTE position window
+		// instead, which is what the replacement will actually shadow.
+		const reNodes = meter.measure(session).nodes;
+		const windowUnchanged =
+			reNodes.length >= selection.endIdx + 1 &&
+			selection.shadowedSeqs.every(
+				(seq, offset) => reNodes[selection.startIdx + offset]?.seq === seq
+			) &&
+			isDeepStrictEqual(
+				[
+					session.surface.nodes[selection.startIdx],
+					session.surface.nodes[selection.endIdx]
+				],
+				[start, end]
+			);
+
+		if (!windowUnchanged) {
+			throw new SpanChangedError("prune: the selected span changed while the summary was being written");
+		}
+
+		// --- 7. Shrink validation: a checkpoint must actually save space. --
+		// Staged as "summary": a checkpoint that fails to shrink is a failure to
+		// produce a useful summary, not a commit failure.
+		const checkpointMessage = createUserMessage({
+			content: frameSummary(summarized.summary),
+			source: compactCheckpointSource(compactionId, sourceCommandId)
+		});
+		const framedTokens = meter.estimateMessage(checkpointMessage);
+		if (framedTokens >= shadowedRouteTokenCount) {
+			throw new Error(
+				`prune: the checkpoint is not smaller than the span it replaces (${framedTokens} >= ${shadowedRouteTokenCount} tokens)`
+			);
+		}
+
+		// From here on the surface is actually mutated, so a failure is a
+		// commit failure.
+		stage = "commit";
+
+		// The metering event must be immediately followed by its replacement:
+		// a consumer pairs a replacement with the shadow-price event directly
+		// before it.
+		const summaryEvent = session.append("compaction/summary", {
+			compactionId,
+			...(sourceCommandId === undefined ? {} : { sourceCommandId }),
+			summary: summarized.summary,
+			...(summarized.rawOutput === undefined ? {} : { rawOutput: summarized.rawOutput }),
+			...(summarized.llmStreamCall === true ? { llmStreamCall: true } : {}),
+			shadowedRange: { start, end },
+			shadowedSeqs: [...selection.shadowedSeqs],
+			shadowedTokenCount,
+			provider: summarized.provider,
+			model: summarized.model,
+			...(summarized.maxTokens === undefined ? {} : { maxTokens: summarized.maxTokens }),
+			...(summarized.usage === undefined ? {} : { usage: summarized.usage })
+		});
+
+		const replacement = session.append("user/message", checkpointMessage, {
+			surfaceOp: { op: "replace", startSeq: start, endSeq: end },
+			sourceEventSeqs: [startEvent.seq, summaryEvent.seq, ...selection.shadowedSeqs]
+		});
+
+		closing = true;
+		const endEvent = session.append("compaction/end", lifecycle);
+		closed = true;
+
+		committed = {
+			compactionId,
+			...(sourceCommandId === undefined ? {} : { sourceCommandId }),
+			startSeq: startEvent.seq,
+			summarySeq: summaryEvent.seq,
+			endSeq: endEvent.seq,
+			summary: summarized.summary,
+			shadowedRange: { start, end },
+			shadowedSeqs: [...selection.shadowedSeqs],
+			shadowedTokenCount,
+			replacementSeq: replacement.seq
+		};
+	} catch (error) {
+		failure = { error, stage: closing ? "commit" : stage };
+		if (!closing) {
+			closing = true;
+			try {
+				session.append("compaction/end", { ...lifecycle, error: String(error?.message ?? error) });
+				closed = true;
+			} catch (closeError) {
+				failure = { error: closeError, stage: "commit" };
+			}
+		}
+	}
+
+	// --- 8. Durability checkpoint. ----------------------------------------
+	let flushFailure;
+	if (closed && flush !== undefined) {
+		try {
+			await flush();
+		} catch (error) {
+			flushFailure = error;
+		}
+	}
+
+	signal?.throwIfAborted();
+
+	if (failure !== undefined) {
+		if (failure.stage === "commit") {
+			throw new ManualCompactionError("commit", "prune did not commit cleanly", { cause: failure.error });
+		}
+		if (failure.error instanceof SpanChangedError) {
+			throw new ManualCompactionError("changed", "the history selected for pruning changed during the operation", {
+				cause: failure.error
+			});
+		}
+		throw new ManualCompactionError("summary", "prune could not produce a useful checkpoint", {
+			cause: failure.error
+		});
+	}
+	if (flushFailure !== undefined) {
+		throw new ManualCompactionError("persistence", "prune durability checkpoint failed", { cause: flushFailure });
+	}
+	if (committed === undefined) throw new Error("prune committed without a result");
+
+	return committed;
+}