@iowarp/clio-coder 0.3.3 → 0.3.6

package/src/domains/context/working-set/replay/controls.ts

@@ -0,0 +1,112 @@
+import type { SessionEntry } from "../../../session/entries.js";
+import type { EvictionCandidate, PolicyInput, WorkingSetPolicy, WorkingSetPolicyId } from "../contract.js";
+import { tokensFreedByEviction } from "../engine.js";
+import { protectionCutoffIndex } from "../horizon.js";
+import { buildPathIndex, callPathsByToolCallId } from "../path-index.js";
+import { isProtected } from "../protect.js";
+import type { ReferenceGraph } from "./reference-graph.js";
+import { countReplayTurns } from "./trace.js";
+
+/** Replay-only diagnostic surface; it does not widen the live policy contract. */
+export interface ReplayCandidatePoolPolicy extends WorkingSetPolicy {
+	/** Number of currently usable candidates before target-based truncation. */
+	replayCandidateCount(input: PolicyInput): number;
+}
+
+function controlId(id: string): WorkingSetPolicyId {
+	return id as WorkingSetPolicyId;
+}
+
+function eligibleToolResults(input: PolicyInput): SessionEntry[] {
+	const cutoff = protectionCutoffIndex(input.entries, input.settings.protectLastTurns);
+	const index = buildPathIndex(input.entries, { cwd: input.cwd });
+	const out: SessionEntry[] = [];
+	for (let entryIndex = cutoff - 1; entryIndex >= 0; entryIndex -= 1) {
+		const entry = input.entries[entryIndex];
+		if (entry?.kind !== "message" || entry.role !== "tool_result") continue;
+		if (input.view.evicted.has(entry.turnId)) continue;
+		if (isProtected(entry, { entryIndex, cutoffIndex: cutoff, input, index })) continue;
+		out.push(entry);
+	}
+	return out;
+}
+
+function takeToTarget(input: PolicyInput, entries: ReadonlyArray<SessionEntry>): EvictionCandidate[] {
+	let tokensNeeded = Math.max(0, input.pressure.tokens - input.pressure.target * input.pressure.contextWindow);
+	if (tokensNeeded <= 0) return [];
+	const selected: EvictionCandidate[] = [];
+	const callPaths = callPathsByToolCallId(input.entries);
+	for (const entry of entries) {
+		const candidate: EvictionCandidate = { ref: { entry: entry.turnId }, reason: "age_horizon" };
+		selected.push(candidate);
+		tokensNeeded -= tokensFreedByEviction(input.estimateTokens, entry, candidate, callPaths);
+		if (tokensNeeded <= 0) break;
+	}
+	return selected;
+}
+
+export function makeOraclePolicy(graph: ReferenceGraph): ReplayCandidatePoolPolicy {
+	let lastInput: PolicyInput | null = null;
+	let lastCandidateCount = 0;
+	const safeEntries = (input: PolicyInput): SessionEntry[] => {
+		const currentTurn = countReplayTurns(input.entries) + 1;
+		return eligibleToolResults(input).filter((entry) => {
+			const futureTurns = graph.futureTurnsOf.get(entry.turnId) ?? [];
+			return futureTurns.every((turn) => turn < currentTurn);
+		});
+	};
+	return {
+		id: controlId("oracle"),
+		select(input): ReadonlyArray<EvictionCandidate> {
+			// Replay calls before the next turn-start entry is appended. A reference
+			// in that next turn is therefore still future from the model's view.
+			const safe = safeEntries(input);
+			lastInput = input;
+			lastCandidateCount = safe.length;
+			return takeToTarget(input, safe);
+		},
+		replayCandidateCount(input): number {
+			return input === lastInput ? lastCandidateCount : safeEntries(input).length;
+		},
+	};
+}
+
+function mulberry32(seed: number): () => number {
+	let value = seed >>> 0;
+	return () => {
+		value = (value + 0x6d2b79f5) >>> 0;
+		let next = value;
+		next = Math.imul(next ^ (next >>> 15), next | 1);
+		next ^= next + Math.imul(next ^ (next >>> 7), next | 61);
+		return ((next ^ (next >>> 14)) >>> 0) / 4_294_967_296;
+	};
+}
+
+export function makeRandomPolicy(seed: number): ReplayCandidatePoolPolicy {
+	let lastInput: PolicyInput | null = null;
+	let lastCandidateCount = 0;
+	return {
+		id: controlId("random"),
+		select(input): ReadonlyArray<EvictionCandidate> {
+			const entries = [...eligibleToolResults(input)];
+			lastInput = input;
+			lastCandidateCount = entries.length;
+			const random = mulberry32(seed);
+			for (let index = entries.length - 1; index > 0; index -= 1) {
+				const swap = Math.floor(random() * (index + 1));
+				const value = entries[index];
+				entries[index] = entries[swap] as SessionEntry;
+				entries[swap] = value as SessionEntry;
+			}
+			return takeToTarget(input, entries);
+		},
+		replayCandidateCount(input): number {
+			return input === lastInput ? lastCandidateCount : eligibleToolResults(input).length;
+		},
+	};
+}
+
+export const nonePolicy: WorkingSetPolicy = {
+	id: controlId("none"),
+	select: () => [],
+};

package/src/domains/context/working-set/replay/load-clio.ts

@@ -0,0 +1,199 @@
+import type { Dirent } from "node:fs";
+import { readdir, readFile, stat } from "node:fs/promises";
+import { basename, dirname, join, resolve } from "node:path";
+import { parseSessionEntries } from "../../../session/archive-readers.js";
+import { isSessionHeader, type SessionEntry } from "../../../session/entries.js";
+import { filterEntriesToActivePath } from "../../../session/tree/active-path.js";
+import { buildPathIndex } from "../path-index.js";
+import { buildReferenceGraph } from "./reference-graph.js";
+import { countReplayTurns, type Trace } from "./trace.js";
+
+export interface ReplayFilterOptions {
+	minTurns?: number;
+	minToolResults?: number;
+	requireFileReread?: boolean;
+}
+
+export interface LoadClioTraceOptions {
+	/** False is the CLI's --no-filter behavior. */
+	filter?: false | ReplayFilterOptions;
+}
+
+export interface ReplayLoadCascade {
+	found: number;
+	unreadable: number;
+	filtered: Record<string, number>;
+	kept: number;
+}
+
+const DEFAULT_FILTER = {
+	minTurns: 8,
+	minToolResults: 8,
+	requireFileReread: true,
+} as const;
+
+async function collectLedgerFiles(input: string, out: Set<string>): Promise<void> {
+	const path = resolve(input);
+	let facts: Awaited<ReturnType<typeof stat>>;
+	try {
+		facts = await stat(path);
+	} catch {
+		return;
+	}
+	if (facts.isFile()) {
+		// Directory discovery is intentionally limited to current.jsonl below;
+		// an explicit JSONL path may also be a frozen synthetic fixture.
+		if (path.endsWith(".jsonl")) out.add(path);
+		return;
+	}
+	if (!facts.isDirectory()) return;
+
+	const direct = join(path, "current.jsonl");
+	try {
+		if ((await stat(direct)).isFile()) {
+			out.add(direct);
+			return;
+		}
+	} catch {
+		// A sessions root or cwd-hash directory has no direct ledger.
+	}
+
+	let children: Dirent[];
+	try {
+		children = await readdir(path, { withFileTypes: true });
+	} catch {
+		return;
+	}
+	for (const child of children.sort((a, b) => a.name.localeCompare(b.name))) {
+		if (!child.isDirectory()) continue;
+		await collectLedgerFiles(join(path, child.name), out);
+	}
+}
+
+function sessionFacts(raw: string, source: string): { id: string; cwd: string | null } {
+	const fallback = { id: basename(dirname(source)), cwd: null };
+	for (const line of raw.split("\n")) {
+		if (line.trim().length === 0) continue;
+		try {
+			const value = JSON.parse(line) as unknown;
+			if (isSessionHeader(value)) return { id: value.id, cwd: value.cwd.length > 0 ? value.cwd : null };
+		} catch {
+			return fallback;
+		}
+	}
+	return fallback;
+}
+
+async function pinnedLeafTurnId(source: string): Promise<string | undefined> {
+	try {
+		const raw = await readFile(join(dirname(source), "meta.json"), "utf8");
+		const value = JSON.parse(raw) as unknown;
+		if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined;
+		const pinned = (value as Record<string, unknown>).pinnedLeafTurnId;
+		return typeof pinned === "string" && pinned.length > 0 ? pinned : undefined;
+	} catch {
+		return undefined;
+	}
+}
+
+function cleanActiveEntries(entries: ReadonlyArray<SessionEntry>, activeLeafTurnId?: string): SessionEntry[] {
+	return filterEntriesToActivePath(entries, activeLeafTurnId).filter(
+		(entry) => entry.kind !== "contextEviction" && entry.kind !== "contextRecall",
+	);
+}
+
+function toolResultCount(entries: ReadonlyArray<SessionEntry>): number {
+	let count = 0;
+	for (const entry of entries) {
+		if (entry.kind === "message" && entry.role === "tool_result") count += 1;
+	}
+	return count;
+}
+
+/**
+ * Load only Clio ledgers. Inputs may be a current.jsonl file, one session
+ * directory, a cwd-hash directory, or the sessions root.
+ */
+export async function loadClioTraces(
+	paths: ReadonlyArray<string>,
+	options: LoadClioTraceOptions = {},
+): Promise<{ traces: Trace[]; cascade: ReplayLoadCascade }> {
+	const discovered = new Set<string>();
+	let missingInputs = 0;
+	for (const path of paths) {
+		const before = discovered.size;
+		await collectLedgerFiles(path, discovered);
+		if (discovered.size === before) {
+			try {
+				await stat(resolve(path));
+			} catch {
+				missingInputs += 1;
+			}
+		}
+	}
+
+	const filtered: Record<string, number> = {
+		turns_lt_8: 0,
+		tool_results_lt_8: 0,
+		no_file_reread: 0,
+	};
+	const cascade: ReplayLoadCascade = {
+		found: discovered.size,
+		unreadable: missingInputs,
+		filtered,
+		kept: 0,
+	};
+	const traces: Trace[] = [];
+	const filter =
+		options.filter === false
+			? null
+			: {
+					minTurns: options.filter?.minTurns ?? DEFAULT_FILTER.minTurns,
+					minToolResults: options.filter?.minToolResults ?? DEFAULT_FILTER.minToolResults,
+					requireFileReread: options.filter?.requireFileReread ?? DEFAULT_FILTER.requireFileReread,
+				};
+
+	for (const source of [...discovered].sort((a, b) => a.localeCompare(b))) {
+		let raw: string;
+		try {
+			raw = await readFile(source, "utf8");
+		} catch {
+			cascade.unreadable += 1;
+			continue;
+		}
+		const parsed = parseSessionEntries(raw, source);
+		if (parsed.errors.length > 0) {
+			cascade.unreadable += 1;
+			continue;
+		}
+		const entries = cleanActiveEntries(parsed.entries, await pinnedLeafTurnId(source));
+		const facts = sessionFacts(raw, source);
+		const trace: Trace = {
+			id: facts.id,
+			source,
+			cwd: facts.cwd,
+			entries,
+			turnCount: countReplayTurns(entries),
+		};
+		if (filter !== null) {
+			if (trace.turnCount < filter.minTurns) {
+				filtered.turns_lt_8 = (filtered.turns_lt_8 ?? 0) + 1;
+				continue;
+			}
+			if (toolResultCount(entries) < filter.minToolResults) {
+				filtered.tool_results_lt_8 = (filtered.tool_results_lt_8 ?? 0) + 1;
+				continue;
+			}
+			if (filter.requireFileReread) {
+				const graph = buildReferenceGraph(trace, buildPathIndex(entries, { cwd: trace.cwd }));
+				if (!graph.edges.some((edge) => edge.kind === "file_reread")) {
+					filtered.no_file_reread = (filtered.no_file_reread ?? 0) + 1;
+					continue;
+				}
+			}
+		}
+		traces.push(trace);
+	}
+	cascade.kept = traces.length;
+	return { traces, cascade };
+}

package/src/domains/context/working-set/replay/metrics.ts

@@ -0,0 +1,185 @@
+import { covers, type PathIndex, type PathObservation } from "../path-index.js";
+import type { ReferenceGraph } from "./reference-graph.js";
+import type { ReplayTraceResult } from "./runner.js";
+import type { Trace } from "./trace.js";
+
+export interface ReplayMetrics {
+	traces: number;
+	retention: number;
+	/** Retention after crediting a surviving newer read that covers the original range. */
+	retentionCovered: number;
+	retentionAt10: number;
+	evictionPrecision: number;
+	tokensEvicted: number;
+	/** Tokens freed by evicting items a later turn referenced again: the re-discovery bill a perfect recall would pay. */
+	recallTokens: number;
+	/** Sum over events of the projected tokens after the earliest evicted position: exact-prefix cache re-prefill cost. */
+	coldPrefixTokens: number;
+	evictionEvents: number;
+	/** Fraction of applied events that exhausted the policy's usable candidates. */
+	saturatedEvents: number;
+	turnsToFirstSummary: number | null;
+	/** Summary compactions the modeled summary stage applied; what a policy exists to make rare. */
+	summaries: number;
+}
+
+export interface ReplayMeasurement {
+	trace: Trace;
+	index: PathIndex;
+	graph: ReferenceGraph;
+	replay: ReplayTraceResult;
+}
+
+export interface ReplayMetricAggregate {
+	/** Arithmetic mean of the per-trace metrics; `traces` is the sample size. */
+	mean: ReplayMetrics;
+	/** Number of traces contributing to the nullable `turnsToFirstSummary` mean. */
+	turnsToFirstSummaryCount: number;
+	/** Headline pair-level retention pooled across every critical future reference. */
+	pooledRetention: number;
+	pooledRetentionCovered: number;
+	pooledRetentionAt10: number;
+}
+
+interface MeasuredTrace {
+	metrics: ReplayMetrics;
+	pairs: number;
+	retainedPairs: number;
+	coveredRetainedPairs: number;
+	pairsAt10: number;
+	retainedPairsAt10: number;
+	saturatedEventCount: number;
+}
+
+function safeFraction(numerator: number, denominator: number, empty: number): number {
+	return denominator === 0 ? empty : numerator / denominator;
+}
+
+function survivesThrough(
+	observation: PathObservation,
+	referenceTurn: number,
+	evictedAtTurn: ReadonlyMap<string, number>,
+): boolean {
+	const evictedAt = evictedAtTurn.get(observation.ref.entry);
+	return evictedAt === undefined || evictedAt > referenceTurn;
+}
+
+function hasSurvivingCoveringRead(input: ReplayMeasurement, ref: string, referenceTurn: number): boolean {
+	const original = input.index.byRef.get(ref);
+	if (original?.op !== "read" || original.path.length === 0) return false;
+	return (input.index.byPath.get(original.path) ?? []).some(
+		(later) =>
+			later.op === "read" &&
+			!later.isError &&
+			later.entryIndex > original.entryIndex &&
+			later.turnIndex < referenceTurn &&
+			covers(later.range, original.range) &&
+			survivesThrough(later, referenceTurn, input.replay.evictedAtTurn),
+	);
+}
+
+function measure(input: ReplayMeasurement): MeasuredTrace {
+	let pairs = 0;
+	let retainedPairs = 0;
+	let coveredRetainedPairs = 0;
+	let pairsAt10 = 0;
+	let retainedPairsAt10 = 0;
+	for (const [ref, futureTurns] of input.graph.futureTurnsOf) {
+		const observationTurn = input.index.byRef.get(ref)?.turnIndex;
+		const evictedAt = input.replay.evictedAtTurn.get(ref);
+		for (const referenceTurn of futureTurns) {
+			pairs += 1;
+			const retained = evictedAt === undefined || evictedAt > referenceTurn;
+			if (retained) retainedPairs += 1;
+			if (retained || hasSurvivingCoveringRead(input, ref, referenceTurn)) coveredRetainedPairs += 1;
+			if (observationTurn !== undefined && referenceTurn - observationTurn <= 10) {
+				pairsAt10 += 1;
+				if (retained) retainedPairsAt10 += 1;
+			}
+		}
+	}
+
+	// Precision is the share of evicted items never referenced again; the
+	// complement (items the session came back to) is what live churn would
+	// count as recalls, so it is not reported as a second column.
+	let evictedItems = 0;
+	let safelyEvictedItems = 0;
+	let tokensEvicted = 0;
+	let recallTokens = 0;
+	let coldPrefixTokens = 0;
+	let saturatedEventCount = 0;
+	for (const event of input.replay.events) {
+		if (event.saturated) saturatedEventCount += 1;
+		coldPrefixTokens += event.coldPrefixTokens;
+		for (const item of event.items) {
+			evictedItems += 1;
+			tokensEvicted += item.tokensFreed;
+			const future = input.graph.futureTurnsOf.get(item.ref.entry) ?? [];
+			if (!future.some((turn) => turn > event.turnIndex)) safelyEvictedItems += 1;
+			else recallTokens += item.tokensFreed;
+		}
+	}
+
+	return {
+		metrics: {
+			traces: 1,
+			retention: safeFraction(retainedPairs, pairs, 1),
+			retentionCovered: safeFraction(coveredRetainedPairs, pairs, 1),
+			retentionAt10: safeFraction(retainedPairsAt10, pairsAt10, 1),
+			evictionPrecision: safeFraction(safelyEvictedItems, evictedItems, 1),
+			tokensEvicted,
+			recallTokens,
+			coldPrefixTokens,
+			evictionEvents: input.replay.events.length,
+			saturatedEvents: safeFraction(saturatedEventCount, input.replay.events.length, 0),
+			turnsToFirstSummary: input.replay.turnsToFirstSummary,
+			summaries: input.replay.summaries,
+		},
+		pairs,
+		retainedPairs,
+		coveredRetainedPairs,
+		pairsAt10,
+		retainedPairsAt10,
+		saturatedEventCount,
+	};
+}
+
+export function measureReplayTrace(input: ReplayMeasurement): ReplayMetrics {
+	return measure(input).metrics;
+}
+
+function mean(values: ReadonlyArray<number>): number {
+	return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+
+export function aggregateReplayMetrics(inputs: ReadonlyArray<ReplayMeasurement>): ReplayMetricAggregate {
+	const measured = inputs.map(measure);
+	const firstSummaries = measured
+		.map((entry) => entry.metrics.turnsToFirstSummary)
+		.filter((value): value is number => value !== null);
+	const sum = (field: "pairs" | "retainedPairs" | "coveredRetainedPairs" | "pairsAt10" | "retainedPairsAt10"): number =>
+		measured.reduce((total, entry) => total + entry[field], 0);
+	const totalEvents = measured.reduce((total, entry) => total + entry.metrics.evictionEvents, 0);
+	const saturatedEvents = measured.reduce((total, entry) => total + entry.saturatedEventCount, 0);
+	return {
+		mean: {
+			traces: measured.length,
+			retention: measured.length === 0 ? 1 : mean(measured.map((entry) => entry.metrics.retention)),
+			retentionCovered: measured.length === 0 ? 1 : mean(measured.map((entry) => entry.metrics.retentionCovered)),
+			retentionAt10: measured.length === 0 ? 1 : mean(measured.map((entry) => entry.metrics.retentionAt10)),
+			evictionPrecision: measured.length === 0 ? 1 : mean(measured.map((entry) => entry.metrics.evictionPrecision)),
+			tokensEvicted: mean(measured.map((entry) => entry.metrics.tokensEvicted)),
+			recallTokens: mean(measured.map((entry) => entry.metrics.recallTokens)),
+			coldPrefixTokens: mean(measured.map((entry) => entry.metrics.coldPrefixTokens)),
+			evictionEvents: mean(measured.map((entry) => entry.metrics.evictionEvents)),
+			// Event-pooled: zero-event traces must not dilute the saturation rate.
+			saturatedEvents: safeFraction(saturatedEvents, totalEvents, 0),
+			turnsToFirstSummary: firstSummaries.length === 0 ? null : mean(firstSummaries),
+			summaries: mean(measured.map((entry) => entry.metrics.summaries)),
+		},
+		turnsToFirstSummaryCount: firstSummaries.length,
+		pooledRetention: safeFraction(sum("retainedPairs"), sum("pairs"), 1),
+		pooledRetentionCovered: safeFraction(sum("coveredRetainedPairs"), sum("pairs"), 1),
+		pooledRetentionAt10: safeFraction(sum("retainedPairsAt10"), sum("pairsAt10"), 1),
+	};
+}

package/src/domains/context/working-set/replay/reference-graph.ts

@@ -0,0 +1,79 @@
+import type { SessionEntry } from "../../../session/entries.js";
+import type { PathIndex, PathObservation } from "../path-index.js";
+import type { Trace } from "./trace.js";
+
+export interface ReferenceEdge {
+	/** Ref key of the earlier tool_result. */
+	from: string;
+	toTurnIndex: number;
+	kind: "file_reread" | "file_discovery" | "file_rewrite";
+}
+
+export interface ReferenceGraph {
+	edges: ReadonlyArray<ReferenceEdge>;
+	/** Critical future references only: file_reread and file_discovery. */
+	futureTurnsOf: ReadonlyMap<string, ReadonlyArray<number>>;
+}
+
+const READ_CLASS_OPS = new Set<PathObservation["op"]>(["read", "grep", "find", "ls", "code_nav"]);
+const MUTATION_OPS = new Set<PathObservation["op"]>(["write", "edit"]);
+
+function isToolResult(entry: SessionEntry | undefined): boolean {
+	return entry?.kind === "message" && entry.role === "tool_result";
+}
+
+function isReadableObservation(observation: PathObservation): boolean {
+	return !observation.isError && observation.path.length > 0 && READ_CLASS_OPS.has(observation.op);
+}
+
+function edgeKey(edge: ReferenceEdge): string {
+	return `${edge.from}\u0000${edge.toTurnIndex}\u0000${edge.kind}`;
+}
+
+/**
+ * Label future path use without inspecting result prose. Rewrites are emitted
+ * for diagnosis but deliberately stay out of `futureTurnsOf`: a mutation makes
+ * the earlier read stale rather than critical to retain.
+ */
+export function buildReferenceGraph(trace: Trace, index: PathIndex): ReferenceGraph {
+	const entryById = new Map(trace.entries.map((entry) => [entry.turnId, entry]));
+	const edges: ReferenceEdge[] = [];
+	const seenEdges = new Set<string>();
+	const future = new Map<string, Set<number>>();
+
+	const add = (edge: ReferenceEdge): void => {
+		const key = edgeKey(edge);
+		if (seenEdges.has(key)) return;
+		seenEdges.add(key);
+		edges.push(edge);
+		if (edge.kind === "file_rewrite") return;
+		const turns = future.get(edge.from);
+		if (turns === undefined) future.set(edge.from, new Set([edge.toTurnIndex]));
+		else turns.add(edge.toTurnIndex);
+	};
+
+	for (const earlier of index.observations) {
+		if (!isToolResult(entryById.get(earlier.ref.entry)) || !isReadableObservation(earlier)) continue;
+		const surfaced = new Set(earlier.surfaced);
+		for (const later of index.observations) {
+			if (later.entryIndex <= earlier.entryIndex) continue;
+			if (isReadableObservation(later) && later.path === earlier.path) {
+				add({ from: earlier.ref.entry, toTurnIndex: later.turnIndex, kind: "file_reread" });
+			}
+			if (isReadableObservation(later) && surfaced.has(later.path)) {
+				add({ from: earlier.ref.entry, toTurnIndex: later.turnIndex, kind: "file_discovery" });
+			}
+			if (later.path === earlier.path && MUTATION_OPS.has(later.op)) {
+				add({ from: earlier.ref.entry, toTurnIndex: later.turnIndex, kind: "file_rewrite" });
+			}
+		}
+	}
+
+	const futureTurnsOf = new Map<string, ReadonlyArray<number>>();
+	for (const [ref, turns] of future)
+		futureTurnsOf.set(
+			ref,
+			[...turns].sort((a, b) => a - b),
+		);
+	return { edges, futureTurnsOf };
+}