@iowarp/clio-coder 0.3.7 → 0.3.9

package/src/domains/middleware/memory-intervention.ts

@@ -4,7 +4,7 @@ import {
 	sanitizeToolResultDigest,
 	type ToolResultDigest,
 } from "../../tools/result-disposition.js";
-import { TASK_MEMORY_DEFAULT_PROCEDURAL_CAP, type TaskMemoryBank } from "../memory/task-bank.js";
+import { TASK_MEMORY_DEFAULT_PROCEDURAL_CAP, type TaskMemoryBank, type TaskMemoryEntry } from "../memory/task-bank.js";
 import {
 	runTaskMemoryPolicy,
 	TASK_MEMORY_POLICY_DEFAULT_TIMEOUT_MS,
@@ -12,6 +12,7 @@ import {
 	type TaskMemoryModelClient,
 	type TaskMemoryPolicyReason,
 	type TaskMemoryPolicyResult,
+	type TaskMemoryStepUsage,
 	type TaskMemoryTrajectoryStep,
 } from "../memory/task-memory-policy.js";
 import type { TaskMemoryActivityEvent } from "../memory/task-memory-status.js";
@@ -75,6 +76,25 @@ export interface MemoryInterventionDeps {
 	everyNTools?: number;
 	/** Lazily resolves the explicitly configured background role. Null means rules-only. */
 	getModelClient?: () => TaskMemoryModelClient | null;
+	/**
+	 * True when the background target's inference endpoint is already serving the
+	 * chat target's stream. A memory step then either queues behind the operator's
+	 * own turn on a single-slot server or makes the server evict the resident
+	 * model to serve it, so the boundary is skipped and the skip is recorded.
+	 */
+	backgroundEndpointBusy?: () => boolean;
+	/**
+	 * Provider-reported spend for one completed model step. The composition root
+	 * routes it to the cost ledger; the middleware never prices anything.
+	 */
+	onStepUsage?: (usage: TaskMemoryStepUsage) => void;
+	/**
+	 * Bank entries an llm-tier reminder cited when it reached the operator. The
+	 * composition root proposes them into the durable memory store, which is what
+	 * connects the session bank to the records `/memory` and `clio-coder memory`
+	 * read. Best effort and content-bearing, exactly like `onEnvelope`.
+	 */
+	onInjectedEntries?: (entries: ReadonlyArray<TaskMemoryEntry>) => void;
 	/** Live next-turn settings view; individual fields above remain test-friendly fallbacks. */
 	getSettings?: () => Readonly<MemoryInterventionSettings>;
 	/** Best-effort content-free telemetry; sink failures never affect intervention. */
@@ -348,6 +368,7 @@ export function createMemoryInterventionRegistration(deps: MemoryInterventionDep
 			reminder: null,
 			inputTokens: 0,
 			outputTokens: 0,
+			usage: null,
 			effects: NO_EFFECTS,
 		});
 		const live = settings();
@@ -361,10 +382,20 @@ export function createMemoryInterventionRegistration(deps: MemoryInterventionDep
 		let tier: TaskMemoryTelemetryTier = "rules";
 		promptedStepTier = tier;
 		let promptedResult: TaskMemoryPolicyResult;
+		// A boundary that never reached the model is a skipped boundary rather than
+		// a policy decision, and `dropped` is the telemetry outcome for exactly
+		// that. The returned decision stays `silent`, since no reminder was
+		// produced and the policy vocabulary has no value for a step that never ran.
+		let telemetryDecision: TaskMemoryTelemetryDecision | null = null;
 		try {
 			const client = deps.getModelClient?.() ?? null;
 			if (client === null) {
 				promptedResult = silent("no_client");
+			} else if (deps.backgroundEndpointBusy?.() === true) {
+				tier = "llm";
+				promptedStepTier = tier;
+				promptedResult = silent("endpoint_busy");
+				telemetryDecision = "dropped";
 			} else {
 				tier = "llm";
 				promptedStepTier = tier;
@@ -396,10 +427,18 @@ export function createMemoryInterventionRegistration(deps: MemoryInterventionDep
 			});
 		}
 		lastDecision = promptedResult.decision;
+		if (promptedResult.usage !== null) {
+			try {
+				deps.onStepUsage?.(promptedResult.usage);
+			} catch {
+				// Accounting is bookkeeping. A failing sink never changes memory behavior.
+			}
+		}
+		if (promptedResult.decision === "injected") reportInjectedEntries(promptedResult.reminder);
 		emitTelemetry(
 			triggers,
 			tier,
-			promptedResult.decision,
+			telemetryDecision ?? promptedResult.decision,
 			promptedResult.reason,
 			citedEntryCount(promptedResult.reminder),
 			promptedResult.inputTokens,
@@ -462,10 +501,35 @@ export function createMemoryInterventionRegistration(deps: MemoryInterventionDep
 		if (activity.length > MEMORY_INTERVENTION_ACTIVITY_LIMIT) activity.length = MEMORY_INTERVENTION_ACTIVITY_LIMIT;
 	}
 
-	function citedEntryCount(message: string | null): number {
-		if (message === null) return 0;
+	function citedEntries(message: string | null): TaskMemoryEntry[] {
+		if (message === null) return [];
 		const snapshot = deps.bank.snapshot();
-		return [...snapshot.knowledge, ...snapshot.procedural].filter((entry) => message.includes(`[${entry.id}]`)).length;
+		return [...snapshot.knowledge, ...snapshot.procedural].filter((entry) => message.includes(`[${entry.id}]`));
+	}
+
+	function citedEntryCount(message: string | null): number {
+		return citedEntries(message).length;
+	}
+
+	/**
+	 * Hand the cited entries to the durable store.
+	 *
+	 * Only the model tier reports here. A rules-tier reminder cites an entry this
+	 * middleware wrote itself out of one repeated tool failure, and proposing
+	 * every one of those as a durable lesson would fill the operator's review
+	 * queue with rows nobody asked for. The model tier's entries are the ones the
+	 * background plane produced, which is what #229 asks to be reviewable.
+	 */
+	function reportInjectedEntries(reminder: string | null): void {
+		if (deps.onInjectedEntries === undefined) return;
+		const entries = citedEntries(reminder);
+		if (entries.length === 0) return;
+		try {
+			deps.onInjectedEntries(entries);
+		} catch {
+			// A failing store is the composition root's problem to report, never a
+			// reason to fail the step that produced the reminder.
+		}
 	}
 
 	function observeBeforeTool(input: MiddlewareHookInput): void {

package/src/domains/middleware/memory-step-endpoint.ts

@@ -0,0 +1,71 @@
+/**
+ * Publishing and accounting for the endpoint a background memory step uses.
+ *
+ * A proactive-memory step that runs on the chat target's own endpoint moves
+ * that server's prefix cache, so the next turn is cold for a reason Clio caused
+ * and can name. The turn context turns {@link BusChannels.MemoryStepCompleted}
+ * into the `background_memory` expected-cold reason; without it `/context`
+ * reports the turn as an unexplained provider re-prefill.
+ *
+ * The disturbance is a property of the request having left the process, not of
+ * the answer coming back. A step that times out still had its whole trajectory
+ * prefilled into the single prefix slot, because aborting the HTTP request does
+ * not stop a llama.cpp prefill (the measurement is recorded in
+ * `src/interactive/turn-prewarm.ts`). Publishing from the usage sink instead
+ * missed exactly that case, since usage is only assigned once the call resolves.
+ *
+ * The same request also occupies real endpoint capacity. Both responsibilities
+ * live here around the client's `complete`: the slot is registered only after
+ * the middleware's `endpoint_busy` admission check, so a step never sees its
+ * own slot, then released in the `finally` that also announces the disturbance.
+ * Both happen on resolve, timeout, and transport throw, and neither happens for
+ * a boundary that skipped the model altogether (`endpoint_busy`, no configured
+ * route), because those never call `complete`.
+ */
+
+import { BusChannels } from "../../core/bus-events.js";
+import type { SafeEventBus } from "../../core/event-bus.js";
+import { registerForegroundStream } from "../providers/endpoint-capacity.js";
+
+/** What the announcer needs: somewhere to publish, and the route to name. */
+export interface MemoryStepEndpointAnnouncerDeps {
+	bus: Pick<SafeEventBus, "emit"> | null;
+	/** Canonical endpoint the step calls, or null when the route has none. */
+	endpointKey: string | null;
+	targetId: string;
+}
+
+/**
+ * Wrap one background memory `complete` so every request that leaves the
+ * process holds one endpoint slot and publishes its endpoint exactly once,
+ * whatever the call does next.
+ *
+ * A route with no canonical endpoint key is unchanged. A null bus suppresses
+ * the announcement but still counts capacity for a route whose key is known.
+ */
+export function announceMemoryStepEndpoint<Request, Response>(
+	deps: MemoryStepEndpointAnnouncerDeps,
+	complete: (request: Request) => Promise<Response>,
+): (request: Request) => Promise<Response> {
+	const { bus, endpointKey, targetId } = deps;
+	if (endpointKey === null) return complete;
+	return async (request: Request): Promise<Response> => {
+		// `runPromptedStep` performs its busy check before it invokes `complete`,
+		// so this registration counts the outgoing request without self-refusing.
+		const releaseEndpointSlot = registerForegroundStream(endpointKey);
+		try {
+			return await complete(request);
+		} finally {
+			releaseEndpointSlot();
+			// Bookkeeping never changes memory behavior, and a crashing subscriber
+			// must not turn a resolved step into a failed one.
+			if (bus !== null) {
+				try {
+					bus.emit(BusChannels.MemoryStepCompleted, { endpointKey, targetId });
+				} catch {
+					// The safe bus already contains listener errors; this covers the rest.
+				}
+			}
+		}
+	};
+}

package/src/domains/observability/background-memory-usage.ts

@@ -0,0 +1,140 @@
+/**
+ * Accounting for one proactive-memory step.
+ *
+ * A background memory step is a priced model call that never enters the
+ * session: it appends nothing to the session JSONL, exactly as a `/btw` side
+ * question does not. Until this existed the spend was recorded nowhere at all.
+ * The operator's own telemetry ledger held 60 model steps, 137,205 tokens, and
+ * 1,666 seconds of model time across 14 days while `/cost` showed no row for
+ * any of it and `clio-coder usage report` could not see it either (#229).
+ *
+ * The two writes are the same pair the side-question path makes: the
+ * in-process cost tracker under the `background-memory` label, which is what
+ * `/cost` folds, and one durable row in the out-of-turn usage store, which is
+ * what an archive reader folds after the process is gone.
+ */
+
+import { uncachedPrefillTokens } from "../../core/cache-telemetry.js";
+import type { CostProvenance } from "../providers/index.js";
+import type { CostEntryLabel, UsageBreakdown } from "./cost.js";
+import { appendOutOfTurnUsageRow, type OutOfTurnUsageRow } from "./out-of-turn-usage.js";
+
+/** Provider-reported spend for one memory step, as the calling client observed it. */
+export interface BackgroundMemoryStepUsage {
+	targetId: string;
+	attributedModelId: string;
+	input: number;
+	output: number;
+	cacheRead: number;
+	cacheWrite: number;
+	reasoning: number;
+	totalTokens: number;
+	costUsd: number;
+	costProvenance: CostProvenance;
+	durationMs: number;
+	backend: {
+		promptTokens: number;
+		cachedTokens: number | null;
+		promptMs: number;
+		source: string;
+	} | null;
+}
+
+/** The subset of the observability contract this accounting needs. */
+export interface BackgroundMemoryUsageSink {
+	recordTokens(
+		providerId: string,
+		attributedModelId: string,
+		tokens: number,
+		costUsd?: number,
+		breakdown?: Partial<UsageBreakdown>,
+		costProvenance?: CostProvenance,
+		modelIdFacts?: unknown,
+		label?: CostEntryLabel,
+	): void;
+}
+
+export interface RecordBackgroundMemoryStepInput {
+	usage: BackgroundMemoryStepUsage;
+	stateDir: string;
+	sessionId: string | null;
+	/** The cwd hash the session ledger is filed under, so `usage report --repo` selects it. */
+	repoIdentity: string | null;
+	observability?: BackgroundMemoryUsageSink | undefined;
+	now?: Date;
+	/** Test seam; production appends to the out-of-turn store under the state dir. */
+	appendRow?: (stateDir: string, row: OutOfTurnUsageRow) => void;
+}
+
+/** Build the durable row for one memory step without writing it. */
+export function backgroundMemoryUsageRow(
+	usage: BackgroundMemoryStepUsage,
+	options: { sessionId: string | null; repoIdentity: string | null; now?: Date },
+): OutOfTurnUsageRow {
+	return {
+		label: "background-memory",
+		sessionId: options.sessionId,
+		repoIdentity: options.repoIdentity,
+		timestamp: (options.now ?? new Date()).toISOString(),
+		target: usage.targetId,
+		attributedModelId: usage.attributedModelId,
+		usage: {
+			input: usage.input,
+			output: usage.output,
+			cacheRead: usage.cacheRead,
+			cacheWrite: usage.cacheWrite,
+			reasoning: usage.reasoning,
+			totalTokens: usage.totalTokens,
+			costUsd: usage.costUsd,
+			costProvenance: usage.costProvenance,
+		},
+		timing: { durationMs: usage.durationMs },
+		...(usage.backend === null
+			? {}
+			: {
+					promptCache: {
+						promptTokens: usage.backend.promptTokens,
+						cachedTokens: usage.backend.cachedTokens,
+						uncachedPrefillTokens: uncachedPrefillTokens({
+							promptTokens: usage.backend.promptTokens,
+							cachedTokens: usage.backend.cachedTokens,
+							predictedTokens: 0,
+							promptMs: usage.backend.promptMs,
+							predictedMs: 0,
+							source: "llamacpp-timings",
+						}),
+						promptMs: usage.backend.promptMs,
+						source: usage.backend.source,
+					},
+				}),
+	};
+}
+
+/** Record one memory step in the session's cost totals and in the durable store. */
+export function recordBackgroundMemoryStep(input: RecordBackgroundMemoryStepInput): OutOfTurnUsageRow {
+	input.observability?.recordTokens(
+		input.usage.targetId,
+		input.usage.attributedModelId,
+		input.usage.totalTokens,
+		input.usage.costUsd,
+		{
+			input: input.usage.input,
+			output: input.usage.output,
+			cacheRead: input.usage.cacheRead,
+			cacheWrite: input.usage.cacheWrite,
+			reasoningTokens: input.usage.reasoning,
+			totalTokens: input.usage.totalTokens,
+			apiCalls: 1,
+		},
+		input.usage.costProvenance,
+		undefined,
+		"background-memory",
+	);
+	const row = backgroundMemoryUsageRow(input.usage, {
+		sessionId: input.sessionId,
+		repoIdentity: input.repoIdentity,
+		...(input.now === undefined ? {} : { now: input.now }),
+	});
+	(input.appendRow ?? appendOutOfTurnUsageRow)(input.stateDir, row);
+	return row;
+}

package/src/domains/observability/cost.ts

@@ -116,7 +116,7 @@ export interface UsageBreakdown {
  * the usage surfaces separate these out so an operator can see that money was
  * spent beside the session rather than inside it.
  */
-export type CostEntryLabel = "side-question" | "handoff";
+export type CostEntryLabel = "side-question" | "handoff" | "prewarm" | "background-memory";
 
 export interface CostEntry {
 	providerId: string;

package/src/domains/observability/index.ts

@@ -16,6 +16,13 @@ export function createObservabilityDomainModule(options: ObservabilityBundleOpti
 
 export type { AccountabilitySummary } from "./accountability.js";
 export { readAccountabilitySummary, summarizeEvidenceIndex } from "./accountability.js";
+export {
+	type BackgroundMemoryStepUsage,
+	type BackgroundMemoryUsageSink,
+	backgroundMemoryUsageRow,
+	type RecordBackgroundMemoryStepInput,
+	recordBackgroundMemoryStep,
+} from "./background-memory-usage.js";
 export type {
 	ObservabilityContract,
 	ObservabilityNotice,

package/src/domains/observability/out-of-turn-usage.ts

@@ -1,6 +1,7 @@
 /**
  * Durable record of the model calls that were billed beside a session rather
- * than inside it: `/btw` side questions and `/handoff` extraction rounds.
+ * than inside it: `/btw` side questions, `/handoff` extraction rounds, and the
+ * session pre-warm.
  *
  * These rounds deliberately append nothing to the session JSONL. A fleet run
  * briefs its workers from the transcript, so a question the operator asked to
@@ -50,6 +51,27 @@ export const MAX_OUT_OF_TURN_USAGE_ROWS = 1000;
  */
 const BOUND_CHECK_INTERVAL = 64;
 
+/** Wall-clock facts about one out-of-turn call. */
+export interface OutOfTurnTiming {
+	/** Duration of the model call itself, as the caller measured it. */
+	durationMs: number;
+}
+
+/**
+ * Backend prefill facts for one out-of-turn call, in the shape brief #247
+ * established for assistant entries. A background memory step re-sends a whole
+ * trajectory to a server that has just been serving the chat prefix, so whether
+ * that prefill was cached is the difference between a cheap step and a step
+ * that re-prefills thousands of tokens on the operator's own machine.
+ */
+export interface OutOfTurnPromptCache {
+	promptTokens: number;
+	cachedTokens: number | null;
+	uncachedPrefillTokens: number | null;
+	promptMs: number;
+	source: string;
+}
+
 /** Provider-reported usage for one out-of-turn call. */
 export interface OutOfTurnUsage {
 	input: number;
@@ -75,6 +97,10 @@ export interface OutOfTurnUsageRow {
 	target: string;
 	attributedModelId: string;
 	usage: OutOfTurnUsage;
+	/** Present when the caller measured the call. */
+	timing?: OutOfTurnTiming;
+	/** Present when the serving backend reported prefill facts. */
+	promptCache?: OutOfTurnPromptCache;
 }
 
 export interface OutOfTurnUsageReadResult {
@@ -179,10 +205,13 @@ export function readOutOfTurnUsageRows(stateDir: string): OutOfTurnUsageReadResu
 function asOutOfTurnUsageRow(value: unknown): OutOfTurnUsageRow | null {
 	if (!isRecord(value)) return null;
 	const label = value.label;
-	if (label !== "side-question" && label !== "handoff") return null;
+	if (label !== "side-question" && label !== "handoff" && label !== "prewarm" && label !== "background-memory")
+		return null;
 	if (typeof value.timestamp !== "string" || value.timestamp.length === 0) return null;
 	if (!isRecord(value.usage)) return null;
 	const usage = value.usage;
+	const timing = asTiming(value.timing);
+	const promptCache = asPromptCache(value.promptCache);
 	return {
 		label,
 		sessionId: typeof value.sessionId === "string" ? value.sessionId : null,
@@ -203,6 +232,26 @@ function asOutOfTurnUsageRow(value: unknown): OutOfTurnUsageRow | null {
 			costUsd: numberOr0(usage.costUsd),
 			costProvenance: asCostProvenance(usage.costProvenance),
 		},
+		...(timing === null ? {} : { timing }),
+		...(promptCache === null ? {} : { promptCache }),
+	};
+}
+
+function asTiming(value: unknown): OutOfTurnTiming | null {
+	if (!isRecord(value)) return null;
+	const durationMs = numberOr0(value.durationMs);
+	return durationMs === 0 ? null : { durationMs };
+}
+
+function asPromptCache(value: unknown): OutOfTurnPromptCache | null {
+	if (!isRecord(value)) return null;
+	if (typeof value.source !== "string" || value.source.length === 0) return null;
+	return {
+		promptTokens: numberOr0(value.promptTokens),
+		cachedTokens: typeof value.cachedTokens === "number" ? value.cachedTokens : null,
+		uncachedPrefillTokens: typeof value.uncachedPrefillTokens === "number" ? value.uncachedPrefillTokens : null,
+		promptMs: numberOr0(value.promptMs),
+		source: value.source,
 	};
 }