@iowarp/clio-coder 0.4.2 → 0.4.3

package/src/entry/orchestrator.ts

@@ -1,3 +1,5 @@
+import { randomUUID } from "node:crypto";
+import { readFileSync } from "node:fs";
 import { join } from "node:path";
 import chalk from "chalk";
 import { modelBootstrapGenerate, resolveBootstrapRoute } from "../cli/bootstrap-generate.js";
@@ -44,6 +46,7 @@ import { ConfigDomainModule, createConfigDomainModule } from "../domains/config/
 import type { ContextContract } from "../domains/context/contract.js";
 import { bootstrapInputFromInitOptions } from "../domains/context/init-options.js";
 import { createContextDomainModule } from "../domains/context/runtime.js";
+import { endpointCapacityUsage } from "../domains/dispatch/capacity-lease.js";
 import type { DispatchContract } from "../domains/dispatch/contract.js";
 import { createDispatchDedupRegistration } from "../domains/dispatch/dedup.js";
 import { agentRoleFactsResolver } from "../domains/dispatch/execution-role.js";
@@ -77,6 +80,7 @@ import {
 	taskMemoryTracePath,
 } from "../domains/memory/index.js";
 import { TaskMemoryBank } from "../domains/memory/task-bank.js";
+import { TaskMemoryEndpointBusyError } from "../domains/memory/task-memory-policy.js";
 import {
 	createDetachedDispatchNudgeRegistration,
 	createReadOnlyExplorationNudgeRegistration,
@@ -99,8 +103,10 @@ import { createTaskBoardReminderRegistration } from "../domains/middleware/task-
 import { createTaskNudgeRegistration } from "../domains/middleware/task-nudge.js";
 import { createWatchdogRegistration } from "../domains/middleware/watchdog.js";
 import type { MuxContract } from "../domains/mux/index.js";
+import type { BackgroundMemoryUsageSink } from "../domains/observability/background-memory-usage.js";
+import { recordFailedCompactionCalls } from "../domains/observability/compaction-usage.js";
 import type { ObservabilityContract } from "../domains/observability/index.js";
-import { ObservabilityDomainModule, recordBackgroundMemoryStep } from "../domains/observability/index.js";
+import { ObservabilityDomainModule } from "../domains/observability/index.js";
 import type { PromptsContract } from "../domains/prompts/contract.js";
 import { createPromptsDomainModule } from "../domains/prompts/index.js";
 import type { ProvidersContract, TargetDescriptor, ThinkingLevel } from "../domains/providers/index.js";
@@ -109,12 +115,12 @@ import {
 	applyModelCapabilityPatch,
 	canonicalEndpointKey,
 	firstRuntimeResolutionError,
-	foregroundStreamUsage,
 	isOrchestratorEligibleRuntime,
 	normalizeCostProvenance,
 	ProvidersDomainModule,
 	refineRuntimeTargetWithModelHints,
 	registerForegroundStream,
+	resolveEndpointCapacities,
 	resolveModelCapabilities,
 	resolveModelRuntimeCapabilitiesForProviders,
 	resolveRuntimeTarget,
@@ -122,8 +128,10 @@ import {
 	targetRequiresAuth,
 	VALID_THINKING_LEVELS,
 } from "../domains/providers/index.js";
+import { hasLiveModelCatalog, modelResidencyForStatus } from "../domains/providers/model-discovery.js";
 import { memoryInterventionModelMaxTokens } from "../domains/providers/model-runtime-capabilities.js";
 import { getRuntimeRegistry } from "../domains/providers/registry.js";
+import { resolveModelReference } from "../domains/providers/resolver.js";
 import { registerBuiltinRuntimes } from "../domains/providers/runtimes/builtins.js";
 import {
 	createResourcesDomainModule,
@@ -135,7 +143,12 @@ import {
 import { DEFAULT_RECENT_ENTRY_LIMIT } from "../domains/safety/finish-contract.js";
 import { createFinishContractRegistration } from "../domains/safety/finish-contract-registration.js";
 import type { AutonomyLevel, SafetyContract } from "../domains/safety/index.js";
-import { parseRigorOverride, resolveRigor, SafetyDomainModule } from "../domains/safety/index.js";
+import {
+	modelMayActivateSkills,
+	parseRigorOverride,
+	resolveRigor,
+	SafetyDomainModule,
+} from "../domains/safety/index.js";
 import type { ProtectedArtifactState } from "../domains/safety/protected-artifacts.js";
 import {
 	createProtectedArtifactsRegistration,
@@ -143,6 +156,7 @@ import {
 } from "../domains/safety/protected-artifacts-registration.js";
 import type { SchedulingContract } from "../domains/scheduling/contract.js";
 import { SchedulingDomainModule } from "../domains/scheduling/index.js";
+import type { CompactionCallObservation } from "../domains/session/compaction/compact.js";
 import { type CompactResult, compact } from "../domains/session/compaction/compact.js";
 import { collectSessionEntries } from "../domains/session/compaction/session-entries.js";
 import { estimateTokens } from "../domains/session/compaction/tokens.js";
@@ -167,7 +181,7 @@ import { ToolchainDomainModule } from "../domains/toolchain/index.js";
 import { createUserTasksStore } from "../domains/user-tasks/store.js";
 import { type AcpSafeSettingsPatch, type AcpSafeSettingsSnapshot, serveClioAcpAgent } from "../engine/acp/server.js";
 import { createStdioServerTransport } from "../engine/acp/transport.js";
-import { completeEngineText } from "../engine/ai.js";
+import { completeEngineText, type EngineTextCompletionResult } from "../engine/ai.js";
 import { setProtectedModelsProvider } from "../engine/apis/residency.js";
 import {
 	createLoopGuardRegistration,
@@ -179,9 +193,12 @@ import type { EngineModel } from "../engine/types.js";
 import { createChatLoop } from "../interactive/chat-loop.js";
 import { type RunIo, startInteractive } from "../interactive/index.js";
 import { buildModelReplayAgentMessagesFromTurns } from "../interactive/model-session-replay.js";
+import { prepareBackgroundModelMetadata } from "./background-model-metadata.js";
 import type { BootOptions } from "./boot-options.js";
+import { readCompactionSystemPrompt } from "./compaction-prompt.js";
 import { createExtensionReloadCoordinator } from "./extension-reload.js";
 import { resolvePanesEnablement } from "./panes-activation.js";
+import { bindTaskMemoryLifecycle, captureTaskMemoryUsage } from "./task-memory-lifecycle.js";
 
 export type { BootOptions, HeadlessSamplingOverrides } from "./boot-options.js";
 
@@ -287,6 +304,8 @@ interface CompactionResolution {
 	model: EngineModel;
 	targetId: string;
 	endpointKey: string | null;
+	wireModelId: string;
+	headers?: Record<string, string>;
 	apiKey?: string;
 }
 
@@ -312,53 +331,71 @@ function advanceThinkingLevel(current: ThinkingLevel, available: ReadonlyArray<T
  */
 const LOCAL_API_KEY_FALLBACK = "clio-coder-local-target";
 
-async function resolveApiKeyForTarget(
-	target: TargetDescriptor,
-	providers: ProvidersContract,
-	signal?: AbortSignal,
-): Promise<string | undefined> {
-	const runtime = providers.getRuntime(target.runtime);
-	if (!runtime) return undefined;
-	// A target that needs no credential still needs the placeholder. Returning
-	// nothing here left compaction as the one path that did not send it, so
-	// `/context compact` and every automatic compaction at the window threshold
-	// failed on exactly the local runtimes Clio is built for, while ordinary
-	// turns against the same target succeeded.
-	if (!targetRequiresAuth(target, runtime)) return LOCAL_API_KEY_FALLBACK;
-	const resolved = await providers.auth.resolveForTarget(target, runtime, signal ? { signal } : undefined);
-	return resolved.apiKey;
-}
-
 /**
  * The background memory role, resolved to the endpoint it would call.
  *
- * The endpoint key is what lets the middleware decline a step that would land
- * on the same inference scheduler the chat target is streaming against (#229,
- * #250), and it is the same key dispatch admission counts slots on.
+ * The endpoint key joins the existing dispatch capacity evidence and local
+ * request holds. A gateway key does not identify one physical model server or
+ * imply a one-slot limit; the configured protocol owns downstream routing.
  */
 interface BackgroundMemoryRoute {
 	client: TaskMemoryModelClient;
+	selection: "dedicated" | "chat-fallback";
+	fallbackReason?: string;
 	targetId: string;
 	wireModelId: string;
 	endpointKey: string | null;
 	modelMaxTokens(configuredMaxTokens: number): number;
 }
 
-function createBackgroundMemoryModelClient(
+export function createBackgroundMemoryModelClient(
 	providers: ProvidersContract,
 	settings: Readonly<ClioSettings>,
 	timeoutMs: number,
 	bus: Pick<SafeEventBus, "emit"> | null,
+	fallbackOnly = false,
 ): BackgroundMemoryRoute | null {
-	const targetId = settings.context.memory.target?.trim();
-	const wireModelId = settings.context.memory.model?.trim();
-	if (!targetId || !wireModelId) return null;
-	// One model cannot both drive the action agent and think its way through a
-	// memory step: the memory call would contend with the agent's own decoding on
-	// the same server, and a reasoning model spends most of a memory step
-	// deliberating over a task it is not executing. Memory falls back to its free
-	// deterministic tier rather than degrading the work the operator asked for.
-	if (backgroundSharesReasoningModelWithOrchestrator(providers, settings)) return null;
+	const configuredTarget = settings.context.memory.target?.trim();
+	const configuredModel = settings.context.memory.model?.trim();
+	if (!configuredTarget || !configuredModel) return null;
+	const chatTarget = settings.chat.target?.trim();
+	const chatModel = settings.chat.model?.trim();
+	const sameRoute = configuredTarget === chatTarget && configuredModel === chatModel;
+	if (fallbackOnly && sameRoute) return null;
+	let fallbackReason: string | undefined;
+	if (!fallbackOnly) {
+		try {
+			return prepareBackgroundMemoryRoute(providers, configuredTarget, configuredModel, timeoutMs, bus, "dedicated");
+		} catch {
+			fallbackReason = "configured route unavailable";
+		}
+	} else fallbackReason = "configured route client error";
+	if (!chatTarget || !chatModel || sameRoute) {
+		if (fallbackOnly) return null;
+		throw new Error("configured memory route is unavailable and no distinct chat fallback can run");
+	}
+	try {
+		return {
+			...prepareBackgroundMemoryRoute(providers, chatTarget, chatModel, timeoutMs, bus, "chat-fallback"),
+			fallbackReason,
+		};
+	} catch {
+		if (fallbackOnly) return null;
+		throw new Error("neither the configured memory route nor the active chat fallback is available");
+	}
+}
+
+function prepareBackgroundMemoryModel(providers: ProvidersContract, targetId: string, wireModelId: string) {
+	const status = providers.list().find((entry) => entry.target.id === targetId);
+	if (
+		status &&
+		(!status.available ||
+			status.health.status === "down" ||
+			(hasLiveModelCatalog(status) && !status.discoveredModels.includes(wireModelId)) ||
+			modelResidencyForStatus(status, wireModelId) === "absent" ||
+			modelResidencyForStatus(status, wireModelId) === "loading")
+	)
+		throw new Error("background memory route is unavailable");
 	const resolved = resolveRuntimeTarget(providers, {
 		targetId,
 		wireModelId,
@@ -378,9 +415,37 @@ function createBackgroundMemoryModelClient(
 	const model = resolved.target.runtime.synthesizeModel(resolved.target.target, resolved.target.wireModelId, kbHit);
 	const refined = refineRuntimeTargetWithModelHints(resolved.target, model, providers.knowledgeBase);
 	applyModelCapabilityPatch(model, refined.capabilities);
-	const costProvenance = normalizeCostProvenance(refined.costProvenance);
+	return { model, refined };
+}
+
+function backgroundMemoryEndpointBusy(
+	providers: ProvidersContract,
+	endpointKey: string | null,
+	targets: ReadonlyArray<TargetDescriptor>,
+): boolean {
+	if (endpointKey === null) return false;
+	const capacity = resolveEndpointCapacities({
+		statuses: providers.list(),
+		targets,
+		runtimeFor: (id) => providers.getRuntime(id),
+	})[endpointKey];
+	// Gateways and non-fixed schedulers have no invented local one-slot cap.
+	return capacity !== undefined && (endpointCapacityUsage()[endpointKey] ?? 0) >= capacity.limit;
+}
+
+function prepareBackgroundMemoryRoute(
+	providers: ProvidersContract,
+	targetId: string,
+	wireModelId: string,
+	timeoutMs: number,
+	bus: Pick<SafeEventBus, "emit"> | null,
+	selection: BackgroundMemoryRoute["selection"],
+): BackgroundMemoryRoute {
+	const initial = prepareBackgroundMemoryModel(providers, targetId, wireModelId);
+	const { refined } = initial;
 	const endpointKey = canonicalEndpointKey(refined.target);
 	return {
+		selection,
 		targetId,
 		wireModelId: refined.wireModelId,
 		endpointKey,
@@ -394,47 +459,121 @@ function createBackgroundMemoryModelClient(
 			// Wrapped at the layer that knows a request left the process: the step
 			// holds endpoint capacity while it is out and publishes the cache
 			// disturbance even when a timeout means the usage sink never sees it.
-			complete: announceMemoryStepEndpoint({ bus, endpointKey, targetId }, async (request) => {
+			complete: async (request) => {
+				await prepareBackgroundModelMetadata(providers, targetId, request.signal);
+				const { model, refined } = prepareBackgroundMemoryModel(providers, targetId, wireModelId);
+				if (refined.runtimeId !== initial.refined.runtimeId || canonicalEndpointKey(refined.target) !== endpointKey) {
+					throw new Error("background memory target changed during preparation");
+				}
+				const costProvenance = normalizeCostProvenance(refined.costProvenance);
 				const apiKey = targetRequiresAuth(refined.target, refined.runtime)
 					? (await providers.auth.resolveForTarget(refined.target, refined.runtime, { signal: request.signal })).apiKey
 					: LOCAL_API_KEY_FALLBACK;
-				const startedAt = Date.now();
-				const completion = await completeEngineText({
-					model,
-					systemPrompt: request.systemPrompt,
-					userPrompt: request.userPrompt,
-					maxTokens: request.maxTokens,
-					// Always off, never the operator's chat thinking level. A model that
-					// reasons anyway still works: `completeEngineText` keeps only text
-					// blocks, and the memory output budget leaves room for the preamble.
-					thinkingLevel: "off",
-					signal: request.signal,
-					timeoutMs,
-					...(apiKey === undefined ? {} : { apiKey }),
-				});
-				// The step is billed here whatever the policy later decides about the
-				// answer. A model that read a trajectory and chose silence spent the
-				// same prefill as one that produced a reminder.
-				return {
-					text: completion.text,
-					inputTokens: completion.inputTokens,
-					outputTokens: completion.outputTokens,
-					usage: {
+				request.signal.throwIfAborted();
+				// Preparation yielded after the middleware's first check. Recheck current
+				// occupancy immediately before the synchronous foreground-slot registration;
+				// no await may separate this admission from announceMemoryStepEndpoint.
+				if (backgroundMemoryEndpointBusy(providers, endpointKey, [refined.target])) {
+					throw new TaskMemoryEndpointBusyError("background memory endpoint is busy");
+				}
+				return announceMemoryStepEndpoint({ bus, endpointKey, targetId }, async () => {
+					const startedAt = Date.now();
+					let observedUsage: TaskMemoryStepUsage | undefined;
+					const mapUsage = (completion: Pick<EngineTextCompletionResult, "usage" | "backend">): TaskMemoryStepUsage => ({
+						...completion.usage,
 						targetId,
 						attributedModelId: refined.wireModelId,
-						input: completion.usage.input,
-						output: completion.usage.output,
-						cacheRead: completion.usage.cacheRead,
-						cacheWrite: completion.usage.cacheWrite,
-						reasoning: completion.usage.reasoning,
-						totalTokens: completion.usage.totalTokens,
-						costUsd: completion.usage.costUsd,
 						costProvenance,
 						durationMs: Date.now() - startedAt,
 						backend: completion.backend,
-					},
-				};
-			}),
+					});
+					const completion = await completeEngineText({
+						model,
+						systemPrompt: request.systemPrompt,
+						userPrompt: request.userPrompt,
+						maxTokens:
+							refined.capabilityDecisions.maxTokens > 0
+								? Math.min(request.maxTokens, refined.capabilityDecisions.maxTokens)
+								: request.maxTokens,
+						// Always off, never the operator's chat thinking level. A model that
+						// reasons anyway still works: `completeEngineText` keeps only text
+						// blocks, and the memory output budget leaves room for the preamble.
+						thinkingLevel: "off",
+						signal: request.signal,
+						timeoutMs,
+						onUsage: (observation) => {
+							observedUsage = mapUsage(observation);
+							request.onUsage?.(observedUsage);
+						},
+						...(apiKey === undefined ? {} : { apiKey }),
+						...(refined.target.auth?.headers ? { headers: refined.target.auth.headers } : {}),
+					});
+					// The step is billed here whatever the policy later decides about the
+					// answer. A model that read a trajectory and chose silence spent the
+					// same prefill as one that produced a reminder.
+					return {
+						text: completion.text,
+						inputTokens: completion.inputTokens,
+						outputTokens: completion.outputTokens,
+						usage: observedUsage ?? mapUsage(completion),
+					};
+				})(request);
+			},
+		},
+	};
+}
+
+/** Production callback composition shared with routing/capacity contracts. */
+export function createBackgroundMemoryRouting(
+	providers: ProvidersContract,
+	getSettings: () => Readonly<ClioSettings> | undefined,
+	bus: Pick<SafeEventBus, "emit"> | null,
+) {
+	let route: BackgroundMemoryRoute | null = null;
+	let snapshot: Readonly<ClioSettings> | undefined;
+	let lastNotice: string | null = null;
+	const noteFallback = (): void => {
+		if (route?.selection !== "chat-fallback") {
+			lastNotice = null;
+			return;
+		}
+		const message = `Memory: ${route.fallbackReason}; selected chat fallback ${route.targetId}/${route.wireModelId} for this step, subject to available endpoint capacity.`;
+		if (message === lastNotice) return;
+		lastNotice = message;
+		try {
+			bus?.emit(BusChannels.RuntimeNotice, {
+				kind: "degraded",
+				level: "info",
+				message,
+				targetId: route.targetId,
+				runtimeId: providers.getTarget(route.targetId)?.runtime ?? "unknown",
+				model: route.wireModelId,
+			});
+		} catch {
+			/* Advisory only. */
+		}
+	};
+	return {
+		getModelClient: (): TaskMemoryModelClient | null => {
+			const current = getSettings();
+			snapshot = current === undefined ? undefined : structuredClone(current);
+			route =
+				snapshot === undefined
+					? null
+					: createBackgroundMemoryModelClient(providers, snapshot, snapshot.context.memory.timeoutMs, bus);
+			noteFallback();
+			return route?.client ?? null;
+		},
+		getFallbackModelClient: (): TaskMemoryModelClient | null => {
+			if (route?.selection !== "dedicated" || snapshot === undefined) return null;
+			route = createBackgroundMemoryModelClient(providers, snapshot, snapshot.context.memory.timeoutMs, bus, true);
+			noteFallback();
+			return route?.client ?? null;
+		},
+		getModelMaxTokens: (configured: number): number => route?.modelMaxTokens(configured) ?? configured,
+		backgroundEndpointBusy: (): boolean => {
+			if (route?.endpointKey == null || snapshot === undefined) return false;
+			return backgroundMemoryEndpointBusy(providers, route.endpointKey, snapshot.targets);
 		},
 	};
 }
@@ -472,74 +611,84 @@ function agentRoleToolWarnings(providers: ProvidersContract, settings: Readonly<
 	return warnings;
 }
 
-/**
- * True when the background role names the same model the orchestrator runs and
- * that model reasons. Distinct models, or a non-reasoning shared model, are both
- * fine; this only catches the one-model-does-everything configuration.
- */
-function backgroundSharesReasoningModelWithOrchestrator(
+async function resolveCompactionModel(
+	settings: ClioSettings,
 	providers: ProvidersContract,
-	settings: Readonly<ClioSettings>,
-): boolean {
-	const backgroundModel = settings.context.memory.model?.trim();
-	const orchestratorModel = settings.chat.model?.trim();
-	if (!backgroundModel || backgroundModel !== orchestratorModel) return false;
-	if (settings.context.memory.target?.trim() !== settings.chat.target?.trim()) return false;
-	try {
-		const status = providers.list().find((entry) => entry.target.id === settings.context.memory.target?.trim());
-		if (!status) return false;
-		return resolveModelCapabilities(status, backgroundModel, providers.knowledgeBase).reasoning === true;
-	} catch {
-		// An unresolvable capability is not evidence of a reasoning model; the
-		// operator's explicit configuration stands.
-		return false;
+): Promise<CompactionResolution | null> {
+	const override = settings.context.compaction.model;
+	let targetId = settings.chat.target;
+	let wireModelId = settings.chat.model;
+	if (override !== undefined && override !== null) {
+		let selected: ReturnType<typeof resolveModelReference>;
+		try {
+			selected = resolveModelReference(override, providers);
+		} catch {
+			throw new Error("context.compaction.model is an invalid pattern; set a unique target/model reference");
+		}
+		if (!selected.ref || selected.warning) {
+			throw new Error(
+				"context.compaction.model must match exactly one configured model; set a unique target/model reference",
+			);
+		}
+		targetId = selected.ref.target;
+		wireModelId = selected.ref.model;
 	}
-}
-
-function synthesizeOrchestratorModel(
-	providers: ProvidersContract,
-	target: TargetDescriptor,
-	wireModelId: string,
-): EngineModel | null {
-	const runtime = providers.getRuntime(target.runtime);
-	if (!runtime) return null;
+	if (!targetId || !wireModelId) return null;
+	await prepareBackgroundModelMetadata(providers, targetId);
+	const status = providers.list().find((entry) => entry.target.id === targetId);
+	if (!status?.available) {
+		throw new Error("context.compaction.model target is unavailable; check the selected target with clio-coder targets");
+	}
+	const resolved = resolveRuntimeTarget(providers, {
+		targetId,
+		wireModelId,
+		use: "orchestrator",
+		requestedThinkingLevel: "off",
+		requireTools: false,
+		requireStreaming: true,
+		requireOutputBudget: true,
+	});
+	if (!resolved.ok) {
+		// Configured identifiers and provider diagnostics can contain terminal controls.
+		// Keep the failure actionable without echoing arbitrary provider/config text.
+		throw new Error(
+			"context.compaction.model cannot run as a summarizer; select an available HTTP chat model with an output budget",
+		);
+	}
+	const route = resolved.target;
 	let model: EngineModel;
 	try {
-		const kbHit = providers.knowledgeBase?.lookup(wireModelId) ?? null;
-		model = runtime.synthesizeModel(target, wireModelId, kbHit);
+		model = route.runtime.synthesizeModel(
+			route.target,
+			route.wireModelId,
+			providers.knowledgeBase?.lookup(route.wireModelId) ?? null,
+		);
 	} catch {
-		return null;
+		throw new Error("context.compaction.model could not be prepared; check the selected target/model configuration");
 	}
-	try {
-		const status = providers.list().find((entry) => entry.target.id === target.id);
-		if (status) {
-			const caps = resolveModelCapabilities(status, wireModelId, providers.knowledgeBase, {
-				detectedReasoning: providers.getDetectedReasoning(target.id, wireModelId),
-			});
-			applyModelCapabilityPatch(model, caps);
+	const refined = refineRuntimeTargetWithModelHints(route, model, providers.knowledgeBase);
+	if (refined.capabilityDecisions.maxTokens <= 0) {
+		throw new Error("context.compaction.model has no output budget; select a model with a known positive output limit");
+	}
+	applyModelCapabilityPatch(model, refined.capabilities);
+	let apiKey: string | undefined = LOCAL_API_KEY_FALLBACK;
+	if (targetRequiresAuth(route.target, route.runtime)) {
+		const auth = await providers.auth.resolveForTarget(route.target, route.runtime);
+		if (!auth.available || !auth.apiKey) {
+			throw new Error(
+				"context.compaction.model authentication is unavailable; authenticate the selected target with clio-coder auth",
+			);
 		}
-	} catch {
-		// Older test doubles and degraded provider bundles may not expose live
-		// status. The synthesized model still carries runtime and catalog caps.
+		apiKey = auth.apiKey;
 	}
-	return model;
-}
-
-async function resolveCompactionModel(
-	settings: ClioSettings,
-	providers: ProvidersContract,
-): Promise<CompactionResolution | null> {
-	const targetId = settings.chat?.target ?? null;
-	const wireModelId = settings.chat?.model ?? null;
-	if (!targetId || !wireModelId) return null;
-	const target = resolveTarget(providers, targetId);
-	if (!target) return null;
-	const model = synthesizeOrchestratorModel(providers, target, wireModelId);
-	if (!model) return null;
-	const apiKey = await resolveApiKeyForTarget(target, providers);
-	const resolution: CompactionResolution = { model, targetId, endpointKey: canonicalEndpointKey(target) };
-	if (apiKey !== undefined) resolution.apiKey = apiKey;
-	return resolution;
+	return {
+		model,
+		targetId: route.targetId,
+		wireModelId: route.wireModelId,
+		endpointKey: canonicalEndpointKey(route.target),
+		apiKey,
+		...(route.target.auth?.headers ? { headers: route.target.auth.headers } : {}),
+	};
 }
 
 function readSessionEntriesForCompact(sessionId: string): SessionEntry[] {
@@ -644,11 +793,19 @@ async function runCompactionFlow(
 	providers: ProvidersContract,
 	instructions?: string,
 	trigger?: CompactionTrigger,
+	observability?: BackgroundMemoryUsageSink,
 ): Promise<CompactResult | null> {
 	const meta = session.current();
 	if (!meta) {
 		throw new Error("no current session to compact; start one with /new or /resume first");
 	}
+	const stateDir = clioStateDir();
+	const activeLeafTurnId = session.tree(meta.id).leafId ?? undefined;
+	const isOriginCurrent = () =>
+		session.current()?.id === meta.id && (session.tree(meta.id).leafId ?? undefined) === activeLeafTurnId;
+	const entries = filterEntriesToActivePath(readSessionEntriesForCompact(meta.id), activeLeafTurnId);
+	if (entries.length === 0) return null;
+	const systemPrompt = await readCompactionSystemPrompt(settings.context.compaction.systemPrompt, meta.cwd);
 	const resolved = await resolveCompactionModel(settings, providers);
 	if (!resolved) {
 		throw new Error("no model configured; set chat.target + chat.model");
@@ -661,11 +818,22 @@ async function runCompactionFlow(
 	// read the full file too (last taskLedger entry in file order, with no
 	// branch filter at all), which was a second, independent instance of this
 	// same bug. See the taskBoard wiring below for the fix.
-	const activeLeafTurnId = session.tree(meta.id).leafId ?? undefined;
-	const entries = filterEntriesToActivePath(readSessionEntriesForCompact(meta.id), activeLeafTurnId);
-	if (entries.length === 0) return null;
+	if (!isOriginCurrent()) throw new Error("compaction session or branch changed before summarization");
+	const calls: CompactionCallObservation[] = [];
+	const preserveCalls = () =>
+		recordFailedCompactionCalls(
+			{
+				stateDir,
+				sessionId: meta.id,
+				repoIdentity: meta.cwdHash || cwdHash(meta.cwd),
+				target: resolved.targetId,
+				model: resolved.wireModelId,
+			},
+			calls,
+			isOriginCurrent() ? observability : undefined,
+		);
 
-	// A compaction summary is a full streamed request against the chat target.
+	// A compaction summary is a full streamed request against its resolved target.
 	// Hold the same canonical endpoint slot as an ordinary turn, /btw round, or
 	// pre-warm so dispatch admission and background memory see its real usage.
 	const releaseEndpointSlot = resolved.endpointKey === null ? () => {} : registerForegroundStream(resolved.endpointKey);
@@ -673,17 +841,39 @@ async function runCompactionFlow(
 	try {
 		result = await compact({
 			entries,
+			onCall: (call) => calls.push(call),
 			model: resolved.model,
+			...(systemPrompt !== undefined ? { systemPrompt } : {}),
+			...(resolved.headers !== undefined ? { headers: resolved.headers } : {}),
 			...(resolved.apiKey !== undefined ? { apiKey: resolved.apiKey } : {}),
 			...(instructions !== undefined ? { instructions } : {}),
 		});
+	} catch (error) {
+		try {
+			preserveCalls();
+		} catch (recordingError) {
+			throw new AggregateError([error, recordingError], "compaction failed and its usage could not be fully recorded");
+		}
+		throw error;
 	} finally {
 		releaseEndpointSlot();
 	}
-	if (result.messagesSummarized === 0 || result.summary.length === 0) return null;
+	if (!isOriginCurrent()) {
+		preserveCalls();
+		throw new Error("compaction session or branch changed; summary discarded and originating usage retained");
+	}
+	if (result.messagesSummarized === 0 || result.summary.length === 0) {
+		preserveCalls();
+		if (calls.length > 0) throw new Error("compaction returned no summary; reported usage retained without a checkpoint");
+		return null;
+	}
+	if (result.usage) {
+		result.usage = { ...result.usage, targetId: resolved.targetId, modelId: resolved.wireModelId };
+	}
 
-	const entry: Omit<CompactionSummaryEntry, "turnId" | "timestamp"> = {
+	const entry: Omit<CompactionSummaryEntry, "timestamp"> = {
 		kind: "compactionSummary",
+		turnId: randomUUID(),
 		parentTurnId: result.firstKeptTurnId ?? null,
 		summary: result.summary,
 		tokensBefore: result.tokensBefore,
@@ -697,22 +887,61 @@ async function runCompactionFlow(
 		...(result.usage !== undefined ? { usage: result.usage } : {}),
 	};
 	if (trigger !== undefined) entry.trigger = trigger;
-	session.appendEntry(entry);
+	try {
+		session.appendEntry(entry);
+	} catch (error) {
+		// A write may have reached the ledger before its caller threw. Reconcile
+		// this exact checkpoint identity before choosing a second accounting store.
+		try {
+			if (!compactionCheckpointWasWritten(sessionPaths(meta).current, entry.turnId)) preserveCalls();
+		} catch (recordingError) {
+			throw new AggregateError(
+				[error, recordingError],
+				"compaction checkpoint persistence is unresolved; usage was not duplicated",
+			);
+		}
+		throw error;
+	}
 	return result;
 }
 
+function compactionCheckpointWasWritten(path: string, turnId: string): boolean {
+	let malformed = false;
+	for (const line of readFileSync(path, "utf8").split("\n")) {
+		if (line.trim().length === 0) continue;
+		try {
+			const entry: unknown = JSON.parse(line);
+			if (
+				entry &&
+				typeof entry === "object" &&
+				"turnId" in entry &&
+				entry.turnId === turnId &&
+				"kind" in entry &&
+				entry.kind === "compactionSummary"
+			)
+				return true;
+		} catch {
+			malformed = true;
+		}
+	}
+	if (malformed) throw new Error("cannot prove checkpoint absence in a malformed session ledger");
+	return false;
+}
+
 /**
  * Compose the production chat-loop compaction callback. Errors intentionally
  * propagate: the chat loop owns activity failure reporting and distinguishes
  * a thrown read/model/persistence failure from the legitimate null no-op that
  * `runCompactionFlow` returns for an empty session or an unavailable cut.
  */
-function createProductionAutoCompact(
+export function createProductionAutoCompact(
 	session: SessionContract,
 	getSettings: () => ClioSettings,
 	providers: ProvidersContract,
+	observability?: BackgroundMemoryUsageSink,
 ): (instructions?: string, trigger?: CompactionTrigger) => Promise<CompactResult | null> {
-	return (instructions, trigger) => runCompactionFlow(session, getSettings(), providers, instructions, trigger);
+	return (instructions, trigger) =>
+		runCompactionFlow(session, getSettings(), providers, instructions, trigger, observability);
 }
 
 function estimateTokensFromSummary(summary: string): number {
@@ -1167,37 +1396,12 @@ export async function bootOrchestrator(options: BootOptions = {}): Promise<BootR
 	middleware.registerHook(
 		createSkillActivationObserver((activation) => appendSkillActivationRegistryEvent(session, activation)),
 	);
-	// First-turn skills reminder: user-message-visible text is the one channel
-	// the battery-tested local models act on; suggestion protocol only, the
-	// operator load gate is untouched.
-	if (resources) {
-		middleware.registerHook(
-			createSkillsReminderRegistration({
-				countModelVisibleSkills: () => modelVisibleSkills(resources.skills(process.cwd()).items).length,
-				// Same lookup context(scope="skills") lists under its Marketplace
-				// heading, minus what is already installed, so the count the
-				// reminder quotes is the count the listing will show.
-				countInstallableSkills: () => {
-					const installed = new Set(resources.skills(process.cwd()).items.map((skill) => skill.name));
-					return discoverMarketplaceSkills({ cwd: process.cwd() }).skills.filter((skill) => !installed.has(skill.name))
-						.length;
-				},
-			}),
-		);
-	}
 	// Task-board reminder: same user-message-visible channel, fired once per
 	// session when a request literally enumerates three or more steps. The
 	// static routing line and tasks hint ask for the same board; battery-tested
 	// local models only comply when the instruction rides the user message.
 	middleware.registerHook(createTaskBoardReminderRegistration());
 	const taskMemoryBank = new TaskMemoryBank();
-	let taskMemorySessionId = session?.current()?.id ?? null;
-	const ensureTaskMemorySession = (): void => {
-		const currentSessionId = session?.current()?.id ?? null;
-		if (currentSessionId === taskMemorySessionId) return;
-		taskMemoryBank.clear();
-		taskMemorySessionId = currentSessionId;
-	};
 	const memorySettings = (config?.get() ?? readSettings()).context.memory;
 	// Bound late: the registration is built here, but the buffer a deferred
 	// reminder lands in belongs to the chat loop that has not been composed yet.
@@ -1212,7 +1416,6 @@ export async function bootOrchestrator(options: BootOptions = {}): Promise<BootR
 	const memoryTrace = memoryTracePath === null ? null : createTaskMemoryTrace(memoryTracePath);
 	// The route the last resolved memory client would call, kept so the endpoint
 	// check and the cost row read the same resolution the step itself used.
-	let backgroundMemoryRoute: BackgroundMemoryRoute | null = null;
 	/**
 	 * Account for one background memory step exactly as a `/btw` side question is
 	 * accounted for: the in-process cost tracker under its own label so `/cost`
@@ -1221,23 +1424,16 @@ export async function bootOrchestrator(options: BootOptions = {}): Promise<BootR
 	 *
 	 * Accounting only. The client wrapper (`announceMemoryStepEndpoint`) holds the
 	 * endpoint slot and publishes the disturbance; it sees the timed-out and
-	 * thrown steps this sink never hears about because their usage stays null.
+	 * thrown steps that report no usage. Late reported usage still reaches this sink.
 	 */
-	const recordBackgroundMemoryUsage = (usage: TaskMemoryStepUsage): void => {
+	const captureBackgroundMemoryUsage = () => {
 		const meta = session?.current() ?? null;
-		try {
-			recordBackgroundMemoryStep({
-				usage,
-				stateDir: clioStateDir(),
-				sessionId: meta?.id ?? null,
-				// The identity the session ledger is filed under, so `usage report
-				// --repo` selects these rows with the same hash it selects ledgers with.
-				repoIdentity: meta ? meta.cwdHash || cwdHash(meta.cwd || process.cwd()) : null,
-				...(observability === undefined ? {} : { observability }),
-			});
-		} catch {
-			// Accounting is bookkeeping. Its failure never reaches the memory step.
-		}
+		return captureTaskMemoryUsage({
+			stateDir: clioStateDir(),
+			sessionId: meta?.id ?? null,
+			repoIdentity: meta ? meta.cwdHash || cwdHash(meta.cwd || process.cwd()) : null,
+			...(observability === undefined ? {} : { observability }),
+		});
 	};
 	const proposeInjectedMemoryEntries = (entries: ReadonlyArray<TaskMemoryEntry>): void => {
 		const meta = session?.current() ?? null;
@@ -1266,7 +1462,6 @@ export async function bootOrchestrator(options: BootOptions = {}): Promise<BootR
 		deliversDeferredReminders: options.headless === undefined,
 		onDeferredReminder: (message) => deferredMemoryReminderSink?.(message),
 		getSettings: () => {
-			ensureTaskMemorySession();
 			const memory = effectiveSettingsForDispatch?.().context.memory ?? memorySettings;
 			return {
 				enabled: memory.enabled,
@@ -1276,34 +1471,17 @@ export async function bootOrchestrator(options: BootOptions = {}): Promise<BootR
 				timeoutMs: memory.timeoutMs,
 			};
 		},
-		getModelClient: () => {
-			const settings = effectiveSettingsForDispatch?.();
-			backgroundMemoryRoute =
-				settings === undefined
-					? null
-					: createBackgroundMemoryModelClient(providers, settings, settings.context.memory.timeoutMs, bus);
-			return backgroundMemoryRoute?.client ?? null;
-		},
-		getModelMaxTokens: (configuredMaxTokens) =>
-			backgroundMemoryRoute?.modelMaxTokens(configuredMaxTokens) ?? configuredMaxTokens,
-		// A single-slot local server serves one request at a time, so a memory step
-		// started while the operator's turn is streaming either waits behind it or
-		// makes the server swap the resident model out to answer. The chat loop
-		// registers its stream on the endpoint it streams against, so the two are
-		// compared by the same canonical key dispatch admission counts slots on.
-		backgroundEndpointBusy: () => {
-			const endpointKey = backgroundMemoryRoute?.endpointKey ?? null;
-			if (endpointKey === null) return false;
-			// This runs before the client wrapper registers the admitted step, so the
-			// count contains other requests only and the step cannot refuse itself.
-			return (foregroundStreamUsage()[endpointKey] ?? 0) > 0;
-		},
-		onStepUsage: (usage) => recordBackgroundMemoryUsage(usage),
+		...createBackgroundMemoryRouting(providers, () => effectiveSettingsForDispatch?.(), bus),
+		captureStepUsage: captureBackgroundMemoryUsage,
 		onInjectedEntries: (entries) => proposeInjectedMemoryEntries(entries),
 	});
 	middleware.registerHook(memoryIntervention);
 	const unsubscribeMemoryLoop = bus.on(BusChannels.LoopBlocked, () => memoryIntervention.signalLoop());
-	termination.onDrain(() => unsubscribeMemoryLoop());
+	const disposeMemoryLifecycle = bindTaskMemoryLifecycle(bus, memoryIntervention);
+	termination.onDrain(() => {
+		unsubscribeMemoryLoop();
+		disposeMemoryLifecycle();
+	});
 	if (contextDomain) {
 		middleware.registerHook(createFileMutationObserver(({ paths }) => contextDomain.noteFileChanges(paths)));
 	}
@@ -1346,17 +1524,34 @@ export async function bootOrchestrator(options: BootOptions = {}): Promise<BootR
 		(config?.get() ?? readSettings()).safety.autonomy ??
 		"auto-edit";
 	const resolveEffectiveAutonomy = (): AutonomyLevel => activeAcpSessionAutonomy ?? resolveBaselineAutonomy();
+	// First-turn skills reminder: user-message-visible text is the one channel
+	// the battery-tested local models act on. Which protocol it teaches follows
+	// the effective autonomy level, resolved one line up: suggest-and-wait at
+	// read-only and suggest, load-it-yourself at auto-edit and full-auto.
+	if (resources) {
+		middleware.registerHook(
+			createSkillsReminderRegistration({
+				countModelVisibleSkills: () => modelVisibleSkills(resources.skills(process.cwd()).items).length,
+				// Same lookup context(scope="skills") lists under its Marketplace
+				// heading, minus what is already installed, so the count the
+				// reminder quotes is the count the listing will show.
+				countInstallableSkills: () => {
+					const installed = new Set(resources.skills(process.cwd()).items.map((skill) => skill.name));
+					return discoverMarketplaceSkills({ cwd: process.cwd() }).skills.filter((skill) => !installed.has(skill.name))
+						.length;
+				},
+				modelMayActivateSkills: () => modelMayActivateSkills(resolveEffectiveAutonomy()),
+			}),
+		);
+	}
 	// Marketplace self-promotion: coordinator-only by this wiring (never a
-	// dispatch worker), local matcher, consented installs and full-auto
-	// autonomous installs both pass the own-marketplace source gate inside the
-	// registration. Registered here because it needs the effective-autonomy
-	// resolution defined one line up.
+	// dispatch worker), local matcher and operator-consented installs at every autonomy level. The
+	// registration also checks the own-marketplace source gate before installing.
 	if (resources) {
 		middleware.registerHook(
 			createMarketplaceOfferRegistration({
 				listInstalledSkillNames: () => resources.skills(process.cwd()).items.map((skill) => skill.name),
 				listMarketplaceEntries: () => discoverMarketplaceSkills({ cwd: process.cwd() }).skills,
-				getAutonomy: resolveEffectiveAutonomy,
 				installEntry: (entry, scope) => {
 					const installed = installSkill({ source: entry.sourceUrl, scope, name: entry.name, cwd: process.cwd() });
 					return { path: installed.path, sourceUrl: installed.sourceUrl, installedHash: installed.installedHash };
@@ -1593,7 +1788,6 @@ export async function bootOrchestrator(options: BootOptions = {}): Promise<BootR
 		return taskMemoryHandoffSeedOffer(process.cwd(), getCurrentSettings().context.memory.enabled);
 	};
 	const seedCurrentTaskMemoryFromHandoff = () => {
-		ensureTaskMemorySession();
 		return seedTaskMemoryFromNewestHandoff(taskMemoryBank, process.cwd(), getCurrentSettings().context.memory.enabled);
 	};
 	if (resumedSessionAtBoot) {
@@ -1835,6 +2029,7 @@ export async function bootOrchestrator(options: BootOptions = {}): Promise<BootR
 			return key === null ? null : registerForegroundStream(key);
 		},
 		getSettings: getCurrentSettings,
+		getAutonomy: resolveEffectiveAutonomy,
 		providers,
 		middleware,
 		middlewareToolChoice,
@@ -1863,7 +2058,6 @@ export async function bootOrchestrator(options: BootOptions = {}): Promise<BootR
 			}
 		},
 		getTaskMemoryHandoffSource: () => {
-			ensureTaskMemorySession();
 			const meta = session?.current();
 			if (!meta) throw new Error("task memory handoff requires an active session");
 			const settings = getCurrentSettings();
@@ -1888,7 +2082,7 @@ export async function bootOrchestrator(options: BootOptions = {}): Promise<BootR
 		...(session
 			? {
 					readSessionEntries: readCurrentSessionEntries,
-					autoCompact: createProductionAutoCompact(session, getCurrentSettings, providers),
+					autoCompact: createProductionAutoCompact(session, getCurrentSettings, providers, observability),
 				}
 			: {}),
 		toolRegistry,
@@ -2165,7 +2359,6 @@ export async function bootOrchestrator(options: BootOptions = {}): Promise<BootR
 		supersedeDecision: (interviewId, key, correction) => decisionBoard.supersede(interviewId, key, correction),
 		userTasks,
 		getTaskMemoryStatus: () => {
-			ensureTaskMemorySession();
 			const settings = getCurrentSettings();
 			const bank = taskMemoryBank.snapshot();
 			return {
@@ -2319,10 +2512,8 @@ export async function bootOrchestrator(options: BootOptions = {}): Promise<BootR
 			? {
 					onResumeSession: (sessionId) => {
 						try {
-							const previousSessionId = session.current()?.id ?? null;
 							session.resume(sessionId);
 							taskBoard.snapshot();
-							if (previousSessionId !== sessionId) ensureTaskMemorySession();
 						} catch (err) {
 							process.stderr.write(
 								`[/resume] failed to resume ${sessionId}: ${err instanceof Error ? err.message : String(err)}\n`,
@@ -2336,13 +2527,11 @@ export async function bootOrchestrator(options: BootOptions = {}): Promise<BootR
 						if (settings.chat.model) input.model = settings.chat.model;
 						session.create(input);
 						taskBoard.snapshot();
-						ensureTaskMemorySession();
 					},
 					onForkSession: (parentTurnId) => {
 						try {
 							session.fork(parentTurnId);
 							taskBoard.snapshot();
-							ensureTaskMemorySession();
 						} catch (err) {
 							process.stderr.write(
 								`[/fork] failed at turn ${parentTurnId}: ${err instanceof Error ? err.message : String(err)}\n`,