@iowarp/clio-coder 0.3.8 → 0.3.9

package/src/engine/provider-payload.ts

@@ -1,4 +1,4 @@
-import { RESPONSE_SCHEMA_RUNTIME_ID } from "../core/response-schema.js";
+import { RESPONSE_SCHEMA_RUNTIME_ID, type ResponseSchemaDialect } from "../core/response-schema.js";
 import type { ThinkingLevel } from "../domains/providers/index.js";
 import type { EngineModel } from "./types.js";
 
@@ -217,6 +217,34 @@ function patchLlamaCppResponseSchemaPayload(
 	};
 }
 
+/**
+ * Apply a JSON-schema response constraint in the dialect the runtime takes.
+ *
+ * Unlike the worker patcher above, this one is for a seam that treats native
+ * enforcement as an optimization: the caller looks the dialect up first and
+ * simply does not call this when there is none, so an unconstrained request
+ * still goes out and the prompt-level instruction carries it (issue #223).
+ * Returns undefined when the payload is not an object, which leaves it alone.
+ */
+export function patchResponseSchemaPayloadForDialect(
+	payload: unknown,
+	dialect: ResponseSchemaDialect,
+	responseSchema: Record<string, unknown>,
+	schemaName: string,
+): unknown | undefined {
+	if (!isRecord(payload)) return undefined;
+	if (dialect === "llamacpp-json-object") {
+		return { ...payload, response_format: { type: "json_object", schema: responseSchema } };
+	}
+	return {
+		...payload,
+		response_format: {
+			type: "json_schema",
+			json_schema: { name: schemaName, strict: true, schema: responseSchema },
+		},
+	};
+}
+
 export interface WorkerPayloadPatchOptions {
 	runtimeId: string;
 	thinkingLevel?: ThinkingLevel;

package/src/entry/orchestrator.ts

@@ -8,6 +8,7 @@ import { installBusTracer } from "../core/bus-trace.js";
 import { type ClioSettings, readSettings, type SettingsMutator, updateSettings } from "../core/config.js";
 import { DEFAULT_DELEGATION_PERMISSION_TIMEOUT_MS } from "../core/defaults.js";
 import { loadDomains } from "../core/domain-loader.js";
+import type { SafeEventBus } from "../core/event-bus.js";
 import { expandInlineFileReferencesAsync } from "../core/file-references.js";
 import { setGitCommitAttributionEnabled } from "../core/git-commit-attribution.js";
 import { configureGuardrails } from "../core/guardrails.js";
@@ -63,9 +64,13 @@ import {
 	createTaskMemoryTelemetrySink,
 	createTaskMemoryTrace,
 	loadMemoryRecordsSync,
+	proposeInjectedTaskMemory,
+	readTaskMemorySpendSummary,
 	renderTaskMemoryHandoffSource,
 	seedTaskMemoryFromNewestHandoff,
+	type TaskMemoryEntry,
 	type TaskMemoryModelClient,
+	type TaskMemoryStepUsage,
 	taskMemoryBankSize,
 	taskMemoryHandoffSeedOffer,
 	taskMemoryTracePath,
@@ -88,21 +93,26 @@ import {
 	writeMiddlewareDiagnosticToStderr,
 } from "../domains/middleware/index.js";
 import { createMemoryInterventionRegistration } from "../domains/middleware/memory-intervention.js";
+import { announceMemoryStepEndpoint } from "../domains/middleware/memory-step-endpoint.js";
 import { createTaskBoardReminderRegistration } from "../domains/middleware/task-board-reminder.js";
 import { createTaskNudgeRegistration } from "../domains/middleware/task-nudge.js";
 import { createWatchdogRegistration } from "../domains/middleware/watchdog.js";
 import type { ObservabilityContract } from "../domains/observability/index.js";
-import { ObservabilityDomainModule } from "../domains/observability/index.js";
+import { ObservabilityDomainModule, recordBackgroundMemoryStep } from "../domains/observability/index.js";
 import type { PromptsContract } from "../domains/prompts/contract.js";
 import { createPromptsDomainModule } from "../domains/prompts/index.js";
 import type { ProvidersContract, TargetDescriptor, ThinkingLevel } from "../domains/providers/index.js";
 import {
 	AGENT_ROLE_TOOLS_REQUIRED_REASON,
 	applyModelCapabilityPatch,
+	canonicalEndpointKey,
 	firstRuntimeResolutionError,
+	foregroundStreamUsage,
 	isOrchestratorEligibleRuntime,
+	normalizeCostProvenance,
 	ProvidersDomainModule,
 	refineRuntimeTargetWithModelHints,
+	registerForegroundStream,
 	resolveModelCapabilities,
 	resolveModelRuntimeCapabilitiesForProviders,
 	resolveRuntimeTarget,
@@ -159,7 +169,7 @@ import {
 	INTERACTIVE_LOOP_BLOCK_BUDGET,
 	readOrchTurnToolCallBudget,
 } from "../engine/loop-guard.js";
-import { openSession, readSessionTailTurns, sessionCurrentPath, sessionPaths } from "../engine/session.js";
+import { cwdHash, openSession, readSessionTailTurns, sessionCurrentPath, sessionPaths } from "../engine/session.js";
 import type { EngineModel } from "../engine/types.js";
 import { createChatLoop } from "../interactive/chat-loop.js";
 import { type RunIo, startInteractive } from "../interactive/index.js";
@@ -268,6 +278,7 @@ function applyHeadlessSettingsOverlay(
 interface CompactionResolution {
 	model: EngineModel;
 	targetId: string;
+	endpointKey: string | null;
 	apiKey?: string;
 }
 
@@ -310,11 +321,26 @@ export async function resolveApiKeyForTarget(
 	return resolved.apiKey;
 }
 
+/**
+ * The background memory role, resolved to the endpoint it would call.
+ *
+ * The endpoint key is what lets the middleware decline a step that would land
+ * on the same inference scheduler the chat target is streaming against (#229,
+ * #250), and it is the same key dispatch admission counts slots on.
+ */
+interface BackgroundMemoryRoute {
+	client: TaskMemoryModelClient;
+	targetId: string;
+	wireModelId: string;
+	endpointKey: string | null;
+}
+
 function createBackgroundMemoryModelClient(
 	providers: ProvidersContract,
 	settings: Readonly<ClioSettings>,
 	timeoutMs: number,
-): TaskMemoryModelClient | null {
+	bus: Pick<SafeEventBus, "emit"> | null,
+): BackgroundMemoryRoute | null {
 	const targetId = settings.background.target?.trim();
 	const wireModelId = settings.background.model?.trim();
 	if (!targetId || !wireModelId) return null;
@@ -343,24 +369,57 @@ function createBackgroundMemoryModelClient(
 	const model = resolved.target.runtime.synthesizeModel(resolved.target.target, resolved.target.wireModelId, kbHit);
 	const refined = refineRuntimeTargetWithModelHints(resolved.target, model, providers.knowledgeBase);
 	applyModelCapabilityPatch(model, refined.capabilities);
+	const costProvenance = normalizeCostProvenance(refined.costProvenance);
+	const endpointKey = canonicalEndpointKey(refined.target);
 	return {
-		async complete(request) {
-			const apiKey = targetRequiresAuth(refined.target, refined.runtime)
-				? (await providers.auth.resolveForTarget(refined.target, refined.runtime, { signal: request.signal })).apiKey
-				: LOCAL_API_KEY_FALLBACK;
-			return completeEngineText({
-				model,
-				systemPrompt: request.systemPrompt,
-				userPrompt: request.userPrompt,
-				maxTokens: request.maxTokens,
-				// Always off, never the operator's chat thinking level. A model that
-				// reasons anyway still works: `completeEngineText` keeps only text
-				// blocks, and the memory output budget leaves room for the preamble.
-				thinkingLevel: "off",
-				signal: request.signal,
-				timeoutMs,
-				...(apiKey === undefined ? {} : { apiKey }),
-			});
+		targetId,
+		wireModelId: refined.wireModelId,
+		endpointKey,
+		client: {
+			// Wrapped at the layer that knows a request left the process: the step
+			// holds endpoint capacity while it is out and publishes the cache
+			// disturbance even when a timeout means the usage sink never sees it.
+			complete: announceMemoryStepEndpoint({ bus, endpointKey, targetId }, async (request) => {
+				const apiKey = targetRequiresAuth(refined.target, refined.runtime)
+					? (await providers.auth.resolveForTarget(refined.target, refined.runtime, { signal: request.signal })).apiKey
+					: LOCAL_API_KEY_FALLBACK;
+				const startedAt = Date.now();
+				const completion = await completeEngineText({
+					model,
+					systemPrompt: request.systemPrompt,
+					userPrompt: request.userPrompt,
+					maxTokens: request.maxTokens,
+					// Always off, never the operator's chat thinking level. A model that
+					// reasons anyway still works: `completeEngineText` keeps only text
+					// blocks, and the memory output budget leaves room for the preamble.
+					thinkingLevel: "off",
+					signal: request.signal,
+					timeoutMs,
+					...(apiKey === undefined ? {} : { apiKey }),
+				});
+				// The step is billed here whatever the policy later decides about the
+				// answer. A model that read a trajectory and chose silence spent the
+				// same prefill as one that produced a reminder.
+				return {
+					text: completion.text,
+					inputTokens: completion.inputTokens,
+					outputTokens: completion.outputTokens,
+					usage: {
+						targetId,
+						attributedModelId: refined.wireModelId,
+						input: completion.usage.input,
+						output: completion.usage.output,
+						cacheRead: completion.usage.cacheRead,
+						cacheWrite: completion.usage.cacheWrite,
+						reasoning: completion.usage.reasoning,
+						totalTokens: completion.usage.totalTokens,
+						costUsd: completion.usage.costUsd,
+						costProvenance,
+						durationMs: Date.now() - startedAt,
+						backend: completion.backend,
+					},
+				};
+			}),
 		},
 	};
 }
@@ -463,7 +522,7 @@ async function resolveCompactionModel(
 	const model = synthesizeOrchestratorModel(providers, target, wireModelId);
 	if (!model) return null;
 	const apiKey = await resolveApiKeyForTarget(target, providers);
-	const resolution: CompactionResolution = { model, targetId };
+	const resolution: CompactionResolution = { model, targetId, endpointKey: canonicalEndpointKey(target) };
 	if (apiKey !== undefined) resolution.apiKey = apiKey;
 	return resolution;
 }
@@ -591,12 +650,21 @@ async function runCompactionFlow(
 	const entries = filterEntriesToActivePath(readSessionEntriesForCompact(meta.id), activeLeafTurnId);
 	if (entries.length === 0) return null;
 
-	const result = await compact({
-		entries,
-		model: resolved.model,
-		...(resolved.apiKey !== undefined ? { apiKey: resolved.apiKey } : {}),
-		...(instructions !== undefined ? { instructions } : {}),
-	});
+	// A compaction summary is a full streamed request against the chat target.
+	// Hold the same canonical endpoint slot as an ordinary turn, /btw round, or
+	// pre-warm so dispatch admission and background memory see its real usage.
+	const releaseEndpointSlot = resolved.endpointKey === null ? () => {} : registerForegroundStream(resolved.endpointKey);
+	let result: CompactResult;
+	try {
+		result = await compact({
+			entries,
+			model: resolved.model,
+			...(resolved.apiKey !== undefined ? { apiKey: resolved.apiKey } : {}),
+			...(instructions !== undefined ? { instructions } : {}),
+		});
+	} finally {
+		releaseEndpointSlot();
+	}
 	if (result.messagesSummarized === 0 || result.summary.length === 0) return null;
 
 	const entry: Omit<CompactionSummaryEntry, "turnId" | "timestamp"> = {
@@ -1079,6 +1147,53 @@ export async function bootOrchestrator(options: BootOptions = {}): Promise<BootR
 	// telemetry row says which silence happened; this says what the model wrote.
 	const memoryTracePath = taskMemoryTracePath();
 	const memoryTrace = memoryTracePath === null ? null : createTaskMemoryTrace(memoryTracePath);
+	// The route the last resolved memory client would call, kept so the endpoint
+	// check and the cost row read the same resolution the step itself used.
+	let backgroundMemoryRoute: BackgroundMemoryRoute | null = null;
+	/**
+	 * Account for one background memory step exactly as a `/btw` side question is
+	 * accounted for: the in-process cost tracker under its own label so `/cost`
+	 * shows it while the session lives, and one durable out-of-turn row so
+	 * `clio-coder usage report` can still see it afterwards.
+	 *
+	 * Accounting only. The client wrapper (`announceMemoryStepEndpoint`) holds the
+	 * endpoint slot and publishes the disturbance; it sees the timed-out and
+	 * thrown steps this sink never hears about because their usage stays null.
+	 */
+	const recordBackgroundMemoryUsage = (usage: TaskMemoryStepUsage): void => {
+		const meta = session?.current() ?? null;
+		try {
+			recordBackgroundMemoryStep({
+				usage,
+				stateDir: clioStateDir(),
+				sessionId: meta?.id ?? null,
+				// The identity the session ledger is filed under, so `usage report
+				// --repo` selects these rows with the same hash it selects ledgers with.
+				repoIdentity: meta ? meta.cwdHash || cwdHash(meta.cwd || process.cwd()) : null,
+				...(observability === undefined ? {} : { observability }),
+			});
+		} catch {
+			// Accounting is bookkeeping. Its failure never reaches the memory step.
+		}
+	};
+	const proposeInjectedMemoryEntries = (entries: ReadonlyArray<TaskMemoryEntry>): void => {
+		const meta = session?.current() ?? null;
+		void proposeInjectedTaskMemory(clioDataDir(), {
+			sessionId: meta?.id ?? null,
+			cwd: meta?.cwd || process.cwd(),
+			entries,
+		})
+			.then((result) => {
+				for (const error of result.errors) {
+					process.stderr.write(`[clio:memory] proposed record not written for ${error}\n`);
+				}
+			})
+			.catch((error: unknown) => {
+				process.stderr.write(
+					`[clio:memory] proposed records not written: ${error instanceof Error ? error.message : String(error)}\n`,
+				);
+			});
+	};
 	const memoryIntervention = createMemoryInterventionRegistration({
 		bank: taskMemoryBank,
 		telemetry: createTaskMemoryTelemetrySink(),
@@ -1093,10 +1208,26 @@ export async function bootOrchestrator(options: BootOptions = {}): Promise<BootR
 		},
 		getModelClient: () => {
 			const settings = effectiveSettingsForDispatch?.();
-			return settings === undefined
-				? null
-				: createBackgroundMemoryModelClient(providers, settings, settings.memory.intervention.timeoutMs);
+			backgroundMemoryRoute =
+				settings === undefined
+					? null
+					: createBackgroundMemoryModelClient(providers, settings, settings.memory.intervention.timeoutMs, bus);
+			return backgroundMemoryRoute?.client ?? null;
+		},
+		// A single-slot local server serves one request at a time, so a memory step
+		// started while the operator's turn is streaming either waits behind it or
+		// makes the server swap the resident model out to answer. The chat loop
+		// registers its stream on the endpoint it streams against, so the two are
+		// compared by the same canonical key dispatch admission counts slots on.
+		backgroundEndpointBusy: () => {
+			const endpointKey = backgroundMemoryRoute?.endpointKey ?? null;
+			if (endpointKey === null) return false;
+			// This runs before the client wrapper registers the admitted step, so the
+			// count contains other requests only and the step cannot refuse itself.
+			return (foregroundStreamUsage()[endpointKey] ?? 0) > 0;
 		},
+		onStepUsage: (usage) => recordBackgroundMemoryUsage(usage),
+		onInjectedEntries: (entries) => proposeInjectedMemoryEntries(entries),
 	});
 	middleware.registerHook(memoryIntervention);
 	const unsubscribeMemoryLoop = bus.on(BusChannels.LoopBlocked, () => memoryIntervention.signalLoop());
@@ -1569,6 +1700,12 @@ export async function bootOrchestrator(options: BootOptions = {}): Promise<BootR
 	}
 
 	const chat = createChatLoop({
+		// The pre-warm holds one slot on its endpoint while it runs, so dispatch
+		// admission (#250) sees it exactly as it sees the orchestrator's own turn.
+		registerPrewarmEndpointSlot: (runtime) => {
+			const key = canonicalEndpointKey(runtime.runtimeResolution.target);
+			return key === null ? null : registerForegroundStream(key);
+		},
 		getSettings: getCurrentSettings,
 		providers,
 		middleware,
@@ -1628,6 +1765,11 @@ export async function bootOrchestrator(options: BootOptions = {}): Promise<BootR
 			: {}),
 		toolRegistry,
 		hasAttachedDispatch: () => dispatchBackground.size() > 0,
+		// The pre-warm buys latency for a person about to type the next turn. A
+		// headless `run` submits its one prompt immediately and an unattended boot
+		// never submits at all, so neither has latency to buy; the ACP surface has
+		// an operator on the other end of the client and keeps it.
+		isLatencySurface: () => interactive || acpMode,
 	});
 
 	// Coordinated shutdown (SIGINT/SIGTERM, TUI quit) must abort any in-flight
@@ -1891,6 +2033,10 @@ export async function bootOrchestrator(options: BootOptions = {}): Promise<BootR
 				bank,
 				activity: memoryIntervention.recentActivity(),
 				stepInFlight: memoryIntervention.stepInFlight(),
+				// Folded from the telemetry ledger, which is durable across sessions,
+				// so `/memory` answers what the tier has cost since it was turned on
+				// rather than what it cost since this process started.
+				spend: readTaskMemorySpendSummary(clioStateDir()),
 			};
 		},
 		getTaskMemorySeedOffer,

package/src/interactive/chat-loop-messages.ts

@@ -6,6 +6,11 @@
  */
 
 import { randomUUID } from "node:crypto";
+import {
+	type BackendCacheVerdict,
+	type BackendCompletionTimings,
+	uncachedPrefillTokens,
+} from "../core/cache-telemetry.js";
 import { settingsPath } from "../core/config.js";
 import {
 	addResponseModelIdObservationCount,
@@ -446,20 +451,34 @@ export function sumRunUsage(messages: ReadonlyArray<AgentMessage>): RunUsageSumm
 	return summary;
 }
 
-export type BackendCacheVerdict = "hot" | "partial" | "cold" | "small";
+export type { BackendCacheVerdict } from "../core/cache-telemetry.js";
+
+function classifyCacheUsage(input: number, cacheRead: number): BackendCacheVerdict {
+	if (cacheRead > 0) return input >= 2000 ? "partial" : "hot";
+	return input >= 2000 ? "cold" : "small";
+}
 
 /**
- * Classify one API call's provider-reported usage. This is the single
- * definition of a cache verdict; the ledger, the overlay, and any forensics
- * reader all consume the persisted value rather than reclassifying:
+ * Classify one API call's provider-reported usage. When normalized provider
+ * usage reports no cache read but the serving backend supplies cache_n, use
+ * that direct observation instead. This is the single definition of a cache
+ * verdict; the ledger, the overlay, and any forensics reader all consume the
+ * persisted value rather than reclassifying:
  *   hot      cacheRead > 0  and input < 2000   (prefix reused, prefill ≈ user text)
  *   partial  cacheRead > 0  and input >= 2000  (prefix reused up to a divergence point)
  *   cold     cacheRead == 0 and input >= 2000  (full re-prefill)
  *   small    cacheRead == 0 and input < 2000   (too small to judge)
  */
-export function backendCacheVerdict(input: number, cacheRead: number): BackendCacheVerdict {
-	if (cacheRead > 0) return input >= 2000 ? "partial" : "hot";
-	return input >= 2000 ? "cold" : "small";
+export function backendCacheVerdict(
+	input: number,
+	cacheRead: number,
+	backend?: BackendCompletionTimings | null,
+): BackendCacheVerdict {
+	if (cacheRead === 0 && backend?.cachedTokens !== null && backend?.cachedTokens !== undefined) {
+		const backendInput = uncachedPrefillTokens(backend);
+		if (backendInput !== null) return classifyCacheUsage(backendInput, backend.cachedTokens);
+	}
+	return classifyCacheUsage(input, cacheRead);
 }
 
 /** Per-API-call latency captured from the agent event stream (T3.2). */