@iowarp/clio-coder 0.3.8 → 0.4.0

package/src/domains/dispatch/reservation-store.ts

@@ -1,5 +1,6 @@
 import { randomBytes } from "node:crypto";
 import { withStateFileLockSync } from "../../core/state-file-lock.js";
+import { endpointLabel } from "../providers/endpoint-capacity.js";
 import {
 	acquireCapacityLease,
 	activeCapacityDrainUnsafe,
@@ -7,21 +8,38 @@ import {
 	type CapacityLimits,
 	type CapacityStateFile,
 	capacityStateLockPath,
+	describeEndpointCapacityHolders,
+	endpointCapacityRemedy,
 	readCapacityStateUnsafe,
 	writeCapacityStateUnsafe,
 } from "./capacity-lease.js";
+import { COUNCIL_MIN_MEMBERS } from "./gate-role-prompts.js";
 
 const MAX_RESERVATION_RECORDS = 500;
 export const DEFAULT_RESERVATION_TTL_MS = 15 * 60_000;
 
-export type ReservationTopology = "parallel" | "detached" | "sequential" | "pipeline" | "review" | "compete";
+/**
+ * `council` is carried rather than flattened into `parallel` because it is the
+ * one topology with a floor on its own width. Every other plan can answer a
+ * capacity denial by dispatching fewer at once; a council of two cannot, and a
+ * denial that tells it to is advice it cannot take.
+ */
+export type ReservationTopology =
+	| "parallel"
+	| "detached"
+	| "sequential"
+	| "pipeline"
+	| "review"
+	| "compete"
+	| "council";
 export type ReservationMemberStatus = "held" | "consumed" | "released";
 export type ReservationStatus = "active" | "released" | "rolled_back" | "expired";
 
 /**
- * A reservation holds three scarce things and nothing else: a global
- * concurrency slot, a per-node slot, and a budget upper bound. Node identity
- * matters only because slots are per-node. Agent, target, model, and runtime
+ * A reservation holds four scarce things and nothing else: a global
+ * concurrency slot, a per-node slot, an inference-endpoint slot, and a budget
+ * upper bound. Node and endpoint identity matter only because slots bind on
+ * those dimensions. Agent, target, model, and runtime
  * identity are route identity, which the failover envelope owns
  * (`assertRouteWithinApprovedEnvelope`); pinning them here made a
  * plan-approved dispatch unable to fail over at all.
@@ -30,6 +48,7 @@ export interface ReservationPlanTask {
 	memberId: string;
 	wave: number;
 	nodeId: string;
+	endpointKey?: string;
 	costUpperBoundUsd: number;
 }
 
@@ -54,17 +73,33 @@ export interface DispatchReservationRecord {
 export interface ReservationCapacitySnapshot {
 	global: { active: number; limit: number };
 	nodes: Readonly<Record<string, { active: number; limit: number }>>;
+	endpoints: Readonly<Record<string, { active: number; limit: number; leases?: number; foregroundStreams?: number }>>;
 	budget: { currentUsd: number; ceilingUsd: number };
 }
 
 export interface ReservationAllocation {
 	globalSlots: number;
 	nodeSlots: Readonly<Record<string, number>>;
+	endpointSlots: Readonly<Record<string, number>>;
 	budgetUsd: number;
 }
 
 export type ReservationPlanResult = { ok: true; allocation: ReservationAllocation } | { ok: false; reason: string };
 
+function endpointCapacityDenial(input: {
+	endpointKey: string;
+	limit: number;
+	used: number;
+	requested: number;
+	leases: number;
+	reservations: number;
+	foregroundStreams: number;
+	topology?: ReservationTopology;
+}): string {
+	const remedy = endpointCapacityRemedy(input.topology === "council" ? { rosterFloor: COUNCIL_MIN_MEMBERS } : undefined);
+	return `dispatch: admission denied: endpoint '${endpointLabel(input.endpointKey)}' capacity exceeded (${input.used + input.requested}/${input.limit} slots): ${describeEndpointCapacityHolders(input)}; ${remedy}`;
+}
+
 function copyRecord(record: DispatchReservationRecord): DispatchReservationRecord {
 	return { ...record, members: record.members.map((member) => ({ ...member })) };
 }
@@ -82,7 +117,8 @@ function validRecord(value: unknown): value is DispatchReservationRecord {
 			record.topology === "sequential" ||
 			record.topology === "pipeline" ||
 			record.topology === "review" ||
-			record.topology === "compete") &&
+			record.topology === "compete" ||
+			record.topology === "council") &&
 		(record.status === "active" ||
 			record.status === "released" ||
 			record.status === "rolled_back" ||
@@ -96,6 +132,7 @@ function validRecord(value: unknown): value is DispatchReservationRecord {
 				typeof member.memberId === "string" &&
 				Number.isInteger(member.wave) &&
 				typeof member.nodeId === "string" &&
+				(member.endpointKey === undefined || typeof member.endpointKey === "string") &&
 				typeof member.costUpperBoundUsd === "number" &&
 				Number.isFinite(member.costUpperBoundUsd) &&
 				member.costUpperBoundUsd >= 0 &&
@@ -145,9 +182,13 @@ function reservationAllocation(tasks: ReadonlyArray<ReservationPlanTask>): Reser
 		),
 	);
 	const nodeSlots = peakBy(tasks.map((task) => ({ wave: task.wave, key: task.nodeId })));
+	const endpointSlots = peakBy(
+		tasks.flatMap((task) => (task.endpointKey === undefined ? [] : [{ wave: task.wave, key: task.endpointKey }])),
+	);
 	return {
 		globalSlots,
 		nodeSlots,
+		endpointSlots,
 		budgetUsd: tasks.reduce((sum, task) => sum + task.costUpperBoundUsd, 0),
 	};
 }
@@ -168,6 +209,7 @@ function allocateReservation(
 	tasks: ReadonlyArray<ReservationPlanTask>,
 	existing: ReadonlyArray<DispatchReservationRecord>,
 	capacity: ReservationCapacitySnapshot,
+	topology?: ReservationTopology,
 ): ReservationPlanResult {
 	if (tasks.length === 0) return { ok: false, reason: "reservation requires at least one task" };
 	const duplicate = tasks.find((task, index) => tasks.slice(0, index).some((entry) => entry.memberId === task.memberId));
@@ -191,6 +233,27 @@ function allocateReservation(
 			return { ok: false, reason: `node '${nodeId}' capacity exceeded (${used + slots}/${node.limit})` };
 		}
 	}
+	for (const [endpointKey, slots] of Object.entries(requested.endpointSlots)) {
+		const endpoint = capacity.endpoints[endpointKey];
+		if (!endpoint || !Number.isFinite(endpoint.limit)) continue;
+		const heldSlots = held.reduce((sum, allocation) => sum + (allocation.endpointSlots[endpointKey] ?? 0), 0);
+		const used = endpoint.active + heldSlots;
+		if (used + slots > endpoint.limit) {
+			return {
+				ok: false,
+				reason: endpointCapacityDenial({
+					endpointKey,
+					limit: endpoint.limit,
+					used,
+					requested: slots,
+					leases: endpoint.leases ?? endpoint.active - (endpoint.foregroundStreams ?? 0),
+					reservations: heldSlots,
+					foregroundStreams: endpoint.foregroundStreams ?? 0,
+					...(topology === undefined ? {} : { topology }),
+				}),
+			};
+		}
+	}
 	const existingBudget = activeRecords.reduce((sum, record) => sum + outstandingBudget(record), 0);
 	const projected = capacity.budget.currentUsd + existingBudget + requested.budgetUsd;
 	if (projected >= capacity.budget.ceilingUsd) {
@@ -221,11 +284,13 @@ export function createDispatchReservation(input: {
 				`dispatch: reservation denied: capacity is draining (requested by pid ${drain.requestedByPid} at ${drain.requestedAt})`,
 			);
 		}
-		const planned = allocateReservation(input.tasks, records, input.capacity);
+		const planned = allocateReservation(input.tasks, records, input.capacity, input.topology);
 		if (!planned.ok) {
 			replaceReservations(state, records);
 			writeCapacityStateUnsafe(state);
-			throw new Error(`dispatch: reservation denied: ${planned.reason}`);
+			throw new Error(
+				planned.reason.startsWith("dispatch:") ? planned.reason : `dispatch: reservation denied: ${planned.reason}`,
+			);
 		}
 		const createdAt = new Date(nowMs).toISOString();
 		const record: DispatchReservationRecord = {
@@ -250,6 +315,7 @@ export function transferDispatchReservationToLease(input: {
 	memberId: string;
 	assignmentId: string;
 	nodeId: string;
+	endpointKey?: string;
 	limits: CapacityLimits;
 	nowMs?: number;
 	ttlMs?: number;
@@ -258,6 +324,7 @@ export function transferDispatchReservationToLease(input: {
 	return acquireCapacityLease({
 		assignmentId: input.assignmentId,
 		nodeId: input.nodeId,
+		...(input.endpointKey !== undefined ? { endpointKey: input.endpointKey } : {}),
 		limits: input.limits,
 		nowMs,
 		...(input.ttlMs !== undefined ? { ttlMs: input.ttlMs } : {}),
@@ -290,6 +357,7 @@ export function rebindDispatchReservationMember(input: {
 	ownerId: string;
 	memberId: string;
 	nodeId: string;
+	endpointKey?: string;
 	costUpperBoundUsd: number;
 	capacity: ReservationCapacitySnapshot;
 	nowMs?: number;
@@ -304,11 +372,42 @@ export function rebindDispatchReservationMember(input: {
 		if (member.status === "released") {
 			throw new Error(`dispatch: reservation member '${input.memberId}' was already released`);
 		}
-		if (member.nodeId === input.nodeId && member.costUpperBoundUsd === input.costUpperBoundUsd) {
+		if (
+			member.nodeId === input.nodeId &&
+			member.endpointKey === input.endpointKey &&
+			member.costUpperBoundUsd === input.costUpperBoundUsd
+		) {
 			return copyRecord(record);
 		}
-
 		const others = records.filter((entry) => entry.status === "active" && entry.ownerId !== input.ownerId);
+		if (member.endpointKey !== input.endpointKey && input.endpointKey !== undefined) {
+			const endpointKey = input.endpointKey;
+			const endpoint = input.capacity.endpoints[endpointKey];
+			if (endpoint !== undefined && Number.isFinite(endpoint.limit)) {
+				const siblingSlots = record.members.filter(
+					(entry) => entry !== member && entry.status !== "released" && entry.endpointKey === endpointKey,
+				).length;
+				const otherHeld = others.reduce(
+					(sum, entry) => sum + (reservationAllocation(outstandingTasks(entry)).endpointSlots[endpointKey] ?? 0),
+					0,
+				);
+				const used = endpoint.active + otherHeld + siblingSlots;
+				if (used + 1 > endpoint.limit) {
+					throw new Error(
+						endpointCapacityDenial({
+							endpointKey,
+							limit: endpoint.limit,
+							used,
+							requested: 1,
+							leases: endpoint.leases ?? endpoint.active - (endpoint.foregroundStreams ?? 0),
+							reservations: otherHeld + siblingSlots,
+							foregroundStreams: endpoint.foregroundStreams ?? 0,
+						}),
+					);
+				}
+			}
+		}
+
 		if (member.nodeId !== input.nodeId) {
 			const node = input.capacity.nodes[input.nodeId];
 			if (node !== undefined && Number.isFinite(node.limit)) {
@@ -342,6 +441,8 @@ export function rebindDispatchReservationMember(input: {
 		}
 
 		member.nodeId = input.nodeId;
+		if (input.endpointKey === undefined) delete member.endpointKey;
+		else member.endpointKey = input.endpointKey;
 		member.costUpperBoundUsd = Math.max(0, input.costUpperBoundUsd);
 		writeStore(records);
 		return copyRecord(record);
@@ -370,6 +471,26 @@ export function releaseDispatchReservationMember(
 	});
 }
 
+/** Settle every member after the scheduler has finished the reserved plan. */
+export function releaseDispatchReservation(ownerId: string, nowMs = Date.now()): DispatchReservationRecord | null {
+	return withStateFileLockSync(capacityStateLockPath(), () => {
+		const records = readStore();
+		const record = records.find((entry) => entry.ownerId === ownerId);
+		if (!record) return null;
+		if (record.status !== "active") return copyRecord(record);
+		const settledAt = new Date(nowMs).toISOString();
+		for (const member of record.members) {
+			if (member.status === "released") continue;
+			member.status = "released";
+			member.releasedAt = settledAt;
+		}
+		record.status = "released";
+		record.settledAt = settledAt;
+		writeStore(records);
+		return copyRecord(record);
+	});
+}
+
 export function rollbackDispatchReservation(ownerId: string, nowMs = Date.now()): DispatchReservationRecord | null {
 	return withStateFileLockSync(capacityStateLockPath(), () => {
 		const records = readStore();
@@ -488,14 +609,21 @@ export function reservedBudgetUsd(): number {
 		.reduce((sum, record) => sum + outstandingBudget(record), 0);
 }
 
-export function reservedCapacity(): { globalSlots: number; nodeSlots: Readonly<Record<string, number>> } {
+export function reservedCapacity(): {
+	globalSlots: number;
+	nodeSlots: Readonly<Record<string, number>>;
+	endpointSlots: Readonly<Record<string, number>>;
+} {
 	let globalSlots = 0;
 	const nodeSlots: Record<string, number> = {};
+	const endpointSlots: Record<string, number> = {};
 	for (const record of readStore().filter((entry) => entry.status === "active")) {
 		const allocation = reservationAllocation(outstandingTasks(record));
 		globalSlots += allocation.globalSlots;
 		for (const [nodeId, count] of Object.entries(allocation.nodeSlots))
 			nodeSlots[nodeId] = (nodeSlots[nodeId] ?? 0) + count;
+		for (const [endpointKey, count] of Object.entries(allocation.endpointSlots))
+			endpointSlots[endpointKey] = (endpointSlots[endpointKey] ?? 0) + count;
 	}
-	return { globalSlots, nodeSlots };
+	return { globalSlots, nodeSlots, endpointSlots };
 }

package/src/domains/dispatch/run-event-journal-bridge.ts

@@ -0,0 +1,149 @@
+/**
+ * Domain-owned writer for the run event journal.
+ *
+ * The journal was originally teed from `createDispatchRunEventRegistry`
+ * (src/tools/dispatch-run-events.ts), the display-tail helper behind the
+ * model-facing `dispatch` tool. That helper is not on the operator paths:
+ * `clio-coder run --agent` (src/cli/run.ts), `clio-coder fleet run`
+ * (src/cli/fleet.ts), and the TUI `/run` slash command each take a handle
+ * straight off `DispatchContract.dispatch` and iterate `handle.events`
+ * themselves, so no registry is ever built and the sink never fired. The
+ * writer had to move to something every dispatch reaches.
+ *
+ * `BusChannels.DispatchProgress` is that thing. The dispatch domain publishes
+ * it from inside the event pump for every consumer-visible event of every run,
+ * "attached, detached, batched, and retry runs, exactly once" (see
+ * src/core/bus-events.ts), whether or not anyone iterates the handle. The
+ * terminal pair `DispatchCompleted`/`DispatchFailed` closes every run the same
+ * way. Subscribing here makes the journal a property of dispatching, not of
+ * which caller happened to build a registry.
+ *
+ * Ownership is explicit rather than ambient: the bridge attaches only when a
+ * composition root asks for it (`DispatchBundleOptions.journalRunEvents`), and
+ * `registerAllTools` hands its registry `journal: null` in that same process.
+ * One writer per file, decided at one place, so a tool-path run in the TUI is
+ * not transcribed twice.
+ *
+ * Every write is best-effort. The sink degrades itself on I/O failure and a
+ * listener that throws is contained by the safe bus, so nothing here can fail a
+ * dispatch.
+ */
+
+import type {
+	DispatchCompletedPayload,
+	DispatchFailedPayload,
+	DispatchProgressPayload,
+} from "../../core/bus-events.js";
+import { BusChannels } from "../../core/bus-events.js";
+import type { SafeEventBus } from "../../core/event-bus.js";
+import { runTailEntryFromEvent } from "../../tools/dispatch-run-events.js";
+import { defaultRunEventJournal, type RunEventJournalSink } from "./run-event-journal.js";
+
+/**
+ * Runs whose `open` line this bridge has already written. Bounded so a
+ * long-lived orchestrator cannot grow it without limit; the journal writer
+ * keeps its own bound for the same reason. Evicting a run only forgets that it
+ * was opened, and a later event for it writes a second `open` line, which the
+ * reader and the writer both tolerate.
+ */
+const OPENED_RUN_LIMIT = 512;
+
+export interface RunEventJournalBridge {
+	/** Unsubscribe from the bus. Idempotent. */
+	stop(): void;
+}
+
+export interface AttachRunEventJournalBridgeOptions {
+	/** Sink override; defaults to the process-wide journal. */
+	journal?: RunEventJournalSink;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function runIdentity(payload: unknown): { runId: string; agentId: string } | null {
+	if (!isRecord(payload)) return null;
+	if (typeof payload.runId !== "string" || payload.runId.length === 0) return null;
+	return { runId: payload.runId, agentId: typeof payload.agentId === "string" ? payload.agentId : "unknown" };
+}
+
+/**
+ * Subscribe the journal to a dispatch bus. Returns a handle whose `stop()` the
+ * owning bundle calls on shutdown so a second bundle in the same process (tests
+ * do this) never inherits the first one's listeners.
+ */
+export function attachRunEventJournalBridge(
+	bus: SafeEventBus,
+	options: AttachRunEventJournalBridgeOptions = {},
+): RunEventJournalBridge {
+	const journal = options.journal ?? defaultRunEventJournal();
+	const opened = new Set<string>();
+
+	const ensureOpen = (runId: string, agentId: string): void => {
+		if (opened.has(runId)) return;
+		opened.add(runId);
+		while (opened.size > OPENED_RUN_LIMIT) {
+			const oldest = opened.values().next();
+			if (oldest.done === true) break;
+			opened.delete(oldest.value);
+		}
+		journal.open(runId, agentId);
+	};
+
+	/**
+	 * Write the receipt facts and close the run. Read defensively for the same
+	 * reason the registry's seal is: a terminal payload that is missing a field
+	 * it is typed to have must degrade the transcript, never throw back into the
+	 * finalizer that emitted it.
+	 */
+	const seal = (outcome: string, detail: string | null | undefined, payload: unknown): void => {
+		const identity = runIdentity(payload);
+		if (identity === null) return;
+		ensureOpen(identity.runId, identity.agentId);
+		const exitCode = isRecord(payload) && typeof payload.exitCode === "number" ? payload.exitCode : null;
+		journal.receipt(identity.runId, { outcome, exitCode });
+		journal.terminal(identity.runId, outcome, detail ?? undefined);
+		opened.delete(identity.runId);
+	};
+
+	const unsubscribes = [
+		bus.on(BusChannels.DispatchProgress, (payload: DispatchProgressPayload) => {
+			const identity = runIdentity(payload);
+			if (identity === null) return;
+			// The display projection decides what a transcript line is; a
+			// heartbeat or a streaming delta is not one. Opening the run only
+			// once an event survives that filter keeps a heartbeat-only run from
+			// creating an empty journal directory.
+			const entry = runTailEntryFromEvent(payload.event);
+			if (entry === null) return;
+			ensureOpen(identity.runId, identity.agentId);
+			// Route resolution warnings use `message`, while worker transcript
+			// events generally project their text through `detail`. Preserve the
+			// warning at this dispatch-owned durability seam so fleet view does not
+			// render a content-free `route_warning` line.
+			const message =
+				isRecord(payload.event) && typeof payload.event.message === "string" ? payload.event.message : undefined;
+			journal.append(
+				identity.runId,
+				entry.detail === undefined && message !== undefined ? { ...entry, detail: message } : entry,
+			);
+		}),
+		bus.on(BusChannels.DispatchCompleted, (payload: DispatchCompletedPayload) => {
+			seal(payload.outcome, payload.outcomeDetail, payload);
+		}),
+		bus.on(BusChannels.DispatchFailed, (payload: DispatchFailedPayload) => {
+			seal(payload.outcome, payload.outcomeDetail, payload);
+		}),
+	];
+
+	let stopped = false;
+	return {
+		stop(): void {
+			if (stopped) return;
+			stopped = true;
+			for (const unsubscribe of unsubscribes) unsubscribe();
+			opened.clear();
+		},
+	};
+}