@iowarp/clio-coder 0.3.7 → 0.3.9

package/src/domains/dispatch/admission.ts

@@ -1,4 +1,5 @@
 import { randomBytes } from "node:crypto";
+import { endpointLabel } from "../providers/endpoint-capacity.js";
 import type { ActionClass } from "../safety/action-classifier.js";
 import type { ScopeSpec } from "../safety/scope.js";
 import { createAdmissionQueue } from "./admission-queue.js";
@@ -7,6 +8,8 @@ import {
 	type CapacityLease,
 	type CapacityLimits,
 	capacityLeaseUsage,
+	describeEndpointCapacityHolders,
+	type EndpointCapacityHolders,
 	heartbeatCapacityLease,
 	releaseCapacityLease,
 	renameCapacityLeaseAssignment,
@@ -41,6 +44,7 @@ export interface CapacityAdmissionController {
 	admit(input: {
 		assignmentId: string;
 		nodeId: string;
+		endpointKey?: string;
 		deadlineAt: number;
 		priority?: number;
 		planId?: string;
@@ -105,13 +109,31 @@ const PUMP_MAX_MS = 500;
 function describeAdmissionFailure(input: {
 	state: "canceled" | "timed_out";
 	nodeId: string;
+	endpointKey?: string;
 	waitedMs: number;
 	overdueAtQueueMs: number;
 	queueDepth: number;
 	limits: CapacityLimits;
-	usage: { global: number; nodes: Readonly<Record<string, number>> };
+	usage: {
+		global: number;
+		nodes: Readonly<Record<string, number>>;
+		endpoints: Readonly<Record<string, number>>;
+		endpointHolders?: Readonly<Record<string, EndpointCapacityHolders>>;
+	};
 }): string {
 	if (input.state === "canceled") return "dispatch: admission canceled before a capacity slot opened";
+	if (input.endpointKey !== undefined) {
+		const endpointLimit = input.limits.endpoints[input.endpointKey];
+		const endpointActive = input.usage.endpoints[input.endpointKey] ?? 0;
+		if (endpointLimit !== undefined && endpointActive >= endpointLimit) {
+			const holders = input.usage.endpointHolders?.[input.endpointKey];
+			const occupancy =
+				holders === undefined
+					? "the supplied capacity snapshot does not attribute the occupied slots"
+					: describeEndpointCapacityHolders(holders);
+			return `dispatch: admission denied: endpoint '${endpointLabel(input.endpointKey)}' capacity reached (${endpointActive}/${endpointLimit} slots): ${occupancy}; collect in-flight runs or point workers at a second server`;
+		}
+	}
 	const nodeLimit = input.limits.nodes[input.nodeId];
 	const nodeActive = input.usage.nodes[input.nodeId] ?? 0;
 	const where =
@@ -142,7 +164,12 @@ function describeAdmissionFailure(input: {
 export function createCapacityAdmissionController(options: {
 	limits: () => CapacityLimits;
 	/** Live lease counts for the timeout message; defaults to the durable state file. */
-	usage?: () => { global: number; nodes: Readonly<Record<string, number>> };
+	usage?: () => {
+		global: number;
+		nodes: Readonly<Record<string, number>>;
+		endpoints: Readonly<Record<string, number>>;
+		endpointHolders?: Readonly<Record<string, EndpointCapacityHolders>>;
+	};
 	now?: () => number;
 	maxQueueSize?: number;
 	queueCeilingMs?: number;
@@ -175,7 +202,14 @@ export function createCapacityAdmissionController(options: {
 					admittedLease = request.value();
 					return true;
 				} catch (error) {
-					// Capacity is a wait, not a failure. Anything else fails the request.
+					// Endpoint saturation is a named refusal because waiting would keep a
+					// model request queued behind the same inference scheduler. Existing
+					// node and global admission retain their bounded wait behavior.
+					if (error instanceof Error && /endpoint '.*' capacity reached/u.test(error.message)) {
+						queue.fail(request.requestId, error);
+						return false;
+					}
+					// Node and global capacity can wait. Anything else fails the request.
 					if (error instanceof Error && /capacity reached/.test(error.message)) return false;
 					queue.fail(request.requestId, error instanceof Error ? error : new Error(String(error)));
 					return false;
@@ -229,12 +263,14 @@ export function createCapacityAdmissionController(options: {
 									memberId: input.reservation.memberId,
 									assignmentId: input.assignmentId,
 									nodeId: input.nodeId,
+									...(input.endpointKey !== undefined ? { endpointKey: input.endpointKey } : {}),
 									limits: options.limits(),
 									nowMs: now(),
 								})
 							: acquireCapacityLease({
 									assignmentId: input.assignmentId,
 									nodeId: input.nodeId,
+									...(input.endpointKey !== undefined ? { endpointKey: input.endpointKey } : {}),
 									limits: options.limits(),
 									nowMs: now(),
 								}),
@@ -246,6 +282,7 @@ export function createCapacityAdmissionController(options: {
 						describeAdmissionFailure({
 							state: outcome.state,
 							nodeId: input.nodeId,
+							...(input.endpointKey !== undefined ? { endpointKey: input.endpointKey } : {}),
 							waitedMs: now() - queuedAt,
 							overdueAtQueueMs: queuedAt - input.deadlineAt,
 							queueDepth: queue.size(),

package/src/domains/dispatch/assignment-reconcile.ts

@@ -1,8 +1,9 @@
 /**
  * Startup reconciliation for durable assignments. The in-memory retry queue and
- * assignment registry do not survive a restart (Symphony §14.3), so every
- * durable record still marked `running` after a crash is orphaned: no active
- * attempt, no queued retry, and no attached terminal promise can ever settle it.
+ * assignment registry do not survive a restart (Symphony §14.3), so a durable
+ * record whose process owner is gone is orphaned: no active attempt, no queued
+ * retry, and no attached terminal promise can ever settle it. Running records
+ * owned by a live sibling process are deliberately left alone.
  *
  * This pass resolves each orphan deterministically against durable ledger
  * evidence so a later `wait`/`collect` can never observe a stuck `running`
@@ -11,7 +12,7 @@
  */
 
 import type { AssignmentStatus } from "./assignment.js";
-import type { DurableAssignmentRecord } from "./assignment-store.js";
+import type { AssignmentVerdictOwner, DurableAssignmentRecord } from "./assignment-store.js";
 
 /** Minimal terminal view of one attempt's durable ledger row. */
 export interface ReconcileAttemptView {
@@ -25,13 +26,16 @@ export interface ReconcileAttemptView {
 export interface AssignmentReconcileDeps {
 	/** Durable records currently marked `running`. */
 	listRunning: () => ReadonlyArray<DurableAssignmentRecord>;
+	/** Decide liveness from the record's durable pid and process birth token. */
+	ownerAlive: (record: DurableAssignmentRecord) => boolean;
 	/** Resolve one attempt's terminal view from the ledger, or null if pruned. */
 	lookupAttempt: (runId: string) => ReconcileAttemptView | null;
-	/** Persist the reconciled terminal status. */
+	/** Persist the reconciled terminal status, as the record's verdict owner when it has one. */
 	settle: (
 		assignmentId: string,
 		terminalRunId: string,
 		status: Exclude<AssignmentStatus, "running">,
+		owner?: AssignmentVerdictOwner,
 	) => Promise<unknown>;
 }
 
@@ -44,6 +48,10 @@ export interface AssignmentReconcileSummary {
 
 /**
  * Deterministic rule, in priority order:
+ *  -1. a record owned by a live process is not an orphan and remains running;
+ *   0. a record whose verdict belongs to its opener is failed (abandoned): the
+ *      opener died before reaching a verdict, and a green attempt under it is
+ *      one step of a run, never the run's answer;
  *   1. a recovered succeeded terminal attempt wins (`succeeded`);
  *   2. otherwise the last terminal attempt marks the assignment `failed`;
  *   3. otherwise no attempt is recoverable and the assignment is failed
@@ -53,6 +61,15 @@ export async function reconcileOrphanAssignments(deps: AssignmentReconcileDeps):
 	const summary: AssignmentReconcileSummary = { recovered: 0, abandoned: 0 };
 	for (const record of deps.listRunning()) {
 		if (record.status !== "running") continue;
+		if (deps.ownerAlive(record)) continue;
+		if (record.verdictOwner !== undefined) {
+			// Attempts under a claimed row are fleet steps, not the fleet verdict.
+			// Filing a green step as the terminal id of an abandoned fleet would
+			// contradict that step's immutable ledger result.
+			await deps.settle(record.assignmentId, record.assignmentId, "failed", record.verdictOwner);
+			summary.abandoned += 1;
+			continue;
+		}
 		const views = record.attempts
 			.map((runId) => deps.lookupAttempt(runId))
 			.filter((view): view is ReconcileAttemptView => view !== null);

package/src/domains/dispatch/assignment-store.ts

@@ -5,6 +5,7 @@
 
 import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
+import { BIRTH_TOKEN_SOURCE_AVAILABLE, processAlive, processBirthToken } from "../../core/process-identity.js";
 import { withStateFileLock } from "../../core/state-file-lock.js";
 import { clioStateDir } from "../../core/xdg.js";
 import { atomicWrite } from "../../engine/session.js";
@@ -12,11 +13,54 @@ import type { AssignmentStatus } from "./assignment.js";
 
 const MAX_ASSIGNMENT_RECORDS = 1_000;
 
+/**
+ * Who decides a record's status.
+ *
+ * Absent is the ordinary case: the record is one logical dispatch, so the
+ * attempts filed under it are the only thing that can settle it. A fleet claims
+ * the verdict instead, because a fleet root gathers every step of a run under
+ * one id and no single step is the run's answer. Without the claim the last
+ * step to settle writes the row, which reports a green step of an aborted run
+ * as the run's outcome.
+ */
+export type AssignmentVerdictOwner = "fleet";
+
+/**
+ * Durable identity of the process executing a running assignment. A pid alone
+ * is not enough: the OS may recycle it after a crash, so the process birth
+ * token must still match before another extension treats the owner as live.
+ */
+export interface AssignmentProcessOwner {
+	pid: number;
+	processBirthToken: string;
+	acquiredAt: string;
+}
+
+export interface AssignmentProcessOwnerProbe {
+	birthToken(pid: number): string | null;
+	/** True when a missing birth token proves that the process is gone. */
+	tokenProvesDeath?: boolean;
+	/** Liveness fallback for platforms whose birth token is synthetic. */
+	alive?(pid: number): boolean;
+}
+
+/**
+ * One durable logical-dispatch row.
+ *
+ * `attempts` holds terminal run ids: an agent attempt's receipt id, and for a
+ * fleet also each deterministic step's `code-*` run id, whose evidence lives
+ * under `code-steps/<assignmentId>/`. `status` answers for the whole record,
+ * so for a fleet it answers for the whole run rather than for its last step.
+ */
 export interface DurableAssignmentRecord {
 	assignmentId: string;
 	attempts: string[];
 	terminalRunId: string | null;
 	status: AssignmentStatus;
+	/** Set when something other than the attempts decides `status`. */
+	verdictOwner?: AssignmentVerdictOwner;
+	/** Process lease held only while this record is running. */
+	processOwner?: AssignmentProcessOwner;
 }
 
 interface AssignmentStoreFile {
@@ -28,6 +72,18 @@ function storePath(): string {
 	return join(clioStateDir(), "assignments.json");
 }
 
+function validProcessOwner(value: unknown): value is AssignmentProcessOwner {
+	if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
+	const owner = value as Partial<AssignmentProcessOwner>;
+	return (
+		Number.isSafeInteger(owner.pid) &&
+		(owner.pid ?? 0) > 0 &&
+		typeof owner.processBirthToken === "string" &&
+		typeof owner.acquiredAt === "string" &&
+		Number.isFinite(Date.parse(owner.acquiredAt))
+	);
+}
+
 function validRecord(value: unknown): value is DurableAssignmentRecord {
 	if (typeof value !== "object" || value === null) return false;
 	const record = value as Partial<DurableAssignmentRecord>;
@@ -36,6 +92,8 @@ function validRecord(value: unknown): value is DurableAssignmentRecord {
 		Array.isArray(record.attempts) &&
 		record.attempts.every((runId) => typeof runId === "string") &&
 		(record.terminalRunId === null || typeof record.terminalRunId === "string") &&
+		(record.verdictOwner === undefined || record.verdictOwner === "fleet") &&
+		(record.processOwner === undefined || validProcessOwner(record.processOwner)) &&
 		(record.status === "running" ||
 			record.status === "succeeded" ||
 			record.status === "failed" ||
@@ -57,7 +115,32 @@ function readStore(): DurableAssignmentRecord[] {
 }
 
 function copyRecord(record: DurableAssignmentRecord): DurableAssignmentRecord {
-	return { ...record, attempts: [...record.attempts] };
+	return {
+		...record,
+		attempts: [...record.attempts],
+		...(record.processOwner === undefined ? {} : { processOwner: { ...record.processOwner } }),
+	};
+}
+
+const defaultProcessOwnerProbe: AssignmentProcessOwnerProbe = {
+	birthToken: processBirthToken,
+	tokenProvesDeath: BIRTH_TOKEN_SOURCE_AVAILABLE,
+	alive: processAlive,
+};
+
+/** True only when the durable identity still names the same live process. */
+export function assignmentProcessOwnerAlive(
+	record: Pick<DurableAssignmentRecord, "processOwner">,
+	probe: AssignmentProcessOwnerProbe = defaultProcessOwnerProbe,
+): boolean {
+	const owner = record.processOwner;
+	if (owner === undefined) return false;
+	const current = probe.birthToken(owner.pid);
+	if (current === null || current !== owner.processBirthToken) return false;
+	// A synthetic `pid-N` token cannot distinguish pid reuse, so require an
+	// independent liveness signal on platforms without a real birth-token source.
+	if (probe.tokenProvesDeath === false) return probe.alive?.(owner.pid) ?? false;
+	return true;
 }
 
 function writeStore(assignments: ReadonlyArray<DurableAssignmentRecord>): void {
@@ -114,49 +197,103 @@ export async function renameStoredAssignment(
 	return copyRecord(result);
 }
 
-export function registerAssignment(assignmentId: string): Promise<DurableAssignmentRecord> {
-	return updateRecord(
+function currentProcessOwner(): AssignmentProcessOwner {
+	const token = processBirthToken();
+	if (token === null) throw new Error("dispatch: cannot establish assignment owner process birth token");
+	return {
+		pid: process.pid,
+		processBirthToken: token,
+		acquiredAt: new Date().toISOString(),
+	};
+}
+
+function openRecord(assignmentId: string): DurableAssignmentRecord {
+	return {
 		assignmentId,
-		(current) => current ?? { assignmentId, attempts: [], terminalRunId: null, status: "running" },
-	);
+		attempts: [],
+		terminalRunId: null,
+		status: "running",
+		processOwner: currentProcessOwner(),
+	};
+}
+
+function ensureRunningProcessOwner(record: DurableAssignmentRecord): DurableAssignmentRecord {
+	if (record.status !== "running" || record.processOwner !== undefined) return record;
+	return { ...record, processOwner: currentProcessOwner() };
+}
+
+function clearProcessOwner(record: DurableAssignmentRecord): DurableAssignmentRecord {
+	const result = { ...record };
+	delete result.processOwner;
+	return result;
+}
+
+export function registerAssignment(assignmentId: string): Promise<DurableAssignmentRecord> {
+	return updateRecord(assignmentId, (current) => ensureRunningProcessOwner(current ?? openRecord(assignmentId)));
+}
+
+/**
+ * Reserve this record's status for `owner`, opening the record if it is new.
+ * Attempts filed by anyone are still recorded; only the verdict is reserved,
+ * and until the owner settles it the record stays `running`.
+ */
+export function claimAssignmentVerdict(
+	assignmentId: string,
+	owner: AssignmentVerdictOwner,
+): Promise<DurableAssignmentRecord> {
+	return updateRecord(assignmentId, (current) => ({
+		...ensureRunningProcessOwner(current ?? openRecord(assignmentId)),
+		verdictOwner: owner,
+	}));
 }
 
 export function recordAssignmentAttempt(assignmentId: string, runId: string): Promise<DurableAssignmentRecord> {
 	return updateRecord(assignmentId, (current) => {
-		const base = current ?? { assignmentId, attempts: [], terminalRunId: null, status: "running" as const };
+		const base = ensureRunningProcessOwner(current ?? openRecord(assignmentId));
 		return base.attempts.includes(runId) ? base : { ...base, attempts: [...base.attempts, runId] };
 	});
 }
 
 export function failQueuedAssignment(assignmentId: string): Promise<DurableAssignmentRecord> {
 	return updateRecord(assignmentId, (current) => {
-		const base = current ?? { assignmentId, attempts: [], terminalRunId: null, status: "running" as const };
-		return base.status === "running" ? { ...base, status: "failed" } : base;
+		const base = current ?? openRecord(assignmentId);
+		if (base.verdictOwner !== undefined) return base;
+		return base.status === "running" ? clearProcessOwner({ ...base, status: "failed" }) : base;
 	});
 }
 
 export function timeoutStoredAssignment(assignmentId: string): Promise<DurableAssignmentRecord> {
 	return updateRecord(assignmentId, (current) => {
-		const base = current ?? { assignmentId, attempts: [], terminalRunId: null, status: "running" as const };
-		return base.status === "running" ? { ...base, status: "timed_out" } : base;
+		const base = current ?? openRecord(assignmentId);
+		if (base.verdictOwner !== undefined) return base;
+		return base.status === "running" ? clearProcessOwner({ ...base, status: "timed_out" }) : base;
 	});
 }
 
 export function cancelStoredAssignment(assignmentId: string): Promise<DurableAssignmentRecord> {
 	return updateRecord(assignmentId, (current) => {
-		const base = current ?? { assignmentId, attempts: [], terminalRunId: null, status: "running" as const };
-		return base.status === "running" ? { ...base, status: "canceled" } : base;
+		const base = current ?? openRecord(assignmentId);
+		if (base.verdictOwner !== undefined) return base;
+		return base.status === "running" ? clearProcessOwner({ ...base, status: "canceled" }) : base;
 	});
 }
 
+/**
+ * File `terminalRunId` as an attempt and, when the caller owns the verdict, set
+ * the record's status. An attempt settling under a claimed record settles
+ * itself and not the record: one fleet step reaching a green receipt says
+ * nothing about the six steps behind it that never ran.
+ */
 export function settleStoredAssignment(
 	assignmentId: string,
 	terminalRunId: string,
 	status: Exclude<AssignmentStatus, "running">,
+	owner?: AssignmentVerdictOwner,
 ): Promise<DurableAssignmentRecord> {
 	return updateRecord(assignmentId, (current) => {
-		const base = current ?? { assignmentId, attempts: [], terminalRunId: null, status: "running" as const };
+		const base = current ?? openRecord(assignmentId);
 		const attempts = base.attempts.includes(terminalRunId) ? base.attempts : [...base.attempts, terminalRunId];
-		return { ...base, attempts, terminalRunId, status };
+		if (base.verdictOwner !== owner) return { ...base, attempts };
+		return clearProcessOwner({ ...base, attempts, terminalRunId, status });
 	});
 }

package/src/domains/dispatch/capacity-lease.ts

@@ -6,6 +6,7 @@ import { BIRTH_TOKEN_SOURCE_AVAILABLE, processAlive, processBirthToken } from ".
 import { FILE_LOCK_ACQUIRE_TIMEOUT_MS, withStateFileLockSync } from "../../core/state-file-lock.js";
 import { clioStateDir } from "../../core/xdg.js";
 import { atomicWrite } from "../../engine/session.js";
+import { endpointLabel, foregroundStreamUsage } from "../providers/endpoint-capacity.js";
 
 export { processBirthToken } from "../../core/process-identity.js";
 
@@ -30,6 +31,8 @@ export interface CapacityLease {
 	leaseId: string;
 	assignmentId: string;
 	nodeId: string;
+	/** Canonical inference endpoint. Absent only on leases written before endpoint capacity existed. */
+	endpointKey?: string;
 	/** Owner host. Absent only on records written before the field existed; those are read as this host. */
 	host?: string;
 	ownerPid: number;
@@ -43,6 +46,7 @@ export interface CapacityLease {
 export interface CapacityLimits {
 	global: number;
 	nodes: Readonly<Record<string, number>>;
+	endpoints: Readonly<Record<string, number>>;
 }
 export interface CapacityDrain {
 	requestedByPid: number;
@@ -81,6 +85,7 @@ function valid(value: unknown): value is CapacityLease {
 		typeof lease.leaseId === "string" &&
 		typeof lease.assignmentId === "string" &&
 		typeof lease.nodeId === "string" &&
+		(lease.endpointKey === undefined || typeof lease.endpointKey === "string") &&
 		(lease.host === undefined || typeof lease.host === "string") &&
 		Number.isInteger(lease.ownerPid) &&
 		typeof lease.processBirthToken === "string" &&
@@ -182,9 +187,14 @@ const MEMBER_STATUSES = new Set(["held", "consumed", "released"]);
  * rejected here as well: silently skipping an unreadable record would under-count
  * capacity and admit past the cap.
  */
-function heldReservationUsage(values: ReadonlyArray<unknown>): { global: number; nodes: Record<string, number> } {
+function heldReservationUsage(values: ReadonlyArray<unknown>): {
+	global: number;
+	nodes: Record<string, number>;
+	endpoints: Record<string, number>;
+} {
 	let global = 0;
 	const nodes: Record<string, number> = {};
+	const endpoints: Record<string, number> = {};
 	for (const value of values) {
 		if (typeof value !== "object" || value === null)
 			throw new Error("dispatch capacity store has invalid reservation records");
@@ -194,10 +204,11 @@ function heldReservationUsage(values: ReadonlyArray<unknown>): { global: number;
 		if (record.status !== "active") continue;
 		const byWave = new Map<number, number>();
 		const byNodeWave = new Map<string, Map<number, number>>();
+		const byEndpointWave = new Map<string, Map<number, number>>();
 		for (const raw of record.members) {
 			if (typeof raw !== "object" || raw === null)
 				throw new Error("dispatch capacity store has invalid reservation records");
-			const member = raw as { status?: unknown; wave?: unknown; nodeId?: unknown };
+			const member = raw as { status?: unknown; wave?: unknown; nodeId?: unknown; endpointKey?: unknown };
 			if (
 				typeof member.status !== "string" ||
 				!MEMBER_STATUSES.has(member.status) ||
@@ -211,13 +222,53 @@ function heldReservationUsage(values: ReadonlyArray<unknown>): { global: number;
 			const nodeWaves = byNodeWave.get(member.nodeId) ?? new Map<number, number>();
 			nodeWaves.set(wave, (nodeWaves.get(wave) ?? 0) + 1);
 			byNodeWave.set(member.nodeId, nodeWaves);
+			if (member.endpointKey !== undefined && typeof member.endpointKey !== "string")
+				throw new Error("dispatch capacity store has invalid reservation records");
+			if (typeof member.endpointKey === "string") {
+				const endpointWaves = byEndpointWave.get(member.endpointKey) ?? new Map<number, number>();
+				endpointWaves.set(wave, (endpointWaves.get(wave) ?? 0) + 1);
+				byEndpointWave.set(member.endpointKey, endpointWaves);
+			}
 		}
 		global += Math.max(0, ...byWave.values());
 		for (const [id, waves] of byNodeWave) nodes[id] = (nodes[id] ?? 0) + Math.max(0, ...waves.values());
+		for (const [key, waves] of byEndpointWave) endpoints[key] = (endpoints[key] ?? 0) + Math.max(0, ...waves.values());
 	}
-	return { global, nodes };
+	return { global, nodes, endpoints };
 }
-function assertCapacity(file: CapacityStateFile, nodeId: string, limits: CapacityLimits): void {
+
+export interface EndpointCapacityHolders {
+	leases: number;
+	reservations: number;
+	foregroundStreams: number;
+}
+
+/** Name the endpoint-capacity addends that are actually nonzero. */
+export function describeEndpointCapacityHolders(holders: EndpointCapacityHolders): string {
+	const parts: string[] = [];
+	const add = (count: number, singular: string, plural: string): void => {
+		if (count > 0) parts.push(`${count} ${count === 1 ? singular : plural}`);
+	};
+	add(holders.leases, "active lease", "active leases");
+	add(holders.reservations, "held reservation", "held reservations");
+	add(holders.foregroundStreams, "foreground stream", "foreground streams");
+	if (parts.length === 0) return "no active lease, held reservation, or foreground stream currently holds a slot";
+	const named =
+		parts.length === 1
+			? parts[0]
+			: parts.length === 2
+				? `${parts[0]} and ${parts[1]}`
+				: `${parts.slice(0, -1).join(", ")}, and ${parts.at(-1)}`;
+	const total = holders.leases + holders.reservations + holders.foregroundStreams;
+	return `${named} ${total === 1 ? "holds" : "hold"} ${total === 1 ? "the slot" : "the slots"}`;
+}
+
+function assertCapacity(
+	file: CapacityStateFile,
+	nodeId: string,
+	endpointKey: string | undefined,
+	limits: CapacityLimits,
+): void {
 	const held = heldReservationUsage(file.reservations);
 	const globalUsed = file.leases.length + held.global;
 	// The denial names the compliant next move, not just the gate. A model told
@@ -235,10 +286,28 @@ function assertCapacity(file: CapacityStateFile, nodeId: string, limits: Capacit
 		if (used >= nodeLimit)
 			throw new Error(`dispatch: admission denied: node '${nodeId}' capacity reached (${used}/${nodeLimit})`);
 	}
+	if (endpointKey !== undefined) {
+		const endpointLimit = limits.endpoints[endpointKey];
+		if (endpointLimit !== undefined) {
+			if (endpointLimit < 1)
+				throw new Error(`dispatch: admission denied: endpoint '${endpointLabel(endpointKey)}' capacity unavailable`);
+			const holders: EndpointCapacityHolders = {
+				leases: file.leases.filter((lease) => lease.endpointKey === endpointKey).length,
+				reservations: held.endpoints[endpointKey] ?? 0,
+				foregroundStreams: foregroundStreamUsage()[endpointKey] ?? 0,
+			};
+			const used = holders.leases + holders.reservations + holders.foregroundStreams;
+			if (used >= endpointLimit)
+				throw new Error(
+					`dispatch: admission denied: endpoint '${endpointLabel(endpointKey)}' capacity reached (${used}/${endpointLimit} slots): ${describeEndpointCapacityHolders(holders)}; collect in-flight runs or point workers at a second server`,
+				);
+		}
+	}
 }
 export function acquireCapacityLease(input: {
 	assignmentId: string;
 	nodeId: string;
+	endpointKey?: string;
 	limits: CapacityLimits;
 	nowMs?: number;
 	ttlMs?: number;
@@ -259,10 +328,12 @@ export function acquireCapacityLease(input: {
 			);
 		const existing = file.leases.find((lease) => lease.assignmentId === input.assignmentId);
 		if (existing) {
-			if (existing.nodeId !== input.nodeId) {
+			if (existing.nodeId !== input.nodeId || existing.endpointKey !== input.endpointKey) {
 				const without = { ...file, leases: file.leases.filter((lease) => lease !== existing) };
-				assertCapacity(without, input.nodeId, input.limits);
+				assertCapacity(without, input.nodeId, input.endpointKey, input.limits);
 				existing.nodeId = input.nodeId;
+				if (input.endpointKey === undefined) delete existing.endpointKey;
+				else existing.endpointKey = input.endpointKey;
 			}
 			existing.heartbeatAt = new Date(nowMs).toISOString();
 			existing.expiresAt = new Date(nowMs + (input.ttlMs ?? DEFAULT_CAPACITY_LEASE_TTL_MS)).toISOString();
@@ -270,7 +341,7 @@ export function acquireCapacityLease(input: {
 			return clone(existing);
 		}
 		input.onAcquiredUnderLock?.(file);
-		assertCapacity(file, input.nodeId, input.limits);
+		assertCapacity(file, input.nodeId, input.endpointKey, input.limits);
 		const token = input.processBirthToken ?? processBirthToken(input.ownerPid ?? process.pid);
 		if (!token) throw new Error("dispatch: cannot establish process birth token");
 		const at = new Date(nowMs).toISOString();
@@ -278,6 +349,7 @@ export function acquireCapacityLease(input: {
 			leaseId: `lease-${nowMs.toString(36)}-${randomBytes(6).toString("hex")}`,
 			assignmentId: input.assignmentId,
 			nodeId: input.nodeId,
+			...(input.endpointKey !== undefined ? { endpointKey: input.endpointKey } : {}),
 			host: hostname(),
 			ownerPid: input.ownerPid ?? process.pid,
 			processBirthToken: token,
@@ -365,11 +437,28 @@ export function capacityDrain(nowMs = Date.now()): CapacityDrain | null {
 export function capacityLeaseUsage(options?: { nowMs?: number; probe?: LeaseOwnerProbe }): {
 	global: number;
 	nodes: Readonly<Record<string, number>>;
+	endpoints: Readonly<Record<string, number>>;
+	endpointHolders: Readonly<Record<string, EndpointCapacityHolders>>;
 } {
 	const leases = listCapacityLeases(options);
 	const nodes: Record<string, number> = {};
-	for (const lease of leases) nodes[lease.nodeId] = (nodes[lease.nodeId] ?? 0) + 1;
-	return { global: leases.length, nodes };
+	const foreground = foregroundStreamUsage();
+	const endpoints: Record<string, number> = { ...foreground };
+	const endpointHolders: Record<string, EndpointCapacityHolders> = Object.fromEntries(
+		Object.entries(foreground).map(([endpointKey, foregroundStreams]) => [
+			endpointKey,
+			{ leases: 0, reservations: 0, foregroundStreams },
+		]),
+	);
+	for (const lease of leases) {
+		nodes[lease.nodeId] = (nodes[lease.nodeId] ?? 0) + 1;
+		if (lease.endpointKey !== undefined) {
+			endpoints[lease.endpointKey] = (endpoints[lease.endpointKey] ?? 0) + 1;
+			const holders = endpointHolders[lease.endpointKey] ?? { leases: 0, reservations: 0, foregroundStreams: 0 };
+			endpointHolders[lease.endpointKey] = { ...holders, leases: holders.leases + 1 };
+		}
+	}
+	return { global: leases.length, nodes, endpoints, endpointHolders };
 }
 
 /**