@iowarp/clio-coder 0.3.0 → 0.3.2

package/src/domains/dispatch/failure-classification.ts

@@ -0,0 +1,147 @@
+import { isResponseSchemaRejection } from "../../core/response-schema.js";
+import { WORKER_EXIT_PERMISSION_REQUIRED } from "../../worker/spec-contract.js";
+import { isDeterministicOutcomeCode } from "./backoff.js";
+import type { RunTerminationEvidence } from "./outcome.js";
+import type { RunOutcome, RunOutcomeCode } from "./types.js";
+import type { SpawnedWorkerResult } from "./worker-spawn.js";
+
+export type FailureClass =
+	| "operator-cancel"
+	| "policy"
+	| "permission"
+	| "deterministic-task"
+	| "model-quality"
+	| "target-auth"
+	| "target-rate-limit"
+	| "target-transient"
+	| "capacity"
+	| "node-channel"
+	| "node-resource"
+	| "worker-runtime"
+	| "internal";
+
+export type RoutePart = "agent" | "target" | "model" | "node" | "runtime";
+
+export interface RetryDecision {
+	retry: boolean;
+	retryAfterMs?: number;
+	excludedRouteParts: RoutePart[];
+	/** Typed authority to consider another agent; only model-quality evidence can carry it. */
+	qualityEscalation: null | { kind: "model-quality"; allowAgentChange: true };
+	reasonCode: string;
+}
+
+function resultText(result: SpawnedWorkerResult | null): string {
+	return result?.stderrTail?.toLowerCase() ?? "";
+}
+
+/** Classify coordinator-owned termination evidence without mutating routing state. */
+export function classifyFailure(
+	evidence: RunTerminationEvidence,
+	result: SpawnedWorkerResult | null,
+	outcome: RunOutcome,
+	code: RunOutcomeCode | null | undefined,
+): FailureClass {
+	if (evidence.abortedByOperator || outcome === "canceled") return "operator-cancel";
+	if (evidence.policyDenied !== null || outcome === "denied_by_policy") return "policy";
+	if (evidence.permissionFailure || evidence.exitCode === WORKER_EXIT_PERMISSION_REQUIRED) return "permission";
+	if (isDeterministicOutcomeCode(code)) return "deterministic-task";
+	if (evidence.qualityGateFailure === true) return "model-quality";
+	// A typed control-channel failure is node evidence on its own. It is checked
+	// before the diagnostic text so a stderr tail that happens to mention a
+	// target error cannot reclassify a channel that demonstrably failed.
+	if (result?.channelFailure !== undefined) return "node-channel";
+	if (evidence.stallKilled || outcome === "stalled" || outcome === "spawn_failed" || result?.exitCode === 255) {
+		return "node-channel";
+	}
+	const diagnostic = resultText(result);
+	// A response schema the server will not compile into a grammar is a verdict
+	// on the request Clio sent, not on the target. Retrying the identical bytes
+	// earns the identical 400, and letting it reach the target breaker parks a
+	// healthy endpoint for every other run in the window. Deterministic ends the
+	// attempt here and leaves the caller free to redispatch without the schema.
+	if (isResponseSchemaRejection(diagnostic)) return "deterministic-task";
+	// The worker rejected the specification the orchestrator wrote. A retry sends
+	// a document built the same way from the same settings and earns the same
+	// rejection, so three attempts only tripled the cost of one configuration
+	// error. Ending it here surfaces the contract message the operator has to act
+	// on instead of burying it under two more identical failures.
+	if (/\[worker\] fatal: workerspec/.test(diagnostic)) return "deterministic-task";
+	if (/\b(?:401|403)\b|unauthorized|forbidden|invalid api key|authentication/.test(diagnostic)) return "target-auth";
+	if (/\b429\b|rate[ -]?limit|too many requests/.test(diagnostic)) return "target-rate-limit";
+	if (/\bvram\b|\bgpu\b|\bcuda\b|\boom\b|out of memory/.test(diagnostic)) return "node-resource";
+	if (/capacity|overloaded|queue full/.test(diagnostic)) return "capacity";
+	if (evidence.timedOut || outcome === "timed_out" || /timeout|temporar|unavailable|\b50[234]\b/.test(diagnostic)) {
+		return "target-transient";
+	}
+	if (outcome === "failed") return "worker-runtime";
+	return "internal";
+}
+
+/** Pure bounded retry policy. attempt is the zero-based lineage attempt. */
+export function decideRetry(failureClass: FailureClass, attempt: number, maxRetries: number): RetryDecision {
+	const exhausted = maxRetries <= 0 || attempt >= maxRetries;
+	const base = (
+		excludedRouteParts: RoutePart[],
+		reasonCode: string,
+		qualityEscalation: RetryDecision["qualityEscalation"] = null,
+	): RetryDecision => ({
+		retry: !exhausted,
+		excludedRouteParts,
+		qualityEscalation,
+		reasonCode: exhausted ? "retry-exhausted" : reasonCode,
+	});
+
+	switch (failureClass) {
+		case "operator-cancel":
+		case "policy":
+		case "permission":
+		case "deterministic-task":
+			return {
+				retry: false,
+				excludedRouteParts: [],
+				qualityEscalation: null,
+				reasonCode: `non-retryable-${failureClass}`,
+			};
+		case "model-quality":
+			return base(["agent", "model"], "retry-model-quality", {
+				kind: "model-quality",
+				allowAgentChange: true,
+			});
+		case "node-channel":
+			return base(["node"], "retry-node-channel");
+		case "node-resource":
+			return base(["node"], "retry-node-resource");
+		case "target-auth":
+			return base(["target"], "retry-target-auth");
+		case "target-rate-limit": {
+			const decision = base(["target"], "retry-target-rate-limit");
+			return decision.retry ? { ...decision, retryAfterMs: 1_000 } : decision;
+		}
+		case "target-transient":
+			return base(["target"], "retry-target-transient");
+		case "capacity":
+			return base(["node"], "retry-capacity");
+		case "worker-runtime":
+			return base(["runtime"], "retry-worker-runtime");
+		case "internal":
+			return base([], "retry-internal");
+	}
+}
+
+export function affectsTargetBreaker(failureClass: FailureClass): boolean {
+	return (
+		failureClass === "target-auth" ||
+		failureClass === "target-rate-limit" ||
+		failureClass === "target-transient" ||
+		failureClass === "worker-runtime"
+	);
+}
+
+export function affectsNodeBreaker(failureClass: FailureClass): boolean {
+	return failureClass === "node-channel" || failureClass === "node-resource" || failureClass === "capacity";
+}
+
+export function isInfrastructureFailure(failureClass: FailureClass): boolean {
+	return affectsTargetBreaker(failureClass) || affectsNodeBreaker(failureClass) || failureClass === "internal";
+}

package/src/domains/dispatch/fleet-commit-attribution.ts

@@ -0,0 +1,56 @@
+import type { CommitAttributionEvidence } from "../../core/commit-attribution.js";
+import type { ExecutionPlan, ExecutionPlanCodeStep } from "./execution-plan.js";
+import { RUN_RECEIPT_INTEGRITY_ALGORITHM, RUN_RECEIPT_INTEGRITY_VERSION } from "./receipt-integrity.js";
+
+export interface FleetCommitPriorResult {
+	succeeded: boolean;
+	integrityValid: boolean;
+	receiptDigest: string;
+}
+
+/**
+ * Derive one fleet commit's semantic attribution from scheduler-owned facts.
+ * Agent output text is deliberately absent from this input: prose cannot turn
+ * into testing or review evidence.
+ */
+export function deriveFleetCommitAttribution(input: {
+	plan: ExecutionPlan;
+	step: ExecutionPlanCodeStep;
+	priorResults: ReadonlyMap<string, FleetCommitPriorResult>;
+	validationFresh: boolean;
+	independentReviewFresh: boolean;
+}): CommitAttributionEvidence {
+	const byId = new Map(input.plan.steps.map((step) => [step.id, step] as const));
+	let materiallyAuthored = false;
+	let receipt: CommitAttributionEvidence["receipt"];
+	for (const candidate of input.step.commitFrom ?? []) {
+		const source = byId.get(candidate);
+		const result = input.priorResults.get(candidate);
+		if (
+			source?.kind !== "agent" ||
+			source.scope !== "workspace" ||
+			result?.succeeded !== true ||
+			result.integrityValid !== true
+		) {
+			continue;
+		}
+		materiallyAuthored = true;
+		if (/^[0-9a-f]{64}$/u.test(result.receiptDigest)) {
+			receipt = {
+				version: RUN_RECEIPT_INTEGRITY_VERSION,
+				algorithm: RUN_RECEIPT_INTEGRITY_ALGORITHM,
+				digest: result.receiptDigest,
+				integrityValid: true,
+				directlyRelevant: true,
+			};
+		}
+		break;
+	}
+	return {
+		materiallyAssisted: materiallyAuthored,
+		materiallyAuthored,
+		validationSucceeded: input.validationFresh,
+		independentReviewPassed: input.independentReviewFresh,
+		...(receipt === undefined ? {} : { receipt }),
+	};
+}

package/src/domains/dispatch/fleet-plan.ts

@@ -0,0 +1,225 @@
+/**
+ * Fleet contract to execution plan.
+ *
+ * Bounded loops are unrolled here, at compile time, into statically declared
+ * conditional nodes: `maxAttempts` verifications and one fewer repair, chained
+ * so attempt `n + 1` only exists behind repair `n`. Nothing about the graph is
+ * decided while it runs, so the plan keeps the properties every other part of
+ * dispatch depends on: one deterministic hashed DAG, waves computed once,
+ * whole-plan admission and reservation before the first spawn, and a separate
+ * receipt for every attempt. The scheduler's only runtime decision is whether
+ * a declared node is still needed, which is a question about results rather
+ * than about structure.
+ *
+ * A dynamic loop executor would have to admit work mid-run, which is exactly
+ * the property fleet admission exists to deny.
+ */
+
+import {
+	type FleetContract,
+	type FleetContractStep,
+	type FleetStepScope,
+	fleetLoopCheckStepId,
+	fleetLoopRepairStepId,
+	fleetStepWriteBoundary,
+} from "../agents/fleet-contract.js";
+import type { ResultContract } from "../agents/result-contract.js";
+import type { AgentAutomationAuthority } from "../agents/spec.js";
+import {
+	compileExecutionPlan,
+	type ExecutionPlan,
+	type ExecutionPlanLoop,
+	type ExecutionPlanStepInput,
+} from "./execution-plan.js";
+import type { ExecutionRole } from "./execution-role.js";
+
+/** Where an agent node sits, so the caller can resolve its role and authority. */
+export interface FleetPlanAgentContext {
+	stepId: string;
+	agentId: string;
+	scope: "readonly" | "workspace";
+	/** Zero for declared work; a loop repair is attempt n and therefore recovery. */
+	attempt: number;
+	/** Set when this node is a loop's verification, which is a gate position. */
+	gateRole?: "reviewer";
+}
+
+export interface FleetPlanAgentResolution {
+	requestedAuthority: AgentAutomationAuthority;
+	/** Null makes the plan an approval request that cannot execute. */
+	approvedAuthority: AgentAutomationAuthority | null;
+	expectedResultContract: ResultContract["kind"];
+	executionRole: ExecutionRole;
+}
+
+export interface CompileFleetPlanInput {
+	contract: FleetContract;
+	/** Rendered prompt body; every agent node carries it as its task. */
+	task: string;
+	resolveAgent(context: FleetPlanAgentContext): FleetPlanAgentResolution;
+}
+
+/** Expand a declared dependency: a loop stands for all of its verifications. */
+function expandDependencies(
+	dependencies: ReadonlyArray<string>,
+	loopMembers: ReadonlyMap<string, ExecutionPlanLoop>,
+): string[] {
+	return dependencies.flatMap((id) => {
+		const loop = loopMembers.get(id);
+		return loop === undefined ? [id] : [...loop.checkStepIds];
+	});
+}
+
+/**
+ * Expand a commit's message sources into concrete node ids, most recent first.
+ * A loop stands for its repair attempts in reverse order, because the newest
+ * repair is the one whose words describe the tree being committed.
+ */
+function expandCommitSources(
+	sources: ReadonlyArray<string>,
+	loopMembers: ReadonlyMap<string, ExecutionPlanLoop>,
+): string[] {
+	return sources.flatMap((id) => {
+		const loop = loopMembers.get(id);
+		return loop === undefined ? [id] : [...loop.repairStepIds].reverse();
+	});
+}
+
+function declaredLoops(steps: ReadonlyArray<FleetContractStep>): Map<string, ExecutionPlanLoop> {
+	const loops = new Map<string, ExecutionPlanLoop>();
+	for (const step of steps) {
+		if (step.kind !== "loop") continue;
+		const checkStepIds: string[] = [];
+		const repairStepIds: string[] = [];
+		for (let attempt = 1; attempt <= step.maxAttempts; attempt++) {
+			checkStepIds.push(fleetLoopCheckStepId(step.id, attempt));
+			if (attempt < step.maxAttempts) repairStepIds.push(fleetLoopRepairStepId(step.id, attempt));
+		}
+		loops.set(step.id, {
+			id: step.id,
+			checkKind: step.check.kind,
+			maxAttempts: step.maxAttempts,
+			checkStepIds,
+			repairStepIds,
+		});
+	}
+	return loops;
+}
+
+export function compileFleetExecutionPlan(input: CompileFleetPlanInput): ExecutionPlan {
+	const { contract, task, resolveAgent } = input;
+	const loops = declaredLoops(contract.steps);
+	const steps: ExecutionPlanStepInput[] = [];
+
+	/**
+	 * The declared boundary of one position, carried onto every node the
+	 * position unrolls to. Each of a loop's attempts inherits the same
+	 * declaration: the bound changes how many times work may be retried, never
+	 * what it may touch.
+	 */
+	const boundary = (scope: FleetStepScope, writes: ReadonlyArray<string> | undefined): { writes?: string[] } => {
+		const resolved = fleetStepWriteBoundary(contract.version, scope, writes);
+		return resolved === undefined ? {} : { writes: [...resolved] };
+	};
+
+	const agentNode = (
+		context: FleetPlanAgentContext,
+		dependencies: ReadonlyArray<string>,
+		writes: ReadonlyArray<string> | undefined,
+		loop?: { loopId: string; role: "check" | "repair"; attempt: number },
+	): ExecutionPlanStepInput => {
+		const resolved = resolveAgent(context);
+		return {
+			kind: "agent",
+			id: context.stepId,
+			agentId: context.agentId,
+			scope: context.scope,
+			dependencies: [...dependencies],
+			task,
+			expectedResultContract: resolved.expectedResultContract,
+			requestedAuthority: resolved.requestedAuthority,
+			approvedAuthority: resolved.approvedAuthority,
+			executionRole: resolved.executionRole,
+			...boundary(context.scope, writes),
+			...(loop !== undefined ? { loop } : {}),
+		};
+	};
+
+	for (const step of contract.steps) {
+		const dependencies = expandDependencies(step.dependencies, loops);
+		if (step.kind === "agent") {
+			steps.push(
+				agentNode({ stepId: step.id, agentId: step.agent, scope: step.scope, attempt: 0 }, dependencies, step.writes),
+			);
+			continue;
+		}
+		if (step.kind === "code") {
+			steps.push({
+				kind: "code",
+				id: step.id,
+				commandId: step.command,
+				scope: step.scope,
+				dependencies,
+				...boundary(step.scope, step.writes),
+				...(step.commitFrom !== undefined ? { commitFrom: expandCommitSources(step.commitFrom, loops) } : {}),
+			});
+			continue;
+		}
+		const loop = loops.get(step.id);
+		if (loop === undefined) throw new Error(`fleet plan: loop '${step.id}' failed to compile`);
+		for (let attempt = 1; attempt <= step.maxAttempts; attempt++) {
+			const checkId = loop.checkStepIds[attempt - 1] ?? "";
+			const previousRepair = attempt === 1 ? null : (loop.repairStepIds[attempt - 2] ?? "");
+			const checkDependencies = previousRepair === null ? dependencies : [previousRepair];
+			const membership = { loopId: loop.id, role: "check" as const, attempt };
+			if (step.check.kind === "code") {
+				steps.push({
+					kind: "code",
+					id: checkId,
+					commandId: step.check.command,
+					scope: step.check.scope,
+					dependencies: checkDependencies,
+					...boundary(step.check.scope, step.check.writes),
+					loop: membership,
+					// The green a later step may rely on; the scheduler re-runs it
+					// when a workspace step lands after it.
+					verification: true,
+				});
+			} else {
+				steps.push(
+					agentNode(
+						{
+							stepId: checkId,
+							agentId: step.check.agent,
+							scope: step.check.scope,
+							attempt: 0,
+							gateRole: "reviewer",
+						},
+						checkDependencies,
+						step.check.writes,
+						membership,
+					),
+				);
+			}
+			if (attempt >= step.maxAttempts) continue;
+			const repairId = loop.repairStepIds[attempt - 1] ?? "";
+			steps.push(
+				agentNode(
+					{ stepId: repairId, agentId: step.repair.agent, scope: step.repair.scope, attempt },
+					[checkId],
+					step.repair.writes,
+					{ loopId: loop.id, role: "repair", attempt },
+				),
+			);
+		}
+	}
+
+	return compileExecutionPlan({
+		topology: "fleet",
+		rootTask: task,
+		maxWorkers: contract.maxWorkers,
+		onFailure: contract.onFailure,
+		steps,
+		loops: [...loops.values()],
+	});
+}