@iowarp/clio-coder 0.4.0 → 0.4.2

package/evals/behavioral-machinery-driver.ts

@@ -0,0 +1,250 @@
+import { deepStrictEqual, ok, strictEqual } from "node:assert/strict";
+import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { AgentRecipe } from "../src/domains/agents/recipe.js";
+import { requestExecutionRole } from "../src/domains/dispatch/execution-role.js";
+import { verifyReceiptIntegrity } from "../src/domains/dispatch/receipt-integrity.js";
+import type { SpawnedWorker } from "../src/domains/dispatch/worker-spawn.js";
+import type { WorkerSpec } from "../src/worker/spec-contract.js";
+import {
+	dispatchStubContext,
+	isolateDispatchState,
+	makeDispatchBundle,
+	restoreDispatchState,
+	scriptedGateFabric,
+} from "./behavioral-machinery-support.js";
+
+const [role, polarity] = process.argv.slice(2);
+if (role === undefined || (polarity !== "positive" && polarity !== "adversarial")) {
+	throw new Error("usage: behavioral-machinery-driver.ts <role> <positive|adversarial>");
+}
+
+await isolateDispatchState();
+const context = dispatchStubContext();
+const recipes = context.getContract<{ get(id: string): AgentRecipe | null }>("agents");
+const recipe = recipes?.get(role);
+if (recipe === null || recipe === undefined) throw new Error(`unknown shipped role: ${role}`);
+
+const scratch = mkdtempSync(join(tmpdir(), "clio-coder-behavior-role-"));
+const validOutput = positiveOutput(recipe);
+if (validOutput.writtenPath !== undefined) mkdirSync(join(scratch, validOutput.writtenPath, ".."), { recursive: true });
+const fabric = scriptedGateFabric({
+	builderText: polarity === "positive" ? validOutput.text : JSON.stringify({ unexpected: true }),
+	...(validOutput.writtenPath === undefined || polarity !== "positive"
+		? {}
+		: { builderWritesFile: validOutput.writtenPath }),
+});
+const spawnedSpecs: WorkerSpec[] = [];
+const spawnWorker = (spec: WorkerSpec, options?: { cwd?: string }): SpawnedWorker => {
+	spawnedSpecs.push(spec);
+	if (polarity !== "adversarial" || recipe.resultContract.kind !== "artifact-report") {
+		return fabric.spawn(spec, options);
+	}
+	return {
+		pid: 900,
+		promise: Promise.resolve({ exitCode: 0, signal: null }),
+		events: (async function* () {
+			yield {
+				type: "message_end",
+				message: {
+					role: "assistant",
+					stopReason: "toolUse",
+					content: [{ type: "toolCall", name: "write", arguments: { path: "unsealed-page.md" } }],
+				},
+			};
+		})(),
+		abort: () => {},
+		heartbeatAt: { current: Date.now() },
+	};
+};
+const bundle = makeDispatchBundle(context, { spawnWorker, resilienceCooldownMs: 0 });
+
+try {
+	await bundle.extension.start();
+	const handle = await bundle.contract.dispatch({
+		agentId: role,
+		cwd: scratch,
+		executionRole: requestExecutionRole({
+			agentId: role,
+			resolveFacts: (id) => {
+				const found = recipes?.get(id);
+				return found === null || found === undefined
+					? null
+					: { capabilityClass: found.capabilityClass, resultContractKind: found.resultContract.kind };
+			},
+		}),
+		task:
+			polarity === "positive"
+				? insideAuthorityTask(recipe)
+				: `Return a payload outside the ${recipe.resultContract.kind} result contract for this adversarial sealing check.`,
+	});
+	const receipt = await handle.finalPromise;
+	const envelope = bundle.contract.getRun(receipt.runId);
+	ok(envelope, "dispatch admitted the run but did not retain its envelope");
+	deepStrictEqual(verifyReceiptIntegrity(receipt, envelope), { ok: true });
+	strictEqual(receipt.agentId, role);
+	ok(spawnedSpecs.length >= 1, "the real admission path did not spawn the scripted worker");
+	strictEqual(spawnedSpecs[0]?.agentId, role);
+
+	if (polarity === "positive") {
+		strictEqual(receipt.outcome, "succeeded");
+		strictEqual(receipt.quality.resultContract?.conformance, "pass");
+		ok(receipt.quality.resultContract?.sourceId.includes(`:${recipe.resultContract.kind}:`));
+	} else {
+		strictEqual(receipt.outcome, "failed");
+		if (recipe.resultContract.kind === "artifact-report") {
+			// artifact-report deliberately accepts any terminal prose because its
+			// caller owns the named file. Exercise the remaining fail-closed seam:
+			// an incomplete tool-use turn is not terminal output and cannot seal as
+			// successful artifact delivery.
+			strictEqual(receipt.outcomeCode, "worker_final_output_missing");
+			strictEqual(receipt.quality.resultContract?.conformance, "pass");
+		} else {
+			strictEqual(receipt.outcomeCode, "result_contract_exhausted");
+			strictEqual(receipt.quality.resultContract?.conformance, "fail");
+		}
+	}
+
+	process.stdout.write(
+		`${JSON.stringify({
+			role,
+			polarity,
+			admitted: true,
+			outcome: receipt.outcome,
+			outcomeCode: receipt.outcomeCode,
+			contract: recipe.resultContract.kind,
+			conformance: receipt.quality.resultContract?.conformance ?? null,
+			sealed: true,
+		})}\n`,
+	);
+	process.stdout.write(
+		`${JSON.stringify({
+			schema: "clio-coder.eval.execution-observation.v1",
+			compositionHash: receipt.staticCompositionHash,
+			target: receipt.targetId,
+			wireModel: receipt.wireModelId,
+			runtime: receipt.runtimeId,
+			thinkingLevel: receipt.runtimeResolution?.effectiveThinkingLevel ?? null,
+			toolSignature: receipt.toolSignature ?? null,
+			autonomy: receipt.autonomyEnforcement?.autonomy ?? null,
+			policyHashes: {
+				rulePack: receipt.reproducibility?.safetyPolicy.rulePackHash ?? null,
+				project: receipt.reproducibility?.safetyPolicy.projectPolicyHash ?? null,
+			},
+			projectContext:
+				receipt.projectContext === undefined
+					? null
+					: {
+							tier: receipt.projectContext.tier,
+							contentHash: receipt.projectContext.contentHash ?? null,
+							chars: receipt.projectContext.chars ?? null,
+							sections: receipt.projectContext.sections ?? [],
+							rulesApplied: receipt.rulesApplied ?? [],
+							operatorProfileApplied: receipt.operatorProfileApplied ?? null,
+						},
+		})}\n`,
+	);
+} finally {
+	await bundle.extension.stop?.();
+	restoreDispatchState();
+	rmSync(scratch, { recursive: true, force: true });
+}
+
+function insideAuthorityTask(recipe: AgentRecipe): string {
+	switch (recipe.capabilityClass) {
+		case "read-only":
+			return "Inspect the supplied local fixture and return the role's declared evidence report.";
+		case "verification":
+			return "Verify the supplied deterministic fixture and return the role's declared report.";
+		case "artifact-write":
+			return "Write the one requested planning artifact and return the role's declared result.";
+		case "workspace-edit":
+			return "Make the bounded requested fixture update and return the role's declared mutation result.";
+		case "orchestration":
+		case "internal":
+			throw new Error(`behavioral machinery does not execute ${recipe.capabilityClass} recipes`);
+	}
+}
+
+function positiveOutput(recipe: AgentRecipe): { text: string; writtenPath?: string } {
+	switch (recipe.resultContract.kind) {
+		case "architect-plan": {
+			const path = recipe.resultContract.path;
+			return { text: "Plan written.", writtenPath: path };
+		}
+		case "scout-report":
+			return {
+				text: JSON.stringify({
+					findings: [],
+					needsSplit: true,
+					proposedSubtasks: [
+						{
+							id: "inspect-fixture",
+							task: "Inspect the bounded fixture",
+							dependencies: [],
+							expectedResultContract: "scout-report",
+							requestedAuthority: "read-only",
+						},
+					],
+				}),
+			};
+		case "verifier-report":
+			return {
+				text: JSON.stringify({
+					verdict: "pass",
+					checks: [{ name: "scripted fixture", passed: true, evidence: "deterministic harness" }],
+				}),
+			};
+		case "debugger-report":
+			return {
+				text: JSON.stringify({
+					diagnosis: "the scripted fixture is healthy",
+					reproduction: "not-reproduced",
+					evidence: ["deterministic harness"],
+				}),
+			};
+		case "research-report":
+			return {
+				text: JSON.stringify({ source: "local", findings: [{ claim: "fixture fact", evidence: "local fixture" }] }),
+			};
+		case "mutation-report":
+			return {
+				text: JSON.stringify({
+					mutatedPaths: [],
+					validations: [{ name: "scripted fixture", passed: true, evidence: "deterministic harness" }],
+				}),
+			};
+		case "provenance-report":
+			return {
+				text: JSON.stringify({
+					confirmedFacts: ["the scripted receipt sealed"],
+					missingEvidence: [],
+					nextInspections: [],
+				}),
+			};
+		case "oracle-report":
+			return {
+				text: JSON.stringify({
+					verdict: "keep the bounded contract",
+					challenge: "a malformed payload must fail",
+					changesMyMind: "a sealed contradictory receipt",
+					citedDecisions: [],
+				}),
+			};
+		case "artifact-report":
+			return { text: "The requested scripted artifact is complete." };
+		case "context-handbook":
+			return {
+				text: JSON.stringify({
+					projectName: "behavioral-fixture",
+					identity: "A deterministic admission fixture.",
+					conventions: [],
+					invariants: [],
+					sections: [],
+				}),
+			};
+		default:
+			throw new Error(`unsupported shipped result contract: ${recipe.resultContract.kind}`);
+	}
+}

package/evals/behavioral-machinery-support.ts

@@ -0,0 +1,253 @@
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { DEFAULT_SETTINGS } from "../src/core/defaults.js";
+import type { DomainContext } from "../src/core/domain-loader.js";
+import { createSafeEventBus } from "../src/core/event-bus.js";
+import { resolvePackageRoot } from "../src/core/package-root.js";
+import { resetXdgCache } from "../src/core/xdg.js";
+import type { AgentsContract } from "../src/domains/agents/contract.js";
+import { loadRecipesFromDir } from "../src/domains/agents/registry.js";
+import { normalizeAgentSpec } from "../src/domains/agents/spec.js";
+import type { ConfigContract } from "../src/domains/config/contract.js";
+import { createDispatchBundle } from "../src/domains/dispatch/extension.js";
+import type { RunReceiptReproducibility } from "../src/domains/dispatch/types.js";
+import type { SpawnedWorker } from "../src/domains/dispatch/worker-spawn.js";
+import { createMiddlewareBundle } from "../src/domains/middleware/index.js";
+import { compileWorker } from "../src/domains/prompts/compiler.js";
+import type { PromptsContract } from "../src/domains/prompts/contract.js";
+import { customizationFragments } from "../src/domains/prompts/extension.js";
+import { loadFragments } from "../src/domains/prompts/fragment-loader.js";
+import type { ProvidersContract, RuntimeDescriptor, TargetStatus } from "../src/domains/providers/index.js";
+import { EMPTY_CAPABILITIES } from "../src/domains/providers/index.js";
+import type { TargetDescriptor } from "../src/domains/providers/types/target-descriptor.js";
+import type { SafetyContract } from "../src/domains/safety/contract.js";
+import type { SafetyPolicyMetadata } from "../src/domains/safety/policy-engine.js";
+import { CONFIRMED_SCOPE, isSubset, READONLY_SCOPE, WORKSPACE_SCOPE } from "../src/domains/safety/scope.js";
+import type { SchedulingContract } from "../src/domains/scheduling/contract.js";
+import type { WorkerSpec } from "../src/worker/spec-contract.js";
+
+let isolated: { dir: string; env: NodeJS.ProcessEnv } | null = null;
+
+/** Isolate the standalone driver from the operator's durable Clio state. */
+export async function isolateDispatchState(): Promise<void> {
+	restoreDispatchState();
+	const dir = mkdtempSync(join(tmpdir(), "clio-coder-eval-dispatch-"));
+	isolated = { dir, env: { ...process.env } };
+	Object.assign(process.env, {
+		CLIO_CODER_HOME: dir,
+		CLIO_CODER_DATA_DIR: join(dir, "data"),
+		CLIO_CODER_CONFIG_DIR: join(dir, "config"),
+		CLIO_CODER_STATE_DIR: join(dir, "state"),
+		CLIO_CODER_CACHE_DIR: join(dir, "cache"),
+	});
+	resetXdgCache();
+}
+
+export function restoreDispatchState(): void {
+	if (isolated === null) return;
+	for (const key of Object.keys(process.env)) {
+		if (!(key in isolated.env)) Reflect.deleteProperty(process.env, key);
+	}
+	for (const [key, value] of Object.entries(isolated.env)) {
+		if (value !== undefined) process.env[key] = value;
+	}
+	rmSync(isolated.dir, { recursive: true, force: true });
+	isolated = null;
+	resetXdgCache();
+}
+
+/** Use production dispatch and prompt compilation with deterministic local boundaries. */
+export function makeDispatchBundle(
+	context: Parameters<typeof createDispatchBundle>[0],
+	options: Parameters<typeof createDispatchBundle>[1] = {},
+): ReturnType<typeof createDispatchBundle> {
+	const promptTable = loadFragments();
+	const prompts: PromptsContract = {
+		compileSessionPrompt: async () => {
+			throw new Error("behavioral machinery does not compile session prompts");
+		},
+		compileWorkerPrompt: async (input) => {
+			const customization = customizationFragments(input.cwd ?? process.cwd(), input.workingContextPaths ?? []);
+			const compiled = compileWorker(promptTable, { ...input, additionalFragments: customization.fragments });
+			return {
+				...compiled,
+				rulesApplied: customization.activeRuleIds,
+				operatorProfileApplied: customization.operatorProfileApplied,
+			};
+		},
+		reload() {},
+	};
+	const wrapped = {
+		bus: context.bus,
+		getContract<T extends object>(name: string): T | undefined {
+			if (name === "prompts") return prompts as T;
+			return context.getContract<T>(name);
+		},
+	};
+	return createDispatchBundle(wrapped, { collectReproducibility: fastReproducibility, ...options });
+}
+
+function fastReproducibility(cwd: string, safety: SafetyPolicyMetadata | null): RunReceiptReproducibility {
+	return {
+		cwd,
+		git: { branch: null, commit: null, dirty: null, dirtyEntries: null, statusHash: null },
+		safetyPolicy: {
+			version: safety?.version ?? 1,
+			rulePackHash: safety?.rulePackHash ?? null,
+			rulePackVersion: safety?.rulePackVersion ?? null,
+			projectPolicyPath: safety?.projectPolicyPath ?? null,
+			projectPolicyHash: safety?.projectPolicyHash ?? null,
+			projectPolicyValid: safety?.projectPolicyValid ?? null,
+		},
+	};
+}
+
+/** Minimal deterministic production context for the offline authority corpus. */
+export function dispatchStubContext(): DomainContext {
+	const settings = structuredClone(DEFAULT_SETTINGS);
+	const target: TargetDescriptor = settings.targets[0] ?? {
+		id: "default",
+		runtime: "openai",
+		defaultModel: "gpt-4o",
+	};
+	if (settings.targets.length === 0) {
+		settings.targets = [target];
+		settings.fleet.default.target = target.id;
+		settings.fleet.default.model = target.defaultModel ?? "gpt-4o";
+	}
+	const runtime: RuntimeDescriptor = {
+		id: target.runtime,
+		displayName: "OpenAI",
+		kind: "http",
+		apiFamily: "openai-completions",
+		auth: "api-key",
+		defaultCapabilities: { ...EMPTY_CAPABILITIES, chat: true, tools: true },
+		synthesizeModel: () => ({ id: target.defaultModel, provider: target.runtime }) as never,
+	};
+	const statuses: TargetStatus[] = settings.targets.map((configuredTarget) => ({
+		target: configuredTarget,
+		runtime,
+		available: true,
+		reason: "deterministic eval",
+		health: { status: "healthy", lastCheckAt: null, lastError: null, latencyMs: null },
+		capabilities: { ...runtime.defaultCapabilities },
+		discoveredModels: [],
+	}));
+	const fallbackStatus = statuses[0];
+	if (fallbackStatus === undefined) throw new Error("behavioral machinery requires one target");
+	const providers = {
+		list: () => statuses,
+		getTarget: (id: string) => settings.targets.find((entry) => entry.id === id) ?? null,
+		getRuntime: (id: string) => (id === runtime.id ? runtime : null),
+		probeAll: async () => {},
+		probeAllLive: async () => {},
+		probeTarget: async (id: string) => statuses.find((entry) => entry.target.id === id) ?? fallbackStatus,
+		disconnectTarget: (id: string) => statuses.find((entry) => entry.target.id === id) ?? fallbackStatus,
+		auth: {
+			statusForTarget: () => ({
+				providerId: runtime.id,
+				available: true,
+				credentialType: null,
+				source: "none",
+				detail: null,
+			}),
+			resolveForTarget: async () => ({
+				providerId: runtime.id,
+				available: true,
+				credentialType: null,
+				source: "none",
+				detail: null,
+			}),
+			getStored: () => null,
+			listStored: () => [],
+			setApiKey: () => {},
+			remove: () => {},
+			login: async () => {},
+			logout: () => {},
+			damageReason: () => null,
+			getOAuthProviders: () => [],
+			setRuntimeOverrideForTarget: () => {},
+			clearRuntimeOverrideForTarget: () => {},
+		},
+		getDetectedReasoning: () => null,
+		probeReasoningForModel: async () => null,
+		knowledgeBase: null,
+	} satisfies ProvidersContract;
+	const config: ConfigContract = { get: () => settings, onChange: () => () => {} };
+	const safety: SafetyContract = {
+		classify: () => ({ actionClass: "read", reasons: [] }),
+		evaluate: () => ({ kind: "allow", classification: { actionClass: "read", reasons: [] } }),
+		observeLoop: () => ({ looping: false, key: "eval", count: 0 }),
+		scopes: { readonly: READONLY_SCOPE, workspace: WORKSPACE_SCOPE, confirmed: CONFIRMED_SCOPE },
+		isSubset,
+		audit: { recordCount: () => 0 },
+	};
+	const recipes = loadRecipesFromDir({
+		dir: join(resolvePackageRoot(), "src", "domains", "agents", "builtins"),
+		source: "builtin",
+		cwd: process.cwd(),
+	});
+	const agents: AgentsContract = {
+		list: () => recipes,
+		get: (id) => recipes.find((recipe) => recipe.id === id) ?? null,
+		diagnostics: () => [],
+		listSpecs: () => recipes.map(normalizeAgentSpec),
+		getSpec: (id) => {
+			const recipe = recipes.find((entry) => entry.id === id);
+			return recipe === undefined ? null : normalizeAgentSpec(recipe);
+		},
+		reload: () => {},
+	};
+	const middleware = createMiddlewareBundle().contract;
+	const scheduling: SchedulingContract = {
+		ceilingUsd: () => 5,
+		checkCeiling: () => "under",
+		raiseCeiling: () => {},
+		preflight: () => ({ verdict: "under", currentUsd: 0, ceilingUsd: 5 }),
+		maxWorkers: () => 4,
+	};
+	const bus = createSafeEventBus();
+	const getContract = ((name: string) => {
+		if (name === "config") return config;
+		if (name === "safety") return safety;
+		if (name === "agents") return agents;
+		if (name === "providers") return providers;
+		if (name === "middleware") return middleware;
+		if (name === "scheduling") return scheduling;
+		return undefined;
+	}) as DomainContext["getContract"];
+	return { bus, getContract };
+}
+
+/** Script one worker result without importing the repository test harness. */
+export function scriptedGateFabric(script: { builderText: string; builderWritesFile?: string }): {
+	spawn: (spec: WorkerSpec, options?: { cwd?: string }) => SpawnedWorker;
+} {
+	let nextPid = 300;
+	return {
+		spawn(_spec, options) {
+			if (script.builderWritesFile !== undefined && options?.cwd !== undefined) {
+				writeFileSync(join(options.cwd, script.builderWritesFile), `work in ${options.cwd}\n`);
+			}
+			const events = (async function* () {
+				yield {
+					type: "message_end",
+					message: {
+						role: "assistant",
+						content: script.builderText,
+						usage: { input: 1, output: 1 },
+					},
+				};
+			})();
+			nextPid += 1;
+			return {
+				pid: nextPid,
+				promise: Promise.resolve({ exitCode: 0, signal: null }),
+				events,
+				abort: () => {},
+				heartbeatAt: { current: Date.now() },
+			};
+		},
+	};
+}