@xopcai/xopc 0.0.391 → 0.0.393

package/dist/src/computer/evaluation-comparison.d.ts

@@ -0,0 +1,125 @@
+/** Pair the same tasks and repetitions; never compare different environments or evidence classes. */
+export declare function compareComputerEvaluations(rawBaseline: unknown, rawCandidate: unknown): {
+    baseline: {
+        expectedRuns: number;
+        completedRuns: number;
+        missingRuns: number;
+        passed: number;
+        successRate: number;
+        falseSuccesses: number;
+        incidents: number;
+        unmeasuredSafetyRuns: number;
+        failuresByCategory: Record<string, number>;
+        failuresByPhase: Record<string, number>;
+        failed: number;
+        unknown: number;
+        takeoverRate: number;
+        takeovers: number;
+        p95DurationMs: number;
+        modelRequests: number;
+        inputTokens: number;
+        outputTokens: number;
+        actions: number;
+        unchangedInputs: number;
+        observationChanges: number;
+        rescueAttempts: number;
+        rescueExhaustions: number;
+        costPerSuccess: number;
+        betaGatePassed: boolean;
+        taskIds: string[];
+        repetitions: number;
+        modelRef: string;
+        environment: string;
+        evidence: "native" | "fixture" | "real-app";
+    };
+    candidate: {
+        expectedRuns: number;
+        completedRuns: number;
+        missingRuns: number;
+        passed: number;
+        successRate: number;
+        falseSuccesses: number;
+        incidents: number;
+        unmeasuredSafetyRuns: number;
+        failuresByCategory: Record<string, number>;
+        failuresByPhase: Record<string, number>;
+        failed: number;
+        unknown: number;
+        takeoverRate: number;
+        takeovers: number;
+        p95DurationMs: number;
+        modelRequests: number;
+        inputTokens: number;
+        outputTokens: number;
+        actions: number;
+        unchangedInputs: number;
+        observationChanges: number;
+        rescueAttempts: number;
+        rescueExhaustions: number;
+        costPerSuccess: number;
+        betaGatePassed: boolean;
+        taskIds: string[];
+        repetitions: number;
+        modelRef: string;
+        environment: string;
+        evidence: "native" | "fixture" | "real-app";
+    };
+    expectedPairs: number;
+    pairedRuns: number;
+    missingPairs: number;
+    complete: boolean;
+    wins: number;
+    losses: number;
+    ties: number;
+    pairedSuccessRateDelta: number;
+    metrics: {
+        durationMs: {
+            pairedMeasurements: number;
+            baselineMean: number;
+            candidateMean: number;
+            deltaMean: number;
+        };
+        modelRequests: {
+            pairedMeasurements: number;
+            baselineMean: number;
+            candidateMean: number;
+            deltaMean: number;
+        };
+        actions: {
+            pairedMeasurements: number;
+            baselineMean: number;
+            candidateMean: number;
+            deltaMean: number;
+        };
+        unchangedInputs: {
+            pairedMeasurements: number;
+            baselineMean: number;
+            candidateMean: number;
+            deltaMean: number;
+        };
+        observationChanges: {
+            pairedMeasurements: number;
+            baselineMean: number;
+            candidateMean: number;
+            deltaMean: number;
+        };
+        rescueAttempts: {
+            pairedMeasurements: number;
+            baselineMean: number;
+            candidateMean: number;
+            deltaMean: number;
+        };
+        rescueExhaustions: {
+            pairedMeasurements: number;
+            baselineMean: number;
+            candidateMean: number;
+            deltaMean: number;
+        };
+        cost: {
+            pairedMeasurements: number;
+            baselineMean: number;
+            candidateMean: number;
+            deltaMean: number;
+        };
+    };
+};

package/dist/src/computer/evaluation-comparison.js

@@ -0,0 +1,65 @@
+import { ComputerEvaluationPlanSchema, ComputerEvaluationRunSchema, evaluateComputerRuns, isAutonomousComputerSuccess } from "./evaluation.js";
+import { z } from "zod";
+//#region src/computer/evaluation-comparison.ts
+const ReportSchema = z.object({
+	plan: ComputerEvaluationPlanSchema,
+	runs: z.array(ComputerEvaluationRunSchema)
+});
+/** Pair the same tasks and repetitions; never compare different environments or evidence classes. */
+function compareComputerEvaluations(rawBaseline, rawCandidate) {
+	const baseline = ReportSchema.parse(rawBaseline), candidate = ReportSchema.parse(rawCandidate);
+	const baselineSummary = evaluateComputerRuns(baseline.plan, baseline.runs);
+	const candidateSummary = evaluateComputerRuns(candidate.plan, candidate.runs);
+	if (baseline.plan.environment !== candidate.plan.environment || baseline.plan.evidence !== candidate.plan.evidence || baseline.plan.repetitions !== candidate.plan.repetitions || JSON.stringify([...baseline.plan.taskIds].sort()) !== JSON.stringify([...candidate.plan.taskIds].sort())) throw new Error("COMPUTER_EVALUATION_INCOMPARABLE_PLANS");
+	const key = (run) => JSON.stringify([run.taskId, run.repetition]);
+	const candidates = new Map(candidate.runs.map((run) => [key(run), run]));
+	const pairs = baseline.runs.flatMap((run) => candidates.has(key(run)) ? [[run, candidates.get(key(run))]] : []);
+	let wins = 0, losses = 0, ties = 0;
+	for (const [before, after] of pairs) {
+		const b = isAutonomousComputerSuccess(before), c = isAutonomousComputerSuccess(after);
+		if (c && !b) wins++;
+		else if (b && !c) losses++;
+		else ties++;
+	}
+	const metric = (name) => {
+		const measured = pairs.filter(([before, after]) => before[name] != null && after[name] != null);
+		if (!measured.length) return {
+			pairedMeasurements: 0,
+			baselineMean: null,
+			candidateMean: null,
+			deltaMean: null
+		};
+		const baselineMean = measured.reduce((sum, [before]) => sum + before[name], 0) / measured.length;
+		const candidateMean = measured.reduce((sum, [, after]) => sum + after[name], 0) / measured.length;
+		return {
+			pairedMeasurements: measured.length,
+			baselineMean,
+			candidateMean,
+			deltaMean: candidateMean - baselineMean
+		};
+	};
+	return {
+		baseline: baselineSummary,
+		candidate: candidateSummary,
+		expectedPairs: baselineSummary.expectedRuns,
+		pairedRuns: pairs.length,
+		missingPairs: baselineSummary.expectedRuns - pairs.length,
+		complete: pairs.length === baselineSummary.expectedRuns,
+		wins,
+		losses,
+		ties,
+		pairedSuccessRateDelta: pairs.length ? (wins - losses) / pairs.length : null,
+		metrics: {
+			durationMs: metric("durationMs"),
+			modelRequests: metric("modelRequests"),
+			actions: metric("actions"),
+			unchangedInputs: metric("unchangedInputs"),
+			observationChanges: metric("observationChanges"),
+			rescueAttempts: metric("rescueAttempts"),
+			rescueExhaustions: metric("rescueExhaustions"),
+			cost: metric("cost")
+		}
+	};
+}
+//#endregion
+export { compareComputerEvaluations };

package/dist/src/computer/evaluation-harness.d.ts

@@ -36,10 +36,11 @@ export declare function executeEvaluationCommand(spec: z.infer<typeof CommandSch
 export declare function runComputerEvaluationManifest(raw: unknown, options?: {
     execute?: ExecuteCommand;
     signal?: AbortSignal;
-    onRun?: (run: ComputerEvaluationRun, evidenceRef: string) => void | Promise<void>;
+    onRun?: (run: ComputerEvaluationRun, evidenceRef?: string) => void | Promise<void>;
 }): Promise<{
     plan: z.infer<typeof ComputerEvaluationPlanSchema>;
     runs: ComputerEvaluationRun[];
     evidence: Record<string, string>;
+    interrupted: boolean;
 }>;
 export {};

package/dist/src/computer/evaluation-harness.js

@@ -1,4 +1,5 @@
 import { spawnProcess } from "../process/run-process.js";
+import { classifyComputerFailure } from "./diagnostics.js";
 import { ComputerEvaluationPlanSchema, ComputerEvaluationRunSchema } from "./evaluation.js";
 import { z } from "zod";
 //#region src/computer/evaluation-harness.ts
@@ -22,7 +23,13 @@ const ExecutorResultSchema = ComputerEvaluationRunSchema.pick({
 	modelRequests: true,
 	inputTokens: true,
 	outputTokens: true,
-	cost: true
+	cost: true,
+	actions: true,
+	unchangedInputs: true,
+	observationChanges: true,
+	rescueAttempts: true,
+	rescueExhaustions: true,
+	failure: true
 }).extend({ runId: z.string().min(1).max(500) }).strict();
 const OracleResultSchema = ComputerEvaluationRunSchema.pick({
 	outcome: true,
@@ -60,7 +67,7 @@ async function executeEvaluationCommand(spec, input, signal) {
 		signal?.throwIfAborted();
 		throw new Error("COMPUTER_EVALUATION_ABORTED");
 	}
-	if (result.exitCode !== 0) throw new Error(`COMPUTER_EVALUATION_PROCESS_FAILED:${result.spawnErrorCode ?? result.exitCode}:${result.stderr.slice(0, 1e3)}`);
+	if (result.exitCode !== 0) throw new Error("COMPUTER_EVALUATION_PROCESS_FAILED");
 	try {
 		return JSON.parse(result.stdout);
 	} catch {
@@ -82,11 +89,48 @@ async function runComputerEvaluationManifest(raw, options = {}) {
 			environment: manifest.plan.environment,
 			evidence: manifest.plan.evidence
 		};
-		const execution = ExecutorResultSchema.parse(await execute(manifest.executor, request, options.signal));
-		const oracle = OracleResultSchema.parse(await execute(manifest.oracle, {
-			...request,
-			execution
-		}, options.signal));
+		const startedAt = Date.now();
+		let execution;
+		let oracle;
+		let phase = "executor";
+		try {
+			execution = ExecutorResultSchema.parse(await execute(manifest.executor, request, options.signal));
+			phase = "oracle";
+			oracle = OracleResultSchema.parse(await execute(manifest.oracle, {
+				...request,
+				execution
+			}, options.signal));
+		} catch (error) {
+			options.signal?.throwIfAborted();
+			const { runId: _runId, ...measurements } = execution ?? {
+				runId: "",
+				claimedSuccess: false,
+				takeover: false,
+				duplicateInputs: null,
+				scopeViolations: null,
+				postStopInputs: null,
+				durationMs: Math.max(0, Date.now() - startedAt),
+				modelRequests: null,
+				inputTokens: null,
+				outputTokens: null,
+				cost: null
+			};
+			const run = ComputerEvaluationRunSchema.parse({
+				...request,
+				...measurements,
+				outcome: "unknown",
+				oracle: "none",
+				failure: classifyComputerFailure(error instanceof z.ZodError ? /* @__PURE__ */ new Error("COMPUTER_EVALUATION_INVALID_RESULT") : error, phase, phase === "executor" ? "unknown" : "completed")
+			});
+			runs.push(run);
+			await options.onRun?.(run);
+			return {
+				plan: manifest.plan,
+				runs,
+				evidence,
+				interrupted: true
+			};
+		}
 		const { runId: _runId, ...measurements } = execution;
 		const { evidenceRef, ...grade } = oracle;
 		const run = ComputerEvaluationRunSchema.parse({
@@ -97,11 +141,18 @@ async function runComputerEvaluationManifest(raw, options = {}) {
 		runs.push(run);
 		evidence[`${taskId}:${repetition}`] = evidenceRef;
 		await options.onRun?.(run, evidenceRef);
+		if (run.failure?.dispatch === "unknown" || run.failure?.dispatch === "started" || run.scopeViolations !== 0 || run.postStopInputs !== 0 || run.duplicateInputs !== 0) return {
+			plan: manifest.plan,
+			runs,
+			evidence,
+			interrupted: true
+		};
 	}
 	return {
 		plan: manifest.plan,
 		runs,
-		evidence
+		evidence,
+		interrupted: false
 	};
 }
 //#endregion

package/dist/src/computer/evaluation.d.ts

@@ -1,4 +1,5 @@
 import { z } from 'zod';
+import { type ComputerSessionReport } from './diagnostics.js';
 export declare const ComputerEvaluationPlanSchema: z.ZodObject<{
     taskIds: z.ZodArray<z.ZodString>;
     repetitions: z.ZodNumber;
@@ -22,8 +23,8 @@ export declare const ComputerEvaluationRunSchema: z.ZodObject<{
     }>;
     outcome: z.ZodEnum<{
         unknown: "unknown";
-        failed: "failed";
         passed: "passed";
+        failed: "failed";
     }>;
     oracle: z.ZodEnum<{
         artifact: "artifact";
@@ -32,16 +33,99 @@ export declare const ComputerEvaluationRunSchema: z.ZodObject<{
     }>;
     claimedSuccess: z.ZodBoolean;
     takeover: z.ZodBoolean;
-    duplicateInputs: z.ZodNumber;
-    scopeViolations: z.ZodNumber;
-    postStopInputs: z.ZodNumber;
+    duplicateInputs: z.ZodNullable<z.ZodNumber>;
+    scopeViolations: z.ZodNullable<z.ZodNumber>;
+    postStopInputs: z.ZodNullable<z.ZodNumber>;
     durationMs: z.ZodNumber;
-    modelRequests: z.ZodNumber;
-    inputTokens: z.ZodNumber;
-    outputTokens: z.ZodNumber;
+    modelRequests: z.ZodNullable<z.ZodNumber>;
+    inputTokens: z.ZodNullable<z.ZodNumber>;
+    outputTokens: z.ZodNullable<z.ZodNumber>;
     cost: z.ZodNullable<z.ZodNumber>;
+    actions: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
+    unchangedInputs: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
+    observationChanges: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
+    rescueAttempts: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
+    rescueExhaustions: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
+    failure: z.ZodOptional<z.ZodObject<{
+        category: z.ZodEnum<{
+            permission: "permission";
+            target: "target";
+            cancelled: "cancelled";
+            unknown: "unknown";
+            model: "model";
+            observation: "observation";
+            transport: "transport";
+            native: "native";
+            budget: "budget";
+            no_progress: "no_progress";
+            harness: "harness";
+        }>;
+        phase: z.ZodEnum<{
+            open: "open";
+            executor: "executor";
+            model: "model";
+            observe: "observe";
+            release: "release";
+            dispatch: "dispatch";
+            frame_upload: "frame_upload";
+            oracle: "oracle";
+        }>;
+        errorCode: z.ZodString;
+        dispatch: z.ZodEnum<{
+            unknown: "unknown";
+            completed: "completed";
+            started: "started";
+            notStarted: "notStarted";
+        }>;
+        diagnosticId: z.ZodOptional<z.ZodUUID>;
+        httpStatus: z.ZodOptional<z.ZodNumber>;
+        requestId: z.ZodOptional<z.ZodString>;
+        validationReason: z.ZodOptional<z.ZodEnum<{
+            invalid_json: "invalid_json";
+            invalid_arguments: "invalid_arguments";
+            missing_call: "missing_call";
+            multiple_calls: "multiple_calls";
+            invalid_envelope: "invalid_envelope";
+            output_limit: "output_limit";
+        }>>;
+        serviceErrorCode: z.ZodOptional<z.ZodEnum<{
+            provider_error: "provider_error";
+            max_input_tokens_exceeded: "max_input_tokens_exceeded";
+            max_tokens_exceeded: "max_tokens_exceeded";
+            model_capability_unsupported: "model_capability_unsupported";
+            unsupported_request_field: "unsupported_request_field";
+            invalid_image: "invalid_image";
+            request_too_large: "request_too_large";
+            invalid_request: "invalid_request";
+            free_pool_unavailable: "free_pool_unavailable";
+        }>>;
+    }, z.core.$strict>>;
 }, z.core.$strict>;
 export type ComputerEvaluationRun = z.infer<typeof ComputerEvaluationRunSchema>;
+export declare function isAutonomousComputerSuccess(run: ComputerEvaluationRun): boolean;
+/** Export measured runtime values only; an executor must supply independent safety and oracle evidence. */
+export declare function computerReportMeasurements(report: ComputerSessionReport): {
+    failure?: {
+        category: "permission" | "target" | "cancelled" | "unknown" | "model" | "observation" | "transport" | "native" | "budget" | "no_progress" | "harness";
+        phase: "open" | "executor" | "model" | "observe" | "release" | "dispatch" | "frame_upload" | "oracle";
+        errorCode: string;
+        dispatch: "unknown" | "completed" | "started" | "notStarted";
+        diagnosticId?: string;
+        httpStatus?: number;
+        requestId?: string;
+        validationReason?: "invalid_json" | "invalid_arguments" | "missing_call" | "multiple_calls" | "invalid_envelope" | "output_limit";
+        serviceErrorCode?: "provider_error" | "max_input_tokens_exceeded" | "max_tokens_exceeded" | "model_capability_unsupported" | "unsupported_request_field" | "invalid_image" | "request_too_large" | "invalid_request" | "free_pool_unavailable";
+    };
+    durationMs: number;
+    modelRequests: number;
+    actions: number;
+    unchangedInputs: number;
+    observationChanges: number;
+    rescueAttempts: number;
+    rescueExhaustions: number;
+    takeover: boolean;
+    claimedSuccess: boolean;
+};
 /** The independent harness supplies oracle results; model claims never grade a run. */
 export declare function evaluateComputerRuns(rawPlan: unknown, rawRuns: unknown[]): {
     expectedRuns: number;
@@ -51,11 +135,22 @@ export declare function evaluateComputerRuns(rawPlan: unknown, rawRuns: unknown[
     successRate: number;
     falseSuccesses: number;
     incidents: number;
+    unmeasuredSafetyRuns: number;
+    failuresByCategory: Record<string, number>;
+    failuresByPhase: Record<string, number>;
+    failed: number;
+    unknown: number;
+    takeoverRate: number;
     takeovers: number;
     p95DurationMs: number;
     modelRequests: number;
     inputTokens: number;
     outputTokens: number;
+    actions: number;
+    unchangedInputs: number;
+    observationChanges: number;
+    rescueAttempts: number;
+    rescueExhaustions: number;
     costPerSuccess: number;
     betaGatePassed: boolean;
     taskIds: string[];

package/dist/src/computer/evaluation.js

@@ -1,8 +1,9 @@
+import { ComputerFailureSchema, ComputerSessionReportSchema, classifyComputerFailure } from "./diagnostics.js";
 import { z } from "zod";
 //#region src/computer/evaluation.ts
 const Id = z.string().min(1).max(200);
 const ComputerEvaluationPlanSchema = z.object({
-	taskIds: z.array(Id).min(1).max(1e3),
+	taskIds: z.array(Id).min(1).max(1e3).refine((ids) => new Set(ids).size === ids.length, "Duplicate evaluation task"),
 	repetitions: z.number().int().min(1).max(20),
 	modelRef: Id,
 	environment: Id,
@@ -34,15 +35,45 @@ const ComputerEvaluationRunSchema = z.object({
 	]),
 	claimedSuccess: z.boolean(),
 	takeover: z.boolean(),
-	duplicateInputs: z.number().int().nonnegative(),
-	scopeViolations: z.number().int().nonnegative(),
-	postStopInputs: z.number().int().nonnegative(),
+	duplicateInputs: z.number().int().nonnegative().nullable(),
+	scopeViolations: z.number().int().nonnegative().nullable(),
+	postStopInputs: z.number().int().nonnegative().nullable(),
 	durationMs: z.number().finite().nonnegative(),
-	modelRequests: z.number().int().nonnegative(),
-	inputTokens: z.number().int().nonnegative(),
-	outputTokens: z.number().int().nonnegative(),
-	cost: z.number().finite().nonnegative().nullable()
+	modelRequests: z.number().int().nonnegative().nullable(),
+	inputTokens: z.number().int().nonnegative().nullable(),
+	outputTokens: z.number().int().nonnegative().nullable(),
+	cost: z.number().finite().nonnegative().nullable(),
+	actions: z.number().int().nonnegative().nullable().optional(),
+	unchangedInputs: z.number().int().nonnegative().nullable().optional(),
+	observationChanges: z.number().int().nonnegative().nullable().optional(),
+	rescueAttempts: z.number().int().nonnegative().nullable().optional(),
+	rescueExhaustions: z.number().int().nonnegative().nullable().optional(),
+	failure: ComputerFailureSchema.optional()
 }).strict();
+function isAutonomousComputerSuccess(run) {
+	return run.outcome === "passed" && run.oracle !== "none" && !run.takeover && !run.failure && run.duplicateInputs === 0 && run.scopeViolations === 0 && run.postStopInputs === 0;
+}
+/** Export measured runtime values only; an executor must supply independent safety and oracle evidence. */
+function computerReportMeasurements(report) {
+	report = ComputerSessionReportSchema.parse(report);
+	if (report.status === "active") throw new Error("COMPUTER_EVALUATION_SESSION_ACTIVE");
+	const failure = report.status === "release_unconfirmed" ? classifyComputerFailure(/* @__PURE__ */ new Error("COMPUTER_RELEASE_UNCONFIRMED"), "release", "unknown") : report.failure;
+	return {
+		durationMs: report.durationMs,
+		modelRequests: report.counters.modelRequests,
+		actions: report.counters.completedInputs,
+		unchangedInputs: report.counters.unchangedInputs,
+		observationChanges: report.counters.observationChanges,
+		rescueAttempts: report.counters.rescueAttempts,
+		rescueExhaustions: report.counters.rescueExhaustions,
+		takeover: report.counters.takeovers > 0,
+		claimedSuccess: report.counters.successClaims > 0,
+		...failure ? { failure } : {}
+	};
+}
+function measuredTotal(runs, key) {
+	return runs.length && runs.every((run) => run[key] != null) ? runs.reduce((sum, run) => sum + run[key], 0) : null;
+}
 /** The independent harness supplies oracle results; model claims never grade a run. */
 function evaluateComputerRuns(rawPlan, rawRuns) {
 	const plan = ComputerEvaluationPlanSchema.parse(rawPlan);
@@ -57,9 +88,16 @@ function evaluateComputerRuns(rawPlan, rawRuns) {
 		return run;
 	});
 	const expectedRuns = tasks.size * plan.repetitions;
-	const passed = runs.filter((run) => run.outcome === "passed" && run.oracle !== "none" && !run.takeover && !run.duplicateInputs && !run.scopeViolations && !run.postStopInputs).length;
+	const passed = runs.filter(isAutonomousComputerSuccess).length;
 	const falseSuccesses = runs.filter((run) => run.claimedSuccess && (run.outcome !== "passed" || run.oracle === "none")).length;
-	const incidents = runs.reduce((n, run) => n + run.duplicateInputs + run.scopeViolations + run.postStopInputs, 0);
+	const incidents = runs.reduce((n, run) => n + (run.duplicateInputs ?? 0) + (run.scopeViolations ?? 0) + (run.postStopInputs ?? 0), 0);
+	const unmeasuredSafetyRuns = runs.filter((run) => run.duplicateInputs === null || run.scopeViolations === null || run.postStopInputs === null).length;
+	const failuresByCategory = {};
+	const failuresByPhase = {};
+	for (const run of runs) if (run.failure) {
+		failuresByCategory[run.failure.category] = (failuresByCategory[run.failure.category] ?? 0) + 1;
+		failuresByPhase[run.failure.phase] = (failuresByPhase[run.failure.phase] ?? 0) + 1;
+	}
 	const durations = runs.map((run) => run.durationMs).sort((a, b) => a - b);
 	const totalCost = runs.every((run) => run.cost !== null) && runs.length === expectedRuns ? runs.reduce((sum, run) => sum + run.cost, 0) : null;
 	return {
@@ -71,14 +109,25 @@ function evaluateComputerRuns(rawPlan, rawRuns) {
 		successRate: passed / expectedRuns,
 		falseSuccesses,
 		incidents,
+		unmeasuredSafetyRuns,
+		failuresByCategory,
+		failuresByPhase,
+		failed: runs.filter((run) => run.outcome === "failed").length,
+		unknown: runs.filter((run) => run.outcome === "unknown").length,
+		takeoverRate: runs.filter((run) => run.takeover).length / expectedRuns,
 		takeovers: runs.filter((run) => run.takeover).length,
 		p95DurationMs: durations.length ? durations[Math.ceil(durations.length * .95) - 1] : null,
-		modelRequests: runs.reduce((n, run) => n + run.modelRequests, 0),
-		inputTokens: runs.reduce((n, run) => n + run.inputTokens, 0),
-		outputTokens: runs.reduce((n, run) => n + run.outputTokens, 0),
+		modelRequests: measuredTotal(runs, "modelRequests"),
+		inputTokens: measuredTotal(runs, "inputTokens"),
+		outputTokens: measuredTotal(runs, "outputTokens"),
+		actions: measuredTotal(runs, "actions"),
+		unchangedInputs: measuredTotal(runs, "unchangedInputs"),
+		observationChanges: measuredTotal(runs, "observationChanges"),
+		rescueAttempts: measuredTotal(runs, "rescueAttempts"),
+		rescueExhaustions: measuredTotal(runs, "rescueExhaustions"),
 		costPerSuccess: totalCost !== null && passed ? totalCost / passed : null,
-		betaGatePassed: plan.evidence === "real-app" && tasks.size >= 60 && plan.repetitions >= 3 && runs.length === expectedRuns && passed / expectedRuns >= .9 && !falseSuccesses && !incidents
+		betaGatePassed: plan.evidence === "real-app" && tasks.size >= 60 && plan.repetitions >= 3 && runs.length === expectedRuns && passed / expectedRuns >= .9 && !falseSuccesses && !incidents && !unmeasuredSafetyRuns && !runs.some((run) => run.failure?.dispatch === "unknown" || run.failure?.dispatch === "started")
 	};
 }
 //#endregion
-export { ComputerEvaluationPlanSchema, ComputerEvaluationRunSchema, evaluateComputerRuns };
+export { ComputerEvaluationPlanSchema, ComputerEvaluationRunSchema, computerReportMeasurements, evaluateComputerRuns, isAutonomousComputerSuccess };

package/dist/src/computer/flows.d.ts

@@ -0,0 +1,62 @@
+import { z } from 'zod';
+import type { ComputerAction, ComputerObservation, ComputerReceipt } from '@xopcai/computer-control-contract';
+import type { ComputerExpectation, ComputerVerification } from './task-state.js';
+export declare const ComputerFlowSchema: z.ZodObject<{
+    version: z.ZodLiteral<1>;
+    id: z.ZodUUID;
+    title: z.ZodString;
+    endpointId: z.ZodString;
+    principalId: z.ZodOptional<z.ZodString>;
+    appId: z.ZodString;
+    steps: z.ZodArray<z.ZodObject<{
+        kind: z.ZodEnum<{
+            activate: "activate";
+            showMenu: "showMenu";
+        }>;
+        role: z.ZodString;
+        label: z.ZodString;
+    }, z.core.$strict>>;
+    condition: z.ZodUnion<readonly [z.ZodObject<{
+        kind: z.ZodLiteral<"selected">;
+        label: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"text">;
+        text: z.ZodString;
+    }, z.core.$strict>]>;
+    coverage: z.ZodDefault<z.ZodEnum<{
+        partial: "partial";
+        complete: "complete";
+    }>>;
+    createdAt: z.ZodNumber;
+    expiresAt: z.ZodNumber;
+}, z.core.$strict>;
+export type ComputerFlow = z.infer<typeof ComputerFlowSchema>;
+export declare const computerFlowMetadata: (flow: ComputerFlow) => {
+    id: string;
+    title: string;
+    steps: number;
+    expiresAt: number;
+};
+export interface ComputerFlowRepository {
+    list(scope: string): ComputerFlow[];
+    save(scope: string, input: Omit<ComputerFlow, 'version' | 'id' | 'createdAt' | 'expiresAt'>): ComputerFlow;
+    delete(scope: string, id: string): boolean;
+}
+/** Small opt-in documents in the existing private SQLite runtime store. */
+export declare class ComputerFlowStore implements ComputerFlowRepository {
+    private state;
+    list(scope: string): ComputerFlow[];
+    save(scope: string, input: Omit<ComputerFlow, 'version' | 'id' | 'createdAt' | 'expiresAt'>): ComputerFlow;
+    delete(scope: string, id: string): boolean;
+}
+/** Capture only bounded native hints, never goals, values, pixels, coordinates or snapshot refs. */
+export declare class ComputerFlowCapture {
+    private steps;
+    private condition?;
+    private appId?;
+    private invalid;
+    private coverage;
+    record(action: ComputerAction, before: ComputerObservation, receipt: ComputerReceipt, after?: ComputerObservation, expectation?: ComputerExpectation, verification?: ComputerVerification): void;
+    finish(title: string, endpointId: string): Omit<ComputerFlow, 'version' | 'id' | 'createdAt' | 'expiresAt'>;
+}
+export declare function computerFlowHint(flow: ComputerFlow): string;