@velum-labs/routekit-eval-setup 1.4.0 → 1.5.0

package/dist/eval-capability-execution-envelope.d.ts

@@ -0,0 +1,64 @@
+import type { AgenticCapabilityNameV1 } from "./agentic-capabilities-protocol.js";
+import { type EvalCapabilityScientificPhaseV1 } from "./eval-capability-policy.js";
+export type EvalCapabilityScientificReserveV1 = Readonly<{
+    calls: number;
+    wallTimeMs: number;
+    attempts: number;
+}>;
+export type EvalCapabilityExecutionBudgetV1 = Readonly<{
+    calls: number;
+    wallTimeMs: number;
+}>;
+/**
+ * Time kept outside scientific execution for checkpoint/archive persistence
+ * and the host's durable capability-completion receipt.
+ */
+export declare const EVAL_CAPABILITY_DURABLE_FINALIZATION_RESERVE_MS_V1 = 30000;
+export type EvalCapabilityExecutionEnvelopeDecisionV1 = Readonly<{
+    version: 1;
+    capability: AgenticCapabilityNameV1;
+    phase: EvalCapabilityScientificPhaseV1 | null;
+    available: EvalCapabilityExecutionBudgetV1;
+    requestedMaximum: EvalCapabilityExecutionBudgetV1;
+    downstreamReserve: EvalCapabilityScientificReserveV1;
+    finalizationReserve: Readonly<{
+        wallTimeMs: number;
+    }>;
+    minimumRequired: EvalCapabilityExecutionBudgetV1;
+    availableForCapability: EvalCapabilityExecutionBudgetV1;
+}> & (Readonly<{
+    admitted: true;
+    envelope: Readonly<{
+        maximumModelCalls: number;
+        maximumWallTimeMs: number;
+    }>;
+}> | Readonly<{
+    admitted: false;
+    dimension: "calls" | "wall_time";
+    envelope: null;
+}>);
+/**
+ * The immutable downstream reserve for a scientific capability. This is the
+ * single package-owned source used by admission and execution; campaign-size
+ * ratios must not weaken these fixed phase reserves.
+ */
+export declare const evalCapabilityScientificDownstreamReserveV1: (capability: AgenticCapabilityNameV1) => Readonly<{
+    phase: EvalCapabilityScientificPhaseV1 | null;
+    reserve: EvalCapabilityScientificReserveV1;
+}>;
+/**
+ * Computes the positive execution slice that fits after preserving the fixed
+ * downstream reserve and durable finalization time. Requested maxima are
+ * ceilings, not all-or-nothing reservations: a smaller positive slice is
+ * admitted whenever one fits.
+ *
+ * The structured decision retains every value needed to explain or persist the
+ * decision without reconstructing it from prose.
+ */
+export declare const evalCapabilityExecutionEnvelopeV1: (input: {
+    readonly capability: AgenticCapabilityNameV1;
+    readonly requestedMaximumModelCalls?: number;
+    readonly requestedMaximumWallMs?: number;
+    readonly campaignModelCallsRemaining: number;
+    readonly campaignWallTimeRemainingMs: number;
+}) => EvalCapabilityExecutionEnvelopeDecisionV1;

package/dist/eval-capability-execution-envelope.js

@@ -0,0 +1,98 @@
+import { EVAL_CAPABILITY_SCIENTIFIC_PHASE_POLICY_V1, evalCapabilityScientificMaximumModelCallsV1, evalCapabilityScientificMaximumWallTimeMsV1, evalCapabilityScientificPhaseV1 } from "./eval-capability-policy.js";
+/**
+ * Time kept outside scientific execution for checkpoint/archive persistence
+ * and the host's durable capability-completion receipt.
+ */
+export const EVAL_CAPABILITY_DURABLE_FINALIZATION_RESERVE_MS_V1 = 30_000;
+const positiveInteger = (value, fallback) => Number.isSafeInteger(value) && value > 0 ? value : fallback;
+const availableInteger = (value) => Number.isSafeInteger(value) && value > 0 ? value : 0;
+/**
+ * The immutable downstream reserve for a scientific capability. This is the
+ * single package-owned source used by admission and execution; campaign-size
+ * ratios must not weaken these fixed phase reserves.
+ */
+export const evalCapabilityScientificDownstreamReserveV1 = (capability) => {
+    const phase = evalCapabilityScientificPhaseV1(capability);
+    const reserve = phase === null
+        ? { calls: 0, wallTimeMs: 0, attempts: 0 }
+        : EVAL_CAPABILITY_SCIENTIFIC_PHASE_POLICY_V1[phase].downstream;
+    return { phase, reserve };
+};
+/**
+ * Computes the positive execution slice that fits after preserving the fixed
+ * downstream reserve and durable finalization time. Requested maxima are
+ * ceilings, not all-or-nothing reservations: a smaller positive slice is
+ * admitted whenever one fits.
+ *
+ * The structured decision retains every value needed to explain or persist the
+ * decision without reconstructing it from prose.
+ */
+export const evalCapabilityExecutionEnvelopeV1 = (input) => {
+    const { phase, reserve } = evalCapabilityScientificDownstreamReserveV1(input.capability);
+    const scientific = phase !== null;
+    const finalizationReserveWallTimeMs = scientific
+        ? EVAL_CAPABILITY_DURABLE_FINALIZATION_RESERVE_MS_V1
+        : 0;
+    const available = {
+        calls: availableInteger(input.campaignModelCallsRemaining),
+        wallTimeMs: availableInteger(input.campaignWallTimeRemainingMs)
+    };
+    const requestedMaximum = {
+        calls: Math.min(scientific
+            ? evalCapabilityScientificMaximumModelCallsV1(input.capability)
+            : 12, positiveInteger(input.requestedMaximumModelCalls ?? 6, 6)),
+        wallTimeMs: Math.min(scientific
+            ? evalCapabilityScientificMaximumWallTimeMsV1(input.capability)
+            : 300_000, positiveInteger(input.requestedMaximumWallMs ?? 120_000, 120_000))
+    };
+    const minimumRequired = {
+        calls: reserve.calls + 1,
+        wallTimeMs: reserve.wallTimeMs + finalizationReserveWallTimeMs + 1
+    };
+    const availableForCapability = {
+        calls: Math.max(0, available.calls - reserve.calls),
+        wallTimeMs: Math.max(0, available.wallTimeMs -
+            reserve.wallTimeMs -
+            finalizationReserveWallTimeMs)
+    };
+    const common = {
+        version: 1,
+        capability: input.capability,
+        phase,
+        available,
+        requestedMaximum,
+        downstreamReserve: reserve,
+        finalizationReserve: {
+            wallTimeMs: finalizationReserveWallTimeMs
+        },
+        minimumRequired,
+        availableForCapability
+    };
+    // Read-only/control capabilities remain reachable after scientific exhaustion
+    // so callers can inspect progress and finish the campaign. They receive the
+    // legacy minimal positive execution fence and no downstream reserve.
+    if (!scientific) {
+        return {
+            ...common,
+            admitted: true,
+            envelope: {
+                maximumModelCalls: Math.max(1, Math.min(requestedMaximum.calls, available.calls)),
+                maximumWallTimeMs: Math.max(1, Math.min(requestedMaximum.wallTimeMs, available.wallTimeMs))
+            }
+        };
+    }
+    if (availableForCapability.calls < 1) {
+        return { ...common, admitted: false, dimension: "calls", envelope: null };
+    }
+    if (availableForCapability.wallTimeMs < 1) {
+        return { ...common, admitted: false, dimension: "wall_time", envelope: null };
+    }
+    return {
+        ...common,
+        admitted: true,
+        envelope: {
+            maximumModelCalls: Math.min(requestedMaximum.calls, availableForCapability.calls),
+            maximumWallTimeMs: Math.min(requestedMaximum.wallTimeMs, availableForCapability.wallTimeMs)
+        }
+    };
+};

package/dist/eval-capability-policy.d.ts

@@ -0,0 +1,90 @@
+import type { AgenticCapabilityNameV1, AgenticProgressFingerprintV1, AgenticScientificProgressReceiptV1 } from "./agentic-capabilities-protocol.js";
+/**
+ * A scheduling slice that only advances verified durable work is not another
+ * scientific attempt. Charging it makes sufficiently slow but productive
+ * cases impossible, regardless of their remaining campaign budget.
+ * New decisions, resets, legacy/unbound receipts, and unchanged work still
+ * consume attempts. Calls, spend, wall time and the no-progress circuit are
+ * independent and are never refunded.
+ */
+export declare const evalCapabilityScientificAttemptCostV1: (receipt: Pick<AgenticScientificProgressReceiptV1, "previousFingerprint" | "resultingFingerprint" | "inputRevision" | "outputRevision" | "gatesCompleted" | "gatesInvalidated" | "sameFingerprintCount">) => 0 | 1;
+/**
+ * No-op calls to other tools cannot reset a capability's stall counter. A
+ * genuine accepted-state change does reset it, even if a later repair returns
+ * to the same state. Worker admission and Cloud receipt validation share this
+ * rule; the global previous-fingerprint chain is verified independently.
+ * History must be ordered oldest first and scoped to one candidate revision.
+ */
+export declare const evalCapabilityScientificProgressPredecessorV1: (input: {
+    readonly capability: AgenticCapabilityNameV1;
+    readonly fingerprint: AgenticProgressFingerprintV1;
+    readonly history: readonly AgenticScientificProgressReceiptV1[];
+}) => AgenticScientificProgressReceiptV1 | undefined;
+/**
+ * One scientific capability is allowed enough time to finish a complete
+ * repository-grounded role chain (analysis, visible authoring, and private
+ * review) under normal provider tail latency. Cloud admission, the sandbox
+ * executor, and the native tool import this dependency-light policy module so
+ * loading an OpenCode tool never initializes the server-side foundry runtime.
+ */
+export declare const EVAL_CAPABILITY_SCIENTIFIC_MAXIMUM_MODEL_CALLS_V1 = 12;
+export declare const EVAL_CAPABILITY_SCIENTIFIC_MAXIMUM_WALL_TIME_MS_V1: number;
+/**
+ * Versioned capability slices are derived from the maximum number of native
+ * foundry turns that must remain contiguous to preserve tool state. Most
+ * capabilities fit one base slice. Oracle authoring and repair own longer
+ * tool-using conversations and therefore receive multiple contiguous slices
+ * instead of repeatedly restarting after the base ceiling.
+ */
+export declare const EVAL_CAPABILITY_SCIENTIFIC_SLICE_PLAN_V1: {
+    readonly author_oracle: {
+        readonly modelCallSlices: 3;
+        readonly wallTimeSlices: 3;
+    };
+    readonly repair_oracle: {
+        readonly modelCallSlices: 2;
+        readonly wallTimeSlices: 2;
+    };
+};
+export declare const evalCapabilityScientificMaximumModelCallsV1: (capability: AgenticCapabilityNameV1) => number;
+export declare const evalCapabilityScientificMaximumWallTimeMsV1: (capability: AgenticCapabilityNameV1) => number;
+export declare const EVAL_CAPABILITY_SCIENTIFIC_PHASE_POLICY_V1: {
+    readonly specification: {
+        readonly capabilities: readonly ["draft_specification"];
+        readonly downstream: {
+            readonly calls: 24;
+            readonly wallTimeMs: 4500000;
+            readonly attempts: 6;
+        };
+        readonly maximumAttempts: 8;
+    };
+    readonly environment: {
+        readonly capabilities: readonly ["plan_environment", "probe_fixture", "qualify_reference"];
+        readonly downstream: {
+            readonly calls: 16;
+            readonly wallTimeMs: 3600000;
+            readonly attempts: 4;
+        };
+        readonly maximumAttempts: 8;
+    };
+    readonly oracle: {
+        readonly capabilities: readonly ["author_oracle", "generate_controls", "evaluate_oracle", "repair_oracle"];
+        readonly downstream: {
+            readonly calls: 8;
+            readonly wallTimeMs: 1800000;
+            readonly attempts: 2;
+        };
+        readonly maximumAttempts: 12;
+    };
+    readonly admission: {
+        readonly capabilities: readonly ["freeze_candidate", "run_held_out_challenge", "request_admission"];
+        readonly downstream: {
+            readonly calls: 0;
+            readonly wallTimeMs: 0;
+            readonly attempts: 0;
+        };
+        readonly maximumAttempts: 6;
+    };
+};
+export type EvalCapabilityScientificPhaseV1 = keyof typeof EVAL_CAPABILITY_SCIENTIFIC_PHASE_POLICY_V1;
+export declare const evalCapabilityScientificPhaseV1: (capability: AgenticCapabilityNameV1) => EvalCapabilityScientificPhaseV1 | null;

package/dist/eval-capability-policy.js

@@ -0,0 +1,107 @@
+/**
+ * A scheduling slice that only advances verified durable work is not another
+ * scientific attempt. Charging it makes sufficiently slow but productive
+ * cases impossible, regardless of their remaining campaign budget.
+ * New decisions, resets, legacy/unbound receipts, and unchanged work still
+ * consume attempts. Calls, spend, wall time and the no-progress circuit are
+ * independent and are never refunded.
+ */
+export const evalCapabilityScientificAttemptCostV1 = (receipt) => {
+    const previous = receipt.previousFingerprint;
+    const next = receipt.resultingFingerprint;
+    return previous !== null &&
+        receipt.inputRevision !== null &&
+        receipt.inputRevision === receipt.outputRevision &&
+        receipt.sameFingerprintCount === 0 &&
+        receipt.gatesCompleted.length === 0 &&
+        receipt.gatesInvalidated.length === 0 &&
+        next.continuationProgressDigest != null &&
+        next.continuationProgressDigest !== (previous.continuationProgressDigest ?? null) &&
+        next.completedGateDigest === previous.completedGateDigest &&
+        next.environmentDigest === previous.environmentDigest
+        ? 0
+        : 1;
+};
+/**
+ * No-op calls to other tools cannot reset a capability's stall counter. A
+ * genuine accepted-state change does reset it, even if a later repair returns
+ * to the same state. Worker admission and Cloud receipt validation share this
+ * rule; the global previous-fingerprint chain is verified independently.
+ * History must be ordered oldest first and scoped to one candidate revision.
+ */
+export const evalCapabilityScientificProgressPredecessorV1 = (input) => {
+    for (let index = input.history.length - 1; index >= 0; index -= 1) {
+        const receipt = input.history[index];
+        const fingerprint = receipt.resultingFingerprint;
+        if (fingerprint.acceptedEvidenceDigest !== input.fingerprint.acceptedEvidenceDigest ||
+            fingerprint.completedGateDigest !== input.fingerprint.completedGateDigest ||
+            fingerprint.environmentDigest !== input.fingerprint.environmentDigest ||
+            (fingerprint.continuationProgressDigest ?? null) !==
+                (input.fingerprint.continuationProgressDigest ?? null))
+            return undefined;
+        if (receipt.capability === input.capability)
+            return receipt;
+    }
+    return undefined;
+};
+/**
+ * One scientific capability is allowed enough time to finish a complete
+ * repository-grounded role chain (analysis, visible authoring, and private
+ * review) under normal provider tail latency. Cloud admission, the sandbox
+ * executor, and the native tool import this dependency-light policy module so
+ * loading an OpenCode tool never initializes the server-side foundry runtime.
+ */
+export const EVAL_CAPABILITY_SCIENTIFIC_MAXIMUM_MODEL_CALLS_V1 = 12;
+export const EVAL_CAPABILITY_SCIENTIFIC_MAXIMUM_WALL_TIME_MS_V1 = 10 * 60_000;
+/**
+ * Versioned capability slices are derived from the maximum number of native
+ * foundry turns that must remain contiguous to preserve tool state. Most
+ * capabilities fit one base slice. Oracle authoring and repair own longer
+ * tool-using conversations and therefore receive multiple contiguous slices
+ * instead of repeatedly restarting after the base ceiling.
+ */
+export const EVAL_CAPABILITY_SCIENTIFIC_SLICE_PLAN_V1 = {
+    author_oracle: { modelCallSlices: 3, wallTimeSlices: 3 },
+    repair_oracle: { modelCallSlices: 2, wallTimeSlices: 2 }
+};
+export const evalCapabilityScientificMaximumModelCallsV1 = (capability) => EVAL_CAPABILITY_SCIENTIFIC_MAXIMUM_MODEL_CALLS_V1 *
+    (capability in EVAL_CAPABILITY_SCIENTIFIC_SLICE_PLAN_V1
+        ? EVAL_CAPABILITY_SCIENTIFIC_SLICE_PLAN_V1[capability].modelCallSlices
+        : 1);
+export const evalCapabilityScientificMaximumWallTimeMsV1 = (capability) => EVAL_CAPABILITY_SCIENTIFIC_MAXIMUM_WALL_TIME_MS_V1 *
+    (capability in EVAL_CAPABILITY_SCIENTIFIC_SLICE_PLAN_V1
+        ? EVAL_CAPABILITY_SCIENTIFIC_SLICE_PLAN_V1[capability].wallTimeSlices
+        : 1);
+export const EVAL_CAPABILITY_SCIENTIFIC_PHASE_POLICY_V1 = {
+    specification: {
+        capabilities: ["draft_specification"],
+        // This is the minimum protected envelope for environment qualification,
+        // oracle/control construction, held-out challenge, and admission. It is not
+        // a per-turn timeout. A smaller campaign can start useful work but cannot
+        // complete the mandatory scientific path.
+        downstream: { calls: 24, wallTimeMs: 4_500_000, attempts: 6 },
+        maximumAttempts: 8
+    },
+    environment: {
+        capabilities: ["plan_environment", "probe_fixture", "qualify_reference"],
+        downstream: { calls: 16, wallTimeMs: 3_600_000, attempts: 4 },
+        maximumAttempts: 8
+    },
+    oracle: {
+        capabilities: ["author_oracle", "generate_controls", "evaluate_oracle", "repair_oracle"],
+        downstream: { calls: 8, wallTimeMs: 1_800_000, attempts: 2 },
+        maximumAttempts: 12
+    },
+    admission: {
+        capabilities: ["freeze_candidate", "run_held_out_challenge", "request_admission"],
+        downstream: { calls: 0, wallTimeMs: 0, attempts: 0 },
+        maximumAttempts: 6
+    }
+};
+export const evalCapabilityScientificPhaseV1 = (capability) => {
+    for (const [phase, policy] of Object.entries(EVAL_CAPABILITY_SCIENTIFIC_PHASE_POLICY_V1)) {
+        if (policy.capabilities.includes(capability))
+            return phase;
+    }
+    return null;
+};

package/dist/eval-event-log.d.ts

@@ -1,4 +1,6 @@
 import { Schema } from "effect";
+export declare const EvalRunFailurePhase: Schema.Literals<readonly ["target", "classifier", "suite-inspection", "comparison", "activation", "accounting", "completion"]>;
+export type EvalRunFailurePhase = typeof EvalRunFailurePhase.Type;
 export declare const EvalNamedEvent: Schema.Union<readonly [Schema.Struct<{
     readonly name: Schema.Literal<"plan">;
     readonly at: Schema.String;
@@ -18,6 +20,30 @@ export declare const EvalNamedEvent: Schema.Union<readonly [Schema.Struct<{
     readonly caseId: Schema.String;
     readonly sequence: Schema.Finite;
     readonly name: Schema.Literal<"judge#">;
+}>, Schema.Struct<{
+    readonly name: Schema.Literal<"failure-detail">;
+    readonly at: Schema.String;
+    readonly runId: Schema.String;
+    readonly phase: Schema.Literals<readonly ["target", "classifier", "suite-inspection", "comparison", "activation", "accounting", "completion"]>;
+    readonly reason: Schema.String;
+    readonly retryable: Schema.Boolean;
+    readonly callId: Schema.optionalKey<Schema.String>;
+    readonly reservationId: Schema.optionalKey<Schema.String>;
+    readonly accounting: Schema.Struct<{
+        readonly expectedCalls: Schema.Finite;
+        readonly observedCalls: Schema.Finite;
+        readonly knownInputTokens: Schema.Finite;
+        readonly knownOutputTokens: Schema.Finite;
+        readonly unknownTokenMeasurements: Schema.Finite;
+        readonly knownPricedSubtotalUsd: Schema.Finite;
+        readonly unpricedCalls: Schema.Finite;
+        readonly subscriptionUnitCalls: Schema.Finite;
+    }>;
+    readonly cleanup: Schema.Struct<{
+        readonly sessionOpened: Schema.Boolean;
+        readonly sessionClosed: Schema.Boolean;
+        readonly detail: Schema.optionalKey<Schema.String>;
+    }>;
 }>, Schema.Struct<{
     readonly at: Schema.String;
     readonly operationId: Schema.String;
@@ -51,6 +77,30 @@ export declare const EvalNamedEventLog: Schema.$Array<Schema.Union<readonly [Sch
     readonly caseId: Schema.String;
     readonly sequence: Schema.Finite;
     readonly name: Schema.Literal<"judge#">;
+}>, Schema.Struct<{
+    readonly name: Schema.Literal<"failure-detail">;
+    readonly at: Schema.String;
+    readonly runId: Schema.String;
+    readonly phase: Schema.Literals<readonly ["target", "classifier", "suite-inspection", "comparison", "activation", "accounting", "completion"]>;
+    readonly reason: Schema.String;
+    readonly retryable: Schema.Boolean;
+    readonly callId: Schema.optionalKey<Schema.String>;
+    readonly reservationId: Schema.optionalKey<Schema.String>;
+    readonly accounting: Schema.Struct<{
+        readonly expectedCalls: Schema.Finite;
+        readonly observedCalls: Schema.Finite;
+        readonly knownInputTokens: Schema.Finite;
+        readonly knownOutputTokens: Schema.Finite;
+        readonly unknownTokenMeasurements: Schema.Finite;
+        readonly knownPricedSubtotalUsd: Schema.Finite;
+        readonly unpricedCalls: Schema.Finite;
+        readonly subscriptionUnitCalls: Schema.Finite;
+    }>;
+    readonly cleanup: Schema.Struct<{
+        readonly sessionOpened: Schema.Boolean;
+        readonly sessionClosed: Schema.Boolean;
+        readonly detail: Schema.optionalKey<Schema.String>;
+    }>;
 }>, Schema.Struct<{
     readonly at: Schema.String;
     readonly operationId: Schema.String;
@@ -86,4 +136,5 @@ export type EvalEventLogPresentation = {
 export declare function presentEvalEventLog(log: EvalNamedEventLog, options?: {
     readonly failHoles?: boolean;
 }): EvalEventLogPresentation;
+export declare function assertEvalRunCanFail(log: EvalNamedEventLog): void;
 export declare function assertEvalRunCanComplete(log: EvalNamedEventLog): void;

package/dist/eval-event-log.js

@@ -1,6 +1,10 @@
 import { Schema } from "effect";
 const NonNegativeInteger = Schema.Finite.pipe(Schema.check(Schema.makeFilter((value) => value >= 0 && Number.isInteger(value) ? undefined : "value must be a non-negative integer")));
 const PositiveInteger = NonNegativeInteger.pipe(Schema.check(Schema.makeFilter((value) => value >= 1 ? undefined : "value must be a positive integer")));
+const NonNegativeFinite = Schema.Finite.pipe(Schema.check(Schema.makeFilter((value) => value >= 0 ? undefined : "value must be a non-negative finite number")));
+const BoundedFailureReason = Schema.String.pipe(Schema.check(Schema.makeFilter((value) => value.length >= 1 && value.length <= 2_000
+    ? undefined
+    : "failure reason must contain between 1 and 2000 characters")));
 const RunPlanEvent = Schema.Struct({
     name: Schema.Literal("plan"),
     at: Schema.String,
@@ -22,6 +26,40 @@ const RunJudgeEvent = Schema.Struct({
     name: Schema.Literal("judge#"),
     ...RunCallEventFields
 });
+export const EvalRunFailurePhase = Schema.Literals([
+    "target",
+    "classifier",
+    "suite-inspection",
+    "comparison",
+    "activation",
+    "accounting",
+    "completion"
+]);
+const RunFailureDetailEvent = Schema.Struct({
+    name: Schema.Literal("failure-detail"),
+    at: Schema.String,
+    runId: Schema.String,
+    phase: EvalRunFailurePhase,
+    reason: BoundedFailureReason,
+    retryable: Schema.Boolean,
+    callId: Schema.optionalKey(Schema.String),
+    reservationId: Schema.optionalKey(Schema.String),
+    accounting: Schema.Struct({
+        expectedCalls: NonNegativeInteger,
+        observedCalls: NonNegativeInteger,
+        knownInputTokens: NonNegativeInteger,
+        knownOutputTokens: NonNegativeInteger,
+        unknownTokenMeasurements: NonNegativeInteger,
+        knownPricedSubtotalUsd: NonNegativeFinite,
+        unpricedCalls: NonNegativeInteger,
+        subscriptionUnitCalls: NonNegativeInteger
+    }),
+    cleanup: Schema.Struct({
+        sessionOpened: Schema.Boolean,
+        sessionClosed: Schema.Boolean,
+        detail: Schema.optionalKey(Schema.String)
+    })
+});
 const AuthoringEventFields = {
     at: Schema.String,
     operationId: Schema.String
@@ -42,6 +80,7 @@ export const EvalNamedEvent = Schema.Union([
     RunPlanEvent,
     RunCandidateEvent,
     RunJudgeEvent,
+    RunFailureDetailEvent,
     AuthoringBasisEvent,
     AuthoringDimensionsEvent,
     AuthoringActivationEvent
@@ -125,6 +164,9 @@ export function presentEvalEventLog(log, options = {}) {
     const cursors = resumeEvalCaseCursors(log);
     const failHoles = options.failHoles ?? true;
     const lastEvent = log[log.length - 1];
+    const failureLine = lastEvent?.name === "failure-detail"
+        ? `run fail ${lastEvent.phase}: ${lastEvent.reason}`
+        : undefined;
     const authoringLine = lastEvent?.name === "basis" ||
         lastEvent?.name === "dimensions" ||
         lastEvent?.name === "activation"
@@ -137,23 +179,40 @@ export function presentEvalEventLog(log, options = {}) {
             : `fail ${label} missing judge#${String(cursor.hole.sequence)}`}`;
     });
     return {
-        failed: failHoles && cursors.some((cursor) => cursor.hole !== undefined),
-        lines: cursors.length === 0
-            ? authoringLine === undefined
-                ? plan === undefined
-                    ? []
-                    : [`run ${evalEventLabel(plan)}`]
-                : [authoringLine]
-            : authoringLine === undefined
-                ? cursorLines
-                : [...cursorLines, authoringLine]
+        failed: failureLine !== undefined ||
+            (failHoles && cursors.some((cursor) => cursor.hole !== undefined)),
+        lines: failureLine !== undefined
+            ? [...cursorLines, failureLine]
+            : cursors.length === 0
+                ? authoringLine === undefined
+                    ? plan === undefined
+                        ? []
+                        : [`run ${evalEventLabel(plan)}`]
+                    : [authoringLine]
+                : authoringLine === undefined
+                    ? cursorLines
+                    : [...cursorLines, authoringLine]
     };
 }
+export function assertEvalRunCanFail(log) {
+    const plan = lastPlanEvent(log);
+    if (plan === undefined) {
+        throw new Error("failed eval run cannot terminate without its plan event");
+    }
+    const terminal = log[log.length - 1];
+    if (terminal?.name !== "failure-detail" || terminal.runId !== plan.runId) {
+        throw new Error("failed eval run must end with a matching terminal failure-detail event");
+    }
+}
 export function assertEvalRunCanComplete(log) {
     const plan = lastPlanEvent(log);
     if (plan === undefined || plan.plannedCalls < 1) {
         throw new Error("eval run cannot complete without a plan containing at least one call");
     }
+    const failure = log.find((event) => event.name === "failure-detail" && event.runId === plan.runId);
+    if (failure !== undefined) {
+        throw new Error(`eval run cannot complete after terminal failure in phase ${JSON.stringify(failure.phase)}`);
+    }
     const hole = resumeEvalCaseCursors(log).find((cursor) => cursor.hole !== undefined);
     if (hole?.hole !== undefined) {
         throw new Error(`eval case ${JSON.stringify(hole.dimensionId)}/${JSON.stringify(hole.caseId)} cannot complete after ${evalEventLabel(hole.hole)} without judge#${String(hole.hole.sequence)}`);

package/dist/evaluation-authoring-policy.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Provisional transport limits for compiler-backed evaluation authoring.
+ *
+ * These values bound model input/output and persisted candidate context; they
+ * are not semantic-quality thresholds. Live preflight and later empirical
+ * calibration must determine whether they produce coherent, useful cases.
+ */
+export declare const EVAL_AUTHORING_EVIDENCE_BLOCK_BYTES = 3000;
+export declare const EVAL_AUTHORING_CASE_EVIDENCE_BYTES = 12000;
+/**
+ * Maximum compiler blocks exposed as one mechanically safe author selection.
+ *
+ * A block may consume the complete block-byte allowance, so this cardinality
+ * guarantees that selecting every exposed block cannot exceed the case context
+ * allowance. The exact byte check remains authoritative because smaller blocks
+ * do not make every same-cardinality combination equivalent.
+ */
+export declare const EVAL_AUTHORING_CASE_EVIDENCE_BLOCKS: number;

package/dist/evaluation-authoring-policy.js

@@ -0,0 +1,19 @@
+/**
+ * Provisional transport limits for compiler-backed evaluation authoring.
+ *
+ * These values bound model input/output and persisted candidate context; they
+ * are not semantic-quality thresholds. Live preflight and later empirical
+ * calibration must determine whether they produce coherent, useful cases.
+ */
+export const EVAL_AUTHORING_EVIDENCE_BLOCK_BYTES = 3_000;
+export const EVAL_AUTHORING_CASE_EVIDENCE_BYTES = 12_000;
+/**
+ * Maximum compiler blocks exposed as one mechanically safe author selection.
+ *
+ * A block may consume the complete block-byte allowance, so this cardinality
+ * guarantees that selecting every exposed block cannot exceed the case context
+ * allowance. The exact byte check remains authoritative because smaller blocks
+ * do not make every same-cardinality combination equivalent.
+ */
+export const EVAL_AUTHORING_CASE_EVIDENCE_BLOCKS = Math.floor(EVAL_AUTHORING_CASE_EVIDENCE_BYTES /
+    EVAL_AUTHORING_EVIDENCE_BLOCK_BYTES);

package/dist/evaluation-authoring-validation.d.ts

@@ -0,0 +1,22 @@
+export type EvalAuthoringValidationCategory = "schema-constraint" | "schema-decode" | "case-shape" | "criterion-topology" | "control-topology" | "evidence-selection";
+export declare class EvalAuthoringValidationIssue extends Error {
+    readonly category: EvalAuthoringValidationCategory;
+    readonly path: string;
+    constructor(input: {
+        readonly category: EvalAuthoringValidationCategory;
+        readonly path: string;
+        readonly detail: string;
+        readonly cause?: unknown;
+    });
+}
+export declare const evalAuthoringValidationDiagnostic: (cause: unknown) => {
+    readonly category: EvalAuthoringValidationCategory;
+    readonly path: string;
+    readonly detail: string;
+};
+export declare const prefixEvalAuthoringValidationIssue: (cause: unknown, prefix: string) => EvalAuthoringValidationIssue;
+export declare const renderEvalAuthoringValidationFailure: (input: {
+    readonly scope: string;
+    readonly cause: unknown;
+    readonly callId?: string;
+}) => string;