@openpond/evals 0.4.2 → 0.5.0

package/dist/rollouts.js

@@ -0,0 +1,241 @@
+import { z } from "zod";
+import { ImmutableArtifactRefSchema, ImmutableReleaseRefSchema, MetadataSchema, ModelRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, contentHash, } from "@openpond/harness";
+import { AttemptOutcomeClassSchema, FailureOwnerSchema, RewardReceiptSchema, ScoringStatusSchema, } from "./execution-contracts.js";
+import { TaskSplitSchema } from "./tasksets.js";
+import { AttemptReceiptSchema, verifyAttemptReceipt } from "./runs.js";
+export const OptimizerTrainingSampleSchema = z
+    .object({
+    schemaVersion: z.literal("openpond.optimizerTrainingSample.v1"),
+    tokenIds: z.array(z.number().int().nonnegative()).min(2).max(32_768),
+    mask: z.array(z.boolean()).min(2).max(32_768),
+    logprobs: z.array(z.number().finite()).min(2).max(32_768),
+    temperatures: z.array(z.number().positive().finite()).min(2).max(32_768),
+    envName: z.string().trim().min(1).max(200),
+    modelRequestId: z.string().trim().min(1).max(1_000),
+    promptTokenCount: z.number().int().positive(),
+    completionTokenCount: z.number().int().positive(),
+    servedPolicyVersion: z.number().int().nonnegative(),
+})
+    .strict()
+    .superRefine((sample, context) => {
+    const length = sample.tokenIds.length;
+    for (const [name, values] of [
+        ["mask", sample.mask],
+        ["logprobs", sample.logprobs],
+        ["temperatures", sample.temperatures],
+    ]) {
+        if (values.length !== length) {
+            context.addIssue({
+                code: "custom",
+                path: [name],
+                message: `${name} must align with tokenIds`,
+            });
+        }
+    }
+    if (sample.promptTokenCount + sample.completionTokenCount !== length) {
+        context.addIssue({
+            code: "custom",
+            path: ["completionTokenCount"],
+            message: "prompt and completion token counts must span tokenIds",
+        });
+    }
+    if (sample.mask.filter((trainable) => !trainable).length !== sample.promptTokenCount) {
+        context.addIssue({
+            code: "custom",
+            path: ["mask"],
+            message: "promptTokenCount must equal the non-trainable mask count",
+        });
+    }
+    if (sample.mask.filter(Boolean).length !== sample.completionTokenCount) {
+        context.addIssue({
+            code: "custom",
+            path: ["mask"],
+            message: "completionTokenCount must equal the trainable mask count",
+        });
+    }
+});
+export const EnvironmentExecutionEvidenceSchema = z.object({
+    id: ReleaseIdSchema,
+    environmentRelease: ImmutableReleaseRefSchema,
+    status: z.enum(["completed", "failed", "timed_out", "cancelled"]),
+    startedAt: ReleaseTimestampSchema,
+    completedAt: ReleaseTimestampSchema,
+    traceRefs: z.array(ImmutableArtifactRefSchema).max(10_000),
+    metadata: MetadataSchema,
+}).strict();
+const RolloutRewardProjectionSchema = z.object({
+    receiptRef: ImmutableReleaseRefSchema,
+    status: ScoringStatusSchema,
+    value: z.number().min(0).max(1).nullable(),
+    learningEligible: z.boolean(),
+    passed: z.boolean(),
+    outcomeClass: AttemptOutcomeClassSchema,
+    failureOwner: FailureOwnerSchema.nullable(),
+    components: z.record(ReleaseIdSchema, z.number().min(0).max(1).nullable()),
+}).strict();
+const CanonicalRolloutRecordFieldsSchema = z.object({
+    schemaVersion: z.literal("openpond.canonicalRolloutRecord.v1"),
+    id: ReleaseIdSchema,
+    attemptRef: ImmutableReleaseRefSchema,
+    artifactManifestRef: ImmutableReleaseRefSchema,
+    tasksetRelease: ImmutableReleaseRefSchema,
+    environmentRelease: ImmutableReleaseRefSchema,
+    verifierSetRelease: ImmutableReleaseRefSchema,
+    harnessRelease: ImmutableReleaseRefSchema,
+    taskId: ReleaseIdSchema,
+    split: TaskSplitSchema,
+    model: ModelRefSchema,
+    seed: z.string().trim().min(1).max(500),
+    reward: RolloutRewardProjectionSchema,
+    traceRef: ImmutableArtifactRefSchema,
+    optimizerSample: OptimizerTrainingSampleSchema.nullable(),
+    environmentExecutions: z.array(EnvironmentExecutionEvidenceSchema).min(1).max(100_000),
+    startedAt: ReleaseTimestampSchema,
+    completedAt: ReleaseTimestampSchema,
+    metadata: MetadataSchema,
+}).strict();
+function validateCanonicalRolloutRecord(record, context) {
+    if (record.reward.status === "scored" && (record.reward.value === null || !record.reward.learningEligible)) {
+        context.addIssue({
+            code: "custom",
+            path: ["reward", "status"],
+            message: "A scored rollout requires a numeric, learning-eligible reward.",
+        });
+    }
+    if (record.reward.status === "unscorable" && (record.reward.value !== null || record.reward.learningEligible)) {
+        context.addIssue({
+            code: "custom",
+            path: ["reward", "status"],
+            message: "An unscorable rollout has no reward and cannot be learning-eligible.",
+        });
+    }
+}
+const CanonicalRolloutRecordBaseSchema = CanonicalRolloutRecordFieldsSchema
+    .superRefine(validateCanonicalRolloutRecord);
+export const CanonicalRolloutRecordSchema = z.object({
+    ...CanonicalRolloutRecordFieldsSchema.shape,
+    contentHash: ReleaseHashSchema,
+}).strict().superRefine(validateCanonicalRolloutRecord);
+export const RolloutQualificationSchema = z.object({
+    schemaVersion: z.literal("openpond.rolloutQualification.v1"),
+    rolloutCount: z.number().int().nonnegative(),
+    scoredCount: z.number().int().nonnegative(),
+    optimizerEligibleCount: z.number().int().nonnegative(),
+    unscorableCount: z.number().int().nonnegative(),
+    zeroRewardCount: z.number().int().nonnegative(),
+    rewardMean: z.number().min(0).max(1).nullable(),
+    rewardVariance: z.number().nonnegative().nullable(),
+    distinctRewardCount: z.number().int().nonnegative(),
+    eligibleForRl: z.boolean(),
+    reasons: z.array(z.string().trim().min(1).max(1_000)).max(100),
+}).strict();
+export function createCanonicalRolloutRecord(input) {
+    const attemptReceipt = AttemptReceiptSchema.parse(input.attemptReceipt);
+    if (!verifyAttemptReceipt(attemptReceipt)) {
+        throw new Error("Rollout Attempt Receipt failed content-hash verification.");
+    }
+    const rewardReceipt = RewardReceiptSchema.parse(input.rewardReceipt);
+    if (rewardReceipt.attemptRef.id !== attemptReceipt.id
+        || rewardReceipt.attemptRef.contentHash !== attemptReceipt.contentHash) {
+        throw new Error("Rollout Attempt Receipt does not match its Reward Receipt.");
+    }
+    if (rewardReceipt.artifactManifestRef.id !== input.artifactManifestRef.id
+        || rewardReceipt.artifactManifestRef.contentHash !== input.artifactManifestRef.contentHash) {
+        throw new Error("Rollout Artifact Manifest does not match its Reward Receipt.");
+    }
+    if (input.environmentExecutions.some((execution) => execution.environmentRelease.id !== input.environmentRelease.id
+        || execution.environmentRelease.contentHash !== input.environmentRelease.contentHash)) {
+        throw new Error("Rollout Environment execution does not match its admitted Environment Release.");
+    }
+    const content = CanonicalRolloutRecordBaseSchema.parse({
+        schemaVersion: "openpond.canonicalRolloutRecord.v1",
+        id: input.id,
+        attemptRef: rewardReceipt.attemptRef,
+        artifactManifestRef: input.artifactManifestRef,
+        tasksetRelease: input.tasksetRelease,
+        environmentRelease: input.environmentRelease,
+        verifierSetRelease: rewardReceipt.verifierSetRef,
+        harnessRelease: input.harnessRelease,
+        taskId: input.taskId,
+        split: input.split,
+        model: input.model,
+        seed: input.seed,
+        reward: {
+            receiptRef: { id: rewardReceipt.id, contentHash: rewardReceipt.contentHash },
+            status: rewardReceipt.status,
+            value: rewardReceipt.reward,
+            learningEligible: rewardReceipt.learningEligible,
+            passed: rewardReceipt.passed,
+            outcomeClass: rewardReceipt.outcomeClass,
+            failureOwner: rewardReceipt.failureOwner,
+            components: Object.fromEntries(rewardReceipt.components.map((component) => [
+                component.verifierId,
+                component.rewardContribution,
+            ])),
+        },
+        traceRef: input.traceRef,
+        optimizerSample: input.optimizerSample,
+        environmentExecutions: input.environmentExecutions,
+        startedAt: input.startedAt,
+        completedAt: input.completedAt,
+        metadata: input.metadata ?? {},
+    });
+    return CanonicalRolloutRecordSchema.parse({
+        ...content,
+        contentHash: contentHash(content),
+    });
+}
+export function verifyCanonicalRolloutRecord(value) {
+    const parsed = CanonicalRolloutRecordSchema.safeParse(value);
+    if (!parsed.success)
+        return false;
+    const { contentHash: actual, ...content } = parsed.data;
+    const canonical = CanonicalRolloutRecordBaseSchema.safeParse(content);
+    return canonical.success && contentHash(canonical.data) === actual;
+}
+export function optimizerEligibleRollouts(records) {
+    return records.filter((record) => record.reward.status === "scored"
+        && record.reward.learningEligible
+        && record.reward.value !== null
+        && record.optimizerSample !== null);
+}
+export function qualifyRolloutBatch(input) {
+    const records = input.records.map((record) => CanonicalRolloutRecordSchema.parse(record));
+    const scored = records.filter((record) => record.reward.status === "scored"
+        && record.reward.learningEligible
+        && record.reward.value !== null);
+    const optimizerEligible = optimizerEligibleRollouts(records);
+    const rewards = scored.map((record) => record.reward.value);
+    const mean = rewards.length
+        ? rewards.reduce((total, reward) => total + reward, 0) / rewards.length
+        : null;
+    const variance = mean === null
+        ? null
+        : rewards.reduce((total, reward) => total + (reward - mean) ** 2, 0) / rewards.length;
+    const distinctRewardCount = new Set(rewards.map((reward) => reward.toPrecision(12))).size;
+    const minimumScoredRollouts = input.minimumScoredRollouts ?? 2;
+    const minimumDistinctRewards = input.minimumDistinctRewards ?? 2;
+    const minimumRewardVariance = input.minimumRewardVariance ?? Number.EPSILON;
+    const reasons = [];
+    if (scored.length < minimumScoredRollouts)
+        reasons.push(`Requires at least ${minimumScoredRollouts} scored rollouts.`);
+    if (optimizerEligible.length !== scored.length)
+        reasons.push("Every scored rollout requires aligned optimizer token evidence.");
+    if (distinctRewardCount < minimumDistinctRewards)
+        reasons.push(`Requires at least ${minimumDistinctRewards} distinct reward values.`);
+    if (variance === null || variance < minimumRewardVariance)
+        reasons.push(`Reward variance must be at least ${minimumRewardVariance}.`);
+    return RolloutQualificationSchema.parse({
+        schemaVersion: "openpond.rolloutQualification.v1",
+        rolloutCount: records.length,
+        scoredCount: scored.length,
+        optimizerEligibleCount: optimizerEligible.length,
+        unscorableCount: records.length - scored.length,
+        zeroRewardCount: rewards.filter((reward) => reward === 0).length,
+        rewardMean: mean,
+        rewardVariance: variance,
+        distinctRewardCount,
+        eligibleForRl: reasons.length === 0,
+        reasons,
+    });
+}

package/dist/tasksets.js

@@ -1,6 +1,13 @@
 import { z } from "zod";
 import { CapabilityRequirementSchema, ImmutableAssetRefSchema, MetadataSchema, ReleaseHashSchema, ReleaseIdSchema, ToolDeclarationSchema, assertContentHash, contentHash, } from "@openpond/harness";
 export const TaskSplitSchema = z.enum(["train", "validation", "test", "frozen_eval"]);
+export const RequiredOutputContractSchema = z.object({
+    path: z.string().trim().min(1).max(2_000).refine(safeRelativePath),
+    mediaType: z.string().trim().min(1).max(200),
+    schemaRef: ImmutableAssetRefSchema.nullable().default(null),
+    maxBytes: z.number().int().positive().max(250_000_000).nullable().default(null),
+    metadata: MetadataSchema,
+}).strict();
 export const PolicyBoundarySchema = z.object({
     policyVisibleFields: z.array(ReleaseIdSchema).max(1_000).default([]),
     privilegedFields: z.array(ReleaseIdSchema).max(1_000).default([]),
@@ -60,6 +67,7 @@ export const TaskRecordSchema = z.object({
     policyVisibleContext: z.record(z.string(), z.unknown()).default({}),
     privilegedContextRef: ReleaseIdSchema.nullable(),
     artifactRefs: z.array(ImmutableAssetRefSchema).max(1_000).default([]),
+    requiredOutputs: z.array(RequiredOutputContractSchema).max(1_000).optional(),
     tags: z.array(ReleaseIdSchema).max(100).default([]),
 }).strict();
 export const TasksetReleaseContentSchema = z.object({
@@ -68,10 +76,12 @@ export const TasksetReleaseContentSchema = z.object({
     revision: z.number().int().positive(),
     policy: PolicyBoundarySchema,
     environment: EnvironmentContractSchema,
+    environmentRelease: z.object({ id: ReleaseIdSchema, contentHash: ReleaseHashSchema }).strict().optional(),
     tools: z.array(ToolDeclarationSchema).max(200),
     capabilities: z.array(CapabilityRequirementSchema).max(200),
     tasks: z.array(TaskRecordSchema).min(1).max(1_000_000),
     graders: z.array(GraderSpecSchema).min(1).max(1_000),
+    verifierSetRelease: z.object({ id: ReleaseIdSchema, contentHash: ReleaseHashSchema }).strict().optional(),
     metadata: MetadataSchema,
 }).strict();
 export const TasksetReleaseSchema = TasksetReleaseContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
@@ -92,6 +102,14 @@ export function validateTasksetRelease(input) {
     }
     const taskset = parsed.data;
     const issues = [];
+    if (Boolean(taskset.environmentRelease) !== Boolean(taskset.verifierSetRelease)) {
+        issues.push({
+            code: "execution_release_binding_incomplete",
+            severity: "error",
+            message: "A Taskset Release must bind both Environment and Verifier Set releases or neither during v2 migration.",
+            path: "environmentRelease",
+        });
+    }
     const clusterSplits = new Map();
     for (const task of taskset.tasks) {
         const splits = clusterSplits.get(task.clusterKey) ?? new Set();
@@ -147,3 +165,9 @@ export function trainingPolicyTaskViews(taskset) {
         .map(policyTaskView);
 }
 export { CapabilityRequirementSchema, ToolDeclarationSchema, } from "@openpond/harness";
+function safeRelativePath(value) {
+    const normalized = value.replaceAll("\\", "/");
+    if (!normalized || normalized.startsWith("/") || normalized.includes("\0"))
+        return false;
+    return !normalized.split("/").some((part) => !part || part === "." || part === "..");
+}

package/dist/types/artifact-verification.d.ts

@@ -0,0 +1,31 @@
+import type { ImmutableArtifactRef } from "@openpond/harness";
+import { type ArtifactManifest, type RewardComponentReceipt } from "./execution-contracts.js";
+import type { RequiredOutputContract } from "./tasksets.js";
+export type CollectedArtifact = {
+    path: string;
+    artifact: ImmutableArtifactRef | null;
+    detectedMediaType: string | null;
+    status: "collected" | "failed";
+    parseStatus?: "not_requested" | "passed" | "failed";
+    schemaStatus?: "not_requested" | "passed" | "failed";
+    errorCode?: string | null;
+    failureOwner?: "policy" | "collector" | "environment" | null;
+    evidenceRefs?: ImmutableArtifactRef[];
+    metadata?: Record<string, unknown>;
+};
+export declare function buildArtifactManifest(input: {
+    id: string;
+    attemptRef: {
+        id: string;
+        contentHash: string;
+    };
+    requiredOutputs: RequiredOutputContract[];
+    collectedArtifacts: CollectedArtifact[];
+    createdAt: string;
+    metadata?: Record<string, unknown>;
+}): ArtifactManifest;
+export declare function verifyRequiredOutputs(input: {
+    requiredOutputs: RequiredOutputContract[];
+    manifest: ArtifactManifest;
+}): RewardComponentReceipt[];
+//# sourceMappingURL=artifact-verification.d.ts.map

package/dist/types/artifact-verification.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"artifact-verification.d.ts","sourceRoot":"","sources":["../../../../src/artifact-verification.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAE9D,OAAO,EAGL,KAAK,gBAAgB,EAGrB,KAAK,sBAAsB,EAC5B,MAAM,0BAA0B,CAAC;AAElC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAE5D,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACtC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,MAAM,EAAE,WAAW,GAAG,QAAQ,CAAC;IAC/B,WAAW,CAAC,EAAE,eAAe,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACpD,YAAY,CAAC,EAAE,eAAe,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACrD,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,YAAY,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,aAAa,GAAG,IAAI,CAAC;IAC7D,YAAY,CAAC,EAAE,oBAAoB,EAAE,CAAC;IACtC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,CAAC;AAEF,wBAAgB,qBAAqB,CAAC,KAAK,EAAE;IAC3C,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,eAAe,EAAE,sBAAsB,EAAE,CAAC;IAC1C,kBAAkB,EAAE,iBAAiB,EAAE,CAAC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,gBAAgB,CAwEnB;AAED,wBAAgB,qBAAqB,CAAC,KAAK,EAAE;IAC3C,eAAe,EAAE,sBAAsB,EAAE,CAAC;IAC1C,QAAQ,EAAE,gBAAgB,CAAC;CAC5B,GAAG,sBAAsB,EAAE,CAqC3B"}

package/dist/types/builtin-benchmarks/harness-refiner.d.ts

@@ -49,6 +49,20 @@ export declare const harnessRefinerBenchmarkRelease: {
             visibility: "policy" | "verifier" | "host_private";
         }[];
         tags: string[];
+        requiredOutputs?: {
+            path: string;
+            mediaType: string;
+            schemaRef: {
+                id: string;
+                path: string;
+                contentHash: string;
+                sizeBytes: number;
+                mediaType: string;
+                visibility: "policy" | "verifier" | "host_private";
+            } | null;
+            maxBytes: number | null;
+            metadata: Record<string, unknown>;
+        }[] | undefined;
     }[];
     graders: ({
         id: string;
@@ -114,6 +128,14 @@ export declare const harnessRefinerBenchmarkRelease: {
     })[];
     metadata: Record<string, unknown>;
     contentHash: string;
+    environmentRelease?: {
+        id: string;
+        contentHash: string;
+    } | undefined;
+    verifierSetRelease?: {
+        id: string;
+        contentHash: string;
+    } | undefined;
 };
 export declare const harnessRefinerBenchmarkAssets: Readonly<Record<string, string>>;
 //# sourceMappingURL=harness-refiner.d.ts.map

package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"harness-refiner.d.ts","sourceRoot":"","sources":["../../../../../src/builtin-benchmarks/harness-refiner.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwuC1C,CAAC;AAEF,eAAO,MAAM,6BAA6B,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CASzE,CAAC"}
+{"version":3,"file":"harness-refiner.d.ts","sourceRoot":"","sources":["../../../../../src/builtin-benchmarks/harness-refiner.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqqD1C,CAAC;AAEF,eAAO,MAAM,6BAA6B,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CASzE,CAAC"}

package/dist/types/conformance.d.ts

@@ -159,6 +159,20 @@ export declare const genericToolConformance: {
                 visibility: "policy" | "verifier" | "host_private";
             }[];
             tags: string[];
+            requiredOutputs?: {
+                path: string;
+                mediaType: string;
+                schemaRef: {
+                    id: string;
+                    path: string;
+                    contentHash: string;
+                    sizeBytes: number;
+                    mediaType: string;
+                    visibility: "policy" | "verifier" | "host_private";
+                } | null;
+                maxBytes: number | null;
+                metadata: Record<string, unknown>;
+            }[] | undefined;
         }[];
         graders: ({
             id: string;
@@ -224,6 +238,14 @@ export declare const genericToolConformance: {
         })[];
         metadata: Record<string, unknown>;
         contentHash: string;
+        environmentRelease?: {
+            id: string;
+            contentHash: string;
+        } | undefined;
+        verifierSetRelease?: {
+            id: string;
+            contentHash: string;
+        } | undefined;
     };
     manifest: {
         schemaVersion: "openpond.runManifest.v1";
@@ -426,6 +448,20 @@ export declare const marketingPortfolioConformance: {
                 visibility: "policy" | "verifier" | "host_private";
             }[];
             tags: string[];
+            requiredOutputs?: {
+                path: string;
+                mediaType: string;
+                schemaRef: {
+                    id: string;
+                    path: string;
+                    contentHash: string;
+                    sizeBytes: number;
+                    mediaType: string;
+                    visibility: "policy" | "verifier" | "host_private";
+                } | null;
+                maxBytes: number | null;
+                metadata: Record<string, unknown>;
+            }[] | undefined;
         }[];
         graders: ({
             id: string;
@@ -491,6 +527,14 @@ export declare const marketingPortfolioConformance: {
         })[];
         metadata: Record<string, unknown>;
         contentHash: string;
+        environmentRelease?: {
+            id: string;
+            contentHash: string;
+        } | undefined;
+        verifierSetRelease?: {
+            id: string;
+            contentHash: string;
+        } | undefined;
     };
     manifest: {
         schemaVersion: "openpond.runManifest.v1";

package/dist/types/conformance.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"conformance.d.ts","sourceRoot":"","sources":["../../../../src/conformance.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAEjC,CAAC;AAEH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAGxC,CAAC"}
+{"version":3,"file":"conformance.d.ts","sourceRoot":"","sources":["../../../../src/conformance.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAEjC,CAAC;AAEH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAGxC,CAAC"}

package/dist/types/evidence/conformance.d.ts

@@ -9,9 +9,9 @@ export declare const completeWorkProcessTraceFixture: {
         sequence: number;
         timestamp: string;
         layer: "agent" | "environment";
-        kind: "approval" | "validation" | "artifact" | "tool" | "state_transition" | "question" | "cleanup";
+        kind: "validation" | "artifact" | "approval" | "tool" | "state_transition" | "question" | "cleanup";
         action: "turn_started" | "turn_completed" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
-        status: "cancelled" | "failed" | "started" | "completed";
+        status: "failed" | "completed" | "cancelled" | "started";
         inputHash: string | null;
         outputHash: string | null;
         receiptHash: string | null;
@@ -25,14 +25,14 @@ export declare const completeWorkProcessTraceFixture: {
         attributes: {
             toolCategory: "agent" | "command" | "filesystem" | "source_control" | "browser" | "connected_app" | "sandbox" | "other" | null;
             validationKind: "test" | "structural" | "visual" | "user_review" | "other" | null;
-            transitionState: "timeout" | "cancelled" | "failed" | "completed" | "running" | null;
+            transitionState: "failed" | "completed" | "cancelled" | "timeout" | "running" | null;
             interventionOutcome: "requested" | "approved" | "denied" | "answered" | "dismissed" | null;
             artifactCount: number;
             exitCode: number | null;
             durationMs: number | null;
             cpuTimeMs: number | null;
             memoryPeakBytes: number | null;
-            errorClass: "unknown" | "timeout" | "cancelled" | "validation" | "policy" | "environment" | "infrastructure" | null;
+            errorClass: "validation" | "policy" | "unknown" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
         };
     }[];
     contentHash: string;
@@ -70,8 +70,8 @@ export declare const completeWorkEvidenceFixture: {
     };
     inputHash: string;
     terminal: {
-        status: "timeout" | "cancelled" | "failed" | "completed";
-        failureClass: "unknown" | "policy_failure" | "environment_failure" | "infrastructure_failure" | "timeout" | "cancelled" | "validation_failure" | "model_failure" | null;
+        status: "failed" | "completed" | "cancelled" | "timeout";
+        failureClass: "unknown" | "policy_failure" | "environment_failure" | "cancelled" | "infrastructure_failure" | "timeout" | "validation_failure" | "model_failure" | null;
     };
     trace: {
         sanitizedRef: {
@@ -186,9 +186,9 @@ export declare const incompleteWorkProcessTraceFixture: {
         sequence: number;
         timestamp: string;
         layer: "agent" | "environment";
-        kind: "approval" | "validation" | "artifact" | "tool" | "state_transition" | "question" | "cleanup";
+        kind: "validation" | "artifact" | "approval" | "tool" | "state_transition" | "question" | "cleanup";
         action: "turn_started" | "turn_completed" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
-        status: "cancelled" | "failed" | "started" | "completed";
+        status: "failed" | "completed" | "cancelled" | "started";
         inputHash: string | null;
         outputHash: string | null;
         receiptHash: string | null;
@@ -202,14 +202,14 @@ export declare const incompleteWorkProcessTraceFixture: {
         attributes: {
             toolCategory: "agent" | "command" | "filesystem" | "source_control" | "browser" | "connected_app" | "sandbox" | "other" | null;
             validationKind: "test" | "structural" | "visual" | "user_review" | "other" | null;
-            transitionState: "timeout" | "cancelled" | "failed" | "completed" | "running" | null;
+            transitionState: "failed" | "completed" | "cancelled" | "timeout" | "running" | null;
             interventionOutcome: "requested" | "approved" | "denied" | "answered" | "dismissed" | null;
             artifactCount: number;
             exitCode: number | null;
             durationMs: number | null;
             cpuTimeMs: number | null;
             memoryPeakBytes: number | null;
-            errorClass: "unknown" | "timeout" | "cancelled" | "validation" | "policy" | "environment" | "infrastructure" | null;
+            errorClass: "validation" | "policy" | "unknown" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
         };
     }[];
     contentHash: string;
@@ -308,8 +308,8 @@ export declare const invalidRawEvidenceFixture: {
     };
     inputHash: string;
     terminal: {
-        status: "timeout" | "cancelled" | "failed" | "completed";
-        failureClass: "unknown" | "policy_failure" | "environment_failure" | "infrastructure_failure" | "timeout" | "cancelled" | "validation_failure" | "model_failure" | null;
+        status: "failed" | "completed" | "cancelled" | "timeout";
+        failureClass: "unknown" | "policy_failure" | "environment_failure" | "cancelled" | "infrastructure_failure" | "timeout" | "validation_failure" | "model_failure" | null;
     };
     trace: {
         sanitizedRef: {
@@ -392,9 +392,9 @@ export declare const workEvidenceConformance: {
             sequence: number;
             timestamp: string;
             layer: "agent" | "environment";
-            kind: "approval" | "validation" | "artifact" | "tool" | "state_transition" | "question" | "cleanup";
+            kind: "validation" | "artifact" | "approval" | "tool" | "state_transition" | "question" | "cleanup";
             action: "turn_started" | "turn_completed" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
-            status: "cancelled" | "failed" | "started" | "completed";
+            status: "failed" | "completed" | "cancelled" | "started";
             inputHash: string | null;
             outputHash: string | null;
             receiptHash: string | null;
@@ -408,14 +408,14 @@ export declare const workEvidenceConformance: {
             attributes: {
                 toolCategory: "agent" | "command" | "filesystem" | "source_control" | "browser" | "connected_app" | "sandbox" | "other" | null;
                 validationKind: "test" | "structural" | "visual" | "user_review" | "other" | null;
-                transitionState: "timeout" | "cancelled" | "failed" | "completed" | "running" | null;
+                transitionState: "failed" | "completed" | "cancelled" | "timeout" | "running" | null;
                 interventionOutcome: "requested" | "approved" | "denied" | "answered" | "dismissed" | null;
                 artifactCount: number;
                 exitCode: number | null;
                 durationMs: number | null;
                 cpuTimeMs: number | null;
                 memoryPeakBytes: number | null;
-                errorClass: "unknown" | "timeout" | "cancelled" | "validation" | "policy" | "environment" | "infrastructure" | null;
+                errorClass: "validation" | "policy" | "unknown" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
             };
         }[];
         contentHash: string;
@@ -453,8 +453,8 @@ export declare const workEvidenceConformance: {
         };
         inputHash: string;
         terminal: {
-            status: "timeout" | "cancelled" | "failed" | "completed";
-            failureClass: "unknown" | "policy_failure" | "environment_failure" | "infrastructure_failure" | "timeout" | "cancelled" | "validation_failure" | "model_failure" | null;
+            status: "failed" | "completed" | "cancelled" | "timeout";
+            failureClass: "unknown" | "policy_failure" | "environment_failure" | "cancelled" | "infrastructure_failure" | "timeout" | "validation_failure" | "model_failure" | null;
         };
         trace: {
             sanitizedRef: {
@@ -625,9 +625,9 @@ export declare const workEvidenceConformance: {
             sequence: number;
             timestamp: string;
             layer: "agent" | "environment";
-            kind: "approval" | "validation" | "artifact" | "tool" | "state_transition" | "question" | "cleanup";
+            kind: "validation" | "artifact" | "approval" | "tool" | "state_transition" | "question" | "cleanup";
             action: "turn_started" | "turn_completed" | "turn_failed" | "turn_cancelled" | "turn_timed_out" | "tool_invoked" | "tool_completed" | "tool_failed" | "validation_completed" | "validation_failed" | "approval_requested" | "approval_resolved" | "question_asked" | "question_answered" | "question_dismissed" | "artifact_created" | "workspace_changed" | "environment_created" | "environment_reset" | "environment_destroyed" | "cleanup_completed" | "cleanup_failed";
-            status: "cancelled" | "failed" | "started" | "completed";
+            status: "failed" | "completed" | "cancelled" | "started";
             inputHash: string | null;
             outputHash: string | null;
             receiptHash: string | null;
@@ -641,14 +641,14 @@ export declare const workEvidenceConformance: {
             attributes: {
                 toolCategory: "agent" | "command" | "filesystem" | "source_control" | "browser" | "connected_app" | "sandbox" | "other" | null;
                 validationKind: "test" | "structural" | "visual" | "user_review" | "other" | null;
-                transitionState: "timeout" | "cancelled" | "failed" | "completed" | "running" | null;
+                transitionState: "failed" | "completed" | "cancelled" | "timeout" | "running" | null;
                 interventionOutcome: "requested" | "approved" | "denied" | "answered" | "dismissed" | null;
                 artifactCount: number;
                 exitCode: number | null;
                 durationMs: number | null;
                 cpuTimeMs: number | null;
                 memoryPeakBytes: number | null;
-                errorClass: "unknown" | "timeout" | "cancelled" | "validation" | "policy" | "environment" | "infrastructure" | null;
+                errorClass: "validation" | "policy" | "unknown" | "environment" | "cancelled" | "timeout" | "infrastructure" | null;
             };
         }[];
         contentHash: string;
@@ -691,8 +691,8 @@ export declare const workEvidenceConformance: {
         };
         inputHash: string;
         terminal: {
-            status: "timeout" | "cancelled" | "failed" | "completed";
-            failureClass: "unknown" | "policy_failure" | "environment_failure" | "infrastructure_failure" | "timeout" | "cancelled" | "validation_failure" | "model_failure" | null;
+            status: "failed" | "completed" | "cancelled" | "timeout";
+            failureClass: "unknown" | "policy_failure" | "environment_failure" | "cancelled" | "infrastructure_failure" | "timeout" | "validation_failure" | "model_failure" | null;
         };
         trace: {
             sanitizedRef: {