@openpond/evals 0.4.2 → 0.6.0

package/dist/execution-contracts.js

@@ -0,0 +1,153 @@
+import { z } from "zod";
+import { ImmutableArtifactRefSchema, ImmutableAssetRefSchema, ImmutableReleaseRefSchema, MetadataSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, } from "@openpond/harness";
+import { EnvironmentContractSchema, GraderSpecSchema } from "./tasksets.js";
+export const ScoringStatusSchema = z.enum(["scored", "unscorable"]);
+export const FailureOwnerSchema = z.enum([
+    "policy",
+    "environment",
+    "collector",
+    "verifier",
+    "provider",
+    "host",
+    "user",
+    "scheduler",
+]);
+export const AttemptOutcomeClassSchema = z.enum([
+    "completed",
+    "policy_failure",
+    "incomplete_output",
+    "task_deadline",
+    "environment_failure",
+    "collector_failure",
+    "verifier_failure",
+    "provider_failure",
+    "host_failure",
+    "infrastructure_timeout",
+    "cancelled",
+]);
+export const EnvironmentReleaseContentSchema = z.object({
+    schemaVersion: z.literal("openpond.environmentRelease.v1"),
+    id: ReleaseIdSchema,
+    revision: z.number().int().positive(),
+    contract: EnvironmentContractSchema,
+    actionSchemaRef: ImmutableAssetRefSchema.nullable(),
+    observationSchemaRef: ImmutableAssetRefSchema.nullable(),
+    stateSchemaRef: ImmutableAssetRefSchema.nullable(),
+    artifactCollection: z.object({
+        maxArtifacts: z.number().int().positive().max(100_000),
+        maxTotalBytes: z.number().int().positive().max(10_000_000_000),
+    }).strict(),
+    adapterConformanceHashes: z.record(ReleaseIdSchema, ReleaseHashSchema),
+    metadata: MetadataSchema,
+}).strict();
+export const EnvironmentReleaseSchema = EnvironmentReleaseContentSchema
+    .extend({ contentHash: ReleaseHashSchema })
+    .strict();
+export const ArtifactCollectionStatusSchema = z.enum([
+    "collected",
+    "missing",
+    "skipped",
+    "failed",
+]);
+export const ArtifactValidationStatusSchema = z.enum([
+    "not_requested",
+    "passed",
+    "failed",
+]);
+export const ArtifactManifestEntrySchema = z.object({
+    requiredOutputPath: z.string().trim().min(1).max(2_000).nullable(),
+    collectedPath: z.string().trim().min(1).max(4_000).nullable(),
+    declaredMediaType: z.string().trim().min(1).max(200).nullable(),
+    detectedMediaType: z.string().trim().min(1).max(200).nullable(),
+    artifact: ImmutableArtifactRefSchema.nullable(),
+    status: ArtifactCollectionStatusSchema,
+    parseStatus: ArtifactValidationStatusSchema,
+    schemaStatus: ArtifactValidationStatusSchema,
+    errorCode: ReleaseIdSchema.nullable(),
+    failureOwner: FailureOwnerSchema.nullable(),
+    evidenceRefs: z.array(ImmutableArtifactRefSchema).max(10_000),
+    metadata: MetadataSchema,
+}).strict();
+export const ArtifactManifestContentSchema = z.object({
+    schemaVersion: z.literal("openpond.artifactManifest.v1"),
+    id: ReleaseIdSchema,
+    attemptRef: ImmutableReleaseRefSchema,
+    entries: z.array(ArtifactManifestEntrySchema).max(100_000),
+    createdAt: ReleaseTimestampSchema,
+    metadata: MetadataSchema,
+}).strict();
+export const ArtifactManifestSchema = ArtifactManifestContentSchema
+    .extend({ contentHash: ReleaseHashSchema })
+    .strict();
+export const VerifierSetReleaseContentSchema = z.object({
+    schemaVersion: z.literal("openpond.verifierSetRelease.v1"),
+    id: ReleaseIdSchema,
+    revision: z.number().int().positive(),
+    graders: z.array(GraderSpecSchema).min(1).max(1_000),
+    isolation: z.object({
+        processBoundary: z.enum(["same_process", "isolated_process", "container"]),
+        networkPolicy: z.literal("none"),
+        defaultTimeoutMs: z.number().int().positive().max(300_000),
+    }).strict(),
+    calibrationReceiptRefs: z.array(ImmutableReleaseRefSchema).max(10_000),
+    metadata: MetadataSchema,
+}).strict();
+export const VerifierSetReleaseSchema = VerifierSetReleaseContentSchema
+    .extend({ contentHash: ReleaseHashSchema })
+    .strict();
+export const RewardComponentReceiptSchema = z.object({
+    verifierId: ReleaseIdSchema,
+    verifierVersion: z.string().trim().min(1).max(100),
+    status: ScoringStatusSchema,
+    rawScore: z.number().finite().nullable(),
+    normalizedScore: z.number().min(0).max(1).nullable(),
+    weight: z.number().nonnegative().max(1_000),
+    passed: z.boolean(),
+    hardGate: z.boolean(),
+    rewardEligible: z.boolean(),
+    rewardContribution: z.number().min(0).max(1).nullable(),
+    failureOwner: FailureOwnerSchema.nullable(),
+    feedback: z.array(z.string().max(20_000)).max(1_000),
+    visibleEvidenceRefs: z.array(ImmutableArtifactRefSchema).max(10_000),
+    privilegedEvidenceRefs: z.array(ImmutableArtifactRefSchema).max(10_000),
+    metadata: MetadataSchema,
+}).strict().superRefine((component, context) => {
+    if (component.status === "scored" && component.normalizedScore === null) {
+        context.addIssue({ code: "custom", message: "A scored component requires a normalized score.", path: ["normalizedScore"] });
+    }
+    if (component.status === "unscorable" && (component.normalizedScore !== null || component.rewardEligible)) {
+        context.addIssue({ code: "custom", message: "An unscorable component cannot contribute reward.", path: ["status"] });
+    }
+});
+const RewardReceiptBaseSchema = z.object({
+    schemaVersion: z.literal("openpond.rewardReceipt.v1"),
+    id: ReleaseIdSchema,
+    attemptRef: ImmutableReleaseRefSchema,
+    verifierSetRef: ImmutableReleaseRefSchema,
+    artifactManifestRef: ImmutableReleaseRefSchema,
+    status: ScoringStatusSchema,
+    reward: z.number().min(0).max(1).nullable(),
+    learningEligible: z.boolean(),
+    passed: z.boolean(),
+    outcomeClass: AttemptOutcomeClassSchema,
+    failureOwner: FailureOwnerSchema.nullable(),
+    components: z.array(RewardComponentReceiptSchema).min(1).max(10_000),
+    visibleEvidenceRefs: z.array(ImmutableArtifactRefSchema).max(100_000),
+    privilegedEvidenceRefs: z.array(ImmutableArtifactRefSchema).max(100_000),
+    supersedes: ImmutableReleaseRefSchema.nullable(),
+    createdAt: ReleaseTimestampSchema,
+    metadata: MetadataSchema,
+}).strict();
+function validateRewardReceipt(receipt, context) {
+    if (receipt.status === "scored" && (receipt.reward === null || !receipt.learningEligible)) {
+        context.addIssue({ code: "custom", message: "A scored receipt requires a reward and is learning-eligible.", path: ["status"] });
+    }
+    if (receipt.status === "unscorable" && (receipt.reward !== null || receipt.learningEligible)) {
+        context.addIssue({ code: "custom", message: "An unscorable receipt has null reward and is not learning-eligible.", path: ["status"] });
+    }
+}
+export const RewardReceiptContentSchema = RewardReceiptBaseSchema.superRefine(validateRewardReceipt);
+export const RewardReceiptSchema = RewardReceiptBaseSchema
+    .extend({ contentHash: ReleaseHashSchema })
+    .strict()
+    .superRefine(validateRewardReceipt);

package/dist/execution-receipts.js

@@ -0,0 +1,185 @@
+import { contentHash } from "@openpond/harness";
+import { ArtifactManifestContentSchema, ArtifactManifestSchema, EnvironmentReleaseContentSchema, EnvironmentReleaseSchema, RewardComponentReceiptSchema, RewardReceiptContentSchema, RewardReceiptSchema, VerifierSetReleaseContentSchema, VerifierSetReleaseSchema, } from "./execution-contracts.js";
+import { TasksetReleaseContentSchema, TasksetReleaseSchema, } from "./tasksets.js";
+export function createEnvironmentRelease(input) {
+    const content = EnvironmentReleaseContentSchema.parse(input);
+    return EnvironmentReleaseSchema.parse({ ...content, contentHash: contentHash(content) });
+}
+export function createArtifactManifest(input) {
+    const content = ArtifactManifestContentSchema.parse(input);
+    return ArtifactManifestSchema.parse({ ...content, contentHash: contentHash(content) });
+}
+export function createVerifierSetRelease(input) {
+    const content = VerifierSetReleaseContentSchema.parse(input);
+    return VerifierSetReleaseSchema.parse({ ...content, contentHash: contentHash(content) });
+}
+export function bindTasksetExecutionReleases(input) {
+    verifyContentHash(input.environment, EnvironmentReleaseContentSchema, "Environment Release");
+    verifyContentHash(input.verifierSet, VerifierSetReleaseContentSchema, "Verifier Set Release");
+    if (contentHash(input.taskset.environment) !== contentHash(input.environment.contract)) {
+        throw new Error("Environment Release does not match the Taskset execution contract.");
+    }
+    if (contentHash(input.taskset.graders) !== contentHash(input.verifierSet.graders)) {
+        throw new Error("Verifier Set Release does not match the Taskset graders.");
+    }
+    const { contentHash: _previousHash, ...previousContent } = input.taskset;
+    const content = TasksetReleaseContentSchema.parse({
+        ...previousContent,
+        environmentRelease: {
+            id: input.environment.id,
+            contentHash: input.environment.contentHash,
+        },
+        verifierSetRelease: {
+            id: input.verifierSet.id,
+            contentHash: input.verifierSet.contentHash,
+        },
+    });
+    return TasksetReleaseSchema.parse({ ...content, contentHash: contentHash(content) });
+}
+export function classifyAttemptOutcome(input) {
+    switch (input.failureClass) {
+        case null:
+            return { outcomeClass: "completed", failureOwner: null };
+        case "policy_failure":
+            return { outcomeClass: "policy_failure", failureOwner: "policy" };
+        case "environment_failure":
+            return { outcomeClass: "environment_failure", failureOwner: "environment" };
+        case "grader_failure":
+            return { outcomeClass: "verifier_failure", failureOwner: "verifier" };
+        case "infrastructure_failure":
+            return { outcomeClass: "host_failure", failureOwner: "host" };
+        case "cancelled":
+            return { outcomeClass: "cancelled", failureOwner: "user" };
+        case "timeout":
+            return input.timeoutKind === "task_deadline"
+                ? { outcomeClass: "task_deadline", failureOwner: "policy" }
+                : { outcomeClass: "infrastructure_timeout", failureOwner: "host" };
+    }
+}
+export function createRewardReceipt(input) {
+    verifyContentHash(input.verifierSet, VerifierSetReleaseContentSchema, "Verifier Set Release");
+    verifyContentHash(input.artifactManifest, ArtifactManifestContentSchema, "Artifact Manifest");
+    if (input.artifactManifest.attemptRef.id !== input.attemptRef.id
+        || input.artifactManifest.attemptRef.contentHash !== input.attemptRef.contentHash) {
+        throw new Error("Artifact Manifest does not belong to the Reward Receipt Attempt.");
+    }
+    const outcomeScorable = isScorableOutcome(input.outcomeClass);
+    const components = input.components.map((raw) => {
+        const component = RewardComponentReceiptSchema.parse(raw);
+        const rewardEligible = outcomeScorable
+            && component.status === "scored"
+            && component.normalizedScore !== null
+            && component.rewardEligible;
+        return RewardComponentReceiptSchema.parse({
+            ...component,
+            rewardEligible,
+            rewardContribution: rewardEligible ? component.normalizedScore : null,
+        });
+    });
+    const eligible = components.filter((component) => component.rewardEligible && component.normalizedScore !== null);
+    const status = eligible.length > 0 ? "scored" : "unscorable";
+    const hardGateFailed = eligible.some((component) => component.hardGate && !component.passed);
+    const totalWeight = eligible.reduce((total, component) => total + component.weight, 0);
+    const weightedReward = totalWeight > 0
+        ? eligible.reduce((total, component) => total + component.normalizedScore * component.weight, 0) / totalWeight
+        : 0;
+    const reward = status === "scored" ? hardGateFailed ? 0 : weightedReward : null;
+    const passed = status === "scored" && eligible.every((component) => component.passed);
+    const content = RewardReceiptContentSchema.parse({
+        schemaVersion: "openpond.rewardReceipt.v1",
+        id: input.id,
+        attemptRef: input.attemptRef,
+        verifierSetRef: {
+            id: input.verifierSet.id,
+            contentHash: input.verifierSet.contentHash,
+        },
+        artifactManifestRef: {
+            id: input.artifactManifest.id,
+            contentHash: input.artifactManifest.contentHash,
+        },
+        status,
+        reward,
+        learningEligible: status === "scored",
+        passed,
+        outcomeClass: input.outcomeClass,
+        failureOwner: input.failureOwner,
+        components,
+        visibleEvidenceRefs: uniqueArtifactRefs([
+            ...(input.visibleEvidenceRefs ?? []),
+            ...components.flatMap((component) => component.visibleEvidenceRefs),
+        ]),
+        privilegedEvidenceRefs: uniqueArtifactRefs([
+            ...(input.privilegedEvidenceRefs ?? []),
+            ...components.flatMap((component) => component.privilegedEvidenceRefs),
+        ]),
+        supersedes: input.supersedes ?? null,
+        createdAt: input.createdAt,
+        metadata: input.metadata ?? {},
+    });
+    return RewardReceiptSchema.parse({ ...content, contentHash: contentHash(content) });
+}
+export function regradeRewardReceipt(input) {
+    if (!verifyRewardReceipt(input.original)) {
+        throw new Error("Cannot regrade an invalid Reward Receipt.");
+    }
+    return createRewardReceipt({
+        id: input.id,
+        attemptRef: input.original.attemptRef,
+        verifierSet: input.verifierSet,
+        artifactManifest: input.artifactManifest,
+        outcomeClass: input.outcomeClass ?? input.original.outcomeClass,
+        failureOwner: input.failureOwner === undefined
+            ? input.original.failureOwner
+            : input.failureOwner,
+        components: input.components,
+        supersedes: {
+            id: input.original.id,
+            contentHash: input.original.contentHash,
+        },
+        createdAt: input.createdAt,
+        metadata: {
+            ...input.metadata,
+            regradeOf: input.original.id,
+        },
+    });
+}
+export function verifyEnvironmentRelease(release) {
+    return hasValidContentHash(release, EnvironmentReleaseContentSchema);
+}
+export function verifyArtifactManifest(manifest) {
+    return hasValidContentHash(manifest, ArtifactManifestContentSchema);
+}
+export function verifyVerifierSetRelease(release) {
+    return hasValidContentHash(release, VerifierSetReleaseContentSchema);
+}
+export function verifyRewardReceipt(receipt) {
+    const parsed = RewardReceiptSchema.safeParse(receipt);
+    if (!parsed.success)
+        return false;
+    const { contentHash: actual, ...content } = parsed.data;
+    return contentHash(RewardReceiptContentSchema.parse(content)) === actual;
+}
+export function isScorableOutcome(outcome) {
+    return outcome === "completed"
+        || outcome === "policy_failure"
+        || outcome === "incomplete_output"
+        || outcome === "task_deadline";
+}
+function uniqueArtifactRefs(refs) {
+    return [...new Map(refs.map((ref) => [`${ref.id}:${ref.contentHash}`, ref])).values()];
+}
+function verifyContentHash(value, schema, label) {
+    const { contentHash: actual, ...content } = value;
+    const expected = contentHash(schema.parse(content));
+    if (actual !== expected)
+        throw new Error(`${label} content hash mismatch.`);
+}
+function hasValidContentHash(value, schema) {
+    try {
+        verifyContentHash(value, schema, "Receipt");
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/dist/graders.js

@@ -68,7 +68,7 @@ function gradeDeterministic(grader, task, attempt) {
     return {
         score: passed ? 1 : 0,
         passed,
-        rewardEligible: grader.rewardEligible && passed,
+        rewardEligible: grader.rewardEligible,
         failureClass: passed ? null : "policy_failure",
         feedback: [passed ? "Deterministic grader passed." : "Deterministic grader failed."],
         visibleEvidenceRefs: [...attempt.runtimeEventRefs, ...attempt.artifactRefs],
@@ -76,7 +76,13 @@ function gradeDeterministic(grader, task, attempt) {
     };
 }
 function evidence(grader, result) {
-    const content = GraderEvidenceContentSchema.parse({ schemaVersion: "openpond.graderEvidence.v1", graderId: grader.id, graderVersion: grader.version, ...result });
+    const content = GraderEvidenceContentSchema.parse({
+        schemaVersion: "openpond.graderEvidence.v1",
+        graderId: grader.id,
+        graderVersion: grader.version,
+        ...result,
+        rewardEligible: grader.rewardEligible && result.score !== null,
+    });
     return GraderEvidenceSchema.parse({ ...content, contentHash: contentHash(content) });
 }
 function unavailable(message) {

package/dist/index.js

@@ -1,9 +1,19 @@
 export * from "./compatibility.js";
 export * from "./benchmarks.js";
+export * from "./artifact-verification.js";
 export * from "./evidence/index.js";
+export * from "./execution-contracts.js";
+export * from "./execution-receipts.js";
 export * from "./graders.js";
 export * from "./harness.js";
+export * from "./learned-preference.js";
 export * from "./runs.js";
 export * from "./model-improvement-qualification.js";
+export * from "./preferences.js";
 export * from "./review-conformance.js";
+export * from "./rollouts.js";
+export * from "./telemetry.js";
+export * from "./telemetry-catalog.js";
+export * from "./telemetry-analysis.js";
+export * from "./telemetry-bundle.js";
 export * from "./tasksets.js";