@openpond/evals 0.4.2 → 0.6.0

package/dist/types/execution-contracts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"execution-contracts.d.ts","sourceRoot":"","sources":["../../../../src/execution-contracts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAcxB,eAAO,MAAM,mBAAmB;;;EAAmC,CAAC;AACpE,eAAO,MAAM,kBAAkB;;;;;;;;;EAS7B,CAAC;AACH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;EAYpC,CAAC;AAEH,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAcjC,CAAC;AACZ,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAE1B,CAAC;AAEZ,eAAO,MAAM,8BAA8B;;;;;EAKzC,CAAC;AACH,eAAO,MAAM,8BAA8B;;;;EAIzC,CAAC;AACH,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAa7B,CAAC;AACZ,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAO/B,CAAC;AACZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAExB,CAAC;AAEZ,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAYjC,CAAC;AACZ,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAE1B,CAAC;AAEZ,eAAO,MAAM,4BAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAuBvC,CAAC;AAkCH,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAA6D,CAAC;AACrG,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAGK,CAAC;AAEtC,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAChE,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAC9D,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAC1E,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC;AAChF,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACtE,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAC1E,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAClF,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC"}

package/dist/types/execution-receipts.d.ts

@@ -0,0 +1,55 @@
+import type { FailureClass, ImmutableArtifactRef } from "@openpond/harness";
+import { ArtifactManifestContentSchema, EnvironmentReleaseContentSchema, VerifierSetReleaseContentSchema, type ArtifactManifest, type AttemptOutcomeClass, type EnvironmentRelease, type FailureOwner, type RewardComponentReceipt, type RewardReceipt, type VerifierSetRelease } from "./execution-contracts.js";
+import { type TasksetRelease } from "./tasksets.js";
+export declare function createEnvironmentRelease(input: Parameters<typeof EnvironmentReleaseContentSchema.parse>[0]): EnvironmentRelease;
+export declare function createArtifactManifest(input: Parameters<typeof ArtifactManifestContentSchema.parse>[0]): ArtifactManifest;
+export declare function createVerifierSetRelease(input: Parameters<typeof VerifierSetReleaseContentSchema.parse>[0]): VerifierSetRelease;
+export declare function bindTasksetExecutionReleases(input: {
+    taskset: TasksetRelease;
+    environment: EnvironmentRelease;
+    verifierSet: VerifierSetRelease;
+}): TasksetRelease;
+export declare function classifyAttemptOutcome(input: {
+    failureClass: FailureClass | null;
+    timeoutKind?: "task_deadline" | "infrastructure_timeout";
+}): {
+    outcomeClass: AttemptOutcomeClass;
+    failureOwner: FailureOwner | null;
+};
+export declare function createRewardReceipt(input: {
+    id: string;
+    attemptRef: {
+        id: string;
+        contentHash: string;
+    };
+    verifierSet: VerifierSetRelease;
+    artifactManifest: ArtifactManifest;
+    outcomeClass: AttemptOutcomeClass;
+    failureOwner: FailureOwner | null;
+    components: RewardComponentReceipt[];
+    visibleEvidenceRefs?: ImmutableArtifactRef[];
+    privilegedEvidenceRefs?: ImmutableArtifactRef[];
+    supersedes?: {
+        id: string;
+        contentHash: string;
+    } | null;
+    createdAt: string;
+    metadata?: Record<string, unknown>;
+}): RewardReceipt;
+export declare function regradeRewardReceipt(input: {
+    original: RewardReceipt;
+    id: string;
+    verifierSet: VerifierSetRelease;
+    artifactManifest: ArtifactManifest;
+    components: RewardComponentReceipt[];
+    outcomeClass?: AttemptOutcomeClass;
+    failureOwner?: FailureOwner | null;
+    createdAt: string;
+    metadata?: Record<string, unknown>;
+}): RewardReceipt;
+export declare function verifyEnvironmentRelease(release: EnvironmentRelease): boolean;
+export declare function verifyArtifactManifest(manifest: ArtifactManifest): boolean;
+export declare function verifyVerifierSetRelease(release: VerifierSetRelease): boolean;
+export declare function verifyRewardReceipt(receipt: RewardReceipt): boolean;
+export declare function isScorableOutcome(outcome: AttemptOutcomeClass): boolean;
+//# sourceMappingURL=execution-receipts.d.ts.map

package/dist/types/execution-receipts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"execution-receipts.d.ts","sourceRoot":"","sources":["../../../../src/execution-receipts.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAE5E,OAAO,EACL,6BAA6B,EAE7B,+BAA+B,EAK/B,+BAA+B,EAE/B,KAAK,gBAAgB,EACrB,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACvB,KAAK,YAAY,EACjB,KAAK,sBAAsB,EAC3B,KAAK,aAAa,EAClB,KAAK,kBAAkB,EACxB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAGL,KAAK,cAAc,EACpB,MAAM,eAAe,CAAC;AAEvB,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,UAAU,CAAC,OAAO,+BAA+B,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GACjE,kBAAkB,CAGpB;AAED,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,UAAU,CAAC,OAAO,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAC/D,gBAAgB,CAGlB;AAED,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,UAAU,CAAC,OAAO,+BAA+B,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GACjE,kBAAkB,CAGpB;AAED,wBAAgB,4BAA4B,CAAC,KAAK,EAAE;IAClD,OAAO,EAAE,cAAc,CAAC;IACxB,WAAW,EAAE,kBAAkB,CAAC;IAChC,WAAW,EAAE,kBAAkB,CAAC;CACjC,GAAG,cAAc,CAsBjB;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE;IAC5C,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAClC,WAAW,CAAC,EAAE,eAAe,GAAG,wBAAwB,CAAC;CAC1D,GAAG;IAAE,YAAY,EAAE,mBAAmB,CAAC;IAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAA;CAAE,CAmB3E;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE;IACzC,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,WAAW,EAAE,kBAAkB,CAAC;IAChC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,YAAY,EAAE,mBAAmB,CAAC;IAClC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAClC,UAAU,EAAE,sBAAsB,EAAE,CAAC;IACrC,mBAAmB,CAAC,EAAE,oBAAoB,EAAE,CAAC;IAC7C,sBAAsB,CAAC,EAAE,oBAAoB,EAAE,CAAC;IAChD,UAAU,CAAC,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACxD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,aAAa,CAoEhB;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE;IAC1C,QAAQ,EAAE,aAAa,CAAC;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,kBAAkB,CAAC;IAChC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,UAAU,EAAE,sBAAsB,EAAE,CAAC;IACrC,YAAY,CAAC,EAAE,mBAAmB,CAAC;IACnC,YAAY,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,aAAa,CAwBhB;AAED,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAE7E;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,gBAAgB,GAAG,OAAO,CAE1E;AAED,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAE7E;AAED,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAKnE;AAED,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAKvE"}

package/dist/types/graders.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"graders.d.ts","sourceRoot":"","sources":["../../../../src/graders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,KAAK,EAA2B,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAErF,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;kBAW7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC,CAAC;AACF,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAC/O,MAAM,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,iBAAiB,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAEvP,wBAAsB,aAAa,CAAC,KAAK,EAAE;IACzC,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,eAAe,CAAC;IAC1B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACvC,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAwB5B;AA4CD,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC"}
+{"version":3,"file":"graders.d.ts","sourceRoot":"","sources":["../../../../src/graders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,KAAK,EAA2B,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAErF,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;kBAW7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC,CAAC;AACF,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAC/O,MAAM,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,iBAAiB,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAEvP,wBAAsB,aAAa,CAAC,KAAK,EAAE;IACzC,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,eAAe,CAAC;IAC1B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACvC,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAwB5B;AAkDD,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC"}

package/dist/types/index.d.ts

@@ -1,10 +1,20 @@
 export * from "./compatibility.js";
 export * from "./benchmarks.js";
+export * from "./artifact-verification.js";
 export * from "./evidence/index.js";
+export * from "./execution-contracts.js";
+export * from "./execution-receipts.js";
 export * from "./graders.js";
 export * from "./harness.js";
+export * from "./learned-preference.js";
 export * from "./runs.js";
 export * from "./model-improvement-qualification.js";
+export * from "./preferences.js";
 export * from "./review-conformance.js";
+export * from "./rollouts.js";
+export * from "./telemetry.js";
+export * from "./telemetry-catalog.js";
+export * from "./telemetry-analysis.js";
+export * from "./telemetry-bundle.js";
 export * from "./tasksets.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAC1B,cAAc,sCAAsC,CAAC;AACrD,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC"}

package/dist/types/learned-preference.d.ts

@@ -0,0 +1,282 @@
+import { z } from "zod";
+import { type ComparisonAssignment, type PreferenceComparisonRelease, type PreferenceReceipt } from "./preferences.js";
+import type { TasksetRelease } from "./tasksets.js";
+declare const PreferenceDatasetPartitionSchema: z.ZodEnum<{
+    reward_train: "reward_train";
+    reward_validation: "reward_validation";
+    reward_qualification: "reward_qualification";
+}>;
+export declare const PreferenceEvidenceAuthoritySchema: z.ZodEnum<{
+    human: "human";
+    synthetic_fixture: "synthetic_fixture";
+}>;
+export declare const RewardModelQualificationKindSchema: z.ZodEnum<{
+    synthetic_smoke: "synthetic_smoke";
+    human_heldout: "human_heldout";
+}>;
+export declare const PreferenceDatasetReleaseContentSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.preferenceDatasetRelease.v1">;
+    id: z.ZodString;
+    revision: z.ZodNumber;
+    tasksetRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    comparisonRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    authority: z.ZodEnum<{
+        human: "human";
+        synthetic_fixture: "synthetic_fixture";
+    }>;
+    qualificationEligibility: z.ZodEnum<{
+        smoke_only: "smoke_only";
+        human_heldout: "human_heldout";
+    }>;
+    fixtureRelease: z.ZodNullable<z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>>;
+    groups: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        assignmentRef: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        scenarioRef: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        scenarioSplit: z.ZodEnum<{
+            train: "train";
+            validation: "validation";
+        }>;
+        preferenceResultRef: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        receiptRefs: z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>>;
+        attemptRefs: z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>>;
+        artifactManifestRefs: z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>>;
+        orderedBuckets: z.ZodArray<z.ZodArray<z.ZodString>>;
+        rejectAll: z.ZodBoolean;
+        partition: z.ZodEnum<{
+            reward_train: "reward_train";
+            reward_validation: "reward_validation";
+            reward_qualification: "reward_qualification";
+        }>;
+        metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, z.core.$strict>>;
+    derivedPairs: z.ZodArray<z.ZodObject<{
+        groupId: z.ZodString;
+        preferredAttemptRef: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        dispreferredAttemptRef: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        relation: z.ZodEnum<{
+            preferred: "preferred";
+            tie: "tie";
+        }>;
+    }, z.core.$strict>>;
+    createdAt: z.ZodString;
+    metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$strict>;
+export declare const PreferenceDatasetReleaseSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.preferenceDatasetRelease.v1">;
+    id: z.ZodString;
+    revision: z.ZodNumber;
+    tasksetRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    comparisonRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    authority: z.ZodEnum<{
+        human: "human";
+        synthetic_fixture: "synthetic_fixture";
+    }>;
+    qualificationEligibility: z.ZodEnum<{
+        smoke_only: "smoke_only";
+        human_heldout: "human_heldout";
+    }>;
+    fixtureRelease: z.ZodNullable<z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>>;
+    groups: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        assignmentRef: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        scenarioRef: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        scenarioSplit: z.ZodEnum<{
+            train: "train";
+            validation: "validation";
+        }>;
+        preferenceResultRef: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        receiptRefs: z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>>;
+        attemptRefs: z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>>;
+        artifactManifestRefs: z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>>;
+        orderedBuckets: z.ZodArray<z.ZodArray<z.ZodString>>;
+        rejectAll: z.ZodBoolean;
+        partition: z.ZodEnum<{
+            reward_train: "reward_train";
+            reward_validation: "reward_validation";
+            reward_qualification: "reward_qualification";
+        }>;
+        metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, z.core.$strict>>;
+    derivedPairs: z.ZodArray<z.ZodObject<{
+        groupId: z.ZodString;
+        preferredAttemptRef: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        dispreferredAttemptRef: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        relation: z.ZodEnum<{
+            preferred: "preferred";
+            tie: "tie";
+        }>;
+    }, z.core.$strict>>;
+    createdAt: z.ZodString;
+    metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    contentHash: z.ZodString;
+}, z.core.$strict>;
+export declare const RewardModelQualificationReportContentSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.rewardModelQualificationReport.v1">;
+    id: z.ZodString;
+    kind: z.ZodEnum<{
+        synthetic_smoke: "synthetic_smoke";
+        human_heldout: "human_heldout";
+    }>;
+    rewardModelVersion: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    preferenceDatasetRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    tasksetRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    processorRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    metrics: z.ZodObject<{
+        sampleCount: z.ZodNumber;
+        finiteScoreRate: z.ZodNumber;
+        scoreVariance: z.ZodNumber;
+        checkpointReloadPassed: z.ZodBoolean;
+        processorCompatibilityPassed: z.ZodBoolean;
+        invalidAttemptExclusionPassed: z.ZodBoolean;
+        orderedPairAccuracy: z.ZodNullable<z.ZodNumber>;
+        bucketAccuracy: z.ZodNullable<z.ZodNumber>;
+        tieAgreement: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>;
+    passed: z.ZodBoolean;
+    productionRewardEligible: z.ZodBoolean;
+    createdAt: z.ZodString;
+    metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$strict>;
+export declare const RewardModelQualificationReportSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.rewardModelQualificationReport.v1">;
+    id: z.ZodString;
+    kind: z.ZodEnum<{
+        synthetic_smoke: "synthetic_smoke";
+        human_heldout: "human_heldout";
+    }>;
+    rewardModelVersion: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    preferenceDatasetRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    tasksetRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    processorRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    metrics: z.ZodObject<{
+        sampleCount: z.ZodNumber;
+        finiteScoreRate: z.ZodNumber;
+        scoreVariance: z.ZodNumber;
+        checkpointReloadPassed: z.ZodBoolean;
+        processorCompatibilityPassed: z.ZodBoolean;
+        invalidAttemptExclusionPassed: z.ZodBoolean;
+        orderedPairAccuracy: z.ZodNullable<z.ZodNumber>;
+        bucketAccuracy: z.ZodNullable<z.ZodNumber>;
+        tieAgreement: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>;
+    passed: z.ZodBoolean;
+    productionRewardEligible: z.ZodBoolean;
+    createdAt: z.ZodString;
+    metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    contentHash: z.ZodString;
+}, z.core.$strict>;
+export declare function createPreferenceDatasetRelease(input: z.input<typeof PreferenceDatasetReleaseContentSchema>): PreferenceDatasetRelease;
+export declare function materializePreferenceDatasetRelease(input: {
+    id: string;
+    revision: number;
+    tasksetRelease: TasksetRelease;
+    comparisonRelease: PreferenceComparisonRelease;
+    authority: "human" | "synthetic_fixture";
+    groups: ReadonlyArray<{
+        assignment: ComparisonAssignment;
+        receipt: PreferenceReceipt;
+        partition: z.infer<typeof PreferenceDatasetPartitionSchema>;
+    }>;
+    createdAt: string;
+    metadata?: Record<string, unknown>;
+}): PreferenceDatasetRelease;
+export declare function verifyPreferenceDatasetRelease(value: unknown): value is PreferenceDatasetRelease;
+export declare function createRewardModelQualificationReport(input: z.input<typeof RewardModelQualificationReportContentSchema>): RewardModelQualificationReport;
+export declare function verifyRewardModelQualificationReport(value: unknown): value is RewardModelQualificationReport;
+export type PreferenceEvidenceAuthority = z.infer<typeof PreferenceEvidenceAuthoritySchema>;
+export type PreferenceDatasetRelease = z.infer<typeof PreferenceDatasetReleaseSchema>;
+export type RewardModelQualificationKind = z.infer<typeof RewardModelQualificationKindSchema>;
+export type RewardModelQualificationReport = z.infer<typeof RewardModelQualificationReportSchema>;
+export {};
+//# sourceMappingURL=learned-preference.d.ts.map

package/dist/types/learned-preference.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"learned-preference.d.ts","sourceRoot":"","sources":["../../../../src/learned-preference.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,OAAO,EAGL,KAAK,oBAAoB,EACzB,KAAK,2BAA2B,EAChC,KAAK,iBAAiB,EACvB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAEpD,QAAA,MAAM,gCAAgC;;;;EAIpC,CAAC;AAEH,eAAO,MAAM,iCAAiC;;;EAG5C,CAAC;AAEH,eAAO,MAAM,kCAAkC;;;EAG7C,CAAC;AAwEH,eAAO,MAAM,qCAAqC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAuEhD,CAAC;AAEH,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAEhC,CAAC;AAcZ,eAAO,MAAM,2CAA2C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBA+BtD,CAAC;AAEH,eAAO,MAAM,oCAAoC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAGpC,CAAC;AAEd,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,qCAAqC,CAAC,GAC3D,wBAAwB,CAM1B;AAED,wBAAgB,mCAAmC,CAAC,KAAK,EAAE;IACzD,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,cAAc,CAAC;IAC/B,iBAAiB,EAAE,2BAA2B,CAAC;IAC/C,SAAS,EAAE,OAAO,GAAG,mBAAmB,CAAC;IACzC,MAAM,EAAE,aAAa,CAAC;QACpB,UAAU,EAAE,oBAAoB,CAAC;QACjC,OAAO,EAAE,iBAAiB,CAAC;QAC3B,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,gCAAgC,CAAC,CAAC;KAC7D,CAAC,CAAC;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,wBAAwB,CA4E3B;AAED,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,OAAO,GACb,KAAK,IAAI,wBAAwB,CAMnC;AAED,wBAAgB,oCAAoC,CAClD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,2CAA2C,CAAC,GACjE,8BAA8B,CAMhC;AAED,wBAAgB,oCAAoC,CAClD,KAAK,EAAE,OAAO,GACb,KAAK,IAAI,8BAA8B,CAMzC;AAgED,MAAM,MAAM,2BAA2B,GAAG,CAAC,CAAC,KAAK,CAC/C,OAAO,iCAAiC,CACzC,CAAC;AACF,MAAM,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAC5C,OAAO,8BAA8B,CACtC,CAAC;AACF,MAAM,MAAM,4BAA4B,GAAG,CAAC,CAAC,KAAK,CAChD,OAAO,kCAAkC,CAC1C,CAAC;AACF,MAAM,MAAM,8BAA8B,GAAG,CAAC,CAAC,KAAK,CAClD,OAAO,oCAAoC,CAC5C,CAAC"}