@openpond/evals 0.11.16 → 0.11.17

package/dist/types/profile-external-evaluation.d.ts

@@ -0,0 +1,404 @@
+import { z } from "zod";
+export declare const ProfileExternalFieldMappingsSchema: z.ZodArray<z.ZodObject<{
+    graderId: z.ZodString;
+    fields: z.ZodArray<z.ZodObject<{
+        destination: z.ZodEnum<{
+            output: "output";
+            input: "input";
+            expectedOutput: "expectedOutput";
+            evaluatorContext: "evaluatorContext";
+        }>;
+        path: z.ZodArray<z.ZodString>;
+        source: z.ZodEnum<{
+            output: "output";
+            input: "input";
+            expectedOutput: "expectedOutput";
+            evaluatorContext: "evaluatorContext";
+        }>;
+        sourcePath: z.ZodArray<z.ZodString>;
+    }, z.core.$strict>>;
+}, z.core.$strict>>;
+export declare const ProfileExternalDatasetBindingContentSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.profileExternalDatasetBinding.v1">;
+    dataset: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+        revision: z.ZodNumber;
+    }, z.core.$strict>;
+    packageHash: z.ZodString;
+    profileId: z.ZodString;
+    target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"profile">;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"workflow">;
+        workflowId: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"skill">;
+        skillPath: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"agent_action">;
+        actionId: z.ZodString;
+    }, z.core.$strict>], "kind">;
+    declaredProfileCatalogHash: z.ZodNullable<z.ZodString>;
+    protectedProfileClosureHash: z.ZodString;
+    recordedOrigin: z.ZodOptional<z.ZodObject<{
+        execution: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        originalEvidence: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        members: z.ZodArray<z.ZodObject<{
+            caseId: z.ZodString;
+            seed: z.ZodString;
+            fixtureId: z.ZodString;
+            sourceId: z.ZodString;
+            snapshotHash: z.ZodString;
+            boundaryRevisionHash: z.ZodString;
+        }, z.core.$strict>>;
+    }, z.core.$strict>>;
+    gradingSource: z.ZodOptional<z.ZodObject<{
+        pass: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        evidence: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+    }, z.core.$strict>>;
+    split: z.ZodEnum<{
+        train: "train";
+        validation: "validation";
+        test: "test";
+        frozen_eval: "frozen_eval";
+    }>;
+    population: z.ZodArray<z.ZodObject<{
+        taskId: z.ZodString;
+        seed: z.ZodString;
+        fixtureId: z.ZodNullable<z.ZodString>;
+    }, z.core.$strict>>;
+    evaluators: z.ZodArray<z.ZodObject<{
+        release: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+            revision: z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>;
+        }, z.core.$strict>;
+        feedbackKey: z.ZodString;
+        configurationHash: z.ZodString;
+        output: z.ZodEnum<{
+            boolean: "boolean";
+            score: "score";
+            category: "category";
+        }>;
+        categories: z.ZodArray<z.ZodString>;
+    }, z.core.$strict>>;
+    fieldMappingsHash: z.ZodString;
+    fieldMappings: z.ZodArray<z.ZodObject<{
+        graderId: z.ZodString;
+        fields: z.ZodArray<z.ZodObject<{
+            destination: z.ZodEnum<{
+                output: "output";
+                input: "input";
+                expectedOutput: "expectedOutput";
+                evaluatorContext: "evaluatorContext";
+            }>;
+            path: z.ZodArray<z.ZodString>;
+            source: z.ZodEnum<{
+                output: "output";
+                input: "input";
+                expectedOutput: "expectedOutput";
+                evaluatorContext: "evaluatorContext";
+            }>;
+            sourcePath: z.ZodArray<z.ZodString>;
+        }, z.core.$strict>>;
+    }, z.core.$strict>>;
+    environmentHash: z.ZodString;
+    privateDatasetClosureHash: z.ZodString;
+    criterion: z.ZodObject<{
+        minimumPassRate: z.ZodNumber;
+        requireComplete: z.ZodBoolean;
+    }, z.core.$strict>;
+}, z.core.$strict>;
+export declare const ProfileExternalDatasetBindingSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.profileExternalDatasetBinding.v1">;
+    dataset: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+        revision: z.ZodNumber;
+    }, z.core.$strict>;
+    packageHash: z.ZodString;
+    profileId: z.ZodString;
+    target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"profile">;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"workflow">;
+        workflowId: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"skill">;
+        skillPath: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"agent_action">;
+        actionId: z.ZodString;
+    }, z.core.$strict>], "kind">;
+    declaredProfileCatalogHash: z.ZodNullable<z.ZodString>;
+    protectedProfileClosureHash: z.ZodString;
+    recordedOrigin: z.ZodOptional<z.ZodObject<{
+        execution: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        originalEvidence: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        members: z.ZodArray<z.ZodObject<{
+            caseId: z.ZodString;
+            seed: z.ZodString;
+            fixtureId: z.ZodString;
+            sourceId: z.ZodString;
+            snapshotHash: z.ZodString;
+            boundaryRevisionHash: z.ZodString;
+        }, z.core.$strict>>;
+    }, z.core.$strict>>;
+    gradingSource: z.ZodOptional<z.ZodObject<{
+        pass: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        evidence: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+    }, z.core.$strict>>;
+    split: z.ZodEnum<{
+        train: "train";
+        validation: "validation";
+        test: "test";
+        frozen_eval: "frozen_eval";
+    }>;
+    population: z.ZodArray<z.ZodObject<{
+        taskId: z.ZodString;
+        seed: z.ZodString;
+        fixtureId: z.ZodNullable<z.ZodString>;
+    }, z.core.$strict>>;
+    evaluators: z.ZodArray<z.ZodObject<{
+        release: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+            revision: z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>;
+        }, z.core.$strict>;
+        feedbackKey: z.ZodString;
+        configurationHash: z.ZodString;
+        output: z.ZodEnum<{
+            boolean: "boolean";
+            score: "score";
+            category: "category";
+        }>;
+        categories: z.ZodArray<z.ZodString>;
+    }, z.core.$strict>>;
+    fieldMappingsHash: z.ZodString;
+    fieldMappings: z.ZodArray<z.ZodObject<{
+        graderId: z.ZodString;
+        fields: z.ZodArray<z.ZodObject<{
+            destination: z.ZodEnum<{
+                output: "output";
+                input: "input";
+                expectedOutput: "expectedOutput";
+                evaluatorContext: "evaluatorContext";
+            }>;
+            path: z.ZodArray<z.ZodString>;
+            source: z.ZodEnum<{
+                output: "output";
+                input: "input";
+                expectedOutput: "expectedOutput";
+                evaluatorContext: "evaluatorContext";
+            }>;
+            sourcePath: z.ZodArray<z.ZodString>;
+        }, z.core.$strict>>;
+    }, z.core.$strict>>;
+    environmentHash: z.ZodString;
+    privateDatasetClosureHash: z.ZodString;
+    criterion: z.ZodObject<{
+        minimumPassRate: z.ZodNumber;
+        requireComplete: z.ZodBoolean;
+    }, z.core.$strict>;
+    contentHash: z.ZodString;
+}, z.core.$strict>;
+export type ProfileExternalDatasetBinding = z.infer<typeof ProfileExternalDatasetBindingSchema>;
+export declare function verifyProfileExternalDatasetBinding(raw: unknown): {
+    schemaVersion: "openpond.profileExternalDatasetBinding.v1";
+    dataset: {
+        id: string;
+        contentHash: string;
+        revision: number;
+    };
+    packageHash: string;
+    profileId: string;
+    target: {
+        kind: "profile";
+    } | {
+        kind: "workflow";
+        workflowId: string;
+    } | {
+        kind: "skill";
+        skillPath: string;
+    } | {
+        kind: "agent_action";
+        actionId: string;
+    };
+    declaredProfileCatalogHash: string | null;
+    protectedProfileClosureHash: string;
+    split: "train" | "validation" | "test" | "frozen_eval";
+    population: {
+        taskId: string;
+        seed: string;
+        fixtureId: string | null;
+    }[];
+    evaluators: {
+        release: {
+            id: string;
+            contentHash: string;
+            revision: string | number;
+        };
+        feedbackKey: string;
+        configurationHash: string;
+        output: "boolean" | "score" | "category";
+        categories: string[];
+    }[];
+    fieldMappingsHash: string;
+    fieldMappings: {
+        graderId: string;
+        fields: {
+            destination: "output" | "input" | "expectedOutput" | "evaluatorContext";
+            path: string[];
+            source: "output" | "input" | "expectedOutput" | "evaluatorContext";
+            sourcePath: string[];
+        }[];
+    }[];
+    environmentHash: string;
+    privateDatasetClosureHash: string;
+    criterion: {
+        minimumPassRate: number;
+        requireComplete: boolean;
+    };
+    contentHash: string;
+    recordedOrigin?: {
+        execution: {
+            id: string;
+            contentHash: string;
+        };
+        originalEvidence: {
+            id: string;
+            contentHash: string;
+        };
+        members: {
+            caseId: string;
+            seed: string;
+            fixtureId: string;
+            sourceId: string;
+            snapshotHash: string;
+            boundaryRevisionHash: string;
+        }[];
+    } | undefined;
+    gradingSource?: {
+        pass: {
+            id: string;
+            contentHash: string;
+        };
+        evidence: {
+            id: string;
+            contentHash: string;
+        };
+    } | undefined;
+};
+export declare function createProfileExternalDatasetBinding(raw: z.input<typeof ProfileExternalDatasetBindingContentSchema>): {
+    schemaVersion: "openpond.profileExternalDatasetBinding.v1";
+    dataset: {
+        id: string;
+        contentHash: string;
+        revision: number;
+    };
+    packageHash: string;
+    profileId: string;
+    target: {
+        kind: "profile";
+    } | {
+        kind: "workflow";
+        workflowId: string;
+    } | {
+        kind: "skill";
+        skillPath: string;
+    } | {
+        kind: "agent_action";
+        actionId: string;
+    };
+    declaredProfileCatalogHash: string | null;
+    protectedProfileClosureHash: string;
+    split: "train" | "validation" | "test" | "frozen_eval";
+    population: {
+        taskId: string;
+        seed: string;
+        fixtureId: string | null;
+    }[];
+    evaluators: {
+        release: {
+            id: string;
+            contentHash: string;
+            revision: string | number;
+        };
+        feedbackKey: string;
+        configurationHash: string;
+        output: "boolean" | "score" | "category";
+        categories: string[];
+    }[];
+    fieldMappingsHash: string;
+    fieldMappings: {
+        graderId: string;
+        fields: {
+            destination: "output" | "input" | "expectedOutput" | "evaluatorContext";
+            path: string[];
+            source: "output" | "input" | "expectedOutput" | "evaluatorContext";
+            sourcePath: string[];
+        }[];
+    }[];
+    environmentHash: string;
+    privateDatasetClosureHash: string;
+    criterion: {
+        minimumPassRate: number;
+        requireComplete: boolean;
+    };
+    contentHash: string;
+    recordedOrigin?: {
+        execution: {
+            id: string;
+            contentHash: string;
+        };
+        originalEvidence: {
+            id: string;
+            contentHash: string;
+        };
+        members: {
+            caseId: string;
+            seed: string;
+            fixtureId: string;
+            sourceId: string;
+            snapshotHash: string;
+            boundaryRevisionHash: string;
+        }[];
+    } | undefined;
+    gradingSource?: {
+        pass: {
+            id: string;
+            contentHash: string;
+        };
+        evidence: {
+            id: string;
+            contentHash: string;
+        };
+    } | undefined;
+};
+export declare const externalDatasetDefinitionId: (binding: ProfileExternalDatasetBinding) => string;
+//# sourceMappingURL=profile-external-evaluation.d.ts.map

package/dist/types/profile-external-evaluation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"profile-external-evaluation.d.ts","sourceRoot":"","sources":["../../../../src/profile-external-evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,eAAO,MAAM,kCAAkC;;;;;;;;;;;;;;;;;;mBAAoN,CAAC;AAKpQ,eAAO,MAAM,0CAA0C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAUnD,CAAC;AACL,eAAO,MAAM,mCAAmC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAgG,CAAC;AACjJ,MAAM,MAAM,6BAA6B,GAAC,CAAC,CAAC,KAAK,CAAC,OAAO,mCAAmC,CAAC,CAAC;AAC9F,wBAAgB,mCAAmC,CAAC,GAAG,EAAC,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAAsM;AACrQ,wBAAgB,mCAAmC,CAAC,GAAG,EAAC,CAAC,CAAC,KAAK,CAAC,OAAO,0CAA0C,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAA2J;AAC7Q,eAAO,MAAM,2BAA2B,GAAE,SAAQ,6BAA6B,WAA6F,CAAC"}

package/dist/types/profile-source-candidate.d.ts

@@ -0,0 +1,15 @@
+import { z } from "zod";
+/** An identity to resolve through the actual private hosted source owner. It is
+ * never itself an authorization or a model-artifact declaration. */
+export declare const ProfileSourceCandidateSchema: z.ZodObject<{
+    candidateId: z.ZodString;
+    frozenRevision: z.ZodNumber;
+    frozenHash: z.ZodString;
+    harnessRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    protectedClosureHash: z.ZodString;
+}, z.core.$strict>;
+export type ProfileSourceCandidate = z.infer<typeof ProfileSourceCandidateSchema>;
+//# sourceMappingURL=profile-source-candidate.d.ts.map

package/dist/types/profile-source-candidate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"profile-source-candidate.d.ts","sourceRoot":"","sources":["../../../../src/profile-source-candidate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,CAAC,EAAC,MAAM,KAAK,CAAC;AAEtB;oEACoE;AACpE,eAAO,MAAM,4BAA4B;;;;;;;;;kBAAsP,CAAC;AAChS,MAAM,MAAM,sBAAsB,GAAC,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC"}

package/dist/types/review-conformance.d.ts

@@ -3,7 +3,7 @@ export declare const harnessEvaluationReviewConformance: {
         schemaVersion: "openpond.harnessEvaluationReviewReceipt.v1";
         id: string;
         ownerScope: {
-            kind: "personal" | "team";
+            kind: "team" | "personal";
             id: string;
         };
         workspaceRef: string;
@@ -94,7 +94,7 @@ export declare const harnessEvaluationReviewConformance: {
         schemaVersion: "openpond.harnessEvaluationReviewReceipt.v1";
         id: string;
         ownerScope: {
-            kind: "personal" | "team";
+            kind: "team" | "personal";
             id: string;
         };
         workspaceRef: string;
@@ -185,7 +185,7 @@ export declare const harnessEvaluationReviewConformance: {
         schemaVersion: "openpond.harnessEvaluationReviewReceipt.v1";
         id: string;
         ownerScope: {
-            kind: "personal" | "team";
+            kind: "team" | "personal";
             id: string;
         };
         workspaceRef: string;
@@ -276,7 +276,7 @@ export declare const harnessEvaluationReviewConformance: {
         schemaVersion: "openpond.harnessEvaluationReviewReceipt.v1";
         id: string;
         ownerScope: {
-            kind: "personal" | "team";
+            kind: "team" | "personal";
             id: string;
         };
         workspaceRef: string;
@@ -434,7 +434,7 @@ export declare const harnessEvaluationReviewConformance: {
                 contentHash: string;
             }[];
         };
-        decision: "no_training" | "sft" | "preference" | "rl";
+        decision: "preference" | "no_training" | "sft" | "rl";
         reasons: string[];
         createdAt: string;
         metadata: Record<string, unknown>;
@@ -511,7 +511,7 @@ export declare const harnessEvaluationReviewConformance: {
                 contentHash: string;
             }[];
         };
-        decision: "no_training" | "sft" | "preference" | "rl";
+        decision: "preference" | "no_training" | "sft" | "rl";
         reasons: string[];
         createdAt: string;
         metadata: Record<string, unknown>;
@@ -521,7 +521,7 @@ export declare const harnessEvaluationReviewConformance: {
         schemaVersion: "openpond.harnessEvaluationReviewReceipt.v1";
         id: string;
         ownerScope: {
-            kind: "personal" | "team";
+            kind: "team" | "personal";
             id: string;
         };
         workspaceRef: string;