@openpond/evals 0.11.21 → 0.12.1

package/dist/types/profile-external-evaluation.d.ts

@@ -3,17 +3,17 @@ export declare const ProfileExternalFieldMappingsSchema: z.ZodArray<z.ZodObject<
     graderId: z.ZodString;
     fields: z.ZodArray<z.ZodObject<{
         destination: z.ZodEnum<{
-            output: "output";
-            input: "input";
-            expectedOutput: "expectedOutput";
             evaluatorContext: "evaluatorContext";
+            expectedOutput: "expectedOutput";
+            input: "input";
+            output: "output";
         }>;
         path: z.ZodArray<z.ZodString>;
         source: z.ZodEnum<{
-            output: "output";
-            input: "input";
-            expectedOutput: "expectedOutput";
             evaluatorContext: "evaluatorContext";
+            expectedOutput: "expectedOutput";
+            input: "input";
+            output: "output";
         }>;
         sourcePath: z.ZodArray<z.ZodString>;
     }, z.core.$strict>>;
@@ -70,10 +70,10 @@ export declare const ProfileExternalDatasetBindingContentSchema: z.ZodObject<{
         }, z.core.$strict>;
     }, z.core.$strict>>;
     split: z.ZodEnum<{
+        frozen_eval: "frozen_eval";
+        test: "test";
         train: "train";
         validation: "validation";
-        test: "test";
-        frozen_eval: "frozen_eval";
     }>;
     population: z.ZodArray<z.ZodObject<{
         taskId: z.ZodString;
@@ -90,8 +90,8 @@ export declare const ProfileExternalDatasetBindingContentSchema: z.ZodObject<{
         configurationHash: z.ZodString;
         output: z.ZodEnum<{
             boolean: "boolean";
-            score: "score";
             category: "category";
+            score: "score";
         }>;
         categories: z.ZodArray<z.ZodString>;
     }, z.core.$strict>>;
@@ -100,17 +100,17 @@ export declare const ProfileExternalDatasetBindingContentSchema: z.ZodObject<{
         graderId: z.ZodString;
         fields: z.ZodArray<z.ZodObject<{
             destination: z.ZodEnum<{
-                output: "output";
-                input: "input";
-                expectedOutput: "expectedOutput";
                 evaluatorContext: "evaluatorContext";
+                expectedOutput: "expectedOutput";
+                input: "input";
+                output: "output";
             }>;
             path: z.ZodArray<z.ZodString>;
             source: z.ZodEnum<{
-                output: "output";
-                input: "input";
-                expectedOutput: "expectedOutput";
                 evaluatorContext: "evaluatorContext";
+                expectedOutput: "expectedOutput";
+                input: "input";
+                output: "output";
             }>;
             sourcePath: z.ZodArray<z.ZodString>;
         }, z.core.$strict>>;
@@ -174,10 +174,10 @@ export declare const ProfileExternalDatasetBindingSchema: z.ZodObject<{
         }, z.core.$strict>;
     }, z.core.$strict>>;
     split: z.ZodEnum<{
+        frozen_eval: "frozen_eval";
+        test: "test";
         train: "train";
         validation: "validation";
-        test: "test";
-        frozen_eval: "frozen_eval";
     }>;
     population: z.ZodArray<z.ZodObject<{
         taskId: z.ZodString;
@@ -194,8 +194,8 @@ export declare const ProfileExternalDatasetBindingSchema: z.ZodObject<{
         configurationHash: z.ZodString;
         output: z.ZodEnum<{
             boolean: "boolean";
-            score: "score";
             category: "category";
+            score: "score";
         }>;
         categories: z.ZodArray<z.ZodString>;
     }, z.core.$strict>>;
@@ -204,17 +204,17 @@ export declare const ProfileExternalDatasetBindingSchema: z.ZodObject<{
         graderId: z.ZodString;
         fields: z.ZodArray<z.ZodObject<{
             destination: z.ZodEnum<{
-                output: "output";
-                input: "input";
-                expectedOutput: "expectedOutput";
                 evaluatorContext: "evaluatorContext";
+                expectedOutput: "expectedOutput";
+                input: "input";
+                output: "output";
             }>;
             path: z.ZodArray<z.ZodString>;
             source: z.ZodEnum<{
-                output: "output";
-                input: "input";
-                expectedOutput: "expectedOutput";
                 evaluatorContext: "evaluatorContext";
+                expectedOutput: "expectedOutput";
+                input: "input";
+                output: "output";
             }>;
             sourcePath: z.ZodArray<z.ZodString>;
         }, z.core.$strict>>;
@@ -251,7 +251,35 @@ export declare function verifyProfileExternalDatasetBinding(raw: unknown): {
     };
     declaredProfileCatalogHash: string | null;
     protectedProfileClosureHash: string;
-    split: "train" | "validation" | "test" | "frozen_eval";
+    recordedOrigin?: {
+        execution: {
+            id: string;
+            contentHash: string;
+        };
+        originalEvidence: {
+            id: string;
+            contentHash: string;
+        };
+        members: {
+            caseId: string;
+            seed: string;
+            fixtureId: string;
+            sourceId: string;
+            snapshotHash: string;
+            boundaryRevisionHash: string;
+        }[];
+    } | undefined;
+    gradingSource?: {
+        pass: {
+            id: string;
+            contentHash: string;
+        };
+        evidence: {
+            id: string;
+            contentHash: string;
+        };
+    } | undefined;
+    split: "frozen_eval" | "test" | "train" | "validation";
     population: {
         taskId: string;
         seed: string;
@@ -265,16 +293,16 @@ export declare function verifyProfileExternalDatasetBinding(raw: unknown): {
         };
         feedbackKey: string;
         configurationHash: string;
-        output: "boolean" | "score" | "category";
+        output: "boolean" | "category" | "score";
         categories: string[];
     }[];
     fieldMappingsHash: string;
     fieldMappings: {
         graderId: string;
         fields: {
-            destination: "output" | "input" | "expectedOutput" | "evaluatorContext";
+            destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
             path: string[];
-            source: "output" | "input" | "expectedOutput" | "evaluatorContext";
+            source: "evaluatorContext" | "expectedOutput" | "input" | "output";
             sourcePath: string[];
         }[];
     }[];
@@ -285,6 +313,30 @@ export declare function verifyProfileExternalDatasetBinding(raw: unknown): {
         requireComplete: boolean;
     };
     contentHash: string;
+};
+export declare function createProfileExternalDatasetBinding(raw: z.input<typeof ProfileExternalDatasetBindingContentSchema>): {
+    schemaVersion: "openpond.profileExternalDatasetBinding.v1";
+    dataset: {
+        id: string;
+        contentHash: string;
+        revision: number;
+    };
+    packageHash: string;
+    profileId: string;
+    target: {
+        kind: "profile";
+    } | {
+        kind: "workflow";
+        workflowId: string;
+    } | {
+        kind: "skill";
+        skillPath: string;
+    } | {
+        kind: "agent_action";
+        actionId: string;
+    };
+    declaredProfileCatalogHash: string | null;
+    protectedProfileClosureHash: string;
     recordedOrigin?: {
         execution: {
             id: string;
@@ -313,31 +365,7 @@ export declare function verifyProfileExternalDatasetBinding(raw: unknown): {
             contentHash: string;
         };
     } | undefined;
-};
-export declare function createProfileExternalDatasetBinding(raw: z.input<typeof ProfileExternalDatasetBindingContentSchema>): {
-    schemaVersion: "openpond.profileExternalDatasetBinding.v1";
-    dataset: {
-        id: string;
-        contentHash: string;
-        revision: number;
-    };
-    packageHash: string;
-    profileId: string;
-    target: {
-        kind: "profile";
-    } | {
-        kind: "workflow";
-        workflowId: string;
-    } | {
-        kind: "skill";
-        skillPath: string;
-    } | {
-        kind: "agent_action";
-        actionId: string;
-    };
-    declaredProfileCatalogHash: string | null;
-    protectedProfileClosureHash: string;
-    split: "train" | "validation" | "test" | "frozen_eval";
+    split: "frozen_eval" | "test" | "train" | "validation";
     population: {
         taskId: string;
         seed: string;
@@ -351,16 +379,16 @@ export declare function createProfileExternalDatasetBinding(raw: z.input<typeof
         };
         feedbackKey: string;
         configurationHash: string;
-        output: "boolean" | "score" | "category";
+        output: "boolean" | "category" | "score";
         categories: string[];
     }[];
     fieldMappingsHash: string;
     fieldMappings: {
         graderId: string;
         fields: {
-            destination: "output" | "input" | "expectedOutput" | "evaluatorContext";
+            destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
             path: string[];
-            source: "output" | "input" | "expectedOutput" | "evaluatorContext";
+            source: "evaluatorContext" | "expectedOutput" | "input" | "output";
             sourcePath: string[];
         }[];
     }[];
@@ -371,34 +399,6 @@ export declare function createProfileExternalDatasetBinding(raw: z.input<typeof
         requireComplete: boolean;
     };
     contentHash: string;
-    recordedOrigin?: {
-        execution: {
-            id: string;
-            contentHash: string;
-        };
-        originalEvidence: {
-            id: string;
-            contentHash: string;
-        };
-        members: {
-            caseId: string;
-            seed: string;
-            fixtureId: string;
-            sourceId: string;
-            snapshotHash: string;
-            boundaryRevisionHash: string;
-        }[];
-    } | undefined;
-    gradingSource?: {
-        pass: {
-            id: string;
-            contentHash: string;
-        };
-        evidence: {
-            id: string;
-            contentHash: string;
-        };
-    } | undefined;
 };
 export declare const externalDatasetDefinitionId: (binding: ProfileExternalDatasetBinding) => string;
 //# sourceMappingURL=profile-external-evaluation.d.ts.map

package/dist/types/profile-external-evaluation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"profile-external-evaluation.d.ts","sourceRoot":"","sources":["../../../../src/profile-external-evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,eAAO,MAAM,kCAAkC;;;;;;;;;;;;;;;;;;mBAAoN,CAAC;AAKpQ,eAAO,MAAM,0CAA0C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAUnD,CAAC;AACL,eAAO,MAAM,mCAAmC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAgG,CAAC;AACjJ,MAAM,MAAM,6BAA6B,GAAC,CAAC,CAAC,KAAK,CAAC,OAAO,mCAAmC,CAAC,CAAC;AAC9F,wBAAgB,mCAAmC,CAAC,GAAG,EAAC,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAAsM;AACrQ,wBAAgB,mCAAmC,CAAC,GAAG,EAAC,CAAC,CAAC,KAAK,CAAC,OAAO,0CAA0C,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAA2J;AAC7Q,eAAO,MAAM,2BAA2B,GAAE,SAAQ,6BAA6B,WAA6F,CAAC"}
+{"version":3,"file":"profile-external-evaluation.d.ts","sourceRoot":"","sources":["../../src/profile-external-evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,eAAO,MAAM,kCAAkC;;;;;;;;;;;;;;;;;;mBAAoN,CAAC;AAKpQ,eAAO,MAAM,0CAA0C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAUnD,CAAC;AACL,eAAO,MAAM,mCAAmC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAgG,CAAC;AACjJ,MAAM,MAAM,6BAA6B,GAAC,CAAC,CAAC,KAAK,CAAC,OAAO,mCAAmC,CAAC,CAAC;AAC9F,wBAAgB,mCAAmC,CAAC,GAAG,EAAC,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAAsM;AACrQ,wBAAgB,mCAAmC,CAAC,GAAG,EAAC,CAAC,CAAC,KAAK,CAAC,OAAO,0CAA0C,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAA2J;AAC7Q,eAAO,MAAM,2BAA2B,YAAU,6BAA6B,WAA6F,CAAC"}

package/dist/types/profile-source-candidate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"profile-source-candidate.d.ts","sourceRoot":"","sources":["../../../../src/profile-source-candidate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,CAAC,EAAC,MAAM,KAAK,CAAC;AAEtB;oEACoE;AACpE,eAAO,MAAM,4BAA4B;;;;;;;;;kBAAsP,CAAC;AAChS,MAAM,MAAM,sBAAsB,GAAC,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC"}
+{"version":3,"file":"profile-source-candidate.d.ts","sourceRoot":"","sources":["../../src/profile-source-candidate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,CAAC,EAAC,MAAM,KAAK,CAAC;AAEtB;oEACoE;AACpE,eAAO,MAAM,4BAA4B;;;;;;;;;kBAAsP,CAAC;AAChS,MAAM,MAAM,sBAAsB,GAAC,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC"}