@jean.gnc/harness-kit 0.12.8 → 0.14.0

package/dist/eval/schema.d.ts

@@ -33,9 +33,131 @@ declare const Expectation: z.ZodUnion<[z.ZodObject<{
 }, {
     noSkill: true;
 }>]>;
-declare const Case: z.ZodObject<{
-    id: z.ZodString;
-    prompt: z.ZodString;
+declare const Assertion: z.ZodDiscriminatedUnion<"kind", [z.ZodObject<{
+    kind: z.ZodLiteral<"outputMatches">;
+    pattern: z.ZodString;
+    regex: z.ZodDefault<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    regex: boolean;
+    kind: "outputMatches";
+    pattern: string;
+}, {
+    kind: "outputMatches";
+    pattern: string;
+    regex?: boolean | undefined;
+}>, z.ZodObject<{
+    kind: z.ZodLiteral<"outputExcludes">;
+    pattern: z.ZodString;
+    regex: z.ZodDefault<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    regex: boolean;
+    kind: "outputExcludes";
+    pattern: string;
+}, {
+    kind: "outputExcludes";
+    pattern: string;
+    regex?: boolean | undefined;
+}>, z.ZodObject<{
+    kind: z.ZodLiteral<"usedTool">;
+    tool: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    kind: "usedTool";
+    tool: string;
+}, {
+    kind: "usedTool";
+    tool: string;
+}>, z.ZodObject<{
+    kind: z.ZodLiteral<"didNotUseTool">;
+    tool: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    kind: "didNotUseTool";
+    tool: string;
+}, {
+    kind: "didNotUseTool";
+    tool: string;
+}>, z.ZodObject<{
+    kind: z.ZodLiteral<"wroteFile">;
+    path: z.ZodString;
+    contentMatches: z.ZodOptional<z.ZodString>;
+    regex: z.ZodDefault<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    path: string;
+    regex: boolean;
+    kind: "wroteFile";
+    contentMatches?: string | undefined;
+}, {
+    path: string;
+    kind: "wroteFile";
+    regex?: boolean | undefined;
+    contentMatches?: string | undefined;
+}>]>;
+declare const CombineRule: z.ZodDiscriminatedUnion<"combine", [z.ZodObject<{
+    combine: z.ZodLiteral<"all">;
+}, "strip", z.ZodTypeAny, {
+    combine: "all";
+}, {
+    combine: "all";
+}>, z.ZodObject<{
+    combine: z.ZodLiteral<"fraction">;
+    threshold: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    combine: "fraction";
+    threshold: number;
+}, {
+    combine: "fraction";
+    threshold: number;
+}>]>;
+declare const Rubric: z.ZodObject<{
+    dimensions: z.ZodArray<z.ZodObject<{
+        dimension: z.ZodString;
+        criterion: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        dimension: string;
+        criterion: string;
+    }, {
+        dimension: string;
+        criterion: string;
+    }>, "many">;
+    combine: z.ZodDefault<z.ZodDiscriminatedUnion<"combine", [z.ZodObject<{
+        combine: z.ZodLiteral<"all">;
+    }, "strip", z.ZodTypeAny, {
+        combine: "all";
+    }, {
+        combine: "all";
+    }>, z.ZodObject<{
+        combine: z.ZodLiteral<"fraction">;
+        threshold: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        combine: "fraction";
+        threshold: number;
+    }, {
+        combine: "fraction";
+        threshold: number;
+    }>]>>;
+}, "strip", z.ZodTypeAny, {
+    combine: {
+        combine: "all";
+    } | {
+        combine: "fraction";
+        threshold: number;
+    };
+    dimensions: {
+        dimension: string;
+        criterion: string;
+    }[];
+}, {
+    dimensions: {
+        dimension: string;
+        criterion: string;
+    }[];
+    combine?: {
+        combine: "all";
+    } | {
+        combine: "fraction";
+        threshold: number;
+    } | undefined;
+}>;
+declare const RoutingCase: z.ZodObject<{
     expect: z.ZodUnion<[z.ZodObject<{
         first: z.ZodString;
         not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
@@ -70,13 +192,13 @@ declare const Case: z.ZodObject<{
     }, {
         noSkill: true;
     }>]>;
+    id: z.ZodString;
+    prompt: z.ZodString;
     cwd: z.ZodOptional<z.ZodString>;
     runs: z.ZodOptional<z.ZodNumber>;
     threshold: z.ZodOptional<z.ZodNumber>;
     note: z.ZodOptional<z.ZodString>;
-}, "strip", z.ZodTypeAny, {
-    id: string;
-    prompt: string;
+}, "strict", z.ZodTypeAny, {
     expect: {
         first: string;
         not?: string[] | undefined;
@@ -89,13 +211,13 @@ declare const Case: z.ZodObject<{
     } | {
         noSkill: true;
     };
+    id: string;
+    prompt: string;
+    threshold?: number | undefined;
     cwd?: string | undefined;
     runs?: number | undefined;
-    threshold?: number | undefined;
     note?: string | undefined;
 }, {
-    id: string;
-    prompt: string;
     expect: {
         first: string;
         not?: string[] | undefined;
@@ -108,18 +230,215 @@ declare const Case: z.ZodObject<{
     } | {
         noSkill: true;
     };
+    id: string;
+    prompt: string;
+    threshold?: number | undefined;
     cwd?: string | undefined;
     runs?: number | undefined;
+    note?: string | undefined;
+}>;
+declare const SolvingCase: z.ZodObject<{
+    expectSkill: z.ZodOptional<z.ZodString>;
+    assert: z.ZodDefault<z.ZodArray<z.ZodDiscriminatedUnion<"kind", [z.ZodObject<{
+        kind: z.ZodLiteral<"outputMatches">;
+        pattern: z.ZodString;
+        regex: z.ZodDefault<z.ZodBoolean>;
+    }, "strip", z.ZodTypeAny, {
+        regex: boolean;
+        kind: "outputMatches";
+        pattern: string;
+    }, {
+        kind: "outputMatches";
+        pattern: string;
+        regex?: boolean | undefined;
+    }>, z.ZodObject<{
+        kind: z.ZodLiteral<"outputExcludes">;
+        pattern: z.ZodString;
+        regex: z.ZodDefault<z.ZodBoolean>;
+    }, "strip", z.ZodTypeAny, {
+        regex: boolean;
+        kind: "outputExcludes";
+        pattern: string;
+    }, {
+        kind: "outputExcludes";
+        pattern: string;
+        regex?: boolean | undefined;
+    }>, z.ZodObject<{
+        kind: z.ZodLiteral<"usedTool">;
+        tool: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        kind: "usedTool";
+        tool: string;
+    }, {
+        kind: "usedTool";
+        tool: string;
+    }>, z.ZodObject<{
+        kind: z.ZodLiteral<"didNotUseTool">;
+        tool: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        kind: "didNotUseTool";
+        tool: string;
+    }, {
+        kind: "didNotUseTool";
+        tool: string;
+    }>, z.ZodObject<{
+        kind: z.ZodLiteral<"wroteFile">;
+        path: z.ZodString;
+        contentMatches: z.ZodOptional<z.ZodString>;
+        regex: z.ZodDefault<z.ZodBoolean>;
+    }, "strip", z.ZodTypeAny, {
+        path: string;
+        regex: boolean;
+        kind: "wroteFile";
+        contentMatches?: string | undefined;
+    }, {
+        path: string;
+        kind: "wroteFile";
+        regex?: boolean | undefined;
+        contentMatches?: string | undefined;
+    }>]>, "many">>;
+    rubric: z.ZodOptional<z.ZodObject<{
+        dimensions: z.ZodArray<z.ZodObject<{
+            dimension: z.ZodString;
+            criterion: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            dimension: string;
+            criterion: string;
+        }, {
+            dimension: string;
+            criterion: string;
+        }>, "many">;
+        combine: z.ZodDefault<z.ZodDiscriminatedUnion<"combine", [z.ZodObject<{
+            combine: z.ZodLiteral<"all">;
+        }, "strip", z.ZodTypeAny, {
+            combine: "all";
+        }, {
+            combine: "all";
+        }>, z.ZodObject<{
+            combine: z.ZodLiteral<"fraction">;
+            threshold: z.ZodNumber;
+        }, "strip", z.ZodTypeAny, {
+            combine: "fraction";
+            threshold: number;
+        }, {
+            combine: "fraction";
+            threshold: number;
+        }>]>>;
+    }, "strip", z.ZodTypeAny, {
+        combine: {
+            combine: "all";
+        } | {
+            combine: "fraction";
+            threshold: number;
+        };
+        dimensions: {
+            dimension: string;
+            criterion: string;
+        }[];
+    }, {
+        dimensions: {
+            dimension: string;
+            criterion: string;
+        }[];
+        combine?: {
+            combine: "all";
+        } | {
+            combine: "fraction";
+            threshold: number;
+        } | undefined;
+    }>>;
+    id: z.ZodString;
+    prompt: z.ZodString;
+    cwd: z.ZodOptional<z.ZodString>;
+    runs: z.ZodOptional<z.ZodNumber>;
+    threshold: z.ZodOptional<z.ZodNumber>;
+    note: z.ZodOptional<z.ZodString>;
+}, "strict", z.ZodTypeAny, {
+    id: string;
+    prompt: string;
+    assert: ({
+        regex: boolean;
+        kind: "outputMatches";
+        pattern: string;
+    } | {
+        regex: boolean;
+        kind: "outputExcludes";
+        pattern: string;
+    } | {
+        kind: "usedTool";
+        tool: string;
+    } | {
+        kind: "didNotUseTool";
+        tool: string;
+    } | {
+        path: string;
+        regex: boolean;
+        kind: "wroteFile";
+        contentMatches?: string | undefined;
+    })[];
+    threshold?: number | undefined;
+    cwd?: string | undefined;
+    runs?: number | undefined;
+    note?: string | undefined;
+    expectSkill?: string | undefined;
+    rubric?: {
+        combine: {
+            combine: "all";
+        } | {
+            combine: "fraction";
+            threshold: number;
+        };
+        dimensions: {
+            dimension: string;
+            criterion: string;
+        }[];
+    } | undefined;
+}, {
+    id: string;
+    prompt: string;
     threshold?: number | undefined;
+    cwd?: string | undefined;
+    runs?: number | undefined;
     note?: string | undefined;
+    expectSkill?: string | undefined;
+    assert?: ({
+        kind: "outputMatches";
+        pattern: string;
+        regex?: boolean | undefined;
+    } | {
+        kind: "outputExcludes";
+        pattern: string;
+        regex?: boolean | undefined;
+    } | {
+        kind: "usedTool";
+        tool: string;
+    } | {
+        kind: "didNotUseTool";
+        tool: string;
+    } | {
+        path: string;
+        kind: "wroteFile";
+        regex?: boolean | undefined;
+        contentMatches?: string | undefined;
+    })[] | undefined;
+    rubric?: {
+        dimensions: {
+            dimension: string;
+            criterion: string;
+        }[];
+        combine?: {
+            combine: "all";
+        } | {
+            combine: "fraction";
+            threshold: number;
+        } | undefined;
+    } | undefined;
 }>;
 export declare const TIERS: readonly ["routing", "solving"];
-export declare const CaseFileSchema: z.ZodObject<{
+export declare const CaseFileSchema: z.ZodDiscriminatedUnion<"tier", [z.ZodObject<{
     suite: z.ZodString;
-    tier: z.ZodEnum<["routing", "solving"]>;
+    tier: z.ZodLiteral<"routing">;
     cases: z.ZodArray<z.ZodObject<{
-        id: z.ZodString;
-        prompt: z.ZodString;
         expect: z.ZodUnion<[z.ZodObject<{
             first: z.ZodString;
             not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
@@ -154,13 +473,13 @@ export declare const CaseFileSchema: z.ZodObject<{
         }, {
             noSkill: true;
         }>]>;
+        id: z.ZodString;
+        prompt: z.ZodString;
         cwd: z.ZodOptional<z.ZodString>;
         runs: z.ZodOptional<z.ZodNumber>;
         threshold: z.ZodOptional<z.ZodNumber>;
         note: z.ZodOptional<z.ZodString>;
-    }, "strip", z.ZodTypeAny, {
-        id: string;
-        prompt: string;
+    }, "strict", z.ZodTypeAny, {
         expect: {
             first: string;
             not?: string[] | undefined;
@@ -173,13 +492,13 @@ export declare const CaseFileSchema: z.ZodObject<{
         } | {
             noSkill: true;
         };
+        id: string;
+        prompt: string;
+        threshold?: number | undefined;
         cwd?: string | undefined;
         runs?: number | undefined;
-        threshold?: number | undefined;
         note?: string | undefined;
     }, {
-        id: string;
-        prompt: string;
         expect: {
             first: string;
             not?: string[] | undefined;
@@ -192,17 +511,17 @@ export declare const CaseFileSchema: z.ZodObject<{
         } | {
             noSkill: true;
         };
+        id: string;
+        prompt: string;
+        threshold?: number | undefined;
         cwd?: string | undefined;
         runs?: number | undefined;
-        threshold?: number | undefined;
         note?: string | undefined;
     }>, "many">;
 }, "strip", z.ZodTypeAny, {
     suite: string;
-    tier: "routing" | "solving";
+    tier: "routing";
     cases: {
-        id: string;
-        prompt: string;
         expect: {
             first: string;
             not?: string[] | undefined;
@@ -215,17 +534,17 @@ export declare const CaseFileSchema: z.ZodObject<{
         } | {
             noSkill: true;
         };
+        id: string;
+        prompt: string;
+        threshold?: number | undefined;
         cwd?: string | undefined;
         runs?: number | undefined;
-        threshold?: number | undefined;
         note?: string | undefined;
     }[];
 }, {
     suite: string;
-    tier: "routing" | "solving";
+    tier: "routing";
     cases: {
-        id: string;
-        prompt: string;
         expect: {
             first: string;
             not?: string[] | undefined;
@@ -238,16 +557,312 @@ export declare const CaseFileSchema: z.ZodObject<{
         } | {
             noSkill: true;
         };
+        id: string;
+        prompt: string;
+        threshold?: number | undefined;
+        cwd?: string | undefined;
+        runs?: number | undefined;
+        note?: string | undefined;
+    }[];
+}>, z.ZodObject<{
+    suite: z.ZodString;
+    tier: z.ZodLiteral<"solving">;
+    cases: z.ZodArray<z.ZodObject<{
+        expectSkill: z.ZodOptional<z.ZodString>;
+        assert: z.ZodDefault<z.ZodArray<z.ZodDiscriminatedUnion<"kind", [z.ZodObject<{
+            kind: z.ZodLiteral<"outputMatches">;
+            pattern: z.ZodString;
+            regex: z.ZodDefault<z.ZodBoolean>;
+        }, "strip", z.ZodTypeAny, {
+            regex: boolean;
+            kind: "outputMatches";
+            pattern: string;
+        }, {
+            kind: "outputMatches";
+            pattern: string;
+            regex?: boolean | undefined;
+        }>, z.ZodObject<{
+            kind: z.ZodLiteral<"outputExcludes">;
+            pattern: z.ZodString;
+            regex: z.ZodDefault<z.ZodBoolean>;
+        }, "strip", z.ZodTypeAny, {
+            regex: boolean;
+            kind: "outputExcludes";
+            pattern: string;
+        }, {
+            kind: "outputExcludes";
+            pattern: string;
+            regex?: boolean | undefined;
+        }>, z.ZodObject<{
+            kind: z.ZodLiteral<"usedTool">;
+            tool: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            kind: "usedTool";
+            tool: string;
+        }, {
+            kind: "usedTool";
+            tool: string;
+        }>, z.ZodObject<{
+            kind: z.ZodLiteral<"didNotUseTool">;
+            tool: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            kind: "didNotUseTool";
+            tool: string;
+        }, {
+            kind: "didNotUseTool";
+            tool: string;
+        }>, z.ZodObject<{
+            kind: z.ZodLiteral<"wroteFile">;
+            path: z.ZodString;
+            contentMatches: z.ZodOptional<z.ZodString>;
+            regex: z.ZodDefault<z.ZodBoolean>;
+        }, "strip", z.ZodTypeAny, {
+            path: string;
+            regex: boolean;
+            kind: "wroteFile";
+            contentMatches?: string | undefined;
+        }, {
+            path: string;
+            kind: "wroteFile";
+            regex?: boolean | undefined;
+            contentMatches?: string | undefined;
+        }>]>, "many">>;
+        rubric: z.ZodOptional<z.ZodObject<{
+            dimensions: z.ZodArray<z.ZodObject<{
+                dimension: z.ZodString;
+                criterion: z.ZodString;
+            }, "strip", z.ZodTypeAny, {
+                dimension: string;
+                criterion: string;
+            }, {
+                dimension: string;
+                criterion: string;
+            }>, "many">;
+            combine: z.ZodDefault<z.ZodDiscriminatedUnion<"combine", [z.ZodObject<{
+                combine: z.ZodLiteral<"all">;
+            }, "strip", z.ZodTypeAny, {
+                combine: "all";
+            }, {
+                combine: "all";
+            }>, z.ZodObject<{
+                combine: z.ZodLiteral<"fraction">;
+                threshold: z.ZodNumber;
+            }, "strip", z.ZodTypeAny, {
+                combine: "fraction";
+                threshold: number;
+            }, {
+                combine: "fraction";
+                threshold: number;
+            }>]>>;
+        }, "strip", z.ZodTypeAny, {
+            combine: {
+                combine: "all";
+            } | {
+                combine: "fraction";
+                threshold: number;
+            };
+            dimensions: {
+                dimension: string;
+                criterion: string;
+            }[];
+        }, {
+            dimensions: {
+                dimension: string;
+                criterion: string;
+            }[];
+            combine?: {
+                combine: "all";
+            } | {
+                combine: "fraction";
+                threshold: number;
+            } | undefined;
+        }>>;
+        id: z.ZodString;
+        prompt: z.ZodString;
+        cwd: z.ZodOptional<z.ZodString>;
+        runs: z.ZodOptional<z.ZodNumber>;
+        threshold: z.ZodOptional<z.ZodNumber>;
+        note: z.ZodOptional<z.ZodString>;
+    }, "strict", z.ZodTypeAny, {
+        id: string;
+        prompt: string;
+        assert: ({
+            regex: boolean;
+            kind: "outputMatches";
+            pattern: string;
+        } | {
+            regex: boolean;
+            kind: "outputExcludes";
+            pattern: string;
+        } | {
+            kind: "usedTool";
+            tool: string;
+        } | {
+            kind: "didNotUseTool";
+            tool: string;
+        } | {
+            path: string;
+            regex: boolean;
+            kind: "wroteFile";
+            contentMatches?: string | undefined;
+        })[];
+        threshold?: number | undefined;
         cwd?: string | undefined;
         runs?: number | undefined;
+        note?: string | undefined;
+        expectSkill?: string | undefined;
+        rubric?: {
+            combine: {
+                combine: "all";
+            } | {
+                combine: "fraction";
+                threshold: number;
+            };
+            dimensions: {
+                dimension: string;
+                criterion: string;
+            }[];
+        } | undefined;
+    }, {
+        id: string;
+        prompt: string;
         threshold?: number | undefined;
+        cwd?: string | undefined;
+        runs?: number | undefined;
         note?: string | undefined;
+        expectSkill?: string | undefined;
+        assert?: ({
+            kind: "outputMatches";
+            pattern: string;
+            regex?: boolean | undefined;
+        } | {
+            kind: "outputExcludes";
+            pattern: string;
+            regex?: boolean | undefined;
+        } | {
+            kind: "usedTool";
+            tool: string;
+        } | {
+            kind: "didNotUseTool";
+            tool: string;
+        } | {
+            path: string;
+            kind: "wroteFile";
+            regex?: boolean | undefined;
+            contentMatches?: string | undefined;
+        })[] | undefined;
+        rubric?: {
+            dimensions: {
+                dimension: string;
+                criterion: string;
+            }[];
+            combine?: {
+                combine: "all";
+            } | {
+                combine: "fraction";
+                threshold: number;
+            } | undefined;
+        } | undefined;
+    }>, "many">;
+}, "strip", z.ZodTypeAny, {
+    suite: string;
+    tier: "solving";
+    cases: {
+        id: string;
+        prompt: string;
+        assert: ({
+            regex: boolean;
+            kind: "outputMatches";
+            pattern: string;
+        } | {
+            regex: boolean;
+            kind: "outputExcludes";
+            pattern: string;
+        } | {
+            kind: "usedTool";
+            tool: string;
+        } | {
+            kind: "didNotUseTool";
+            tool: string;
+        } | {
+            path: string;
+            regex: boolean;
+            kind: "wroteFile";
+            contentMatches?: string | undefined;
+        })[];
+        threshold?: number | undefined;
+        cwd?: string | undefined;
+        runs?: number | undefined;
+        note?: string | undefined;
+        expectSkill?: string | undefined;
+        rubric?: {
+            combine: {
+                combine: "all";
+            } | {
+                combine: "fraction";
+                threshold: number;
+            };
+            dimensions: {
+                dimension: string;
+                criterion: string;
+            }[];
+        } | undefined;
     }[];
-}>;
+}, {
+    suite: string;
+    tier: "solving";
+    cases: {
+        id: string;
+        prompt: string;
+        threshold?: number | undefined;
+        cwd?: string | undefined;
+        runs?: number | undefined;
+        note?: string | undefined;
+        expectSkill?: string | undefined;
+        assert?: ({
+            kind: "outputMatches";
+            pattern: string;
+            regex?: boolean | undefined;
+        } | {
+            kind: "outputExcludes";
+            pattern: string;
+            regex?: boolean | undefined;
+        } | {
+            kind: "usedTool";
+            tool: string;
+        } | {
+            kind: "didNotUseTool";
+            tool: string;
+        } | {
+            path: string;
+            kind: "wroteFile";
+            regex?: boolean | undefined;
+            contentMatches?: string | undefined;
+        })[] | undefined;
+        rubric?: {
+            dimensions: {
+                dimension: string;
+                criterion: string;
+            }[];
+            combine?: {
+                combine: "all";
+            } | {
+                combine: "fraction";
+                threshold: number;
+            } | undefined;
+        } | undefined;
+    }[];
+}>]>;
 export type Tier = (typeof TIERS)[number];
 export type Expectation = z.infer<typeof Expectation>;
-export type EvalCase = z.infer<typeof Case>;
+export type Assertion = z.infer<typeof Assertion>;
+export type Rubric = z.infer<typeof Rubric>;
+export type CombineRule = z.infer<typeof CombineRule>;
+export type RoutingCase = z.infer<typeof RoutingCase>;
+export type SolvingCase = z.infer<typeof SolvingCase>;
+export type EvalCase = RoutingCase | SolvingCase;
 export type CaseFile = z.infer<typeof CaseFileSchema>;
 export declare function expectedSkills(expectation: Expectation): readonly string[];
+export declare function caseExpectedSkills(evalCase: EvalCase): readonly string[];
 export {};
 //# sourceMappingURL=schema.d.ts.map