@jean.gnc/harness-kit 0.11.1 → 0.12.0

package/dist/eval/schema.d.ts

@@ -0,0 +1,253 @@
+import { z } from "zod";
+declare const Expectation: z.ZodUnion<[z.ZodObject<{
+    first: z.ZodString;
+    not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+}, "strip", z.ZodTypeAny, {
+    first: string;
+    not?: string[] | undefined;
+}, {
+    first: string;
+    not?: string[] | undefined;
+}>, z.ZodObject<{
+    anyOf: z.ZodArray<z.ZodString, "many">;
+    not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+}, "strip", z.ZodTypeAny, {
+    anyOf: string[];
+    not?: string[] | undefined;
+}, {
+    anyOf: string[];
+    not?: string[] | undefined;
+}>, z.ZodObject<{
+    path: z.ZodArray<z.ZodString, "many">;
+    not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+}, "strip", z.ZodTypeAny, {
+    path: string[];
+    not?: string[] | undefined;
+}, {
+    path: string[];
+    not?: string[] | undefined;
+}>, z.ZodObject<{
+    noSkill: z.ZodLiteral<true>;
+}, "strip", z.ZodTypeAny, {
+    noSkill: true;
+}, {
+    noSkill: true;
+}>]>;
+declare const Case: z.ZodObject<{
+    id: z.ZodString;
+    prompt: z.ZodString;
+    expect: z.ZodUnion<[z.ZodObject<{
+        first: z.ZodString;
+        not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, "strip", z.ZodTypeAny, {
+        first: string;
+        not?: string[] | undefined;
+    }, {
+        first: string;
+        not?: string[] | undefined;
+    }>, z.ZodObject<{
+        anyOf: z.ZodArray<z.ZodString, "many">;
+        not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, "strip", z.ZodTypeAny, {
+        anyOf: string[];
+        not?: string[] | undefined;
+    }, {
+        anyOf: string[];
+        not?: string[] | undefined;
+    }>, z.ZodObject<{
+        path: z.ZodArray<z.ZodString, "many">;
+        not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, "strip", z.ZodTypeAny, {
+        path: string[];
+        not?: string[] | undefined;
+    }, {
+        path: string[];
+        not?: string[] | undefined;
+    }>, z.ZodObject<{
+        noSkill: z.ZodLiteral<true>;
+    }, "strip", z.ZodTypeAny, {
+        noSkill: true;
+    }, {
+        noSkill: true;
+    }>]>;
+    cwd: z.ZodOptional<z.ZodString>;
+    runs: z.ZodOptional<z.ZodNumber>;
+    threshold: z.ZodOptional<z.ZodNumber>;
+    note: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    id: string;
+    prompt: string;
+    expect: {
+        first: string;
+        not?: string[] | undefined;
+    } | {
+        anyOf: string[];
+        not?: string[] | undefined;
+    } | {
+        path: string[];
+        not?: string[] | undefined;
+    } | {
+        noSkill: true;
+    };
+    cwd?: string | undefined;
+    runs?: number | undefined;
+    threshold?: number | undefined;
+    note?: string | undefined;
+}, {
+    id: string;
+    prompt: string;
+    expect: {
+        first: string;
+        not?: string[] | undefined;
+    } | {
+        anyOf: string[];
+        not?: string[] | undefined;
+    } | {
+        path: string[];
+        not?: string[] | undefined;
+    } | {
+        noSkill: true;
+    };
+    cwd?: string | undefined;
+    runs?: number | undefined;
+    threshold?: number | undefined;
+    note?: string | undefined;
+}>;
+export declare const TIERS: readonly ["routing", "solving"];
+export declare const CaseFileSchema: z.ZodObject<{
+    suite: z.ZodString;
+    tier: z.ZodEnum<["routing", "solving"]>;
+    cases: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        prompt: z.ZodString;
+        expect: z.ZodUnion<[z.ZodObject<{
+            first: z.ZodString;
+            not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strip", z.ZodTypeAny, {
+            first: string;
+            not?: string[] | undefined;
+        }, {
+            first: string;
+            not?: string[] | undefined;
+        }>, z.ZodObject<{
+            anyOf: z.ZodArray<z.ZodString, "many">;
+            not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strip", z.ZodTypeAny, {
+            anyOf: string[];
+            not?: string[] | undefined;
+        }, {
+            anyOf: string[];
+            not?: string[] | undefined;
+        }>, z.ZodObject<{
+            path: z.ZodArray<z.ZodString, "many">;
+            not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strip", z.ZodTypeAny, {
+            path: string[];
+            not?: string[] | undefined;
+        }, {
+            path: string[];
+            not?: string[] | undefined;
+        }>, z.ZodObject<{
+            noSkill: z.ZodLiteral<true>;
+        }, "strip", z.ZodTypeAny, {
+            noSkill: true;
+        }, {
+            noSkill: true;
+        }>]>;
+        cwd: z.ZodOptional<z.ZodString>;
+        runs: z.ZodOptional<z.ZodNumber>;
+        threshold: z.ZodOptional<z.ZodNumber>;
+        note: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        id: string;
+        prompt: string;
+        expect: {
+            first: string;
+            not?: string[] | undefined;
+        } | {
+            anyOf: string[];
+            not?: string[] | undefined;
+        } | {
+            path: string[];
+            not?: string[] | undefined;
+        } | {
+            noSkill: true;
+        };
+        cwd?: string | undefined;
+        runs?: number | undefined;
+        threshold?: number | undefined;
+        note?: string | undefined;
+    }, {
+        id: string;
+        prompt: string;
+        expect: {
+            first: string;
+            not?: string[] | undefined;
+        } | {
+            anyOf: string[];
+            not?: string[] | undefined;
+        } | {
+            path: string[];
+            not?: string[] | undefined;
+        } | {
+            noSkill: true;
+        };
+        cwd?: string | undefined;
+        runs?: number | undefined;
+        threshold?: number | undefined;
+        note?: string | undefined;
+    }>, "many">;
+}, "strip", z.ZodTypeAny, {
+    suite: string;
+    tier: "routing" | "solving";
+    cases: {
+        id: string;
+        prompt: string;
+        expect: {
+            first: string;
+            not?: string[] | undefined;
+        } | {
+            anyOf: string[];
+            not?: string[] | undefined;
+        } | {
+            path: string[];
+            not?: string[] | undefined;
+        } | {
+            noSkill: true;
+        };
+        cwd?: string | undefined;
+        runs?: number | undefined;
+        threshold?: number | undefined;
+        note?: string | undefined;
+    }[];
+}, {
+    suite: string;
+    tier: "routing" | "solving";
+    cases: {
+        id: string;
+        prompt: string;
+        expect: {
+            first: string;
+            not?: string[] | undefined;
+        } | {
+            anyOf: string[];
+            not?: string[] | undefined;
+        } | {
+            path: string[];
+            not?: string[] | undefined;
+        } | {
+            noSkill: true;
+        };
+        cwd?: string | undefined;
+        runs?: number | undefined;
+        threshold?: number | undefined;
+        note?: string | undefined;
+    }[];
+}>;
+export type Tier = (typeof TIERS)[number];
+export type Expectation = z.infer<typeof Expectation>;
+export type EvalCase = z.infer<typeof Case>;
+export type CaseFile = z.infer<typeof CaseFileSchema>;
+export declare function expectedSkills(expectation: Expectation): readonly string[];
+export {};
+//# sourceMappingURL=schema.d.ts.map

package/dist/eval/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/eval/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAyBxB,QAAA,MAAM,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAKf,CAAC;AAEH,QAAA,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAQR,CAAC;AAEH,eAAO,MAAM,KAAK,iCAAkC,CAAC;AAErD,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAIzB,CAAC;AAEH,MAAM,MAAM,IAAI,GAAG,CAAC,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC;AAC1C,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AACtD,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;AAC5C,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAEtD,wBAAgB,cAAc,CAAC,WAAW,EAAE,WAAW,GAAG,SAAS,MAAM,EAAE,CAS1E"}

package/dist/eval/schema.js

@@ -0,0 +1,50 @@
+import { z } from "zod";
+import { FQ_ID } from "../ids.js";
+const FqId = z.string().regex(FQ_ID, "must be a `plugin:name` id");
+const FirstExpectation = z.object({
+    first: FqId,
+    not: z.array(FqId).optional(),
+});
+const AnyOfExpectation = z.object({
+    anyOf: z.array(FqId).min(1),
+    not: z.array(FqId).optional(),
+});
+const PathExpectation = z.object({
+    path: z.array(FqId).min(2),
+    not: z.array(FqId).optional(),
+});
+const NoSkillExpectation = z.object({
+    noSkill: z.literal(true),
+});
+const Expectation = z.union([
+    FirstExpectation,
+    AnyOfExpectation,
+    PathExpectation,
+    NoSkillExpectation,
+]);
+const Case = z.object({
+    id: z.string().min(1),
+    prompt: z.string().min(1),
+    expect: Expectation,
+    cwd: z.string().optional(),
+    runs: z.number().int().positive().optional(),
+    threshold: z.number().min(0).max(1).optional(),
+    note: z.string().optional(),
+});
+export const TIERS = ["routing", "solving"];
+export const CaseFileSchema = z.object({
+    suite: z.string().min(1),
+    tier: z.enum(TIERS),
+    cases: z.array(Case).min(1),
+});
+export function expectedSkills(expectation) {
+    if ("noSkill" in expectation)
+        return [];
+    const positive = "first" in expectation
+        ? [expectation.first]
+        : "anyOf" in expectation
+            ? expectation.anyOf
+            : expectation.path;
+    return [...positive, ...(expectation.not ?? [])];
+}
+//# sourceMappingURL=schema.js.map

package/dist/eval/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/eval/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAElC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,4BAA4B,CAAC,CAAC;AAEnE,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,KAAK,EAAE,IAAI;IACX,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AAEH,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3B,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AAEH,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1B,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AAEH,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;CACzB,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC;IAC1B,gBAAgB;IAChB,gBAAgB;IAChB,eAAe;IACf,kBAAkB;CACnB,CAAC,CAAC;AAEH,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC;IACpB,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACrB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,MAAM,EAAE,WAAW;IACnB,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC1B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC5C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC9C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC5B,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,SAAS,EAAE,SAAS,CAAU,CAAC;AAErD,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxB,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;IACnB,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CAC5B,CAAC,CAAC;AAOH,MAAM,UAAU,cAAc,CAAC,WAAwB;IACrD,IAAI,SAAS,IAAI,WAAW;QAAE,OAAO,EAAE,CAAC;IACxC,MAAM,QAAQ,GACZ,OAAO,IAAI,WAAW;QACpB,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC;QACrB,CAAC,CAAC,OAAO,IAAI,WAAW;YACtB,CAAC,CAAC,WAAW,CAAC,KAAK;YACnB,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC;IACzB,OAAO,CAAC,GAAG,QAAQ,EAAE,GAAG,CAAC,WAAW,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC;AACnD,CAAC"}

package/dist/eval/score.d.ts

@@ -0,0 +1,13 @@
+import type { DetectionResult } from "./detect.js";
+import type { Expectation } from "./schema.js";
+export interface CaseScore {
+    readonly matched: number;
+    readonly runs: number;
+    readonly triggerRate: number;
+    readonly threshold: number;
+    readonly pass: boolean;
+    readonly histogram: ReadonlyMap<string, number>;
+}
+export declare function matchesExpectation(expectation: Expectation, run: DetectionResult): boolean;
+export declare function scoreCase(expectation: Expectation, runs: readonly DetectionResult[], threshold?: number): CaseScore;
+//# sourceMappingURL=score.d.ts.map

package/dist/eval/score.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../../src/eval/score.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAI/C,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjD;AAED,wBAAgB,kBAAkB,CAAC,WAAW,EAAE,WAAW,EAAE,GAAG,EAAE,eAAe,GAAG,OAAO,CAa1F;AAED,wBAAgB,SAAS,CACvB,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,SAAS,eAAe,EAAE,EAChC,SAAS,SAAoB,GAC5B,SAAS,CAWX"}

package/dist/eval/score.js

@@ -0,0 +1,52 @@
+const DEFAULT_THRESHOLD = 1.0;
+export function matchesExpectation(expectation, run) {
+    if (violatesNot(expectation, run))
+        return false;
+    if ("noSkill" in expectation) {
+        return run.firstSkill === null;
+    }
+    if ("first" in expectation) {
+        return run.firstSkill === expectation.first;
+    }
+    if ("anyOf" in expectation) {
+        return run.firstSkill !== null && expectation.anyOf.includes(run.firstSkill);
+    }
+    return isOrderedSubsequence(expectation.path, run.observed);
+}
+export function scoreCase(expectation, runs, threshold = DEFAULT_THRESHOLD) {
+    const matched = runs.filter((run) => matchesExpectation(expectation, run)).length;
+    const triggerRate = runs.length === 0 ? 0 : matched / runs.length;
+    return {
+        matched,
+        runs: runs.length,
+        triggerRate,
+        threshold,
+        pass: triggerRate >= threshold,
+        histogram: histogramOf(runs),
+    };
+}
+function violatesNot(expectation, run) {
+    const forbidden = "noSkill" in expectation ? undefined : expectation.not;
+    if (!forbidden || forbidden.length === 0)
+        return false;
+    return run.firstSkill !== null && forbidden.includes(run.firstSkill);
+}
+function isOrderedSubsequence(needle, haystack) {
+    let cursor = 0;
+    for (const id of haystack) {
+        if (id === needle[cursor])
+            cursor += 1;
+        if (cursor === needle.length)
+            return true;
+    }
+    return cursor === needle.length;
+}
+function histogramOf(runs) {
+    const counts = new Map();
+    for (const run of runs) {
+        const key = run.firstSkill ?? "(no skill)";
+        counts.set(key, (counts.get(key) ?? 0) + 1);
+    }
+    return counts;
+}
+//# sourceMappingURL=score.js.map

package/dist/eval/score.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.js","sourceRoot":"","sources":["../../src/eval/score.ts"],"names":[],"mappings":"AAGA,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAW9B,MAAM,UAAU,kBAAkB,CAAC,WAAwB,EAAE,GAAoB;IAC/E,IAAI,WAAW,CAAC,WAAW,EAAE,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAEhD,IAAI,SAAS,IAAI,WAAW,EAAE,CAAC;QAC7B,OAAO,GAAG,CAAC,UAAU,KAAK,IAAI,CAAC;IACjC,CAAC;IACD,IAAI,OAAO,IAAI,WAAW,EAAE,CAAC;QAC3B,OAAO,GAAG,CAAC,UAAU,KAAK,WAAW,CAAC,KAAK,CAAC;IAC9C,CAAC;IACD,IAAI,OAAO,IAAI,WAAW,EAAE,CAAC;QAC3B,OAAO,GAAG,CAAC,UAAU,KAAK,IAAI,IAAI,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,oBAAoB,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,WAAwB,EACxB,IAAgC,EAChC,SAAS,GAAG,iBAAiB;IAE7B,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,kBAAkB,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAClF,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;IAClE,OAAO;QACL,OAAO;QACP,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,WAAW;QACX,SAAS;QACT,IAAI,EAAE,WAAW,IAAI,SAAS;QAC9B,SAAS,EAAE,WAAW,CAAC,IAAI,CAAC;KAC7B,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,WAAwB,EAAE,GAAoB;IACjE,MAAM,SAAS,GAAG,SAAS,IAAI,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC;IACzE,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACvD,OAAO,GAAG,CAAC,UAAU,KAAK,IAAI,IAAI,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;AACvE,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAyB,EAAE,QAA2B;IAClF,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,IAAI,EAAE,KAAK,MAAM,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,CAAC,CAAC;QACvC,IAAI,MAAM,KAAK,MAAM,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;IAC5C,CAAC;IACD,OAAO,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC;AAClC,CAAC;AAED,SAAS,WAAW,CAAC,IAAgC;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,GAAG,CAAC,UAAU,IAAI,YAAY,CAAC;QAC3C,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/index.d.ts

@@ -27,6 +27,10 @@ export { resolveVendors } from "./vendor/registry.js";
 export type { DiscoveredVendorPlugin, LinkedFile, Vendor, VendorEmitContext, VendorInstallContext, } from "./vendor/schema.js";
 export { check } from "./check/index.js";
 export type { CheckOptions, CheckResult, ReferenceViolation, ReferenceViolationKind, SourceSummary, } from "./check/index.js";
+export { runEval, formatConsole, toJson } from "./eval/index.js";
+export type { EvalOptions, EvalReport, CaseReport, LoadedCase, CaseLoadError, } from "./eval/index.js";
+export { CaseFileSchema, TIERS } from "./eval/schema.js";
+export type { CaseFile, EvalCase, Expectation, Tier } from "./eval/schema.js";
 export { defaultSources, discoverInstalled, indexInstalled } from "./installed.js";
 export type { InstalledAgent, InstalledArtifacts, InstalledCommand, InstalledIndex, InstalledSkill, PluginSource, } from "./installed.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,WAAW,EACX,mBAAmB,EACnB,WAAW,EACX,aAAa,EACb,oBAAoB,EACpB,2BAA2B,EAC3B,SAAS,GACV,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,SAAS,EACT,kBAAkB,EAClB,WAAW,EACX,cAAc,EACd,KAAK,EACL,SAAS,EACT,WAAW,GACZ,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAC/D,YAAY,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAC9E,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAE9E,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AACtC,YAAY,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE1C,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACxE,YAAY,EACV,WAAW,EACX,gBAAgB,EAChB,SAAS,EACT,iBAAiB,EACjB,eAAe,GAChB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,YAAY,EAAE,qBAAqB,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAE7E,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,YAAY,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEzD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACxD,YAAY,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACnF,YAAY,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,YAAY,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAEnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,YAAY,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,YAAY,EACV,sBAAsB,EACtB,UAAU,EACV,MAAM,EACN,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,YAAY,EACV,YAAY,EACZ,WAAW,EACX,kBAAkB,EAClB,sBAAsB,EACtB,aAAa,GACd,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACnF,YAAY,EACV,cAAc,EACd,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,cAAc,EACd,YAAY,GACb,MAAM,gBAAgB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,WAAW,EACX,mBAAmB,EACnB,WAAW,EACX,aAAa,EACb,oBAAoB,EACpB,2BAA2B,EAC3B,SAAS,GACV,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,SAAS,EACT,kBAAkB,EAClB,WAAW,EACX,cAAc,EACd,KAAK,EACL,SAAS,EACT,WAAW,GACZ,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAC/D,YAAY,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAC9E,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAE9E,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AACtC,YAAY,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE1C,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACxE,YAAY,EACV,WAAW,EACX,gBAAgB,EAChB,SAAS,EACT,iBAAiB,EACjB,eAAe,GAChB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,YAAY,EAAE,qBAAqB,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAE7E,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,YAAY,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEzD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACxD,YAAY,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACnF,YAAY,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,YAAY,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAEnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,YAAY,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,YAAY,EACV,sBAAsB,EACtB,UAAU,EACV,MAAM,EACN,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,YAAY,EACV,YAAY,EACZ,WAAW,EACX,kBAAkB,EAClB,sBAAsB,EACtB,aAAa,GACd,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACjE,YAAY,EACV,WAAW,EACX,UAAU,EACV,UAAU,EACV,UAAU,EACV,aAAa,GACd,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzD,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAE9E,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACnF,YAAY,EACV,cAAc,EACd,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,cAAc,EACd,YAAY,GACb,MAAM,gBAAgB,CAAC"}

package/dist/index.js

@@ -13,5 +13,7 @@ export { loadHarnessConfig } from "./config/harness.js";
 export { builtinVendors } from "./vendor/builtins.js";
 export { resolveVendors } from "./vendor/registry.js";
 export { check } from "./check/index.js";
+export { runEval, formatConsole, toJson } from "./eval/index.js";
+export { CaseFileSchema, TIERS } from "./eval/schema.js";
 export { defaultSources, discoverInstalled, indexInstalled } from "./installed.js";
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,WAAW,EACX,mBAAmB,EACnB,WAAW,EACX,aAAa,EACb,oBAAoB,EACpB,2BAA2B,EAC3B,SAAS,GACV,MAAM,kBAAkB,CAAC;AAW1B,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAG/D,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAG9E,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AAGtC,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AASxE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGpD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAGvC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAExD,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAGnF,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAG9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAGxD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAStD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AASzC,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,WAAW,EACX,mBAAmB,EACnB,WAAW,EACX,aAAa,EACb,oBAAoB,EACpB,2BAA2B,EAC3B,SAAS,GACV,MAAM,kBAAkB,CAAC;AAW1B,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAG/D,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAG9E,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AAGtC,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AASxE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGpD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAGvC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAExD,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAGnF,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAG9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAGxD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAStD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AASzC,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAQjE,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAGzD,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jean.gnc/harness-kit",
-  "version": "0.11.1",
+  "version": "0.12.0",
   "type": "module",
   "description": "Build your own multi-agent harness: typed toolkit for authoring plugins (skills, agents, commands, hooks) and shipping them to Claude Code and Codex from a single source tree.",
   "license": "MIT",