@roackb2/heddle 0.0.37 → 1.0.0

package/dist/src/core/eval/report-writer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-writer.js","sourceRoot":"","sources":["../../../../src/core/eval/report-writer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAG3C,MAAM,UAAU,oBAAoB,CAAC,MAAuB;IAC1D,SAAS,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IACxD,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IAC1D,aAAa,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACxE,aAAa,CAAC,YAAY,EAAE,uBAAuB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;IACrE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,MAAuB;IAC7D,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IACpF,MAAM,KAAK,GAAG;QACZ,4BAA4B;QAC5B,EAAE;QACF,WAAW,MAAM,CAAC,MAAM,EAAE;QAC1B,YAAY,MAAM,CAAC,SAAS,EAAE;QAC9B,aAAa,MAAM,CAAC,UAAU,EAAE;QAChC,YAAY,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,SAAS;QACpD,EAAE;QACF,gGAAgG;QAChG,uDAAuD;QACvD,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;QACvC,EAAE;KACH,CAAC;IAEF,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC;AAC3C,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAqB;IAC7C,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC1E,OAAO;QACL,MAAM,CAAC,MAAM;QACb,MAAM,CAAC,MAAM;QACb,MAAM,CAAC,KAAK,IAAI,SAAS;QACzB,GAAG,YAAY,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE;QACzC,MAAM,CAAC,OAAO,CAAC,OAAO,IAAI,QAAQ,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,SAAS,EAAE;QACtE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC;QAChC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,iCAAiC,CAAC;KACzD,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,eAAe,CAAC,MAAqB,EAAE,UAAkB;IAChE,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC;IAC3F,MAAM,KAAK,GAAG;QACZ,MAAM,MAAM,CAAC,MAAM,EAAE;QACrB,EAAE;QACF,mBAAmB;QACnB,eAAe;QACf,cAAc,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI;QAC3C,aAAa,UAAU,CAAC,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC,IAAI;QACtD,iBAAiB,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,IAAI,SAAS,CAAC,CAAC,IAAI;QACrE,kBAAkB,UAAU,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI;QACvH,eAAe,UAAU,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI;QACpD,mBAAmB,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC,MAAM;QACjF,gBAAgB,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,MAAM;QAC1E,cAAc,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC,MAAM;QACpF,mBAAmB,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC,MAAM;QAC7F,4BAA4B,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,gBAAgB,EAAE,UAAU,CAAC,CAAC,MAAM;QACvG,oBAAoB,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC,MAAM;QAC5F,gBAAgB,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI;QACvI,uBAAuB,MAAM,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,kBAAkB,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI;QAC1J,mBAAmB,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI;QACpH,EAAE;QACF,sBAAsB;QACtB,EAAE;KACH,CAAC;IAEF,KAAK,CAAC,IAAI,CAAC,GAAG,mBAAmB,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,mBAAmB,EAAE,EAAE,CAAC,CAAC;IACxE,IAAI,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/C,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvB,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,2BAA2B,EAAE,6BAA6B,CAAC,CAAC;QACvE,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC;YACjD,KAAK,CAAC,IAAI,CAAC,KAAK,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,SAAS,IAAI,EAAE,MAAM,IAAI,CAAC,SAAS,IAAI,EAAE,IAAI,CAAC,CAAC;QAC9H,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,qBAAqB,EAAE,EAAE,CAAC,CAAC;IAC1C,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvB,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,OAAO,OAAO,KAAK,CAAC,QAAQ,IAAI,SAAS,KAAK,KAAK,CAAC,UAAU,KAAK,CAAC,CAAC;QAChJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CACR,EAAE,EACF,aAAa,EACb,EAAE,EACF,oBAAoB,EACpB,gBAAgB,EAChB,uBAAuB,MAAM,CAAC,OAAO,CAAC,cAAc,IAAI,EACxD,kBAAkB,MAAM,CAAC,OAAO,CAAC,SAAS,IAAI,EAC9C,iBAAiB,MAAM,CAAC,OAAO,CAAC,SAAS,IAAI,EAC7C,mCAAmC,MAAM,CAAC,OAAO,CAAC,iCAAiC,IAAI,EACvF,2BAA2B,MAAM,CAAC,OAAO,CAAC,kBAAkB,IAAI,EAChE,0BAA0B,MAAM,CAAC,OAAO,CAAC,iBAAiB,IAAI,EAC9D,mBAAmB,MAAM,CAAC,OAAO,CAAC,UAAU,IAAI,CACjD,CAAC;IAEF,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,iCAAiC,EAAE,EAAE,CAAC,CAAC;IACtD,IAAI,MAAM,CAAC,OAAO,CAAC,0BAA0B,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;IACrD,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,0BAA0B,EAAE,CAAC;YAChE,KAAK,CAAC,IAAI,CAAC,OAAO,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,CAAC,CAAC;IACjC,IAAI,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7F,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvB,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;YACrD,KAAK,CAAC,IAAI,CAAC,SAAS,OAAO,EAAE,CAAC,CAAC;QACjC,CAAC;QACD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YACpD,KAAK,CAAC,IAAI,CAAC,SAAS,QAAQ,EAAE,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAqB;IAChD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACvC,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,0BAA0B,EAAE,eAAe,CAAC,CAAC;IAC5D,KAAK,CAAC,IAAI,CAAC,yBAAyB,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IACxF,KAAK,CAAC,IAAI,CAAC,qBAAqB,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAChF,KAAK,CAAC,IAAI,CAAC,oBAAoB,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAC7E,KAAK,CAAC,IAAI,CAAC,uBAAuB,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IACpF,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,KAAe;IACrC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;AAC3F,CAAC;AAED,SAAS,aAAa,CAAC,MAAqB;IAC1C,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;QAC3C,OAAO;YACL,cAAc;YACd,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS;YAC5D,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS;YACzF,MAAM,CAAC,OAAO,CAAC,cAAc,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,KAAK,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;gBAC7F,YAAY,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE;gBACvD,CAAC,CAAC,SAAS;SACZ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,oBAAoB,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;AAClH,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,QAAgB;IAChD,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,OAAO,GAAG,CAAC;IACb,CAAC;IAED,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC9C,OAAO,YAAY,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9E,CAAC"}

package/dist/src/core/eval/schema.d.ts

@@ -0,0 +1,206 @@
+import { z } from 'zod';
+export declare const evalCheckSchema: z.ZodObject<{
+    name: z.ZodOptional<z.ZodString>;
+    command: z.ZodString;
+    timeoutMs: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+export declare const evalSetupSchema: z.ZodObject<{
+    copyFiles: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    files: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    commands: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        name: z.ZodOptional<z.ZodString>;
+        command: z.ZodString;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>>;
+    commitMessage: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export declare const evalInlineFixtureSchema: z.ZodObject<{
+    type: z.ZodLiteral<"inline">;
+}, z.core.$strip>;
+export declare const evalGitWorktreeFixtureSchema: z.ZodObject<{
+    type: z.ZodLiteral<"git-worktree">;
+    repo: z.ZodDefault<z.ZodString>;
+    ref: z.ZodString;
+}, z.core.$strip>;
+export declare const evalFixtureSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    type: z.ZodLiteral<"inline">;
+}, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"git-worktree">;
+    repo: z.ZodDefault<z.ZodString>;
+    ref: z.ZodString;
+}, z.core.$strip>], "type">;
+export declare const evalMilestoneReviewSchema: z.ZodObject<{
+    milestone: z.ZodOptional<z.ZodString>;
+    intent: z.ZodOptional<z.ZodString>;
+    requiredOutcomes: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    allowedScope: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    outOfScope: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    humanQuestions: z.ZodDefault<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+export declare const agentEvalCaseSchema: z.ZodObject<{
+    id: z.ZodString;
+    kind: z.ZodLiteral<"coding">;
+    description: z.ZodOptional<z.ZodString>;
+    prompt: z.ZodString;
+    model: z.ZodOptional<z.ZodString>;
+    maxSteps: z.ZodOptional<z.ZodNumber>;
+    setup: z.ZodDefault<z.ZodObject<{
+        copyFiles: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        files: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        commands: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodOptional<z.ZodString>;
+            command: z.ZodString;
+            timeoutMs: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strip>>>;
+        commitMessage: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
+    fixture: z.ZodDefault<z.ZodDiscriminatedUnion<[z.ZodObject<{
+        type: z.ZodLiteral<"inline">;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"git-worktree">;
+        repo: z.ZodDefault<z.ZodString>;
+        ref: z.ZodString;
+    }, z.core.$strip>], "type">>;
+    review: z.ZodDefault<z.ZodObject<{
+        milestone: z.ZodOptional<z.ZodString>;
+        intent: z.ZodOptional<z.ZodString>;
+        requiredOutcomes: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        allowedScope: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        outOfScope: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        humanQuestions: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>>;
+    checks: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        name: z.ZodOptional<z.ZodString>;
+        command: z.ZodString;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>>;
+    rubric: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    tags: z.ZodDefault<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+export declare const evalCaseSchema: z.ZodObject<{
+    id: z.ZodString;
+    kind: z.ZodLiteral<"coding">;
+    description: z.ZodOptional<z.ZodString>;
+    prompt: z.ZodString;
+    model: z.ZodOptional<z.ZodString>;
+    maxSteps: z.ZodOptional<z.ZodNumber>;
+    setup: z.ZodDefault<z.ZodObject<{
+        copyFiles: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        files: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+        commands: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodOptional<z.ZodString>;
+            command: z.ZodString;
+            timeoutMs: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strip>>>;
+        commitMessage: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
+    fixture: z.ZodDefault<z.ZodDiscriminatedUnion<[z.ZodObject<{
+        type: z.ZodLiteral<"inline">;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"git-worktree">;
+        repo: z.ZodDefault<z.ZodString>;
+        ref: z.ZodString;
+    }, z.core.$strip>], "type">>;
+    review: z.ZodDefault<z.ZodObject<{
+        milestone: z.ZodOptional<z.ZodString>;
+        intent: z.ZodOptional<z.ZodString>;
+        requiredOutcomes: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        allowedScope: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        outOfScope: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        humanQuestions: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>>;
+    checks: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        name: z.ZodOptional<z.ZodString>;
+        command: z.ZodString;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>>;
+    rubric: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    tags: z.ZodDefault<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+export type EvalCheck = z.infer<typeof evalCheckSchema>;
+export type EvalSetup = z.infer<typeof evalSetupSchema>;
+export type EvalFixture = z.infer<typeof evalFixtureSchema>;
+export type EvalMilestoneReview = z.infer<typeof evalMilestoneReviewSchema>;
+export type AgentEvalCase = z.infer<typeof agentEvalCaseSchema>;
+export type EvalCase = z.infer<typeof evalCaseSchema>;
+export type EvalCheckResult = {
+    name: string;
+    command: string;
+    exitCode: number | null;
+    stdout: string;
+    stderr: string;
+    durationMs: number;
+    passed: boolean;
+    timedOut: boolean;
+};
+export type EvalChangedFile = {
+    path: string;
+    status: string;
+    additions?: number;
+    deletions?: number;
+};
+export type EvalTraceMetrics = {
+    assistantTurns: number;
+    toolCalls: number;
+    toolResults: number;
+    mutations: number;
+    approvalsRequested: number;
+    approvalsResolved: number;
+    toolErrors: number;
+    verificationCommandsAfterMutation: number;
+    firstMutationStep?: number;
+    outcome?: string;
+    summary?: string;
+    toolsByName: Record<string, number>;
+    readOrSearchBeforeMutation: string[];
+    verificationCommandDetails: string[];
+};
+export type EvalRunResult = {
+    caseId: string;
+    target: string;
+    status: 'passed' | 'failed';
+    workspaceRoot: string;
+    outputDir: string;
+    fixture: {
+        type: EvalFixture['type'];
+        repo?: string;
+        ref?: string;
+        resolvedRef?: string;
+        baselineCommit?: string;
+    };
+    startedAt: string;
+    finishedAt: string;
+    durationMs: number;
+    agent: {
+        command: string[];
+        exitCode: number | null;
+        stdoutPath: string;
+        stderrPath: string;
+        timedOut: boolean;
+    };
+    artifacts: {
+        gitStatusPath: string;
+        gitDiffPath: string;
+        gitDiffStatPath: string;
+        changedFilesPath: string;
+        progressPath?: string;
+        sessionCatalogPath?: string;
+        traceFiles: string[];
+        changedFiles: EvalChangedFile[];
+    };
+    checks: EvalCheckResult[];
+    metrics: EvalTraceMetrics;
+    review: EvalMilestoneReview;
+    model?: string;
+    maxSteps?: number;
+};
+export type EvalSuiteReport = {
+    version: 1;
+    target: string;
+    repoRoot: string;
+    startedAt: string;
+    finishedAt: string;
+    resultsDir: string;
+    results: EvalRunResult[];
+};
+//# sourceMappingURL=schema.d.ts.map

package/dist/src/core/eval/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../../src/core/eval/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,eAAe;;;;iBASoG,CAAC;AAEjI,eAAO,MAAM,eAAe;;;;;;;;;iBAmB8E,CAAC;AAE3G,eAAO,MAAM,uBAAuB;;iBAG+C,CAAC;AAEpF,eAAO,MAAM,4BAA4B;;;;iBAQ8C,CAAC;AAExF,eAAO,MAAM,iBAAiB;;;;;;2BAGyC,CAAC;AAExE,eAAO,MAAM,yBAAyB;;;;;;;iBA2BoE,CAAC;AAE3G,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA2CmF,CAAC;AAEpH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAAsB,CAAC;AAElD,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAChE,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAEtD,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,iCAAiC,EAAE,MAAM,CAAC;IAC1C,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,0BAA0B,EAAE,MAAM,EAAE,CAAC;IACrC,0BAA0B,EAAE,MAAM,EAAE,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE;QACP,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;QAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE;QACL,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,SAAS,EAAE;QACT,aAAa,EAAE,MAAM,CAAC;QACtB,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,CAAC;QACxB,gBAAgB,EAAE,MAAM,CAAC;QACzB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,YAAY,EAAE,eAAe,EAAE,CAAC;KACjC,CAAC;IACF,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,OAAO,EAAE,gBAAgB,CAAC;IAC1B,MAAM,EAAE,mBAAmB,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,OAAO,EAAE,CAAC,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,aAAa,EAAE,CAAC;CAC1B,CAAC"}

package/dist/src/core/eval/schema.js

@@ -0,0 +1,104 @@
+import { z } from 'zod';
+export const evalCheckSchema = z.object({
+    name: z.string().trim().min(1)
+        .describe('Human-readable label for this check in reports. Defaults to the command when omitted.')
+        .optional(),
+    command: z.string().trim().min(1)
+        .describe('Shell command to run inside the disposable eval workspace after the agent finishes. Exit code 0 means the check passed.'),
+    timeoutMs: z.number().int().positive()
+        .describe('Optional timeout for this check command in milliseconds.')
+        .optional(),
+}).describe('A deterministic post-run command used to decide whether the agent-produced workspace passes objective validation.');
+export const evalSetupSchema = z.object({
+    copyFiles: z.record(z.string().trim().min(1).describe('Repository-relative source file path to copy from the eval harness repo.'), z.string().trim().min(1).describe('Workspace-relative destination file path to create before the eval starts.'))
+        .describe('Fixture files to copy from the eval harness repo into the disposable workspace before committing the initial Git baseline.')
+        .optional(),
+    files: z.record(z.string().trim().min(1).describe('Workspace-relative file path to create before the eval starts.'), z.string().describe('Exact UTF-8 file contents to write for the fixture file.'))
+        .describe('Fixture files to write into the disposable workspace before committing the initial Git baseline.')
+        .optional(),
+    commands: z.array(evalCheckSchema)
+        .describe('Setup commands to run after fixture files are written and before the initial Git baseline is committed.')
+        .optional(),
+    commitMessage: z.string().trim().min(1)
+        .describe('Commit message for the initial fixture Git baseline. Defaults to a generated eval fixture message.')
+        .optional(),
+}).describe('Instructions for creating the disposable repository state that the agent will work against.');
+export const evalInlineFixtureSchema = z.object({
+    type: z.literal('inline')
+        .describe('Create a small disposable Git repository from the case setup files and commands.'),
+}).describe('A synthetic disposable repository built from inline eval setup data.');
+export const evalGitWorktreeFixtureSchema = z.object({
+    type: z.literal('git-worktree')
+        .describe('Create a disposable Git worktree from an existing repository at a pinned ref.'),
+    repo: z.string().trim().min(1)
+        .describe('Repository path to create the worktree from. Relative paths are resolved from the Heddle repo root.')
+        .default('.'),
+    ref: z.string().trim().min(1)
+        .describe('Pinned target ref for the worktree, such as a release tag or commit SHA. Avoid moving HEAD for comparable evals.'),
+}).describe('A realistic disposable repository fixture created from a pinned Git ref.');
+export const evalFixtureSchema = z.discriminatedUnion('type', [
+    evalInlineFixtureSchema,
+    evalGitWorktreeFixtureSchema,
+]).describe('How to prepare the disposable workspace the agent edits.');
+export const evalMilestoneReviewSchema = z.object({
+    milestone: z.string().trim().min(1)
+        .describe('Short name for the user-intended milestone this case evaluates.')
+        .optional(),
+    intent: z.string().trim().min(1)
+        .describe('Human-readable statement of what the agent should accomplish beyond merely passing checks.')
+        .optional(),
+    requiredOutcomes: z.array(z.string().trim().min(1).describe('Observable outcome a human reviewer should look for in the final diff, trace, or answer.'))
+        .describe('Milestone outcomes expected for a high-quality completion.')
+        .default([]),
+    allowedScope: z.array(z.string().trim().min(1).describe('Files, modules, or behavior areas the agent is allowed or expected to touch.'))
+        .describe('Expected implementation scope for judging whether the diff stayed on task.')
+        .default([]),
+    outOfScope: z.array(z.string().trim().min(1).describe('Files, modules, or behavior areas that should not be changed for this case.'))
+        .describe('Boundaries a human reviewer should use to spot unrelated churn.')
+        .default([]),
+    humanQuestions: z.array(z.string().trim().min(1).describe('Question for human review after the run completes.'))
+        .describe('Review prompts that help judge task completion quality beyond deterministic checks.')
+        .default([]),
+}).describe('Human-review metadata for milestone-style eval cases where pass/fail checks are not enough.');
+export const agentEvalCaseSchema = z.object({
+    id: z.string().trim().regex(/^[a-zA-Z0-9._-]+$/, 'Use a filesystem-safe case id.')
+        .describe('Stable filesystem-safe case id used in result paths, filtering, and reports.'),
+    kind: z.literal('coding')
+        .describe('Eval case type. The first harness slice supports coding cases run through ask --new-session.'),
+    description: z.string().trim()
+        .describe('Optional short explanation of what behavior this case is meant to exercise.')
+        .optional(),
+    prompt: z.string().trim().min(1)
+        .describe('User prompt sent to Heddle in the disposable workspace. This should ask for real coding work, not just Q&A.'),
+    model: z.string().trim().min(1)
+        .describe('Optional model override for this case. The CLI-level --model takes precedence when supplied.')
+        .optional(),
+    maxSteps: z.number().int().positive()
+        .describe('Optional maximum agent loop steps for this case. The CLI-level --max-steps takes precedence when supplied.')
+        .optional(),
+    setup: evalSetupSchema
+        .describe('Disposable workspace setup for this case.')
+        .default({}),
+    fixture: evalFixtureSchema
+        .describe('Workspace fixture source. Defaults to an inline synthetic repository.')
+        .default({ type: 'inline' }),
+    review: evalMilestoneReviewSchema
+        .describe('Optional milestone-completion review guidance included in reports.')
+        .default({
+        requiredOutcomes: [],
+        allowedScope: [],
+        outOfScope: [],
+        humanQuestions: [],
+    }),
+    checks: z.array(evalCheckSchema)
+        .describe('Deterministic post-agent commands that must pass for the case to be marked passed.')
+        .default([]),
+    rubric: z.array(z.string().trim().min(1).describe('Qualitative behavior criterion for human or future LLM judging.'))
+        .describe('Non-deterministic quality criteria preserved in reports for manual or future judge review.')
+        .default([]),
+    tags: z.array(z.string().trim().min(1).describe('Free-form label for filtering or grouping eval cases.'))
+        .describe('Case labels such as bugfix, refactor, verification, multi-file, or tui.')
+        .default([]),
+}).describe('A live coding-task eval case run in a disposable Git workspace through Heddle ask/session execution.');
+export const evalCaseSchema = agentEvalCaseSchema;
+//# sourceMappingURL=schema.js.map

package/dist/src/core/eval/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../../src/core/eval/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;SAC3B,QAAQ,CAAC,uFAAuF,CAAC;SACjG,QAAQ,EAAE;IACb,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;SAC9B,QAAQ,CAAC,yHAAyH,CAAC;IACtI,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;SACnC,QAAQ,CAAC,0DAA0D,CAAC;SACpE,QAAQ,EAAE;CACd,CAAC,CAAC,QAAQ,CAAC,mHAAmH,CAAC,CAAC;AAEjI,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,SAAS,EAAE,CAAC,CAAC,MAAM,CACjB,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,0EAA0E,CAAC,EAC7G,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,4EAA4E,CAAC,CAChH;SACE,QAAQ,CAAC,4HAA4H,CAAC;SACtI,QAAQ,EAAE;IACb,KAAK,EAAE,CAAC,CAAC,MAAM,CACb,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,gEAAgE,CAAC,EACnG,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0DAA0D,CAAC,CAChF;SACE,QAAQ,CAAC,kGAAkG,CAAC;SAC5G,QAAQ,EAAE;IACb,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC;SAC/B,QAAQ,CAAC,yGAAyG,CAAC;SACnH,QAAQ,EAAE;IACb,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;SACpC,QAAQ,CAAC,oGAAoG,CAAC;SAC9G,QAAQ,EAAE;CACd,CAAC,CAAC,QAAQ,CAAC,6FAA6F,CAAC,CAAC;AAE3G,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;SACtB,QAAQ,CAAC,kFAAkF,CAAC;CAChG,CAAC,CAAC,QAAQ,CAAC,sEAAsE,CAAC,CAAC;AAEpF,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAAC,CAAC,MAAM,CAAC;IACnD,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC;SAC5B,QAAQ,CAAC,+EAA+E,CAAC;IAC5F,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;SAC3B,QAAQ,CAAC,qGAAqG,CAAC;SAC/G,OAAO,CAAC,GAAG,CAAC;IACf,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;SAC1B,QAAQ,CAAC,kHAAkH,CAAC;CAChI,CAAC,CAAC,QAAQ,CAAC,0EAA0E,CAAC,CAAC;AAExF,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE;IAC5D,uBAAuB;IACvB,4BAA4B;CAC7B,CAAC,CAAC,QAAQ,CAAC,0DAA0D,CAAC,CAAC;AAExE,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;SAChC,QAAQ,CAAC,iEAAiE,CAAC;SAC3E,QAAQ,EAAE;IACb,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;SAC7B,QAAQ,CAAC,4FAA4F,CAAC;SACtG,QAAQ,EAAE;IACb,gBAAgB,EAAE,CAAC,CAAC,KAAK,CACvB,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,0FAA0F,CAAC,CAC9H;SACE,QAAQ,CAAC,4DAA4D,CAAC;SACtE,OAAO,CAAC,EAAE,CAAC;IACd,YAAY,EAAE,CAAC,CAAC,KAAK,CACnB,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,8EAA8E,CAAC,CAClH;SACE,QAAQ,CAAC,4EAA4E,CAAC;SACtF,OAAO,CAAC,EAAE,CAAC;IACd,UAAU,EAAE,CAAC,CAAC,KAAK,CACjB,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,6EAA6E,CAAC,CACjH;SACE,QAAQ,CAAC,iEAAiE,CAAC;SAC3E,OAAO,CAAC,EAAE,CAAC;IACd,cAAc,EAAE,CAAC,CAAC,KAAK,CACrB,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,oDAAoD,CAAC,CACxF;SACE,QAAQ,CAAC,qFAAqF,CAAC;SAC/F,OAAO,CAAC,EAAE,CAAC;CACf,CAAC,CAAC,QAAQ,CAAC,6FAA6F,CAAC,CAAC;AAE3G,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1C,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,mBAAmB,EAAE,gCAAgC,CAAC;SAC/E,QAAQ,CAAC,8EAA8E,CAAC;IAC3F,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;SACtB,QAAQ,CAAC,8FAA8F,CAAC;IAC3G,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE;SAC3B,QAAQ,CAAC,6EAA6E,CAAC;SACvF,QAAQ,EAAE;IACb,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;SAC7B,QAAQ,CAAC,6GAA6G,CAAC;IAC1H,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;SAC5B,QAAQ,CAAC,8FAA8F,CAAC;SACxG,QAAQ,EAAE;IACb,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;SAClC,QAAQ,CAAC,4GAA4G,CAAC;SACtH,QAAQ,EAAE;IACb,KAAK,EAAE,eAAe;SACnB,QAAQ,CAAC,2CAA2C,CAAC;SACrD,OAAO,CAAC,EAAE,CAAC;IACd,OAAO,EAAE,iBAAiB;SACvB,QAAQ,CAAC,uEAAuE,CAAC;SACjF,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAC9B,MAAM,EAAE,yBAAyB;SAC9B,QAAQ,CAAC,oEAAoE,CAAC;SAC9E,OAAO,CAAC;QACP,gBAAgB,EAAE,EAAE;QACpB,YAAY,EAAE,EAAE;QAChB,UAAU,EAAE,EAAE;QACd,cAAc,EAAE,EAAE;KACnB,CAAC;IACJ,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC;SAC7B,QAAQ,CAAC,oFAAoF,CAAC;SAC9F,OAAO,CAAC,EAAE,CAAC;IACd,MAAM,EAAE,CAAC,CAAC,KAAK,CACb,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,iEAAiE,CAAC,CACrG;SACE,QAAQ,CAAC,4FAA4F,CAAC;SACtG,OAAO,CAAC,EAAE,CAAC;IACd,IAAI,EAAE,CAAC,CAAC,KAAK,CACX,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,uDAAuD,CAAC,CAC3F;SACE,QAAQ,CAAC,yEAAyE,CAAC;SACnF,OAAO,CAAC,EAAE,CAAC;CACf,CAAC,CAAC,QAAQ,CAAC,sGAAsG,CAAC,CAAC;AAEpH,MAAM,CAAC,MAAM,cAAc,GAAG,mBAAmB,CAAC"}

package/dist/src/core/eval/trace-analyzer.d.ts

@@ -0,0 +1,6 @@
+import type { TraceEvent } from '../types.js';
+import type { EvalTraceMetrics } from './schema.js';
+export declare function analyzeTraceFiles(paths: string[]): EvalTraceMetrics;
+export declare function analyzeTrace(trace: TraceEvent[]): EvalTraceMetrics;
+export declare function readTraceFile(path: string): TraceEvent[];
+//# sourceMappingURL=trace-analyzer.d.ts.map

package/dist/src/core/eval/trace-analyzer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"trace-analyzer.d.ts","sourceRoot":"","sources":["../../../../src/core/eval/trace-analyzer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAKpD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,gBAAgB,CAEnE;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,gBAAgB,CAiFlE;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,EAAE,CAMxD"}

package/dist/src/core/eval/trace-analyzer.js

@@ -0,0 +1,106 @@
+import { existsSync, readFileSync } from 'node:fs';
+const MUTATION_TOOLS = new Set(['edit_file', 'delete_file', 'move_file', 'run_shell_mutate']);
+const READ_OR_SEARCH_TOOLS = new Set(['read_file', 'list_files', 'search_files', 'run_shell_inspect']);
+export function analyzeTraceFiles(paths) {
+    return analyzeTrace(paths.flatMap(readTraceFile));
+}
+export function analyzeTrace(trace) {
+    const toolsByName = {};
+    const readOrSearchBeforeMutation = [];
+    let assistantTurns = 0;
+    let toolCalls = 0;
+    let toolResults = 0;
+    let mutations = 0;
+    let approvalsRequested = 0;
+    let approvalsResolved = 0;
+    let toolErrors = 0;
+    let verificationCommandsAfterMutation = 0;
+    const verificationCommandDetails = [];
+    let firstMutationStep;
+    let outcome;
+    let summary;
+    for (const event of trace) {
+        if (event.type === 'assistant.turn') {
+            assistantTurns++;
+            for (const call of event.toolCalls ?? []) {
+                toolCalls++;
+                toolsByName[call.tool] = (toolsByName[call.tool] ?? 0) + 1;
+                if (READ_OR_SEARCH_TOOLS.has(call.tool) && firstMutationStep === undefined) {
+                    readOrSearchBeforeMutation.push(summarizeToolInput(call.tool, call.input));
+                }
+                if (MUTATION_TOOLS.has(call.tool)) {
+                    mutations++;
+                    firstMutationStep ??= event.step;
+                }
+                if (firstMutationStep !== undefined
+                    && (call.tool === 'run_shell_mutate' || call.tool === 'run_shell_inspect')
+                    && isVerificationCommand(call.input)) {
+                    verificationCommandsAfterMutation++;
+                    verificationCommandDetails.push(summarizeToolInput(call.tool, call.input));
+                }
+            }
+            continue;
+        }
+        if (event.type === 'tool.result') {
+            toolResults++;
+            if (!event.result.ok) {
+                toolErrors++;
+            }
+        }
+        if (event.type === 'tool.approval_requested') {
+            approvalsRequested++;
+        }
+        if (event.type === 'tool.approval_resolved') {
+            approvalsResolved++;
+        }
+        if (event.type === 'run.finished') {
+            outcome = event.outcome;
+            summary = event.summary;
+        }
+    }
+    return {
+        assistantTurns,
+        toolCalls,
+        toolResults,
+        mutations,
+        approvalsRequested,
+        approvalsResolved,
+        toolErrors,
+        verificationCommandsAfterMutation,
+        verificationCommandDetails: [...new Set(verificationCommandDetails)],
+        firstMutationStep,
+        outcome,
+        summary,
+        toolsByName,
+        readOrSearchBeforeMutation: [...new Set(readOrSearchBeforeMutation)],
+    };
+}
+export function readTraceFile(path) {
+    if (!existsSync(path)) {
+        return [];
+    }
+    const parsed = JSON.parse(readFileSync(path, 'utf8'));
+    return Array.isArray(parsed) ? parsed.filter(isTraceEvent) : [];
+}
+function isTraceEvent(value) {
+    return Boolean(value && typeof value === 'object' && !Array.isArray(value) && typeof value.type === 'string');
+}
+function summarizeToolInput(tool, input) {
+    const object = input && typeof input === 'object' && !Array.isArray(input) ? input : undefined;
+    const path = object && typeof object.path === 'string' ? object.path : undefined;
+    const query = object && typeof object.query === 'string' ? object.query : undefined;
+    const command = object && typeof object.command === 'string' ? object.command : undefined;
+    return `${tool}:${path ?? query ?? command ?? ''}`.slice(0, 160);
+}
+function isVerificationCommand(input) {
+    if (!input || typeof input !== 'object' || Array.isArray(input)) {
+        return false;
+    }
+    const command = input.command;
+    if (typeof command !== 'string') {
+        return false;
+    }
+    return /\b(yarn|npm|pnpm|vitest|jest|mocha|tsc|eslint|cargo|go|pytest|python|ruff)\b/.test(command)
+        && /\b(test|build|lint|typecheck|check|vitest|tsc|pytest|ruff)\b/.test(command);
+}
+//# sourceMappingURL=trace-analyzer.js.map

package/dist/src/core/eval/trace-analyzer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"trace-analyzer.js","sourceRoot":"","sources":["../../../../src/core/eval/trace-analyzer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAInD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,aAAa,EAAE,WAAW,EAAE,kBAAkB,CAAC,CAAC,CAAC;AAC9F,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,YAAY,EAAE,cAAc,EAAE,mBAAmB,CAAC,CAAC,CAAC;AAEvG,MAAM,UAAU,iBAAiB,CAAC,KAAe;IAC/C,OAAO,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAmB;IAC9C,MAAM,WAAW,GAA2B,EAAE,CAAC;IAC/C,MAAM,0BAA0B,GAAa,EAAE,CAAC;IAChD,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,iCAAiC,GAAG,CAAC,CAAC;IAC1C,MAAM,0BAA0B,GAAa,EAAE,CAAC;IAChD,IAAI,iBAAqC,CAAC;IAC1C,IAAI,OAA2B,CAAC;IAChC,IAAI,OAA2B,CAAC;IAEhC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,EAAE,CAAC;YACpC,cAAc,EAAE,CAAC;YACjB,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,SAAS,IAAI,EAAE,EAAE,CAAC;gBACzC,SAAS,EAAE,CAAC;gBACZ,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;gBAE3D,IAAI,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;oBAC3E,0BAA0B,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7E,CAAC;gBAED,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAClC,SAAS,EAAE,CAAC;oBACZ,iBAAiB,KAAK,KAAK,CAAC,IAAI,CAAC;gBACnC,CAAC;gBAED,IACE,iBAAiB,KAAK,SAAS;uBAC5B,CAAC,IAAI,CAAC,IAAI,KAAK,kBAAkB,IAAI,IAAI,CAAC,IAAI,KAAK,mBAAmB,CAAC;uBACvE,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,EACpC,CAAC;oBACD,iCAAiC,EAAE,CAAC;oBACpC,0BAA0B,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7E,CAAC;YACH,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;YACjC,WAAW,EAAE,CAAC;YACd,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;gBACrB,UAAU,EAAE,CAAC;YACf,CAAC;QACH,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,KAAK,yBAAyB,EAAE,CAAC;YAC7C,kBAAkB,EAAE,CAAC;QACvB,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,KAAK,wBAAwB,EAAE,CAAC;YAC5C,iBAAiB,EAAE,CAAC;QACtB,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAClC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;YACxB,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc;QACd,SAAS;QACT,WAAW;QACX,SAAS;QACT,kBAAkB;QAClB,iBAAiB;QACjB,UAAU;QACV,iCAAiC;QACjC,0BAA0B,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,0BAA0B,CAAC,CAAC;QACpE,iBAAiB;QACjB,OAAO;QACP,OAAO;QACP,WAAW;QACX,0BAA0B,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,0BAA0B,CAAC,CAAC;KACrE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAY,CAAC;IACjE,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAClE,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,OAAO,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,OAAQ,KAA4B,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;AACxI,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY,EAAE,KAAc;IACtD,MAAM,MAAM,GAAG,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAgC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1H,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IACjF,MAAM,KAAK,GAAG,MAAM,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IACpF,MAAM,OAAO,GAAG,MAAM,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1F,OAAO,GAAG,IAAI,IAAI,IAAI,IAAI,KAAK,IAAI,OAAO,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAc;IAC3C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAChE,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,OAAO,GAAI,KAA+B,CAAC,OAAO,CAAC;IACzD,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,8EAA8E,CAAC,IAAI,CAAC,OAAO,CAAC;WAC9F,8DAA8D,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACpF,CAAC"}

package/dist/src/core/eval/workspace-fixture.d.ts

@@ -0,0 +1,14 @@
+import type { EvalProgressReporter } from './progress.js';
+import type { AgentEvalCase, EvalCheckResult, EvalRunResult } from './schema.js';
+export type PreparedEvalWorkspace = {
+    workspaceRoot: string;
+    setupResults: EvalCheckResult[];
+    fixture: EvalRunResult['fixture'];
+};
+export declare function prepareEvalWorkspace(args: {
+    testCase: AgentEvalCase;
+    repoRoot: string;
+    workRoot?: string;
+    progress?: EvalProgressReporter;
+}): Promise<PreparedEvalWorkspace>;
+//# sourceMappingURL=workspace-fixture.d.ts.map

package/dist/src/core/eval/workspace-fixture.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"workspace-fixture.d.ts","sourceRoot":"","sources":["../../../../src/core/eval/workspace-fixture.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,KAAK,EAAE,aAAa,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjF,MAAM,MAAM,qBAAqB,GAAG;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,eAAe,EAAE,CAAC;IAChC,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,CAAC;CACnC,CAAC;AAEF,wBAAsB,oBAAoB,CAAC,IAAI,EAAE;IAC/C,QAAQ,EAAE,aAAa,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC,GAAG,OAAO,CAAC,qBAAqB,CAAC,CA8BjC"}