@vercel/agent-eval 1.2.0 → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/cli.js +318 -102
  2. package/dist/cli.js.map +1 -1
  3. package/dist/lib/agents/claude-code/agent.d.ts.map +1 -1
  4. package/dist/lib/agents/claude-code/agent.js +1 -0
  5. package/dist/lib/agents/claude-code/agent.js.map +1 -1
  6. package/dist/lib/agents/codex/agent.d.ts.map +1 -1
  7. package/dist/lib/agents/codex/agent.js +1 -0
  8. package/dist/lib/agents/codex/agent.js.map +1 -1
  9. package/dist/lib/agents/cursor/agent.d.ts.map +1 -1
  10. package/dist/lib/agents/cursor/agent.js +1 -0
  11. package/dist/lib/agents/cursor/agent.js.map +1 -1
  12. package/dist/lib/agents/eval-helper.mjs +10 -6
  13. package/dist/lib/agents/gemini/agent.d.ts.map +1 -1
  14. package/dist/lib/agents/gemini/agent.js +1 -0
  15. package/dist/lib/agents/gemini/agent.js.map +1 -1
  16. package/dist/lib/agents/opencode/agent.d.ts +22 -0
  17. package/dist/lib/agents/opencode/agent.d.ts.map +1 -1
  18. package/dist/lib/agents/opencode/agent.js +42 -0
  19. package/dist/lib/agents/opencode/agent.js.map +1 -1
  20. package/dist/lib/agents/opencode/run.mjs +44 -6
  21. package/dist/lib/agents/plugin/orchestrator.d.ts +37 -0
  22. package/dist/lib/agents/plugin/orchestrator.d.ts.map +1 -1
  23. package/dist/lib/agents/plugin/orchestrator.js +71 -11
  24. package/dist/lib/agents/plugin/orchestrator.js.map +1 -1
  25. package/dist/lib/agents/shared.d.ts +11 -1
  26. package/dist/lib/agents/shared.d.ts.map +1 -1
  27. package/dist/lib/agents/shared.js +13 -1
  28. package/dist/lib/agents/shared.js.map +1 -1
  29. package/dist/lib/agents/types.d.ts +8 -1
  30. package/dist/lib/agents/types.d.ts.map +1 -1
  31. package/dist/lib/config.d.ts.map +1 -1
  32. package/dist/lib/config.js +19 -0
  33. package/dist/lib/config.js.map +1 -1
  34. package/dist/lib/fingerprint.d.ts +39 -1
  35. package/dist/lib/fingerprint.d.ts.map +1 -1
  36. package/dist/lib/fingerprint.js +62 -7
  37. package/dist/lib/fingerprint.js.map +1 -1
  38. package/dist/lib/results.d.ts +5 -1
  39. package/dist/lib/results.d.ts.map +1 -1
  40. package/dist/lib/results.js +8 -2
  41. package/dist/lib/results.js.map +1 -1
  42. package/dist/lib/runner.d.ts +3 -0
  43. package/dist/lib/runner.d.ts.map +1 -1
  44. package/dist/lib/runner.js +4 -1
  45. package/dist/lib/runner.js.map +1 -1
  46. package/dist/lib/types.d.ts +20 -0
  47. package/dist/lib/types.d.ts.map +1 -1
  48. package/dist/lib/types.js.map +1 -1
  49. package/package.json +1 -1
@@ -70,6 +70,21 @@ export type RunCompleteHook = (context: RunCompleteContext) => Promise<EvalRunDa
70
70
  * Sandbox backend type.
71
71
  */
72
72
  export type SandboxBackend = 'vercel' | 'docker';
73
+ /**
74
+ * Configures the agentic LLM judge used by `expect(environment|transcript)`
75
+ * matchers in EVAL.ts. By default the judge runs as the SAME agent+model that
76
+ * generated the code (self-grading). Pin it here to grade every model with one
77
+ * fixed judge — the apples-to-apples choice for cross-model dashboards.
78
+ */
79
+ export interface JudgeConfig {
80
+ /** Judge harness. Defaults to the codegen agent (same harness, model pinned).
81
+ * Set a Claude variant (e.g. 'vercel-ai-gateway/claude-code') to judge codex,
82
+ * gemini, etc. runs with Claude. */
83
+ agent?: AgentType;
84
+ /** Model the judge grades with (e.g. 'claude-opus-4-8'). Required — pinning the
85
+ * judge model is the whole point. */
86
+ model: ModelTier;
87
+ }
73
88
  /**
74
89
  * Experiment configuration.
75
90
  * Defines what to test and how.
@@ -115,6 +130,9 @@ export interface ExperimentConfig {
115
130
  brands?: BrandConfig[];
116
131
  /** Optional hook for custom post-run analysis before results are saved. */
117
132
  onRunComplete?: RunCompleteHook;
133
+ /** Pin the agentic LLM judge to a fixed agent+model. @default undefined
134
+ * (judge self-grades with the codegen agent+model). */
135
+ judge?: JudgeConfig;
118
136
  }
119
137
  /**
120
138
  * Resolved experiment config with all defaults applied.
@@ -137,6 +155,7 @@ export interface ResolvedExperimentConfig {
137
155
  webResearch?: boolean;
138
156
  brands?: BrandConfig[];
139
157
  onRunComplete?: RunCompleteHook;
158
+ judge?: JudgeConfig;
140
159
  }
141
160
  /**
142
161
  * Resolved experiment config with all defaults applied.
@@ -159,6 +178,7 @@ export interface RunnableExperimentConfig {
159
178
  webResearch?: boolean;
160
179
  brands?: BrandConfig[];
161
180
  onRunComplete?: RunCompleteHook;
181
+ judge?: JudgeConfig;
162
182
  }
163
183
  /**
164
184
  * Required files for a valid eval fixture.
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,SAAS,GACjB,+BAA+B,GAC/B,aAAa,GACb,yBAAyB,GACzB,OAAO,GACP,4BAA4B,GAC5B,QAAQ,GACR,QAAQ,CAAC;AAEb;;;GAGG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC;AAE/B;;;;GAIG;AACH,MAAM,MAAM,WAAW,GAAG,eAAe,GAAG,gBAAgB,CAAC;AAE7D,kFAAkF;AAClF,eAAO,MAAM,oBAAoB,EAAG,gBAAyB,CAAC;AAE9D;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;AAEnD,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,MAAM,CAAC;AAE/C,MAAM,WAAW,WAAW;IAC1B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,mCAAmC;IACnC,UAAU,CACR,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,GACvD,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjE,mCAAmC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACxC,iCAAiC;IACjC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,wCAAwC;IACxC,mBAAmB,IAAI,MAAM,CAAC;IAC9B,gEAAgE;IAChE,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACzC;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAEhE,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,WAAW,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,wBAAwB,CAAC;IACjC,OAAO,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,MAAM,eAAe,GAAG,CAC5B,OAAO,EAAE,kBAAkB,KACxB,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,WAAW,GAAG,IAAI,CAAC;AAEtD;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEjD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,4BAA4B;IAC5B,KAAK,EAAE,SAAS,CAAC;IAEjB;gFAC4E;IAC5E,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC;IAEhC,+FAA+F;IAC/F,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IAEvC,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,qDAAqD;IACrD,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB,kEAAkE;IAClE,UAAU,CAAC,EAAE,cAAc,CAAC;IAE5B,8EAA8E;IAC9E,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,uEAAuE;IACvE,KAAK,CAAC,EAAE,aAAa,CAAC;IAEtB,qFAAqF;IACrF,OAAO,CAAC,EAAE,cAAc,GAAG,MAAM,CAAC;IAElC,+FAA+F;IAC/F,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IAExC;;;;yBAIqB;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,KAAK,CAAC;IAEvC;6EACyE;IACzE,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEvC;;0EAEsE;IACtE,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB,4EAA4E;IAC5E,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IAEvB,2EAA2E;IAC3E,aAAa,CAAC,EAAE,eAAe,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC;IACjC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IACxC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,KAAK,CAAC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IACvB,aAAa,CAAC,EAAE,eAAe,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC;IACjC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IACxC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,KAAK,CAAC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IACvB,aAAa,CAAC,EAAE,eAAe,CAAC;CACjC;AAED;;;GAGG;AACH,eAAO,MAAM,mBAAmB,mDAAoD,CAAC;AAErF;;;;GAIG;AACH,eAAO,MAAM,cAAc,uEAAwE,CAAC;AAEpG;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,qDAAqD;IACrD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,8DAA8D;IAC9D,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wDAAwD;IACxD,WAAW,CAAC,EAAE;QACZ,kCAAkC;QAClC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,8DAA8D;QAC9D,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;IACF,qEAAqE;IACrE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,gEAAgE;IAChE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,0DAA0D;IAC1D,MAAM,EAAE,aAAa,CAAC;IACtB,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,aAAa,CAAC,EAAE;QACd,0BAA0B;QAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,qDAAqD;QACrD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;IACF,0FAA0F;IAC1F,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC,6DAA6D;IAC7D,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,IAAI,EAAE,WAAW,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,WAAW,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,wGAAwG;IACxG,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC1F;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC9E;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,aAAa,CAAA;CAAE,GACxG;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC3D;IAAE,IAAI,EAAE,sBAAsB,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACrE;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC/C;IAAE,IAAI,EAAE,oBAAoB,CAAC;IAAC,OAAO,EAAE,iBAAiB,CAAA;CAAE,CAAC;AAE/D;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,wCAAwC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,0CAA0C;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,oCAAoC;IACpC,MAAM,EAAE,wBAAwB,CAAC;IACjC,4BAA4B;IAC5B,KAAK,EAAE,WAAW,EAAE,CAAC;CACtB"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,SAAS,GACjB,+BAA+B,GAC/B,aAAa,GACb,yBAAyB,GACzB,OAAO,GACP,4BAA4B,GAC5B,QAAQ,GACR,QAAQ,CAAC;AAEb;;;GAGG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC;AAE/B;;;;GAIG;AACH,MAAM,MAAM,WAAW,GAAG,eAAe,GAAG,gBAAgB,CAAC;AAE7D,kFAAkF;AAClF,eAAO,MAAM,oBAAoB,EAAG,gBAAyB,CAAC;AAE9D;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;AAEnD,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,MAAM,CAAC;AAE/C,MAAM,WAAW,WAAW;IAC1B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,mCAAmC;IACnC,UAAU,CACR,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,GACvD,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjE,mCAAmC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACxC,iCAAiC;IACjC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,wCAAwC;IACxC,mBAAmB,IAAI,MAAM,CAAC;IAC9B,gEAAgE;IAChE,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACzC;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAEhE,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,WAAW,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,wBAAwB,CAAC;IACjC,OAAO,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,MAAM,eAAe,GAAG,CAC5B,OAAO,EAAE,kBAAkB,KACxB,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,WAAW,GAAG,IAAI,CAAC;AAEtD;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEjD;;;;;GAKG;AACH,MAAM,WAAW,WAAW;IAC1B;;wCAEoC;IACpC,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB;yCACqC;IACrC,KAAK,EAAE,SAAS,CAAC;CAClB;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,4BAA4B;IAC5B,KAAK,EAAE,SAAS,CAAC;IAEjB;gFAC4E;IAC5E,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC;IAEhC,+FAA+F;IAC/F,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IAEvC,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,qDAAqD;IACrD,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB,kEAAkE;IAClE,UAAU,CAAC,EAAE,cAAc,CAAC;IAE5B,8EAA8E;IAC9E,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,uEAAuE;IACvE,KAAK,CAAC,EAAE,aAAa,CAAC;IAEtB,qFAAqF;IACrF,OAAO,CAAC,EAAE,cAAc,GAAG,MAAM,CAAC;IAElC,+FAA+F;IAC/F,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IAExC;;;;yBAIqB;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,KAAK,CAAC;IAEvC;6EACyE;IACzE,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEvC;;0EAEsE;IACtE,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB,4EAA4E;IAC5E,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IAEvB,2EAA2E;IAC3E,aAAa,CAAC,EAAE,eAAe,CAAC;IAEhC;2DACuD;IACvD,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC;IACjC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IACxC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,KAAK,CAAC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IACvB,aAAa,CAAC,EAAE,eAAe,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,SAAS,CAAC;IACjB,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC;IACjC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IACxC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,KAAK,CAAC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IACvB,aAAa,CAAC,EAAE,eAAe,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;;GAGG;AACH,eAAO,MAAM,mBAAmB,mDAAoD,CAAC;AAErF;;;;GAIG;AACH,eAAO,MAAM,cAAc,uEAAwE,CAAC;AAEpG;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,qDAAqD;IACrD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,8DAA8D;IAC9D,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wDAAwD;IACxD,WAAW,CAAC,EAAE;QACZ,kCAAkC;QAClC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,8DAA8D;QAC9D,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;IACF,qEAAqE;IACrE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,gEAAgE;IAChE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,0DAA0D;IAC1D,MAAM,EAAE,aAAa,CAAC;IACtB,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,aAAa,CAAC,EAAE;QACd,0BAA0B;QAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,qDAAqD;QACrD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAClC,CAAC;IACF,0FAA0F;IAC1F,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC,6DAA6D;IAC7D,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,IAAI,EAAE,WAAW,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,WAAW,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,wGAAwG;IACxG,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC1F;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC9E;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,aAAa,CAAA;CAAE,GACxG;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC3D;IAAE,IAAI,EAAE,sBAAsB,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACrE;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC/C;IAAE,IAAI,EAAE,oBAAoB,CAAC;IAAC,OAAO,EAAE,iBAAiB,CAAA;CAAE,CAAC;AAE/D;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,wCAAwC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,0CAA0C;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,oCAAoC;IACpC,MAAM,EAAE,wBAAwB,CAAC;IACjC,4BAA4B;IAC5B,KAAK,EAAE,WAAW,EAAE,CAAC;CACtB"}
@@ -1 +1 @@
1
- {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AA2BH,kFAAkF;AAClF,MAAM,CAAC,MAAM,oBAAoB,GAAG,gBAAyB,CAAC;AAyK9D;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,cAAc,CAAU,CAAC;AAErF;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,CAAU,CAAC"}
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AA2BH,kFAAkF;AAClF,MAAM,CAAC,MAAM,oBAAoB,GAAG,gBAAyB,CAAC;AA+L9D;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,cAAc,CAAU,CAAC;AAErF;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,CAAU,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vercel/agent-eval",
3
- "version": "1.2.0",
3
+ "version": "1.3.1",
4
4
  "description": "Framework for testing AI coding agents in isolated sandboxes",
5
5
  "repository": {
6
6
  "type": "git",