@vercel/agent-eval 0.0.15 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/config.d.ts +1 -0
- package/dist/lib/config.d.ts.map +1 -1
- package/dist/lib/config.js +53 -6
- package/dist/lib/config.js.map +1 -1
- package/dist/lib/fixture.d.ts +2 -1
- package/dist/lib/fixture.d.ts.map +1 -1
- package/dist/lib/fixture.js +24 -11
- package/dist/lib/fixture.js.map +1 -1
- package/dist/lib/init.js +2 -2
- package/package.json +2 -1
package/dist/lib/config.d.ts
CHANGED
|
@@ -30,6 +30,7 @@ export declare function resolveConfig(config: ExperimentConfig): ResolvedExperim
|
|
|
30
30
|
export declare function loadConfig(configPath: string): Promise<ResolvedExperimentConfig>;
|
|
31
31
|
/**
|
|
32
32
|
* Resolves the evals filter to a list of eval names.
|
|
33
|
+
* Supports glob patterns like "vercel-cli/*" for nested directories.
|
|
33
34
|
*/
|
|
34
35
|
export declare function resolveEvalNames(filter: string | string[] | EvalFilter, availableEvals: string[]): string[];
|
|
35
36
|
//# sourceMappingURL=config.d.ts.map
|
package/dist/lib/config.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EACV,gBAAgB,EAChB,wBAAwB,EACxB,UAAU,EACX,MAAM,YAAY,CAAC;AAGpB;;GAEG;AACH,eAAO,MAAM,eAAe;;;;;aAKX,MAAM,EAAE;;;CAGxB,CAAC;AA0BF;;;GAGG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,OAAO,GAAG,gBAAgB,CAWhE;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,gBAAgB,GAAG,wBAAwB,CAmBhF;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,wBAAwB,CAAC,CA8BtF;AAgBD;;;GAGG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,EACtC,cAAc,EAAE,MAAM,EAAE,GACvB,MAAM,EAAE,CA0DV"}
|
package/dist/lib/config.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* Experiment configuration validation and defaults.
|
|
3
3
|
*/
|
|
4
4
|
import { z } from 'zod';
|
|
5
|
+
import { minimatch } from 'minimatch';
|
|
5
6
|
import { getAgent } from './agents/index.js';
|
|
6
7
|
/**
|
|
7
8
|
* Default configuration values.
|
|
@@ -107,27 +108,73 @@ export async function loadConfig(configPath) {
|
|
|
107
108
|
throw error;
|
|
108
109
|
}
|
|
109
110
|
}
|
|
111
|
+
/**
|
|
112
|
+
* Check if a name matches a glob-style pattern.
|
|
113
|
+
* Supports:
|
|
114
|
+
* - "*" matches everything
|
|
115
|
+
* - "vercel-cli/*" matches all under vercel-cli/
|
|
116
|
+
* - "* /deploy" matches any deploy in any folder
|
|
117
|
+
* - "vercel-cli/deploy" exact match
|
|
118
|
+
*
|
|
119
|
+
* Uses minimatch for robust glob matching.
|
|
120
|
+
*/
|
|
121
|
+
function matchesPattern(name, pattern) {
|
|
122
|
+
return minimatch(name, pattern);
|
|
123
|
+
}
|
|
110
124
|
/**
|
|
111
125
|
* Resolves the evals filter to a list of eval names.
|
|
126
|
+
* Supports glob patterns like "vercel-cli/*" for nested directories.
|
|
112
127
|
*/
|
|
113
128
|
export function resolveEvalNames(filter, availableEvals) {
|
|
114
|
-
// Single eval name
|
|
129
|
+
// Single eval name or pattern
|
|
115
130
|
if (typeof filter === 'string') {
|
|
116
131
|
if (filter === '*') {
|
|
117
132
|
return availableEvals;
|
|
118
133
|
}
|
|
134
|
+
// Check if it's a glob pattern
|
|
135
|
+
if (filter.includes('*')) {
|
|
136
|
+
const matched = availableEvals.filter((name) => matchesPattern(name, filter));
|
|
137
|
+
if (matched.length === 0) {
|
|
138
|
+
throw new Error(`No evals matched pattern "${filter}". Available evals: ${availableEvals.join(', ')}`);
|
|
139
|
+
}
|
|
140
|
+
return matched;
|
|
141
|
+
}
|
|
142
|
+
// Exact match
|
|
119
143
|
if (!availableEvals.includes(filter)) {
|
|
120
144
|
throw new Error(`Eval "${filter}" not found. Available evals: ${availableEvals.join(', ')}`);
|
|
121
145
|
}
|
|
122
146
|
return [filter];
|
|
123
147
|
}
|
|
124
|
-
// Array of eval names
|
|
148
|
+
// Array of eval names or patterns
|
|
125
149
|
if (Array.isArray(filter)) {
|
|
126
|
-
const
|
|
127
|
-
|
|
128
|
-
|
|
150
|
+
const result = [];
|
|
151
|
+
const seen = new Set();
|
|
152
|
+
for (const item of filter) {
|
|
153
|
+
// Handle glob patterns in arrays
|
|
154
|
+
if (item.includes('*')) {
|
|
155
|
+
const matched = availableEvals.filter((name) => matchesPattern(name, item));
|
|
156
|
+
if (matched.length === 0) {
|
|
157
|
+
throw new Error(`No evals matched pattern "${item}". Available evals: ${availableEvals.join(', ')}`);
|
|
158
|
+
}
|
|
159
|
+
for (const name of matched) {
|
|
160
|
+
if (!seen.has(name)) {
|
|
161
|
+
result.push(name);
|
|
162
|
+
seen.add(name);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
else {
|
|
167
|
+
// Exact match
|
|
168
|
+
if (!availableEvals.includes(item)) {
|
|
169
|
+
throw new Error(`Eval "${item}" not found. Available evals: ${availableEvals.join(', ')}`);
|
|
170
|
+
}
|
|
171
|
+
if (!seen.has(item)) {
|
|
172
|
+
result.push(item);
|
|
173
|
+
seen.add(item);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
129
176
|
}
|
|
130
|
-
return
|
|
177
|
+
return result;
|
|
131
178
|
}
|
|
132
179
|
// Filter function
|
|
133
180
|
return availableEvals.filter(filter);
|
package/dist/lib/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAMtC,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAE7C;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,KAAK,EAAE,MAAe;IACtB,KAAK,EAAE,GAAY;IACnB,IAAI,EAAE,CAAC;IACP,SAAS,EAAE,IAAI;IACf,OAAO,EAAE,EAAc;IACvB,OAAO,EAAE,GAAG,EAAE,aAAa;IAC3B,OAAO,EAAE,MAAe;CACzB,CAAC;AAEF;;GAEG;AACH,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC;QACZ,+BAA+B;QAC/B,aAAa;QACb,yBAAyB;QACzB,OAAO;QACP,4BAA4B;KAC7B,CAAC;IACF,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5D,KAAK,EAAE,CAAC;SACL,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;SAC5F,QAAQ,EAAE;IACb,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC5C,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACjC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACvC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACzC,KAAK,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC9B,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;IACxD,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;CACzE,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,MAAe;IAC5C,MAAM,MAAM,GAAG,sBAAsB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAExD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM;aAC/B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;aAC/D,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,sCAAsC,MAAM,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,MAAM,CAAC,IAAwB,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,MAAwB;IACpD,wBAAwB;IACxB,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAErC,gDAAgD;IAChD,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;IAE7D,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,KAAK,EAAE,YAAY;QACnB,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,GAAG;QAC1B,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,eAAe,CAAC,IAAI;QACzC,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,eAAe,CAAC,SAAS;QACxD,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,eAAe,CAAC,OAAO;QAClD,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,eAAe,CAAC,OAAO;QAClD,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,eAAe,CAAC,OAAO;QAClD,UAAU,EAAE,MAAM,CAAC,UAAU;KAC9B,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,UAAkB;IACjD,IAAI,CAAC;QACH,IAAI,SAAkB,CAAC;QAEvB,gCAAgC;QAChC,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5C,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;gBACvC,cAAc,EAAE,IAAI;gBACpB,WAAW,EAAE,KAAK;aACnB,CAAC,CAAC;YACH,SAAS,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC5C,CAAC;aAAM,CAAC;YACN,sCAAsC;YACtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC;YACxC,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,CAAC;QAED,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;QACzC,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,8BAA8B,UAAU,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAChF,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;GASG;AACH,SAAS,cAAc,CAAC,IAAY,EAAE,OAAe;IACnD,OAAO,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAClC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAC9B,MAAsC,EACtC,cAAwB;IAExB,8BAA8B;IAC9B,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACnB,OAAO,cAAc,CAAC;QACxB,CAAC;QAED,+BAA+B;QAC/B,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;YAC9E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,uBAAuB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACzG,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,cAAc;QACd,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,SAAS,MAAM,iCAAiC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/F,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,CAAC;IAClB,CAAC;IAED,kCAAkC;IAClC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAE/B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,iCAAiC;YACjC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC5E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACzB,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,uBAAuB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACvG,CAAC;gBACD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;oBAC3B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;wBACpB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;wBAClB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBACjB,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,cAAc;gBACd,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBACnC,MAAM,IAAI,KAAK,CAAC,SAAS,IAAI,iCAAiC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC7F,CAAC;gBACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBACpB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAClB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,kBAAkB;IAClB,OAAO,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AACvC,CAAC"}
|
package/dist/lib/fixture.d.ts
CHANGED
|
@@ -11,7 +11,8 @@ export declare class FixtureValidationError extends Error {
|
|
|
11
11
|
}
|
|
12
12
|
/**
|
|
13
13
|
* Discovers all eval fixtures in a directory.
|
|
14
|
-
*
|
|
14
|
+
* Recursively searches for directories containing PROMPT.md.
|
|
15
|
+
* Supports nested organization like "vercel-cli/deploy" or "flags/create-flag".
|
|
15
16
|
*/
|
|
16
17
|
export declare function discoverFixtures(evalsDir: string): string[];
|
|
17
18
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fixture.d.ts","sourceRoot":"","sources":["../../src/lib/fixture.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAG9C;;GAEG;AACH,qBAAa,sBAAuB,SAAQ,KAAK;IAEtC,WAAW,EAAE,MAAM;gBAAnB,WAAW,EAAE,MAAM,EAC1B,OAAO,EAAE,MAAM;CAKlB;AAeD
|
|
1
|
+
{"version":3,"file":"fixture.d.ts","sourceRoot":"","sources":["../../src/lib/fixture.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAG9C;;GAEG;AACH,qBAAa,sBAAuB,SAAQ,KAAK;IAEtC,WAAW,EAAE,MAAM;gBAAnB,WAAW,EAAE,MAAM,EAC1B,OAAO,EAAE,MAAM;CAKlB;AAeD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAuC3D;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,EAAE,CAmBlE;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,WAAW,EAAE,MAAM,GAAG;IAAE,QAAQ,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAwB9F;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAmCvE;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG;IACjD,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,MAAM,EAAE,sBAAsB,EAAE,CAAC;CAClC,CAmBA;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,WAAW,EAAE,MAAM,EACnB,eAAe,GAAE,SAAS,MAAM,EAAmB,GAClD,MAAM,EAAE,CA2BV;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAC9B,WAAW,EAAE,MAAM,EACnB,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,GAClC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAUrB"}
|
package/dist/lib/fixture.js
CHANGED
|
@@ -30,26 +30,39 @@ function existsWithExactCase(dirPath, fileName) {
|
|
|
30
30
|
}
|
|
31
31
|
/**
|
|
32
32
|
* Discovers all eval fixtures in a directory.
|
|
33
|
-
*
|
|
33
|
+
* Recursively searches for directories containing PROMPT.md.
|
|
34
|
+
* Supports nested organization like "vercel-cli/deploy" or "flags/create-flag".
|
|
34
35
|
*/
|
|
35
36
|
export function discoverFixtures(evalsDir) {
|
|
36
37
|
const absolutePath = resolve(evalsDir);
|
|
37
38
|
if (!existsSync(absolutePath)) {
|
|
38
39
|
throw new Error(`Evals directory not found: ${absolutePath}`);
|
|
39
40
|
}
|
|
40
|
-
const entries = readdirSync(absolutePath);
|
|
41
41
|
const fixtures = [];
|
|
42
|
-
|
|
43
|
-
const
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
42
|
+
function walk(dir, basePath = '') {
|
|
43
|
+
const entries = readdirSync(dir);
|
|
44
|
+
for (const entry of entries) {
|
|
45
|
+
// Skip hidden directories and files
|
|
46
|
+
if (entry.startsWith('.')) {
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
const entryPath = join(dir, entry);
|
|
50
|
+
// Only consider directories
|
|
51
|
+
if (!statSync(entryPath).isDirectory()) {
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
const relativePath = basePath ? `${basePath}/${entry}` : entry;
|
|
55
|
+
// Check if this directory contains PROMPT.md (potential fixture)
|
|
56
|
+
if (existsWithExactCase(entryPath, 'PROMPT.md')) {
|
|
57
|
+
fixtures.push(relativePath);
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
// Not a fixture directory, recurse into it
|
|
61
|
+
walk(entryPath, relativePath);
|
|
62
|
+
}
|
|
51
63
|
}
|
|
52
64
|
}
|
|
65
|
+
walk(absolutePath);
|
|
53
66
|
return fixtures.sort();
|
|
54
67
|
}
|
|
55
68
|
/**
|
package/dist/lib/fixture.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fixture.js","sourceRoot":"","sources":["../../src/lib/fixture.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AACrE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAErC,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjE;;GAEG;AACH,MAAM,OAAO,sBAAuB,SAAQ,KAAK;IAEtC;IADT,YACS,WAAmB,EAC1B,OAAe;QAEf,KAAK,CAAC,SAAS,WAAW,MAAM,OAAO,EAAE,CAAC,CAAC;QAHpC,gBAAW,GAAX,WAAW,CAAQ;QAI1B,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;IACvC,CAAC;CACF;AAED;;;GAGG;AACH,SAAS,mBAAmB,CAAC,OAAe,EAAE,QAAgB;IAC5D,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QACnC,OAAO,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED
|
|
1
|
+
{"version":3,"file":"fixture.js","sourceRoot":"","sources":["../../src/lib/fixture.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AACrE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAErC,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjE;;GAEG;AACH,MAAM,OAAO,sBAAuB,SAAQ,KAAK;IAEtC;IADT,YACS,WAAmB,EAC1B,OAAe;QAEf,KAAK,CAAC,SAAS,WAAW,MAAM,OAAO,EAAE,CAAC,CAAC;QAHpC,gBAAW,GAAX,WAAW,CAAQ;QAI1B,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;IACvC,CAAC;CACF;AAED;;;GAGG;AACH,SAAS,mBAAmB,CAAC,OAAe,EAAE,QAAgB;IAC5D,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QACnC,OAAO,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAC/C,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEvC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,8BAA8B,YAAY,EAAE,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,SAAS,IAAI,CAAC,GAAW,EAAE,WAAmB,EAAE;QAC9C,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAEjC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,oCAAoC;YACpC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1B,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAEnC,4BAA4B;YAC5B,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvC,SAAS;YACX,CAAC;YAED,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;YAE/D,iEAAiE;YACjE,IAAI,mBAAmB,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,CAAC;gBAChD,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,2CAA2C;gBAC3C,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,YAAY,CAAC,CAAC;IACnB,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAAC,WAAmB;IACtD,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,KAAK,MAAM,IAAI,IAAI,mBAAmB,EAAE,CAAC;QACvC,wEAAwE;QACxE,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,SAAS,GAAG,mBAAmB,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAC9D,MAAM,UAAU,GAAG,mBAAmB,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;YAChE,IAAI,CAAC,SAAS,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC9B,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,mBAAmB,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,CAAC;gBAC5C,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,WAAmB;IACrD,MAAM,eAAe,GAAG,IAAI,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAE1D,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,YAAY,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;QACvD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEhC,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC1B,OAAO;gBACL,QAAQ,EAAE,KAAK;gBACf,KAAK,EAAE,yCAAyC;aACjD,CAAC;QACJ,CAAC;QAED,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,WAAW,EAAE,CAAC;YACjC,OAAO;gBACL,QAAQ,EAAE,KAAK;gBACf,KAAK,EAAE,gCAAgC;aACxC,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,QAAgB,EAAE,IAAY;IACxD,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAE5C,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,sBAAsB,CAAC,IAAI,EAAE,wBAAwB,WAAW,EAAE,CAAC,CAAC;IAChF,CAAC;IAED,0BAA0B;IAC1B,MAAM,YAAY,GAAG,oBAAoB,CAAC,WAAW,CAAC,CAAC;IACvD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,sBAAsB,CAC9B,IAAI,EACJ,2BAA2B,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACrD,CAAC;IACJ,CAAC;IAED,wBAAwB;IACxB,MAAM,aAAa,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;IACvD,IAAI,aAAa,CAAC,KAAK,EAAE,CAAC;QACxB,MAAM,IAAI,sBAAsB,CAAC,IAAI,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED,qCAAqC;IACrC,IAAI,CAAC,mBAAmB,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,sBAAsB,CAAC,IAAI,EAAE,yDAAyD,CAAC,CAAC;IACpG,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAEjD,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,WAAW;QACjB,MAAM;QACN,QAAQ,EAAE,aAAa,CAAC,QAAQ;KACjC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,QAAgB;IAI9C,MAAM,YAAY,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,MAAM,MAAM,GAA6B,EAAE,CAAC;IAE5C,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC5C,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,sBAAsB,EAAE,CAAC;gBAC5C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,WAAmB,EACnB,kBAAqC,cAAc;IAEnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,SAAS,IAAI,CAAC,GAAW,EAAE,WAAmB,EAAE;QAC9C,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAEjC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;YAE/D,8BAA8B;YAC9B,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,KAAK,OAAO,IAAI,KAAK,KAAK,OAAO,CAAC,EAAE,CAAC;gBACrF,SAAS;YACX,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAClC,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAEhC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;YAC/B,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,WAAW,CAAC,CAAC;IAClB,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;AACtB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAC9B,WAAmB,EACnB,eAAmC;IAEnC,MAAM,KAAK,GAAG,eAAe,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QACzC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
package/dist/lib/init.js
CHANGED
|
@@ -118,7 +118,7 @@ Open [http://localhost:3000](http://localhost:3000) to browse results.
|
|
|
118
118
|
* Get the default experiment configuration template (Claude Code).
|
|
119
119
|
*/
|
|
120
120
|
function getCCExperiment() {
|
|
121
|
-
return `import type { ExperimentConfig } from 'agent-eval';
|
|
121
|
+
return `import type { ExperimentConfig } from '@vercel/agent-eval';
|
|
122
122
|
|
|
123
123
|
const config: ExperimentConfig = {
|
|
124
124
|
agent: 'vercel-ai-gateway/claude-code',
|
|
@@ -135,7 +135,7 @@ export default config;
|
|
|
135
135
|
* Get the Codex experiment configuration template.
|
|
136
136
|
*/
|
|
137
137
|
function getCodexExperiment() {
|
|
138
|
-
return `import type { ExperimentConfig } from 'agent-eval';
|
|
138
|
+
return `import type { ExperimentConfig } from '@vercel/agent-eval';
|
|
139
139
|
|
|
140
140
|
const config: ExperimentConfig = {
|
|
141
141
|
agent: 'vercel-ai-gateway/codex',
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vercel/agent-eval",
|
|
3
|
-
"version": "0.0
|
|
3
|
+
"version": "0.1.0",
|
|
4
4
|
"description": "Framework for testing AI coding agents in isolated sandboxes",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -37,6 +37,7 @@
|
|
|
37
37
|
"dotenv": "^16.4.5",
|
|
38
38
|
"glob": "^11.0.0",
|
|
39
39
|
"jiti": "^2.6.1",
|
|
40
|
+
"minimatch": "^10.1.2",
|
|
40
41
|
"tar-stream": "^3.1.7",
|
|
41
42
|
"zod": "^3.23.8"
|
|
42
43
|
},
|