@vercel/agent-eval 0.13.1 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +3 -1
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/agents/claude-code.d.ts.map +1 -1
- package/dist/lib/agents/claude-code.js +12 -9
- package/dist/lib/agents/claude-code.js.map +1 -1
- package/dist/lib/agents/codex.d.ts.map +1 -1
- package/dist/lib/agents/codex.js +12 -9
- package/dist/lib/agents/codex.js.map +1 -1
- package/dist/lib/agents/cursor.d.ts.map +1 -1
- package/dist/lib/agents/cursor.js +3 -1
- package/dist/lib/agents/cursor.js.map +1 -1
- package/dist/lib/agents/gemini.d.ts.map +1 -1
- package/dist/lib/agents/gemini.js +3 -1
- package/dist/lib/agents/gemini.js.map +1 -1
- package/dist/lib/agents/opencode.d.ts.map +1 -1
- package/dist/lib/agents/opencode.js +12 -8
- package/dist/lib/agents/opencode.js.map +1 -1
- package/dist/lib/agents/registry.js +1 -1
- package/dist/lib/agents/registry.js.map +1 -1
- package/dist/lib/agents/shared.d.ts +7 -1
- package/dist/lib/agents/shared.d.ts.map +1 -1
- package/dist/lib/agents/shared.js +34 -11
- package/dist/lib/agents/shared.js.map +1 -1
- package/dist/lib/agents/types.d.ts +3 -1
- package/dist/lib/agents/types.d.ts.map +1 -1
- package/dist/lib/config.d.ts +1 -0
- package/dist/lib/config.d.ts.map +1 -1
- package/dist/lib/config.js +14 -0
- package/dist/lib/config.js.map +1 -1
- package/dist/lib/docker-sandbox.d.ts +7 -1
- package/dist/lib/docker-sandbox.d.ts.map +1 -1
- package/dist/lib/docker-sandbox.js +12 -4
- package/dist/lib/docker-sandbox.js.map +1 -1
- package/dist/lib/fixture.d.ts +7 -4
- package/dist/lib/fixture.d.ts.map +1 -1
- package/dist/lib/fixture.js +9 -5
- package/dist/lib/fixture.js.map +1 -1
- package/dist/lib/runner.d.ts +1 -0
- package/dist/lib/runner.d.ts.map +1 -1
- package/dist/lib/runner.js +26 -1
- package/dist/lib/runner.js.map +1 -1
- package/dist/lib/sandbox.d.ts +7 -1
- package/dist/lib/sandbox.d.ts.map +1 -1
- package/dist/lib/sandbox.js +15 -4
- package/dist/lib/sandbox.js.map +1 -1
- package/dist/lib/types.d.ts +35 -0
- package/dist/lib/types.d.ts.map +1 -1
- package/dist/lib/types.js.map +1 -1
- package/package.json +1 -1
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
import type { ScriptResult } from './types.js';
|
|
5
5
|
import type { SandboxManager } from '../sandbox.js';
|
|
6
6
|
import type { DockerSandboxManager } from '../docker-sandbox.js';
|
|
7
|
+
import type { ValidationMode } from '../types.js';
|
|
7
8
|
/** Union type for sandbox implementations */
|
|
8
9
|
type AnySandbox = SandboxManager | DockerSandboxManager;
|
|
9
10
|
/**
|
|
@@ -22,10 +23,15 @@ export interface ValidationResults {
|
|
|
22
23
|
test?: ScriptResult;
|
|
23
24
|
scripts: Record<string, ScriptResult>;
|
|
24
25
|
}
|
|
26
|
+
export interface NeutralWorkspace {
|
|
27
|
+
cwd: string;
|
|
28
|
+
env: Record<string, string>;
|
|
29
|
+
}
|
|
25
30
|
/**
|
|
26
31
|
* Run validation scripts in the sandbox.
|
|
27
32
|
*/
|
|
28
|
-
export declare function runValidation(sandbox: AnySandbox, scripts: string[]): Promise<ValidationResults>;
|
|
33
|
+
export declare function runValidation(sandbox: AnySandbox, scripts: string[], validation?: ValidationMode): Promise<ValidationResults>;
|
|
34
|
+
export declare function prepareNeutralWorkspace(sandbox: AnySandbox): Promise<NeutralWorkspace>;
|
|
29
35
|
export declare function initGitAndCommit(sandbox: AnySandbox): Promise<void>;
|
|
30
36
|
/**
|
|
31
37
|
* Capture source files generated by the agent.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../../src/lib/agents/shared.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../../src/lib/agents/shared.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAEjE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,6CAA6C;AAC7C,KAAK,UAAU,GAAG,cAAc,GAAG,oBAAoB,CAAC;AAExD;;;;GAIG;AACH,eAAO,MAAM,sBAAsB,mBAAmB,CAAC;AAEvD,mDAAmD;AACnD,eAAO,MAAM,uBAAuB,gCAA2C,CAAC;AAEhF;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC7B;AAgCD;;GAEG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,UAAU,EACnB,OAAO,EAAE,MAAM,EAAE,EACjB,UAAU,GAAE,cAAyB,GACpC,OAAO,CAAC,iBAAiB,CAAC,CAqC5B;AAED,wBAAsB,uBAAuB,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA0B5F;AAED,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAUzE;AAED;;;GAGG;AACH,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC;IAAE,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,YAAY,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAmC7E;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAe3E;AAED;;;;;GAKG;AACH,wBAAsB,uBAAuB,CAC3C,OAAO,EAAE,UAAU,EACnB,aAAa,EAAE,MAAM,GAAG,SAAS,EACjC,SAAS,EAAE,MAAM,EACjB,KAAK,CAAC,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC,CAgBf;AAED;;GAEG;AACH,eAAO,MAAM,UAAU;;;;CAIb,CAAC;AAEX;;GAEG;AACH,eAAO,MAAM,gBAAgB;;CAEnB,CAAC;AAEX;;GAEG;AACH,eAAO,MAAM,aAAa;;;CAGhB,CAAC;AAEX;;GAEG;AACH,eAAO,MAAM,aAAa;;CAEhB,CAAC;AAEX;;GAEG;AACH,eAAO,MAAM,aAAa;;CAEhB,CAAC"}
|
|
@@ -40,21 +40,23 @@ async function detectEvalFile(sandbox) {
|
|
|
40
40
|
/**
|
|
41
41
|
* Run validation scripts in the sandbox.
|
|
42
42
|
*/
|
|
43
|
-
export async function runValidation(sandbox, scripts) {
|
|
43
|
+
export async function runValidation(sandbox, scripts, validation = 'vitest') {
|
|
44
44
|
const results = {
|
|
45
45
|
allPassed: true,
|
|
46
46
|
scripts: {},
|
|
47
47
|
};
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
results.
|
|
48
|
+
if (validation === 'vitest') {
|
|
49
|
+
// Detect which eval file exists (EVAL.ts or EVAL.tsx)
|
|
50
|
+
const evalFile = await detectEvalFile(sandbox);
|
|
51
|
+
// Always run vitest for the eval file (explicitly specify the file)
|
|
52
|
+
const testResult = await sandbox.runCommand('npx', ['vitest', 'run', evalFile]);
|
|
53
|
+
results.test = {
|
|
54
|
+
success: testResult.exitCode === 0,
|
|
55
|
+
output: testResult.stdout + testResult.stderr,
|
|
56
|
+
};
|
|
57
|
+
if (!results.test.success) {
|
|
58
|
+
results.allPassed = false;
|
|
59
|
+
}
|
|
58
60
|
}
|
|
59
61
|
// Run configured scripts
|
|
60
62
|
for (const script of scripts) {
|
|
@@ -70,6 +72,27 @@ export async function runValidation(sandbox, scripts) {
|
|
|
70
72
|
}
|
|
71
73
|
return results;
|
|
72
74
|
}
|
|
75
|
+
export async function prepareNeutralWorkspace(sandbox) {
|
|
76
|
+
const neutralEnv = { USER: 'user', LOGNAME: 'user' };
|
|
77
|
+
const currentWorkingDirectory = sandbox.getWorkingDirectory();
|
|
78
|
+
await sandbox.runShell('git remote remove origin 2>/dev/null || true; rm -rf .git/logs');
|
|
79
|
+
if (!currentWorkingDirectory.includes('/vercel/')) {
|
|
80
|
+
return { cwd: currentWorkingDirectory, env: neutralEnv };
|
|
81
|
+
}
|
|
82
|
+
const neutralWorkspacePath = '/workspace';
|
|
83
|
+
const copyResult = await sandbox.runShell([
|
|
84
|
+
`sudo rm -rf ${neutralWorkspacePath}`,
|
|
85
|
+
`sudo mkdir -p ${neutralWorkspacePath}`,
|
|
86
|
+
`sudo cp -a . ${neutralWorkspacePath}/`,
|
|
87
|
+
`sudo chown -R "$(id -u):$(id -g)" ${neutralWorkspacePath}`,
|
|
88
|
+
].join(' && '));
|
|
89
|
+
if (copyResult.exitCode !== 0) {
|
|
90
|
+
const output = (copyResult.stdout + copyResult.stderr).trim().split('\n').slice(-10).join('\n');
|
|
91
|
+
throw new Error(`Failed to prepare neutral workspace:\n${output}`);
|
|
92
|
+
}
|
|
93
|
+
sandbox.setWorkingDirectory(neutralWorkspacePath);
|
|
94
|
+
return { cwd: neutralWorkspacePath, env: neutralEnv };
|
|
95
|
+
}
|
|
73
96
|
export async function initGitAndCommit(sandbox) {
|
|
74
97
|
await sandbox.writeFiles({
|
|
75
98
|
".gitignore": "node_modules/\n",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../../src/lib/agents/shared.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../../src/lib/agents/shared.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMnD;;;;GAIG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,gBAAgB,CAAC;AAEvD,mDAAmD;AACnD,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,sBAAsB,eAAe,CAAC;AAgBhF;;;;GAIG;AACH,KAAK,UAAU,cAAc,CAAC,OAAmB;IAC/C,IAAI,CAAC;QACH,iEAAiE;QACjE,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACjD,IAAI,QAAQ,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAE/D,sDAAsD;YACtD,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC/B,OAAO,UAAU,CAAC;YACpB,CAAC;YAED,oBAAoB;YACpB,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC9B,OAAO,SAAS,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,gBAAgB;IAClB,CAAC;IAED,2DAA2D;IAC3D,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAmB,EACnB,OAAiB,EACjB,aAA6B,QAAQ;IAErC,MAAM,OAAO,GAAsB;QACjC,SAAS,EAAE,IAAI;QACf,OAAO,EAAE,EAAE;KACZ,CAAC;IAEF,IAAI,UAAU,KAAK,QAAQ,EAAE,CAAC;QAC5B,sDAAsD;QACtD,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;QAE/C,oEAAoE;QACpE,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;QAChF,OAAO,CAAC,IAAI,GAAG;YACb,OAAO,EAAE,UAAU,CAAC,QAAQ,KAAK,CAAC;YAClC,MAAM,EAAE,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM;SAC9C,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAC1B,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QACtE,MAAM,MAAM,GAAiB;YAC3B,OAAO,EAAE,YAAY,CAAC,QAAQ,KAAK,CAAC;YACpC,MAAM,EAAE,YAAY,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM;SAClD,CAAC;QAEF,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;QAEjC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,OAAmB;IAC/D,MAAM,UAAU,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IACrD,MAAM,uBAAuB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAE9D,MAAM,OAAO,CAAC,QAAQ,CAAC,gEAAgE,CAAC,CAAC;IAEzF,IAAI,CAAC,uBAAuB,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAClD,OAAO,EAAE,GAAG,EAAE,uBAAuB,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC;IAC3D,CAAC;IAED,MAAM,oBAAoB,GAAG,YAAY,CAAC;IAC1C,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,QAAQ,CACvC;QACE,eAAe,oBAAoB,EAAE;QACrC,iBAAiB,oBAAoB,EAAE;QACvC,gBAAgB,oBAAoB,GAAG;QACvC,qCAAqC,oBAAoB,EAAE;KAC5D,CAAC,IAAI,CAAC,MAAM,CAAC,CACf,CAAC;IACF,IAAI,UAAU,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,CAAC,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChG,MAAM,IAAI,KAAK,CAAC,yCAAyC,MAAM,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,OAAO,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,CAAC;IAClD,OAAO,EAAE,GAAG,EAAE,oBAAoB,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC;AACxD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAAmB;IACxD,MAAM,OAAO,CAAC,UAAU,CAAC;QACvB,YAAY,EAAE,iBAAiB;KAChC,CAAC,CAAC;IAEH,4HAA4H;IAC5H,sBAAsB;IACtB,MAAM,OAAO,CAAC,QAAQ,CACpB,oIAAoI,CACrI,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,OAAmB;IAEnB,MAAM,cAAc,GAA2B,EAAE,CAAC;IAClD,MAAM,YAAY,GAAa,EAAE,CAAC;IAElC,IAAI,CAAC;QACH,+DAA+D;QAC/D,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,0CAA0C,CAAC,CAAC;QAEtF,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM;aAC5B,IAAI,EAAE;aACN,KAAK,CAAC,IAAI,CAAC;aACX,MAAM,CAAC,OAAO,CAAC,CAAC;QAEnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEjC,IAAI,CAAC,QAAQ;gBAAE,SAAS;YAExB,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;gBACnB,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;oBACjD,cAAc,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC;gBACrC,CAAC;gBAAC,MAAM,CAAC;oBACP,wBAAwB;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,yCAAyC;IAC3C,CAAC;IAED,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAmB;IAC1D,gCAAgC;IAChC,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAE/C,MAAM,OAAO,CAAC,UAAU,CAAC;QACvB,kBAAkB,EAAE;;;;iBAIP,QAAQ;;;;CAIxB;KACE,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,OAAmB,EACnB,aAAiC,EACjC,SAAiB,EACjB,KAAc;IAEd,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,aAAa;YAC9B,CAAC,CAAC,eAAe,CAAC,aAAa,EAAE,SAAS,EAAE,KAAK,CAAC;YAClD,CAAC,CAAC,IAAI,CAAC;QAET,MAAM,OAAO,GAAG;YACd,IAAI,EAAE,UAAU,EAAE,OAAO,IAAI,IAAI;SAClC,CAAC;QAEF,MAAM,OAAO,CAAC,UAAU,CAAC;YACvB,CAAC,uBAAuB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;SAC5D,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,8DAA8D;IAChE,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,OAAO,EAAE,8BAA8B;IACvC,aAAa,EAAE,iCAAiC;IAChD,YAAY,EAAE,oBAAoB;CAC1B,CAAC;AAEX;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,YAAY,EAAE,mBAAmB;CACzB,CAAC;AAEX;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,OAAO,EAAE,2BAA2B;IACpC,YAAY,EAAE,gBAAgB;CACtB,CAAC;AAEX;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,YAAY,EAAE,gBAAgB;CACtB,CAAC;AAEX;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,YAAY,EAAE,gBAAgB;CACtB,CAAC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Agent interface and common types for all agents.
|
|
3
3
|
*/
|
|
4
|
-
import type { ModelTier, SetupFunction, SandboxBackend } from '../types.js';
|
|
4
|
+
import type { ModelTier, SetupFunction, SandboxBackend, ValidationMode } from '../types.js';
|
|
5
5
|
/**
|
|
6
6
|
* Common options for all agents.
|
|
7
7
|
*/
|
|
@@ -18,6 +18,8 @@ export interface AgentRunOptions {
|
|
|
18
18
|
setup?: SetupFunction;
|
|
19
19
|
/** npm scripts to run after agent completes */
|
|
20
20
|
scripts?: string[];
|
|
21
|
+
/** Validation mode after agent completes */
|
|
22
|
+
validation?: ValidationMode;
|
|
21
23
|
/** Abort signal to cancel the run */
|
|
22
24
|
signal?: AbortSignal;
|
|
23
25
|
/** Sandbox backend to use */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/lib/agents/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/lib/agents/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE5F;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,oCAAoC;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,oCAAoC;IACpC,KAAK,EAAE,SAAS,CAAC;IACjB,8BAA8B;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,kDAAkD;IAClD,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,4CAA4C;IAC5C,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B,qCAAqC;IACrC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,6BAA6B;IAC7B,OAAO,CAAC,EAAE,cAAc,GAAG,MAAM,CAAC;IAClC,4EAA4E;IAC5E,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,qCAAqC;IACrC,OAAO,EAAE,OAAO,CAAC;IACjB,gDAAgD;IAChD,MAAM,EAAE,MAAM,CAAC;IACf,sEAAsE;IACtE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,CAAC,EAAE,YAAY,CAAC;IAC1B,sCAAsC;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAC9C,+BAA+B;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mCAAmC;IACnC,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC,iCAAiC;IACjC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,KAAK;IACpB,sCAAsC;IACtC,IAAI,EAAE,MAAM,CAAC;IAEb,kCAAkC;IAClC,WAAW,EAAE,MAAM,CAAC;IAEpB,iCAAiC;IACjC,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAE5E,+DAA+D;IAC/D,eAAe,IAAI,MAAM,CAAC;IAE1B,2CAA2C;IAC3C,eAAe,IAAI,SAAS,CAAC;CAC9B"}
|
package/dist/lib/config.d.ts
CHANGED
package/dist/lib/config.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EACV,gBAAgB,EAChB,wBAAwB,EACxB,UAAU,EACX,MAAM,YAAY,CAAC;AAGpB;;GAEG;AACH,eAAO,MAAM,eAAe;;;;;aAKX,MAAM,EAAE
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EACV,gBAAgB,EAChB,wBAAwB,EACxB,UAAU,EACX,MAAM,YAAY,CAAC;AAGpB;;GAEG;AACH,eAAO,MAAM,eAAe;;;;;aAKX,MAAM,EAAE;;;;;CAKxB,CAAC;AAwCF;;;GAGG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,OAAO,GAAG,gBAAgB,CAWhE;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,gBAAgB,GAAG,wBAAwB,CAwBhF;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,wBAAwB,CAAC,CA8BtF;AAgBD;;;GAGG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,EACtC,cAAc,EAAE,MAAM,EAAE,GACvB,MAAM,EAAE,CA0DV"}
|
package/dist/lib/config.js
CHANGED
|
@@ -13,6 +13,7 @@ export const CONFIG_DEFAULTS = {
|
|
|
13
13
|
runs: 1,
|
|
14
14
|
earlyExit: true,
|
|
15
15
|
scripts: [],
|
|
16
|
+
validation: 'vitest',
|
|
16
17
|
timeout: 600, // 10 minutes
|
|
17
18
|
sandbox: 'auto',
|
|
18
19
|
copyFiles: 'none',
|
|
@@ -37,12 +38,22 @@ const experimentConfigSchema = z.object({
|
|
|
37
38
|
runs: z.number().int().positive().optional(),
|
|
38
39
|
earlyExit: z.boolean().optional(),
|
|
39
40
|
scripts: z.array(z.string()).optional(),
|
|
41
|
+
validation: z.enum(['vitest', 'none']).optional(),
|
|
40
42
|
timeout: z.number().positive().optional(),
|
|
41
43
|
setup: z.function().optional(),
|
|
42
44
|
sandbox: z.enum(['vercel', 'docker', 'auto']).optional(),
|
|
43
45
|
editPrompt: z.function().args(z.string()).returns(z.string()).optional(),
|
|
44
46
|
copyFiles: z.enum(['none', 'changed', 'all']).optional(),
|
|
45
47
|
agentOptions: z.record(z.unknown()).optional(),
|
|
48
|
+
brands: z.array(z.object({
|
|
49
|
+
id: z.string().optional(),
|
|
50
|
+
name: z.string().min(1),
|
|
51
|
+
displayName: z.string().optional(),
|
|
52
|
+
domain: z.string().optional(),
|
|
53
|
+
aliases: z.array(z.string()).optional(),
|
|
54
|
+
isYourBrand: z.boolean().optional(),
|
|
55
|
+
})).optional(),
|
|
56
|
+
onRunComplete: z.function().optional(),
|
|
46
57
|
});
|
|
47
58
|
/**
|
|
48
59
|
* Validates an experiment configuration object.
|
|
@@ -73,12 +84,15 @@ export function resolveConfig(config) {
|
|
|
73
84
|
runs: config.runs ?? CONFIG_DEFAULTS.runs,
|
|
74
85
|
earlyExit: config.earlyExit ?? CONFIG_DEFAULTS.earlyExit,
|
|
75
86
|
scripts: config.scripts ?? CONFIG_DEFAULTS.scripts,
|
|
87
|
+
validation: config.validation ?? CONFIG_DEFAULTS.validation,
|
|
76
88
|
timeout: config.timeout ?? CONFIG_DEFAULTS.timeout,
|
|
77
89
|
setup: config.setup,
|
|
78
90
|
sandbox: config.sandbox ?? CONFIG_DEFAULTS.sandbox,
|
|
79
91
|
editPrompt: config.editPrompt,
|
|
80
92
|
copyFiles: config.copyFiles ?? CONFIG_DEFAULTS.copyFiles,
|
|
81
93
|
agentOptions: config.agentOptions,
|
|
94
|
+
brands: config.brands,
|
|
95
|
+
onRunComplete: config.onRunComplete,
|
|
82
96
|
};
|
|
83
97
|
}
|
|
84
98
|
/**
|
package/dist/lib/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAMtC,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAE7C;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,KAAK,EAAE,MAAe;IACtB,KAAK,EAAE,GAAY;IACnB,IAAI,EAAE,CAAC;IACP,SAAS,EAAE,IAAI;IACf,OAAO,EAAE,EAAc;IACvB,OAAO,EAAE,GAAG,EAAE,aAAa;IAC3B,OAAO,EAAE,MAAe;IACxB,SAAS,EAAE,MAAe;CAC3B,CAAC;AAEF;;GAEG;AACH,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC;QACZ,+BAA+B;QAC/B,aAAa;QACb,yBAAyB;QACzB,OAAO;QACP,4BAA4B;QAC5B,QAAQ;QACR,QAAQ;KACT,CAAC;IACF,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5D,KAAK,EAAE,CAAC;SACL,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;SAC5F,QAAQ,EAAE;IACb,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC5C,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACjC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACvC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACzC,KAAK,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC9B,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;IACxD,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACxE,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE;IACxD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAMtC,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAE7C;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,KAAK,EAAE,MAAe;IACtB,KAAK,EAAE,GAAY;IACnB,IAAI,EAAE,CAAC;IACP,SAAS,EAAE,IAAI;IACf,OAAO,EAAE,EAAc;IACvB,UAAU,EAAE,QAAiB;IAC7B,OAAO,EAAE,GAAG,EAAE,aAAa;IAC3B,OAAO,EAAE,MAAe;IACxB,SAAS,EAAE,MAAe;CAC3B,CAAC;AAEF;;GAEG;AACH,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC;QACZ,+BAA+B;QAC/B,aAAa;QACb,yBAAyB;QACzB,OAAO;QACP,4BAA4B;QAC5B,QAAQ;QACR,QAAQ;KACT,CAAC;IACF,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5D,KAAK,EAAE,CAAC;SACL,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;SAC5F,QAAQ,EAAE;IACb,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC5C,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACjC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACvC,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;IACjD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACzC,KAAK,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC9B,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;IACxD,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACxE,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE;IACxD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC9C,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QACvB,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QACzB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACvB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC7B,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;QACvC,WAAW,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;KACpC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACd,aAAa,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CACvC,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,MAAe;IAC5C,MAAM,MAAM,GAAG,sBAAsB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAExD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM;aAC/B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;aAC/D,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,sCAAsC,MAAM,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,MAAM,CAAC,IAAwB,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,MAAwB;IACpD,wBAAwB;IACxB,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAErC,gDAAgD;IAChD,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;IAE7D,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,KAAK,EAAE,YAAY;QACnB,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,GAAG;QAC1B,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,eAAe,CAAC,IAAI;QACzC,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,eAAe,CAAC,SAAS;QACxD,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,eAAe,CAAC,OAAO;QAClD,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,eAAe,CAAC,UAAU;QAC3D,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,eAAe,CAAC,OAAO;QAClD,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,eAAe,CAAC,OAAO;QAClD,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,eAAe,CAAC,SAAS;QACxD,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,aAAa,EAAE,MAAM,CAAC,aAAa;KACpC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,UAAkB;IACjD,IAAI,CAAC;QACH,IAAI,SAAkB,CAAC;QAEvB,gCAAgC;QAChC,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5C,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;gBACvC,cAAc,EAAE,IAAI;gBACpB,WAAW,EAAE,KAAK;aACnB,CAAC,CAAC;YACH,SAAS,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC5C,CAAC;aAAM,CAAC;YACN,sCAAsC;YACtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC;YACxC,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,CAAC;QAED,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;QACzC,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,8BAA8B,UAAU,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAChF,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;GASG;AACH,SAAS,cAAc,CAAC,IAAY,EAAE,OAAe;IACnD,OAAO,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAClC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAC9B,MAAsC,EACtC,cAAwB;IAExB,8BAA8B;IAC9B,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACnB,OAAO,cAAc,CAAC;QACxB,CAAC;QAED,+BAA+B;QAC/B,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;YAC9E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,uBAAuB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACzG,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,cAAc;QACd,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,SAAS,MAAM,iCAAiC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/F,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,CAAC;IAClB,CAAC;IAED,kCAAkC;IAClC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAE/B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,iCAAiC;YACjC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC5E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACzB,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,uBAAuB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACvG,CAAC;gBACD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;oBAC3B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;wBACpB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;wBAClB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBACjB,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,cAAc;gBACd,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBACnC,MAAM,IAAI,KAAK,CAAC,SAAS,IAAI,iCAAiC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC7F,CAAC;gBACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBACpB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAClB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,kBAAkB;IAClB,OAAO,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AACvC,CAAC"}
|
|
@@ -23,6 +23,7 @@ export declare class DockerSandboxManager implements Sandbox {
|
|
|
23
23
|
private _containerId;
|
|
24
24
|
private timeout;
|
|
25
25
|
private runtime;
|
|
26
|
+
private _workingDirectory;
|
|
26
27
|
constructor(options?: DockerSandboxOptions);
|
|
27
28
|
/**
|
|
28
29
|
* Create and start a new Docker sandbox.
|
|
@@ -49,6 +50,7 @@ export declare class DockerSandboxManager implements Sandbox {
|
|
|
49
50
|
*/
|
|
50
51
|
runCommand(command: string, args?: string[], options?: {
|
|
51
52
|
env?: Record<string, string>;
|
|
53
|
+
cwd?: string;
|
|
52
54
|
}): Promise<CommandResult>;
|
|
53
55
|
/**
|
|
54
56
|
* Run a command in the container as root.
|
|
@@ -62,7 +64,7 @@ export declare class DockerSandboxManager implements Sandbox {
|
|
|
62
64
|
/**
|
|
63
65
|
* Run a shell command (through bash).
|
|
64
66
|
*/
|
|
65
|
-
runShell(command: string, env?: Record<string, string
|
|
67
|
+
runShell(command: string, env?: Record<string, string>, cwd?: string): Promise<CommandResult>;
|
|
66
68
|
/**
|
|
67
69
|
* Read a file from the container.
|
|
68
70
|
*/
|
|
@@ -83,6 +85,10 @@ export declare class DockerSandboxManager implements Sandbox {
|
|
|
83
85
|
* Get the working directory.
|
|
84
86
|
*/
|
|
85
87
|
getWorkingDirectory(): string;
|
|
88
|
+
/**
|
|
89
|
+
* Set the working directory.
|
|
90
|
+
*/
|
|
91
|
+
setWorkingDirectory(path: string): void;
|
|
86
92
|
/**
|
|
87
93
|
* Stop and clean up the container.
|
|
88
94
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"docker-sandbox.d.ts","sourceRoot":"","sources":["../../src/lib/docker-sandbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAmC/D;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,8BAA8B;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,0BAA0B;IAC1B,OAAO,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC;CAC/B;AAED;;;GAGG;AACH,qBAAa,oBAAqB,YAAW,OAAO;IAClD,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAiC;IAClD,OAAO,CAAC,YAAY,CAAc;IAClC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAS;
|
|
1
|
+
{"version":3,"file":"docker-sandbox.d.ts","sourceRoot":"","sources":["../../src/lib/docker-sandbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAmC/D;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,8BAA8B;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,0BAA0B;IAC1B,OAAO,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC;CAC/B;AAED;;;GAGG;AACH,qBAAa,oBAAqB,YAAW,OAAO;IAClD,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAiC;IAClD,OAAO,CAAC,YAAY,CAAc;IAClC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,iBAAiB,CAA6B;gBAE1C,OAAO,GAAE,oBAAyB;IAM9C;;OAEG;WACU,MAAM,CAAC,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAMtF;;OAEG;YACW,UAAU;IA0CxB;;OAEG;YACW,WAAW;IAazB;;OAEG;YACW,SAAS;IAyBvB;;OAEG;IACH,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED;;OAEG;IACG,UAAU,CACd,OAAO,EAAE,MAAM,EACf,IAAI,GAAE,MAAM,EAAO,EACnB,OAAO,GAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAO,GAC3D,OAAO,CAAC,aAAa,CAAC;IAczB;;;OAGG;YACW,gBAAgB;IAW9B;;OAEG;YACW,WAAW;IAiGzB;;OAEG;IACG,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;IAInG;;OAEG;IACG,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAQ7C;;OAEG;IACG,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAKhD;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAS9D;;OAEG;IACG,WAAW,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA8BtD;;OAEG;IACH,mBAAmB,IAAI,MAAM;IAI7B;;OAEG;IACH,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAIvC;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAU5B"}
|
|
@@ -42,6 +42,7 @@ export class DockerSandboxManager {
|
|
|
42
42
|
_containerId = '';
|
|
43
43
|
timeout;
|
|
44
44
|
runtime;
|
|
45
|
+
_workingDirectory = CONTAINER_WORKDIR;
|
|
45
46
|
constructor(options = {}) {
|
|
46
47
|
this.docker = new Docker();
|
|
47
48
|
this.timeout = options.timeout ?? DEFAULT_TIMEOUT;
|
|
@@ -148,6 +149,7 @@ export class DockerSandboxManager {
|
|
|
148
149
|
};
|
|
149
150
|
return this.execCommand(command, args, {
|
|
150
151
|
env,
|
|
152
|
+
cwd: options.cwd,
|
|
151
153
|
user: `${SANDBOX_UID}:${SANDBOX_GID}`,
|
|
152
154
|
});
|
|
153
155
|
}
|
|
@@ -176,7 +178,7 @@ export class DockerSandboxManager {
|
|
|
176
178
|
Cmd: cmd,
|
|
177
179
|
AttachStdout: true,
|
|
178
180
|
AttachStderr: true,
|
|
179
|
-
WorkingDir:
|
|
181
|
+
WorkingDir: options.cwd ?? this._workingDirectory,
|
|
180
182
|
Env: env,
|
|
181
183
|
User: options.user,
|
|
182
184
|
});
|
|
@@ -247,8 +249,8 @@ export class DockerSandboxManager {
|
|
|
247
249
|
/**
|
|
248
250
|
* Run a shell command (through bash).
|
|
249
251
|
*/
|
|
250
|
-
async runShell(command, env) {
|
|
251
|
-
return this.runCommand('bash', ['-c', command], { env });
|
|
252
|
+
async runShell(command, env, cwd) {
|
|
253
|
+
return this.runCommand('bash', ['-c', command], { env, cwd });
|
|
252
254
|
}
|
|
253
255
|
/**
|
|
254
256
|
* Read a file from the container.
|
|
@@ -306,7 +308,13 @@ export class DockerSandboxManager {
|
|
|
306
308
|
* Get the working directory.
|
|
307
309
|
*/
|
|
308
310
|
getWorkingDirectory() {
|
|
309
|
-
return
|
|
311
|
+
return this._workingDirectory;
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Set the working directory.
|
|
315
|
+
*/
|
|
316
|
+
setWorkingDirectory(path) {
|
|
317
|
+
this._workingDirectory = path;
|
|
310
318
|
}
|
|
311
319
|
/**
|
|
312
320
|
* Stop and clean up the container.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"docker-sandbox.js","sourceRoot":"","sources":["../../src/lib/docker-sandbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,MAAM,MAAM,WAAW,CAAC;AAC/B,OAAO,KAAK,GAAG,MAAM,YAAY,CAAC;AAIlC;;;GAGG;AACH,MAAM,aAAa,GAA2B;IAC5C,MAAM,EAAE,cAAc;IACtB,MAAM,EAAE,cAAc;CACvB,CAAC;AAEF;;GAEG;AACH,MAAM,eAAe,GAAG,MAAM,CAAC;AAE/B;;GAEG;AACH,MAAM,iBAAiB,GAAG,yBAAyB,CAAC;AAEpD;;;;;GAKG;AACH,MAAM,WAAW,GAAG,IAAI,CAAC;AACzB,MAAM,WAAW,GAAG,IAAI,CAAC;AAEzB;;GAEG;AACH,MAAM,cAAc,GAAG,wBAAwB,CAAC;AAYhD;;;GAGG;AACH,MAAM,OAAO,oBAAoB;IACvB,MAAM,CAAS;IACf,SAAS,GAA4B,IAAI,CAAC;IAC1C,YAAY,GAAW,EAAE,CAAC;IAC1B,OAAO,CAAS;IAChB,OAAO,CAAS;
|
|
1
|
+
{"version":3,"file":"docker-sandbox.js","sourceRoot":"","sources":["../../src/lib/docker-sandbox.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,MAAM,MAAM,WAAW,CAAC;AAC/B,OAAO,KAAK,GAAG,MAAM,YAAY,CAAC;AAIlC;;;GAGG;AACH,MAAM,aAAa,GAA2B;IAC5C,MAAM,EAAE,cAAc;IACtB,MAAM,EAAE,cAAc;CACvB,CAAC;AAEF;;GAEG;AACH,MAAM,eAAe,GAAG,MAAM,CAAC;AAE/B;;GAEG;AACH,MAAM,iBAAiB,GAAG,yBAAyB,CAAC;AAEpD;;;;;GAKG;AACH,MAAM,WAAW,GAAG,IAAI,CAAC;AACzB,MAAM,WAAW,GAAG,IAAI,CAAC;AAEzB;;GAEG;AACH,MAAM,cAAc,GAAG,wBAAwB,CAAC;AAYhD;;;GAGG;AACH,MAAM,OAAO,oBAAoB;IACvB,MAAM,CAAS;IACf,SAAS,GAA4B,IAAI,CAAC;IAC1C,YAAY,GAAW,EAAE,CAAC;IAC1B,OAAO,CAAS;IAChB,OAAO,CAAS;IAChB,iBAAiB,GAAW,iBAAiB,CAAC;IAEtD,YAAY,UAAgC,EAAE;QAC5C,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,eAAe,CAAC;QAClD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAAgC,EAAE;QACpD,MAAM,OAAO,GAAG,IAAI,oBAAoB,CAAC,OAAO,CAAC,CAAC;QAClD,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC3B,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU;QACtB,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAC1D,CAAC;QAED,gCAAgC;QAChC,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAElC,kFAAkF;QAClF,IAAI,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC;YACjD,KAAK,EAAE,SAAS;YAChB,GAAG,EAAE,CAAC,OAAO,EAAE,UAAU,CAAC,EAAE,yBAAyB;YACrD,UAAU,EAAE,iBAAiB;YAC7B,GAAG,EAAE,IAAI;YACT,UAAU,EAAE;gBACV,UAAU,EAAE,IAAI,EAAE,wBAAwB;aAC3C;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QAEtC,sBAAsB;QACtB,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAE7B,qEAAqE;QACrE,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,mFAAmF,CAAC,CAAC,CAAC;QAEjI,sFAAsF;QACtF,gEAAgE;QAChE,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAChE,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,WAAW,IAAI,WAAW,EAAE,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAEjG,6CAA6C;QAC7C,oDAAoD;QACpD,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;QAC7D,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,WAAW,IAAI,WAAW,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;QAE9F,sCAAsC;QACtC,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW,CAAC,SAAiB;QACzC,IAAI,CAAC;YACH,wBAAwB;YACxB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YAC9C,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,+BAA+B;YAC/B,OAAO,CAAC,GAAG,CAAC,yBAAyB,SAAS,KAAK,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC,uBAAuB,SAAS,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,SAAS,CAAC,SAAiB;QACvC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,GAAiB,EAAE,MAA6B,EAAE,EAAE;gBAC/E,IAAI,GAAG,EAAE,CAAC;oBACR,MAAM,CAAC,GAAG,CAAC,CAAC;oBACZ,OAAO;gBACT,CAAC;gBAED,2BAA2B;gBAC3B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAC9B,MAAM,EACN,CAAC,GAAiB,EAAE,EAAE;oBACpB,IAAI,GAAG,EAAE,CAAC;wBACR,MAAM,CAAC,GAAG,CAAC,CAAC;oBACd,CAAC;yBAAM,CAAC;wBACN,OAAO,EAAE,CAAC;oBACZ,CAAC;gBACH,CAAC;gBACD,4DAA4D;gBAC5D,GAAG,EAAE,GAAE,CAAC,CACT,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,2BAA2B;IACpE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CACd,OAAe,EACf,OAAiB,EAAE,EACnB,UAA0D,EAAE;QAE5D,yCAAyC;QACzC,MAAM,GAAG,GAAG;YACV,GAAG,OAAO,CAAC,GAAG;YACd,IAAI,EAAE,GAAG,cAAc,mEAAmE;SAC3F,CAAC;QAEF,OAAO,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE;YACrC,GAAG;YACH,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,IAAI,EAAE,GAAG,WAAW,IAAI,WAAW,EAAE;SACtC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,gBAAgB,CAC5B,OAAe,EACf,OAAiB,EAAE,EACnB,UAA0D,EAAE;QAE5D,OAAO,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE;YACrC,GAAG,OAAO;YACV,IAAI,EAAE,MAAM;SACb,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW,CACvB,OAAe,EACf,OAAiB,EAAE,EACnB,UAAyE,EAAE;QAE3E,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QAED,MAAM,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;QAC/B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG;YACrB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1D,CAAC,CAAC,SAAS,CAAC;QAEd,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YACrC,GAAG,EAAE,GAAG;YACR,YAAY,EAAE,IAAI;YAClB,YAAY,EAAE,IAAI;YAClB,UAAU,EAAE,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,iBAAiB;YACjD,GAAG,EAAE,GAAG;YACR,IAAI,EAAE,OAAO,CAAC,IAAI;SACnB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QAEhE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,iDAAiD;YACjD,sBAAsB;YACtB,MAAM,YAAY,GAAa,EAAE,CAAC;YAClC,MAAM,YAAY,GAAa,EAAE,CAAC;YAElC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;gBAClC,gDAAgD;gBAChD,0DAA0D;gBAC1D,sCAAsC;gBACtC,IAAI,MAAM,GAAG,CAAC,CAAC;gBACf,OAAO,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;oBAC7B,IAAI,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;wBAC9B,0CAA0C;wBAC1C,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;wBACvC,MAAM;oBACR,CAAC;oBAED,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;oBACjC,MAAM,IAAI,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBAE5C,IAAI,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;wBACrC,2CAA2C;wBAC3C,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;wBAC3C,MAAM;oBACR,CAAC;oBAED,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;oBAC3D,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;wBACrB,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC7B,CAAC;yBAAM,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;wBAC5B,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC7B,CAAC;yBAAM,CAAC;wBACN,8BAA8B;wBAC9B,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC7B,CAAC;oBAED,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC;gBACrB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,IAAI,EAAE;gBAC1B,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACvD,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBAEvD,IAAI,CAAC;oBACH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;oBACxC,OAAO,CAAC;wBACN,MAAM;wBACN,MAAM;wBACN,QAAQ,EAAE,UAAU,CAAC,QAAQ,IAAI,CAAC;qBACnC,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CAAC,KAAK,CAAC,CAAC;gBAChB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAE3B,mBAAmB;YACnB,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;gBAChC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACjB,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;YACjE,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YAEjB,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,OAAe,EAAE,GAA4B,EAAE,GAAY;QACxE,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;IAChE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAY;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACpD,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACnE,CAAC;QACD,OAAO,MAAM,CAAC,MAAM,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,IAAY;QAC3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QAC3D,OAAO,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,KAA6B;QAC5C,MAAM,YAAY,GAAkB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YAClF,IAAI;YACJ,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC;SACvC,CAAC,CAAC,CAAC;QAEJ,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,KAAoB;QACpC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QAED,uBAAuB;QACvB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QAExB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;gBAC9C,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC;gBACpC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;YAEjB,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEhB,sBAAsB;QACtB,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAEnE,mFAAmF;QACnF,kDAAkD;QAClD,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,GAAG,WAAW,IAAI,WAAW,EAAE,EAAE,iBAAiB,CAAC,CAAC,CAAC;IACnG,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,mBAAmB,CAAC,IAAY;QAC9B,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,iBAAiB;YACxD,CAAC;YAAC,MAAM,CAAC;gBACP,8CAA8C;YAChD,CAAC;YACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;CACF"}
|
package/dist/lib/fixture.d.ts
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Eval fixture discovery and validation.
|
|
3
3
|
*/
|
|
4
|
-
import type { EvalFixture } from './types.js';
|
|
4
|
+
import type { EvalFixture, ValidationMode } from './types.js';
|
|
5
|
+
export interface FixtureLoadOptions {
|
|
6
|
+
validation?: ValidationMode;
|
|
7
|
+
}
|
|
5
8
|
/**
|
|
6
9
|
* Error thrown when an eval fixture is invalid.
|
|
7
10
|
*/
|
|
@@ -21,7 +24,7 @@ export declare function discoverFixtures(evalsDir: string): string[];
|
|
|
21
24
|
* Note: Accepts either EVAL.ts or EVAL.tsx for the eval file.
|
|
22
25
|
* Case-sensitive: 'prompt.md' will fail even on Mac/Windows.
|
|
23
26
|
*/
|
|
24
|
-
export declare function validateFixtureFiles(fixturePath: string): string[];
|
|
27
|
+
export declare function validateFixtureFiles(fixturePath: string, options?: FixtureLoadOptions): string[];
|
|
25
28
|
/**
|
|
26
29
|
* Validates the package.json of a fixture.
|
|
27
30
|
* Ensures it has "type": "module".
|
|
@@ -33,12 +36,12 @@ export declare function validatePackageJson(fixturePath: string): {
|
|
|
33
36
|
/**
|
|
34
37
|
* Loads a single eval fixture with full validation.
|
|
35
38
|
*/
|
|
36
|
-
export declare function loadFixture(evalsDir: string, name: string): EvalFixture;
|
|
39
|
+
export declare function loadFixture(evalsDir: string, name: string, options?: FixtureLoadOptions): EvalFixture;
|
|
37
40
|
/**
|
|
38
41
|
* Discovers and loads all valid eval fixtures from a directory.
|
|
39
42
|
* Returns both valid fixtures and any validation errors encountered.
|
|
40
43
|
*/
|
|
41
|
-
export declare function loadAllFixtures(evalsDir: string): {
|
|
44
|
+
export declare function loadAllFixtures(evalsDir: string, options?: FixtureLoadOptions): {
|
|
42
45
|
fixtures: EvalFixture[];
|
|
43
46
|
errors: FixtureValidationError[];
|
|
44
47
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fixture.d.ts","sourceRoot":"","sources":["../../src/lib/fixture.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"fixture.d.ts","sourceRoot":"","sources":["../../src/lib/fixture.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAG9D,MAAM,WAAW,kBAAkB;IACjC,UAAU,CAAC,EAAE,cAAc,CAAC;CAC7B;AAED;;GAEG;AACH,qBAAa,sBAAuB,SAAQ,KAAK;IAEtC,WAAW,EAAE,MAAM;gBAAnB,WAAW,EAAE,MAAM,EAC1B,OAAO,EAAE,MAAM;CAKlB;AAeD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAuC3D;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,kBAAuB,GAC/B,MAAM,EAAE,CAuBV;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,WAAW,EAAE,MAAM,GAAG;IAAE,QAAQ,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAwB9F;AAED;;GAEG;AACH,wBAAgB,WAAW,CACzB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,kBAAuB,GAC/B,WAAW,CAmCb;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,kBAAuB,GAC/B;IACD,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,MAAM,EAAE,sBAAsB,EAAE,CAAC;CAClC,CAmBA;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,WAAW,EAAE,MAAM,EACnB,eAAe,GAAE,SAAS,MAAM,EAAmB,GAClD,MAAM,EAAE,CA2BV;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAC9B,WAAW,EAAE,MAAM,EACnB,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,GAClC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAUrB"}
|
package/dist/lib/fixture.js
CHANGED
|
@@ -71,11 +71,15 @@ export function discoverFixtures(evalsDir) {
|
|
|
71
71
|
* Note: Accepts either EVAL.ts or EVAL.tsx for the eval file.
|
|
72
72
|
* Case-sensitive: 'prompt.md' will fail even on Mac/Windows.
|
|
73
73
|
*/
|
|
74
|
-
export function validateFixtureFiles(fixturePath) {
|
|
74
|
+
export function validateFixtureFiles(fixturePath, options = {}) {
|
|
75
75
|
const missing = [];
|
|
76
|
+
const validation = options.validation ?? 'vitest';
|
|
76
77
|
for (const file of REQUIRED_EVAL_FILES) {
|
|
77
78
|
// Special case: Accept either EVAL.ts or EVAL.tsx (both case-sensitive)
|
|
78
79
|
if (file === 'EVAL.ts') {
|
|
80
|
+
if (validation === 'none') {
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
79
83
|
const hasEvalTs = existsWithExactCase(fixturePath, 'EVAL.ts');
|
|
80
84
|
const hasEvalTsx = existsWithExactCase(fixturePath, 'EVAL.tsx');
|
|
81
85
|
if (!hasEvalTs && !hasEvalTsx) {
|
|
@@ -120,13 +124,13 @@ export function validatePackageJson(fixturePath) {
|
|
|
120
124
|
/**
|
|
121
125
|
* Loads a single eval fixture with full validation.
|
|
122
126
|
*/
|
|
123
|
-
export function loadFixture(evalsDir, name) {
|
|
127
|
+
export function loadFixture(evalsDir, name, options = {}) {
|
|
124
128
|
const fixturePath = resolve(evalsDir, name);
|
|
125
129
|
if (!existsSync(fixturePath)) {
|
|
126
130
|
throw new FixtureValidationError(name, `Directory not found: ${fixturePath}`);
|
|
127
131
|
}
|
|
128
132
|
// Validate required files
|
|
129
|
-
const missingFiles = validateFixtureFiles(fixturePath);
|
|
133
|
+
const missingFiles = validateFixtureFiles(fixturePath, options);
|
|
130
134
|
if (missingFiles.length > 0) {
|
|
131
135
|
throw new FixtureValidationError(name, `Missing required files: ${missingFiles.join(', ')}`);
|
|
132
136
|
}
|
|
@@ -152,13 +156,13 @@ export function loadFixture(evalsDir, name) {
|
|
|
152
156
|
* Discovers and loads all valid eval fixtures from a directory.
|
|
153
157
|
* Returns both valid fixtures and any validation errors encountered.
|
|
154
158
|
*/
|
|
155
|
-
export function loadAllFixtures(evalsDir) {
|
|
159
|
+
export function loadAllFixtures(evalsDir, options = {}) {
|
|
156
160
|
const fixtureNames = discoverFixtures(evalsDir);
|
|
157
161
|
const fixtures = [];
|
|
158
162
|
const errors = [];
|
|
159
163
|
for (const name of fixtureNames) {
|
|
160
164
|
try {
|
|
161
|
-
const fixture = loadFixture(evalsDir, name);
|
|
165
|
+
const fixture = loadFixture(evalsDir, name, options);
|
|
162
166
|
fixtures.push(fixture);
|
|
163
167
|
}
|
|
164
168
|
catch (error) {
|
package/dist/lib/fixture.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fixture.js","sourceRoot":"","sources":["../../src/lib/fixture.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AACrE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAErC,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"fixture.js","sourceRoot":"","sources":["../../src/lib/fixture.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AACrE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAErC,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAMjE;;GAEG;AACH,MAAM,OAAO,sBAAuB,SAAQ,KAAK;IAEtC;IADT,YACS,WAAmB,EAC1B,OAAe;QAEf,KAAK,CAAC,SAAS,WAAW,MAAM,OAAO,EAAE,CAAC,CAAC;QAHpC,gBAAW,GAAX,WAAW,CAAQ;QAI1B,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;IACvC,CAAC;CACF;AAED;;;GAGG;AACH,SAAS,mBAAmB,CAAC,OAAe,EAAE,QAAgB;IAC5D,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QACnC,OAAO,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAC/C,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEvC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,8BAA8B,YAAY,EAAE,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,SAAS,IAAI,CAAC,GAAW,EAAE,WAAmB,EAAE;QAC9C,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAEjC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,oCAAoC;YACpC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1B,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAEnC,4BAA4B;YAC5B,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvC,SAAS;YACX,CAAC;YAED,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;YAE/D,iEAAiE;YACjE,IAAI,mBAAmB,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,CAAC;gBAChD,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,2CAA2C;gBAC3C,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,YAAY,CAAC,CAAC;IACnB,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAClC,WAAmB,EACnB,UAA8B,EAAE;IAEhC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,QAAQ,CAAC;IAElD,KAAK,MAAM,IAAI,IAAI,mBAAmB,EAAE,CAAC;QACvC,wEAAwE;QACxE,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;gBAC1B,SAAS;YACX,CAAC;YACD,MAAM,SAAS,GAAG,mBAAmB,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAC9D,MAAM,UAAU,GAAG,mBAAmB,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;YAChE,IAAI,CAAC,SAAS,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC9B,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,mBAAmB,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,CAAC;gBAC5C,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,WAAmB;IACrD,MAAM,eAAe,GAAG,IAAI,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAE1D,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,YAAY,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;QACvD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEhC,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC1B,OAAO;gBACL,QAAQ,EAAE,KAAK;gBACf,KAAK,EAAE,yCAAyC;aACjD,CAAC;QACJ,CAAC;QAED,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,WAAW,EAAE,CAAC;YACjC,OAAO;gBACL,QAAQ,EAAE,KAAK;gBACf,KAAK,EAAE,gCAAgC;aACxC,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CACzB,QAAgB,EAChB,IAAY,EACZ,UAA8B,EAAE;IAEhC,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAE5C,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,sBAAsB,CAAC,IAAI,EAAE,wBAAwB,WAAW,EAAE,CAAC,CAAC;IAChF,CAAC;IAED,0BAA0B;IAC1B,MAAM,YAAY,GAAG,oBAAoB,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IAChE,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,sBAAsB,CAC9B,IAAI,EACJ,2BAA2B,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACrD,CAAC;IACJ,CAAC;IAED,wBAAwB;IACxB,MAAM,aAAa,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;IACvD,IAAI,aAAa,CAAC,KAAK,EAAE,CAAC;QACxB,MAAM,IAAI,sBAAsB,CAAC,IAAI,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED,qCAAqC;IACrC,IAAI,CAAC,mBAAmB,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,sBAAsB,CAAC,IAAI,EAAE,yDAAyD,CAAC,CAAC;IACpG,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAEjD,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,WAAW;QACjB,MAAM;QACN,QAAQ,EAAE,aAAa,CAAC,QAAQ;KACjC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,QAAgB,EAChB,UAA8B,EAAE;IAKhC,MAAM,YAAY,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,MAAM,MAAM,GAA6B,EAAE,CAAC;IAE5C,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YACrD,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,sBAAsB,EAAE,CAAC;gBAC5C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,WAAmB,EACnB,kBAAqC,cAAc;IAEnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,SAAS,IAAI,CAAC,GAAW,EAAE,WAAmB,EAAE;QAC9C,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAEjC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;YAE/D,8BAA8B;YAC9B,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,KAAK,OAAO,IAAI,KAAK,KAAK,OAAO,CAAC,EAAE,CAAC;gBACrF,SAAS;YACX,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAClC,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAEhC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;YAC/B,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,WAAW,CAAC,CAAC;IAClB,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;AACtB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAC9B,WAAmB,EACnB,eAAmC;IAEnC,MAAM,KAAK,GAAG,eAAe,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QACzC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
package/dist/lib/runner.d.ts
CHANGED
|
@@ -64,6 +64,7 @@ export declare function runSingleEval<T extends ResolvedExperimentConfig['model'
|
|
|
64
64
|
apiKey: string;
|
|
65
65
|
setup?: ResolvedExperimentConfig['setup'];
|
|
66
66
|
scripts?: string[];
|
|
67
|
+
validation?: ResolvedExperimentConfig['validation'];
|
|
67
68
|
sandbox?: ResolvedExperimentConfig['sandbox'];
|
|
68
69
|
editPrompt?: (prompt: string) => string;
|
|
69
70
|
verbose?: boolean;
|
package/dist/lib/runner.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/lib/runner.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,wBAAwB,EACxB,WAAW,EACX,WAAW,EAEX,iBAAiB,EACjB,wBAAwB,EACxB,aAAa,EACd,MAAM,YAAY,CAAC;AASpB;;;;GAIG;AACH,qBAAa,gBAAgB;IAMzB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAN3B,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,KAAK,CAA+C;gBAGzC,eAAe,EAAE,MAAM,EACvB,QAAQ,EAAE,MAAM;IAGnC;;;OAGG;IACG,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAWlC,OAAO,CAAC,WAAW;CAgBpB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,wCAAwC;IACxC,MAAM,EAAE,wBAAwB,CAAC;IACjC,sBAAsB;IACtB,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gCAAgC;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,sBAAsB;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,iEAAiE;IACjE,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;IAC5C,qCAAqC;IACrC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,uCAAuC;IACvC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,iFAAiF;IACjF,WAAW,CAAC,EAAE,gBAAgB,CAAC;CAChC;AAoBD;;;GAGG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,iBAAiB,CAAC,
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/lib/runner.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,wBAAwB,EACxB,WAAW,EACX,WAAW,EAEX,iBAAiB,EACjB,wBAAwB,EACxB,aAAa,EACd,MAAM,YAAY,CAAC;AASpB;;;;GAIG;AACH,qBAAa,gBAAgB;IAMzB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAN3B,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,KAAK,CAA+C;gBAGzC,eAAe,EAAE,MAAM,EACvB,QAAQ,EAAE,MAAM;IAGnC;;;OAGG;IACG,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAWlC,OAAO,CAAC,WAAW;CAgBpB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,wCAAwC;IACxC,MAAM,EAAE,wBAAwB,CAAC;IACjC,sBAAsB;IACtB,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,gCAAgC;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,sBAAsB;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,iEAAiE;IACjE,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;IAC5C,qCAAqC;IACrC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,uCAAuC;IACvC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,iFAAiF;IACjF,WAAW,CAAC,EAAE,gBAAgB,CAAC;CAChC;AAoBD;;;GAGG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,iBAAiB,CAAC,CAoP5B;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,CAAC,SAAS,wBAAwB,CAAC,OAAO,CAAC,EAC7E,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE;IACP,KAAK,CAAC,EAAE,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAC1C,KAAK,EAAE,CAAC,CAAC;IACT,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAC1C,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,wBAAwB,CAAC,YAAY,CAAC,CAAC;IACpD,OAAO,CAAC,EAAE,wBAAwB,CAAC,SAAS,CAAC,CAAC;IAC9C,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;IACxC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,YAAY,CAAC,EAAE,wBAAwB,CAAC,cAAc,CAAC,CAAC;CACzD,GACA,OAAO,CAAC,CAAC,SAAS,KAAK,CAAC,OAAO,CAAC,GAAG,WAAW,EAAE,GAAG,WAAW,CAAC,CA+BjE"}
|
package/dist/lib/runner.js
CHANGED
|
@@ -112,6 +112,7 @@ export async function runExperiment(options) {
|
|
|
112
112
|
apiKey,
|
|
113
113
|
setup: config.setup,
|
|
114
114
|
scripts: config.scripts,
|
|
115
|
+
validation: config.validation,
|
|
115
116
|
signal: attemptController.signal,
|
|
116
117
|
sandbox: config.sandbox,
|
|
117
118
|
agentOptions: config.agentOptions,
|
|
@@ -148,7 +149,30 @@ export async function runExperiment(options) {
|
|
|
148
149
|
aborted: true,
|
|
149
150
|
};
|
|
150
151
|
}
|
|
151
|
-
|
|
152
|
+
let runData = agentResultToEvalRunData(agentResult);
|
|
153
|
+
if (config.onRunComplete) {
|
|
154
|
+
try {
|
|
155
|
+
const updatedRunData = await config.onRunComplete({
|
|
156
|
+
fixture,
|
|
157
|
+
runIndex,
|
|
158
|
+
config,
|
|
159
|
+
runData,
|
|
160
|
+
});
|
|
161
|
+
if (updatedRunData) {
|
|
162
|
+
runData = updatedRunData;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
catch (error) {
|
|
166
|
+
runData = {
|
|
167
|
+
...runData,
|
|
168
|
+
result: {
|
|
169
|
+
...runData.result,
|
|
170
|
+
status: 'failed',
|
|
171
|
+
error: `onRunComplete failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
172
|
+
},
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
}
|
|
152
176
|
return {
|
|
153
177
|
fixtureName: fixture.name,
|
|
154
178
|
runIndex,
|
|
@@ -255,6 +279,7 @@ export async function runSingleEval(fixture, options) {
|
|
|
255
279
|
apiKey: options.apiKey,
|
|
256
280
|
setup: options.setup,
|
|
257
281
|
scripts: options.scripts,
|
|
282
|
+
validation: options.validation,
|
|
258
283
|
sandbox: options.sandbox,
|
|
259
284
|
agentOptions: options.agentOptions,
|
|
260
285
|
});
|