@vercel/agent-eval 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/README.md +370 -0
  2. package/dist/cli.d.ts +6 -0
  3. package/dist/cli.d.ts.map +1 -0
  4. package/dist/cli.js +166 -0
  5. package/dist/cli.js.map +1 -0
  6. package/dist/index.d.ts +21 -0
  7. package/dist/index.d.ts.map +1 -0
  8. package/dist/index.js +17 -0
  9. package/dist/index.js.map +1 -0
  10. package/dist/lib/agents/claude-code.d.ts +12 -0
  11. package/dist/lib/agents/claude-code.d.ts.map +1 -0
  12. package/dist/lib/agents/claude-code.js +203 -0
  13. package/dist/lib/agents/claude-code.js.map +1 -0
  14. package/dist/lib/agents/codex.d.ts +12 -0
  15. package/dist/lib/agents/codex.d.ts.map +1 -0
  16. package/dist/lib/agents/codex.js +247 -0
  17. package/dist/lib/agents/codex.js.map +1 -0
  18. package/dist/lib/agents/index.d.ts +7 -0
  19. package/dist/lib/agents/index.d.ts.map +1 -0
  20. package/dist/lib/agents/index.js +14 -0
  21. package/dist/lib/agents/index.js.map +1 -0
  22. package/dist/lib/agents/registry.d.ts +23 -0
  23. package/dist/lib/agents/registry.d.ts.map +1 -0
  24. package/dist/lib/agents/registry.js +35 -0
  25. package/dist/lib/agents/registry.js.map +1 -0
  26. package/dist/lib/agents/shared.d.ts +47 -0
  27. package/dist/lib/agents/shared.d.ts.map +1 -0
  28. package/dist/lib/agents/shared.js +99 -0
  29. package/dist/lib/agents/shared.js.map +1 -0
  30. package/dist/lib/agents/types.d.ts +69 -0
  31. package/dist/lib/agents/types.d.ts.map +1 -0
  32. package/dist/lib/agents/types.js +5 -0
  33. package/dist/lib/agents/types.js.map +1 -0
  34. package/dist/lib/config.d.ts +34 -0
  35. package/dist/lib/config.d.ts.map +1 -0
  36. package/dist/lib/config.js +117 -0
  37. package/dist/lib/config.js.map +1 -0
  38. package/dist/lib/fixture.d.ts +52 -0
  39. package/dist/lib/fixture.d.ts.map +1 -0
  40. package/dist/lib/fixture.js +175 -0
  41. package/dist/lib/fixture.js.map +1 -0
  42. package/dist/lib/init.d.ts +21 -0
  43. package/dist/lib/init.d.ts.map +1 -0
  44. package/dist/lib/init.js +250 -0
  45. package/dist/lib/init.js.map +1 -0
  46. package/dist/lib/results.d.ts +54 -0
  47. package/dist/lib/results.d.ts.map +1 -0
  48. package/dist/lib/results.js +186 -0
  49. package/dist/lib/results.js.map +1 -0
  50. package/dist/lib/runner.d.ts +43 -0
  51. package/dist/lib/runner.d.ts.map +1 -0
  52. package/dist/lib/runner.js +142 -0
  53. package/dist/lib/runner.js.map +1 -0
  54. package/dist/lib/sandbox.d.ts +117 -0
  55. package/dist/lib/sandbox.d.ts.map +1 -0
  56. package/dist/lib/sandbox.js +248 -0
  57. package/dist/lib/sandbox.js.map +1 -0
  58. package/dist/lib/types.d.ts +166 -0
  59. package/dist/lib/types.d.ts.map +1 -0
  60. package/dist/lib/types.js +14 -0
  61. package/dist/lib/types.js.map +1 -0
  62. package/dist/test-setup.d.ts +2 -0
  63. package/dist/test-setup.d.ts.map +1 -0
  64. package/dist/test-setup.js +6 -0
  65. package/dist/test-setup.js.map +1 -0
  66. package/package.json +58 -0
@@ -0,0 +1,99 @@
1
+ /**
2
+ * Shared utilities for agent implementations.
3
+ */
4
+ /**
5
+ * Run validation scripts in the sandbox.
6
+ */
7
+ export async function runValidation(sandbox, scripts) {
8
+ const results = {
9
+ allPassed: true,
10
+ scripts: {},
11
+ };
12
+ // Always run vitest for EVAL.ts (explicitly specify the file)
13
+ const testResult = await sandbox.runCommand('npx', ['vitest', 'run', 'EVAL.ts']);
14
+ results.test = {
15
+ success: testResult.exitCode === 0,
16
+ output: testResult.stdout + testResult.stderr,
17
+ };
18
+ if (!results.test.success) {
19
+ results.allPassed = false;
20
+ }
21
+ // Run configured scripts
22
+ for (const script of scripts) {
23
+ const scriptResult = await sandbox.runCommand('npm', ['run', script]);
24
+ const result = {
25
+ success: scriptResult.exitCode === 0,
26
+ output: scriptResult.stdout + scriptResult.stderr,
27
+ };
28
+ results.scripts[script] = result;
29
+ if (!result.success) {
30
+ results.allPassed = false;
31
+ }
32
+ }
33
+ return results;
34
+ }
35
+ /**
36
+ * Capture source files generated by the agent.
37
+ */
38
+ export async function captureGeneratedFiles(sandbox) {
39
+ const files = {};
40
+ try {
41
+ // Find all source files
42
+ const findResult = await sandbox.runShell("find . -path './node_modules' -prune -o \\( -name '*.ts' -o -name '*.tsx' -o -name '*.js' -o -name '*.jsx' \\) -type f -print");
43
+ const filePaths = findResult.stdout
44
+ .trim()
45
+ .split('\n')
46
+ .filter(Boolean);
47
+ for (const filePath of filePaths) {
48
+ try {
49
+ const content = await sandbox.readFile(filePath);
50
+ files[filePath] = content;
51
+ }
52
+ catch {
53
+ // Skip unreadable files
54
+ }
55
+ }
56
+ }
57
+ catch {
58
+ // If capture fails, return empty object
59
+ }
60
+ return files;
61
+ }
62
+ /**
63
+ * Create vitest config for running EVAL.ts.
64
+ */
65
+ export async function createVitestConfig(sandbox) {
66
+ await sandbox.writeFiles({
67
+ 'vitest.config.ts': `
68
+ import { defineConfig } from 'vitest/config';
69
+ export default defineConfig({
70
+ test: {
71
+ include: ['EVAL.ts'],
72
+ globals: false,
73
+ },
74
+ });
75
+ `,
76
+ });
77
+ }
78
+ /**
79
+ * AI Gateway configuration.
80
+ */
81
+ export const AI_GATEWAY = {
82
+ baseUrl: 'https://ai-gateway.vercel.sh',
83
+ openAiBaseUrl: 'https://ai-gateway.vercel.sh/v1',
84
+ apiKeyEnvVar: 'AI_GATEWAY_API_KEY',
85
+ };
86
+ /**
87
+ * Direct API configuration for Anthropic.
88
+ */
89
+ export const ANTHROPIC_DIRECT = {
90
+ apiKeyEnvVar: 'ANTHROPIC_API_KEY',
91
+ };
92
+ /**
93
+ * Direct API configuration for OpenAI.
94
+ */
95
+ export const OPENAI_DIRECT = {
96
+ baseUrl: 'https://api.openai.com/v1',
97
+ apiKeyEnvVar: 'OPENAI_API_KEY',
98
+ };
99
+ //# sourceMappingURL=shared.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"shared.js","sourceRoot":"","sources":["../../../src/lib/agents/shared.ts"],"names":[],"mappings":"AAAA;;GAEG;AAcH;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAuB,EACvB,OAAiB;IAEjB,MAAM,OAAO,GAAsB;QACjC,SAAS,EAAE,IAAI;QACf,OAAO,EAAE,EAAE;KACZ,CAAC;IAEF,8DAA8D;IAC9D,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;IACjF,OAAO,CAAC,IAAI,GAAG;QACb,OAAO,EAAE,UAAU,CAAC,QAAQ,KAAK,CAAC;QAClC,MAAM,EAAE,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM;KAC9C,CAAC;IACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QAC1B,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC;IAC5B,CAAC;IAED,yBAAyB;IACzB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QACtE,MAAM,MAAM,GAAiB;YAC3B,OAAO,EAAE,YAAY,CAAC,QAAQ,KAAK,CAAC;YACpC,MAAM,EAAE,YAAY,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM;SAClD,CAAC;QAEF,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;QAEjC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,OAAuB;IACjE,MAAM,KAAK,GAA2B,EAAE,CAAC;IAEzC,IAAI,CAAC;QACH,wBAAwB;QACxB,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,QAAQ,CACvC,+HAA+H,CAChI,CAAC;QAEF,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM;aAChC,IAAI,EAAE;aACN,KAAK,CAAC,IAAI,CAAC;aACX,MAAM,CAAC,OAAO,CAAC,CAAC;QAEnB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBACjD,KAAK,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC;YAC5B,CAAC;YAAC,MAAM,CAAC;gBACP,wBAAwB;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,wCAAwC;IAC1C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAuB;IAC9D,MAAM,OAAO,CAAC,UAAU,CAAC;QACvB,kBAAkB,EAAE;;;;;;;;CAQvB;KACE,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,OAAO,EAAE,8BAA8B;IACvC,aAAa,EAAE,iCAAiC;IAChD,YAAY,EAAE,oBAAoB;CAC1B,CAAC;AAEX;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,YAAY,EAAE,mBAAmB;CACzB,CAAC;AAEX;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,OAAO,EAAE,2BAA2B;IACpC,YAAY,EAAE,gBAAgB;CACtB,CAAC"}
@@ -0,0 +1,69 @@
1
+ /**
2
+ * Agent interface and common types for all agents.
3
+ */
4
+ import type { ModelTier, SetupFunction } from '../types.js';
5
+ /**
6
+ * Common options for all agents.
7
+ */
8
+ export interface AgentRunOptions {
9
+ /** The prompt/task for the agent */
10
+ prompt: string;
11
+ /** Model to use (agent-specific) */
12
+ model: ModelTier;
13
+ /** Timeout in milliseconds */
14
+ timeout: number;
15
+ /** API key for the agent */
16
+ apiKey: string;
17
+ /** Optional setup function to run before agent */
18
+ setup?: SetupFunction;
19
+ /** npm scripts to run after agent completes */
20
+ scripts?: string[];
21
+ /** Abort signal to cancel the run */
22
+ signal?: AbortSignal;
23
+ }
24
+ /**
25
+ * Result from running a script.
26
+ */
27
+ export interface ScriptResult {
28
+ success: boolean;
29
+ output: string;
30
+ }
31
+ /**
32
+ * Result of an agent run.
33
+ */
34
+ export interface AgentRunResult {
35
+ /** Whether the run was successful */
36
+ success: boolean;
37
+ /** Raw output from the agent (stdout/stderr) */
38
+ output: string;
39
+ /** Structured transcript (JSONL format for Claude, JSON for Codex) */
40
+ transcript?: string;
41
+ /** Error message if failed */
42
+ error?: string;
43
+ /** Duration in milliseconds */
44
+ duration: number;
45
+ /** Test result (EVAL.ts always runs) */
46
+ testResult?: ScriptResult;
47
+ /** Results from configured scripts */
48
+ scriptsResults?: Record<string, ScriptResult>;
49
+ /** Sandbox ID for debugging */
50
+ sandboxId?: string;
51
+ /** Files generated by the agent */
52
+ generatedFiles?: Record<string, string>;
53
+ }
54
+ /**
55
+ * Agent interface that all agents must implement.
56
+ */
57
+ export interface Agent {
58
+ /** Unique identifier for the agent */
59
+ name: string;
60
+ /** Human-readable display name */
61
+ displayName: string;
62
+ /** Run the agent on a fixture */
63
+ run(fixturePath: string, options: AgentRunOptions): Promise<AgentRunResult>;
64
+ /** Get agent-specific environment variable name for API key */
65
+ getApiKeyEnvVar(): string;
66
+ /** Get the default model for this agent */
67
+ getDefaultModel(): ModelTier;
68
+ }
69
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/lib/agents/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5D;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,oCAAoC;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,oCAAoC;IACpC,KAAK,EAAE,SAAS,CAAC;IACjB,8BAA8B;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,kDAAkD;IAClD,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,qCAAqC;IACrC,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,qCAAqC;IACrC,OAAO,EAAE,OAAO,CAAC;IACjB,gDAAgD;IAChD,MAAM,EAAE,MAAM,CAAC;IACf,sEAAsE;IACtE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,CAAC,EAAE,YAAY,CAAC;IAC1B,sCAAsC;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAC9C,+BAA+B;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mCAAmC;IACnC,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,KAAK;IACpB,sCAAsC;IACtC,IAAI,EAAE,MAAM,CAAC;IAEb,kCAAkC;IAClC,WAAW,EAAE,MAAM,CAAC;IAEpB,iCAAiC;IACjC,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAE5E,+DAA+D;IAC/D,eAAe,IAAI,MAAM,CAAC;IAE1B,2CAA2C;IAC3C,eAAe,IAAI,SAAS,CAAC;CAC9B"}
@@ -0,0 +1,5 @@
1
+ /**
2
+ * Agent interface and common types for all agents.
3
+ */
4
+ export {};
5
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/lib/agents/types.ts"],"names":[],"mappings":"AAAA;;GAEG"}
@@ -0,0 +1,34 @@
1
+ /**
2
+ * Experiment configuration validation and defaults.
3
+ */
4
+ import type { ExperimentConfig, ResolvedExperimentConfig, EvalFilter } from './types.js';
5
+ /**
6
+ * Default configuration values.
7
+ */
8
+ export declare const CONFIG_DEFAULTS: {
9
+ model: "opus";
10
+ evals: "*";
11
+ runs: number;
12
+ earlyExit: boolean;
13
+ scripts: string[];
14
+ timeout: number;
15
+ };
16
+ /**
17
+ * Validates an experiment configuration object.
18
+ * Throws a descriptive error if validation fails.
19
+ */
20
+ export declare function validateConfig(config: unknown): ExperimentConfig;
21
+ /**
22
+ * Resolves an experiment configuration by applying defaults.
23
+ */
24
+ export declare function resolveConfig(config: ExperimentConfig): ResolvedExperimentConfig;
25
+ /**
26
+ * Loads an experiment configuration from a file path.
27
+ * Supports TypeScript and JavaScript files with default exports.
28
+ */
29
+ export declare function loadConfig(configPath: string): Promise<ResolvedExperimentConfig>;
30
+ /**
31
+ * Resolves the evals filter to a list of eval names.
32
+ */
33
+ export declare function resolveEvalNames(filter: string | string[] | EvalFilter, availableEvals: string[]): string[];
34
+ //# sourceMappingURL=config.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EACV,gBAAgB,EAChB,wBAAwB,EACxB,UAAU,EACX,MAAM,YAAY,CAAC;AAGpB;;GAEG;AACH,eAAO,MAAM,eAAe;;;;;aAKX,MAAM,EAAE;;CAExB,CAAC;AAuBF;;;GAGG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,OAAO,GAAG,gBAAgB,CAWhE;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,gBAAgB,GAAG,wBAAwB,CAiBhF;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAkBtF;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,EACtC,cAAc,EAAE,MAAM,EAAE,GACvB,MAAM,EAAE,CAyBV"}
@@ -0,0 +1,117 @@
1
+ /**
2
+ * Experiment configuration validation and defaults.
3
+ */
4
+ import { z } from 'zod';
5
+ import { getAgent } from './agents/index.js';
6
+ /**
7
+ * Default configuration values.
8
+ */
9
+ export const CONFIG_DEFAULTS = {
10
+ model: 'opus',
11
+ evals: '*',
12
+ runs: 1,
13
+ earlyExit: true,
14
+ scripts: [],
15
+ timeout: 300, // 5 minutes
16
+ };
17
+ /**
18
+ * Zod schema for validating experiment configuration.
19
+ */
20
+ const experimentConfigSchema = z.object({
21
+ agent: z.enum([
22
+ 'vercel-ai-gateway/claude-code',
23
+ 'claude-code',
24
+ 'vercel-ai-gateway/codex',
25
+ 'codex',
26
+ ]),
27
+ model: z.string().optional(),
28
+ evals: z
29
+ .union([z.string(), z.array(z.string()), z.function().args(z.string()).returns(z.boolean())])
30
+ .optional(),
31
+ runs: z.number().int().positive().optional(),
32
+ earlyExit: z.boolean().optional(),
33
+ scripts: z.array(z.string()).optional(),
34
+ timeout: z.number().positive().optional(),
35
+ setup: z.function().optional(),
36
+ });
37
+ /**
38
+ * Validates an experiment configuration object.
39
+ * Throws a descriptive error if validation fails.
40
+ */
41
+ export function validateConfig(config) {
42
+ const result = experimentConfigSchema.safeParse(config);
43
+ if (!result.success) {
44
+ const errors = result.error.issues
45
+ .map((issue) => ` - ${issue.path.join('.')}: ${issue.message}`)
46
+ .join('\n');
47
+ throw new Error(`Invalid experiment configuration:\n${errors}`);
48
+ }
49
+ return result.data;
50
+ }
51
+ /**
52
+ * Resolves an experiment configuration by applying defaults.
53
+ */
54
+ export function resolveConfig(config) {
55
+ // Validate agent exists
56
+ const agent = getAgent(config.agent);
57
+ // Get the default model based on the agent type
58
+ const defaultModel = config.model ?? agent.getDefaultModel();
59
+ return {
60
+ agent: config.agent,
61
+ model: defaultModel,
62
+ evals: config.evals ?? '*',
63
+ runs: config.runs ?? CONFIG_DEFAULTS.runs,
64
+ earlyExit: config.earlyExit ?? CONFIG_DEFAULTS.earlyExit,
65
+ scripts: config.scripts ?? CONFIG_DEFAULTS.scripts,
66
+ timeout: config.timeout ?? CONFIG_DEFAULTS.timeout,
67
+ setup: config.setup,
68
+ };
69
+ }
70
+ /**
71
+ * Loads an experiment configuration from a file path.
72
+ * Supports TypeScript and JavaScript files with default exports.
73
+ */
74
+ export async function loadConfig(configPath) {
75
+ try {
76
+ // Dynamic import of the config file
77
+ const module = await import(configPath);
78
+ const rawConfig = module.default;
79
+ if (!rawConfig) {
80
+ throw new Error(`Config file must have a default export`);
81
+ }
82
+ const config = validateConfig(rawConfig);
83
+ return resolveConfig(config);
84
+ }
85
+ catch (error) {
86
+ if (error instanceof Error) {
87
+ throw new Error(`Failed to load config from ${configPath}: ${error.message}`);
88
+ }
89
+ throw error;
90
+ }
91
+ }
92
+ /**
93
+ * Resolves the evals filter to a list of eval names.
94
+ */
95
+ export function resolveEvalNames(filter, availableEvals) {
96
+ // Single eval name
97
+ if (typeof filter === 'string') {
98
+ if (filter === '*') {
99
+ return availableEvals;
100
+ }
101
+ if (!availableEvals.includes(filter)) {
102
+ throw new Error(`Eval "${filter}" not found. Available evals: ${availableEvals.join(', ')}`);
103
+ }
104
+ return [filter];
105
+ }
106
+ // Array of eval names
107
+ if (Array.isArray(filter)) {
108
+ const missing = filter.filter((name) => !availableEvals.includes(name));
109
+ if (missing.length > 0) {
110
+ throw new Error(`Evals not found: ${missing.join(', ')}. Available evals: ${availableEvals.join(', ')}`);
111
+ }
112
+ return filter;
113
+ }
114
+ // Filter function
115
+ return availableEvals.filter(filter);
116
+ }
117
+ //# sourceMappingURL=config.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/lib/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAMxB,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAE7C;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,KAAK,EAAE,MAAe;IACtB,KAAK,EAAE,GAAY;IACnB,IAAI,EAAE,CAAC;IACP,SAAS,EAAE,IAAI;IACf,OAAO,EAAE,EAAc;IACvB,OAAO,EAAE,GAAG,EAAE,YAAY;CAC3B,CAAC;AAEF;;GAEG;AACH,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC;QACZ,+BAA+B;QAC/B,aAAa;QACb,yBAAyB;QACzB,OAAO;KACR,CAAC;IACF,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,KAAK,EAAE,CAAC;SACL,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;SAC5F,QAAQ,EAAE;IACb,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC5C,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACjC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACvC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACzC,KAAK,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAC/B,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,MAAe;IAC5C,MAAM,MAAM,GAAG,sBAAsB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAExD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM;aAC/B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;aAC/D,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,sCAAsC,MAAM,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,MAAM,CAAC,IAAwB,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,MAAwB;IACpD,wBAAwB;IACxB,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAErC,gDAAgD;IAChD,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;IAE7D,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,KAAK,EAAE,YAAY;QACnB,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,GAAG;QAC1B,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,eAAe,CAAC,IAAI;QACzC,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,eAAe,CAAC,SAAS;QACxD,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,eAAe,CAAC,OAAO;QAClD,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,eAAe,CAAC,OAAO;QAClD,KAAK,EAAE,MAAM,CAAC,KAAK;KACpB,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,UAAkB;IACjD,IAAI,CAAC;QACH,oCAAoC;QACpC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC;QAEjC,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,CAAC;QAED,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;QACzC,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,8BAA8B,UAAU,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAChF,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAC9B,MAAsC,EACtC,cAAwB;IAExB,mBAAmB;IACnB,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACnB,OAAO,cAAc,CAAC;QACxB,CAAC;QACD,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,SAAS,MAAM,iCAAiC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/F,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,CAAC;IAClB,CAAC;IAED,sBAAsB;IACtB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACxE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CACb,oBAAoB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACxF,CAAC;QACJ,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,kBAAkB;IAClB,OAAO,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AACvC,CAAC"}
@@ -0,0 +1,52 @@
1
+ /**
2
+ * Eval fixture discovery and validation.
3
+ */
4
+ import type { EvalFixture } from './types.js';
5
+ /**
6
+ * Error thrown when an eval fixture is invalid.
7
+ */
8
+ export declare class FixtureValidationError extends Error {
9
+ fixtureName: string;
10
+ constructor(fixtureName: string, message: string);
11
+ }
12
+ /**
13
+ * Discovers all eval fixtures in a directory.
14
+ * Each subdirectory is considered a potential eval fixture.
15
+ */
16
+ export declare function discoverFixtures(evalsDir: string): string[];
17
+ /**
18
+ * Validates that a fixture has all required files.
19
+ * Returns an array of missing file names, or empty array if valid.
20
+ */
21
+ export declare function validateFixtureFiles(fixturePath: string): string[];
22
+ /**
23
+ * Validates the package.json of a fixture.
24
+ * Ensures it has "type": "module".
25
+ */
26
+ export declare function validatePackageJson(fixturePath: string): {
27
+ isModule: boolean;
28
+ error?: string;
29
+ };
30
+ /**
31
+ * Loads a single eval fixture with full validation.
32
+ */
33
+ export declare function loadFixture(evalsDir: string, name: string): EvalFixture;
34
+ /**
35
+ * Discovers and loads all valid eval fixtures from a directory.
36
+ * Returns both valid fixtures and any validation errors encountered.
37
+ */
38
+ export declare function loadAllFixtures(evalsDir: string): {
39
+ fixtures: EvalFixture[];
40
+ errors: FixtureValidationError[];
41
+ };
42
+ /**
43
+ * Gets a list of all files in a fixture directory.
44
+ * Excludes PROMPT.md, EVAL.ts, node_modules, and .git.
45
+ */
46
+ export declare function getFixtureFiles(fixturePath: string, excludePatterns?: readonly string[]): string[];
47
+ /**
48
+ * Reads all fixture files into a map.
49
+ * Keys are relative paths, values are file contents.
50
+ */
51
+ export declare function readFixtureFiles(fixturePath: string, excludePatterns?: readonly string[]): Map<string, string>;
52
+ //# sourceMappingURL=fixture.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fixture.d.ts","sourceRoot":"","sources":["../../src/lib/fixture.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAG9C;;GAEG;AACH,qBAAa,sBAAuB,SAAQ,KAAK;IAEtC,WAAW,EAAE,MAAM;gBAAnB,WAAW,EAAE,MAAM,EAC1B,OAAO,EAAE,MAAM;CAKlB;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAyB3D;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,EAAE,CAWlE;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,WAAW,EAAE,MAAM,GAAG;IAAE,QAAQ,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAwB9F;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAgCvE;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG;IACjD,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,MAAM,EAAE,sBAAsB,EAAE,CAAC;CAClC,CAmBA;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,WAAW,EAAE,MAAM,EACnB,eAAe,GAAE,SAAS,MAAM,EAAmB,GAClD,MAAM,EAAE,CA2BV;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAC9B,WAAW,EAAE,MAAM,EACnB,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,GAClC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAUrB"}
@@ -0,0 +1,175 @@
1
+ /**
2
+ * Eval fixture discovery and validation.
3
+ */
4
+ import { readFileSync, readdirSync, statSync, existsSync } from 'fs';
5
+ import { join, resolve } from 'path';
6
+ import { REQUIRED_EVAL_FILES, EXCLUDED_FILES } from './types.js';
7
+ /**
8
+ * Error thrown when an eval fixture is invalid.
9
+ */
10
+ export class FixtureValidationError extends Error {
11
+ fixtureName;
12
+ constructor(fixtureName, message) {
13
+ super(`Eval "${fixtureName}": ${message}`);
14
+ this.fixtureName = fixtureName;
15
+ this.name = 'FixtureValidationError';
16
+ }
17
+ }
18
+ /**
19
+ * Discovers all eval fixtures in a directory.
20
+ * Each subdirectory is considered a potential eval fixture.
21
+ */
22
+ export function discoverFixtures(evalsDir) {
23
+ const absolutePath = resolve(evalsDir);
24
+ if (!existsSync(absolutePath)) {
25
+ throw new Error(`Evals directory not found: ${absolutePath}`);
26
+ }
27
+ const entries = readdirSync(absolutePath);
28
+ const fixtures = [];
29
+ for (const entry of entries) {
30
+ const entryPath = join(absolutePath, entry);
31
+ // Skip hidden directories and files
32
+ if (entry.startsWith('.')) {
33
+ continue;
34
+ }
35
+ // Only consider directories
36
+ if (statSync(entryPath).isDirectory()) {
37
+ fixtures.push(entry);
38
+ }
39
+ }
40
+ return fixtures.sort();
41
+ }
42
+ /**
43
+ * Validates that a fixture has all required files.
44
+ * Returns an array of missing file names, or empty array if valid.
45
+ */
46
+ export function validateFixtureFiles(fixturePath) {
47
+ const missing = [];
48
+ for (const file of REQUIRED_EVAL_FILES) {
49
+ const filePath = join(fixturePath, file);
50
+ if (!existsSync(filePath)) {
51
+ missing.push(file);
52
+ }
53
+ }
54
+ return missing;
55
+ }
56
+ /**
57
+ * Validates the package.json of a fixture.
58
+ * Ensures it has "type": "module".
59
+ */
60
+ export function validatePackageJson(fixturePath) {
61
+ const packageJsonPath = join(fixturePath, 'package.json');
62
+ try {
63
+ const content = readFileSync(packageJsonPath, 'utf-8');
64
+ const pkg = JSON.parse(content);
65
+ if (pkg.type !== 'module') {
66
+ return {
67
+ isModule: false,
68
+ error: 'package.json must have "type": "module"',
69
+ };
70
+ }
71
+ return { isModule: true };
72
+ }
73
+ catch (error) {
74
+ if (error instanceof SyntaxError) {
75
+ return {
76
+ isModule: false,
77
+ error: 'package.json is not valid JSON',
78
+ };
79
+ }
80
+ throw error;
81
+ }
82
+ }
83
+ /**
84
+ * Loads a single eval fixture with full validation.
85
+ */
86
+ export function loadFixture(evalsDir, name) {
87
+ const fixturePath = resolve(evalsDir, name);
88
+ if (!existsSync(fixturePath)) {
89
+ throw new FixtureValidationError(name, `Directory not found: ${fixturePath}`);
90
+ }
91
+ // Validate required files
92
+ const missingFiles = validateFixtureFiles(fixturePath);
93
+ if (missingFiles.length > 0) {
94
+ throw new FixtureValidationError(name, `Missing required files: ${missingFiles.join(', ')}`);
95
+ }
96
+ // Validate package.json
97
+ const pkgValidation = validatePackageJson(fixturePath);
98
+ if (pkgValidation.error) {
99
+ throw new FixtureValidationError(name, pkgValidation.error);
100
+ }
101
+ // Read prompt
102
+ const promptPath = join(fixturePath, 'PROMPT.md');
103
+ const prompt = readFileSync(promptPath, 'utf-8');
104
+ return {
105
+ name,
106
+ path: fixturePath,
107
+ prompt,
108
+ isModule: pkgValidation.isModule,
109
+ };
110
+ }
111
+ /**
112
+ * Discovers and loads all valid eval fixtures from a directory.
113
+ * Returns both valid fixtures and any validation errors encountered.
114
+ */
115
+ export function loadAllFixtures(evalsDir) {
116
+ const fixtureNames = discoverFixtures(evalsDir);
117
+ const fixtures = [];
118
+ const errors = [];
119
+ for (const name of fixtureNames) {
120
+ try {
121
+ const fixture = loadFixture(evalsDir, name);
122
+ fixtures.push(fixture);
123
+ }
124
+ catch (error) {
125
+ if (error instanceof FixtureValidationError) {
126
+ errors.push(error);
127
+ }
128
+ else {
129
+ throw error;
130
+ }
131
+ }
132
+ }
133
+ return { fixtures, errors };
134
+ }
135
+ /**
136
+ * Gets a list of all files in a fixture directory.
137
+ * Excludes PROMPT.md, EVAL.ts, node_modules, and .git.
138
+ */
139
+ export function getFixtureFiles(fixturePath, excludePatterns = EXCLUDED_FILES) {
140
+ const files = [];
141
+ function walk(dir, basePath = '') {
142
+ const entries = readdirSync(dir);
143
+ for (const entry of entries) {
144
+ const relativePath = basePath ? `${basePath}/${entry}` : entry;
145
+ // Check if should be excluded
146
+ if (excludePatterns.some((pattern) => relativePath === pattern || entry === pattern)) {
147
+ continue;
148
+ }
149
+ const fullPath = join(dir, entry);
150
+ const stat = statSync(fullPath);
151
+ if (stat.isDirectory()) {
152
+ walk(fullPath, relativePath);
153
+ }
154
+ else {
155
+ files.push(relativePath);
156
+ }
157
+ }
158
+ }
159
+ walk(fixturePath);
160
+ return files.sort();
161
+ }
162
+ /**
163
+ * Reads all fixture files into a map.
164
+ * Keys are relative paths, values are file contents.
165
+ */
166
+ export function readFixtureFiles(fixturePath, excludePatterns) {
167
+ const files = getFixtureFiles(fixturePath, excludePatterns);
168
+ const contents = new Map();
169
+ for (const file of files) {
170
+ const fullPath = join(fixturePath, file);
171
+ contents.set(file, readFileSync(fullPath, 'utf-8'));
172
+ }
173
+ return contents;
174
+ }
175
+ //# sourceMappingURL=fixture.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fixture.js","sourceRoot":"","sources":["../../src/lib/fixture.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AACrE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAErC,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjE;;GAEG;AACH,MAAM,OAAO,sBAAuB,SAAQ,KAAK;IAEtC;IADT,YACS,WAAmB,EAC1B,OAAe;QAEf,KAAK,CAAC,SAAS,WAAW,MAAM,OAAO,EAAE,CAAC,CAAC;QAHpC,gBAAW,GAAX,WAAW,CAAQ;QAI1B,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;IACvC,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAC/C,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEvC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,8BAA8B,YAAY,EAAE,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,OAAO,GAAG,WAAW,CAAC,YAAY,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;QAE5C,oCAAoC;QACpC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,SAAS;QACX,CAAC;QAED,4BAA4B;QAC5B,IAAI,QAAQ,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;YACtC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,WAAmB;IACtD,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,KAAK,MAAM,IAAI,IAAI,mBAAmB,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QACzC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,WAAmB;IACrD,MAAM,eAAe,GAAG,IAAI,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAE1D,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,YAAY,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;QACvD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEhC,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC1B,OAAO;gBACL,QAAQ,EAAE,KAAK;gBACf,KAAK,EAAE,yCAAyC;aACjD,CAAC;QACJ,CAAC;QAED,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,WAAW,EAAE,CAAC;YACjC,OAAO;gBACL,QAAQ,EAAE,KAAK;gBACf,KAAK,EAAE,gCAAgC;aACxC,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,QAAgB,EAAE,IAAY;IACxD,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAE5C,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,sBAAsB,CAAC,IAAI,EAAE,wBAAwB,WAAW,EAAE,CAAC,CAAC;IAChF,CAAC;IAED,0BAA0B;IAC1B,MAAM,YAAY,GAAG,oBAAoB,CAAC,WAAW,CAAC,CAAC;IACvD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,sBAAsB,CAC9B,IAAI,EACJ,2BAA2B,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACrD,CAAC;IACJ,CAAC;IAED,wBAAwB;IACxB,MAAM,aAAa,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;IACvD,IAAI,aAAa,CAAC,KAAK,EAAE,CAAC;QACxB,MAAM,IAAI,sBAAsB,CAAC,IAAI,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED,cAAc;IACd,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAEjD,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,WAAW;QACjB,MAAM;QACN,QAAQ,EAAE,aAAa,CAAC,QAAQ;KACjC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,QAAgB;IAI9C,MAAM,YAAY,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,MAAM,MAAM,GAA6B,EAAE,CAAC;IAE5C,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC5C,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,sBAAsB,EAAE,CAAC;gBAC5C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,WAAmB,EACnB,kBAAqC,cAAc;IAEnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,SAAS,IAAI,CAAC,GAAW,EAAE,WAAmB,EAAE;QAC9C,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAEjC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;YAE/D,8BAA8B;YAC9B,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,KAAK,OAAO,IAAI,KAAK,KAAK,OAAO,CAAC,EAAE,CAAC;gBACrF,SAAS;YACX,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAClC,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAEhC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;YAC/B,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,WAAW,CAAC,CAAC;IAClB,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;AACtB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAC9B,WAAmB,EACnB,eAAmC;IAEnC,MAAM,KAAK,GAAG,eAAe,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QACzC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
@@ -0,0 +1,21 @@
1
+ /**
2
+ * Project initialization - create new eval projects.
3
+ */
4
+ /**
5
+ * Options for initializing a new project.
6
+ */
7
+ export interface InitOptions {
8
+ /** Project name */
9
+ name: string;
10
+ /** Target directory (defaults to current working directory) */
11
+ targetDir?: string;
12
+ }
13
+ /**
14
+ * Initialize a new eval project.
15
+ */
16
+ export declare function initProject(options: InitOptions): string;
17
+ /**
18
+ * Get instructions for after project creation.
19
+ */
20
+ export declare function getPostInitInstructions(projectDir: string, projectName: string): string;
21
+ //# sourceMappingURL=init.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/lib/init.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,mBAAmB;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAiPD;;GAEG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM,CA0BxD;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAavF"}