@vitest-evals/harness-ai-sdk 0.9.0-beta.2 → 0.9.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/README.md +56 -15
  2. package/dist/index.d.mts +52 -15
  3. package/dist/index.d.ts +52 -15
  4. package/dist/index.js +72 -48
  5. package/dist/index.js.map +1 -1
  6. package/dist/index.mjs +74 -50
  7. package/dist/index.mjs.map +1 -1
  8. package/package.json +2 -2
package/README.md CHANGED
@@ -15,7 +15,12 @@ import { expect } from "vitest";
15
15
  import { generateText, stepCountIs } from "ai";
16
16
  import { openai } from "@ai-sdk/openai";
17
17
  import { aiSdkHarness } from "@vitest-evals/harness-ai-sdk";
18
- import { describeEval, toolCalls } from "vitest-evals";
18
+ import {
19
+ createJudge,
20
+ describeEval,
21
+ toolCalls,
22
+ type JudgeContext,
23
+ } from "vitest-evals";
19
24
 
20
25
  const tools = {
21
26
  lookupInvoice: {
@@ -29,19 +34,14 @@ const harness = aiSdkHarness({
29
34
  toolReplay: {
30
35
  lookupInvoice: true,
31
36
  },
32
- prompt: (input, options) =>
33
- generateText({
34
- model: openai("gpt-4o-mini"),
35
- system: options?.system,
36
- prompt: input,
37
- }).then((result) => result.text),
38
- task: ({ input, runtime }) =>
37
+ run: ({ input, runtime }) =>
39
38
  generateText({
40
39
  model: openai("gpt-4o-mini"),
41
40
  prompt: input,
42
41
  tools: runtime.tools,
43
42
  stopWhen: stepCountIs(5),
44
43
  }),
44
+ output: ({ result }) => parseRefundDecision(result.text),
45
45
  });
46
46
 
47
47
  describeEval("refund agent", { harness }, (it) => {
@@ -58,26 +58,67 @@ describeEval("refund agent", { harness }, (it) => {
58
58
  });
59
59
  ```
60
60
 
61
+ If `run()` already returns `{ output }` or a full `HarnessRun`, that typed
62
+ output is used directly. The `output` selector above is only for the raw
63
+ `generateText(...)` result path where the adapter should keep AI SDK
64
+ diagnostics while projecting provider text into app output.
65
+
61
66
  If your existing AI SDK app exposes its own entrypoint, wire that in directly:
62
67
 
63
68
  ```ts
64
69
  const harness = aiSdkHarness({
65
70
  tools,
66
- prompt: sharedJudgePrompt,
67
- task: ({ input, runtime }) => createRefundAgent().run(input, runtime),
71
+ run: ({ input, runtime }) => createRefundAgent().run(input, runtime),
72
+ });
73
+ ```
74
+
75
+ If your app exposes an agent object instead, `agent` can be either that object
76
+ or a per-run factory. Factories receive the eval input and harness context so
77
+ input-dependent instructions, metadata, or seeded state do not require
78
+ side-channel setup:
79
+
80
+ ```ts
81
+ const harness = aiSdkHarness({
82
+ tools,
83
+ agent: ({ input, context }) =>
84
+ createRefundAgent({
85
+ instructions: buildInstructions(input),
86
+ metadata: context.metadata,
87
+ }),
68
88
  });
69
89
  ```
70
90
 
71
- The required `prompt` callback is passed to harness-backed judges as
72
- `JudgeContext.harness.prompt`, which lets rubric or factuality judges share the
73
- same provider/model configuration as the suite harness.
91
+ `run` executes the system under test. Judges are created separately; keep judge
92
+ prompts and model calls in the judge instead of putting a judge model call on
93
+ the app harness.
94
+
95
+ ```ts
96
+ const FactualityJudge = createJudge(
97
+ "FactualityJudge",
98
+ async (ctx: JudgeContext<string, RefundDecision>) => {
99
+ const verdict = await generateText({
100
+ model: openai("gpt-4o-mini"),
101
+ prompt: formatJudgePrompt({
102
+ input: ctx.input,
103
+ output: ctx.output,
104
+ }),
105
+ }).then((result) => result.text);
106
+
107
+ return parseJudgeVerdict(verdict);
108
+ },
109
+ );
110
+ ```
74
111
 
75
112
  The adapter infers:
76
113
 
77
114
  - normalized session and tool-call traces from AI SDK `steps`
78
115
  - usage diagnostics from `totalUsage` / `usage`
79
- - `run.output` from common AI SDK result fields such as `output`, `object`, and
80
- `text`
116
+ - typed `run.output` from explicit `run()` results that return `output`, from
117
+ common AI SDK provider fields such as `object` and `text`, or from a typed
118
+ `output` selector when the app deliberately returns a raw provider result
119
+ - native app output is accepted only when it is already JSON-safe; arbitrary
120
+ fields, primitive raw results, and non-JSON values require an explicit
121
+ `output` selector
81
122
  - replay/cassette metadata for local tools configured with `toolReplay`
82
123
 
83
124
  See the workspace demo app in `apps/demo-ai-sdk` and the RFC notes in
package/dist/index.d.mts CHANGED
@@ -1,10 +1,19 @@
1
- import { HarnessMetadata, HarnessContext, JsonValue, NormalizedSession, UsageSummary, TimingSummary, HarnessPrompt, HarnessRun, Harness } from 'vitest-evals/harness';
1
+ import { HarnessMetadata, HarnessContext, JsonValue, HarnessRun, Harness } from 'vitest-evals/harness';
2
2
  import { ReplayMode, ToolRecording, ToolReplayConfig } from 'vitest-evals/replay';
3
3
  import { ToolExecutionOptions, Tool, ToolExecuteFunction } from 'ai';
4
4
 
5
5
  type MaybePromise<T> = T | Promise<T>;
6
- type AgentSource<TAgent> = TAgent | (() => MaybePromise<TAgent>);
6
+ type JsonOutput<TValue> = [TValue] extends [JsonValue | undefined] ? TValue : undefined;
7
+ type ResultFieldOutput<TResult, TKey extends string> = TKey extends keyof TResult ? Record<string, never> extends Pick<TResult, TKey> ? JsonOutput<TResult[TKey]> | undefined : JsonOutput<TResult[TKey]> : undefined;
8
+ type AgentSource<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = TAgent | ((args: AiSdkCreateAgentArgs<TInput, TMetadata>) => MaybePromise<TAgent>);
7
9
  type AnyAiSdkToolset<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = Record<string, AiSdkToolDefinition<any, any, TInput, TMetadata>>;
10
+ type AiSdkResultOutput<TResult> = TResult extends HarnessRun<infer TOutput> ? TOutput : "output" extends keyof TResult ? ResultFieldOutput<TResult, "output"> : "object" extends keyof TResult ? ResultFieldOutput<TResult, "object"> : "text" extends keyof TResult ? ResultFieldOutput<TResult, "text"> : undefined;
11
+ type AiSdkAgentResult<TAgent, TInput, TMetadata extends HarnessMetadata, TTools extends AiSdkToolset<TInput, TMetadata>> = TAgent extends {
12
+ run: (input: TInput, runtime: AiSdkRuntime<TTools, TInput, TMetadata>) => MaybePromise<infer TResult>;
13
+ } ? Awaited<TResult> : TAgent extends {
14
+ generate: (input: TInput, runtime: AiSdkRuntime<TTools, TInput, TMetadata>) => MaybePromise<infer TResult>;
15
+ } ? Awaited<TResult> : unknown;
16
+ /** Context passed to instrumented AI SDK tool executions. */
8
17
  interface AiSdkToolContext<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> {
9
18
  input: TInput;
10
19
  metadata: HarnessContext<TMetadata>["metadata"];
@@ -12,22 +21,37 @@ interface AiSdkToolContext<TInput = string, TMetadata extends HarnessMetadata =
12
21
  setArtifact: HarnessContext<TMetadata>["setArtifact"];
13
22
  execution: ToolExecutionOptions;
14
23
  }
24
+ /** Replay mode alias used by the AI SDK harness package. */
15
25
  type AiSdkReplayMode = ReplayMode;
26
+ /** Tool replay recording shape for AI SDK tools. */
16
27
  type AiSdkToolRecording<TArgs extends JsonValue = JsonValue, TResult extends JsonValue = JsonValue> = ToolRecording<TArgs, TResult>;
28
+ /** Replay configuration for an AI SDK tool. */
17
29
  type AiSdkToolReplayConfig<TArgs extends JsonValue = JsonValue, TResult extends JsonValue = JsonValue, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = ToolReplayConfig<TArgs, TResult, AiSdkToolContext<TInput, TMetadata>>;
30
+ /** AI SDK tool definition accepted by the harness. */
18
31
  type AiSdkToolDefinition<TArgs extends JsonValue = JsonValue, TResult extends JsonValue = JsonValue, _TInput = string, _TMetadata extends HarnessMetadata = HarnessMetadata> = Tool<TArgs, TResult>;
32
+ /** Replay policy for one AI SDK tool. */
19
33
  type AiSdkToolReplayPolicy<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = boolean | AiSdkToolReplayConfig<JsonValue, JsonValue, TInput, TMetadata>;
34
+ /** Replay policy map keyed by AI SDK tool name. */
20
35
  type AiSdkToolReplayPolicies<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = Record<string, AiSdkToolReplayPolicy<TInput, TMetadata>>;
36
+ /** Toolset shape accepted by the AI SDK harness. */
21
37
  type AiSdkToolset<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = AnyAiSdkToolset<TInput, TMetadata>;
38
+ /** Runtime toolset exposed to the system under test. */
22
39
  type AiSdkRuntimeToolset<TTools extends AnyAiSdkToolset<any, any>> = {
23
40
  [K in keyof TTools]: TTools[K] extends AiSdkToolDefinition<infer TArgs extends JsonValue, infer TResult extends JsonValue, infer _TInput, infer _TMetadata> ? Omit<TTools[K], "execute"> & {
24
41
  execute?: ToolExecuteFunction<TArgs, TResult>;
25
42
  } : TTools[K];
26
43
  };
44
+ /** Runtime object passed into AI SDK-style agent entrypoints. */
27
45
  interface AiSdkRuntime<TTools extends AiSdkToolset<TInput, TMetadata>, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> {
28
46
  tools: AiSdkRuntimeToolset<TTools>;
29
47
  signal?: AbortSignal;
30
48
  }
49
+ /** Arguments passed to per-run AI SDK agent factories. */
50
+ interface AiSdkCreateAgentArgs<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> {
51
+ input: TInput;
52
+ context: HarnessContext<TMetadata>;
53
+ }
54
+ /** Arguments passed to custom AI SDK harness run callbacks. */
31
55
  interface AiSdkHarnessRunArgs<TAgent, TInput, TMetadata extends HarnessMetadata, TTools extends AiSdkToolset<TInput, TMetadata>> {
32
56
  agent: TAgent | undefined;
33
57
  input: TInput;
@@ -35,27 +59,40 @@ interface AiSdkHarnessRunArgs<TAgent, TInput, TMetadata extends HarnessMetadata,
35
59
  runtime: AiSdkRuntime<TTools, TInput, TMetadata>;
36
60
  tools: AiSdkRuntimeToolset<TTools>;
37
61
  }
62
+ /** Arguments passed to AI SDK harness output selectors. */
38
63
  interface AiSdkHarnessResultArgs<TAgent, TInput, TMetadata extends HarnessMetadata, TResult, TTools extends AiSdkToolset<TInput, TMetadata>> extends AiSdkHarnessRunArgs<TAgent, TInput, TMetadata, TTools> {
39
64
  result: TResult;
40
65
  }
41
- type AiSdkHarnessOptions<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>> = AiSdkHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools> & ({
42
- agent: AgentSource<TAgent>;
43
- task?: never;
66
+ type AiSdkHarnessOutputSelector<TAgent, TInput, TMetadata extends HarnessMetadata, TResult, TTools extends AiSdkToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined> = (args: AiSdkHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<TOutput>;
67
+ type AiSdkHarnessOptions<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined = JsonValue | undefined> = AiSdkHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools, TOutput> & ({
68
+ agent: AgentSource<TAgent, TInput, TMetadata>;
69
+ run?: never;
44
70
  } | {
45
- task: (args: AiSdkHarnessRunArgs<TAgent, TInput, TMetadata, TTools>) => MaybePromise<TResult | HarnessRun>;
71
+ run: (args: AiSdkHarnessRunArgs<TAgent, TInput, TMetadata, TTools>) => MaybePromise<TResult | HarnessRun<TOutput>>;
46
72
  agent?: never;
47
73
  });
48
- interface AiSdkHarnessBaseOptions<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>> {
74
+ type AiSdkHarnessOptionsWithOutput<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined = JsonValue | undefined> = AiSdkHarnessOptions<TAgent, TInput, TMetadata, TResult, TTools, TOutput> & {
75
+ output: AiSdkHarnessOutputSelector<TAgent, TInput, TMetadata, TResult, TTools, TOutput>;
76
+ };
77
+ type AiSdkHarnessRunOptionsWithoutOutput<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>> = AiSdkHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools, JsonValue | undefined> & {
78
+ run: (args: AiSdkHarnessRunArgs<TAgent, TInput, TMetadata, TTools>) => MaybePromise<TResult>;
79
+ agent?: never;
80
+ output?: never;
81
+ };
82
+ type AiSdkHarnessAgentOptionsWithoutOutput<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>> = AiSdkHarnessBaseOptions<TAgent, TInput, TMetadata, unknown, TTools, JsonValue | undefined> & {
83
+ agent: AgentSource<TAgent, TInput, TMetadata>;
84
+ run?: never;
85
+ output?: never;
86
+ };
87
+ interface AiSdkHarnessBaseOptions<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined = JsonValue | undefined> {
49
88
  tools?: TTools;
50
89
  toolReplay?: AiSdkToolReplayPolicies<TInput, TMetadata>;
51
- session?: (args: AiSdkHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<NormalizedSession>;
52
- output?: (args: AiSdkHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<JsonValue | undefined>;
53
- usage?: (args: AiSdkHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<UsageSummary>;
54
- timings?: (args: AiSdkHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<TimingSummary | undefined>;
55
- errors?: (args: AiSdkHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<Array<Record<string, JsonValue>>>;
56
- prompt: HarnessPrompt;
90
+ output?: AiSdkHarnessOutputSelector<TAgent, TInput, TMetadata, TResult, TTools, TOutput>;
57
91
  name?: string;
58
92
  }
59
- declare function aiSdkHarness<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>>(options: AiSdkHarnessOptions<TAgent, TInput, TMetadata, TResult, TTools>): Harness<TInput, TMetadata>;
93
+ /** Adapts an AI SDK-style workflow into a normalized eval harness. */
94
+ declare function aiSdkHarness<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined = JsonValue | undefined>(options: AiSdkHarnessOptionsWithOutput<TAgent, TInput, TMetadata, TResult, TTools, TOutput>): Harness<TInput, TOutput, TMetadata>;
95
+ declare function aiSdkHarness<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>>(options: AiSdkHarnessRunOptionsWithoutOutput<TAgent, TInput, TMetadata, TResult, TTools>): Harness<TInput, AiSdkResultOutput<Awaited<TResult>>, TMetadata>;
96
+ declare function aiSdkHarness<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>>(options: AiSdkHarnessAgentOptionsWithoutOutput<TAgent, TInput, TMetadata, TTools>): Harness<TInput, AiSdkResultOutput<AiSdkAgentResult<TAgent, TInput, TMetadata, TTools>>, TMetadata>;
60
97
 
61
- export { type AiSdkHarnessOptions, type AiSdkHarnessResultArgs, type AiSdkHarnessRunArgs, type AiSdkReplayMode, type AiSdkRuntime, type AiSdkRuntimeToolset, type AiSdkToolContext, type AiSdkToolDefinition, type AiSdkToolRecording, type AiSdkToolReplayConfig, type AiSdkToolReplayPolicies, type AiSdkToolReplayPolicy, type AiSdkToolset, aiSdkHarness };
98
+ export { type AiSdkCreateAgentArgs, type AiSdkHarnessResultArgs, type AiSdkHarnessRunArgs, type AiSdkReplayMode, type AiSdkRuntime, type AiSdkRuntimeToolset, type AiSdkToolContext, type AiSdkToolDefinition, type AiSdkToolRecording, type AiSdkToolReplayConfig, type AiSdkToolReplayPolicies, type AiSdkToolReplayPolicy, type AiSdkToolset, aiSdkHarness };
package/dist/index.d.ts CHANGED
@@ -1,10 +1,19 @@
1
- import { HarnessMetadata, HarnessContext, JsonValue, NormalizedSession, UsageSummary, TimingSummary, HarnessPrompt, HarnessRun, Harness } from 'vitest-evals/harness';
1
+ import { HarnessMetadata, HarnessContext, JsonValue, HarnessRun, Harness } from 'vitest-evals/harness';
2
2
  import { ReplayMode, ToolRecording, ToolReplayConfig } from 'vitest-evals/replay';
3
3
  import { ToolExecutionOptions, Tool, ToolExecuteFunction } from 'ai';
4
4
 
5
5
  type MaybePromise<T> = T | Promise<T>;
6
- type AgentSource<TAgent> = TAgent | (() => MaybePromise<TAgent>);
6
+ type JsonOutput<TValue> = [TValue] extends [JsonValue | undefined] ? TValue : undefined;
7
+ type ResultFieldOutput<TResult, TKey extends string> = TKey extends keyof TResult ? Record<string, never> extends Pick<TResult, TKey> ? JsonOutput<TResult[TKey]> | undefined : JsonOutput<TResult[TKey]> : undefined;
8
+ type AgentSource<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = TAgent | ((args: AiSdkCreateAgentArgs<TInput, TMetadata>) => MaybePromise<TAgent>);
7
9
  type AnyAiSdkToolset<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = Record<string, AiSdkToolDefinition<any, any, TInput, TMetadata>>;
10
+ type AiSdkResultOutput<TResult> = TResult extends HarnessRun<infer TOutput> ? TOutput : "output" extends keyof TResult ? ResultFieldOutput<TResult, "output"> : "object" extends keyof TResult ? ResultFieldOutput<TResult, "object"> : "text" extends keyof TResult ? ResultFieldOutput<TResult, "text"> : undefined;
11
+ type AiSdkAgentResult<TAgent, TInput, TMetadata extends HarnessMetadata, TTools extends AiSdkToolset<TInput, TMetadata>> = TAgent extends {
12
+ run: (input: TInput, runtime: AiSdkRuntime<TTools, TInput, TMetadata>) => MaybePromise<infer TResult>;
13
+ } ? Awaited<TResult> : TAgent extends {
14
+ generate: (input: TInput, runtime: AiSdkRuntime<TTools, TInput, TMetadata>) => MaybePromise<infer TResult>;
15
+ } ? Awaited<TResult> : unknown;
16
+ /** Context passed to instrumented AI SDK tool executions. */
8
17
  interface AiSdkToolContext<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> {
9
18
  input: TInput;
10
19
  metadata: HarnessContext<TMetadata>["metadata"];
@@ -12,22 +21,37 @@ interface AiSdkToolContext<TInput = string, TMetadata extends HarnessMetadata =
12
21
  setArtifact: HarnessContext<TMetadata>["setArtifact"];
13
22
  execution: ToolExecutionOptions;
14
23
  }
24
+ /** Replay mode alias used by the AI SDK harness package. */
15
25
  type AiSdkReplayMode = ReplayMode;
26
+ /** Tool replay recording shape for AI SDK tools. */
16
27
  type AiSdkToolRecording<TArgs extends JsonValue = JsonValue, TResult extends JsonValue = JsonValue> = ToolRecording<TArgs, TResult>;
28
+ /** Replay configuration for an AI SDK tool. */
17
29
  type AiSdkToolReplayConfig<TArgs extends JsonValue = JsonValue, TResult extends JsonValue = JsonValue, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = ToolReplayConfig<TArgs, TResult, AiSdkToolContext<TInput, TMetadata>>;
30
+ /** AI SDK tool definition accepted by the harness. */
18
31
  type AiSdkToolDefinition<TArgs extends JsonValue = JsonValue, TResult extends JsonValue = JsonValue, _TInput = string, _TMetadata extends HarnessMetadata = HarnessMetadata> = Tool<TArgs, TResult>;
32
+ /** Replay policy for one AI SDK tool. */
19
33
  type AiSdkToolReplayPolicy<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = boolean | AiSdkToolReplayConfig<JsonValue, JsonValue, TInput, TMetadata>;
34
+ /** Replay policy map keyed by AI SDK tool name. */
20
35
  type AiSdkToolReplayPolicies<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = Record<string, AiSdkToolReplayPolicy<TInput, TMetadata>>;
36
+ /** Toolset shape accepted by the AI SDK harness. */
21
37
  type AiSdkToolset<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = AnyAiSdkToolset<TInput, TMetadata>;
38
+ /** Runtime toolset exposed to the system under test. */
22
39
  type AiSdkRuntimeToolset<TTools extends AnyAiSdkToolset<any, any>> = {
23
40
  [K in keyof TTools]: TTools[K] extends AiSdkToolDefinition<infer TArgs extends JsonValue, infer TResult extends JsonValue, infer _TInput, infer _TMetadata> ? Omit<TTools[K], "execute"> & {
24
41
  execute?: ToolExecuteFunction<TArgs, TResult>;
25
42
  } : TTools[K];
26
43
  };
44
+ /** Runtime object passed into AI SDK-style agent entrypoints. */
27
45
  interface AiSdkRuntime<TTools extends AiSdkToolset<TInput, TMetadata>, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> {
28
46
  tools: AiSdkRuntimeToolset<TTools>;
29
47
  signal?: AbortSignal;
30
48
  }
49
+ /** Arguments passed to per-run AI SDK agent factories. */
50
+ interface AiSdkCreateAgentArgs<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> {
51
+ input: TInput;
52
+ context: HarnessContext<TMetadata>;
53
+ }
54
+ /** Arguments passed to custom AI SDK harness run callbacks. */
31
55
  interface AiSdkHarnessRunArgs<TAgent, TInput, TMetadata extends HarnessMetadata, TTools extends AiSdkToolset<TInput, TMetadata>> {
32
56
  agent: TAgent | undefined;
33
57
  input: TInput;
@@ -35,27 +59,40 @@ interface AiSdkHarnessRunArgs<TAgent, TInput, TMetadata extends HarnessMetadata,
35
59
  runtime: AiSdkRuntime<TTools, TInput, TMetadata>;
36
60
  tools: AiSdkRuntimeToolset<TTools>;
37
61
  }
62
+ /** Arguments passed to AI SDK harness output selectors. */
38
63
  interface AiSdkHarnessResultArgs<TAgent, TInput, TMetadata extends HarnessMetadata, TResult, TTools extends AiSdkToolset<TInput, TMetadata>> extends AiSdkHarnessRunArgs<TAgent, TInput, TMetadata, TTools> {
39
64
  result: TResult;
40
65
  }
41
- type AiSdkHarnessOptions<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>> = AiSdkHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools> & ({
42
- agent: AgentSource<TAgent>;
43
- task?: never;
66
+ type AiSdkHarnessOutputSelector<TAgent, TInput, TMetadata extends HarnessMetadata, TResult, TTools extends AiSdkToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined> = (args: AiSdkHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<TOutput>;
67
+ type AiSdkHarnessOptions<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined = JsonValue | undefined> = AiSdkHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools, TOutput> & ({
68
+ agent: AgentSource<TAgent, TInput, TMetadata>;
69
+ run?: never;
44
70
  } | {
45
- task: (args: AiSdkHarnessRunArgs<TAgent, TInput, TMetadata, TTools>) => MaybePromise<TResult | HarnessRun>;
71
+ run: (args: AiSdkHarnessRunArgs<TAgent, TInput, TMetadata, TTools>) => MaybePromise<TResult | HarnessRun<TOutput>>;
46
72
  agent?: never;
47
73
  });
48
- interface AiSdkHarnessBaseOptions<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>> {
74
+ type AiSdkHarnessOptionsWithOutput<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined = JsonValue | undefined> = AiSdkHarnessOptions<TAgent, TInput, TMetadata, TResult, TTools, TOutput> & {
75
+ output: AiSdkHarnessOutputSelector<TAgent, TInput, TMetadata, TResult, TTools, TOutput>;
76
+ };
77
+ type AiSdkHarnessRunOptionsWithoutOutput<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>> = AiSdkHarnessBaseOptions<TAgent, TInput, TMetadata, TResult, TTools, JsonValue | undefined> & {
78
+ run: (args: AiSdkHarnessRunArgs<TAgent, TInput, TMetadata, TTools>) => MaybePromise<TResult>;
79
+ agent?: never;
80
+ output?: never;
81
+ };
82
+ type AiSdkHarnessAgentOptionsWithoutOutput<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>> = AiSdkHarnessBaseOptions<TAgent, TInput, TMetadata, unknown, TTools, JsonValue | undefined> & {
83
+ agent: AgentSource<TAgent, TInput, TMetadata>;
84
+ run?: never;
85
+ output?: never;
86
+ };
87
+ interface AiSdkHarnessBaseOptions<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined = JsonValue | undefined> {
49
88
  tools?: TTools;
50
89
  toolReplay?: AiSdkToolReplayPolicies<TInput, TMetadata>;
51
- session?: (args: AiSdkHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<NormalizedSession>;
52
- output?: (args: AiSdkHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<JsonValue | undefined>;
53
- usage?: (args: AiSdkHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<UsageSummary>;
54
- timings?: (args: AiSdkHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<TimingSummary | undefined>;
55
- errors?: (args: AiSdkHarnessResultArgs<TAgent, TInput, TMetadata, TResult, TTools>) => MaybePromise<Array<Record<string, JsonValue>>>;
56
- prompt: HarnessPrompt;
90
+ output?: AiSdkHarnessOutputSelector<TAgent, TInput, TMetadata, TResult, TTools, TOutput>;
57
91
  name?: string;
58
92
  }
59
- declare function aiSdkHarness<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>>(options: AiSdkHarnessOptions<TAgent, TInput, TMetadata, TResult, TTools>): Harness<TInput, TMetadata>;
93
+ /** Adapts an AI SDK-style workflow into a normalized eval harness. */
94
+ declare function aiSdkHarness<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>, TOutput extends JsonValue | undefined = JsonValue | undefined>(options: AiSdkHarnessOptionsWithOutput<TAgent, TInput, TMetadata, TResult, TTools, TOutput>): Harness<TInput, TOutput, TMetadata>;
95
+ declare function aiSdkHarness<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>>(options: AiSdkHarnessRunOptionsWithoutOutput<TAgent, TInput, TMetadata, TResult, TTools>): Harness<TInput, AiSdkResultOutput<Awaited<TResult>>, TMetadata>;
96
+ declare function aiSdkHarness<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>>(options: AiSdkHarnessAgentOptionsWithoutOutput<TAgent, TInput, TMetadata, TTools>): Harness<TInput, AiSdkResultOutput<AiSdkAgentResult<TAgent, TInput, TMetadata, TTools>>, TMetadata>;
60
97
 
61
- export { type AiSdkHarnessOptions, type AiSdkHarnessResultArgs, type AiSdkHarnessRunArgs, type AiSdkReplayMode, type AiSdkRuntime, type AiSdkRuntimeToolset, type AiSdkToolContext, type AiSdkToolDefinition, type AiSdkToolRecording, type AiSdkToolReplayConfig, type AiSdkToolReplayPolicies, type AiSdkToolReplayPolicy, type AiSdkToolset, aiSdkHarness };
98
+ export { type AiSdkCreateAgentArgs, type AiSdkHarnessResultArgs, type AiSdkHarnessRunArgs, type AiSdkReplayMode, type AiSdkRuntime, type AiSdkRuntimeToolset, type AiSdkToolContext, type AiSdkToolDefinition, type AiSdkToolRecording, type AiSdkToolReplayConfig, type AiSdkToolReplayPolicies, type AiSdkToolReplayPolicy, type AiSdkToolset, aiSdkHarness };
package/dist/index.js CHANGED
@@ -27,14 +27,17 @@ var import_harness = require("vitest-evals/harness");
27
27
  var import_replay = require("vitest-evals/replay");
28
28
  function aiSdkHarness(options) {
29
29
  validateOptions(options);
30
- return {
30
+ const harness = {
31
31
  name: options.name ?? "ai-sdk",
32
- prompt: options.prompt,
33
32
  run: async (input, context) => {
34
- const agent = await resolveAgent(options);
33
+ const agent = await resolveAgent(options, {
34
+ input,
35
+ context
36
+ });
35
37
  return runAiSdkHarness(options, agent, input, context);
36
38
  }
37
39
  };
40
+ return harness;
38
41
  }
39
42
  async function runAiSdkHarness(options, agent, input, context) {
40
43
  const replayMetadataByToolCallId = /* @__PURE__ */ new Map();
@@ -59,7 +62,7 @@ async function runAiSdkHarness(options, agent, input, context) {
59
62
  runtime,
60
63
  tools
61
64
  });
62
- if ((0, import_harness.isHarnessRun)(result) && !hasResultOverrides(options)) {
65
+ if ((0, import_harness.isHarnessRun)(result) && !hasOutputSelector(options)) {
63
66
  if (Object.keys(context.artifacts).length > 0 && !result.artifacts) {
64
67
  result.artifacts = context.artifacts;
65
68
  }
@@ -74,21 +77,21 @@ async function runAiSdkHarness(options, agent, input, context) {
74
77
  result
75
78
  };
76
79
  const output = options.output ? await options.output(resultArgs) : resolveOutput(result);
77
- const usage = options.usage ? await options.usage(resultArgs) : resolveUsage(result, runtimeToolCalls.length);
78
- const session = options.session ? await options.session(resultArgs) : resolveSession(
80
+ const usage = resolveUsage(result, runtimeToolCalls.length);
81
+ const session = resolveSession(
79
82
  input,
80
83
  result,
81
84
  output,
82
85
  replayMetadataByToolCallId,
83
86
  runtimeToolCalls
84
87
  );
88
+ const errors = (0, import_harness.resolveHarnessRunErrors)(result);
85
89
  return {
86
90
  session,
87
91
  output,
88
92
  usage,
89
- timings: options.timings ? await options.timings(resultArgs) : void 0,
90
93
  artifacts: Object.keys(context.artifacts).length > 0 ? context.artifacts : void 0,
91
- errors: options.errors ? await options.errors(resultArgs) : (0, import_harness.resolveHarnessRunErrors)(result)
94
+ errors
92
95
  };
93
96
  } catch (error) {
94
97
  const run = {
@@ -107,48 +110,49 @@ async function runAiSdkHarness(options, agent, input, context) {
107
110
  throw (0, import_harness.attachHarnessRunToError)(error, run);
108
111
  }
109
112
  }
110
- function hasResultOverrides(options) {
111
- return Boolean(
112
- options.output ?? options.session ?? options.usage ?? options.timings ?? options.errors
113
- );
113
+ function hasOutputSelector(options) {
114
+ return Boolean(options.output);
114
115
  }
115
- async function resolveAgent(options) {
116
- return hasAgentSource(options) ? await resolveAgentSource(options.agent) : void 0;
116
+ async function resolveAgent(options, args) {
117
+ return hasAgentSource(options) ? await resolveAgentSource(options.agent, args) : void 0;
117
118
  }
118
119
  async function runAgent(options, args) {
119
- if (options.task) {
120
- return options.task(args);
120
+ if (options.run) {
121
+ return options.run(args);
121
122
  }
122
123
  if (hasAiSdkRunMethod(args.agent)) {
123
124
  return args.agent.run(args.input, args.runtime);
124
125
  }
125
- if (hasAiSdkGenerateMethod(args.agent)) {
126
+ if (hasAiSdkGenerateMethod(
127
+ args.agent
128
+ )) {
126
129
  return args.agent.generate(args.input, args.runtime);
127
130
  }
128
131
  throw new Error(
129
- "aiSdkHarness agent must expose run(input, runtime) or generate(input, runtime), or use task() for a custom entrypoint."
132
+ "aiSdkHarness agent must expose run(input, runtime) or generate(input, runtime), or use run() for a custom entrypoint."
130
133
  );
131
134
  }
132
135
  function validateOptions(options) {
133
136
  const hasAgent = hasAgentSource(options);
134
- const hasTask = typeof options.task === "function";
135
- if (hasAgent && hasTask) {
137
+ const hasRun = typeof options.run === "function";
138
+ const entrypoints = [hasAgent, hasRun].filter(Boolean).length;
139
+ if (entrypoints > 1) {
136
140
  throw new Error(
137
- "aiSdkHarness accepts either agent or task, not both. Use agent for the zero-glue run(input, runtime) or generate(input, runtime) path, or task for a custom entrypoint."
141
+ "aiSdkHarness accepts exactly one of agent or run. Use agent for the zero-glue run(input, runtime) or generate(input, runtime) path, or run for a custom entrypoint."
138
142
  );
139
143
  }
140
- if (!hasAgent && !hasTask) {
144
+ if (entrypoints === 0) {
141
145
  throw new Error(
142
- "aiSdkHarness requires either agent or task. Use agent for objects with run(input, runtime) or generate(input, runtime), or task for a custom entrypoint."
146
+ "aiSdkHarness requires agent or run. Use agent for objects with run(input, runtime) or generate(input, runtime), or run for a custom entrypoint."
143
147
  );
144
148
  }
145
149
  }
146
150
  function hasAgentSource(options) {
147
151
  return "agent" in options && options.agent !== void 0;
148
152
  }
149
- async function resolveAgentSource(agent) {
153
+ async function resolveAgentSource(agent, args) {
150
154
  if (isAgentFactory(agent)) {
151
- return agent();
155
+ return agent(args);
152
156
  }
153
157
  return agent;
154
158
  }
@@ -282,21 +286,52 @@ async function executeToolWithReplay({
282
286
  }
283
287
  function resolveOutput(result) {
284
288
  if (!result || typeof result !== "object") {
285
- return (0, import_harness.toJsonValue)(result);
289
+ return void 0;
286
290
  }
287
- const candidates = [
288
- "output",
289
- "object",
290
- "experimental_output",
291
- "result",
292
- "text"
293
- ];
291
+ const candidates = ["output", "object", "text"];
292
+ const resultRecord = result;
294
293
  for (const key of candidates) {
295
- const value = result[key];
296
- const normalized = (0, import_harness.toJsonValue)(value);
297
- if (normalized !== void 0) {
298
- return normalized;
294
+ if (Object.prototype.hasOwnProperty.call(resultRecord, key)) {
295
+ return toOutputValue(resultRecord[key]);
296
+ }
297
+ }
298
+ return void 0;
299
+ }
300
+ function toOutputValue(value) {
301
+ if (value === void 0) {
302
+ return void 0;
303
+ }
304
+ if (value === null || typeof value === "string" || typeof value === "boolean") {
305
+ return value;
306
+ }
307
+ if (typeof value === "number") {
308
+ return Number.isFinite(value) ? value : void 0;
309
+ }
310
+ if (Array.isArray(value)) {
311
+ const items = [];
312
+ for (const item of value) {
313
+ const normalized = toOutputValue(item);
314
+ if (normalized === void 0) {
315
+ return void 0;
316
+ }
317
+ items.push(normalized);
299
318
  }
319
+ return items;
320
+ }
321
+ if (typeof value === "object") {
322
+ const prototype = Object.getPrototypeOf(value);
323
+ if (prototype !== Object.prototype && prototype !== null) {
324
+ return void 0;
325
+ }
326
+ const record = {};
327
+ for (const [key, entry] of Object.entries(value)) {
328
+ const normalized = toOutputValue(entry);
329
+ if (normalized === void 0) {
330
+ return void 0;
331
+ }
332
+ record[key] = normalized;
333
+ }
334
+ return record;
300
335
  }
301
336
  return void 0;
302
337
  }
@@ -425,10 +460,8 @@ function resolveSession(input, result, output, replayMetadataByToolCallId, runti
425
460
  });
426
461
  }
427
462
  const lastStep = steps.length > 0 ? steps[steps.length - 1] : void 0;
428
- const outputText = resolveOutputText(result, output, lastStep);
429
463
  return {
430
464
  messages,
431
- outputText,
432
465
  provider: lastStep?.model.provider,
433
466
  model: lastStep?.model.modelId
434
467
  };
@@ -470,15 +503,6 @@ function resolveLanguageModelUsage(result) {
470
503
  const aiResult = result;
471
504
  return aiResult.totalUsage ?? aiResult.usage;
472
505
  }
473
- function resolveOutputText(result, output, lastStep) {
474
- if (lastStep?.text) {
475
- return lastStep.text;
476
- }
477
- if (result && typeof result === "object" && typeof result.text === "string") {
478
- return result.text;
479
- }
480
- return typeof output === "string" ? output : void 0;
481
- }
482
506
  function normalizeStep(step, replayMetadataByToolCallId) {
483
507
  const toolResultsById = new Map(
484
508
  (step.toolResults ?? []).map((toolResult) => [