@vitest-evals/harness-ai-sdk 0.9.0 → 0.11.0

package/README.md

@@ -89,24 +89,24 @@ const harness = aiSdkHarness({
 ```
 
 `run` executes the system under test. Judges are created separately; keep judge
-prompts and model calls in the judge instead of putting a judge model call on
-the app harness.
+prompts and model calls on a judge harness instead of putting them on the app
+harness.
 
 ```ts
-const FactualityJudge = createJudge(
-  "FactualityJudge",
-  async (ctx: JudgeContext<string, RefundDecision>) => {
-    const verdict = await generateText({
-      model: openai("gpt-4o-mini"),
-      prompt: formatJudgePrompt({
-        input: ctx.input,
-        output: ctx.output,
-      }),
-    }).then((result) => result.text);
+import { openai } from "@ai-sdk/openai";
+import { aiSdkJudgeHarness } from "@vitest-evals/harness-ai-sdk";
+import { describeEval, FactualityJudge } from "vitest-evals";
 
-    return parseJudgeVerdict(verdict);
-  },
-);
+const judgeHarness = aiSdkJudgeHarness({
+  model: openai("gpt-4.1-mini"),
+  temperature: 0,
+});
+const factualityJudge = FactualityJudge({ judgeHarness });
+
+describeEval("refund agent", {
+  harness,
+  judges: [factualityJudge],
+});
 ```
 
 The adapter infers:

package/dist/index.d.mts

@@ -1,12 +1,62 @@
+import * as vitest_evals_judges from 'vitest-evals/judges';
 import { HarnessMetadata, HarnessContext, JsonValue, HarnessRun, Harness } from 'vitest-evals/harness';
+import { LanguageModel, generateText, ToolExecutionOptions, Tool, ToolExecuteFunction } from 'ai';
 import { ReplayMode, ToolRecording, ToolReplayConfig } from 'vitest-evals/replay';
-import { ToolExecutionOptions, Tool, ToolExecuteFunction } from 'ai';
 
 type MaybePromise<T> = T | Promise<T>;
+type AiSdkProviderOptions = Parameters<typeof generateText>[0]["providerOptions"];
 type JsonOutput<TValue> = [TValue] extends [JsonValue | undefined] ? TValue : undefined;
 type ResultFieldOutput<TResult, TKey extends string> = TKey extends keyof TResult ? Record<string, never> extends Pick<TResult, TKey> ? JsonOutput<TResult[TKey]> | undefined : JsonOutput<TResult[TKey]> : undefined;
 type AgentSource<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = TAgent | ((args: AiSdkCreateAgentArgs<TInput, TMetadata>) => MaybePromise<TAgent>);
 type AnyAiSdkToolset<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = Record<string, AiSdkToolDefinition<any, any, TInput, TMetadata>>;
+/**
+ * Configuration for adapting an AI SDK language model into a judge harness.
+ *
+ * @example
+ * ```ts
+ * import { anthropic } from "@ai-sdk/anthropic";
+ * import { aiSdkJudgeHarness } from "@vitest-evals/harness-ai-sdk";
+ *
+ * const judgeHarness = aiSdkJudgeHarness({
+ *   model: anthropic("claude-sonnet-4-5"),
+ *   temperature: 0,
+ * });
+ * ```
+ */
+interface AiSdkJudgeHarnessConfig {
+    /** AI SDK language model used for judge prompts. */
+    model: LanguageModel;
+    /** Optional display name for diagnostics. */
+    name?: string;
+    /** Optional judge-model temperature. */
+    temperature?: number;
+    /** Optional judge-model token cap. */
+    maxOutputTokens?: number;
+    /** Optional provider-specific options forwarded to the AI SDK. */
+    providerOptions?: AiSdkProviderOptions;
+}
+/**
+ * Adapts an AI SDK language model into the provider-neutral judge harness API.
+ *
+ * @example
+ * ```ts
+ * import { anthropic } from "@ai-sdk/anthropic";
+ * import { aiSdkJudgeHarness } from "@vitest-evals/harness-ai-sdk";
+ * import { describeEval, FactualityJudge } from "vitest-evals";
+ *
+ * const judgeHarness = aiSdkJudgeHarness({
+ *   model: anthropic("claude-sonnet-4-5"),
+ *   temperature: 0,
+ * });
+ *
+ * describeEval("qa agent", {
+ *   harness: qaHarness,
+ *   judgeHarness,
+ *   judges: [FactualityJudge()],
+ * }, (it) => {});
+ * ```
+ */
+declare function aiSdkJudgeHarness(config: AiSdkJudgeHarnessConfig): vitest_evals_judges.JudgeHarness;
 type AiSdkResultOutput<TResult> = TResult extends HarnessRun<infer TOutput> ? TOutput : "output" extends keyof TResult ? ResultFieldOutput<TResult, "output"> : "object" extends keyof TResult ? ResultFieldOutput<TResult, "object"> : "text" extends keyof TResult ? ResultFieldOutput<TResult, "text"> : undefined;
 type AiSdkAgentResult<TAgent, TInput, TMetadata extends HarnessMetadata, TTools extends AiSdkToolset<TInput, TMetadata>> = TAgent extends {
     run: (input: TInput, runtime: AiSdkRuntime<TTools, TInput, TMetadata>) => MaybePromise<infer TResult>;
@@ -95,4 +145,4 @@ declare function aiSdkHarness<TAgent = unknown, TInput = string, TMetadata exten
 declare function aiSdkHarness<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>>(options: AiSdkHarnessRunOptionsWithoutOutput<TAgent, TInput, TMetadata, TResult, TTools>): Harness<TInput, AiSdkResultOutput<Awaited<TResult>>, TMetadata>;
 declare function aiSdkHarness<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>>(options: AiSdkHarnessAgentOptionsWithoutOutput<TAgent, TInput, TMetadata, TTools>): Harness<TInput, AiSdkResultOutput<AiSdkAgentResult<TAgent, TInput, TMetadata, TTools>>, TMetadata>;
 
-export { type AiSdkCreateAgentArgs, type AiSdkHarnessResultArgs, type AiSdkHarnessRunArgs, type AiSdkReplayMode, type AiSdkRuntime, type AiSdkRuntimeToolset, type AiSdkToolContext, type AiSdkToolDefinition, type AiSdkToolRecording, type AiSdkToolReplayConfig, type AiSdkToolReplayPolicies, type AiSdkToolReplayPolicy, type AiSdkToolset, aiSdkHarness };
+export { type AiSdkCreateAgentArgs, type AiSdkHarnessResultArgs, type AiSdkHarnessRunArgs, type AiSdkJudgeHarnessConfig, type AiSdkReplayMode, type AiSdkRuntime, type AiSdkRuntimeToolset, type AiSdkToolContext, type AiSdkToolDefinition, type AiSdkToolRecording, type AiSdkToolReplayConfig, type AiSdkToolReplayPolicies, type AiSdkToolReplayPolicy, type AiSdkToolset, aiSdkHarness, aiSdkJudgeHarness };

package/dist/index.d.ts

@@ -1,12 +1,62 @@
+import * as vitest_evals_judges from 'vitest-evals/judges';
 import { HarnessMetadata, HarnessContext, JsonValue, HarnessRun, Harness } from 'vitest-evals/harness';
+import { LanguageModel, generateText, ToolExecutionOptions, Tool, ToolExecuteFunction } from 'ai';
 import { ReplayMode, ToolRecording, ToolReplayConfig } from 'vitest-evals/replay';
-import { ToolExecutionOptions, Tool, ToolExecuteFunction } from 'ai';
 
 type MaybePromise<T> = T | Promise<T>;
+type AiSdkProviderOptions = Parameters<typeof generateText>[0]["providerOptions"];
 type JsonOutput<TValue> = [TValue] extends [JsonValue | undefined] ? TValue : undefined;
 type ResultFieldOutput<TResult, TKey extends string> = TKey extends keyof TResult ? Record<string, never> extends Pick<TResult, TKey> ? JsonOutput<TResult[TKey]> | undefined : JsonOutput<TResult[TKey]> : undefined;
 type AgentSource<TAgent, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = TAgent | ((args: AiSdkCreateAgentArgs<TInput, TMetadata>) => MaybePromise<TAgent>);
 type AnyAiSdkToolset<TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata> = Record<string, AiSdkToolDefinition<any, any, TInput, TMetadata>>;
+/**
+ * Configuration for adapting an AI SDK language model into a judge harness.
+ *
+ * @example
+ * ```ts
+ * import { anthropic } from "@ai-sdk/anthropic";
+ * import { aiSdkJudgeHarness } from "@vitest-evals/harness-ai-sdk";
+ *
+ * const judgeHarness = aiSdkJudgeHarness({
+ *   model: anthropic("claude-sonnet-4-5"),
+ *   temperature: 0,
+ * });
+ * ```
+ */
+interface AiSdkJudgeHarnessConfig {
+    /** AI SDK language model used for judge prompts. */
+    model: LanguageModel;
+    /** Optional display name for diagnostics. */
+    name?: string;
+    /** Optional judge-model temperature. */
+    temperature?: number;
+    /** Optional judge-model token cap. */
+    maxOutputTokens?: number;
+    /** Optional provider-specific options forwarded to the AI SDK. */
+    providerOptions?: AiSdkProviderOptions;
+}
+/**
+ * Adapts an AI SDK language model into the provider-neutral judge harness API.
+ *
+ * @example
+ * ```ts
+ * import { anthropic } from "@ai-sdk/anthropic";
+ * import { aiSdkJudgeHarness } from "@vitest-evals/harness-ai-sdk";
+ * import { describeEval, FactualityJudge } from "vitest-evals";
+ *
+ * const judgeHarness = aiSdkJudgeHarness({
+ *   model: anthropic("claude-sonnet-4-5"),
+ *   temperature: 0,
+ * });
+ *
+ * describeEval("qa agent", {
+ *   harness: qaHarness,
+ *   judgeHarness,
+ *   judges: [FactualityJudge()],
+ * }, (it) => {});
+ * ```
+ */
+declare function aiSdkJudgeHarness(config: AiSdkJudgeHarnessConfig): vitest_evals_judges.JudgeHarness;
 type AiSdkResultOutput<TResult> = TResult extends HarnessRun<infer TOutput> ? TOutput : "output" extends keyof TResult ? ResultFieldOutput<TResult, "output"> : "object" extends keyof TResult ? ResultFieldOutput<TResult, "object"> : "text" extends keyof TResult ? ResultFieldOutput<TResult, "text"> : undefined;
 type AiSdkAgentResult<TAgent, TInput, TMetadata extends HarnessMetadata, TTools extends AiSdkToolset<TInput, TMetadata>> = TAgent extends {
     run: (input: TInput, runtime: AiSdkRuntime<TTools, TInput, TMetadata>) => MaybePromise<infer TResult>;
@@ -95,4 +145,4 @@ declare function aiSdkHarness<TAgent = unknown, TInput = string, TMetadata exten
 declare function aiSdkHarness<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TResult = unknown, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>>(options: AiSdkHarnessRunOptionsWithoutOutput<TAgent, TInput, TMetadata, TResult, TTools>): Harness<TInput, AiSdkResultOutput<Awaited<TResult>>, TMetadata>;
 declare function aiSdkHarness<TAgent = unknown, TInput = string, TMetadata extends HarnessMetadata = HarnessMetadata, TTools extends AiSdkToolset<TInput, TMetadata> = AiSdkToolset<TInput, TMetadata>>(options: AiSdkHarnessAgentOptionsWithoutOutput<TAgent, TInput, TMetadata, TTools>): Harness<TInput, AiSdkResultOutput<AiSdkAgentResult<TAgent, TInput, TMetadata, TTools>>, TMetadata>;
 
-export { type AiSdkCreateAgentArgs, type AiSdkHarnessResultArgs, type AiSdkHarnessRunArgs, type AiSdkReplayMode, type AiSdkRuntime, type AiSdkRuntimeToolset, type AiSdkToolContext, type AiSdkToolDefinition, type AiSdkToolRecording, type AiSdkToolReplayConfig, type AiSdkToolReplayPolicies, type AiSdkToolReplayPolicy, type AiSdkToolset, aiSdkHarness };
+export { type AiSdkCreateAgentArgs, type AiSdkHarnessResultArgs, type AiSdkHarnessRunArgs, type AiSdkJudgeHarnessConfig, type AiSdkReplayMode, type AiSdkRuntime, type AiSdkRuntimeToolset, type AiSdkToolContext, type AiSdkToolDefinition, type AiSdkToolRecording, type AiSdkToolReplayConfig, type AiSdkToolReplayPolicies, type AiSdkToolReplayPolicy, type AiSdkToolset, aiSdkHarness, aiSdkJudgeHarness };

package/dist/index.js

@@ -20,26 +20,97 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
-  aiSdkHarness: () => aiSdkHarness
+  aiSdkHarness: () => aiSdkHarness,
+  aiSdkJudgeHarness: () => aiSdkJudgeHarness
 });
 module.exports = __toCommonJS(index_exports);
 var import_harness = require("vitest-evals/harness");
+var import_ai = require("ai");
+var import_judges = require("vitest-evals/judges");
 var import_replay = require("vitest-evals/replay");
+var nextTraceId = 0;
+function aiSdkJudgeHarness(config) {
+  return (0, import_judges.createJudgeHarness)({
+    name: config.name ?? "ai-sdk",
+    run: (input, options) => runAiSdkJudgeHarness(config, input, options)
+  });
+}
+async function runAiSdkJudgeHarness(config, input, options) {
+  const system = formatAiSdkJudgeSystemPrompt(
+    input.system,
+    input.responseFormat
+  );
+  const requestOptions = {
+    model: config.model,
+    prompt: input.prompt,
+    ...system !== void 0 ? { system } : {},
+    ...config.temperature !== void 0 ? { temperature: config.temperature } : {},
+    ...config.maxOutputTokens !== void 0 ? { maxOutputTokens: config.maxOutputTokens } : {},
+    ...config.providerOptions !== void 0 ? { providerOptions: config.providerOptions } : {},
+    ...options.signal ? { abortSignal: options.signal } : {}
+  };
+  if (input.responseFormat?.type === "json" && input.responseFormat.schema !== void 0) {
+    const { object } = await (0, import_ai.generateObject)({
+      ...requestOptions,
+      schema: (0, import_ai.jsonSchema)(
+        input.responseFormat.schema
+      )
+    });
+    return object;
+  }
+  const { text } = await (0, import_ai.generateText)(requestOptions);
+  return text;
+}
+function formatAiSdkJudgeSystemPrompt(system, responseFormat) {
+  if (responseFormat?.type !== "json" || responseFormat.schema !== void 0) {
+    return system;
+  }
+  const jsonInstruction = "Return only valid JSON. Do not include markdown fences or explanatory prose.";
+  return system ? `${system}
+
+${jsonInstruction}` : jsonInstruction;
+}
 function aiSdkHarness(options) {
   validateOptions(options);
+  const harnessName = options.name ?? "ai-sdk";
   const harness = {
-    name: options.name ?? "ai-sdk",
+    name: harnessName,
     run: async (input, context) => {
-      const agent = await resolveAgent(options, {
-        input,
-        context
-      });
-      return runAiSdkHarness(options, agent, input, context);
+      const startedAt = /* @__PURE__ */ new Date();
+      try {
+        const agent = await resolveAgent(options, {
+          input,
+          context
+        });
+        return await runAiSdkHarness(options, agent, input, context);
+      } catch (error) {
+        if ((0, import_harness.getHarnessRunFromError)(error)) {
+          throw error;
+        }
+        throw (0, import_harness.attachHarnessRunToError)(
+          error,
+          createFailedAiSdkRun(input, context, error, harnessName, startedAt)
+        );
+      }
     }
   };
   return harness;
 }
+function createFailedAiSdkRun(input, context, error, harnessName, startedAt) {
+  const run = (0, import_harness.createFailedHarnessRun)(input, error, {
+    artifacts: context.artifacts
+  });
+  (0, import_harness.ensureRunTrace)(run, {
+    name: harnessName,
+    startedAt,
+    finishedAt: /* @__PURE__ */ new Date(),
+    operationName: "invoke_workflow",
+    source: "harness-ai-sdk"
+  });
+  return run;
+}
 async function runAiSdkHarness(options, agent, input, context) {
+  const trace = createTraceRecorder(options.name ?? "ai-sdk");
   const replayMetadataByToolCallId = /* @__PURE__ */ new Map();
   const runtimeToolCalls = [];
   const tools = createToolset({
@@ -66,6 +137,7 @@ async function runAiSdkHarness(options, agent, input, context) {
       if (Object.keys(context.artifacts).length > 0 && !result.artifacts) {
         result.artifacts = context.artifacts;
       }
+      attachAiSdkTrace(result, trace, result, /* @__PURE__ */ new Date());
       return result;
     }
     const resultArgs = {
@@ -86,26 +158,48 @@ async function runAiSdkHarness(options, agent, input, context) {
       runtimeToolCalls
     );
     const errors = (0, import_harness.resolveHarnessRunErrors)(result);
+    const finishedAt = /* @__PURE__ */ new Date();
     return {
       session,
       output,
       usage,
       artifacts: Object.keys(context.artifacts).length > 0 ? context.artifacts : void 0,
-      errors
+      errors,
+      traces: [
+        finishAiSdkTrace(trace, {
+          result,
+          session,
+          usage,
+          errors,
+          finishedAt
+        })
+      ]
     };
   } catch (error) {
+    const finishedAt = /* @__PURE__ */ new Date();
+    const serializedError = (0, import_harness.serializeError)(error);
+    const usage = runtimeToolCalls.length > 0 ? { toolCalls: runtimeToolCalls.length } : {};
+    const session = resolveSession(
+      input,
+      void 0,
+      void 0,
+      replayMetadataByToolCallId,
+      runtimeToolCalls
+    );
     const run = {
-      session: resolveSession(
-        input,
-        void 0,
-        void 0,
-        replayMetadataByToolCallId,
-        runtimeToolCalls
-      ),
+      session,
       output: void 0,
-      usage: runtimeToolCalls.length > 0 ? { toolCalls: runtimeToolCalls.length } : {},
+      usage,
       artifacts: Object.keys(context.artifacts).length > 0 ? context.artifacts : void 0,
-      errors: [(0, import_harness.serializeError)(error)]
+      errors: [serializedError],
+      traces: [
+        finishAiSdkTrace(trace, {
+          session,
+          usage,
+          errors: [serializedError],
+          finishedAt
+        })
+      ]
     };
     throw (0, import_harness.attachHarnessRunToError)(error, run);
   }
@@ -165,6 +259,119 @@ function hasAiSdkGenerateMethod(agent) {
 function isAgentFactory(agent) {
   return typeof agent === "function" && !(0, import_harness.hasCallableMethod)(agent, "run") && !(0, import_harness.hasCallableMethod)(agent, "generate");
 }
+function createTraceRecorder(name) {
+  const id = `ai_sdk_trace_${++nextTraceId}`;
+  return {
+    id,
+    rootSpanId: `${id}:run`,
+    name,
+    startedAt: /* @__PURE__ */ new Date()
+  };
+}
+function attachAiSdkTrace(run, trace, result, finishedAt) {
+  const errors = run.errors ?? [];
+  const nativeTrace = finishAiSdkTrace(trace, {
+    result,
+    session: run.session,
+    usage: run.usage,
+    errors,
+    finishedAt
+  });
+  run.traces = [...run.traces ?? [], nativeTrace];
+}
+function finishAiSdkTrace(trace, options) {
+  const modelSpans = createAiSdkModelSpans(
+    trace,
+    options.result,
+    options.usage
+  );
+  const toolSpans = (0, import_harness.createToolCallSpans)((0, import_harness.toolCalls)(options.session), {
+    traceId: trace.id,
+    parentId: trace.rootSpanId,
+    spanIdPrefix: `${trace.id}:tool`
+  });
+  const finishedAt = options.finishedAt;
+  const durationMs = finishedAt.getTime() - trace.startedAt.getTime();
+  const rootError = options.errors?.[0] ? (0, import_harness.normalizeSpanError)(options.errors[0]) : void 0;
+  const rootSpan = {
+    id: trace.rootSpanId,
+    traceId: trace.id,
+    name: trace.name,
+    kind: "run",
+    startedAt: trace.startedAt.toISOString(),
+    finishedAt: finishedAt.toISOString(),
+    durationMs,
+    status: rootError ? "error" : "ok",
+    ...rootError ? { error: rootError } : {},
+    attributes: (0, import_harness.normalizeSpanAttributes)({
+      "gen_ai.operation.name": "invoke_workflow",
+      "gen_ai.workflow.name": trace.name,
+      ...(0, import_harness.createGenAiUsageAttributes)(options.usage)
+    })
+  };
+  const spans = [rootSpan, ...modelSpans, ...toolSpans];
+  return {
+    id: trace.id,
+    name: trace.name,
+    startedAt: trace.startedAt.toISOString(),
+    finishedAt: finishedAt.toISOString(),
+    durationMs,
+    metadata: {
+      source: "harness-ai-sdk"
+    },
+    spans
+  };
+}
+function createAiSdkModelSpans(trace, result, usage) {
+  const steps = resolveSteps(result);
+  if (steps.length === 0) {
+    const fallback = createUsageModelSpan(trace, usage);
+    return fallback ? [fallback] : [];
+  }
+  return steps.map((step, index) => {
+    const stepUsage = step.usage;
+    return {
+      id: `${trace.id}:model:${index + 1}`,
+      traceId: trace.id,
+      parentId: trace.rootSpanId,
+      name: `ai-sdk step ${step.stepNumber ?? index}`,
+      kind: "model",
+      status: "ok",
+      attributes: (0, import_harness.normalizeSpanAttributes)({
+        "gen_ai.operation.name": "chat",
+        "gen_ai.provider.name": step.model?.provider,
+        "gen_ai.request.model": step.model?.modelId,
+        "gen_ai.response.model": step.model?.modelId,
+        "gen_ai.response.finish_reasons": step.finishReason ? [String(step.finishReason)] : void 0,
+        "gen_ai.usage.input_tokens": stepUsage?.inputTokens,
+        "gen_ai.usage.output_tokens": stepUsage?.outputTokens,
+        "gen_ai.usage.reasoning.output_tokens": stepUsage?.outputTokenDetails?.reasoningTokens ?? stepUsage?.reasoningTokens,
+        "gen_ai.usage.cache_read.input_tokens": stepUsage?.inputTokenDetails?.cacheReadTokens ?? stepUsage?.cachedInputTokens,
+        "gen_ai.usage.cache_creation.input_tokens": stepUsage?.inputTokenDetails?.cacheWriteTokens,
+        "ai.step.number": step.stepNumber,
+        "ai.finish_reason": step.finishReason,
+        "ai.raw_finish_reason": step.rawFinishReason
+      })
+    };
+  });
+}
+function createUsageModelSpan(trace, usage) {
+  if (!usage?.provider && !usage?.model) {
+    return void 0;
+  }
+  return {
+    id: `${trace.id}:model:1`,
+    traceId: trace.id,
+    parentId: trace.rootSpanId,
+    name: usage.model ? `ai-sdk ${usage.model}` : "ai-sdk model",
+    kind: "model",
+    status: "ok",
+    attributes: (0, import_harness.normalizeSpanAttributes)({
+      "gen_ai.operation.name": "chat",
+      ...(0, import_harness.createGenAiUsageAttributes)(usage)
+    })
+  };
+}
 function createToolset({
   input,
   context,
@@ -614,6 +821,7 @@ function isAsyncIterable(value) {
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  aiSdkHarness
+  aiSdkHarness,
+  aiSdkJudgeHarness
 });
 //# sourceMappingURL=index.js.map