@vitest-evals/harness-ai-sdk 0.9.0-beta.0

package/dist/index.js

@@ -0,0 +1,598 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  aiSdkHarness: () => aiSdkHarness
+});
+module.exports = __toCommonJS(index_exports);
+var import_harness = require("vitest-evals/harness");
+var import_replay = require("vitest-evals/replay");
+function aiSdkHarness(options) {
+  validateOptions(options);
+  return {
+    name: options.name ?? "ai-sdk",
+    prompt: options.prompt,
+    run: async (input, context) => {
+      const agent = await resolveAgent(options);
+      return runAiSdkHarness(options, agent, input, context);
+    }
+  };
+}
+async function runAiSdkHarness(options, agent, input, context) {
+  const replayMetadataByToolCallId = /* @__PURE__ */ new Map();
+  const runtimeToolCalls = [];
+  const tools = createToolset({
+    input,
+    context,
+    tools: options.tools,
+    replayMetadataByToolCallId,
+    runtimeToolCalls
+  });
+  const runtime = {
+    tools,
+    signal: context.signal
+  };
+  try {
+    const result = await runAgent(options, {
+      agent,
+      input,
+      context,
+      runtime,
+      tools
+    });
+    if ((0, import_harness.isHarnessRun)(result) && !hasResultOverrides(options)) {
+      if (Object.keys(context.artifacts).length > 0 && !result.artifacts) {
+        result.artifacts = context.artifacts;
+      }
+      return result;
+    }
+    const resultArgs = {
+      agent,
+      input,
+      context,
+      runtime,
+      tools,
+      result
+    };
+    const output = options.output ? await options.output(resultArgs) : resolveOutput(result);
+    const usage = options.usage ? await options.usage(resultArgs) : resolveUsage(result, runtimeToolCalls.length);
+    const session = options.session ? await options.session(resultArgs) : resolveSession(
+      input,
+      result,
+      output,
+      replayMetadataByToolCallId,
+      runtimeToolCalls
+    );
+    return {
+      session,
+      output,
+      usage,
+      timings: options.timings ? await options.timings(resultArgs) : void 0,
+      artifacts: Object.keys(context.artifacts).length > 0 ? context.artifacts : void 0,
+      errors: options.errors ? await options.errors(resultArgs) : (0, import_harness.resolveHarnessRunErrors)(result)
+    };
+  } catch (error) {
+    const run = {
+      session: resolveSession(
+        input,
+        void 0,
+        void 0,
+        replayMetadataByToolCallId,
+        runtimeToolCalls
+      ),
+      output: void 0,
+      usage: runtimeToolCalls.length > 0 ? { toolCalls: runtimeToolCalls.length } : {},
+      artifacts: Object.keys(context.artifacts).length > 0 ? context.artifacts : void 0,
+      errors: [(0, import_harness.serializeError)(error)]
+    };
+    throw (0, import_harness.attachHarnessRunToError)(error, run);
+  }
+}
+function hasResultOverrides(options) {
+  return Boolean(
+    options.output ?? options.session ?? options.usage ?? options.timings ?? options.errors
+  );
+}
+async function resolveAgent(options) {
+  return hasAgentSource(options) ? await resolveAgentSource(options.agent) : void 0;
+}
+async function runAgent(options, args) {
+  if (options.task) {
+    return options.task(args);
+  }
+  if (hasAiSdkRunMethod(args.agent)) {
+    return args.agent.run(args.input, args.runtime);
+  }
+  if (hasAiSdkGenerateMethod(args.agent)) {
+    return args.agent.generate(args.input, args.runtime);
+  }
+  throw new Error(
+    "aiSdkHarness agent must expose run(input, runtime) or generate(input, runtime), or use task() for a custom entrypoint."
+  );
+}
+function validateOptions(options) {
+  const hasAgent = hasAgentSource(options);
+  const hasTask = typeof options.task === "function";
+  if (hasAgent && hasTask) {
+    throw new Error(
+      "aiSdkHarness accepts either agent or task, not both. Use agent for the zero-glue run(input, runtime) or generate(input, runtime) path, or task for a custom entrypoint."
+    );
+  }
+  if (!hasAgent && !hasTask) {
+    throw new Error(
+      "aiSdkHarness requires either agent or task. Use agent for objects with run(input, runtime) or generate(input, runtime), or task for a custom entrypoint."
+    );
+  }
+}
+function hasAgentSource(options) {
+  return "agent" in options && options.agent !== void 0;
+}
+async function resolveAgentSource(agent) {
+  if (isAgentFactory(agent)) {
+    return agent();
+  }
+  return agent;
+}
+function hasAiSdkRunMethod(agent) {
+  return (0, import_harness.hasCallableMethod)(agent, "run");
+}
+function hasAiSdkGenerateMethod(agent) {
+  return (0, import_harness.hasCallableMethod)(agent, "generate");
+}
+function isAgentFactory(agent) {
+  return typeof agent === "function" && !(0, import_harness.hasCallableMethod)(agent, "run") && !(0, import_harness.hasCallableMethod)(agent, "generate");
+}
+function createToolset({
+  input,
+  context,
+  tools,
+  replayMetadataByToolCallId,
+  runtimeToolCalls
+}) {
+  return Object.fromEntries(
+    Object.entries(tools ?? {}).map(([toolName, tool]) => {
+      if (tool.replay && !tool.execute) {
+        throw new Error(
+          `Tool replay requires execute() for ${toolName}. Provider-executed tools cannot be recorded automatically.`
+        );
+      }
+      if (!tool.execute) {
+        return [toolName, tool];
+      }
+      const execute = tool.execute;
+      const wrappedTool = {
+        ...tool,
+        execute: async (toolInput, execution) => {
+          const startedAt = /* @__PURE__ */ new Date();
+          const normalizedArgs = normalizeArguments(toolInput);
+          const replayContext = {
+            input,
+            metadata: context.metadata,
+            signal: context.signal,
+            setArtifact: context.setArtifact,
+            execution
+          };
+          try {
+            const executionResult = tool.replay ? await executeToolWithReplay({
+              toolName,
+              toolInput,
+              execute,
+              execution,
+              context: replayContext,
+              replay: tool.replay
+            }) : {
+              result: await execute(toolInput, execution),
+              replay: void 0
+            };
+            const finishedAt = /* @__PURE__ */ new Date();
+            const normalizedResult = (0, import_harness.toJsonValue)(executionResult.result);
+            const replayMetadata = (0, import_replay.normalizeReplayMetadata)(
+              executionResult.replay
+            );
+            if (executionResult.replay) {
+              replayMetadataByToolCallId.set(
+                execution.toolCallId,
+                executionResult.replay
+              );
+            }
+            runtimeToolCalls.push({
+              id: execution.toolCallId,
+              name: toolName,
+              ...normalizedArgs ? { arguments: normalizedArgs } : {},
+              ...normalizedResult !== void 0 ? { result: normalizedResult } : {},
+              startedAt: startedAt.toISOString(),
+              finishedAt: finishedAt.toISOString(),
+              durationMs: finishedAt.getTime() - startedAt.getTime(),
+              ...replayMetadata ? { metadata: replayMetadata } : {}
+            });
+            return executionResult.result;
+          } catch (error) {
+            const replay = (0, import_replay.getReplayMetadataFromError)(error);
+            const finishedAt = /* @__PURE__ */ new Date();
+            const replayMetadata = (0, import_replay.normalizeReplayMetadata)(replay);
+            if (replay) {
+              replayMetadataByToolCallId.set(execution.toolCallId, replay);
+            }
+            runtimeToolCalls.push({
+              id: execution.toolCallId,
+              name: toolName,
+              ...normalizedArgs ? { arguments: normalizedArgs } : {},
+              error: normalizeError(error),
+              startedAt: startedAt.toISOString(),
+              finishedAt: finishedAt.toISOString(),
+              durationMs: finishedAt.getTime() - startedAt.getTime(),
+              ...replayMetadata ? { metadata: replayMetadata } : {}
+            });
+            throw error;
+          }
+        }
+      };
+      return [toolName, wrappedTool];
+    })
+  );
+}
+async function executeToolWithReplay({
+  toolName,
+  toolInput,
+  execute,
+  execution,
+  context,
+  replay
+}) {
+  const replayInput = toReplayJsonValue(
+    toolInput,
+    `${toolName} tool input`
+  );
+  return (0, import_replay.executeWithReplay)({
+    toolName,
+    args: replayInput,
+    context,
+    execute: async (replayedInput) => {
+      const output = await execute(
+        replayedInput,
+        execution
+      );
+      if (isAsyncIterable(output)) {
+        throw new Error(
+          `Tool replay only supports JSON-serializable outputs. ${toolName} returned an async iterable.`
+        );
+      }
+      return toReplayJsonValue(
+        output,
+        `${toolName} tool output`
+      );
+    },
+    replay
+  });
+}
+function resolveOutput(result) {
+  if (!result || typeof result !== "object") {
+    return (0, import_harness.toJsonValue)(result);
+  }
+  const candidates = [
+    "output",
+    "object",
+    "experimental_output",
+    "result",
+    "text"
+  ];
+  for (const key of candidates) {
+    const value = result[key];
+    const normalized = (0, import_harness.toJsonValue)(value);
+    if (normalized !== void 0) {
+      return normalized;
+    }
+  }
+  return void 0;
+}
+function resolveUsage(result, runtimeToolCallCount = 0) {
+  const steps = resolveSteps(result);
+  const usage = resolveLanguageModelUsage(result) ?? resolveStepUsage(steps);
+  const lastStep = steps.length > 0 ? steps[steps.length - 1] : void 0;
+  if (!usage) {
+    if (steps.length > 0) {
+      const toolCallCount2 = countStepToolCalls(steps);
+      return {
+        provider: lastStep?.model.provider,
+        model: lastStep?.model.modelId,
+        ...toolCallCount2 > 0 ? { toolCalls: toolCallCount2 } : {}
+      };
+    }
+    return runtimeToolCallCount > 0 ? { toolCalls: runtimeToolCallCount } : {};
+  }
+  const stepToolCallCount = countStepToolCalls(steps);
+  const toolCallCount = stepToolCallCount > 0 ? stepToolCallCount : runtimeToolCallCount;
+  return {
+    provider: lastStep?.model.provider,
+    model: lastStep?.model.modelId,
+    inputTokens: usage.inputTokens,
+    outputTokens: usage.outputTokens,
+    reasoningTokens: usage.outputTokenDetails?.reasoningTokens ?? usage.reasoningTokens,
+    totalTokens: usage.totalTokens ?? (usage.inputTokens ?? 0) + (usage.outputTokens ?? 0),
+    toolCalls: toolCallCount > 0 ? toolCallCount : void 0,
+    metadata: (0, import_harness.normalizeMetadata)({
+      cacheReadTokens: usage.inputTokenDetails?.cacheReadTokens ?? usage.cachedInputTokens,
+      cacheWriteTokens: usage.inputTokenDetails?.cacheWriteTokens,
+      raw: usage.raw
+    })
+  };
+}
+function resolveStepUsage(steps) {
+  const usages = steps.map((step) => step.usage).filter((usage) => Boolean(usage));
+  if (usages.length === 0) {
+    return void 0;
+  }
+  return usages.reduce(addLanguageModelUsage);
+}
+function addLanguageModelUsage(left, right) {
+  return {
+    inputTokens: addTokenCounts(left.inputTokens, right.inputTokens),
+    inputTokenDetails: {
+      noCacheTokens: addTokenCounts(
+        left.inputTokenDetails?.noCacheTokens,
+        right.inputTokenDetails?.noCacheTokens
+      ),
+      cacheReadTokens: addTokenCounts(
+        left.inputTokenDetails?.cacheReadTokens,
+        right.inputTokenDetails?.cacheReadTokens
+      ),
+      cacheWriteTokens: addTokenCounts(
+        left.inputTokenDetails?.cacheWriteTokens,
+        right.inputTokenDetails?.cacheWriteTokens
+      )
+    },
+    outputTokens: addTokenCounts(left.outputTokens, right.outputTokens),
+    outputTokenDetails: {
+      textTokens: addTokenCounts(
+        left.outputTokenDetails?.textTokens,
+        right.outputTokenDetails?.textTokens
+      ),
+      reasoningTokens: addTokenCounts(
+        left.outputTokenDetails?.reasoningTokens,
+        right.outputTokenDetails?.reasoningTokens
+      )
+    },
+    totalTokens: addTokenCounts(left.totalTokens, right.totalTokens),
+    reasoningTokens: addTokenCounts(
+      left.reasoningTokens,
+      right.reasoningTokens
+    ),
+    cachedInputTokens: addTokenCounts(
+      left.cachedInputTokens,
+      right.cachedInputTokens
+    )
+  };
+}
+function addTokenCounts(left, right) {
+  return left == null && right == null ? void 0 : (left ?? 0) + (right ?? 0);
+}
+function countStepToolCalls(steps) {
+  return steps.reduce(
+    (count, step) => count + (step.toolCalls?.length ?? 0),
+    0
+  );
+}
+function resolveSession(input, result, output, replayMetadataByToolCallId, runtimeToolCalls = []) {
+  if ((0, import_harness.isNormalizedSession)(
+    result?.session
+  )) {
+    return result.session;
+  }
+  if ((0, import_harness.isNormalizedSession)(result?.trace)) {
+    return result.trace;
+  }
+  const steps = resolveSteps(result);
+  const messages = [
+    {
+      role: "user",
+      content: (0, import_harness.normalizeContent)(input)
+    }
+  ];
+  const stepToolCallIds = /* @__PURE__ */ new Set();
+  for (const step of steps) {
+    for (const toolCall of step.toolCalls ?? []) {
+      stepToolCallIds.add(toolCall.toolCallId);
+    }
+    messages.push(...normalizeStep(step, replayMetadataByToolCallId));
+  }
+  const unmatchedRuntimeToolCalls = runtimeToolCalls.filter(
+    (call) => call.id === void 0 || !stepToolCallIds.has(call.id)
+  );
+  if (unmatchedRuntimeToolCalls.length > 0) {
+    messages.push(...normalizeRuntimeToolCalls(unmatchedRuntimeToolCalls));
+  }
+  if (output !== void 0 && !messages.some(
+    (message) => message.role === "assistant" && message.content !== void 0
+  )) {
+    messages.push({
+      role: "assistant",
+      content: output
+    });
+  }
+  const lastStep = steps.length > 0 ? steps[steps.length - 1] : void 0;
+  const outputText = resolveOutputText(result, output, lastStep);
+  return {
+    messages,
+    outputText,
+    provider: lastStep?.model.provider,
+    model: lastStep?.model.modelId
+  };
+}
+function normalizeRuntimeToolCalls(runtimeToolCalls) {
+  const messages = [
+    {
+      role: "assistant",
+      toolCalls: runtimeToolCalls
+    }
+  ];
+  for (const call of runtimeToolCalls) {
+    if (call.result === void 0 && !call.error) {
+      continue;
+    }
+    const content = call.result !== void 0 ? call.result : call.error && call.error.message.length > 0 ? call.error.message : void 0;
+    messages.push({
+      role: "tool",
+      ...content !== void 0 ? { content } : {},
+      metadata: (0, import_harness.normalizeMetadata)({
+        name: call.name,
+        toolCallId: call.id,
+        isError: Boolean(call.error)
+      })
+    });
+  }
+  return messages;
+}
+function resolveSteps(result) {
+  if (!result || typeof result !== "object") {
+    return [];
+  }
+  return Array.isArray(result.steps) ? result.steps ?? [] : [];
+}
+function resolveLanguageModelUsage(result) {
+  if (!result || typeof result !== "object") {
+    return void 0;
+  }
+  const aiResult = result;
+  return aiResult.totalUsage ?? aiResult.usage;
+}
+function resolveOutputText(result, output, lastStep) {
+  if (lastStep?.text) {
+    return lastStep.text;
+  }
+  if (result && typeof result === "object" && typeof result.text === "string") {
+    return result.text;
+  }
+  return typeof output === "string" ? output : void 0;
+}
+function normalizeStep(step, replayMetadataByToolCallId) {
+  const toolResultsById = new Map(
+    (step.toolResults ?? []).map((toolResult) => [
+      toolResult.toolCallId,
+      toolResult
+    ])
+  );
+  const normalizedCalls = (step.toolCalls ?? []).map(
+    (toolCall) => normalizeToolCall(toolCall, toolResultsById, replayMetadataByToolCallId)
+  );
+  const normalizedCallsById = new Map(
+    normalizedCalls.map((toolCall) => [toolCall.id, toolCall])
+  );
+  const assistantMetadata = (0, import_harness.normalizeMetadata)({
+    stepNumber: step.stepNumber,
+    finishReason: step.finishReason,
+    rawFinishReason: step.rawFinishReason,
+    reasoningText: step.reasoningText,
+    response: step.response
+  });
+  const messages = [];
+  if (step.text || normalizedCalls.length > 0 || assistantMetadata) {
+    messages.push({
+      role: "assistant",
+      ...step.text ? { content: step.text } : {},
+      ...normalizedCalls.length > 0 ? { toolCalls: normalizedCalls } : {},
+      ...assistantMetadata ? { metadata: assistantMetadata } : {}
+    });
+  }
+  for (const toolResult of step.toolResults ?? []) {
+    const content = toolResult.output === void 0 ? void 0 : (0, import_harness.normalizeContent)(toolResult.output);
+    messages.push({
+      role: "tool",
+      ...content !== void 0 ? { content } : {},
+      metadata: (0, import_harness.normalizeMetadata)({
+        name: toolResult.toolName,
+        toolCallId: toolResult.toolCallId,
+        isError: Boolean(normalizedCallsById.get(toolResult.toolCallId)?.error),
+        preliminary: toolResult.preliminary,
+        providerExecuted: toolResult.providerExecuted,
+        title: toolResult.title,
+        providerMetadata: toolResult.providerMetadata
+      })
+    });
+  }
+  return messages;
+}
+function normalizeToolCall(toolCall, toolResultsById, replayMetadataByToolCallId) {
+  const toolResult = toolResultsById.get(toolCall.toolCallId);
+  const normalizedArguments = normalizeArguments(toolCall.input);
+  const normalizedResult = toolResult !== void 0 ? (0, import_harness.toJsonValue)(toolResult.output) : void 0;
+  const errorValue = toolCall.invalid || toolCall.error !== void 0 ? normalizeError(toolCall.error ?? toolCall.invalid) : void 0;
+  const replayMetadata = (0, import_replay.normalizeReplayMetadata)(
+    replayMetadataByToolCallId.get(toolCall.toolCallId)
+  );
+  return {
+    id: toolCall.toolCallId,
+    name: toolCall.toolName,
+    ...normalizedArguments ? { arguments: normalizedArguments } : {},
+    ...toolResult && normalizedResult !== void 0 ? { result: normalizedResult } : {},
+    ...errorValue ? { error: errorValue } : {},
+    metadata: (0, import_harness.normalizeMetadata)({
+      providerExecuted: toolCall.providerExecuted ?? toolResult?.providerExecuted,
+      title: toolCall.title ?? toolResult?.title,
+      dynamic: toolCall.dynamic,
+      invalid: toolCall.invalid,
+      preliminary: toolResult?.preliminary,
+      providerMetadata: toolCall.providerMetadata ?? toolResult?.providerMetadata,
+      ...replayMetadata ?? {}
+    })
+  };
+}
+function normalizeArguments(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return void 0;
+  }
+  return (0, import_harness.normalizeRecord)(value);
+}
+function normalizeError(error) {
+  if (error instanceof Error) {
+    return {
+      type: error.name,
+      message: error.message
+    };
+  }
+  const normalized = (0, import_harness.toJsonValue)(error);
+  if (normalized && typeof normalized === "object" && !Array.isArray(normalized) && typeof normalized.message === "string") {
+    return {
+      message: normalized.message,
+      type: typeof normalized.type === "string" ? normalized.type : "Error"
+    };
+  }
+  return {
+    type: "Error",
+    message: String(error ?? "Unknown tool call error")
+  };
+}
+function toReplayJsonValue(value, label) {
+  const normalized = (0, import_harness.toJsonValue)(value);
+  if (normalized === void 0) {
+    throw new Error(
+      `Tool replay only supports JSON-serializable values. ${label} could not be normalized.`
+    );
+  }
+  return normalized;
+}
+function isAsyncIterable(value) {
+  return value !== null && (typeof value === "object" || typeof value === "function") && Symbol.asyncIterator in value;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  aiSdkHarness
+});
+//# sourceMappingURL=index.js.map