@vitest-evals/harness-pi-ai 0.9.0-beta.0

package/dist/index.mjs

@@ -0,0 +1,657 @@
+// src/index.ts
+import {
+  attachHarnessRunToError,
+  isHarnessRun,
+  isNormalizedSession,
+  normalizeContent,
+  resolveHarnessRunErrors,
+  serializeError,
+  toJsonValue
+} from "vitest-evals/harness";
+import {
+  executeWithReplay,
+  getReplayMetadataFromError,
+  normalizeReplayMetadata
+} from "vitest-evals/replay";
+var ORIGINAL_NATIVE_EXECUTE = Symbol("vitest-evals.originalNativeExecute");
+function piAiHarness(options) {
+  return {
+    name: options.name ?? "pi-ai",
+    run: async (input, context) => {
+      const agent = await resolveAgent(options);
+      const messages = [
+        {
+          role: "user",
+          content: normalizeContent(input)
+        }
+      ];
+      const inferredTools = resolveInferredToolSurfaces(
+        agent
+      );
+      if (hasExplicitToolset(options)) {
+        return executePiHarnessRun(
+          options,
+          agent,
+          input,
+          context,
+          messages,
+          options.tools,
+          inferredTools.nativeToolsets
+        );
+      }
+      return executePiHarnessRun(
+        options,
+        agent,
+        input,
+        context,
+        messages,
+        inferredTools.runtimeTools,
+        inferredTools.nativeToolsets
+      );
+    }
+  };
+}
+async function executePiHarnessRun(options, agent, input, context, messages, runtimeTools, nativeToolsets) {
+  const runtime = createRuntime({
+    input,
+    context,
+    tools: runtimeTools,
+    messages
+  });
+  try {
+    const result = await withInstrumentedAgentTools(
+      agent,
+      nativeToolsets,
+      {
+        input,
+        context,
+        messages,
+        toolCalls: runtime.toolCalls
+      },
+      () => runAgent(options, {
+        agent,
+        input,
+        context,
+        runtime
+      })
+    );
+    if (isHarnessRun(result) && !hasResultOverrides(options)) {
+      if (Object.keys(context.artifacts).length > 0 && !result.artifacts) {
+        result.artifacts = context.artifacts;
+      }
+      return result;
+    }
+    const normalizeResult = result;
+    const resultArgs = {
+      agent,
+      input,
+      context,
+      runtime,
+      result: normalizeResult
+    };
+    const output = options.normalize?.output ? await options.normalize.output(resultArgs) : resolveOutput(normalizeResult);
+    const usage = options.normalize?.usage ? await options.normalize.usage(resultArgs) : resolveUsage(normalizeResult, runtime.toolCalls.length);
+    const session = options.normalize?.session ? await options.normalize.session(resultArgs) : resolveSession(normalizeResult, messages, output, usage);
+    return {
+      session,
+      output,
+      usage,
+      timings: options.normalize?.timings ? await options.normalize.timings(resultArgs) : void 0,
+      artifacts: Object.keys(context.artifacts).length > 0 ? context.artifacts : void 0,
+      errors: options.normalize?.errors ? await options.normalize.errors(resultArgs) : resolveErrors(normalizeResult)
+    };
+  } catch (error) {
+    const usage = resolveUsage(void 0, runtime.toolCalls.length);
+    const run = {
+      session: resolveSession(void 0, messages, void 0, usage),
+      output: void 0,
+      usage,
+      artifacts: Object.keys(context.artifacts).length > 0 ? context.artifacts : void 0,
+      errors: [serializeError(error)]
+    };
+    throw attachHarnessRunToError(error, run);
+  }
+}
+async function resolveAgent(options) {
+  if (options.agent !== void 0) {
+    return options.agent;
+  }
+  if (options.createAgent) {
+    return options.createAgent();
+  }
+  throw new Error(
+    "piAiHarness requires either an agent instance or a createAgent() function."
+  );
+}
+function hasResultOverrides(options) {
+  return Boolean(
+    options.normalize?.output ?? options.normalize?.session ?? options.normalize?.usage ?? options.normalize?.timings ?? options.normalize?.errors
+  );
+}
+function resolveInferredToolSurfaces(agent) {
+  let runtimeTools;
+  const nativeToolsets = [];
+  const seenToolsets = /* @__PURE__ */ new Set();
+  for (const candidate of getAgentToolCandidates(agent)) {
+    const nextRuntimeTools = getRuntimeToolset(candidate);
+    if (runtimeTools === void 0 && nextRuntimeTools !== void 0) {
+      runtimeTools = nextRuntimeTools;
+    }
+    const nativeTools = getNativeToolArray(candidate);
+    if (nativeTools && !seenToolsets.has(nativeTools)) {
+      seenToolsets.add(nativeTools);
+      nativeToolsets.push(nativeTools);
+    }
+  }
+  return {
+    ...runtimeTools ? { runtimeTools } : {},
+    ...nativeToolsets.length > 0 ? { nativeToolsets } : {}
+  };
+}
+function getAgentToolCandidates(agent) {
+  const roots = getAgentRoots(agent);
+  const candidates = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const root of roots) {
+    addUniqueObject(candidates, seen, root);
+    addUniqueObject(candidates, seen, getObjectProperty(root, "state"));
+    addUniqueObject(candidates, seen, getObjectProperty(root, "initialState"));
+  }
+  return candidates;
+}
+function getAgentRoots(agent) {
+  return [asObject(agent)].concat(asObject(getObjectProperty(agent, "agent"))).filter((value) => value !== void 0);
+}
+function addUniqueObject(candidates, seen, value) {
+  if (!value || typeof value !== "object" || seen.has(value)) {
+    return;
+  }
+  seen.add(value);
+  candidates.push(value);
+}
+function asObject(value) {
+  return value && typeof value === "object" ? value : void 0;
+}
+function getObjectProperty(value, key) {
+  return value && typeof value === "object" ? value[key] : void 0;
+}
+function getRuntimeToolset(value) {
+  const candidate = getObjectProperty(value, "tools") ?? getObjectProperty(value, "toolset");
+  return isPiAiToolset(candidate) ? candidate : void 0;
+}
+function getNativeToolArray(value) {
+  const candidate = getObjectProperty(value, "tools");
+  if (isAgentToolArray(candidate)) {
+    return candidate;
+  }
+  return void 0;
+}
+async function runAgent(options, args) {
+  if (options.run) {
+    return options.run(args);
+  }
+  if (hasPiAiRunMethod(args.agent)) {
+    return args.agent.run(args.input, args.runtime);
+  }
+  throw new Error(
+    "piAiHarness requires a run() function unless the provided agent exposes run(input, runtime)."
+  );
+}
+function hasExplicitToolset(options) {
+  return options.tools !== void 0;
+}
+function hasPiAiRunMethod(agent) {
+  if (!agent || typeof agent !== "object") {
+    return false;
+  }
+  return "run" in agent && typeof agent.run === "function";
+}
+function isPiAiToolset(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return false;
+  }
+  const tools = Object.values(value);
+  return tools.length > 0 && tools.every(
+    (tool) => Boolean(
+      tool && typeof tool === "object" && "execute" in tool && typeof tool.execute === "function"
+    )
+  );
+}
+function isAgentToolArray(value) {
+  return Array.isArray(value) && value.every(
+    (tool) => Boolean(
+      tool && typeof tool === "object" && "name" in tool && typeof tool.name === "string" && "execute" in tool && typeof tool.execute === "function"
+    )
+  );
+}
+async function withInstrumentedAgentTools(agent, toolsets, args, callback) {
+  if (!toolsets || toolsets.length === 0) {
+    return callback();
+  }
+  const originalExecutions = /* @__PURE__ */ new Map();
+  const originalResets = /* @__PURE__ */ new Map();
+  const patchTool = (tool) => {
+    if (originalExecutions.has(tool)) {
+      return;
+    }
+    const originalExecute = getNativeToolExecuteOrigin(tool.execute);
+    originalExecutions.set(tool, originalExecute);
+    const instrumentedExecute = async (toolCallId, rawArgs) => {
+      const startedAt = /* @__PURE__ */ new Date();
+      const toolContext = {
+        input: args.input,
+        metadata: args.context.metadata,
+        signal: args.context.signal,
+        setArtifact: args.context.setArtifact
+      };
+      try {
+        const execution = await executeNativeToolWithReplay({
+          toolName: tool.name,
+          toolCallId,
+          execute: originalExecute,
+          replay: tool.replay,
+          args: rawArgs,
+          context: toolContext
+        });
+        const finishedAt = /* @__PURE__ */ new Date();
+        const call = {
+          name: tool.name,
+          arguments: rawArgs,
+          result: execution.normalizedResult,
+          startedAt: startedAt.toISOString(),
+          finishedAt: finishedAt.toISOString(),
+          durationMs: finishedAt.getTime() - startedAt.getTime(),
+          metadata: normalizeReplayMetadata(execution.replay)
+        };
+        args.toolCalls.push(call);
+        args.messages.push({
+          role: "assistant",
+          toolCalls: [call]
+        });
+        args.messages.push({
+          role: "tool",
+          content: execution.normalizedResult,
+          metadata: {
+            name: tool.name
+          }
+        });
+        return execution.result;
+      } catch (error) {
+        const finishedAt = /* @__PURE__ */ new Date();
+        const call = {
+          name: tool.name,
+          arguments: rawArgs,
+          error: serializeToolCallError(error),
+          startedAt: startedAt.toISOString(),
+          finishedAt: finishedAt.toISOString(),
+          durationMs: finishedAt.getTime() - startedAt.getTime(),
+          metadata: normalizeReplayMetadata(getReplayMetadataFromError(error))
+        };
+        args.toolCalls.push(call);
+        args.messages.push({
+          role: "assistant",
+          toolCalls: [call]
+        });
+        throw error;
+      }
+    };
+    instrumentedExecute[ORIGINAL_NATIVE_EXECUTE] = originalExecute;
+    tool.execute = instrumentedExecute;
+  };
+  const patchToolsets = (nextToolsets) => {
+    for (const toolset of nextToolsets) {
+      for (const tool of toolset) {
+        patchTool(tool);
+      }
+    }
+  };
+  patchToolsets(toolsets);
+  for (const target of getAgentResetTargets(agent)) {
+    const originalReset = target.reset;
+    originalResets.set(target, originalReset);
+    target.reset = function patchedReset(...resetArgs) {
+      const resetResult = originalReset.apply(this, resetArgs);
+      if (isPromiseLike(resetResult)) {
+        return resetResult.finally(() => {
+          patchToolsets(
+            resolveInferredNativeToolsets(agent)
+          );
+        });
+      }
+      patchToolsets(resolveInferredNativeToolsets(agent));
+      return resetResult;
+    };
+  }
+  try {
+    return await callback();
+  } finally {
+    for (const [target, originalReset] of originalResets) {
+      target.reset = originalReset;
+    }
+    for (const [tool, originalExecute] of originalExecutions) {
+      tool.execute = originalExecute;
+    }
+  }
+}
+function getAgentResetTargets(agent) {
+  return getAgentRoots(agent).filter(isResettableAgent);
+}
+function isResettableAgent(value) {
+  return "reset" in value && typeof value.reset === "function";
+}
+function resolveInferredNativeToolsets(agent) {
+  const toolsets = [];
+  const seenToolsets = /* @__PURE__ */ new Set();
+  for (const candidate of getAgentToolCandidates(agent)) {
+    const nativeTools = getNativeToolArray(candidate);
+    if (nativeTools && !seenToolsets.has(nativeTools)) {
+      seenToolsets.add(nativeTools);
+      toolsets.push(nativeTools);
+    }
+  }
+  return toolsets;
+}
+function isPromiseLike(value) {
+  return Boolean(
+    value && typeof value.then === "function"
+  );
+}
+function serializeToolCallError(error) {
+  const serialized = serializeError(error);
+  const { message, type, ...details } = serialized;
+  return {
+    ...details,
+    message: typeof message === "string" ? message : String(message),
+    ...typeof type === "string" ? { type } : {}
+  };
+}
+function getNativeToolExecuteOrigin(execute) {
+  const nativeExecute = execute;
+  return nativeExecute[ORIGINAL_NATIVE_EXECUTE] ?? nativeExecute;
+}
+async function executeNativeToolWithReplay({
+  toolName,
+  toolCallId,
+  execute,
+  replay,
+  args,
+  context
+}) {
+  let didExecute = false;
+  let liveResult;
+  const execution = await executeWithReplay({
+    toolName,
+    args,
+    context,
+    execute: async (toolArgs) => {
+      didExecute = true;
+      liveResult = await execute(toolCallId, toolArgs);
+      return createNativeToolReplayEnvelope(liveResult);
+    },
+    replay
+  });
+  if (didExecute) {
+    return {
+      result: liveResult,
+      normalizedResult: normalizeReplayToolResult(liveResult),
+      replay: execution.replay
+    };
+  }
+  return {
+    ...resolveNativeToolReplayResult(execution.result),
+    replay: execution.replay
+  };
+}
+function createRuntime({
+  input,
+  context,
+  tools,
+  messages
+}) {
+  const toolCalls = [];
+  const eventSink = {
+    message: (message) => {
+      messages.push(message);
+    },
+    system: (content, metadata) => {
+      messages.push({
+        role: "system",
+        content,
+        metadata
+      });
+    },
+    user: (content, metadata) => {
+      messages.push({
+        role: "user",
+        content,
+        metadata
+      });
+    },
+    assistant: (content, metadata) => {
+      messages.push({
+        role: "assistant",
+        content,
+        metadata
+      });
+    },
+    tool: (name, content, metadata) => {
+      messages.push({
+        role: "tool",
+        content,
+        metadata: {
+          name,
+          ...metadata ?? {}
+        }
+      });
+    }
+  };
+  const runtimeTools = Object.fromEntries(
+    Object.entries(tools ?? {}).map(([toolName, tool]) => [
+      toolName,
+      async (args) => {
+        const startedAt = /* @__PURE__ */ new Date();
+        const toolContext = {
+          input,
+          metadata: context.metadata,
+          signal: context.signal,
+          setArtifact: context.setArtifact
+        };
+        try {
+          const execution = await executeToolWithReplay({
+            toolName,
+            tool,
+            args,
+            context: toolContext
+          });
+          const finishedAt = /* @__PURE__ */ new Date();
+          const call = {
+            name: toolName,
+            arguments: args,
+            result: execution.result,
+            startedAt: startedAt.toISOString(),
+            finishedAt: finishedAt.toISOString(),
+            durationMs: finishedAt.getTime() - startedAt.getTime(),
+            metadata: normalizeReplayMetadata(execution.replay)
+          };
+          toolCalls.push(call);
+          messages.push({
+            role: "assistant",
+            toolCalls: [call]
+          });
+          messages.push({
+            role: "tool",
+            content: execution.result,
+            metadata: {
+              name: toolName
+            }
+          });
+          return execution.result;
+        } catch (error) {
+          const finishedAt = /* @__PURE__ */ new Date();
+          const call = {
+            name: toolName,
+            arguments: args,
+            error: serializeToolCallError(error),
+            startedAt: startedAt.toISOString(),
+            finishedAt: finishedAt.toISOString(),
+            durationMs: finishedAt.getTime() - startedAt.getTime(),
+            metadata: normalizeReplayMetadata(
+              getReplayMetadataFromError(error)
+            )
+          };
+          toolCalls.push(call);
+          messages.push({
+            role: "assistant",
+            toolCalls: [call]
+          });
+          throw error;
+        }
+      }
+    ])
+  );
+  return {
+    tools: runtimeTools,
+    events: eventSink,
+    signal: context.signal,
+    toolCalls
+  };
+}
+function resolveOutput(result) {
+  if (!result || typeof result !== "object") {
+    return toJsonValue(result);
+  }
+  const candidates = [
+    "output",
+    "decision",
+    "result",
+    "final"
+  ];
+  for (const key of candidates) {
+    const value = result[key];
+    const normalized = toJsonValue(value);
+    if (normalized !== void 0) {
+      return normalized;
+    }
+  }
+  return void 0;
+}
+function normalizeToolResult(result) {
+  const details = result && typeof result === "object" ? toJsonValue(result.details) : void 0;
+  if (details !== void 0) {
+    return details;
+  }
+  return toJsonValue(result) ?? (result === void 0 ? void 0 : String(result));
+}
+function normalizeReplayToolResult(result) {
+  return normalizeToolResult(result) ?? null;
+}
+function createNativeToolReplayEnvelope(result) {
+  const normalizedResult = normalizeReplayToolResult(result);
+  return {
+    __vitestEvals: {
+      kind: "pi-ai-native-tool-result",
+      version: 2
+    },
+    agentResult: toJsonValue(result) ?? normalizedResult,
+    normalizedResult
+  };
+}
+function resolveNativeToolReplayResult(result) {
+  if (isNativeToolReplayEnvelope(result)) {
+    return {
+      result: result.agentResult,
+      normalizedResult: result.normalizedResult
+    };
+  }
+  if (isLegacyNativeToolReplayEnvelope(result)) {
+    return {
+      result: result.agentResult ?? result.normalizedResult,
+      normalizedResult: result.normalizedResult
+    };
+  }
+  return {
+    result,
+    normalizedResult: normalizeReplayToolResult(result)
+  };
+}
+function isNativeToolReplayEnvelope(value) {
+  return Boolean(
+    value && typeof value === "object" && "__vitestEvals" in value && isNativeToolReplayMarker(
+      value.__vitestEvals
+    ) && "agentResult" in value && "normalizedResult" in value
+  );
+}
+function isNativeToolReplayMarker(value) {
+  return Boolean(
+    value && typeof value === "object" && "kind" in value && value.kind === "pi-ai-native-tool-result" && "version" in value && value.version === 2
+  );
+}
+function isLegacyNativeToolReplayEnvelope(value) {
+  return Boolean(
+    value && typeof value === "object" && "__vitestEvals" in value && isLegacyNativeToolReplayMarker(
+      value.__vitestEvals
+    ) && "normalizedResult" in value
+  );
+}
+function isLegacyNativeToolReplayMarker(value) {
+  return Boolean(
+    value && typeof value === "object" && "kind" in value && value.kind === "pi-ai-native-tool-result" && "version" in value && value.version === 1
+  );
+}
+function resolveUsage(result, toolCallCount) {
+  if (!result || typeof result !== "object") {
+    return toolCallCount > 0 ? { toolCalls: toolCallCount } : {};
+  }
+  const usageValue = result.usage ?? result.metrics;
+  const usage = usageValue && typeof usageValue === "object" ? { ...usageValue } : {};
+  if (usage.toolCalls === void 0 && toolCallCount > 0) {
+    usage.toolCalls = toolCallCount;
+  }
+  return usage;
+}
+function resolveSession(result, messages, output, usage) {
+  if (isNormalizedSession(
+    result?.session
+  )) {
+    return result.session;
+  }
+  if (isNormalizedSession(result?.trace)) {
+    return result.trace;
+  }
+  const sessionMessages = [...messages];
+  if (output !== void 0 && !sessionMessages.some(
+    (message) => message.role === "assistant" && message.content !== void 0
+  )) {
+    sessionMessages.push({
+      role: "assistant",
+      content: output
+    });
+  }
+  return {
+    messages: sessionMessages,
+    outputText: typeof output === "string" ? output : void 0,
+    provider: result?.provider ?? usage.provider,
+    model: result?.model ?? usage.model
+  };
+}
+function resolveErrors(result) {
+  return resolveHarnessRunErrors(result);
+}
+async function executeToolWithReplay({
+  toolName,
+  tool,
+  args,
+  context
+}) {
+  return executeWithReplay({
+    toolName,
+    args,
+    context,
+    execute: tool.execute,
+    replay: tool.replay
+  });
+}
+export {
+  piAiHarness
+};
+//# sourceMappingURL=index.mjs.map