@orq-ai/evaluatorq 1.2.0-rc.2 → 1.2.0-rc.4

package/README.md

@@ -10,6 +10,7 @@ An evaluation framework library that provides a flexible way to run parallel eva
 - **Orq Platform Integration**: Seamlessly fetch and evaluate datasets from Orq AI (optional)
 - **OpenTelemetry Tracing**: Built-in observability with automatic span creation for jobs and evaluators
 - **Pass/Fail Tracking**: Evaluators can return pass/fail status for CI/CD integration
+- **Integrations**: LangChain, LangGraph, and Vercel AI SDK agent integration
 
 ## 📥 Installation
 
@@ -35,6 +36,18 @@ For OpenTelemetry tracing (optional):
 npm install @opentelemetry/api @opentelemetry/sdk-node @opentelemetry/sdk-trace-base @opentelemetry/exporter-trace-otlp-http @opentelemetry/resources @opentelemetry/semantic-conventions
 ```
 
+For LangChain/LangGraph integration:
+
+```bash
+npm install langchain @langchain/core @langchain/langgraph
+```
+
+For Vercel AI SDK integration:
+
+```bash
+npm install ai 
+```
+
 ## 🚀 Quick Start
 
 ### Basic Usage
@@ -221,6 +234,38 @@ const conversationJob = job("assistant", async (data) => {
 
 The `invoke()` function returns the text content directly, while `deployment()` returns an object with both `content` and `raw` response for more control.
 
+## 🔗 LangChain Integration
+
+Evaluatorq provides integration with LangChain and LangGraph agents, converting their outputs to the OpenResponses format for standardized evaluation.
+
+The LangChain integration allows you to:
+- Wrap LangChain agents created with `createAgent()` for use in evaluatorq jobs
+- Wrap LangGraph compiled graphs for stateful agent evaluation
+- Automatically convert agent outputs to OpenResponses format
+- Evaluate agent behavior using standard evaluatorq evaluators
+
+### Examples
+
+Complete examples are available in the examples folder:
+
+- **LangChain Agent**: [`examples/src/lib/integrations/langchain-agent-eval.ts`](../../examples/src/lib/integrations/langchain-agent-eval.ts)
+- **LangGraph Agent**: [`examples/src/lib/integrations/langgraph-agent-eval.ts`](../../examples/src/lib/integrations/langgraph-agent-eval.ts)
+
+## 🤖 Vercel AI SDK Integration
+
+Evaluatorq integrates with the Vercel AI SDK, allowing you to wrap AI SDK agents and evaluate them using the standard evaluatorq framework.
+
+The Vercel AI SDK integration allows you to:
+- Wrap Vercel AI SDK `ToolLoopAgent` instances for use in evaluatorq jobs
+- Automatically convert agent outputs to OpenResponses format
+- Evaluate agent behavior using standard evaluatorq evaluators
+
+### Examples
+
+Complete examples are available in the examples folder:
+
+- **Vercel AI SDK Agent**: [`examples/src/lib/integrations/vercel_ai_sdk_integration_example.ts`](../../examples/src/lib/integrations/vercel_ai_sdk_integration_example.ts)
+
 ## 🔧 Configuration
 
 ### Environment Variables
@@ -412,6 +457,11 @@ type Scorer = (
   EvaluationResult<string | number | boolean | EvaluationResultCell>
 >;
 
+// Integration wrappers
+import { wrapLangChainAgent, wrapLangGraphAgent } from "@orq-ai/evaluatorq/langchain";
+import { wrapAISdkAgent } from "@orq-ai/evaluatorq/ai-sdk";
+import type { ResponseResource } from "@orq-ai/evaluatorq/openresponses";
+
 // Deployment helper types
 interface DeploymentOptions {
   inputs?: Record<string, unknown>;

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 export { type DeploymentOptions, type DeploymentResponse, deployment, invoke, } from "./lib/deployment-helper.js";
 export * from "./lib/evaluatorq.js";
+export * from "./lib/integrations/openresponses/index.js";
 export { job } from "./lib/job-helper.js";
 export { sendResultsToOrqEffect } from "./lib/send-results.js";
 export { displayResultsTableEffect } from "./lib/table-display.js";

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,UAAU,EACV,MAAM,GACP,MAAM,4BAA4B,CAAC;AACpC,cAAc,qBAAqB,CAAC;AACpC,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAC1C,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AAEnE,OAAO,EACL,mBAAmB,EACnB,gBAAgB,EAChB,eAAe,EACf,KAAK,cAAc,GACpB,MAAM,wBAAwB,CAAC;AAChC,cAAc,gBAAgB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,UAAU,EACV,MAAM,GACP,MAAM,4BAA4B,CAAC;AACpC,cAAc,qBAAqB,CAAC;AAEpC,cAAc,2CAA2C,CAAC;AAC1D,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAC1C,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AAEnE,OAAO,EACL,mBAAmB,EACnB,gBAAgB,EAChB,eAAe,EACf,KAAK,cAAc,GACpB,MAAM,wBAAwB,CAAC;AAChC,cAAc,gBAAgB,CAAC"}

package/dist/index.js

@@ -1,5 +1,7 @@
 export { deployment, invoke, } from "./lib/deployment-helper.js";
 export * from "./lib/evaluatorq.js";
+// OpenResponses types
+export * from "./lib/integrations/openresponses/index.js";
 export { job } from "./lib/job-helper.js";
 export { sendResultsToOrqEffect } from "./lib/send-results.js";
 export { displayResultsTableEffect } from "./lib/table-display.js";

package/dist/lib/integrations/ai-sdk/convert.d.ts

@@ -0,0 +1,22 @@
+import type { Agent, ToolSet } from "ai";
+import type { ResponseResource } from "../openresponses/index.js";
+/**
+ * Builds the input array for OpenResponses format.
+ *
+ * Resolves item_reference entries to actual function_call items using data from steps.
+ * The input should contain:
+ * - User message
+ * - All function calls made during the conversation
+ * - All function call outputs returned
+ */
+export declare function buildInputFromSteps<TOOLS extends ToolSet>(result: Awaited<ReturnType<Agent<never, TOOLS, never>["generate"]>>, prompt: string | undefined): unknown[];
+/**
+ * Converts Vercel AI SDK agent result to OpenResponses format.
+ */
+export declare function convertToOpenResponses<TOOLS extends ToolSet>(result: Awaited<ReturnType<Agent<never, TOOLS, never>["generate"]>>, agent: Agent<never, TOOLS, never>, prompt?: string): ResponseResource;
+/**
+ * Fallback function to manually build OpenResponses format from steps.
+ * Used when the provider doesn't return native OpenResponses format.
+ */
+export declare function buildOpenResponsesFromSteps<TOOLS extends ToolSet>(result: Awaited<ReturnType<Agent<never, TOOLS, never>["generate"]>>, agent: Agent<never, TOOLS, never>, _prompt?: string): ResponseResource;
+//# sourceMappingURL=convert.d.ts.map

package/dist/lib/integrations/ai-sdk/convert.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert.d.ts","sourceRoot":"","sources":["../../../../src/lib/integrations/ai-sdk/convert.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAc,OAAO,EAAE,MAAM,IAAI,CAAC;AAOrD,OAAO,KAAK,EAMV,gBAAgB,EAEjB,MAAM,2BAA2B,CAAC;AASnC;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,SAAS,OAAO,EACvD,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EACnE,MAAM,EAAE,MAAM,GAAG,SAAS,GACzB,OAAO,EAAE,CA6EX;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,SAAS,OAAO,EAC1D,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EACnE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,EACjC,MAAM,CAAC,EAAE,MAAM,GACd,gBAAgB,CAElB;AAwCD;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,SAAS,OAAO,EAC/D,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EACnE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,EACjC,OAAO,CAAC,EAAE,MAAM,GACf,gBAAgB,CA4JlB"}

package/dist/lib/integrations/ai-sdk/convert.js

@@ -0,0 +1,230 @@
+import { generateItemId, getResponseStatus, serializeArgs, } from "../common/index.js";
+/**
+ * Builds the input array for OpenResponses format.
+ *
+ * Resolves item_reference entries to actual function_call items using data from steps.
+ * The input should contain:
+ * - User message
+ * - All function calls made during the conversation
+ * - All function call outputs returned
+ */
+export function buildInputFromSteps(result, prompt) {
+    const input = [];
+    // Add the initial user message
+    if (prompt) {
+        input.push({
+            type: "message",
+            id: generateItemId("msg"),
+            status: "completed",
+            role: "user",
+            content: [
+                {
+                    type: "input_text",
+                    text: prompt,
+                },
+            ],
+        });
+    }
+    // Collect all function calls and outputs from steps
+    for (const step of result.steps) {
+        const stepData = step;
+        // Primary: Extract from content array (ToolLoopAgent format)
+        if (stepData.content && stepData.content.length > 0) {
+            for (const item of stepData.content) {
+                if (item.type === "tool-call" && item.toolCallId && item.toolName) {
+                    input.push({
+                        type: "function_call",
+                        id: generateItemId("fc"),
+                        call_id: item.toolCallId,
+                        name: item.toolName,
+                        arguments: serializeArgs(item.input),
+                        status: "completed",
+                    });
+                }
+                if (item.type === "tool-result" && item.toolCallId) {
+                    input.push({
+                        type: "function_call_output",
+                        id: generateItemId("fco"),
+                        call_id: item.toolCallId,
+                        output: serializeArgs(item.output),
+                        status: "completed",
+                    });
+                }
+            }
+        }
+        // Fallback: Extract from toolCalls/toolResults arrays (alternative AI SDK format)
+        else {
+            if (stepData.toolCalls && stepData.toolCalls.length > 0) {
+                for (const toolCall of stepData.toolCalls) {
+                    input.push({
+                        type: "function_call",
+                        id: generateItemId("fc"),
+                        call_id: toolCall.toolCallId,
+                        name: toolCall.toolName,
+                        arguments: serializeArgs(toolCall.input),
+                        status: "completed",
+                    });
+                }
+            }
+            if (stepData.toolResults && stepData.toolResults.length > 0) {
+                for (const toolResult of stepData.toolResults) {
+                    input.push({
+                        type: "function_call_output",
+                        id: generateItemId("fco"),
+                        call_id: toolResult.toolCallId,
+                        output: serializeArgs(toolResult.output),
+                        status: "completed",
+                    });
+                }
+            }
+        }
+    }
+    return input;
+}
+/**
+ * Converts Vercel AI SDK agent result to OpenResponses format.
+ */
+export function convertToOpenResponses(result, agent, prompt) {
+    return buildOpenResponsesFromSteps(result, agent, prompt);
+}
+/**
+ * Fallback function to manually build OpenResponses format from steps.
+ * Used when the provider doesn't return native OpenResponses format.
+ */
+export function buildOpenResponsesFromSteps(result, agent, _prompt) {
+    const now = Math.floor(Date.now() / 1000);
+    // Extract configuration from first step's request body
+    const firstStep = result.steps[0];
+    const lastStep = result.steps[result.steps.length - 1];
+    const requestBody = firstStep?.request?.body;
+    // Extract timestamps from response (handle both Date and string formats)
+    const getTimestamp = (ts) => {
+        if (!ts)
+            return now;
+        return Math.floor((ts instanceof Date ? ts : new Date(ts)).getTime() / 1000);
+    };
+    const createdAt = getTimestamp(firstStep?.response?.timestamp);
+    const completedAt = getTimestamp(lastStep?.response?.timestamp);
+    // Extract service tier from provider metadata
+    const serviceTier = lastStep?.providerMetadata?.openai?.serviceTier ?? "default";
+    // Convert tools from agent configuration
+    const tools = [];
+    if (agent.tools) {
+        for (const [toolName, toolDef] of Object.entries(agent.tools)) {
+            const toolConfig = toolDef;
+            tools.push({
+                type: "function",
+                name: toolName,
+                description: toolConfig.description ?? null,
+                parameters: toolConfig.parameters ?? null,
+                strict: null,
+            });
+        }
+    }
+    // Build output items from steps
+    const output = [];
+    let callIdCounter = 0;
+    for (const step of result.steps) {
+        const stepData = step;
+        // Add function calls from this step
+        if (stepData.toolCalls && stepData.toolCalls.length > 0) {
+            for (const toolCall of stepData.toolCalls) {
+                const functionCall = {
+                    type: "function_call",
+                    id: generateItemId("fc"),
+                    call_id: toolCall.toolCallId || `call_${callIdCounter++}`,
+                    name: toolCall.toolName,
+                    arguments: serializeArgs(toolCall.input),
+                    status: "completed",
+                };
+                output.push(functionCall);
+            }
+        }
+        // Add function call outputs from this step
+        if (stepData.toolResults && stepData.toolResults.length > 0) {
+            for (const toolResult of stepData.toolResults) {
+                const functionCallOutput = {
+                    type: "function_call_output",
+                    id: generateItemId("fco"),
+                    call_id: toolResult.toolCallId,
+                    output: serializeArgs(toolResult.output),
+                    status: "completed",
+                };
+                output.push(functionCallOutput);
+            }
+        }
+    }
+    // Add final message with text response
+    if (result.text) {
+        const message = {
+            type: "message",
+            id: generateItemId("msg"),
+            status: "completed",
+            role: "assistant",
+            content: [
+                {
+                    type: "output_text",
+                    text: result.text,
+                    annotations: [],
+                    logprobs: [],
+                },
+            ],
+        };
+        output.push(message);
+    }
+    // Convert usage
+    const usage = result.totalUsage
+        ? {
+            input_tokens: result.totalUsage.inputTokens ?? 0,
+            input_tokens_details: {
+                cached_tokens: result.totalUsage
+                    .cachedInputTokens ?? 0,
+            },
+            output_tokens: result.totalUsage.outputTokens ?? 0,
+            output_tokens_details: {
+                reasoning_tokens: result.totalUsage
+                    .reasoningTokens ?? 0,
+            },
+            total_tokens: result.totalUsage.totalTokens ?? 0,
+        }
+        : null;
+    const status = getResponseStatus(result.finishReason);
+    return {
+        id: result.response.id || generateItemId("resp"),
+        object: "response",
+        created_at: createdAt,
+        completed_at: status === "completed" ? completedAt : null,
+        status,
+        incomplete_details: status === "incomplete" ? { reason: result.finishReason } : null,
+        model: result.response.modelId,
+        previous_response_id: null,
+        instructions: null,
+        output,
+        error: status === "failed" ? { message: "Agent execution failed" } : null,
+        tools,
+        tool_choice: requestBody?.tool_choice ?? "auto",
+        truncation: "disabled",
+        parallel_tool_calls: true,
+        text: {
+            format: {
+                type: "text",
+            },
+        },
+        top_p: requestBody?.top_p ?? 1,
+        presence_penalty: requestBody?.presence_penalty ?? 0,
+        frequency_penalty: requestBody?.frequency_penalty ?? 0,
+        top_logprobs: 0,
+        temperature: requestBody?.temperature ?? 1,
+        reasoning: null,
+        user: null,
+        usage,
+        max_output_tokens: requestBody?.max_output_tokens ?? null,
+        max_tool_calls: null,
+        store: false,
+        background: false,
+        service_tier: serviceTier,
+        metadata: { framework: "vercel-ai-sdk" },
+        safety_identifier: null,
+        prompt_cache_key: null,
+    };
+}

package/dist/lib/integrations/ai-sdk/index.d.ts

@@ -0,0 +1,5 @@
+export { generateItemId } from "../common/index.js";
+export { buildInputFromSteps, buildOpenResponsesFromSteps, convertToOpenResponses, } from "./convert.js";
+export type { AgentJobOptions, StepData } from "./types.js";
+export { wrapAISdkAgent } from "./wrap-agent.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/integrations/ai-sdk/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/integrations/ai-sdk/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EACL,mBAAmB,EACnB,2BAA2B,EAC3B,sBAAsB,GACvB,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/lib/integrations/ai-sdk/index.js

@@ -0,0 +1,4 @@
+// Re-export common utilities for backwards compatibility
+export { generateItemId } from "../common/index.js";
+export { buildInputFromSteps, buildOpenResponsesFromSteps, convertToOpenResponses, } from "./convert.js";
+export { wrapAISdkAgent } from "./wrap-agent.js";

package/dist/lib/integrations/ai-sdk/types.d.ts

@@ -0,0 +1,56 @@
+import type { StepResult } from "ai";
+import type { FunctionTool, ResponseResource } from "../openresponses/index.js";
+export type { StepResult };
+/**
+ * Type definition for step data extracted from AI SDK results.
+ * This interface extends the AI SDK's StepResult type with additional fields
+ * that may be present in certain AI SDK versions or providers.
+ */
+export interface StepData {
+    content?: Array<{
+        type: string;
+        toolCallId?: string;
+        toolName?: string;
+        input?: unknown;
+        output?: unknown;
+    }>;
+    request?: {
+        body?: {
+            input?: unknown[];
+            tools?: FunctionTool[];
+        };
+    };
+    response?: {
+        body?: ResponseResource;
+        messages?: Array<{
+            role: string;
+            content: Array<{
+                type: string;
+                toolCallId?: string;
+                toolName?: string;
+                input?: unknown;
+                text?: string;
+                providerOptions?: {
+                    openai?: {
+                        itemId?: string;
+                    };
+                };
+            }>;
+        }>;
+    };
+    providerMetadata?: {
+        openai?: {
+            itemId?: string;
+        };
+    };
+}
+/**
+ * Options for creating an evaluatorq Job from an AI SDK Agent.
+ */
+export interface AgentJobOptions {
+    /** The name of the job (defaults to agent.id or "agent") */
+    name?: string;
+    /** The key in data.inputs to use as the prompt (defaults to "prompt") */
+    promptKey?: string;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/lib/integrations/ai-sdk/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/lib/integrations/ai-sdk/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAErC,OAAO,KAAK,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAGhF,YAAY,EAAE,UAAU,EAAE,CAAC;AAE3B;;;;GAIG;AACH,MAAM,WAAW,QAAQ;IAEvB,OAAO,CAAC,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,MAAM,CAAC;QACb,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,MAAM,CAAC,EAAE,OAAO,CAAC;KAClB,CAAC,CAAC;IAEH,OAAO,CAAC,EAAE;QACR,IAAI,CAAC,EAAE;YACL,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC;YAClB,KAAK,CAAC,EAAE,YAAY,EAAE,CAAC;SACxB,CAAC;KACH,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,IAAI,CAAC,EAAE,gBAAgB,CAAC;QACxB,QAAQ,CAAC,EAAE,KAAK,CAAC;YACf,IAAI,EAAE,MAAM,CAAC;YACb,OAAO,EAAE,KAAK,CAAC;gBACb,IAAI,EAAE,MAAM,CAAC;gBACb,UAAU,CAAC,EAAE,MAAM,CAAC;gBACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;gBAClB,KAAK,CAAC,EAAE,OAAO,CAAC;gBAChB,IAAI,CAAC,EAAE,MAAM,CAAC;gBACd,eAAe,CAAC,EAAE;oBAChB,MAAM,CAAC,EAAE;wBACP,MAAM,CAAC,EAAE,MAAM,CAAC;qBACjB,CAAC;iBACH,CAAC;aACH,CAAC,CAAC;SACJ,CAAC,CAAC;KACJ,CAAC;IACF,gBAAgB,CAAC,EAAE;QACjB,MAAM,CAAC,EAAE;YACP,MAAM,CAAC,EAAE,MAAM,CAAC;SACjB,CAAC;KACH,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,4DAA4D;IAC5D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,yEAAyE;IACzE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB"}

package/dist/lib/integrations/ai-sdk/types.js

@@ -0,0 +1 @@
+export {};

package/dist/lib/integrations/ai-sdk/wrap-agent.d.ts

@@ -0,0 +1,58 @@
+import type { Agent, ToolSet } from "ai";
+import type { Job } from "../../types.js";
+import type { AgentJobOptions } from "./types.js";
+/**
+ * Creates an evaluatorq Job from any AI SDK Agent.
+ *
+ * Supports:
+ * - `Agent` (base interface)
+ * - `ToolLoopAgent`
+ * - `Experimental_Agent` (deprecated alias for ToolLoopAgent)
+ *
+ * The job will:
+ * - Execute the agent with the prompt from data.inputs
+ * - Convert the result to OpenResponses format (industry standard)
+ * - Return the OpenResponses resource for backend integration
+ *
+ * @example
+ * ```typescript
+ * import { wrapAISdkAgent } from "@orq-ai/evaluatorq/ai-sdk";
+ * import { ToolLoopAgent, tool } from "ai";
+ *
+ * const weatherAgent = new ToolLoopAgent({
+ *   model: openai("gpt-4o"),
+ *   tools: {
+ *     weather: tool({
+ *       description: "Get the weather in a location",
+ *       inputSchema: z.object({ location: z.string() }),
+ *       execute: async ({ location }) => ({ location, temperature: 72 }),
+ *     }),
+ *   },
+ * });
+ *
+ * await evaluatorq("weather-agent-eval", {
+ *   data: [
+ *     { inputs: { prompt: "What is the weather in SF?" } },
+ *   ],
+ *   jobs: [wrapAISdkAgent(weatherAgent)],
+ *   evaluators: [
+ *     {
+ *       name: "response-quality",
+ *       scorer: async ({ output }) => {
+ *         const result = output as unknown as ResponseResource;
+ *         // Access the final message text
+ *         const lastMessage = result.output.find(
+ *           (item) => item.type === "message"
+ *         );
+ *         return {
+ *           value: lastMessage ? 1 : 0,
+ *           explanation: "Agent produced a response",
+ *         };
+ *       },
+ *     },
+ *   ],
+ * });
+ * ```
+ */
+export declare function wrapAISdkAgent<TOOLS extends ToolSet>(agent: Agent<never, TOOLS, never>, options?: AgentJobOptions): Job;
+//# sourceMappingURL=wrap-agent.d.ts.map

package/dist/lib/integrations/ai-sdk/wrap-agent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"wrap-agent.d.ts","sourceRoot":"","sources":["../../../../src/lib/integrations/ai-sdk/wrap-agent.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAEzC,OAAO,KAAK,EAAa,GAAG,EAAU,MAAM,gBAAgB,CAAC;AAG7D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoDG;AACH,wBAAgB,cAAc,CAAC,KAAK,SAAS,OAAO,EAClD,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,EACjC,OAAO,GAAE,eAAoB,GAC5B,GAAG,CAeL"}

package/dist/lib/integrations/ai-sdk/wrap-agent.js

@@ -0,0 +1,68 @@
+import { extractPromptFromData } from "../common/index.js";
+import { convertToOpenResponses } from "./convert.js";
+/**
+ * Creates an evaluatorq Job from any AI SDK Agent.
+ *
+ * Supports:
+ * - `Agent` (base interface)
+ * - `ToolLoopAgent`
+ * - `Experimental_Agent` (deprecated alias for ToolLoopAgent)
+ *
+ * The job will:
+ * - Execute the agent with the prompt from data.inputs
+ * - Convert the result to OpenResponses format (industry standard)
+ * - Return the OpenResponses resource for backend integration
+ *
+ * @example
+ * ```typescript
+ * import { wrapAISdkAgent } from "@orq-ai/evaluatorq/ai-sdk";
+ * import { ToolLoopAgent, tool } from "ai";
+ *
+ * const weatherAgent = new ToolLoopAgent({
+ *   model: openai("gpt-4o"),
+ *   tools: {
+ *     weather: tool({
+ *       description: "Get the weather in a location",
+ *       inputSchema: z.object({ location: z.string() }),
+ *       execute: async ({ location }) => ({ location, temperature: 72 }),
+ *     }),
+ *   },
+ * });
+ *
+ * await evaluatorq("weather-agent-eval", {
+ *   data: [
+ *     { inputs: { prompt: "What is the weather in SF?" } },
+ *   ],
+ *   jobs: [wrapAISdkAgent(weatherAgent)],
+ *   evaluators: [
+ *     {
+ *       name: "response-quality",
+ *       scorer: async ({ output }) => {
+ *         const result = output as unknown as ResponseResource;
+ *         // Access the final message text
+ *         const lastMessage = result.output.find(
+ *           (item) => item.type === "message"
+ *         );
+ *         return {
+ *           value: lastMessage ? 1 : 0,
+ *           explanation: "Agent produced a response",
+ *         };
+ *       },
+ *     },
+ *   ],
+ * });
+ * ```
+ */
+export function wrapAISdkAgent(agent, options = {}) {
+    const { name = agent.id ?? "agent", promptKey = "prompt" } = options;
+    return async (data, _row) => {
+        const prompt = extractPromptFromData(data, promptKey);
+        const result = await agent.generate({ prompt });
+        // Convert to OpenResponses format
+        const openResponsesOutput = convertToOpenResponses(result, agent, prompt);
+        return {
+            name,
+            output: openResponsesOutput,
+        };
+    };
+}

package/dist/lib/integrations/common/index.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Common utilities shared across agent integrations.
+ */
+export { extractPromptFromData, generateItemId, getResponseStatus, serializeArgs, } from "./utils.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/integrations/common/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/integrations/common/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EACL,qBAAqB,EACrB,cAAc,EACd,iBAAiB,EACjB,aAAa,GACd,MAAM,YAAY,CAAC"}

package/dist/lib/integrations/common/index.js

@@ -0,0 +1,4 @@
+/**
+ * Common utilities shared across agent integrations.
+ */
+export { extractPromptFromData, generateItemId, getResponseStatus, serializeArgs, } from "./utils.js";

package/dist/lib/integrations/common/utils.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Shared utilities for agent integrations.
+ */
+import type { DataPoint } from "../../types.js";
+import type { ResponseResource } from "../openresponses/index.js";
+/**
+ * Generates a unique ID for OpenResponses items.
+ *
+ * @param prefix - The prefix for the ID (e.g., "fc" for function_call, "msg" for message)
+ * @returns A unique string ID with the given prefix
+ */
+export declare function generateItemId(prefix: string): string;
+/**
+ * Serializes tool arguments to a JSON string.
+ *
+ * @param args - The arguments to serialize (can be any type)
+ * @returns A JSON string representation of the arguments
+ */
+export declare function serializeArgs(args: unknown): string;
+/**
+ * Maps a finish reason to an OpenResponses status.
+ * Handles common finish reasons from various providers.
+ *
+ * @param finishReason - The finish reason from the LLM response
+ * @returns The corresponding OpenResponses status
+ */
+export declare function getResponseStatus(finishReason: string | undefined): ResponseResource["status"];
+/**
+ * Extracts and validates a prompt string from a DataPoint.
+ *
+ * @param data - The data point containing inputs
+ * @param promptKey - The key to look up in data.inputs
+ * @returns The prompt string
+ * @throws Error if the prompt is not a string
+ */
+export declare function extractPromptFromData(data: DataPoint, promptKey: string): string;
+//# sourceMappingURL=utils.d.ts.map

package/dist/lib/integrations/common/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../../src/lib/integrations/common/utils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAElE;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAErD;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAiBnD;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAC/B,YAAY,EAAE,MAAM,GAAG,SAAS,GAC/B,gBAAgB,CAAC,QAAQ,CAAC,CAa5B;AAED;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,SAAS,EACf,SAAS,EAAE,MAAM,GAChB,MAAM,CAQR"}