@orq-ai/evaluatorq 1.2.0-rc.4 → 1.2.1

package/dist/lib/integrations/langchain/wrap-agent.js

@@ -2,7 +2,7 @@
  * Wrapper functions for LangChain/LangGraph agents to integrate with evaluatorq.
  */
 import { toJsonSchema } from "@langchain/core/utils/json_schema";
-import { extractPromptFromData } from "../common/index.js";
+import { extractMessagesFromData, extractPromptFromData, } from "../common/utils.js";
 import { convertToOpenResponses } from "./convert.js";
 /**
  * Extract JSON schema parameters from a schema object.
@@ -85,13 +85,62 @@ function convertToolToDefinition(tool) {
  * @returns An evaluatorq Job function
  */
 export function wrapLangChainAgent(agent, options = {}) {
-    const { name = "agent", promptKey = "prompt", tools } = options;
+    const { name = "agent", promptKey = "prompt", instructions, tools } = options;
     return async (data, _row) => {
-        const prompt = extractPromptFromData(data, promptKey);
-        // Invoke the LangChain agent with messages format
-        const result = await agent.invoke({
-            messages: [{ role: "user", content: prompt }],
-        });
+        const inputMessages = extractMessagesFromData(data);
+        const hasMessages = inputMessages !== undefined;
+        const hasPrompt = typeof data.inputs[promptKey] === "string" && data.inputs[promptKey];
+        let result;
+        if (instructions) {
+            // Resolve instructions (static string or dynamic function)
+            const resolvedInstructions = typeof instructions === "function" ? instructions(data) : instructions;
+            const systemMessage = { role: "system", content: resolvedInstructions };
+            let messages;
+            if (hasMessages && hasPrompt) {
+                messages = [
+                    systemMessage,
+                    ...inputMessages,
+                    { role: "user", content: data.inputs[promptKey] },
+                ];
+            }
+            else if (hasPrompt) {
+                messages = [
+                    systemMessage,
+                    { role: "user", content: data.inputs[promptKey] },
+                ];
+            }
+            else if (hasMessages) {
+                messages = [systemMessage, ...inputMessages];
+            }
+            else {
+                throw new Error("Expected data.inputs.messages (array) or data.inputs.prompt (string), but neither was provided");
+            }
+            result = await agent.invoke({ messages });
+        }
+        else if (hasMessages && hasPrompt) {
+            // Both exist — merge messages + prompt appended as user message
+            const messages = [
+                ...inputMessages,
+                { role: "user", content: data.inputs[promptKey] },
+            ];
+            result = await agent.invoke({ messages });
+        }
+        else if (hasPrompt) {
+            // Prompt only — wrap as single user message
+            const prompt = extractPromptFromData(data, promptKey);
+            result = await agent.invoke({
+                messages: [{ role: "user", content: prompt }],
+            });
+        }
+        else if (hasMessages) {
+            // Messages only — pass directly
+            result = await agent.invoke({
+                messages: inputMessages,
+            });
+        }
+        else {
+            throw new Error("Expected data.inputs.messages (array) or data.inputs.prompt (string), but neither was provided");
+        }
         // Extract messages from result
         const messages = result.messages ?? [];
         // Get tools from agent if not provided explicitly

package/dist/lib/integrations/openresponses/index.d.ts

@@ -188,6 +188,8 @@ export interface ResponseResource {
     previous_response_id: string | null;
     /** System instructions provided to the model. */
     instructions: string | null;
+    /** The input items that were sent to the model. */
+    input: ItemField[];
     /** The output items that were generated by the model. */
     output: ItemField[];
     /** Error information if the response failed. */
@@ -233,4 +235,14 @@ export interface ResponseResource {
     /** Prompt cache key if caching was used. */
     prompt_cache_key: string | null;
 }
+/**
+ * Extracts the text content from an Output value that is a ResponseResource.
+ *
+ * Finds the first assistant message with `output_text` content and returns its text.
+ * Returns an empty string if the output is not a ResponseResource or contains no text.
+ *
+ * @param output - The output from a job (can be any Output type)
+ * @returns The extracted text string, or empty string if not found
+ */
+export declare function extractText(output: unknown): string;
 //# sourceMappingURL=index.d.ts.map

package/dist/lib/integrations/openresponses/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/integrations/openresponses/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAMH,eAAO,MAAM,eAAe;;;;;CAKlB,CAAC;AAEX,MAAM,MAAM,WAAW,GACrB,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,OAAO,eAAe,CAAC,CAAC;AAEzD,eAAO,MAAM,iBAAiB;;;;CAIpB,CAAC;AAEX,MAAM,MAAM,aAAa,GACvB,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,OAAO,iBAAiB,CAAC,CAAC;AAE7D,eAAO,MAAM,sBAAsB;;;;CAIzB,CAAC;AAEX,MAAM,MAAM,kBAAkB,GAC5B,CAAC,OAAO,sBAAsB,CAAC,CAAC,MAAM,OAAO,sBAAsB,CAAC,CAAC;AAEvE,eAAO,MAAM,4BAA4B;;;;CAI/B,CAAC;AAEX,MAAM,MAAM,4BAA4B,GACtC,CAAC,OAAO,4BAA4B,CAAC,CAAC,MAAM,OAAO,4BAA4B,CAAC,CAAC;AAMnF;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,uDAAuD;IACvD,IAAI,EAAE,YAAY,CAAC;IACnB,mCAAmC;IACnC,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,yDAAyD;IACzD,IAAI,EAAE,aAAa,CAAC;IACpB,sCAAsC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,0CAA0C;IAC1C,WAAW,EAAE,OAAO,EAAE,CAAC;IACvB,+CAA+C;IAC/C,QAAQ,EAAE,OAAO,EAAE,CAAC;CACrB;AAMD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,8DAA8D;IAC9D,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,qEAAqE;IACrE,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,sDAAsD;IACtD,MAAM,EAAE,MAAM,CAAC;CAChB;AAMD;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,oDAAoD;IACpD,IAAI,EAAE,eAAe,CAAC;IACtB,+CAA+C;IAC/C,EAAE,EAAE,MAAM,CAAC;IACX,kEAAkE;IAClE,OAAO,EAAE,MAAM,CAAC;IAChB,gDAAgD;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,uCAAuC;IACvC,MAAM,EAAE,kBAAkB,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,gFAAgF;IAChF,IAAI,EAAE,sBAAsB,CAAC;IAC7B,sDAAsD;IACtD,EAAE,EAAE,MAAM,CAAC;IACX,sEAAsE;IACtE,OAAO,EAAE,MAAM,CAAC;IAChB,yCAAyC;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,MAAM,EAAE,4BAA4B,CAAC;CACtC;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,wDAAwD;IACxD,IAAI,EAAE,UAAU,CAAC;IACjB,wCAAwC;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,+CAA+C;IAC/C,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,oEAAoE;IACpE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC3C,iDAAiD;IACjD,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,wDAAwD;IACxD,IAAI,EAAE,SAAS,CAAC;IAChB,oCAAoC;IACpC,EAAE,EAAE,MAAM,CAAC;IACX,iCAAiC;IACjC,MAAM,EAAE,aAAa,CAAC;IACtB,sCAAsC;IACtC,IAAI,EAAE,WAAW,CAAC;IAClB,kCAAkC;IAClC,OAAO,EAAE,KAAK,CACZ,gBAAgB,GAAG,iBAAiB,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC/D,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,KAAK;IACpB,0EAA0E;IAC1E,YAAY,EAAE,MAAM,CAAC;IACrB,oEAAoE;IACpE,aAAa,EAAE,MAAM,CAAC;IACtB,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,wCAAwC;IACxC,oBAAoB,EAAE,kBAAkB,CAAC;IACzC,yCAAyC;IACzC,qBAAqB,EAAE,mBAAmB,CAAC;CAC5C;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,gDAAgD;IAChD,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,2BAA2B;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG,OAAO,GAAG,YAAY,GAAG,kBAAkB,CAAC;AAEpE;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,sDAAsD;IACtD,EAAE,EAAE,MAAM,CAAC;IACX,0CAA0C;IAC1C,MAAM,EAAE,UAAU,CAAC;IACnB,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,gDAAgD;IAChD,MAAM,EAAE,MAAM,CAAC;IACf,qDAAqD;IACrD,kBAAkB,EAAE,iBAAiB,GAAG,IAAI,CAAC;IAC7C,8CAA8C;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,yDAAyD;IACzD,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,iDAAiD;IACjD,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,yDAAyD;IACzD,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,gDAAgD;IAChD,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACtC,6EAA6E;IAC7E,KAAK,EAAE,YAAY,EAAE,CAAC;IACtB,qDAAqD;IACrD,WAAW,EAAE,MAAM,CAAC;IACpB,+DAA+D;IAC/D,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,mBAAmB,EAAE,OAAO,CAAC;IAC7B,iCAAiC;IACjC,IAAI,EAAE,SAAS,CAAC;IAChB,sEAAsE;IACtE,KAAK,EAAE,MAAM,CAAC;IACd,0CAA0C;IAC1C,gBAAgB,EAAE,MAAM,CAAC;IACzB,2CAA2C;IAC3C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kEAAkE;IAClE,YAAY,EAAE,MAAM,CAAC;IACrB,gEAAgE;IAChE,WAAW,EAAE,MAAM,CAAC;IACpB,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC1C,8BAA8B;IAC9B,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IACpB,qCAAqC;IACrC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,kCAAkC;IAClC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,4DAA4D;IAC5D,KAAK,EAAE,OAAO,CAAC;IACf,sDAAsD;IACtD,UAAU,EAAE,OAAO,CAAC;IACpB,wDAAwD;IACxD,YAAY,EAAE,MAAM,CAAC;IACrB,+DAA+D;IAC/D,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,0CAA0C;IAC1C,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,4CAA4C;IAC5C,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/integrations/openresponses/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAMH,eAAO,MAAM,eAAe;;;;;CAKlB,CAAC;AAEX,MAAM,MAAM,WAAW,GACrB,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,OAAO,eAAe,CAAC,CAAC;AAEzD,eAAO,MAAM,iBAAiB;;;;CAIpB,CAAC;AAEX,MAAM,MAAM,aAAa,GACvB,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,OAAO,iBAAiB,CAAC,CAAC;AAE7D,eAAO,MAAM,sBAAsB;;;;CAIzB,CAAC;AAEX,MAAM,MAAM,kBAAkB,GAC5B,CAAC,OAAO,sBAAsB,CAAC,CAAC,MAAM,OAAO,sBAAsB,CAAC,CAAC;AAEvE,eAAO,MAAM,4BAA4B;;;;CAI/B,CAAC;AAEX,MAAM,MAAM,4BAA4B,GACtC,CAAC,OAAO,4BAA4B,CAAC,CAAC,MAAM,OAAO,4BAA4B,CAAC,CAAC;AAMnF;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,uDAAuD;IACvD,IAAI,EAAE,YAAY,CAAC;IACnB,mCAAmC;IACnC,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,yDAAyD;IACzD,IAAI,EAAE,aAAa,CAAC;IACpB,sCAAsC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,0CAA0C;IAC1C,WAAW,EAAE,OAAO,EAAE,CAAC;IACvB,+CAA+C;IAC/C,QAAQ,EAAE,OAAO,EAAE,CAAC;CACrB;AAMD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,8DAA8D;IAC9D,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,qEAAqE;IACrE,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,sDAAsD;IACtD,MAAM,EAAE,MAAM,CAAC;CAChB;AAMD;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,oDAAoD;IACpD,IAAI,EAAE,eAAe,CAAC;IACtB,+CAA+C;IAC/C,EAAE,EAAE,MAAM,CAAC;IACX,kEAAkE;IAClE,OAAO,EAAE,MAAM,CAAC;IAChB,gDAAgD;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,uCAAuC;IACvC,MAAM,EAAE,kBAAkB,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,gFAAgF;IAChF,IAAI,EAAE,sBAAsB,CAAC;IAC7B,sDAAsD;IACtD,EAAE,EAAE,MAAM,CAAC;IACX,sEAAsE;IACtE,OAAO,EAAE,MAAM,CAAC;IAChB,yCAAyC;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,MAAM,EAAE,4BAA4B,CAAC;CACtC;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,wDAAwD;IACxD,IAAI,EAAE,UAAU,CAAC;IACjB,wCAAwC;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,+CAA+C;IAC/C,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,oEAAoE;IACpE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC3C,iDAAiD;IACjD,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,wDAAwD;IACxD,IAAI,EAAE,SAAS,CAAC;IAChB,oCAAoC;IACpC,EAAE,EAAE,MAAM,CAAC;IACX,iCAAiC;IACjC,MAAM,EAAE,aAAa,CAAC;IACtB,sCAAsC;IACtC,IAAI,EAAE,WAAW,CAAC;IAClB,kCAAkC;IAClC,OAAO,EAAE,KAAK,CACZ,gBAAgB,GAAG,iBAAiB,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC/D,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,KAAK;IACpB,0EAA0E;IAC1E,YAAY,EAAE,MAAM,CAAC;IACrB,oEAAoE;IACpE,aAAa,EAAE,MAAM,CAAC;IACtB,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,wCAAwC;IACxC,oBAAoB,EAAE,kBAAkB,CAAC;IACzC,yCAAyC;IACzC,qBAAqB,EAAE,mBAAmB,CAAC;CAC5C;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,gDAAgD;IAChD,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,2BAA2B;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG,OAAO,GAAG,YAAY,GAAG,kBAAkB,CAAC;AAEpE;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,sDAAsD;IACtD,EAAE,EAAE,MAAM,CAAC;IACX,0CAA0C;IAC1C,MAAM,EAAE,UAAU,CAAC;IACnB,yEAAyE;IACzE,UAAU,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,gDAAgD;IAChD,MAAM,EAAE,MAAM,CAAC;IACf,qDAAqD;IACrD,kBAAkB,EAAE,iBAAiB,GAAG,IAAI,CAAC;IAC7C,8CAA8C;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,yDAAyD;IACzD,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,iDAAiD;IACjD,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,mDAAmD;IACnD,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,yDAAyD;IACzD,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,gDAAgD;IAChD,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACtC,6EAA6E;IAC7E,KAAK,EAAE,YAAY,EAAE,CAAC;IACtB,qDAAqD;IACrD,WAAW,EAAE,MAAM,CAAC;IACpB,+DAA+D;IAC/D,UAAU,EAAE,MAAM,CAAC;IACnB,wEAAwE;IACxE,mBAAmB,EAAE,OAAO,CAAC;IAC7B,iCAAiC;IACjC,IAAI,EAAE,SAAS,CAAC;IAChB,sEAAsE;IACtE,KAAK,EAAE,MAAM,CAAC;IACd,0CAA0C;IAC1C,gBAAgB,EAAE,MAAM,CAAC;IACzB,2CAA2C;IAC3C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kEAAkE;IAClE,YAAY,EAAE,MAAM,CAAC;IACrB,gEAAgE;IAChE,WAAW,EAAE,MAAM,CAAC;IACpB,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC1C,8BAA8B;IAC9B,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IACpB,qCAAqC;IACrC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,kCAAkC;IAClC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,4DAA4D;IAC5D,KAAK,EAAE,OAAO,CAAC;IACf,sDAAsD;IACtD,UAAU,EAAE,OAAO,CAAC;IACpB,wDAAwD;IACxD,YAAY,EAAE,MAAM,CAAC;IACrB,+DAA+D;IAC/D,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,0CAA0C;IAC1C,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,4CAA4C;IAC5C,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,CAWnD"}

package/dist/lib/integrations/openresponses/index.js

@@ -29,3 +29,22 @@ export const functionCallOutputStatusEnum = {
     completed: "completed",
     incomplete: "incomplete",
 };
+/**
+ * Extracts the text content from an Output value that is a ResponseResource.
+ *
+ * Finds the first assistant message with `output_text` content and returns its text.
+ * Returns an empty string if the output is not a ResponseResource or contains no text.
+ *
+ * @param output - The output from a job (can be any Output type)
+ * @returns The extracted text string, or empty string if not found
+ */
+export function extractText(output) {
+    const res = output;
+    if (!res?.output)
+        return "";
+    const message = res.output.find((item) => item.type === "message");
+    if (!message)
+        return "";
+    const textContent = message.content.find((c) => c.type === "output_text");
+    return textContent?.text ?? "";
+}