@assay-ai/ai-sdk 0.1.0-beta

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Assay AI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/index.cjs

@@ -0,0 +1,150 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  fromGenerateText: () => fromGenerateText,
+  fromMessages: () => fromMessages,
+  fromStreamText: () => fromStreamText
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/adapter.ts
+function fromGenerateText(result, input, options) {
+  const testCase = {
+    input,
+    actualOutput: result.text
+  };
+  if (result.usage?.totalTokens != null) {
+    testCase.tokenCost = result.usage.totalTokens;
+  }
+  if (options?.expectedOutput != null) {
+    testCase.expectedOutput = options.expectedOutput;
+  }
+  if (options?.context != null) {
+    testCase.context = options.context;
+  }
+  if (options?.retrievalContext != null) {
+    testCase.retrievalContext = options.retrievalContext;
+  }
+  if (options?.name != null) {
+    testCase.name = options.name;
+  }
+  if (options?.tags != null) {
+    testCase.tags = options.tags;
+  }
+  return testCase;
+}
+async function fromStreamText(result, input, options) {
+  const [text, usage] = await Promise.all([
+    Promise.resolve(result.text),
+    result.usage != null ? Promise.resolve(result.usage) : Promise.resolve(void 0)
+  ]);
+  const testCase = {
+    input,
+    actualOutput: text
+  };
+  if (usage?.totalTokens != null) {
+    testCase.tokenCost = usage.totalTokens;
+  }
+  if (options?.expectedOutput != null) {
+    testCase.expectedOutput = options.expectedOutput;
+  }
+  if (options?.context != null) {
+    testCase.context = options.context;
+  }
+  if (options?.retrievalContext != null) {
+    testCase.retrievalContext = options.retrievalContext;
+  }
+  if (options?.name != null) {
+    testCase.name = options.name;
+  }
+  if (options?.tags != null) {
+    testCase.tags = options.tags;
+  }
+  return testCase;
+}
+function fromMessages(messages, options) {
+  if (messages.length === 0) {
+    throw new Error("fromMessages requires at least one message");
+  }
+  let lastUserMessage;
+  let lastAssistantMessage;
+  const contextParts = [];
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const msg = messages[i];
+    if (msg.role === "assistant" && lastAssistantMessage == null) {
+      lastAssistantMessage = msg.content;
+    } else if (msg.role === "user" && lastUserMessage == null) {
+      lastUserMessage = msg.content;
+    }
+    if (lastUserMessage != null && lastAssistantMessage != null) {
+      break;
+    }
+  }
+  let foundLastUser = false;
+  let foundLastAssistant = false;
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const msg = messages[i];
+    if (msg.role === "user" && msg.content === lastUserMessage && !foundLastUser) {
+      foundLastUser = true;
+      continue;
+    }
+    if (msg.role === "assistant" && msg.content === lastAssistantMessage && !foundLastAssistant) {
+      foundLastAssistant = true;
+      continue;
+    }
+    contextParts.unshift(`[${msg.role}]: ${msg.content}`);
+  }
+  const input = lastUserMessage ?? messages[0]?.content ?? "";
+  const testCase = {
+    input
+  };
+  if (lastAssistantMessage != null) {
+    testCase.actualOutput = lastAssistantMessage;
+  }
+  const allContext = [
+    ...contextParts.length > 0 ? contextParts : [],
+    ...options?.context ?? []
+  ];
+  if (allContext.length > 0) {
+    testCase.context = allContext;
+  }
+  if (options?.expectedOutput != null) {
+    testCase.expectedOutput = options.expectedOutput;
+  }
+  if (options?.retrievalContext != null) {
+    testCase.retrievalContext = options.retrievalContext;
+  }
+  if (options?.name != null) {
+    testCase.name = options.name;
+  }
+  if (options?.tags != null) {
+    testCase.tags = options.tags;
+  }
+  return testCase;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  fromGenerateText,
+  fromMessages,
+  fromStreamText
+});
+//# sourceMappingURL=index.cjs.map

package/dist/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts","../src/adapter.ts"],"sourcesContent":["export { fromGenerateText, fromStreamText, fromMessages } from \"./adapter\";\n","import type { LLMTestCase } from \"@assay-ai/core\";\n\n/**\n * A duck-typed interface matching the result of `generateText()` from the\n * Vercel AI SDK. We use structural typing so users don't need to install\n * a specific version of the AI SDK.\n */\ninterface GenerateTextResultLike {\n  text: string;\n  usage?: {\n    totalTokens?: number;\n    promptTokens?: number;\n    completionTokens?: number;\n  };\n  finishReason?: string;\n}\n\n/**\n * A duck-typed interface matching the result of `streamText()` from the\n * Vercel AI SDK. The `text` and `usage` properties may be promises that\n * resolve once the stream completes.\n */\ninterface StreamTextResultLike {\n  text: Promise<string> | string;\n  usage?:\n    | Promise<{ totalTokens?: number; promptTokens?: number; completionTokens?: number }>\n    | { totalTokens?: number; promptTokens?: number; completionTokens?: number };\n}\n\ninterface AdapterOptions {\n  /** Expected output for comparison-based metrics. */\n  expectedOutput?: string;\n  /** Context passages for retrieval-based metrics (e.g., RAG). */\n  context?: string[];\n  /** Retrieved context for faithfulness/hallucination evaluation. */\n  retrievalContext?: string[];\n  /** Human-readable name for the test case. */\n  name?: string;\n  /** Tags for filtering/grouping test cases. */\n  tags?: string[];\n}\n\n/**\n * Convert a `generateText()` result into an `LLMTestCase` for evaluation.\n *\n * @param result - The result from Vercel AI SDK's `generateText()`.\n * @param input - The original prompt/input that was sent to the model.\n * @param options - Additional fields for the test case.\n * @returns A fully populated `LLMTestCase`.\n *\n * @example\n * ```ts\n * import { generateText } from \"ai\";\n * import { fromGenerateText } from \"@assay-ai/ai-sdk\";\n *\n * const result = await generateText({ model, prompt: \"What is 2+2?\" });\n * const testCase = fromGenerateText(result, \"What is 2+2?\", {\n *   expectedOutput: \"4\",\n * });\n * ```\n */\nexport function fromGenerateText(\n  result: GenerateTextResultLike,\n  input: string,\n  options?: AdapterOptions,\n): LLMTestCase {\n  const testCase: LLMTestCase = {\n    input,\n    actualOutput: result.text,\n  };\n\n  if (result.usage?.totalTokens != null) {\n    testCase.tokenCost = result.usage.totalTokens;\n  }\n\n  if (options?.expectedOutput != null) {\n    testCase.expectedOutput = options.expectedOutput;\n  }\n  if (options?.context != null) {\n    testCase.context = options.context;\n  }\n  if (options?.retrievalContext != null) {\n    testCase.retrievalContext = options.retrievalContext;\n  }\n  if (options?.name != null) {\n    testCase.name = options.name;\n  }\n  if (options?.tags != null) {\n    testCase.tags = options.tags;\n  }\n\n  return testCase;\n}\n\n/**\n * Convert a `streamText()` result into an `LLMTestCase` for evaluation.\n * This awaits the streamed text and usage before building the test case.\n *\n * @param result - The result from Vercel AI SDK's `streamText()`.\n * @param input - The original prompt/input that was sent to the model.\n * @param options - Additional fields for the test case.\n * @returns A promise that resolves to a fully populated `LLMTestCase`.\n *\n * @example\n * ```ts\n * import { streamText } from \"ai\";\n * import { fromStreamText } from \"@assay-ai/ai-sdk\";\n *\n * const result = streamText({ model, prompt: \"Explain gravity.\" });\n * const testCase = await fromStreamText(result, \"Explain gravity.\");\n * ```\n */\nexport async function fromStreamText(\n  result: StreamTextResultLike,\n  input: string,\n  options?: AdapterOptions,\n): Promise<LLMTestCase> {\n  const [text, usage] = await Promise.all([\n    Promise.resolve(result.text),\n    result.usage != null ? Promise.resolve(result.usage) : Promise.resolve(undefined),\n  ]);\n\n  const testCase: LLMTestCase = {\n    input,\n    actualOutput: text,\n  };\n\n  if (usage?.totalTokens != null) {\n    testCase.tokenCost = usage.totalTokens;\n  }\n\n  if (options?.expectedOutput != null) {\n    testCase.expectedOutput = options.expectedOutput;\n  }\n  if (options?.context != null) {\n    testCase.context = options.context;\n  }\n  if (options?.retrievalContext != null) {\n    testCase.retrievalContext = options.retrievalContext;\n  }\n  if (options?.name != null) {\n    testCase.name = options.name;\n  }\n  if (options?.tags != null) {\n    testCase.tags = options.tags;\n  }\n\n  return testCase;\n}\n\n/**\n * Convert an array of chat messages into an `LLMTestCase`.\n * The last assistant message is treated as `actualOutput`, and the last\n * user message is treated as `input`. Earlier messages are combined into\n * context for reference.\n *\n * @param messages - Array of chat messages with role and content.\n * @param options - Additional fields for the test case.\n * @returns A fully populated `LLMTestCase`.\n *\n * @example\n * ```ts\n * import { fromMessages } from \"@assay-ai/ai-sdk\";\n *\n * const testCase = fromMessages([\n *   { role: \"user\", content: \"What is the capital of France?\" },\n *   { role: \"assistant\", content: \"The capital of France is Paris.\" },\n * ]);\n * ```\n */\nexport function fromMessages(\n  messages: Array<{ role: string; content: string }>,\n  options?: AdapterOptions,\n): LLMTestCase {\n  if (messages.length === 0) {\n    throw new Error(\"fromMessages requires at least one message\");\n  }\n\n  // Find the last user message and last assistant message\n  let lastUserMessage: string | undefined;\n  let lastAssistantMessage: string | undefined;\n  const contextParts: string[] = [];\n\n  // Walk backward to find the last assistant and user messages\n  for (let i = messages.length - 1; i >= 0; i--) {\n    const msg = messages[i]!;\n    if (msg.role === \"assistant\" && lastAssistantMessage == null) {\n      lastAssistantMessage = msg.content;\n    } else if (msg.role === \"user\" && lastUserMessage == null) {\n      lastUserMessage = msg.content;\n    }\n    if (lastUserMessage != null && lastAssistantMessage != null) {\n      break;\n    }\n  }\n\n  // Build conversation context from all messages except the ones we extracted\n  // as input/output\n  let foundLastUser = false;\n  let foundLastAssistant = false;\n\n  for (let i = messages.length - 1; i >= 0; i--) {\n    const msg = messages[i]!;\n    if (msg.role === \"user\" && msg.content === lastUserMessage && !foundLastUser) {\n      foundLastUser = true;\n      continue;\n    }\n    if (msg.role === \"assistant\" && msg.content === lastAssistantMessage && !foundLastAssistant) {\n      foundLastAssistant = true;\n      continue;\n    }\n    contextParts.unshift(`[${msg.role}]: ${msg.content}`);\n  }\n\n  const input = lastUserMessage ?? messages[0]?.content ?? \"\";\n\n  const testCase: LLMTestCase = {\n    input,\n  };\n\n  if (lastAssistantMessage != null) {\n    testCase.actualOutput = lastAssistantMessage;\n  }\n\n  // Merge conversation context with any user-provided context\n  const allContext = [\n    ...(contextParts.length > 0 ? contextParts : []),\n    ...(options?.context ?? []),\n  ];\n  if (allContext.length > 0) {\n    testCase.context = allContext;\n  }\n\n  if (options?.expectedOutput != null) {\n    testCase.expectedOutput = options.expectedOutput;\n  }\n  if (options?.retrievalContext != null) {\n    testCase.retrievalContext = options.retrievalContext;\n  }\n  if (options?.name != null) {\n    testCase.name = options.name;\n  }\n  if (options?.tags != null) {\n    testCase.tags = options.tags;\n  }\n\n  return testCase;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;AC6DO,SAAS,iBACd,QACA,OACA,SACa;AACb,QAAM,WAAwB;AAAA,IAC5B;AAAA,IACA,cAAc,OAAO;AAAA,EACvB;AAEA,MAAI,OAAO,OAAO,eAAe,MAAM;AACrC,aAAS,YAAY,OAAO,MAAM;AAAA,EACpC;AAEA,MAAI,SAAS,kBAAkB,MAAM;AACnC,aAAS,iBAAiB,QAAQ;AAAA,EACpC;AACA,MAAI,SAAS,WAAW,MAAM;AAC5B,aAAS,UAAU,QAAQ;AAAA,EAC7B;AACA,MAAI,SAAS,oBAAoB,MAAM;AACrC,aAAS,mBAAmB,QAAQ;AAAA,EACtC;AACA,MAAI,SAAS,QAAQ,MAAM;AACzB,aAAS,OAAO,QAAQ;AAAA,EAC1B;AACA,MAAI,SAAS,QAAQ,MAAM;AACzB,aAAS,OAAO,QAAQ;AAAA,EAC1B;AAEA,SAAO;AACT;AAoBA,eAAsB,eACpB,QACA,OACA,SACsB;AACtB,QAAM,CAAC,MAAM,KAAK,IAAI,MAAM,QAAQ,IAAI;AAAA,IACtC,QAAQ,QAAQ,OAAO,IAAI;AAAA,IAC3B,OAAO,SAAS,OAAO,QAAQ,QAAQ,OAAO,KAAK,IAAI,QAAQ,QAAQ,MAAS;AAAA,EAClF,CAAC;AAED,QAAM,WAAwB;AAAA,IAC5B;AAAA,IACA,cAAc;AAAA,EAChB;AAEA,MAAI,OAAO,eAAe,MAAM;AAC9B,aAAS,YAAY,MAAM;AAAA,EAC7B;AAEA,MAAI,SAAS,kBAAkB,MAAM;AACnC,aAAS,iBAAiB,QAAQ;AAAA,EACpC;AACA,MAAI,SAAS,WAAW,MAAM;AAC5B,aAAS,UAAU,QAAQ;AAAA,EAC7B;AACA,MAAI,SAAS,oBAAoB,MAAM;AACrC,aAAS,mBAAmB,QAAQ;AAAA,EACtC;AACA,MAAI,SAAS,QAAQ,MAAM;AACzB,aAAS,OAAO,QAAQ;AAAA,EAC1B;AACA,MAAI,SAAS,QAAQ,MAAM;AACzB,aAAS,OAAO,QAAQ;AAAA,EAC1B;AAEA,SAAO;AACT;AAsBO,SAAS,aACd,UACA,SACa;AACb,MAAI,SAAS,WAAW,GAAG;AACzB,UAAM,IAAI,MAAM,4CAA4C;AAAA,EAC9D;AAGA,MAAI;AACJ,MAAI;AACJ,QAAM,eAAyB,CAAC;AAGhC,WAAS,IAAI,SAAS,SAAS,GAAG,KAAK,GAAG,KAAK;AAC7C,UAAM,MAAM,SAAS,CAAC;AACtB,QAAI,IAAI,SAAS,eAAe,wBAAwB,MAAM;AAC5D,6BAAuB,IAAI;AAAA,IAC7B,WAAW,IAAI,SAAS,UAAU,mBAAmB,MAAM;AACzD,wBAAkB,IAAI;AAAA,IACxB;AACA,QAAI,mBAAmB,QAAQ,wBAAwB,MAAM;AAC3D;AAAA,IACF;AAAA,EACF;AAIA,MAAI,gBAAgB;AACpB,MAAI,qBAAqB;AAEzB,WAAS,IAAI,SAAS,SAAS,GAAG,KAAK,GAAG,KAAK;AAC7C,UAAM,MAAM,SAAS,CAAC;AACtB,QAAI,IAAI,SAAS,UAAU,IAAI,YAAY,mBAAmB,CAAC,eAAe;AAC5E,sBAAgB;AAChB;AAAA,IACF;AACA,QAAI,IAAI,SAAS,eAAe,IAAI,YAAY,wBAAwB,CAAC,oBAAoB;AAC3F,2BAAqB;AACrB;AAAA,IACF;AACA,iBAAa,QAAQ,IAAI,IAAI,IAAI,MAAM,IAAI,OAAO,EAAE;AAAA,EACtD;AAEA,QAAM,QAAQ,mBAAmB,SAAS,CAAC,GAAG,WAAW;AAEzD,QAAM,WAAwB;AAAA,IAC5B;AAAA,EACF;AAEA,MAAI,wBAAwB,MAAM;AAChC,aAAS,eAAe;AAAA,EAC1B;AAGA,QAAM,aAAa;AAAA,IACjB,GAAI,aAAa,SAAS,IAAI,eAAe,CAAC;AAAA,IAC9C,GAAI,SAAS,WAAW,CAAC;AAAA,EAC3B;AACA,MAAI,WAAW,SAAS,GAAG;AACzB,aAAS,UAAU;AAAA,EACrB;AAEA,MAAI,SAAS,kBAAkB,MAAM;AACnC,aAAS,iBAAiB,QAAQ;AAAA,EACpC;AACA,MAAI,SAAS,oBAAoB,MAAM;AACrC,aAAS,mBAAmB,QAAQ;AAAA,EACtC;AACA,MAAI,SAAS,QAAQ,MAAM;AACzB,aAAS,OAAO,QAAQ;AAAA,EAC1B;AACA,MAAI,SAAS,QAAQ,MAAM;AACzB,aAAS,OAAO,QAAQ;AAAA,EAC1B;AAEA,SAAO;AACT;","names":[]}

package/dist/index.d.cts

@@ -0,0 +1,110 @@
+import { LLMTestCase } from '@assay-ai/core';
+
+/**
+ * A duck-typed interface matching the result of `generateText()` from the
+ * Vercel AI SDK. We use structural typing so users don't need to install
+ * a specific version of the AI SDK.
+ */
+interface GenerateTextResultLike {
+    text: string;
+    usage?: {
+        totalTokens?: number;
+        promptTokens?: number;
+        completionTokens?: number;
+    };
+    finishReason?: string;
+}
+/**
+ * A duck-typed interface matching the result of `streamText()` from the
+ * Vercel AI SDK. The `text` and `usage` properties may be promises that
+ * resolve once the stream completes.
+ */
+interface StreamTextResultLike {
+    text: Promise<string> | string;
+    usage?: Promise<{
+        totalTokens?: number;
+        promptTokens?: number;
+        completionTokens?: number;
+    }> | {
+        totalTokens?: number;
+        promptTokens?: number;
+        completionTokens?: number;
+    };
+}
+interface AdapterOptions {
+    /** Expected output for comparison-based metrics. */
+    expectedOutput?: string;
+    /** Context passages for retrieval-based metrics (e.g., RAG). */
+    context?: string[];
+    /** Retrieved context for faithfulness/hallucination evaluation. */
+    retrievalContext?: string[];
+    /** Human-readable name for the test case. */
+    name?: string;
+    /** Tags for filtering/grouping test cases. */
+    tags?: string[];
+}
+/**
+ * Convert a `generateText()` result into an `LLMTestCase` for evaluation.
+ *
+ * @param result - The result from Vercel AI SDK's `generateText()`.
+ * @param input - The original prompt/input that was sent to the model.
+ * @param options - Additional fields for the test case.
+ * @returns A fully populated `LLMTestCase`.
+ *
+ * @example
+ * ```ts
+ * import { generateText } from "ai";
+ * import { fromGenerateText } from "@assay-ai/ai-sdk";
+ *
+ * const result = await generateText({ model, prompt: "What is 2+2?" });
+ * const testCase = fromGenerateText(result, "What is 2+2?", {
+ *   expectedOutput: "4",
+ * });
+ * ```
+ */
+declare function fromGenerateText(result: GenerateTextResultLike, input: string, options?: AdapterOptions): LLMTestCase;
+/**
+ * Convert a `streamText()` result into an `LLMTestCase` for evaluation.
+ * This awaits the streamed text and usage before building the test case.
+ *
+ * @param result - The result from Vercel AI SDK's `streamText()`.
+ * @param input - The original prompt/input that was sent to the model.
+ * @param options - Additional fields for the test case.
+ * @returns A promise that resolves to a fully populated `LLMTestCase`.
+ *
+ * @example
+ * ```ts
+ * import { streamText } from "ai";
+ * import { fromStreamText } from "@assay-ai/ai-sdk";
+ *
+ * const result = streamText({ model, prompt: "Explain gravity." });
+ * const testCase = await fromStreamText(result, "Explain gravity.");
+ * ```
+ */
+declare function fromStreamText(result: StreamTextResultLike, input: string, options?: AdapterOptions): Promise<LLMTestCase>;
+/**
+ * Convert an array of chat messages into an `LLMTestCase`.
+ * The last assistant message is treated as `actualOutput`, and the last
+ * user message is treated as `input`. Earlier messages are combined into
+ * context for reference.
+ *
+ * @param messages - Array of chat messages with role and content.
+ * @param options - Additional fields for the test case.
+ * @returns A fully populated `LLMTestCase`.
+ *
+ * @example
+ * ```ts
+ * import { fromMessages } from "@assay-ai/ai-sdk";
+ *
+ * const testCase = fromMessages([
+ *   { role: "user", content: "What is the capital of France?" },
+ *   { role: "assistant", content: "The capital of France is Paris." },
+ * ]);
+ * ```
+ */
+declare function fromMessages(messages: Array<{
+    role: string;
+    content: string;
+}>, options?: AdapterOptions): LLMTestCase;
+
+export { fromGenerateText, fromMessages, fromStreamText };

package/dist/index.d.ts

@@ -0,0 +1,110 @@
+import { LLMTestCase } from '@assay-ai/core';
+
+/**
+ * A duck-typed interface matching the result of `generateText()` from the
+ * Vercel AI SDK. We use structural typing so users don't need to install
+ * a specific version of the AI SDK.
+ */
+interface GenerateTextResultLike {
+    text: string;
+    usage?: {
+        totalTokens?: number;
+        promptTokens?: number;
+        completionTokens?: number;
+    };
+    finishReason?: string;
+}
+/**
+ * A duck-typed interface matching the result of `streamText()` from the
+ * Vercel AI SDK. The `text` and `usage` properties may be promises that
+ * resolve once the stream completes.
+ */
+interface StreamTextResultLike {
+    text: Promise<string> | string;
+    usage?: Promise<{
+        totalTokens?: number;
+        promptTokens?: number;
+        completionTokens?: number;
+    }> | {
+        totalTokens?: number;
+        promptTokens?: number;
+        completionTokens?: number;
+    };
+}
+interface AdapterOptions {
+    /** Expected output for comparison-based metrics. */
+    expectedOutput?: string;
+    /** Context passages for retrieval-based metrics (e.g., RAG). */
+    context?: string[];
+    /** Retrieved context for faithfulness/hallucination evaluation. */
+    retrievalContext?: string[];
+    /** Human-readable name for the test case. */
+    name?: string;
+    /** Tags for filtering/grouping test cases. */
+    tags?: string[];
+}
+/**
+ * Convert a `generateText()` result into an `LLMTestCase` for evaluation.
+ *
+ * @param result - The result from Vercel AI SDK's `generateText()`.
+ * @param input - The original prompt/input that was sent to the model.
+ * @param options - Additional fields for the test case.
+ * @returns A fully populated `LLMTestCase`.
+ *
+ * @example
+ * ```ts
+ * import { generateText } from "ai";
+ * import { fromGenerateText } from "@assay-ai/ai-sdk";
+ *
+ * const result = await generateText({ model, prompt: "What is 2+2?" });
+ * const testCase = fromGenerateText(result, "What is 2+2?", {
+ *   expectedOutput: "4",
+ * });
+ * ```
+ */
+declare function fromGenerateText(result: GenerateTextResultLike, input: string, options?: AdapterOptions): LLMTestCase;
+/**
+ * Convert a `streamText()` result into an `LLMTestCase` for evaluation.
+ * This awaits the streamed text and usage before building the test case.
+ *
+ * @param result - The result from Vercel AI SDK's `streamText()`.
+ * @param input - The original prompt/input that was sent to the model.
+ * @param options - Additional fields for the test case.
+ * @returns A promise that resolves to a fully populated `LLMTestCase`.
+ *
+ * @example
+ * ```ts
+ * import { streamText } from "ai";
+ * import { fromStreamText } from "@assay-ai/ai-sdk";
+ *
+ * const result = streamText({ model, prompt: "Explain gravity." });
+ * const testCase = await fromStreamText(result, "Explain gravity.");
+ * ```
+ */
+declare function fromStreamText(result: StreamTextResultLike, input: string, options?: AdapterOptions): Promise<LLMTestCase>;
+/**
+ * Convert an array of chat messages into an `LLMTestCase`.
+ * The last assistant message is treated as `actualOutput`, and the last
+ * user message is treated as `input`. Earlier messages are combined into
+ * context for reference.
+ *
+ * @param messages - Array of chat messages with role and content.
+ * @param options - Additional fields for the test case.
+ * @returns A fully populated `LLMTestCase`.
+ *
+ * @example
+ * ```ts
+ * import { fromMessages } from "@assay-ai/ai-sdk";
+ *
+ * const testCase = fromMessages([
+ *   { role: "user", content: "What is the capital of France?" },
+ *   { role: "assistant", content: "The capital of France is Paris." },
+ * ]);
+ * ```
+ */
+declare function fromMessages(messages: Array<{
+    role: string;
+    content: string;
+}>, options?: AdapterOptions): LLMTestCase;
+
+export { fromGenerateText, fromMessages, fromStreamText };

package/dist/index.js

@@ -0,0 +1,121 @@
+// src/adapter.ts
+function fromGenerateText(result, input, options) {
+  const testCase = {
+    input,
+    actualOutput: result.text
+  };
+  if (result.usage?.totalTokens != null) {
+    testCase.tokenCost = result.usage.totalTokens;
+  }
+  if (options?.expectedOutput != null) {
+    testCase.expectedOutput = options.expectedOutput;
+  }
+  if (options?.context != null) {
+    testCase.context = options.context;
+  }
+  if (options?.retrievalContext != null) {
+    testCase.retrievalContext = options.retrievalContext;
+  }
+  if (options?.name != null) {
+    testCase.name = options.name;
+  }
+  if (options?.tags != null) {
+    testCase.tags = options.tags;
+  }
+  return testCase;
+}
+async function fromStreamText(result, input, options) {
+  const [text, usage] = await Promise.all([
+    Promise.resolve(result.text),
+    result.usage != null ? Promise.resolve(result.usage) : Promise.resolve(void 0)
+  ]);
+  const testCase = {
+    input,
+    actualOutput: text
+  };
+  if (usage?.totalTokens != null) {
+    testCase.tokenCost = usage.totalTokens;
+  }
+  if (options?.expectedOutput != null) {
+    testCase.expectedOutput = options.expectedOutput;
+  }
+  if (options?.context != null) {
+    testCase.context = options.context;
+  }
+  if (options?.retrievalContext != null) {
+    testCase.retrievalContext = options.retrievalContext;
+  }
+  if (options?.name != null) {
+    testCase.name = options.name;
+  }
+  if (options?.tags != null) {
+    testCase.tags = options.tags;
+  }
+  return testCase;
+}
+function fromMessages(messages, options) {
+  if (messages.length === 0) {
+    throw new Error("fromMessages requires at least one message");
+  }
+  let lastUserMessage;
+  let lastAssistantMessage;
+  const contextParts = [];
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const msg = messages[i];
+    if (msg.role === "assistant" && lastAssistantMessage == null) {
+      lastAssistantMessage = msg.content;
+    } else if (msg.role === "user" && lastUserMessage == null) {
+      lastUserMessage = msg.content;
+    }
+    if (lastUserMessage != null && lastAssistantMessage != null) {
+      break;
+    }
+  }
+  let foundLastUser = false;
+  let foundLastAssistant = false;
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const msg = messages[i];
+    if (msg.role === "user" && msg.content === lastUserMessage && !foundLastUser) {
+      foundLastUser = true;
+      continue;
+    }
+    if (msg.role === "assistant" && msg.content === lastAssistantMessage && !foundLastAssistant) {
+      foundLastAssistant = true;
+      continue;
+    }
+    contextParts.unshift(`[${msg.role}]: ${msg.content}`);
+  }
+  const input = lastUserMessage ?? messages[0]?.content ?? "";
+  const testCase = {
+    input
+  };
+  if (lastAssistantMessage != null) {
+    testCase.actualOutput = lastAssistantMessage;
+  }
+  const allContext = [
+    ...contextParts.length > 0 ? contextParts : [],
+    ...options?.context ?? []
+  ];
+  if (allContext.length > 0) {
+    testCase.context = allContext;
+  }
+  if (options?.expectedOutput != null) {
+    testCase.expectedOutput = options.expectedOutput;
+  }
+  if (options?.retrievalContext != null) {
+    testCase.retrievalContext = options.retrievalContext;
+  }
+  if (options?.name != null) {
+    testCase.name = options.name;
+  }
+  if (options?.tags != null) {
+    testCase.tags = options.tags;
+  }
+  return testCase;
+}
+export {
+  fromGenerateText,
+  fromMessages,
+  fromStreamText
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/adapter.ts"],"sourcesContent":["import type { LLMTestCase } from \"@assay-ai/core\";\n\n/**\n * A duck-typed interface matching the result of `generateText()` from the\n * Vercel AI SDK. We use structural typing so users don't need to install\n * a specific version of the AI SDK.\n */\ninterface GenerateTextResultLike {\n  text: string;\n  usage?: {\n    totalTokens?: number;\n    promptTokens?: number;\n    completionTokens?: number;\n  };\n  finishReason?: string;\n}\n\n/**\n * A duck-typed interface matching the result of `streamText()` from the\n * Vercel AI SDK. The `text` and `usage` properties may be promises that\n * resolve once the stream completes.\n */\ninterface StreamTextResultLike {\n  text: Promise<string> | string;\n  usage?:\n    | Promise<{ totalTokens?: number; promptTokens?: number; completionTokens?: number }>\n    | { totalTokens?: number; promptTokens?: number; completionTokens?: number };\n}\n\ninterface AdapterOptions {\n  /** Expected output for comparison-based metrics. */\n  expectedOutput?: string;\n  /** Context passages for retrieval-based metrics (e.g., RAG). */\n  context?: string[];\n  /** Retrieved context for faithfulness/hallucination evaluation. */\n  retrievalContext?: string[];\n  /** Human-readable name for the test case. */\n  name?: string;\n  /** Tags for filtering/grouping test cases. */\n  tags?: string[];\n}\n\n/**\n * Convert a `generateText()` result into an `LLMTestCase` for evaluation.\n *\n * @param result - The result from Vercel AI SDK's `generateText()`.\n * @param input - The original prompt/input that was sent to the model.\n * @param options - Additional fields for the test case.\n * @returns A fully populated `LLMTestCase`.\n *\n * @example\n * ```ts\n * import { generateText } from \"ai\";\n * import { fromGenerateText } from \"@assay-ai/ai-sdk\";\n *\n * const result = await generateText({ model, prompt: \"What is 2+2?\" });\n * const testCase = fromGenerateText(result, \"What is 2+2?\", {\n *   expectedOutput: \"4\",\n * });\n * ```\n */\nexport function fromGenerateText(\n  result: GenerateTextResultLike,\n  input: string,\n  options?: AdapterOptions,\n): LLMTestCase {\n  const testCase: LLMTestCase = {\n    input,\n    actualOutput: result.text,\n  };\n\n  if (result.usage?.totalTokens != null) {\n    testCase.tokenCost = result.usage.totalTokens;\n  }\n\n  if (options?.expectedOutput != null) {\n    testCase.expectedOutput = options.expectedOutput;\n  }\n  if (options?.context != null) {\n    testCase.context = options.context;\n  }\n  if (options?.retrievalContext != null) {\n    testCase.retrievalContext = options.retrievalContext;\n  }\n  if (options?.name != null) {\n    testCase.name = options.name;\n  }\n  if (options?.tags != null) {\n    testCase.tags = options.tags;\n  }\n\n  return testCase;\n}\n\n/**\n * Convert a `streamText()` result into an `LLMTestCase` for evaluation.\n * This awaits the streamed text and usage before building the test case.\n *\n * @param result - The result from Vercel AI SDK's `streamText()`.\n * @param input - The original prompt/input that was sent to the model.\n * @param options - Additional fields for the test case.\n * @returns A promise that resolves to a fully populated `LLMTestCase`.\n *\n * @example\n * ```ts\n * import { streamText } from \"ai\";\n * import { fromStreamText } from \"@assay-ai/ai-sdk\";\n *\n * const result = streamText({ model, prompt: \"Explain gravity.\" });\n * const testCase = await fromStreamText(result, \"Explain gravity.\");\n * ```\n */\nexport async function fromStreamText(\n  result: StreamTextResultLike,\n  input: string,\n  options?: AdapterOptions,\n): Promise<LLMTestCase> {\n  const [text, usage] = await Promise.all([\n    Promise.resolve(result.text),\n    result.usage != null ? Promise.resolve(result.usage) : Promise.resolve(undefined),\n  ]);\n\n  const testCase: LLMTestCase = {\n    input,\n    actualOutput: text,\n  };\n\n  if (usage?.totalTokens != null) {\n    testCase.tokenCost = usage.totalTokens;\n  }\n\n  if (options?.expectedOutput != null) {\n    testCase.expectedOutput = options.expectedOutput;\n  }\n  if (options?.context != null) {\n    testCase.context = options.context;\n  }\n  if (options?.retrievalContext != null) {\n    testCase.retrievalContext = options.retrievalContext;\n  }\n  if (options?.name != null) {\n    testCase.name = options.name;\n  }\n  if (options?.tags != null) {\n    testCase.tags = options.tags;\n  }\n\n  return testCase;\n}\n\n/**\n * Convert an array of chat messages into an `LLMTestCase`.\n * The last assistant message is treated as `actualOutput`, and the last\n * user message is treated as `input`. Earlier messages are combined into\n * context for reference.\n *\n * @param messages - Array of chat messages with role and content.\n * @param options - Additional fields for the test case.\n * @returns A fully populated `LLMTestCase`.\n *\n * @example\n * ```ts\n * import { fromMessages } from \"@assay-ai/ai-sdk\";\n *\n * const testCase = fromMessages([\n *   { role: \"user\", content: \"What is the capital of France?\" },\n *   { role: \"assistant\", content: \"The capital of France is Paris.\" },\n * ]);\n * ```\n */\nexport function fromMessages(\n  messages: Array<{ role: string; content: string }>,\n  options?: AdapterOptions,\n): LLMTestCase {\n  if (messages.length === 0) {\n    throw new Error(\"fromMessages requires at least one message\");\n  }\n\n  // Find the last user message and last assistant message\n  let lastUserMessage: string | undefined;\n  let lastAssistantMessage: string | undefined;\n  const contextParts: string[] = [];\n\n  // Walk backward to find the last assistant and user messages\n  for (let i = messages.length - 1; i >= 0; i--) {\n    const msg = messages[i]!;\n    if (msg.role === \"assistant\" && lastAssistantMessage == null) {\n      lastAssistantMessage = msg.content;\n    } else if (msg.role === \"user\" && lastUserMessage == null) {\n      lastUserMessage = msg.content;\n    }\n    if (lastUserMessage != null && lastAssistantMessage != null) {\n      break;\n    }\n  }\n\n  // Build conversation context from all messages except the ones we extracted\n  // as input/output\n  let foundLastUser = false;\n  let foundLastAssistant = false;\n\n  for (let i = messages.length - 1; i >= 0; i--) {\n    const msg = messages[i]!;\n    if (msg.role === \"user\" && msg.content === lastUserMessage && !foundLastUser) {\n      foundLastUser = true;\n      continue;\n    }\n    if (msg.role === \"assistant\" && msg.content === lastAssistantMessage && !foundLastAssistant) {\n      foundLastAssistant = true;\n      continue;\n    }\n    contextParts.unshift(`[${msg.role}]: ${msg.content}`);\n  }\n\n  const input = lastUserMessage ?? messages[0]?.content ?? \"\";\n\n  const testCase: LLMTestCase = {\n    input,\n  };\n\n  if (lastAssistantMessage != null) {\n    testCase.actualOutput = lastAssistantMessage;\n  }\n\n  // Merge conversation context with any user-provided context\n  const allContext = [\n    ...(contextParts.length > 0 ? contextParts : []),\n    ...(options?.context ?? []),\n  ];\n  if (allContext.length > 0) {\n    testCase.context = allContext;\n  }\n\n  if (options?.expectedOutput != null) {\n    testCase.expectedOutput = options.expectedOutput;\n  }\n  if (options?.retrievalContext != null) {\n    testCase.retrievalContext = options.retrievalContext;\n  }\n  if (options?.name != null) {\n    testCase.name = options.name;\n  }\n  if (options?.tags != null) {\n    testCase.tags = options.tags;\n  }\n\n  return testCase;\n}\n"],"mappings":";AA6DO,SAAS,iBACd,QACA,OACA,SACa;AACb,QAAM,WAAwB;AAAA,IAC5B;AAAA,IACA,cAAc,OAAO;AAAA,EACvB;AAEA,MAAI,OAAO,OAAO,eAAe,MAAM;AACrC,aAAS,YAAY,OAAO,MAAM;AAAA,EACpC;AAEA,MAAI,SAAS,kBAAkB,MAAM;AACnC,aAAS,iBAAiB,QAAQ;AAAA,EACpC;AACA,MAAI,SAAS,WAAW,MAAM;AAC5B,aAAS,UAAU,QAAQ;AAAA,EAC7B;AACA,MAAI,SAAS,oBAAoB,MAAM;AACrC,aAAS,mBAAmB,QAAQ;AAAA,EACtC;AACA,MAAI,SAAS,QAAQ,MAAM;AACzB,aAAS,OAAO,QAAQ;AAAA,EAC1B;AACA,MAAI,SAAS,QAAQ,MAAM;AACzB,aAAS,OAAO,QAAQ;AAAA,EAC1B;AAEA,SAAO;AACT;AAoBA,eAAsB,eACpB,QACA,OACA,SACsB;AACtB,QAAM,CAAC,MAAM,KAAK,IAAI,MAAM,QAAQ,IAAI;AAAA,IACtC,QAAQ,QAAQ,OAAO,IAAI;AAAA,IAC3B,OAAO,SAAS,OAAO,QAAQ,QAAQ,OAAO,KAAK,IAAI,QAAQ,QAAQ,MAAS;AAAA,EAClF,CAAC;AAED,QAAM,WAAwB;AAAA,IAC5B;AAAA,IACA,cAAc;AAAA,EAChB;AAEA,MAAI,OAAO,eAAe,MAAM;AAC9B,aAAS,YAAY,MAAM;AAAA,EAC7B;AAEA,MAAI,SAAS,kBAAkB,MAAM;AACnC,aAAS,iBAAiB,QAAQ;AAAA,EACpC;AACA,MAAI,SAAS,WAAW,MAAM;AAC5B,aAAS,UAAU,QAAQ;AAAA,EAC7B;AACA,MAAI,SAAS,oBAAoB,MAAM;AACrC,aAAS,mBAAmB,QAAQ;AAAA,EACtC;AACA,MAAI,SAAS,QAAQ,MAAM;AACzB,aAAS,OAAO,QAAQ;AAAA,EAC1B;AACA,MAAI,SAAS,QAAQ,MAAM;AACzB,aAAS,OAAO,QAAQ;AAAA,EAC1B;AAEA,SAAO;AACT;AAsBO,SAAS,aACd,UACA,SACa;AACb,MAAI,SAAS,WAAW,GAAG;AACzB,UAAM,IAAI,MAAM,4CAA4C;AAAA,EAC9D;AAGA,MAAI;AACJ,MAAI;AACJ,QAAM,eAAyB,CAAC;AAGhC,WAAS,IAAI,SAAS,SAAS,GAAG,KAAK,GAAG,KAAK;AAC7C,UAAM,MAAM,SAAS,CAAC;AACtB,QAAI,IAAI,SAAS,eAAe,wBAAwB,MAAM;AAC5D,6BAAuB,IAAI;AAAA,IAC7B,WAAW,IAAI,SAAS,UAAU,mBAAmB,MAAM;AACzD,wBAAkB,IAAI;AAAA,IACxB;AACA,QAAI,mBAAmB,QAAQ,wBAAwB,MAAM;AAC3D;AAAA,IACF;AAAA,EACF;AAIA,MAAI,gBAAgB;AACpB,MAAI,qBAAqB;AAEzB,WAAS,IAAI,SAAS,SAAS,GAAG,KAAK,GAAG,KAAK;AAC7C,UAAM,MAAM,SAAS,CAAC;AACtB,QAAI,IAAI,SAAS,UAAU,IAAI,YAAY,mBAAmB,CAAC,eAAe;AAC5E,sBAAgB;AAChB;AAAA,IACF;AACA,QAAI,IAAI,SAAS,eAAe,IAAI,YAAY,wBAAwB,CAAC,oBAAoB;AAC3F,2BAAqB;AACrB;AAAA,IACF;AACA,iBAAa,QAAQ,IAAI,IAAI,IAAI,MAAM,IAAI,OAAO,EAAE;AAAA,EACtD;AAEA,QAAM,QAAQ,mBAAmB,SAAS,CAAC,GAAG,WAAW;AAEzD,QAAM,WAAwB;AAAA,IAC5B;AAAA,EACF;AAEA,MAAI,wBAAwB,MAAM;AAChC,aAAS,eAAe;AAAA,EAC1B;AAGA,QAAM,aAAa;AAAA,IACjB,GAAI,aAAa,SAAS,IAAI,eAAe,CAAC;AAAA,IAC9C,GAAI,SAAS,WAAW,CAAC;AAAA,EAC3B;AACA,MAAI,WAAW,SAAS,GAAG;AACzB,aAAS,UAAU;AAAA,EACrB;AAEA,MAAI,SAAS,kBAAkB,MAAM;AACnC,aAAS,iBAAiB,QAAQ;AAAA,EACpC;AACA,MAAI,SAAS,oBAAoB,MAAM;AACrC,aAAS,mBAAmB,QAAQ;AAAA,EACtC;AACA,MAAI,SAAS,QAAQ,MAAM;AACzB,aAAS,OAAO,QAAQ;AAAA,EAC1B;AACA,MAAI,SAAS,QAAQ,MAAM;AACzB,aAAS,OAAO,QAAQ;AAAA,EAC1B;AAEA,SAAO;AACT;","names":[]}

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@assay-ai/ai-sdk",
+  "version": "0.1.0-beta",
+  "description": "Vercel AI SDK adapter for the Assay LLM evaluation framework",
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist/index.d.cts",
+        "default": "./dist/index.cjs"
+      }
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "peerDependencies": {
+    "ai": ">=4.0.0",
+    "@assay-ai/core": "0.1.0-beta"
+  },
+  "devDependencies": {
+    "ai": "^4.0.0",
+    "tsup": "^8.3.0",
+    "typescript": "^5.7.0",
+    "@assay-ai/tsconfig": "0.0.0",
+    "@assay-ai/core": "0.1.0-beta"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/assay-ai/assay",
+    "directory": "packages/ai-sdk"
+  },
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "typecheck": "tsc --noEmit",
+    "clean": "rm -rf dist .turbo"
+  }
+}