@syntheticlab/synbad 0.0.5 → 0.0.7

package/dist/evals/tools/octo-list-no-optional-args.js

@@ -0,0 +1,73 @@
+import * as assert from "../../source/asserts.js";
+export function test({ tool_calls }) {
+    assert.isNotNullish(tool_calls);
+    assert.isNotEmptyArray(tool_calls);
+    assert.strictEqual(tool_calls.length, 1);
+    assert.strictEqual(tool_calls[0].type, "function");
+    const parsed = JSON.parse(tool_calls[0].function.arguments);
+    assert.or(() => assert.isNullish(parsed), () => assert.deepEqual(parsed, {}), () => assert.isNullish(parsed.dirPath), () => assert.strictEqual(parsed.dirPath, ""));
+}
+export const json = {
+    "messages": [
+        {
+            "role": "system",
+            "content": "You are a coding assistant called Octo."
+        },
+        {
+            "role": "user",
+            "content": "call the list tool with no args"
+        }
+    ],
+    tools: [
+        {
+            "type": "function",
+            "function": {
+                "name": "append",
+                "description": "The append tool",
+                "parameters": {
+                    "type": "object",
+                    "required": ["filePath", "text"],
+                    "properties": {
+                        "filePath": {
+                            "description": "The path to the file",
+                            "type": "string"
+                        },
+                        "text": {
+                            "description": "The text to append",
+                            "type": "string"
+                        }
+                    }
+                },
+                "strict": true
+            }
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "create",
+                "description": "The create tool",
+                "parameters": {
+                    "type": "object",
+                    "required": ["filePath", "content"],
+                    "properties": {
+                        "filePath": {
+                            "description": "Path where the file should be created",
+                            "type": "string"
+                        }, "content": {
+                            "description": "Content to write to the file",
+                            "type": "string"
+                        }
+                    }
+                },
+                "strict": true
+            }
+        },
+        { "type": "function", "function": { "name": "edit", "description": "The edit tool", "parameters": { "type": "object", "required": ["filePath", "search", "replace"], "properties": { "filePath": { "description": "The path to the file", "type": "string" }, "search": { "description": "The search string to replace. Must EXACTLY match the text you intend to replace, including\nwhitespace, punctuation, etc. Make sure to give a few lines of context above and below so you\ndon't accidentally replace a different matching substring in the same file.", "type": "string" }, "replace": { "description": "The string you want to insert into the file", "type": "string" } } }, "strict": true } },
+        { "type": "function", "function": { "name": "fetch", "description": "The fetch tool", "parameters": { "type": "object", "required": ["url"], "properties": { "url": { "description": "Full url to fetch, e.g. https://...", "type": "string" }, "includeMarkup": { "description": "Include the HTML markup? Defaults to false. By default or when set to false, markup will be\nstripped and converted to plain text. Prefer markup stripping, and only set this to true if the\noutput is confusing: otherwise you may download a massive amount of data", "type": "boolean" } } }, "strict": true } },
+        { "type": "function", "function": { "name": "list", "description": "The list tool", "parameters": { "type": "object", "required": [], "properties": { "dirPath": { "description": "Path to the directory", "type": "string" } } }, "strict": true } },
+        { "type": "function", "function": { "name": "prepend", "description": "The prepend tool", "parameters": { "type": "object", "required": ["filePath", "text"], "properties": { "filePath": { "description": "The path to the file", "type": "string" }, "text": { "description": "The text to prepend", "type": "string" } } }, "strict": true } },
+        { "type": "function", "function": { "name": "read", "description": "The read tool", "parameters": { "type": "object", "required": ["filePath"], "properties": { "filePath": { "description": "Path to file to read", "type": "string" } } }, "strict": true } },
+        { "type": "function", "function": { "name": "rewrite", "description": "The rewrite tool", "parameters": { "type": "object", "required": ["filePath", "text"], "properties": { "filePath": { "description": "The path to the file", "type": "string" }, "text": { "description": "The replaced file contents. This will rewrite and replace the entire file", "type": "string" } } }, "strict": true } },
+        { "type": "function", "function": { "name": "shell", "description": "The shell tool", "parameters": { "type": "object", "required": ["timeout", "cmd"], "properties": { "timeout": { "description": "A timeout for the command, in milliseconds. Be generous. You MUST specify this.", "type": "number" }, "cmd": { "description": "The command to run", "type": "string" } } }, "strict": true } }
+    ],
+};

package/dist/evals/tools/parallel-tool.d.ts

@@ -1,5 +1,5 @@
-import OpenAI from "openai";
-export declare function test(response: OpenAI.ChatCompletion): void;
+import { ChatMessage } from "../../source/chat-completion.ts";
+export declare function test({ tool_calls }: ChatMessage): void;
 export declare const json: {
     messages: {
         role: string;

package/dist/evals/tools/parallel-tool.js

@@ -1,6 +1,5 @@
 import * as assert from "../../source/asserts.js";
-export function test(response) {
-    const { tool_calls } = response.choices[0].message;
+export function test({ tool_calls }) {
     assert.isNotNullish(tool_calls);
     assert.isNotEmptyArray(tool_calls);
     assert.strictEqual(tool_calls.length, 2);

package/dist/evals/tools/simple-tool.d.ts

@@ -1,5 +1,5 @@
-import { ChatResponse } from "../../source/chat-completion.ts";
-export declare function test(response: ChatResponse): void;
+import { ChatMessage } from "../../source/chat-completion.ts";
+export declare function test({ content, tool_calls }: ChatMessage): void;
 export declare const json: {
     messages: {
         role: string;

package/dist/evals/tools/simple-tool.js

@@ -1,6 +1,5 @@
 import * as assert from "../../source/asserts.js";
-export function test(response) {
-    const { tool_calls } = response.choices[0].message;
+export function test({ content, tool_calls }) {
     assert.isNotNullish(tool_calls);
     assert.isNotEmptyArray(tool_calls);
     assert.strictEqual(tool_calls.length, 1);
@@ -8,6 +7,8 @@ export function test(response) {
     assert.strictEqual(tool_calls[0].function.name, "get_weather");
     const args = JSON.parse(tool_calls[0].function.arguments);
     assert.match(args.location.toLowerCase(), /paris/);
+    // Assert the tool call didn't leak into the content
+    assert.doesNotMatch(content || "", /get_weather/);
 }
 export const json = {
     "messages": [

package/dist/evals/tools/tool-dash-underscore.d.ts

@@ -0,0 +1,26 @@
+import { ChatMessage } from "../../source/chat-completion.ts";
+export declare function test({ content, tool_calls }: ChatMessage): void;
+export declare const json: {
+    messages: {
+        role: string;
+        content: string;
+    }[];
+    tools: {
+        type: string;
+        function: {
+            name: string;
+            description: string;
+            parameters: {
+                type: string;
+                properties: {
+                    location: {
+                        type: string;
+                        description: string;
+                    };
+                };
+                required: string[];
+            };
+        };
+    }[];
+    tool_choice: string;
+};

package/dist/evals/tools/tool-dash-underscore.js

@@ -0,0 +1,37 @@
+import * as assert from "../../source/asserts.js";
+export function test({ content, tool_calls }) {
+    assert.isNotNullish(tool_calls);
+    assert.isNotEmptyArray(tool_calls);
+    assert.strictEqual(tool_calls.length, 1);
+    assert.strictEqual(tool_calls[0].type, "function");
+    assert.strictEqual(tool_calls[0].function.name, "get-weather__v1");
+    const args = JSON.parse(tool_calls[0].function.arguments);
+    assert.match(args.location.toLowerCase(), /paris/);
+    // Assert the tool call didn't leak into the content
+    assert.doesNotMatch(content || "", /get_weather/);
+}
+export const json = {
+    "messages": [
+        { "role": "user", "content": "What's the weather in Paris?" }
+    ],
+    "tools": [
+        {
+            "type": "function",
+            "function": {
+                "name": "get-weather__v1",
+                "description": "Get current weather for a location",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "location": {
+                            "type": "string",
+                            "description": "City name"
+                        }
+                    },
+                    "required": ["location"]
+                }
+            }
+        }
+    ],
+    "tool_choice": "auto",
+};

package/dist/evals/tools/tool-path-corruption.d.ts

@@ -0,0 +1,26 @@
+import { ChatMessage } from "../../source/chat-completion.ts";
+export declare function test({ tool_calls }: ChatMessage): void;
+export declare const json: {
+    messages: {
+        role: string;
+        content: string;
+    }[];
+    tools: {
+        type: string;
+        function: {
+            name: string;
+            description: string;
+            parameters: {
+                type: string;
+                required: string[];
+                properties: {
+                    filePath: {
+                        description: string;
+                        type: string;
+                    };
+                };
+            };
+            strict: boolean;
+        };
+    }[];
+};

package/dist/evals/tools/tool-path-corruption.js

@@ -0,0 +1,41 @@
+import * as assert from "../../source/asserts.js";
+const PATH = "/development/evals/reasoning/Scratch/reasoning-claude-tool-call.ts";
+export function test({ tool_calls }) {
+    assert.isNotNullish(tool_calls);
+    assert.isNotEmptyArray(tool_calls);
+    assert.strictEqual(tool_calls.length, 1);
+    assert.strictEqual(tool_calls[0].type, "function");
+    assert.strictEqual(tool_calls[0].function.name, "read");
+    const args = JSON.parse(tool_calls[0].function.arguments);
+    assert.stringContains(args.filePath, PATH);
+}
+export const json = {
+    "messages": [
+        {
+            "role": "user",
+            "content": "Read and summarize the file /development/evals/reasoning/Scratch/reasoning-claude-tool-call.ts"
+        }
+    ],
+    "tools": [
+        {
+            "type": "function",
+            "function": {
+                "name": "read",
+                "description": "The read tool",
+                "parameters": {
+                    "type": "object",
+                    "required": [
+                        "filePath"
+                    ],
+                    "properties": {
+                        "filePath": {
+                            "description": "Path to file to read",
+                            "type": "string"
+                        }
+                    }
+                },
+                "strict": true
+            }
+        },
+    ],
+};

package/dist/source/asserts.d.ts

@@ -3,5 +3,8 @@ export declare function or(a: () => void, ...rest: Array<() => void>): void;
 export declare function isNullish(a: unknown): asserts a is null | undefined;
 export declare function isNotNullish<T extends any>(a: T): asserts a is Exclude<T, null | undefined>;
 export declare function isEmptyArray(a: any[]): boolean;
-export declare function isNotEmptyArray(a: any[]): boolean;
+export declare function isNotEmptyArray(a: any[] | undefined): boolean;
 export declare function startsWith(a: string, prefix: string): boolean;
+export declare function gt(num: number, target: number): boolean;
+export declare function gte(num: number, target: number): boolean;
+export declare function stringContains(str: string, expected: string): boolean;

package/dist/source/asserts.js

@@ -45,6 +45,12 @@ export function isEmptyArray(a) {
     });
 }
 export function isNotEmptyArray(a) {
+    if (a == null) {
+        throw new assert.AssertionError({
+            message: "Expected a non-empty array",
+            actual: a,
+        });
+    }
     if (a.length !== 0)
         return true;
     throw new assert.AssertionError({
@@ -60,3 +66,33 @@ export function startsWith(a, prefix) {
         actual: a,
     });
 }
+export function gt(num, target) {
+    if (num > target)
+        return true;
+    throw new assert.AssertionError({
+        message: `Expected ${num} > ${target}`,
+        actual: num,
+    });
+}
+export function gte(num, target) {
+    if (num >= target)
+        return true;
+    throw new assert.AssertionError({
+        message: `Expected ${num} >= ${target}`,
+        actual: num,
+    });
+}
+export function stringContains(str, expected) {
+    if (typeof str !== "string") {
+        throw new assert.AssertionError({
+            message: "Expected input to be of type string.",
+            actual: typeof str,
+        });
+    }
+    if (str.includes(expected))
+        return true;
+    throw new assert.AssertionError({
+        message: `Expected string to contain: "${expected}"`,
+        actual: str,
+    });
+}

package/dist/source/chat-completion.d.ts

@@ -9,9 +9,13 @@ export type ChatResponse = OpenAI.ChatCompletion & {
         message: {
             reasoning_content?: string;
             reasoning?: string;
+            tool_calls?: Array<{
+                index: number;
+            }>;
         };
     }>;
 };
+export type ChatMessage = ChatResponse["choices"][number]["message"];
 export declare const ChatCompletion: t.Struct<{
     messages: t.Arr<t.UnwrappedTypeStruct<{
         content: t.Type<string | t.UnwrappedTypeStruct<{
@@ -54,6 +58,7 @@ export declare const ChatCompletion: t.Struct<{
             name: t.TypeOf<string>;
         }>>;
         reasoning_content: t.OptionalKey<t.Type<string | null>>;
+        reasoning: t.OptionalKey<t.Type<string | null>>;
     }> | t.UnwrappedTypeStruct<{
         role: t.Value<"tool">;
         content: t.Type<string | t.UnwrappedTypeStruct<{

package/dist/source/chat-completion.js

@@ -41,6 +41,7 @@ const AssistantMessageSchema = t.subtype({
         name: t.str,
     })),
     reasoning_content: t.optional(t.str.or(t.nil)),
+    reasoning: t.optional(t.str.or(t.nil)),
 });
 const UserMessageSchema = t.subtype({
     content: UserContent,

package/dist/source/evals.d.ts

@@ -0,0 +1,9 @@
+import { ChatMessage } from "./chat-completion.ts";
+export type Eval = {
+    test: (response: ChatMessage) => any;
+    json: any;
+    name: string;
+};
+export declare function getEvals(): Promise<Eval[]>;
+export declare function evalName(file: string): string;
+export declare function findTestFiles(dir: string, skipReasoning: boolean): AsyncGenerator<string>;

package/dist/source/evals.js

@@ -0,0 +1,53 @@
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+import fs from "fs/promises";
+import path from "path";
+export async function getEvals() {
+    const evals = [];
+    const evalsPath = path.join(import.meta.dirname, "..", "evals");
+    for await (const testFile of findTestFiles(evalsPath, false)) {
+        const { test, json } = await import(__rewriteRelativeImportExtension(testFile));
+        evals.push({ test, json, name: evalName(testFile) });
+    }
+    return evals;
+}
+export function evalName(file) {
+    return `${path.basename(path.dirname(file))}/${path.basename(file).replace(/.js$/, "")}`;
+}
+export async function* findTestFiles(dir, skipReasoning) {
+    try {
+        await fs.stat(dir);
+    }
+    catch (e) {
+        const pathname = `${dir}.js`;
+        const stat = await fs.stat(pathname);
+        if (stat.isFile()) {
+            yield pathname;
+            return;
+        }
+        throw e;
+    }
+    const entryNames = await fs.readdir(dir);
+    const entries = await Promise.all(entryNames.map(async (entry) => {
+        return {
+            path: path.join(dir, entry),
+            stat: await fs.stat(path.join(dir, entry)),
+        };
+    }));
+    for (const entry of entries) {
+        if (entry.stat.isFile() && entry.path.endsWith(".js")) {
+            yield entry.path;
+        }
+        if (entry.stat.isDirectory()) {
+            if (skipReasoning && path.basename(entry.path) === "reasoning")
+                continue;
+            yield* findTestFiles(entry.path, skipReasoning);
+        }
+    }
+}

package/dist/source/evals.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/source/evals.test.js

@@ -0,0 +1,12 @@
+import { describe, expect, it } from "vitest";
+import { getEvals } from "./evals.js";
+describe("get-evals", () => {
+    it("works", async () => {
+        const evals = await getEvals();
+        evals.map(({ test, json, name }) => {
+            expect(name).toBeTypeOf("string");
+            expect(json).toBeTruthy();
+            expect(test).toBeTypeOf("function");
+        });
+    });
+});

package/dist/source/exports.d.ts

@@ -0,0 +1,2 @@
+export { getEvals, type Eval as SynbadEval } from "./evals.ts";
+export { ChatMessage as SynbadChatMessage } from "./chat-completion.ts";

package/dist/source/exports.js

@@ -0,0 +1 @@
+export { getEvals } from "./evals.js";

package/dist/source/index.js

@@ -9,10 +9,11 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
 };
 import * as http from "http";
 import * as https from "https";
-import fs from "fs/promises";
 import path from "path";
 import { Command } from "@commander-js/extra-typings";
 import OpenAI from "openai";
+import { getReasoning } from "./chat-completion.js";
+import { findTestFiles, evalName } from "./evals.js";
 const cli = new Command()
     .name("synbad")
     .description("A set of evals for LLM inference providers");
@@ -21,10 +22,12 @@ cli.command("eval")
     .requiredOption("--env-var <env var name>", "The env var to use to authenticate with the inference provider")
     .requiredOption("--base-url <base url>", "The base URL for the inference provider")
     .option("--skip-reasoning", "Skip reasoning evals (set this for non-reasoning models)")
+    .option("--reasoning-effort <level>", "Set the reasoning effort to high, medium, or low")
     .option("--only <eval path within synbad>", "Specific evals you want to run, e.g. evals/reasoning or evals/tools/claude-dash")
     .option("--count <num times>", "Number of times to run the eval. Any failures count as an overall failure")
+    .option("--stream", "Test streaming API calls")
     .requiredOption("--model <model name>", "The model name to test")
-    .action(async ({ model, envVar, baseUrl, only, count }) => {
+    .action(async ({ model, envVar, baseUrl, only, count, skipReasoning, reasoningEffort, stream }) => {
     if (!process.env[envVar]) {
         console.error(`No env var named ${envVar} exists for the current process`);
         process.exit(1);
@@ -35,29 +38,111 @@ cli.command("eval")
     });
     let found = 0;
     const failures = new Set();
-    const evalPath = only ? path.join(import.meta.dirname, "..", only) : path.join(import.meta.dirname, "../evals");
+    const evalPath = only ? path.join(import.meta.dirname, "..", only) : path.join(import.meta.dirname, "..", "evals");
     const maxRuns = count == null ? 1 : parseInt(count, 10);
-    for await (const testFile of findTestFiles(evalPath)) {
+    for await (const testFile of findTestFiles(evalPath, !!skipReasoning)) {
         found++;
         const test = await import(__rewriteRelativeImportExtension(testFile));
         const json = test.json;
         const name = evalName(testFile);
         process.stdout.write(`Running ${name}...`);
+        async function respond() {
+            const reasoning = reasoningEffort == null ? {} : {
+                reasoning_effort: reasoningEffort,
+            };
+            if (!stream) {
+                const response = await client.chat.completions.create({
+                    ...json,
+                    ...reasoning,
+                    stream: false,
+                    model,
+                });
+                return response.choices[0].message;
+            }
+            const msg = {};
+            const chunkStream = await client.chat.completions.create({
+                ...json,
+                ...reasoning,
+                model,
+                stream: true,
+            });
+            let lastIndex = null;
+            let toolBuffer = null;
+            for await (const chunk of chunkStream) {
+                if (!chunk.choices)
+                    continue;
+                const choice = chunk.choices[0];
+                if (!choice)
+                    continue;
+                const content = choice.delta.content;
+                const tools = choice.delta.tool_calls;
+                const reasoning = getReasoning(choice.delta);
+                if (content) {
+                    if (!msg.content)
+                        msg.content = "";
+                    msg.content += content;
+                }
+                if (tools) {
+                    for (const toolDelta of tools) {
+                        if (lastIndex == null)
+                            lastIndex = toolDelta.index;
+                        if (lastIndex !== toolDelta.index && toolBuffer != null) {
+                            msg.tool_calls ||= [];
+                            // @ts-ignore
+                            msg.tool_calls.push(toolBuffer);
+                            toolBuffer = {
+                                index: toolDelta.index,
+                                type: "function",
+                                function: {},
+                            };
+                        }
+                        if (!toolBuffer) {
+                            toolBuffer = {
+                                index: toolDelta.index,
+                                type: "function",
+                                function: {}
+                            };
+                        }
+                        lastIndex = toolDelta.index;
+                        if (toolDelta.id)
+                            toolBuffer.id = toolDelta.id;
+                        if (toolDelta.function) {
+                            if (toolDelta.function.name) {
+                                toolBuffer.function.name ||= "";
+                                toolBuffer.function.name += toolDelta.function.name;
+                            }
+                            if (toolDelta.function.arguments) {
+                                toolBuffer.function.arguments ||= "";
+                                toolBuffer.function.arguments += toolDelta.function.arguments;
+                            }
+                        }
+                    }
+                }
+                if (reasoning) {
+                    if (!msg.reasoning_content)
+                        msg.reasoning_content = "";
+                    msg.reasoning_content += reasoning;
+                }
+            }
+            if (toolBuffer) {
+                msg.tool_calls ||= [];
+                // @ts-ignore
+                msg.tool_calls.push(toolBuffer);
+            }
+            return msg;
+        }
         try {
             for (let i = 0; i < maxRuns; i++) {
                 if (maxRuns > 1) {
                     process.stdout.write(` ${i + 1}/${maxRuns}`);
                 }
-                const response = await client.chat.completions.create({
-                    model,
-                    ...json,
-                });
+                const response = await respond();
                 try {
                     test.test(response);
                 }
                 catch (e) {
                     console.error("Response:");
-                    console.error(JSON.stringify(response.choices[0], null, 2));
+                    console.error(JSON.stringify(response, null, 2));
                     throw e;
                 }
             }
@@ -84,6 +169,7 @@ ${passed}/${found} evals passed. Failures:
 cli.command("proxy")
     .requiredOption("-p, --port <number>", "Port to listen on")
     .requiredOption("-t, --target <url>", "Target URL to proxy to")
+    .option("--pretty", "Pretty-print the JSON")
     .action(async (options) => {
     const port = parseInt(options.port, 10);
     const targetUrl = new URL(options.target);
@@ -108,6 +194,7 @@ cli.command("proxy")
             stderrLog(`[${timestamp}] 📦 Writing request data to stdout...`);
             // Choose the right module based on target protocol
             const httpModule = targetUrl.protocol === "https:" ? https : http;
+            const buffer = [];
             // Create proxy request
             const proxyReq = httpModule.request({
                 hostname: targetUrl.hostname,
@@ -151,11 +238,16 @@ cli.command("proxy")
                 }
             });
             req.on("data", (chunk) => {
-                process.stdout.write(chunk);
+                buffer.push(chunk);
+                if (!options.pretty)
+                    process.stdout.write(chunk);
                 proxyReq.write(chunk);
             });
             req.on("end", () => {
-                process.stdout.write("\n");
+                if (options.pretty)
+                    console.log(JSON.stringify(JSON.parse(buffer.join()), null, 2));
+                else
+                    process.stdout.write("\n");
                 console.log(`[${timestamp}] ✅ Request complete`);
                 proxyReq.end();
             });
@@ -178,38 +270,6 @@ cli.command("proxy")
         stderrLog("🤓 Terminal UI messages (such as this one) will be logged to stderr");
     });
 });
-function evalName(file) {
-    return `${path.basename(path.dirname(file))}/${path.basename(file).replace(/.js$/, "")}`;
-}
-async function* findTestFiles(dir) {
-    try {
-        await fs.stat(dir);
-    }
-    catch (e) {
-        const pathname = `${dir}.js`;
-        const stat = await fs.stat(pathname);
-        if (stat.isFile()) {
-            yield pathname;
-            return;
-        }
-        throw e;
-    }
-    const entryNames = await fs.readdir(dir);
-    const entries = await Promise.all(entryNames.map(async (entry) => {
-        return {
-            path: path.join(dir, entry),
-            stat: await fs.stat(path.join(dir, entry)),
-        };
-    }));
-    for (const entry of entries) {
-        if (entry.stat.isFile() && entry.path.endsWith(".js")) {
-            yield entry.path;
-        }
-        if (entry.stat.isDirectory()) {
-            yield* findTestFiles(entry.path);
-        }
-    }
-}
 function stderrLog(item, ...items) {
     let formatted = item;
     if (items.length > 0) {

package/evals/reasoning/multiturn-reasoning-parsing.ts

@@ -1,8 +1,8 @@
 import * as assert from "../../source/asserts.ts";
-import { ChatResponse, getReasoning } from "../../source/chat-completion.ts";
+import { ChatMessage, getReasoning } from "../../source/chat-completion.ts";
 
-export function test(response: ChatResponse) {
-  const reasoning = getReasoning(response.choices[0].message);
+export function test(message: ChatMessage) {
+  const reasoning = getReasoning(message);
   assert.isNotNullish(reasoning);
 }