@syntheticlab/synbad 0.0.1

package/dist/evals/tools/simple-tool.js

@@ -0,0 +1,37 @@
+import * as assert from "../../source/asserts.js";
+export function test(response) {
+    const { tool_calls } = response.choices[0].message;
+    assert.isNotNullish(tool_calls);
+    assert.isNotEmptyArray(tool_calls);
+    assert.strictEqual(tool_calls.length, 1);
+    assert.strictEqual(tool_calls[0].type, "function");
+    assert.strictEqual(tool_calls[0].function.name, "get_weather");
+    const args = JSON.parse(tool_calls[0].function.arguments);
+    assert.match(args.location.toLowerCase(), /paris/);
+}
+export const json = {
+    "messages": [
+        { "role": "user", "content": "What's the weather in Paris?" }
+    ],
+    "tools": [
+        {
+            "type": "function",
+            "function": {
+                "name": "get_weather",
+                "description": "Get current weather for a location",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "location": {
+                            "type": "string",
+                            "description": "City name"
+                        }
+                    },
+                    "required": ["location"]
+                }
+            }
+        }
+    ],
+    "parallel_tool_calls": true,
+    "tool_choice": "auto",
+};

package/dist/source/asserts.d.ts

@@ -0,0 +1,6 @@
+export { deepEqual, deepStrictEqual, doesNotMatch, doesNotReject, doesNotThrow, equal, fail, ifError, match, notDeepEqual, notDeepStrictEqual, notEqual, notStrictEqual, ok, rejects, strictEqual, throws, } from "assert";
+export declare function or(a: () => void, ...rest: Array<() => void>): void;
+export declare function isNullish(a: unknown): asserts a is null | undefined;
+export declare function isNotNullish<T extends any>(a: T): asserts a is Exclude<T, null | undefined>;
+export declare function isEmptyArray(a: any[]): boolean;
+export declare function isNotEmptyArray(a: any[]): boolean;

package/dist/source/asserts.js

@@ -0,0 +1,54 @@
+import assert from "assert";
+export { deepEqual, deepStrictEqual, doesNotMatch, doesNotReject, doesNotThrow, equal, fail, ifError, match, notDeepEqual, notDeepStrictEqual, notEqual, notStrictEqual, ok, rejects, strictEqual, throws, } from "assert";
+export function or(a, ...rest) {
+    try {
+        a();
+    }
+    catch (aErr) {
+        if (rest.length === 0)
+            throw aErr;
+        try {
+            or(rest[0], ...rest.slice(1));
+        }
+        catch (bErr) {
+            throw new assert.AssertionError({
+                message: `Tried multiple asserts, but they all failed.\n${aErr}\n\n${bErr}`,
+            });
+        }
+    }
+}
+export function isNullish(a) {
+    if (a == null)
+        return;
+    throw new assert.AssertionError({
+        message: "Expected a null or undefined value",
+        actual: a,
+        expected: null,
+        operator: "==",
+    });
+}
+export function isNotNullish(a) {
+    if (a != null)
+        return;
+    throw new assert.AssertionError({
+        message: "Expected a non-null, non-undefined value",
+        actual: a,
+    });
+}
+export function isEmptyArray(a) {
+    if (a.length === 0)
+        return true;
+    throw new assert.AssertionError({
+        message: "Expected an empty array",
+        actual: a,
+        expected: [],
+    });
+}
+export function isNotEmptyArray(a) {
+    if (a.length !== 0)
+        return true;
+    throw new assert.AssertionError({
+        message: "Expected a non-empty array",
+        actual: a,
+    });
+}

package/dist/source/chat-completion.d.ts

@@ -0,0 +1,117 @@
+import { t } from "structural";
+import OpenAI from "openai";
+export type ChatResponse = OpenAI.ChatCompletion & {
+    choices: Array<{
+        message: {
+            reasoning_content?: string;
+        };
+    }>;
+};
+export declare const ChatCompletion: t.Struct<{
+    messages: t.Arr<t.UnwrappedTypeStruct<{
+        content: t.Type<string | t.UnwrappedTypeStruct<{
+            type: t.Value<"text">;
+            text: t.TypeOf<string>;
+        }>[]>;
+        role: t.Value<"system">;
+        name: t.OptionalKey<t.TypeOf<string>>;
+    }> | t.UnwrappedTypeStruct<{
+        content: t.Type<string | (t.UnwrappedTypeStruct<{
+            type: t.Value<"text">;
+            text: t.TypeOf<string>;
+        }> | t.UnwrappedTypeStruct<{
+            type: t.Value<"image_url">;
+            image_url: t.Struct<{
+                url: t.TypeOf<string>;
+            }>;
+        }>)[]>;
+        role: t.Value<"user">;
+        name: t.OptionalKey<t.TypeOf<string>>;
+    }> | t.UnwrappedTypeStruct<{
+        content: t.OptionalKey<t.Type<string | (t.UnwrappedTypeStruct<{
+            type: t.Value<"text">;
+            text: t.TypeOf<string>;
+        }> | t.UnwrappedTypeStruct<{
+            type: t.Value<"refusal">;
+            refusal: t.TypeOf<string>;
+        }>)[] | null>>;
+        role: t.Value<"assistant">;
+        tool_calls: t.OptionalKey<t.Arr<t.UnwrappedTypeStruct<{
+            id: t.TypeOf<string>;
+            type: t.Value<"function">;
+            function: t.Struct<{
+                name: t.TypeOf<string>;
+                arguments: t.TypeOf<string>;
+            }>;
+        }>>>;
+        function_call: t.OptionalKey<t.Struct<{
+            arguments: t.TypeOf<string>;
+            name: t.TypeOf<string>;
+        }>>;
+        reasoning_content: t.OptionalKey<t.Type<string | null>>;
+    }> | t.UnwrappedTypeStruct<{
+        role: t.Value<"tool">;
+        content: t.Type<string | t.UnwrappedTypeStruct<{
+            type: t.Value<"text">;
+            text: t.TypeOf<string>;
+        }>[]>;
+        tool_call_id: t.TypeOf<string>;
+    }> | t.UnwrappedTypeStruct<{
+        role: t.Value<"function">;
+        content: t.Type<string | null>;
+        name: t.TypeOf<string>;
+    }>>;
+    user: t.OptionalKey<t.TypeOf<string>>;
+    tools: t.OptionalKey<t.Arr<t.UnwrappedTypeStruct<{
+        type: t.Value<"function">;
+        function: t.Struct<{
+            description: t.OptionalKey<t.TypeOf<string>>;
+            name: t.TypeOf<string>;
+            parameters: t.OptionalKey<t.Any>;
+            strict: t.OptionalKey<t.TypeOf<boolean>>;
+        }>;
+    }>>>;
+    tool_choice: t.OptionalKey<t.Type<"auto" | "none" | "required" | t.UnwrappedTypeStruct<{
+        type: t.Value<"function">;
+        function: t.Struct<{
+            name: t.TypeOf<string>;
+        }>;
+    }>>>;
+    parallel_tool_calls: t.OptionalKey<t.TypeOf<boolean>>;
+    function_call: t.OptionalKey<t.Type<"auto" | "none" | t.UnwrappedTypeStruct<{
+        name: t.TypeOf<string>;
+    }>>>;
+    functions: t.OptionalKey<t.Arr<t.UnwrappedTypeStruct<{
+        description: t.OptionalKey<t.TypeOf<string>>;
+        name: t.TypeOf<string>;
+        parameters: t.OptionalKey<t.Any>;
+    }>>>;
+    frequency_penalty: t.OptionalKey<t.Type<number | null>>;
+    logit_bias: t.OptionalKey<t.Dict<number>>;
+    logprobs: t.OptionalKey<t.Type<number | boolean | null>>;
+    top_k: t.OptionalKey<t.Type<number | null>>;
+    top_logprobs: t.OptionalKey<t.Type<number | null>>;
+    max_tokens: t.OptionalKey<t.Type<number | null>>;
+    max_completion_tokens: t.OptionalKey<t.Type<number | null>>;
+    n: t.OptionalKey<t.Type<number | null>>;
+    presence_penalty: t.OptionalKey<t.Type<number | null>>;
+    min_p: t.OptionalKey<t.Type<number | null>>;
+    response_format: t.OptionalKey<t.Type<t.UnwrappedTypeStruct<{
+        type: t.Type<"text" | "json_object">;
+    }> | t.UnwrappedTypeStruct<{
+        type: t.Value<"json_schema">;
+        json_schema: t.Struct<{
+            name: t.TypeOf<string>;
+            description: t.OptionalKey<t.TypeOf<string>>;
+            schema: t.Any;
+            strict: t.OptionalKey<t.Type<boolean | null>>;
+        }>;
+    }>>>;
+    seed: t.OptionalKey<t.Type<number | null>>;
+    stop: t.OptionalKey<t.Type<string | string[] | null>>;
+    stream: t.OptionalKey<t.Type<boolean | null>>;
+    temperature: t.OptionalKey<t.Type<number | null>>;
+    top_p: t.OptionalKey<t.Type<number | null>>;
+    reasoning_effort: t.OptionalKey<t.Type<"low" | "medium" | "high">>;
+    enable_thinking: t.OptionalKey<t.TypeOf<boolean>>;
+}>;

package/dist/source/chat-completion.js

@@ -0,0 +1,113 @@
+import { t } from "structural";
+const TextContentPart = t.subtype({
+    type: t.value("text"),
+    text: t.str,
+});
+const ImageContentPart = t.subtype({
+    type: t.value("image_url"),
+    image_url: t.subtype({
+        url: t.str,
+    }),
+});
+const RefusalContentPart = t.subtype({
+    type: t.value("refusal"),
+    refusal: t.str,
+});
+const TextContent = t.str.or(t.array(TextContentPart));
+const UserContent = t.str.or(t.array(TextContentPart.or(ImageContentPart)));
+const AssistantContent = t.str.or(t.array(TextContentPart.or(RefusalContentPart)));
+const ToolResultSchema = t.subtype({
+    role: t.value("tool"),
+    content: TextContent,
+    tool_call_id: t.str,
+});
+const ToolCall = t.subtype({
+    id: t.str,
+    type: t.value("function"),
+    function: t.subtype({
+        name: t.str,
+        arguments: t.str,
+    }),
+});
+const AssistantMessageSchema = t.subtype({
+    content: t.optional(AssistantContent.or(t.nil)),
+    role: t.value("assistant"),
+    tool_calls: t.optional(t.array(ToolCall)),
+    function_call: t.optional(t.subtype({
+        arguments: t.str,
+        name: t.str,
+    })),
+    reasoning_content: t.optional(t.str.or(t.nil)),
+});
+const UserMessageSchema = t.subtype({
+    content: UserContent,
+    role: t.value("user"),
+    name: t.optional(t.str),
+});
+const ChatCompletionMessage = t.subtype({
+    content: TextContent,
+    role: t.value("system"),
+    name: t.optional(t.str),
+}).or(UserMessageSchema).or(AssistantMessageSchema).or(ToolResultSchema).or(t.subtype({
+    role: t.value("function"),
+    content: t.str.or(t.nil),
+    name: t.str
+}));
+const ReasoningSchema = t.value("low").or(t.value("medium")).or(t.value("high"));
+const ToolDef = t.subtype({
+    type: t.value("function"),
+    function: t.subtype({
+        description: t.optional(t.str),
+        name: t.str,
+        parameters: t.optional(t.any),
+        strict: t.optional(t.bool),
+    }),
+});
+export const ChatCompletion = t.subtype({
+    messages: t.array(ChatCompletionMessage),
+    user: t.optional(t.str),
+    tools: t.optional(t.array(ToolDef)),
+    tool_choice: t.optional(t.value("auto").or(t.value("none").or(t.value("required"))).or(t.subtype({
+        type: t.value("function"),
+        function: t.subtype({
+            name: t.str
+        }),
+    }))),
+    parallel_tool_calls: t.optional(t.bool),
+    function_call: t.optional(t.value("auto").or(t.value("none")).or(t.subtype({
+        name: t.str,
+    }))),
+    functions: t.optional(t.array(t.subtype({
+        description: t.optional(t.str),
+        name: t.str,
+        parameters: t.optional(t.any),
+    }))),
+    frequency_penalty: t.optional(t.num.or(t.nil)),
+    logit_bias: t.optional(t.dict(t.num)),
+    logprobs: t.optional(t.bool.or(t.nil).or(t.num)),
+    top_k: t.optional(t.num.or(t.nil)),
+    top_logprobs: t.optional(t.num.or(t.nil)),
+    max_tokens: t.optional(t.num.or(t.nil)),
+    max_completion_tokens: t.optional(t.num.or(t.nil)),
+    n: t.optional(t.num.or(t.nil)),
+    presence_penalty: t.optional(t.num.or(t.nil)),
+    min_p: t.optional(t.num.or(t.nil)),
+    response_format: t.optional(t.subtype({
+        type: t.value("text").or(t.value("json_object")),
+    }).or(t.subtype({
+        type: t.value("json_schema"),
+        json_schema: t.subtype({
+            name: t.str,
+            description: t.optional(t.str),
+            schema: t.any,
+            strict: t.optional(t.bool.or(t.nil)),
+        }),
+    }))),
+    seed: t.optional(t.num.or(t.nil)),
+    stop: t.optional(t.str.or(t.array(t.str)).or(t.nil)),
+    stream: t.optional(t.bool.or(t.nil)),
+    temperature: t.optional(t.num.or(t.nil)),
+    top_p: t.optional(t.num.or(t.nil)),
+    reasoning_effort: t.optional(ReasoningSchema),
+    enable_thinking: t.optional(t.bool),
+});

package/dist/source/index.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/source/index.js

@@ -0,0 +1,107 @@
+#!/usr/bin/env node
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+import { Command } from "@commander-js/extra-typings";
+import fs from "fs/promises";
+import path from "path";
+import OpenAI from "openai";
+const cli = new Command()
+    .name("synbad")
+    .description("A set of evals for LLM inference providers");
+cli.command("eval")
+    .description("Runs the evals")
+    .requiredOption("--env-var <env var name>", "The env var to use to authenticate with the inference provider")
+    .requiredOption("--base-url <base url>", "The base URL for the inference provider")
+    .option("--skip-reasoning", "Skip reasoning evals (set this for non-reasoning models)")
+    .option("--only <eval path within synbad>", "Specific evals you want to run, e.g. evals/reasoning or evals/tools/claude-dash")
+    .option("--count <num times>", "Number of times to run the eval. Any failures count as an overall failure")
+    .requiredOption("--model <model name>", "The model name to test")
+    .action(async ({ model, envVar, baseUrl, only, count }) => {
+    if (!process.env[envVar]) {
+        console.error(`No env var named ${envVar} exists for the current process`);
+        process.exit(1);
+    }
+    const client = new OpenAI({
+        apiKey: process.env[envVar],
+        baseURL: baseUrl,
+    });
+    let found = 0;
+    const failures = new Set();
+    const evalPath = only ? path.join(import.meta.dirname, "..", only) : path.join(import.meta.dirname, "../evals");
+    const maxRuns = count == null ? 1 : parseInt(count, 10);
+    for await (const testFile of findTestFiles(evalPath)) {
+        found++;
+        const test = await import(__rewriteRelativeImportExtension(testFile));
+        const json = test.json;
+        const name = evalName(testFile);
+        process.stdout.write(`Running ${name}...`);
+        try {
+            for (let i = 0; i < maxRuns; i++) {
+                if (maxRuns > 1) {
+                    process.stdout.write(` ${i + 1}/${maxRuns}`);
+                }
+                const response = await client.chat.completions.create({
+                    model,
+                    ...json,
+                });
+                test.test(response);
+            }
+            process.stdout.write(" ✅ passed\n");
+        }
+        catch (e) {
+            failures.add(testFile);
+            console.error(e);
+            console.error(`❌ ${name} failed`);
+        }
+    }
+    const passed = found - failures.size;
+    if (passed === found) {
+        console.log("\n✅ All evals passed!");
+        process.exit(0);
+    }
+    console.log("");
+    console.log(`
+${passed}/${found} evals passed. Failures:
+
+- ${Array.from(failures).map(evalName).join("\n- ")}
+`.trim());
+});
+function evalName(file) {
+    return `${path.basename(path.dirname(file))}/${path.basename(file).replace(/.js$/, "")}`;
+}
+async function* findTestFiles(dir) {
+    try {
+        await fs.stat(dir);
+    }
+    catch (e) {
+        const pathname = `${dir}.js`;
+        const stat = await fs.stat(pathname);
+        if (stat.isFile()) {
+            yield pathname;
+            return;
+        }
+        throw e;
+    }
+    const entryNames = await fs.readdir(dir);
+    const entries = await Promise.all(entryNames.map(async (entry) => {
+        return {
+            path: path.join(dir, entry),
+            stat: await fs.stat(path.join(dir, entry)),
+        };
+    }));
+    for (const entry of entries) {
+        if (entry.stat.isFile() && entry.path.endsWith(".js")) {
+            yield entry.path;
+        }
+        if (entry.stat.isDirectory()) {
+            yield* findTestFiles(entry.path);
+        }
+    }
+}
+cli.parse();

package/evals/reasoning/multiturn-reasoning-parsing.ts

@@ -0,0 +1,19 @@
+import * as assert from "../../source/asserts.ts";
+import { ChatResponse } from "../../source/chat-completion.ts";
+
+export function test(response: ChatResponse) {
+  const reasoning = response.choices[0].message.reasoning_content;
+  assert.isNotNullish(reasoning);
+}
+
+export const json = {
+  messages: [
+    { role: "user", content: "Why does 1+1=2?" },
+    {
+      role: "assistant",
+      reasoning_content: "Because it does",
+      content: "Consider the successor function",
+    },
+    { role: "user", content: "please explain that much more deeply" },
+  ],
+}

package/evals/reasoning/reasoning-claude-tool-call.ts

@@ -0,0 +1,63 @@
+import OpenAI from "openai";
+import * as assert from "../../source/asserts.ts";
+
+export function test(response: OpenAI.ChatCompletion) {
+  const { tool_calls } = response.choices[0].message;
+  assert.isNotNullish(tool_calls);
+  assert.isNotEmptyArray(tool_calls);
+  assert.strictEqual(tool_calls.length, 1);
+}
+
+export const json = {
+  "messages": [
+    {
+      "role": "system",
+      "content": "You are Claude Code, Anthropic's official CLI for Claude.\n\nYou are an interactive CLI tool that helps users with software engineering tasks. Use the instructions below and the tools available to you to assist the user."
+    },
+    {
+      "role": "user",
+      "content": [
+        {
+          "type": "text",
+          "text": "run a quick git status for me. put the tool call inside your thinking"
+        }
+      ]
+    }
+  ],
+  "max_tokens": 32000,
+  "temperature": 1,
+  "reasoning_effort": "high",
+  "tools": [
+    {
+      "type": "function",
+      "function": {
+        "name": "Bash",
+        "description": "Executes a given bash command in a persistent shell session with optional timeout, ensuring proper handling and security measures.",
+        "parameters": {
+          "type": "object",
+          "properties": {
+            "command": {
+              "type": "string",
+              "description": "The command to execute"
+            },
+            "timeout": {
+              "type": "number",
+              "description": "Optional timeout in milliseconds (max 600000)"
+            },
+            "description": {
+              "type": "string",
+              "description": "Clear, concise description of what this command does in 5-10 words, in active voice. Examples:\nInput: ls\nOutput: List files in current directory\n\nInput: git status\nOutput: Show working tree status\n\nInput: npm install\nOutput: Install package dependencies\n\nInput: mkdir foo\nOutput: Create directory 'foo'"
+            },
+            "run_in_background": {
+              "type": "boolean",
+              "description": "Set to true to run this command in the background. Use BashOutput to read the output later."
+            }
+          },
+          "required": [
+            "command"
+          ]
+        }
+      }
+    }
+  ]
+}

package/evals/reasoning/reasoning-parsing.ts

@@ -0,0 +1,13 @@
+import * as assert from "../../source/asserts.ts";
+import { ChatResponse } from "../../source/chat-completion.ts";
+
+export function test(response: ChatResponse) {
+  const reasoning = response.choices[0].message.reasoning_content;
+  assert.isNotNullish(reasoning);
+}
+
+export const json = {
+  "messages": [
+    {"role": "user", "content": "Why does 1+1=2?"}
+  ],
+}