@syntheticlab/synbad 0.0.4 → 0.0.6

package/README.md

@@ -8,32 +8,70 @@ inference quality as high as possible.
 If you find bugs in Synthetic's model hosting, please contribute the bugs here!
 We will fix them.
 
+## Install
+
+Synbad is distributed through npm. Install it with:
+
+```bash
+npm install -g @syntheticlab/synbad
+```
+
 ## Results
 
-We keep a running tally of provider+model results for GLM-4.6, Kimi K2
-Thinking, and MiniMax M2. Feel free to add more provider results!
+We keep a running tally of provider+model results for tool calling and
+reasoning parsing for GLM-4.7, Kimi K2 Thinking, and MiniMax M2. Feel free to
+add more provider results!
 
 |Provider |Model           |Success Rate|
 |---------|----------------|------------|
-|Synthetic|GLM-4.6         |:white_check_mark: 100%|
-|Synthetic|Kimi K2 Thinking|:white_check_mark: 100%|
-|Synthetic|MiniMax M2      |:white_check_mark: 100%|
+|Synthetic.new|GLM-4.7         |:white_check_mark: 100%|
+|Synthetic.new|Kimi K2 Thinking|:white_check_mark: 100%|
+|Synthetic.new|MiniMax M2      |:white_check_mark: 100%|
 
 |Provider |Model           |Success Rate|
 |---------|----------------|------------|
-|Fireworks|GLM-4.6         |:white_check_mark: 100%|
-|Fireworks|Kimi K2 Thinking|:x: 86%|
-|Fireworks|MiniMax M2      |:x: 29%|
+|Fireworks|GLM-4.7         |:x: 83%|
+|Fireworks|Kimi K2 Thinking|:x: 92%|
+|Fireworks|MiniMax M2      |:white_check_mark: 100%|
 
 |Provider |Model           |Success Rate|
 |---------|----------------|------------|
-|Together |GLM-4.6         |:white_check_mark: 100%|
-|Together |Kimi K2 Thinking|:x: 71%|
+|Together |Kimi K2 Thinking|:x: 66%|
 
 |Provider |Model           |Success Rate|
 |---------|----------------|------------|
-|Parasail |GLM-4.6         |:x: 71%|
-|Parasail |Kimi K2 Thinking|:x: 57%|
+|Parasail |GLM-4.7         |:x: 83%|
+|Parasail |Kimi K2 Thinking|:x: 75%|
+
+Note for attempting reproductions: generally all tests are reproducible with
+`--count 1` and `--count 1 --stream`, but for evaluating the
+response-in-reasoning eval, you generally will need a high count to reproduce
+the bug: `--count 40` and `--count 40 --stream` typically is sufficient.
+
+All evals must pass both with and without Synbad's `--stream` parameter (which
+tests streaming APIs) to be considered a pass.
+
+## How do I contribute inference bugs?
+
+If you already have some problematic JSON, head over to the
+[Contributing](#Contributing) section. If you don't, don't worry! Synbad makes
+it easy to capture the problematic JSON you're encountering.
+
+First, run the Synbad Proxy, specifying the local port you want to use and the
+inference host you want to target. For example, to forward requests from
+`localhost:3000` to Synthetic's API, you'd do:
+
+```bash
+synbad proxy -p 3000 -t https://api.synthetic.new/openai/v1
+```
+
+Then, configure your coding agent — or whichever local tool you're using — to
+point to `http://localhost:3000` (or whichever port you selected). The Synbad
+Proxy will log all request bodies to `stdout`, so all you need to do is
+reproduce the bug by using your tool or coding agent, and then copy the JSON it
+printed to `stdout`.
+
+Now you have reproducible JSON to file a bug via Synbad!
 
 ## Contributing
 
@@ -49,25 +87,47 @@ TypeScript. You need to export two things from an eval:
 1. The JSON that reproduces the problem, as the const `json`. It doesn't have to
    reproduce it 100% of the time; if the bug appears even 5% of the time,
    that's fine.
-2. A `test` function that runs some asserts on the output of the response,
+2. A `test` function that runs some asserts on the returned assistant message,
    which detect the error.
 
-For example, we can test reasoning parsing very simply (as we do in the
-`evals/reasoning/reasoning-parsing.ts` file):
+For example, we can test parallel tool call support very simply (as we do in the
+`evals/tools/parallel-tool.ts` file):
 
 ```typescript
 import * as assert from "../../source/asserts.ts";
-import { ChatResponse } from "../../source/chat-completion.ts";
+import { ChatMessage } from "../../source/chat-completion.ts";
 
-export function test(response: ChatResponse) {
-  const reasoning = response.choices[0].message.reasoning_content;
-  assert.isNotNullish(reasoning);
+export function test({ tool_calls }: ChatMessage) {
+  assert.isNotNullish(tool_calls);
+  assert.isNotEmptyArray(tool_calls);
+  assert.strictEqual(tool_calls.length, 2);
 }
 
 export const json = {
-  messages: [
-    { role: "user", content: "Why does 1+1=2?" },
+  "messages": [
+    {"role": "user", "content": "What's the weather in Paris and London?"}
   ],
+  "tools": [
+    {
+      "type": "function",
+      "function": {
+        "name": "get_weather",
+        "description": "Get current weather for a location",
+        "parameters": {
+          "type": "object",
+          "properties": {
+            "location": {
+              "type": "string",
+              "description": "City name"
+            }
+          },
+          "required": ["location"]
+        }
+      }
+    }
+  ],
+  "parallel_tool_calls": true,
+  "tool_choice": "auto",
 }
 ```
 
@@ -75,18 +135,43 @@ The `asserts.ts` file re-exports all of the built-in NodeJS assertion
 functions, and also adds a few extra ones, e.g. `isNotNullish` which checks
 whether an object is `null` or `undefined`.
 
-To run your new eval, use the `synbad.sh` script in this repo. Assuming you're
-testing the `evals/reasoning/reasoning-parsing` test, for GLM-4.6 on Synthetic,
-and you want to run it 5 times since it isn't consistently failing:
+To run your new eval, use the `synbad.sh` script in this repo, which
+auto-recompiles everything (including your new test!) before running the evals.
+Assuming you're testing the `evals/reasoning/reasoning-parsing` test, for
+GLM-4.6 on Synthetic, and you want to run it 5 times since it isn't
+consistently failing:
 
 ```bash
-./synbad.sh --env-var SYNTHETIC_API_KEY \
+./synbad.sh eval --env-var SYNTHETIC_API_KEY \
   --base-url "https://api.synthetic.new/openai/v1" \
   --only evals/reasoning/reasoning-parsing \
   --model "hf:zai-org/GLM-4.6" \
   --count 5
 ```
 
+### Handling reasoning parsing
+
+The OpenAI spec didn't originally include reasoning content parsing, since the
+original OpenAI models didn't reason. The open-source community added support
+for reasoning later, but there are two competing specs:
+
+1. Storing the reasoning content in `message.reasoning_content`, or
+2. Storing the reasoning content in `message.reasoning`.
+
+To make sure your evals work with a wider range of inference providers, use
+the `getReasoning` function when testing reasoning parsing like so:
+
+```typescript
+import { getReasoning } from "../../source/chat-completion.ts";
+
+// In your test:
+
+const reasoning = getReasoning(message);
+```
+
+This ensures your test will use the correct reasoning content data regardless
+of which spec the underlying inference provider is using.
+
 ## Running Synbad
 
 First, install it:
@@ -98,7 +183,7 @@ npm install -g @syntheticlab/synbad
 Then run:
 
 ```bash
-synbad --env-var SYNTHETIC_API_KEY \
+synbad eval --env-var SYNTHETIC_API_KEY \
   --base-url "https://api.synthetic.new/openai/v1" \
   --model "hf:zai-org/GLM-4.6"
 ```

package/dist/evals/reasoning/multiturn-reasoning-parsing.d.ts

@@ -1,5 +1,5 @@
-import { ChatResponse } from "../../source/chat-completion.ts";
-export declare function test(response: ChatResponse): void;
+import { ChatMessage } from "../../source/chat-completion.ts";
+export declare function test(message: ChatMessage): void;
 export declare const json: {
     messages: ({
         role: string;

package/dist/evals/reasoning/multiturn-reasoning-parsing.js

@@ -1,7 +1,7 @@
 import * as assert from "../../source/asserts.js";
 import { getReasoning } from "../../source/chat-completion.js";
-export function test(response) {
-    const reasoning = getReasoning(response.choices[0].message);
+export function test(message) {
+    const reasoning = getReasoning(message);
     assert.isNotNullish(reasoning);
 }
 export const json = {

package/dist/evals/reasoning/reasoning-claude-tool-call.d.ts

@@ -1,5 +1,5 @@
-import { ChatResponse } from "../../source/chat-completion.ts";
-export declare function test(response: ChatResponse): void;
+import { ChatMessage } from "../../source/chat-completion.ts";
+export declare function test({ tool_calls }: ChatMessage): void;
 export declare const json: {
     messages: ({
         role: string;

package/dist/evals/reasoning/reasoning-claude-tool-call.js

@@ -1,6 +1,5 @@
 import * as assert from "../../source/asserts.js";
-export function test(response) {
-    const { tool_calls } = response.choices[0].message;
+export function test({ tool_calls }) {
     assert.isNotNullish(tool_calls);
     assert.isNotEmptyArray(tool_calls);
     assert.strictEqual(tool_calls.length, 1);

package/dist/evals/reasoning/reasoning-parsing.d.ts

@@ -1,5 +1,5 @@
-import { ChatResponse } from "../../source/chat-completion.ts";
-export declare function test(response: ChatResponse): void;
+import { ChatMessage } from "../../source/chat-completion.ts";
+export declare function test(message: ChatMessage): void;
 export declare const json: {
     messages: {
         role: string;

package/dist/evals/reasoning/reasoning-parsing.js

@@ -1,11 +1,11 @@
 import * as assert from "../../source/asserts.js";
 import { getReasoning } from "../../source/chat-completion.js";
-export function test(response) {
-    const reasoning = getReasoning(response.choices[0].message);
+export function test(message) {
+    const reasoning = getReasoning(message);
     assert.isNotNullish(reasoning);
 }
 export const json = {
-    "messages": [
-        { "role": "user", "content": "Why does 1+1=2?" }
+    messages: [
+        { role: "user", content: "Why does 1+1=2?" }
     ],
 };

package/dist/evals/reasoning/response-in-reasoning.d.ts

@@ -0,0 +1,45 @@
+import { ChatMessage } from "../../source/chat-completion.ts";
+export declare function test(message: ChatMessage): void;
+export declare const json: {
+    messages: ({
+        role: string;
+        content: string;
+    } | {
+        role: string;
+        content: {
+            type: string;
+            text: string;
+        }[];
+    })[];
+    temperature: number;
+    tools: {
+        type: string;
+        function: {
+            name: string;
+            description: string;
+            parameters: {
+                type: string;
+                properties: {
+                    description: {
+                        description: string;
+                        type: string;
+                    };
+                    prompt: {
+                        description: string;
+                        type: string;
+                    };
+                    subagent_type: {
+                        description: string;
+                        type: string;
+                    };
+                    session_id: {
+                        description: string;
+                        type: string;
+                    };
+                };
+                required: string[];
+            };
+        };
+    }[];
+    tool_choice: string;
+};

package/dist/evals/reasoning/response-in-reasoning.js

@@ -0,0 +1,59 @@
+import * as assert from "../../source/asserts.js";
+export function test(message) {
+    const content = message.content;
+    assert.or(() => assert.isNotNullish(content), () => assert.isNotEmptyArray(message.tool_calls));
+}
+export const json = {
+    "messages": [
+        {
+            "role": "system",
+            "content": "When I ask you to add a feature or resolve a problem: ALWAYS start the project explorer sub-agent to build complete understanding"
+        },
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "Hello"
+                }
+            ]
+        }
+    ],
+    "temperature": 1,
+    "tools": [
+        {
+            "type": "function",
+            "function": {
+                "name": "task",
+                "description": "Launch a new agent to handle complex, multi-step tasks autonomously.\n\nAvailable agent types and the tools they have access to:\n- general: General-purpose agent for researching complex questions and executing multi-step tasks. Use this agent to execute multiple units of work in parallel.\n- explore: Fast agent specialized for exploring codebases. Use this when you need to quickly find files by patterns (eg. \"src/components/**/*.tsx\"), search code for keywords (eg. \"API endpoints\"), or answer questions about the codebase (eg. \"how do API endpoints work?\"). When calling this agent, specify the desired thoroughness level: \"quick\" for basic searches, \"medium\" for moderate exploration, or \"very thorough\" for comprehensive analysis across multiple locations and naming conventions.\n- code-reviewer: Expert code review specialist. Proactively reviews code for quality, security, and maintainability. Use immediately after writing or modifying code.\n\nWhen using the Task tool, you must specify a subagent_type parameter to select which agent type to use.\n\nWhen to use the Task tool:\n- When you are instructed to execute custom slash commands. Use the Task tool with the slash command invocation as the entire prompt. The slash command can take arguments. For example: Task(description=\"Check the file\", prompt=\"/check-file path/to/file.py\")\n\nWhen NOT to use the Task tool:\n- If you want to read a specific file path, use the Read or Glob tool instead of the Task tool, to find the match more quickly\n- If you are searching for a specific class definition like \"class Foo\", use the Glob tool instead, to find the match more quickly\n- If you are searching for code within a specific file or set of 2-3 files, use the Read tool instead of the Task tool, to find the match more quickly\n- Other tasks that are not related to the agent descriptions above\n\n\nUsage notes:\n1. Launch multiple agents concurrently whenever possible, to maximize performance; to do that, use a single message with multiple tool uses\n2. When the agent is done, it will return a single message back to you. The result returned by the agent is not visible to the user. To show the user the result, you should send a text message back to the user with a concise summary of the result.\n3. Each agent invocation is stateless unless you provide a session_id. Your prompt should contain a highly detailed task description for the agent to perform autonomously and you should specify exactly what information the agent should return back to you in its final and only message to you.\n4. The agent's outputs should generally be trusted\n5. Clearly tell the agent whether you expect it to write code or just to do research (search, file reads, web fetches, etc.), since it is not aware of the user's intent\n6. If the agent description mentions that it should be used proactively, then you should try your best to use it without the user having to ask for it first. Use your judgement.\n\n",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "description": {
+                            "description": "A short (3-5 words) description of the task",
+                            "type": "string"
+                        },
+                        "prompt": {
+                            "description": "The task for the agent to perform",
+                            "type": "string"
+                        },
+                        "subagent_type": {
+                            "description": "The type of specialized agent to use for this task",
+                            "type": "string"
+                        },
+                        "session_id": {
+                            "description": "Existing Task session to continue",
+                            "type": "string"
+                        }
+                    },
+                    "required": [
+                        "description",
+                        "prompt",
+                        "subagent_type"
+                    ]
+                }
+            }
+        },
+    ],
+    "tool_choice": "auto"
+};

package/dist/evals/tools/claude-dash.d.ts

@@ -1,5 +1,5 @@
-import OpenAI from "openai";
-export declare function test(response: OpenAI.ChatCompletion): void;
+import { ChatMessage } from "../../source/chat-completion.ts";
+export declare function test({ tool_calls }: ChatMessage): void;
 export declare const json: {
     messages: ({
         role: string;

package/dist/evals/tools/claude-dash.js

@@ -1,6 +1,5 @@
 import * as assert from "../../source/asserts.js";
-export function test(response) {
-    const { tool_calls } = response.choices[0].message;
+export function test({ tool_calls }) {
     assert.isNotNullish(tool_calls);
     assert.isNotEmptyArray(tool_calls);
 }

package/dist/evals/tools/crush-list-files.d.ts

@@ -1,5 +1,5 @@
-import OpenAI from "openai";
-export declare function test(response: OpenAI.ChatCompletion): void;
+import { ChatMessage } from "../../source/chat-completion.ts";
+export declare function test({ tool_calls }: ChatMessage): void;
 export declare const json: {
     messages: {
         content: string;
@@ -670,7 +670,4 @@ export declare const json: {
     })[];
     tool_choice: string;
     max_tokens: number;
-    stream_options: {
-        include_usage: boolean;
-    };
 };

package/dist/evals/tools/crush-list-files.js

@@ -1,15 +1,16 @@
 import * as assert from "../../source/asserts.js";
-export function test(response) {
-    const { tool_calls } = response.choices[0].message;
+export function test({ tool_calls }) {
     assert.isNotNullish(tool_calls);
     assert.isNotEmptyArray(tool_calls);
-    assert.strictEqual(tool_calls.length, 1);
+    assert.gte(tool_calls.length, 1);
     assert.strictEqual(tool_calls[0].type, "function");
     const fn = tool_calls[0].function;
     assert.or(() => {
         assert.strictEqual(fn.name, "ls");
-        const args = JSON.parse(fn.arguments);
-        assert.or(() => assert.strictEqual(args.path, "/home/reissbaker/Hack/scratch-scripts"), () => assert.strictEqual(args.path, "."), () => assert.isNullish(args.path));
+        if (fn.arguments) {
+            const args = JSON.parse(fn.arguments);
+            assert.or(() => assert.strictEqual(args.path, "/home/reissbaker/Hack/scratch-scripts"), () => assert.strictEqual(args.path, "."), () => assert.isNullish(args.path));
+        }
     }, () => {
         assert.strictEqual(fn.name, "bash");
         const args = JSON.parse(fn.arguments);
@@ -435,7 +436,4 @@ export const json = {
     ],
     "tool_choice": "auto",
     "max_tokens": 60000,
-    "stream_options": {
-        "include_usage": true
-    }
 };

package/dist/evals/tools/multi-turn-tools.d.ts

@@ -0,0 +1,46 @@
+import { ChatMessage } from "../../source/chat-completion.ts";
+export declare function test({ tool_calls }: ChatMessage): void;
+export declare const json: {
+    messages: ({
+        role: string;
+        content: string;
+        tool_calls?: undefined;
+        tool_call_id?: undefined;
+    } | {
+        role: string;
+        tool_calls: {
+            id: string;
+            type: string;
+            function: {
+                name: string;
+                arguments: string;
+            };
+        }[];
+        content?: undefined;
+        tool_call_id?: undefined;
+    } | {
+        role: string;
+        tool_call_id: string;
+        content: string;
+        tool_calls?: undefined;
+    })[];
+    tools: {
+        type: string;
+        function: {
+            name: string;
+            description: string;
+            parameters: {
+                type: string;
+                properties: {
+                    location: {
+                        type: string;
+                        description: string;
+                    };
+                };
+                required: string[];
+            };
+        };
+    }[];
+    parallel_tool_calls: boolean;
+    tool_choice: string;
+};

package/dist/evals/tools/multi-turn-tools.js

@@ -0,0 +1,100 @@
+import * as assert from "../../source/asserts.js";
+export function test({ tool_calls }) {
+    assert.isNotNullish(tool_calls);
+    assert.isNotEmptyArray(tool_calls);
+    assert.gte(tool_calls.length, 1);
+    assert.ok(tool_calls.some(tool_call => {
+        if (tool_call.type === "function" && tool_call.function.name === "get_weather") {
+            const location = JSON.parse(tool_call.function.arguments).location;
+            if (typeof location === "string") {
+                return location.toLowerCase().match(/las vegas/);
+            }
+        }
+        return false;
+    }), "At least one tool call must be get_weather({ location: 'las_vegas' })");
+}
+export const json = {
+    "messages": [
+        {
+            role: "user",
+            content: "What's the weather in Paris?"
+        },
+        {
+            role: "assistant",
+            tool_calls: [
+                {
+                    id: "gw1",
+                    type: "function",
+                    function: {
+                        name: "get_weather",
+                        arguments: JSON.stringify({
+                            location: "Paris, France",
+                        }),
+                    },
+                },
+            ],
+        },
+        {
+            role: "tool",
+            tool_call_id: "gw1",
+            content: "The weather in Paris is 24 degrees Celsius",
+        },
+        {
+            role: "assistant",
+            content: "I've looked up the weather in Paris, and it's a comfy 24 degrees Celsius today.",
+        },
+        {
+            role: "user",
+            content: "I meant Paris, Texas",
+        },
+        {
+            role: "assistant",
+            tool_calls: [
+                {
+                    id: "gw2",
+                    type: "function",
+                    function: {
+                        name: "get_weather",
+                        arguments: JSON.stringify({
+                            location: "Paris, Texas",
+                        }),
+                    },
+                },
+            ],
+        },
+        {
+            role: "tool",
+            tool_call_id: "gw2",
+            content: "The weather in Paris, Texas is 34 degrees Celsius",
+        },
+        {
+            role: "assistant",
+            content: "I've looked up the weather in Paris, Texas and it's a scorching 24 degrees Celsius today.",
+        },
+        {
+            role: "user",
+            content: "How about Las Vegas",
+        },
+    ],
+    "tools": [
+        {
+            "type": "function",
+            "function": {
+                "name": "get_weather",
+                "description": "Get current weather for a location",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "location": {
+                            "type": "string",
+                            "description": "City name"
+                        }
+                    },
+                    "required": ["location"]
+                }
+            }
+        }
+    ],
+    "parallel_tool_calls": true,
+    "tool_choice": "auto",
+};

package/dist/evals/tools/no-fn-args.d.ts

@@ -0,0 +1,22 @@
+import { ChatMessage } from "../../source/chat-completion.ts";
+export declare function test({ tool_calls }: ChatMessage): void;
+export declare const json: {
+    messages: {
+        role: string;
+        content: string;
+    }[];
+    tools: {
+        type: string;
+        function: {
+            name: string;
+            description: string;
+            parameters: {
+                $schema: string;
+                type: string;
+                properties: {};
+                additionalProperties: boolean;
+            };
+        };
+    }[];
+    tool_choice: string;
+};

package/dist/evals/tools/no-fn-args.js

@@ -0,0 +1,31 @@
+import * as assert from "../../source/asserts.js";
+export function test({ tool_calls }) {
+    assert.isNotNullish(tool_calls);
+    assert.isNotEmptyArray(tool_calls);
+    assert.strictEqual(tool_calls.length, 1);
+    assert.strictEqual(tool_calls[0].type, "function");
+}
+export const json = {
+    "messages": [
+        {
+            "role": "user",
+            "content": "read the todos",
+        },
+    ],
+    "tools": [
+        {
+            "type": "function",
+            "function": {
+                "name": "get_todo_items",
+                "description": "Retrieves the current list of todo items, including their names and completion statuses.",
+                "parameters": {
+                    "$schema": "http://json-schema.org/draft-07/schema#",
+                    "type": "object",
+                    "properties": {},
+                    "additionalProperties": false
+                }
+            }
+        },
+    ],
+    "tool_choice": "auto",
+};