@syntheticlab/synbad 0.0.1

package/README.md

@@ -0,0 +1,77 @@
+![Synbad the legendary sailor](https://raw.githubusercontent.com/synthetic-lab/synbad/main/synbad.png)
+
+Synbad is a tool for detecting bugs in LLM inference providers, especially
+open-source ones. Synbad is maintained by
+[Synthetic](https://synthetic.new), as part of our efforts to keep our
+inference quality as high as possible.
+
+If you find bugs in Synthetic's model hosting, please contribute the bugs here!
+We will fix them.
+
+## Contributing
+
+First, clone this repo from Github. Then `cd` into it and run:
+
+```bash
+npm install
+```
+
+All inference evals are stored in the `evals/` directory. They're written in
+TypeScript. You need to export two things from an eval:
+
+1. The JSON that reproduces the problem, as the const `json`. It doesn't have to
+   reproduce it 100% of the time; if the bug appears even 5% of the time,
+   that's fine.
+2. A `test` function that runs some asserts on the output of the response,
+   which detect the error.
+
+For example, we can test reasoning parsing very simply (as we do in the
+`evals/reasoning/reasoning-parsing.ts` file):
+
+```typescript
+import * as assert from "../../source/asserts.ts";
+import { ChatResponse } from "../../source/chat-completion.ts";
+
+export function test(response: ChatResponse) {
+  const reasoning = response.choices[0].message.reasoning_content;
+  assert.isNotNullish(reasoning);
+}
+
+export const json = {
+  messages: [
+    { role: "user", content: "Why does 1+1=2?" },
+  ],
+}
+```
+
+The `asserts.ts` file re-exports all of the built-in NodeJS assertion
+functions, and also adds a few extra ones, e.g. `isNotNullish` which checks
+whether an object is `null` or `undefined`.
+
+To run your new eval, use the `synbad.sh` script in this repo. Assuming you're
+testing the `evals/reasoning/reasoning-parsing` test, for GLM-4.6 on Synthetic,
+and you want to run it 5 times since it isn't consistently failing:
+
+```bash
+synbad.sh --env-var SYNTHETIC_API_KEY \
+  --base-url "https://api.synthetic.new/openai/v1" \
+  --only evals/reasoning/reasoning-parsing \
+  --model "hf:zai-org/GLM-4.6" \
+  --count 5
+```
+
+## Running Synbad
+
+First, install it:
+
+```bash
+npm install synbad
+```
+
+Then run:
+
+```bash
+synbad --env-var SYNTHETIC_API_KEY \
+  --base-url "https://api.synthetic.new/openai/v1" \
+  --model "hf:zai-org/GLM-4.6"
+```

package/dist/evals/reasoning/multiturn-reasoning-parsing.d.ts

@@ -0,0 +1,13 @@
+import { ChatResponse } from "../../source/chat-completion.ts";
+export declare function test(response: ChatResponse): void;
+export declare const json: {
+    messages: ({
+        role: string;
+        content: string;
+        reasoning_content?: undefined;
+    } | {
+        role: string;
+        reasoning_content: string;
+        content: string;
+    })[];
+};

package/dist/evals/reasoning/multiturn-reasoning-parsing.js

@@ -0,0 +1,16 @@
+import * as assert from "../../source/asserts.js";
+export function test(response) {
+    const reasoning = response.choices[0].message.reasoning_content;
+    assert.isNotNullish(reasoning);
+}
+export const json = {
+    messages: [
+        { role: "user", content: "Why does 1+1=2?" },
+        {
+            role: "assistant",
+            reasoning_content: "Because it does",
+            content: "Consider the successor function",
+        },
+        { role: "user", content: "please explain that much more deeply" },
+    ],
+};

package/dist/evals/reasoning/reasoning-claude-tool-call.d.ts

@@ -0,0 +1,46 @@
+import OpenAI from "openai";
+export declare function test(response: OpenAI.ChatCompletion): void;
+export declare const json: {
+    messages: ({
+        role: string;
+        content: string;
+    } | {
+        role: string;
+        content: {
+            type: string;
+            text: string;
+        }[];
+    })[];
+    max_tokens: number;
+    temperature: number;
+    reasoning_effort: string;
+    tools: {
+        type: string;
+        function: {
+            name: string;
+            description: string;
+            parameters: {
+                type: string;
+                properties: {
+                    command: {
+                        type: string;
+                        description: string;
+                    };
+                    timeout: {
+                        type: string;
+                        description: string;
+                    };
+                    description: {
+                        type: string;
+                        description: string;
+                    };
+                    run_in_background: {
+                        type: string;
+                        description: string;
+                    };
+                };
+                required: string[];
+            };
+        };
+    }[];
+};

package/dist/evals/reasoning/reasoning-claude-tool-call.js

@@ -0,0 +1,60 @@
+import * as assert from "../../source/asserts.js";
+export function test(response) {
+    const { tool_calls } = response.choices[0].message;
+    assert.isNotNullish(tool_calls);
+    assert.isNotEmptyArray(tool_calls);
+    assert.strictEqual(tool_calls.length, 1);
+}
+export const json = {
+    "messages": [
+        {
+            "role": "system",
+            "content": "You are Claude Code, Anthropic's official CLI for Claude.\n\nYou are an interactive CLI tool that helps users with software engineering tasks. Use the instructions below and the tools available to you to assist the user."
+        },
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "run a quick git status for me. put the tool call inside your thinking"
+                }
+            ]
+        }
+    ],
+    "max_tokens": 32000,
+    "temperature": 1,
+    "reasoning_effort": "high",
+    "tools": [
+        {
+            "type": "function",
+            "function": {
+                "name": "Bash",
+                "description": "Executes a given bash command in a persistent shell session with optional timeout, ensuring proper handling and security measures.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "command": {
+                            "type": "string",
+                            "description": "The command to execute"
+                        },
+                        "timeout": {
+                            "type": "number",
+                            "description": "Optional timeout in milliseconds (max 600000)"
+                        },
+                        "description": {
+                            "type": "string",
+                            "description": "Clear, concise description of what this command does in 5-10 words, in active voice. Examples:\nInput: ls\nOutput: List files in current directory\n\nInput: git status\nOutput: Show working tree status\n\nInput: npm install\nOutput: Install package dependencies\n\nInput: mkdir foo\nOutput: Create directory 'foo'"
+                        },
+                        "run_in_background": {
+                            "type": "boolean",
+                            "description": "Set to true to run this command in the background. Use BashOutput to read the output later."
+                        }
+                    },
+                    "required": [
+                        "command"
+                    ]
+                }
+            }
+        }
+    ]
+};

package/dist/evals/reasoning/reasoning-parsing.d.ts

@@ -0,0 +1,8 @@
+import { ChatResponse } from "../../source/chat-completion.ts";
+export declare function test(response: ChatResponse): void;
+export declare const json: {
+    messages: {
+        role: string;
+        content: string;
+    }[];
+};

package/dist/evals/reasoning/reasoning-parsing.js

@@ -0,0 +1,10 @@
+import * as assert from "../../source/asserts.js";
+export function test(response) {
+    const reasoning = response.choices[0].message.reasoning_content;
+    assert.isNotNullish(reasoning);
+}
+export const json = {
+    "messages": [
+        { "role": "user", "content": "Why does 1+1=2?" }
+    ],
+};