@syntheticlab/synbad 0.0.2 → 0.0.4

package/README.md

@@ -8,6 +8,33 @@ inference quality as high as possible.
 If you find bugs in Synthetic's model hosting, please contribute the bugs here!
 We will fix them.
 
+## Results
+
+We keep a running tally of provider+model results for GLM-4.6, Kimi K2
+Thinking, and MiniMax M2. Feel free to add more provider results!
+
+|Provider |Model           |Success Rate|
+|---------|----------------|------------|
+|Synthetic|GLM-4.6         |:white_check_mark: 100%|
+|Synthetic|Kimi K2 Thinking|:white_check_mark: 100%|
+|Synthetic|MiniMax M2      |:white_check_mark: 100%|
+
+|Provider |Model           |Success Rate|
+|---------|----------------|------------|
+|Fireworks|GLM-4.6         |:white_check_mark: 100%|
+|Fireworks|Kimi K2 Thinking|:x: 86%|
+|Fireworks|MiniMax M2      |:x: 29%|
+
+|Provider |Model           |Success Rate|
+|---------|----------------|------------|
+|Together |GLM-4.6         |:white_check_mark: 100%|
+|Together |Kimi K2 Thinking|:x: 71%|
+
+|Provider |Model           |Success Rate|
+|---------|----------------|------------|
+|Parasail |GLM-4.6         |:x: 71%|
+|Parasail |Kimi K2 Thinking|:x: 57%|
+
 ## Contributing
 
 First, clone this repo from Github. Then `cd` into it and run:
@@ -53,7 +80,7 @@ testing the `evals/reasoning/reasoning-parsing` test, for GLM-4.6 on Synthetic,
 and you want to run it 5 times since it isn't consistently failing:
 
 ```bash
-synbad.sh --env-var SYNTHETIC_API_KEY \
+./synbad.sh --env-var SYNTHETIC_API_KEY \
   --base-url "https://api.synthetic.new/openai/v1" \
   --only evals/reasoning/reasoning-parsing \
   --model "hf:zai-org/GLM-4.6" \
@@ -65,7 +92,7 @@ synbad.sh --env-var SYNTHETIC_API_KEY \
 First, install it:
 
 ```bash
-npm install @syntheticlab/synbad
+npm install -g @syntheticlab/synbad
 ```
 
 Then run:

package/dist/evals/reasoning/multiturn-reasoning-parsing.js

@@ -1,6 +1,7 @@
 import * as assert from "../../source/asserts.js";
+import { getReasoning } from "../../source/chat-completion.js";
 export function test(response) {
-    const reasoning = response.choices[0].message.reasoning_content;
+    const reasoning = getReasoning(response.choices[0].message);
     assert.isNotNullish(reasoning);
 }
 export const json = {

package/dist/evals/reasoning/reasoning-claude-tool-call.d.ts

@@ -1,5 +1,5 @@
-import OpenAI from "openai";
-export declare function test(response: OpenAI.ChatCompletion): void;
+import { ChatResponse } from "../../source/chat-completion.ts";
+export declare function test(response: ChatResponse): void;
 export declare const json: {
     messages: ({
         role: string;

package/dist/evals/reasoning/reasoning-parsing.js

@@ -1,6 +1,7 @@
 import * as assert from "../../source/asserts.js";
+import { getReasoning } from "../../source/chat-completion.js";
 export function test(response) {
-    const reasoning = response.choices[0].message.reasoning_content;
+    const reasoning = getReasoning(response.choices[0].message);
     assert.isNotNullish(reasoning);
 }
 export const json = {

package/dist/evals/tools/crush-list-files.js

@@ -9,11 +9,11 @@ export function test(response) {
     assert.or(() => {
         assert.strictEqual(fn.name, "ls");
         const args = JSON.parse(fn.arguments);
-        assert.or(() => assert.deepStrictEqual(args, {}), () => assert.strictEqual(args.path, "/home/reissbaker/Hack/scratch-scripts"), () => assert.strictEqual(args.path, "."));
+        assert.or(() => assert.strictEqual(args.path, "/home/reissbaker/Hack/scratch-scripts"), () => assert.strictEqual(args.path, "."), () => assert.isNullish(args.path));
     }, () => {
         assert.strictEqual(fn.name, "bash");
         const args = JSON.parse(fn.arguments);
-        assert.ok(args.command.startsWith("ls"));
+        assert.or(() => assert.startsWith(args.command, "ls"), () => assert.startsWith(args.command, "find"));
     });
 }
 export const json = {

package/dist/source/asserts.d.ts

@@ -4,3 +4,4 @@ export declare function isNullish(a: unknown): asserts a is null | undefined;
 export declare function isNotNullish<T extends any>(a: T): asserts a is Exclude<T, null | undefined>;
 export declare function isEmptyArray(a: any[]): boolean;
 export declare function isNotEmptyArray(a: any[]): boolean;
+export declare function startsWith(a: string, prefix: string): boolean;

package/dist/source/asserts.js

@@ -52,3 +52,11 @@ export function isNotEmptyArray(a) {
         actual: a,
     });
 }
+export function startsWith(a, prefix) {
+    if (a.startsWith(prefix))
+        return true;
+    throw new assert.AssertionError({
+        message: "Expected to start with: " + prefix,
+        actual: a,
+    });
+}

package/dist/source/chat-completion.d.ts

@@ -1,9 +1,14 @@
 import { t } from "structural";
 import OpenAI from "openai";
+export declare function getReasoning(msg: {
+    reasoning_content?: string;
+    reasoning?: string;
+}): string | undefined;
 export type ChatResponse = OpenAI.ChatCompletion & {
     choices: Array<{
         message: {
             reasoning_content?: string;
+            reasoning?: string;
         };
     }>;
 };

package/dist/source/chat-completion.js

@@ -1,4 +1,7 @@
 import { t } from "structural";
+export function getReasoning(msg) {
+    return msg.reasoning_content || msg.reasoning;
+}
 const TextContentPart = t.subtype({
     type: t.value("text"),
     text: t.str,

package/dist/source/index.js

@@ -50,7 +50,14 @@ cli.command("eval")
                     model,
                     ...json,
                 });
-                test.test(response);
+                try {
+                    test.test(response);
+                }
+                catch (e) {
+                    console.error("Response:");
+                    console.error(JSON.stringify(response.choices[0], null, 2));
+                    throw e;
+                }
             }
             process.stdout.write(" ✅ passed\n");
         }

package/evals/reasoning/multiturn-reasoning-parsing.ts

@@ -1,8 +1,8 @@
 import * as assert from "../../source/asserts.ts";
-import { ChatResponse } from "../../source/chat-completion.ts";
+import { ChatResponse, getReasoning } from "../../source/chat-completion.ts";
 
 export function test(response: ChatResponse) {
-  const reasoning = response.choices[0].message.reasoning_content;
+  const reasoning = getReasoning(response.choices[0].message);
   assert.isNotNullish(reasoning);
 }
 

package/evals/reasoning/reasoning-claude-tool-call.ts

@@ -1,7 +1,7 @@
-import OpenAI from "openai";
+import { ChatResponse } from "../../source/chat-completion.ts";
 import * as assert from "../../source/asserts.ts";
 
-export function test(response: OpenAI.ChatCompletion) {
+export function test(response: ChatResponse) {
   const { tool_calls } = response.choices[0].message;
   assert.isNotNullish(tool_calls);
   assert.isNotEmptyArray(tool_calls);

package/evals/reasoning/reasoning-parsing.ts

@@ -1,8 +1,8 @@
 import * as assert from "../../source/asserts.ts";
-import { ChatResponse } from "../../source/chat-completion.ts";
+import { ChatResponse, getReasoning } from "../../source/chat-completion.ts";
 
 export function test(response: ChatResponse) {
-  const reasoning = response.choices[0].message.reasoning_content;
+  const reasoning = getReasoning(response.choices[0].message);
   assert.isNotNullish(reasoning);
 }
 

package/evals/tools/crush-list-files.ts

@@ -13,15 +13,18 @@ export function test(response: OpenAI.ChatCompletion) {
       assert.strictEqual(fn.name, "ls");
       const args = JSON.parse(fn.arguments);
       assert.or(
-        () => assert.deepStrictEqual(args, {}),
         () => assert.strictEqual(args.path, "/home/reissbaker/Hack/scratch-scripts"),
         () => assert.strictEqual(args.path, "."),
+        () => assert.isNullish(args.path),
       );
     },
     () => {
       assert.strictEqual(fn.name, "bash");
       const args = JSON.parse(fn.arguments);
-      assert.ok(args.command.startsWith("ls"));
+      assert.or(
+        () => assert.startsWith(args.command, "ls"),
+        () => assert.startsWith(args.command, "find"),
+      );
     },
   );
 }

package/package.json

@@ -1,17 +1,17 @@
 {
   "name": "@syntheticlab/synbad",
-  "version": "0.0.2",
+  "version": "0.0.4",
   "description": "LLM inference provider evals",
   "main": "dist/source/index.js",
   "bin": {
-		"synbad": "dist/source/index.js"
+    "synbad": "dist/source/index.js"
   },
-	"preferGlobal": true,
+  "preferGlobal": true,
   "type": "module",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
-		"build": "tsc",
-		"prepublishOnly": "tsc"
+    "build": "tsc",
+    "prepublishOnly": "tsc"
   },
   "author": "@reissbaker",
   "license": "MIT",

package/source/asserts.ts

@@ -68,3 +68,11 @@ export function isNotEmptyArray(a: any[]) {
     actual: a,
   });
 }
+
+export function startsWith(a: string, prefix: string) {
+  if(a.startsWith(prefix)) return true;
+  throw new assert.AssertionError({
+    message: "Expected to start with: " + prefix,
+    actual: a,
+  });
+}

package/source/chat-completion.ts

@@ -1,10 +1,15 @@
 import { t } from "structural";
 import OpenAI from "openai";
 
+export function getReasoning(msg: { reasoning_content?: string, reasoning?: string }) {
+  return msg.reasoning_content || msg.reasoning;
+}
+
 export type ChatResponse = OpenAI.ChatCompletion & {
   choices: Array<{
     message: {
       reasoning_content?: string,
+      reasoning?: string,
     },
   }>
 };

package/source/index.ts

@@ -56,7 +56,13 @@ cli.command("eval")
           model,
           ...json,
         });
-        test.test(response);
+        try {
+          test.test(response);
+        } catch(e) {
+          console.error("Response:");
+          console.error(JSON.stringify(response.choices[0], null, 2));
+          throw e;
+        }
       }
       process.stdout.write(" ✅ passed\n");
     } catch(e) {