@syntheticlab/synbad 0.0.3 → 0.0.4

package/README.md

@@ -8,6 +8,33 @@ inference quality as high as possible.
 If you find bugs in Synthetic's model hosting, please contribute the bugs here!
 We will fix them.
 
+## Results
+
+We keep a running tally of provider+model results for GLM-4.6, Kimi K2
+Thinking, and MiniMax M2. Feel free to add more provider results!
+
+|Provider |Model           |Success Rate|
+|---------|----------------|------------|
+|Synthetic|GLM-4.6         |:white_check_mark: 100%|
+|Synthetic|Kimi K2 Thinking|:white_check_mark: 100%|
+|Synthetic|MiniMax M2      |:white_check_mark: 100%|
+
+|Provider |Model           |Success Rate|
+|---------|----------------|------------|
+|Fireworks|GLM-4.6         |:white_check_mark: 100%|
+|Fireworks|Kimi K2 Thinking|:x: 86%|
+|Fireworks|MiniMax M2      |:x: 29%|
+
+|Provider |Model           |Success Rate|
+|---------|----------------|------------|
+|Together |GLM-4.6         |:white_check_mark: 100%|
+|Together |Kimi K2 Thinking|:x: 71%|
+
+|Provider |Model           |Success Rate|
+|---------|----------------|------------|
+|Parasail |GLM-4.6         |:x: 71%|
+|Parasail |Kimi K2 Thinking|:x: 57%|
+
 ## Contributing
 
 First, clone this repo from Github. Then `cd` into it and run:

package/dist/evals/reasoning/multiturn-reasoning-parsing.js

@@ -1,6 +1,7 @@
 import * as assert from "../../source/asserts.js";
+import { getReasoning } from "../../source/chat-completion.js";
 export function test(response) {
-    const reasoning = response.choices[0].message.reasoning_content;
+    const reasoning = getReasoning(response.choices[0].message);
     assert.isNotNullish(reasoning);
 }
 export const json = {

package/dist/evals/reasoning/reasoning-claude-tool-call.d.ts

@@ -1,5 +1,5 @@
-import OpenAI from "openai";
-export declare function test(response: OpenAI.ChatCompletion): void;
+import { ChatResponse } from "../../source/chat-completion.ts";
+export declare function test(response: ChatResponse): void;
 export declare const json: {
     messages: ({
         role: string;

package/dist/evals/reasoning/reasoning-parsing.js

@@ -1,6 +1,7 @@
 import * as assert from "../../source/asserts.js";
+import { getReasoning } from "../../source/chat-completion.js";
 export function test(response) {
-    const reasoning = response.choices[0].message.reasoning_content;
+    const reasoning = getReasoning(response.choices[0].message);
     assert.isNotNullish(reasoning);
 }
 export const json = {

package/dist/source/chat-completion.d.ts

@@ -1,9 +1,14 @@
 import { t } from "structural";
 import OpenAI from "openai";
+export declare function getReasoning(msg: {
+    reasoning_content?: string;
+    reasoning?: string;
+}): string | undefined;
 export type ChatResponse = OpenAI.ChatCompletion & {
     choices: Array<{
         message: {
             reasoning_content?: string;
+            reasoning?: string;
         };
     }>;
 };

package/dist/source/chat-completion.js

@@ -1,4 +1,7 @@
 import { t } from "structural";
+export function getReasoning(msg) {
+    return msg.reasoning_content || msg.reasoning;
+}
 const TextContentPart = t.subtype({
     type: t.value("text"),
     text: t.str,

package/evals/reasoning/multiturn-reasoning-parsing.ts

@@ -1,8 +1,8 @@
 import * as assert from "../../source/asserts.ts";
-import { ChatResponse } from "../../source/chat-completion.ts";
+import { ChatResponse, getReasoning } from "../../source/chat-completion.ts";
 
 export function test(response: ChatResponse) {
-  const reasoning = response.choices[0].message.reasoning_content;
+  const reasoning = getReasoning(response.choices[0].message);
   assert.isNotNullish(reasoning);
 }
 

package/evals/reasoning/reasoning-claude-tool-call.ts

@@ -1,7 +1,7 @@
-import OpenAI from "openai";
+import { ChatResponse } from "../../source/chat-completion.ts";
 import * as assert from "../../source/asserts.ts";
 
-export function test(response: OpenAI.ChatCompletion) {
+export function test(response: ChatResponse) {
   const { tool_calls } = response.choices[0].message;
   assert.isNotNullish(tool_calls);
   assert.isNotEmptyArray(tool_calls);

package/evals/reasoning/reasoning-parsing.ts

@@ -1,8 +1,8 @@
 import * as assert from "../../source/asserts.ts";
-import { ChatResponse } from "../../source/chat-completion.ts";
+import { ChatResponse, getReasoning } from "../../source/chat-completion.ts";
 
 export function test(response: ChatResponse) {
-  const reasoning = response.choices[0].message.reasoning_content;
+  const reasoning = getReasoning(response.choices[0].message);
   assert.isNotNullish(reasoning);
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syntheticlab/synbad",
-  "version": "0.0.3",
+  "version": "0.0.4",
   "description": "LLM inference provider evals",
   "main": "dist/source/index.js",
   "bin": {

package/source/chat-completion.ts

@@ -1,10 +1,15 @@
 import { t } from "structural";
 import OpenAI from "openai";
 
+export function getReasoning(msg: { reasoning_content?: string, reasoning?: string }) {
+  return msg.reasoning_content || msg.reasoning;
+}
+
 export type ChatResponse = OpenAI.ChatCompletion & {
   choices: Array<{
     message: {
       reasoning_content?: string,
+      reasoning?: string,
     },
   }>
 };