@syntheticlab/synbad 0.0.1 → 0.0.3

package/README.md

@@ -53,7 +53,7 @@ testing the `evals/reasoning/reasoning-parsing` test, for GLM-4.6 on Synthetic,
 and you want to run it 5 times since it isn't consistently failing:
 
 ```bash
-synbad.sh --env-var SYNTHETIC_API_KEY \
+./synbad.sh --env-var SYNTHETIC_API_KEY \
   --base-url "https://api.synthetic.new/openai/v1" \
   --only evals/reasoning/reasoning-parsing \
   --model "hf:zai-org/GLM-4.6" \
@@ -65,7 +65,7 @@ synbad.sh --env-var SYNTHETIC_API_KEY \
 First, install it:
 
 ```bash
-npm install synbad
+npm install -g @syntheticlab/synbad
 ```
 
 Then run:

package/dist/evals/tools/crush-list-files.js

@@ -9,11 +9,11 @@ export function test(response) {
     assert.or(() => {
         assert.strictEqual(fn.name, "ls");
         const args = JSON.parse(fn.arguments);
-        assert.or(() => assert.deepStrictEqual(args, {}), () => assert.strictEqual(args.path, "/home/reissbaker/Hack/scratch-scripts"), () => assert.strictEqual(args.path, "."));
+        assert.or(() => assert.strictEqual(args.path, "/home/reissbaker/Hack/scratch-scripts"), () => assert.strictEqual(args.path, "."), () => assert.isNullish(args.path));
     }, () => {
         assert.strictEqual(fn.name, "bash");
         const args = JSON.parse(fn.arguments);
-        assert.ok(args.command.startsWith("ls"));
+        assert.or(() => assert.startsWith(args.command, "ls"), () => assert.startsWith(args.command, "find"));
     });
 }
 export const json = {

package/dist/source/asserts.d.ts

@@ -4,3 +4,4 @@ export declare function isNullish(a: unknown): asserts a is null | undefined;
 export declare function isNotNullish<T extends any>(a: T): asserts a is Exclude<T, null | undefined>;
 export declare function isEmptyArray(a: any[]): boolean;
 export declare function isNotEmptyArray(a: any[]): boolean;
+export declare function startsWith(a: string, prefix: string): boolean;

package/dist/source/asserts.js

@@ -52,3 +52,11 @@ export function isNotEmptyArray(a) {
         actual: a,
     });
 }
+export function startsWith(a, prefix) {
+    if (a.startsWith(prefix))
+        return true;
+    throw new assert.AssertionError({
+        message: "Expected to start with: " + prefix,
+        actual: a,
+    });
+}

package/dist/source/index.js

@@ -50,7 +50,14 @@ cli.command("eval")
                     model,
                     ...json,
                 });
-                test.test(response);
+                try {
+                    test.test(response);
+                }
+                catch (e) {
+                    console.error("Response:");
+                    console.error(JSON.stringify(response.choices[0], null, 2));
+                    throw e;
+                }
             }
             process.stdout.write(" ✅ passed\n");
         }

package/evals/tools/crush-list-files.ts

@@ -13,15 +13,18 @@ export function test(response: OpenAI.ChatCompletion) {
       assert.strictEqual(fn.name, "ls");
       const args = JSON.parse(fn.arguments);
       assert.or(
-        () => assert.deepStrictEqual(args, {}),
         () => assert.strictEqual(args.path, "/home/reissbaker/Hack/scratch-scripts"),
         () => assert.strictEqual(args.path, "."),
+        () => assert.isNullish(args.path),
       );
     },
     () => {
       assert.strictEqual(fn.name, "bash");
       const args = JSON.parse(fn.arguments);
-      assert.ok(args.command.startsWith("ls"));
+      assert.or(
+        () => assert.startsWith(args.command, "ls"),
+        () => assert.startsWith(args.command, "find"),
+      );
     },
   );
 }

package/package.json

@@ -1,17 +1,17 @@
 {
   "name": "@syntheticlab/synbad",
-  "version": "0.0.1",
+  "version": "0.0.3",
   "description": "LLM inference provider evals",
   "main": "dist/source/index.js",
   "bin": {
-		"synbad": "dist/source/index.js"
+    "synbad": "dist/source/index.js"
   },
-	"preferGlobal": true,
+  "preferGlobal": true,
   "type": "module",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
-		"build": "tsc",
-		"prepublishOnly": "tsc"
+    "build": "tsc",
+    "prepublishOnly": "tsc"
   },
   "author": "@reissbaker",
   "license": "MIT",

package/source/asserts.ts

@@ -68,3 +68,11 @@ export function isNotEmptyArray(a: any[]) {
     actual: a,
   });
 }
+
+export function startsWith(a: string, prefix: string) {
+  if(a.startsWith(prefix)) return true;
+  throw new assert.AssertionError({
+    message: "Expected to start with: " + prefix,
+    actual: a,
+  });
+}

package/source/index.ts

@@ -56,7 +56,13 @@ cli.command("eval")
           model,
           ...json,
         });
-        test.test(response);
+        try {
+          test.test(response);
+        } catch(e) {
+          console.error("Response:");
+          console.error(JSON.stringify(response.choices[0], null, 2));
+          throw e;
+        }
       }
       process.stdout.write(" ✅ passed\n");
     } catch(e) {