@syntheticlab/synbad 0.0.1

package/source/asserts.ts

@@ -0,0 +1,70 @@
+import assert from "assert";
+export {
+  deepEqual,
+  deepStrictEqual,
+  doesNotMatch,
+  doesNotReject,
+  doesNotThrow,
+  equal,
+  fail,
+  ifError,
+  match,
+  notDeepEqual,
+  notDeepStrictEqual,
+  notEqual,
+  notStrictEqual,
+  ok,
+  rejects,
+  strictEqual,
+  throws,
+} from "assert";
+
+export function or(a: () => void, ...rest: Array<() => void>) {
+  try {
+    a();
+  } catch(aErr) {
+    if(rest.length === 0) throw aErr;
+    try {
+      or(rest[0], ...rest.slice(1));
+    } catch(bErr) {
+      throw new assert.AssertionError({
+        message: `Tried multiple asserts, but they all failed.\n${aErr}\n\n${bErr}`,
+      });
+    }
+  }
+}
+
+export function isNullish(a: unknown): asserts a is null | undefined {
+  if(a == null) return;
+  throw new assert.AssertionError({
+    message: "Expected a null or undefined value",
+    actual: a,
+    expected: null,
+    operator: "==",
+  });
+}
+
+export function isNotNullish<T extends any>(a: T): asserts a is Exclude<T, null | undefined> {
+  if(a != null) return;
+  throw new assert.AssertionError({
+    message: "Expected a non-null, non-undefined value",
+    actual: a,
+  });
+}
+
+export function isEmptyArray(a: any[]) {
+  if(a.length === 0) return true;
+  throw new assert.AssertionError({
+    message: "Expected an empty array",
+    actual: a,
+    expected: [],
+  });
+}
+
+export function isNotEmptyArray(a: any[]) {
+  if(a.length !== 0) return true;
+  throw new assert.AssertionError({
+    message: "Expected a non-empty array",
+    actual: a,
+  });
+}

package/source/chat-completion.ts

@@ -0,0 +1,140 @@
+import { t } from "structural";
+import OpenAI from "openai";
+
+export type ChatResponse = OpenAI.ChatCompletion & {
+  choices: Array<{
+    message: {
+      reasoning_content?: string,
+    },
+  }>
+};
+
+const TextContentPart =  t.subtype({
+  type: t.value("text"),
+  text: t.str,
+});
+const ImageContentPart =  t.subtype({
+  type: t.value("image_url"),
+  image_url: t.subtype({
+    url: t.str,
+  }),
+});
+const RefusalContentPart = t.subtype({
+  type: t.value("refusal"),
+  refusal: t.str,
+});
+
+const TextContent = t.str.or(t.array(TextContentPart));
+const UserContent = t.str.or(t.array(TextContentPart.or(ImageContentPart)));
+const AssistantContent = t.str.or(t.array(TextContentPart.or(RefusalContentPart)));
+
+const ToolResultSchema = t.subtype({
+  role: t.value("tool"),
+  content: TextContent,
+  tool_call_id: t.str,
+});
+
+const ToolCall = t.subtype({
+  id: t.str,
+  type: t.value("function"),
+  function: t.subtype({
+    name: t.str,
+    arguments: t.str,
+  }),
+});
+const AssistantMessageSchema = t.subtype({
+  content: t.optional(AssistantContent.or(t.nil)),
+  role: t.value("assistant"),
+  tool_calls: t.optional(t.array(ToolCall)),
+  function_call: t.optional(t.subtype({
+    arguments: t.str,
+    name: t.str,
+  })),
+  reasoning_content: t.optional(t.str.or(t.nil)),
+});
+
+const UserMessageSchema = t.subtype({
+  content: UserContent,
+  role: t.value("user"),
+  name: t.optional(t.str),
+});
+
+const ChatCompletionMessage = t.subtype({
+  content: TextContent,
+  role: t.value("system"),
+  name: t.optional(t.str),
+}).or(
+  UserMessageSchema
+).or(
+  AssistantMessageSchema
+).or(
+  ToolResultSchema
+).or(t.subtype({
+  role: t.value("function"),
+  content: t.str.or(t.nil),
+  name: t.str
+}));
+
+const ReasoningSchema = t.value("low").or(t.value("medium")).or(t.value("high"));
+
+const ToolDef = t.subtype({
+  type: t.value("function"),
+  function: t.subtype({
+    description: t.optional(t.str),
+    name: t.str,
+    parameters: t.optional(t.any),
+    strict: t.optional(t.bool),
+  }),
+});
+
+export const ChatCompletion = t.subtype({
+  messages: t.array(ChatCompletionMessage),
+  user: t.optional(t.str),
+
+  tools: t.optional(t.array(ToolDef)),
+  tool_choice: t.optional(t.value("auto").or(t.value("none").or(t.value("required"))).or(t.subtype({
+    type: t.value("function"),
+    function: t.subtype({
+      name: t.str
+    }),
+  }))),
+  parallel_tool_calls: t.optional(t.bool),
+  function_call: t.optional(t.value("auto").or(t.value("none")).or(t.subtype({
+    name: t.str,
+  }))),
+  functions: t.optional(t.array(t.subtype({
+    description: t.optional(t.str),
+    name: t.str,
+    parameters: t.optional(t.any),
+  }))),
+
+  frequency_penalty: t.optional(t.num.or(t.nil)),
+  logit_bias: t.optional(t.dict(t.num)),
+  logprobs: t.optional(t.bool.or(t.nil).or(t.num)),
+  top_k: t.optional(t.num.or(t.nil)),
+  top_logprobs: t.optional(t.num.or(t.nil)),
+  max_tokens: t.optional(t.num.or(t.nil)),
+  max_completion_tokens: t.optional(t.num.or(t.nil)),
+  n: t.optional(t.num.or(t.nil)),
+  presence_penalty: t.optional(t.num.or(t.nil)),
+  min_p: t.optional(t.num.or(t.nil)),
+  response_format: t.optional(t.subtype({
+    type: t.value("text").or(t.value("json_object")),
+  }).or(t.subtype({
+    type: t.value("json_schema"),
+    json_schema: t.subtype({
+      name: t.str,
+      description: t.optional(t.str),
+      schema: t.any,
+      strict: t.optional(t.bool.or(t.nil)),
+    }),
+  }))),
+  seed: t.optional(t.num.or(t.nil)),
+  stop: t.optional(t.str.or(t.array(t.str)).or(t.nil)),
+  stream: t.optional(t.bool.or(t.nil)),
+  temperature: t.optional(t.num.or(t.nil)),
+  top_p: t.optional(t.num.or(t.nil)),
+
+  reasoning_effort: t.optional(ReasoningSchema),
+  enable_thinking: t.optional(t.bool),
+});

package/source/index.ts

@@ -0,0 +1,115 @@
+#!/usr/bin/env node
+import { Command } from "@commander-js/extra-typings";
+import fs from "fs/promises";
+import path from "path";
+import OpenAI from "openai";
+
+const cli = new Command()
+.name("synbad")
+.description("A set of evals for LLM inference providers");
+
+cli.command("eval")
+.description("Runs the evals")
+.requiredOption(
+  "--env-var <env var name>", "The env var to use to authenticate with the inference provider"
+)
+.requiredOption(
+  "--base-url <base url>", "The base URL for the inference provider"
+)
+.option(
+  "--skip-reasoning", "Skip reasoning evals (set this for non-reasoning models)"
+)
+.option(
+  "--only <eval path within synbad>", "Specific evals you want to run, e.g. evals/reasoning or evals/tools/claude-dash"
+)
+.option(
+  "--count <num times>", "Number of times to run the eval. Any failures count as an overall failure",
+)
+.requiredOption("--model <model name>", "The model name to test")
+.action(async ({ model, envVar, baseUrl, only, count }) => {
+  if(!process.env[envVar]) {
+    console.error(`No env var named ${envVar} exists for the current process`);
+    process.exit(1);
+  }
+  const client = new OpenAI({
+    apiKey: process.env[envVar],
+    baseURL: baseUrl,
+  });
+  let found = 0;
+  const failures = new Set<string>();
+  const evalPath = only ? path.join(
+    import.meta.dirname, "..", only
+  ) : path.join(import.meta.dirname, "../evals");
+  const maxRuns = count == null ? 1 : parseInt(count, 10);
+  for await(const testFile of findTestFiles(evalPath)) {
+    found++;
+    const test = await import(testFile);
+    const json = test.json;
+    const name = evalName(testFile);
+    process.stdout.write(`Running ${name}...`);
+    try {
+      for(let i = 0; i < maxRuns; i++) {
+        if(maxRuns > 1) {
+          process.stdout.write(` ${i + 1}/${maxRuns}`);
+        }
+        const response = await client.chat.completions.create({
+          model,
+          ...json,
+        });
+        test.test(response);
+      }
+      process.stdout.write(" ✅ passed\n");
+    } catch(e) {
+      failures.add(testFile);
+      console.error(e);
+      console.error(`❌ ${name} failed`);
+    }
+  }
+  const passed = found - failures.size
+  if(passed === found) {
+    console.log("\n✅ All evals passed!");
+    process.exit(0);
+  }
+
+  console.log("");
+  console.log(`
+${passed}/${found} evals passed. Failures:
+
+- ${Array.from(failures).map(evalName).join("\n- ")}
+`.trim());
+});
+
+function evalName(file: string) {
+  return `${path.basename(path.dirname(file))}/${path.basename(file).replace(/.js$/, "")}`
+}
+
+async function* findTestFiles(dir: string): AsyncGenerator<string> {
+  try {
+    await fs.stat(dir);
+  } catch(e) {
+    const pathname = `${dir}.js`;
+    const stat = await fs.stat(pathname);
+    if(stat.isFile()) {
+      yield pathname;
+      return;
+    }
+    throw e;
+  }
+  const entryNames = await fs.readdir(dir);
+  const entries = await Promise.all(entryNames.map(async (entry) => {
+    return {
+      path: path.join(dir, entry),
+      stat: await fs.stat(path.join(dir, entry)),
+    };
+  }));
+  for(const entry of entries) {
+    if(entry.stat.isFile() && entry.path.endsWith(".js")) {
+      yield entry.path;
+    }
+    if(entry.stat.isDirectory()) {
+      yield* findTestFiles(entry.path);
+    }
+  }
+}
+
+cli.parse();

package/tsconfig.json

@@ -0,0 +1,32 @@
+{
+	"include": [ "source", "evals" ],
+  "exclude": [ "node_modules" ],
+	"compilerOptions": {
+    "skipLibCheck": true,
+		"outDir": "dist",
+		"module": "node16",
+		"moduleResolution": "node16",
+		"moduleDetection": "force",
+		"target": "esnext",
+		"lib": [
+			"DOM",
+			"DOM.Iterable",
+			"ES2022" // Node.js 18
+		],
+		"resolveJsonModule": false, // ESM doesn't yet support JSON modules.
+		"jsx": "react",
+		"declaration": true,
+		"newLine": "lf",
+		"stripInternal": true,
+		"strict": true,
+		"noErrorTruncation": true,
+		"noImplicitReturns": true,
+		"noImplicitOverride": true,
+		"noFallthroughCasesInSwitch": true,
+		"noPropertyAccessFromIndexSignature": true,
+		"noUncheckedSideEffectImports": true,
+		"forceConsistentCasingInFileNames": true,
+    "rewriteRelativeImportExtensions": true,
+    "allowImportingTsExtensions": true
+	}
+}