@botpress/vai 0.0.1 → 0.0.2

package/dist/models.js

@@ -0,0 +1,388 @@
+"use strict";
+export const Models = [
+  {
+    "id": "anthropic__claude-3-haiku-20240307",
+    "name": "Claude 3 Haiku",
+    "integration": "anthropic",
+    "input": {
+      "maxTokens": 2e5
+    },
+    "output": {
+      "maxTokens": 4096
+    }
+  },
+  {
+    "id": "anthropic__claude-3-5-sonnet-20240620",
+    "name": "Claude 3.5 Sonnet",
+    "integration": "anthropic",
+    "input": {
+      "maxTokens": 2e5
+    },
+    "output": {
+      "maxTokens": 4096
+    }
+  },
+  {
+    "id": "cerebras__llama3.1-70b",
+    "name": "Llama 3.1 70B",
+    "integration": "cerebras",
+    "input": {
+      "maxTokens": 8192
+    },
+    "output": {
+      "maxTokens": 8192
+    }
+  },
+  {
+    "id": "cerebras__llama3.1-8b",
+    "name": "Llama 3.1 8B",
+    "integration": "cerebras",
+    "input": {
+      "maxTokens": 8192
+    },
+    "output": {
+      "maxTokens": 8192
+    }
+  },
+  {
+    "id": "fireworks-ai__accounts/fireworks/models/deepseek-coder-v2-instruct",
+    "name": "DeepSeek Coder V2 Instruct",
+    "integration": "fireworks-ai",
+    "input": {
+      "maxTokens": 131072
+    },
+    "output": {
+      "maxTokens": 131072
+    }
+  },
+  {
+    "id": "fireworks-ai__accounts/fireworks/models/deepseek-coder-v2-lite-instruct",
+    "name": "DeepSeek Coder V2 Lite",
+    "integration": "fireworks-ai",
+    "input": {
+      "maxTokens": 163840
+    },
+    "output": {
+      "maxTokens": 163840
+    }
+  },
+  {
+    "id": "fireworks-ai__accounts/fireworks/models/firellava-13b",
+    "name": "FireLLaVA-13B",
+    "integration": "fireworks-ai",
+    "input": {
+      "maxTokens": 4096
+    },
+    "output": {
+      "maxTokens": 4096
+    }
+  },
+  {
+    "id": "fireworks-ai__accounts/fireworks/models/firefunction-v2",
+    "name": "Firefunction V2",
+    "integration": "fireworks-ai",
+    "input": {
+      "maxTokens": 8192
+    },
+    "output": {
+      "maxTokens": 8192
+    }
+  },
+  {
+    "id": "fireworks-ai__accounts/fireworks/models/gemma2-9b-it",
+    "name": "Gemma 2 9B Instruct",
+    "integration": "fireworks-ai",
+    "input": {
+      "maxTokens": 8192
+    },
+    "output": {
+      "maxTokens": 8192
+    }
+  },
+  {
+    "id": "fireworks-ai__accounts/fireworks/models/llama-v3p1-405b-instruct",
+    "name": "Llama 3.1 405B Instruct",
+    "integration": "fireworks-ai",
+    "input": {
+      "maxTokens": 131072
+    },
+    "output": {
+      "maxTokens": 131072
+    }
+  },
+  {
+    "id": "fireworks-ai__accounts/fireworks/models/llama-v3p1-70b-instruct",
+    "name": "Llama 3.1 70B Instruct",
+    "integration": "fireworks-ai",
+    "input": {
+      "maxTokens": 131072
+    },
+    "output": {
+      "maxTokens": 131072
+    }
+  },
+  {
+    "id": "fireworks-ai__accounts/fireworks/models/llama-v3p1-8b-instruct",
+    "name": "Llama 3.1 8B Instruct",
+    "integration": "fireworks-ai",
+    "input": {
+      "maxTokens": 131072
+    },
+    "output": {
+      "maxTokens": 131072
+    }
+  },
+  {
+    "id": "fireworks-ai__accounts/fireworks/models/mixtral-8x22b-instruct",
+    "name": "Mixtral MoE 8x22B Instruct",
+    "integration": "fireworks-ai",
+    "input": {
+      "maxTokens": 65536
+    },
+    "output": {
+      "maxTokens": 65536
+    }
+  },
+  {
+    "id": "fireworks-ai__accounts/fireworks/models/mixtral-8x7b-instruct",
+    "name": "Mixtral MoE 8x7B Instruct",
+    "integration": "fireworks-ai",
+    "input": {
+      "maxTokens": 32768
+    },
+    "output": {
+      "maxTokens": 32768
+    }
+  },
+  {
+    "id": "fireworks-ai__accounts/fireworks/models/mythomax-l2-13b",
+    "name": "MythoMax L2 13b",
+    "integration": "fireworks-ai",
+    "input": {
+      "maxTokens": 4096
+    },
+    "output": {
+      "maxTokens": 4096
+    }
+  },
+  {
+    "id": "fireworks-ai__accounts/fireworks/models/qwen2-72b-instruct",
+    "name": "Qwen2 72b Instruct",
+    "integration": "fireworks-ai",
+    "input": {
+      "maxTokens": 32768
+    },
+    "output": {
+      "maxTokens": 32768
+    }
+  },
+  {
+    "id": "groq__gemma2-9b-it",
+    "name": "Gemma2 9B",
+    "integration": "groq",
+    "input": {
+      "maxTokens": 8192
+    },
+    "output": {
+      "maxTokens": 8192
+    }
+  },
+  {
+    "id": "groq__llama3-70b-8192",
+    "name": "LLaMA 3 70B",
+    "integration": "groq",
+    "input": {
+      "maxTokens": 8192
+    },
+    "output": {
+      "maxTokens": 8192
+    }
+  },
+  {
+    "id": "groq__llama3-8b-8192",
+    "name": "LLaMA 3 8B",
+    "integration": "groq",
+    "input": {
+      "maxTokens": 8192
+    },
+    "output": {
+      "maxTokens": 8192
+    }
+  },
+  {
+    "id": "groq__llama-3.1-70b-versatile",
+    "name": "LLaMA 3.1 70B",
+    "integration": "groq",
+    "input": {
+      "maxTokens": 128e3
+    },
+    "output": {
+      "maxTokens": 8192
+    }
+  },
+  {
+    "id": "groq__llama-3.1-8b-instant",
+    "name": "LLaMA 3.1 8B",
+    "integration": "groq",
+    "input": {
+      "maxTokens": 128e3
+    },
+    "output": {
+      "maxTokens": 8192
+    }
+  },
+  {
+    "id": "groq__llama-3.2-11b-vision-preview",
+    "name": "LLaMA 3.2 11B Vision",
+    "integration": "groq",
+    "input": {
+      "maxTokens": 128e3
+    },
+    "output": {
+      "maxTokens": 8192
+    }
+  },
+  {
+    "id": "groq__llama-3.2-1b-preview",
+    "name": "LLaMA 3.2 1B",
+    "integration": "groq",
+    "input": {
+      "maxTokens": 128e3
+    },
+    "output": {
+      "maxTokens": 8192
+    }
+  },
+  {
+    "id": "groq__llama-3.2-3b-preview",
+    "name": "LLaMA 3.2 3B",
+    "integration": "groq",
+    "input": {
+      "maxTokens": 128e3
+    },
+    "output": {
+      "maxTokens": 8192
+    }
+  },
+  {
+    "id": "groq__llama-3.2-90b-vision-preview",
+    "name": "LLaMA 3.2 90B Vision",
+    "integration": "groq",
+    "input": {
+      "maxTokens": 128e3
+    },
+    "output": {
+      "maxTokens": 8192
+    }
+  },
+  {
+    "id": "groq__llama-3.3-70b-versatile",
+    "name": "LLaMA 3.3 70B",
+    "integration": "groq",
+    "input": {
+      "maxTokens": 128e3
+    },
+    "output": {
+      "maxTokens": 32768
+    }
+  },
+  {
+    "id": "groq__mixtral-8x7b-32768",
+    "name": "Mixtral 8x7B",
+    "integration": "groq",
+    "input": {
+      "maxTokens": 32768
+    },
+    "output": {
+      "maxTokens": 32768
+    }
+  },
+  {
+    "id": "openai__o1-2024-12-17",
+    "name": "GPT o1",
+    "integration": "openai",
+    "input": {
+      "maxTokens": 2e5
+    },
+    "output": {
+      "maxTokens": 1e5
+    }
+  },
+  {
+    "id": "openai__o1-mini-2024-09-12",
+    "name": "GPT o1-mini",
+    "integration": "openai",
+    "input": {
+      "maxTokens": 128e3
+    },
+    "output": {
+      "maxTokens": 65536
+    }
+  },
+  {
+    "id": "openai__gpt-3.5-turbo-0125",
+    "name": "GPT-3.5 Turbo",
+    "integration": "openai",
+    "input": {
+      "maxTokens": 128e3
+    },
+    "output": {
+      "maxTokens": 4096
+    }
+  },
+  {
+    "id": "openai__gpt-4-turbo-2024-04-09",
+    "name": "GPT-4 Turbo",
+    "integration": "openai",
+    "input": {
+      "maxTokens": 128e3
+    },
+    "output": {
+      "maxTokens": 4096
+    }
+  },
+  {
+    "id": "openai__gpt-4o-2024-08-06",
+    "name": "GPT-4o (August 2024)",
+    "integration": "openai",
+    "input": {
+      "maxTokens": 128e3
+    },
+    "output": {
+      "maxTokens": 16384
+    }
+  },
+  {
+    "id": "openai__gpt-4o-2024-05-13",
+    "name": "GPT-4o (May 2024)",
+    "integration": "openai",
+    "input": {
+      "maxTokens": 128e3
+    },
+    "output": {
+      "maxTokens": 4096
+    }
+  },
+  {
+    "id": "openai__gpt-4o-2024-11-20",
+    "name": "GPT-4o (November 2024)",
+    "integration": "openai",
+    "input": {
+      "maxTokens": 128e3
+    },
+    "output": {
+      "maxTokens": 16384
+    }
+  },
+  {
+    "id": "openai__gpt-4o-mini-2024-07-18",
+    "name": "GPT-4o Mini",
+    "integration": "openai",
+    "input": {
+      "maxTokens": 128e3
+    },
+    "output": {
+      "maxTokens": 16384
+    }
+  }
+];

package/dist/scripts/update-models.js

@@ -0,0 +1,58 @@
+"use strict";
+import { Client } from "@botpress/client";
+import _ from "lodash";
+import fs from "node:fs";
+const LLM_LIST_MODELS = "listLanguageModels";
+const client = new Client({
+  apiUrl: process.env.CLOUD_API_ENDPOINT,
+  botId: process.env.CLOUD_BOT_ID,
+  token: process.env.CLOUD_PAT
+});
+const { bot } = await client.getBot({
+  id: process.env.CLOUD_BOT_ID
+});
+const models = [];
+for (const integrationId in bot.integrations) {
+  const botIntegration = bot.integrations[integrationId];
+  if (botIntegration?.public && botIntegration?.enabled && botIntegration?.status === "registered") {
+    try {
+      const { integration } = await client.getPublicIntegrationById({
+        id: botIntegration.id
+      });
+      const canListModels = Object.keys(integration.actions).includes(LLM_LIST_MODELS);
+      if (!canListModels) {
+        continue;
+      }
+      const { output } = await client.callAction({
+        type: `${integration.name}:${LLM_LIST_MODELS}`,
+        input: {}
+      });
+      if (_.isArray(output?.models)) {
+        for (const model of output.models) {
+          models.push({
+            id: `${integration.name}__${model.id}`,
+            name: model.name,
+            integration: integration.name,
+            input: { maxTokens: model.input.maxTokens },
+            output: { maxTokens: model.output.maxTokens }
+          });
+        }
+      }
+    } catch (err) {
+      console.error("Error fetching integration:", err instanceof Error ? err.message : `${err}`);
+    }
+  }
+}
+const content = JSON.stringify(_.orderBy(models, ["integration", "name"]), null, 2);
+fs.writeFileSync(
+  "./src/models.ts",
+  `
+// This file is generated. Do not edit it manually.
+// See 'scripts/update-models.ts'
+
+/* eslint-disable */
+/* tslint:disable */
+
+export const Models = ${content} as const`,
+  "utf-8"
+);

package/dist/scripts/update-types.js

@@ -0,0 +1,42 @@
+"use strict";
+import { Client } from "@botpress/client";
+import { z } from "@bpinternal/zui";
+import _ from "lodash";
+import fs from "node:fs";
+import path from "node:path";
+const Interfaces = ["llm"];
+const client = new Client({
+  apiUrl: process.env.CLOUD_API_ENDPOINT,
+  botId: process.env.CLOUD_BOT_ID,
+  token: process.env.CLOUD_PAT
+});
+for (const name of Interfaces) {
+  const { interfaces } = await client.listInterfaces({
+    name
+  });
+  const { interface: latest } = await client.getInterface({
+    id: _.maxBy(interfaces, "version").id
+  });
+  for (const action of Object.keys(latest.actions)) {
+    const references = Object.keys(latest.entities).reduce((acc, key) => {
+      return { ...acc, [key]: z.fromJsonSchema(latest.entities?.[key]?.schema) };
+    }, {});
+    const input = latest.actions[action]?.input.schema;
+    const output = latest.actions[action]?.output.schema;
+    const types = `
+// This file is generated. Do not edit it manually.
+// See 'scripts/update-models.ts'
+
+/* eslint-disable */
+/* tslint:disable */
+
+export namespace ${name} {
+    export namespace ${action} {
+      export ${z.fromJsonSchema(input).title("Input").dereference(references).toTypescript({ declaration: "type" })};
+      export ${z.fromJsonSchema(output).title("Output").dereference(references).toTypescript({ declaration: "type" })};
+    }
+}`;
+    fs.mkdirSync(path.resolve(`./src/sdk-interfaces/${name}`), { recursive: true });
+    fs.writeFileSync(path.resolve(`./src/sdk-interfaces/${name}/${action}.ts`), types);
+  }
+}

package/dist/task/compare.js

@@ -0,0 +1,45 @@
+"use strict";
+import { z } from "@bpinternal/zui";
+import { createTaskCollector, getCurrentSuite } from "vitest/suite";
+import { Deferred } from "../utils/deferred";
+const scenarioId = z.string().trim().min(1, "Scenario ID/name must not be empty").max(50, "Scenario ID/name is too long");
+const ScenarioLike = z.union([
+  scenarioId,
+  z.object({ name: scenarioId }).passthrough(),
+  z.object({ id: scenarioId }).passthrough()
+]);
+const getScenarioName = (scenario) => typeof scenario === "string" ? scenario : "name" in scenario ? scenario?.name : scenario?.id;
+const scenarioArgs = z.array(ScenarioLike).min(2, "You need at least two scenarios to compare").max(10, "You can only compare up to 10 scenarios").refine((scenarios) => {
+  const set = /* @__PURE__ */ new Set();
+  scenarios.forEach((scenario) => set.add(getScenarioName(scenario)));
+  return set.size === scenarios.length;
+}, "Scenarios names must be unique");
+export function compare(name, scenarios, fn) {
+  scenarios = scenarioArgs.parse(scenarios);
+  return createTaskCollector((_name, fn2, timeout) => {
+    const currentSuite = getCurrentSuite();
+    let completedCount = 0;
+    const finished = new Deferred();
+    for (const scenario of scenarios) {
+      const key = getScenarioName(scenario);
+      currentSuite.task(key, {
+        meta: {
+          scenario: key,
+          isVaiTest: true
+        },
+        handler: async (context) => {
+          const extendedContext = Object.freeze({
+            scenario
+          });
+          context.onTestFinished(() => {
+            if (++completedCount === scenarios.length) {
+              finished.resolve();
+            }
+          });
+          await fn2({ ...context, ...extendedContext });
+        },
+        timeout: timeout ?? 1e4
+      });
+    }
+  })(name, fn);
+}

package/dist/utils/asyncAssertion.js

@@ -0,0 +1,37 @@
+"use strict";
+import { expect } from "vitest";
+import { getCurrentTest } from "vitest/suite";
+import { Context } from "../context";
+export class AsyncExpectError extends Error {
+  constructor(message, output) {
+    super(message);
+    this.output = output;
+    this.name = "AsyncExpectError";
+  }
+}
+const getErrorMessages = (e) => {
+  if (e instanceof Error) {
+    return e.message;
+  } else if (typeof e === "string") {
+    return e;
+  } else if (typeof e === "object" && e !== null) {
+    return JSON.stringify(e);
+  }
+  return `Unknown error: ${e}`;
+};
+export const asyncExpect = (output, assertion) => {
+  const promise = output.then((x) => {
+    try {
+      assertion(expect(x.result, x.reason));
+    } catch (e) {
+      if (Context.wrapError) {
+        return new AsyncExpectError(getErrorMessages(e), x);
+      }
+      throw e;
+    }
+    return x;
+  });
+  getCurrentTest().promises ??= [];
+  getCurrentTest().promises.push(promise);
+  return promise;
+};

package/dist/utils/deferred.js

@@ -0,0 +1,15 @@
+"use strict";
+export class Deferred {
+  constructor() {
+    this.promise = new Promise((resolve, reject) => {
+      this._resolve = resolve;
+      this._reject = reject;
+    });
+  }
+  resolve(value) {
+    this._resolve(value);
+  }
+  reject(reason) {
+    this._reject(reason);
+  }
+}

package/dist/utils/predictJson.js

@@ -0,0 +1,75 @@
+"use strict";
+import { z } from "@bpinternal/zui";
+import JSON5 from "json5";
+import { Context } from "../context";
+const nonEmptyString = z.string().trim().min(1);
+const nonEmptyObject = z.object({}).passthrough().refine((value) => Object.keys(value).length > 0, {
+  message: "Expected a non-empty object"
+});
+const Input = nonEmptyString.or(nonEmptyObject).or(z.array(z.any()));
+const Output = z.object({
+  reason: nonEmptyString.describe("A human-readable explanation of the result"),
+  result: z.any().describe(
+    "Your best guess at the output according to the instructions provided, rooted in the context of the input and the reason above"
+  )
+});
+const Example = z.object({
+  input: Input,
+  output: Output
+});
+const Options = z.object({
+  systemMessage: z.string(),
+  examples: z.array(Example).default([]),
+  input: Input,
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  outputSchema: z.custom((value) => typeof value === "object" && value !== null && "_def" in value),
+  model: z.string()
+});
+const isValidExample = (outputSchema) => (example) => Input.safeParse(example.input).success && Output.safeParse(example.output).success && outputSchema.safeParse(example.output.result).success;
+export async function predictJson(_options) {
+  const options = Options.parse(_options);
+  const [integration, model] = options.model.split("__");
+  if (!model?.length) {
+    throw new Error("Invalid model");
+  }
+  const exampleMessages = options.examples.filter(isValidExample(options.outputSchema)).flatMap(({ input, output: output2 }) => [
+    { role: "user", content: JSON.stringify(input, null, 2) },
+    { role: "assistant", content: JSON.stringify(output2, null, 2) }
+  ]);
+  const outputSchema = Output.extend({
+    result: options.outputSchema.describe(Output.shape.result.description)
+  });
+  const result = await Context.client.callAction({
+    type: `${integration}:generateContent`,
+    input: {
+      systemPrompt: `
+${options.systemMessage}
+
+---
+Please generate a JSON response with the following format:
+\`\`\`typescript
+${await outputSchema.toTypescriptAsync()}
+\`\`\`
+`.trim(),
+      messages: [
+        ...exampleMessages,
+        {
+          role: "user",
+          content: JSON.stringify(options.input, null, 2)
+        }
+      ],
+      temperature: 0,
+      responseFormat: "json_object",
+      model: { id: model }
+    }
+  });
+  const output = result.output;
+  if (!output.choices.length || typeof output.choices?.[0]?.content !== "string") {
+    throw new Error("Invalid response from the model");
+  }
+  const json = output.choices[0].content.trim();
+  if (!json.length) {
+    throw new Error("No response from the model");
+  }
+  return outputSchema.parse(JSON5.parse(json));
+}

package/package.json

@@ -1,17 +1,18 @@
 {
   "name": "@botpress/vai",
-  "version": "0.0.1",
-  "type": "module",
+  "version": "0.0.2",
   "description": "Vitest AI (vai) – a vitest extension for testing with LLMs",
   "exports": {
     ".": {
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js",
-      "require": "./dist/index.cjs"
+      "require": "./dist/index.js"
     }
   },
   "scripts": {
-    "build": "tsup",
+    "build": "npm run build:types && npm run build:neutral",
+    "build:neutral": "esbuild src/**/*.ts src/*.ts --platform=neutral --outdir=dist",
+    "build:types": "tsup",
     "watch": "tsup --watch",
     "test": "vitest run --config vitest.config.ts",
     "test:update": "vitest -u run --config vitest.config.ts",
@@ -30,15 +31,16 @@
   "devDependencies": {
     "@types/lodash": "^4.17.0",
     "dotenv": "^16.3.1",
+    "esbuild": "^0.24.2",
     "ts-node": "^10.9.2",
     "tsup": "^8.3.5",
     "typescript": "^5.7.2"
   },
   "peerDependencies": {
+    "@botpress/client": "^0.40.0",
+    "@botpress/wasm": "^1.0.1",
+    "@bpinternal/zui": "^0.13.4",
     "lodash": "^4.17.21",
-    "@botpress/client": "^0.36.2",
-    "@botpress/sdk": "^1.6.1",
-    "@botpress/wasm": "^1.0.0",
     "vitest": "^2 || ^3 || ^4 || ^5"
   }
 }