@botpress/vai 0.0.1-beta.1 → 0.0.1-beta.3

package/package.json

@@ -1,14 +1,13 @@
 {
   "name": "@botpress/vai",
-  "version": "0.0.1-beta.1",
+  "version": "0.0.1-beta.3",
   "type": "module",
   "description": "Vitest AI (vai) – a vitest extension for testing with LLMs",
-  "main": "src/index.ts",
   "exports": {
     ".": {
-      "types": "./dist/src/index.d.ts",
-      "import": "./dist/src/index.js",
-      "require": "./dist/src/index.js"
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
     }
   },
   "scripts": {
@@ -40,6 +39,6 @@
     "@botpress/client": "^0.36.2",
     "@botpress/sdk": "^1.6.1",
     "@botpress/wasm": "^1.0.0",
-    "vitest": "^2.0.5"
+    "vitest": "^2 || ^3 || ^4 || ^5"
   }
 }

package/.env

@@ -1,3 +0,0 @@
-CLOUD_API_ENDPOINT=https://api.botpress.dev
-CLOUD_BOT_ID=151f5562-dfe8-4261-a49c-acf308807d15
-CLOUD_PAT=bp_pat_GxLLMYV5KlxFnb0ubCmQVpiMd5YwlpEG6j9A

package/src/assertions/check.ts

@@ -1,28 +0,0 @@
-import { z } from '@botpress/sdk'
-import { Context } from '../context'
-import { asyncExpect } from '../utils/asyncAssertion'
-import { Input, predictJson } from '../utils/predictJson'
-import { makeToMatchInlineSnapshot, toAssertion } from './extension'
-
-export type CheckOptions<T> = {
-  examples?: { value: T; expected: boolean; reason: string }[]
-}
-
-export function check<T extends Input>(value: T, condition: string, options?: CheckOptions<T>) {
-  const promise = predictJson({
-    systemMessage: `Check that the value meets the condition: ${condition}`,
-    examples: options?.examples?.map(({ value, reason, expected }) => ({
-      input: value,
-      output: { reason, result: expected }
-    })),
-    outputSchema: z.boolean(),
-    model: Context.evaluatorModel,
-    input: value
-  })
-
-  return {
-    ...toAssertion(promise),
-    toBe: (expected: boolean) => asyncExpect(promise, (expect) => expect.toEqual(expected)),
-    toMatchInlineSnapshot: makeToMatchInlineSnapshot(promise)
-  }
-}

package/src/assertions/extension.ts

@@ -1,51 +0,0 @@
-import json5 from 'json5'
-import { expect } from 'vitest'
-import { getCurrentTest } from 'vitest/suite'
-
-import { asyncExpect } from '../utils/asyncAssertion'
-import { Output } from '../utils/predictJson'
-
-export type ExtendedPromise<T> = PromiseLike<Output<T>> & {
-  value: PromiseLike<T>
-}
-
-export const toAssertion = <T>(promise: Promise<Output<T>>): ExtendedPromise<T> => {
-  return {
-    then: promise.then.bind(promise),
-    value: promise.then((value) => value.result)
-  }
-}
-
-export const makeToMatchInlineSnapshot =
-  <T>(promise: Promise<Output<T>>) =>
-  async (expected?: string) => {
-    const stack = new Error().stack!.split('\n')[2]
-    const newStack = `
-at __INLINE_SNAPSHOT__ (node:internal/process/task_queues:1:1)
-at randomLine (node:internal/process/task_queues:1:1)
-${stack}
-`.trim()
-
-    const obj = json5.parse(expected ?? '""')
-    const expectation = asyncExpect(promise, (expect) => expect.toMatchObject(obj)).catch(() => {
-      // we swallow the error here, as we're going to throw a new one with the correct stack
-      // this is just to make vitest happy and show a nice error message
-    })
-
-    try {
-      expect((await promise).result).toMatchObject(obj)
-    } catch (err) {
-      const newError = new Error()
-      newError.stack = newStack
-
-      expect.getState().snapshotState.match({
-        isInline: true,
-        received: (await promise).result,
-        testName: getCurrentTest()!.name,
-        error: newError,
-        inlineSnapshot: expected
-      })
-    }
-
-    return expectation
-  }

package/src/assertions/extract.ts

@@ -1,39 +0,0 @@
-import { z } from '@botpress/sdk'
-
-import { Context } from '../context'
-import { asyncExpect } from '../utils/asyncAssertion'
-import { Input, predictJson } from '../utils/predictJson'
-import { makeToMatchInlineSnapshot, toAssertion } from './extension'
-
-export type ExtractOptions<T, S> = {
-  description?: string
-  examples?: { value: T; extracted: S; reason: string }[]
-}
-
-export function extract<T extends Input, S extends z.AnyZodObject>(
-  value: T,
-  shape: S,
-  options?: ExtractOptions<T, z.infer<S>>
-) {
-  const additionalMessage = options?.description
-    ? `\nIn order to extract the right information, follow these instructions:\n${options.description}\n`
-    : ''
-  const promise = predictJson({
-    systemMessage:
-      'From the given input, extract the required information into the requested format.' + additionalMessage.trim(),
-    examples: options?.examples?.map(({ value, reason, extracted }) => ({
-      input: value,
-      output: { reason, result: extracted }
-    })),
-    outputSchema: shape,
-    model: Context.evaluatorModel,
-    input: value
-  })
-
-  return {
-    ...toAssertion(promise),
-    toBe: (expected: z.infer<S>) => asyncExpect(promise, (expect) => expect.toEqual(expected)),
-    toMatchObject: (expected: Partial<z.infer<S>>) => asyncExpect(promise, (expect) => expect.toMatchObject(expected)),
-    toMatchInlineSnapshot: makeToMatchInlineSnapshot(promise)
-  }
-}

package/src/assertions/filter.ts

@@ -1,86 +0,0 @@
-import { literal, z } from '@botpress/sdk'
-
-import { Context } from '../context'
-import { asyncExpect } from '../utils/asyncAssertion'
-import { predictJson } from '../utils/predictJson'
-import { makeToMatchInlineSnapshot, toAssertion } from './extension'
-
-export type FilterOptions<T> = {
-  examples?: { value: T; reason: string; keep: boolean }[]
-}
-
-export function filter<U>(values: U[], condition: string, options?: FilterOptions<U>) {
-  const mappedValues = values.map((_, idx) =>
-    z.object({
-      index: literal(idx),
-      reason: z.string(),
-      keep: z.boolean()
-    })
-  )
-
-  const input = values.map((value, idx) => ({
-    index: idx,
-    value
-  }))
-
-  const schema = z
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    .tuple(mappedValues as any)
-    .describe(
-      'An array of the objects with the index and a boolean value indicating if the object should be kept or not'
-    )
-
-  const promise = predictJson({
-    systemMessage: `
-Based on the following qualification criteria, you need to filter the given list of objects.
-Citeria: ${condition}
-
----
-You need to return an array of objects with the index and a boolean value indicating if the object should be kept or not.
-`.trim(),
-    examples: options?.examples
-      ? [
-          {
-            input: options?.examples?.map((v, index) => ({
-              index,
-              value: v.value
-            })),
-            output: {
-              reason: 'Here are some examples',
-              result: options?.examples?.map((v, idx) => ({
-                index: idx,
-                reason: v.reason,
-                keep: v.keep
-              }))
-            }
-          }
-        ]
-      : undefined,
-    outputSchema: schema,
-    model: Context.evaluatorModel,
-    input
-  }).then((x) => {
-    const results = schema.parse(x.result) as { index: number; keep: boolean }[]
-    return {
-      result: values.filter((_, idx) => results.find((r) => r.index === idx)?.keep),
-      reason: x.reason
-    }
-  })
-
-  return {
-    ...toAssertion(promise),
-    toBe: (expected: U[]) => asyncExpect(promise, (expect) => expect.toEqual(expected)),
-    toMatchInlineSnapshot: makeToMatchInlineSnapshot(promise),
-    toHaveNoneFiltered: () => asyncExpect(promise, (expect) => expect.toEqual(values)),
-    toHaveSomeFiltered: () => asyncExpect(promise, (expect) => expect.not.toEqual(values)),
-    toBeEmpty: () => asyncExpect(promise, (expect) => expect.toHaveLength(0)),
-    length: {
-      toBe: (expected: number) => asyncExpect(promise, (expect) => expect.toHaveLength(expected)),
-      toBeGreaterThanOrEqual: (expected: number) =>
-        asyncExpect(promise, (expect) => expect.length.greaterThanOrEqual(expected)),
-      toBeLessThanOrEqual: (expected: number) =>
-        asyncExpect(promise, (expect) => expect.length.lessThanOrEqual(expected)),
-      toBeBetween: (min: number, max: number) => asyncExpect(promise, (expect) => expect.length.within(min, max))
-    }
-  }
-}

package/src/assertions/rate.ts

@@ -1,40 +0,0 @@
-import { z } from '@botpress/sdk'
-
-import { Context } from '../context'
-import { asyncExpect } from '../utils/asyncAssertion'
-import { Input, predictJson } from '../utils/predictJson'
-import { makeToMatchInlineSnapshot, toAssertion } from './extension'
-
-export type RatingScore = 1 | 2 | 3 | 4 | 5
-export type RateOptions<T> = {
-  examples?: { value: T; rating: number; reason: string }[]
-}
-
-export function rate<T extends Input>(value: T, condition: string, options?: RateOptions<T>) {
-  const schema = z.number().min(1).max(5).describe('Rating score, higher is better (1 is the worst, 5 is the best)')
-  const promise = predictJson({
-    systemMessage: `Based on the following qualification criteria, you need to rate the given situation from a score of 1 to 5.\nScoring: 1 is the worst score, 5 is the best score possible.\nCriteria: ${condition}`,
-    examples: options?.examples?.map(({ value, reason, rating }) => ({
-      input: value,
-      output: { reason, result: rating }
-    })),
-    outputSchema: schema,
-    model: Context.evaluatorModel,
-    input: value
-  }).then((x) => {
-    return {
-      result: typeof x.result === 'number' ? x.result : parseInt(x.result, 10),
-      reason: x.reason
-    }
-  })
-
-  return {
-    ...toAssertion(promise),
-    toBe: (expected: number) => asyncExpect(promise, (expect) => expect.toEqual(expected)),
-    toMatchInlineSnapshot: makeToMatchInlineSnapshot(promise),
-    toBeGreaterThanOrEqual: (expected: RatingScore) =>
-      asyncExpect(promise, (expect) => expect.toBeGreaterThanOrEqual(expected)),
-    toBeLessThanOrEqual: (expected: RatingScore) =>
-      asyncExpect(promise, (expect) => expect.toBeLessThanOrEqual(expected))
-  }
-}

package/src/context.ts

@@ -1,65 +0,0 @@
-import type { Client } from '@botpress/client'
-import { onTestFinished } from 'vitest'
-import { getCurrentTest } from 'vitest/suite'
-import { Models } from './models'
-
-export type EvaluatorModel = (typeof Models)[number]['id']
-
-export type TestMetadata = {
-  isVaiTest: boolean
-  scenario?: string
-  evaluatorModel?: EvaluatorModel
-}
-
-const getTestMetadata = (): TestMetadata => {
-  const test = getCurrentTest()
-  return (test?.meta ?? {
-    isVaiTest: false
-  }) as TestMetadata
-}
-
-class VaiContext {
-  #client: Client | null = null
-  #wrapError = false
-
-  get wrapError() {
-    return this.#wrapError
-  }
-
-  get client() {
-    if (!this.#client) {
-      throw new Error('Botpress client is not set')
-    }
-
-    return this.#client
-  }
-
-  get evaluatorModel(): EvaluatorModel {
-    return getTestMetadata().evaluatorModel ?? 'openai__gpt-4o-mini-2024-07-18'
-  }
-
-  get scenario() {
-    return getTestMetadata().scenario
-  }
-
-  get isVaiTest() {
-    return getTestMetadata().isVaiTest
-  }
-
-  setClient(cognitive: Client) {
-    this.#client = cognitive
-  }
-
-  swallowErrors() {
-    if (!getCurrentTest()) {
-      throw new Error('cancelBail is a Vitest hook and must be called within a test')
-    }
-
-    this.#wrapError = true
-    onTestFinished(() => {
-      this.#wrapError = false
-    })
-  }
-}
-
-export const Context = new VaiContext()

package/src/hooks/setEvaluator.ts

@@ -1,13 +0,0 @@
-import { getCurrentTest } from 'vitest/suite'
-import { EvaluatorModel, TestMetadata } from '../context'
-
-export const setEvaluator = (model: EvaluatorModel) => {
-  const test = getCurrentTest()
-
-  if (!test) {
-    throw new Error('setEvaluator is a Vitest hook and must be called within a test')
-  }
-
-  const meta = test.meta as TestMetadata
-  meta.evaluatorModel = model
-}

package/src/hooks/setupClient.ts

@@ -1,6 +0,0 @@
-import { Client } from '@botpress/client'
-import { Context } from '../context'
-
-export const setupClient = (client: Client) => {
-  Context.setClient(client)
-}

package/src/index.ts

@@ -1,9 +0,0 @@
-export { compare } from './task/compare'
-
-export { check } from './assertions/check'
-export { extract } from './assertions/extract'
-export { filter } from './assertions/filter'
-export { rate } from './assertions/rate'
-
-export { setEvaluator } from './hooks/setEvaluator'
-export { setupClient } from './hooks/setupClient'

package/src/models.ts

@@ -1,394 +0,0 @@
-
-// This file is generated. Do not edit it manually.
-// See 'scripts/update-models.ts'
-
-/* eslint-disable */
-/* tslint:disable */
-
-export const Models = [
-  {
-    "id": "anthropic__claude-3-haiku-20240307",
-    "name": "Claude 3 Haiku",
-    "integration": "anthropic",
-    "input": {
-      "maxTokens": 200000
-    },
-    "output": {
-      "maxTokens": 4096
-    }
-  },
-  {
-    "id": "anthropic__claude-3-5-sonnet-20240620",
-    "name": "Claude 3.5 Sonnet",
-    "integration": "anthropic",
-    "input": {
-      "maxTokens": 200000
-    },
-    "output": {
-      "maxTokens": 4096
-    }
-  },
-  {
-    "id": "cerebras__llama3.1-70b",
-    "name": "Llama 3.1 70B",
-    "integration": "cerebras",
-    "input": {
-      "maxTokens": 8192
-    },
-    "output": {
-      "maxTokens": 8192
-    }
-  },
-  {
-    "id": "cerebras__llama3.1-8b",
-    "name": "Llama 3.1 8B",
-    "integration": "cerebras",
-    "input": {
-      "maxTokens": 8192
-    },
-    "output": {
-      "maxTokens": 8192
-    }
-  },
-  {
-    "id": "fireworks-ai__accounts/fireworks/models/deepseek-coder-v2-instruct",
-    "name": "DeepSeek Coder V2 Instruct",
-    "integration": "fireworks-ai",
-    "input": {
-      "maxTokens": 131072
-    },
-    "output": {
-      "maxTokens": 131072
-    }
-  },
-  {
-    "id": "fireworks-ai__accounts/fireworks/models/deepseek-coder-v2-lite-instruct",
-    "name": "DeepSeek Coder V2 Lite",
-    "integration": "fireworks-ai",
-    "input": {
-      "maxTokens": 163840
-    },
-    "output": {
-      "maxTokens": 163840
-    }
-  },
-  {
-    "id": "fireworks-ai__accounts/fireworks/models/firellava-13b",
-    "name": "FireLLaVA-13B",
-    "integration": "fireworks-ai",
-    "input": {
-      "maxTokens": 4096
-    },
-    "output": {
-      "maxTokens": 4096
-    }
-  },
-  {
-    "id": "fireworks-ai__accounts/fireworks/models/firefunction-v2",
-    "name": "Firefunction V2",
-    "integration": "fireworks-ai",
-    "input": {
-      "maxTokens": 8192
-    },
-    "output": {
-      "maxTokens": 8192
-    }
-  },
-  {
-    "id": "fireworks-ai__accounts/fireworks/models/gemma2-9b-it",
-    "name": "Gemma 2 9B Instruct",
-    "integration": "fireworks-ai",
-    "input": {
-      "maxTokens": 8192
-    },
-    "output": {
-      "maxTokens": 8192
-    }
-  },
-  {
-    "id": "fireworks-ai__accounts/fireworks/models/llama-v3p1-405b-instruct",
-    "name": "Llama 3.1 405B Instruct",
-    "integration": "fireworks-ai",
-    "input": {
-      "maxTokens": 131072
-    },
-    "output": {
-      "maxTokens": 131072
-    }
-  },
-  {
-    "id": "fireworks-ai__accounts/fireworks/models/llama-v3p1-70b-instruct",
-    "name": "Llama 3.1 70B Instruct",
-    "integration": "fireworks-ai",
-    "input": {
-      "maxTokens": 131072
-    },
-    "output": {
-      "maxTokens": 131072
-    }
-  },
-  {
-    "id": "fireworks-ai__accounts/fireworks/models/llama-v3p1-8b-instruct",
-    "name": "Llama 3.1 8B Instruct",
-    "integration": "fireworks-ai",
-    "input": {
-      "maxTokens": 131072
-    },
-    "output": {
-      "maxTokens": 131072
-    }
-  },
-  {
-    "id": "fireworks-ai__accounts/fireworks/models/mixtral-8x22b-instruct",
-    "name": "Mixtral MoE 8x22B Instruct",
-    "integration": "fireworks-ai",
-    "input": {
-      "maxTokens": 65536
-    },
-    "output": {
-      "maxTokens": 65536
-    }
-  },
-  {
-    "id": "fireworks-ai__accounts/fireworks/models/mixtral-8x7b-instruct",
-    "name": "Mixtral MoE 8x7B Instruct",
-    "integration": "fireworks-ai",
-    "input": {
-      "maxTokens": 32768
-    },
-    "output": {
-      "maxTokens": 32768
-    }
-  },
-  {
-    "id": "fireworks-ai__accounts/fireworks/models/mythomax-l2-13b",
-    "name": "MythoMax L2 13b",
-    "integration": "fireworks-ai",
-    "input": {
-      "maxTokens": 4096
-    },
-    "output": {
-      "maxTokens": 4096
-    }
-  },
-  {
-    "id": "fireworks-ai__accounts/fireworks/models/qwen2-72b-instruct",
-    "name": "Qwen2 72b Instruct",
-    "integration": "fireworks-ai",
-    "input": {
-      "maxTokens": 32768
-    },
-    "output": {
-      "maxTokens": 32768
-    }
-  },
-  {
-    "id": "groq__gemma2-9b-it",
-    "name": "Gemma2 9B",
-    "integration": "groq",
-    "input": {
-      "maxTokens": 8192
-    },
-    "output": {
-      "maxTokens": 8192
-    }
-  },
-  {
-    "id": "groq__llama3-70b-8192",
-    "name": "LLaMA 3 70B",
-    "integration": "groq",
-    "input": {
-      "maxTokens": 8192
-    },
-    "output": {
-      "maxTokens": 8192
-    }
-  },
-  {
-    "id": "groq__llama3-8b-8192",
-    "name": "LLaMA 3 8B",
-    "integration": "groq",
-    "input": {
-      "maxTokens": 8192
-    },
-    "output": {
-      "maxTokens": 8192
-    }
-  },
-  {
-    "id": "groq__llama-3.1-70b-versatile",
-    "name": "LLaMA 3.1 70B",
-    "integration": "groq",
-    "input": {
-      "maxTokens": 128000
-    },
-    "output": {
-      "maxTokens": 8192
-    }
-  },
-  {
-    "id": "groq__llama-3.1-8b-instant",
-    "name": "LLaMA 3.1 8B",
-    "integration": "groq",
-    "input": {
-      "maxTokens": 128000
-    },
-    "output": {
-      "maxTokens": 8192
-    }
-  },
-  {
-    "id": "groq__llama-3.2-11b-vision-preview",
-    "name": "LLaMA 3.2 11B Vision",
-    "integration": "groq",
-    "input": {
-      "maxTokens": 128000
-    },
-    "output": {
-      "maxTokens": 8192
-    }
-  },
-  {
-    "id": "groq__llama-3.2-1b-preview",
-    "name": "LLaMA 3.2 1B",
-    "integration": "groq",
-    "input": {
-      "maxTokens": 128000
-    },
-    "output": {
-      "maxTokens": 8192
-    }
-  },
-  {
-    "id": "groq__llama-3.2-3b-preview",
-    "name": "LLaMA 3.2 3B",
-    "integration": "groq",
-    "input": {
-      "maxTokens": 128000
-    },
-    "output": {
-      "maxTokens": 8192
-    }
-  },
-  {
-    "id": "groq__llama-3.2-90b-vision-preview",
-    "name": "LLaMA 3.2 90B Vision",
-    "integration": "groq",
-    "input": {
-      "maxTokens": 128000
-    },
-    "output": {
-      "maxTokens": 8192
-    }
-  },
-  {
-    "id": "groq__llama-3.3-70b-versatile",
-    "name": "LLaMA 3.3 70B",
-    "integration": "groq",
-    "input": {
-      "maxTokens": 128000
-    },
-    "output": {
-      "maxTokens": 32768
-    }
-  },
-  {
-    "id": "groq__mixtral-8x7b-32768",
-    "name": "Mixtral 8x7B",
-    "integration": "groq",
-    "input": {
-      "maxTokens": 32768
-    },
-    "output": {
-      "maxTokens": 32768
-    }
-  },
-  {
-    "id": "openai__o1-2024-12-17",
-    "name": "GPT o1",
-    "integration": "openai",
-    "input": {
-      "maxTokens": 200000
-    },
-    "output": {
-      "maxTokens": 100000
-    }
-  },
-  {
-    "id": "openai__o1-mini-2024-09-12",
-    "name": "GPT o1-mini",
-    "integration": "openai",
-    "input": {
-      "maxTokens": 128000
-    },
-    "output": {
-      "maxTokens": 65536
-    }
-  },
-  {
-    "id": "openai__gpt-3.5-turbo-0125",
-    "name": "GPT-3.5 Turbo",
-    "integration": "openai",
-    "input": {
-      "maxTokens": 128000
-    },
-    "output": {
-      "maxTokens": 4096
-    }
-  },
-  {
-    "id": "openai__gpt-4-turbo-2024-04-09",
-    "name": "GPT-4 Turbo",
-    "integration": "openai",
-    "input": {
-      "maxTokens": 128000
-    },
-    "output": {
-      "maxTokens": 4096
-    }
-  },
-  {
-    "id": "openai__gpt-4o-2024-08-06",
-    "name": "GPT-4o (August 2024)",
-    "integration": "openai",
-    "input": {
-      "maxTokens": 128000
-    },
-    "output": {
-      "maxTokens": 16384
-    }
-  },
-  {
-    "id": "openai__gpt-4o-2024-05-13",
-    "name": "GPT-4o (May 2024)",
-    "integration": "openai",
-    "input": {
-      "maxTokens": 128000
-    },
-    "output": {
-      "maxTokens": 4096
-    }
-  },
-  {
-    "id": "openai__gpt-4o-2024-11-20",
-    "name": "GPT-4o (November 2024)",
-    "integration": "openai",
-    "input": {
-      "maxTokens": 128000
-    },
-    "output": {
-      "maxTokens": 16384
-    }
-  },
-  {
-    "id": "openai__gpt-4o-mini-2024-07-18",
-    "name": "GPT-4o Mini",
-    "integration": "openai",
-    "input": {
-      "maxTokens": 128000
-    },
-    "output": {
-      "maxTokens": 16384
-    }
-  }
-] as const

package/src/scripts/update-models.ts

@@ -1,76 +0,0 @@
-import { Client } from '@botpress/client'
-
-import _ from 'lodash'
-import fs from 'node:fs'
-
-const LLM_LIST_MODELS = 'listLanguageModels'
-
-const client = new Client({
-  apiUrl: process.env.CLOUD_API_ENDPOINT,
-  botId: process.env.CLOUD_BOT_ID,
-  token: process.env.CLOUD_PAT
-})
-
-const { bot } = await client.getBot({
-  id: process.env.CLOUD_BOT_ID!
-})
-
-type Model = {
-  id: string
-  name: string
-  integration: string
-  input: { maxTokens: number }
-  output: { maxTokens: number }
-}
-
-const models: Model[] = []
-
-for (const integrationId in bot.integrations) {
-  const botIntegration = bot.integrations[integrationId]
-  if (botIntegration?.public && botIntegration?.enabled && botIntegration?.status === 'registered') {
-    try {
-      const { integration } = await client.getPublicIntegrationById({
-        id: botIntegration.id
-      })
-
-      const canListModels = Object.keys(integration.actions).includes(LLM_LIST_MODELS)
-      if (!canListModels) {
-        continue
-      }
-
-      const { output } = await client.callAction({
-        type: `${integration.name}:${LLM_LIST_MODELS}`,
-        input: {}
-      })
-
-      if (_.isArray(output?.models)) {
-        for (const model of output.models) {
-          models.push({
-            id: `${integration.name}__${model.id}`,
-            name: model.name,
-            integration: integration.name,
-            input: { maxTokens: model.input.maxTokens },
-            output: { maxTokens: model.output.maxTokens }
-          })
-        }
-      }
-    } catch (err: unknown) {
-      console.error('Error fetching integration:', err instanceof Error ? err.message : `${err}`)
-    }
-  }
-}
-
-const content = JSON.stringify(_.orderBy(models, ['integration', 'name']), null, 2)
-
-fs.writeFileSync(
-  './src/models.ts',
-  `
-// This file is generated. Do not edit it manually.
-// See 'scripts/update-models.ts'
-
-/* eslint-disable */
-/* tslint:disable */
-
-export const Models = ${content} as const`,
-  'utf-8'
-)

package/src/scripts/update-types.ts

@@ -1,59 +0,0 @@
-import { Client } from '@botpress/client'
-import sdk from '@botpress/sdk'
-
-import _ from 'lodash'
-import fs from 'node:fs'
-import path from 'node:path'
-
-const { z } = sdk
-
-const Interfaces = ['llm'] as const
-
-const client = new Client({
-  apiUrl: process.env.CLOUD_API_ENDPOINT,
-  botId: process.env.CLOUD_BOT_ID,
-  token: process.env.CLOUD_PAT
-})
-
-for (const name of Interfaces) {
-  const { interfaces } = await client.listInterfaces({
-    name
-  })
-
-  const { interface: latest } = await client.getInterface({
-    id: _.maxBy(interfaces, 'version')!.id
-  })
-
-  for (const action of Object.keys(latest.actions)) {
-    const references = Object.keys(latest.entities).reduce((acc, key) => {
-      return { ...acc, [key]: z.fromJsonSchema(latest.entities?.[key]?.schema!) }
-    }, {})
-    const input = latest.actions[action]?.input.schema!
-    const output = latest.actions[action]?.output.schema!
-
-    const types = `
-// This file is generated. Do not edit it manually.
-// See 'scripts/update-models.ts'
-
-/* eslint-disable */
-/* tslint:disable */
-
-export namespace ${name} {
-    export namespace ${action} {
-      export ${sdk.z
-        .fromJsonSchema(input)
-        .title('Input')
-        .dereference(references)
-        .toTypescript({ declaration: 'type' })};
-      export ${sdk.z
-        .fromJsonSchema(output)
-        .title('Output')
-        .dereference(references)
-        .toTypescript({ declaration: 'type' })};
-    }
-}`
-
-    fs.mkdirSync(path.resolve(`./src/sdk-interfaces/${name}`), { recursive: true })
-    fs.writeFileSync(path.resolve(`./src/sdk-interfaces/${name}/${action}.ts`), types)
-  }
-}

package/src/sdk-interfaces/llm/generateContent.ts

@@ -1,127 +0,0 @@
-
-// This file is generated. Do not edit it manually.
-// See 'scripts/update-models.ts'
-
-/* eslint-disable */
-/* tslint:disable */
-
-export namespace llm {
-    export namespace generateContent {
-      export type Input = { 
-/** Model to use for content generation */
-model?: { id: string }
-; 
-/** Optional system prompt to guide the model */
-systemPrompt?: string
-; 
-/** Array of messages for the model to process */
-messages: Array<{ role: 'user' | 'assistant'; type?: 'text' | 'tool_calls' | 'tool_result' | 'multipart'; 
-/** Required if `type` is "tool_calls" */
-toolCalls?: Array<{ id: string; type: 'function'; function: { name: string; 
-/** Some LLMs may generate invalid JSON for a tool call, so this will be `null` when it happens. */
-arguments: 
- { [key: string]: any } | null
- } }>
-; 
-/** Required if `type` is "tool_result" */
-toolResultCallId?: string
-; 
-/** Required unless `type` is "tool_call". If `type` is "multipart", this field must be an array of content objects. If `type` is "tool_result" then this field should be the result of the tool call (a plain string or a JSON-encoded array or object). If `type` is "tool_call" then the `toolCalls` field should be used instead. */
-content: 
-
-string | Array<{ type: 'text' | 'image'; 
-/** Indicates the MIME type of the content. If not provided it will be detected from the content-type header of the provided URL. */
-mimeType?: string
-; 
-/** Required if part type is "text" */
-text?: string
-; 
-/** Required if part type is "image" */
-url?: string
- }> | null
- }>
-; 
-/** Response format expected from the model. If "json_object" is chosen, you must instruct the model to generate JSON either via the system prompt or a user message. */
-responseFormat?: 'text' | 'json_object'
-; 
-/** Maximum number of tokens allowed in the generated response */
-maxTokens?: number
-; 
-/** Sampling temperature for the model. Higher values result in more random outputs. */
-temperature?: /** Sampling temperature for the model. Higher values result in more random outputs. */ number
-; 
-/** Top-p sampling parameter. Limits sampling to the smallest set of tokens with a cumulative probability above the threshold. */
-topP?: /** Top-p sampling parameter. Limits sampling to the smallest set of tokens with a cumulative probability above the threshold. */ number
-; 
-/** Sequences where the model should stop generating further tokens. */
-stopSequences?: string[]
-; tools?: Array<{ type: 'function'; function: { 
-/** Function name */
-name: string
-; description?: string; 
-/** JSON schema of the function arguments */
-argumentsSchema?: { }
- } }>; toolChoice?: { type?: 'auto' | 'specific' | 'any' | 'none' | ''; 
-/** Required if `type` is "specific" */
-functionName?: string
- }; userId?: string; 
-/** Set to `true` to output debug information to the bot logs */
-debug?: boolean
-; meta?: { 
-/** Source of the prompt, e.g. agent/:id/:version cards/ai-generate, cards/ai-task, nodes/autonomous, etc. */
-promptSource?: string
-; promptCategory?: string; 
-/** Name of the integration that originally received the message that initiated this action */
-integrationName?: string
- } };;
-      export type Output = { 
-/** Response ID from LLM provider */
-id: string
-; 
-/** LLM provider name */
-provider: string
-; 
-/** Model name */
-model: string
-; choices: Array<{ type?: 'text' | 'tool_calls' | 'tool_result' | 'multipart'; 
-/** Required if `type` is "tool_calls" */
-toolCalls?: Array<{ id: string; type: 'function'; function: { name: string; 
-/** Some LLMs may generate invalid JSON for a tool call, so this will be `null` when it happens. */
-arguments: 
- { [key: string]: any } | null
- } }>
-; 
-/** Required if `type` is "tool_result" */
-toolResultCallId?: string
-; 
-/** Required unless `type` is "tool_call". If `type` is "multipart", this field must be an array of content objects. If `type` is "tool_result" then this field should be the result of the tool call (a plain string or a JSON-encoded array or object). If `type` is "tool_call" then the `toolCalls` field should be used instead. */
-content: 
-
-string | Array<{ type: 'text' | 'image'; 
-/** Indicates the MIME type of the content. If not provided it will be detected from the content-type header of the provided URL. */
-mimeType?: string
-; 
-/** Required if part type is "text" */
-text?: string
-; 
-/** Required if part type is "image" */
-url?: string
- }> | null
-; role: 'assistant'; index: number; stopReason: 'stop' | 'max_tokens' | 'tool_calls' | 'content_filter' | 'other' }>; usage: { 
-/** Number of input tokens used by the model */
-inputTokens: number
-; 
-/** Cost of the input tokens received by the model, in U.S. dollars */
-inputCost: number
-; 
-/** Number of output tokens used by the model */
-outputTokens: number
-; 
-/** Cost of the output tokens generated by the model, in U.S. dollars */
-outputCost: number
- }; botpress: { 
-/** Total cost of the content generation, in U.S. dollars */
-cost: number
- } };;
-    }
-}

package/src/sdk-interfaces/llm/listLanguageModels.ts

@@ -1,19 +0,0 @@
-
-// This file is generated. Do not edit it manually.
-// See 'scripts/update-models.ts'
-
-/* eslint-disable */
-/* tslint:disable */
-
-export namespace llm {
-    export namespace listLanguageModels {
-      export type Input = { };;
-      export type Output = { models: Array<{ id: string; name: string; description: string; tags: Array<'recommended' | 'deprecated' | 'general-purpose' | 'low-cost' | 'vision' | 'coding' | 'agents' | 'function-calling' | 'roleplay' | 'storytelling' | 'reasoning'>; input: { maxTokens: number; 
-/** Cost per 1 million tokens, in U.S. dollars */
-costPer1MTokens: number
- }; output: { maxTokens: number; 
-/** Cost per 1 million tokens, in U.S. dollars */
-costPer1MTokens: number
- } } & { id: string }> };;
-    }
-}

package/src/task/compare.ts

@@ -1,72 +0,0 @@
-import { z } from '@botpress/sdk'
-import { TestFunction } from 'vitest'
-import { createTaskCollector, getCurrentSuite } from 'vitest/suite'
-import { TestMetadata } from '../context'
-import { Deferred } from '../utils/deferred'
-
-const scenarioId = z
-  .string()
-  .trim()
-  .min(1, 'Scenario ID/name must not be empty')
-  .max(50, 'Scenario ID/name is too long')
-
-export type ScenarioLike = z.infer<typeof ScenarioLike>
-const ScenarioLike = z.union([
-  scenarioId,
-  z.object({ name: scenarioId }).passthrough(),
-  z.object({ id: scenarioId }).passthrough()
-])
-
-const getScenarioName = (scenario: ScenarioLike) =>
-  (typeof scenario === 'string' ? scenario : 'name' in scenario ? scenario?.name : scenario?.id) as string
-
-const scenarioArgs = z
-  .array(ScenarioLike)
-  .min(2, 'You need at least two scenarios to compare')
-  .max(10, 'You can only compare up to 10 scenarios')
-  .refine((scenarios) => {
-    const set = new Set<string>()
-    scenarios.forEach((scenario) => set.add(getScenarioName(scenario)))
-    return set.size === scenarios.length
-  }, 'Scenarios names must be unique')
-
-export function compare<T extends ReadonlyArray<ScenarioLike>>(
-  name: string | Function,
-  scenarios: T,
-  fn?: TestFunction<{
-    scenario: T[number]
-  }>
-) {
-  scenarios = scenarioArgs.parse(scenarios) as unknown as T
-
-  return createTaskCollector((_name, fn, timeout) => {
-    const currentSuite = getCurrentSuite()
-
-    let completedCount = 0
-    const finished = new Deferred<void>()
-
-    for (const scenario of scenarios) {
-      const key = getScenarioName(scenario)
-
-      currentSuite.task(key, {
-        meta: {
-          scenario: key,
-          isVaiTest: true
-        } satisfies TestMetadata,
-        handler: async (context) => {
-          const extendedContext = Object.freeze({
-            scenario
-          })
-          context.onTestFinished(() => {
-            if (++completedCount === scenarios.length) {
-              finished.resolve()
-            }
-          })
-
-          await fn({ ...context, ...extendedContext })
-        },
-        timeout: timeout ?? 10_000
-      })
-    }
-  })(name, fn)
-}

package/src/utils/asyncAssertion.ts

@@ -1,40 +0,0 @@
-import { Assertion, expect } from 'vitest'
-import { getCurrentTest } from 'vitest/suite'
-import { Context } from '../context'
-import { Output } from './predictJson'
-
-export class AsyncExpectError<T> extends Error {
-  constructor(message: string, public readonly output: Output<T>) {
-    super(message)
-    this.name = 'AsyncExpectError'
-  }
-}
-
-const getErrorMessages = (e: unknown): string => {
-  if (e instanceof Error) {
-    return e.message
-  } else if (typeof e === 'string') {
-    return e
-  } else if (typeof e === 'object' && e !== null) {
-    return JSON.stringify(e)
-  }
-
-  return `Unknown error: ${e}`
-}
-
-export const asyncExpect = <T>(output: Promise<Output<T>>, assertion: (assert: Assertion<T>) => void) => {
-  const promise = output.then((x) => {
-    try {
-      assertion(expect(x.result, x.reason))
-    } catch (e: unknown) {
-      if (Context.wrapError) {
-        return new AsyncExpectError<T>(getErrorMessages(e), x)
-      }
-      throw e
-    }
-    return x
-  })
-  getCurrentTest()!.promises ??= []
-  getCurrentTest()!.promises!.push(promise)
-  return promise
-}

package/src/utils/deferred.ts

@@ -1,20 +0,0 @@
-export class Deferred<T> {
-  promise: Promise<T>
-  private _resolve!: (value: T | PromiseLike<T>) => void
-  private _reject!: (reason?: unknown) => void
-
-  constructor() {
-    this.promise = new Promise<T>((resolve, reject) => {
-      this._resolve = resolve
-      this._reject = reject
-    })
-  }
-
-  resolve(value: T | PromiseLike<T>): void {
-    this._resolve(value)
-  }
-
-  reject(reason?: unknown): void {
-    this._reject(reason)
-  }
-}

package/src/utils/predictJson.ts

@@ -1,114 +0,0 @@
-import { z, ZodSchema } from '@botpress/sdk'
-import JSON5 from 'json5'
-import { Context } from '../context'
-import { llm } from '../sdk-interfaces/llm/generateContent'
-
-const nonEmptyString = z.string().trim().min(1)
-const nonEmptyObject = z
-  .object({})
-  .passthrough()
-  .refine((value) => Object.keys(value).length > 0, {
-    message: 'Expected a non-empty object'
-  })
-
-export type Input = z.infer<typeof Input>
-const Input = nonEmptyString.or(nonEmptyObject).or(z.array(z.any()))
-
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-export type Output<T = any> = z.infer<typeof Output> & { result: T }
-const Output = z.object({
-  reason: nonEmptyString.describe('A human-readable explanation of the result'),
-  result: z
-    .any()
-    .describe(
-      'Your best guess at the output according to the instructions provided, rooted in the context of the input and the reason above'
-    )
-})
-
-type Example = z.infer<typeof Example>
-const Example = z.object({
-  input: Input,
-  output: Output
-})
-
-type InputOptions<T extends ZodSchema = ZodSchema> = z.input<typeof Options> & { outputSchema: T }
-type Options = z.infer<typeof Options>
-const Options = z.object({
-  systemMessage: z.string(),
-  examples: z.array(Example).default([]),
-  input: Input,
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  outputSchema: z.custom<ZodSchema<any>>((value) => value instanceof ZodSchema),
-  model: z.string()
-})
-
-type Message = {
-  role: 'user' | 'assistant' | 'system'
-  content: string
-}
-
-const isValidExample =
-  (outputSchema: ZodSchema) =>
-  (example: Example): example is Example =>
-    Input.safeParse(example.input).success &&
-    Output.safeParse(example.output).success &&
-    outputSchema.safeParse(example.output.result).success
-
-export async function predictJson<T extends ZodSchema>(_options: InputOptions<T>): Promise<Output<z.infer<T>>> {
-  const options = Options.parse(_options)
-  const [integration, model] = options.model.split('__')
-
-  if (!model?.length) {
-    throw new Error('Invalid model')
-  }
-
-  const exampleMessages = options.examples
-    .filter(isValidExample(options.outputSchema))
-    .flatMap(({ input, output }) => [
-      { role: 'user', content: JSON.stringify(input, null, 2) } satisfies Message,
-      { role: 'assistant', content: JSON.stringify(output, null, 2) } satisfies Message
-    ])
-
-  const outputSchema = Output.extend({
-    result: options.outputSchema.describe(Output.shape.result.description!)
-  })
-
-  const result = await Context.client.callAction({
-    type: `${integration}:generateContent`,
-    input: {
-      systemPrompt: `
-${options.systemMessage}
-
----
-Please generate a JSON response with the following format:
-\`\`\`typescript
-${await outputSchema.toTypescriptAsync()}
-\`\`\`
-`.trim(),
-      messages: [
-        ...exampleMessages,
-        {
-          role: 'user',
-          content: JSON.stringify(options.input, null, 2)
-        }
-      ],
-      temperature: 0,
-      responseFormat: 'json_object',
-      model: { id: model! }
-    } satisfies llm.generateContent.Input
-  })
-
-  const output = result.output as llm.generateContent.Output
-
-  if (!output.choices.length || typeof output.choices?.[0]?.content !== 'string') {
-    throw new Error('Invalid response from the model')
-  }
-
-  const json = output.choices[0].content.trim()
-
-  if (!json.length) {
-    throw new Error('No response from the model')
-  }
-
-  return outputSchema.parse(JSON5.parse(json)) as Output<z.infer<T>>
-}