@botpress/vai 0.0.1-beta.1

package/src/task/compare.ts

@@ -0,0 +1,72 @@
+import { z } from '@botpress/sdk'
+import { TestFunction } from 'vitest'
+import { createTaskCollector, getCurrentSuite } from 'vitest/suite'
+import { TestMetadata } from '../context'
+import { Deferred } from '../utils/deferred'
+
+const scenarioId = z
+  .string()
+  .trim()
+  .min(1, 'Scenario ID/name must not be empty')
+  .max(50, 'Scenario ID/name is too long')
+
+export type ScenarioLike = z.infer<typeof ScenarioLike>
+const ScenarioLike = z.union([
+  scenarioId,
+  z.object({ name: scenarioId }).passthrough(),
+  z.object({ id: scenarioId }).passthrough()
+])
+
+const getScenarioName = (scenario: ScenarioLike) =>
+  (typeof scenario === 'string' ? scenario : 'name' in scenario ? scenario?.name : scenario?.id) as string
+
+const scenarioArgs = z
+  .array(ScenarioLike)
+  .min(2, 'You need at least two scenarios to compare')
+  .max(10, 'You can only compare up to 10 scenarios')
+  .refine((scenarios) => {
+    const set = new Set<string>()
+    scenarios.forEach((scenario) => set.add(getScenarioName(scenario)))
+    return set.size === scenarios.length
+  }, 'Scenarios names must be unique')
+
+export function compare<T extends ReadonlyArray<ScenarioLike>>(
+  name: string | Function,
+  scenarios: T,
+  fn?: TestFunction<{
+    scenario: T[number]
+  }>
+) {
+  scenarios = scenarioArgs.parse(scenarios) as unknown as T
+
+  return createTaskCollector((_name, fn, timeout) => {
+    const currentSuite = getCurrentSuite()
+
+    let completedCount = 0
+    const finished = new Deferred<void>()
+
+    for (const scenario of scenarios) {
+      const key = getScenarioName(scenario)
+
+      currentSuite.task(key, {
+        meta: {
+          scenario: key,
+          isVaiTest: true
+        } satisfies TestMetadata,
+        handler: async (context) => {
+          const extendedContext = Object.freeze({
+            scenario
+          })
+          context.onTestFinished(() => {
+            if (++completedCount === scenarios.length) {
+              finished.resolve()
+            }
+          })
+
+          await fn({ ...context, ...extendedContext })
+        },
+        timeout: timeout ?? 10_000
+      })
+    }
+  })(name, fn)
+}

package/src/utils/asyncAssertion.ts

@@ -0,0 +1,40 @@
+import { Assertion, expect } from 'vitest'
+import { getCurrentTest } from 'vitest/suite'
+import { Context } from '../context'
+import { Output } from './predictJson'
+
+export class AsyncExpectError<T> extends Error {
+  constructor(message: string, public readonly output: Output<T>) {
+    super(message)
+    this.name = 'AsyncExpectError'
+  }
+}
+
+const getErrorMessages = (e: unknown): string => {
+  if (e instanceof Error) {
+    return e.message
+  } else if (typeof e === 'string') {
+    return e
+  } else if (typeof e === 'object' && e !== null) {
+    return JSON.stringify(e)
+  }
+
+  return `Unknown error: ${e}`
+}
+
+export const asyncExpect = <T>(output: Promise<Output<T>>, assertion: (assert: Assertion<T>) => void) => {
+  const promise = output.then((x) => {
+    try {
+      assertion(expect(x.result, x.reason))
+    } catch (e: unknown) {
+      if (Context.wrapError) {
+        return new AsyncExpectError<T>(getErrorMessages(e), x)
+      }
+      throw e
+    }
+    return x
+  })
+  getCurrentTest()!.promises ??= []
+  getCurrentTest()!.promises!.push(promise)
+  return promise
+}

package/src/utils/deferred.ts

@@ -0,0 +1,20 @@
+export class Deferred<T> {
+  promise: Promise<T>
+  private _resolve!: (value: T | PromiseLike<T>) => void
+  private _reject!: (reason?: unknown) => void
+
+  constructor() {
+    this.promise = new Promise<T>((resolve, reject) => {
+      this._resolve = resolve
+      this._reject = reject
+    })
+  }
+
+  resolve(value: T | PromiseLike<T>): void {
+    this._resolve(value)
+  }
+
+  reject(reason?: unknown): void {
+    this._reject(reason)
+  }
+}

package/src/utils/predictJson.ts

@@ -0,0 +1,114 @@
+import { z, ZodSchema } from '@botpress/sdk'
+import JSON5 from 'json5'
+import { Context } from '../context'
+import { llm } from '../sdk-interfaces/llm/generateContent'
+
+const nonEmptyString = z.string().trim().min(1)
+const nonEmptyObject = z
+  .object({})
+  .passthrough()
+  .refine((value) => Object.keys(value).length > 0, {
+    message: 'Expected a non-empty object'
+  })
+
+export type Input = z.infer<typeof Input>
+const Input = nonEmptyString.or(nonEmptyObject).or(z.array(z.any()))
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type Output<T = any> = z.infer<typeof Output> & { result: T }
+const Output = z.object({
+  reason: nonEmptyString.describe('A human-readable explanation of the result'),
+  result: z
+    .any()
+    .describe(
+      'Your best guess at the output according to the instructions provided, rooted in the context of the input and the reason above'
+    )
+})
+
+type Example = z.infer<typeof Example>
+const Example = z.object({
+  input: Input,
+  output: Output
+})
+
+type InputOptions<T extends ZodSchema = ZodSchema> = z.input<typeof Options> & { outputSchema: T }
+type Options = z.infer<typeof Options>
+const Options = z.object({
+  systemMessage: z.string(),
+  examples: z.array(Example).default([]),
+  input: Input,
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  outputSchema: z.custom<ZodSchema<any>>((value) => value instanceof ZodSchema),
+  model: z.string()
+})
+
+type Message = {
+  role: 'user' | 'assistant' | 'system'
+  content: string
+}
+
+const isValidExample =
+  (outputSchema: ZodSchema) =>
+  (example: Example): example is Example =>
+    Input.safeParse(example.input).success &&
+    Output.safeParse(example.output).success &&
+    outputSchema.safeParse(example.output.result).success
+
+export async function predictJson<T extends ZodSchema>(_options: InputOptions<T>): Promise<Output<z.infer<T>>> {
+  const options = Options.parse(_options)
+  const [integration, model] = options.model.split('__')
+
+  if (!model?.length) {
+    throw new Error('Invalid model')
+  }
+
+  const exampleMessages = options.examples
+    .filter(isValidExample(options.outputSchema))
+    .flatMap(({ input, output }) => [
+      { role: 'user', content: JSON.stringify(input, null, 2) } satisfies Message,
+      { role: 'assistant', content: JSON.stringify(output, null, 2) } satisfies Message
+    ])
+
+  const outputSchema = Output.extend({
+    result: options.outputSchema.describe(Output.shape.result.description!)
+  })
+
+  const result = await Context.client.callAction({
+    type: `${integration}:generateContent`,
+    input: {
+      systemPrompt: `
+${options.systemMessage}
+
+---
+Please generate a JSON response with the following format:
+\`\`\`typescript
+${await outputSchema.toTypescriptAsync()}
+\`\`\`
+`.trim(),
+      messages: [
+        ...exampleMessages,
+        {
+          role: 'user',
+          content: JSON.stringify(options.input, null, 2)
+        }
+      ],
+      temperature: 0,
+      responseFormat: 'json_object',
+      model: { id: model! }
+    } satisfies llm.generateContent.Input
+  })
+
+  const output = result.output as llm.generateContent.Output
+
+  if (!output.choices.length || typeof output.choices?.[0]?.content !== 'string') {
+    throw new Error('Invalid response from the model')
+  }
+
+  const json = output.choices[0].content.trim()
+
+  if (!json.length) {
+    throw new Error('No response from the model')
+  }
+
+  return outputSchema.parse(JSON5.parse(json)) as Output<z.infer<T>>
+}

package/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "compilerOptions": {
+    "esModuleInterop": true,
+    "module": "ESNext",
+    "moduleResolution": "Node",
+    "target": "ES2017",
+    "strict": true,
+    "jsx": "preserve",
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noUncheckedIndexedAccess": true,
+    "allowSyntheticDefaultImports": true,
+    "lib": ["dom", "ESNext", "dom.iterable"],
+    "allowJs": false,
+    "declaration": false,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": false,
+    "resolveJsonModule": true,
+    "isolatedModules": true
+  },
+  "exclude": ["node_modules", "dist"],
+  "include": ["src/**/*", "vitest.d.ts"],
+  "ts-node": {
+    "esm": true,
+    "require": ["dotenv/config", "./ensure-env.cjs"]
+  }
+}

package/tsup.config.ts

@@ -0,0 +1,16 @@
+import { defineConfig } from 'tsup'
+
+export default defineConfig({
+  entry: ['src/index.ts'],
+  splitting: false,
+  format: ['esm', 'cjs'],
+  sourcemap: true,
+  keepNames: true,
+  dts: true,
+  platform: 'browser',
+  clean: true,
+  shims: true,
+  external: ['react', 'react-dom'],
+  bundle: true,
+  plugins: [],
+})

package/vitest.config.ts

@@ -0,0 +1,9 @@
+import 'dotenv/config'
+import { defineConfig } from 'vitest/config'
+
+export default defineConfig({
+  test: {
+    include: ['./src/**/*.test.ts'],
+    setupFiles: './vitest.setup.ts'
+  }
+})

package/vitest.setup.ts

@@ -0,0 +1,13 @@
+import { beforeAll } from 'vitest'
+import { setupClient } from './src/hooks/setupClient'
+import { Client } from '@botpress/client'
+
+beforeAll(async () => {
+  setupClient(
+    new Client({
+      apiUrl: process.env.CLOUD_API_ENDPOINT ?? 'https://api.botpress.dev',
+      botId: process.env.CLOUD_BOT_ID,
+      token: process.env.CLOUD_PAT
+    })
+  )
+})