@botpress/zai 1.0.0-beta.2 → 1.0.0-beta.4

package/src/operations/__tests/index.ts

@@ -1,30 +0,0 @@
-import { Client } from '@botpress/client'
-import { type TextTokenizer, getWasmTokenizer } from '@botpress/wasm'
-import fs from 'node:fs'
-import path from 'node:path'
-import { beforeAll } from 'vitest'
-
-import { Zai } from '../..'
-
-export const getClient = () => {
-  return new Client({
-    apiUrl: process.env.VITE_CLOUD_API_ENDPOINT,
-    botId: process.env.VITEST_BOT_ID ?? process.env.VITE_BOT_ID,
-    token: process.env.VITEST_CLOUD_PAT ?? process.env.VITE_CLOUD_PAT
-  })
-}
-
-export const getZai = () => {
-  const client = getClient()
-  return new Zai({ client, retry: { maxRetries: 0 } })
-}
-
-export let tokenizer: TextTokenizer = null!
-
-beforeAll(async () => {
-  tokenizer = await getWasmTokenizer()
-})
-
-export const BotpressDocumentation = fs.readFileSync(path.join(__dirname, './botpress_docs.txt'), 'utf-8').trim()
-
-export const metadata = { cost: { input: 1, output: 1 }, latency: 0, model: '', tokens: { input: 1, output: 1 } }

package/src/operations/check.test.ts

@@ -1,155 +0,0 @@
-import { describe, it, expect, afterAll, beforeEach, afterEach } from 'vitest'
-
-import { BotpressDocumentation, getClient, getZai, metadata } from './__tests'
-
-import { TableAdapter } from '../adapters/botpress-table'
-
-describe('zai.check', { timeout: 60_000 }, () => {
-  const zai = getZai()
-
-  it('basic check on a string', async () => {
-    const value = await zai.check('This text is very clearly written in English.', 'is an english sentence')
-    expect(value).toBe(true)
-  })
-
-  it('text that is too long gets truncated', async () => {
-    const isBotpressDocumentation = await zai.check(
-      BotpressDocumentation,
-      'is about botpress and looks like documentation'
-    )
-
-    const isAboutBirds = await zai.check(BotpressDocumentation, 'is a book about birds and their species')
-
-    expect(isBotpressDocumentation).toBe(true)
-    expect(isAboutBirds).toBe(false)
-  })
-
-  it('works with any input type', async () => {
-    const sly = { name: 'Sylvain Perron', age: 30, job: 'CEO', company: 'Botpress', location: 'Quebec' }
-    const american = await zai.check(sly, 'person lives in north america')
-    const european = await zai.check(sly, 'person lives in europe')
-
-    expect(american).toBe(true)
-    expect(european).toBe(false)
-  })
-
-  it('check with examples', async () => {
-    const examples = [
-      {
-        input: 'Rasa (framework)',
-        check: true,
-        reason: 'Rasa is a chatbot framework, so it competes with us (Botpress).'
-      },
-      {
-        input: 'Rasa (coffee company)',
-        check: false,
-        reason:
-          'Rasa (coffee company) is not in the chatbot or AI agent industry, therefore it does not compete with us (Botpress).'
-      },
-      {
-        input: 'Dialogflow',
-        check: true,
-        reason: 'Dialogflow is a chatbot development product, so it competes with us (Botpress).'
-      }
-    ]
-
-    const moveworks = await zai.check('Moveworks', 'competes with us', { examples })
-    const ada = await zai.check('Ada.cx', 'competes with us', { examples })
-    const voiceflow = await zai.check('Voiceflow', 'competes with us', { examples })
-    const nike = await zai.check('Nike', 'competes with us', { examples })
-    const adidas = await zai.check('Adidas', 'competes with us', { examples })
-
-    expect(moveworks).toBe(true)
-    expect(ada).toBe(true)
-    expect(voiceflow).toBe(true)
-
-    expect(nike).toBe(false)
-    expect(adidas).toBe(false)
-  })
-})
-
-describe('zai.learn.check', { timeout: 60_000 }, () => {
-  const client = getClient()
-  let tableName = 'ZaiTestCheckInternalTable'
-  let taskId = 'check'
-  let zai = getZai()
-
-  beforeEach(async () => {
-    zai = getZai().with({
-      activeLearning: {
-        enable: true,
-        taskId,
-        tableName
-      }
-    })
-  })
-
-  afterEach(async () => {
-    try {
-      await client.deleteTableRows({ table: tableName, deleteAllRows: true })
-    } catch (err) {}
-  })
-
-  afterAll(async () => {
-    try {
-      await client.deleteTable({ table: tableName })
-    } catch (err) {}
-  })
-
-  it('learns a contradiction from examples', async () => {
-    const adapter = new TableAdapter({
-      client,
-      tableName
-    })
-
-    const value = await zai.learn(taskId).check(`What's up`, 'is a greeting')
-    expect(value).toBe(true)
-
-    let rows = await client.findTableRows({ table: tableName })
-    expect(rows.rows.length).toBe(1)
-    expect(rows.rows[0].output.value).toEqual(value)
-
-    await adapter.saveExample({
-      key: 't1',
-      taskId: `zai/${taskId}`,
-      taskType: 'zai.check',
-      instructions: 'is a greeting',
-      input: 'what is up',
-      output: false,
-      explanation: `"What's up" is our business scenario is NOT considered an official greeting.`,
-      metadata,
-      status: 'approved'
-    })
-
-    await adapter.saveExample({
-      key: 't2',
-      taskId: `zai/${taskId}`,
-      taskType: 'zai.check',
-      instructions: 'is a greeting',
-      input: 'hello! how are you?',
-      output: true,
-      explanation: `"hello!" is a common greeting in English.`,
-      metadata,
-      status: 'approved'
-    })
-
-    await adapter.saveExample({
-      key: 't3',
-      taskId: `zai/${taskId}`,
-      taskType: 'zai.check',
-      instructions: 'is a greeting',
-      input: 'wassup',
-      output: false,
-      explanation: `"wassup" is a slang term and not considered a formal greeting. It is therefore NOT considered a greeting.`,
-      metadata,
-      status: 'approved'
-    })
-
-    const second = await zai.learn(taskId).check(`What's up`, 'is a greeting')
-    expect(second).toBe(false)
-
-    rows = await client.findTableRows({ table: tableName })
-    expect(rows.rows.length).toBe(4)
-    expect(rows.rows[0].output.value).toEqual(second)
-  })
-})

package/src/operations/check.ts

@@ -1,187 +0,0 @@
-import { z } from '@bpinternal/zui'
-
-import { fastHash, stringify, takeUntilTokens } from '../utils'
-import { Zai } from '../zai'
-import { PROMPT_INPUT_BUFFER } from './constants'
-
-const Example = z.object({
-  input: z.any(),
-  check: z.boolean(),
-  reason: z.string().optional()
-})
-
-export type Options = z.input<typeof Options>
-const Options = z.object({
-  examples: z.array(Example).describe('Examples to check the condition against').default([])
-})
-
-declare module '../zai' {
-  interface Zai {
-    /** Checks wether a condition is true or not */
-    check(input: unknown, condition: string, options?: Options): Promise<boolean>
-  }
-}
-
-const TRUE = '■TRUE■'
-const FALSE = '■FALSE■'
-const END = '■END■'
-
-Zai.prototype.check = async function (this: Zai, input, condition, _options) {
-  const options = Options.parse(_options ?? {})
-  const tokenizer = await this.getTokenizer()
-  const PROMPT_COMPONENT = Math.max(this.Model.input.maxTokens - PROMPT_INPUT_BUFFER, 100)
-
-  const taskId = this.taskId
-  const taskType = 'zai.check'
-
-  const PROMPT_TOKENS = {
-    INPUT: Math.floor(0.5 * PROMPT_COMPONENT),
-    CONDITION: Math.floor(0.2 * PROMPT_COMPONENT)
-  }
-
-  // Truncate the input to fit the model's input size
-  const inputAsString = tokenizer.truncate(stringify(input), PROMPT_TOKENS.INPUT)
-  condition = tokenizer.truncate(condition, PROMPT_TOKENS.CONDITION)
-
-  // All tokens remaining after the input and condition are accounted can be used for examples
-  const EXAMPLES_TOKENS = PROMPT_COMPONENT - tokenizer.count(inputAsString) - tokenizer.count(condition)
-
-  const Key = fastHash(
-    JSON.stringify({
-      taskType,
-      taskId,
-      input: inputAsString,
-      condition
-    })
-  )
-
-  const examples = taskId
-    ? await this.adapter.getExamples<string, boolean>({
-        input: inputAsString,
-        taskType,
-        taskId
-      })
-    : []
-
-  const exactMatch = examples.find((x) => x.key === Key)
-  if (exactMatch) {
-    return exactMatch.output
-  }
-
-  const defaultExamples = [
-    { input: '50 Cent', check: true, reason: '50 Cent is widely recognized as a public personality.' },
-    {
-      input: ['apple', 'banana', 'carrot', 'house'],
-      check: false,
-      reason: 'The list contains a house, which is not a fruit. Also, the list contains a carrot, which is a vegetable.'
-    }
-  ]
-
-  const userExamples = [
-    ...examples.map((e) => ({ input: e.input, check: e.output, reason: e.explanation })),
-    ...options.examples
-  ]
-
-  let exampleId = 1
-
-  const formatInput = (input: string, condition: string) => {
-    const header = userExamples.length ? `Expert Example #${exampleId++}` : `Example of condition: "${condition}"`
-
-    return `
-${header}
-<|start_input|>
-${input.trim()}
-<|end_input|>
-`.trim()
-  }
-
-  const formatOutput = (answer: boolean, justification: string) => {
-    return `
-Analysis: ${justification}
-Final Answer: ${answer ? TRUE : FALSE}
-${END}
-`.trim()
-  }
-
-  const formatExample = (example: { input?: any; check: boolean; reason?: string }) => [
-    { type: 'text' as const, content: formatInput(stringify(example.input ?? null), condition), role: 'user' as const },
-    {
-      type: 'text' as const,
-      content: formatOutput(example.check, example.reason ?? ''),
-      role: 'assistant' as const
-    }
-  ]
-
-  const allExamples = takeUntilTokens(
-    userExamples.length ? userExamples : defaultExamples,
-    EXAMPLES_TOKENS,
-    (el) => tokenizer.count(stringify(el.input)) + tokenizer.count(el.reason ?? '')
-  )
-    .map(formatExample)
-    .flat()
-
-  const specialInstructions = userExamples.length
-    ? `
-- You have been provided with examples from previous experts. Make sure to read them carefully before making your decision.
-- Make sure to refer to the examples provided by the experts to justify your decision (when applicable).
-- When in doubt, ground your decision on the examples provided by the experts instead of your own intuition.
-- When no example is similar to the input, make sure to provide a clear justification for your decision while inferring the decision-making process from the examples provided by the experts.
-`.trim()
-    : ''
-
-  const output = await this.callModel({
-    systemPrompt: `
-Check if the following condition is true or false for the given input. Before answering, make sure to read the input and the condition carefully.
-Justify your answer, then answer with either ${TRUE} or ${FALSE} at the very end, then add ${END} to finish the response.
-IMPORTANT: Make sure to answer with either ${TRUE} or ${FALSE} at the end of your response, but NOT both.
----
-Expert Examples (#1 to #${exampleId - 1}):
-${specialInstructions}
-`.trim(),
-    stopSequences: [END],
-    messages: [
-      ...allExamples,
-      {
-        type: 'text',
-        content: `
-Considering the below input and above examples, is the following condition true or false?
-${formatInput(inputAsString, condition)}
-In your "Analysis", please refer to the Expert Examples # to justify your decision.`.trim(),
-        role: 'user'
-      }
-    ]
-  })
-
-  const answer = output.choices[0].content as string
-
-  const hasTrue = answer.includes(TRUE)
-  const hasFalse = answer.includes(FALSE)
-
-  if (!hasTrue && !hasFalse) {
-    throw new Error(`The model did not return a valid answer. The response was: ${answer}`)
-  }
-
-  let finalAnswer: boolean
-
-  if (hasTrue && hasFalse) {
-    // If both TRUE and FALSE are present, we need to check which one was answered last
-    finalAnswer = answer.lastIndexOf(TRUE) > answer.lastIndexOf(FALSE)
-  } else {
-    finalAnswer = hasTrue
-  }
-
-  if (taskId) {
-    await this.adapter.saveExample({
-      key: Key,
-      taskType,
-      taskId,
-      input: inputAsString,
-      instructions: condition,
-      metadata: output.metadata,
-      output: finalAnswer,
-      explanation: answer.replace(TRUE, '').replace(FALSE, '').replace(END, '').replace('Final Answer:', '').trim()
-    })
-  }
-
-  return finalAnswer
-}

package/src/operations/constants.ts

@@ -1,2 +0,0 @@
-export const PROMPT_INPUT_BUFFER = 1048
-export const PROMPT_OUTPUT_BUFFER = 512

package/src/operations/errors.ts

@@ -1,9 +0,0 @@
-export class JsonParsingError extends Error {
-  constructor(
-    public json: unknown,
-    public error: Error
-  ) {
-    const message = `Error parsing JSON:\n\n---JSON---\n${json}\n\n---Error---\n\n ${error}`
-    super(message)
-  }
-}

package/src/operations/extract.test.ts

@@ -1,209 +0,0 @@
-import { describe, it, expect, beforeEach, afterEach, afterAll } from 'vitest'
-
-import { BotpressDocumentation, getClient, getZai, metadata } from './__tests'
-import { z } from '@bpinternal/zui'
-import { check } from '@botpress/vai'
-import { TableAdapter } from '../adapters/botpress-table'
-
-describe('zai.extract', () => {
-  const zai = getZai()
-
-  it('extract simple object from paragraph', async () => {
-    const person = await zai.extract(
-      'My name is John Doe, I am 30 years old and I live in Quebec',
-      z.object({
-        name: z.string(),
-        age: z.number(),
-        location: z.string()
-      })
-    )
-
-    expect(person).toMatchInlineSnapshot(`
-      {
-        "age": 30,
-        "location": "Quebec",
-        "name": "John Doe",
-      }
-    `)
-  })
-
-  it('extract an array of objects from paragraph', async () => {
-    const people = await zai.extract(
-      `
-      My name is John Doe, I am 30 years old and I live in Quebec.
-      My name is Jane Doe, I am 25 years old and I live in Montreal.
-      His name is Jack Doe, he is 35 years old and he lives in Toronto.
-      Her name is Jill Doe, she is 40 years old and she lives in Vancouver.`,
-      z.array(
-        z.object({
-          name: z.string(),
-          age: z.number(),
-          location: z.string()
-        })
-      )
-    )
-
-    expect(people).toMatchInlineSnapshot(`
-      [
-        {
-          "age": 30,
-          "location": "Quebec",
-          "name": "John Doe",
-        },
-        {
-          "age": 25,
-          "location": "Montreal",
-          "name": "Jane Doe",
-        },
-        {
-          "age": 35,
-          "location": "Toronto",
-          "name": "Jack Doe",
-        },
-        {
-          "age": 40,
-          "location": "Vancouver",
-          "name": "Jill Doe",
-        },
-      ]
-    `)
-  })
-
-  it('extract an object from anything as input', async () => {
-    const person = await zai.extract(
-      {
-        person: { first: 'John', last: 'Doe', age: 30 }
-      },
-      z.object({
-        a: z.string().describe('The full name of the person in the text'),
-        b: z.number().describe('The age of the person in the text')
-      })
-    )
-
-    expect(person).toMatchInlineSnapshot(`
-      {
-        "a": "John Doe",
-        "b": 30,
-      }
-    `)
-  })
-
-  it('extract an array of objects from a super long text', async () => {
-    const features = await zai.extract(
-      BotpressDocumentation,
-      z.array(
-        z
-          .object({
-            feature: z.string().describe('The name of the feature'),
-            parent: z.string().optional().describe('The parent feature').nullable(),
-            description: z.string().describe('The description of the feature')
-          })
-          .describe('A feature of Botpress')
-      ),
-      {
-        instructions:
-          'Extract all things that looks like a Botpress feature in the provided input. You must extract a minimum of one element.'
-      }
-    )
-
-    check(features, 'Contains an element about flows / workflows').toBe(true)
-    check(features, 'Contains an element about tables').toBe(true)
-    check(features, 'Contains an element about webchat').toBe(true)
-    check(features, 'Contains an element about integrations').toBe(true)
-    check(features, 'Contains an element about HITL (human in the loop)').toBe(true)
-    check(features, 'Contains an element about knowledge bases (KB) or Files API').toBe(true)
-  })
-})
-
-describe('zai.learn.extract', () => {
-  const client = getClient()
-  let tableName = 'ZaiTestExtractInternalTable'
-  let taskId = 'extract'
-  let zai = getZai()
-
-  beforeEach(async () => {
-    zai = getZai().with({
-      activeLearning: {
-        enable: true,
-        taskId,
-        tableName
-      }
-    })
-  })
-
-  afterEach(async () => {
-    try {
-      await client.deleteTableRows({ table: tableName, deleteAllRows: true })
-    } catch (err) {}
-  })
-
-  afterAll(async () => {
-    try {
-      await client.deleteTable({ table: tableName })
-    } catch (err) {}
-  })
-
-  it('learns a extraction format from examples', async () => {
-    const adapter = new TableAdapter({
-      client,
-      tableName
-    })
-
-    const value = await zai.learn(taskId).extract(
-      `I really liked Casino Royale`,
-      z.object({
-        name: z.string(),
-        movie: z.string()
-      }),
-      { instructions: 'extract the name of the movie and name of the main character' }
-    )
-
-    check(value, 'extracted james bond and casino royale').toBe(true)
-    check(value, 'the values are NOT IN ALL CAPS').toBe(true)
-
-    let rows = await client.findTableRows({ table: tableName })
-    expect(rows.rows.length).toBe(1)
-
-    await adapter.saveExample({
-      key: 't1',
-      taskId: `zai/${taskId}`,
-      taskType: 'zai.extract',
-      instructions: 'extract name of movie and main character',
-      input: `I went to see the Titanic yesterday and I fell asleep`,
-      output: { name: 'JACK DAWSON', movie: 'TITANIC' },
-      metadata,
-      status: 'approved'
-    })
-
-    await adapter.saveExample({
-      key: 't2',
-      taskId: `zai/${taskId}`,
-      taskType: 'zai.extract',
-      instructions: 'extract name of movie and main character',
-      input: `Did you know that the gladiator movie has a lot of fighting scenes?`,
-      output: { name: 'MAXIMUS DECIMUS MERIDIUS', movie: 'GLADIATOR' },
-      metadata,
-      status: 'approved'
-    })
-
-    const second = await zai.learn(taskId).extract(
-      `I really liked Casino Royale`,
-      z.object({
-        name: z.string(),
-        movie: z.string()
-      }),
-      { instructions: 'extract the name of the movie and name of the main character' }
-    )
-
-    expect(second).toMatchInlineSnapshot(`
-      {
-        "movie": "CASINO ROYALE",
-        "name": "JAMES BOND",
-      }
-    `)
-
-    rows = await client.findTableRows({ table: tableName })
-    expect(rows.rows.length).toBe(3)
-    expect(rows.rows[0].output.value).toMatchObject(second)
-  })
-})