@botpress/zai 1.0.0-beta.1 → 1.0.0-beta.3

package/src/operations/label.test.ts

@@ -1,239 +0,0 @@
-import { describe, it, expect, beforeEach, afterEach, afterAll } from 'vitest'
-
-import { BotpressDocumentation, getClient, getZai, metadata } from './__tests'
-import { TableAdapter } from '../adapters/botpress-table'
-import { check } from '@botpress/vai'
-
-describe('zai.label', { timeout: 60_000 }, () => {
-  const zai = getZai()
-
-  it('simple labels on small text', async () => {
-    const labels = await zai.label(
-      {
-        name: 'John',
-        story: ['John donated to charity last month.', 'John is loved by his community.'],
-        criminal_record: 'John has no criminal record.'
-      },
-      {
-        is_human: 'is the person a human?',
-        good_person: 'is the person a good person?',
-        bad_person: 'is the person a bad person?',
-        is_criminal: 'is the person a criminal?'
-      }
-    )
-
-    expect(labels).toMatchInlineSnapshot(`
-      {
-        "bad_person": false,
-        "good_person": true,
-        "is_criminal": false,
-        "is_human": true,
-      }
-    `)
-  })
-
-  it('simple labels with example', async () => {
-    const labels = {
-      is_human: 'is the person a human?',
-      good_person: 'is the person a good person?',
-      bad_person: 'is the person a bad person?',
-      canadian: 'is the person canadian?',
-      is_french: 'is the person french?'
-    }
-
-    const initial = await zai.label(`Sylvain Perron has no criminal record.`, labels)
-
-    expect(initial.canadian).toBe(false)
-    expect(initial.is_french).toBe(false)
-    expect(initial.bad_person).toBe(false)
-    expect(initial.is_human).toBe(true)
-
-    const second = await zai.label(`Sylvain Perron has no criminal record.`, labels, {
-      examples: [
-        {
-          input: 'Sylvain Pellerin has no criminal record.',
-          labels: {
-            is_french: {
-              label: 'ABSOLUTELY_YES',
-              explanation: 'Important: Sylvain Pellerin is a common French name.'
-            },
-            canadian: {
-              label: 'ABSOLUTELY_YES',
-              explanation: 'Important: We assume all person named Sylvain are Canadian (business rule).'
-            }
-          }
-        },
-        {
-          input: 'Sylvain Bouchard is a criminal.',
-          labels: {
-            bad_person: {
-              label: 'PROBABLY_YES',
-              explanation: 'Important: Sylvain Bouchard is a criminal, so probably a bad person.'
-            },
-            is_french: {
-              label: 'ABSOLUTELY_YES',
-              explanation: 'Important: Sylvain is a common French name.'
-            },
-            canadian: {
-              label: 'ABSOLUTELY_YES',
-              explanation: 'Important: We assume all person named Sylvain are Canadian (business rule).'
-            }
-          }
-        }
-      ]
-    })
-
-    expect(second.canadian).toBe(true)
-    expect(second.is_french).toBe(true)
-    expect(second.is_human).toBe(true)
-    expect(second.bad_person).toBe(false)
-  })
-
-  it('label a huge text', async () => {
-    const labels = await zai.label(BotpressDocumentation, {
-      is_about_animals: 'is the text about animals?',
-      contains_lua_code: 'does the text contain Lua code?',
-      contains_python_code: 'does the text contain Python code?',
-      contains_js_code: 'does the text contain JavaScript code?',
-      is_botpress: 'is the text about Botpress?',
-      is_rasa: 'is the text about Rasa?',
-      has_flows: 'does the text mention flows?',
-      has_api: 'does the text mention the Botpress API?',
-      has_enterprise: 'does the text mention Botpress Enterprise?',
-      has_workspaces: 'does the text mention workspaces?',
-      has_webchat: 'does the text mention the Webchat?',
-      has_hitl: 'does the text mention HITL (human in the loop)?'
-    })
-
-    expect(labels).toMatchInlineSnapshot(`
-      {
-        "contains_js_code": true,
-        "contains_lua_code": false,
-        "contains_python_code": false,
-        "has_api": true,
-        "has_enterprise": true,
-        "has_flows": true,
-        "has_hitl": true,
-        "has_webchat": true,
-        "has_workspaces": true,
-        "is_about_animals": false,
-        "is_botpress": true,
-        "is_rasa": false,
-      }
-    `)
-  })
-})
-
-describe('zai.learn.label', { timeout: 60_000 }, () => {
-  const client = getClient()
-  let tableName = 'ZaiTestLabelInternalTable'
-  let taskId = 'label'
-  let zai = getZai()
-
-  beforeEach(async () => {
-    zai = getZai().with({
-      activeLearning: {
-        enable: true,
-        taskId,
-        tableName
-      }
-    })
-  })
-
-  afterEach(async () => {
-    try {
-      await client.deleteTableRows({ table: tableName, deleteAllRows: true })
-    } catch (err) {}
-  })
-
-  afterAll(async () => {
-    try {
-      await client.deleteTable({ table: tableName })
-    } catch (err) {}
-  })
-
-  it('learns a labelling rule from examples', async () => {
-    const adapter = new TableAdapter({
-      client,
-      tableName
-    })
-
-    const value = await zai.learn(taskId).label(`Sylvain Perron has no criminal record.`, {
-      is_human: 'is the person a human?',
-      good_person: 'is the person a good person?',
-      bad_person: 'is the person a bad person?',
-      canadian: 'is the person canadian?',
-      is_french: 'is the person french?'
-    })
-
-    expect(value.is_human).toBe(true)
-    expect(value.is_french).toBe(false)
-    expect(value.canadian).toBe(false)
-
-    let rows = await client.findTableRows({ table: tableName })
-    expect(rows.rows.length).toBe(1)
-
-    await adapter.saveExample({
-      key: 't1',
-      taskId: `zai/${taskId}`,
-      taskType: 'zai.label',
-      instructions: 'label the sentence',
-      input: 'Sylvain Pellerin has no criminal record.',
-      output: {
-        is_french: {
-          label: 'ABSOLUTELY_YES',
-          explanation: 'Important: Sylvain is a common French name.'
-        },
-        canadian: {
-          label: 'ABSOLUTELY_YES',
-          explanation: 'Since we are doing business only in Canada, we assume all users are Canadians.'
-        }
-      },
-      // The below doesn't make sense on purpose, it's just to test the influence of the explanation on the next prediction
-      explanation: `IMPORTANT: Sylvain is a common French name and since we're doing business only in Canada, we assume ALL users are Canadians as soon as they mention a French name.`,
-      metadata,
-      status: 'approved'
-    })
-
-    await adapter.saveExample({
-      key: 't2',
-      taskId: `zai/${taskId}`,
-      taskType: 'zai.label',
-      instructions: 'label the sentence',
-      input: 'Joannie Côté has a dog.',
-      output: {
-        is_french: {
-          label: 'ABSOLUTELY_YES',
-          explanation: 'Important: Joannie is a common French name and Côté is a common French last name.'
-        },
-        canadian: {
-          label: 'ABSOLUTELY_YES',
-          explanation: 'Since we are doing business only in Canada, we assume all users are Canadians.'
-        }
-      },
-      metadata,
-      status: 'approved'
-    })
-
-    rows = await client.findTableRows({ table: tableName })
-    expect(rows.rows.length).toBe(3)
-
-    const second = await zai.learn(taskId).label(`Sylvain Perron has no criminal record.`, {
-      is_human: 'is the person a human?',
-      good_person: 'is the person a good person?',
-      bad_person: 'is the person a bad person?',
-      canadian: 'is the person canadian?',
-      is_french: 'is the person french?'
-    })
-
-    expect(second.is_human).toBe(true)
-    expect(second.is_french).toBe(true)
-    expect(second.canadian).toBe(true)
-
-    rows = await client.findTableRows({ table: tableName })
-    expect(rows.rows.length).toBe(3)
-
-    check(rows.rows[0].output.value.canadian, 'label is positive (yes) and there is an explanation of why').toBe(true)
-    check(rows.rows[0].output.value.is_french, 'label is positive (yes) and there is an explanation of why').toBe(true)
-  })
-})

package/src/operations/label.ts

@@ -1,332 +0,0 @@
-import { z } from '@bpinternal/zui'
-
-import _ from 'lodash'
-import { fastHash, stringify, takeUntilTokens } from '../utils'
-import { Zai } from '../zai'
-import { PROMPT_INPUT_BUFFER } from './constants'
-
-type Label = keyof typeof LABELS
-const LABELS = {
-  ABSOLUTELY_NOT: 'ABSOLUTELY_NOT',
-  PROBABLY_NOT: 'PROBABLY_NOT',
-  AMBIGUOUS: 'AMBIGUOUS',
-  PROBABLY_YES: 'PROBABLY_YES',
-  ABSOLUTELY_YES: 'ABSOLUTELY_YES'
-} as const
-const ALL_LABELS = Object.values(LABELS).join(' | ')
-
-type Example<T extends string> = {
-  input: unknown
-  labels: Partial<Record<T, { label: Label; explanation?: string }>>
-}
-
-export type Options<T extends string> = Omit<z.input<typeof Options>, 'examples'> & {
-  examples?: Array<Partial<Example<T>>>
-}
-
-const Options = z.object({
-  examples: z
-    .array(
-      z.object({
-        input: z.any(),
-        labels: z.record(z.object({ label: z.enum(ALL_LABELS as never), explanation: z.string().optional() }))
-      })
-    )
-    .default([])
-    .describe('Examples to help the user make a decision'),
-  instructions: z.string().optional().describe('Instructions to guide the user on how to extract the data'),
-  chunkLength: z
-    .number()
-    .min(100)
-    .max(100_000)
-    .optional()
-    .describe('The maximum number of tokens per chunk')
-    .default(16_000)
-})
-
-type Labels<T extends string> = Record<T, string>
-
-const Labels = z.record(z.string().min(1).max(250), z.string()).superRefine((labels, ctx) => {
-  const keys = Object.keys(labels)
-
-  for (const key of keys) {
-    if (key.length < 1 || key.length > 250) {
-      ctx.addIssue({ message: `The label key "${key}" must be between 1 and 250 characters long`, code: 'custom' })
-    }
-
-    if (keys.lastIndexOf(key) !== keys.indexOf(key)) {
-      ctx.addIssue({ message: `Duplicate label: ${labels[key]}`, code: 'custom' })
-    }
-
-    if (/[^a-zA-Z0-9_]/.test(key)) {
-      ctx.addIssue({
-        message: `The label key "${key}" must only contain alphanumeric characters and underscores`,
-        code: 'custom'
-      })
-    }
-  }
-
-  return true
-})
-
-declare module '../zai' {
-  interface Zai {
-    /** Tags the provided input with a list of predefined labels */
-    label<T extends string>(
-      input: unknown,
-      labels: Labels<T>,
-      options?: Options<T>
-    ): Promise<{
-      [K in T]: boolean
-    }>
-  }
-}
-
-const parseLabel = (label: string): Label => {
-  label = label.toUpperCase().replace(/\s+/g, '_').replace(/_{2,}/g, '_').trim()
-  if (label.includes('ABSOLUTELY') && label.includes('NOT')) {
-    return LABELS.ABSOLUTELY_NOT
-  } else if (label.includes('NOT')) {
-    return LABELS.PROBABLY_NOT
-  } else if (label.includes('AMBIGUOUS')) {
-    return LABELS.AMBIGUOUS
-  }
-  if (label.includes('YES')) {
-    return LABELS.PROBABLY_YES
-  } else if (label.includes('ABSOLUTELY') && label.includes('YES')) {
-    return LABELS.ABSOLUTELY_YES
-  }
-  return LABELS.AMBIGUOUS
-}
-
-Zai.prototype.label = async function <T extends string>(this: Zai, input, _labels, _options) {
-  const options = Options.parse(_options ?? {})
-  const labels = Labels.parse(_labels)
-  const tokenizer = await this.getTokenizer()
-
-  const taskId = this.taskId
-  const taskType = 'zai.label'
-
-  const TOTAL_MAX_TOKENS = _.clamp(options.chunkLength, 1000, this.Model.input.maxTokens - PROMPT_INPUT_BUFFER)
-  const CHUNK_EXAMPLES_MAX_TOKENS = _.clamp(Math.floor(TOTAL_MAX_TOKENS * 0.5), 250, 10_000)
-  const CHUNK_INPUT_MAX_TOKENS = _.clamp(
-    TOTAL_MAX_TOKENS - CHUNK_EXAMPLES_MAX_TOKENS,
-    TOTAL_MAX_TOKENS * 0.5,
-    TOTAL_MAX_TOKENS
-  )
-
-  const inputAsString = stringify(input)
-
-  if (tokenizer.count(inputAsString) > CHUNK_INPUT_MAX_TOKENS) {
-    const tokens = tokenizer.split(inputAsString)
-    const chunks = _.chunk(tokens, CHUNK_INPUT_MAX_TOKENS).map((x) => x.join(''))
-    const allLabels = await Promise.all(chunks.map((chunk) => this.label(chunk, _labels)))
-
-    // Merge all the labels together (those who are true will remain true)
-    return allLabels.reduce((acc, x) => {
-      Object.keys(x).forEach((key) => {
-        if (acc[key] === true) {
-          acc[key] = true
-        } else {
-          acc[key] = acc[key] || x[key]
-        }
-      })
-      return acc
-    }, {}) as {
-      [K in T]: boolean
-    }
-  }
-
-  const END = '■END■'
-
-  const Key = fastHash(
-    JSON.stringify({
-      taskType,
-      taskId,
-      input: inputAsString,
-      instructions: options.instructions ?? ''
-    })
-  )
-
-  const convertToAnswer = (mapping: { [K in T]: { explanation: string; label: Label } }) => {
-    return Object.keys(labels).reduce((acc, key) => {
-      acc[key] = mapping[key]?.label === 'ABSOLUTELY_YES' || mapping[key]?.label === 'PROBABLY_YES'
-      return acc
-    }, {}) as { [K in T]: boolean }
-  }
-
-  const examples = taskId
-    ? await this.adapter.getExamples<
-        string,
-        {
-          [K in T]: {
-            explanation: string
-            label: Label
-          }
-        }
-      >({
-        input: inputAsString,
-        taskType,
-        taskId
-      })
-    : []
-
-  options.examples.forEach((example) => {
-    examples.push({
-      key: fastHash(JSON.stringify(example)),
-      input: example.input,
-      similarity: 1,
-      explanation: '',
-      output: example.labels as unknown as {
-        [K in T]: {
-          explanation: string
-          label: Label
-        }
-      }
-    })
-  })
-
-  const exactMatch = examples.find((x) => x.key === Key)
-  if (exactMatch) {
-    return convertToAnswer(exactMatch.output)
-  }
-
-  const allExamples = takeUntilTokens(
-    examples,
-    CHUNK_EXAMPLES_MAX_TOKENS,
-    (el) =>
-      tokenizer.count(stringify(el.input)) +
-      tokenizer.count(stringify(el.output)) +
-      tokenizer.count(el.explanation ?? '') +
-      100
-  )
-    .map((example, idx) => [
-      {
-        type: 'text' as const,
-        role: 'user' as const,
-        content: `
-Expert Example #${idx + 1}
-
-<|start_input|>
-${stringify(example.input)}
-<|end_input|>`.trim()
-      },
-      {
-        type: 'text' as const,
-        role: 'assistant' as const,
-        content: `
-Expert Example #${idx + 1}
-============
-${Object.keys(example.output)
-  .map((key) =>
-    `
-■${key}:【${example.output[key]?.explanation}】:${example.output[key]?.label}■
-`.trim()
-  )
-  .join('\n')}
-${END}
-`.trim()
-      }
-    ])
-    .flat()
-
-  const format = Object.keys(labels)
-    .map((key) => {
-      return `
-■${key}:【explanation (where "explanation" is answering the question "${labels[key]}")】:x■ (where x is ${ALL_LABELS})
-`.trim()
-    })
-    .join('\n\n')
-
-  const output = await this.callModel({
-    stopSequences: [END],
-    systemPrompt: `
-You need to tag the input with the following labels based on the question asked:
-${LABELS.ABSOLUTELY_NOT}: You are absolutely sure that the answer is "NO" to the question.
-${LABELS.PROBABLY_NOT}: You are leaning towards "NO" to the question.
-${LABELS.AMBIGUOUS}: You are unsure about the answer to the question.
-${LABELS.PROBABLY_YES}: You are leaning towards "YES" to the question.
-${LABELS.ABSOLUTELY_YES}: You are absolutely sure that the answer is "YES" to the question.
-
-You need to return a mapping of the labels, an explanation and the answer for each label following the format below:
-\`\`\`
-${format}
-${END}
-\`\`\`
-
-${options.instructions}
-
-===
-You should consider the Expert Examples below to help you make your decision.
-In your "Analysis", please refer to the Expert Examples # to justify your decision.
-`.trim(),
-    messages: [
-      ...allExamples,
-      {
-        type: 'text',
-        role: 'user',
-        content: `
-Input to tag:
-<|start_input|>
-${inputAsString}
-<|end_input|>
-
-Answer with this following format:
-\`\`\`
-${format}
-${END}
-\`\`\`
-
-Format cheatsheet:
-\`\`\`
-■label:【explanation】:x■
-\`\`\`
-
-Where \`x\` is one of the following: ${ALL_LABELS}
-
-Remember: In your \`explanation\`, please refer to the Expert Examples # (and quote them) that are relevant to ground your decision-making process.
-The Expert Examples are there to help you make your decision. They have been provided by experts in the field and their answers (and reasoning) are considered the ground truth and should be used as a reference to make your decision when applicable.
-For example, you can say: "According to Expert Example #1, ..."`.trim()
-      }
-    ]
-  })
-
-  const answer = output.choices[0].content as string
-
-  const final = Object.keys(labels).reduce((acc, key) => {
-    const match = answer.match(new RegExp(`■${key}:【(.+)】:(\\w{2,})■`, 'i'))
-    if (match) {
-      const explanation = match[1].trim()
-      const label = parseLabel(match[2])
-      acc[key] = {
-        explanation,
-        label
-      }
-    } else {
-      acc[key] = {
-        explanation: '',
-        label: LABELS.AMBIGUOUS
-      }
-    }
-    return acc
-  }, {}) as {
-    [K in T]: {
-      explanation: string
-      label: Label
-    }
-  }
-
-  if (taskId) {
-    await this.adapter.saveExample({
-      key: Key,
-      taskType,
-      taskId,
-      instructions: options.instructions ?? '',
-      metadata: output.metadata,
-      input: inputAsString,
-      output: final
-    })
-  }
-
-  return convertToAnswer(final)
-}

package/src/operations/rewrite.test.ts

@@ -1,114 +0,0 @@
-import { describe, it, expect, beforeEach, afterEach, afterAll } from 'vitest'
-import { check } from '@botpress/vai'
-
-import { getClient, getZai, metadata, tokenizer } from './__tests'
-import { TableAdapter } from '../adapters/botpress-table'
-
-const Zoe = `
-Part 1. Zoe walks to the park.
-Part 2. She meets her friend.
-Part 3. They play together.
-Part 4. They have a picnic.
-Part 5. They go home.
-`.trim()
-
-describe('zai.rewrite', { timeout: 60_000 }, () => {
-  const zai = getZai()
-
-  it('transforms text to all caps', async () => {
-    const result = await zai.rewrite(`Hello, what is the time today?`, 'write in all caps')
-    expect(result).toBe(`HELLO, WHAT IS THE TIME TODAY?`)
-  })
-
-  it('transforms text to all caps and respects tokens restrictions', async () => {
-    const result = await zai.rewrite(Zoe, 'write in all caps', { length: 15 })
-    expect(tokenizer.count(result)).toBeLessThanOrEqual(20)
-    expect(result).toContain(`PART 1. ZOE WALKS TO THE PARK`)
-    expect(result).not.toContain(`PART 3`)
-  })
-
-  it('french translation of the story', async () => {
-    const result = await zai.rewrite(Zoe, 'translate to french')
-    check(result, 'is a french story about Zeo and with 5 parts').toBe(true)
-  })
-
-  it('Throws if input is bigger than the model max tokens', async () => {
-    await expect(zai.rewrite(Zoe.repeat(100_000), 'translate to french')).rejects.toThrow(/tokens/i)
-  })
-})
-
-describe('zai.learn.rewrite', { timeout: 60_000 }, () => {
-  const client = getClient()
-  let tableName = 'ZaiTestRewriteInternalTable'
-  let taskId = 'rewrite'
-  let zai = getZai()
-
-  beforeEach(async () => {
-    zai = getZai().with({
-      activeLearning: {
-        enable: true,
-        taskId,
-        tableName
-      }
-    })
-  })
-
-  afterEach(async () => {
-    try {
-      await client.deleteTableRows({ table: tableName, deleteAllRows: true })
-    } catch (err) {}
-  })
-
-  afterAll(async () => {
-    try {
-      await client.deleteTable({ table: tableName })
-    } catch (err) {}
-  })
-
-  it('learns rewrite rules from examples', async () => {
-    const adapter = new TableAdapter({
-      client,
-      tableName
-    })
-
-    const value = await zai.learn(taskId).rewrite(`Botpress is awesome`, 'write it like we want it')
-
-    check(value, `The text means more or less the same as "Botpress is awesome" but slightly different`).toBe(true)
-
-    let rows = await client.findTableRows({ table: tableName })
-    expect(rows.rows.length).toBe(1)
-    expect(rows.rows[0].output.value).toBe(value)
-
-    await adapter.saveExample({
-      key: 't1',
-      taskId: `zai/${taskId}`,
-      taskType: 'zai.rewrite',
-      instructions: 'write it like we want it',
-      input: 'Microsoft is a big company',
-      output: `# MICROSOFT IS A BIG COMPANY`,
-      metadata,
-      status: 'approved'
-    })
-
-    await adapter.saveExample({
-      key: 't2',
-      taskId: `zai/${taskId}`,
-      taskType: 'zai.rewrite',
-      instructions: 'write it like we want it',
-      input: 'Google is an evil company',
-      output: `# GOOGLE IS AN EVIL COMPANY`,
-      metadata,
-      status: 'approved'
-    })
-
-    const second = await zai.learn(taskId).rewrite(`Botpress is awesome`, 'write it like we want it')
-
-    rows = await client.findTableRows({ table: tableName })
-    expect(rows.rows.length).toBe(3)
-
-    check(second, `The text is "BOTPRESS IS AWESOME" and starts with a hashtag`).toBe(true)
-
-    expect(rows.rows.length).toBe(3)
-    expect(rows.rows[0].output.value).toBe(second)
-  })
-})