@botpress/zai 2.0.15 → 2.1.0

package/dist/operations/summarize.js

@@ -1,5 +1,8 @@
 import { z } from "@bpinternal/zui";
 import { chunk } from "lodash-es";
+import { ZaiContext } from "../context";
+import { Response } from "../response";
+import { getTokenizer } from "../tokenizer";
 import { Zai } from "../zai";
 import { PROMPT_INPUT_BUFFER, PROMPT_OUTPUT_BUFFER } from "./constants";
 const Options = z.object({
@@ -17,20 +20,20 @@ const Options = z.object({
 });
 const START = "\u25A0START\u25A0";
 const END = "\u25A0END\u25A0";
-Zai.prototype.summarize = async function(original, _options) {
-  const options = Options.parse(_options ?? {});
-  const tokenizer = await this.getTokenizer();
-  await this.fetchModelDetails();
-  const INPUT_COMPONENT_SIZE = Math.max(100, (this.ModelDetails.input.maxTokens - PROMPT_INPUT_BUFFER) / 4);
+const summarize = async (original, options, ctx) => {
+  ctx.controller.signal.throwIfAborted();
+  const tokenizer = await getTokenizer();
+  const model = await ctx.getModel();
+  const INPUT_COMPONENT_SIZE = Math.max(100, (model.input.maxTokens - PROMPT_INPUT_BUFFER) / 4);
   options.prompt = tokenizer.truncate(options.prompt, INPUT_COMPONENT_SIZE);
   options.format = tokenizer.truncate(options.format, INPUT_COMPONENT_SIZE);
-  const maxOutputSize = this.ModelDetails.output.maxTokens - PROMPT_OUTPUT_BUFFER;
+  const maxOutputSize = model.output.maxTokens - PROMPT_OUTPUT_BUFFER;
   if (options.length > maxOutputSize) {
     throw new Error(
-      `The desired output length is ${maxOutputSize} tokens long, which is more than the maximum of ${this.ModelDetails.output.maxTokens} tokens for this model (${this.ModelDetails.name})`
+      `The desired output length is ${maxOutputSize} tokens long, which is more than the maximum of ${model.output.maxTokens} tokens for this model (${model.name})`
     );
   }
-  options.sliding.window = Math.min(options.sliding.window, this.ModelDetails.input.maxTokens - PROMPT_INPUT_BUFFER);
+  options.sliding.window = Math.min(options.sliding.window, model.input.maxTokens - PROMPT_INPUT_BUFFER);
   options.sliding.overlap = Math.min(options.sliding.overlap, options.sliding.window - 3 * options.sliding.overlap);
   const format = (summary, newText) => {
     return `
@@ -52,8 +55,8 @@ ${newText}
   const chunkSize = Math.ceil(tokens.length / (parts * N));
   if (useMergeSort) {
     const chunks = chunk(tokens, chunkSize).map((x) => x.join(""));
-    const allSummaries = await Promise.all(chunks.map((chunk2) => this.summarize(chunk2, options)));
-    return this.summarize(allSummaries.join("\n\n============\n\n"), options);
+    const allSummaries = (await Promise.allSettled(chunks.map((chunk2) => summarize(chunk2, options, ctx)))).filter((x) => x.status === "fulfilled").map((x) => x.value);
+    return summarize(allSummaries.join("\n\n============\n\n"), options, ctx);
   }
   const summaries = [];
   let currentSummary = "";
@@ -103,7 +106,7 @@ ${newText}
         );
       }
     }
-    const { output } = await this.callModel({
+    let { extracted: result } = await ctx.generateContent({
       systemPrompt: `
 You are summarizing a text. The text is split into ${parts} parts, and you are currently working on part ${iteration}.
 At every step, you will receive the current summary and a new part of the text. You need to amend the summary to include the new information (if needed).
@@ -117,9 +120,14 @@ ${options.format}
 `.trim(),
       messages: [{ type: "text", content: format(currentSummary, slice), role: "user" }],
       maxTokens: generationLength,
-      stopSequences: [END]
+      stopSequences: [END],
+      transform: (text) => {
+        if (!text.trim().length) {
+          throw new Error("The model did not return a valid summary. The response was empty.");
+        }
+        return text;
+      }
     });
-    let result = output?.choices[0]?.content;
     if (result.includes(START)) {
       result = result.slice(result.indexOf(START) + START.length);
     }
@@ -131,3 +139,14 @@ ${options.format}
   }
   return currentSummary.trim();
 };
+Zai.prototype.summarize = function(original, _options) {
+  const options = Options.parse(_options ?? {});
+  const context = new ZaiContext({
+    client: this.client,
+    modelId: this.Model,
+    taskId: this.taskId,
+    taskType: "summarize",
+    adapter: this.adapter
+  });
+  return new Response(context, summarize(original, options, context), (value) => value);
+};

package/dist/operations/text.js

@@ -1,17 +1,21 @@
 import { z } from "@bpinternal/zui";
 import { clamp } from "lodash-es";
+import { ZaiContext } from "../context";
+import { Response } from "../response";
+import { getTokenizer } from "../tokenizer";
 import { Zai } from "../zai";
 import { PROMPT_INPUT_BUFFER, PROMPT_OUTPUT_BUFFER } from "./constants";
 const Options = z.object({
   length: z.number().min(1).max(1e5).optional().describe("The maximum number of tokens to generate")
 });
-Zai.prototype.text = async function(prompt, _options) {
+const text = async (prompt, _options, ctx) => {
+  ctx.controller.signal.throwIfAborted();
   const options = Options.parse(_options ?? {});
-  const tokenizer = await this.getTokenizer();
-  await this.fetchModelDetails();
-  prompt = tokenizer.truncate(prompt, Math.max(this.ModelDetails.input.maxTokens - PROMPT_INPUT_BUFFER, 100));
+  const tokenizer = await getTokenizer();
+  const model = await ctx.getModel();
+  prompt = tokenizer.truncate(prompt, Math.max(model.input.maxTokens - PROMPT_INPUT_BUFFER, 100));
   if (options.length) {
-    options.length = Math.min(this.ModelDetails.output.maxTokens - PROMPT_OUTPUT_BUFFER, options.length);
+    options.length = Math.min(model.output.maxTokens - PROMPT_OUTPUT_BUFFER, options.length);
   }
   const instructions = [];
   let chart = "";
@@ -33,7 +37,7 @@ Zai.prototype.text = async function(prompt, _options) {
 | 200-300 tokens| A medium paragraph (150-200 words) |
 | 300-500 tokens| A long paragraph (200-300 words)   |`.trim();
   }
-  const { output } = await this.callModel({
+  const { extracted } = await ctx.generateContent({
     systemPrompt: `
 Generate a text that fulfills the user prompt below. Answer directly to the prompt, without any acknowledgements or fluff. Also, make sure the text is standalone and complete.
 ${instructions.map((x) => `- ${x}`).join("\n")}
@@ -41,7 +45,23 @@ ${chart}
 `.trim(),
     temperature: 0.7,
     messages: [{ type: "text", content: prompt, role: "user" }],
-    maxTokens: options.length
+    maxTokens: options.length,
+    transform: (text2) => {
+      if (!text2.trim().length) {
+        throw new Error("The model did not return a valid summary. The response was empty.");
+      }
+      return text2;
+    }
   });
-  return output?.choices?.[0]?.content;
+  return extracted;
+};
+Zai.prototype.text = function(prompt, _options) {
+  const context = new ZaiContext({
+    client: this.client,
+    modelId: this.Model,
+    taskId: this.taskId,
+    taskType: "zai.text",
+    adapter: this.adapter
+  });
+  return new Response(context, text(prompt, _options, context), (result) => result);
 };

package/dist/response.js

@@ -0,0 +1,82 @@
+import { EventEmitter } from "./emitter";
+export class Response {
+  _promise;
+  _eventEmitter;
+  _context;
+  _elasped = null;
+  _simplify;
+  constructor(context, promise, simplify) {
+    this._context = context;
+    this._eventEmitter = new EventEmitter();
+    this._simplify = simplify;
+    this._promise = promise.then(
+      (value) => {
+        this._elasped ||= this._context.elapsedTime;
+        this._eventEmitter.emit("complete", value);
+        this._eventEmitter.clear();
+        this._context.clear();
+        return value;
+      },
+      (reason) => {
+        this._elasped ||= this._context.elapsedTime;
+        this._eventEmitter.emit("error", reason);
+        this._eventEmitter.clear();
+        this._context.clear();
+        throw reason;
+      }
+    );
+    this._context.on("update", (usage) => {
+      this._eventEmitter.emit("progress", usage);
+    });
+  }
+  // Event emitter methods
+  on(type, listener) {
+    this._eventEmitter.on(type, listener);
+    return this;
+  }
+  off(type, listener) {
+    this._eventEmitter.off(type, listener);
+    return this;
+  }
+  once(type, listener) {
+    this._eventEmitter.once(type, listener);
+    return this;
+  }
+  bindSignal(signal) {
+    if (signal.aborted) {
+      this.abort(signal.reason);
+    }
+    const signalAbort = () => {
+      this.abort(signal.reason);
+    };
+    signal.addEventListener("abort", () => signalAbort());
+    this.once("complete", () => signal.removeEventListener("abort", signalAbort));
+    this.once("error", () => signal.removeEventListener("abort", signalAbort));
+    return this;
+  }
+  abort(reason) {
+    this._context.controller.abort(reason);
+  }
+  then(onfulfilled, onrejected) {
+    return this._promise.then(
+      (value) => {
+        const simplified = this._simplify(value);
+        return onfulfilled ? onfulfilled(simplified) : simplified;
+      },
+      (reason) => {
+        if (onrejected) {
+          return onrejected(reason);
+        }
+        throw reason;
+      }
+    );
+  }
+  catch(onrejected) {
+    return this._promise.catch(onrejected);
+  }
+  async result() {
+    const output = await this._promise;
+    const usage = this._context.usage;
+    return { output, usage, elapsed: this._elasped };
+  }
+}

package/dist/tokenizer.js

@@ -0,0 +1,11 @@
+import { getWasmTokenizer } from "@bpinternal/thicktoken";
+let tokenizer = null;
+export async function getTokenizer() {
+  if (!tokenizer) {
+    while (!getWasmTokenizer) {
+      await new Promise((resolve) => setTimeout(resolve, 25));
+    }
+    tokenizer = getWasmTokenizer();
+  }
+  return tokenizer;
+}

package/e2e/client.ts

@@ -1,5 +1,5 @@
 import { Client } from '@botpress/client'
-import { Cognitive } from '@botpress/cognitive'
+import { Cognitive, ModelProvider } from '@botpress/cognitive'
 import { diffLines } from 'diff'
 import fs from 'node:fs'
 import path from 'node:path'
@@ -102,6 +102,47 @@ class CachedClient extends Client {
   }
 }
 
+const cognitiveProvider: ModelProvider = {
+  deleteModelPreferences: async () => {},
+  saveModelPreferences: async () => {},
+  fetchInstalledModels: async () => [
+    {
+      ref: 'openai:gpt-4o-2024-11-20',
+      integration: 'openai',
+      id: 'gpt-4o-2024-11-20',
+      name: 'GPT-4o (November 2024)',
+      description:
+        "GPT-4o (“o” for “omni”) is OpenAI's most advanced model. It is multimodal (accepting text or image inputs and outputting text), and it has the same high intelligence as GPT-4 Turbo but is cheaper and more efficient.",
+      input: {
+        costPer1MTokens: 2.5,
+        maxTokens: 128000,
+      },
+      output: {
+        costPer1MTokens: 10,
+        maxTokens: 16384,
+      },
+      tags: ['recommended', 'vision', 'general-purpose', 'coding', 'agents', 'function-calling'],
+    },
+  ],
+  fetchModelPreferences: async () => ({
+    best: ['openai:gpt-4o-2024-11-20'] as const,
+    fast: ['openai:gpt-4o-2024-11-20'] as const,
+    downtimes: [],
+  }),
+}
+
+export const getCognitiveClient = () => {
+  const cognitive = new Cognitive({
+    client: new Client({
+      apiUrl: process.env.CLOUD_API_ENDPOINT ?? 'https://api.botpress.dev',
+      botId: process.env.CLOUD_BOT_ID,
+      token: process.env.CLOUD_PAT,
+    }),
+    provider: cognitiveProvider,
+  })
+  return cognitive
+}
+
 export const getCachedCognitiveClient = () => {
   const cognitive = new Cognitive({
     client: new CachedClient({
@@ -109,34 +150,7 @@ export const getCachedCognitiveClient = () => {
       botId: process.env.CLOUD_BOT_ID,
       token: process.env.CLOUD_PAT,
     }),
-    provider: {
-      deleteModelPreferences: async () => {},
-      saveModelPreferences: async () => {},
-      fetchInstalledModels: async () => [
-        {
-          ref: 'openai:gpt-4o-2024-11-20',
-          integration: 'openai',
-          id: 'gpt-4o-2024-11-20',
-          name: 'GPT-4o (November 2024)',
-          description:
-            "GPT-4o (“o” for “omni”) is OpenAI's most advanced model. It is multimodal (accepting text or image inputs and outputting text), and it has the same high intelligence as GPT-4 Turbo but is cheaper and more efficient.",
-          input: {
-            costPer1MTokens: 2.5,
-            maxTokens: 128000,
-          },
-          output: {
-            costPer1MTokens: 10,
-            maxTokens: 16384,
-          },
-          tags: ['recommended', 'vision', 'general-purpose', 'coding', 'agents', 'function-calling'],
-        },
-      ],
-      fetchModelPreferences: async () => ({
-        best: ['openai:gpt-4o-2024-11-20'] as const,
-        fast: ['openai:gpt-4o-2024-11-20'] as const,
-        downtimes: [],
-      }),
-    },
+    provider: cognitiveProvider,
   })
   return cognitive
 }