@bedrockio/ai 0.3.0 → 0.4.4

package/CHANGELOG.md

@@ -1,3 +1,28 @@
+## 0.4.3
+
+- Moved to files whitelist.
+
+## 0.4.2
+
+- Exclude keys from tarball.
+
+## 0.4.1
+
+- No error on missing API key.
+
+## 0.4.0
+
+- Rewrote OpenAI to use new responses format.
+- Allow structured responses.
+- Allow yada or JSON schema.
+- Allow extracting partial JSON when streaming.
+- Using `template` option only now.
+- Allow specifying model in client options.
+- Store the previous response id for OpenAI.
+- Removed MultiClient for now.
+- Client -> createClient.
+- Added debug feature.
+
 ## 0.3.0
 
 - Added MultiClient.

package/README.md

@@ -20,9 +20,10 @@ yarn install @bedrockio/ai
 ## Usage
 
 ```js
-import { Client } from '@bedrockio/ai';
+import yd from '@bedrockio/yada';
+import { createClient } from '@bedrockio/ai';
 
-const client = new Client({
+const client = createClient({
   // Directory to templates
   templates: './test/templates',
   // Platform: openai|gpt|anthopic|claude
@@ -33,13 +34,18 @@ const client = new Client({
 
 // Get a one time response.
 const response = await client.prompt({
-  // The template file to use.
-  file: 'classify-fruits',
+  // The template to use. If no template is found will
+  // use this string as the template.
+  template: 'classify-fruits',
   // The form of output. May be raw|text|messages|json.
   // Default is "text".
   output: 'json',
-  // A custom template may be passed if "file" is not.
-  template: 'custom',
+
+  // Aa yada schema (or any JSON schema) may be passed
+  // here to define structured output.
+  schema: yd.object({
+    name: yd.string(),
+  })
 
   // All other variables will be
   // interpolated into the template.
@@ -55,27 +61,63 @@ Responses may be streamed:
 ```js
 // Stream the results
 const stream = await client.stream({
-  file: 'classify-fruits',
+  template: 'classify-fruits',
+
+  // See below.
+  extractMessages: 'text',
 });
 
 // Will return an AsyncIterator
-for await (const chunk of stream) {
-  console.info(chunk.text);
+for await (const event of stream) {
+  console.info(event.text);
 }
 ```
 
-## Templates
+Event types:
 
-Template files must be markdown (`.md`) and live in your templates directory.
-They may be a simple text prompt or delineated roles:
+- `start` - Response has been initiated. This event also contains an `id` field.
+  that can be passsed back in as `prevResponseId` (OpenAI/Grok only).
+- `stop` - Response has finished. Contains the `id` field and usage data.
+- `delta`- Main text delta event when a new token is output.
+- `done` - Text has stopped.
+- `extract:delta` - Used with `extractMessages` (see below).
+- `extract:done` - Used with `extractMessages` (see below).
 
-````
---- SYSTEM ---
+### Streaming Structured Data
+
+Often you want prompt responses to be structured JSON, however you still want to
+stream the user-facing message. In this case use the `extractMessages` option to
+define the key of the structured output you want to stream. When this is defined
+you receive additional `extract:delta` and `extract:done` events. These will
+stream even as the partial JSON data comes in.
+
+### Streaming Notes
 
-This is a list of fruits: {{fruits}}
+Note that in addition to streaming partial data above, there are 2 other valid
+approaches:
 
---- USER ---
+1. Send two prompts, one for the message and one for the extracted data. This
+   works, however there are edge cases when there needs to correlation between
+   the responses. For example when asking the user a "next question" in text but
+   extracting the type of question in data, the results may not match depending
+   on the LLM temperament. This also will increase token usage.
 
+2. Use function calls, ie "tools". This approach seems more appropriate as
+   function calls stream separately to text output and can easily be
+   multiplexed, however at the time of this writing there seem to me issues with
+   ensuring tht the LLM actually uses the correct tools and results have been
+   flaky. Depending on the approach this may also increase token usage.
+
+For the reasons above currently the most reliable approach to streaming
+structured data is using `extractMessage` to stream the partial JSON response.
+
+## Templates
+
+Template files must be markdown (`.md`) and live in your templates directory.
+These will be passed as `instructions`, or the equivalent to the `developer`
+role.
+
+````
 Which fruit do you think the following input most closely resembles?
 
 Please provide your response as a JSON object containing:
@@ -95,7 +137,6 @@ Currently supported platforms:
 
 - OpenAI (ChatGPT)
 - Anthropic (Claude)
-- Google (Gemini).
 - xAi (Grok).
 
 ## Models

package/dist/cjs/BaseClient.js

@@ -1,186 +1,246 @@
 "use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = void 0;
-var _mustache = _interopRequireDefault(require("mustache"));
-var _util = require("./util.js");
-function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
-const MESSAGES_REG = /(?:^|\n)-{3,}\s*(\w+)\s*-{3,}(.*?)(?=\n-{3,}|$)/gs;
+Object.defineProperty(exports, "__esModule", { value: true });
+const code_js_1 = require("./utils/code.js");
+const json_js_1 = require("./utils/json.js");
+const templates_js_1 = require("./utils/templates.js");
 class BaseClient {
-  constructor(options) {
-    this.options = options;
-    this.templates = null;
-  }
-
-  /**
-   * Interpolates vars into the provided template and
-   * runs the chat completion. The "output" option may
-   * be omitted and will default to `"text"`.
-   * {@link https://github.com/bedrockio/ai?tab=readme-ov-file#bedrockioai Documentation}
-   *
-   * @param {object} options
-   * @param {string} options.model - The model to use.
-   * @param {"raw" | "text" | "json" | "messages"} [options.output] - The output to use.
-   * @param {Object.<string, any>} [options.other] - Additional props
-   *             will be interpolated in the template.
-   */
-  async prompt(options) {
-    options = {
-      ...this.options,
-      ...options
-    };
-    const messages = await this.getMessages(options);
-    return await this.getCompletion({
-      ...options,
-      messages
-    });
-  }
-
-  /**
-   * Streams the prompt response.
-   * @returns {AsyncIterator}
-   */
-  async *stream(options) {
-    const stream = await this.getStream(options);
-    let started = false;
-
-    // @ts-ignore
-    for await (const chunk of stream) {
-      const resolved = this.getStreamedChunk(chunk, started);
-      started = true;
-
-      // @ts-ignore
-      if (resolved) {
-        yield resolved;
-      }
-    }
-  }
-  async getMessages(options) {
-    const {
-      text
-    } = options;
-    const template = await this.resolveTemplate(options);
-    if (template) {
-      const raw = render(template, options);
-      const messages = [];
-      for (let match of raw.matchAll(MESSAGES_REG)) {
-        const [, role, content] = match;
-        messages.push({
-          role: role.toLowerCase(),
-          content: content.trim()
-        });
-      }
-      if (!messages.length) {
-        messages.push({
-          role: 'user',
-          content: raw.trim()
-        });
-      }
-      return messages;
-    } else if (text) {
-      return [{
-        role: 'user',
-        content: text
-      }];
-    } else {
-      throw new Error('No input provided.');
-    }
-  }
-  async buildTemplate(options) {
-    const template = await this.resolveTemplate(options);
-    return render(template, options);
-  }
-  async loadTemplates() {
-    const {
-      templates
-    } = this.options;
-    this.templates ||= await (0, _util.loadTemplates)(templates);
-  }
-  async resolveTemplate(options) {
-    const {
-      template,
-      file
-    } = options;
-    if (template) {
-      return template;
-    } else if (file?.endsWith('.md')) {
-      return await (0, _util.loadTemplate)(file);
-    } else if (file) {
-      await this.loadTemplates();
-      return this.templates[file];
-    }
-  }
-  async getStream(options) {
-    return await this.prompt({
-      ...options,
-      output: 'raw',
-      stream: true
-    });
-  }
-  getCompletion(options) {
-    void options;
-    new Error('Method not implemented.');
-  }
-  getStreamedChunk(chunk, started) {
-    void chunk;
-    void started;
-    new Error('Method not implemented.');
-  }
+    constructor(options) {
+        this.options = {
+            // @ts-ignore
+            model: this.constructor.DEFAULT_MODEL,
+            ...options,
+        };
+        this.templates = null;
+    }
+    // Public
+    /**
+     * Interpolates vars into the provided template as instructions and runs the
+     * prompt.
+     *
+     * @param {PromptOptions} options
+     */
+    async prompt(options) {
+        options = await this.normalizeOptions(options);
+        const { input, output, stream, schema } = options;
+        const response = await this.runPrompt(options);
+        if (!stream) {
+            this.debug('Response:', response);
+        }
+        if (output === 'raw') {
+            return response;
+        }
+        let result;
+        if (schema) {
+            result = this.getStructuredResponse(response);
+            // @ts-ignore
+            if (options.hasWrappedSchema) {
+                result = result.items;
+            }
+        }
+        else if (output === 'json') {
+            result = JSON.parse((0, code_js_1.parseCode)(this.getTextResponse(response)));
+        }
+        else {
+            result = (0, code_js_1.parseCode)(this.getTextResponse(response));
+        }
+        if (output === 'messages') {
+            return {
+                result,
+                ...this.getMessagesResponse(input, response),
+            };
+        }
+        else {
+            return result;
+        }
+    }
+    /**
+     * Streams the prompt response.
+     *
+     * @param {PromptOptions & StreamOptions} options
+     * @returns {AsyncIterator}
+     */
+    async *stream(options) {
+        options = await this.normalizeOptions(options);
+        const extractor = this.getMessageExtractor(options);
+        try {
+            const stream = await this.runStream(options);
+            // @ts-ignore
+            for await (let event of stream) {
+                this.debug('Event:', event);
+                event = this.normalizeStreamEvent(event);
+                if (event) {
+                    yield event;
+                }
+                const extractedMessages = extractor?.(event) || [];
+                for (let message of extractedMessages) {
+                    const { key, delta, text, done } = message;
+                    let extractEvent;
+                    if (done) {
+                        extractEvent = {
+                            type: 'extract:done',
+                            text,
+                            key,
+                        };
+                    }
+                    else {
+                        extractEvent = {
+                            type: 'extract:delta',
+                            delta,
+                            key,
+                        };
+                    }
+                    this.debug('Extract:', extractEvent);
+                    yield extractEvent;
+                }
+            }
+        }
+        catch (error) {
+            const { message, code } = error;
+            yield {
+                type: 'error',
+                code,
+                message,
+            };
+        }
+    }
+    async buildTemplate(options) {
+        const template = await this.resolveTemplate(options);
+        return (0, templates_js_1.renderTemplate)(template, options);
+    }
+    // Protected
+    runPrompt(options) {
+        void options;
+        throw new Error('Method not implemented.');
+    }
+    runStream(options) {
+        void options;
+        throw new Error('Method not implemented.');
+    }
+    getTextResponse(response) {
+        void response;
+        throw new Error('Method not implemented.');
+    }
+    /**
+     * @returns {Object}
+     */
+    getStructuredResponse(response) {
+        void response;
+        throw new Error('Method not implemented.');
+    }
+    /**
+     * @returns {Object}
+     */
+    getMessagesResponse(input, response) {
+        void response;
+        throw new Error('Method not implemented.');
+    }
+    /**
+     * @returns {Object}
+     */
+    normalizeStreamEvent(event) {
+        void event;
+        throw new Error('Method not implemented.');
+    }
+    // Private
+    async normalizeOptions(options) {
+        options = {
+            input: '',
+            output: 'text',
+            ...this.options,
+            ...options,
+        };
+        options.input = this.normalizeInput(options);
+        options.schema = this.normalizeSchema(options);
+        options.instructions ||= await this.resolveInstructions(options);
+        return options;
+    }
+    normalizeInput(options) {
+        let { input = '', output } = options;
+        if (typeof input === 'string') {
+            if (output === 'json') {
+                input += '\nOutput only valid JSON.';
+            }
+            input = [
+                {
+                    role: 'user',
+                    content: input,
+                },
+            ];
+        }
+        return input;
+    }
+    normalizeSchema(options) {
+        let { schema } = options;
+        if (!schema) {
+            return;
+        }
+        // Convert to JSON schema.
+        schema = schema.toJSON?.() || schema;
+        if (schema?.type === 'array') {
+            schema = {
+                type: 'object',
+                properties: {
+                    items: schema,
+                },
+                required: ['items'],
+                additionalProperties: false,
+            };
+            options.hasWrappedSchema = true;
+        }
+        return schema;
+    }
+    getMessageExtractor(options) {
+        const { extractMessages } = options;
+        if (!extractMessages) {
+            return;
+        }
+        const messageExtractor = (0, json_js_1.createMessageExtractor)([extractMessages]);
+        return (event) => {
+            if (event?.type === 'delta') {
+                return messageExtractor(event.delta);
+            }
+        };
+    }
+    debug(message, arg) {
+        if (this.options.debug) {
+            // TODO: replace with logger when opentelemetry is removed
+            // eslint-disable-next-line
+            console.debug(`${message}\n${JSON.stringify(arg, null, 2)}\n`);
+        }
+    }
+    async resolveInstructions(options) {
+        if (options.template) {
+            const template = await this.resolveTemplate(options);
+            return (0, templates_js_1.renderTemplate)(template, options);
+        }
+    }
+    async resolveTemplate(options) {
+        const { template } = options;
+        await this.loadTemplates();
+        return this.templates[template] || template;
+    }
+    async loadTemplates() {
+        const { templates } = this.options;
+        this.templates ||= await (0, templates_js_1.loadTemplates)(templates);
+    }
 }
 exports.default = BaseClient;
-function render(template, options) {
-  let params = {
-    ...options,
-    ...options.params
-  };
-  params = mapObjects(params);
-  params = wrapProxy(params);
-  return _mustache.default.render(template, params);
-}
-
-// Transform arrays and object to versions
-// that are more understandable in the context
-// of a template that may have meaningful whitespace.
-function mapObjects(params) {
-  const result = {};
-  for (let [key, value] of Object.entries(params)) {
-    if (Array.isArray(value)) {
-      value = mapArray(value);
-    } else if (typeof value === 'object') {
-      value = JSON.stringify(value, null, 2);
-    }
-    result[key] = value;
-  }
-  return result;
-}
-function mapArray(arr) {
-  // Only map simple arrays of primitives.
-  if (typeof arr[0] === 'string') {
-    arr = arr.map(el => {
-      return `- ${el}`;
-    }).join('\n');
-  }
-  return arr;
-}
-
-// Wrap params with a proxy object that reports
-// as having all properties. If one is accessed
-// that does not exist then return the original
-// token. This way templates can be partially
-// interpolated and re-interpolated later.
-function wrapProxy(params) {
-  return new Proxy(params, {
-    has() {
-      return true;
-    },
-    get(target, prop) {
-      if (prop in target) {
-        return target[prop];
-      } else {
-        return `{{{${prop.toString()}}}}`;
-      }
-    }
-  });
-}
+/**
+ * @typedef {Object} PromptOptions
+ * @property {string|PromptMessage[]} input - Input to use.
+ * @property {string} [model] - The model to use.
+ * @property {boolean} stream - Stream response.
+ * @property {Object} [schema] - A JSON schema compatible object that defines the output shape.
+ * @property {"raw" | "text" | "json" | "messages"} [output] - The return value type.
+ * @property {Object} [params] - Params to be interpolated into the template.
+ *                               May also be passed as additional props to options.
+ */
+/**
+ * @typedef {Object} StreamOptions
+ * @property {string} [extractMessages] - Key in JSON response to extract a message stream from.
+ */
+/**
+ * @typedef {Object} PromptMessage
+ * @property {"system" | "user" | "assistant"} role
+ * @property {string} content
+ */