@bedrockio/ai 0.3.0 → 0.4.2

package/.claude/settings.local.json

@@ -0,0 +1,11 @@
+{
+  "permissions": {
+    "allow": [
+      "WebFetch(domain:ai-sdk.dev)",
+      "WebSearch",
+      "Bash(yarn build:*)"
+    ],
+    "deny": [],
+    "ask": []
+  }
+}

package/CHANGELOG.md

@@ -1,3 +1,24 @@
+## 0.4.2
+
+- Exclude keys from tarball.
+
+## 0.4.1
+
+- No error on missing API key.
+
+## 0.4.0
+
+- Rewrote OpenAI to use new responses format.
+- Allow structured responses.
+- Allow yada or JSON schema.
+- Allow extracting partial JSON when streaming.
+- Using `template` option only now.
+- Allow specifying model in client options.
+- Store the previous response id for OpenAI.
+- Removed MultiClient for now.
+- Client -> createClient.
+- Added debug feature.
+
 ## 0.3.0
 
 - Added MultiClient.

package/README.md

@@ -20,9 +20,10 @@ yarn install @bedrockio/ai
 ## Usage
 
 ```js
-import { Client } from '@bedrockio/ai';
+import yd from '@bedrockio/yada';
+import { createClient } from '@bedrockio/ai';
 
-const client = new Client({
+const client = createClient({
   // Directory to templates
   templates: './test/templates',
   // Platform: openai|gpt|anthopic|claude
@@ -33,13 +34,18 @@ const client = new Client({
 
 // Get a one time response.
 const response = await client.prompt({
-  // The template file to use.
-  file: 'classify-fruits',
+  // The template to use. If no template is found will
+  // use this string as the template.
+  template: 'classify-fruits',
   // The form of output. May be raw|text|messages|json.
   // Default is "text".
   output: 'json',
-  // A custom template may be passed if "file" is not.
-  template: 'custom',
+
+  // Aa yada schema (or any JSON schema) may be passed
+  // here to define structured output.
+  schema: yd.object({
+    name: yd.string(),
+  })
 
   // All other variables will be
   // interpolated into the template.
@@ -55,27 +61,63 @@ Responses may be streamed:
 ```js
 // Stream the results
 const stream = await client.stream({
-  file: 'classify-fruits',
+  template: 'classify-fruits',
+
+  // See below.
+  extractMessages: 'text',
 });
 
 // Will return an AsyncIterator
-for await (const chunk of stream) {
-  console.info(chunk.text);
+for await (const event of stream) {
+  console.info(event.text);
 }
 ```
 
-## Templates
+Event types:
 
-Template files must be markdown (`.md`) and live in your templates directory.
-They may be a simple text prompt or delineated roles:
+- `start` - Response has been initiated. This event also contains an `id` field.
+  that can be passsed back in as `prevResponseId` (OpenAI/Grok only).
+- `stop` - Response has finished. Contains the `id` field and usage data.
+- `delta`- Main text delta event when a new token is output.
+- `done` - Text has stopped.
+- `extract:delta` - Used with `extractMessages` (see below).
+- `extract:done` - Used with `extractMessages` (see below).
 
-````
---- SYSTEM ---
+### Streaming Structured Data
+
+Often you want prompt responses to be structured JSON, however you still want to
+stream the user-facing message. In this case use the `extractMessages` option to
+define the key of the structured output you want to stream. When this is defined
+you receive additional `extract:delta` and `extract:done` events. These will
+stream even as the partial JSON data comes in.
+
+### Streaming Notes
 
-This is a list of fruits: {{fruits}}
+Note that in addition to streaming partial data above, there are 2 other valid
+approaches:
 
---- USER ---
+1. Send two prompts, one for the message and one for the extracted data. This
+   works, however there are edge cases when there needs to correlation between
+   the responses. For example when asking the user a "next question" in text but
+   extracting the type of question in data, the results may not match depending
+   on the LLM temperament. This also will increase token usage.
 
+2. Use function calls, ie "tools". This approach seems more appropriate as
+   function calls stream separately to text output and can easily be
+   multiplexed, however at the time of this writing there seem to me issues with
+   ensuring tht the LLM actually uses the correct tools and results have been
+   flaky. Depending on the approach this may also increase token usage.
+
+For the reasons above currently the most reliable approach to streaming
+structured data is using `extractMessage` to stream the partial JSON response.
+
+## Templates
+
+Template files must be markdown (`.md`) and live in your templates directory.
+These will be passed as `instructions`, or the equivalent to the `developer`
+role.
+
+````
 Which fruit do you think the following input most closely resembles?
 
 Please provide your response as a JSON object containing:
@@ -95,7 +137,6 @@ Currently supported platforms:
 
 - OpenAI (ChatGPT)
 - Anthropic (Claude)
-- Google (Gemini).
 - xAi (Grok).
 
 ## Models

package/__mocks__/@anthropic-ai/sdk.js

@@ -1,6 +1,7 @@
 let mock;
+let models;
 
-function MockAnthropicClient() {
+export default function MockAnthropicClient() {
   return {
     messages: {
       create(options) {
@@ -11,33 +12,26 @@ function MockAnthropicClient() {
         }
       },
     },
+    models: {
+      list() {
+        return {
+          data: models,
+        };
+      },
+    },
   };
 }
 
-function setResponse(data) {
+export function setResponse(data) {
   mock = data;
 }
 
-async function* streamMock() {
-  const content = mock.content[0].text;
-  const size = Math.floor(content.length / 3);
-  const one = content.slice(0, size);
-  const two = content.slice(size, 2 * size);
-  const three = content.slice(2 * size);
-  yield wrapChunk(one, 'content_block_start');
-  yield wrapChunk(two, 'content_block_delta');
-  yield wrapChunk(three, 'message_stop');
+export function setModels(data) {
+  models = data;
 }
 
-function wrapChunk(str, type) {
-  return {
-    type,
-    delta: {
-      text: str,
-    },
-  };
+async function* streamMock() {
+  for await (let event of mock) {
+    yield event;
+  }
 }
-
-MockAnthropicClient.setResponse = setResponse;
-
-module.exports = MockAnthropicClient;

package/__mocks__/@google/generative-ai.js

@@ -54,6 +54,6 @@ function wrapChunk(str, finish) {
 }
 
 module.exports = {
-  setResponse,
   GoogleGenerativeAI: MockGoogleClient,
+  setResponse,
 };

package/__mocks__/openai.js

@@ -1,6 +1,7 @@
-let mock;
+let models;
+let responses = {};
 
-function MockOpenAiClient() {
+export default function MockOpenAiClient() {
   return {
     chat: {
       completions: {
@@ -8,41 +9,45 @@ function MockOpenAiClient() {
           if (options.stream) {
             return streamMock();
           } else {
-            return mock;
+            return responses['default'];
           }
         },
       },
     },
+    responses: {
+      create(options) {
+        const { previous_response_id = 'default' } = options;
+        if (!options.input) {
+          throw new Error('Missing parameter "input".');
+        }
+        const response = responses[previous_response_id];
+        if (options.stream) {
+          return streamMock(response);
+        } else {
+          return response;
+        }
+      },
+    },
+    models: {
+      list() {
+        return {
+          data: models,
+        };
+      },
+    },
   };
 }
 
-async function* streamMock() {
-  const content = mock.choices[0].message.content;
-  const size = Math.floor(content.length / 3);
-  const one = content.slice(0, size);
-  const two = content.slice(size, 2 * size);
-  const three = content.slice(2 * size);
-  yield wrapChunk(one);
-  yield wrapChunk(two);
-  yield wrapChunk(three);
+export function setResponse(data, name = 'default') {
+  responses[name] = data;
 }
 
-function wrapChunk(str) {
-  return {
-    choices: [
-      {
-        delta: {
-          content: str,
-        },
-      },
-    ],
-  };
+export function setModels(data) {
+  models = data;
 }
 
-function setResponse(data) {
-  mock = data;
+async function* streamMock(response) {
+  for await (let event of response) {
+    yield event;
+  }
 }
-
-MockOpenAiClient.setResponse = setResponse;
-
-module.exports = MockOpenAiClient;