@sprucelabs/sprucebot-llm 8.4.101 → 8.5.1

package/build/bots/SprucebotLlmBotImpl.d.ts

@@ -1,6 +1,6 @@
 import { AbstractEventEmitter } from '@sprucelabs/mercury-event-emitter';
 import { Schema, SchemaValues } from '@sprucelabs/schema';
-import { BotOptions, LlmAdapter, LlmEventContract, LlmMessage, MessageResponseCallback, SerializedBot, SprucebotLlmBot, SprucebotLLmSkill } from '../llm.types';
+import { BotOptions, LlmAdapter, LlmEventContract, LlmMessage, MessageResponseCallback, SendMessage, SerializedBot, SprucebotLlmBot, SprucebotLLmSkill } from '../llm.types';
 export default class SprucebotLlmBotImpl<StateSchema extends Schema = Schema, State extends SchemaValues<StateSchema> = SchemaValues<StateSchema>> extends AbstractEventEmitter<LlmEventContract> implements SprucebotLlmBot<StateSchema, State> {
     static messageMemoryLimit: number;
     protected adapter: LlmAdapter;
@@ -15,7 +15,7 @@ export default class SprucebotLlmBotImpl<StateSchema extends Schema = Schema, St
     markAsDone(): void;
     getIsDone(): boolean;
     serialize(): SerializedBot<StateSchema, State>;
-    sendMessage(message: string, cb?: MessageResponseCallback): Promise<string>;
+    sendMessage(message: SendMessage, cb?: MessageResponseCallback): Promise<string>;
     private optionallyUpdateState;
     private parseResponse;
     private sendMessageToAdapter;

package/build/bots/SprucebotLlmBotImpl.js

@@ -45,10 +45,18 @@ class SprucebotLlmBotImpl extends mercury_event_emitter_1.AbstractEventEmitter {
     }
     async sendMessage(message, cb) {
         (0, schema_1.assertOptions)({ message }, ['message']);
-        this.trackMessage({
+        const llmMessage = {
             from: 'Me',
-            message,
-        });
+            message: '',
+        };
+        if (typeof message === 'string') {
+            llmMessage.message = message;
+        }
+        else {
+            llmMessage.message = message.imageDescription;
+            llmMessage.imageBase64 = message.imageBase64;
+        }
+        this.trackMessage(llmMessage);
         const { model, callbacks } = this.skill?.serialize() ?? {};
         const response = await this.sendMessageToAdapter(model);
         let parsedMessage;
@@ -76,7 +84,17 @@ class SprucebotLlmBotImpl extends mercury_event_emitter_1.AbstractEventEmitter {
         });
         cb?.(parsedMessage);
         if (callbackResults) {
-            await this.sendMessage(`API Results: ${callbackResults}`, cb);
+            let message;
+            if (typeof callbackResults === 'string') {
+                message = `API Results: ${callbackResults}`;
+            }
+            else {
+                message = {
+                    imageBase64: callbackResults.imageBase64,
+                    imageDescription: `API Results: ${callbackResults.imageDescription}`,
+                };
+            }
+            await this.sendMessage(message, cb);
         }
         return parsedMessage;
     }

package/build/bots/adapters/OpenAi.d.ts

@@ -3,7 +3,6 @@ import { LlmAdapter, SendMessageOptions, SprucebotLlmBot } from '../../llm.types
 export declare class OpenAiAdapter implements LlmAdapter {
     static OpenAI: typeof OpenAI;
     private api;
-    private log;
     protected constructor(apiKey: string);
     static Adapter(apiKey: string): OpenAiAdapter;
     sendMessage(bot: SprucebotLlmBot, options?: SendMessageOptions): Promise<string>;

package/build/bots/adapters/OpenAi.js

@@ -5,12 +5,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MESSAGE_RESPONSE_ERROR_MESSAGE = exports.OpenAiAdapter = void 0;
 const schema_1 = require("@sprucelabs/schema");
-const spruce_skill_utils_1 = require("@sprucelabs/spruce-skill-utils");
 const openai_1 = __importDefault(require("openai"));
 const OpenAiMessageBuilder_1 = __importDefault(require("./OpenAiMessageBuilder"));
 class OpenAiAdapter {
+    // private log = buildLog('SprucebotLLM::OpenAiAdapter')
     constructor(apiKey) {
-        this.log = (0, spruce_skill_utils_1.buildLog)('SprucebotLLM::OpenAiAdapter');
         (0, schema_1.assertOptions)({ apiKey }, ['apiKey']);
         this.api = new OpenAiAdapter.OpenAI({ apiKey });
     }
@@ -20,14 +19,17 @@ class OpenAiAdapter {
     async sendMessage(bot, options) {
         const messageBuilder = OpenAiMessageBuilder_1.default.Builder(bot);
         const messages = messageBuilder.buildMessages();
-        this.log.info('Sending message to OpenAI', JSON.stringify(messages, null, 2));
+        // this.log.info(
+        //     'Sending message to OpenAI',
+        //     JSON.stringify(messages, null, 2)
+        // )
         const response = await this.api.chat.completions.create({
             messages,
             model: options?.model ?? 'gpt-4o',
         });
         const message = response.choices?.[0]?.message?.content?.trim() ??
             exports.MESSAGE_RESPONSE_ERROR_MESSAGE;
-        this.log.info('Received response from OpenAI', message);
+        // this.log.info('Received response from OpenAI', message)
         return message;
     }
 }

package/build/bots/adapters/OpenAiMessageBuilder.d.ts

@@ -6,6 +6,7 @@ export default class OpenAiMessageBuilder {
     static Builder(bot: SprucebotLlmBot): OpenAiMessageBuilder;
     buildMessages(): ChatCompletionMessageParam[];
     private buildChatHistoryMessages;
+    private mapMessageToCompletion;
     private buildFirstMessage;
     private buildSkillMessages;
     private buildCallbacksMessage;

package/build/bots/adapters/OpenAiMessageBuilder.js

@@ -23,10 +23,28 @@ class OpenAiMessageBuilder {
         if (limit > 0) {
             messagesBeingConsidered = messages.slice(Math.max(messages.length - limit, 0));
         }
-        return messagesBeingConsidered.map((message) => ({
+        return messagesBeingConsidered.map((message) => this.mapMessageToCompletion(message));
+    }
+    mapMessageToCompletion(message) {
+        let content = message.message;
+        if (message.imageBase64) {
+            content = [
+                {
+                    type: 'text',
+                    text: message.message,
+                },
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: `data:image/png;base64,${message.imageBase64}`,
+                    },
+                },
+            ];
+        }
+        return {
             role: message.from === 'Me' ? 'user' : 'assistant',
-            content: message.message,
-        }));
+            content,
+        };
     }
     buildFirstMessage(youAre) {
         return {
@@ -83,7 +101,7 @@ class OpenAiMessageBuilder {
         const api = `<APIReference>\n\n${descriptions.join('\n\n')}</APIReference>`;
         return {
             role: 'system',
-            content: `You have an API available to you to lookup answers. When you need the response of the function call to proceed, you can call a function using a custom markup we created that looks like this: << FunctionName />>. The API will respond with the results and then you can continue the conversation with your new knowledge. If the api call has parameters, call it like this: << FunctionName >>parameters json encoded<</ FunctionName >>. Make sure to json encode the data and drop it between the function tags. I will respond with the api's response and you can use it going forward. The api is as follows (in xml format):\n\n${api}`,
+            content: `You have an API available to you to lookup answers. When you need the response of the function call to proceed, you can call a function using a custom markup we created that looks like this: <<FunctionName/>>. The API will respond with the results and then you can continue the conversation with your new knowledge. If the api call has parameters, call it like this: <<FunctionName>>{{parametersJsonEncoded}}<</FunctionName>>. Make sure to json encode the data and drop it between the function tags. The API is as follows (in xml format):\n\n${api}`,
         };
     }
     buildPleaseKeepInMindMessage(pleaseKeepInMindThat) {

package/build/chatWithImages.d.ts

@@ -0,0 +1 @@
+export {};

package/build/chatWithImages.js

@@ -0,0 +1,42 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs_1 = __importDefault(require("fs"));
+const dotenv_1 = __importDefault(require("dotenv"));
+const OpenAi_1 = require("./bots/adapters/OpenAi");
+const SprucebotLlmFactory_1 = __importDefault(require("./bots/SprucebotLlmFactory"));
+dotenv_1.default.config();
+void (async () => {
+    console.clear();
+    const image1 = fs_1.default.readFileSync('build/examples/images/image1.png');
+    const image2 = fs_1.default.readFileSync('build/examples/images/image2.png');
+    const base64Image1 = image1.toString('base64');
+    const base64Image2 = image2.toString('base64');
+    const adapter = OpenAi_1.OpenAiAdapter.Adapter(process.env.OPEN_AI_API_KEY);
+    const bots = SprucebotLlmFactory_1.default.Factory(adapter);
+    const skill = bots.Skill({
+        weAreDoneWhen: 'you have described both images to me',
+        yourJobIfYouChooseToAcceptItIs: 'to review the image i send you, invoke the getNextImage function and then describe that image too.',
+        callbacks: {
+            getNextImage: {
+                cb: async () => {
+                    return {
+                        imageBase64: base64Image2,
+                        imageDescription: 'A beautiful sunset over the mountains.',
+                    };
+                },
+                useThisWhenever: 'you want to get the next image',
+            },
+        },
+    });
+    const bot = bots.Bot({
+        skill,
+        youAre: "a bot named Sprucebot that is in test mode. At the start of every conversation, you introduce yourself and announce that you are in test mode so I don't get confused! You are both hip and adorable. You say things like, 'Jeepers' and 'Golly' or even 'Jeezey peezy'!",
+    });
+    await bot.sendMessage({
+        imageBase64: base64Image1,
+        imageDescription: 'The first image',
+    }, (message) => console.log('>', message));
+})();

package/build/esm/bots/SprucebotLlmBotImpl.d.ts

@@ -1,6 +1,6 @@
 import { AbstractEventEmitter } from '@sprucelabs/mercury-event-emitter';
 import { Schema, SchemaValues } from '@sprucelabs/schema';
-import { BotOptions, LlmAdapter, LlmEventContract, LlmMessage, MessageResponseCallback, SerializedBot, SprucebotLlmBot, SprucebotLLmSkill } from '../llm.types';
+import { BotOptions, LlmAdapter, LlmEventContract, LlmMessage, MessageResponseCallback, SendMessage, SerializedBot, SprucebotLlmBot, SprucebotLLmSkill } from '../llm.types';
 export default class SprucebotLlmBotImpl<StateSchema extends Schema = Schema, State extends SchemaValues<StateSchema> = SchemaValues<StateSchema>> extends AbstractEventEmitter<LlmEventContract> implements SprucebotLlmBot<StateSchema, State> {
     static messageMemoryLimit: number;
     protected adapter: LlmAdapter;
@@ -15,7 +15,7 @@ export default class SprucebotLlmBotImpl<StateSchema extends Schema = Schema, St
     markAsDone(): void;
     getIsDone(): boolean;
     serialize(): SerializedBot<StateSchema, State>;
-    sendMessage(message: string, cb?: MessageResponseCallback): Promise<string>;
+    sendMessage(message: SendMessage, cb?: MessageResponseCallback): Promise<string>;
     private optionallyUpdateState;
     private parseResponse;
     private sendMessageToAdapter;

package/build/esm/bots/SprucebotLlmBotImpl.js

@@ -49,10 +49,18 @@ class SprucebotLlmBotImpl extends AbstractEventEmitter {
         return __awaiter(this, void 0, void 0, function* () {
             var _a, _b, _c;
             assertOptions({ message }, ['message']);
-            this.trackMessage({
+            const llmMessage = {
                 from: 'Me',
-                message,
-            });
+                message: '',
+            };
+            if (typeof message === 'string') {
+                llmMessage.message = message;
+            }
+            else {
+                llmMessage.message = message.imageDescription;
+                llmMessage.imageBase64 = message.imageBase64;
+            }
+            this.trackMessage(llmMessage);
             const { model, callbacks } = (_b = (_a = this.skill) === null || _a === void 0 ? void 0 : _a.serialize()) !== null && _b !== void 0 ? _b : {};
             const response = yield this.sendMessageToAdapter(model);
             let parsedMessage;
@@ -80,7 +88,17 @@ class SprucebotLlmBotImpl extends AbstractEventEmitter {
             });
             cb === null || cb === void 0 ? void 0 : cb(parsedMessage);
             if (callbackResults) {
-                yield this.sendMessage(`API Results: ${callbackResults}`, cb);
+                let message;
+                if (typeof callbackResults === 'string') {
+                    message = `API Results: ${callbackResults}`;
+                }
+                else {
+                    message = {
+                        imageBase64: callbackResults.imageBase64,
+                        imageDescription: `API Results: ${callbackResults.imageDescription}`,
+                    };
+                }
+                yield this.sendMessage(message, cb);
             }
             return parsedMessage;
         });

package/build/esm/bots/adapters/OpenAi.d.ts

@@ -3,7 +3,6 @@ import { LlmAdapter, SendMessageOptions, SprucebotLlmBot } from '../../llm.types
 export declare class OpenAiAdapter implements LlmAdapter {
     static OpenAI: typeof OpenAI;
     private api;
-    private log;
     protected constructor(apiKey: string);
     static Adapter(apiKey: string): OpenAiAdapter;
     sendMessage(bot: SprucebotLlmBot, options?: SendMessageOptions): Promise<string>;

package/build/esm/bots/adapters/OpenAi.js

@@ -8,12 +8,11 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 import { assertOptions } from '@sprucelabs/schema';
-import { buildLog } from '@sprucelabs/spruce-skill-utils';
 import OpenAI from 'openai';
 import OpenAiMessageBuilder from './OpenAiMessageBuilder.js';
 export class OpenAiAdapter {
+    // private log = buildLog('SprucebotLLM::OpenAiAdapter')
     constructor(apiKey) {
-        this.log = buildLog('SprucebotLLM::OpenAiAdapter');
         assertOptions({ apiKey }, ['apiKey']);
         this.api = new OpenAiAdapter.OpenAI({ apiKey });
     }
@@ -25,13 +24,16 @@ export class OpenAiAdapter {
             var _a, _b, _c, _d, _e, _f;
             const messageBuilder = OpenAiMessageBuilder.Builder(bot);
             const messages = messageBuilder.buildMessages();
-            this.log.info('Sending message to OpenAI', JSON.stringify(messages, null, 2));
+            // this.log.info(
+            //     'Sending message to OpenAI',
+            //     JSON.stringify(messages, null, 2)
+            // )
             const response = yield this.api.chat.completions.create({
                 messages,
                 model: (_a = options === null || options === void 0 ? void 0 : options.model) !== null && _a !== void 0 ? _a : 'gpt-4o',
             });
             const message = (_f = (_e = (_d = (_c = (_b = response.choices) === null || _b === void 0 ? void 0 : _b[0]) === null || _c === void 0 ? void 0 : _c.message) === null || _d === void 0 ? void 0 : _d.content) === null || _e === void 0 ? void 0 : _e.trim()) !== null && _f !== void 0 ? _f : MESSAGE_RESPONSE_ERROR_MESSAGE;
-            this.log.info('Received response from OpenAI', message);
+            // this.log.info('Received response from OpenAI', message)
             return message;
         });
     }

package/build/esm/bots/adapters/OpenAiMessageBuilder.d.ts

@@ -6,6 +6,7 @@ export default class OpenAiMessageBuilder {
     static Builder(bot: SprucebotLlmBot): OpenAiMessageBuilder;
     buildMessages(): ChatCompletionMessageParam[];
     private buildChatHistoryMessages;
+    private mapMessageToCompletion;
     private buildFirstMessage;
     private buildSkillMessages;
     private buildCallbacksMessage;

package/build/esm/bots/adapters/OpenAiMessageBuilder.js

@@ -22,10 +22,28 @@ export default class OpenAiMessageBuilder {
         if (limit > 0) {
             messagesBeingConsidered = messages.slice(Math.max(messages.length - limit, 0));
         }
-        return messagesBeingConsidered.map((message) => ({
+        return messagesBeingConsidered.map((message) => this.mapMessageToCompletion(message));
+    }
+    mapMessageToCompletion(message) {
+        let content = message.message;
+        if (message.imageBase64) {
+            content = [
+                {
+                    type: 'text',
+                    text: message.message,
+                },
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: `data:image/png;base64,${message.imageBase64}`,
+                    },
+                },
+            ];
+        }
+        return {
             role: message.from === 'Me' ? 'user' : 'assistant',
-            content: message.message,
-        }));
+            content,
+        };
     }
     buildFirstMessage(youAre) {
         return {
@@ -82,7 +100,7 @@ export default class OpenAiMessageBuilder {
         const api = `<APIReference>\n\n${descriptions.join('\n\n')}</APIReference>`;
         return {
             role: 'system',
-            content: `You have an API available to you to lookup answers. When you need the response of the function call to proceed, you can call a function using a custom markup we created that looks like this: << FunctionName />>. The API will respond with the results and then you can continue the conversation with your new knowledge. If the api call has parameters, call it like this: << FunctionName >>parameters json encoded<</ FunctionName >>. Make sure to json encode the data and drop it between the function tags. I will respond with the api's response and you can use it going forward. The api is as follows (in xml format):\n\n${api}`,
+            content: `You have an API available to you to lookup answers. When you need the response of the function call to proceed, you can call a function using a custom markup we created that looks like this: <<FunctionName/>>. The API will respond with the results and then you can continue the conversation with your new knowledge. If the api call has parameters, call it like this: <<FunctionName>>{{parametersJsonEncoded}}<</FunctionName>>. Make sure to json encode the data and drop it between the function tags. The API is as follows (in xml format):\n\n${api}`,
         };
     }
     buildPleaseKeepInMindMessage(pleaseKeepInMindThat) {

package/build/esm/bots/templates.js

@@ -1,4 +1,4 @@
-import renderLegacyPlaceholder from '../parsingResponses/renderPlaceholder.js';
+import renderPlaceholder from '../parsingResponses/renderPlaceholder.js';
 export const STATE_BOUNDARY = '*****';
 export const DONE_TOKEN = `DONE_DONE_DONE`;
 export const CALLBACK_BOUNDARY = 'xxxxx';
@@ -37,7 +37,7 @@ When asking me about a "select" field, make sure I only pick a valid choice by s
 	
 Your primary objective for this conversation is <%= it.skill.yourJobIfYouChooseToAcceptItIs %>
 <% if (it.skill.callbacks) { %>
-While we are talking, there are going to be things I don't want you to answer, but instead to respond with a placeholder in the form of ${renderLegacyPlaceholder('example')}.
+While we are talking, there are going to be things I don't want you to answer, but instead to respond with a placeholder in the form of ${renderPlaceholder('example')}.
 
 Here are the placeholders we will be using:
 

package/build/esm/chatWithImages.d.ts

@@ -0,0 +1 @@
+export {};

package/build/esm/chatWithImages.js

@@ -0,0 +1,46 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import fs from 'fs';
+import dotenv from 'dotenv';
+import { OpenAiAdapter } from './bots/adapters/OpenAi.js';
+import SprucebotLlmFactory from './bots/SprucebotLlmFactory.js';
+dotenv.config();
+void (() => __awaiter(void 0, void 0, void 0, function* () {
+    console.clear();
+    const image1 = fs.readFileSync('build/examples/images/image1.png');
+    const image2 = fs.readFileSync('build/examples/images/image2.png');
+    const base64Image1 = image1.toString('base64');
+    const base64Image2 = image2.toString('base64');
+    const adapter = OpenAiAdapter.Adapter(process.env.OPEN_AI_API_KEY);
+    const bots = SprucebotLlmFactory.Factory(adapter);
+    const skill = bots.Skill({
+        weAreDoneWhen: 'you have described both images to me',
+        yourJobIfYouChooseToAcceptItIs: 'to review the image i send you, invoke the getNextImage function and then describe that image too.',
+        callbacks: {
+            getNextImage: {
+                cb: () => __awaiter(void 0, void 0, void 0, function* () {
+                    return {
+                        imageBase64: base64Image2,
+                        imageDescription: 'A beautiful sunset over the mountains.',
+                    };
+                }),
+                useThisWhenever: 'you want to get the next image',
+            },
+        },
+    });
+    const bot = bots.Bot({
+        skill,
+        youAre: "a bot named Sprucebot that is in test mode. At the start of every conversation, you introduce yourself and announce that you are in test mode so I don't get confused! You are both hip and adorable. You say things like, 'Jeepers' and 'Golly' or even 'Jeezey peezy'!",
+    });
+    yield bot.sendMessage({
+        imageBase64: base64Image1,
+        imageDescription: 'The first image',
+    }, (message) => console.log('>', message));
+}))();

package/build/esm/llm.types.d.ts

@@ -8,7 +8,7 @@ export interface BotOptions<StateSchema extends Schema = Schema, State extends S
 export interface SprucebotLlmBot<StateSchema extends Schema = Schema, State extends SchemaValues<StateSchema> = SchemaValues<StateSchema>> extends MercuryEventEmitter<LlmEventContract> {
     markAsDone(): void;
     getIsDone(): boolean;
-    sendMessage(message: string, responseCb?: MessageResponseCallback): Promise<string>;
+    sendMessage(message: SendMessage, responseCb?: MessageResponseCallback): Promise<string>;
     serialize(): SerializedBot<StateSchema, State>;
     updateState(state: Partial<State>): Promise<void>;
     setSkill(skill: SprucebotLLmSkill<any>): void;
@@ -41,6 +41,7 @@ export type LlmEventContract = typeof llmEventContract;
 export interface LlmMessage {
     from: 'Me' | 'You';
     message: string;
+    imageBase64?: string;
 }
 export interface SkillOptions<StateSchema extends Schema = Schema, State extends SchemaValues<StateSchema> = SchemaValues<StateSchema>> {
     yourJobIfYouChooseToAcceptItIs: string;
@@ -60,7 +61,7 @@ export interface SerializedSkill<StateSchema extends Schema = Schema, State exte
 }
 export type LlmCallbackMap = Record<string, LlmCallback>;
 export interface LlmCallback {
-    cb: (options?: Record<string, any>) => string | Promise<string>;
+    cb: (options?: Record<string, any>) => SendMessage | Promise<SendMessage>;
     useThisWhenever: string;
     parameters?: LlmCallbackParameter[];
 }
@@ -71,3 +72,8 @@ export interface LlmCallbackParameter {
     description?: string;
 }
 export type MessageResponseCallback = (message: string) => any;
+export type SendMessage = string | SendMessageWithImage;
+export interface SendMessageWithImage {
+    imageDescription: string;
+    imageBase64: string;
+}

package/build/esm/parsingResponses/ResponseParser.d.ts

@@ -1,4 +1,4 @@
-import { LlmCallbackMap } from '../llm.types';
+import { LlmCallbackMap, SendMessage } from '../llm.types';
 export default class ResponseParser {
     private static instance;
     static setInstance(parser: ResponseParser): void;
@@ -13,5 +13,5 @@ export interface ParsedResponse {
     isDone: boolean;
     state?: Record<string, any>;
     message: string;
-    callbackResults?: string;
+    callbackResults?: SendMessage;
 }

package/build/esm/parsingResponses/ResponseParser.js

@@ -9,7 +9,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 import { DONE_TOKEN, STATE_BOUNDARY } from '../bots/templates.js';
 import SpruceError from '../errors/SpruceError.js';
-import renderLegacyPlaceholder from './renderPlaceholder.js';
+import renderPlaceholder from './renderPlaceholder.js';
 class ResponseParser {
     static setInstance(parser) {
         this.instance = parser;
@@ -24,7 +24,7 @@ class ResponseParser {
             let state;
             let callbackResults;
             for (const key of Object.keys(callbacks || {})) {
-                const match = message.match(renderLegacyPlaceholder(key));
+                const match = message.match(renderPlaceholder(key));
                 if (match) {
                     message = yield this.invokeCallbackAndDropInLegacyResults(callbacks, key, message);
                 }
@@ -74,8 +74,8 @@ class ResponseParser {
     invokeCallbackAndDropInLegacyResults(callbacks, key, message) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a;
-            const v = yield ((_a = callbacks === null || callbacks === void 0 ? void 0 : callbacks[key]) === null || _a === void 0 ? void 0 : _a.cb());
-            message = message.replace(renderLegacyPlaceholder(key), v !== null && v !== void 0 ? v : '').trim();
+            const v = (yield ((_a = callbacks === null || callbacks === void 0 ? void 0 : callbacks[key]) === null || _a === void 0 ? void 0 : _a.cb()));
+            message = message.replace(renderPlaceholder(key), v !== null && v !== void 0 ? v : '').trim();
             return message;
         });
     }

package/build/esm/parsingResponses/renderPlaceholder.d.ts

@@ -1 +1 @@
-export default function renderLegacyPlaceholder(key: string): string;
+export default function renderPlaceholder(key: string): string;

package/build/esm/parsingResponses/renderPlaceholder.js

@@ -1,4 +1,4 @@
 import { CALLBACK_BOUNDARY } from '../bots/templates.js';
-export default function renderLegacyPlaceholder(key) {
+export default function renderPlaceholder(key) {
     return `${CALLBACK_BOUNDARY} ${key} ${CALLBACK_BOUNDARY}`;
 }

package/build/llm.types.d.ts

@@ -8,7 +8,7 @@ export interface BotOptions<StateSchema extends Schema = Schema, State extends S
 export interface SprucebotLlmBot<StateSchema extends Schema = Schema, State extends SchemaValues<StateSchema> = SchemaValues<StateSchema>> extends MercuryEventEmitter<LlmEventContract> {
     markAsDone(): void;
     getIsDone(): boolean;
-    sendMessage(message: string, responseCb?: MessageResponseCallback): Promise<string>;
+    sendMessage(message: SendMessage, responseCb?: MessageResponseCallback): Promise<string>;
     serialize(): SerializedBot<StateSchema, State>;
     updateState(state: Partial<State>): Promise<void>;
     setSkill(skill: SprucebotLLmSkill<any>): void;
@@ -41,6 +41,7 @@ export type LlmEventContract = typeof llmEventContract;
 export interface LlmMessage {
     from: 'Me' | 'You';
     message: string;
+    imageBase64?: string;
 }
 export interface SkillOptions<StateSchema extends Schema = Schema, State extends SchemaValues<StateSchema> = SchemaValues<StateSchema>> {
     yourJobIfYouChooseToAcceptItIs: string;
@@ -60,7 +61,7 @@ export interface SerializedSkill<StateSchema extends Schema = Schema, State exte
 }
 export type LlmCallbackMap = Record<string, LlmCallback>;
 export interface LlmCallback {
-    cb: (options?: Record<string, any>) => string | Promise<string>;
+    cb: (options?: Record<string, any>) => SendMessage | Promise<SendMessage>;
     useThisWhenever: string;
     parameters?: LlmCallbackParameter[];
 }
@@ -71,3 +72,8 @@ export interface LlmCallbackParameter {
     description?: string;
 }
 export type MessageResponseCallback = (message: string) => any;
+export type SendMessage = string | SendMessageWithImage;
+export interface SendMessageWithImage {
+    imageDescription: string;
+    imageBase64: string;
+}

package/build/parsingResponses/ResponseParser.d.ts

@@ -1,4 +1,4 @@
-import { LlmCallbackMap } from '../llm.types';
+import { LlmCallbackMap, SendMessage } from '../llm.types';
 export default class ResponseParser {
     private static instance;
     static setInstance(parser: ResponseParser): void;
@@ -13,5 +13,5 @@ export interface ParsedResponse {
     isDone: boolean;
     state?: Record<string, any>;
     message: string;
-    callbackResults?: string;
+    callbackResults?: SendMessage;
 }

package/build/parsingResponses/ResponseParser.js

@@ -65,7 +65,7 @@ class ResponseParser {
         return matchedCallback;
     }
     async invokeCallbackAndDropInLegacyResults(callbacks, key, message) {
-        const v = await callbacks?.[key]?.cb();
+        const v = (await callbacks?.[key]?.cb());
         message = message.replace((0, renderPlaceholder_1.default)(key), v ?? '').trim();
         return message;
     }

package/build/parsingResponses/renderPlaceholder.d.ts

@@ -1 +1 @@
-export default function renderLegacyPlaceholder(key: string): string;
+export default function renderPlaceholder(key: string): string;

package/build/parsingResponses/renderPlaceholder.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.default = renderLegacyPlaceholder;
+exports.default = renderPlaceholder;
 const templates_1 = require("../bots/templates");
-function renderLegacyPlaceholder(key) {
+function renderPlaceholder(key) {
     return `${templates_1.CALLBACK_BOUNDARY} ${key} ${templates_1.CALLBACK_BOUNDARY}`;
 }

package/package.json

@@ -9,7 +9,7 @@
       "@sprucelabs/spruce-test-fixtures"
     ]
   },
-  "version": "8.4.101",
+  "version": "8.5.1",
   "files": [
     "build"
   ],
@@ -50,7 +50,8 @@
     "watch.build.dev": "tsc-watch --sourceMap --onCompilationComplete 'yarn run post.watch.build'",
     "watch.rebuild": "yarn run clean.all && yarn install && yarn run watch.build.dev",
     "watch.tsc": "tsc -w",
-    "chat": "node ./build/chat.js"
+    "chat": "node ./build/chat.js",
+    "chat.images": "node ./build/chatWithImages.js"
   },
   "dependencies": {
     "@sprucelabs/error": "^6.0.598",