@proteinjs/conversation 1.5.2 → 1.6.1

package/dist/src/OpenAiStreamProcessor.js

@@ -0,0 +1,202 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenAiStreamProcessor = void 0;
+var util_1 = require("@proteinjs/util");
+var stream_1 = require("stream");
+/**
+ * Processes streaming responses from OpenAI's `ChatCompletions` api.
+ *   - When a tool call is received, it delegates processing to `onToolCalls`; this can happen recursively
+ *   - When a response to the user is received, it writes to `outputStream`
+ */
+var OpenAiStreamProcessor = /** @class */ (function () {
+    function OpenAiStreamProcessor(inputStream, onToolCalls, logLevel) {
+        this.onToolCalls = onToolCalls;
+        this.accumulatedToolCalls = [];
+        this.toolCallsExecuted = 0;
+        this.currentToolCall = null;
+        this.logger = new util_1.Logger(this.constructor.name, logLevel);
+        this.inputStream = stream_1.Readable.from(inputStream);
+        this.controlStream = this.createControlStream();
+        this.outputStream = new stream_1.PassThrough();
+        this.inputStream.pipe(this.controlStream);
+    }
+    /**
+     * @returns a `Readable` stream that will receive the assistant's text response to the user
+     */
+    OpenAiStreamProcessor.prototype.getOutputStream = function () {
+        return this.outputStream;
+    };
+    /**
+     * @returns a `Transform` that parses the input stream and delegates to tool calls or writes a text response to the user
+     */
+    OpenAiStreamProcessor.prototype.createControlStream = function () {
+        var _this = this;
+        return new stream_1.Transform({
+            objectMode: true,
+            transform: function (chunk, encoding, callback) {
+                var _a, _b, _c, _d, _e, _f, _g, _h;
+                try {
+                    if (_this.outputStream.destroyed) {
+                        _this.logger.warn("Destroying input and control streams since output stream is destroyed");
+                        _this.inputStream.destroy();
+                        _this.controlStream.destroy();
+                        return;
+                    }
+                    if (!chunk || !chunk.choices) {
+                        throw new Error("Received invalid chunk:\n".concat(JSON.stringify(chunk, null, 2)));
+                    }
+                    else if ((_b = (_a = chunk.choices[0]) === null || _a === void 0 ? void 0 : _a.delta) === null || _b === void 0 ? void 0 : _b.content) {
+                        _this.outputStream.push(chunk.choices[0].delta.content);
+                    }
+                    else if ((_d = (_c = chunk.choices[0]) === null || _c === void 0 ? void 0 : _c.delta) === null || _d === void 0 ? void 0 : _d.tool_calls) {
+                        _this.handleToolCallDelta(chunk.choices[0].delta.tool_calls);
+                    }
+                    else if (((_e = chunk.choices[0]) === null || _e === void 0 ? void 0 : _e.finish_reason) === 'tool_calls') {
+                        _this.handleToolCalls();
+                    }
+                    else if (((_f = chunk.choices[0]) === null || _f === void 0 ? void 0 : _f.finish_reason) === 'stop') {
+                        _this.outputStream.push(null);
+                    }
+                    else if (((_g = chunk.choices[0]) === null || _g === void 0 ? void 0 : _g.finish_reason) === 'length') {
+                        _this.logger.warn("The maximum number of tokens specified in the request was reached");
+                        _this.outputStream.push(null);
+                    }
+                    else if (((_h = chunk.choices[0]) === null || _h === void 0 ? void 0 : _h.finish_reason) === 'content_filter') {
+                        _this.logger.error("Content was omitted due to a flag from OpenAI's content filters");
+                        _this.outputStream.push(null);
+                    }
+                    callback();
+                }
+                catch (error) {
+                    _this.logger.error('Error tranforming chunk', error);
+                    _this.destroyStreams(error);
+                }
+            },
+        });
+    };
+    /**
+     * Destroy all streams used by `OpenAiStreamProcessor`
+     */
+    OpenAiStreamProcessor.prototype.destroyStreams = function (error) {
+        this.inputStream.destroy();
+        this.controlStream.destroy();
+        this.outputStream.emit('error', error);
+        this.outputStream.destroy();
+    };
+    /**
+     * Accumulates tool call deltas into complete tool calls.
+     * @param toolCallDeltas `ChatCompletionChunk.Choice.Delta.ToolCall` objects that contain part of a complete `ChatCompletionMessageToolCall`
+     */
+    OpenAiStreamProcessor.prototype.handleToolCallDelta = function (toolCallDeltas) {
+        var _a, _b, _c, _d;
+        for (var _i = 0, toolCallDeltas_1 = toolCallDeltas; _i < toolCallDeltas_1.length; _i++) {
+            var delta = toolCallDeltas_1[_i];
+            if (delta.id) {
+                // Start of a new tool call
+                if (this.currentToolCall) {
+                    this.accumulatedToolCalls.push(this.currentToolCall);
+                }
+                this.currentToolCall = {
+                    id: delta.id,
+                    type: delta.type || 'function',
+                    function: {
+                        name: ((_a = delta.function) === null || _a === void 0 ? void 0 : _a.name) || '',
+                        arguments: ((_b = delta.function) === null || _b === void 0 ? void 0 : _b.arguments) || '',
+                    },
+                };
+            }
+            else {
+                // Continue building the current tool call
+                if ((_c = delta.function) === null || _c === void 0 ? void 0 : _c.name) {
+                    this.currentToolCall.function.name += delta.function.name;
+                }
+                if ((_d = delta.function) === null || _d === void 0 ? void 0 : _d.arguments) {
+                    this.currentToolCall.function.arguments += delta.function.arguments;
+                }
+            }
+        }
+    };
+    /**
+     * Delegates `ChatCompletionMessageToolCall`s to `onToolCalls`.
+     *   - Manages refreshing the `inputStream` and `controlStream`
+     *   - Manages tool call state (such as keeping track of the number of tool calls made)
+     */
+    OpenAiStreamProcessor.prototype.handleToolCalls = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var completedToolCalls, _a, error_1;
+            var _this = this;
+            return __generator(this, function (_b) {
+                switch (_b.label) {
+                    case 0:
+                        if (this.currentToolCall) {
+                            this.accumulatedToolCalls.push(this.currentToolCall);
+                            this.currentToolCall = null;
+                        }
+                        completedToolCalls = this.accumulatedToolCalls.filter(function (tc) {
+                            return tc.id !== undefined && tc.function !== undefined && tc.type !== undefined;
+                        });
+                        this.accumulatedToolCalls = [];
+                        this.inputStream.destroy();
+                        this.controlStream.destroy();
+                        this.controlStream = this.createControlStream();
+                        _b.label = 1;
+                    case 1:
+                        _b.trys.push([1, 3, , 4]);
+                        _a = this;
+                        return [4 /*yield*/, this.onToolCalls(completedToolCalls, this.toolCallsExecuted)];
+                    case 2:
+                        _a.inputStream = _b.sent();
+                        this.inputStream.on('error', function (error) { return _this.destroyStreams(error); });
+                        this.inputStream.pipe(this.controlStream);
+                        this.toolCallsExecuted += completedToolCalls.length;
+                        return [3 /*break*/, 4];
+                    case 3:
+                        error_1 = _b.sent();
+                        this.logger.error('Error processing tool calls:', error_1);
+                        this.destroyStreams(error_1);
+                        return [3 /*break*/, 4];
+                    case 4: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    return OpenAiStreamProcessor;
+}());
+exports.OpenAiStreamProcessor = OpenAiStreamProcessor;
+//# sourceMappingURL=OpenAiStreamProcessor.js.map

package/dist/src/OpenAiStreamProcessor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"OpenAiStreamProcessor.js","sourceRoot":"","sources":["../../src/OpenAiStreamProcessor.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,wCAAmD;AAEnD,iCAA6E;AAE7E;;;;GAIG;AACH;IASE,+BACE,WAAwC,EAChC,WAGc,EACtB,QAAkB;QAJV,gBAAW,GAAX,WAAW,CAGG;QAZhB,yBAAoB,GAA6C,EAAE,CAAC;QACpE,sBAAiB,GAAG,CAAC,CAAC;QACtB,oBAAe,GAAkD,IAAI,CAAC;QAa5E,IAAI,CAAC,MAAM,GAAG,IAAI,aAAM,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC1D,IAAI,CAAC,WAAW,GAAG,iBAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAChD,IAAI,CAAC,YAAY,GAAG,IAAI,oBAAW,EAAE,CAAC;QACtC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,+CAAe,GAAf;QACE,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED;;OAEG;IACK,mDAAmB,GAA3B;QAAA,iBAoCC;QAnCC,OAAO,IAAI,kBAAS,CAAC;YACnB,UAAU,EAAE,IAAI;YAChB,SAAS,EAAE,UAAC,KAA0B,EAAE,QAAgB,EAAE,QAA2B;;gBACnF,IAAI;oBACF,IAAI,KAAI,CAAC,YAAY,CAAC,SAAS,EAAE;wBAC/B,KAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uEAAuE,CAAC,CAAC;wBAC1F,KAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;wBAC3B,KAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;wBAC7B,OAAO;qBACR;oBAED,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE;wBAC5B,MAAM,IAAI,KAAK,CAAC,mCAA4B,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,CAAC;qBAC/E;yBAAM,IAAI,MAAA,MAAA,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,0CAAE,KAAK,0CAAE,OAAO,EAAE;wBAC3C,KAAI,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;qBACxD;yBAAM,IAAI,MAAA,MAAA,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,0CAAE,KAAK,0CAAE,UAAU,EAAE;wBAC9C,KAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;qBAC7D;yBAAM,IAAI,CAAA,MAAA,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,0CAAE,aAAa,MAAK,YAAY,EAAE;wBAC3D,KAAI,CAAC,eAAe,EAAE,CAAC;qBACxB;yBAAM,IAAI,CAAA,MAAA,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,0CAAE,aAAa,MAAK,MAAM,EAAE;wBACrD,KAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;qBAC9B;yBAAM,IAAI,CAAA,MAAA,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,0CAAE,aAAa,MAAK,QAAQ,EAAE;wBACvD,KAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mEAAmE,CAAC,CAAC;wBACtF,KAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;qBAC9B;yBAAM,IAAI,CAAA,MAAA,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,0CAAE,aAAa,MAAK,gBAAgB,EAAE;wBAC/D,KAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iEAAiE,CAAC,CAAC;wBACrF,KAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;qBAC9B;oBACD,QAAQ,EAAE,CAAC;iBACZ;gBAAC,OAAO,KAAU,EAAE;oBACnB,KAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAC;oBACpD,KAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;iBAC5B;YACH,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,8CAAc,GAAtB,UAAuB,KAAa;QAClC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAC3B,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACvC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;IAC9B,CAAC;IAED;;;OAGG;IACK,mDAAmB,GAA3B,UAA4B,cAA2D;;QACrF,KAAoB,UAAc,EAAd,iCAAc,EAAd,4BAAc,EAAd,IAAc,EAAE;YAA/B,IAAM,KAAK,uBAAA;YACd,IAAI,KAAK,CAAC,EAAE,EAAE;gBACZ,2BAA2B;gBAC3B,IAAI,IAAI,CAAC,eAAe,EAAE;oBACxB,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;iBACtD;gBACD,IAAI,CAAC,eAAe,GAAG;oBACrB,EAAE,EAAE,KAAK,CAAC,EAAE;oBACZ,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,UAAU;oBAC9B,QAAQ,EAAE;wBACR,IAAI,EAAE,CAAA,MAAA,KAAK,CAAC,QAAQ,0CAAE,IAAI,KAAI,EAAE;wBAChC,SAAS,EAAE,CAAA,MAAA,KAAK,CAAC,QAAQ,0CAAE,SAAS,KAAI,EAAE;qBAC3C;iBACF,CAAC;aACH;iBAAM;gBACL,0CAA0C;gBAC1C,IAAI,MAAA,KAAK,CAAC,QAAQ,0CAAE,IAAI,EAAE;oBACxB,IAAI,CAAC,eAAgB,CAAC,QAAS,CAAC,IAAI,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;iBAC7D;gBACD,IAAI,MAAA,KAAK,CAAC,QAAQ,0CAAE,SAAS,EAAE;oBAC7B,IAAI,CAAC,eAAgB,CAAC,QAAS,CAAC,SAAS,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;iBACvE;aACF;SACF;IACH,CAAC;IAED;;;;OAIG;IACW,+CAAe,GAA7B;;;;;;;wBACE,IAAI,IAAI,CAAC,eAAe,EAAE;4BACxB,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;4BACrD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;yBAC7B;wBAEK,kBAAkB,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,CACzD,UAAC,EAAE;4BACD,OAAA,EAAE,CAAC,EAAE,KAAK,SAAS,IAAI,EAAE,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,CAAC,IAAI,KAAK,SAAS;wBAAzE,CAAyE,CAC5E,CAAC;wBAEF,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC;wBAC/B,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;wBAC3B,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;wBAC7B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;;;;wBAG9C,KAAA,IAAI,CAAA;wBAAe,qBAAM,IAAI,CAAC,WAAW,CAAC,kBAAkB,EAAE,IAAI,CAAC,iBAAiB,CAAC,EAAA;;wBAArF,GAAK,WAAW,GAAG,SAAkE,CAAC;wBACtF,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,UAAC,KAAK,IAAK,OAAA,KAAI,CAAC,cAAc,CAAC,KAAK,CAAC,EAA1B,CAA0B,CAAC,CAAC;wBACpE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;wBAC1C,IAAI,CAAC,iBAAiB,IAAI,kBAAkB,CAAC,MAAM,CAAC;;;;wBAEpD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,OAAK,CAAC,CAAC;wBACzD,IAAI,CAAC,cAAc,CAAC,OAAK,CAAC,CAAC;;;;;;KAE9B;IACH,4BAAC;AAAD,CAAC,AAhJD,IAgJC;AAhJY,sDAAqB"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@proteinjs/conversation",
-  "version": "1.5.2",
+  "version": "1.6.1",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "publishConfig": {
@@ -36,5 +36,5 @@
     "tiktoken": "1.0.15",
     "typescript": "5.2.2"
   },
-  "gitHead": "19aa49a81de90688b40224120ec611cef65e77f4"
+  "gitHead": "e0bc39f8f16e8e9f062194234cee2c4703336da5"
 }

package/src/Conversation.ts

@@ -209,6 +209,18 @@ export class Conversation {
     );
   }
 
+  async generateStreamingResponse(messages: (string | ChatCompletionMessageParam)[], model?: TiktokenModel) {
+    await this.enforceTokenLimit(messages, model);
+    return await OpenAi.generateStreamingResponse(
+      messages,
+      model,
+      this.history,
+      this.functions,
+      this.messageModerators,
+      this.params.logLevel
+    );
+  }
+
   async generateCode(description: string[], model?: TiktokenModel) {
     this.logger.info(`Generating code for description:\n${description.join('\n')}`);
     const code = await OpenAi.generateCode(

package/src/OpenAi.ts

@@ -1,11 +1,19 @@
 import { OpenAI as OpenAIApi } from 'openai';
-import { ChatCompletionMessageParam, ChatCompletion, ChatCompletionMessageToolCall } from 'openai/resources/chat';
+import {
+  ChatCompletionMessageParam,
+  ChatCompletion,
+  ChatCompletionMessageToolCall,
+  ChatCompletionChunk,
+} from 'openai/resources/chat';
 import { LogLevel, Logger, isInstanceOf } from '@proteinjs/util';
 import { MessageModerator } from './history/MessageModerator';
 import { Function } from './Function';
 import { MessageHistory } from './history/MessageHistory';
 import { TiktokenModel } from 'tiktoken';
 import { ChatCompletionMessageParamFactory } from './ChatCompletionMessageParamFactory';
+import { Stream } from 'openai/streaming';
+import { Readable } from 'stream';
+import { OpenAiStreamProcessor } from './OpenAiStreamProcessor';
 
 function delay(ms: number) {
   return new Promise((resolve) => setTimeout(resolve, ms));
@@ -22,8 +30,9 @@ export class OpenAi {
     logLevel: LogLevel = 'info',
     maxFunctionCalls: number = 50
   ): Promise<string> {
-    return await this.generateResponseHelper(
+    return (await this.generateResponseHelper(
       messages,
+      false,
       0,
       model,
       history,
@@ -31,11 +40,34 @@ export class OpenAi {
       messageModerators,
       logLevel,
       maxFunctionCalls
-    );
+    )) as string;
+  }
+
+  static async generateStreamingResponse(
+    messages: (string | ChatCompletionMessageParam)[],
+    model?: string,
+    history?: MessageHistory,
+    functions?: Omit<Function, 'instructions'>[],
+    messageModerators?: MessageModerator[],
+    logLevel: LogLevel = 'info',
+    maxFunctionCalls: number = 50
+  ): Promise<Readable> {
+    return (await this.generateResponseHelper(
+      messages,
+      true,
+      0,
+      model,
+      history,
+      functions,
+      messageModerators,
+      logLevel,
+      maxFunctionCalls
+    )) as Readable;
   }
 
   static async generateResponseHelper(
     messages: (string | ChatCompletionMessageParam)[],
+    stream: boolean,
     currentFunctionCalls: number,
     model?: string,
     history?: MessageHistory,
@@ -43,38 +75,44 @@ export class OpenAi {
     messageModerators?: MessageModerator[],
     logLevel: LogLevel = 'info',
     maxFunctionCalls: number = 50
-  ): Promise<string> {
-    const logger = new Logger('OpenAi.generateResponse', logLevel);
-    const messageParams: ChatCompletionMessageParam[] = messages.map((message) => {
-      if (typeof message === 'string') {
-        return { role: 'user', content: message };
-      }
+  ): Promise<string | Readable> {
+    const logger = new Logger('OpenAi.generateResponseHelper', logLevel);
+    const updatedHistory = OpenAi.getUpdatedMessageHistory(messages, history, messageModerators);
+    const response = await OpenAi.executeRequest(updatedHistory, stream, logLevel, functions, model);
+    if (stream) {
+      logger.info(`Processing response stream`);
+      const inputStream = response as Stream<ChatCompletionChunk>;
 
-      return message;
-    });
-    if (history) {
-      history.push(messageParams);
-    }
-    let messageParamsWithHistory = history ? history : new MessageHistory().push(messageParams);
-    if (messageModerators) {
-      messageParamsWithHistory = OpenAi.moderateHistory(messageParamsWithHistory, messageModerators);
-    }
-    const response = await OpenAi.executeRequest(messageParamsWithHistory, logLevel, functions, model);
-    const responseMessage = response.choices[0].message;
-    if (responseMessage.tool_calls) {
-      if (currentFunctionCalls >= maxFunctionCalls) {
-        throw new Error(`Max function calls (${maxFunctionCalls}) reached. Stopping execution.`);
+      // For subsequent tool calls, return the raw OpenAI stream to `OpenAiStreamProcessor`
+      if (currentFunctionCalls > 0) {
+        return Readable.from(inputStream);
       }
 
-      messageParamsWithHistory.push([responseMessage]);
-      const toolMessageParams = await this.callTools(logLevel, responseMessage.tool_calls, functions);
-      messageParamsWithHistory.push([...toolMessageParams]);
+      // For the initial call to `generateResponseHelper`, return the `OpenAiStreamProcessor` output stream
+      const onToolCalls = ((toolCalls, currentFunctionCalls) =>
+        OpenAi.handleToolCalls(
+          toolCalls,
+          true,
+          currentFunctionCalls,
+          updatedHistory,
+          model,
+          functions,
+          messageModerators,
+          logLevel,
+          maxFunctionCalls
+        )) as (toolCalls: ChatCompletionMessageToolCall[], currentFunctionCalls: number) => Promise<Readable>;
+      const streamProcessor = new OpenAiStreamProcessor(inputStream, onToolCalls, logLevel);
+      return streamProcessor.getOutputStream();
+    }
 
-      return await this.generateResponseHelper(
-        [],
-        currentFunctionCalls + responseMessage.tool_calls.length,
+    const responseMessage = (response as ChatCompletion).choices[0].message;
+    if (responseMessage.tool_calls) {
+      return await OpenAi.handleToolCalls(
+        responseMessage.tool_calls,
+        stream,
+        currentFunctionCalls,
+        updatedHistory,
         model,
-        messageParamsWithHistory,
         functions,
         messageModerators,
         logLevel,
@@ -87,10 +125,33 @@ export class OpenAi {
       throw new Error(`Response was empty for messages: ${messages.join('\n')}`);
     }
 
-    messageParamsWithHistory.push([responseMessage]);
+    updatedHistory.push([responseMessage]);
     return responseText;
   }
 
+  private static getUpdatedMessageHistory(
+    messages: (string | ChatCompletionMessageParam)[],
+    history?: MessageHistory,
+    messageModerators?: MessageModerator[]
+  ) {
+    const messageParams: ChatCompletionMessageParam[] = messages.map((message) => {
+      if (typeof message === 'string') {
+        return { role: 'user', content: message };
+      }
+
+      return message;
+    });
+    if (history) {
+      history.push(messageParams);
+    }
+    let messageParamsWithHistory = history ? history : new MessageHistory().push(messageParams);
+    if (messageModerators) {
+      messageParamsWithHistory = OpenAi.moderateHistory(messageParamsWithHistory, messageModerators);
+    }
+
+    return messageParamsWithHistory;
+  }
+
   private static moderateHistory(history: MessageHistory, messageModerators: MessageModerator[]) {
     for (const messageModerator of messageModerators) {
       history.setMessages(messageModerator.observe(history.getMessages()));
@@ -101,24 +162,18 @@ export class OpenAi {
 
   private static async executeRequest(
     messageParamsWithHistory: MessageHistory,
+    stream: boolean,
     logLevel: LogLevel,
     functions?: Omit<Function, 'instructions'>[],
     model?: string
-  ): Promise<ChatCompletion> {
+  ): Promise<ChatCompletion | Stream<ChatCompletionChunk>> {
     const logger = new Logger('OpenAi.executeRequest', logLevel);
     const openaiApi = new OpenAIApi();
-    let response: ChatCompletion;
     try {
       const latestMessage = messageParamsWithHistory.getMessages()[messageParamsWithHistory.getMessages().length - 1];
-      if (latestMessage.content) {
-        logger.info(`Sending request: ${latestMessage.content}`);
-      } else if (latestMessage.role == 'function') {
-        logger.info(`Sending request: returning output of ${latestMessage.name} function`);
-      } else {
-        logger.info(`Sending request`);
-      }
-      logger.debug(`Sending messages: ${JSON.stringify(messageParamsWithHistory.getMessages(), null, 2)}`, true);
-      response = await openaiApi.chat.completions.create({
+      this.logRequestDetails(logger, logLevel, latestMessage, messageParamsWithHistory);
+
+      const response = await openaiApi.chat.completions.create({
         model: model ? model : DEFAULT_MODEL,
         temperature: 0,
         messages: messageParamsWithHistory.getMessages(),
@@ -126,41 +181,130 @@ export class OpenAi {
           type: 'function',
           function: f.definition,
         })),
+        stream: stream,
       });
-      const responseMessage = response.choices[0].message;
-      if (responseMessage.content) {
-        logger.info(`Received response: ${responseMessage.content}`);
-      } else if (responseMessage.tool_calls) {
-        logger.info(
-          `Received response: call functions: ${JSON.stringify(responseMessage.tool_calls.map((toolCall) => toolCall.function.name))}`
-        );
-      } else {
-        logger.info(`Received response`);
-      }
-      if (response.usage) {
-        logger.info(JSON.stringify(response.usage));
-      } else {
-        logger.info(JSON.stringify(`Usage data missing`));
+
+      if (!stream) {
+        this.logResponseDetails(logger, response as ChatCompletion);
       }
+
+      return response;
     } catch (error: any) {
-      logger.info(`Received error response, error type: ${error.type}`);
-      if (typeof error.status !== 'undefined' && error.status == 429) {
-        if (error.type == 'tokens' && typeof error.headers['x-ratelimit-reset-tokens'] === 'string') {
-          const waitTime = parseInt(error.headers['x-ratelimit-reset-tokens']);
-          const remainingTokens = error.headers['x-ratelimit-remaining-tokens'];
-          const delayMs = 15000;
-          logger.warn(
-            `Waiting to retry in ${delayMs / 1000}s, token reset in: ${waitTime}s, remaining tokens: ${remainingTokens}`
-          );
-          await delay(delayMs);
-          return await OpenAi.executeRequest(messageParamsWithHistory, logLevel, functions, model);
-        }
+      return this.handleRequestError(logger, error, messageParamsWithHistory, stream, logLevel, functions, model);
+    }
+  }
+
+  private static logRequestDetails(
+    logger: Logger,
+    logLevel: LogLevel,
+    latestMessage: ChatCompletionMessageParam,
+    messageParamsWithHistory: MessageHistory
+  ) {
+    if (latestMessage.role == 'tool') {
+      logger.info(`Sending request: returning output of tool call (${latestMessage.tool_call_id})`);
+    } else if (latestMessage.content) {
+      const requestContent =
+        typeof latestMessage.content === 'string'
+          ? latestMessage.content
+          : latestMessage.content[0].type === 'text'
+            ? latestMessage.content[0].text
+            : 'image';
+      logger.info(`Sending request: ${requestContent}`);
+    } else {
+      logger.info(`Sending request`);
+    }
+
+    if (logLevel === 'debug') {
+      logger.debug(`Sending messages: ${JSON.stringify(messageParamsWithHistory.getMessages(), null, 2)}`, true);
+    }
+  }
+
+  private static logResponseDetails(logger: Logger, response: ChatCompletion) {
+    const responseMessage = response.choices[0].message;
+    if (responseMessage.content) {
+      logger.info(`Received response: ${responseMessage.content}`);
+    } else if (responseMessage.tool_calls) {
+      logger.info(
+        `Received response: call functions: ${JSON.stringify(responseMessage.tool_calls.map((toolCall) => toolCall.function.name))}`
+      );
+    } else {
+      logger.info(`Received response`);
+    }
+    if (response.usage) {
+      logger.info(JSON.stringify(response.usage));
+    } else {
+      logger.info(JSON.stringify(`Usage data missing`));
+    }
+  }
+
+  private static async handleRequestError(
+    logger: Logger,
+    error: any,
+    messageParamsWithHistory: MessageHistory,
+    stream: boolean,
+    logLevel: LogLevel,
+    functions?: Omit<Function, 'instructions'>[],
+    model?: string
+  ): Promise<ChatCompletion | Stream<ChatCompletionChunk>> {
+    logger.info(`Received error response, error type: ${error.type}`);
+    if (typeof error.status !== 'undefined' && error.status == 429) {
+      if (error.type == 'tokens' && typeof error.headers['x-ratelimit-reset-tokens'] === 'string') {
+        const waitTime = parseInt(error.headers['x-ratelimit-reset-tokens']);
+        const remainingTokens = error.headers['x-ratelimit-remaining-tokens'];
+        const delayMs = 15000;
+        logger.warn(
+          `Waiting to retry in ${delayMs / 1000}s, token reset in: ${waitTime}s, remaining tokens: ${remainingTokens}`
+        );
+        await delay(delayMs);
+        return await OpenAi.executeRequest(messageParamsWithHistory, stream, logLevel, functions, model);
       }
+    }
+    throw error;
+  }
 
-      throw error;
+  private static async handleToolCalls(
+    toolCalls: ChatCompletionMessageToolCall[],
+    stream: boolean,
+    currentFunctionCalls: number,
+    history: MessageHistory,
+    model?: string,
+    functions?: Omit<Function, 'instructions'>[],
+    messageModerators?: MessageModerator[],
+    logLevel: LogLevel = 'info',
+    maxFunctionCalls: number = 50
+  ): Promise<string | Readable> {
+    if (currentFunctionCalls >= maxFunctionCalls) {
+      throw new Error(`Max function calls (${maxFunctionCalls}) reached. Stopping execution.`);
     }
 
-    return response;
+    // Create a message for the tool calls
+    const toolCallMessage: ChatCompletionMessageParam = {
+      role: 'assistant',
+      content: null,
+      tool_calls: toolCalls,
+    };
+
+    // Add the tool call message to the history
+    history.push([toolCallMessage]);
+
+    // Call the tools and get the responses
+    const toolMessageParams = await this.callTools(logLevel, toolCalls, functions);
+
+    // Add the tool responses to the history
+    history.push(toolMessageParams);
+
+    // Generate the next response
+    return this.generateResponseHelper(
+      [],
+      stream,
+      currentFunctionCalls + toolCalls.length,
+      model,
+      history,
+      functions,
+      messageModerators,
+      logLevel,
+      maxFunctionCalls
+    );
   }
 
   private static async callTools(
@@ -201,8 +345,12 @@ export class OpenAi {
     }
 
     try {
-      logger.info(`Assistant calling function: (${toolCallId}) ${f.definition.name}(${functionCall.arguments})`, 1000);
-      const returnObject = await f.call(JSON.parse(functionCall.arguments));
+      const parsedArguments = JSON.parse(functionCall.arguments);
+      logger.info(
+        `Assistant calling function: (${toolCallId}) ${f.definition.name}(${JSON.stringify(parsedArguments, null, 2)})`,
+        1000
+      );
+      const returnObject = await f.call(parsedArguments);
 
       const returnObjectCompletionParams: ChatCompletionMessageParam[] = [];
       if (isInstanceOf(returnObject, ChatCompletionMessageParamFactory)) {
@@ -216,7 +364,7 @@ export class OpenAi {
         };
         returnObjectCompletionParams.push(instructionMessageParam, ...messageParams);
         logger.info(
-          `Assistant called function: (${toolCallId}) ${f.definition.name} => ${JSON.stringify(messageParams)}`,
+          `Assistant called function: (${toolCallId}) ${f.definition.name} => ${JSON.stringify(messageParams, null, 2)}`,
           500
         );
       } else {
@@ -228,7 +376,7 @@ export class OpenAi {
           content: serializedReturnObject,
         });
         logger.info(
-          `Assistant called function: (${toolCallId}) ${f.definition.name} => ${serializedReturnObject}`,
+          `Assistant called function: (${toolCallId}) ${f.definition.name} => ${JSON.stringify(returnObject, null, 2)}`,
           1000
         );
       }
@@ -245,9 +393,9 @@ export class OpenAi {
 
       return returnObjectCompletionParams;
     } catch (error: any) {
-      const errorMessage = `Error occurred while executing function ${f.definition.name}: ${error.message}`;
+      const errorMessage = `Error occurred while executing function ${f.definition.name}: (${toolCallId}) ${error.message}`;
       logger.error(errorMessage);
-      return [{ role: 'tool', tool_call_id: toolCallId, content: JSON.stringify({ error: errorMessage }) }];
+      throw error;
     }
   }