@theia/ai-openai 1.58.2 → 1.59.0-next.62

package/src/node/openai-language-model.ts

@@ -20,17 +20,20 @@ import {
     LanguageModelRequest,
     LanguageModelRequestMessage,
     LanguageModelResponse,
-    LanguageModelStreamResponsePart,
     LanguageModelTextResponse
 } from '@theia/ai-core';
 import { CancellationToken } from '@theia/core';
+import { injectable } from '@theia/core/shared/inversify';
 import { OpenAI, AzureOpenAI } from 'openai';
 import { ChatCompletionStream } from 'openai/lib/ChatCompletionStream';
 import { RunnableToolFunctionWithoutParse } from 'openai/lib/RunnableFunction';
 import { ChatCompletionMessageParam } from 'openai/resources';
+import { StreamingAsyncIterator } from './openai-streaming-iterator';
 
 export const OpenAiModelIdentifier = Symbol('OpenAiModelIdentifier');
 
+export type DeveloperMessageSettings = 'user' | 'system' | 'developer' | 'mergeWithFollowingUserMessage' | 'skip';
+
 export class OpenAiModel implements LanguageModel {
 
     /**
@@ -39,7 +42,7 @@ export class OpenAiModel implements LanguageModel {
      * @param enableStreaming whether the streaming API shall be used
      * @param apiKey a function that returns the API key to use for this model, called on each request
      * @param apiVersion a function that returns the OpenAPI version to use for this model, called on each request
-     * @param supportsDeveloperMessage whether the model supports the `developer` role
+     * @param developerMessageSettings how to handle system messages
      * @param url the OpenAI API compatible endpoint where the model is hosted. If not provided the default OpenAI endpoint will be used.
      * @param defaultRequestSettings optional default settings for requests made using this model.
      */
@@ -49,9 +52,11 @@ export class OpenAiModel implements LanguageModel {
         public enableStreaming: boolean,
         public apiKey: () => string | undefined,
         public apiVersion: () => string | undefined,
-        public supportsDeveloperMessage: boolean,
+        public supportsStructuredOutput: boolean,
         public url: string | undefined,
-        public defaultRequestSettings?: { [key: string]: unknown }
+        public openAiModelUtils: OpenAiModelUtils,
+        public developerMessageSettings: DeveloperMessageSettings = 'developer',
+        public defaultRequestSettings?: { [key: string]: unknown },
     ) { }
 
     protected getSettings(request: LanguageModelRequest): Record<string, unknown> {
@@ -66,23 +71,23 @@ export class OpenAiModel implements LanguageModel {
         const settings = this.getSettings(request);
         const openai = this.initializeOpenAi();
 
+        if (request.response_format?.type === 'json_schema' && this.supportsStructuredOutput) {
+            return this.handleStructuredOutputRequest(openai, request);
+        }
+
         if (this.isNonStreamingModel(this.model) || (typeof settings.stream === 'boolean' && !settings.stream)) {
             return this.handleNonStreamingRequest(openai, request);
         }
 
-        if (request.response_format?.type === 'json_schema' && this.supportsStructuredOutput()) {
-            return this.handleStructuredOutputRequest(openai, request);
-        }
         if (cancellationToken?.isCancellationRequested) {
             return { text: '' };
         }
-
         let runner: ChatCompletionStream;
         const tools = this.createTools(request);
         if (tools) {
             runner = openai.beta.chat.completions.runTools({
                 model: this.model,
-                messages: request.messages.map(this.toOpenAIMessage.bind(this)),
+                messages: this.processMessages(request.messages),
                 stream: true,
                 tools: tools,
                 tool_choice: 'auto',
@@ -91,81 +96,20 @@ export class OpenAiModel implements LanguageModel {
         } else {
             runner = openai.beta.chat.completions.stream({
                 model: this.model,
-                messages: request.messages.map(this.toOpenAIMessage.bind(this)),
+                messages: this.processMessages(request.messages),
                 stream: true,
                 ...settings
             });
         }
-        cancellationToken?.onCancellationRequested(() => {
-            runner.abort();
-        });
 
-        let runnerEnd = false;
-
-        let resolve: ((part: LanguageModelStreamResponsePart) => void) | undefined;
-        runner.on('error', error => {
-            console.error('Error in OpenAI chat completion stream:', error);
-            runnerEnd = true;
-            resolve?.({ content: error.message });
-        });
-        // we need to also listen for the emitted errors, as otherwise any error actually thrown by the API will not be caught
-        runner.emitted('error').then(error => {
-            console.error('Error in OpenAI chat completion stream:', error);
-            runnerEnd = true;
-            resolve?.({ content: error.message });
-        });
-        runner.emitted('abort').then(() => {
-            // cancel async iterator
-            runnerEnd = true;
-        });
-        runner.on('message', message => {
-            if (message.role === 'tool') {
-                resolve?.({ tool_calls: [{ id: message.tool_call_id, finished: true, result: this.getCompletionContent(message) }] });
-            }
-            console.debug('Received Open AI message', JSON.stringify(message));
-        });
-        runner.once('end', () => {
-            runnerEnd = true;
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            resolve?.(runner.finalChatCompletion as any);
-        });
-        if (cancellationToken?.isCancellationRequested) {
-            return { text: '' };
-        }
-        const asyncIterator = {
-            async *[Symbol.asyncIterator](): AsyncIterator<LanguageModelStreamResponsePart> {
-                runner.on('chunk', chunk => {
-                    if (cancellationToken?.isCancellationRequested) {
-                        resolve = undefined;
-                        return;
-                    }
-                    if (resolve && chunk.choices[0]?.delta) {
-                        resolve({ ...chunk.choices[0]?.delta });
-                    }
-                });
-                while (!runnerEnd) {
-                    if (cancellationToken?.isCancellationRequested) {
-                        throw new Error('Iterator canceled');
-                    }
-                    const promise = new Promise<LanguageModelStreamResponsePart>((res, rej) => {
-                        resolve = res;
-                        cancellationToken?.onCancellationRequested(() => {
-                            rej(new Error('Canceled'));
-                            runnerEnd = true; // Stop the iterator
-                        });
-                    });
-                    yield promise;
-                }
-            }
-        };
-        return { stream: asyncIterator };
+        return { stream: new StreamingAsyncIterator(runner, cancellationToken) };
     }
 
     protected async handleNonStreamingRequest(openai: OpenAI, request: LanguageModelRequest): Promise<LanguageModelTextResponse> {
         const settings = this.getSettings(request);
         const response = await openai.chat.completions.create({
             model: this.model,
-            messages: request.messages.map(this.toOpenAIMessage.bind(this)),
+            messages: this.processMessages(request.messages),
             ...settings
         });
 
@@ -176,43 +120,16 @@ export class OpenAiModel implements LanguageModel {
         };
     }
 
-    protected toOpenAIMessage(message: LanguageModelRequestMessage): ChatCompletionMessageParam {
-        return {
-            role: this.toOpenAiRole(message),
-            content: message.query || ''
-        };
-    }
-
-    protected toOpenAiRole(message: LanguageModelRequestMessage): 'developer' | 'user' | 'assistant' {
-        switch (message.actor) {
-            case 'system':
-                return this.supportsDeveloperMessage ? 'developer' : 'user';
-            case 'ai':
-                return 'assistant';
-            default:
-                return 'user';
-        }
-    }
-
     protected isNonStreamingModel(_model: string): boolean {
         return !this.enableStreaming;
     }
 
-    protected supportsStructuredOutput(): boolean {
-        // see https://platform.openai.com/docs/models/gpt-4o
-        return [
-            'gpt-4o',
-            'gpt-4o-2024-08-06',
-            'gpt-4o-mini'
-        ].includes(this.model);
-    }
-
     protected async handleStructuredOutputRequest(openai: OpenAI, request: LanguageModelRequest): Promise<LanguageModelParsedResponse> {
         const settings = this.getSettings(request);
         // TODO implement tool support for structured output (parse() seems to require different tool format)
         const result = await openai.beta.chat.completions.parse({
             model: this.model,
-            messages: request.messages.map(this.toOpenAIMessage.bind(this)),
+            messages: this.processMessages(request.messages),
             response_format: request.response_format,
             ...settings
         });
@@ -226,13 +143,6 @@ export class OpenAiModel implements LanguageModel {
         };
     }
 
-    private getCompletionContent(message: OpenAI.Chat.Completions.ChatCompletionToolMessageParam): string {
-        if (Array.isArray(message.content)) {
-            return message.content.join('');
-        }
-        return message.content;
-    }
-
     protected createTools(request: LanguageModelRequest): RunnableToolFunctionWithoutParse[] | undefined {
         return request.tools?.map(tool => ({
             type: 'function',
@@ -260,4 +170,92 @@ export class OpenAiModel implements LanguageModel {
             return new OpenAI({ apiKey: apiKey ?? 'no-key', baseURL: this.url });
         }
     }
+
+    protected processMessages(messages: LanguageModelRequestMessage[]): ChatCompletionMessageParam[] {
+        return this.openAiModelUtils.processMessages(messages, this.developerMessageSettings, this.model);
+    }
+}
+
+/**
+ * Utility class for processing messages for the OpenAI language model.
+ *
+ * Adopters can rebind this class to implement custom message processing behavior.
+ */
+@injectable()
+export class OpenAiModelUtils {
+
+    protected processSystemMessages(
+        messages: LanguageModelRequestMessage[],
+        developerMessageSettings: DeveloperMessageSettings
+    ): LanguageModelRequestMessage[] {
+        if (developerMessageSettings === 'skip') {
+            return messages.filter(message => message.actor !== 'system');
+        } else if (developerMessageSettings === 'mergeWithFollowingUserMessage') {
+            const updated = messages.slice();
+            for (let i = updated.length - 1; i >= 0; i--) {
+                if (updated[i].actor === 'system') {
+                    if (i + 1 < updated.length && updated[i + 1].actor === 'user') {
+                        // Merge system message with the next user message
+                        updated[i + 1] = {
+                            ...updated[i + 1],
+                            query: updated[i].query + '\n' + updated[i + 1].query
+                        };
+                        updated.splice(i, 1);
+                    } else {
+                        // The message directly after is not a user message (or none exists), so create a new user message right after
+                        updated.splice(i + 1, 0, { actor: 'user', type: 'text', query: updated[i].query });
+                        updated.splice(i, 1);
+                    }
+                }
+            }
+            return updated;
+        }
+        return messages;
+    }
+
+    protected toOpenAiRole(
+        message: LanguageModelRequestMessage,
+        developerMessageSettings: DeveloperMessageSettings
+    ): 'developer' | 'user' | 'assistant' | 'system' {
+        if (message.actor === 'system') {
+            if (developerMessageSettings === 'user' || developerMessageSettings === 'system' || developerMessageSettings === 'developer') {
+                return developerMessageSettings;
+            } else {
+                return 'developer';
+            }
+        } else if (message.actor === 'ai') {
+            return 'assistant';
+        }
+        return 'user';
+    }
+
+    protected toOpenAIMessage(
+        message: LanguageModelRequestMessage,
+        developerMessageSettings: DeveloperMessageSettings
+    ): ChatCompletionMessageParam {
+        return {
+            role: this.toOpenAiRole(message, developerMessageSettings),
+            content: message.query || ''
+        };
+    }
+
+    /**
+     * Processes the provided list of messages by applying system message adjustments and converting
+     * them to the format expected by the OpenAI API.
+     *
+     * Adopters can rebind this processing to implement custom behavior.
+     *
+     * @param messages the list of messages to process.
+     * @param developerMessageSettings how system and developer messages are handled during processing.
+     * @param model the OpenAI model identifier. Currently not used, but allows subclasses to implement model-specific behavior.
+     * @returns an array of messages formatted for the OpenAI API.
+     */
+    processMessages(
+        messages: LanguageModelRequestMessage[],
+        developerMessageSettings: DeveloperMessageSettings,
+        model: string
+    ): ChatCompletionMessageParam[] {
+        const processed = this.processSystemMessages(messages, developerMessageSettings);
+        return processed.map(m => this.toOpenAIMessage(m, developerMessageSettings));
+    }
 }

package/src/node/openai-language-models-manager-impl.ts

@@ -16,12 +16,15 @@
 
 import { LanguageModelRegistry } from '@theia/ai-core';
 import { inject, injectable } from '@theia/core/shared/inversify';
-import { OpenAiModel } from './openai-language-model';
+import { OpenAiModel, OpenAiModelUtils } from './openai-language-model';
 import { OpenAiLanguageModelsManager, OpenAiModelDescription } from '../common';
 
 @injectable()
 export class OpenAiLanguageModelsManagerImpl implements OpenAiLanguageModelsManager {
 
+    @inject(OpenAiModelUtils)
+    protected readonly openAiModelUtils: OpenAiModelUtils;
+
     protected _apiKey: string | undefined;
     protected _apiVersion: string | undefined;
 
@@ -70,7 +73,8 @@ export class OpenAiLanguageModelsManagerImpl implements OpenAiLanguageModelsMana
                 model.url = modelDescription.url;
                 model.apiKey = apiKeyProvider;
                 model.apiVersion = apiVersionProvider;
-                model.supportsDeveloperMessage = modelDescription.supportsDeveloperMessage;
+                model.developerMessageSettings = modelDescription.developerMessageSettings || 'developer';
+                model.supportsStructuredOutput = modelDescription.supportsStructuredOutput;
                 model.defaultRequestSettings = modelDescription.defaultRequestSettings;
             } else {
                 this.languageModelRegistry.addLanguageModels([
@@ -80,8 +84,10 @@ export class OpenAiLanguageModelsManagerImpl implements OpenAiLanguageModelsMana
                         modelDescription.enableStreaming,
                         apiKeyProvider,
                         apiVersionProvider,
-                        modelDescription.supportsDeveloperMessage,
+                        modelDescription.supportsStructuredOutput,
                         modelDescription.url,
+                        this.openAiModelUtils,
+                        modelDescription.developerMessageSettings,
                         modelDescription.defaultRequestSettings
                     )
                 ]);

package/src/node/openai-model-utils.spec.ts

@@ -0,0 +1,164 @@
+// *****************************************************************************
+// Copyright (C) 2024 EclipseSource GmbH.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+const { expect } = require('chai');
+const { OpenAiModelUtils } = require('./openai-language-model');
+const utils = new OpenAiModelUtils();
+
+describe('OpenAiModelUtils - processMessages', () => {
+    describe("when developerMessageSettings is 'skip'", () => {
+        it('should remove all system messages', () => {
+            const messages = [
+                { actor: 'system', type: 'text', query: 'system message' },
+                { actor: 'user', type: 'text', query: 'user message' },
+                { actor: 'system', type: 'text', query: 'another system message' },
+            ];
+            const result = utils.processMessages(messages, 'skip');
+            expect(result).to.deep.equal([
+                { role: 'user', content: 'user message' }
+            ]);
+        });
+
+        it('should do nothing if there is no system message', () => {
+            const messages = [
+                { actor: 'user', type: 'text', query: 'user message' },
+                { actor: 'user', type: 'text', query: 'another user message' },
+                { actor: 'ai', type: 'text', query: 'ai message' }
+            ];
+            const result = utils.processMessages(messages, 'skip');
+            expect(result).to.deep.equal([
+                { role: 'user', content: 'user message' },
+                { role: 'user', content: 'another user message' },
+                { role: 'assistant', content: 'ai message' }
+            ]);
+        });
+    });
+
+    describe("when developerMessageSettings is 'mergeWithFollowingUserMessage'", () => {
+        it('should merge the system message with the next user message, assign role user, and remove the system message', () => {
+            const messages = [
+                { actor: 'system', type: 'text', query: 'system msg' },
+                { actor: 'user', type: 'text', query: 'user msg' },
+                { actor: 'ai', type: 'text', query: 'ai message' }
+            ];
+            const result = utils.processMessages(messages, 'mergeWithFollowingUserMessage');
+            expect(result).to.deep.equal([
+                { role: 'user', content: 'system msg\nuser msg' },
+                { role: 'assistant', content: 'ai message' }
+            ]);
+        });
+
+        it('should create a new user message if no user message exists, and remove the system message', () => {
+            const messages = [
+                { actor: 'system', type: 'text', query: 'system only msg' },
+                { actor: 'ai', type: 'text', query: 'ai message' }
+            ];
+            const result = utils.processMessages(messages, 'mergeWithFollowingUserMessage');
+            expect(result).to.deep.equal([
+                { role: 'user', content: 'system only msg' },
+                { role: 'assistant', content: 'ai message' }
+            ]);
+        });
+
+        it('should create a merge multiple system message with the next user message', () => {
+            const messages = [
+                { actor: 'user', type: 'text', query: 'user message' },
+                { actor: 'system', type: 'text', query: 'system message' },
+                { actor: 'system', type: 'text', query: 'system message2' },
+                { actor: 'user', type: 'text', query: 'user message2' },
+                { actor: 'ai', type: 'text', query: 'ai message' }
+            ];
+            const result = utils.processMessages(messages, 'mergeWithFollowingUserMessage');
+            expect(result).to.deep.equal([
+                { role: 'user', content: 'user message' },
+                { role: 'user', content: 'system message\nsystem message2\nuser message2' },
+                { role: 'assistant', content: 'ai message' }
+            ]);
+        });
+
+        it('should create a new user message from several system messages if the next message is not a user message', () => {
+            const messages = [
+                { actor: 'user', type: 'text', query: 'user message' },
+                { actor: 'system', type: 'text', query: 'system message' },
+                { actor: 'system', type: 'text', query: 'system message2' },
+                { actor: 'ai', type: 'text', query: 'ai message' }
+            ];
+            const result = utils.processMessages(messages, 'mergeWithFollowingUserMessage');
+            expect(result).to.deep.equal([
+                { role: 'user', content: 'user message' },
+                { role: 'user', content: 'system message\nsystem message2' },
+                { role: 'assistant', content: 'ai message' }
+            ]);
+        });
+    });
+
+    describe('when no special merging or skipping is needed', () => {
+        it('should leave messages unchanged in ordering and assign roles based on developerMessageSettings', () => {
+            const messages = [
+                { actor: 'user', type: 'text', query: 'user message' },
+                { actor: 'system', type: 'text', query: 'system message' },
+                { actor: 'ai', type: 'text', query: 'ai message' }
+            ];
+            // Using a developerMessageSettings that is not merge/skip, e.g., 'developer'
+            const result = utils.processMessages(messages, 'developer');
+            expect(result).to.deep.equal([
+                { role: 'user', content: 'user message' },
+                { role: 'developer', content: 'system message' },
+                { role: 'assistant', content: 'ai message' }
+            ]);
+        });
+    });
+
+    describe('role assignment for system messages when developerMessageSettings is one of the role strings', () => {
+        it('should assign role as specified for a system message when developerMessageSettings is "user"', () => {
+            const messages = [
+                { actor: 'system', type: 'text', query: 'system msg' },
+                { actor: 'ai', type: 'text', query: 'ai msg' }
+            ];
+            // Since the first message is system and developerMessageSettings is not merge/skip, ordering is not adjusted
+            const result = utils.processMessages(messages, 'user');
+            expect(result).to.deep.equal([
+                { role: 'user', content: 'system msg' },
+                { role: 'assistant', content: 'ai msg' }
+            ]);
+        });
+
+        it('should assign role as specified for a system message when developerMessageSettings is "system"', () => {
+            const messages = [
+                { actor: 'system', type: 'text', query: 'system msg' },
+                { actor: 'ai', type: 'text', query: 'ai msg' }
+            ];
+            const result = utils.processMessages(messages, 'system');
+            expect(result).to.deep.equal([
+                { role: 'system', content: 'system msg' },
+                { role: 'assistant', content: 'ai msg' }
+            ]);
+        });
+
+        it('should assign role as specified for a system message when developerMessageSettings is "developer"', () => {
+            const messages = [
+                { actor: 'system', type: 'text', query: 'system msg' },
+                { actor: 'user', type: 'text', query: 'user msg' },
+                { actor: 'ai', type: 'text', query: 'ai msg' }
+            ];
+            const result = utils.processMessages(messages, 'developer');
+            expect(result).to.deep.equal([
+                { role: 'developer', content: 'system msg' },
+                { role: 'user', content: 'user msg' },
+                { role: 'assistant', content: 'ai msg' }
+            ]);
+        });
+    });
+});