@theia/ai-openai 1.55.1 → 1.57.0-next.112

package/src/browser/openai-preferences.ts

@@ -34,7 +34,7 @@ export const OpenAiPreferencesSchema: PreferenceSchema = {
             type: 'array',
             description: 'Official OpenAI models to use',
             title: AI_CORE_PREFERENCES_TITLE,
-            default: ['gpt-4o', 'gpt-4o-2024-08-06', 'gpt-4o-2024-05-13', 'gpt-4o-mini', 'gpt-4-turbo', 'gpt-4', 'gpt-3.5-turbo', 'o1-preview'],
+            default: ['gpt-4o', 'gpt-4o-2024-11-20', 'gpt-4o-2024-08-06', 'gpt-4o-mini', 'o1', 'o1-mini', 'o3-mini'],
             items: {
                 type: 'string'
             }
@@ -50,6 +50,14 @@ export const OpenAiPreferencesSchema: PreferenceSchema = {
             \n\
             - provide an `apiKey` to access the API served at the given url. Use `true` to indicate the use of the global OpenAI API key.\
             \n\
+            - provide an `apiVersion` to access the API served at the given url in Azure. Use `true` to indicate the use of the global OpenAI API version.\
+            \n\
+            - specify `supportsDeveloperMessage: false` to indicate that the developer role shall not be used.\
+            \n\
+            - specify `supportsStructuredOutput: false` to indicate that structured output shall not be used.\
+            \n\
+            - specify `enableStreaming: false` to indicate that streaming shall not be used.\
+            \n\
             Refer to [our documentation](https://theia-ide.org/docs/user_ai/#openai-compatible-models-eg-via-vllm) for more information.',
             default: [],
             items: {
@@ -71,6 +79,22 @@ export const OpenAiPreferencesSchema: PreferenceSchema = {
                         type: ['string', 'boolean'],
                         title: 'Either the key to access the API served at the given url or `true` to use the global OpenAI API key',
                     },
+                    apiVersion: {
+                        type: ['string', 'boolean'],
+                        title: 'Either the version to access the API served at the given url in Azure or `true` to use the global OpenAI API version',
+                    },
+                    supportsDeveloperMessage: {
+                        type: 'boolean',
+                        title: 'Indicates whether the model supports the `developer` role. `true` by default.',
+                    },
+                    supportsStructuredOutput: {
+                        type: 'boolean',
+                        title: 'Indicates whether the model supports structured output. `true` by default.',
+                    },
+                    enableStreaming: {
+                        type: 'boolean',
+                        title: 'Indicates whether the streaming API shall be used. `true` by default.',
+                    }
                 }
             }
         }

package/src/common/openai-language-models-manager.ts

@@ -32,10 +32,31 @@ export interface OpenAiModelDescription {
      * The key for the model. If 'true' is provided the global OpenAI API key will be used.
      */
     apiKey: string | true | undefined;
+    /**
+     * The version for the api. If 'true' is provided the global OpenAI version will be used.
+     */
+    apiVersion: string | true | undefined;
+    /**
+     * Indicate whether the streaming API shall be used.
+     */
+    enableStreaming: boolean;
+    /**
+     * Flag to configure whether the OpenAPI model supports the `developer` role. Default is `true`.
+     */
+    supportsDeveloperMessage: boolean;
+    /**
+     * Flag to configure whether the OpenAPI model supports structured output. Default is `true`.
+     */
+    supportsStructuredOutput: boolean;
+    /**
+     * Default request settings for the OpenAI model.
+     */
+    defaultRequestSettings?: { [key: string]: unknown };
 }
 export interface OpenAiLanguageModelsManager {
     apiKey: string | undefined;
     setApiKey(key: string | undefined): void;
+    setApiVersion(version: string | undefined): void;
     createOrUpdateLanguageModels(...models: OpenAiModelDescription[]): Promise<void>;
     removeLanguageModels(...modelIds: string[]): void
 }

package/src/node/openai-backend-module.ts

@@ -18,13 +18,19 @@ import { ContainerModule } from '@theia/core/shared/inversify';
 import { OPENAI_LANGUAGE_MODELS_MANAGER_PATH, OpenAiLanguageModelsManager } from '../common/openai-language-models-manager';
 import { ConnectionHandler, RpcConnectionHandler } from '@theia/core';
 import { OpenAiLanguageModelsManagerImpl } from './openai-language-models-manager-impl';
+import { ConnectionContainerModule } from '@theia/core/lib/node/messaging/connection-container-module';
 
 export const OpenAiModelFactory = Symbol('OpenAiModelFactory');
 
-export default new ContainerModule(bind => {
+// We use a connection module to handle AI services separately for each frontend.
+const openAiConnectionModule = ConnectionContainerModule.create(({ bind, bindBackendService, bindFrontendService }) => {
     bind(OpenAiLanguageModelsManagerImpl).toSelf().inSingletonScope();
     bind(OpenAiLanguageModelsManager).toService(OpenAiLanguageModelsManagerImpl);
     bind(ConnectionHandler).toDynamicValue(ctx =>
         new RpcConnectionHandler(OPENAI_LANGUAGE_MODELS_MANAGER_PATH, () => ctx.container.get(OpenAiLanguageModelsManager))
     ).inSingletonScope();
 });
+
+export default new ContainerModule(bind => {
+    bind(ConnectionContainerModule).toConstantValue(openAiConnectionModule);
+});

package/src/node/openai-language-model.ts

@@ -24,49 +24,59 @@ import {
     LanguageModelTextResponse
 } from '@theia/ai-core';
 import { CancellationToken } from '@theia/core';
-import OpenAI from 'openai';
+import { OpenAI, AzureOpenAI } from 'openai';
 import { ChatCompletionStream } from 'openai/lib/ChatCompletionStream';
 import { RunnableToolFunctionWithoutParse } from 'openai/lib/RunnableFunction';
 import { ChatCompletionMessageParam } from 'openai/resources';
 
 export const OpenAiModelIdentifier = Symbol('OpenAiModelIdentifier');
 
-function toOpenAIMessage(message: LanguageModelRequestMessage): ChatCompletionMessageParam {
-    return {
-        role: toOpenAiRole(message),
-        content: message.query || ''
-    };
-}
-
-function toOpenAiRole(message: LanguageModelRequestMessage): 'system' | 'user' | 'assistant' {
-    switch (message.actor) {
-        case 'system':
-            return 'system';
-        case 'ai':
-            return 'assistant';
-        default:
-            return 'user';
-    }
-}
-
 export class OpenAiModel implements LanguageModel {
 
     /**
      * @param id the unique id for this language model. It will be used to identify the model in the UI.
      * @param model the model id as it is used by the OpenAI API
-     * @param openAIInitializer initializer for the OpenAI client, used for each request.
+     * @param enableStreaming whether the streaming API shall be used
+     * @param apiKey a function that returns the API key to use for this model, called on each request
+     * @param apiVersion a function that returns the OpenAPI version to use for this model, called on each request
+     * @param supportsDeveloperMessage whether the model supports the `developer` role
+     * @param url the OpenAI API compatible endpoint where the model is hosted. If not provided the default OpenAI endpoint will be used.
+     * @param defaultRequestSettings optional default settings for requests made using this model.
      */
-    constructor(public readonly id: string, public model: string, public apiKey: () => string | undefined, public url: string | undefined) { }
+    constructor(
+        public readonly id: string,
+        public model: string,
+        public enableStreaming: boolean,
+        public apiKey: () => string | undefined,
+        public apiVersion: () => string | undefined,
+        public supportsDeveloperMessage: boolean,
+        public supportsStructuredOutput: boolean,
+        public url: string | undefined,
+        public defaultRequestSettings?: { [key: string]: unknown }
+    ) { }
+
+    protected getSettings(request: LanguageModelRequest): Record<string, unknown> {
+        const settings = request.settings ? request.settings : this.defaultRequestSettings;
+        if (!settings) {
+            return {};
+        }
+        return settings;
+    }
 
     async request(request: LanguageModelRequest, cancellationToken?: CancellationToken): Promise<LanguageModelResponse> {
+        const settings = this.getSettings(request);
         const openai = this.initializeOpenAi();
 
-        if (this.isNonStreamingModel(this.model)) {
+        if (request.response_format?.type === 'json_schema' && this.supportsStructuredOutput) {
+            return this.handleStructuredOutputRequest(openai, request);
+        }
+
+        if (this.isNonStreamingModel(this.model) || (typeof settings.stream === 'boolean' && !settings.stream)) {
             return this.handleNonStreamingRequest(openai, request);
         }
 
-        if (request.response_format?.type === 'json_schema' && this.supportsStructuredOutput()) {
-            return this.handleStructuredOutputRequest(openai, request);
+        if (cancellationToken?.isCancellationRequested) {
+            return { text: '' };
         }
 
         let runner: ChatCompletionStream;
@@ -74,18 +84,18 @@ export class OpenAiModel implements LanguageModel {
         if (tools) {
             runner = openai.beta.chat.completions.runTools({
                 model: this.model,
-                messages: request.messages.map(toOpenAIMessage),
+                messages: request.messages.map(this.toOpenAIMessage.bind(this)),
                 stream: true,
                 tools: tools,
                 tool_choice: 'auto',
-                ...request.settings
+                ...settings
             });
         } else {
             runner = openai.beta.chat.completions.stream({
                 model: this.model,
-                messages: request.messages.map(toOpenAIMessage),
+                messages: request.messages.map(this.toOpenAIMessage.bind(this)),
                 stream: true,
-                ...request.settings
+                ...settings
             });
         }
         cancellationToken?.onCancellationRequested(() => {
@@ -94,42 +104,57 @@ export class OpenAiModel implements LanguageModel {
 
         let runnerEnd = false;
 
-        let resolve: (part: LanguageModelStreamResponsePart) => void;
+        let resolve: ((part: LanguageModelStreamResponsePart) => void) | undefined;
         runner.on('error', error => {
             console.error('Error in OpenAI chat completion stream:', error);
             runnerEnd = true;
-            resolve({ content: error.message });
+            resolve?.({ content: error.message });
         });
         // we need to also listen for the emitted errors, as otherwise any error actually thrown by the API will not be caught
         runner.emitted('error').then(error => {
             console.error('Error in OpenAI chat completion stream:', error);
             runnerEnd = true;
-            resolve({ content: error.message });
+            resolve?.({ content: error.message });
         });
         runner.emitted('abort').then(() => {
-            // do nothing, as the abort event is only emitted when the runner is aborted by us
+            // cancel async iterator
+            runnerEnd = true;
         });
         runner.on('message', message => {
             if (message.role === 'tool') {
-                resolve({ tool_calls: [{ id: message.tool_call_id, finished: true, result: this.getCompletionContent(message) }] });
+                resolve?.({ tool_calls: [{ id: message.tool_call_id, finished: true, result: this.getCompletionContent(message) }] });
             }
             console.debug('Received Open AI message', JSON.stringify(message));
         });
         runner.once('end', () => {
             runnerEnd = true;
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            resolve(runner.finalChatCompletion as any);
+            resolve?.(runner.finalChatCompletion as any);
         });
+        if (cancellationToken?.isCancellationRequested) {
+            return { text: '' };
+        }
         const asyncIterator = {
             async *[Symbol.asyncIterator](): AsyncIterator<LanguageModelStreamResponsePart> {
                 runner.on('chunk', chunk => {
-                    if (chunk.choices[0]?.delta) {
+                    if (cancellationToken?.isCancellationRequested) {
+                        resolve = undefined;
+                        return;
+                    }
+                    if (resolve && chunk.choices[0]?.delta) {
                         resolve({ ...chunk.choices[0]?.delta });
                     }
                 });
                 while (!runnerEnd) {
+                    if (cancellationToken?.isCancellationRequested) {
+                        throw new Error('Iterator canceled');
+                    }
                     const promise = new Promise<LanguageModelStreamResponsePart>((res, rej) => {
                         resolve = res;
+                        cancellationToken?.onCancellationRequested(() => {
+                            rej(new Error('Canceled'));
+                            runnerEnd = true; // Stop the iterator
+                        });
                     });
                     yield promise;
                 }
@@ -139,10 +164,11 @@ export class OpenAiModel implements LanguageModel {
     }
 
     protected async handleNonStreamingRequest(openai: OpenAI, request: LanguageModelRequest): Promise<LanguageModelTextResponse> {
+        const settings = this.getSettings(request);
         const response = await openai.chat.completions.create({
             model: this.model,
-            messages: request.messages.map(toOpenAIMessage),
-            ...request.settings
+            messages: request.messages.map(this.toOpenAIMessage.bind(this)),
+            ...settings
         });
 
         const message = response.choices[0].message;
@@ -152,26 +178,36 @@ export class OpenAiModel implements LanguageModel {
         };
     }
 
-    protected isNonStreamingModel(model: string): boolean {
-        return ['o1-preview'].includes(model);
+    protected toOpenAIMessage(message: LanguageModelRequestMessage): ChatCompletionMessageParam {
+        return {
+            role: this.toOpenAiRole(message),
+            content: message.query || ''
+        };
+    }
+
+    protected toOpenAiRole(message: LanguageModelRequestMessage): 'developer' | 'user' | 'assistant' {
+        switch (message.actor) {
+            case 'system':
+                return this.supportsDeveloperMessage ? 'developer' : 'user';
+            case 'ai':
+                return 'assistant';
+            default:
+                return 'user';
+        }
     }
 
-    protected supportsStructuredOutput(): boolean {
-        // see https://platform.openai.com/docs/models/gpt-4o
-        return [
-            'gpt-4o',
-            'gpt-4o-2024-08-06',
-            'gpt-4o-mini'
-        ].includes(this.model);
+    protected isNonStreamingModel(_model: string): boolean {
+        return !this.enableStreaming;
     }
 
     protected async handleStructuredOutputRequest(openai: OpenAI, request: LanguageModelRequest): Promise<LanguageModelParsedResponse> {
+        const settings = this.getSettings(request);
         // TODO implement tool support for structured output (parse() seems to require different tool format)
         const result = await openai.beta.chat.completions.parse({
             model: this.model,
-            messages: request.messages.map(toOpenAIMessage),
+            messages: request.messages.map(this.toOpenAIMessage.bind(this)),
             response_format: request.response_format,
-            ...request.settings
+            ...settings
         });
         const message = result.choices[0].message;
         if (message.refusal || message.parsed === undefined) {
@@ -207,7 +243,14 @@ export class OpenAiModel implements LanguageModel {
         if (!apiKey && !(this.url)) {
             throw new Error('Please provide OPENAI_API_KEY in preferences or via environment variable');
         }
-        // We need to hand over "some" key, even if a custom url is not key protected as otherwise the OpenAI client will throw an error
-        return new OpenAI({ apiKey: apiKey ?? 'no-key', baseURL: this.url });
+
+        const apiVersion = this.apiVersion();
+        if (apiVersion) {
+            // We need to hand over "some" key, even if a custom url is not key protected as otherwise the OpenAI client will throw an error
+            return new AzureOpenAI({ apiKey: apiKey ?? 'no-key', baseURL: this.url, apiVersion: apiVersion });
+        } else {
+            // We need to hand over "some" key, even if a custom url is not key protected as otherwise the OpenAI client will throw an error
+            return new OpenAI({ apiKey: apiKey ?? 'no-key', baseURL: this.url });
+        }
     }
 }

package/src/node/openai-language-models-manager-impl.ts

@@ -23,6 +23,7 @@ import { OpenAiLanguageModelsManager, OpenAiModelDescription } from '../common';
 export class OpenAiLanguageModelsManagerImpl implements OpenAiLanguageModelsManager {
 
     protected _apiKey: string | undefined;
+    protected _apiVersion: string | undefined;
 
     @inject(LanguageModelRegistry)
     protected readonly languageModelRegistry: LanguageModelRegistry;
@@ -31,6 +32,10 @@ export class OpenAiLanguageModelsManagerImpl implements OpenAiLanguageModelsMana
         return this._apiKey ?? process.env.OPENAI_API_KEY;
     }
 
+    get apiVersion(): string | undefined {
+        return this._apiVersion ?? process.env.OPENAI_API_VERSION;
+    }
+
     // Triggered from frontend. In case you want to use the models on the backend
     // without a frontend then call this yourself
     async createOrUpdateLanguageModels(...modelDescriptions: OpenAiModelDescription[]): Promise<void> {
@@ -45,21 +50,43 @@ export class OpenAiLanguageModelsManagerImpl implements OpenAiLanguageModelsMana
                 }
                 return undefined;
             };
+            const apiVersionProvider = () => {
+                if (modelDescription.apiVersion === true) {
+                    return this.apiVersion;
+                }
+                if (modelDescription.apiVersion) {
+                    return modelDescription.apiVersion;
+                }
+                return undefined;
+            };
+
             if (model) {
                 if (!(model instanceof OpenAiModel)) {
-                    console.warn(`Open AI: model ${modelDescription.id} is not an OpenAI model`);
+                    console.warn(`OpenAI: model ${modelDescription.id} is not an OpenAI model`);
                     continue;
                 }
-                if (!modelDescription.url) {
-                    // This seems to be an official model, but it was already created. This can happen during the initializing of more than one frontend.
-                    console.info(`Open AI: skip creating model ${modelDescription.id} because it already exists`);
-                    continue;
-                }
-                model.url = modelDescription.url;
                 model.model = modelDescription.model;
+                model.enableStreaming = modelDescription.enableStreaming;
+                model.url = modelDescription.url;
                 model.apiKey = apiKeyProvider;
+                model.apiVersion = apiVersionProvider;
+                model.supportsDeveloperMessage = modelDescription.supportsDeveloperMessage;
+                model.supportsStructuredOutput = modelDescription.supportsStructuredOutput;
+                model.defaultRequestSettings = modelDescription.defaultRequestSettings;
             } else {
-                this.languageModelRegistry.addLanguageModels([new OpenAiModel(modelDescription.id, modelDescription.model, apiKeyProvider, modelDescription.url)]);
+                this.languageModelRegistry.addLanguageModels([
+                    new OpenAiModel(
+                        modelDescription.id,
+                        modelDescription.model,
+                        modelDescription.enableStreaming,
+                        apiKeyProvider,
+                        apiVersionProvider,
+                        modelDescription.supportsDeveloperMessage,
+                        modelDescription.supportsStructuredOutput,
+                        modelDescription.url,
+                        modelDescription.defaultRequestSettings
+                    )
+                ]);
             }
         }
     }
@@ -75,4 +102,12 @@ export class OpenAiLanguageModelsManagerImpl implements OpenAiLanguageModelsMana
             this._apiKey = undefined;
         }
     }
+
+    setApiVersion(apiVersion: string | undefined): void {
+        if (apiVersion) {
+            this._apiVersion = apiVersion;
+        } else {
+            this._apiVersion = undefined;
+        }
+    }
 }