@jupyterlite/ai 0.7.0 → 0.8.0

package/lib/default-providers/OpenAI/settings-schema.json

@@ -42,10 +42,7 @@
                                                     "description": "The type of the content part."
                                                 }
                                             },
-                                            "required": [
-                                                "text",
-                                                "type"
-                                            ],
+                                            "required": ["text", "type"],
                                             "additionalProperties": false,
                                             "description": "Learn about [text inputs](https://platform.openai.com/docs/guides/text-generation)."
                                         }
@@ -63,10 +60,7 @@
                                 "description": "An optional name for the participant. Provides the model information to differentiate between participants of the same role."
                             }
                         },
-                        "required": [
-                            "content",
-                            "role"
-                        ],
+                        "required": ["content", "role"],
                         "additionalProperties": false,
                         "description": "Developer-provided instructions that the model should follow, regardless of messages sent by the user. With o1 models and newer, `developer` messages replace the previous `system` messages."
                     },
@@ -93,10 +87,7 @@
                                                     "description": "The type of the content part."
                                                 }
                                             },
-                                            "required": [
-                                                "text",
-                                                "type"
-                                            ],
+                                            "required": ["text", "type"],
                                             "additionalProperties": false,
                                             "description": "Learn about [text inputs](https://platform.openai.com/docs/guides/text-generation)."
                                         }
@@ -114,10 +105,7 @@
                                 "description": "An optional name for the participant. Provides the model information to differentiate between participants of the same role."
                             }
                         },
-                        "required": [
-                            "content",
-                            "role"
-                        ],
+                        "required": ["content", "role"],
                         "additionalProperties": false,
                         "description": "Developer-provided instructions that the model should follow, regardless of messages sent by the user. With o1 models and newer, use `developer` messages for this purpose instead."
                     },
@@ -146,10 +134,7 @@
                                                             "description": "The type of the content part."
                                                         }
                                                     },
-                                                    "required": [
-                                                        "text",
-                                                        "type"
-                                                    ],
+                                                    "required": ["text", "type"],
                                                     "additionalProperties": false,
                                                     "description": "Learn about [text inputs](https://platform.openai.com/docs/guides/text-generation)."
                                                 },
@@ -165,17 +150,11 @@
                                                                 },
                                                                 "detail": {
                                                                     "type": "string",
-                                                                    "enum": [
-                                                                        "auto",
-                                                                        "low",
-                                                                        "high"
-                                                                    ],
+                                                                    "enum": ["auto", "low", "high"],
                                                                     "description": "Specifies the detail level of the image. Learn more in the [Vision guide](https://platform.openai.com/docs/guides/vision#low-or-high-fidelity-image-understanding)."
                                                                 }
                                                             },
-                                                            "required": [
-                                                                "url"
-                                                            ],
+                                                            "required": ["url"],
                                                             "additionalProperties": false
                                                         },
                                                         "type": {
@@ -184,10 +163,7 @@
                                                             "description": "The type of the content part."
                                                         }
                                                     },
-                                                    "required": [
-                                                        "image_url",
-                                                        "type"
-                                                    ],
+                                                    "required": ["image_url", "type"],
                                                     "additionalProperties": false,
                                                     "description": "Learn about [image inputs](https://platform.openai.com/docs/guides/vision)."
                                                 },
@@ -203,17 +179,11 @@
                                                                 },
                                                                 "format": {
                                                                     "type": "string",
-                                                                    "enum": [
-                                                                        "wav",
-                                                                        "mp3"
-                                                                    ],
+                                                                    "enum": ["wav", "mp3"],
                                                                     "description": "The format of the encoded audio data. Currently supports \"wav\" and \"mp3\"."
                                                                 }
                                                             },
-                                                            "required": [
-                                                                "data",
-                                                                "format"
-                                                            ],
+                                                            "required": ["data", "format"],
                                                             "additionalProperties": false
                                                         },
                                                         "type": {
@@ -222,12 +192,40 @@
                                                             "description": "The type of the content part. Always `input_audio`."
                                                         }
                                                     },
-                                                    "required": [
-                                                        "input_audio",
-                                                        "type"
-                                                    ],
+                                                    "required": ["input_audio", "type"],
                                                     "additionalProperties": false,
                                                     "description": "Learn about [audio inputs](https://platform.openai.com/docs/guides/audio)."
+                                                },
+                                                {
+                                                    "type": "object",
+                                                    "properties": {
+                                                        "file": {
+                                                            "type": "object",
+                                                            "properties": {
+                                                                "file_data": {
+                                                                    "type": "string",
+                                                                    "description": "The base64 encoded file data, used when passing the file to the model as a string."
+                                                                },
+                                                                "file_id": {
+                                                                    "type": "string",
+                                                                    "description": "The ID of an uploaded file to use as input."
+                                                                },
+                                                                "filename": {
+                                                                    "type": "string",
+                                                                    "description": "The name of the file, used when passing the file to the model as a string."
+                                                                }
+                                                            },
+                                                            "additionalProperties": false
+                                                        },
+                                                        "type": {
+                                                            "type": "string",
+                                                            "const": "file",
+                                                            "description": "The type of the content part. Always `file`."
+                                                        }
+                                                    },
+                                                    "required": ["file", "type"],
+                                                    "additionalProperties": false,
+                                                    "description": "Learn about [file inputs](https://platform.openai.com/docs/guides/text) for text generation."
                                                 }
                                             ],
                                             "description": "Learn about [text inputs](https://platform.openai.com/docs/guides/text-generation)."
@@ -246,10 +244,7 @@
                                 "description": "An optional name for the participant. Provides the model information to differentiate between participants of the same role."
                             }
                         },
-                        "required": [
-                            "content",
-                            "role"
-                        ],
+                        "required": ["content", "role"],
                         "additionalProperties": false,
                         "description": "Messages sent by an end user, containing prompts or additional context information."
                     },
@@ -271,9 +266,7 @@
                                                 "description": "Unique identifier for a previous audio response from the model."
                                             }
                                         },
-                                        "required": [
-                                            "id"
-                                        ],
+                                        "required": ["id"],
                                         "additionalProperties": false,
                                         "description": "Data about a previous audio response from the model. [Learn more](https://platform.openai.com/docs/guides/audio)."
                                     },
@@ -305,10 +298,7 @@
                                                             "description": "The type of the content part."
                                                         }
                                                     },
-                                                    "required": [
-                                                        "text",
-                                                        "type"
-                                                    ],
+                                                    "required": ["text", "type"],
                                                     "additionalProperties": false,
                                                     "description": "Learn about [text inputs](https://platform.openai.com/docs/guides/text-generation)."
                                                 },
@@ -325,10 +315,7 @@
                                                             "description": "The type of the content part."
                                                         }
                                                     },
-                                                    "required": [
-                                                        "refusal",
-                                                        "type"
-                                                    ],
+                                                    "required": ["refusal", "type"],
                                                     "additionalProperties": false
                                                 }
                                             ]
@@ -354,10 +341,7 @@
                                                 "description": "The name of the function to call."
                                             }
                                         },
-                                        "required": [
-                                            "arguments",
-                                            "name"
-                                        ],
+                                        "required": ["arguments", "name"],
                                         "additionalProperties": false,
                                         "deprecated": "Deprecated and replaced by `tool_calls`. The name and arguments of a\nfunction that should be called, as generated by the model."
                                     },
@@ -372,10 +356,7 @@
                                 "description": "An optional name for the participant. Provides the model information to differentiate between participants of the same role."
                             },
                             "refusal": {
-                                "type": [
-                                    "string",
-                                    "null"
-                                ],
+                                "type": ["string", "null"],
                                 "description": "The refusal message by the assistant."
                             },
                             "tool_calls": {
@@ -399,10 +380,7 @@
                                                     "description": "The name of the function to call."
                                                 }
                                             },
-                                            "required": [
-                                                "arguments",
-                                                "name"
-                                            ],
+                                            "required": ["arguments", "name"],
                                             "additionalProperties": false,
                                             "description": "The function that the model called."
                                         },
@@ -412,19 +390,13 @@
                                             "description": "The type of the tool. Currently, only `function` is supported."
                                         }
                                     },
-                                    "required": [
-                                        "id",
-                                        "function",
-                                        "type"
-                                    ],
+                                    "required": ["id", "function", "type"],
                                     "additionalProperties": false
                                 },
                                 "description": "The tool calls generated by the model, such as function calls."
                             }
                         },
-                        "required": [
-                            "role"
-                        ],
+                        "required": ["role"],
                         "additionalProperties": false,
                         "description": "Messages sent by the model in response to user messages."
                     },
@@ -451,10 +423,7 @@
                                                     "description": "The type of the content part."
                                                 }
                                             },
-                                            "required": [
-                                                "text",
-                                                "type"
-                                            ],
+                                            "required": ["text", "type"],
                                             "additionalProperties": false,
                                             "description": "Learn about [text inputs](https://platform.openai.com/docs/guides/text-generation)."
                                         }
@@ -472,21 +441,14 @@
                                 "description": "Tool call that this message is responding to."
                             }
                         },
-                        "required": [
-                            "content",
-                            "role",
-                            "tool_call_id"
-                        ],
+                        "required": ["content", "role", "tool_call_id"],
                         "additionalProperties": false
                     },
                     {
                         "type": "object",
                         "properties": {
                             "content": {
-                                "type": [
-                                    "string",
-                                    "null"
-                                ],
+                                "type": ["string", "null"],
                                 "description": "The contents of the function message."
                             },
                             "name": {
@@ -499,11 +461,7 @@
                                 "description": "The role of the messages author, in this case `function`."
                             }
                         },
-                        "required": [
-                            "content",
-                            "name",
-                            "role"
-                        ],
+                        "required": ["content", "name", "role"],
                         "additionalProperties": false,
                         "deprecated": true
                     }
@@ -524,10 +482,7 @@
             "type": "array",
             "items": {
                 "type": "string",
-                "enum": [
-                    "text",
-                    "audio"
-                ]
+                "enum": ["text", "audio"]
             },
             "description": "Output types that you would like the model to generate for this request. Most models are capable of generating text, which is the default:\n\n`[\"text\"]`\n\nThe `gpt-4o-audio-preview` model can also be used to [generate audio](https://platform.openai.com/docs/guides/audio). To request that this model generate both text and audio responses, you can use:\n\n`[\"text\", \"audio\"]`"
         },
@@ -536,43 +491,48 @@
             "properties": {
                 "format": {
                     "type": "string",
-                    "enum": [
-                        "wav",
-                        "mp3",
-                        "flac",
-                        "opus",
-                        "pcm16"
-                    ],
+                    "enum": ["wav", "aac", "mp3", "flac", "opus", "pcm16"],
                     "description": "Specifies the output audio format. Must be one of `wav`, `mp3`, `flac`, `opus`, or `pcm16`."
                 },
                 "voice": {
-                    "type": "string",
-                    "enum": [
-                        "alloy",
-                        "ash",
-                        "ballad",
-                        "coral",
-                        "echo",
-                        "sage",
-                        "shimmer",
-                        "verse"
+                    "anyOf": [
+                        {
+                            "type": "string"
+                        },
+                        {
+                            "type": "string",
+                            "enum": [
+                                "alloy",
+                                "ash",
+                                "ballad",
+                                "coral",
+                                "echo",
+                                "fable",
+                                "onyx",
+                                "nova",
+                                "sage",
+                                "shimmer",
+                                "verse"
+                            ]
+                        }
                     ],
-                    "description": "The voice the model uses to respond. Supported voices are `ash`, `ballad`, `coral`, `sage`, and `verse` (also supported but not recommended are `alloy`, `echo`, and `shimmer`; these voices are less expressive)."
+                    "description": "The voice the model uses to respond. Supported voices are `alloy`, `ash`, `ballad`, `coral`, `echo`, `fable`, `nova`, `onyx`, `sage`, and `shimmer`."
                 }
             },
-            "required": [
-                "format",
-                "voice"
-            ],
+            "required": ["format", "voice"],
             "additionalProperties": false,
             "description": "Parameters for audio output. Required when audio output is requested with `modalities: [\"audio\"]`. [Learn more](https://platform.openai.com/docs/guides/audio)."
         },
         "reasoningEffort": {
-            "type": "string",
-            "enum": [
-                "low",
-                "medium",
-                "high"
+            "anyOf": [
+                {
+                    "type": ["string", "null"],
+                    "enum": ["low", "medium", "high", null],
+                    "description": "**o-series models only**\n\nConstrains effort on reasoning for [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently supported values are `low`, `medium`, and `high`. Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning in a response."
+                },
+                {
+                    "type": "null"
+                }
             ],
             "description": "Constrains effort on reasoning for reasoning models. Currently supported values are low, medium, and high. Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning in a response."
         },

package/lib/default-providers/WebLLM/completer.d.ts

@@ -1,25 +1,19 @@
 import { CompletionHandler, IInlineCompletionContext } from '@jupyterlab/completer';
 import { ChatWebLLM } from '@langchain/community/chat_models/webllm';
-import { BaseCompleter, IBaseCompleter } from '../../base-completer';
-export declare class WebLLMCompleter implements IBaseCompleter {
+import { BaseCompleter } from '../../base-completer';
+export declare class WebLLMCompleter extends BaseCompleter {
     constructor(options: BaseCompleter.IOptions);
     /**
      * Initialize the WebLLM model
      */
     private _initializeModel;
-    /**
-     * Getter and setter for the initial prompt.
-     */
-    get prompt(): string;
-    set prompt(value: string);
     get provider(): ChatWebLLM;
     fetch(request: CompletionHandler.IRequest, context: IInlineCompletionContext): Promise<{
         items: {
             insertText: string;
         }[];
     }>;
-    private _completer;
-    private _prompt;
+    protected _completer: ChatWebLLM;
     private _isInitialized;
     private _isInitializing;
     private _initError;

package/lib/default-providers/WebLLM/completer.js

@@ -1,6 +1,6 @@
 import { HumanMessage, SystemMessage } from '@langchain/core/messages';
 import { ChatWebLLM } from '@langchain/community/chat_models/webllm';
-import { COMPLETION_SYSTEM_PROMPT } from '../../provider';
+import { BaseCompleter } from '../../base-completer';
 /**
  * Regular expression to match the '```' string at the start of a string.
  * So the completions returned by the LLM can still be kept after removing the code block formatting.
@@ -20,8 +20,9 @@ const CODE_BLOCK_START_REGEX = /^```(?:[a-zA-Z]+)?\n?/;
  * Regular expression to match the '```' string at the end of a string.
  */
 const CODE_BLOCK_END_REGEX = /```$/;
-export class WebLLMCompleter {
+export class WebLLMCompleter extends BaseCompleter {
     constructor(options) {
+        super(options);
         const model = options.settings.model;
         // provide model separately since ChatWebLLM expects it
         this._completer = new ChatWebLLM({
@@ -57,15 +58,6 @@ export class WebLLMCompleter {
             console.error('Failed to initialize WebLLM model:', error);
         }
     }
-    /**
-     * Getter and setter for the initial prompt.
-     */
-    get prompt() {
-        return this._prompt;
-    }
-    set prompt(value) {
-        this._prompt = value;
-    }
     get provider() {
         return this._completer;
     }
@@ -99,7 +91,7 @@ export class WebLLMCompleter {
         const prompt = text.slice(0, cursorOffset);
         const trimmedPrompt = prompt.trim();
         const messages = [
-            new SystemMessage(this._prompt),
+            new SystemMessage(this.systemPrompt),
             new HumanMessage(trimmedPrompt)
         ];
         try {
@@ -128,7 +120,6 @@ export class WebLLMCompleter {
         }
     }
     _completer;
-    _prompt = COMPLETION_SYSTEM_PROMPT;
     _isInitialized = false;
     _isInitializing = false;
     _initError = null;

package/lib/default-providers/WebLLM/settings-schema.json

@@ -13,9 +13,7 @@
             "type": "string"
         }
     },
-    "required": [
-        "model"
-    ],
+    "required": ["model"],
     "additionalProperties": false,
     "definitions": {}
 }

package/lib/default-providers/index.js

@@ -2,12 +2,14 @@ import { Notification } from '@jupyterlab/apputils';
 import { ChatAnthropic } from '@langchain/anthropic';
 import { ChatWebLLM } from '@langchain/community/chat_models/webllm';
 import { ChromeAI } from '@langchain/community/experimental/llms/chrome_ai';
+import { ChatGoogleGenerativeAI } from '@langchain/google-genai';
 import { ChatMistralAI } from '@langchain/mistralai';
 import { ChatOllama } from '@langchain/ollama';
 import { ChatOpenAI } from '@langchain/openai';
 // Import completers
 import { AnthropicCompleter } from './Anthropic/completer';
 import { ChromeCompleter } from './ChromeAI/completer';
+import { GeminiCompleter } from './Gemini/completer';
 import { CodestralCompleter } from './MistralAI/completer';
 import { OllamaCompleter } from './Ollama/completer';
 import { OpenAICompleter } from './OpenAI/completer';
@@ -15,12 +17,14 @@ import { WebLLMCompleter } from './WebLLM/completer';
 // Import Settings
 import AnthropicSettings from './Anthropic/settings-schema.json';
 import ChromeAISettings from './ChromeAI/settings-schema.json';
+import GeminiSettings from './Gemini/settings-schema.json';
 import MistralAISettings from './MistralAI/settings-schema.json';
 import OllamaAISettings from './Ollama/settings-schema.json';
 import OpenAISettings from './OpenAI/settings-schema.json';
 import WebLLMSettings from './WebLLM/settings-schema.json';
 // Import instructions
 import ChromeAIInstructions, { compatibilityCheck as chromeAICompatibilityCheck } from './ChromeAI/instructions';
+import GeminiInstructions from './Gemini/instructions';
 import MistralAIInstructions from './MistralAI/instructions';
 import OllamaInstructions from './Ollama/instructions';
 import WebLLMInstructions, { compatibilityCheck as webLLMCompatibilityCheck } from './WebLLM/instructions';
@@ -30,7 +34,7 @@ import { IAIProviderRegistry } from '../tokens';
 const AIProviders = [
     {
         name: 'Anthropic',
-        chatModel: ChatAnthropic,
+        chat: ChatAnthropic,
         completer: AnthropicCompleter,
         settingsSchema: AnthropicSettings,
         errorMessage: (error) => error.error.error.message
@@ -39,7 +43,7 @@ const AIProviders = [
         name: 'ChromeAI',
         // TODO: fix
         // @ts-expect-error: missing properties
-        chatModel: ChromeAI,
+        chat: ChromeAI,
         completer: ChromeCompleter,
         instructions: ChromeAIInstructions,
         settingsSchema: ChromeAISettings,
@@ -47,21 +51,28 @@ const AIProviders = [
     },
     {
         name: 'MistralAI',
-        chatModel: ChatMistralAI,
+        chat: ChatMistralAI,
         completer: CodestralCompleter,
         instructions: MistralAIInstructions,
         settingsSchema: MistralAISettings
     },
     {
         name: 'Ollama',
-        chatModel: ChatOllama,
+        chat: ChatOllama,
         completer: OllamaCompleter,
         instructions: OllamaInstructions,
         settingsSchema: OllamaAISettings
     },
+    {
+        name: 'Gemini',
+        chat: ChatGoogleGenerativeAI,
+        completer: GeminiCompleter,
+        instructions: GeminiInstructions,
+        settingsSchema: GeminiSettings
+    },
     {
         name: 'OpenAI',
-        chatModel: ChatOpenAI,
+        chat: ChatOpenAI,
         completer: OpenAICompleter,
         settingsSchema: OpenAISettings
     }
@@ -78,30 +89,23 @@ const webLLMProviderPlugin = {
     activate: (app, registry) => {
         registry.add({
             name: 'WebLLM',
-            chatModel: ChatWebLLM,
+            chat: ChatWebLLM,
             completer: WebLLMCompleter,
             settingsSchema: WebLLMSettings,
             instructions: WebLLMInstructions,
             compatibilityCheck: webLLMCompatibilityCheck,
             exposeChatModel: true
         });
-        registry.providerChanged.connect(async (sender, args) => {
-            const { currentName, currentChatModel, chatError } = registry;
-            if (currentChatModel === null) {
-                Notification.emit(chatError, 'error', {
-                    autoClose: 2000
-                });
-                return;
-            }
+        registry.providerChanged.connect(async (sender, role) => {
+            const { currentChatModel } = registry;
             // TODO: implement a proper way to handle models that may need to be initialized before being used.
             // Mostly applies to WebLLM and ChromeAI as they may need to download the model in the browser first.
-            if (currentName === 'WebLLM') {
+            if (registry.currentName(role) === 'WebLLM') {
+                // Leaving this check here, but it should never happen, this check is done in
+                // the provider registry, and the current name is set to 'None' if there is a
+                // compatibility error.
                 const compatibilityError = await webLLMCompatibilityCheck();
                 if (compatibilityError) {
-                    Notification.dismiss();
-                    Notification.emit(compatibilityError, 'error', {
-                        autoClose: 2000
-                    });
                     return;
                 }
                 const model = currentChatModel;

package/lib/index.d.ts

@@ -4,3 +4,4 @@ import { IAIProviderRegistry } from './tokens';
 declare const _default: (JupyterFrontEndPlugin<void> | JupyterFrontEndPlugin<IChatCommandRegistry> | JupyterFrontEndPlugin<IAIProviderRegistry>)[];
 export default _default;
 export { IAIProviderRegistry } from './tokens';
+export * from './base-completer';