@jupyterlite/ai 0.17.0 → 0.18.0

package/lib/chat-model.js

@@ -5,6 +5,7 @@ import { UUID } from '@lumino/coreutils';
 import { Debouncer } from '@lumino/polling';
 import { Signal } from '@lumino/signaling';
 import { AI_AVATAR } from './icons';
+import { modelSupportsAudio, modelSupportsImages, modelSupportsPdf } from './providers/model-info';
 /**
  * AI Chat Model implementation that provides chat functionality tool integration,
  * and MCP server support.
@@ -27,10 +28,14 @@ export class AIChatModel extends AbstractChatModel {
         this._user = options.user;
         this._agentManager = options.agentManager;
         this._contentsManager = options.contentsManager;
+        this._providerRegistry = options.providerRegistry;
         // Listen for agent events
         this._agentManager.agentEvent.connect(this._onAgentEvent, this);
         // Listen for settings changes to update chat behavior
         this._settingsModel.stateChanged.connect(this._onSettingsChanged, this);
+        // Rebuild history when the model changes
+        this._agentManager.activeProviderChanged.connect(this._onModelChanged, this);
+        this._settingsModel.stateChanged.connect(this._onModelChanged, this);
         this._autosaveDebouncer = new Debouncer(this.save, 3000);
     }
     /**
@@ -81,17 +86,20 @@ export class AIChatModel extends AbstractChatModel {
         return this._autosave;
     }
     set autosave(value) {
+        if (value === this._autosave) {
+            return;
+        }
         this._autosave = value;
         this._autosaveChanged.emit(value);
         if (value) {
             this.messagesUpdated.connect(this._autosaveDebouncer.invoke, this._autosaveDebouncer);
             this.messageChanged.connect(this._autosaveDebouncer.invoke, this._autosaveDebouncer);
-            this._autosaveDebouncer.invoke();
         }
         else {
             this.messagesUpdated.disconnect(this._autosaveDebouncer.invoke, this._autosaveDebouncer);
             this.messageChanged.disconnect(this._autosaveDebouncer.invoke, this._autosaveDebouncer);
         }
+        this._autosaveDebouncer.invoke();
     }
     /**
      * A signal emitting when the autosave flag changed.
@@ -112,7 +120,7 @@ export class AIChatModel extends AbstractChatModel {
         return this._agentManager.tokenUsageChanged;
     }
     /**
-     * Get the agent manager associated to the model.
+     * The agent manager used in the model.
      */
     get agentManager() {
         return this._agentManager;
@@ -127,6 +135,7 @@ export class AIChatModel extends AbstractChatModel {
      * Dispose of the model.
      */
     dispose() {
+        this.stopStreaming();
         this.messagesUpdated.disconnect(this._autosaveDebouncer.invoke, this._autosaveDebouncer);
         super.dispose();
     }
@@ -142,7 +151,7 @@ export class AIChatModel extends AbstractChatModel {
             stopStreaming: () => this.stopStreaming(),
             clearMessages: () => this.clearMessages(),
             agentManager: this._agentManager,
-            addSystemMessage: (body) => this.addSystemMessage(body)
+            addSystemMessage: (body) => this._addSystemMessage(body)
         };
     }
     /**
@@ -155,14 +164,29 @@ export class AIChatModel extends AbstractChatModel {
      * Clears all messages from the chat and resets conversation state.
      */
     clearMessages = async () => {
+        this.stopStreaming();
+        this._messageQueue = [];
+        this._isBusy = false;
+        this._queueMessageId = null;
+        this._currentStreamingMessage = null;
         this.messagesDeleted(0, this.messages.length);
+        this.title = null;
         this._toolContexts.clear();
         await this._agentManager.clearHistory();
     };
+    /**
+     * Overrides messageAdded to ensure queued messages stay at the bottom.
+     */
+    messageAdded(message) {
+        super.messageAdded(message);
+        if (this._queueMessageId && message.id !== this._queueMessageId) {
+            this._updateQueueUI();
+        }
+    }
     /**
      * Adds a non-user message to the chat (used by chat commands).
      */
-    addSystemMessage(body) {
+    _addSystemMessage(body) {
         const message = {
             body,
             sender: this._getAIUser(),
@@ -193,9 +217,9 @@ export class AIChatModel extends AbstractChatModel {
             raw_time: false,
             attachments: [...this.input.attachments]
         };
-        this.messageAdded(userMessage);
         // Check if we have valid configuration
         if (!this._agentManager.hasValidConfig()) {
+            this.messageAdded(userMessage);
             const errorMessage = {
                 body: 'Please configure your AI settings first. Open the AI Settings to set your API key and model.',
                 sender: this._getAIUser(),
@@ -207,30 +231,48 @@ export class AIChatModel extends AbstractChatModel {
             this.messageAdded(errorMessage);
             return;
         }
+        if (this._isBusy) {
+            this._messageQueue.push({
+                id: UUID.uuid4(),
+                body: message.body,
+                _originalMsg: userMessage
+            });
+            this.input.clearAttachments();
+            this._updateQueueUI();
+            return;
+        }
+        this._isBusy = true;
+        this.messageAdded(userMessage);
+        this.input.clearAttachments();
+        await this._processMessage(userMessage);
+    }
+    /**
+     * Internal method to process attachments and send the message to the agent.
+     */
+    async _processMessage(userMessage) {
         try {
-            // Process attachments and add their content to the message
-            let enhancedMessage = message.body;
-            if (this.input.attachments.length > 0) {
-                const { textContents, binaryParts } = await Private.processAttachments(this.input.attachments, this.input.documentManager);
-                this.input.clearAttachments();
-                let textPart = message.body;
-                if (textContents.length > 0) {
-                    textPart +=
-                        '\n\n--- Attached Files ---\n' + textContents.join('\n\n');
-                }
-                if (binaryParts.length > 0) {
-                    enhancedMessage = [{ type: 'text', text: textPart }, ...binaryParts];
-                }
-                else {
-                    enhancedMessage = textPart;
-                }
-            }
             this.updateWriters([{ user: this._getAIUser() }]);
+            let enhancedMessage = userMessage.body;
+            if (userMessage.attachments && userMessage.attachments.length > 0) {
+                const providerConfig = this._settingsModel.getProvider(this._agentManager.activeProvider);
+                const supportsImages = modelSupportsImages(providerConfig, this._providerRegistry);
+                const supportsPdf = modelSupportsPdf(providerConfig, this._providerRegistry);
+                const supportsAudio = modelSupportsAudio(providerConfig, this._providerRegistry);
+                enhancedMessage = await Private.processAttachments(userMessage.attachments, this.input.documentManager, userMessage.body, supportsImages, supportsPdf, supportsAudio);
+            }
             await this._agentManager.generateResponse(enhancedMessage);
         }
         catch (error) {
             const errorMessage = {
-                body: `Error generating AI response: ${error.message}`,
+                body: '',
+                mime_model: {
+                    data: {
+                        'application/vnd.jupyter.chat.components': 'error'
+                    },
+                    metadata: {
+                        errorMessage: `Error generating AI response: ${error.message}`
+                    }
+                },
                 sender: this._getAIUser(),
                 id: UUID.uuid4(),
                 time: Date.now() / 1000,
@@ -240,8 +282,89 @@ export class AIChatModel extends AbstractChatModel {
             this.messageAdded(errorMessage);
         }
         finally {
+            this._drainQueue();
+            if (this._settingsModel.config.autoTitle &&
+                (this.messages.length <= 5 || this.title === null)) {
+                try {
+                    this.title = await this.requestTitle();
+                }
+                catch (e) {
+                    console.warn('Error while generating a title\n', e);
+                }
+            }
+        }
+    }
+    /**
+     * Removes the message-queue chat component.
+     */
+    _removeQueueUI() {
+        if (this._queueMessageId) {
+            const existingMsg = this.messages.find(msg => msg.id === this._queueMessageId);
+            if (existingMsg) {
+                const idx = this.messages.indexOf(existingMsg);
+                if (idx !== -1) {
+                    this.messagesDeleted(idx, 1);
+                }
+            }
+            this._queueMessageId = null;
+        }
+    }
+    /**
+     * Creates or updates the message-queue chat component.
+     */
+    _updateQueueUI() {
+        this._removeQueueUI();
+        if (this._messageQueue.length === 0) {
+            return;
+        }
+        const queueBody = {
+            data: {
+                'application/vnd.jupyter.chat.components': 'message-queue'
+            },
+            metadata: {
+                messages: this._messageQueue.map(m => ({
+                    id: m.id,
+                    body: m.body,
+                    attachments: m._originalMsg.attachments
+                })),
+                targetId: this.name
+            }
+        };
+        this._queueMessageId = UUID.uuid4();
+        const queueMessage = {
+            body: '',
+            mime_model: queueBody,
+            sender: { username: 'system', display_name: '' },
+            id: this._queueMessageId,
+            time: Date.now() / 1000,
+            type: 'msg',
+            raw_time: false
+        };
+        this.messageAdded(queueMessage);
+    }
+    /**
+     * Processes the next message in the queue, or marks the agent as idle.
+     */
+    async _drainQueue() {
+        if (this._messageQueue.length === 0) {
+            this._isBusy = false;
             this.updateWriters([]);
+            this._removeQueueUI();
+            return;
         }
+        // Dequeue and push to chat
+        const next = this._messageQueue.shift();
+        next._originalMsg.time = Date.now() / 1000;
+        this.messageAdded(next._originalMsg);
+        await this._processMessage(next._originalMsg);
+    }
+    /**
+     * Removes a queued message by its ID.
+     * @param messageId The ID of the queued message to remove
+     */
+    removeQueuedMessage(messageId) {
+        this._messageQueue = this._messageQueue.filter(msg => msg.id !== messageId);
+        this._updateQueueUI();
     }
     /**
      * Save the chat as json file.
@@ -327,7 +450,7 @@ export class AIChatModel extends AbstractChatModel {
         });
         await this.clearMessages();
         this.messagesInserted(0, messages);
-        this._agentManager.setHistory(messages);
+        await this._rebuildHistory();
         this.autosave = content.metadata?.autosave ?? false;
         this.title = content.metadata?.title ?? null;
         return true;
@@ -343,7 +466,7 @@ export class AIChatModel extends AbstractChatModel {
         const messages = [
             {
                 role: 'system',
-                content: "Generate a concise title (no more than 10 words) for the following conversation. Do not use formatting. Focus on the user's main intent."
+                content: "Generate a concise title (no more than 10 words) for the following conversation. Do not use formatting, quotes, or punctuation. Focus on the subject matter and specific content the user is working on, not on the actions taken (e.g. prefer 'Pandas DataFrame filtering' over 'Opening a notebook'). The title should be a noun phrase describing the topic."
             },
             {
                 role: 'user',
@@ -362,6 +485,9 @@ export class AIChatModel extends AbstractChatModel {
         const attachmentMap = new Map(); // JSON → index
         const attachmentsList = []; // Actual attachments
         this.messages.forEach(message => {
+            if (message.content?.mime_model?.data?.['application/vnd.jupyter.chat.components'] === 'message-queue') {
+                return;
+            }
             let attachmentIndexes = [];
             if (message.attachments) {
                 attachmentIndexes = message.attachments.map(attachment => {
@@ -420,6 +546,49 @@ export class AIChatModel extends AbstractChatModel {
         this.config = { ...config, enableCodeToolbar: true };
         // Agent manager handles agent recreation automatically via its own settings listener
     }
+    /**
+     * Rebuild history when the active model changes.
+     */
+    _onModelChanged() {
+        const providerConfig = this._settingsModel.getProvider(this._agentManager.activeProvider);
+        const modelKey = providerConfig
+            ? `${providerConfig.provider}:${providerConfig.model}`
+            : undefined;
+        if (modelKey && modelKey !== this._currentModelKey) {
+            this._currentModelKey = modelKey;
+            this._rebuildHistory().catch(e => console.warn('Failed to rebuild history on model change:', e));
+        }
+    }
+    /**
+     * Rebuilds the agent history from the current messages.
+     * For vision-capable models, re-reads binary attachments from disk.
+     * For text-only models, uses message text only.
+     */
+    async _rebuildHistory() {
+        const providerConfig = this._settingsModel.getProvider(this._agentManager.activeProvider);
+        const supportsImages = modelSupportsImages(providerConfig, this._providerRegistry);
+        const supportsPdf = modelSupportsPdf(providerConfig, this._providerRegistry);
+        const supportsAudio = modelSupportsAudio(providerConfig, this._providerRegistry);
+        const modelMessages = [];
+        for (const msg of this.messages) {
+            const isAI = msg.sender.username === 'ai-assistant';
+            if (!isAI && msg.attachments?.length) {
+                const enhancedContent = await Private.processAttachments(msg.attachments, this.input.documentManager, msg.body, supportsImages, supportsPdf, supportsAudio);
+                modelMessages.push({
+                    role: 'user',
+                    content: enhancedContent
+                });
+            }
+            else if (msg.body) {
+                modelMessages.push({
+                    role: isAI ? 'assistant' : 'user',
+                    content: msg.body
+                });
+            }
+            // Skip messages with empty body like tool calls
+        }
+        this._agentManager.setHistory(modelMessages);
+    }
     /**
      * Handles events emitted by the agent manager.
      * @param event The event data containing type and payload
@@ -583,13 +752,18 @@ export class AIChatModel extends AbstractChatModel {
             body: '',
             mime_model: {
                 data: {
-                    'application/vnd.jupyter.chat.components': 'tool-call'
+                    'application/vnd.jupyter.chat.components': 'grouped-tool-calls'
                 },
                 metadata: {
-                    toolName: context.toolName,
-                    input: context.input,
-                    status: context.status,
-                    summary: context.summary
+                    toolCalls: [
+                        {
+                            toolCallId: context.toolCallId,
+                            title: `${context.toolName}${context.summary ? ' : ' + context.summary : ''}`,
+                            kind: context.toolName,
+                            status: 'in_progress',
+                            rawInput: context.input
+                        }
+                    ]
                 }
             },
             sender: this._getAIUser(),
@@ -642,7 +816,15 @@ export class AIChatModel extends AbstractChatModel {
      */
     _handleErrorEvent(event) {
         this.messageAdded({
-            body: `Error generating response: ${event.data.error.message}`,
+            body: '',
+            mime_model: {
+                data: {
+                    'application/vnd.jupyter.chat.components': 'error'
+                },
+                metadata: {
+                    errorMessage: `Error generating response: ${event.data.error.message}`
+                }
+            },
             sender: this._getAIUser(),
             id: UUID.uuid4(),
             time: Date.now() / 1000,
@@ -658,7 +840,6 @@ export class AIChatModel extends AbstractChatModel {
         if (!context) {
             return;
         }
-        context.approvalId = event.data.approvalId;
         context.input = JSON.stringify(event.data.args, null, 2);
         this._updateToolCallUI(event.data.toolCallId, 'awaiting_approval');
     }
@@ -666,12 +847,12 @@ export class AIChatModel extends AbstractChatModel {
      * Handles tool approval resolved events from the AI agent.
      */
     _handleToolApprovalResolved(event) {
-        const context = Array.from(this._toolContexts.values()).find(ctx => ctx.approvalId === event.data.approvalId);
+        const context = this._toolContexts.get(event.data.toolCallId);
         if (!context) {
             return;
         }
         const status = event.data.approved ? 'approved' : 'rejected';
-        this._updateToolCallUI(context.toolCallId, status);
+        this._updateToolCallUI(event.data.toolCallId, status);
         if (!event.data.approved) {
             this._toolContexts.delete(context.toolCallId);
         }
@@ -692,31 +873,69 @@ export class AIChatModel extends AbstractChatModel {
         existingMessage.update({
             mime_model: {
                 data: {
-                    'application/vnd.jupyter.chat.components': 'tool-call'
+                    'application/vnd.jupyter.chat.components': 'grouped-tool-calls'
                 },
                 metadata: {
-                    toolName: context.toolName,
-                    input: context.input,
-                    status: context.status,
-                    summary: context.summary,
-                    output,
-                    targetId: this.name,
-                    approvalId: context.approvalId
+                    toolCalls: [
+                        {
+                            toolCallId: context.toolCallId,
+                            title: `${context.toolName}${context.summary ? ' : ' + context.summary : ''}`,
+                            kind: context.toolName,
+                            status: context.status,
+                            rawInput: context.input,
+                            rawOutput: output,
+                            sessionId: this.name,
+                            permissionStatus: status === 'awaiting_approval' ? 'pending' : 'resolved',
+                            ...(status === 'awaiting_approval' && {
+                                permissionOptions: [
+                                    { optionId: 'approve', name: 'Approve', kind: 'allow_once' },
+                                    { optionId: 'reject', name: 'Reject', kind: 'reject_once' }
+                                ]
+                            })
+                        }
+                    ]
                 }
             }
         });
     }
+    /**
+     * The current message queue
+     */
+    get messageQueue() {
+        return this._messageQueue;
+    }
+    set messageQueue(value) {
+        this._messageQueue = value;
+        this._updateQueueUI();
+        if (this._messageQueue.length > 0 && !this._isBusy) {
+            this._drainQueue();
+        }
+    }
+    /**
+     * Whether the chat is busy
+     */
+    get isBusy() {
+        return this._isBusy;
+    }
+    set isBusy(value) {
+        this._isBusy = value;
+    }
     // Private fields
     _settingsModel;
     _user;
     _toolContexts = new Map();
     _agentManager;
+    _providerRegistry;
+    _currentModelKey;
     _currentStreamingMessage = null;
     _nameChanged = new Signal(this);
     _contentsManager;
     _autosave = false;
     _autosaveChanged = new Signal(this);
     _autosaveDebouncer;
+    _messageQueue = [];
+    _isBusy = false;
+    _queueMessageId = null;
     _title = null;
     _titleChanged = new Signal(this);
 }
@@ -804,16 +1023,21 @@ var Private;
     }
     Private.formatToolOutput = formatToolOutput;
     /**
-     * Processes file attachments and returns text contents and binary parts separately.
+     * Processes file attachments and returns the message content with the attachments.
      * @param attachments Array of file attachments to process
      * @param documentManager Optional document manager for file operations
-     * @returns Text contents and binary parts
+     * @param body The message body
+     * @param supportsImages Whether the model supports images
+     * @param supportsPdf Whether the model supports pdfs
+     * @param supportsAudio Whether the model supports audio
+     * @returns Enhanced message content
      */
-    async function processAttachments(attachments, documentManager) {
+    async function processAttachments(attachments, documentManager, body, supportsImages, supportsPdf, supportsAudio) {
         const textContents = [];
-        const binaryParts = [];
+        const includedParts = [];
+        const omittedNames = [];
         if (!documentManager) {
-            return { textContents, binaryParts };
+            return body;
         }
         for (const attachment of attachments) {
             try {
@@ -837,24 +1061,50 @@ var Private;
                         }
                     }
                     if (mimetype?.startsWith('image/')) {
-                        const data = await readBinaryAttachment(attachment, documentManager);
-                        if (data) {
-                            binaryParts.push({
-                                type: 'image',
-                                image: data,
-                                mediaType: mimetype
-                            });
+                        if (supportsImages) {
+                            const data = await readBinaryAttachment(attachment, documentManager);
+                            if (data) {
+                                includedParts.push({
+                                    type: 'image',
+                                    image: data,
+                                    mediaType: mimetype
+                                });
+                            }
+                        }
+                        else {
+                            omittedNames.push(PathExt.basename(attachment.value));
                         }
                     }
                     else if (mimetype === 'application/pdf') {
-                        const data = await readBinaryAttachment(attachment, documentManager);
-                        if (data) {
-                            binaryParts.push({
-                                type: 'file',
-                                data,
-                                mediaType: mimetype,
-                                filename: PathExt.basename(attachment.value)
-                            });
+                        if (supportsPdf) {
+                            const data = await readBinaryAttachment(attachment, documentManager);
+                            if (data) {
+                                includedParts.push({
+                                    type: 'file',
+                                    data,
+                                    mediaType: mimetype,
+                                    filename: PathExt.basename(attachment.value)
+                                });
+                            }
+                        }
+                        else {
+                            omittedNames.push(PathExt.basename(attachment.value));
+                        }
+                    }
+                    else if (mimetype?.startsWith('audio/')) {
+                        if (supportsAudio) {
+                            const data = await readBinaryAttachment(attachment, documentManager);
+                            if (data) {
+                                includedParts.push({
+                                    type: 'file',
+                                    data,
+                                    mediaType: mimetype,
+                                    filename: PathExt.basename(attachment.value)
+                                });
+                            }
+                        }
+                        else {
+                            omittedNames.push(PathExt.basename(attachment.value));
                         }
                     }
                     else {
@@ -874,7 +1124,16 @@ var Private;
                 textContents.push(`**File: ${attachment.value}** (Could not read file)`);
             }
         }
-        return { textContents, binaryParts };
+        let textPart = body;
+        if (textContents.length > 0) {
+            textPart += '\n\n--- Attached Files ---\n' + textContents.join('\n\n');
+        }
+        if (omittedNames.length > 0) {
+            textPart += `\n[Attachments omitted (not supported by this model): ${omittedNames.join(', ')}.]`;
+        }
+        return includedParts.length > 0
+            ? [{ type: 'text', text: textPart }, ...includedParts]
+            : textPart;
     }
     Private.processAttachments = processAttachments;
     /**

package/lib/components/save-button.d.ts

@@ -1,7 +1,7 @@
 import { ReactWidget } from '@jupyterlab/ui-components';
 import type { TranslationBundle } from '@jupyterlab/translation';
 import React from 'react';
-import { AIChatModel } from '../chat-model';
+import { IAIChatModel } from '../tokens';
 /**
  * Properties for the SaveButton component.
  */
@@ -9,7 +9,7 @@ export interface ISaveButtonProps {
     /**
      * The chat model, used to listen for message changes for auto-save.
      */
-    model: AIChatModel;
+    model: IAIChatModel;
     /**
      * The application language translator.
      */