@jupyterlite/ai 0.16.0 → 0.18.0

package/lib/agent.d.ts

@@ -1,5 +1,5 @@
-import type { IMessageContent } from '@jupyter/chat';
 import { ISignal } from '@lumino/signaling';
+import { type ModelMessage, type UserContent } from 'ai';
 import { ISecretsManager } from 'jupyter-secrets-manager';
 import { type IAgentManager, type IAgentManagerFactory, type IAISettingsModel, type ISkillRegistry, type ITokenUsage, type ToolMap } from './tokens';
 /**
@@ -132,10 +132,10 @@ export declare class AgentManager implements IAgentManager {
      */
     clearHistory(): Promise<void>;
     /**
-     * Sets the history with a list of messages from the chat.
-     * @param messages The chat messages to set as history
+     * Sets the history from already-processed model messages.
+     * @param messages Pre-built model messages (may include binary content)
      */
-    setHistory(messages: IMessageContent[]): void;
+    setHistory(messages: ModelMessage[]): void;
     /**
      * Stops the current streaming response by aborting the request.
      * Resolve any pending approval.
@@ -143,26 +143,35 @@ export declare class AgentManager implements IAgentManager {
     stopStreaming(reason?: string): void;
     /**
      * Approves a pending tool call.
-     * @param approvalId The approval ID to approve
+     * @param toolCallId The tool call ID to approve
      * @param reason Optional reason for approval
      */
-    approveToolCall(approvalId: string, reason?: string): void;
+    approveToolCall(toolCallId: string, reason?: string): void;
     /**
      * Rejects a pending tool call.
-     * @param approvalId The approval ID to reject
+     * @param toolCallId The tool call ID to reject
      * @param reason Optional reason for rejection
      */
-    rejectToolCall(approvalId: string, reason?: string): void;
+    rejectToolCall(toolCallId: string, reason?: string): void;
     /**
      * Generates AI response to user message using the agent.
      * Handles the complete execution cycle including tool calls.
      * @param message The user message to respond to (may include processed attachment content)
      */
-    generateResponse(message: string): Promise<void>;
+    generateResponse(message: UserContent): Promise<void>;
+    /**
+     * Create a transient language model to request a text response which won't be added to history.
+     * @param messages - the messages sequence to send to the model.
+     */
+    textResponse(messages: ModelMessage[]): Promise<string>;
     /**
      * Updates cumulative token usage statistics from a completed model step.
      */
     private _updateTokenUsage;
+    /**
+     * Removes image and file parts from all user messages in the given list.
+     */
+    private _stripAttachments;
     /**
      * Gets the configured context window for the active provider.
      */
@@ -217,7 +226,7 @@ export declare class AgentManager implements IAgentManager {
     private _handleApprovalRequest;
     /**
      * Waits for user approval of a tool call.
-     * @param approvalId The approval ID to wait for
+     * @param toolCallId The tool call ID to wait for approval
      * @returns Promise that resolves to true if approved, false if rejected
      */
     private _waitForApproval;

package/lib/agent.js

@@ -1,7 +1,7 @@
 import { createMCPClient } from '@ai-sdk/mcp';
 import { PromiseDelegate } from '@lumino/coreutils';
 import { Signal } from '@lumino/signaling';
-import { ToolLoopAgent, stepCountIs } from 'ai';
+import { generateText, ToolLoopAgent, stepCountIs, APICallError } from 'ai';
 import { createModel } from './providers/models';
 import { getEffectiveContextWindow } from './providers/model-info';
 import { createProviderTools } from './providers/provider-tools';
@@ -331,29 +331,12 @@ export class AgentManager {
         this._tokenUsageChanged.emit(this._tokenUsage);
     }
     /**
-     * Sets the history with a list of messages from the chat.
-     * @param messages The chat messages to set as history
+     * Sets the history from already-processed model messages.
+     * @param messages Pre-built model messages (may include binary content)
      */
     setHistory(messages) {
-        // Stop any ongoing streaming and reject awaiting approvals
-        this.stopStreaming();
-        for (const [approvalId, pending] of this._pendingApprovals) {
-            pending.resolve(false, 'Chat history changed');
-            this._agentEvent.emit({
-                type: 'tool_approval_resolved',
-                data: { approvalId, approved: false }
-            });
-        }
-        this._pendingApprovals.clear();
-        // Convert chat messages to model messages
-        const modelMessages = messages.map(msg => {
-            const isAIMessage = msg.sender.username === 'ai-assistant';
-            return {
-                role: isAIMessage ? 'assistant' : 'user',
-                content: msg.body
-            };
-        });
-        this._history = Private.sanitizeModelMessages(modelMessages);
+        this.stopStreaming('Chat history changed');
+        this._history = Private.sanitizeModelMessages(messages);
     }
     /**
      * Stops the current streaming response by aborting the request.
@@ -362,44 +345,44 @@ export class AgentManager {
     stopStreaming(reason) {
         this._controller?.abort();
         // Reject any pending approvals
-        for (const [approvalId, pending] of this._pendingApprovals) {
+        for (const [toolCallId, pending] of this._pendingApprovals) {
             pending.resolve(false, reason ?? 'Stream ended by user');
             this._agentEvent.emit({
                 type: 'tool_approval_resolved',
-                data: { approvalId, approved: false }
+                data: { toolCallId, approved: false }
             });
         }
         this._pendingApprovals.clear();
     }
     /**
      * Approves a pending tool call.
-     * @param approvalId The approval ID to approve
+     * @param toolCallId The tool call ID to approve
      * @param reason Optional reason for approval
      */
-    approveToolCall(approvalId, reason) {
-        const pending = this._pendingApprovals.get(approvalId);
+    approveToolCall(toolCallId, reason) {
+        const pending = this._pendingApprovals.get(toolCallId);
         if (pending) {
             pending.resolve(true, reason);
-            this._pendingApprovals.delete(approvalId);
+            this._pendingApprovals.delete(toolCallId);
             this._agentEvent.emit({
                 type: 'tool_approval_resolved',
-                data: { approvalId, approved: true }
+                data: { toolCallId, approved: true }
             });
         }
     }
     /**
      * Rejects a pending tool call.
-     * @param approvalId The approval ID to reject
+     * @param toolCallId The tool call ID to reject
      * @param reason Optional reason for rejection
      */
-    rejectToolCall(approvalId, reason) {
-        const pending = this._pendingApprovals.get(approvalId);
+    rejectToolCall(toolCallId, reason) {
+        const pending = this._pendingApprovals.get(toolCallId);
         if (pending) {
             pending.resolve(false, reason);
-            this._pendingApprovals.delete(approvalId);
+            this._pendingApprovals.delete(toolCallId);
             this._agentEvent.emit({
                 type: 'tool_approval_resolved',
-                data: { approvalId, approved: false }
+                data: { toolCallId, approved: false }
             });
         }
     }
@@ -412,6 +395,11 @@ export class AgentManager {
         this._streaming = new PromiseDelegate();
         this._controller = new AbortController();
         const responseHistory = [];
+        // Add user message to history
+        responseHistory.push({
+            role: 'user',
+            content: message
+        });
         try {
             // Ensure we have an agent
             if (!this._agent) {
@@ -420,11 +408,6 @@ export class AgentManager {
             if (!this._agent) {
                 throw new Error('Failed to initialize agent');
             }
-            // Add user message to history
-            responseHistory.push({
-                role: 'user',
-                content: message
-            });
             let continueLoop = true;
             while (continueLoop) {
                 const result = await this._agent.stream({
@@ -473,9 +456,26 @@ export class AgentManager {
         }
         catch (error) {
             if (error.name !== 'AbortError') {
+                let helpMessage = `${error.message}`;
+                // Remove attachments from history on payload rejection errors
+                if (APICallError.isInstance(error) &&
+                    (error.statusCode === 400 ||
+                        error.statusCode === 404 ||
+                        error.statusCode === 413 ||
+                        error.statusCode === 415 ||
+                        error.statusCode === 422)) {
+                    this._stripAttachments([...this._history, ...responseHistory], '_Attachment removed due to error_');
+                    helpMessage +=
+                        '\n\nAttachments have been removed from history. Please send your prompt again.';
+                }
                 this._agentEvent.emit({
                     type: 'error',
-                    data: { error: error }
+                    data: { error: new Error(helpMessage) }
+                });
+                this._history.push(...Private.sanitizeModelMessages(responseHistory));
+                this._history.push({
+                    role: 'assistant',
+                    content: helpMessage
                 });
             }
         }
@@ -484,6 +484,24 @@ export class AgentManager {
             this._streaming.resolve();
         }
     }
+    /**
+     * Create a transient language model to request a text response which won't be added to history.
+     * @param messages - the messages sequence to send to the model.
+     */
+    async textResponse(messages) {
+        try {
+            const model = await this._createModel();
+            const result = await generateText({
+                model,
+                messages
+            });
+            this._updateTokenUsage(result.totalUsage, result.totalUsage.inputTokens);
+            return result.text;
+        }
+        catch (e) {
+            throw `Error while getting the topic of the chat\n${e}`;
+        }
+    }
     /**
      * Updates cumulative token usage statistics from a completed model step.
      */
@@ -498,6 +516,23 @@ export class AgentManager {
         this._tokenUsage.contextWindow = contextWindow;
         this._tokenUsageChanged.emit(this._tokenUsage);
     }
+    /**
+     * Removes image and file parts from all user messages in the given list.
+     */
+    _stripAttachments(messages, placeholder) {
+        for (const msg of messages) {
+            if (msg.role === 'user' && Array.isArray(msg.content)) {
+                const hasMedia = msg.content.some(p => p.type !== 'text');
+                if (hasMedia) {
+                    const textContent = msg.content
+                        .filter(p => p.type === 'text')
+                        .map(p => p.text)
+                        .join('\n');
+                    msg.content = textContent || placeholder;
+                }
+            }
+        }
+    }
     /**
      * Gets the configured context window for the active provider.
      */
@@ -694,13 +729,15 @@ ${richOutputWorkflowInstruction}`;
                     }
                     await this._handleApprovalRequest(part, processResult);
                     break;
+                case 'error':
+                    throw part.error;
                 case 'finish-step':
                     this._updateTokenUsage(part.usage, part.usage.inputTokens);
                     break;
                 case 'abort':
                     processResult.aborted = true;
                     break;
-                // Ignore: text-start, text-end, finish, error, and others
+                // Ignore: text-start, text-end, finish, and others
                 default:
                     break;
             }
@@ -779,13 +816,12 @@ ${richOutputWorkflowInstruction}`;
         this._agentEvent.emit({
             type: 'tool_approval_request',
             data: {
-                approvalId,
                 toolCallId: toolCall.toolCallId,
                 toolName: toolCall.toolName,
                 args: toolCall.input
             }
         });
-        const approved = await this._waitForApproval(approvalId);
+        const approved = await this._waitForApproval(toolCall.toolCallId);
         result.approvalProcessed = true;
         result.approvalResponse = {
             role: 'tool',
@@ -800,12 +836,12 @@ ${richOutputWorkflowInstruction}`;
     }
     /**
      * Waits for user approval of a tool call.
-     * @param approvalId The approval ID to wait for
+     * @param toolCallId The tool call ID to wait for approval
      * @returns Promise that resolves to true if approved, false if rejected
      */
-    _waitForApproval(approvalId) {
+    _waitForApproval(toolCallId) {
         return new Promise(resolve => {
-            this._pendingApprovals.set(approvalId, {
+            this._pendingApprovals.set(toolCallId, {
                 resolve: (approved) => {
                     resolve(approved);
                 }

package/lib/chat-commands/clear.js

@@ -16,7 +16,7 @@ export class ClearCommandProvider {
             return;
         }
         const context = inputModel.chatContext;
-        context?.clearMessages?.();
+        await context?.clearMessages?.();
         inputModel.value = '';
         inputModel.clearAttachments();
         inputModel.clearMentions();

package/lib/chat-model-handler.d.ts

@@ -2,14 +2,13 @@ import { ActiveCellManager } from '@jupyter/chat';
 import { IDocumentManager } from '@jupyterlab/docmanager';
 import { IRenderMimeRegistry } from '@jupyterlab/rendermime';
 import { Contents } from '@jupyterlab/services';
-import { AIChatModel } from './chat-model';
-import type { IAgentManagerFactory, IAISettingsModel, IChatModelHandler, ICreateChatOptions, IProviderRegistry, IToolRegistry } from './tokens';
+import type { IAgentManagerFactory, IAIChatModel, IAISettingsModel, IChatModelHandler, ICreateChatOptions, IProviderRegistry, IToolRegistry } from './tokens';
 /**
  * The chat model handler.
  */
 export declare class ChatModelHandler implements IChatModelHandler {
     constructor(options: ChatModelHandler.IOptions);
-    createModel(options: ICreateChatOptions): AIChatModel;
+    createModel(options: ICreateChatOptions): IAIChatModel;
     /**
      * Getter/setter for the active cell manager.
      */

package/lib/chat-model-handler.js

@@ -14,7 +14,7 @@ export class ChatModelHandler {
         this._contentsManager = options.contentsManager;
     }
     createModel(options) {
-        const { name, activeProvider, tokenUsage, messages, autosave } = options;
+        const { name, activeProvider, tokenUsage, messages, autosave, title } = options;
         // Create Agent Manager first so it can be shared
         const agentManager = this._agentManagerFactory.createAgent({
             settingsModel: this._settingsModel,
@@ -31,13 +31,17 @@ export class ChatModelHandler {
             agentManager,
             activeCellManager: this._activeCellManager,
             documentManager: this._docManager,
-            contentsManager: this._contentsManager
+            contentsManager: this._contentsManager,
+            providerRegistry: this._providerRegistry
         });
         messages?.forEach(message => {
             model.messageAdded({ ...message.content });
         });
         model.autosave = autosave ?? false;
         model.name = name;
+        if (title) {
+            model.title = title;
+        }
         return model;
     }
     /**

package/lib/chat-model.d.ts

@@ -1,13 +1,14 @@
-import { AbstractChatModel, IActiveCellManager, IAttachment, IChatContext, IMessageContent, INewMessage, IUser } from '@jupyter/chat';
+import { AbstractChatModel, IActiveCellManager, IAttachment, IChatContext, IMessageContent, IMimeModelBody, INewMessage, IUser } from '@jupyter/chat';
 import { IDocumentManager } from '@jupyterlab/docmanager';
 import { Contents } from '@jupyterlab/services';
 import { ISignal } from '@lumino/signaling';
-import type { IAgentManager, IAISettingsModel, ITokenUsage } from './tokens';
+import type { UserContent } from 'ai';
+import type { IAgentManager, IAIChatModel, IAISettingsModel, IProviderRegistry, ITokenUsage } from './tokens';
 /**
  * AI Chat Model implementation that provides chat functionality tool integration,
  * and MCP server support.
  */
-export declare class AIChatModel extends AbstractChatModel {
+export declare class AIChatModel extends AbstractChatModel implements IAIChatModel {
     /**
      * Constructs a new AIChatModel instance.
      * @param options Configuration options for the chat model
@@ -18,6 +19,19 @@ export declare class AIChatModel extends AbstractChatModel {
      */
     get name(): string;
     set name(value: string);
+    /**
+     * A signal emitting when the chat name has changed.
+     */
+    get nameChanged(): ISignal<IAIChatModel, string>;
+    /**
+     * The title of the chat.
+     */
+    get title(): string | null;
+    set title(value: string | null);
+    /**
+     * A signal emitting when the chat title has changed.
+     */
+    get titleChanged(): ISignal<IAIChatModel, string | null>;
     /**
      * Whether to save the chat automatically.
      */
@@ -26,11 +40,7 @@ export declare class AIChatModel extends AbstractChatModel {
     /**
      * A signal emitting when the autosave flag changed.
      */
-    get autosaveChanged(): ISignal<AIChatModel, boolean>;
-    /**
-     * A signal emitting when the chat name has changed.
-     */
-    get nameChanged(): ISignal<AIChatModel, string>;
+    get autosaveChanged(): ISignal<IAIChatModel, boolean>;
     /**
      * Gets the current user information.
      */
@@ -40,7 +50,7 @@ export declare class AIChatModel extends AbstractChatModel {
      */
     get tokenUsageChanged(): ISignal<IAgentManager, ITokenUsage>;
     /**
-     * Get the agent manager associated to the model.
+     * The agent manager used in the model.
      */
     get agentManager(): IAgentManager;
     /**
@@ -62,16 +72,41 @@ export declare class AIChatModel extends AbstractChatModel {
     /**
      * Clears all messages from the chat and resets conversation state.
      */
-    clearMessages: () => void;
+    clearMessages: () => Promise<void>;
+    /**
+     * Overrides messageAdded to ensure queued messages stay at the bottom.
+     */
+    messageAdded(message: IMessageContent): void;
     /**
      * Adds a non-user message to the chat (used by chat commands).
      */
-    addSystemMessage(body: string): void;
+    private _addSystemMessage;
     /**
      * Sends a message to the AI and generates a response.
      * @param message The user message to send
      */
     sendMessage(message: INewMessage): Promise<void>;
+    /**
+     * Internal method to process attachments and send the message to the agent.
+     */
+    private _processMessage;
+    /**
+     * Removes the message-queue chat component.
+     */
+    private _removeQueueUI;
+    /**
+     * Creates or updates the message-queue chat component.
+     */
+    private _updateQueueUI;
+    /**
+     * Processes the next message in the queue, or marks the agent as idle.
+     */
+    private _drainQueue;
+    /**
+     * Removes a queued message by its ID.
+     * @param messageId The ID of the queued message to remove
+     */
+    removeQueuedMessage(messageId: string): void;
     /**
      * Save the chat as json file.
      */
@@ -83,6 +118,10 @@ export declare class AIChatModel extends AbstractChatModel {
      * restoration is not possible.
      */
     restore: (filepath: string, silent?: boolean) => Promise<boolean>;
+    /**
+     * Request a title to this chat, regarding the message history.
+     */
+    requestTitle(): Promise<string>;
     /**
      * Serialize the model for backup
      */
@@ -95,6 +134,16 @@ export declare class AIChatModel extends AbstractChatModel {
      * Handles settings changes and updates chat configuration accordingly.
      */
     private _onSettingsChanged;
+    /**
+     * Rebuild history when the active model changes.
+     */
+    private _onModelChanged;
+    /**
+     * Rebuilds the agent history from the current messages.
+     * For vision-capable models, re-reads binary attachments from disk.
+     * For text-only models, uses message text only.
+     */
+    private _rebuildHistory;
     /**
      * Handles events emitted by the agent manager.
      * @param event The event data containing type and payload
@@ -156,33 +205,78 @@ export declare class AIChatModel extends AbstractChatModel {
      */
     private _updateToolCallUI;
     /**
-     * Processes file attachments and returns their content as formatted strings.
-     * @param attachments Array of file attachments to process
-     * @returns Array of formatted attachment contents
-     */
-    private _processAttachments;
-    /**
-     * Reads the content of a notebook cell.
-     * @param attachment The notebook attachment to read
-     * @returns Cell content as string or null if unable to read
+     * The current message queue
      */
-    private _readNotebookCells;
+    get messageQueue(): Private.IQueuedItem[];
+    set messageQueue(value: Private.IQueuedItem[]);
     /**
-     * Reads the content of a file attachment.
-     * @param attachment The file attachment to read
-     * @returns File content as string or null if unable to read
+     * Whether the chat is busy
      */
-    private _readFileAttachment;
+    get isBusy(): boolean;
+    set isBusy(value: boolean);
     private _settingsModel;
     private _user;
     private _toolContexts;
     private _agentManager;
+    private _providerRegistry?;
+    private _currentModelKey;
     private _currentStreamingMessage;
     private _nameChanged;
     private _contentsManager?;
     private _autosave;
     private _autosaveChanged;
     private _autosaveDebouncer;
+    private _messageQueue;
+    private _isBusy;
+    private _queueMessageId;
+    private _title;
+    private _titleChanged;
+}
+declare namespace Private {
+    interface IQueuedItem {
+        id: string;
+        body: string;
+        _originalMsg: IMessageContent;
+    }
+    /**
+     * Extract rendermime-ready mime bundles from arbitrary tool results.
+     */
+    function extractMimeBundlesFromUnknown(content: unknown, options?: {
+        trustedMimeTypes?: ReadonlyArray<string>;
+    }): IMimeModelBody[];
+    function formatToolOutput(outputData: unknown): string;
+    /**
+     * Processes file attachments and returns the message content with the attachments.
+     * @param attachments Array of file attachments to process
+     * @param documentManager Optional document manager for file operations
+     * @param body The message body
+     * @param supportsImages Whether the model supports images
+     * @param supportsPdf Whether the model supports pdfs
+     * @param supportsAudio Whether the model supports audio
+     * @returns Enhanced message content
+     */
+    function processAttachments(attachments: IAttachment[], documentManager: IDocumentManager | null | undefined, body: string, supportsImages: boolean, supportsPdf: boolean, supportsAudio: boolean): Promise<UserContent>;
+    /**
+     * Reads a binary attachment and returns its base64-encoded content.
+     * @param attachment The attachment to read
+     * @param documentManager Optional document manager for file operations
+     * @returns Base64 string or null if unable to read
+     */
+    function readBinaryAttachment(attachment: IAttachment, documentManager: IDocumentManager | null | undefined): Promise<string | null>;
+    /**
+     * Reads the content of a notebook cell.
+     * @param attachment The notebook attachment to read
+     * @param documentManager Optional document manager for file operations
+     * @returns Cell content as string or null if unable to read
+     */
+    function readNotebookCells(attachment: IAttachment, documentManager: IDocumentManager | null | undefined): Promise<string | null>;
+    /**
+     * Reads the content of a file attachment.
+     * @param attachment The file attachment to read
+     * @param documentManager Optional document manager for file operations
+     * @returns File content as string or null if unable to read
+     */
+    function readFileAttachment(attachment: IAttachment, documentManager: IDocumentManager | null | undefined): Promise<string | null>;
 }
 /**
  * Namespace containing types and interfaces for AIChatModel.
@@ -216,6 +310,10 @@ export declare namespace AIChatModel {
          * The contents manager.
          */
         contentsManager?: Contents.IManager;
+        /**
+         * Optional provider registry for model capability lookups.
+         */
+        providerRegistry?: IProviderRegistry;
         /**
          * Whether to restore or not the message (default to true)
          */
@@ -232,7 +330,7 @@ export declare namespace AIChatModel {
         /**
          * The clear messages callback.
          */
-        clearMessages: () => void;
+        clearMessages: () => Promise<void>;
         /**
          * Adds an assistant/system message to the chat.
          */
@@ -274,6 +372,11 @@ export declare namespace AIChatModel {
              * Whether the chat is automatically saved.
              */
             autosave?: boolean;
+            /**
+             * An optional title of the chat.
+             */
+            title?: string;
         };
     };
 }
+export {};