@jupyterlite/ai 0.17.0 → 0.18.0

package/lib/agent.d.ts

@@ -1,4 +1,3 @@
-import type { IMessageContent } from '@jupyter/chat';
 import { ISignal } from '@lumino/signaling';
 import { type ModelMessage, type UserContent } from 'ai';
 import { ISecretsManager } from 'jupyter-secrets-manager';
@@ -133,10 +132,10 @@ export declare class AgentManager implements IAgentManager {
      */
     clearHistory(): Promise<void>;
     /**
-     * Sets the history with a list of messages from the chat.
-     * @param messages The chat messages to set as history
+     * Sets the history from already-processed model messages.
+     * @param messages Pre-built model messages (may include binary content)
      */
-    setHistory(messages: IMessageContent[]): void;
+    setHistory(messages: ModelMessage[]): void;
     /**
      * Stops the current streaming response by aborting the request.
      * Resolve any pending approval.
@@ -144,16 +143,16 @@ export declare class AgentManager implements IAgentManager {
     stopStreaming(reason?: string): void;
     /**
      * Approves a pending tool call.
-     * @param approvalId The approval ID to approve
+     * @param toolCallId The tool call ID to approve
      * @param reason Optional reason for approval
      */
-    approveToolCall(approvalId: string, reason?: string): void;
+    approveToolCall(toolCallId: string, reason?: string): void;
     /**
      * Rejects a pending tool call.
-     * @param approvalId The approval ID to reject
+     * @param toolCallId The tool call ID to reject
      * @param reason Optional reason for rejection
      */
-    rejectToolCall(approvalId: string, reason?: string): void;
+    rejectToolCall(toolCallId: string, reason?: string): void;
     /**
      * Generates AI response to user message using the agent.
      * Handles the complete execution cycle including tool calls.
@@ -169,6 +168,10 @@ export declare class AgentManager implements IAgentManager {
      * Updates cumulative token usage statistics from a completed model step.
      */
     private _updateTokenUsage;
+    /**
+     * Removes image and file parts from all user messages in the given list.
+     */
+    private _stripAttachments;
     /**
      * Gets the configured context window for the active provider.
      */
@@ -223,7 +226,7 @@ export declare class AgentManager implements IAgentManager {
     private _handleApprovalRequest;
     /**
      * Waits for user approval of a tool call.
-     * @param approvalId The approval ID to wait for
+     * @param toolCallId The tool call ID to wait for approval
      * @returns Promise that resolves to true if approved, false if rejected
      */
     private _waitForApproval;

package/lib/agent.js

@@ -331,29 +331,12 @@ export class AgentManager {
         this._tokenUsageChanged.emit(this._tokenUsage);
     }
     /**
-     * Sets the history with a list of messages from the chat.
-     * @param messages The chat messages to set as history
+     * Sets the history from already-processed model messages.
+     * @param messages Pre-built model messages (may include binary content)
      */
     setHistory(messages) {
-        // Stop any ongoing streaming and reject awaiting approvals
-        this.stopStreaming();
-        for (const [approvalId, pending] of this._pendingApprovals) {
-            pending.resolve(false, 'Chat history changed');
-            this._agentEvent.emit({
-                type: 'tool_approval_resolved',
-                data: { approvalId, approved: false }
-            });
-        }
-        this._pendingApprovals.clear();
-        // Convert chat messages to model messages
-        const modelMessages = messages.map(msg => {
-            const role = msg.sender.username === 'ai-assistant' ? 'assistant' : 'user';
-            return {
-                role,
-                content: msg.body
-            };
-        });
-        this._history = Private.sanitizeModelMessages(modelMessages);
+        this.stopStreaming('Chat history changed');
+        this._history = Private.sanitizeModelMessages(messages);
     }
     /**
      * Stops the current streaming response by aborting the request.
@@ -362,44 +345,44 @@ export class AgentManager {
     stopStreaming(reason) {
         this._controller?.abort();
         // Reject any pending approvals
-        for (const [approvalId, pending] of this._pendingApprovals) {
+        for (const [toolCallId, pending] of this._pendingApprovals) {
             pending.resolve(false, reason ?? 'Stream ended by user');
             this._agentEvent.emit({
                 type: 'tool_approval_resolved',
-                data: { approvalId, approved: false }
+                data: { toolCallId, approved: false }
             });
         }
         this._pendingApprovals.clear();
     }
     /**
      * Approves a pending tool call.
-     * @param approvalId The approval ID to approve
+     * @param toolCallId The tool call ID to approve
      * @param reason Optional reason for approval
      */
-    approveToolCall(approvalId, reason) {
-        const pending = this._pendingApprovals.get(approvalId);
+    approveToolCall(toolCallId, reason) {
+        const pending = this._pendingApprovals.get(toolCallId);
         if (pending) {
             pending.resolve(true, reason);
-            this._pendingApprovals.delete(approvalId);
+            this._pendingApprovals.delete(toolCallId);
             this._agentEvent.emit({
                 type: 'tool_approval_resolved',
-                data: { approvalId, approved: true }
+                data: { toolCallId, approved: true }
             });
         }
     }
     /**
      * Rejects a pending tool call.
-     * @param approvalId The approval ID to reject
+     * @param toolCallId The tool call ID to reject
      * @param reason Optional reason for rejection
      */
-    rejectToolCall(approvalId, reason) {
-        const pending = this._pendingApprovals.get(approvalId);
+    rejectToolCall(toolCallId, reason) {
+        const pending = this._pendingApprovals.get(toolCallId);
         if (pending) {
             pending.resolve(false, reason);
-            this._pendingApprovals.delete(approvalId);
+            this._pendingApprovals.delete(toolCallId);
             this._agentEvent.emit({
                 type: 'tool_approval_resolved',
-                data: { approvalId, approved: false }
+                data: { toolCallId, approved: false }
             });
         }
     }
@@ -481,19 +464,7 @@ export class AgentManager {
                         error.statusCode === 413 ||
                         error.statusCode === 415 ||
                         error.statusCode === 422)) {
-                    for (const msg of [...this._history, ...responseHistory]) {
-                        if (msg.role === 'user' && Array.isArray(msg.content)) {
-                            const hasMedia = msg.content.some(p => p.type !== 'text');
-                            if (hasMedia) {
-                                const textContent = msg.content
-                                    .filter(p => p.type === 'text')
-                                    .map(p => p.text)
-                                    .join('\n');
-                                msg.content =
-                                    textContent || '_Attachment removed due to error_';
-                            }
-                        }
-                    }
+                    this._stripAttachments([...this._history, ...responseHistory], '_Attachment removed due to error_');
                     helpMessage +=
                         '\n\nAttachments have been removed from history. Please send your prompt again.';
                 }
@@ -545,6 +516,23 @@ export class AgentManager {
         this._tokenUsage.contextWindow = contextWindow;
         this._tokenUsageChanged.emit(this._tokenUsage);
     }
+    /**
+     * Removes image and file parts from all user messages in the given list.
+     */
+    _stripAttachments(messages, placeholder) {
+        for (const msg of messages) {
+            if (msg.role === 'user' && Array.isArray(msg.content)) {
+                const hasMedia = msg.content.some(p => p.type !== 'text');
+                if (hasMedia) {
+                    const textContent = msg.content
+                        .filter(p => p.type === 'text')
+                        .map(p => p.text)
+                        .join('\n');
+                    msg.content = textContent || placeholder;
+                }
+            }
+        }
+    }
     /**
      * Gets the configured context window for the active provider.
      */
@@ -828,13 +816,12 @@ ${richOutputWorkflowInstruction}`;
         this._agentEvent.emit({
             type: 'tool_approval_request',
             data: {
-                approvalId,
                 toolCallId: toolCall.toolCallId,
                 toolName: toolCall.toolName,
                 args: toolCall.input
             }
         });
-        const approved = await this._waitForApproval(approvalId);
+        const approved = await this._waitForApproval(toolCall.toolCallId);
         result.approvalProcessed = true;
         result.approvalResponse = {
             role: 'tool',
@@ -849,12 +836,12 @@ ${richOutputWorkflowInstruction}`;
     }
     /**
      * Waits for user approval of a tool call.
-     * @param approvalId The approval ID to wait for
+     * @param toolCallId The tool call ID to wait for approval
      * @returns Promise that resolves to true if approved, false if rejected
      */
-    _waitForApproval(approvalId) {
+    _waitForApproval(toolCallId) {
         return new Promise(resolve => {
-            this._pendingApprovals.set(approvalId, {
+            this._pendingApprovals.set(toolCallId, {
                 resolve: (approved) => {
                     resolve(approved);
                 }

package/lib/chat-model-handler.d.ts

@@ -2,14 +2,13 @@ import { ActiveCellManager } from '@jupyter/chat';
 import { IDocumentManager } from '@jupyterlab/docmanager';
 import { IRenderMimeRegistry } from '@jupyterlab/rendermime';
 import { Contents } from '@jupyterlab/services';
-import { AIChatModel } from './chat-model';
-import type { IAgentManagerFactory, IAISettingsModel, IChatModelHandler, ICreateChatOptions, IProviderRegistry, IToolRegistry } from './tokens';
+import type { IAgentManagerFactory, IAIChatModel, IAISettingsModel, IChatModelHandler, ICreateChatOptions, IProviderRegistry, IToolRegistry } from './tokens';
 /**
  * The chat model handler.
  */
 export declare class ChatModelHandler implements IChatModelHandler {
     constructor(options: ChatModelHandler.IOptions);
-    createModel(options: ICreateChatOptions): AIChatModel;
+    createModel(options: ICreateChatOptions): IAIChatModel;
     /**
      * Getter/setter for the active cell manager.
      */

package/lib/chat-model-handler.js

@@ -31,7 +31,8 @@ export class ChatModelHandler {
             agentManager,
             activeCellManager: this._activeCellManager,
             documentManager: this._docManager,
-            contentsManager: this._contentsManager
+            contentsManager: this._contentsManager,
+            providerRegistry: this._providerRegistry
         });
         messages?.forEach(message => {
             model.messageAdded({ ...message.content });

package/lib/chat-model.d.ts

@@ -1,13 +1,14 @@
-import { AbstractChatModel, IActiveCellManager, IAttachment, IChatContext, IMessageContent, INewMessage, IUser } from '@jupyter/chat';
+import { AbstractChatModel, IActiveCellManager, IAttachment, IChatContext, IMessageContent, IMimeModelBody, INewMessage, IUser } from '@jupyter/chat';
 import { IDocumentManager } from '@jupyterlab/docmanager';
 import { Contents } from '@jupyterlab/services';
 import { ISignal } from '@lumino/signaling';
-import type { IAgentManager, IAISettingsModel, ITokenUsage } from './tokens';
+import type { UserContent } from 'ai';
+import type { IAgentManager, IAIChatModel, IAISettingsModel, IProviderRegistry, ITokenUsage } from './tokens';
 /**
  * AI Chat Model implementation that provides chat functionality tool integration,
  * and MCP server support.
  */
-export declare class AIChatModel extends AbstractChatModel {
+export declare class AIChatModel extends AbstractChatModel implements IAIChatModel {
     /**
      * Constructs a new AIChatModel instance.
      * @param options Configuration options for the chat model
@@ -21,7 +22,7 @@ export declare class AIChatModel extends AbstractChatModel {
     /**
      * A signal emitting when the chat name has changed.
      */
-    get nameChanged(): ISignal<AIChatModel, string>;
+    get nameChanged(): ISignal<IAIChatModel, string>;
     /**
      * The title of the chat.
      */
@@ -30,7 +31,7 @@ export declare class AIChatModel extends AbstractChatModel {
     /**
      * A signal emitting when the chat title has changed.
      */
-    get titleChanged(): ISignal<AIChatModel, string | null>;
+    get titleChanged(): ISignal<IAIChatModel, string | null>;
     /**
      * Whether to save the chat automatically.
      */
@@ -39,7 +40,7 @@ export declare class AIChatModel extends AbstractChatModel {
     /**
      * A signal emitting when the autosave flag changed.
      */
-    get autosaveChanged(): ISignal<AIChatModel, boolean>;
+    get autosaveChanged(): ISignal<IAIChatModel, boolean>;
     /**
      * Gets the current user information.
      */
@@ -49,7 +50,7 @@ export declare class AIChatModel extends AbstractChatModel {
      */
     get tokenUsageChanged(): ISignal<IAgentManager, ITokenUsage>;
     /**
-     * Get the agent manager associated to the model.
+     * The agent manager used in the model.
      */
     get agentManager(): IAgentManager;
     /**
@@ -72,15 +73,40 @@ export declare class AIChatModel extends AbstractChatModel {
      * Clears all messages from the chat and resets conversation state.
      */
     clearMessages: () => Promise<void>;
+    /**
+     * Overrides messageAdded to ensure queued messages stay at the bottom.
+     */
+    messageAdded(message: IMessageContent): void;
     /**
      * Adds a non-user message to the chat (used by chat commands).
      */
-    addSystemMessage(body: string): void;
+    private _addSystemMessage;
     /**
      * Sends a message to the AI and generates a response.
      * @param message The user message to send
      */
     sendMessage(message: INewMessage): Promise<void>;
+    /**
+     * Internal method to process attachments and send the message to the agent.
+     */
+    private _processMessage;
+    /**
+     * Removes the message-queue chat component.
+     */
+    private _removeQueueUI;
+    /**
+     * Creates or updates the message-queue chat component.
+     */
+    private _updateQueueUI;
+    /**
+     * Processes the next message in the queue, or marks the agent as idle.
+     */
+    private _drainQueue;
+    /**
+     * Removes a queued message by its ID.
+     * @param messageId The ID of the queued message to remove
+     */
+    removeQueuedMessage(messageId: string): void;
     /**
      * Save the chat as json file.
      */
@@ -108,6 +134,16 @@ export declare class AIChatModel extends AbstractChatModel {
      * Handles settings changes and updates chat configuration accordingly.
      */
     private _onSettingsChanged;
+    /**
+     * Rebuild history when the active model changes.
+     */
+    private _onModelChanged;
+    /**
+     * Rebuilds the agent history from the current messages.
+     * For vision-capable models, re-reads binary attachments from disk.
+     * For text-only models, uses message text only.
+     */
+    private _rebuildHistory;
     /**
      * Handles events emitted by the agent manager.
      * @param event The event data containing type and payload
@@ -168,19 +204,80 @@ export declare class AIChatModel extends AbstractChatModel {
      * Updates a tool call's UI with new status and optional output.
      */
     private _updateToolCallUI;
+    /**
+     * The current message queue
+     */
+    get messageQueue(): Private.IQueuedItem[];
+    set messageQueue(value: Private.IQueuedItem[]);
+    /**
+     * Whether the chat is busy
+     */
+    get isBusy(): boolean;
+    set isBusy(value: boolean);
     private _settingsModel;
     private _user;
     private _toolContexts;
     private _agentManager;
+    private _providerRegistry?;
+    private _currentModelKey;
     private _currentStreamingMessage;
     private _nameChanged;
     private _contentsManager?;
     private _autosave;
     private _autosaveChanged;
     private _autosaveDebouncer;
+    private _messageQueue;
+    private _isBusy;
+    private _queueMessageId;
     private _title;
     private _titleChanged;
 }
+declare namespace Private {
+    interface IQueuedItem {
+        id: string;
+        body: string;
+        _originalMsg: IMessageContent;
+    }
+    /**
+     * Extract rendermime-ready mime bundles from arbitrary tool results.
+     */
+    function extractMimeBundlesFromUnknown(content: unknown, options?: {
+        trustedMimeTypes?: ReadonlyArray<string>;
+    }): IMimeModelBody[];
+    function formatToolOutput(outputData: unknown): string;
+    /**
+     * Processes file attachments and returns the message content with the attachments.
+     * @param attachments Array of file attachments to process
+     * @param documentManager Optional document manager for file operations
+     * @param body The message body
+     * @param supportsImages Whether the model supports images
+     * @param supportsPdf Whether the model supports pdfs
+     * @param supportsAudio Whether the model supports audio
+     * @returns Enhanced message content
+     */
+    function processAttachments(attachments: IAttachment[], documentManager: IDocumentManager | null | undefined, body: string, supportsImages: boolean, supportsPdf: boolean, supportsAudio: boolean): Promise<UserContent>;
+    /**
+     * Reads a binary attachment and returns its base64-encoded content.
+     * @param attachment The attachment to read
+     * @param documentManager Optional document manager for file operations
+     * @returns Base64 string or null if unable to read
+     */
+    function readBinaryAttachment(attachment: IAttachment, documentManager: IDocumentManager | null | undefined): Promise<string | null>;
+    /**
+     * Reads the content of a notebook cell.
+     * @param attachment The notebook attachment to read
+     * @param documentManager Optional document manager for file operations
+     * @returns Cell content as string or null if unable to read
+     */
+    function readNotebookCells(attachment: IAttachment, documentManager: IDocumentManager | null | undefined): Promise<string | null>;
+    /**
+     * Reads the content of a file attachment.
+     * @param attachment The file attachment to read
+     * @param documentManager Optional document manager for file operations
+     * @returns File content as string or null if unable to read
+     */
+    function readFileAttachment(attachment: IAttachment, documentManager: IDocumentManager | null | undefined): Promise<string | null>;
+}
 /**
  * Namespace containing types and interfaces for AIChatModel.
  */
@@ -213,6 +310,10 @@ export declare namespace AIChatModel {
          * The contents manager.
          */
         contentsManager?: Contents.IManager;
+        /**
+         * Optional provider registry for model capability lookups.
+         */
+        providerRegistry?: IProviderRegistry;
         /**
          * Whether to restore or not the message (default to true)
          */
@@ -278,3 +379,4 @@ export declare namespace AIChatModel {
         };
     };
 }
+export {};