@jupyterlite/ai 0.17.0 → 0.18.0

package/lib/providers/model-info.js

@@ -45,6 +45,39 @@ export function getProviderModelInfo(providerInfo, model) {
         return normalizeModelId(candidateId) === normalizedModelId;
     })?.[1];
 }
+export function modelSupportsImages(providerConfig, providerRegistry) {
+    if (!providerConfig) {
+        return true;
+    }
+    const providerInfo = providerRegistry?.getProviderInfo(providerConfig.provider);
+    const modelInfo = getProviderModelInfo(providerInfo, providerConfig.model);
+    if (!modelInfo) {
+        return true;
+    }
+    return modelInfo.supportsImages ?? true;
+}
+export function modelSupportsPdf(providerConfig, providerRegistry) {
+    if (!providerConfig) {
+        return true;
+    }
+    const providerInfo = providerRegistry?.getProviderInfo(providerConfig.provider);
+    const modelInfo = getProviderModelInfo(providerInfo, providerConfig.model);
+    if (!modelInfo) {
+        return true;
+    }
+    return modelInfo.supportsPdf ?? true;
+}
+export function modelSupportsAudio(providerConfig, providerRegistry) {
+    if (!providerConfig) {
+        return true;
+    }
+    const providerInfo = providerRegistry?.getProviderInfo(providerConfig.provider);
+    const modelInfo = getProviderModelInfo(providerInfo, providerConfig.model);
+    if (!modelInfo) {
+        return true;
+    }
+    return modelInfo.supportsAudio ?? true;
+}
 export function getEffectiveContextWindow(providerConfig, providerRegistry) {
     if (!providerConfig) {
         return undefined;

package/lib/tokens.d.ts

@@ -1,4 +1,4 @@
-import { ActiveCellManager, IMessage, IMessageContent } from '@jupyter/chat';
+import { ActiveCellManager, IChatModel, IMessage } from '@jupyter/chat';
 import { VDomRenderer } from '@jupyterlab/apputils';
 import { IRenderMimeRegistry } from '@jupyterlab/rendermime';
 import { Token } from '@lumino/coreutils';
@@ -7,7 +7,6 @@ import { ISignal } from '@lumino/signaling';
 import type { Tool, LanguageModel, UserContent, ModelMessage } from 'ai';
 import { ISecretsManager } from 'jupyter-secrets-manager';
 import type { IModelOptions } from './providers/models';
-import { AIChatModel } from './chat-model';
 import type { ISkillDefinition, ISkillRegistration, ISkillResourceResult, ISkillSummary } from './skills/types';
 export type { ISkillDefinition, ISkillRegistration, ISkillResourceResult, ISkillSummary } from './skills/types';
 /**
@@ -159,6 +158,18 @@ export interface IProviderModelInfo {
      * Default context window for the model in tokens.
      */
     contextWindow?: number;
+    /**
+     * Whether the model supports image inputs.
+     */
+    supportsImages?: boolean;
+    /**
+     * Whether the model supports PDF inputs.
+     */
+    supportsPdf?: boolean;
+    /**
+     * Whether the model supports audio inputs.
+     */
+    supportsAudio?: boolean;
 }
 export interface IProviderInfo {
     /**
@@ -303,6 +314,7 @@ export interface IAIConfig {
     diffDisplayMode: 'split' | 'unified';
     skillsPaths: string[];
     chatBackupDirectory: string;
+    autoTitle: boolean;
 }
 export interface IAISettingsModel extends VDomRenderer.IModel {
     readonly config: IAIConfig;
@@ -401,13 +413,12 @@ export declare namespace IAgentManager {
             isError: boolean;
         };
         tool_approval_request: {
-            approvalId: string;
             toolCallId: string;
             toolName: string;
             args: unknown;
         };
         tool_approval_resolved: {
-            approvalId: string;
+            toolCallId: string;
             approved: boolean;
         };
         error: {
@@ -468,26 +479,26 @@ export interface IAgentManager {
      */
     clearHistory(): Promise<void>;
     /**
-     * Sets the conversation history with a list of messages from the chat.
-     * @param messages The chat messages to set as history
+     * Sets the history from already-processed model messages.
+     * @param messages Pre-built model messages (may include binary content)
      */
-    setHistory(messages: IMessageContent[]): void;
+    setHistory(messages: ModelMessage[]): void;
     /**
      * Stops the current streaming response by aborting the request.
      */
     stopStreaming(): void;
     /**
      * Approves a pending tool call.
-     * @param approvalId The approval ID to approve
+     * @param toolCallId The tool call ID to approve
      * @param reason Optional reason for approval
      */
-    approveToolCall(approvalId: string, reason?: string): void;
+    approveToolCall(toolCallId: string, reason?: string): void;
     /**
      * Rejects a pending tool call.
-     * @param approvalId The approval ID to reject
+     * @param toolCallId The tool call ID to reject
      * @param reason Optional reason for rejection
      */
-    rejectToolCall(approvalId: string, reason?: string): void;
+    rejectToolCall(toolCallId: string, reason?: string): void;
     /**
      * Generates AI response to user message using the agent.
      * Handles the complete execution cycle including tool calls.
@@ -533,6 +544,68 @@ export interface IAgentManagerFactory {
     getMCPTools(): Promise<ToolMap>;
 }
 export declare const IAgentManagerFactory: Token<IAgentManagerFactory>;
+export interface IAIChatModel extends IChatModel {
+    /**
+     * A signal emitting when the chat name has changed.
+     */
+    readonly nameChanged: ISignal<IAIChatModel, string>;
+    /**
+     * The title of the chat.
+     */
+    title: string | null;
+    /**
+     * A signal emitting when the chat title has changed.
+     */
+    readonly titleChanged: ISignal<IAIChatModel, string | null>;
+    /**
+     * Whether to save the chat automatically.
+     */
+    autosave: boolean;
+    /**
+     * A signal emitting when the autosave flag changed.
+     */
+    readonly autosaveChanged: ISignal<IAIChatModel, boolean>;
+    /**
+     * Whether save/restore is available.
+     */
+    readonly saveAvailable: boolean;
+    /**
+     * A signal emitting when the token usage changed.
+     */
+    readonly tokenUsageChanged: ISignal<IAgentManager, ITokenUsage>;
+    /**
+     * The agent manager used in the model.
+     */
+    readonly agentManager: IAgentManager;
+    /**
+     * Save the chat as json file.
+     */
+    save(): Promise<void>;
+    /**
+     * Restore the chat from a json file.
+     *
+     * @param silent - Whether a log should be displayed in the console if the
+     * restoration is not possible.
+     */
+    restore(filepath: string, silent?: boolean): Promise<boolean>;
+    /**
+     * Request a title to this chat, regarding the message history.
+     */
+    requestTitle(): Promise<string>;
+    /**
+     * Removes a queued message by its ID.
+     * @param messageId The ID of the queued message to remove
+     */
+    removeQueuedMessage(messageId: string): void;
+    /**
+     * The current message queue
+     */
+    messageQueue: any[];
+    /**
+     * Whether the chat is currently busy processing a message
+     */
+    isBusy: boolean;
+}
 /**
  * The interface for the chat model handler.
  */
@@ -540,7 +613,7 @@ export interface IChatModelHandler {
     /**
      * The function to create a new model.
      */
-    createModel(options: ICreateChatOptions): AIChatModel;
+    createModel(options: ICreateChatOptions): IAIChatModel;
     /**
      * The active cell manager (to copy code from chat to cell).
      */

package/lib/widgets/ai-settings.js

@@ -482,6 +482,11 @@ const AISettingsComponent = ({ model, agentManagerFactory, themeManager, provide
                                 }), color: "primary" }), label: React.createElement(Box, null,
                                 React.createElement(Typography, { variant: "body1" }, trans.__('Send with Shift+Enter')),
                                 React.createElement(Typography, { variant: "caption", color: "text.secondary" }, trans.__('Use Shift+Enter to send messages (Enter creates new line)'))) }),
+                        React.createElement(FormControlLabel, { control: React.createElement(Switch, { checked: config.autoTitle, onChange: e => handleConfigUpdate({
+                                    autoTitle: e.target.checked
+                                }), color: "primary" }), label: React.createElement(Box, null,
+                                React.createElement(Typography, { variant: "body1" }, trans.__('Auto Title')),
+                                React.createElement(Typography, { variant: "caption", color: "text.secondary" }, trans.__('Automatically generate a chat title from the model for every message until there are 5 messages'))) }),
                         React.createElement(FormControlLabel, { control: React.createElement(Switch, { checked: config.showTokenUsage, onChange: e => handleConfigUpdate({
                                     showTokenUsage: e.target.checked
                                 }), color: "primary" }), label: React.createElement(Box, null,

package/lib/widgets/main-area-chat.d.ts

@@ -2,8 +2,7 @@ import { ChatWidget } from '@jupyter/chat';
 import { MainAreaWidget } from '@jupyterlab/apputils';
 import type { TranslationBundle } from '@jupyterlab/translation';
 import { CommandRegistry } from '@lumino/commands';
-import { AIChatModel } from '../chat-model';
-import { type IAISettingsModel } from '../tokens';
+import { IAIChatModel, type IAISettingsModel } from '../tokens';
 export declare namespace MainAreaChat {
     interface IOptions extends MainAreaWidget.IOptions<ChatWidget> {
         commands: CommandRegistry;
@@ -20,7 +19,7 @@ export declare class MainAreaChat extends MainAreaWidget<ChatWidget> {
     /**
      * Get the model of the chat.
      */
-    get model(): AIChatModel;
+    get model(): IAIChatModel;
     /**
      * Get the area of the chat.
      */

package/lib/widgets/main-area-chat.js

@@ -43,14 +43,14 @@ export class MainAreaChat extends MainAreaWidget {
         this._outputAreaCompat = new RenderedMessageOutputAreaCompat({
             chatPanel: this.content
         });
-        this.model.writersChanged.connect(this._writersChanged);
+        this.model.writersChanged?.connect(this._writersChanged);
         this.model.titleChanged.connect(this._titleChanged);
     }
     dispose() {
         super.dispose();
         // Dispose of the approval buttons widget when the chat is disposed.
         this._outputAreaCompat.dispose();
-        this.model.writersChanged.disconnect(this._writersChanged);
+        this.model.writersChanged?.disconnect(this._writersChanged);
         this.model.titleChanged.disconnect(this._titleChanged);
     }
     /**
@@ -69,12 +69,10 @@ export class MainAreaChat extends MainAreaWidget {
         // Check if AI is currently writing (streaming)
         const aiWriting = writers.some(writer => writer.user.username === 'ai-assistant');
         if (aiWriting) {
-            this.content.inputToolbarRegistry?.hide('send');
             this.content.inputToolbarRegistry?.show('stop');
         }
         else {
             this.content.inputToolbarRegistry?.hide('stop');
-            this.content.inputToolbarRegistry?.show('send');
         }
     };
     _titleChanged = () => {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@jupyterlite/ai",
-    "version": "0.17.0",
+    "version": "0.18.0",
     "description": "AI code completions and chat for JupyterLite",
     "keywords": [
         "jupyter",
@@ -54,7 +54,7 @@
         "watch:labextension": "jupyter labextension watch .",
         "docs": "jupyter book start",
         "docs:build": "sed -e 's/\\[@/[/g' -e 's/@/\\@/g' CHANGELOG.md > docs/_changelog_content.md && jupyter book build --html",
-        "sync:model-context-windows": "node scripts/sync-model-context-windows.mjs && prettier --write src/providers/generated-context-windows.ts && eslint --fix src/providers/generated-context-windows.ts"
+        "sync:model-info": "node scripts/sync-model-info.mjs && prettier --write src/providers/generated-model-info.ts && eslint --fix src/providers/generated-model-info.ts"
     },
     "dependencies": {
         "@ai-sdk/anthropic": "^3.0.58",
@@ -89,7 +89,7 @@
         "@mui/icons-material": "^7",
         "@mui/material": "^7",
         "ai": "^6.0.116",
-        "jupyter-chat-components": "^0.2.0",
+        "jupyter-chat-components": "^0.5.0",
         "jupyter-secrets-manager": "^0.5.0",
         "yaml": "^2.8.1",
         "zod": "^4.3.6"

package/schema/settings-model.json

@@ -210,6 +210,12 @@
       "type": "boolean",
       "default": false
     },
+    "autoTitle": {
+      "title": "Auto Title",
+      "description": "Automatically request a title from the model for every message until there are 5 messages",
+      "type": "boolean",
+      "default": false
+    },
     "showTokenUsage": {
       "title": "Show Token Usage",
       "description": "Display token usage information in the chat toolbar",

package/src/agent.ts

@@ -1,5 +1,4 @@
 import { createMCPClient, type MCPClient } from '@ai-sdk/mcp';
-import type { IMessageContent } from '@jupyter/chat';
 import { IRenderMimeRegistry } from '@jupyterlab/rendermime';
 import { PromiseDelegate } from '@lumino/coreutils';
 import { ISignal, Signal } from '@lumino/signaling';
@@ -490,32 +489,12 @@ export class AgentManager implements IAgentManager {
   }
 
   /**
-   * Sets the history with a list of messages from the chat.
-   * @param messages The chat messages to set as history
+   * Sets the history from already-processed model messages.
+   * @param messages Pre-built model messages (may include binary content)
    */
-  setHistory(messages: IMessageContent[]): void {
-    // Stop any ongoing streaming and reject awaiting approvals
-    this.stopStreaming();
-
-    for (const [approvalId, pending] of this._pendingApprovals) {
-      pending.resolve(false, 'Chat history changed');
-      this._agentEvent.emit({
-        type: 'tool_approval_resolved',
-        data: { approvalId, approved: false }
-      });
-    }
-    this._pendingApprovals.clear();
-
-    // Convert chat messages to model messages
-    const modelMessages: ModelMessage[] = messages.map(msg => {
-      const role =
-        msg.sender.username === 'ai-assistant' ? 'assistant' : 'user';
-      return {
-        role,
-        content: msg.body
-      };
-    });
-    this._history = Private.sanitizeModelMessages(modelMessages);
+  setHistory(messages: ModelMessage[]): void {
+    this.stopStreaming('Chat history changed');
+    this._history = Private.sanitizeModelMessages(messages);
   }
 
   /**
@@ -526,11 +505,11 @@ export class AgentManager implements IAgentManager {
     this._controller?.abort();
 
     // Reject any pending approvals
-    for (const [approvalId, pending] of this._pendingApprovals) {
+    for (const [toolCallId, pending] of this._pendingApprovals) {
       pending.resolve(false, reason ?? 'Stream ended by user');
       this._agentEvent.emit({
         type: 'tool_approval_resolved',
-        data: { approvalId, approved: false }
+        data: { toolCallId, approved: false }
       });
     }
     this._pendingApprovals.clear();
@@ -538,34 +517,34 @@ export class AgentManager implements IAgentManager {
 
   /**
    * Approves a pending tool call.
-   * @param approvalId The approval ID to approve
+   * @param toolCallId The tool call ID to approve
    * @param reason Optional reason for approval
    */
-  approveToolCall(approvalId: string, reason?: string): void {
-    const pending = this._pendingApprovals.get(approvalId);
+  approveToolCall(toolCallId: string, reason?: string): void {
+    const pending = this._pendingApprovals.get(toolCallId);
     if (pending) {
       pending.resolve(true, reason);
-      this._pendingApprovals.delete(approvalId);
+      this._pendingApprovals.delete(toolCallId);
       this._agentEvent.emit({
         type: 'tool_approval_resolved',
-        data: { approvalId, approved: true }
+        data: { toolCallId, approved: true }
       });
     }
   }
 
   /**
    * Rejects a pending tool call.
-   * @param approvalId The approval ID to reject
+   * @param toolCallId The tool call ID to reject
    * @param reason Optional reason for rejection
    */
-  rejectToolCall(approvalId: string, reason?: string): void {
-    const pending = this._pendingApprovals.get(approvalId);
+  rejectToolCall(toolCallId: string, reason?: string): void {
+    const pending = this._pendingApprovals.get(toolCallId);
     if (pending) {
       pending.resolve(false, reason);
-      this._pendingApprovals.delete(approvalId);
+      this._pendingApprovals.delete(toolCallId);
       this._agentEvent.emit({
         type: 'tool_approval_resolved',
-        data: { approvalId, approved: false }
+        data: { toolCallId, approved: false }
       });
     }
   }
@@ -663,19 +642,10 @@ export class AgentManager implements IAgentManager {
             error.statusCode === 415 ||
             error.statusCode === 422)
         ) {
-          for (const msg of [...this._history, ...responseHistory]) {
-            if (msg.role === 'user' && Array.isArray(msg.content)) {
-              const hasMedia = msg.content.some(p => p.type !== 'text');
-              if (hasMedia) {
-                const textContent = msg.content
-                  .filter(p => p.type === 'text')
-                  .map(p => (p as { text: string }).text)
-                  .join('\n');
-                msg.content =
-                  textContent || '_Attachment removed due to error_';
-              }
-            }
-          }
+          this._stripAttachments(
+            [...this._history, ...responseHistory],
+            '_Attachment removed due to error_'
+          );
           helpMessage +=
             '\n\nAttachments have been removed from history. Please send your prompt again.';
         }
@@ -735,6 +705,27 @@ export class AgentManager implements IAgentManager {
     this._tokenUsageChanged.emit(this._tokenUsage);
   }
 
+  /**
+   * Removes image and file parts from all user messages in the given list.
+   */
+  private _stripAttachments(
+    messages: ModelMessage[],
+    placeholder: string
+  ): void {
+    for (const msg of messages) {
+      if (msg.role === 'user' && Array.isArray(msg.content)) {
+        const hasMedia = msg.content.some(p => p.type !== 'text');
+        if (hasMedia) {
+          const textContent = msg.content
+            .filter(p => p.type === 'text')
+            .map(p => (p as { text: string }).text)
+            .join('\n');
+          msg.content = textContent || placeholder;
+        }
+      }
+    }
+  }
+
   /**
    * Gets the configured context window for the active provider.
    */
@@ -1092,14 +1083,13 @@ ${richOutputWorkflowInstruction}`;
     this._agentEvent.emit({
       type: 'tool_approval_request',
       data: {
-        approvalId,
         toolCallId: toolCall.toolCallId,
         toolName: toolCall.toolName,
         args: toolCall.input
       }
     });
 
-    const approved = await this._waitForApproval(approvalId);
+    const approved = await this._waitForApproval(toolCall.toolCallId);
 
     result.approvalProcessed = true;
     result.approvalResponse = {
@@ -1116,12 +1106,12 @@ ${richOutputWorkflowInstruction}`;
 
   /**
    * Waits for user approval of a tool call.
-   * @param approvalId The approval ID to wait for
+   * @param toolCallId The tool call ID to wait for approval
    * @returns Promise that resolves to true if approved, false if rejected
    */
-  private _waitForApproval(approvalId: string): Promise<boolean> {
+  private _waitForApproval(toolCallId: string): Promise<boolean> {
     return new Promise(resolve => {
-      this._pendingApprovals.set(approvalId, {
+      this._pendingApprovals.set(toolCallId, {
         resolve: (approved: boolean) => {
           resolve(approved);
         }

package/src/chat-model-handler.ts

@@ -6,6 +6,7 @@ import { Contents } from '@jupyterlab/services';
 import { AIChatModel } from './chat-model';
 import type {
   IAgentManagerFactory,
+  IAIChatModel,
   IAISettingsModel,
   IChatModelHandler,
   ICreateChatOptions,
@@ -28,7 +29,7 @@ export class ChatModelHandler implements IChatModelHandler {
     this._contentsManager = options.contentsManager;
   }
 
-  createModel(options: ICreateChatOptions): AIChatModel {
+  createModel(options: ICreateChatOptions): IAIChatModel {
     const { name, activeProvider, tokenUsage, messages, autosave, title } =
       options;
 
@@ -49,7 +50,8 @@ export class ChatModelHandler implements IChatModelHandler {
       agentManager,
       activeCellManager: this._activeCellManager,
       documentManager: this._docManager,
-      contentsManager: this._contentsManager
+      contentsManager: this._contentsManager,
+      providerRegistry: this._providerRegistry
     });
 
     messages?.forEach(message => {