@jupyterlite/ai 0.12.0 → 0.14.0

package/lib/chat-model.js

@@ -1,5 +1,6 @@
 import { AbstractChatModel } from '@jupyter/chat';
 import { PathExt } from '@jupyterlab/coreutils';
+import * as nbformat from '@jupyterlab/nbformat';
 import { UUID } from '@lumino/coreutils';
 import { Signal } from '@lumino/signaling';
 import { AI_AVATAR } from './icons';
@@ -236,8 +237,9 @@ export class AIChatModel extends AbstractChatModel {
             type: 'msg',
             raw_time: false
         };
-        this._currentStreamingMessage = aiMessage;
         this.messageAdded(aiMessage);
+        this._currentStreamingMessage =
+            this.messages.find(message => message.id === aiMessage.id) ?? null;
     }
     /**
      * Handles streaming message chunks from the AI agent.
@@ -246,8 +248,7 @@ export class AIChatModel extends AbstractChatModel {
     _handleMessageChunk(event) {
         if (this._currentStreamingMessage &&
             this._currentStreamingMessage.id === event.data.messageId) {
-            this._currentStreamingMessage.body = event.data.fullContent;
-            this.messageAdded(this._currentStreamingMessage);
+            this._currentStreamingMessage.update({ body: event.data.fullContent });
         }
     }
     /**
@@ -257,8 +258,7 @@ export class AIChatModel extends AbstractChatModel {
     _handleMessageComplete(event) {
         if (this._currentStreamingMessage &&
             this._currentStreamingMessage.id === event.data.messageId) {
-            this._currentStreamingMessage.body = event.data.content;
-            this.messageAdded(this._currentStreamingMessage);
+            this._currentStreamingMessage.update({ body: event.data.content });
             this._currentStreamingMessage = null;
         }
     }
@@ -295,6 +295,21 @@ export class AIChatModel extends AbstractChatModel {
                         return parsedInput.name;
                     }
                     break;
+                case 'browser_fetch':
+                    if (parsedInput.url) {
+                        return parsedInput.url;
+                    }
+                    break;
+                case 'web_fetch':
+                    if (parsedInput.url) {
+                        return parsedInput.url;
+                    }
+                    break;
+                case 'web_search':
+                    if (parsedInput.query) {
+                        return `query: "${parsedInput.query}"`;
+                    }
+                    break;
             }
         }
         catch {
@@ -302,6 +317,22 @@ export class AIChatModel extends AbstractChatModel {
         }
         return '';
     }
+    /**
+     * Determine whether this tool call should auto-render trusted MIME bundles.
+     */
+    _computeShouldAutoRenderMimeBundles(toolName, input) {
+        if (toolName !== 'execute_command') {
+            return false;
+        }
+        try {
+            const parsedInput = JSON.parse(input);
+            return (typeof parsedInput.commandId === 'string' &&
+                this._settingsModel.config.commandsAutoRenderMimeBundles.includes(parsedInput.commandId));
+        }
+        catch {
+            return false;
+        }
+    }
     /**
      * Handles the start of a tool call execution.
      * @param event Event containing the tool call start data
@@ -309,13 +340,15 @@ export class AIChatModel extends AbstractChatModel {
     _handleToolCallStartEvent(event) {
         const messageId = UUID.uuid4();
         const summary = this._extractToolSummary(event.data.toolName, event.data.input);
+        const shouldAutoRenderMimeBundles = this._computeShouldAutoRenderMimeBundles(event.data.toolName, event.data.input);
         const context = {
             toolCallId: event.data.callId,
             messageId,
             toolName: event.data.toolName,
             input: event.data.input,
             status: 'pending',
-            summary
+            summary,
+            shouldAutoRenderMimeBundles
         };
         this._toolContexts.set(event.data.callId, context);
         const toolCallMessage = {
@@ -338,10 +371,38 @@ export class AIChatModel extends AbstractChatModel {
      * Handles the completion of a tool call execution.
      */
     _handleToolCallCompleteEvent(event) {
+        const context = this._toolContexts.get(event.data.callId);
         const status = event.data.isError ? 'error' : 'completed';
-        this._updateToolCallUI(event.data.callId, status, event.data.output);
+        this._updateToolCallUI(event.data.callId, status, Private.formatToolOutput(event.data.outputData));
+        if (!event.data.isError && this._shouldAutoRenderMimeBundles(context)) {
+            // Tool results are arbitrary command payloads (often wrapped in
+            // { success, result, outputs, ... }), so extract display outputs
+            // defensively instead of assuming a raw kernel message shape.
+            const mimeBundles = Private.extractMimeBundlesFromUnknown(event.data.outputData, {
+                trustedMimeTypes: this._settingsModel.config.trustedMimeTypesForAutoRender
+            });
+            for (const bundle of mimeBundles) {
+                this.messageAdded({
+                    body: bundle,
+                    sender: this._getAIUser(),
+                    id: UUID.uuid4(),
+                    time: Date.now() / 1000,
+                    type: 'msg',
+                    raw_time: false
+                });
+            }
+        }
         this._toolContexts.delete(event.data.callId);
     }
+    /**
+     * Determine whether a tool call output should auto-render MIME bundles.
+     */
+    _shouldAutoRenderMimeBundles(context) {
+        if (!context) {
+            return false;
+        }
+        return !!context.shouldAutoRenderMimeBundles;
+    }
     /**
      * Handles error events from the AI agent.
      */
@@ -638,6 +699,87 @@ export class AIChatModel extends AbstractChatModel {
 }
 var Private;
 (function (Private) {
+    const isPlainObject = (value) => {
+        return typeof value === 'object' && value !== null && !Array.isArray(value);
+    };
+    const isDisplayOutput = (value) => {
+        if (!isPlainObject(value)) {
+            return false;
+        }
+        const output = value;
+        return (nbformat.isDisplayData(output) ||
+            nbformat.isDisplayUpdate(output) ||
+            nbformat.isExecuteResult(output));
+    };
+    const toMimeBundle = (value, trustedMimeTypes) => {
+        const data = value.data;
+        if (!isPlainObject(data) || Object.keys(data).length === 0) {
+            return null;
+        }
+        return {
+            data: data,
+            ...(isPlainObject(value.metadata)
+                ? { metadata: value.metadata }
+                : {}),
+            // MIME auto-rendering only runs for explicitly configured command IDs.
+            // Trust handling is configurable to keep risky MIME execution opt-in.
+            ...(Object.keys(data).some(m => trustedMimeTypes.has(m))
+                ? { trusted: true }
+                : {})
+        };
+    };
+    /**
+     * Normalize arbitrary tool payloads into canonical display outputs.
+     *
+     * Tool outputs are not guaranteed to be raw Jupyter IOPub messages; they are
+     * often wrapped objects (for example `{ success, result: { outputs: [...] } }`).
+     */
+    const toDisplayOutputs = (value) => {
+        if (isDisplayOutput(value)) {
+            return [value];
+        }
+        if (Array.isArray(value)) {
+            return value.filter(isDisplayOutput);
+        }
+        if (!isPlainObject(value)) {
+            return [];
+        }
+        if (Array.isArray(value.outputs)) {
+            return value.outputs.filter(isDisplayOutput);
+        }
+        if ('result' in value) {
+            return toDisplayOutputs(value.result);
+        }
+        return [];
+    };
+    /**
+     * Extract rendermime-ready mime bundles from arbitrary tool results.
+     */
+    function extractMimeBundlesFromUnknown(content, options = {}) {
+        const bundles = [];
+        const outputs = toDisplayOutputs(content);
+        const trustedMimeTypes = new Set(options.trustedMimeTypes ?? []);
+        for (const output of outputs) {
+            const bundle = toMimeBundle(output, trustedMimeTypes);
+            if (bundle) {
+                bundles.push(bundle);
+            }
+        }
+        return bundles;
+    }
+    Private.extractMimeBundlesFromUnknown = extractMimeBundlesFromUnknown;
+    function formatToolOutput(outputData) {
+        if (typeof outputData === 'string') {
+            return outputData;
+        }
+        try {
+            return JSON.stringify(outputData, null, 2);
+        }
+        catch {
+            return '[Complex object - cannot serialize]';
+        }
+    }
+    Private.formatToolOutput = formatToolOutput;
     function escapeHtml(value) {
         // Prefer the same native escaping approach used in JupyterLab itself
         // (e.g. `@jupyterlab/completer`).
@@ -737,7 +879,7 @@ var Private;
 <pre class="jp-ai-tool-code"><code>${escapedOutput}</code></pre>
 </div>`;
         }
-        return `<details class="jp-ai-tool-call ${config.cssClass}"${openAttr}>
+        const HTMLContent = `<details class="jp-ai-tool-call ${config.cssClass}"${openAttr}>
 <summary class="jp-ai-tool-header">
 <div class="jp-ai-tool-icon">⚡</div>
 <div class="jp-ai-tool-title">${escapedToolName}${summaryHtml}</div>
@@ -746,6 +888,12 @@ var Private;
 <div class="jp-ai-tool-body">${bodyContent}
 </div>
 </details>`;
+        return {
+            trusted: true,
+            data: {
+                'text/html': HTMLContent
+            }
+        };
     }
     Private.buildToolCallHtml = buildToolCallHtml;
 })(Private || (Private = {}));

package/lib/completion/completion-provider.d.ts

@@ -80,6 +80,6 @@ export declare namespace AICompletionProvider {
         /**
          * The token used to request the secrets manager.
          */
-        token: symbol;
+        token: symbol | null;
     }
 }

package/lib/completion/completion-provider.js

@@ -22,6 +22,10 @@ export class AICompletionProvider {
             this._updateModel();
         });
         this._updateModel();
+        // Disable the secrets manager if the token is empty.
+        if (!options.token) {
+            this._secretsManager = undefined;
+        }
     }
     /**
      * The unique identifier of the provider.
@@ -115,8 +119,16 @@ export class AICompletionProvider {
         const baseURL = activeProvider.baseURL;
         let apiKey;
         if (this._secretsManager && this._settingsModel.config.useSecretsManager) {
-            apiKey =
-                (await this._secretsManager.get(Private.getToken(), SECRETS_NAMESPACE, `${provider}:apiKey`))?.value ?? '';
+            const token = Private.getToken();
+            if (!token) {
+                // This should never happen, the secrets manager should be disabled.
+                console.error('@jupyterlite/ai::AICompletionProvider error: the settings manager token is not set.\nYou should disable the secrets manager from the AI settings.');
+                apiKey = '';
+            }
+            else {
+                apiKey =
+                    (await this._secretsManager.get(token, SECRETS_NAMESPACE, `${provider}:apiKey`))?.value ?? '';
+            }
         }
         else {
             apiKey = this._settingsModel.getApiKey(activeProvider.id);

package/lib/components/model-select.js

@@ -115,11 +115,11 @@ export function ModelSelect(props) {
                 e.stopPropagation();
             }, sx: {
                 backgroundColor: isSelected
-                    ? 'var(--jp-brand-color3, rgba(33, 150, 243, 0.1))'
+                    ? 'var(--mui-palette-primary-main)'
                     : 'transparent',
                 '&:hover': {
                     backgroundColor: isSelected
-                        ? 'var(--jp-brand-color3, rgba(33, 150, 243, 0.15))'
+                        ? 'var(--mui-palette-primary-main)'
                         : 'var(--jp-layout-color1)'
                 },
                 display: 'flex',
@@ -127,13 +127,13 @@ export function ModelSelect(props) {
                 gap: '8px'
             } },
             isSelected ? (React.createElement(CheckIcon, { sx: {
-                    color: 'var(--jp-brand-color1, #2196F3)',
+                    color: 'var(--jp-ui-inverse-font-color1)',
                     fontSize: 16
                 } })) : (React.createElement("div", { style: { width: '16px' } })),
             React.createElement(Typography, { variant: "body2", component: "div", sx: {
                     fontWeight: isSelected ? 600 : 400,
                     color: isSelected
-                        ? 'var(--jp-brand-color1, #2196F3)'
+                        ? 'var(--jp-ui-inverse-font-color1)'
                         : 'inherit'
                 } }, providerLabel)))))));
 }

package/lib/components/tool-select.d.ts

@@ -1,6 +1,7 @@
 import { InputToolbarRegistry } from '@jupyter/chat';
 import type { TranslationBundle } from '@jupyterlab/translation';
-import { IToolRegistry } from '../tokens';
+import { IProviderRegistry, IToolRegistry } from '../tokens';
+import { AISettingsModel } from '../models/settings-model';
 /**
  * Properties for the tool select component.
  */
@@ -17,6 +18,14 @@ export interface IToolSelectProps extends InputToolbarRegistry.IToolbarItemProps
      * Function to handle tool selection changes.
      */
     onToolSelectionChange: (selectedToolNames: string[]) => void;
+    /**
+     * The settings model to compute provider-level web tools.
+     */
+    settingsModel: AISettingsModel;
+    /**
+     * Registry for provider metadata used to resolve provider tool capabilities.
+     */
+    providerRegistry: IProviderRegistry;
     /**
      * The application language translator.
      */
@@ -29,4 +38,4 @@ export declare function ToolSelect(props: IToolSelectProps): JSX.Element;
 /**
  * Factory function returning the toolbar item for tool selection.
  */
-export declare function createToolSelectItem(toolRegistry: IToolRegistry, toolsEnabled: boolean | undefined, translator: TranslationBundle): InputToolbarRegistry.IToolbarItem;
+export declare function createToolSelectItem(toolRegistry: IToolRegistry, settingsModel: AISettingsModel, providerRegistry: IProviderRegistry, toolsEnabled: boolean | undefined, translator: TranslationBundle): InputToolbarRegistry.IToolbarItem;

package/lib/components/tool-select.js

@@ -1,16 +1,20 @@
 import { TooltippedButton } from '@jupyter/chat';
 import BuildIcon from '@mui/icons-material/Build';
 import CheckIcon from '@mui/icons-material/Check';
-import { Menu, MenuItem, Tooltip, Typography } from '@mui/material';
+import { Divider, Menu, MenuItem, Tooltip, Typography } from '@mui/material';
 import React, { useCallback, useEffect, useState } from 'react';
+import { createProviderTools } from '../providers/provider-tools';
 const SELECT_ITEM_CLASS = 'jp-AIToolSelect-item';
 /**
  * The tool select component for choosing AI tools.
  */
 export function ToolSelect(props) {
-    const { toolRegistry, onToolSelectionChange, toolsEnabled, translator: trans } = props;
+    const { toolRegistry, onToolSelectionChange, toolsEnabled, settingsModel, providerRegistry, model, translator: trans } = props;
+    const chatContext = model.chatContext;
+    const agentManager = chatContext.agentManager;
     const [selectedToolNames, setSelectedToolNames] = useState([]);
     const [tools, setTools] = useState(toolRegistry?.namedTools || []);
+    const [providerToolNames, setProviderToolNames] = useState([]);
     const [menuAnchorEl, setMenuAnchorEl] = useState(null);
     const [menuOpen, setMenuOpen] = useState(false);
     const openMenu = useCallback((el) => {
@@ -48,6 +52,41 @@ export function ToolSelect(props) {
             };
         }
     }, [toolRegistry]);
+    // Track provider-level tools (e.g. web_search/web_fetch).
+    useEffect(() => {
+        if (!agentManager || !toolsEnabled) {
+            setProviderToolNames([]);
+            return;
+        }
+        const updateProviderTools = () => {
+            const activeProviderId = agentManager.activeProvider;
+            const providerConfig = settingsModel.getProvider(activeProviderId);
+            if (!providerConfig) {
+                setProviderToolNames([]);
+                return;
+            }
+            const providerInfo = providerRegistry.getProviderInfo(providerConfig.provider);
+            const providerTools = createProviderTools({
+                providerInfo,
+                customSettings: providerConfig.customSettings,
+                hasFunctionTools: selectedToolNames.length > 0
+            });
+            setProviderToolNames(Object.keys(providerTools));
+        };
+        updateProviderTools();
+        settingsModel.stateChanged.connect(updateProviderTools);
+        agentManager.activeProviderChanged.connect(updateProviderTools);
+        return () => {
+            settingsModel.stateChanged.disconnect(updateProviderTools);
+            agentManager.activeProviderChanged.disconnect(updateProviderTools);
+        };
+    }, [
+        settingsModel,
+        providerRegistry,
+        agentManager,
+        selectedToolNames.length,
+        toolsEnabled
+    ]);
     // Initialize selected tools to all tools by default
     useEffect(() => {
         if (tools.length > 0 && selectedToolNames.length === 0) {
@@ -57,14 +96,16 @@ export function ToolSelect(props) {
         }
     }, [tools, selectedToolNames.length, onToolSelectionChange]);
     // Don't render if tools are disabled or no tools available
-    if (!toolsEnabled || tools.length === 0) {
+    if (!toolsEnabled || (tools.length === 0 && providerToolNames.length === 0)) {
         return React.createElement(React.Fragment, null);
     }
+    const selectedCount = selectedToolNames.length + providerToolNames.length;
+    const totalCount = tools.length + providerToolNames.length;
     return (React.createElement(React.Fragment, null,
         React.createElement(TooltippedButton, { onClick: e => {
                 openMenu(e.currentTarget);
-            }, tooltip: trans.__('Tools (%1/%2 selected)', selectedToolNames.length.toString(), tools.length.toString()), buttonProps: {
-                ...(selectedToolNames.length === 0 && {
+            }, tooltip: trans.__('Tools (%1/%2 selected)', selectedCount.toString(), totalCount.toString()), buttonProps: {
+                ...(selectedCount === 0 && {
                     variant: 'outlined'
                 }),
                 title: trans.__('Select AI Tools'),
@@ -76,7 +117,7 @@ export function ToolSelect(props) {
                     // Stop propagation to prevent sending message
                     e.stopPropagation();
                 }
-            }, sx: selectedToolNames.length === 0
+            }, sx: selectedCount === 0
                 ? { backgroundColor: 'var(--jp-layout-color3)' }
                 : {} },
             React.createElement(BuildIcon, { sx: { fontSize: 'small' } })),
@@ -91,22 +132,38 @@ export function ToolSelect(props) {
                     padding: '0.5em',
                     paddingRight: '2em'
                 }
-            } }, tools.map(namedTool => (React.createElement(Tooltip, { key: namedTool.name, title: namedTool.tool.description || namedTool.name, placement: "left" },
-            React.createElement(MenuItem, { className: SELECT_ITEM_CLASS, onClick: e => {
-                    toggleTool(namedTool.name);
-                    // Prevent sending message on tool selection
-                    e.stopPropagation();
-                } },
-                selectedToolNames.includes(namedTool.name) ? (React.createElement(CheckIcon, { sx: {
-                        marginRight: '8px',
-                        color: 'var(--jp-brand-color1, #2196F3)'
-                    } })) : (React.createElement("div", { style: { width: '24px', marginRight: '8px' } })),
-                React.createElement(Typography, { variant: "body2" }, namedTool.name))))))));
+            } },
+            tools.map(namedTool => (React.createElement(Tooltip, { key: namedTool.name, title: namedTool.tool.description || namedTool.name, placement: "left" },
+                React.createElement(MenuItem, { className: SELECT_ITEM_CLASS, onClick: e => {
+                        toggleTool(namedTool.name);
+                        // Prevent sending message on tool selection
+                        e.stopPropagation();
+                    } },
+                    selectedToolNames.includes(namedTool.name) ? (React.createElement(CheckIcon, { sx: {
+                            marginRight: '8px',
+                            color: 'var(--jp-brand-color1, #2196F3)'
+                        } })) : (React.createElement("div", { style: { width: '24px', marginRight: '8px' } })),
+                    React.createElement(Typography, { variant: "body2" }, namedTool.name))))),
+            providerToolNames.length > 0 && tools.length > 0 && React.createElement(Divider, null),
+            providerToolNames.length > 0 && (React.createElement(MenuItem, { disabled: true },
+                React.createElement(Typography, { variant: "caption" }, trans.__('Provider Tools')))),
+            providerToolNames.map(toolName => {
+                return (React.createElement(Tooltip, { key: toolName, title: trans.__('Enabled via provider settings.'), placement: "left" },
+                    React.createElement(MenuItem, { className: SELECT_ITEM_CLASS, onClick: e => {
+                            // Keep provider-managed tools read-only from this menu.
+                            e.stopPropagation();
+                        } },
+                        React.createElement(CheckIcon, { sx: {
+                                marginRight: '8px',
+                                color: 'text.disabled'
+                            } }),
+                        React.createElement(Typography, { variant: "body2" }, toolName))));
+            }))));
 }
 /**
  * Factory function returning the toolbar item for tool selection.
  */
-export function createToolSelectItem(toolRegistry, toolsEnabled = true, translator) {
+export function createToolSelectItem(toolRegistry, settingsModel, providerRegistry, toolsEnabled = true, translator) {
     return {
         element: (props) => {
             const onToolSelectionChange = (tools) => {
@@ -120,6 +177,8 @@ export function createToolSelectItem(toolRegistry, toolsEnabled = true, translat
             const toolSelectProps = {
                 ...props,
                 toolRegistry,
+                settingsModel,
+                providerRegistry,
                 onToolSelectionChange,
                 toolsEnabled,
                 translator

package/lib/index.d.ts

@@ -1,9 +1,9 @@
 import { JupyterFrontEndPlugin } from '@jupyterlab/application';
-import { IChatCommandRegistry, IInputToolbarRegistryFactory } from '@jupyter/chat';
+import { IChatCommandRegistry, IChatTracker, IInputToolbarRegistryFactory } from '@jupyter/chat';
 import { AgentManagerFactory } from './agent';
-import { IProviderRegistry, IToolRegistry, ISkillRegistry, IChatModelRegistry, IDiffManager } from './tokens';
+import { IProviderRegistry, IToolRegistry, ISkillRegistry, IChatModelHandler, IDiffManager } from './tokens';
 import { AISettingsModel } from './models/settings-model';
-declare const _default: (JupyterFrontEndPlugin<IProviderRegistry> | JupyterFrontEndPlugin<void> | JupyterFrontEndPlugin<IChatCommandRegistry> | JupyterFrontEndPlugin<IChatModelRegistry> | JupyterFrontEndPlugin<AgentManagerFactory> | JupyterFrontEndPlugin<AISettingsModel> | JupyterFrontEndPlugin<IDiffManager> | JupyterFrontEndPlugin<ISkillRegistry> | JupyterFrontEndPlugin<IToolRegistry> | JupyterFrontEndPlugin<IInputToolbarRegistryFactory>)[];
+declare const _default: (JupyterFrontEndPlugin<IProviderRegistry> | JupyterFrontEndPlugin<void> | JupyterFrontEndPlugin<IChatCommandRegistry> | JupyterFrontEndPlugin<IChatModelHandler> | JupyterFrontEndPlugin<IChatTracker> | JupyterFrontEndPlugin<AgentManagerFactory> | JupyterFrontEndPlugin<AISettingsModel> | JupyterFrontEndPlugin<IDiffManager> | JupyterFrontEndPlugin<ISkillRegistry> | JupyterFrontEndPlugin<IToolRegistry> | JupyterFrontEndPlugin<IInputToolbarRegistryFactory>)[];
 export default _default;
 export * from './tokens';
 export * from './icons';