@theia/ai-chat 1.63.0-next.24 → 1.63.0-next.52

package/src/browser/file-chat-variable-contribution.ts

@@ -17,13 +17,14 @@
 import { AIVariableContext, AIVariableResolutionRequest, PromptText } from '@theia/ai-core';
 import { AIVariableCompletionContext, AIVariableDropResult, FrontendVariableContribution, FrontendVariableService } from '@theia/ai-core/lib/browser';
 import { FILE_VARIABLE } from '@theia/ai-core/lib/browser/file-variable-contribution';
-import { CancellationToken, QuickInputService, URI } from '@theia/core';
+import { CancellationToken, ILogger, QuickInputService, URI } from '@theia/core';
 import { inject, injectable } from '@theia/core/shared/inversify';
 import * as monaco from '@theia/monaco-editor-core';
 import { FileQuickPickItem, QuickFileSelectService } from '@theia/file-search/lib/browser/quick-file-select-service';
 import { WorkspaceService } from '@theia/workspace/lib/browser';
 import { FileService } from '@theia/filesystem/lib/browser/file-service';
 import { VARIABLE_ADD_CONTEXT_COMMAND } from './ai-chat-frontend-contribution';
+import { IMAGE_CONTEXT_VARIABLE, ImageContextVariable } from '../common/image-context-variable';
 
 @injectable()
 export class FileChatVariableContribution implements FrontendVariableContribution {
@@ -39,8 +40,12 @@ export class FileChatVariableContribution implements FrontendVariableContributio
     @inject(QuickFileSelectService)
     protected readonly quickFileSelectService: QuickFileSelectService;
 
+    @inject(ILogger)
+    protected readonly logger: ILogger;
+
     registerVariables(service: FrontendVariableService): void {
         service.registerArgumentPicker(FILE_VARIABLE, this.triggerArgumentPicker.bind(this));
+        service.registerArgumentPicker(IMAGE_CONTEXT_VARIABLE, this.imageArgumentPicker.bind(this));
         service.registerArgumentCompletionProvider(FILE_VARIABLE, this.provideArgumentCompletionItems.bind(this));
         service.registerDropHandler(this.handleDrop.bind(this));
     }
@@ -68,6 +73,57 @@ export class FileChatVariableContribution implements FrontendVariableContributio
         });
     }
 
+    protected async imageArgumentPicker(): Promise<string | undefined> {
+        const quickPick = this.quickInputService.createQuickPick();
+        quickPick.title = 'Select an image file';
+
+        // Get all files and filter only image files
+        const allPicks = await this.quickFileSelectService.getPicks();
+        quickPick.items = allPicks.filter(item => {
+            if (FileQuickPickItem.is(item)) {
+                return this.isImageFile(item.uri.path.toString());
+            }
+            return false;
+        });
+
+        const updateItems = async (value: string) => {
+            const filteredPicks = await this.quickFileSelectService.getPicks(value, CancellationToken.None);
+            quickPick.items = filteredPicks.filter(item => {
+                if (FileQuickPickItem.is(item)) {
+                    return this.isImageFile(item.uri.path.toString());
+                }
+                return false;
+            });
+        };
+
+        const onChangeListener = quickPick.onDidChangeValue(updateItems);
+        quickPick.show();
+
+        return new Promise(resolve => {
+            quickPick.onDispose(onChangeListener.dispose);
+            quickPick.onDidAccept(async () => {
+                const selectedItem = quickPick.selectedItems[0];
+                if (selectedItem && FileQuickPickItem.is(selectedItem)) {
+                    quickPick.dispose();
+                    const filePath = await this.wsService.getWorkspaceRelativePath(selectedItem.uri);
+                    const fileName = selectedItem.uri.displayName;
+                    const base64Data = await this.fileToBase64(selectedItem.uri);
+                    const mimeType = this.getMimeTypeFromExtension(selectedItem.uri.path.toString());
+
+                    // Create the argument string in the required format
+                    const imageVarArgs: ImageContextVariable = {
+                        name: fileName,
+                        wsRelativePath: filePath,
+                        data: base64Data,
+                        mimeType: mimeType
+                    };
+
+                    resolve(ImageContextVariable.createArgString(imageVarArgs));
+                }
+            });
+        });
+    }
+
     protected async provideArgumentCompletionItems(
         model: monaco.editor.ITextModel,
         position: monaco.Position,
@@ -106,6 +162,50 @@ export class FileChatVariableContribution implements FrontendVariableContributio
         );
     }
 
+    /**
+     * Checks if a file is an image based on its extension.
+     */
+    protected isImageFile(filePath: string): boolean {
+        const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.svg', '.webp'];
+        const extension = filePath.toLowerCase().substring(filePath.lastIndexOf('.'));
+        return imageExtensions.includes(extension);
+    }
+
+    /**
+     * Determines the MIME type based on file extension.
+     */
+    protected getMimeTypeFromExtension(filePath: string): string {
+        const extension = filePath.toLowerCase().substring(filePath.lastIndexOf('.'));
+        const mimeTypes: { [key: string]: string } = {
+            '.jpg': 'image/jpeg',
+            '.jpeg': 'image/jpeg',
+            '.png': 'image/png',
+            '.gif': 'image/gif',
+            '.bmp': 'image/bmp',
+            '.svg': 'image/svg+xml',
+            '.webp': 'image/webp'
+        };
+        return mimeTypes[extension] || 'application/octet-stream';
+    }
+
+    /**
+     * Converts a file to base64 data URL.
+     */
+    protected async fileToBase64(uri: URI): Promise<string> {
+        try {
+            const fileContent = await this.fileService.readFile(uri);
+            const uint8Array = new Uint8Array(fileContent.value.buffer);
+            let binary = '';
+            for (let i = 0; i < uint8Array.length; i++) {
+                binary += String.fromCharCode(uint8Array[i]);
+            }
+            return btoa(binary);
+        } catch (error) {
+            this.logger.error('Error reading file content:', error);
+            return '';
+        }
+    }
+
     protected async handleDrop(event: DragEvent, _: AIVariableContext): Promise<AIVariableDropResult | undefined> {
         const data = event.dataTransfer?.getData('selected-tree-nodes');
         if (!data) {
@@ -126,11 +226,25 @@ export class FileChatVariableContribution implements FrontendVariableContributio
                 const uri = URI.fromFilePath(filePath);
                 if (await this.fileService.exists(uri)) {
                     const wsRelativePath = await this.wsService.getWorkspaceRelativePath(uri);
-                    variables.push({
-                        variable: FILE_VARIABLE,
-                        arg: wsRelativePath
-                    });
-                    texts.push(`${PromptText.VARIABLE_CHAR}${FILE_VARIABLE.name}${PromptText.VARIABLE_SEPARATOR_CHAR}${wsRelativePath}`);
+                    const fileName = uri.displayName;
+
+                    if (this.isImageFile(filePath)) {
+                        const base64Data = await this.fileToBase64(uri);
+                        const mimeType = this.getMimeTypeFromExtension(filePath);
+                        variables.push(ImageContextVariable.createRequest({
+                            [ImageContextVariable.name]: fileName,
+                            [ImageContextVariable.wsRelativePath]: wsRelativePath,
+                            [ImageContextVariable.data]: base64Data,
+                            [ImageContextVariable.mimeType]: mimeType
+                        }));
+                        // we do not want to push a text for image variables
+                    } else {
+                        variables.push({
+                            variable: FILE_VARIABLE,
+                            arg: wsRelativePath
+                        });
+                        texts.push(`${PromptText.VARIABLE_CHAR}${FILE_VARIABLE.name}${PromptText.VARIABLE_SEPARATOR_CHAR}${wsRelativePath}`);
+                    }
                 }
             }
 

package/src/browser/image-context-variable-contribution.ts

@@ -0,0 +1,153 @@
+// *****************************************************************************
+// Copyright (C) 2025 EclipseSource GmbH and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import {
+    AIVariableContext, AIVariableContribution,
+    AIVariableOpener, AIVariableResolutionRequest, AIVariableResolver, ResolvedAIContextVariable
+} from '@theia/ai-core';
+import { FrontendVariableService, AIVariablePasteResult } from '@theia/ai-core/lib/browser';
+import { Path, URI } from '@theia/core';
+import { LabelProvider, LabelProviderContribution, open, OpenerService } from '@theia/core/lib/browser';
+import { inject, injectable } from '@theia/core/shared/inversify';
+import { FileService } from '@theia/filesystem/lib/browser/file-service';
+import { WorkspaceService } from '@theia/workspace/lib/browser';
+import { IMAGE_CONTEXT_VARIABLE, ImageContextVariable, ImageContextVariableRequest } from '../common/image-context-variable';
+
+@injectable()
+export class ImageContextVariableContribution implements AIVariableContribution, AIVariableResolver, AIVariableOpener, LabelProviderContribution {
+    @inject(FileService)
+    protected readonly fileService: FileService;
+
+    @inject(WorkspaceService)
+    protected readonly wsService: WorkspaceService;
+
+    @inject(OpenerService)
+    protected readonly openerService: OpenerService;
+
+    @inject(LabelProvider)
+    protected readonly labelProvider: LabelProvider;
+
+    registerVariables(service: FrontendVariableService): void {
+        service.registerResolver(IMAGE_CONTEXT_VARIABLE, this);
+        service.registerOpener(IMAGE_CONTEXT_VARIABLE, this);
+        service.registerPasteHandler(this.handlePaste.bind(this));
+    }
+
+    async canResolve(request: AIVariableResolutionRequest, _: AIVariableContext): Promise<number> {
+        return ImageContextVariable.isImageContextRequest(request) ? 1 : 0;
+    }
+
+    async resolve(request: AIVariableResolutionRequest, _: AIVariableContext): Promise<ResolvedAIContextVariable | undefined> {
+        return ImageContextVariable.resolve(request as ImageContextVariableRequest);
+    }
+
+    async canOpen(request: AIVariableResolutionRequest, context: AIVariableContext): Promise<number> {
+        return ImageContextVariable.isImageContextRequest(request) && !!ImageContextVariable.parseRequest(request)?.wsRelativePath ? 1 : 0;
+    }
+
+    async open(request: ImageContextVariableRequest, context: AIVariableContext): Promise<void> {
+        const uri = await this.toUri(request);
+        if (!uri) {
+            throw new Error('Unable to resolve URI for request.');
+        }
+        await open(this.openerService, uri);
+    }
+
+    protected async toUri(request: ImageContextVariableRequest): Promise<URI | undefined> {
+        const variable = ImageContextVariable.parseRequest(request);
+        return variable?.wsRelativePath ? this.makeAbsolute(variable.wsRelativePath) : undefined;
+    }
+
+    async handlePaste(event: ClipboardEvent, context: AIVariableContext): Promise<AIVariablePasteResult | undefined> {
+        if (!event.clipboardData?.items) { return undefined; }
+
+        const variables: AIVariableResolutionRequest[] = [];
+
+        for (const item of event.clipboardData.items) {
+            if (item.type.startsWith('image/')) {
+                const blob = item.getAsFile();
+                if (blob) {
+                    try {
+                        const dataUrl = await this.readFileAsDataURL(blob);
+                        // Extract the base64 data by removing the data URL prefix
+                        // Format is like: data:image/png;base64,BASE64DATA
+                        const imageData = dataUrl.substring(dataUrl.indexOf(',') + 1);
+                        variables.push(ImageContextVariable.createRequest({
+                            data: imageData,
+                            name: blob.name || `pasted-image-${Date.now()}.png`,
+                            mimeType: blob.type
+                        }));
+                    } catch (error) {
+                        console.error('Failed to process pasted image:', error);
+                    }
+                }
+            }
+        }
+
+        return variables.length > 0 ? { variables } : undefined;
+    }
+
+    private readFileAsDataURL(blob: Blob): Promise<string> {
+        return new Promise((resolve, reject) => {
+            const reader = new FileReader();
+            reader.onload = e => {
+                if (!e.target?.result) {
+                    reject(new Error('Failed to read file as data URL'));
+                    return;
+                }
+                resolve(e.target.result as string);
+            };
+            reader.onerror = () => reject(reader.error);
+            reader.readAsDataURL(blob);
+        });
+    }
+
+    protected async makeAbsolute(pathStr: string): Promise<URI | undefined> {
+        const path = new Path(Path.normalizePathSeparator(pathStr));
+        if (!path.isAbsolute) {
+            const workspaceRoots = this.wsService.tryGetRoots();
+            const wsUris = workspaceRoots.map(root => root.resource.resolve(path));
+            for (const uri of wsUris) {
+                if (await this.fileService.exists(uri)) {
+                    return uri;
+                }
+            }
+        }
+        const argUri = new URI(pathStr);
+        if (await this.fileService.exists(argUri)) {
+            return argUri;
+        }
+        return undefined;
+    }
+
+    canHandle(element: object): number {
+        return ImageContextVariable.isImageContextRequest(element) ? 10 : -1;
+    }
+
+    getIcon(element: ImageContextVariableRequest): string | undefined {
+        const path = ImageContextVariable.parseArg(element.arg).wsRelativePath;
+        return path ? this.labelProvider.getIcon(new URI(path)) : undefined;
+    }
+
+    getName(element: ImageContextVariableRequest): string | undefined {
+        return ImageContextVariable.parseArg(element.arg).name;
+    }
+
+    getDetails(element: ImageContextVariableRequest): string | undefined {
+        const path = ImageContextVariable.parseArg(element.arg).wsRelativePath;
+        return path ? this.labelProvider.getDetails(new URI(path)) : '[pasted]';
+    }
+}

package/src/browser/task-context-service.ts

@@ -15,12 +15,15 @@
 // *****************************************************************************
 
 import { inject, injectable } from '@theia/core/shared/inversify';
-import { MaybePromise, ProgressService, URI, generateUuid, Event } from '@theia/core';
+import { MaybePromise, ProgressService, URI, generateUuid, Event, EOL } from '@theia/core';
 import { ChatAgent, ChatAgentLocation, ChatService, ChatSession, MutableChatModel, MutableChatRequestModel, ParsedChatRequestTextPart } from '../common';
+import { PreferenceService } from '@theia/core/lib/browser';
 import { ChatSessionSummaryAgent } from '../common/chat-session-summary-agent';
 import { Deferred } from '@theia/core/lib/common/promise-util';
-import { AgentService, PromptService } from '@theia/ai-core';
+import { AgentService, PromptService, ResolvedPromptFragment } from '@theia/ai-core';
 import { CHAT_SESSION_SUMMARY_PROMPT } from '../common/chat-session-summary-agent-prompt';
+import { ChangeSetFileElementFactory } from './change-set-file-element';
+import * as yaml from 'js-yaml';
 
 export interface SummaryMetadata {
     label: string;
@@ -33,7 +36,7 @@ export interface Summary extends SummaryMetadata {
     id: string;
 }
 
-export const TaskContextStorageService = Symbol('TextContextStorageService');
+export const TaskContextStorageService = Symbol('TaskContextStorageService');
 export interface TaskContextStorageService {
     onDidChange: Event<void>;
     store(summary: Summary): MaybePromise<void>;
@@ -53,6 +56,9 @@ export class TaskContextService {
     @inject(PromptService) protected readonly promptService: PromptService;
     @inject(TaskContextStorageService) protected readonly storageService: TaskContextStorageService;
     @inject(ProgressService) protected readonly progressService: ProgressService;
+    @inject(PreferenceService) protected readonly preferenceService: PreferenceService;
+    @inject(ChangeSetFileElementFactory)
+    protected readonly fileChangeFactory: ChangeSetFileElementFactory;
 
     get onDidChange(): Event<void> {
         return this.storageService.onDidChange;
@@ -77,18 +83,19 @@ export class TaskContextService {
     }
 
     /** Returns an ID that can be used to refer to the summary in the future. */
-    async summarize(session: ChatSession, promptId?: string, agent?: ChatAgent): Promise<string> {
+    async summarize(session: ChatSession, promptId?: string, agent?: ChatAgent, override = true): Promise<string> {
         const pending = this.pendingSummaries.get(session.id);
         if (pending) { return pending.then(({ id }) => id); }
         const existing = this.getSummaryForSession(session);
-        if (existing) { return existing.id; }
+        if (existing && !override) { return existing.id; }
         const summaryId = generateUuid();
         const summaryDeferred = new Deferred<Summary>();
         const progress = await this.progressService.showProgress({ text: `Summarize: ${session.title || session.id}`, options: { location: 'ai-chat' } });
         this.pendingSummaries.set(session.id, summaryDeferred.promise);
         try {
+            const prompt = await this.getSystemPrompt(session, promptId);
             const newSummary: Summary = {
-                summary: await this.getLlmSummary(session, promptId, agent),
+                summary: await this.getLlmSummary(session, prompt, agent),
                 label: session.title || session.id,
                 sessionId: session.id,
                 id: summaryId
@@ -97,6 +104,13 @@ export class TaskContextService {
             return summaryId;
         } catch (err) {
             summaryDeferred.reject(err);
+            const errorSummary: Summary = {
+                summary: `Summary creation failed: ${err instanceof Error ? err.message : typeof err === 'string' ? err : 'Unknown error'}`,
+                label: session.title || session.id,
+                sessionId: session.id,
+                id: summaryId
+            };
+            await this.storageService.store(errorSummary);
             throw err;
         } finally {
             progress.cancel();
@@ -104,7 +118,95 @@ export class TaskContextService {
         }
     }
 
-    protected async getLlmSummary(session: ChatSession, promptId: string = CHAT_SESSION_SUMMARY_PROMPT.id, agent?: ChatAgent): Promise<string> {
+    async update(session: ChatSession, promptId?: string, agent?: ChatAgent, override = true): Promise<string> {
+        // Get the existing summary for the session
+        const existingSummary = this.getSummaryForSession(session);
+        if (!existingSummary) {
+            // If no summary exists, create one instead
+            // TODO: Maybe we could also look into the task context folder and ask for the existing ones with an additional menu to create a new one?
+            return this.summarize(session, promptId, agent, override);
+        }
+
+        const progress = await this.progressService.showProgress({ text: `Updating: ${session.title || session.id}`, options: { location: 'ai-chat' } });
+        try {
+            const prompt = await this.getSystemPrompt(session, promptId);
+            if (!prompt) {
+                return '';
+            }
+
+            // Get the task context file path
+            const taskContextStorageDirectory = this.preferenceService.get(
+                // preference key is defined in TASK_CONTEXT_STORAGE_DIRECTORY_PREF in @theia/ai-ide
+                'ai-features.promptTemplates.taskContextStorageDirectory',
+                '.prompts/task-contexts'
+            );
+            const taskContextFileVariable = session.model.context.getVariables().find(variableReq => variableReq.variable.id === 'file-provider' &&
+                typeof variableReq.arg === 'string' &&
+                (variableReq.arg.startsWith(taskContextStorageDirectory)));
+
+            // Check if we have a document path to update
+            if (taskContextFileVariable && typeof taskContextFileVariable.arg === 'string') {
+                // Set document path in prompt template
+                const documentPath = taskContextFileVariable.arg;
+
+                // Modify prompt to include the document path and content
+                prompt.text = prompt.text + '\nThe document to update is: ' + documentPath + '\n\n## Current Document Content\n\n' + existingSummary.summary;
+
+                // Get updated document content from LLM
+                const updatedDocumentContent = await this.getLlmSummary(session, prompt, agent);
+
+                if (existingSummary.uri) {
+                    // updated document metadata shall be updated.
+                    // otherwise, frontmatter won't be set
+                    const frontmatter = {
+                        sessionId: existingSummary.sessionId,
+                        date: new Date().toISOString(),
+                        label: existingSummary.label,
+                    };
+                    const content = yaml.dump(frontmatter).trim() + `${EOL}---${EOL}` + updatedDocumentContent;
+
+                    session.model.changeSet.addElements(this.fileChangeFactory({
+                        uri: existingSummary.uri,
+                        type: 'modify',
+                        state: 'pending',
+                        targetState: content,
+                        requestId: session.model.id, // not a request id, as no changeRequest made yet.
+                        chatSessionId: session.id
+                    }));
+                } else {
+                    const updatedSummary: Summary = {
+                        ...existingSummary,
+                        summary: updatedDocumentContent
+                    };
+
+                    // Store the updated summary
+                    await this.storageService.store(updatedSummary);
+                }
+                return existingSummary.id;
+            } else {
+                // Fall back to standard update if no document path is found
+                const updatedSummaryText = await this.getLlmSummary(session, prompt, agent);
+                const updatedSummary: Summary = {
+                    ...existingSummary,
+                    summary: updatedSummaryText
+                };
+                await this.storageService.store(updatedSummary);
+                return updatedSummary.id;
+            }
+        } catch (err) {
+            const errorSummary: Summary = {
+                ...existingSummary,
+                summary: `Summary update failed: ${err instanceof Error ? err.message : typeof err === 'string' ? err : 'Unknown error'}`
+            };
+            await this.storageService.store(errorSummary);
+            throw err;
+        } finally {
+            progress.cancel();
+        }
+    }
+
+    protected async getLlmSummary(session: ChatSession, prompt: ResolvedPromptFragment | undefined, agent?: ChatAgent): Promise<string> {
+        if (!prompt) { return ''; }
         agent = agent || this.agentService.getAgents().find<ChatAgent>((candidate): candidate is ChatAgent =>
             'invoke' in candidate
             && typeof candidate.invoke === 'function'
@@ -112,8 +214,7 @@ export class TaskContextService {
         );
         if (!agent) { throw new Error('Unable to identify agent for summary.'); }
         const model = new MutableChatModel(ChatAgentLocation.Panel);
-        const prompt = await this.promptService.getResolvedPromptFragment(promptId || CHAT_SESSION_SUMMARY_PROMPT.id, undefined, { model: session.model });
-        if (!prompt) { return ''; }
+
         const messages = session.model.getRequests().filter((candidate): candidate is MutableChatRequestModel => candidate instanceof MutableChatRequestModel);
         messages.forEach(message => model['_hierarchy'].append(message));
         const summaryRequest = model.addRequest({
@@ -126,6 +227,11 @@ export class TaskContextService {
         return summaryRequest.response.response.asDisplayString();
     }
 
+    protected async getSystemPrompt(session: ChatSession, promptId: string = CHAT_SESSION_SUMMARY_PROMPT.id): Promise<ResolvedPromptFragment | undefined> {
+        const prompt = await this.promptService.getResolvedPromptFragment(promptId || CHAT_SESSION_SUMMARY_PROMPT.id, undefined, { model: session.model });
+        return prompt;
+    }
+
     hasSummary(chatSession: ChatSession): boolean {
         return !!this.getSummaryForSession(chatSession);
     }

package/src/browser/task-context-storage-service.ts

@@ -28,6 +28,10 @@ export class InMemoryTaskContextStorage implements TaskContextStorageService {
     protected readonly onDidChangeEmitter = new Emitter<void>();
     readonly onDidChange = this.onDidChangeEmitter.event;
 
+    protected sanitizeLabel(label: string): string {
+        return label.replace(/^[^\p{L}\p{N}]+/vg, '');
+    }
+
     @inject(AIVariableResourceResolver)
     protected readonly variableResourceResolver: AIVariableResourceResolver;
 
@@ -35,7 +39,7 @@ export class InMemoryTaskContextStorage implements TaskContextStorageService {
     protected readonly openerService: OpenerService;
 
     store(summary: Summary): void {
-        this.summaries.set(summary.id, summary);
+        this.summaries.set(summary.id, { ...summary, label: this.sanitizeLabel(summary.label) });
         this.onDidChangeEmitter.fire();
     }
 

package/src/common/change-set.ts

@@ -142,7 +142,7 @@ export class ChangeSetImpl implements ChangeSet {
         const added = [];
         const modified = [];
         const removed = [];
-        const toHandle = new Set(...this._elements.keys());
+        const toHandle = new Set(this._elements.keys());
         for (const element of elements) {
             toHandle.delete(element.uri.toString());
             if (this.doAdd(element)) {

package/src/common/chat-agent-service.ts

@@ -25,6 +25,7 @@ import { ChatAgent } from './chat-agents';
 import { AgentService } from '@theia/ai-core';
 
 export const ChatAgentService = Symbol('ChatAgentService');
+export const ChatAgentServiceFactory = Symbol('ChatAgentServiceFactory');
 /**
  * The ChatAgentService provides access to the available chat agents.
  */

package/src/common/chat-agents.ts

@@ -54,18 +54,19 @@ import { inject, injectable, named, postConstruct } from '@theia/core/shared/inv
 import { ChatAgentService } from './chat-agent-service';
 import {
     ChatModel,
-    MutableChatRequestModel,
+    ChatRequestModel,
     ChatResponseContent,
     ErrorChatResponseContentImpl,
     MarkdownChatResponseContentImpl,
-    ToolCallChatResponseContentImpl,
-    ChatRequestModel,
+    MutableChatRequestModel,
     ThinkingChatResponseContentImpl,
+    ToolCallChatResponseContentImpl,
     ErrorChatResponseContent,
 } from './chat-model';
+import { ChatToolRequest, ChatToolRequestService } from './chat-tool-request-service';
 import { parseContents } from './parse-contents';
 import { DefaultResponseContentFactory, ResponseContentMatcher, ResponseContentMatcherProvider } from './response-content-matcher';
-import { ChatToolRequest, ChatToolRequestService } from './chat-tool-request-service';
+import { ImageContextVariable } from './image-context-variable';
 
 /**
  * System message content, enriched with function descriptions.
@@ -254,11 +255,27 @@ export abstract class AbstractChatAgent implements ChatAgent {
         const requestMessages = model.getRequests().flatMap(request => {
             const messages: LanguageModelMessage[] = [];
             const text = request.message.parts.map(part => part.promptText).join('');
-            messages.push({
-                actor: 'user',
-                type: 'text',
-                text: text,
-            });
+            if (text.length > 0) {
+                messages.push({
+                    actor: 'user',
+                    type: 'text',
+                    text: text,
+                });
+            }
+            const imageMessages = request.context.variables
+                .filter(variable => ImageContextVariable.isResolvedImageContext(variable))
+                .map(variable => ImageContextVariable.parseResolved(variable))
+                .filter(content => content !== undefined)
+                .map(content => ({
+                    actor: 'user' as const,
+                    type: 'image' as const,
+                    image: {
+                        base64data: content!.data,
+                        mimeType: content!.mimeType
+                    }
+                }));
+            messages.push(...imageMessages);
+
             if (request.response.isComplete || includeResponseInProgress) {
                 const responseMessages: LanguageModelMessage[] = request.response.response.content
                     .filter(c => !ErrorChatResponseContent.is(c))

package/src/common/chat-model.ts

@@ -19,15 +19,23 @@
  *--------------------------------------------------------------------------------------------*/
 // Partially copied from https://github.com/microsoft/vscode/blob/a2cab7255c0df424027be05d58e1b7b941f4ea60/src/vs/workbench/contrib/chat/common/chatModel.ts
 
-import { AIVariableResolutionRequest, LanguageModelMessage, ResolvedAIContextVariable, TextMessage, ThinkingMessage, ToolResultMessage, ToolUseMessage } from '@theia/ai-core';
+import {
+    AIVariableResolutionRequest,
+    LanguageModelMessage,
+    ResolvedAIContextVariable,
+    TextMessage,
+    ThinkingMessage,
+    ToolResultMessage,
+    ToolUseMessage
+} from '@theia/ai-core';
 import { ArrayUtils, CancellationToken, CancellationTokenSource, Command, Disposable, DisposableCollection, Emitter, Event, generateUuid, URI } from '@theia/core';
 import { MarkdownString, MarkdownStringImpl } from '@theia/core/lib/common/markdown-rendering';
 import { Position } from '@theia/core/shared/vscode-languageserver-protocol';
+import { ChangeSet, ChangeSetElement, ChangeSetImpl, ChatUpdateChangeSetEvent } from './change-set';
 import { ChatAgentLocation } from './chat-agents';
 import { ParsedChatRequest } from './parsed-chat-request';
-import { ChangeSet, ChangeSetImpl, ChangeSetElement, ChatUpdateChangeSetEvent } from './change-set';
 import debounce = require('@theia/core/shared/lodash.debounce');
-export { ChangeSet, ChangeSetImpl, ChangeSetElement };
+export { ChangeSet, ChangeSetElement, ChangeSetImpl };
 
 /**********************
  * INTERFACES AND TYPE GUARDS

package/src/common/chat-request-parser.ts

@@ -104,6 +104,9 @@ export class ChatRequestParserImpl implements ChatRequestParser {
         const parts: ParsedChatRequestPart[] = [];
         const variables = new Map<string, AIVariable>();
         const toolRequests = new Map<string, ToolRequest>();
+        if (!request.text) {
+            return { parts, toolRequests, variables };
+        }
         const message = request.text;
         for (let i = 0; i < message.length; i++) {
             const previousChar = message.charAt(i - 1);