@theia/ai-copilot 1.76.0-next.7 → 1.76.0

package/src/node/copilot-sdk-mappers.ts

@@ -14,14 +14,8 @@
 // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
 // *****************************************************************************
 
-import { LanguageModelMessage } from '@theia/ai-core';
-import type { ModelInfo, SystemMessageConfig } from './copilot-sdk-types';
-
-/**
- * Recognizes an id that names a dated release of another model, such as
- * `gpt-4o-2024-11-20` or `claude-sonnet-5-20260514` for `gpt-4o` and `claude-sonnet-5`.
- */
-const DATED_MODEL_ID = /^(.+?)-(\d{4}-\d{2}-\d{2}|\d{8})$/;
+import { Base64ImageContent, ImageContent, ImageMessage, LanguageModelMessage } from '@theia/ai-core';
+import type { BlobMessageAttachment, ModelInfo, SystemMessageConfig } from './copilot-sdk-types';
 
 /**
  * Selects the model IDs that should be surfaced to Theia from the list returned
@@ -30,12 +24,12 @@ const DATED_MODEL_ID = /^(.+?)-(\d{4}-\d{2}-\d{2}|\d{8})$/;
  * Models whose policy is explicitly `disabled` are filtered out; `enabled` and
  * `unconfigured` models are kept. Order is preserved and duplicates are removed.
  *
- * The list can name the same model both by its family and by its dated releases. Offering the user
- * several entries that select the same model is noise, so a dated release is dropped when the family
- * it belongs to is offered as well. It is kept when it is the only way to select that model.
+ * The list can name the same model both by its family and by its dated releases, and both are kept:
+ * the family is what the model pickers offer, while a dated release stays selectable for anyone who
+ * wants to pin one. This mirrors the other providers, where discovery reports every release and only
+ * the undated ids are featured.
  */
 export function selectSdkModelIds(models: ModelInfo[]): string[] {
-    const available = new Set(models.filter(model => model.id && model.policy?.state !== 'disabled').map(model => model.id));
     const result: string[] = [];
     const seen = new Set<string>();
     for (const model of models) {
@@ -45,10 +39,6 @@ export function selectSdkModelIds(models: ModelInfo[]): string[] {
         if (model.policy && model.policy.state === 'disabled') {
             continue;
         }
-        const family = model.id.match(DATED_MODEL_ID)?.[1];
-        if (family && available.has(family)) {
-            continue;
-        }
         seen.add(model.id);
         result.push(model.id);
     }
@@ -64,6 +54,8 @@ export interface SdkPrompt {
     systemText: string;
     /** The user-facing prompt body derived from the non-system messages. */
     prompt: string;
+    /** Base64-encoded images pulled out of the user's messages, sent alongside the prompt. */
+    attachments?: BlobMessageAttachment[];
 }
 
 function roleLabel(message: LanguageModelMessage): string {
@@ -77,6 +69,17 @@ function roleLabel(message: LanguageModelMessage): string {
     }
 }
 
+/**
+ * Whether a message is a user image the SDK can be sent as a blob attachment.
+ *
+ * URL images are not covered: the SDK's attachments carry inline data, not a remote reference,
+ * and fetching the URL on the backend to inline it would add a server-side-request-forgery
+ * surface for what is, today, not a use case Theia exercises.
+ */
+function isAttachableImageMessage(message: LanguageModelMessage): message is ImageMessage & { image: Base64ImageContent } {
+    return LanguageModelMessage.isImageMessage(message) && message.actor === 'user' && ImageContent.isBase64(message.image);
+}
+
 function messageToText(message: LanguageModelMessage): string {
     if (LanguageModelMessage.isTextMessage(message)) {
         return message.text;
@@ -91,6 +94,7 @@ function messageToText(message: LanguageModelMessage): string {
         return `[tool result: ${content}]`;
     }
     if (LanguageModelMessage.isImageMessage(message)) {
+        // Reached for URL images and non-user image messages; base64 user images are attachments instead, see buildSdkPrompt.
         return '[image omitted]';
     }
     return '';
@@ -105,16 +109,23 @@ function messageToText(message: LanguageModelMessage): string {
  * see {@link buildSdkSystemMessage}. A lone user turn is forwarded verbatim;
  * richer histories are rendered as a role-labelled transcript.
  *
- * This is a lossy mapping by design, see the limitations documented in the package
- * README, and is intended for single-turn requests.
+ * Base64 user images are pulled out of the history and sent as blob attachments instead of
+ * being flattened into the prompt text, see {@link isAttachableImageMessage}. This is otherwise
+ * a lossy mapping by design, see the limitations documented in the package README, and is
+ * intended for single-turn requests.
  */
 export function buildSdkPrompt(messages: LanguageModelMessage[]): SdkPrompt {
     const systemParts: string[] = [];
     const conversation: LanguageModelMessage[] = [];
+    const attachments: BlobMessageAttachment[] = [];
     for (const message of messages) {
         if (message.actor === 'system' && LanguageModelMessage.isTextMessage(message)) {
             systemParts.push(message.text);
-        } else if (message.type !== 'thinking') {
+        } else if (message.type === 'thinking') {
+            continue;
+        } else if (isAttachableImageMessage(message)) {
+            attachments.push({ type: 'blob', data: message.image.base64data, mimeType: message.image.mimeType });
+        } else {
             conversation.push(message);
         }
     }
@@ -131,7 +142,7 @@ export function buildSdkPrompt(messages: LanguageModelMessage[]): SdkPrompt {
             .trim();
     }
 
-    return { systemText, prompt };
+    return { systemText, prompt, attachments: attachments.length > 0 ? attachments : undefined };
 }
 
 /**

package/src/node/copilot-sdk-types.ts

@@ -191,9 +191,23 @@ export interface CopilotSessionEvents {
 export type CopilotSessionEventType = keyof CopilotSessionEvents;
 
 // ============================================================================
-// Session, from `dist/session.d.ts`
+// Session, from `dist/session.d.ts` and `dist/types.d.ts`
 // ============================================================================
 
+/** The one attachment kind this integration produces: an inline image, from `MessageOptions['attachments']`. */
+export interface BlobMessageAttachment {
+    type: 'blob';
+    data: string;
+    mimeType: string;
+    displayName?: string;
+}
+
+/** The options this integration passes to {@link CopilotSession.send}. */
+export interface MessageOptions {
+    prompt: string;
+    attachments?: BlobMessageAttachment[];
+}
+
 /** A conversation with the runtime, upstream a class with resume, fork and sharing on top. */
 export interface CopilotSession {
     readonly sessionId: string;
@@ -202,7 +216,7 @@ export interface CopilotSession {
      */
     on<K extends CopilotSessionEventType>(eventType: K, handler: (event: CopilotSessionEvents[K]) => void): () => void;
     /** Sends a turn and resolves with the id of the message it created. */
-    send(options: { prompt: string }): Promise<string>;
+    send(options: MessageOptions): Promise<string>;
     /** Stops the turn in flight. */
     abort(): Promise<void>;
     /** Releases the session in memory, leaving what it persisted in place. */