gsd-pi 2.73.1-dev.9a4cd44 → 2.73.1-dev.a2eb797

package/src/resources/extensions/claude-code-cli/stream-adapter.ts

@@ -97,6 +97,31 @@ interface ParsedTextInputField {
 	secure: boolean;
 }
 
+interface SDKInputImageBlock {
+	type: "image";
+	source: {
+		type: "base64";
+		media_type: string;
+		data: string;
+	};
+}
+
+interface SDKInputTextBlock {
+	type: "text";
+	text: string;
+}
+
+type SDKInputUserContentBlock = SDKInputImageBlock | SDKInputTextBlock;
+
+interface SDKInputUserMessage {
+	type: "user";
+	message: {
+		role: "user";
+		content: SDKInputUserContentBlock[];
+	};
+	parent_tool_use_id: null;
+}
+
 const OTHER_OPTION_LABEL = "None of the above";
 const SENSITIVE_FIELD_PATTERN = /(password|passphrase|secret|token|api[_\s-]*key|private[_\s-]*key|credential)/i;
 
@@ -223,6 +248,74 @@ export function buildPromptFromContext(context: Context): string {
 	return parts.join("\n\n");
 }
 
+function stripDataUriPrefix(value: string): string {
+	const commaIndex = value.indexOf(",");
+	if (value.startsWith("data:") && commaIndex !== -1) {
+		return value.slice(commaIndex + 1);
+	}
+	return value;
+}
+
+function inferMimeTypeFromDataUri(value: string): string | null {
+	const match = /^data:([^;,]+);base64,/.exec(value);
+	return match?.[1] ?? null;
+}
+
+export function extractImageBlocksFromContext(context: Context): SDKInputImageBlock[] {
+	const imageBlocks: SDKInputImageBlock[] = [];
+
+	for (const msg of context.messages) {
+		if (msg.role !== "user" || !Array.isArray(msg.content)) continue;
+		for (const part of msg.content) {
+			if (!part || typeof part !== "object") continue;
+			const block = part as { type?: unknown; data?: unknown; mimeType?: unknown };
+			if (block.type !== "image" || typeof block.data !== "string") continue;
+
+			const mimeType =
+				typeof block.mimeType === "string" && block.mimeType.length > 0
+					? block.mimeType
+					: inferMimeTypeFromDataUri(block.data);
+			if (!mimeType) continue;
+
+			imageBlocks.push({
+				type: "image",
+				source: {
+					type: "base64",
+					media_type: mimeType,
+					data: stripDataUriPrefix(block.data),
+				},
+			});
+		}
+	}
+
+	return imageBlocks;
+}
+
+export function buildSdkQueryPrompt(
+	context: Context,
+	textPrompt: string = buildPromptFromContext(context),
+): string | AsyncIterable<SDKInputUserMessage> {
+	const imageBlocks = extractImageBlocksFromContext(context);
+	if (imageBlocks.length === 0) {
+		return textPrompt;
+	}
+
+	const content: SDKInputUserContentBlock[] = [...imageBlocks];
+	if (textPrompt) {
+		content.push({ type: "text", text: textPrompt });
+	}
+
+	const sdkMessage: SDKInputUserMessage = {
+		type: "user",
+		message: { role: "user", content },
+		parent_tool_use_id: null,
+	};
+
+	return (async function* () {
+		yield sdkMessage;
+	})();
+}
+
 // ---------------------------------------------------------------------------
 // Error helper
 // ---------------------------------------------------------------------------
@@ -828,6 +921,7 @@ async function pumpSdkMessages(
 		}
 
 		const prompt = buildPromptFromContext(context);
+		const queryPrompt = buildSdkQueryPrompt(context, prompt);
 		const permissionMode = await resolveClaudePermissionMode();
 		const sdkOpts = buildSdkOptions(
 			modelId,
@@ -844,7 +938,7 @@ async function pumpSdkMessages(
 		);
 
 		const queryResult = sdk.query({
-			prompt,
+			prompt: queryPrompt,
 			options: {
 				...sdkOpts,
 				abortController: controller,

package/src/resources/extensions/claude-code-cli/tests/stream-adapter.test.ts

@@ -10,8 +10,10 @@ import {
 	mergePendingToolCalls,
 	resolveClaudePermissionMode,
 	buildPromptFromContext,
+	buildSdkQueryPrompt,
 	buildSdkOptions,
 	createClaudeCodeElicitationHandler,
+	extractImageBlocksFromContext,
 	extractToolResultsFromSdkUserMessage,
 	getClaudeLookupCommand,
 	parseAskUserQuestionsElicitation,
@@ -167,6 +169,92 @@ describe("stream-adapter — full context prompt (#2859)", () => {
 	});
 });
 
+describe("stream-adapter — image prompt forwarding (#4183)", () => {
+	test("extractImageBlocksFromContext maps user image parts to Anthropic base64 image blocks", () => {
+		const context: Context = {
+			messages: [
+				{
+					role: "user",
+					content: [
+						{ type: "text", text: "look" },
+						{
+							type: "image",
+							data: "data:image/png;base64,abc123",
+							mimeType: "image/png",
+						},
+					],
+				} as Message,
+			],
+		};
+
+		const imageBlocks = extractImageBlocksFromContext(context);
+		assert.deepEqual(imageBlocks, [
+			{
+				type: "image",
+				source: {
+					type: "base64",
+					media_type: "image/png",
+					data: "abc123",
+				},
+			},
+		]);
+	});
+
+	test("buildSdkQueryPrompt returns plain string when no images exist in context", () => {
+		const context: Context = {
+			messages: [{ role: "user", content: "hello" } as Message],
+		};
+		const textPrompt = buildPromptFromContext(context);
+
+		const prompt = buildSdkQueryPrompt(context, textPrompt);
+		assert.equal(typeof prompt, "string");
+		assert.equal(prompt, textPrompt);
+	});
+
+	test("buildSdkQueryPrompt wraps images and prompt text in an SDK user message iterable", async () => {
+		const context: Context = {
+			messages: [
+				{
+					role: "user",
+					content: [
+						{ type: "image", data: "ZmFrZQ==", mimeType: "image/jpeg" },
+						{ type: "text", text: "What is in this image?" },
+					],
+				} as Message,
+			],
+		};
+		const textPrompt = buildPromptFromContext(context);
+
+		const prompt = buildSdkQueryPrompt(context, textPrompt);
+		assert.notEqual(typeof prompt, "string");
+		assert.ok(prompt && typeof (prompt as any)[Symbol.asyncIterator] === "function");
+
+		const messages: any[] = [];
+		for await (const item of prompt as AsyncIterable<any>) {
+			messages.push(item);
+		}
+		assert.equal(messages.length, 1);
+		assert.deepEqual(messages[0], {
+			type: "user",
+			message: {
+				role: "user",
+				content: [
+					{
+						type: "image",
+						source: {
+							type: "base64",
+							media_type: "image/jpeg",
+							data: "ZmFrZQ==",
+						},
+					},
+					{ type: "text", text: textPrompt },
+				],
+			},
+			parent_tool_use_id: null,
+		});
+	});
+});
+
 // ---------------------------------------------------------------------------
 // Bug #4102 — transcript fabrication regression tests
 // ---------------------------------------------------------------------------