@cartanova/qgrid-ai-sdk 2.4.0 → 2.4.2

package/README.ko.md

@@ -203,8 +203,27 @@ const result = await generateText({
 const image = result.files[0]; // mediaType: "image/png", base64
 ```
 
+레퍼런스 이미지는 일반 AI SDK multimodal message part로 전달할 수 있습니다:
+
+```typescript
+const result = await generateText({
+  model: qgrid("openai/gpt-5.4"),
+  messages: [
+    {
+      role: "user",
+      content: [
+        { type: "text", text: "이 이미지를 스타일 레퍼런스로 사용해서 포스터를 만들어줘" },
+        { type: "file", mediaType: "image/png", data: referenceImageBase64 },
+      ],
+    },
+  ],
+  providerOptions: { qgrid: { imageGeneration: true } },
+});
+```
+
 - `streamText`에서는 거부됩니다 (non-stream 전용).
 - 이미지 생성 요청은 항상 cold one-shot thread로 실행되며 thread reuse 대상에서 제외됩니다.
+- 레퍼런스 이미지는 JSON data URL로 전송됩니다. 큰 사진은 압축하거나 리사이즈해서 전달하세요. 과도하게 큰 base64 입력은 SDK가 거부하며, 사진에는 WebP/JPEG를 권장합니다.
 - 이미지 비용은 `gpt-image-2` 공개 단가표 기반 **추정치**로 request log의 `image_cost_usd`에 별도 기록됩니다 (codex가 정확한 이미지 tool 사용량을 노출하지 않음).
 
 ## Telemetry Logger

package/README.md

@@ -203,8 +203,27 @@ const result = await generateText({
 const image = result.files[0]; // mediaType: "image/png", base64
 ```
 
+Reference images can be passed through normal AI SDK multimodal message parts:
+
+```typescript
+const result = await generateText({
+  model: qgrid("openai/gpt-5.4"),
+  messages: [
+    {
+      role: "user",
+      content: [
+        { type: "text", text: "Use this image as a style reference and create a poster" },
+        { type: "file", mediaType: "image/png", data: referenceImageBase64 },
+      ],
+    },
+  ],
+  providerOptions: { qgrid: { imageGeneration: true } },
+});
+```
+
 - Rejected in `streamText` (non-stream only).
 - Image-generation requests always run on a cold one-shot thread and are excluded from thread reuse.
+- Reference images are sent as JSON data URLs. Compress or resize large photos before passing them in; oversized base64 inputs are rejected by the SDK. WebP/JPEG is recommended for photos.
 - The image cost is an **estimate** based on the public `gpt-image-2` price table, recorded separately as `image_cost_usd` on the request log (codex does not expose exact image-tool usage).
 
 ## Telemetry Logger

package/dist/index.js

@@ -3,7 +3,7 @@ const QGRID_PROVIDER_NAME = "qgrid";
 const THREAD_COORD_TTL_MS = 10 * 6e4;
 const TEXT_STREAM_ID_PREFIX = "text";
 const TOP_LEVEL_SCHEMA_WARNING = "OpenAI structured output requires \"object\". Falling back to client-side parsing.";
-const PENDING_TOOL_RESULTS_WARNING = "[qgrid] pending tool results not found in prompt, clearing client run state";
+const PENDING_TOOL_RESULTS_WARNING = "[qgrid] tool results did not match any pending qgrid run; sending request without runContext";
 //#endregion
 //#region src/utils.ts
 async function createRun(serverUrl, body) {
@@ -84,7 +84,10 @@ async function* parseSSE(body) {
 		}
 	}
 }
-function extractPromptAndHistory(messages) {
+function extractPromptAndHistory(messages, options = {}) {
+	const includeImages = options.includeImages ?? true;
+	const imageUrls = [];
+	let droppedImageCount = 0;
 	let system;
 	const nonSystem = [];
 	for (const msg of messages) if (msg.role === "system") system = extractTextFromContent(msg.content);
@@ -92,28 +95,51 @@ function extractPromptAndHistory(messages) {
 	if (nonSystem.length === 0) return {
 		prompt: "",
 		system,
-		history: []
-	};
-	if (nonSystem.length === 1 && nonSystem[0].role === "user") return {
-		prompt: extractTextFromContent(nonSystem[0].content),
-		system,
-		history: []
+		history: [],
+		imageUrls,
+		droppedImageCount
 	};
+	if (nonSystem.length === 1 && nonSystem[0].role === "user") {
+		const extracted = extractUserContent(nonSystem[0].content, includeImages);
+		imageUrls.push(...extracted.imageUrls);
+		droppedImageCount += extracted.droppedImageCount;
+		return {
+			prompt: extracted.text,
+			system,
+			history: [],
+			imageUrls,
+			droppedImageCount,
+			...extracted.input ? { input: extracted.input } : {}
+		};
+	}
 	const last = nonSystem[nonSystem.length - 1];
-	const prompt = last.role === "user" ? extractTextFromContent(last.content) : "";
+	let prompt = "";
+	let input;
+	if (last.role === "user") {
+		const extracted = extractUserContent(last.content, includeImages);
+		prompt = extracted.text;
+		imageUrls.push(...extracted.imageUrls);
+		droppedImageCount += extracted.droppedImageCount;
+		input = extracted.input;
+	}
 	const historyEnd = last.role === "user" ? nonSystem.length - 1 : nonSystem.length;
 	const history = [];
 	for (let i = 0; i < historyEnd; i++) {
 		const msg = nonSystem[i];
-		if (msg.role === "user") history.push({
-			type: "message",
-			role: "user",
-			content: [{
-				type: "input_text",
-				text: extractTextFromContent(msg.content)
-			}]
-		});
-		else if (msg.role === "assistant") {
+		if (msg.role === "user") {
+			const extracted = extractUserContent(msg.content, includeImages);
+			imageUrls.push(...extracted.imageUrls);
+			droppedImageCount += extracted.droppedImageCount;
+			const content = codexContentItemsFromExtracted(extracted.parts);
+			history.push({
+				type: "message",
+				role: "user",
+				content: content.length > 0 ? content : [{
+					type: "input_text",
+					text: extracted.text
+				}]
+			});
+		} else if (msg.role === "assistant") {
 			for (const part of msg.content) if ("text" in part && typeof part.text === "string") history.push({
 				type: "message",
 				role: "assistant",
@@ -143,7 +169,10 @@ function extractPromptAndHistory(messages) {
 	return {
 		prompt,
 		system,
-		history
+		history,
+		imageUrls,
+		droppedImageCount,
+		...input ? { input } : {}
 	};
 }
 function extractTextFromContent(content) {
@@ -152,6 +181,108 @@ function extractTextFromContent(content) {
 	for (const part of content) if ("text" in part && typeof part.text === "string") parts.push(part.text);
 	return parts.join("\n");
 }
+function extractUserContent(content, includeImages) {
+	const extracted = extractMessageParts(content, { includeImages });
+	return {
+		text: extractTextFromContent(content),
+		parts: extracted.parts,
+		input: inputPartsFromExtracted(extracted.parts),
+		imageUrls: extracted.imageUrls,
+		droppedImageCount: extracted.droppedImageCount
+	};
+}
+function extractMessageParts(content, options) {
+	if (typeof content === "string") return {
+		parts: [{
+			kind: "text",
+			text: content
+		}],
+		imageUrls: [],
+		droppedImageCount: 0
+	};
+	const parts = [];
+	const imageUrls = [];
+	let droppedImageCount = 0;
+	for (const part of content) {
+		if ("text" in part && typeof part.text === "string") {
+			parts.push({
+				kind: "text",
+				text: part.text
+			});
+			continue;
+		}
+		const url = extractImageUrl(part);
+		if (!url) continue;
+		if (options.includeImages) {
+			parts.push({
+				kind: "image",
+				url
+			});
+			imageUrls.push(url);
+		} else droppedImageCount++;
+	}
+	return {
+		parts,
+		imageUrls,
+		droppedImageCount
+	};
+}
+function inputPartsFromExtracted(extractedParts) {
+	const parts = [];
+	for (const part of extractedParts) if (part.kind === "text") {
+		if (part.text.length > 0) parts.push({
+			type: "text",
+			text: part.text,
+			text_elements: []
+		});
+	} else parts.push({
+		type: "image",
+		url: part.url
+	});
+	return parts.some((part) => part.type === "image") ? parts : void 0;
+}
+function codexContentItemsFromExtracted(parts) {
+	const items = [];
+	for (const part of parts) if (part.kind === "text") items.push({
+		type: "input_text",
+		text: part.text
+	});
+	else items.push({
+		type: "input_image",
+		image_url: part.url
+	});
+	return items;
+}
+function extractImageUrl(part) {
+	if (!part || typeof part !== "object") return void 0;
+	const record = part;
+	const type = record.type;
+	if (type !== "file" && type !== "image") return void 0;
+	const mediaType = typeof record.mediaType === "string" ? record.mediaType : "image/png";
+	if (type === "file" && !mediaType.toLowerCase().startsWith("image/")) return void 0;
+	const data = record.data ?? record.image;
+	if (typeof data === "string") {
+		if (isSupportedImageUrl(data)) return data;
+		if (/^[a-z][a-z0-9+.-]*:/i.test(data)) return void 0;
+		return `data:${mediaType};base64,${data}`;
+	}
+	if (data instanceof URL) {
+		const url = data.toString();
+		return isSupportedImageUrl(url) ? url : void 0;
+	}
+	if (data instanceof Uint8Array) return `data:${mediaType};base64,${bytesToBase64(data)}`;
+	if (data instanceof ArrayBuffer) return `data:${mediaType};base64,${bytesToBase64(new Uint8Array(data))}`;
+}
+function isSupportedImageUrl(value) {
+	return /^(https?:|data:|blob:)/i.test(value);
+}
+function bytesToBase64(bytes) {
+	if (typeof Buffer !== "undefined") return Buffer.from(bytes).toString("base64");
+	const chunkSize = 32768;
+	let binary = "";
+	for (let i = 0; i < bytes.length; i += chunkSize) binary += String.fromCharCode(...bytes.subarray(i, i + chunkSize));
+	return btoa(binary);
+}
 function safeStringify(value) {
 	try {
 		const json = JSON.stringify(value);
@@ -471,6 +602,21 @@ function createQgridLogger(config = {}) {
 //#endregion
 //#region src/index.ts
 const threadCoordStore = /* @__PURE__ */ new Map();
+const MAX_IMAGE_INPUT_DATA_URL_CHARS = 9e5;
+const MAX_IMAGE_INPUT_DATA_URL_CHARS_LABEL = `${MAX_IMAGE_INPUT_DATA_URL_CHARS / 1e3}k`;
+function assertImageInputFitsJsonTransport(imageUrls) {
+	let totalDataUrlChars = 0;
+	for (const url of imageUrls) {
+		if (!url.toLowerCase().startsWith("data:")) continue;
+		totalDataUrlChars += url.length;
+		if (url.length > MAX_IMAGE_INPUT_DATA_URL_CHARS) throw new Error(`qgrid: reference image input is too large for JSON transport (${url.length} chars). Compress or resize it before passing it to imageGeneration, for example as WebP/JPEG under ${MAX_IMAGE_INPUT_DATA_URL_CHARS_LABEL} total base64 data-url chars.`);
+	}
+	if (totalDataUrlChars > MAX_IMAGE_INPUT_DATA_URL_CHARS) throw new Error(`qgrid: reference image inputs are too large for JSON transport (${totalDataUrlChars} total chars). Compress or resize them before passing them to imageGeneration, for example as WebP/JPEG under ${MAX_IMAGE_INPUT_DATA_URL_CHARS_LABEL} total base64 data-url chars.`);
+}
+function warnDroppedImages(droppedImageCount) {
+	if (droppedImageCount <= 0) return;
+	console.warn(`[qgrid] ${droppedImageCount} image message part(s) were ignored because providerOptions.qgrid.imageGeneration is not enabled.`);
+}
 function getThreadCoord(sessionKey) {
 	const entry = threadCoordStore.get(sessionKey);
 	if (!entry) return void 0;
@@ -513,7 +659,39 @@ function qgrid(modelId, config) {
 	const serverUrl = config?.serverUrl ?? process.env.QGRID_URL ?? "http://localhost:44900";
 	const effort = config?.defaultEffort ?? "low";
 	const projectName = config?.projectName ?? process.env.QGRID_PROJECT_NAME;
-	let clientRun = null;
+	const clientRuns = /* @__PURE__ */ new Map();
+	let anonymousClientRunId = 0;
+	function clientRunKey(runContext) {
+		if (runContext.requestLogId !== void 0) return `request:${runContext.requestLogId}`;
+		if (runContext.threadCoord) {
+			const { workerId, epoch, threadId } = runContext.threadCoord;
+			return `thread:${workerId}:${epoch}:${threadId}`;
+		}
+		anonymousClientRunId++;
+		return `anonymous:${anonymousClientRunId}`;
+	}
+	function rememberClientRun(runContext, toolCallIds) {
+		if (toolCallIds.length === 0) return;
+		clientRuns.set(clientRunKey(runContext), {
+			runContext,
+			pendingToolCallIds: new Set(toolCallIds)
+		});
+	}
+	function matchClientRun(messages) {
+		const toolResults = extractToolResultsFromHistory(messages);
+		if (toolResults.length === 0) return null;
+		const resultIds = new Set(toolResults.map((r) => r.callId));
+		for (const [key, run] of clientRuns) if (run.pendingToolCallIds.size > 0 && [...run.pendingToolCallIds].every((id) => resultIds.has(id))) return {
+			key,
+			runContext: run.runContext,
+			toolResultsPayload: toolResults.filter((r) => run.pendingToolCallIds.has(r.callId)).map((r) => ({
+				toolCallId: r.callId,
+				output: r.result
+			}))
+		};
+		if (clientRuns.size > 0) console.warn(PENDING_TOOL_RESULTS_WARNING);
+		return null;
+	}
 	return {
 		specificationVersion: "v3",
 		provider: QGRID_PROVIDER_NAME,
@@ -522,14 +700,16 @@ function qgrid(modelId, config) {
 		async doGenerate(options) {
 			const tools = options.tools?.filter((t) => t.type === "function");
 			const hasTools = tools && tools.length > 0;
-			const { prompt, system, history } = extractPromptAndHistory(options.prompt);
 			const qgridOptions = getQgridProviderOptions(options);
+			const imageGeneration = qgridOptions.imageGeneration;
+			const { prompt, system, history, input, imageUrls, droppedImageCount } = extractPromptAndHistory(options.prompt, { includeImages: imageGeneration === true });
+			if (!imageGeneration) warnDroppedImages(droppedImageCount);
 			const effectiveEffort = qgridOptions.effort ?? effort;
 			const verbosity = qgridOptions.verbosity;
 			const reasoningSummary = qgridOptions.reasoningSummary;
 			const serviceTier = qgridOptions.serviceTier;
-			const imageGeneration = qgridOptions.imageGeneration;
 			const imageGenerationOptions = qgridOptions.imageGenerationOptions;
+			if (imageGeneration) assertImageInputFitsJsonTransport(imageUrls);
 			const sessionKey = qgridOptions.sessionKey;
 			const reuseSessionKey = reusableSessionKey(modelId, sessionKey);
 			const rawSchema = options.responseFormat?.type === "json" ? options.responseFormat.schema : void 0;
@@ -539,20 +719,11 @@ function qgrid(modelId, config) {
 			let runContext;
 			let toolResultsPayload;
 			let logMode;
-			if (clientRun) {
-				const toolResults = extractToolResultsFromHistory(options.prompt);
-				const resultIds = new Set(toolResults.map((r) => r.callId));
-				if (clientRun.pendingToolCallIds.size > 0 && [...clientRun.pendingToolCallIds].every((id) => resultIds.has(id))) {
-					runContext = clientRun.runContext;
-					toolResultsPayload = toolResults.filter((r) => clientRun.pendingToolCallIds.has(r.callId)).map((r) => ({
-						toolCallId: r.callId,
-						output: r.result
-					}));
-					logMode = "run";
-				} else {
-					console.warn(PENDING_TOOL_RESULTS_WARNING);
-					clientRun = null;
-				}
+			const matchedClientRun = matchClientRun(options.prompt);
+			if (matchedClientRun) {
+				runContext = matchedClientRun.runContext;
+				toolResultsPayload = matchedClientRun.toolResultsPayload;
+				logMode = "run";
 			}
 			const storedCoord = reuseSessionKey ? getThreadCoord(reuseSessionKey) : void 0;
 			if (!runContext && storedCoord) runContext = { threadCoord: storedCoord };
@@ -562,6 +733,7 @@ function qgrid(modelId, config) {
 				headers: { "Content-Type": "application/json" },
 				body: JSON.stringify({ args: {
 					prompt,
+					...imageGeneration && input ? { input } : {},
 					model: modelId,
 					system,
 					effort: effectiveEffort,
@@ -616,11 +788,8 @@ function qgrid(modelId, config) {
 				text: data.text
 			});
 			if (imageGeneration && !content.some((c) => c.type === "file")) throw new Error("qgrid: imageGeneration was requested but the response contained no image — the server may be an older version that does not support image generation.");
-			if (data.runContext && finishReason.unified === "tool-calls") clientRun = {
-				runContext: data.runContext,
-				pendingToolCallIds: new Set(content.filter((c) => c.type === "tool-call").map((c) => c.toolCallId))
-			};
-			else clientRun = null;
+			if (matchedClientRun) clientRuns.delete(matchedClientRun.key);
+			if (data.runContext && finishReason.unified === "tool-calls") rememberClientRun(data.runContext, content.filter((c) => c.type === "tool-call").map((c) => c.toolCallId));
 			if (reuseSessionKey && data.runContext?.threadCoord) setThreadCoord(reuseSessionKey, data.runContext.threadCoord);
 			return {
 				content,
@@ -639,13 +808,14 @@ function qgrid(modelId, config) {
 		async doStream(options) {
 			const tools = options.tools?.filter((t) => t.type === "function");
 			const hasTools = tools && tools.length > 0;
-			const { prompt, system, history } = extractPromptAndHistory(options.prompt);
 			const qgridOptions = getQgridProviderOptions(options);
 			const effectiveEffort = qgridOptions.effort ?? effort;
 			const verbosity = qgridOptions.verbosity;
 			const reasoningSummary = qgridOptions.reasoningSummary;
 			const serviceTier = qgridOptions.serviceTier;
 			if (qgridOptions.imageGeneration) throw new Error("qgrid: imageGeneration is not supported with streamText — use generateText instead.");
+			const { prompt, system, history, droppedImageCount } = extractPromptAndHistory(options.prompt, { includeImages: false });
+			warnDroppedImages(droppedImageCount);
 			const sessionKey = qgridOptions.sessionKey;
 			const reuseSessionKey = reusableSessionKey(modelId, sessionKey);
 			const rawSchema = options.responseFormat?.type === "json" ? options.responseFormat.schema : void 0;
@@ -653,20 +823,11 @@ function qgrid(modelId, config) {
 			let runContext;
 			let toolResultsPayload;
 			let logMode;
-			if (clientRun) {
-				const toolResults = extractToolResultsFromHistory(options.prompt);
-				const resultIds = new Set(toolResults.map((r) => r.callId));
-				if (clientRun.pendingToolCallIds.size > 0 && [...clientRun.pendingToolCallIds].every((id) => resultIds.has(id))) {
-					runContext = clientRun.runContext;
-					toolResultsPayload = toolResults.filter((r) => clientRun.pendingToolCallIds.has(r.callId)).map((r) => ({
-						toolCallId: r.callId,
-						output: r.result
-					}));
-					logMode = "run";
-				} else {
-					console.warn(PENDING_TOOL_RESULTS_WARNING);
-					clientRun = null;
-				}
+			const matchedClientRun = matchClientRun(options.prompt);
+			if (matchedClientRun) {
+				runContext = matchedClientRun.runContext;
+				toolResultsPayload = matchedClientRun.toolResultsPayload;
+				logMode = "run";
 			}
 			const storedCoord = reuseSessionKey ? getThreadCoord(reuseSessionKey) : void 0;
 			if (!runContext && storedCoord) runContext = { threadCoord: storedCoord };
@@ -733,11 +894,8 @@ function qgrid(modelId, config) {
 							textStarted = false;
 						}
 						const done = event.data;
-						if (done.runContext && done.finishReason === "tool-calls") clientRun = {
-							runContext: done.runContext,
-							pendingToolCallIds: new Set((done.content ?? []).filter((c) => c.type === "tool-call").map((c) => c.toolCallId))
-						};
-						else clientRun = null;
+						if (matchedClientRun) clientRuns.delete(matchedClientRun.key);
+						if (done.runContext && done.finishReason === "tool-calls") rememberClientRun(done.runContext, (done.content ?? []).filter((c) => c.type === "tool-call").map((c) => c.toolCallId));
 						if (reuseSessionKey && done.runContext?.threadCoord) setThreadCoord(reuseSessionKey, done.runContext.threadCoord);
 						if (done.content) {
 							for (const item of done.content) if (item.type === "text" && !deltaTextEmitted) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cartanova/qgrid-ai-sdk",
-  "version": "2.4.0",
+  "version": "2.4.2",
   "description": "AI SDK LanguageModelV3 provider for qgrid",
   "repository": {
     "type": "git",