@cartanova/qgrid-ai-sdk 2.10.8 → 2.10.9

package/README.ko.md

@@ -323,7 +323,9 @@ const { image } = await generateImage({
 정확한 최종 크기는 소비 앱의 명시적인 crop/resize 정책으로 처리합니다.
 
 `generateText`와 `streamText`는 이미지 생성 옵션이 없는 대화 모델 인터페이스입니다.
-일반적인 이미지 인식도 지원하지 않아 image part는 경고와 함께 무시됩니다. 이미지 생성·편집이
+`openai/` 모델은 user 메시지의 image part를 모델에 전달하므로 이미지 인식(image→text)이 됩니다.
+`anthropic/` 모델과 assistant 메시지의 이미지는 버려지고 `result.warnings`에 `unsupported` 경고가
+남습니다. 이미지 생성·편집이
 목적이면 `generateImage`의 참조 입력을 사용하세요. 대시보드 `qgrid_chat`은 내부 이미지
 도구를 기본으로 등록합니다. 채팅 모델이 생성 여부를 판단하고 완결된 프롬프트를 작성하며,
 편집 시 최근 대화 이미지 중 최대 5개를 선택합니다. 서버는 일반·투명 이미지 모두 구독

package/README.md

@@ -332,7 +332,9 @@ alpha when saving. Fully opaque, entirely empty, or invalid PNG results fail exp
 not remove backgrounds or resize output. Apply any final crop/resize policy in the consuming app.
 
 `generateText` and `streamText` use the language-model interface, which has no image-generation
-options. General vision is also unsupported there: image parts are ignored with a warning. Use
+options. Image input (image→text) works on `openai/` models: user-message image parts are sent
+to the model. `anthropic/` models and assistant-message images drop image parts and report an
+`unsupported` warning in `result.warnings`. Use
 `generateImage` references when the intended result is a generated or edited image. Dashboard
 `qgrid_chat` registers an internal image tool by default: its chat model decides whether to
 generate an image, writes a self-contained prompt, and selects up to five recent conversation

package/dist/index.js

@@ -434,42 +434,41 @@ async function* parseSSE(body, signal) {
 		}
 	}
 }
-function extractPromptAndHistory(messages) {
+function extractPromptAndHistory(messages, options = {}) {
+	const includeImages = options.includeImages ?? false;
+	const imageUrls = [];
 	let droppedImageCount = 0;
 	let system;
 	const nonSystem = [];
-	for (const msg of messages) if (msg.role === "system") system = extractPromptTextParts(msg.content).text;
+	const extractUser = (content) => {
+		const extracted = extractPromptParts(content, includeImages);
+		imageUrls.push(...extracted.imageUrls);
+		droppedImageCount += extracted.droppedImageCount;
+		return extracted;
+	};
+	for (const msg of messages) if (msg.role === "system") system = extractPromptParts(msg.content, false).text;
 	else nonSystem.push(msg);
 	if (nonSystem.length === 0) return {
 		prompt: "",
 		system,
 		history: [],
+		imageUrls,
 		droppedImageCount
 	};
-	if (nonSystem.length === 1 && nonSystem[0].role === "user") {
-		const extracted = extractPromptTextParts(nonSystem[0].content);
-		droppedImageCount += extracted.droppedImageCount;
-		return {
-			prompt: extracted.text,
-			system,
-			history: [],
-			droppedImageCount
-		};
-	}
 	const last = nonSystem[nonSystem.length - 1];
 	let prompt = "";
+	let input;
 	if (last.role === "user") {
-		const extracted = extractPromptTextParts(last.content);
+		const extracted = extractUser(last.content);
 		prompt = extracted.text;
-		droppedImageCount += extracted.droppedImageCount;
+		input = inputPartsFromPromptParts(extracted.parts);
 	}
 	const historyEnd = last.role === "user" ? nonSystem.length - 1 : nonSystem.length;
 	const history = [];
 	for (let i = 0; i < historyEnd; i++) {
 		const msg = nonSystem[i];
 		if (msg.role === "user") {
-			const extracted = extractPromptTextParts(msg.content);
-			droppedImageCount += extracted.droppedImageCount;
+			const extracted = extractUser(msg.content);
 			const content = extracted.parts;
 			history.push({
 				type: "message",
@@ -511,36 +510,78 @@ function extractPromptAndHistory(messages) {
 		prompt,
 		system,
 		history,
-		droppedImageCount
+		imageUrls,
+		droppedImageCount,
+		...input ? { input } : {}
 	};
 }
-function extractPromptTextParts(content) {
+function extractPromptParts(content, includeImages) {
 	if (typeof content === "string") return {
 		text: content,
 		parts: [{
 			type: "input_text",
 			text: content
 		}],
+		imageUrls: [],
 		droppedImageCount: 0
 	};
 	const parts = [];
+	const texts = [];
+	const imageUrls = [];
 	let droppedImageCount = 0;
-	for (const part of content) if ("text" in part && typeof part.text === "string") parts.push({
-		type: "input_text",
-		text: part.text
-	});
-	else if (isImagePart(part)) droppedImageCount++;
+	for (const part of content) if ("text" in part && typeof part.text === "string") {
+		parts.push({
+			type: "input_text",
+			text: part.text
+		});
+		texts.push(part.text);
+	} else if (isImagePart(part)) {
+		const url = includeImages ? imageUrl(part) : void 0;
+		if (url) {
+			parts.push({
+				type: "input_image",
+				image_url: url
+			});
+			imageUrls.push(url);
+		} else droppedImageCount++;
+	}
 	return {
-		text: parts.map((part) => part.text).join("\n"),
+		text: texts.join("\n"),
 		parts,
+		imageUrls,
 		droppedImageCount
 	};
 }
+function inputPartsFromPromptParts(parts) {
+	if (!parts.some((part) => part.type === "input_image")) return void 0;
+	return parts.flatMap((part) => {
+		if (part.type === "input_image") return [{
+			type: "image",
+			url: part.image_url
+		}];
+		return part.text ? [{
+			type: "text",
+			text: part.text,
+			text_elements: []
+		}] : [];
+	});
+}
 function isImagePart(part) {
 	if (!part || typeof part !== "object") return false;
 	const record = part;
 	return record.type === "image" || record.type === "file" && typeof record.mediaType === "string" && record.mediaType.toLowerCase().startsWith("image/");
 }
+function imageUrl(part) {
+	const record = part;
+	const mediaType = typeof record.mediaType === "string" && record.mediaType !== "image/*" ? record.mediaType : "image/png";
+	const data = record.data ?? record.image;
+	if (data instanceof URL) return /^https?:$/i.test(data.protocol) ? data.toString() : void 0;
+	if (data instanceof Uint8Array) return `data:${mediaType};base64,${bytesToBase64(data)}`;
+	if (data instanceof ArrayBuffer) return `data:${mediaType};base64,${bytesToBase64(new Uint8Array(data))}`;
+	if (typeof data !== "string") return void 0;
+	if (/^(https?:|data:)/i.test(data)) return data;
+	return /^[a-z][a-z0-9+.-]*:/i.test(data) ? void 0 : `data:${mediaType};base64,${data}`;
+}
 function bytesToBase64(bytes) {
 	if (typeof Buffer !== "undefined") return Buffer.from(bytes).toString("base64");
 	const chunkSize = 32768;
@@ -555,9 +596,9 @@ function assertImageInputFitsJsonTransport(imageUrls) {
 	for (const url of imageUrls) {
 		if (!url.toLowerCase().startsWith("data:")) continue;
 		totalDataUrlChars += url.length;
-		if (url.length > MAX_IMAGE_INPUT_DATA_URL_CHARS) throw new Error(`qgrid: image input is too large for JSON transport (${url.length} chars). Compress or resize it before passing it to generateImage, for example as WebP/JPEG under ${MAX_IMAGE_INPUT_DATA_URL_CHARS_LABEL} total base64 data-url chars.`);
+		if (url.length > MAX_IMAGE_INPUT_DATA_URL_CHARS) throw new Error(`qgrid: image input is too large for JSON transport (${url.length} chars). Compress or resize it before passing it to qgrid, for example as WebP/JPEG under ${MAX_IMAGE_INPUT_DATA_URL_CHARS_LABEL} total base64 data-url chars.`);
 	}
-	if (totalDataUrlChars > MAX_IMAGE_INPUT_DATA_URL_CHARS) throw new Error(`qgrid: image inputs are too large for JSON transport (${totalDataUrlChars} total chars). Compress or resize them before passing them to generateImage, for example as WebP/JPEG under ${MAX_IMAGE_INPUT_DATA_URL_CHARS_LABEL} total base64 data-url chars.`);
+	if (totalDataUrlChars > MAX_IMAGE_INPUT_DATA_URL_CHARS) throw new Error(`qgrid: image inputs are too large for JSON transport (${totalDataUrlChars} total chars). Compress or resize them before passing them to qgrid, for example as WebP/JPEG under ${MAX_IMAGE_INPUT_DATA_URL_CHARS_LABEL} total base64 data-url chars.`);
 }
 function safeStringify(value) {
 	try {
@@ -1076,10 +1117,6 @@ function createQgridLogger(config = {}) {
 const threadCoordStore = /* @__PURE__ */ new Map();
 const CACHE_AFFINITY_NAMESPACE = "qgrid-cache-affinity-v1";
 const MAX_THREAD_COORDS = 1e3;
-function warnDroppedImages(droppedImageCount) {
-	if (droppedImageCount <= 0) return;
-	console.warn(`[qgrid] ${droppedImageCount} image message part(s) were ignored because qgrid language models do not support image input. For image generation or editing, use generateImage with qgrid.image().`);
-}
 function serializeObjectResponseFormat(responseFormat) {
 	const rawSchema = responseFormat?.type === "json" ? responseFormat.schema : void 0;
 	if (rawSchema === void 0) return void 0;
@@ -1109,12 +1146,19 @@ function setThreadCoord(sessionKey, coord) {
 	});
 	if (threadCoordStore.size > MAX_THREAD_COORDS) threadCoordStore.delete(threadCoordStore.keys().next().value);
 }
-function providerWarnings(options) {
-	return options.fallbackModels === void 0 ? [] : [{
+function providerWarnings(options, droppedImageCount) {
+	const warnings = [];
+	if (options.fallbackModels !== void 0) warnings.push({
 		type: "unsupported",
 		feature: "providerOptions.qgrid.fallbackModels",
 		details: "fallbackModels is deprecated and ignored; qgrid does not implement model fallback routing."
-	}];
+	});
+	if (droppedImageCount > 0) warnings.push({
+		type: "unsupported",
+		feature: "image input",
+		details: `${droppedImageCount} image part(s) were ignored. Only user-message images on openai/ models are sent; for image generation or editing, use generateImage with qgrid.image().`
+	});
+	return warnings;
 }
 async function deriveCacheAffinityKey(modelId, sessionKey) {
 	if (!sessionKey || modelId.startsWith("anthropic/")) return void 0;
@@ -1206,8 +1250,8 @@ function qgrid(modelId, config) {
 			const tools = options.tools?.filter((t) => t.type === "function");
 			const hasTools = tools && tools.length > 0;
 			const qgridOptions = getQgridProviderOptions(options);
-			const { prompt, system, history, droppedImageCount } = extractPromptAndHistory(options.prompt);
-			warnDroppedImages(droppedImageCount);
+			const { prompt, system, history, input, imageUrls, droppedImageCount } = extractPromptAndHistory(options.prompt, { includeImages: modelId.startsWith("openai/") });
+			assertImageInputFitsJsonTransport(imageUrls);
 			const effectiveEffort = qgridOptions.thinking === false ? void 0 : qgridOptions.effort ?? effort;
 			const verbosity = qgridOptions.verbosity;
 			const reasoningSummary = qgridOptions.reasoningSummary;
@@ -1233,6 +1277,7 @@ function qgrid(modelId, config) {
 				headers: { "Content-Type": "application/json" },
 				body: JSON.stringify({ args: {
 					prompt,
+					...input ? { input } : {},
 					model: modelId,
 					system,
 					effort: effectiveEffort,
@@ -1295,7 +1340,7 @@ function qgrid(modelId, config) {
 				content,
 				finishReason,
 				usage: toAiSdkUsage(data.usage),
-				warnings: providerWarnings(qgridOptions),
+				warnings: providerWarnings(qgridOptions, droppedImageCount),
 				providerMetadata: qgridProviderMetadata(data),
 				response: { modelId: data.model }
 			};
@@ -1311,8 +1356,8 @@ function qgrid(modelId, config) {
 			const timeoutMs = qgridOptions.timeoutMs;
 			const logger = qgridOptions.logger;
 			const tokenName = qgridOptions.tokenName;
-			const { prompt, system, history, droppedImageCount } = extractPromptAndHistory(options.prompt);
-			warnDroppedImages(droppedImageCount);
+			const { prompt, system, history, input, imageUrls, droppedImageCount } = extractPromptAndHistory(options.prompt, { includeImages: modelId.startsWith("openai/") });
+			assertImageInputFitsJsonTransport(imageUrls);
 			const sessionKey = qgridOptions.sessionKey;
 			const cacheAffinityKey = await deriveCacheAffinityKey(modelId, sessionKey);
 			const jsonSchema = serializeObjectResponseFormat(options.responseFormat);
@@ -1332,6 +1377,7 @@ function qgrid(modelId, config) {
 				headers: { "Content-Type": "application/json" },
 				body: JSON.stringify({ args: {
 					prompt,
+					...input ? { input } : {},
 					model: modelId,
 					system,
 					effort: effectiveEffort,
@@ -1382,7 +1428,7 @@ function qgrid(modelId, config) {
 				try {
 					controller.enqueue({
 						type: "stream-start",
-						warnings: providerWarnings(qgridOptions)
+						warnings: providerWarnings(qgridOptions, droppedImageCount)
 					});
 					let streamCompleted = false;
 					for await (const event of parseSSE(streamRes.body, signal)) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cartanova/qgrid-ai-sdk",
-  "version": "2.10.8",
+  "version": "2.10.9",
   "description": "AI SDK LanguageModelV3 and ImageModelV3 provider for qgrid",
   "repository": {
     "type": "git",