ima2-gen 1.1.13 → 1.1.16

package/lib/responsesFallback.ts

@@ -0,0 +1,114 @@
+import { logEvent } from "./logger.js";
+import type { ParsedResponsesResult } from "./responsesParse.js";
+import type { RouteRuntimeContext } from "./runtimeContext.js";
+import { imageToolChoice, tools } from "./responsesTools.js";
+import { emptyResponseError } from "./responsesErrors.js";
+import { buildUserTextPrompt } from "./oauthProxy.js";
+
+type PostResponses = (args: {
+  ctx: RouteRuntimeContext;
+  provider: string | undefined;
+  scope: string;
+  payload: unknown;
+  requestId?: string | null;
+  maxImages?: number;
+  signal?: AbortSignal | null;
+}) => Promise<ParsedResponsesResult>;
+
+export async function retryPromptOnlyJsonImage({
+  postResponses,
+  ctx,
+  provider,
+  prompt,
+  mode,
+  model,
+  quality,
+  size,
+  moderation,
+  requestId,
+  signal,
+  initial,
+  referencesDroppedOnRetry,
+  webSearchDroppedOnRetry,
+  reasoningEffort,
+}: {
+  postResponses: PostResponses;
+  ctx: RouteRuntimeContext;
+  provider: string | undefined;
+  prompt: string | undefined;
+  mode: string;
+  model: string;
+  quality?: string;
+  size?: string;
+  moderation?: string;
+  requestId: string | null;
+  signal?: AbortSignal | null;
+  initial: ParsedResponsesResult;
+  referencesDroppedOnRetry: boolean;
+  webSearchDroppedOnRetry: boolean;
+  reasoningEffort?: string;
+}) {
+  if (provider === "api") return null;
+  const retryKind = "prompt_only_json_image_tool";
+  const retryMeta = {
+    retryKind,
+    initialEventCount: initial.eventCount,
+    initialEventTypes: initial.eventTypes,
+    referencesDroppedOnRetry,
+    developerPromptDroppedOnRetry: true,
+    webSearchDroppedOnRetry,
+  };
+  logEvent("oauth", "retry_json", { requestId, ...retryMeta });
+  let retry: ParsedResponsesResult;
+  try {
+    retry = await postResponses({
+      ctx,
+      provider,
+      scope: "oauth-fallback",
+      requestId,
+      maxImages: 1,
+      signal,
+      payload: {
+        model,
+        input: [{ role: "user", content: buildUserTextPrompt(prompt, mode, { webSearchEnabled: false }) }],
+        tools: tools(false, { quality, size, moderation }),
+        tool_choice: imageToolChoice(true),
+        reasoning: { effort: reasoningEffort || "low" },
+        stream: false,
+      },
+    });
+  } catch (e) {
+    if (e && typeof e === "object") Object.assign(e, retryMeta);
+    throw e;
+  }
+  const image = retry.images[0];
+  if (image?.b64) {
+    logEvent("oauth", "retry_image", { requestId, retryKind, imageChars: image.b64.length });
+    return { b64: image.b64, usage: retry.usage, webSearchCalls: initial.webSearchCalls, revisedPrompt: image.revisedPrompt, text: retry.text, ...retryMeta };
+  }
+  logEvent("oauth", "retry_no_image", {
+    requestId,
+    retryKind,
+    fallbackEventCount: retry.eventCount,
+    fallbackImageCallSeen: retry.diagnostics.imageCallSeen,
+    fallbackImageResultCount: retry.diagnostics.imageResultCount,
+  });
+  throw emptyResponseError("No image data received from Responses API fallback", retry, {
+    provider,
+    model,
+    quality,
+    size,
+    moderation,
+    webSearchEnabled: false,
+    refsCount: 0,
+    inputImageCount: 0,
+    promptChars: typeof prompt === "string" ? prompt.length : 0,
+    toolTypes: ["image_generation"],
+    toolChoiceKind: "image_generation",
+    ...retryMeta,
+    fallbackEventCount: retry.eventCount,
+    fallbackEventTypes: retry.eventTypes,
+    fallbackImageCallSeen: retry.diagnostics.imageCallSeen,
+    fallbackImageResultCount: retry.diagnostics.imageResultCount,
+  });
+}

package/lib/responsesImageAdapter.js

@@ -1,11 +1,16 @@
-import { setJobPhase } from "./inflight.js";
 import { logEvent } from "./logger.js";
 import { classifyUpstreamError, classifyUpstreamErrorCode } from "./errorClassify.js";
 import { compressReferenceB64ForOAuth } from "./referenceImageCompress.js";
 import { detectImageMimeFromB64 } from "./refs.js";
 import { errInfo } from "./errInfo.js";
+import { setJobPhase } from "./inflight.js";
 import { requireRuntimeContext } from "./runtimeContext.js";
+import { parseJson, parseStream, safeDiagnosticLabel, } from "./responsesParse.js";
+import { imageToolChoice, imageToolChoiceKind, tools, toolTypes, } from "./responsesTools.js";
+import { emptyResponseError } from "./responsesErrors.js";
+import { retryPromptOnlyJsonImage } from "./responsesFallback.js";
 import { AUTO_PROMPT_FIDELITY_SUFFIX, DIRECT_PROMPT_FIDELITY_SUFFIX, EDIT_DEVELOPER_PROMPT, EDIT_NO_SEARCH_DEVELOPER_PROMPT, GENERATE_DEVELOPER_PROMPT, GENERATE_NO_SEARCH_DEVELOPER_PROMPT, MULTIMODE_DEVELOPER_PROMPT, MULTIMODE_NO_SEARCH_DEVELOPER_PROMPT, buildEditTextPrompt, buildMultimodeSequencePrompt, buildUserTextPrompt, waitForOAuthReady, } from "./oauthProxy.js";
+const RESPONSES_ERROR_MARKER = "ima2ResponsesError";
 function makeError(message, { status = 500, code = "RESPONSES_IMAGE_ERROR", cause, ...rest } = {}) {
     const err = new Error(message);
     err.status = status;
@@ -13,6 +18,7 @@ function makeError(message, { status = 500, code = "RESPONSES_IMAGE_ERROR", caus
     if (cause)
         err.cause = cause;
     Object.assign(err, rest);
+    Object.defineProperty(err, RESPONSES_ERROR_MARKER, { value: true });
     return err;
 }
 function parseOpenAIErrorBody(text) {
@@ -21,9 +27,9 @@ function parseOpenAIErrorBody(text) {
         const error = parsed?.error || {};
         return {
             message: typeof error.message === "string" && error.message ? error.message : "OpenAI request failed",
-            code: typeof error.code === "string" ? error.code : null,
-            type: typeof error.type === "string" ? error.type : null,
-            param: typeof error.param === "string" ? error.param : null,
+            code: safeDiagnosticLabel(error.code),
+            type: safeDiagnosticLabel(error.type),
+            param: safeDiagnosticLabel(error.param),
         };
     }
     catch {
@@ -54,36 +60,56 @@ function safeUpstreamClientMessage(upstream, status) {
         return "OpenAI rate limited the image request.";
     return "OpenAI rejected the image request.";
 }
+function safeBaseUrl(value) {
+    try {
+        const parsed = new URL(value);
+        parsed.username = "";
+        parsed.password = "";
+        return parsed.toString().replace(/\/$/, "");
+    }
+    catch {
+        return value.replace(/\/$/, "");
+    }
+}
+function apiAuthorizationHeader(apiKey) {
+    const key = typeof apiKey === "string" ? apiKey.trim() : "";
+    if (!key) {
+        throw makeError("API key is required for API provider image generation", {
+            status: 401,
+            code: "API_KEY_REQUIRED",
+        });
+    }
+    if (/[\u0000-\u001f\u007f]/.test(key)) {
+        throw makeError("API key contains invalid characters.", {
+            status: 401,
+            code: "AUTH_API_KEY_INVALID",
+        });
+    }
+    return `Bearer ${key}`;
+}
+function isKnownResponsesError(value) {
+    return Boolean(value &&
+        typeof value === "object" &&
+        value.ima2ResponsesError === true);
+}
 async function getEndpoint(ctx, provider, _scope) {
     if (provider === "api") {
-        if (!ctx?.apiKey) {
-            throw makeError("API key is required for API provider image generation", {
-                status: 401,
-                code: "API_KEY_REQUIRED",
-            });
-        }
         return {
             url: "https://api.openai.com/v1/responses",
             headers: {
                 "Content-Type": "application/json",
                 Accept: "text/event-stream",
-                Authorization: `Bearer ${ctx.apiKey}`,
+                Authorization: apiAuthorizationHeader(ctx.apiKey),
             },
         };
     }
     await waitForOAuthReady(ctx);
     const port = ctx?.config?.oauth?.proxyPort || 10531;
     return {
-        url: `${ctx?.oauthUrl || `http://127.0.0.1:${port}`}/v1/responses`,
+        url: `${safeBaseUrl(ctx?.oauthUrl || `http://127.0.0.1:${port}`)}/v1/responses`,
         headers: { "Content-Type": "application/json", Accept: "text/event-stream" },
     };
 }
-function tools(webSearchEnabled, imageOptions) {
-    return [
-        ...(webSearchEnabled ? [{ type: "web_search" }] : []),
-        { type: "image_generation", ...imageOptions },
-    ];
-}
 function normalizeRef(ref) {
     const b64 = typeof ref === "string" ? ref : ref?.b64;
     const detectedMime = typeof ref === "object" && ref?.detectedMime
@@ -97,149 +123,6 @@ function normalizeRef(ref) {
             : "image/png";
     return { type: "input_image", image_url: `data:${mime};base64,${b64}` };
 }
-function extractSseData(block) {
-    let eventData = "";
-    for (const line of block.split("\n")) {
-        if (line.startsWith("data: "))
-            eventData += line.slice(6);
-    }
-    return eventData;
-}
-function extractPartialImage(data) {
-    if (typeof data?.type !== "string" || !data.type.includes("partial"))
-        return null;
-    const item = data.item || {};
-    const b64 = data.partial_image || data.image || data.result || item.partial_image || item.image || item.result;
-    if (typeof b64 !== "string" || b64.length === 0)
-        return null;
-    const index = Number.isFinite(data.index) ? data.index : Number.isFinite(item.index) ? item.index : null;
-    return { b64, index };
-}
-function extractTextDelta(data) {
-    if (data.type === "response.output_text.delta" && typeof data.delta === "string")
-        return data.delta;
-    return null;
-}
-function extractFinalText(data) {
-    if (data.type === "response.output_text.done" && typeof data.text === "string")
-        return cleanTextOutput(data.text);
-    if (data.type === "response.output_item.done" && data.item?.type === "message") {
-        return extractJsonItemText(data.item);
-    }
-    return null;
-}
-function extractJsonItemText(item) {
-    if (item.type === "output_text" && typeof item.text === "string")
-        return cleanTextOutput(item.text);
-    if (!Array.isArray(item.content))
-        return null;
-    const text = item.content
-        .filter((part) => part.type === "output_text" && typeof part.text === "string")
-        .map((part) => part.text)
-        .join("\n\n");
-    return cleanTextOutput(text);
-}
-function cleanTextOutput(value) {
-    const trimmed = value.trim();
-    return trimmed ? trimmed.slice(0, 4_000) : null;
-}
-async function parseStream(res, { requestId, scope, maxImages = 1, onPartialImage = null, onFinalImage = null, }) {
-    const reader = res.body.getReader();
-    const decoder = new TextDecoder();
-    const images = [];
-    const eventTypes = {};
-    let buffer = "";
-    let usage = null;
-    let textOutput = "";
-    let finalTextOutput = null;
-    let webSearchCalls = 0;
-    let eventCount = 0;
-    let extraIgnored = 0;
-    while (true) {
-        const { done, value } = await reader.read();
-        if (done)
-            break;
-        buffer += decoder.decode(value, { stream: true });
-        let boundary;
-        while ((boundary = buffer.indexOf("\n\n")) !== -1) {
-            const block = buffer.slice(0, boundary);
-            buffer = buffer.slice(boundary + 2);
-            const eventData = extractSseData(block);
-            if (!eventData || eventData === "[DONE]")
-                continue;
-            let data;
-            try {
-                data = JSON.parse(eventData);
-            }
-            catch {
-                continue;
-            }
-            eventCount++;
-            eventTypes[data.type || "_unknown"] = (eventTypes[data.type || "_unknown"] || 0) + 1;
-            const delta = extractTextDelta(data);
-            if (delta)
-                textOutput += delta;
-            const finalText = extractFinalText(data);
-            if (finalText)
-                finalTextOutput = finalText;
-            const partial = extractPartialImage(data);
-            if (partial && typeof onPartialImage === "function")
-                onPartialImage(partial);
-            if (data.type === "response.output_item.done" && data.item?.type === "image_generation_call") {
-                if (data.item.result && images.length < maxImages) {
-                    const image = {
-                        b64: data.item.result,
-                        revisedPrompt: typeof data.item.revised_prompt === "string" ? data.item.revised_prompt : null,
-                    };
-                    const index = images.length;
-                    images.push(image);
-                    if (requestId)
-                        setJobPhase(requestId, "decoding");
-                    await onFinalImage?.(image, index);
-                }
-                else if (data.item.result)
-                    extraIgnored++;
-            }
-            if (data.type === "response.output_item.done" && data.item?.type === "web_search_call")
-                webSearchCalls++;
-            if (data.type === "response.completed") {
-                usage = data.response?.usage || null;
-                const wsNum = data.response?.tool_usage?.web_search?.num_requests;
-                if (typeof wsNum === "number" && wsNum > webSearchCalls)
-                    webSearchCalls = wsNum;
-            }
-            if (data.type === "error") {
-                throw makeError("Responses stream returned an error", {
-                    code: data.error?.code || "RESPONSES_STREAM_ERROR",
-                    eventCount,
-                    eventType: data.type,
-                });
-            }
-        }
-    }
-    logEvent(scope, "stream_end", { requestId, events: eventCount, imageCount: images.length });
-    return { images, usage, webSearchCalls, eventCount, eventTypes, extraIgnored, text: finalTextOutput ?? cleanTextOutput(textOutput) };
-}
-async function parseJson(res, maxImages) {
-    const json = await res.json();
-    const images = [];
-    const textParts = [];
-    let webSearchCalls = 0;
-    for (const item of json.output || []) {
-        if (item.type === "image_generation_call" && item.result && images.length < maxImages) {
-            images.push({
-                b64: item.result,
-                revisedPrompt: typeof item.revised_prompt === "string" ? item.revised_prompt : null,
-            });
-        }
-        if (item.type === "web_search_call")
-            webSearchCalls++;
-        const itemText = extractJsonItemText(item);
-        if (itemText)
-            textParts.push(itemText);
-    }
-    return { images, usage: json.usage || null, webSearchCalls, eventCount: 0, eventTypes: {}, extraIgnored: 0, text: cleanTextOutput(textParts.join("\n\n")) };
-}
 function combineAbortSignals(signals) {
     if (signals.length === 1)
         return signals[0];
@@ -307,7 +190,14 @@ async function postResponses({ ctx, provider, scope, payload, requestId, maxImag
             }
             throw makeError("Responses image generation timed out", { status: 504, code: "RESPONSES_IMAGE_TIMEOUT", cause: err.raw });
         }
-        throw err.raw;
+        if (isKnownResponsesError(err.raw))
+            throw err.raw;
+        throw makeError("Responses request failed before receiving a response", {
+            status: 502,
+            code: "NETWORK_FAILED",
+            errorName: err.name,
+            upstreamMessageRedacted: true,
+        });
     }
     finally {
         clearTimeout(timer);
@@ -315,7 +205,11 @@ async function postResponses({ ctx, provider, scope, payload, requestId, maxImag
 }
 export async function generateViaResponses(provider, prompt, quality, size, moderation = "low", references = [], requestId = null, mode = "auto", ctxRaw = {}, options = {}) {
     const ctx = requireRuntimeContext(ctxRaw);
+    const model = options.model || ctx.config?.imageModels?.default || "gpt-5.4-mini";
     const webSearchEnabled = options.webSearchEnabled !== false && options.searchMode !== "off";
+    const requestTools = tools(webSearchEnabled, { quality, size, moderation, ...(options.partialImages ? { partial_images: options.partialImages } : {}) });
+    const toolChoice = imageToolChoice(options.forceImageToolChoice ?? ctx.config?.oauth?.forceImageToolChoice !== false);
+    const toolChoiceKind = imageToolChoiceKind(toolChoice);
     const referenceInputs = references.map(normalizeRef);
     const userContent = referenceInputs.length
         ? [...referenceInputs, { type: "input_text", text: buildUserTextPrompt(prompt, mode, { webSearchEnabled }) }]
@@ -330,26 +224,62 @@ export async function generateViaResponses(provider, prompt, quality, size, mode
         onPartialImage: options.onPartialImage,
         onFinalImage: options.onFinalImage,
         payload: {
-            model: options.model || ctx.config?.imageModels?.default || "gpt-5.4-mini",
+            model,
             input: [
                 { role: "developer", content: webSearchEnabled ? GENERATE_DEVELOPER_PROMPT : GENERATE_NO_SEARCH_DEVELOPER_PROMPT },
                 { role: "user", content: userContent },
             ],
-            tools: tools(webSearchEnabled, { quality, size, moderation, ...(options.partialImages ? { partial_images: options.partialImages } : {}) }),
-            tool_choice: "required",
+            tools: requestTools,
+            tool_choice: toolChoice,
             reasoning: { effort: options.reasoningEffort || "low" },
             stream: true,
         },
     });
     const image = result.images[0];
-    if (!image?.b64)
-        throw makeError("No image data received from Responses API", { code: "EMPTY_RESPONSE", eventCount: result.eventCount });
+    if (!image?.b64) {
+        if (options.allowPromptOnlyOAuthFallback === true) {
+            const fallback = await retryPromptOnlyJsonImage({
+                postResponses,
+                ctx,
+                provider,
+                prompt,
+                mode,
+                model,
+                quality,
+                size,
+                moderation,
+                requestId,
+                signal: options.signal,
+                initial: result,
+                referencesDroppedOnRetry: referenceInputs.length > 0,
+                webSearchDroppedOnRetry: webSearchEnabled,
+                reasoningEffort: options.reasoningEffort,
+            });
+            if (fallback)
+                return fallback;
+        }
+        throw emptyResponseError("No image data received from Responses API", result, {
+            provider,
+            model,
+            quality,
+            size,
+            moderation,
+            webSearchEnabled,
+            refsCount: referenceInputs.length,
+            inputImageCount: referenceInputs.length,
+            promptChars: typeof prompt === "string" ? prompt.length : 0,
+            toolTypes: toolTypes(requestTools),
+            toolChoiceKind,
+        });
+    }
     return { b64: image.b64, usage: result.usage, webSearchCalls: result.webSearchCalls, revisedPrompt: image.revisedPrompt, text: result.text };
 }
 export async function generateMultimodeViaResponses(provider, prompt, quality, size, moderation = "low", references = [], requestId = null, mode = "auto", ctxRaw = {}, options = {}) {
     const ctx = requireRuntimeContext(ctxRaw);
     const maxImages = Math.min(8, Math.max(1, Math.trunc(Number(options.maxImages) || 1)));
+    const model = options.model || ctx.config?.imageModels?.default || "gpt-5.4-mini";
     const webSearchEnabled = options.webSearchEnabled !== false && options.searchMode !== "off";
+    const requestTools = tools(webSearchEnabled, { quality, size, moderation, ...(options.partialImages ? { partial_images: options.partialImages } : {}) });
     const userText = buildMultimodeSequencePrompt(mode === "direct"
         ? `${prompt}${DIRECT_PROMPT_FIDELITY_SUFFIX}`
         : `${prompt}${webSearchEnabled ? "" : ""}${AUTO_PROMPT_FIDELITY_SUFFIX}`, maxImages, { webSearchEnabled });
@@ -367,12 +297,12 @@ export async function generateMultimodeViaResponses(provider, prompt, quality, s
         onPartialImage: options.onPartialImage,
         onFinalImage: options.onFinalImage,
         payload: {
-            model: options.model || ctx.config?.imageModels?.default || "gpt-5.4-mini",
+            model,
             input: [
                 { role: "developer", content: webSearchEnabled ? MULTIMODE_DEVELOPER_PROMPT : MULTIMODE_NO_SEARCH_DEVELOPER_PROMPT },
                 { role: "user", content: userContent },
             ],
-            tools: tools(webSearchEnabled, { quality, size, moderation, ...(options.partialImages ? { partial_images: options.partialImages } : {}) }),
+            tools: requestTools,
             tool_choice: "required",
             reasoning: { effort: options.reasoningEffort || "low" },
             stream: true,
@@ -381,7 +311,11 @@ export async function generateMultimodeViaResponses(provider, prompt, quality, s
 }
 export async function editViaResponses(provider, prompt, imageB64, quality, size, moderation = "low", mode = "auto", ctxRaw = {}, requestId = null, options = {}) {
     const ctx = requireRuntimeContext(ctxRaw);
+    const model = options.model || ctx.config?.imageModels?.default || "gpt-5.4-mini";
     const webSearchEnabled = options.webSearchEnabled !== false && options.searchMode !== "off";
+    const requestTools = tools(webSearchEnabled, { quality, size, moderation });
+    const toolChoice = imageToolChoice(options.forceImageToolChoice ?? ctx.config?.oauth?.forceImageToolChoice !== false);
+    const toolChoiceKind = imageToolChoiceKind(toolChoice);
     const imageForRequest = await compressReferenceB64ForOAuth(imageB64, {
         maxB64Bytes: ctx.config?.limits?.maxRefB64Bytes,
         force: true,
@@ -410,19 +344,32 @@ export async function editViaResponses(provider, prompt, imageB64, quality, size
         maxImages: 1,
         signal: options.signal,
         payload: {
-            model: options.model || ctx.config?.imageModels?.default || "gpt-5.4-mini",
+            model,
             input: [
                 { role: "developer", content: webSearchEnabled ? EDIT_DEVELOPER_PROMPT : EDIT_NO_SEARCH_DEVELOPER_PROMPT },
                 { role: "user", content: userContent },
             ],
-            tools: tools(webSearchEnabled, { quality, size, moderation }),
-            tool_choice: "required",
+            tools: requestTools,
+            tool_choice: toolChoice,
             reasoning: { effort: options.reasoningEffort || "low" },
             stream: true,
         },
     });
     const image = result.images[0];
-    if (!image?.b64)
-        throw makeError("No image data received from Responses edit", { code: "EMPTY_RESPONSE", eventCount: result.eventCount });
+    if (!image?.b64) {
+        throw emptyResponseError("No image data received from Responses edit", result, {
+            provider,
+            model,
+            quality,
+            size,
+            moderation,
+            webSearchEnabled,
+            refsCount: referenceImages.length,
+            inputImageCount: 1 + referenceImages.length + (maskContent.length ? 1 : 0),
+            promptChars: typeof prompt === "string" ? prompt.length : 0,
+            toolTypes: toolTypes(requestTools),
+            toolChoiceKind,
+        });
+    }
     return { b64: image.b64, usage: result.usage, revisedPrompt: image.revisedPrompt, webSearchCalls: result.webSearchCalls };
 }