@cartanova/qgrid-cli 2.9.7 → 2.9.9

package/bundle/src/utils/providers/openai/openai-backend-protocol.test.ts

@@ -8,6 +8,11 @@ import {
 } from "./openai-backend-protocol";
 
 describe("OpenAI Codex backend protocol", () => {
+  it("distinguishes partial function items from completed arguments", () => {
+    const item = { type: "function_call", call_id: "call", name: "generate_transparent_image", arguments: "{}" };
+    expect(normalizeOpenAIEvent({ type: "response.output_item.added", item })).toEqual({ type: "output-item", item, completed: false });
+    expect(normalizeOpenAIEvent({ type: "response.output_item.done", item })).toEqual({ type: "output-item", item, completed: true });
+  });
   it("uses returned bytes rather than a mismatched format label", () => {
     expect(normalizeOpenAIEvent({ type: "response.output_item.done", item: {
       type: "image_generation_call", result: "UklGRgAAAABXRUJQ", output_format: "png",

package/bundle/src/utils/providers/openai/openai-backend-protocol.ts

@@ -179,7 +179,7 @@ export type OpenAINormalizedEvent =
       revisedPrompt?: string;
       generation?: ImageGenerationMetadata;
     }
-  | { type: "output-item"; item: OpenAIResponseItem }
+  | { type: "output-item"; item: OpenAIResponseItem; completed?: boolean }
   | { type: "completed"; responseId: string; usage?: OpenAIUsage; model?: string }
   | { type: "error"; error: OpenAIProtocolError };
 
@@ -256,7 +256,7 @@ export function normalizeOpenAIEvent(raw: unknown): OpenAINormalizedEvent | unde
             : {}),
       };
     }
-    return { type: "output-item", item };
+    return { type: "output-item", item, completed: type === "response.output_item.done" };
   }
   if (type === "response.completed") {
     const response = (event.response ?? {}) as Record<string, unknown>;

package/bundle/src/utils/providers/openai/openai-dispatcher.test.ts

@@ -295,6 +295,120 @@ describe("OpenAIDispatcher direct runtime", () => {
     expect(complete).toHaveBeenCalledWith(expect.objectContaining({ text: "ab" }));
   });
 
+  it("returns multiple hosted images with one driver request", async () => {
+    const run = vi.fn(() => events(
+      { type: "image", id: "first", base64: "first", mimeType: "image/png" },
+      { type: "image", id: "second", base64: "second", mimeType: "image/webp" },
+      { type: "completed", responseId: "driver" },
+    ));
+    const d = dispatcher(run);
+    await d.onTokenAdded(1, "one", credentials);
+    const result = await d.generate(request({ imageGeneration: "auto" }));
+    expect(result.images).toMatchObject([{ data: "first", mediaType: "image/png" }, { data: "second", mediaType: "image/webp" }]);
+    expect(run).toHaveBeenCalledTimes(1);
+  });
+
+  it("lets automatic mode return text or a hosted image with one driver request", async () => {
+    for (const image of [false, true]) {
+      const run = vi.fn((options: OpenAIResponsesOptions) => {
+        const body = buildOpenAIResponsesRequest(options);
+        expect(body.parallel_tool_calls).toBe(false);
+        expect(body.tools).toEqual(expect.arrayContaining([{ type: "image_generation" }, expect.objectContaining({ name: "generate_transparent_image" })]));
+        return events(...(image ? [{ type: "image" as const, base64: "png", mimeType: "image/png" as const }] : [{ type: "text-delta" as const, text: "hello" }]), { type: "completed", responseId: "r" });
+      });
+      const d = dispatcher(run);
+      await d.onTokenAdded(1, "one", credentials);
+      const result = await d.generate(request({ imageGeneration: "auto" }));
+      expect(run).toHaveBeenCalledTimes(1);
+      expect(image ? result.images?.length : result.text).toBe(image ? 1 : "hello");
+    }
+  });
+
+  it("runs one standalone transparent call after completed arguments and preserves driver accounting", async () => {
+    const calls: OpenAIResponsesOptions[] = [];
+    const signals: Array<AbortSignal | undefined> = [];
+    const item = { type: "function_call", call_id: "call", name: "generate_transparent_image", arguments: JSON.stringify({ prompt: "remove background" }) };
+    const d = dispatcher((options, signal) => {
+      calls.push(options); signals.push(signal);
+      if (calls.length === 1) return events(
+        { type: "output-item", item: { ...item, arguments: "" }, completed: false },
+        { type: "output-item", item, completed: true },
+        { type: "output-item", item, completed: true },
+        { type: "completed", responseId: "driver", model: "actual-driver", usage: { inputTokens: 10, cachedInputTokens: 2, outputTokens: 3, reasoningTokens: 1, totalTokens: 13 } },
+      );
+      return events({ type: "image", base64: "png", mimeType: "image/png", generation: { route: "codex-images", model: "gpt-image-2" } }, { type: "completed", responseId: "image", model: "gpt-image-2", usage: { inputTokens: 999, outputTokens: 999, cachedInputTokens: 0, reasoningTokens: 0, totalTokens: 1998 } });
+    });
+    await d.onTokenAdded(1, "one", credentials);
+    const history = [{ type: "message", role: "user", content: [{ type: "input_image", image_url: "https://example.com/ref.png" }] }];
+    const result = await d.generate(request({ imageGeneration: "auto", systemPrompt: "system", coldHistory: history, imageGenerationOptions: { outputFormat: "webp" } }));
+    expect(calls).toHaveLength(2);
+    expect(calls[1]).toMatchObject({ instructions: "system", imageGeneration: { background: "transparent", output_format: "png" } });
+    expect(calls[1]!.history[0]).toEqual(history[0]);
+    expect(calls[1]!.tools).toBeUndefined();
+    expect(calls[1]!.outputSchema).toBeUndefined();
+    expect(signals[0]).toBe(signals[1]);
+    expect(result).toMatchObject({ model: "actual-driver", usage: { inputTokens: 10, outputTokens: 3, totalTokens: 13 }, images: [{ generation: { model: "gpt-image-2" } }] });
+  });
+
+  it("rejects incomplete, invalid, multiple and mixed automatic image calls before standalone execution", async () => {
+    const item = { type: "function_call", call_id: "call", name: "generate_transparent_image", arguments: JSON.stringify({ prompt: "remove background" }) };
+    const done = { type: "output-item" as const, item, completed: true };
+    const badEvents: OpenAINormalizedEvent[][] = [
+      [{ ...done, completed: false }],
+      [{ ...done, item: { ...item, arguments: '{"prompt":"x","extra":true}' } }],
+      [done, { ...done, item: { ...item, call_id: "second" } }],
+      [done, { type: "image", base64: "png", mimeType: "image/png" }],
+      [{ type: "output-item", item: { type: "image_generation_call" }, completed: false }],
+    ];
+    for (const stream of badEvents) {
+      const run = vi.fn(() => events(...stream, { type: "completed", responseId: "driver" }));
+      const d = dispatcher(run);
+      await d.onTokenAdded(1, "one", credentials);
+      await expect(d.generate(request({ imageGeneration: "auto" }))).rejects.toMatchObject({ kind: "incomplete" });
+      expect(run).toHaveBeenCalledTimes(1);
+    }
+  });
+
+  it("rejects structured output in auto mode before transport", async () => {
+    const run = vi.fn(() => events());
+    const d = dispatcher(run);
+    await d.onTokenAdded(1, "one", credentials);
+    await expect(d.generate(request({ imageGeneration: "auto", outputSchema: { type: "object" } }))).rejects.toMatchObject({ kind: "gate" });
+    expect(run).not.toHaveBeenCalled();
+  });
+
+  it.each(["missing-image", "missing-completion", "cancel"])("rejects standalone %s", async (failure) => {
+    const controller = new AbortController();
+    let count = 0;
+    const d = dispatcher(async function* (_options, signal) {
+      count++;
+      if (count === 1) {
+        yield { type: "output-item", completed: true, item: { type: "function_call", call_id: "call", name: "generate_transparent_image", arguments: '{"prompt":"transparent"}' } };
+        yield { type: "completed", responseId: "driver" };
+      } else {
+        if (failure === "cancel") { controller.abort(); expect(signal?.aborted).toBe(true); throw signal!.reason; }
+        if (failure === "missing-image") yield { type: "completed", responseId: "image" };
+        else yield { type: "image", base64: "png", mimeType: "image/png" };
+      }
+    });
+    await d.onTokenAdded(1, "one", credentials);
+    await expect(d.generate(request({ imageGeneration: "auto", abortSignal: controller.signal }))).rejects.toMatchObject(failure === "cancel" ? { name: "AbortError" } : { kind: "incomplete" });
+    expect(count).toBe(2);
+  });
+
+  it("does not start standalone generation after cancellation", async () => {
+    const controller = new AbortController();
+    const run = vi.fn(async function* () {
+      yield { type: "output-item" as const, completed: true, item: { type: "function_call", call_id: "call", name: "generate_transparent_image", arguments: '{"prompt":"transparent"}' } };
+      yield { type: "completed" as const, responseId: "driver" };
+      controller.abort();
+    });
+    const d = dispatcher(run);
+    await d.onTokenAdded(1, "one", credentials);
+    await expect(d.generate(request({ imageGeneration: "auto", abortSignal: controller.signal }))).rejects.toMatchObject({ name: "AbortError" });
+    expect(run).toHaveBeenCalledTimes(1);
+  });
+
   it("maps images and preserves image failure classifications", async () => {
     const success = dispatcher(() =>
       events(

package/bundle/src/utils/providers/openai/openai-dispatcher.ts

@@ -26,6 +26,7 @@ const logger = getLogger(["qgrid", "openai-dispatcher"]);
 // 교체된 codex worker 가 강제하던 watchdog. 호출자가 timeoutMs 를 생략해도 요청이
 // 영원히 매달리지 않도록 direct transport 도 유한한 기본 상한을 유지한다.
 const DEFAULT_REQUEST_TIMEOUT_MS = 600_000;
+const TRANSPARENT_IMAGE_TOOL = "generate_transparent_image";
 
 export type ImageFailureKind = "gate" | "not_called" | "incomplete";
 
@@ -113,9 +114,48 @@ function asReasoning(req: GenerateRequest): OpenAIResponsesOptions["reasoning"]
 
 function requestOptions(req: GenerateRequest): OpenAIResponsesOptions {
   const { outputFormat, ...imageControls } = req.imageGenerationOptions ?? {};
+  const auto = req.imageGeneration === "auto";
+  if (auto && req.outputSchema) {
+    throw new ImageGenerationError(
+      "gate",
+      "Automatic image generation does not support structured output or external tools",
+    );
+  }
   return {
     model: req.model ?? "",
     ...(req.systemPrompt ? { instructions: req.systemPrompt } : {}),
+    ...(auto
+      ? {
+          instructions: [
+            req.systemPrompt,
+            "Use image_generation for ordinary image generation or editing. Use generate_transparent_image only when a transparent background is requested. Text replies are allowed when no image is needed. Use only one kind of image tool per response, and at most one generate_transparent_image call.",
+          ]
+            .filter(Boolean)
+            .join("\n\n"),
+          parallelToolCalls: false,
+          tools: [
+            {
+              type: "function",
+              name: TRANSPARENT_IMAGE_TOOL,
+              description:
+                "Generate or edit an image with a transparent background using the conversation's reference images.",
+              strict: true,
+              parameters: {
+                type: "object",
+                properties: {
+                  prompt: {
+                    type: "string",
+                    description:
+                      "Complete instructions for the transparent image generation or edit.",
+                  },
+                },
+                required: ["prompt"],
+                additionalProperties: false,
+              },
+            },
+          ],
+        }
+      : {}),
     history: inputItems(req),
     ...(asReasoning(req) ? { reasoning: asReasoning(req) } : {}),
     ...(req.verbosity ? { verbosity: req.verbosity as OpenAIResponsesOptions["verbosity"] } : {}),
@@ -129,6 +169,7 @@ function requestOptions(req: GenerateRequest): OpenAIResponsesOptions {
           imageGeneration: req.imageGenerationOptions
             ? {
                 ...imageControls,
+                ...(auto ? { background: "auto" } : {}),
                 ...(outputFormat ? { output_format: outputFormat } : {}),
               }
             : true,
@@ -312,6 +353,11 @@ export class OpenAIDispatcher implements ProviderDispatcher {
     };
     const images: GeneratedImage[] = [];
     let imageAttempted = false;
+    const auto = req.imageGeneration === "auto";
+    const transparentCalls = new Map<string, string>();
+    const transparentCallIds = new Set<unknown>();
+    let transparentAttempted = false;
+    let driverCompleted = false;
     let servingModel = req.model ?? "";
     const metadata = selection.metadata;
     const clientEntry = this.acquireClient(selection);
@@ -334,7 +380,42 @@ export class OpenAIDispatcher implements ProviderDispatcher {
           });
         } else if (event.type === "output-item" && event.item.type === "image_generation_call") {
           imageAttempted = true;
+        } else if (auto && event.type === "output-item" && event.item.type === "function_call") {
+          if (event.item.name !== TRANSPARENT_IMAGE_TOOL) {
+            throw new ImageGenerationError("incomplete", "Unexpected automatic image tool call");
+          }
+          transparentAttempted = true;
+          transparentCallIds.add(event.item.call_id);
+          if (!event.completed) continue;
+          const id = event.item.call_id;
+          let args: unknown;
+          try {
+            args = JSON.parse(String(event.item.arguments));
+          } catch {
+            throw new ImageGenerationError("incomplete", "Invalid transparent image arguments");
+          }
+          if (
+            typeof id !== "string" ||
+            !id ||
+            !args ||
+            typeof args !== "object" ||
+            Array.isArray(args) ||
+            Object.keys(args).length !== 1 ||
+            !("prompt" in args) ||
+            typeof args.prompt !== "string" ||
+            !args.prompt.trim()
+          ) {
+            throw new ImageGenerationError("incomplete", "Invalid transparent image arguments");
+          }
+          if (transparentCalls.has(id) && transparentCalls.get(id) !== args.prompt) {
+            throw new ImageGenerationError(
+              "incomplete",
+              "Conflicting transparent image tool calls",
+            );
+          }
+          transparentCalls.set(id, args.prompt);
         } else if (event.type === "completed" && event.usage) {
+          driverCompleted = true;
           servingModel = event.model ?? servingModel;
           usage = {
             totalTokens: event.usage.totalTokens,
@@ -344,6 +425,7 @@ export class OpenAIDispatcher implements ProviderDispatcher {
             reasoningOutputTokens: event.usage.reasoningTokens,
           };
         } else if (event.type === "completed") {
+          driverCompleted = true;
           servingModel = event.model ?? servingModel;
         } else if (event.type === "error") {
           if (req.imageGeneration)
@@ -351,10 +433,63 @@ export class OpenAIDispatcher implements ProviderDispatcher {
           throw event.error;
         }
       }
+      if (auto && transparentAttempted) {
+        if (
+          !driverCompleted ||
+          transparentCalls.size !== 1 ||
+          transparentCallIds.size !== 1 ||
+          imageAttempted
+        ) {
+          throw new ImageGenerationError(
+            "incomplete",
+            "Expected one completed transparent image call without another image tool",
+          );
+        }
+        if (req.abortSignal?.aborted) throw abortError(req.abortSignal);
+        const prompt = [...transparentCalls.values()][0]!;
+        let imageCompleted = false;
+        for await (const event of clientEntry.client.responses(
+          {
+            model: req.model ?? "",
+            ...(req.systemPrompt ? { instructions: req.systemPrompt } : {}),
+            history: [
+              ...inputItems(req),
+              { type: "message", role: "user", content: [{ type: "input_text", text: prompt }] },
+            ],
+            imageGeneration: {
+              quality: req.imageGenerationOptions?.quality,
+              size: req.imageGenerationOptions?.size,
+              output_format: "png",
+              background: "transparent",
+            },
+          },
+          req.abortSignal,
+        )) {
+          if (event.type === "image")
+            images.push({
+              data: event.base64,
+              mediaType: event.mimeType,
+              revisedPrompt: event.revisedPrompt ?? null,
+              ...(event.generation ? { generation: event.generation } : {}),
+            });
+          else if (event.type === "completed") imageCompleted = true;
+          else if (event.type === "error")
+            throw new ImageGenerationError("incomplete", event.error.message);
+        }
+        imageAttempted = true;
+        if (!imageCompleted)
+          throw new ImageGenerationError(
+            "incomplete",
+            "Transparent image generation did not complete",
+          );
+      }
+      if (auto && imageAttempted && !driverCompleted)
+        throw new ImageGenerationError("incomplete", "Image generation response did not complete");
+      if (req.abortSignal?.aborted) throw abortError(req.abortSignal);
     } finally {
       this.releaseClient(clientEntry);
     }
-    if (req.imageGeneration && images.length === 0) {
+    if ((req.imageGeneration === true || (auto && imageAttempted)) && images.length === 0) {
       throw new ImageGenerationError(
         imageAttempted ? "incomplete" : "not_called",
         imageAttempted