@kodelyth/codex 2026.5.40 → 2026.5.42

package/media-understanding-provider.test.ts

@@ -0,0 +1,486 @@
+import { describe, expect, it, vi } from "vitest";
+import { buildCodexMediaUnderstandingProvider } from "./media-understanding-provider.js";
+import type { CodexAppServerClient } from "./src/app-server/client.js";
+import type { CodexServerNotification, JsonValue } from "./src/app-server/protocol.js";
+
+function codexModel(inputModalities: string[] = ["text", "image"]) {
+  return {
+    id: "gpt-5.4",
+    model: "gpt-5.4",
+    upgrade: null,
+    upgradeInfo: null,
+    availabilityNux: null,
+    displayName: "gpt-5.4",
+    description: "GPT-5.4",
+    hidden: false,
+    supportedReasoningEfforts: [{ reasoningEffort: "low", description: "fast" }],
+    defaultReasoningEffort: "low",
+    inputModalities,
+    supportsPersonality: false,
+    additionalSpeedTiers: [],
+    isDefault: true,
+  };
+}
+
+function threadStartResult() {
+  return {
+    thread: {
+      id: "thread-1",
+      sessionId: "session-1",
+      forkedFromId: null,
+      preview: "",
+      ephemeral: true,
+      modelProvider: "openai",
+      createdAt: 1,
+      updatedAt: 1,
+      status: { type: "idle" },
+      path: null,
+      cwd: "/tmp/klaw-agent",
+      cliVersion: "0.125.0",
+      source: "unknown",
+      agentNickname: null,
+      agentRole: null,
+      gitInfo: null,
+      name: null,
+      turns: [],
+    },
+    model: "gpt-5.4",
+    modelProvider: "openai",
+    serviceTier: null,
+    cwd: "/tmp/klaw-agent",
+    instructionSources: [],
+    approvalPolicy: "on-request",
+    approvalsReviewer: "user",
+    sandbox: { type: "dangerFullAccess" },
+    permissionProfile: null,
+    reasoningEffort: null,
+  };
+}
+
+function turnStartResult(status = "inProgress", items: JsonValue[] = []) {
+  return {
+    turn: {
+      id: "turn-1",
+      status,
+      items,
+      error: null,
+      startedAt: null,
+      completedAt: null,
+      durationMs: null,
+    },
+  };
+}
+
+function createFakeClient(options?: {
+  inputModalities?: string[];
+  completeWithItems?: boolean;
+  notifyError?: string;
+  approvalRequestMethod?: string;
+  responseText?: string;
+}) {
+  const notifications = new Set<(notification: CodexServerNotification) => void>();
+  const requestHandlers = new Set<(request: { method: string }) => JsonValue | undefined>();
+  const requests: Array<{ method: string; params?: JsonValue }> = [];
+  const approvalResponses: JsonValue[] = [];
+  const request = vi.fn(async (method: string, params?: JsonValue) => {
+    requests.push({ method, params });
+    if (method === "model/list") {
+      return {
+        data: [codexModel(options?.inputModalities)],
+        nextCursor: null,
+      };
+    }
+    if (method === "thread/start") {
+      return threadStartResult();
+    }
+    if (method === "turn/start") {
+      if (options?.approvalRequestMethod) {
+        for (const handler of requestHandlers) {
+          const response = handler({ method: options.approvalRequestMethod });
+          if (response !== undefined) {
+            approvalResponses.push(response);
+          }
+        }
+      }
+      if (options?.notifyError) {
+        for (const notify of notifications) {
+          notify({
+            method: "error",
+            params: {
+              threadId: "thread-1",
+              turnId: "turn-1",
+              error: {
+                message: options.notifyError,
+                codexErrorInfo: null,
+                additionalDetails: null,
+              },
+              willRetry: false,
+            },
+          });
+        }
+      } else if (!options?.completeWithItems) {
+        for (const notify of notifications) {
+          notify({
+            method: "item/agentMessage/delta",
+            params: {
+              threadId: "thread-1",
+              turnId: "turn-1",
+              itemId: "msg-1",
+              delta: options?.responseText ?? "A red square.",
+            },
+          });
+          notify({
+            method: "turn/completed",
+            params: {
+              threadId: "thread-1",
+              turnId: "turn-1",
+              turn: turnStartResult("completed").turn,
+            },
+          });
+        }
+      }
+      return turnStartResult(
+        options?.completeWithItems ? "completed" : "inProgress",
+        options?.completeWithItems
+          ? [
+              {
+                id: "msg-1",
+                type: "agentMessage",
+                text: options?.responseText ?? "A blue circle.",
+                phase: null,
+                memoryCitation: null,
+              },
+            ]
+          : [],
+      );
+    }
+    return {};
+  });
+
+  const client = {
+    request,
+    addNotificationHandler(handler: (notification: CodexServerNotification) => void) {
+      notifications.add(handler);
+      return () => notifications.delete(handler);
+    },
+    addRequestHandler(handler: (request: { method: string }) => JsonValue | undefined) {
+      requestHandlers.add(handler);
+      return () => requestHandlers.delete(handler);
+    },
+  } as unknown as CodexAppServerClient;
+
+  return { client, requests, approvalResponses };
+}
+
+describe("codex media understanding provider", () => {
+  it("runs image understanding through a bounded Codex app-server turn", async () => {
+    const { client, requests } = createFakeClient();
+    const provider = buildCodexMediaUnderstandingProvider({
+      clientFactory: async () => client,
+    });
+
+    const result = await provider.describeImage?.({
+      buffer: Buffer.from("image-bytes"),
+      fileName: "image.png",
+      mime: "image/png",
+      provider: "codex",
+      model: "gpt-5.4",
+      prompt: "Describe briefly.",
+      timeoutMs: 30_000,
+      cfg: {},
+      agentDir: "/tmp/klaw-agent",
+    });
+
+    expect(result).toEqual({ text: "A red square.", model: "gpt-5.4" });
+    expect(requests.map((entry) => entry.method)).toEqual([
+      "model/list",
+      "thread/start",
+      "turn/start",
+    ]);
+    expect(requests[1]?.params).toEqual({
+      model: "gpt-5.4",
+      modelProvider: "openai",
+      cwd: "/tmp/klaw-agent",
+      approvalPolicy: "on-request",
+      sandbox: "read-only",
+      serviceName: "Klaw",
+      developerInstructions:
+        "You are Klaw's bounded image-understanding worker. Describe only the provided image content. Do not call tools, edit files, or ask follow-up questions.",
+      dynamicTools: [],
+      experimentalRawEvents: true,
+      ephemeral: true,
+      persistExtendedHistory: false,
+    });
+    expect(requests[2]?.params).toEqual({
+      threadId: "thread-1",
+      input: [
+        { type: "text", text: "Describe briefly.", text_elements: [] },
+        { type: "image", url: "data:image/png;base64,aW1hZ2UtYnl0ZXM=" },
+      ],
+      cwd: "/tmp/klaw-agent",
+      approvalPolicy: "on-request",
+      model: "gpt-5.4",
+      effort: "low",
+    });
+  });
+
+  it("declines approval requests during image understanding", async () => {
+    const { client, approvalResponses } = createFakeClient({
+      approvalRequestMethod: "item/permissions/requestApproval",
+    });
+    const provider = buildCodexMediaUnderstandingProvider({
+      clientFactory: async () => client,
+    });
+
+    await provider.describeImage?.({
+      buffer: Buffer.from("image-bytes"),
+      fileName: "image.png",
+      mime: "image/png",
+      provider: "codex",
+      model: "gpt-5.4",
+      prompt: "Describe briefly.",
+      timeoutMs: 30_000,
+      cfg: {},
+      agentDir: "/tmp/klaw-agent",
+    });
+
+    expect(approvalResponses).toEqual([{ permissions: {}, scope: "turn" }]);
+  });
+
+  it("extracts text from terminal turn items", async () => {
+    const { client } = createFakeClient({ completeWithItems: true });
+    const provider = buildCodexMediaUnderstandingProvider({
+      clientFactory: async () => client,
+    });
+
+    const result = await provider.describeImages?.({
+      images: [{ buffer: Buffer.from("image-bytes"), fileName: "image.png", mime: "image/png" }],
+      provider: "codex",
+      model: "gpt-5.4",
+      prompt: "Describe briefly.",
+      timeoutMs: 30_000,
+      cfg: {},
+      agentDir: "/tmp/klaw-agent",
+    });
+
+    expect(result).toEqual({ text: "A blue circle.", model: "gpt-5.4" });
+  });
+
+  it("rejects text-only Codex app-server models before starting a turn", async () => {
+    const { client, requests } = createFakeClient({ inputModalities: ["text"] });
+    const provider = buildCodexMediaUnderstandingProvider({
+      clientFactory: async () => client,
+    });
+
+    await expect(
+      provider.describeImage?.({
+        buffer: Buffer.from("image-bytes"),
+        fileName: "image.png",
+        mime: "image/png",
+        provider: "codex",
+        model: "gpt-5.4",
+        timeoutMs: 30_000,
+        cfg: {},
+        agentDir: "/tmp/klaw-agent",
+      }),
+    ).rejects.toThrow("Codex app-server model does not support images: gpt-5.4");
+    expect(requests.map((entry) => entry.method)).toEqual(["model/list"]);
+  });
+
+  it("surfaces Codex app-server turn errors", async () => {
+    const { client } = createFakeClient({ notifyError: "vision unavailable" });
+    const provider = buildCodexMediaUnderstandingProvider({
+      clientFactory: async () => client,
+    });
+
+    await expect(
+      provider.describeImage?.({
+        buffer: Buffer.from("image-bytes"),
+        fileName: "image.png",
+        mime: "image/png",
+        provider: "codex",
+        model: "gpt-5.4",
+        timeoutMs: 30_000,
+        cfg: {},
+        agentDir: "/tmp/klaw-agent",
+      }),
+    ).rejects.toThrow("vision unavailable");
+  });
+
+  it("runs structured extraction through the same bounded Codex app-server path", async () => {
+    const { client, requests } = createFakeClient({
+      responseText: '{"summary":"red square","tags":["shape"]}',
+    });
+    const provider = buildCodexMediaUnderstandingProvider({
+      clientFactory: async () => client,
+    });
+
+    const result = await provider.extractStructured?.({
+      input: [
+        { type: "text", text: "Extract searchable evidence." },
+        {
+          type: "image",
+          buffer: Buffer.from("image-bytes"),
+          fileName: "image.png",
+          mime: "image/png",
+        },
+      ],
+      instructions: "Return a compact evidence object.",
+      schemaName: "example.media",
+      jsonSchema: {
+        type: "object",
+        properties: {
+          summary: { type: "string" },
+          tags: { type: "array", items: { type: "string" } },
+        },
+        required: ["summary"],
+      },
+      provider: "codex",
+      model: "gpt-5.4",
+      timeoutMs: 30_000,
+      cfg: {},
+      agentDir: "/tmp/klaw-agent",
+    });
+
+    expect(result).toEqual({
+      text: '{"summary":"red square","tags":["shape"]}',
+      parsed: { summary: "red square", tags: ["shape"] },
+      model: "gpt-5.4",
+      provider: "codex",
+      contentType: "json",
+    });
+    expect(requests.map((entry) => entry.method)).toEqual([
+      "model/list",
+      "thread/start",
+      "turn/start",
+    ]);
+    expect(requests[1]?.params).toEqual({
+      model: "gpt-5.4",
+      modelProvider: "openai",
+      cwd: "/tmp/klaw-agent",
+      approvalPolicy: "on-request",
+      sandbox: "read-only",
+      serviceName: "Klaw",
+      developerInstructions:
+        "You are Klaw's bounded structured-extraction worker. Return only the requested extraction. Do not call tools, edit files, ask follow-up questions, or include secrets.",
+      dynamicTools: [],
+      experimentalRawEvents: true,
+      ephemeral: true,
+      persistExtendedHistory: false,
+    });
+    const turnParams = requests[2]?.params as
+      | {
+          threadId?: unknown;
+          approvalPolicy?: unknown;
+          model?: unknown;
+          input?: Array<{ type?: unknown; text?: unknown; text_elements?: unknown; url?: unknown }>;
+          cwd?: unknown;
+          effort?: unknown;
+        }
+      | undefined;
+    expect(turnParams?.threadId).toBe("thread-1");
+    expect(turnParams?.approvalPolicy).toBe("on-request");
+    expect(turnParams?.model).toBe("gpt-5.4");
+    expect(turnParams?.cwd).toBe("/tmp/klaw-agent");
+    expect(turnParams?.effort).toBe("low");
+    expect(turnParams?.input).toHaveLength(3);
+    expect(turnParams?.input?.[0]?.type).toBe("text");
+    expect(turnParams?.input?.[0]?.text).toContain("Return valid JSON only");
+    expect(turnParams?.input?.[0]?.text_elements).toStrictEqual([]);
+    expect(turnParams?.input?.[1]).toStrictEqual({
+      type: "text",
+      text: "Extract searchable evidence.",
+      text_elements: [],
+    });
+    expect(turnParams?.input?.[2]).toStrictEqual({
+      type: "image",
+      url: "data:image/png;base64,aW1hZ2UtYnl0ZXM=",
+    });
+  });
+
+  it("rejects text-only structured extraction before starting a turn", async () => {
+    const { client, requests } = createFakeClient({
+      inputModalities: ["text"],
+      responseText: '{"summary":"only text"}',
+    });
+    const provider = buildCodexMediaUnderstandingProvider({
+      clientFactory: async () => client,
+    });
+
+    await expect(
+      provider.extractStructured?.({
+        input: [{ type: "text", text: "The answer is only text." }],
+        instructions: "Return summary JSON.",
+        provider: "codex",
+        model: "gpt-5.4",
+        timeoutMs: 30_000,
+        cfg: {},
+        agentDir: "/tmp/klaw-agent",
+      }),
+    ).rejects.toThrow("Codex structured extraction requires at least one image input.");
+    expect(requests).toEqual([]);
+  });
+
+  it("returns a controlled error when structured JSON parsing fails", async () => {
+    const { client } = createFakeClient({ responseText: "not json" });
+    const provider = buildCodexMediaUnderstandingProvider({
+      clientFactory: async () => client,
+    });
+
+    await expect(
+      provider.extractStructured?.({
+        input: [
+          { type: "text", text: "Extract JSON." },
+          {
+            type: "image",
+            buffer: Buffer.from("image-bytes"),
+            fileName: "image.png",
+            mime: "image/png",
+          },
+        ],
+        instructions: "Return summary JSON.",
+        provider: "codex",
+        model: "gpt-5.4",
+        timeoutMs: 30_000,
+        cfg: {},
+        agentDir: "/tmp/klaw-agent",
+      }),
+    ).rejects.toThrow("Codex structured extraction returned invalid JSON.");
+  });
+
+  it("validates structured extraction JSON against the requested schema", async () => {
+    const { client } = createFakeClient({
+      responseText: '{"summary":123,"tags":["shape"]}',
+    });
+    const provider = buildCodexMediaUnderstandingProvider({
+      clientFactory: async () => client,
+    });
+
+    await expect(
+      provider.extractStructured?.({
+        input: [
+          { type: "text", text: "Extract JSON." },
+          {
+            type: "image",
+            buffer: Buffer.from("image-bytes"),
+            fileName: "image.png",
+            mime: "image/png",
+          },
+        ],
+        instructions: "Return summary JSON.",
+        jsonSchema: {
+          type: "object",
+          properties: {
+            summary: { type: "string" },
+          },
+          required: ["summary"],
+        },
+        provider: "codex",
+        model: "gpt-5.4",
+        timeoutMs: 30_000,
+        cfg: {},
+        agentDir: "/tmp/klaw-agent",
+      }),
+    ).rejects.toThrow("Codex structured extraction JSON did not match schema");
+  });
+});