@openclaw/codex 2026.5.1-beta.1

package/media-understanding-provider.ts

@@ -0,0 +1,369 @@
+import {
+  type ImagesDescriptionRequest,
+  type ImagesDescriptionResult,
+  type MediaUnderstandingProvider,
+} from "openclaw/plugin-sdk/media-understanding";
+import { CODEX_PROVIDER_ID, FALLBACK_CODEX_MODELS } from "./provider-catalog.js";
+import { type CodexAppServerClientFactory } from "./src/app-server/client-factory.js";
+import type { CodexAppServerClient } from "./src/app-server/client.js";
+import { resolveCodexAppServerRuntimeOptions } from "./src/app-server/config.js";
+import { readModelListResult } from "./src/app-server/models.js";
+import {
+  assertCodexThreadStartResponse,
+  assertCodexTurnStartResponse,
+  readCodexErrorNotification,
+  readCodexTurnCompletedNotification,
+} from "./src/app-server/protocol-validators.js";
+import {
+  isJsonObject,
+  type CodexServerNotification,
+  type CodexThreadItem,
+  type CodexThreadStartParams,
+  type CodexTurn,
+  type CodexTurnStartParams,
+  type JsonObject,
+  type JsonValue,
+} from "./src/app-server/protocol.js";
+
+const DEFAULT_CODEX_IMAGE_MODEL =
+  FALLBACK_CODEX_MODELS.find((model) => model.inputModalities.includes("image"))?.id ??
+  FALLBACK_CODEX_MODELS[0]?.id;
+const DEFAULT_CODEX_IMAGE_PROMPT = "Describe the image.";
+
+export type CodexMediaUnderstandingProviderOptions = {
+  pluginConfig?: unknown;
+  clientFactory?: CodexAppServerClientFactory;
+};
+
+export function buildCodexMediaUnderstandingProvider(
+  options: CodexMediaUnderstandingProviderOptions = {},
+): MediaUnderstandingProvider {
+  return {
+    id: CODEX_PROVIDER_ID,
+    capabilities: ["image"],
+    ...(DEFAULT_CODEX_IMAGE_MODEL ? { defaultModels: { image: DEFAULT_CODEX_IMAGE_MODEL } } : {}),
+    describeImage: async (req) =>
+      describeCodexImages(
+        {
+          images: [
+            {
+              buffer: req.buffer,
+              fileName: req.fileName,
+              mime: req.mime,
+            },
+          ],
+          provider: req.provider,
+          model: req.model,
+          prompt: req.prompt,
+          maxTokens: req.maxTokens,
+          timeoutMs: req.timeoutMs,
+          profile: req.profile,
+          preferredProfile: req.preferredProfile,
+          authStore: req.authStore,
+          agentDir: req.agentDir,
+          cfg: req.cfg,
+        },
+        options,
+      ),
+    describeImages: async (req) => describeCodexImages(req, options),
+  };
+}
+
+async function describeCodexImages(
+  req: ImagesDescriptionRequest,
+  options: CodexMediaUnderstandingProviderOptions,
+): Promise<ImagesDescriptionResult> {
+  const model = req.model.trim();
+  if (!model) {
+    throw new Error("Codex image understanding requires model id.");
+  }
+
+  const appServer = resolveCodexAppServerRuntimeOptions({ pluginConfig: options.pluginConfig });
+  const timeoutMs = Math.max(100, req.timeoutMs);
+  const ownsClient = !options.clientFactory;
+  const client = options.clientFactory
+    ? await options.clientFactory(appServer.start, req.profile)
+    : await import("./src/app-server/shared-client.js").then(
+        ({ createIsolatedCodexAppServerClient }) =>
+          createIsolatedCodexAppServerClient({
+            startOptions: appServer.start,
+            timeoutMs,
+            authProfileId: req.profile,
+          }),
+      );
+  const abortController = new AbortController();
+  const timeout = setTimeout(() => abortController.abort("timeout"), timeoutMs);
+  timeout.unref?.();
+
+  try {
+    await assertCodexModelSupportsImage({
+      client,
+      model,
+      timeoutMs,
+      signal: abortController.signal,
+    });
+    const thread = assertCodexThreadStartResponse(
+      await client.request<unknown>(
+        "thread/start",
+        {
+          model,
+          modelProvider: "openai",
+          cwd: req.agentDir || process.cwd(),
+          approvalPolicy: "on-request",
+          sandbox: "read-only",
+          serviceName: "OpenClaw",
+          developerInstructions:
+            "You are OpenClaw's bounded image-understanding worker. Describe only the provided image content. Do not call tools, edit files, or ask follow-up questions.",
+          dynamicTools: [],
+          experimentalRawEvents: true,
+          persistExtendedHistory: false,
+          ephemeral: true,
+        } satisfies CodexThreadStartParams,
+        { timeoutMs, signal: abortController.signal },
+      ),
+    );
+    const collector = createCodexImageTurnCollector(thread.thread.id);
+    const cleanup = client.addNotificationHandler(collector.handleNotification);
+    const requestCleanup = client.addRequestHandler(denyCodexImageApprovalRequest);
+    try {
+      const turn = assertCodexTurnStartResponse(
+        await client.request<unknown>(
+          "turn/start",
+          {
+            threadId: thread.thread.id,
+            input: [
+              { type: "text", text: buildCodexImagePrompt(req), text_elements: [] },
+              ...req.images.map((image) => ({
+                type: "image" as const,
+                url: `data:${image.mime ?? "image/png"};base64,${image.buffer.toString("base64")}`,
+              })),
+            ],
+            cwd: req.agentDir || process.cwd(),
+            approvalPolicy: "on-request",
+            model,
+            effort: "low",
+          } satisfies CodexTurnStartParams,
+          { timeoutMs, signal: abortController.signal },
+        ),
+      );
+      const text = await collector.collect(turn.turn, {
+        timeoutMs,
+        signal: abortController.signal,
+      });
+      return { text, model };
+    } finally {
+      requestCleanup();
+      cleanup();
+    }
+  } finally {
+    clearTimeout(timeout);
+    if (ownsClient) {
+      client.close();
+    }
+  }
+}
+
+function denyCodexImageApprovalRequest(request: { method: string }): JsonValue | undefined {
+  if (
+    request.method === "item/commandExecution/requestApproval" ||
+    request.method === "item/fileChange/requestApproval"
+  ) {
+    return {
+      decision: "decline",
+      reason: "OpenClaw Codex image understanding does not grant tool or file approvals.",
+    };
+  }
+  if (request.method === "item/permissions/requestApproval") {
+    return { permissions: {}, scope: "turn" };
+  }
+  if (request.method.includes("requestApproval")) {
+    return {
+      decision: "decline",
+      reason: "OpenClaw Codex image understanding does not grant native approvals.",
+    };
+  }
+  if (request.method === "mcpServer/elicitation/request") {
+    return { action: "decline" };
+  }
+  return undefined;
+}
+
+async function assertCodexModelSupportsImage(params: {
+  client: CodexAppServerClient;
+  model: string;
+  timeoutMs: number;
+  signal: AbortSignal;
+}): Promise<void> {
+  const result = await params.client.request<unknown>(
+    "model/list",
+    { limit: 100, cursor: null, includeHidden: false },
+    { timeoutMs: Math.min(params.timeoutMs, 5_000), signal: params.signal },
+  );
+  const listed = readModelListResult(result).models;
+  const match = listed.find((entry) => entry.model === params.model || entry.id === params.model);
+  if (!match) {
+    throw new Error(`Codex app-server model not found: ${params.model}`);
+  }
+  if (!match.inputModalities.includes("image")) {
+    throw new Error(`Codex app-server model does not support images: ${params.model}`);
+  }
+}
+
+function buildCodexImagePrompt(req: ImagesDescriptionRequest): string {
+  const prompt = req.prompt?.trim() || DEFAULT_CODEX_IMAGE_PROMPT;
+  if (req.images.length <= 1) {
+    return prompt;
+  }
+  return `${prompt}\n\nAnalyze all ${req.images.length} images together.`;
+}
+
+function createCodexImageTurnCollector(threadId: string) {
+  let turnId: string | undefined;
+  let completedTurn: CodexTurn | undefined;
+  let promptError: string | undefined;
+  const pending: CodexServerNotification[] = [];
+  const assistantTextByItem = new Map<string, string>();
+  const assistantItemOrder: string[] = [];
+  let resolveCompletion: (() => void) | undefined;
+  const completion = new Promise<void>((resolve) => {
+    resolveCompletion = resolve;
+  });
+
+  const rememberAssistantText = (itemId: string, text: string) => {
+    if (!text) {
+      return;
+    }
+    if (!assistantTextByItem.has(itemId)) {
+      assistantItemOrder.push(itemId);
+    }
+    assistantTextByItem.set(itemId, text);
+  };
+
+  const handleNotification = (notification: CodexServerNotification): void => {
+    const params = isJsonObject(notification.params) ? notification.params : undefined;
+    if (!params || readString(params, "threadId") !== threadId) {
+      return;
+    }
+    if (!turnId) {
+      pending.push(notification);
+      return;
+    }
+    const notificationTurnId = readNotificationTurnId(params);
+    if (notificationTurnId !== turnId) {
+      return;
+    }
+    if (notification.method === "item/agentMessage/delta") {
+      const itemId = readString(params, "itemId") ?? readString(params, "id") ?? "assistant";
+      const delta = readString(params, "delta") ?? "";
+      rememberAssistantText(itemId, `${assistantTextByItem.get(itemId) ?? ""}${delta}`);
+      return;
+    }
+    if (notification.method === "turn/completed") {
+      completedTurn =
+        readCodexTurnCompletedNotification(notification.params)?.turn ?? completedTurn;
+      resolveCompletion?.();
+      return;
+    }
+    if (notification.method === "error") {
+      promptError =
+        readCodexErrorNotification(notification.params)?.error.message ??
+        "codex app-server image turn failed";
+      resolveCompletion?.();
+    }
+  };
+
+  return {
+    handleNotification,
+    async collect(
+      startedTurn: CodexTurn,
+      options: { timeoutMs: number; signal: AbortSignal },
+    ): Promise<string> {
+      turnId = startedTurn.id;
+      if (isTerminalTurn(startedTurn)) {
+        completedTurn = startedTurn;
+      }
+      for (const notification of pending.splice(0)) {
+        handleNotification(notification);
+      }
+      if (!completedTurn && !promptError) {
+        await waitForTurnCompletion({
+          completion,
+          timeoutMs: options.timeoutMs,
+          signal: options.signal,
+        });
+      }
+      if (promptError) {
+        throw new Error(promptError);
+      }
+      if (completedTurn?.status === "failed") {
+        throw new Error(completedTurn.error?.message ?? "codex app-server image turn failed");
+      }
+      const itemText = collectAssistantTextFromItems(completedTurn?.items);
+      const deltaText = assistantItemOrder
+        .map((itemId) => assistantTextByItem.get(itemId)?.trim())
+        .filter((text): text is string => Boolean(text))
+        .join("\n\n")
+        .trim();
+      const text = (itemText || deltaText).trim();
+      if (!text) {
+        throw new Error("Codex app-server image turn returned no text.");
+      }
+      return text;
+    },
+  };
+}
+
+async function waitForTurnCompletion(params: {
+  completion: Promise<void>;
+  timeoutMs: number;
+  signal: AbortSignal;
+}): Promise<void> {
+  let timeout: ReturnType<typeof setTimeout> | undefined;
+  let cleanupAbort: (() => void) | undefined;
+  try {
+    await Promise.race([
+      params.completion,
+      new Promise<never>((_, reject) => {
+        timeout = setTimeout(
+          () => reject(new Error("codex app-server image turn timed out")),
+          params.timeoutMs,
+        );
+        timeout.unref?.();
+        const abortListener = () => reject(new Error("codex app-server image turn aborted"));
+        params.signal.addEventListener("abort", abortListener, { once: true });
+        cleanupAbort = () => params.signal.removeEventListener("abort", abortListener);
+      }),
+    ]);
+  } finally {
+    if (timeout) {
+      clearTimeout(timeout);
+    }
+    cleanupAbort?.();
+  }
+}
+
+function collectAssistantTextFromItems(items: CodexThreadItem[] | undefined): string {
+  return (items ?? [])
+    .filter((item) => item.type === "agentMessage")
+    .map((item) => item.text.trim())
+    .filter(Boolean)
+    .join("\n\n")
+    .trim();
+}
+
+function readNotificationTurnId(record: JsonObject): string | undefined {
+  const direct = readString(record, "turnId");
+  if (direct) {
+    return direct;
+  }
+  return isJsonObject(record.turn) ? readString(record.turn, "id") : undefined;
+}
+
+function readString(record: JsonObject, key: string): string | undefined {
+  const value = record[key];
+  return typeof value === "string" ? value : undefined;
+}
+
+function isTerminalTurn(turn: CodexTurn): boolean {
+  return turn.status === "completed" || turn.status === "interrupted" || turn.status === "failed";
+}

package/openclaw.plugin.json

@@ -0,0 +1,299 @@
+{
+  "id": "codex",
+  "name": "Codex",
+  "description": "Codex app-server harness and Codex-managed GPT model catalog.",
+  "providers": ["codex"],
+  "contracts": {
+    "mediaUnderstandingProviders": ["codex"],
+    "migrationProviders": ["codex"]
+  },
+  "mediaUnderstandingProviderMetadata": {
+    "codex": {
+      "capabilities": ["image"],
+      "defaultModels": {
+        "image": "gpt-5.5"
+      }
+    }
+  },
+  "providerDiscoveryEntry": "./provider-discovery.ts",
+  "syntheticAuthRefs": ["codex"],
+  "nonSecretAuthMarkers": ["codex-app-server"],
+  "activation": {
+    "onStartup": false,
+    "onAgentHarnesses": ["codex"]
+  },
+  "commandAliases": [
+    {
+      "name": "codex",
+      "kind": "runtime-slash",
+      "cliCommand": "plugins"
+    }
+  ],
+  "configSchema": {
+    "type": "object",
+    "additionalProperties": false,
+    "properties": {
+      "codexDynamicToolsProfile": {
+        "type": "string",
+        "enum": ["native-first", "openclaw-compat"],
+        "default": "native-first"
+      },
+      "codexDynamicToolsExclude": {
+        "type": "array",
+        "items": { "type": "string" },
+        "default": []
+      },
+      "discovery": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "enabled": { "type": "boolean" },
+          "timeoutMs": {
+            "type": "number",
+            "minimum": 1,
+            "default": 2500
+          }
+        }
+      },
+      "computerUse": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "enabled": {
+            "type": "boolean",
+            "default": false
+          },
+          "autoInstall": {
+            "type": "boolean",
+            "default": false
+          },
+          "marketplaceDiscoveryTimeoutMs": {
+            "type": "number",
+            "minimum": 1,
+            "default": 60000
+          },
+          "marketplaceSource": {
+            "type": "string"
+          },
+          "marketplacePath": {
+            "type": "string"
+          },
+          "marketplaceName": {
+            "type": "string"
+          },
+          "pluginName": {
+            "type": "string",
+            "default": "computer-use"
+          },
+          "mcpServerName": {
+            "type": "string",
+            "default": "computer-use"
+          }
+        }
+      },
+      "appServer": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "mode": {
+            "type": "string",
+            "enum": ["yolo", "guardian"],
+            "default": "yolo"
+          },
+          "transport": {
+            "type": "string",
+            "enum": ["stdio", "websocket"],
+            "default": "stdio"
+          },
+          "command": { "type": "string" },
+          "args": {
+            "oneOf": [
+              {
+                "type": "array",
+                "items": { "type": "string" }
+              },
+              { "type": "string" }
+            ]
+          },
+          "url": { "type": "string" },
+          "authToken": { "type": "string" },
+          "headers": {
+            "type": "object",
+            "additionalProperties": { "type": "string" }
+          },
+          "clearEnv": {
+            "type": "array",
+            "items": { "type": "string" }
+          },
+          "requestTimeoutMs": {
+            "type": "number",
+            "minimum": 1,
+            "default": 60000
+          },
+          "approvalPolicy": {
+            "type": "string",
+            "enum": ["never", "on-request", "on-failure", "untrusted"]
+          },
+          "sandbox": {
+            "type": "string",
+            "enum": ["read-only", "workspace-write", "danger-full-access"]
+          },
+          "approvalsReviewer": {
+            "type": "string",
+            "enum": ["user", "auto_review", "guardian_subagent"]
+          },
+          "serviceTier": { "type": ["string", "null"], "enum": ["fast", "flex", null] },
+          "defaultWorkspaceDir": {
+            "type": "string"
+          }
+        }
+      }
+    }
+  },
+  "uiHints": {
+    "codexDynamicToolsProfile": {
+      "label": "Dynamic Tools Profile",
+      "help": "Select which OpenClaw dynamic tools are exposed to Codex app-server. native-first omits tools Codex already owns.",
+      "advanced": true
+    },
+    "codexDynamicToolsExclude": {
+      "label": "Dynamic Tool Excludes",
+      "help": "Additional OpenClaw dynamic tool names to omit from Codex app-server turns.",
+      "advanced": true
+    },
+    "discovery": {
+      "label": "Model Discovery",
+      "help": "Plugin-owned controls for discovering Codex app-server models."
+    },
+    "discovery.enabled": {
+      "label": "Enable Discovery",
+      "help": "When false, OpenClaw keeps the Codex harness available but uses the bundled fallback model list."
+    },
+    "discovery.timeoutMs": {
+      "label": "Discovery Timeout",
+      "help": "Maximum time to wait for Codex app-server model discovery before falling back to the bundled model list.",
+      "advanced": true
+    },
+    "computerUse": {
+      "label": "Computer Use",
+      "help": "Controls Codex app-server setup for the Computer Use plugin.",
+      "advanced": true
+    },
+    "computerUse.enabled": {
+      "label": "Enable Computer Use",
+      "help": "When true, Codex-mode turns require the configured Computer Use MCP server to be available.",
+      "advanced": true
+    },
+    "computerUse.autoInstall": {
+      "label": "Auto Install",
+      "help": "Install the configured Computer Use plugin when Codex-mode turns start.",
+      "advanced": true
+    },
+    "computerUse.marketplaceDiscoveryTimeoutMs": {
+      "label": "Marketplace Discovery Timeout",
+      "help": "Maximum time to wait for Codex app-server to finish loading marketplaces during Computer Use install.",
+      "advanced": true
+    },
+    "computerUse.marketplaceSource": {
+      "label": "Marketplace Source",
+      "help": "Optional Codex marketplace source to add before installing Computer Use.",
+      "advanced": true
+    },
+    "computerUse.marketplacePath": {
+      "label": "Marketplace Path",
+      "help": "Optional local Codex marketplace file path containing the Computer Use plugin.",
+      "advanced": true
+    },
+    "computerUse.marketplaceName": {
+      "label": "Marketplace Name",
+      "help": "Optional registered Codex marketplace name containing the Computer Use plugin.",
+      "advanced": true
+    },
+    "computerUse.pluginName": {
+      "label": "Plugin Name",
+      "help": "Codex marketplace plugin name for Computer Use.",
+      "advanced": true
+    },
+    "computerUse.mcpServerName": {
+      "label": "MCP Server Name",
+      "help": "MCP server name exposed by the Computer Use plugin.",
+      "advanced": true
+    },
+    "appServer": {
+      "label": "App Server",
+      "help": "Runtime controls for connecting to Codex app-server.",
+      "advanced": true
+    },
+    "appServer.mode": {
+      "label": "Execution Mode",
+      "help": "Use yolo for unchained local execution or guardian for Codex guardian-reviewed approvals.",
+      "advanced": true
+    },
+    "appServer.transport": {
+      "label": "Transport",
+      "help": "Use stdio to spawn Codex locally, or websocket to connect to an already-running app-server.",
+      "advanced": true
+    },
+    "appServer.command": {
+      "label": "Command",
+      "help": "Executable used for stdio transport. Leave unset to use OpenClaw's managed Codex binary.",
+      "advanced": true
+    },
+    "appServer.args": {
+      "label": "Arguments",
+      "help": "Arguments used for stdio transport. Defaults to app-server --listen stdio://.",
+      "advanced": true
+    },
+    "appServer.url": {
+      "label": "WebSocket URL",
+      "help": "Codex app-server WebSocket URL when transport is websocket.",
+      "advanced": true
+    },
+    "appServer.authToken": {
+      "label": "Auth Token",
+      "help": "Bearer token sent to the WebSocket app-server.",
+      "sensitive": true,
+      "advanced": true
+    },
+    "appServer.headers": {
+      "label": "Headers",
+      "help": "Additional headers sent to the WebSocket app-server.",
+      "advanced": true
+    },
+    "appServer.clearEnv": {
+      "label": "Clear Environment",
+      "help": "Environment variable names removed from the spawned stdio app-server process after overrides are applied.",
+      "advanced": true
+    },
+    "appServer.requestTimeoutMs": {
+      "label": "Request Timeout",
+      "help": "Maximum time to wait for Codex app-server control-plane requests.",
+      "advanced": true
+    },
+    "appServer.approvalPolicy": {
+      "label": "Approval Policy",
+      "help": "Codex native approval policy sent to thread start, resume, and turns.",
+      "advanced": true
+    },
+    "appServer.sandbox": {
+      "label": "Sandbox",
+      "help": "Codex native sandbox mode sent to thread start and resume.",
+      "advanced": true
+    },
+    "appServer.approvalsReviewer": {
+      "label": "Approvals Reviewer",
+      "help": "Use user approvals or Codex auto_review for native app-server approvals. guardian_subagent remains accepted for compatibility.",
+      "advanced": true
+    },
+    "appServer.serviceTier": {
+      "label": "Service Tier",
+      "help": "Optional Codex app-server service tier. Use fast, flex, or null.",
+      "advanced": true
+    },
+    "appServer.defaultWorkspaceDir": {
+      "label": "Default Workspace",
+      "help": "Workspace used by /codex bind when --cwd is omitted.",
+      "advanced": true
+    }
+  }
+}