@openclaw/codex 2026.6.8 → 2026.6.9-beta.1

package/dist/media-understanding-provider-CiwPkR58.js

@@ -0,0 +1,463 @@
+import { CODEX_PROVIDER_ID, FALLBACK_CODEX_MODELS } from "./provider-catalog.js";
+import { n as isJsonObject } from "./protocol-dh-ETiNd.js";
+import { M as mergeCodexThreadConfigs, r as buildCodexRuntimeThreadConfig } from "./thread-lifecycle-CbJqqnEw.js";
+import { S as readCodexNotificationItem } from "./attempt-notifications-Meja3TX6.js";
+import { d as resolveCodexAppServerRuntimeOptions } from "./config-BksX0T33.js";
+import { i as assertCodexTurnStartResponse, l as readCodexTurnCompletedNotification, o as readCodexErrorNotification, r as assertCodexThreadStartResponse } from "./protocol-validators-B19q5BIX.js";
+import { i as readModelListResult } from "./models-BsyV1Psm.js";
+import { validateJsonSchemaValue } from "openclaw/plugin-sdk/json-schema-runtime";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { resolveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime";
+import { resolvePreferredOpenClawTmpDir, withTempWorkspace } from "openclaw/plugin-sdk/temp-path";
+//#region extensions/codex/src/app-server/bounded-turn.ts
+const CODEX_PRIVATE_STDIO_ARGS = [
+	"app-server",
+	"--listen",
+	"stdio://"
+];
+const OPENCLAW_CODEX_APP_SERVER_ARGS_ENV_VAR = "OPENCLAW_CODEX_APP_SERVER_ARGS";
+const CODEX_BOUNDED_THREAD_CONFIG = {
+	"features.multi_agent": false,
+	"features.apps": false,
+	"features.plugins": false,
+	"features.image_generation": false,
+	"features.standalone_web_search": false,
+	web_search: "disabled"
+};
+const CODEX_PRIVATE_BOUNDED_THREAD_CONFIG = {
+	"features.hooks": false,
+	notify: []
+};
+async function runBoundedCodexAppServerTurn(params) {
+	const appServer = resolveCodexAppServerRuntimeOptions({ pluginConfig: params.options.pluginConfig });
+	if (params.isolation === "configured-transport") return await runBoundedCodexAppServerTurnInWorkspace(params, appServer, { cwd: params.agentDir?.trim() || process.cwd() });
+	if (appServer.start.transport !== "stdio") throw new Error("Bounded Codex turns require stdio transport so native tools can be isolated.");
+	return await withTempWorkspace({
+		rootDir: resolvePreferredOpenClawTmpDir(),
+		prefix: "codex-bounded-turn-"
+	}, async (workspace) => {
+		const codexHome = path.join(workspace.dir, "codex-home");
+		const cwd = path.join(workspace.dir, "workspace");
+		await Promise.all([fs.mkdir(codexHome, { recursive: true }), fs.mkdir(cwd, { recursive: true })]);
+		return await runBoundedCodexAppServerTurnInWorkspace(params, appServer, {
+			codexHome,
+			cwd
+		});
+	});
+}
+async function runBoundedCodexAppServerTurnInWorkspace(params, appServer, workspace) {
+	const timeoutMs = resolveTimerTimeoutMs(params.timeoutMs, 100, 100);
+	const agentDir = params.agentDir?.trim() || void 0;
+	const startOptions = workspace.codexHome ? buildPrivateCodexAppServerStartOptions(appServer.start, workspace.codexHome) : appServer.start;
+	const ownsClient = !params.options.clientFactory;
+	const client = params.options.clientFactory ? await params.options.clientFactory(startOptions, params.profile, agentDir, params.config, { timeoutMs }) : await import("./shared-client-Bg5d7VV9.js").then((n) => n.c).then(({ createIsolatedCodexAppServerClient }) => createIsolatedCodexAppServerClient({
+		startOptions,
+		timeoutMs,
+		authProfileId: params.profile,
+		agentDir,
+		authProfileStore: params.authProfileStore,
+		config: params.config
+	}));
+	const abortController = new AbortController();
+	const abortFromCaller = () => abortController.abort(params.signal?.reason ?? "aborted");
+	if (params.signal?.aborted) abortFromCaller();
+	else params.signal?.addEventListener("abort", abortFromCaller, { once: true });
+	const timeout = setTimeout(() => abortController.abort("timeout"), timeoutMs);
+	timeout.unref?.();
+	try {
+		const model = await resolveCodexBoundedTurnModel({
+			client,
+			selection: params.model,
+			requiredModalities: params.requiredModalities,
+			timeoutMs,
+			signal: abortController.signal
+		});
+		const thread = assertCodexThreadStartResponse(await client.request("thread/start", {
+			model,
+			modelProvider: "openai",
+			cwd: workspace.cwd,
+			approvalPolicy: "on-request",
+			sandbox: "read-only",
+			serviceName: "OpenClaw",
+			developerInstructions: params.developerInstructions,
+			config: buildCodexRuntimeThreadConfig(resolveBoundedThreadConfig(params, workspace), { nativeCodeModeEnabled: false }),
+			environments: [],
+			dynamicTools: [],
+			experimentalRawEvents: true,
+			persistExtendedHistory: false,
+			ephemeral: true
+		}, {
+			timeoutMs,
+			signal: abortController.signal
+		}));
+		const collector = createCodexBoundedTurnCollector(thread.thread.id, params.taskLabel);
+		const cleanup = client.addNotificationHandler(collector.handleNotification);
+		const requestCleanup = client.addRequestHandler(createCodexBoundedApprovalHandler(params.taskLabel));
+		try {
+			const turn = assertCodexTurnStartResponse(await client.request("turn/start", {
+				threadId: thread.thread.id,
+				input: params.input,
+				cwd: workspace.cwd,
+				approvalPolicy: "on-request",
+				model,
+				effort: "low"
+			}, {
+				timeoutMs,
+				signal: abortController.signal
+			}));
+			return {
+				...await collector.collect(turn.turn, {
+					timeoutMs,
+					signal: abortController.signal
+				}),
+				model
+			};
+		} finally {
+			requestCleanup();
+			cleanup();
+		}
+	} finally {
+		clearTimeout(timeout);
+		params.signal?.removeEventListener("abort", abortFromCaller);
+		if (ownsClient) client.close();
+	}
+}
+function resolveBoundedThreadConfig(params, workspace) {
+	const boundedConfig = mergeCodexThreadConfigs(CODEX_BOUNDED_THREAD_CONFIG, params.threadConfig) ?? CODEX_BOUNDED_THREAD_CONFIG;
+	return workspace.codexHome ? mergeCodexThreadConfigs(boundedConfig, CODEX_PRIVATE_BOUNDED_THREAD_CONFIG) ?? boundedConfig : boundedConfig;
+}
+function buildPrivateCodexAppServerStartOptions(start, codexHome) {
+	const privateEnv = Object.fromEntries(Object.entries(start.env ?? {}).filter(([name]) => name.trim().toUpperCase() !== OPENCLAW_CODEX_APP_SERVER_ARGS_ENV_VAR));
+	const clearEnv = (start.clearEnv ?? []).filter((name) => {
+		const normalized = name.trim().toUpperCase();
+		return normalized !== "CODEX_HOME" && normalized !== OPENCLAW_CODEX_APP_SERVER_ARGS_ENV_VAR;
+	});
+	return {
+		...start,
+		args: [...CODEX_PRIVATE_STDIO_ARGS],
+		env: {
+			...privateEnv,
+			CODEX_HOME: codexHome
+		},
+		clearEnv: [...clearEnv, OPENCLAW_CODEX_APP_SERVER_ARGS_ENV_VAR]
+	};
+}
+function createCodexBoundedApprovalHandler(taskLabel) {
+	return (request) => {
+		if (request.method === "item/commandExecution/requestApproval" || request.method === "item/fileChange/requestApproval") return {
+			decision: "decline",
+			reason: `OpenClaw Codex ${taskLabel} does not grant tool or file approvals.`
+		};
+		if (request.method === "item/permissions/requestApproval") return {
+			permissions: {},
+			scope: "turn"
+		};
+		if (request.method.includes("requestApproval")) return {
+			decision: "decline",
+			reason: `OpenClaw Codex ${taskLabel} does not grant native approvals.`
+		};
+		if (request.method === "mcpServer/elicitation/request") return { action: "decline" };
+	};
+}
+async function resolveCodexBoundedTurnModel(params) {
+	const listed = readModelListResult(await params.client.request("model/list", {
+		limit: null,
+		cursor: null,
+		includeHidden: false
+	}, {
+		timeoutMs: Math.min(params.timeoutMs, 5e3),
+		signal: params.signal
+	})).models;
+	if (params.selection.mode === "live-default") {
+		const supported = listed.filter((entry) => params.requiredModalities.every((modality) => entry.inputModalities.includes(modality)));
+		const selected = supported.find((entry) => entry.isDefault) ?? supported[0];
+		if (!selected) throw new Error(`Codex app-server has no model supporting ${params.requiredModalities.join(" and ")} input.`);
+		return selected.model;
+	}
+	const model = params.selection.id;
+	const match = listed.find((entry) => entry.model === model || entry.id === model);
+	if (!match) throw new Error(`Codex app-server model not found: ${model}`);
+	if (params.requiredModalities.includes("image") && !match.inputModalities.includes("image")) throw new Error(`Codex app-server model does not support images: ${model}`);
+	if (params.requiredModalities.includes("text") && !match.inputModalities.includes("text")) throw new Error(`Codex app-server model does not support text: ${model}`);
+	return model;
+}
+function createCodexBoundedTurnCollector(threadId, taskLabel) {
+	let turnId;
+	let completedTurn;
+	let promptError;
+	const pending = [];
+	const completedItems = /* @__PURE__ */ new Map();
+	const assistantTextByItem = /* @__PURE__ */ new Map();
+	const assistantItemOrder = [];
+	let resolveCompletion;
+	const completion = new Promise((resolve) => {
+		resolveCompletion = resolve;
+	});
+	const rememberAssistantText = (itemId, text) => {
+		if (!text) return;
+		if (!assistantTextByItem.has(itemId)) assistantItemOrder.push(itemId);
+		assistantTextByItem.set(itemId, text);
+	};
+	const handleNotification = (notification) => {
+		const params = isJsonObject(notification.params) ? notification.params : void 0;
+		if (!params || readString(params, "threadId") !== threadId) return;
+		if (!turnId) {
+			pending.push(notification);
+			return;
+		}
+		if (readNotificationTurnId(params) !== turnId) return;
+		if (notification.method === "item/completed") {
+			const item = readCodexNotificationItem(notification.params);
+			if (item) {
+				completedItems.set(item.id, item);
+				if (item.type === "agentMessage" && typeof item.text === "string") rememberAssistantText(item.id, item.text);
+			}
+			return;
+		}
+		if (notification.method === "item/agentMessage/delta") {
+			const itemId = readString(params, "itemId") ?? readString(params, "id") ?? "assistant";
+			const delta = readString(params, "delta") ?? "";
+			rememberAssistantText(itemId, `${assistantTextByItem.get(itemId) ?? ""}${delta}`);
+			return;
+		}
+		if (notification.method === "turn/completed") {
+			completedTurn = readCodexTurnCompletedNotification(notification.params)?.turn ?? completedTurn;
+			resolveCompletion?.();
+			return;
+		}
+		if (notification.method === "error") {
+			promptError = readCodexErrorNotification(notification.params)?.error.message ?? `codex app-server ${taskLabel} turn failed`;
+			resolveCompletion?.();
+		}
+	};
+	return {
+		handleNotification,
+		async collect(startedTurn, options) {
+			turnId = startedTurn.id;
+			if (isTerminalTurn(startedTurn)) completedTurn = startedTurn;
+			for (const notification of pending.splice(0)) handleNotification(notification);
+			if (!completedTurn && !promptError) await waitForTurnCompletion({
+				completion,
+				timeoutMs: options.timeoutMs,
+				signal: options.signal,
+				taskLabel
+			});
+			if (promptError) throw new Error(promptError);
+			if (completedTurn?.status === "failed") throw new Error(completedTurn.error?.message ?? `codex app-server ${taskLabel} turn failed`);
+			const items = collectCompletedItems(completedTurn?.items, completedItems);
+			const itemText = collectAssistantTextFromItems(items);
+			const deltaText = assistantItemOrder.map((itemId) => assistantTextByItem.get(itemId)?.trim()).filter((text) => Boolean(text)).join("\n\n").trim();
+			const text = (itemText || deltaText).trim();
+			if (!text) throw new Error(`Codex app-server ${taskLabel} turn returned no text.`);
+			return {
+				text,
+				items
+			};
+		}
+	};
+}
+function collectCompletedItems(turnItems, notificationItems) {
+	const items = new Map(notificationItems);
+	for (const item of turnItems ?? []) items.set(item.id, item);
+	return [...items.values()];
+}
+async function waitForTurnCompletion(params) {
+	if (params.signal.aborted) throw new Error(`codex app-server ${params.taskLabel} turn aborted`);
+	let timeout;
+	let cleanupAbort;
+	try {
+		await Promise.race([params.completion, new Promise((_, reject) => {
+			timeout = setTimeout(() => reject(/* @__PURE__ */ new Error(`codex app-server ${params.taskLabel} turn timed out`)), params.timeoutMs);
+			timeout.unref?.();
+			const abortListener = () => reject(/* @__PURE__ */ new Error(`codex app-server ${params.taskLabel} turn aborted`));
+			params.signal.addEventListener("abort", abortListener, { once: true });
+			cleanupAbort = () => params.signal.removeEventListener("abort", abortListener);
+		})]);
+	} finally {
+		if (timeout) clearTimeout(timeout);
+		cleanupAbort?.();
+	}
+}
+function collectAssistantTextFromItems(items) {
+	return (items ?? []).filter((item) => item.type === "agentMessage").map((item) => item.text.trim()).filter(Boolean).join("\n\n").trim();
+}
+function readNotificationTurnId(record) {
+	const direct = readString(record, "turnId");
+	if (direct) return direct;
+	return isJsonObject(record.turn) ? readString(record.turn, "id") : void 0;
+}
+function readString(record, key) {
+	const value = record[key];
+	return typeof value === "string" ? value : void 0;
+}
+function isTerminalTurn(turn) {
+	return turn.status === "completed" || turn.status === "interrupted" || turn.status === "failed";
+}
+//#endregion
+//#region extensions/codex/media-understanding-provider.ts
+/**
+* Codex-backed media understanding provider for bounded image description and
+* structured extraction turns.
+*/
+const DEFAULT_CODEX_IMAGE_MODEL = FALLBACK_CODEX_MODELS.find((model) => model.inputModalities.includes("image"))?.id ?? FALLBACK_CODEX_MODELS[0]?.id;
+const DEFAULT_CODEX_IMAGE_PROMPT = "Describe the image.";
+/**
+* Builds the media-understanding provider that delegates image tasks to an
+* isolated Codex app-server session.
+*/
+function buildCodexMediaUnderstandingProvider(options = {}) {
+	return {
+		id: CODEX_PROVIDER_ID,
+		capabilities: ["image"],
+		...DEFAULT_CODEX_IMAGE_MODEL ? { defaultModels: { image: DEFAULT_CODEX_IMAGE_MODEL } } : {},
+		describeImage: async (req) => describeCodexImages({
+			images: [{
+				buffer: req.buffer,
+				fileName: req.fileName,
+				mime: req.mime
+			}],
+			provider: req.provider,
+			model: req.model,
+			prompt: req.prompt,
+			maxTokens: req.maxTokens,
+			timeoutMs: req.timeoutMs,
+			profile: req.profile,
+			preferredProfile: req.preferredProfile,
+			authStore: req.authStore,
+			agentDir: req.agentDir,
+			cfg: req.cfg
+		}, options),
+		describeImages: async (req) => describeCodexImages(req, options),
+		extractStructured: async (req) => extractCodexStructured(req, options)
+	};
+}
+async function describeCodexImages(req, options) {
+	const model = req.model.trim();
+	if (!model) throw new Error("Codex image understanding requires model id.");
+	const { text } = await runBoundedCodexAppServerTurn({
+		config: req.cfg,
+		model: {
+			mode: "required",
+			id: model
+		},
+		profile: req.profile,
+		timeoutMs: req.timeoutMs,
+		agentDir: req.agentDir,
+		authProfileStore: req.authStore,
+		options,
+		taskLabel: "image understanding",
+		developerInstructions: "You are OpenClaw's bounded image-understanding worker. Describe only the provided image content. Do not call tools, edit files, or ask follow-up questions.",
+		input: [{
+			type: "text",
+			text: buildCodexImagePrompt(req),
+			text_elements: []
+		}, ...req.images.map((image) => ({
+			type: "image",
+			url: `data:${image.mime ?? "image/png"};base64,${image.buffer.toString("base64")}`
+		}))],
+		requiredModalities: ["text", "image"],
+		isolation: "configured-transport"
+	});
+	return {
+		text,
+		model
+	};
+}
+async function extractCodexStructured(req, options) {
+	const model = req.model.trim();
+	if (!model) throw new Error("Codex structured extraction requires model id.");
+	if (!req.instructions.trim()) throw new Error("Codex structured extraction requires instructions.");
+	if (req.input.length === 0) throw new Error("Codex structured extraction requires at least one input.");
+	if (!req.input.some((entry) => entry.type === "image")) throw new Error("Codex structured extraction requires at least one image input.");
+	const { text } = await runBoundedCodexAppServerTurn({
+		config: req.cfg,
+		model: {
+			mode: "required",
+			id: model
+		},
+		profile: req.profile,
+		timeoutMs: req.timeoutMs,
+		agentDir: req.agentDir,
+		authProfileStore: req.authStore,
+		options,
+		taskLabel: "structured extraction",
+		developerInstructions: "You are OpenClaw's bounded structured-extraction worker. Return only the requested extraction. Do not call tools, edit files, ask follow-up questions, or include secrets.",
+		input: buildCodexStructuredInput(req),
+		requiredModalities: requiredStructuredModalities(),
+		isolation: "configured-transport"
+	});
+	return normalizeStructuredExtractionResult({
+		text,
+		model,
+		provider: req.provider,
+		req
+	});
+}
+function buildCodexImagePrompt(req) {
+	const prompt = req.prompt?.trim() || DEFAULT_CODEX_IMAGE_PROMPT;
+	if (req.images.length <= 1) return prompt;
+	return `${prompt}\n\nAnalyze all ${req.images.length} images together.`;
+}
+function requiredStructuredModalities() {
+	return ["text", "image"];
+}
+function buildCodexStructuredInput(req) {
+	return [{
+		type: "text",
+		text: buildStructuredExtractionPrompt(req),
+		text_elements: []
+	}, ...req.input.map((entry) => {
+		if (entry.type === "text") return {
+			type: "text",
+			text: entry.text,
+			text_elements: []
+		};
+		return {
+			type: "image",
+			url: `data:${entry.mime ?? "image/png"};base64,${entry.buffer.toString("base64")}`
+		};
+	})];
+}
+function buildStructuredExtractionPrompt(req) {
+	return [
+		req.instructions.trim(),
+		req.schemaName ? `Schema name: ${req.schemaName}` : void 0,
+		req.jsonSchema ? `JSON schema:\n${JSON.stringify(req.jsonSchema)}` : void 0,
+		req.jsonMode === false ? "Return the extraction as concise text." : "Return valid JSON only. Do not wrap the JSON in Markdown fences."
+	].filter((part) => Boolean(part)).join("\n\n");
+}
+function isJsonSchemaObject(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function normalizeStructuredExtractionResult(params) {
+	const result = {
+		text: params.text,
+		model: params.model,
+		provider: params.provider,
+		contentType: params.req.jsonMode === false ? "text" : "json"
+	};
+	if (params.req.jsonMode !== false) {
+		try {
+			result.parsed = JSON.parse(params.text);
+		} catch {
+			throw new Error("Codex structured extraction returned invalid JSON.");
+		}
+		if (isJsonSchemaObject(params.req.jsonSchema)) {
+			const validation = validateJsonSchemaValue({
+				schema: params.req.jsonSchema,
+				cacheKey: "codex.media-understanding.extractStructured",
+				value: result.parsed,
+				cache: false
+			});
+			if (!validation.ok) {
+				const message = validation.errors.map((error) => error.text).join("; ") || "invalid";
+				throw new Error(`Codex structured extraction JSON did not match schema: ${message}`);
+			}
+			result.parsed = validation.value;
+		}
+	}
+	return result;
+}
+//#endregion
+export { runBoundedCodexAppServerTurn as n, buildCodexMediaUnderstandingProvider as t };