@resolveio/server-lib 22.3.125 → 22.3.126

package/src/services/codex-client.ts

@@ -0,0 +1,1231 @@
+import { promises as fs } from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { setTimeout as sleepTimeout } from 'timers/promises';
+export interface CodexConfig {
+	apiKey?: string;
+	baseUrl?: string;
+	model?: string;
+	fallbackModel?: string;
+	fallbackModels?: string[];
+	maxRetries?: number;
+	retryDelayMs?: number;
+}
+
+/* eslint-disable no-unused-vars -- Function type params are required for clarity. */
+type StreamEventHandler = (event: any) => void | Promise<void>;
+type StreamTextHandler = (delta: string, fullText: string) => void | Promise<void>;
+type AgentMessageHandler = (text: string) => void | Promise<void>;
+type UsageHandler = (usage: CodexUsageSummary | null) => void | Promise<void>;
+type DynamicImport = (specifier: string) => Promise<any>;
+/* eslint-enable no-unused-vars */
+
+type CodexModelFallbackPreference = {
+	fallbackModel: string;
+	expiresAt: number;
+	updatedAt: number;
+};
+
+export interface CodexRunOptions {
+	timeoutMs?: number;
+	streamIdleTimeoutMs?: number;
+	maxRetries?: number;
+	processWorkingDirectory?: string;
+	threadOptions?: CodexThreadOptions;
+	threadKey?: string;
+	reuseThread?: boolean;
+	fallbackModels?: string[];
+	failOnStreamHandlerError?: boolean;
+	onStreamEvent?: StreamEventHandler;
+	onStreamText?: StreamTextHandler;
+	onUsage?: UsageHandler;
+}
+
+export interface CodexUsageSummary {
+	inputTokens: number;
+	cachedInputTokens: number;
+	outputTokens: number;
+}
+
+export type CodexThreadOptions = {
+	model?: string;
+	sandboxMode?: 'read-only' | 'workspace-write' | 'danger-full-access';
+	workingDirectory?: string;
+	skipGitRepoCheck?: boolean;
+	modelReasoningEffort?: 'minimal' | 'none' | 'low' | 'medium' | 'high' | 'xhigh';
+	networkAccessEnabled?: boolean;
+	webSearchMode?: 'disabled' | 'cached' | 'live';
+	webSearchEnabled?: boolean;
+	approvalPolicy?: 'never' | 'on-request' | 'on-failure' | 'untrusted';
+	additionalDirectories?: string[];
+};
+
+export class CodexClient {
+	private static readonly PING_PROMPT = "Say 'pong'";
+	private static readonly MODEL_UNAVAILABLE_FALLBACK_FILE = path.join(os.tmpdir(), 'resolveio-codex-model-fallbacks.json');
+	private static readonly DEFAULT_MODEL_UNAVAILABLE_TTL_MS = 6 * 60 * 60 * 1000;
+	private static readonly MIN_MODEL_UNAVAILABLE_TTL_MS = 60 * 1000;
+	private static readonly modelFallbackPreferences = new Map<string, CodexModelFallbackPreference>();
+	private static modelFallbackPreferencesLoaded = false;
+	private static modelFallbackPreferencesLoadPromise: Promise<void> | null = null;
+	private static cwdFallbackRunning = false;
+	private codex: any | null = null;
+	private codexInit: Promise<void> | null = null;
+	private readonly threadCache = new Map<string, { thread: any; optionsKey: string; modelKey: string }>();
+	private readonly debugEnabled = (process.env.AI_DASHBOARD_DEBUG || '').trim().toLowerCase() === 'true';
+	private readonly streamEnabled = this.resolveStreamEnabled();
+	private readonly cleanZdotdirEnabled = this.resolveCleanZdotdirEnabled();
+	private zDotDir: string | null = null;
+	private readonly defaultTimeoutMs = this.resolveDefaultTimeoutMs();
+	private readonly pingEnabled = this.resolvePingEnabled();
+	private readonly pingTimeoutMs = this.resolvePingTimeoutMs();
+	private readonly modelUnavailableTtlMs = this.resolveModelUnavailableTtlMs();
+	private readonly config: CodexConfig;
+	private readonly defaultThreadOptions: CodexThreadOptions;
+
+	constructor(config: CodexConfig) {
+		this.config = config;
+		if (!this.config.apiKey) {
+			throw new Error('AI API key is required for AI client');
+		}
+		this.defaultThreadOptions = this.resolveDefaultThreadOptions();
+	}
+
+	async run(prompt: string, options?: CodexRunOptions): Promise<string> {
+		await this.ensureModelFallbackPreferencesLoaded();
+		const maxRetries = this.normalizeRetryCount(options?.maxRetries ?? this.config.maxRetries);
+		const retryDelayMs = this.normalizeRetryDelay(this.config.retryDelayMs);
+		let lastError: unknown = null;
+		const modelSequence = this.resolveModelSequence(options);
+		for (let modelIndex = 0; modelIndex < modelSequence.length; modelIndex += 1) {
+			const model = modelSequence[modelIndex];
+			const nextModel = modelSequence[modelIndex + 1];
+			const modelOptions = this.applyModelOverride(options, model);
+			let processCwdFallbackApplied = false;
+			let attempt = 0;
+			while (attempt <= maxRetries) {
+				try {
+					this.log(`run model=${model || this.config.model || 'default'} attempt ${attempt + 1}/${maxRetries + 1}`);
+					if (this.pingEnabled && !this.isPingPrompt(prompt)) {
+						await this.runPing(modelOptions?.threadOptions);
+					}
+					if (this.streamEnabled && !this.isPingPrompt(prompt)) {
+						const result = await this.runStreamedOnce(prompt, modelOptions);
+						await this.clearModelFallbackPreference(model);
+						return result;
+					}
+					const result = await this.runOnce(prompt, modelOptions);
+					await this.clearModelFallbackPreference(model);
+					return result;
+				}
+					catch (error) {
+						lastError = error;
+						const message = error instanceof Error ? error.message : String(error || 'Unknown error');
+						this.log(`run model=${model || this.config.model || 'default'} attempt ${attempt + 1} failed: ${message}`);
+						if (!processCwdFallbackApplied && this.shouldApplyProcessCwdFallback(error, modelOptions)) {
+							processCwdFallbackApplied = true;
+							const workingDirectory = this.resolveProcessWorkingDirectory(modelOptions);
+							const fallbackOptions = this.applyProcessCwdFallback(modelOptions);
+							this.log('codex workingDirectory launch failed with os error 2; retrying from process cwd without CLI workingDirectory.');
+							return await this.withProcessWorkingDirectoryFallback(workingDirectory, async () => {
+								if (this.streamEnabled && !this.isPingPrompt(prompt)) {
+									const fallbackResult = await this.runStreamedOnce(prompt, fallbackOptions);
+									await this.clearModelFallbackPreference(model);
+									return fallbackResult;
+								}
+								const fallbackResult = await this.runOnce(prompt, fallbackOptions);
+								await this.clearModelFallbackPreference(model);
+								return fallbackResult;
+							});
+						}
+						if (nextModel && this.isModelAvailabilityError(error)) {
+							await this.rememberModelFallbackPreference(model, nextModel);
+							this.log(`model unavailable; retrying with fallback model ${nextModel}`);
+							break;
+						}
+						if (this.isCodexEnvironmentError(error)) {
+							this.recoverCodexEnvironment(error);
+						}
+						if (attempt >= maxRetries || !this.isRetryableError(error)) {
+							throw lastError;
+						}
+						const delay = retryDelayMs * Math.pow(2, attempt);
+						await this.sleep(delay);
+					attempt += 1;
+				}
+			}
+		}
+		throw lastError;
+	}
+
+	private async runStreamedOnce(prompt: string, options?: CodexRunOptions): Promise<string> {
+		const startedAt = Date.now();
+		this.log(`runStreamed start (promptLength=${prompt.length})`);
+		const thread = await this.startThread(options?.threadOptions, options?.threadKey, options?.reuseThread);
+		const timeoutMs = options?.timeoutMs ?? this.defaultTimeoutMs;
+		const streamIdleTimeoutMs = this.normalizeStreamIdleTimeoutMs(options?.streamIdleTimeoutMs);
+		const controller = typeof AbortController !== 'undefined' ? new AbortController() : null;
+		const timeoutId = controller
+			? setTimeout(() => controller.abort(), timeoutMs)
+			: null;
+		let lastStreamActivityAt = Date.now();
+		const idleTimeoutId = controller && streamIdleTimeoutMs > 0
+			? setInterval(() => {
+				if (Date.now() - lastStreamActivityAt >= streamIdleTimeoutMs) {
+					controller.abort();
+				}
+			}, Math.max(15_000, Math.min(60_000, Math.floor(streamIdleTimeoutMs / 4))))
+			: null;
+		let finalResponse = '';
+		let lastText = '';
+		let usageSummary: CodexUsageSummary | null = null;
+		try {
+			const { events } = await thread.runStreamed(prompt, controller ? { signal: controller.signal } : undefined);
+			for await (const event of events) {
+				lastStreamActivityAt = Date.now();
+				if (event?.type === 'turn.completed') {
+					usageSummary = this.normalizeUsageSummary(event?.usage);
+				}
+				if (options?.onStreamEvent) {
+					try {
+						await options.onStreamEvent(event);
+					}
+					catch (error) {
+						const message = error instanceof Error ? error.message : String(error || 'Unknown error');
+						this.log(`onStreamEvent failed: ${message}`);
+						if (options?.failOnStreamHandlerError) {
+							throw error;
+						}
+					}
+				}
+				await this.handleStreamEvent(event, async (text: string) => {
+					const next = text || '';
+					if (!next) {
+						return;
+					}
+					const delta = next.startsWith(lastText) ? next.slice(lastText.length) : next;
+					if (delta) {
+						this.logStreamText(delta);
+						if (options?.onStreamText) {
+							try {
+								await options.onStreamText(delta, next);
+							}
+							catch (error) {
+								const message = error instanceof Error ? error.message : String(error || 'Unknown error');
+								this.log(`onStreamText failed: ${message}`);
+								if (options?.failOnStreamHandlerError) {
+									throw error;
+								}
+							}
+						}
+					}
+					lastText = next;
+					finalResponse = next;
+				});
+				if (event?.type === 'turn.failed') {
+					throw new Error(event.error?.message || 'AI run failed.');
+				}
+				if (event?.type === 'error') {
+					throw new Error(event.message || 'AI stream error.');
+				}
+			}
+		}
+		catch (error) {
+			if (controller?.signal?.aborted) {
+				const idleForMs = Date.now() - lastStreamActivityAt;
+				if (streamIdleTimeoutMs > 0 && idleForMs >= streamIdleTimeoutMs) {
+					this.log(`runStreamed idle timed out (durationMs=${Date.now() - startedAt}, idleForMs=${idleForMs})`);
+					throw new Error(`AI stream stalled with no activity for ${idleForMs}ms.`);
+				}
+				this.log(`runStreamed timed out (durationMs=${Date.now() - startedAt})`);
+				throw new Error('AI request timed out.');
+			}
+			const message = error instanceof Error ? error.message : String(error || 'Unknown error');
+			this.log(`runStreamed failed (durationMs=${Date.now() - startedAt}): ${message}`);
+			throw error;
+		}
+		finally {
+			if (timeoutId) {
+				clearTimeout(timeoutId);
+			}
+			if (idleTimeoutId) {
+				clearInterval(idleTimeoutId);
+			}
+		}
+		if (!finalResponse) {
+			throw new Error('AI returned no completion content.');
+		}
+		await this.emitUsageSummary(options, usageSummary);
+		this.log(`runStreamed complete (contentLength=${finalResponse.length}, durationMs=${Date.now() - startedAt})`);
+		return finalResponse.trim();
+	}
+
+	private normalizeStreamIdleTimeoutMs(value: any): number {
+		const parsed = Number(value || 0);
+		if (!Number.isFinite(parsed) || parsed < 60_000) {
+			return 0;
+		}
+		return Math.floor(parsed);
+	}
+
+	private async runOnce(prompt: string, options?: CodexRunOptions): Promise<string> {
+		const startedAt = Date.now();
+		this.log(`runOnce start (promptLength=${prompt.length})`);
+		try {
+			const thread = await this.startThread(options?.threadOptions, options?.threadKey, options?.reuseThread);
+			const runPromise = thread.run(prompt);
+			const timeoutMs = options?.timeoutMs ?? this.defaultTimeoutMs;
+			this.log(`runOnce awaiting result (timeoutMs=${timeoutMs})`);
+			const result: any = await this.withTimeout(runPromise, timeoutMs);
+			const content = this.extractContent(result);
+			if (!content) {
+				throw new Error('AI returned no completion content.');
+			}
+			await this.emitUsageSummary(options, this.normalizeUsageSummary(result?.usage));
+			this.log(`runOnce complete (contentLength=${content.length}, durationMs=${Date.now() - startedAt})`);
+			return content;
+		}
+		catch (error) {
+			const message = error instanceof Error ? error.message : String(error || 'Unknown error');
+			this.log(`runOnce failed (durationMs=${Date.now() - startedAt}): ${message}`);
+			throw error;
+		}
+	}
+
+	private async startThread(threadOptions?: CodexThreadOptions, threadKey?: string, reuseThread = true): Promise<any> {
+		await this.ensureCodex();
+		const mergedOptions = this.mergeThreadOptions(threadOptions);
+		const optionsKey = this.buildThreadOptionsKey(mergedOptions);
+		const effectiveModel = mergedOptions?.model || this.config.model || 'default';
+		const modelKey = this.normalizeOptionalString(effectiveModel).toLowerCase() || 'default';
+		const cacheKey = this.buildThreadCacheKey(threadKey, mergedOptions);
+		const explicitThreadKey = (threadKey || '').trim();
+		const hasExplicitThreadKey = explicitThreadKey.length > 0;
+		if (reuseThread && cacheKey) {
+			const existing = this.threadCache.get(cacheKey);
+			if (existing) {
+				const sameOptions = existing.optionsKey === optionsKey;
+				const sameModel = existing.modelKey === modelKey;
+				if (sameOptions || (hasExplicitThreadKey && sameModel)) {
+					if (hasExplicitThreadKey && existing.optionsKey !== optionsKey) {
+						this.log(`startThread reuse explicit key (${cacheKey}) despite option change`);
+					}
+					else {
+						this.log(`startThread reuse (${cacheKey})`);
+					}
+					return existing.thread;
+				}
+				if (hasExplicitThreadKey && !sameModel) {
+					this.log(`startThread explicit key (${cacheKey}) model changed (${existing.modelKey} -> ${modelKey}); creating new thread`);
+				}
+			}
+		}
+		this.log(`startThread (model=${effectiveModel})`);
+		if (mergedOptions?.model) {
+			this.log(`startThread options: ${this.formatThreadOptions(mergedOptions)}`);
+			const thread = this.codex.startThread(mergedOptions);
+			this.log('startThread created with model override');
+			if (cacheKey) {
+				this.threadCache.set(cacheKey, { thread, optionsKey, modelKey });
+			}
+			return thread;
+		}
+		if (mergedOptions && Object.keys(mergedOptions).length) {
+			this.log(`startThread options: ${this.formatThreadOptions(mergedOptions)}`);
+		}
+		const thread = this.codex.startThread(mergedOptions);
+		this.log('startThread created');
+		if (cacheKey) {
+			this.threadCache.set(cacheKey, { thread, optionsKey, modelKey });
+		}
+		return thread;
+	}
+
+	private extractContent(result: any): string {
+		if (!result) {
+			return '';
+		}
+		if (typeof result.finalResponse === 'string') {
+			return result.finalResponse.trim();
+		}
+		if (typeof result === 'string') {
+			return result.trim();
+		}
+		if (Array.isArray(result.items)) {
+			const lastMessage = [...result.items].reverse().find((item: any) => item?.type === 'agent_message' && typeof item.text === 'string');
+			if (lastMessage?.text) {
+				return lastMessage.text.trim();
+			}
+		}
+		if (typeof result.output_text === 'string') {
+			return result.output_text.trim();
+		}
+		if (typeof result.outputText === 'string') {
+			return result.outputText.trim();
+		}
+		if (typeof result.content === 'string') {
+			return result.content.trim();
+		}
+		if (typeof result.text === 'string') {
+			return result.text.trim();
+		}
+		if (Array.isArray(result.content)) {
+			const text = result.content
+				.map((part: any) => this.extractPartText(part))
+				.join('')
+				.trim();
+			if (text) {
+				return text;
+			}
+		}
+		if (Array.isArray(result.output)) {
+			const text = result.output
+				.map((item: any) => {
+					if (!item) {
+						return '';
+					}
+					if (typeof item === 'string') {
+						return item;
+					}
+					if (typeof item.text === 'string') {
+						return item.text;
+					}
+					if (typeof item.content === 'string') {
+						return item.content;
+					}
+					if (Array.isArray(item.content)) {
+						return item.content.map((part: any) => this.extractPartText(part)).join('');
+					}
+					return '';
+				})
+				.join('')
+				.trim();
+			if (text) {
+				return text;
+			}
+		}
+		return '';
+	}
+
+	private async handleStreamEvent(event: any, onAgentMessage: AgentMessageHandler): Promise<void> {
+		if (!event || !event.type) {
+			return;
+		}
+		switch (event.type) {
+			case 'thread.started':
+				this.log(`stream thread.started (id=${event.thread_id || 'unknown'})`);
+				break;
+			case 'turn.started':
+				this.log('stream turn.started');
+				break;
+			case 'turn.completed':
+				this.log('stream turn.completed');
+				break;
+			case 'item.started':
+			case 'item.updated':
+			case 'item.completed':
+				await this.handleStreamItem(event.item, onAgentMessage);
+				break;
+			default:
+				break;
+		}
+	}
+
+	private normalizeUsageSummary(raw: any): CodexUsageSummary | null {
+		if (!raw || typeof raw !== 'object') {
+			return null;
+		}
+		const inputTokens = this.normalizeUsageToken(raw.input_tokens ?? raw.inputTokens);
+		const cachedInputTokens = this.normalizeUsageToken(raw.cached_input_tokens ?? raw.cachedInputTokens);
+		const outputTokens = this.normalizeUsageToken(raw.output_tokens ?? raw.outputTokens);
+		if (inputTokens <= 0 && cachedInputTokens <= 0 && outputTokens <= 0) {
+			return null;
+		}
+		return {
+			inputTokens,
+			cachedInputTokens,
+			outputTokens
+		};
+	}
+
+	private normalizeUsageToken(value: any): number {
+		const parsed = Number(value);
+		if (!Number.isFinite(parsed) || parsed <= 0) {
+			return 0;
+		}
+		return Math.floor(parsed);
+	}
+
+	private async emitUsageSummary(options: CodexRunOptions | undefined, usage: CodexUsageSummary | null): Promise<void> {
+		if (!options?.onUsage) {
+			return;
+		}
+		try {
+			await options.onUsage(usage);
+		}
+		catch (error) {
+			const message = error instanceof Error ? error.message : String(error || 'Unknown error');
+			this.log(`onUsage failed: ${message}`);
+		}
+	}
+
+	private async handleStreamItem(item: any, onAgentMessage: AgentMessageHandler): Promise<void> {
+		if (!item || !item.type) {
+			return;
+		}
+		switch (item.type) {
+			case 'agent_message':
+				await onAgentMessage(item.text || '');
+				break;
+			case 'command_execution':
+				if (item.command) {
+					this.log(`stream command: ${item.command} (${item.status || 'unknown'})`);
+				}
+				if (item.aggregated_output) {
+					const output = item.aggregated_output.length > 800
+						? `${item.aggregated_output.slice(0, 800)}...`
+						: item.aggregated_output;
+					this.log(`stream command output: ${output}`);
+				}
+				break;
+			case 'file_change':
+				if (Array.isArray(item.changes) && item.changes.length) {
+					const summary = item.changes.map((change: any) => `${change.kind}:${change.path}`).join(', ');
+					this.log(`stream file_change (${item.status || 'unknown'}): ${summary}`);
+				}
+				break;
+			case 'mcp_tool_call':
+				if (item.tool) {
+					this.log(`stream tool_call: ${item.tool} (${item.status || 'unknown'})`);
+				}
+				if (item.error?.message) {
+					this.log(`stream tool_call error: ${item.error.message}`);
+				}
+				break;
+			case 'web_search':
+				if (item.query) {
+					this.log(`stream web_search: ${item.query}`);
+				}
+				break;
+			case 'todo_list':
+				if (Array.isArray(item.items)) {
+					const total = item.items.length;
+					const done = item.items.filter((entry: any) => entry?.completed).length;
+					this.log(`stream todo_list: ${done}/${total} completed`);
+				}
+				break;
+			case 'reasoning':
+				if (item.text) {
+					const snippet = item.text.length > 200 ? `${item.text.slice(0, 200)}...` : item.text;
+					this.log(`stream reasoning: ${snippet}`);
+				}
+				break;
+			case 'error':
+				if (item.message) {
+					this.log(`stream error: ${item.message}`);
+				}
+				break;
+			default:
+				break;
+		}
+	}
+
+	private logStreamText(text: string): void {
+		if (!this.debugEnabled) {
+			return;
+		}
+		const lines = text.split(/\r?\n/);
+		for (const line of lines) {
+			this.log(`stream: ${line}`);
+		}
+	}
+
+	private extractPartText(part: any): string {
+		if (!part) {
+			return '';
+		}
+		if (typeof part === 'string') {
+			return part;
+		}
+		if (typeof part.text === 'string') {
+			return part.text;
+		}
+		if (typeof part.content === 'string') {
+			return part.content;
+		}
+		return '';
+	}
+
+	private async withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
+		let done = false;
+		const startedAt = Date.now();
+		const heartbeatMs = Math.max(10000, Math.min(30000, Math.floor(timeoutMs / 4)));
+		const timeoutPromise = (async (): Promise<T> => {
+			while (!done) {
+				const elapsed = Date.now() - startedAt;
+				if (elapsed >= timeoutMs) {
+					throw new Error('AI request timed out.');
+				}
+				const remaining = timeoutMs - elapsed;
+				await sleepTimeout(Math.min(heartbeatMs, remaining));
+				if (done) {
+					return await promise;
+				}
+				const currentElapsed = Date.now() - startedAt;
+				if (currentElapsed >= timeoutMs) {
+					throw new Error('AI request timed out.');
+				}
+				const currentRemaining = Math.max(0, timeoutMs - currentElapsed);
+				this.log(`runOnce waiting (elapsedMs=${currentElapsed}, remainingMs=${currentRemaining})`);
+			}
+			return await promise;
+		})();
+		try {
+			return await Promise.race([promise, timeoutPromise]);
+		}
+		finally {
+			done = true;
+		}
+	}
+
+	private normalizeRetryCount(value?: number): number {
+		if (typeof value !== 'number' || Number.isNaN(value)) {
+			return 2;
+		}
+		return Math.max(0, Math.floor(value));
+	}
+
+	private normalizeRetryDelay(value?: number): number {
+		if (typeof value !== 'number' || Number.isNaN(value)) {
+			return 500;
+		}
+		return Math.max(100, Math.floor(value));
+	}
+
+	private resolveModelSequence(options?: CodexRunOptions): string[] {
+		const primary = this.normalizeOptionalString(options?.threadOptions?.model || this.config.model);
+		const fallbackModels = this.collectFallbackModels(options?.fallbackModels);
+		const sequence: string[] = [];
+		const push = (value: any) => {
+			const normalized = this.normalizeOptionalString(value);
+			if (!normalized || sequence.includes(normalized)) {
+				return;
+			}
+			sequence.push(normalized);
+		};
+		const preferredFallback = this.getModelFallbackPreference(primary);
+		if (preferredFallback) {
+			push(preferredFallback);
+			this.log(`sticky model fallback active: ${primary} -> ${preferredFallback}`);
+		}
+		push(primary);
+		fallbackModels.forEach(push);
+		if (!sequence.length) {
+			sequence.push('');
+		}
+		return sequence;
+	}
+
+	private collectFallbackModels(runFallbackModels?: string[]): string[] {
+		const models: string[] = [];
+		const push = (value: any) => {
+			const normalized = this.normalizeOptionalString(value);
+			if (!normalized || models.includes(normalized)) {
+				return;
+			}
+			models.push(normalized);
+		};
+		this.normalizeModelList(runFallbackModels).forEach(push);
+		this.normalizeModelList(this.config.fallbackModels).forEach(push);
+		push(this.config.fallbackModel);
+		this.normalizeModelList(process.env.AI_ASSISTANT_CODEX_FALLBACK_MODELS).forEach(push);
+		push(process.env.AI_ASSISTANT_CODEX_FALLBACK_MODEL);
+		this.normalizeModelList(process.env.AI_TERMINAL_CODEX_FALLBACK_MODELS).forEach(push);
+		push(process.env.AI_TERMINAL_CODEX_FALLBACK_MODEL);
+		this.normalizeModelList(process.env.AI_DASHBOARD_CODEX_FALLBACK_MODELS).forEach(push);
+		push(process.env.AI_DASHBOARD_CODEX_FALLBACK_MODEL);
+		const primaryModel = this.normalizeOptionalString(
+			this.config.model
+			|| runFallbackModels?.[0]
+			|| ''
+		);
+		if (/^gpt-5(?:\.\d+)?-codex$/i.test(primaryModel)) {
+			push('gpt-5-codex');
+		}
+		if (/^gpt-5(?:-codex|\.\d+-codex)$/i.test(primaryModel)) {
+			push('gpt-5.5');
+		}
+		return models;
+	}
+
+	private normalizeModelList(value: any): string[] {
+		if (Array.isArray(value)) {
+			return value
+				.map(entry => this.normalizeOptionalString(entry))
+				.filter(Boolean);
+		}
+		const raw = this.normalizeOptionalString(value);
+		if (!raw) {
+			return [];
+		}
+		return raw
+			.split(',')
+			.map(entry => this.normalizeOptionalString(entry))
+			.filter(Boolean);
+	}
+
+	private applyModelOverride(options: CodexRunOptions | undefined, model: string): CodexRunOptions {
+		const nextThreadOptions: CodexThreadOptions = {
+			...(options?.threadOptions || {})
+		};
+		if (model) {
+			nextThreadOptions.model = model;
+		}
+		else {
+			delete nextThreadOptions.model;
+		}
+		return {
+			...(options || {}),
+			threadOptions: nextThreadOptions
+		};
+	}
+
+	private isModelAvailabilityError(error: unknown): boolean {
+		const message = error instanceof Error ? error.message : String(error || '');
+		const normalized = message.toLowerCase();
+		if (!normalized.includes('model')) {
+			return false;
+		}
+		if (/does not exist|not found|unknown model|invalid model|unsupported model|model_not_found|no such model/.test(normalized)) {
+			return true;
+		}
+		if (/not available|unavailable|not enabled|not allowed|do not have access|don't have access|insufficient permissions/.test(normalized)) {
+			return true;
+		}
+		return /404/.test(normalized) && /model/.test(normalized);
+	}
+
+	private isRetryableError(error: unknown): boolean {
+		const message = error instanceof Error ? error.message : String(error || '');
+		if (/AI returned no completion content|Codex returned no completion content/i.test(message)) {
+			return true;
+		}
+		if (this.isCodexEnvironmentError(error)) {
+			return true;
+		}
+		const statusMatch = message.match(/\\b(429|5\\d{2})\\b/);
+		if (statusMatch) {
+			return true;
+		}
+		return /ECONNRESET|ETIMEDOUT|EAI_AGAIN|ENOTFOUND|fetch failed|timed out|timeout/i.test(message);
+	}
+
+	private isCodexEnvironmentError(error: unknown): boolean {
+		const message = error instanceof Error ? error.message : String(error || '');
+		return /error finding codex home|could not update path: no such file or directory/i.test(message);
+	}
+
+	private recoverCodexEnvironment(error: unknown): void {
+		const message = error instanceof Error ? error.message : String(error || 'Unknown error');
+		this.log(`recovering codex environment after error: ${message}`);
+		this.threadCache.clear();
+		this.codex = null;
+		this.codexInit = null;
+		this.zDotDir = null;
+	}
+
+	private shouldApplyProcessCwdFallback(error: unknown, options?: CodexRunOptions): boolean {
+		const workingDirectory = this.resolveProcessWorkingDirectory(options);
+		if (!workingDirectory) {
+			return false;
+		}
+		const message = error instanceof Error ? error.message : String(error || '');
+		return /no such file or directory \(os error 2\)/i.test(message);
+	}
+
+	private resolveProcessWorkingDirectory(options?: CodexRunOptions): string {
+		return this.normalizeOptionalString(options?.processWorkingDirectory || options?.threadOptions?.workingDirectory);
+	}
+
+	private applyProcessCwdFallback(options?: CodexRunOptions): CodexRunOptions {
+		const nextThreadOptions: CodexThreadOptions = {
+			...(options?.threadOptions || {})
+		};
+		delete nextThreadOptions.workingDirectory;
+		return {
+			...(options || {}),
+			threadOptions: nextThreadOptions
+		};
+	}
+
+	private async withProcessWorkingDirectoryFallback<T>(workingDirectory: string, runner: () => Promise<T>): Promise<T> {
+		const normalizedWorkingDirectory = this.normalizeOptionalString(workingDirectory);
+		if (!normalizedWorkingDirectory) {
+			return await runner();
+		}
+
+		while (CodexClient.cwdFallbackRunning) {
+			await this.sleep(50);
+		}
+		CodexClient.cwdFallbackRunning = true;
+
+		const originalCwd = process.cwd();
+		try {
+			process.chdir(normalizedWorkingDirectory);
+			this.log(`running Codex fallback from process cwd ${normalizedWorkingDirectory}`);
+			return await runner();
+		}
+		finally {
+			try {
+				process.chdir(originalCwd);
+			}
+			catch (error) {
+				const message = error instanceof Error ? error.message : String(error || 'Unknown error');
+				this.log(`failed to restore process cwd to ${originalCwd}: ${message}`);
+			}
+			CodexClient.cwdFallbackRunning = false;
+		}
+	}
+
+	private normalizeOptionalString(value: any): string {
+		return typeof value === 'string' ? value.trim() : '';
+	}
+
+	private resolveModelUnavailableTtlMs(): number {
+		const raw = Number(
+			process.env.AI_DASHBOARD_CODEX_MODEL_UNAVAILABLE_TTL_MS
+			|| process.env.AI_ASSISTANT_CODEX_MODEL_UNAVAILABLE_TTL_MS
+		);
+		if (Number.isFinite(raw) && raw >= CodexClient.MIN_MODEL_UNAVAILABLE_TTL_MS) {
+			return Math.floor(raw);
+		}
+		return CodexClient.DEFAULT_MODEL_UNAVAILABLE_TTL_MS;
+	}
+
+	private async ensureModelFallbackPreferencesLoaded(): Promise<void> {
+		if (CodexClient.modelFallbackPreferencesLoaded) {
+			this.pruneModelFallbackPreferences();
+			return;
+		}
+		if (CodexClient.modelFallbackPreferencesLoadPromise !== null) {
+			await CodexClient.modelFallbackPreferencesLoadPromise;
+			this.pruneModelFallbackPreferences();
+			return;
+		}
+		CodexClient.modelFallbackPreferencesLoadPromise = (async () => {
+			try {
+				const raw = await fs.readFile(CodexClient.MODEL_UNAVAILABLE_FALLBACK_FILE, 'utf8');
+				const parsed = JSON.parse(raw);
+				if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+					return;
+				}
+				Object.entries(parsed).forEach(([model, entry]) => {
+					const key = this.normalizeOptionalString(model);
+					if (!key) {
+						return;
+					}
+					const fallbackModel = this.normalizeOptionalString((entry as any)?.fallbackModel);
+					const expiresAt = Number((entry as any)?.expiresAt);
+					const updatedAt = Number((entry as any)?.updatedAt);
+					if (!fallbackModel || !Number.isFinite(expiresAt) || expiresAt <= Date.now()) {
+						return;
+					}
+					CodexClient.modelFallbackPreferences.set(key, {
+						fallbackModel,
+						expiresAt: Math.floor(expiresAt),
+						updatedAt: Number.isFinite(updatedAt) ? Math.floor(updatedAt) : Date.now()
+					});
+				});
+			}
+			catch {
+				// Ignore cache load errors.
+			}
+			finally {
+				CodexClient.modelFallbackPreferencesLoaded = true;
+				CodexClient.modelFallbackPreferencesLoadPromise = null;
+			}
+		})();
+		await CodexClient.modelFallbackPreferencesLoadPromise;
+		this.pruneModelFallbackPreferences();
+	}
+
+	private pruneModelFallbackPreferences(now = Date.now()): void {
+		Array.from(CodexClient.modelFallbackPreferences.entries()).forEach(([model, entry]) => {
+			if (!entry || !Number.isFinite(entry.expiresAt) || entry.expiresAt <= now) {
+				CodexClient.modelFallbackPreferences.delete(model);
+			}
+		});
+	}
+
+	private getModelFallbackPreference(model: string): string {
+		const normalizedModel = this.normalizeOptionalString(model);
+		if (!normalizedModel) {
+			return '';
+		}
+		this.pruneModelFallbackPreferences();
+		const entry = CodexClient.modelFallbackPreferences.get(normalizedModel);
+		if (!entry || entry.expiresAt <= Date.now()) {
+			if (entry) {
+				CodexClient.modelFallbackPreferences.delete(normalizedModel);
+			}
+			return '';
+		}
+		return entry.fallbackModel;
+	}
+
+	private async rememberModelFallbackPreference(model: string, fallbackModel: string): Promise<void> {
+		const normalizedModel = this.normalizeOptionalString(model);
+		const normalizedFallback = this.normalizeOptionalString(fallbackModel);
+		if (!normalizedModel || !normalizedFallback || normalizedModel === normalizedFallback) {
+			return;
+		}
+		const now = Date.now();
+		CodexClient.modelFallbackPreferences.set(normalizedModel, {
+			fallbackModel: normalizedFallback,
+			expiresAt: now + this.modelUnavailableTtlMs,
+			updatedAt: now
+		});
+		this.pruneModelFallbackPreferences(now);
+		await this.persistModelFallbackPreferences();
+	}
+
+	private async clearModelFallbackPreference(model: string): Promise<void> {
+		const normalizedModel = this.normalizeOptionalString(model);
+		if (!normalizedModel) {
+			return;
+		}
+		if (!CodexClient.modelFallbackPreferences.has(normalizedModel)) {
+			return;
+		}
+		CodexClient.modelFallbackPreferences.delete(normalizedModel);
+		await this.persistModelFallbackPreferences();
+	}
+
+	private async persistModelFallbackPreferences(): Promise<void> {
+		this.pruneModelFallbackPreferences();
+		if (!CodexClient.modelFallbackPreferences.size) {
+			try {
+				await fs.unlink(CodexClient.MODEL_UNAVAILABLE_FALLBACK_FILE);
+			}
+			catch {
+				// Ignore file cleanup errors.
+			}
+			return;
+		}
+		const serialized: Record<string, CodexModelFallbackPreference> = {};
+		CodexClient.modelFallbackPreferences.forEach((entry, model) => {
+			serialized[model] = entry;
+		});
+		try {
+			await fs.writeFile(CodexClient.MODEL_UNAVAILABLE_FALLBACK_FILE, JSON.stringify(serialized), 'utf8');
+		}
+		catch {
+			// Ignore cache persistence errors.
+		}
+	}
+
+	private async sleep(ms: number): Promise<void> {
+		await sleepTimeout(ms);
+	}
+
+	private async ensureCodex(): Promise<void> {
+		if (this.codex) {
+			return;
+		}
+		if (this.codexInit !== null) {
+			await this.codexInit;
+			return;
+		}
+		this.codexInit = (async () => {
+			this.log('Loading AI SDK module...');
+			const importer = new Function('specifier', 'return import(specifier)') as DynamicImport;
+			const mod = await importer('@openai/codex-sdk');
+			const CodexCtor = mod?.Codex || mod?.default?.Codex || mod?.default || mod;
+			if (!CodexCtor) {
+				throw new Error('AI SDK export not found.');
+			}
+			this.log('AI SDK module loaded.');
+			const env = await this.buildCodexEnv();
+			this.codex = new CodexCtor({
+				apiKey: this.config.apiKey,
+				baseUrl: this.config.baseUrl,
+				model: this.config.model,
+				env
+			});
+			this.log('AI client initialized.');
+		})();
+		await this.codexInit;
+	}
+
+	private log(message: string): void {
+		if (!this.debugEnabled) {
+			return;
+		}
+		console.log(`[AI-Dashboard][AI] ${message}`);
+	}
+
+	private resolveDefaultTimeoutMs(): number {
+		const raw = Number(process.env.AI_DASHBOARD_CODEX_TIMEOUT_MS || process.env.AI_DASHBOARD_AGENT_TIMEOUT_MS);
+		if (Number.isFinite(raw) && raw > 0) {
+			return Math.floor(raw);
+		}
+		return 600000;
+	}
+
+	private resolveStreamEnabled(): boolean {
+		const raw = (process.env.AI_DASHBOARD_CODEX_STREAM || '').trim().toLowerCase();
+		if (!raw) {
+			return this.debugEnabled;
+		}
+		if (['true', '1', 'yes', 'y', 'on'].includes(raw)) {
+			return true;
+		}
+		if (['false', '0', 'no', 'n', 'off'].includes(raw)) {
+			return false;
+		}
+		return this.debugEnabled;
+	}
+
+	private resolveCleanZdotdirEnabled(): boolean {
+		const raw = (process.env.AI_DASHBOARD_CODEX_CLEAN_ZDOTDIR ?? '').trim().toLowerCase();
+		if (!raw) {
+			return true;
+		}
+		if (['false', '0', 'no', 'n', 'off'].includes(raw)) {
+			return false;
+		}
+		if (['true', '1', 'yes', 'y', 'on'].includes(raw)) {
+			return true;
+		}
+		return true;
+	}
+
+	private async buildCodexEnv(): Promise<Record<string, string>> {
+		const env: Record<string, string> = {};
+		for (const [key, value] of Object.entries(process.env)) {
+			if (typeof value === 'string') {
+				env[key] = value;
+			}
+		}
+		if (!this.normalizeOptionalString(env.HOME)) {
+			const fallbackHome = this.normalizeOptionalString(process.env.HOME || os.homedir());
+			if (fallbackHome) {
+				env.HOME = fallbackHome;
+			}
+		}
+		await this.ensureCodexHome(env);
+		if (this.cleanZdotdirEnabled) {
+			env.ZDOTDIR = await this.ensureZdotdir();
+		}
+		return env;
+	}
+
+	private async ensureCodexHome(env: Record<string, string>): Promise<void> {
+		const current = this.normalizeOptionalString(env.CODEX_HOME || process.env.CODEX_HOME);
+		const homeRoot = this.normalizeOptionalString(env.HOME || process.env.HOME || os.homedir());
+		const primary = current || (homeRoot ? path.join(homeRoot, '.codex') : path.join(os.tmpdir(), 'resolveio-codex-home'));
+		try {
+			await fs.mkdir(path.join(primary, 'sessions'), { recursive: true });
+			env.CODEX_HOME = primary;
+			process.env.CODEX_HOME = primary;
+			return;
+		}
+		catch {
+			const fallback = path.join(os.tmpdir(), 'resolveio-codex-home');
+			await fs.mkdir(path.join(fallback, 'sessions'), { recursive: true });
+			env.CODEX_HOME = fallback;
+			process.env.CODEX_HOME = fallback;
+		}
+	}
+
+	private async ensureZdotdir(): Promise<string> {
+		if (this.zDotDir) {
+			return this.zDotDir;
+		}
+		const override = (process.env.AI_DASHBOARD_CODEX_ZDOTDIR || '').trim();
+		const dir = override || path.join(os.tmpdir(), 'codex-zdotdir');
+		await fs.mkdir(dir, { recursive: true });
+		await Promise.allSettled([
+			fs.writeFile(path.join(dir, '.zshrc'), '', { flag: 'a' }),
+			fs.writeFile(path.join(dir, '.zprofile'), '', { flag: 'a' })
+		]);
+		this.zDotDir = dir;
+		return dir;
+	}
+
+	private resolvePingEnabled(): boolean {
+		return this.normalizeBoolean(process.env.AI_DASHBOARD_CODEX_PING, false);
+	}
+
+	private resolvePingTimeoutMs(): number {
+		const raw = Number(process.env.AI_DASHBOARD_CODEX_PING_TIMEOUT_MS);
+		if (Number.isFinite(raw) && raw > 0) {
+			return Math.floor(raw);
+		}
+		return 10000;
+	}
+
+	private resolveDefaultThreadOptions(): CodexThreadOptions {
+		const approvalPolicy = this.normalizeApprovalPolicy(process.env.AI_DASHBOARD_CODEX_APPROVAL_POLICY);
+		const sandboxMode = this.normalizeSandboxMode(process.env.AI_DASHBOARD_CODEX_SANDBOX_MODE, 'workspace-write');
+		const skipGitRepoCheck = this.normalizeBoolean(process.env.AI_DASHBOARD_CODEX_SKIP_GIT_CHECK, true);
+		const webSearchMode = this.normalizeWebSearchMode(process.env.AI_DASHBOARD_CODEX_WEB_SEARCH_MODE);
+		const webSearchEnabled = this.normalizeBoolean(process.env.AI_DASHBOARD_CODEX_WEB_SEARCH_ENABLED, false);
+		const networkAccessEnabled = this.normalizeBoolean(process.env.AI_DASHBOARD_CODEX_NETWORK_ACCESS_ENABLED, false);
+		const modelReasoningEffort = this.normalizeModelReasoningEffort(process.env.AI_DASHBOARD_CODEX_REASONING_EFFORT);
+		return {
+			model: this.config.model,
+			approvalPolicy,
+			sandboxMode,
+			skipGitRepoCheck,
+			webSearchMode,
+			webSearchEnabled,
+			networkAccessEnabled,
+			modelReasoningEffort
+		};
+	}
+
+	private mergeThreadOptions(overrides?: CodexThreadOptions): CodexThreadOptions | undefined {
+		const merged: CodexThreadOptions = {
+			...this.defaultThreadOptions,
+			...(overrides || {})
+		};
+		const effectiveModel = this.normalizeOptionalString(merged.model || this.config.model);
+		const normalizedEffort = this.normalizeModelReasoningEffortForModel(effectiveModel, merged.modelReasoningEffort);
+		if (normalizedEffort && normalizedEffort !== merged.modelReasoningEffort) {
+			this.log(`normalized reasoning effort for model=${effectiveModel || 'default'}: ${merged.modelReasoningEffort} -> ${normalizedEffort}`);
+			merged.modelReasoningEffort = normalizedEffort;
+		}
+		const cleaned = Object.entries(merged).reduce((acc, [key, value]) => {
+			if (value === undefined || value === null || value === '') {
+				return acc;
+			}
+			(acc as any)[key] = value;
+			return acc;
+		}, {} as CodexThreadOptions);
+		return Object.keys(cleaned).length ? cleaned : undefined;
+	}
+
+	private normalizeApprovalPolicy(value?: string): CodexThreadOptions['approvalPolicy'] {
+		const normalized = (value || '').trim().toLowerCase();
+		if (normalized === 'never' || normalized === 'on-request' || normalized === 'on-failure' || normalized === 'untrusted') {
+			return normalized as CodexThreadOptions['approvalPolicy'];
+		}
+		return 'never';
+	}
+
+	private normalizeSandboxMode(value?: string, fallback: CodexThreadOptions['sandboxMode'] = 'read-only'): CodexThreadOptions['sandboxMode'] {
+		const normalized = (value || '').trim().toLowerCase();
+		if (normalized === 'read-only' || normalized === 'workspace-write' || normalized === 'danger-full-access') {
+			return normalized as CodexThreadOptions['sandboxMode'];
+		}
+		return fallback;
+	}
+
+	private normalizeWebSearchMode(value?: string): CodexThreadOptions['webSearchMode'] | undefined {
+		const normalized = (value || '').trim().toLowerCase();
+		if (normalized === 'disabled' || normalized === 'cached' || normalized === 'live') {
+			return normalized as CodexThreadOptions['webSearchMode'];
+		}
+		return undefined;
+	}
+
+	private normalizeModelReasoningEffort(value?: string): CodexThreadOptions['modelReasoningEffort'] | undefined {
+		const normalized = (value || '').trim().toLowerCase();
+		if (normalized === 'minimal' || normalized === 'none' || normalized === 'low' || normalized === 'medium' || normalized === 'high' || normalized === 'xhigh') {
+			return normalized as CodexThreadOptions['modelReasoningEffort'];
+		}
+		return undefined;
+	}
+
+	private normalizeModelReasoningEffortForModel(
+		model: string,
+		effort?: CodexThreadOptions['modelReasoningEffort']
+	): CodexThreadOptions['modelReasoningEffort'] | undefined {
+		if (!effort) {
+			return effort;
+		}
+		const normalizedModel = this.normalizeOptionalString(model).toLowerCase();
+		if (effort === 'minimal' && (!normalizedModel || normalizedModel.startsWith('gpt-5') || normalizedModel.includes('codex'))) {
+			return 'low';
+		}
+		return effort;
+	}
+
+	private normalizeBoolean(value: string | undefined, fallback: boolean): boolean {
+		if (!value) {
+			return fallback;
+		}
+		const normalized = value.trim().toLowerCase();
+		if (['true', '1', 'yes', 'y', 'on'].includes(normalized)) {
+			return true;
+		}
+		if (['false', '0', 'no', 'n', 'off'].includes(normalized)) {
+			return false;
+		}
+		return fallback;
+	}
+
+	private formatThreadOptions(options: CodexThreadOptions): string {
+		const summary: Record<string, string> = {};
+		Object.entries(options).forEach(([key, value]) => {
+			if (value === undefined || value === null) {
+				return;
+			}
+			summary[key] = String(value);
+		});
+		return JSON.stringify(summary);
+	}
+
+	private buildThreadOptionsKey(options?: CodexThreadOptions): string {
+		if (!options) {
+			return '';
+		}
+		const ordered: Record<string, string> = {};
+		Object.keys(options)
+			.sort()
+			.forEach((key) => {
+				const value = (options as Record<string, unknown>)[key];
+				if (value === undefined || value === null) {
+					return;
+				}
+				ordered[key] = Array.isArray(value) ? JSON.stringify(value) : String(value);
+			});
+		return JSON.stringify(ordered);
+	}
+
+	private buildThreadCacheKey(threadKey: string | undefined, options?: CodexThreadOptions): string | null {
+		if (threadKey) {
+			return `thread:${threadKey}`;
+		}
+		const model = options?.model || this.config.model;
+		const workingDirectory = options?.workingDirectory;
+		if (!model && !workingDirectory) {
+			return null;
+		}
+		return `thread:auto:${model || 'default'}:${workingDirectory || 'root'}`;
+	}
+
+	private isPingPrompt(prompt: string): boolean {
+		return prompt.trim() === CodexClient.PING_PROMPT;
+	}
+
+	private async runPing(threadOptions?: CodexThreadOptions): Promise<void> {
+		this.log(`runPing start (timeoutMs=${this.pingTimeoutMs})`);
+		try {
+			const content = await this.runOnce(CodexClient.PING_PROMPT, {
+				timeoutMs: this.pingTimeoutMs,
+				threadOptions,
+				reuseThread: false
+			});
+			const trimmed = content.trim();
+			const preview = trimmed.length > 80 ? `${trimmed.slice(0, 80)}…` : trimmed;
+			this.log(`runPing response: ${preview}`);
+		}
+		catch (error) {
+			const message = error instanceof Error ? error.message : String(error || 'Unknown error');
+			this.log(`runPing failed: ${message}`);
+			throw error;
+		}
+	}
+}