@resolveio/server-lib 22.3.28 → 22.3.29

package/src/managers/ai-assistant-codex-manager.manager.ts

@@ -0,0 +1,1131 @@
+import { AiTerminalConversations } from '../collections/ai-terminal-conversation.collection';
+import { AiTerminalIssueReports } from '../collections/ai-terminal-issue-report.collection';
+import { AiTerminalMessages } from '../collections/ai-terminal-message.collection';
+import { AiTerminalMessageModel } from '../models/ai-terminal-message.model';
+import { AiTerminalConversationModel } from '../models/ai-terminal-conversation.model';
+import { promises as fs } from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { execFile } from 'child_process';
+import { promisify } from 'util';
+
+type AiAssistantCodexManagerRuntime = {
+	isWorkersEnabled: boolean;
+	isWorkerInstance: boolean;
+	workerIndex?: string | null;
+	workerInstance?: string | null;
+};
+
+type AiAssistantCodexManagerConfig = {
+	enabled: boolean;
+	runOnAnyWorker: boolean;
+	targetWorkerIndex: string;
+	targetWorkerInstance: string | null;
+	pollIntervalMs: number;
+	lookbackMinutes: number;
+	maxMessagesPerPoll: number;
+	maxRemediationsPerPoll: number;
+	autoReportEnabled: boolean;
+	autoRemediationEnabled: boolean;
+	remediationHookPath: string;
+	remediationTimeoutMs: number;
+	remediationMaxBufferBytes: number;
+	remediationMaxAttemptsPerIssue: number;
+	remediationCooldownMinutes: number;
+	autoReplayEnabled: boolean;
+	replayCommand: string;
+	replayTimeoutMs: number;
+	replayMaxBufferBytes: number;
+	replayMaxAttemptsPerIssue: number;
+	replayCooldownMinutes: number;
+	debug: boolean;
+};
+
+type AiAssistantQualityDecision = {
+	status: 'pass' | 'needs_review';
+	reason: string;
+	code: string;
+};
+
+const DEFAULT_POLL_INTERVAL_MS = 15000;
+const DEFAULT_LOOKBACK_MINUTES = 120;
+const DEFAULT_MAX_MESSAGES_PER_POLL = 100;
+const DEFAULT_MAX_REMEDIATIONS_PER_POLL = 3;
+const DEFAULT_REMEDIATION_TIMEOUT_MS = 20 * 60 * 1000;
+const DEFAULT_REMEDIATION_MAX_BUFFER_BYTES = 1024 * 1024;
+const DEFAULT_REMEDIATION_MAX_ATTEMPTS_PER_ISSUE = 2;
+const DEFAULT_REMEDIATION_COOLDOWN_MINUTES = 15;
+const DEFAULT_REPLAY_TIMEOUT_MS = 5 * 60 * 1000;
+const DEFAULT_REPLAY_MAX_BUFFER_BYTES = 1024 * 1024;
+const DEFAULT_REPLAY_MAX_ATTEMPTS_PER_ISSUE = 2;
+const DEFAULT_REPLAY_COOLDOWN_MINUTES = 10;
+const MAX_STORED_OUTPUT_CHARS = 4000;
+const execFileAsync = promisify(execFile);
+
+function normalizeOptionalString(value: any): string {
+	return typeof value === 'string' ? value.trim() : '';
+}
+
+function parseBooleanEnv(value: any, fallback: boolean): boolean {
+	const normalized = normalizeOptionalString(value).toLowerCase();
+	if (!normalized) {
+		return fallback;
+	}
+	if (['1', 'true', 'yes', 'on', 'enabled'].includes(normalized)) {
+		return true;
+	}
+	if (['0', 'false', 'no', 'off', 'disabled'].includes(normalized)) {
+		return false;
+	}
+	return fallback;
+}
+
+function parsePositiveInt(value: any, fallback: number, min = 1): number {
+	const parsed = parseInt(String(value ?? '').trim(), 10);
+	if (!Number.isFinite(parsed) || Number.isNaN(parsed) || parsed < min) {
+		return fallback;
+	}
+	return parsed;
+}
+
+function parsePositiveIntWithMax(value: any, fallback: number, min: number, max: number): number {
+	const parsed = parsePositiveInt(value, fallback, min);
+	return Math.min(parsed, max);
+}
+
+function resolveEnabledFlag(): boolean {
+	const explicitValue = process.env.AI_ASSISTANT_CODEX_MANAGER_ENABLED
+		?? process.env.AI_ASSISTANT_QUALITY_MANAGER_ENABLED;
+	if (explicitValue !== undefined && explicitValue !== null && String(explicitValue).trim() !== '') {
+		return parseBooleanEnv(explicitValue, false);
+	}
+	return false;
+}
+
+export class AiAssistantCodexManager {
+	private readonly config: AiAssistantCodexManagerConfig;
+	private readonly runtime: AiAssistantCodexManagerRuntime;
+	private _interval: NodeJS.Timeout | null = null;
+	private _inFlight = false;
+	private _cursorCreatedAt: Date;
+	private _cursorMessageId = '';
+
+	constructor(config: AiAssistantCodexManagerConfig, runtime: AiAssistantCodexManagerRuntime) {
+		this.config = config;
+		this.runtime = runtime;
+		this._cursorCreatedAt = new Date(Date.now() - (config.lookbackMinutes * 60 * 1000));
+	}
+
+	static createIfEnabled(runtime: AiAssistantCodexManagerRuntime): AiAssistantCodexManager | null {
+		const config = this.resolveConfig();
+		if (!config.enabled) {
+			return null;
+		}
+		if (!this.shouldRunOnCurrentProcess(config, runtime)) {
+			return null;
+		}
+		const manager = new AiAssistantCodexManager(config, runtime);
+		manager.initialize();
+		return manager;
+	}
+
+	private static resolveConfig(): AiAssistantCodexManagerConfig {
+		const targetWorkerIndex = normalizeOptionalString(
+			process.env.AI_ASSISTANT_CODEX_MANAGER_WORKER_INDEX
+			|| process.env.AI_ASSISTANT_CODEX_WORKER_INDEX
+			|| process.env.CODEX_WORKER_INDEX
+			|| 'ai-assistant-manager'
+		) || 'ai-assistant-manager';
+		const targetWorkerInstance = normalizeOptionalString(
+			process.env.AI_ASSISTANT_CODEX_MANAGER_WORKER_INSTANCE
+		) || null;
+		return {
+			enabled: resolveEnabledFlag(),
+			runOnAnyWorker: parseBooleanEnv(process.env.AI_ASSISTANT_CODEX_MANAGER_RUN_ANY_WORKER, false),
+			targetWorkerIndex,
+			targetWorkerInstance,
+			pollIntervalMs: parsePositiveInt(process.env.AI_ASSISTANT_CODEX_MANAGER_INTERVAL_MS, DEFAULT_POLL_INTERVAL_MS, 1000),
+			lookbackMinutes: parsePositiveInt(process.env.AI_ASSISTANT_CODEX_MANAGER_LOOKBACK_MINUTES, DEFAULT_LOOKBACK_MINUTES, 1),
+			maxMessagesPerPoll: parsePositiveIntWithMax(
+				process.env.AI_ASSISTANT_CODEX_MANAGER_MAX_MESSAGES_PER_POLL,
+				DEFAULT_MAX_MESSAGES_PER_POLL,
+				1,
+				1000
+			),
+			maxRemediationsPerPoll: parsePositiveIntWithMax(
+				process.env.AI_ASSISTANT_CODEX_MANAGER_MAX_REMEDIATIONS_PER_POLL,
+				DEFAULT_MAX_REMEDIATIONS_PER_POLL,
+				1,
+				100
+			),
+			autoReportEnabled: parseBooleanEnv(process.env.AI_ASSISTANT_CODEX_MANAGER_AUTO_REPORT, true),
+			autoRemediationEnabled: parseBooleanEnv(process.env.AI_ASSISTANT_CODEX_MANAGER_AUTO_REMEDIATE, false),
+			remediationHookPath: normalizeOptionalString(process.env.AI_ASSISTANT_CODEX_MANAGER_REMEDIATE_HOOK),
+			remediationTimeoutMs: parsePositiveInt(
+				process.env.AI_ASSISTANT_CODEX_MANAGER_REMEDIATE_TIMEOUT_MS,
+				DEFAULT_REMEDIATION_TIMEOUT_MS,
+				1000
+			),
+			remediationMaxBufferBytes: parsePositiveInt(
+				process.env.AI_ASSISTANT_CODEX_MANAGER_REMEDIATE_MAX_BUFFER_BYTES,
+				DEFAULT_REMEDIATION_MAX_BUFFER_BYTES,
+				1024
+			),
+			remediationMaxAttemptsPerIssue: parsePositiveInt(
+				process.env.AI_ASSISTANT_CODEX_MANAGER_REMEDIATE_MAX_ATTEMPTS_PER_ISSUE,
+				DEFAULT_REMEDIATION_MAX_ATTEMPTS_PER_ISSUE,
+				1
+			),
+			remediationCooldownMinutes: parsePositiveInt(
+				process.env.AI_ASSISTANT_CODEX_MANAGER_REMEDIATE_COOLDOWN_MINUTES,
+				DEFAULT_REMEDIATION_COOLDOWN_MINUTES,
+				1
+			),
+			autoReplayEnabled: parseBooleanEnv(process.env.AI_ASSISTANT_CODEX_MANAGER_REPLAY_ENABLED, false),
+			replayCommand: normalizeOptionalString(process.env.AI_ASSISTANT_CODEX_MANAGER_REPLAY_COMMAND),
+			replayTimeoutMs: parsePositiveInt(
+				process.env.AI_ASSISTANT_CODEX_MANAGER_REPLAY_TIMEOUT_MS,
+				DEFAULT_REPLAY_TIMEOUT_MS,
+				1000
+			),
+			replayMaxBufferBytes: parsePositiveInt(
+				process.env.AI_ASSISTANT_CODEX_MANAGER_REPLAY_MAX_BUFFER_BYTES,
+				DEFAULT_REPLAY_MAX_BUFFER_BYTES,
+				1024
+			),
+			replayMaxAttemptsPerIssue: parsePositiveInt(
+				process.env.AI_ASSISTANT_CODEX_MANAGER_REPLAY_MAX_ATTEMPTS_PER_ISSUE,
+				DEFAULT_REPLAY_MAX_ATTEMPTS_PER_ISSUE,
+				1
+			),
+			replayCooldownMinutes: parsePositiveInt(
+				process.env.AI_ASSISTANT_CODEX_MANAGER_REPLAY_COOLDOWN_MINUTES,
+				DEFAULT_REPLAY_COOLDOWN_MINUTES,
+				1
+			),
+			debug: parseBooleanEnv(process.env.AI_ASSISTANT_CODEX_MANAGER_DEBUG, false)
+		};
+	}
+
+	private static shouldRunOnCurrentProcess(config: AiAssistantCodexManagerConfig, runtime: AiAssistantCodexManagerRuntime): boolean {
+		const workerIndex = normalizeOptionalString(runtime.workerIndex);
+		const workerInstance = normalizeOptionalString(runtime.workerInstance);
+		if (!runtime.isWorkersEnabled) {
+			return true;
+		}
+		if (!runtime.isWorkerInstance) {
+			return false;
+		}
+		if (config.runOnAnyWorker) {
+			if (config.targetWorkerInstance) {
+				return workerInstance === config.targetWorkerInstance;
+			}
+			return true;
+		}
+		if (workerIndex !== config.targetWorkerIndex) {
+			return false;
+		}
+		if (config.targetWorkerInstance && workerInstance !== config.targetWorkerInstance) {
+			return false;
+		}
+		return true;
+	}
+
+	private initialize(): void {
+		console.log(new Date(), '[AI Assistant Codex Manager] started', {
+			workerIndex: process.env.WORKER_INDEX || null,
+			workerInstance: process.env.NODE_APP_INSTANCE || null,
+			pollIntervalMs: this.config.pollIntervalMs,
+			maxMessagesPerPoll: this.config.maxMessagesPerPoll,
+			lookbackMinutes: this.config.lookbackMinutes,
+			autoRemediationEnabled: this.config.autoRemediationEnabled,
+			remediationHookPath: this.config.remediationHookPath || null,
+			autoReplayEnabled: this.config.autoReplayEnabled
+		});
+
+		setTimeout(async () => {
+			try {
+				await this.pollOnce();
+			}
+			catch (error) {
+				console.error(new Date(), '[AI Assistant Codex Manager] initial poll failed', error);
+			}
+		}, 2500);
+
+		this._interval = setInterval(async () => {
+			try {
+				await this.pollOnce();
+			}
+			catch (error) {
+				console.error(new Date(), '[AI Assistant Codex Manager] poll failed', error);
+			}
+		}, this.config.pollIntervalMs);
+	}
+
+	private async pollOnce(): Promise<void> {
+		if (this._inFlight) {
+			return;
+		}
+		this._inFlight = true;
+		try {
+			const query: Record<string, any> = {
+				role: 'assistant',
+				$or: [
+					{ createdAt: { $gt: this._cursorCreatedAt } },
+					{ createdAt: this._cursorCreatedAt, _id: { $gt: this._cursorMessageId || '' } }
+				]
+			};
+			const messages = await AiTerminalMessages.find(
+				query,
+				{
+					sort: { createdAt: 1, _id: 1 },
+					limit: this.config.maxMessagesPerPoll
+				}
+			) as AiTerminalMessageModel[];
+			if (Array.isArray(messages) && messages.length) {
+				for (const message of messages) {
+					const createdAt = message?.createdAt ? new Date(message.createdAt) : null;
+					if (createdAt && Number.isFinite(createdAt.getTime())) {
+						this._cursorCreatedAt = createdAt;
+					}
+					this._cursorMessageId = normalizeOptionalString(message?._id);
+					await this.processMessage(message);
+				}
+			}
+
+			if (this.config.autoRemediationEnabled) {
+				await this.pollRemediationQueue();
+			}
+		}
+		finally {
+			this._inFlight = false;
+		}
+	}
+
+	private async processMessage(message: AiTerminalMessageModel): Promise<void> {
+		const messageId = normalizeOptionalString(message?._id);
+		const conversationId = normalizeOptionalString(message?.id_conversation);
+		if (!messageId || !conversationId) {
+			return;
+		}
+		const metadata = this.normalizeMetadata(message?.metadata);
+		if (metadata.pending === true) {
+			return;
+		}
+		if (metadata.quality_manager?.checked_at) {
+			return;
+		}
+
+		const decision = this.evaluateMessageQuality(message, metadata);
+		let issueReportId = '';
+		if (decision.status === 'needs_review' && this.config.autoReportEnabled) {
+			issueReportId = await this.ensureAutomatedIssueReport(message, decision);
+		}
+		await this.markMessageQualityReviewed(messageId, decision, issueReportId);
+		if (issueReportId && this.config.autoRemediationEnabled) {
+			await this.runIssueRemediation(issueReportId, { source: 'auto', trigger: 'message_quality' });
+		}
+	}
+
+	private evaluateMessageQuality(message: AiTerminalMessageModel, metadata: Record<string, any>): AiAssistantQualityDecision {
+		const content = normalizeOptionalString(message?.content);
+		const toolError = normalizeOptionalString(metadata?.tool_execution?.outcome?.error);
+		if (metadata.failed === true) {
+			return {
+				status: 'needs_review',
+				reason: 'Assistant response finalized in failed state.',
+				code: 'metadata_failed'
+			};
+		}
+		if (/I ran into an internal error while preparing that response/i.test(content)) {
+			return {
+				status: 'needs_review',
+				reason: 'Assistant returned internal error text.',
+				code: 'internal_error_copy'
+			};
+		}
+		if (/That took longer than expected\./i.test(content)) {
+			return {
+				status: 'needs_review',
+				reason: 'Assistant response timed out.',
+				code: 'timeout_copy'
+			};
+		}
+		if (/I could not finish this request within the assistant time budget/i.test(content)) {
+			return {
+				status: 'needs_review',
+				reason: 'Assistant run exceeded configured time budget.',
+				code: 'run_budget_exceeded_copy'
+			};
+		}
+		if (toolError) {
+			return {
+				status: 'needs_review',
+				reason: `Assistant tool execution error: ${toolError}`.slice(0, 500),
+				code: 'tool_execution_error'
+			};
+		}
+		return {
+			status: 'pass',
+			reason: 'Quality check passed.',
+			code: 'ok'
+		};
+	}
+
+	private async ensureAutomatedIssueReport(message: AiTerminalMessageModel, decision: AiAssistantQualityDecision): Promise<string> {
+		const messageId = normalizeOptionalString(message?._id);
+		const conversationId = normalizeOptionalString(message?.id_conversation);
+		if (!messageId || !conversationId) {
+			return '';
+		}
+		const existing = await AiTerminalIssueReports.findOne({
+			id_message: messageId,
+			source: 'automated'
+		});
+		if (existing?._id) {
+			return normalizeOptionalString(existing._id);
+		}
+
+		const conversation = await AiTerminalConversations.findById(conversationId) as AiTerminalConversationModel | null;
+		if (!conversation) {
+			return '';
+		}
+		const assistantContent = normalizeOptionalString(message?.content);
+		const requestId = normalizeOptionalString((message as any)?.metadata?.request_id);
+		const userPrompt = await this.resolveClosestUserPrompt(conversationId, message?.createdAt);
+		const now = new Date();
+		const doc: Record<string, any> = {
+			id_conversation: conversationId,
+			status: 'open',
+			source: 'automated',
+			reason: decision.reason,
+			...(userPrompt ? { user_prompt: userPrompt } : {}),
+			...(assistantContent ? { assistant_content: assistantContent } : {}),
+			...(requestId ? { request_id: requestId } : {}),
+			id_message: messageId,
+			...(normalizeOptionalString(conversation?.id_client) ? { id_client: normalizeOptionalString(conversation?.id_client) } : {}),
+			...(normalizeOptionalString(conversation?.id_app) ? { id_app: normalizeOptionalString(conversation?.id_app) } : {}),
+			...(normalizeOptionalString(conversation?.id_user) ? { id_user: normalizeOptionalString(conversation?.id_user) } : {}),
+			metadata: {
+				quality_manager: {
+					code: decision.code,
+					checked_at: now,
+					worker_index: normalizeOptionalString(process.env.WORKER_INDEX),
+					worker_instance: normalizeOptionalString(process.env.NODE_APP_INSTANCE)
+				}
+			},
+			createdAt: now,
+			updatedAt: now
+		};
+		const insertResult = await AiTerminalIssueReports.insertOne(doc as any);
+		const issueId = normalizeOptionalString(insertResult?._id);
+		console.log(new Date(), '[AI Assistant Codex Manager] created automated issue report', {
+			id_issue_report: issueId || null,
+			id_conversation: conversationId,
+			id_message: messageId,
+			code: decision.code
+		});
+		return issueId;
+	}
+
+	private async pollRemediationQueue(): Promise<void> {
+		const issues = await AiTerminalIssueReports.find(
+			{
+				source: 'automated',
+				status: { $in: ['open', 'triaged'] },
+				$or: [
+					{ 'metadata.quality_manager.remediation.status': { $exists: false } },
+					{ 'metadata.quality_manager.remediation.status': { $in: ['queued', 'failed', 'skipped'] } }
+				]
+			},
+			{
+				sort: { updatedAt: 1, createdAt: 1 },
+				limit: this.config.maxRemediationsPerPoll
+			}
+		) as Array<Record<string, any>>;
+		if (!Array.isArray(issues) || !issues.length) {
+			return;
+		}
+		for (const issue of issues) {
+			const issueId = normalizeOptionalString(issue?._id);
+			if (!issueId) {
+				continue;
+			}
+			const remediationResult = await this.runIssueRemediation(issueId, { source: 'auto', trigger: 'poll' });
+			if (remediationResult?.status === 'success' || remediationResult?.status === 'running') {
+				break;
+			}
+		}
+	}
+
+	public async runIssueRemediation(
+		issueReportId: string,
+		options: { force?: boolean; source?: 'auto' | 'manual'; trigger?: string } = {}
+	): Promise<Record<string, any>> {
+		const issueId = normalizeOptionalString(issueReportId);
+		if (!issueId) {
+			return {
+				status: 'failed',
+				reason: 'Issue report id is required.'
+			};
+		}
+		if (!this.config.autoRemediationEnabled) {
+			return {
+				status: 'skipped',
+				reason: 'Auto remediation is disabled.'
+			};
+		}
+		if (!this.config.remediationHookPath) {
+			return {
+				status: 'skipped',
+				reason: 'Remediation hook path is not configured.'
+			};
+		}
+
+		const issue = await AiTerminalIssueReports.findById(issueId) as Record<string, any> | null;
+		if (!issue) {
+			return {
+				status: 'not_found',
+				reason: 'Issue report not found.'
+			};
+		}
+
+		const force = options.force === true;
+		const source = options.source === 'manual' ? 'manual' : 'auto';
+		const now = new Date();
+		const metadata = this.normalizeMetadata(issue.metadata);
+		const qualityManager = this.normalizeMetadata(metadata.quality_manager);
+		const remediation = this.normalizeMetadata(qualityManager.remediation);
+		const existingStatus = normalizeOptionalString(remediation.status).toLowerCase();
+		const attemptsUsed = Number.isFinite(Number(remediation.attempt_count)) ? Number(remediation.attempt_count) : 0;
+		const maxAttempts = this.config.remediationMaxAttemptsPerIssue;
+		const cooldownMinutes = this.config.remediationCooldownMinutes;
+		const cooldownMs = cooldownMinutes * 60 * 1000;
+		const lastAttemptAt = this.parseDate(remediation.last_attempt_at);
+		if (!force && existingStatus === 'running') {
+			return {
+				status: 'running',
+				reason: 'Remediation already running.',
+				issueId
+			};
+		}
+		if (!force && maxAttempts > 0 && attemptsUsed >= maxAttempts) {
+			return {
+				status: 'skipped',
+				reason: `Remediation attempt budget reached (${attemptsUsed}/${maxAttempts}).`,
+				issueId
+			};
+		}
+		if (!force && existingStatus === 'failed' && lastAttemptAt) {
+			const ageMs = now.getTime() - lastAttemptAt.getTime();
+			if (ageMs >= 0 && ageMs < cooldownMs) {
+				return {
+					status: 'skipped',
+					reason: 'Remediation cooldown active.',
+					issueId,
+					nextAttemptAt: new Date(lastAttemptAt.getTime() + cooldownMs)
+				};
+			}
+		}
+
+		const attemptCount = attemptsUsed + 1;
+		await AiTerminalIssueReports.updateOne(
+			{ _id: issueId },
+			{
+				$set: {
+					'metadata.quality_manager.remediation': {
+						...remediation,
+						status: 'running',
+						source,
+						trigger: normalizeOptionalString(options.trigger) || 'unspecified',
+						attempt_count: attemptCount,
+						max_attempts: maxAttempts,
+						cooldown_minutes: cooldownMinutes,
+						hook: this.config.remediationHookPath,
+						last_attempt_at: now,
+						updated_at: now,
+						last_error: '',
+						last_exit_code: null
+					},
+					updatedAt: now
+				}
+			}
+		);
+
+		const startedAtMs = Date.now();
+		const contextPath = await this.writeRemediationContextFile(issue, {
+			issueId,
+			attemptCount,
+			source,
+			trigger: normalizeOptionalString(options.trigger) || 'unspecified'
+		});
+		let parsedOutput: Record<string, any> | null = null;
+		try {
+			const execResult = await execFileAsync(
+				this.config.remediationHookPath,
+				[contextPath],
+				{
+					timeout: this.config.remediationTimeoutMs,
+					maxBuffer: this.config.remediationMaxBufferBytes,
+					env: {
+						...process.env,
+						RIO_AI_ASSISTANT_REMEDIATION_CONTEXT: contextPath,
+						RIO_AI_ASSISTANT_REMEDIATION_ISSUE_ID: issueId
+					}
+				}
+			);
+			const finishedAt = new Date();
+			const durationMs = Date.now() - startedAtMs;
+			const stdout = this.toSafeOutput(execResult?.stdout);
+			const stderr = this.toSafeOutput(execResult?.stderr);
+			parsedOutput = this.parseOutputJson(stdout);
+			const outputStatus = normalizeOptionalString(parsedOutput?.status).toLowerCase();
+			const success = outputStatus ? ['success', 'ok', 'completed', 'pass'].includes(outputStatus) : true;
+			const remediationStatus = success ? 'success' : 'failed';
+			const remediationReason = normalizeOptionalString(parsedOutput?.reason)
+				|| (success ? 'Remediation hook completed successfully.' : 'Remediation hook reported a non-success status.');
+
+			await AiTerminalIssueReports.updateOne(
+				{ _id: issueId },
+				{
+					$set: {
+						'metadata.quality_manager.remediation': {
+							...remediation,
+							status: remediationStatus,
+							source,
+							trigger: normalizeOptionalString(options.trigger) || 'unspecified',
+							attempt_count: attemptCount,
+							max_attempts: maxAttempts,
+							cooldown_minutes: cooldownMinutes,
+							hook: this.config.remediationHookPath,
+							last_attempt_at: now,
+							last_success_at: success ? finishedAt : remediation.last_success_at,
+							last_error: success ? '' : remediationReason,
+							last_exit_code: success ? 0 : null,
+							last_duration_ms: durationMs,
+							last_stdout: stdout,
+							last_stderr: stderr,
+							last_output: parsedOutput || undefined,
+							updated_at: finishedAt
+						},
+						...(success ? { status: 'triaged' } : {}),
+						updatedAt: finishedAt
+					}
+				}
+			);
+
+				let replayResult: Record<string, any> | null = null;
+				if (success && this.config.autoReplayEnabled) {
+					replayResult = await this.runIssueReplay(issueId, {
+						source,
+						trigger: normalizeOptionalString(options.trigger) || 'post_remediation'
+					});
+				}
+
+				console.log(new Date(), '[AI Assistant Codex Manager] remediation completed', {
+					id_issue_report: issueId,
+					status: remediationStatus,
+					attempt: attemptCount,
+					durationMs,
+					replayStatus: replayResult?.status || null
+				});
+
+				return {
+					status: remediationStatus,
+					reason: remediationReason,
+					issueId,
+					attemptCount,
+					durationMs,
+					replayStatus: replayResult?.status || '',
+					replayReason: replayResult?.reason || ''
+				};
+		}
+		catch (error) {
+			const finishedAt = new Date();
+			const durationMs = Date.now() - startedAtMs;
+			const err: any = error || {};
+			const stdout = this.toSafeOutput(err.stdout);
+			const stderr = this.toSafeOutput(err.stderr);
+			const exitCode = Number.isFinite(Number(err.code)) ? Number(err.code) : null;
+			const timeoutHit = err?.killed === true || normalizeOptionalString(err?.signal).toUpperCase() === 'SIGTERM';
+			const reason = timeoutHit
+				? `Remediation hook timed out after ${this.config.remediationTimeoutMs}ms.`
+				: (normalizeOptionalString(err?.message) || 'Remediation hook failed.');
+			const nextAttemptAt = new Date(finishedAt.getTime() + cooldownMs);
+
+			await AiTerminalIssueReports.updateOne(
+				{ _id: issueId },
+				{
+					$set: {
+						'metadata.quality_manager.remediation': {
+							...remediation,
+							status: 'failed',
+							source,
+							trigger: normalizeOptionalString(options.trigger) || 'unspecified',
+							attempt_count: attemptCount,
+							max_attempts: maxAttempts,
+							cooldown_minutes: cooldownMinutes,
+							hook: this.config.remediationHookPath,
+							last_attempt_at: now,
+							last_error: reason,
+							last_exit_code: exitCode,
+							last_duration_ms: durationMs,
+							last_stdout: stdout,
+							last_stderr: stderr,
+							last_output: parsedOutput || undefined,
+							next_attempt_at: nextAttemptAt,
+							updated_at: finishedAt
+						},
+						updatedAt: finishedAt
+					}
+				}
+			);
+
+			console.error(new Date(), '[AI Assistant Codex Manager] remediation failed', {
+				id_issue_report: issueId,
+				attempt: attemptCount,
+				reason,
+				exitCode,
+				durationMs
+			});
+
+			return {
+				status: 'failed',
+				reason,
+				issueId,
+				attemptCount,
+				durationMs,
+				exitCode
+			};
+		}
+		finally {
+			if (contextPath) {
+				try {
+					await fs.unlink(contextPath);
+				}
+				catch {
+					// Ignore cleanup failures.
+				}
+			}
+		}
+	}
+
+	public async runIssueReplay(
+		issueReportId: string,
+		options: { force?: boolean; source?: 'auto' | 'manual'; trigger?: string } = {}
+	): Promise<Record<string, any>> {
+		const issueId = normalizeOptionalString(issueReportId);
+		if (!issueId) {
+			return {
+				status: 'failed',
+				reason: 'Issue report id is required.'
+			};
+		}
+
+		const source = options.source === 'manual' ? 'manual' : 'auto';
+		if (!this.config.autoReplayEnabled && source !== 'manual') {
+			return {
+				status: 'skipped',
+				reason: 'Auto replay is disabled.'
+			};
+		}
+		if (!this.config.replayCommand) {
+			return {
+				status: 'skipped',
+				reason: 'Replay command is not configured.'
+			};
+		}
+
+		const issue = await AiTerminalIssueReports.findById(issueId) as Record<string, any> | null;
+		if (!issue) {
+			return {
+				status: 'not_found',
+				reason: 'Issue report not found.'
+			};
+		}
+
+		const force = options.force === true;
+		const now = new Date();
+		const metadata = this.normalizeMetadata(issue.metadata);
+		const qualityManager = this.normalizeMetadata(metadata.quality_manager);
+		const replay = this.normalizeMetadata(qualityManager.replay);
+		const existingStatus = normalizeOptionalString(replay.status).toLowerCase();
+		const attemptsUsed = Number.isFinite(Number(replay.attempt_count)) ? Number(replay.attempt_count) : 0;
+		const maxAttempts = this.config.replayMaxAttemptsPerIssue;
+		const cooldownMinutes = this.config.replayCooldownMinutes;
+		const cooldownMs = cooldownMinutes * 60 * 1000;
+		const lastAttemptAt = this.parseDate(replay.last_attempt_at);
+		if (!force && existingStatus === 'running') {
+			return {
+				status: 'running',
+				reason: 'Replay already running.',
+				issueId
+			};
+		}
+		if (!force && existingStatus === 'success') {
+			return {
+				status: 'skipped',
+				reason: 'Replay already succeeded.',
+				issueId
+			};
+		}
+		if (!force && maxAttempts > 0 && attemptsUsed >= maxAttempts) {
+			return {
+				status: 'skipped',
+				reason: `Replay attempt budget reached (${attemptsUsed}/${maxAttempts}).`,
+				issueId
+			};
+		}
+		if (!force && existingStatus === 'failed' && lastAttemptAt) {
+			const ageMs = now.getTime() - lastAttemptAt.getTime();
+			if (ageMs >= 0 && ageMs < cooldownMs) {
+				return {
+					status: 'skipped',
+					reason: 'Replay cooldown active.',
+					issueId,
+					nextAttemptAt: new Date(lastAttemptAt.getTime() + cooldownMs)
+				};
+			}
+		}
+
+		const attemptCount = attemptsUsed + 1;
+		await AiTerminalIssueReports.updateOne(
+			{ _id: issueId },
+			{
+				$set: {
+					'metadata.quality_manager.replay': {
+						...replay,
+						status: 'running',
+						source,
+						trigger: normalizeOptionalString(options.trigger) || 'unspecified',
+						attempt_count: attemptCount,
+						max_attempts: maxAttempts,
+						cooldown_minutes: cooldownMinutes,
+						command: this.config.replayCommand,
+						last_attempt_at: now,
+						updated_at: now,
+						last_error: '',
+						last_exit_code: null
+					},
+					updatedAt: now
+				}
+			}
+		);
+
+		const startedAtMs = Date.now();
+		const contextPath = await this.writeReplayContextFile(issue, {
+			issueId,
+			attemptCount,
+			source,
+			trigger: normalizeOptionalString(options.trigger) || 'unspecified'
+		});
+		let parsedOutput: Record<string, any> | null = null;
+		try {
+			const execResult = await execFileAsync(
+				'/bin/bash',
+				['-lc', this.config.replayCommand],
+				{
+					timeout: this.config.replayTimeoutMs,
+					maxBuffer: this.config.replayMaxBufferBytes,
+					env: {
+						...process.env,
+						RIO_AI_ASSISTANT_REPLAY_CONTEXT: contextPath,
+						RIO_AI_ASSISTANT_REPLAY_ISSUE_ID: issueId
+					}
+				}
+			);
+			const finishedAt = new Date();
+			const durationMs = Date.now() - startedAtMs;
+			const stdout = this.toSafeOutput(execResult?.stdout);
+			const stderr = this.toSafeOutput(execResult?.stderr);
+			parsedOutput = this.parseOutputJson(stdout);
+			const outputStatus = normalizeOptionalString(parsedOutput?.status).toLowerCase();
+			const success = outputStatus ? ['success', 'ok', 'completed', 'pass'].includes(outputStatus) : true;
+			const replayStatus = success ? 'success' : 'failed';
+			const replayReason = normalizeOptionalString(parsedOutput?.reason)
+				|| (success ? 'Replay command completed successfully.' : 'Replay command reported a non-success status.');
+
+			await AiTerminalIssueReports.updateOne(
+				{ _id: issueId },
+				{
+					$set: {
+						'metadata.quality_manager.replay': {
+							...replay,
+							status: replayStatus,
+							source,
+							trigger: normalizeOptionalString(options.trigger) || 'unspecified',
+							attempt_count: attemptCount,
+							max_attempts: maxAttempts,
+							cooldown_minutes: cooldownMinutes,
+							command: this.config.replayCommand,
+							last_attempt_at: now,
+							last_success_at: success ? finishedAt : replay.last_success_at,
+							last_error: success ? '' : replayReason,
+							last_exit_code: success ? 0 : null,
+							last_duration_ms: durationMs,
+							last_stdout: stdout,
+							last_stderr: stderr,
+							last_output: parsedOutput || undefined,
+							updated_at: finishedAt
+						},
+						updatedAt: finishedAt
+					}
+				}
+			);
+
+			console.log(new Date(), '[AI Assistant Codex Manager] replay completed', {
+				id_issue_report: issueId,
+				status: replayStatus,
+				attempt: attemptCount,
+				durationMs
+			});
+
+			return {
+				status: replayStatus,
+				reason: replayReason,
+				issueId,
+				attemptCount,
+				durationMs
+			};
+		}
+		catch (error) {
+			const finishedAt = new Date();
+			const durationMs = Date.now() - startedAtMs;
+			const err: any = error || {};
+			const stdout = this.toSafeOutput(err.stdout);
+			const stderr = this.toSafeOutput(err.stderr);
+			const exitCode = Number.isFinite(Number(err.code)) ? Number(err.code) : null;
+			const timeoutHit = err?.killed === true || normalizeOptionalString(err?.signal).toUpperCase() === 'SIGTERM';
+			const reason = timeoutHit
+				? `Replay command timed out after ${this.config.replayTimeoutMs}ms.`
+				: (normalizeOptionalString(err?.message) || 'Replay command failed.');
+			const nextAttemptAt = new Date(finishedAt.getTime() + cooldownMs);
+
+			await AiTerminalIssueReports.updateOne(
+				{ _id: issueId },
+				{
+					$set: {
+						'metadata.quality_manager.replay': {
+							...replay,
+							status: 'failed',
+							source,
+							trigger: normalizeOptionalString(options.trigger) || 'unspecified',
+							attempt_count: attemptCount,
+							max_attempts: maxAttempts,
+							cooldown_minutes: cooldownMinutes,
+							command: this.config.replayCommand,
+							last_attempt_at: now,
+							last_error: reason,
+							last_exit_code: exitCode,
+							last_duration_ms: durationMs,
+							last_stdout: stdout,
+							last_stderr: stderr,
+							last_output: parsedOutput || undefined,
+							next_attempt_at: nextAttemptAt,
+							updated_at: finishedAt
+						},
+						updatedAt: finishedAt
+					}
+				}
+			);
+
+			console.error(new Date(), '[AI Assistant Codex Manager] replay failed', {
+				id_issue_report: issueId,
+				attempt: attemptCount,
+				reason,
+				exitCode,
+				durationMs
+			});
+
+			return {
+				status: 'failed',
+				reason,
+				issueId,
+				attemptCount,
+				durationMs,
+				exitCode
+			};
+		}
+		finally {
+			if (contextPath) {
+				try {
+					await fs.unlink(contextPath);
+				}
+				catch {
+					// Ignore cleanup failures.
+				}
+			}
+		}
+	}
+
+	private async markMessageQualityReviewed(messageId: string, decision: AiAssistantQualityDecision, issueReportId = ''): Promise<void> {
+		const now = new Date();
+		const qualityPayload: Record<string, any> = {
+			status: decision.status,
+			reason: decision.reason,
+			code: decision.code,
+			checked_at: now,
+			worker_index: normalizeOptionalString(process.env.WORKER_INDEX),
+			worker_instance: normalizeOptionalString(process.env.NODE_APP_INSTANCE)
+		};
+		if (issueReportId) {
+			qualityPayload.id_issue_report = issueReportId;
+		}
+		await AiTerminalMessages.updateOne(
+			{ _id: messageId },
+			{
+				$set: {
+					'metadata.quality_manager': qualityPayload,
+					updatedAt: now
+				}
+			}
+		);
+		if (this.config.debug) {
+			console.log(new Date(), '[AI Assistant Codex Manager] checked message', {
+				id_message: messageId,
+				status: decision.status,
+				code: decision.code,
+				id_issue_report: issueReportId || null
+			});
+		}
+	}
+
+	private async resolveClosestUserPrompt(conversationId: string, assistantCreatedAt?: Date): Promise<string> {
+		const query: Record<string, any> = {
+			id_conversation: conversationId,
+			role: 'user'
+		};
+		const createdAt = assistantCreatedAt ? new Date(assistantCreatedAt) : null;
+		if (createdAt && Number.isFinite(createdAt.getTime())) {
+			query.createdAt = { $lte: createdAt };
+		}
+		const userMessage = await AiTerminalMessages.findOne(query, { sort: { createdAt: -1 } }) as AiTerminalMessageModel | null;
+		return normalizeOptionalString(userMessage?.content);
+	}
+
+	private async writeRemediationContextFile(issue: Record<string, any>, options: Record<string, any>): Promise<string> {
+		const issueId = normalizeOptionalString(issue?._id);
+		const filePath = path.join(os.tmpdir(), `resolveio-ai-assistant-remediation-${issueId}-${Date.now()}.json`);
+		const payload = {
+			created_at: new Date().toISOString(),
+			issue_report: {
+				_id: issueId,
+				status: normalizeOptionalString(issue?.status),
+				source: normalizeOptionalString(issue?.source),
+				reason: normalizeOptionalString(issue?.reason),
+				id_conversation: normalizeOptionalString(issue?.id_conversation),
+				id_message: normalizeOptionalString(issue?.id_message),
+				id_client: normalizeOptionalString(issue?.id_client),
+				id_app: normalizeOptionalString(issue?.id_app),
+				id_user: normalizeOptionalString(issue?.id_user),
+				request_id: normalizeOptionalString(issue?.request_id),
+				user_prompt: normalizeOptionalString(issue?.user_prompt),
+				assistant_content: normalizeOptionalString(issue?.assistant_content)
+			},
+			remediation: {
+				source: normalizeOptionalString(options?.source),
+				trigger: normalizeOptionalString(options?.trigger),
+				attempt_count: Number(options?.attemptCount) || 0,
+				max_attempts: this.config.remediationMaxAttemptsPerIssue,
+				cooldown_minutes: this.config.remediationCooldownMinutes
+			}
+		};
+		await fs.writeFile(filePath, JSON.stringify(payload, null, 2), 'utf8');
+		return filePath;
+	}
+
+	private async writeReplayContextFile(issue: Record<string, any>, options: Record<string, any>): Promise<string> {
+		const issueId = normalizeOptionalString(issue?._id);
+		const metadata = this.normalizeMetadata(issue?.metadata);
+		const qualityManager = this.normalizeMetadata(metadata.quality_manager);
+		const remediation = this.normalizeMetadata(qualityManager.remediation);
+		const filePath = path.join(os.tmpdir(), `resolveio-ai-assistant-replay-${issueId}-${Date.now()}.json`);
+		const payload = {
+			created_at: new Date().toISOString(),
+			issue_report: {
+				_id: issueId,
+				status: normalizeOptionalString(issue?.status),
+				source: normalizeOptionalString(issue?.source),
+				reason: normalizeOptionalString(issue?.reason),
+				id_conversation: normalizeOptionalString(issue?.id_conversation),
+				id_message: normalizeOptionalString(issue?.id_message),
+				id_client: normalizeOptionalString(issue?.id_client),
+				id_app: normalizeOptionalString(issue?.id_app),
+				id_user: normalizeOptionalString(issue?.id_user),
+				request_id: normalizeOptionalString(issue?.request_id),
+				user_prompt: normalizeOptionalString(issue?.user_prompt),
+				assistant_content: normalizeOptionalString(issue?.assistant_content)
+			},
+			remediation: {
+				status: normalizeOptionalString(remediation.status),
+				last_attempt_at: remediation.last_attempt_at || null,
+				last_success_at: remediation.last_success_at || null,
+				last_error: normalizeOptionalString(remediation.last_error),
+				attempt_count: Number(remediation.attempt_count) || 0
+			},
+			replay: {
+				source: normalizeOptionalString(options?.source),
+				trigger: normalizeOptionalString(options?.trigger),
+				attempt_count: Number(options?.attemptCount) || 0,
+				max_attempts: this.config.replayMaxAttemptsPerIssue,
+				cooldown_minutes: this.config.replayCooldownMinutes,
+				command: this.config.replayCommand
+			}
+		};
+		await fs.writeFile(filePath, JSON.stringify(payload, null, 2), 'utf8');
+		return filePath;
+	}
+
+	private parseOutputJson(stdout: string): Record<string, any> | null {
+		const text = normalizeOptionalString(stdout);
+		if (!text) {
+			return null;
+		}
+		const firstChar = text.charAt(0);
+		if (firstChar !== '{' && firstChar !== '[') {
+			return null;
+		}
+		try {
+			const parsed = JSON.parse(text);
+			if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+				return null;
+			}
+			return parsed;
+		}
+		catch {
+			return null;
+		}
+	}
+
+	private toSafeOutput(value: any): string {
+		const asString = typeof value === 'string'
+			? value
+			: (Buffer.isBuffer(value) ? value.toString('utf8') : String(value || ''));
+		const normalized = asString.trim();
+		if (!normalized) {
+			return '';
+		}
+		if (normalized.length <= MAX_STORED_OUTPUT_CHARS) {
+			return normalized;
+		}
+		return normalized.slice(0, MAX_STORED_OUTPUT_CHARS);
+	}
+
+	private parseDate(value: any): Date | null {
+		if (!value) {
+			return null;
+		}
+		const date = value instanceof Date ? value : new Date(value);
+		if (!Number.isFinite(date.getTime())) {
+			return null;
+		}
+		return date;
+	}
+
+	private normalizeMetadata(value: any): Record<string, any> {
+		if (!value || typeof value !== 'object' || Array.isArray(value)) {
+			return {};
+		}
+		return value;
+	}
+}