@resolveio/server-lib 22.3.18 → 22.3.19

package/tests/ai-assistant-corpus-replay-e2e.ts

@@ -0,0 +1,774 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { ResolveIOServer } from '../src/resolveio-server-app';
+import { classifyAssistantRequestType } from '../src/methods/ai-terminal';
+
+type CorpusPromptEntry = {
+	id?: string;
+	prompt?: string;
+	score?: number;
+};
+
+type CorpusFile = {
+	prompts?: CorpusPromptEntry[];
+};
+
+type ReplayResult = {
+	id: string;
+	prompt: string;
+	pass: boolean;
+	errors: string[];
+	conversationId?: string;
+	classification?: {
+		type?: string;
+		dataQuestion?: boolean;
+		source?: string;
+	};
+	tool?: {
+		type?: string;
+		collection?: string;
+		rowCount?: number;
+		total?: number | null;
+		columns?: string[];
+		source?: string;
+	};
+};
+
+type AssistantDirective = {
+	type: 'read' | 'aggregate';
+	payload: Record<string, any>;
+	source: 'metadata.debug.directive' | 'assistant.directive_line';
+};
+
+function parseBoolean(value: any): boolean {
+	return ['true', '1', 'yes', 'y', 'on'].includes(String(value || '').trim().toLowerCase());
+}
+
+function normalizeOptionalString(value: any): string {
+	return String(value ?? '').trim();
+}
+
+function parseInteger(value: any, fallback: number, min: number, max: number): number {
+	const parsed = Number(value);
+	if (!Number.isFinite(parsed)) {
+		return fallback;
+	}
+	return Math.min(Math.max(Math.round(parsed), min), max);
+}
+
+function loadDotEnvFile(filePath: string): Record<string, string> {
+	const parsed: Record<string, string> = {};
+	if (!fs.existsSync(filePath)) {
+		return parsed;
+	}
+	const content = fs.readFileSync(filePath, 'utf8');
+	content.split(/\r?\n/g).forEach((lineRaw) => {
+		const line = lineRaw.trim();
+		if (!line || line.startsWith('#')) {
+			return;
+		}
+		const jsonLike = line.match(/^"([A-Za-z_][A-Za-z0-9_]*)"\s*:\s*(.+?)\s*,?$/);
+		if (jsonLike?.[1]) {
+			const key = jsonLike[1];
+			let value = String(jsonLike[2] || '').trim();
+			if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith('\'') && value.endsWith('\''))) {
+				value = value.slice(1, -1);
+			}
+			if (key) {
+				parsed[key] = value;
+			}
+			return;
+		}
+		const separator = line.indexOf('=');
+		if (separator <= 0) {
+			return;
+		}
+		const key = line.slice(0, separator).trim();
+		let value = line.slice(separator + 1).trim();
+		if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith('\'') && value.endsWith('\''))) {
+			value = value.slice(1, -1);
+		}
+		if (key) {
+			parsed[key] = value;
+		}
+	});
+	return parsed;
+}
+
+function applyEnvFromServerDir(serverDir: string): void {
+	const envFiles = [
+		path.join(serverDir, '.env.codex'),
+		path.join(serverDir, '.env.production'),
+		path.join(serverDir, '.env')
+	];
+	envFiles.forEach((filePath) => {
+		const values = loadDotEnvFile(filePath);
+		Object.keys(values).forEach((key) => {
+			if (!process.env[key]) {
+				process.env[key] = values[key];
+			}
+		});
+	});
+}
+
+async function sleep(ms: number): Promise<void> {
+	return await new Promise(resolve => setTimeout(resolve, ms));
+}
+
+async function pollFinalAssistantMessage(db: any, idConversation: string, timeoutMs = 180000): Promise<any> {
+	const started = Date.now();
+	while (Date.now() - started < timeoutMs) {
+		const message = await db.collection('ai-terminal-messages').findOne(
+			{ id_conversation: idConversation, role: 'assistant' },
+			{ sort: { createdAt: -1 } }
+		);
+		if (message && message?.metadata?.pending !== true) {
+			return message;
+		}
+		await sleep(1200);
+	}
+	throw new Error(`Timed out waiting for final assistant message (${idConversation}).`);
+}
+
+function buildMethodContext(methodManager: any, idUser: string): any {
+	return Object.assign({}, methodManager, Object.getPrototypeOf(methodManager), {
+		id_user: idUser,
+		user: 'AI Corpus Replay',
+		id_ws: 'ai-assistant-corpus-replay-e2e'
+	});
+}
+
+async function callMethodAsUser(methodManager: any, idUser: string, method: string, payload: Record<string, any>): Promise<any> {
+	const ctx = buildMethodContext(methodManager, idUser);
+	return await methodManager.callMethod.call(ctx, method, payload);
+}
+
+async function findSuperAdminUserId(db: any): Promise<string> {
+	const user = await db.collection('users').findOne(
+		{ 'roles.super_admin': true },
+		{ projection: { _id: 1 }, sort: { updatedAt: -1, createdAt: -1 } }
+	);
+	if (!user?._id) {
+		throw new Error('Could not find super admin user.');
+	}
+	return String(user._id);
+}
+
+async function runPrompt(methodManager: any, db: any, idUser: string, prompt: string, appId: string): Promise<{ conversationId: string; message: any }> {
+	const model = String(process.env.AI_ASSISTANT_CODEX_MODEL || process.env.OPENAI_MODEL || '').trim();
+	const payload = {
+		message: prompt,
+		id_app: appId,
+		max_history: 0,
+		config: {
+			...(model ? { model } : {})
+		}
+	};
+	const response = await callMethodAsUser(methodManager, idUser, 'aiCoderTerminalRunCodex', payload);
+	const conversationId = String(response?.conversation?._id || response?.conversation?.id_conversation || '').trim();
+	if (!conversationId) {
+		throw new Error('No conversation id returned from aiCoderTerminalRunCodex.');
+	}
+	const message = await pollFinalAssistantMessage(db, conversationId);
+	return { conversationId, message };
+}
+
+function parseDirectiveLine(content: string): AssistantDirective | null {
+	const lines = String(content || '').split(/\r?\n/g);
+	for (const lineRaw of lines) {
+		const line = lineRaw.trim();
+		if (!line) {
+			continue;
+		}
+		const readPrefix = 'REPORT_BUILDER_READ:';
+		const aggPrefix = 'REPORT_BUILDER_AGG:';
+		let type: AssistantDirective['type'] | null = null;
+		let jsonText = '';
+		if (line.startsWith(readPrefix)) {
+			type = 'read';
+			jsonText = line.slice(readPrefix.length).trim();
+		}
+		else if (line.startsWith(aggPrefix)) {
+			type = 'aggregate';
+			jsonText = line.slice(aggPrefix.length).trim();
+		}
+		if (!type || !jsonText) {
+			continue;
+		}
+		try {
+			const payload = JSON.parse(jsonText);
+			if (!payload || typeof payload !== 'object' || Array.isArray(payload)) {
+				continue;
+			}
+			return {
+				type,
+				payload,
+				source: 'assistant.directive_line'
+			};
+		}
+		catch {
+			continue;
+		}
+	}
+	return null;
+}
+
+function extractDirectiveFromMessage(message: any): AssistantDirective | null {
+	const debugDirective = message?.metadata?.debug?.directive;
+	if (debugDirective && typeof debugDirective === 'object') {
+		const normalizedType = normalizeOptionalString(debugDirective?.type).toLowerCase();
+		if ((normalizedType === 'read' || normalizedType === 'aggregate')
+			&& debugDirective.payload
+			&& typeof debugDirective.payload === 'object'
+			&& !Array.isArray(debugDirective.payload)) {
+			return {
+				type: normalizedType as AssistantDirective['type'],
+				payload: debugDirective.payload,
+				source: 'metadata.debug.directive'
+			};
+		}
+		const rawLineDirective = parseDirectiveLine(normalizeOptionalString(debugDirective?.rawLine));
+		if (rawLineDirective) {
+			return {
+				...rawLineDirective,
+				source: 'metadata.debug.directive'
+			};
+		}
+	}
+	return parseDirectiveLine(normalizeOptionalString(message?.content));
+}
+
+function buildSyntheticToolResultFromDirective(directive: AssistantDirective, methodResponse: any): any {
+	const display = methodResponse?.display && typeof methodResponse.display === 'object'
+		? methodResponse.display
+		: {
+			columns: [] as string[],
+			rows: [] as Array<Record<string, any>>,
+			rowCount: 0,
+			truncated: false
+		};
+	const rowCount = Array.isArray(methodResponse?.documents)
+		? methodResponse.documents.length
+		: Number(display?.rowCount || 0);
+	const total = typeof methodResponse?.total === 'number' ? methodResponse.total : undefined;
+	const collection = normalizeOptionalString(methodResponse?.debug?.collectionResolved)
+		|| normalizeOptionalString(methodResponse?.debug?.collection)
+		|| normalizeOptionalString(directive.payload?.collection);
+	return {
+		type: directive.type === 'aggregate' ? 'mongo_agg' : 'mongo_read',
+		input: directive.payload,
+		output: {
+			display,
+			total,
+			collection: collection || undefined,
+			rowCount,
+			columns: Array.isArray(display?.columns) ? display.columns : [],
+			truncated: display?.truncated === true
+		}
+	};
+}
+
+function toNormalizedKey(value: string): string {
+	return String(value || '').toLowerCase().replace(/[^a-z0-9]+/g, '');
+}
+
+function parseNumericLoose(value: any): number | null {
+	if (typeof value === 'number' && Number.isFinite(value)) {
+		return value;
+	}
+	if (typeof value !== 'string') {
+		return null;
+	}
+	const trimmed = value.trim();
+	if (!trimmed) {
+		return null;
+	}
+	const normalized = trimmed
+		.replace(/\$/g, '')
+		.replace(/,/g, '')
+		.replace(/%/g, '')
+		.replace(/\s+/g, '');
+	if (!normalized || !/^[-+]?\d*\.?\d+$/.test(normalized)) {
+		return null;
+	}
+	const parsed = Number(normalized);
+	return Number.isFinite(parsed) ? parsed : null;
+}
+
+function normalizeCellValue(value: any): any {
+	const numeric = parseNumericLoose(value);
+	if (numeric !== null) {
+		return Number(numeric.toFixed(6));
+	}
+	if (value instanceof Date) {
+		return value.toISOString();
+	}
+	if (typeof value === 'string') {
+		const trimmed = value.trim();
+		const maybeDate = /[0-9]{4}-[0-9]{2}-[0-9]{2}/.test(trimmed) ? Date.parse(trimmed) : Number.NaN;
+		if (Number.isFinite(maybeDate)) {
+			return new Date(maybeDate).toISOString();
+		}
+		return trimmed;
+	}
+	if (value === undefined) {
+		return null;
+	}
+	return value;
+}
+
+function normalizeDisplayRows(display: any): Array<Record<string, any>> {
+	const rows = Array.isArray(display?.rows) ? display.rows : [];
+	return rows.map((row: any) => {
+		const next: Record<string, any> = {};
+		Object.keys(row || {}).forEach((key) => {
+			next[toNormalizedKey(key)] = normalizeCellValue((row as any)[key]);
+		});
+		return next;
+	});
+}
+
+function normalizeDisplayColumns(display: any): string[] {
+	const columns = Array.isArray(display?.columns) ? display.columns : [];
+	return columns.map((column: any) => toNormalizedKey(String(column || ''))).filter(Boolean).sort();
+}
+
+function collectMonthColumns(columns: string[]): string[] {
+	return columns.filter(column => /^\d{4}-\d{2}$/.test(String(column || '').trim()));
+}
+
+function parseMonthlyTotalsFromDisplay(display: any): Map<string, number> | null {
+	if (!display || !Array.isArray(display.columns) || !Array.isArray(display.rows) || !display.rows.length) {
+		return null;
+	}
+	const columns = display.columns.map((column: any) => String(column || '').trim());
+	const monthColumns = collectMonthColumns(columns);
+	if (monthColumns.length) {
+		const firstColumn = columns[0] || '';
+		const normalizedFirst = toNormalizedKey(firstColumn);
+		if (normalizedFirst === 'metric') {
+			const metricRow = display.rows.find((row: any) => {
+				const metricValue = String(row?.[firstColumn] || '').toLowerCase();
+				return metricValue.includes('revenue') || metricValue.includes('total');
+			}) || display.rows[0];
+			const result = new Map<string, number>();
+			monthColumns.forEach((month) => {
+				const numeric = parseNumericLoose(metricRow?.[month]);
+				if (numeric !== null) {
+					result.set(month, numeric);
+				}
+			});
+			return result.size ? result : null;
+		}
+		const result = new Map<string, number>();
+		monthColumns.forEach((month) => {
+			let sum = 0;
+			display.rows.forEach((row: any) => {
+				const numeric = parseNumericLoose(row?.[month]);
+				if (numeric !== null) {
+					sum += numeric;
+				}
+			});
+			result.set(month, sum);
+		});
+		return result;
+	}
+	const normalizedColumns = columns.map(toNormalizedKey);
+	const monthColumnIndex = normalizedColumns.findIndex((column) => column.includes('month'));
+	const totalColumnIndex = normalizedColumns.findIndex((column) => column.includes('revenue') || column.includes('total'));
+	if (monthColumnIndex === -1 || totalColumnIndex === -1) {
+		return null;
+	}
+	const monthColumn = columns[monthColumnIndex];
+	const totalColumn = columns[totalColumnIndex];
+	const result = new Map<string, number>();
+	display.rows.forEach((row: any) => {
+		const monthValue = String(row?.[monthColumn] || '').trim();
+		if (!/^\d{4}-\d{2}$/.test(monthValue)) {
+			return;
+		}
+		const numeric = parseNumericLoose(row?.[totalColumn]);
+		if (numeric === null) {
+			return;
+		}
+		result.set(monthValue, (result.get(monthValue) || 0) + numeric);
+	});
+	return result.size ? result : null;
+}
+
+function parseCustomerMonthTotalsFromDisplay(display: any): Map<string, number> | null {
+	if (!display || !Array.isArray(display.columns) || !Array.isArray(display.rows) || !display.rows.length) {
+		return null;
+	}
+	const columns = display.columns.map((column: any) => String(column || '').trim());
+	const normalizedColumns = columns.map(toNormalizedKey);
+	const monthColumns = collectMonthColumns(columns);
+	const monthTotals = new Map<string, number>();
+
+	if (monthColumns.length && normalizedColumns[0]?.includes('customer')) {
+		display.rows.forEach((row: any) => {
+			monthColumns.forEach((month) => {
+				const total = parseNumericLoose(row?.[month]);
+				if (total === null) {
+					return;
+				}
+				monthTotals.set(month, (monthTotals.get(month) || 0) + total);
+			});
+		});
+		return monthTotals.size ? monthTotals : null;
+	}
+
+	const monthColumnIndex = normalizedColumns.findIndex((column) => column.includes('month'));
+	const totalColumnIndex = normalizedColumns.findIndex((column) => column.includes('revenue') || column.includes('total'));
+	if (monthColumnIndex === -1 || totalColumnIndex === -1) {
+		return null;
+	}
+	const monthColumn = columns[monthColumnIndex];
+	const totalColumn = columns[totalColumnIndex];
+	display.rows.forEach((row: any) => {
+		const month = String(row?.[monthColumn] || '').trim();
+		if (!/^\d{4}-\d{2}$/.test(month)) {
+			return;
+		}
+		const total = parseNumericLoose(row?.[totalColumn]);
+		if (total === null) {
+			return;
+		}
+		monthTotals.set(month, (monthTotals.get(month) || 0) + total);
+	});
+	return monthTotals.size ? monthTotals : null;
+}
+
+function compareNumericMaps(expected: Map<string, number>, actual: Map<string, number>, tolerance = 0.05): string[] {
+	const errors: string[] = [];
+	const keys = new Set<string>([...Array.from(expected.keys()), ...Array.from(actual.keys())]);
+	keys.forEach((key) => {
+		const left = expected.get(key);
+		const right = actual.get(key);
+		if (left === undefined || right === undefined) {
+			errors.push(`missing_or_extra_key:${key}`);
+			return;
+		}
+		if (Math.abs(left - right) > tolerance) {
+			errors.push(`mismatch:${key}:${left}:${right}`);
+		}
+	});
+	return errors;
+}
+
+function compareReplayDisplay(toolDisplay: any, replayDisplay: any, toolRowCount: number, replayRowCount: number): string[] {
+	const errors: string[] = [];
+	if (toolRowCount === 0 && replayRowCount === 0) {
+		return [];
+	}
+	const toolColumns = normalizeDisplayColumns(toolDisplay);
+	const replayColumns = normalizeDisplayColumns(replayDisplay);
+	if (toolRowCount !== replayRowCount) {
+		errors.push(`Replay rowCount mismatch: tool_result=${toolRowCount}, replay=${replayRowCount}.`);
+	}
+	const sourceRows = normalizeDisplayRows(toolDisplay);
+	const replayRows = normalizeDisplayRows(replayDisplay);
+	const stringifySortedRows = (rows: Array<Record<string, any>>) => rows
+		.map(row => JSON.stringify(row, Object.keys(row).sort()))
+		.sort();
+	const sourceSorted = stringifySortedRows(sourceRows);
+	const replaySorted = stringifySortedRows(replayRows);
+	if (sourceSorted.length === replaySorted.length) {
+		let mismatch = false;
+		for (let index = 0; index < sourceSorted.length; index += 1) {
+			if (sourceSorted[index] !== replaySorted[index]) {
+				mismatch = true;
+				break;
+			}
+		}
+		if (!mismatch) {
+			return [];
+		}
+	}
+	const alignRowsToKeys = (rows: Array<Record<string, any>>, keys: string[]) => rows.map((row) => {
+		const next: Record<string, any> = {};
+		keys.forEach((key) => {
+			next[key] = Object.prototype.hasOwnProperty.call(row, key) ? row[key] : null;
+		});
+		return next;
+	});
+	const allRowKeys = Array.from(new Set([
+		...sourceRows.flatMap(row => Object.keys(row || {})),
+		...replayRows.flatMap(row => Object.keys(row || {}))
+	])).sort();
+	if (allRowKeys.length) {
+		const sourceAligned = stringifySortedRows(alignRowsToKeys(sourceRows, allRowKeys));
+		const replayAligned = stringifySortedRows(alignRowsToKeys(replayRows, allRowKeys));
+		if (sourceAligned.length === replayAligned.length) {
+			let alignedMismatch = false;
+			for (let index = 0; index < sourceAligned.length; index += 1) {
+				if (sourceAligned[index] !== replayAligned[index]) {
+					alignedMismatch = true;
+					break;
+				}
+			}
+			if (!alignedMismatch) {
+				return [];
+			}
+		}
+	}
+	const sourceMonthly = parseMonthlyTotalsFromDisplay(toolDisplay);
+	const replayMonthly = parseMonthlyTotalsFromDisplay(replayDisplay);
+	if (sourceMonthly && replayMonthly && !compareNumericMaps(sourceMonthly, replayMonthly, 0.05).length) {
+		return errors;
+	}
+	const sourceCustomerMonthly = parseCustomerMonthTotalsFromDisplay(toolDisplay);
+	const replayCustomerMonthly = parseCustomerMonthTotalsFromDisplay(replayDisplay);
+	if (sourceCustomerMonthly && replayCustomerMonthly && !compareNumericMaps(sourceCustomerMonthly, replayCustomerMonthly, 0.05).length) {
+		return errors;
+	}
+	if (toolColumns.join('|') !== replayColumns.join('|')) {
+		errors.push(`Replay columns mismatch: tool=[${toolColumns.join(',')}], replay=[${replayColumns.join(',')}].`);
+	}
+	errors.push(`Replay display mismatch: tool_result rows=${sourceRows.length}, replay rows=${replayRows.length}.`);
+	return errors;
+}
+
+function containsAssistantInternalError(content: string): boolean {
+	const text = String(content || '').toLowerCase();
+	if (!text) {
+		return false;
+	}
+	return text.includes('i ran into an internal error while preparing that response');
+}
+
+async function runReplayCase(
+	methodManager: any,
+	db: any,
+	idUser: string,
+	appId: string,
+	entry: CorpusPromptEntry,
+	index: number
+): Promise<ReplayResult> {
+	const id = normalizeOptionalString(entry.id) || `corpus_case_${String(index + 1).padStart(3, '0')}`;
+	const prompt = normalizeOptionalString(entry.prompt);
+	const result: ReplayResult = {
+		id,
+		prompt,
+		pass: false,
+		errors: []
+	};
+	if (!prompt) {
+		result.errors.push('Prompt is empty.');
+		return result;
+	}
+	try {
+		const ai = await runPrompt(methodManager, db, idUser, prompt, appId);
+		result.conversationId = ai.conversationId;
+		const message = ai.message || {};
+		const assistantContent = normalizeOptionalString(message?.content);
+		const classification = classifyAssistantRequestType(prompt);
+		result.classification = {
+			type: classification.type,
+			dataQuestion: classification.dataQuestion,
+			source: classification.source
+		};
+		if (containsAssistantInternalError(assistantContent)) {
+			result.errors.push('Assistant returned internal error response.');
+		}
+		let toolResult = message?.metadata?.tool_result;
+		let toolSource = 'metadata.tool_result';
+		if ((!toolResult?.input || !toolResult?.output) && classification.dataQuestion) {
+			const directive = extractDirectiveFromMessage(message);
+			if (!directive) {
+				result.errors.push('No tool_result found for data question and no directive was recoverable.');
+			}
+			else {
+				const replayMethod = directive.type === 'aggregate' ? 'aiAssistantMongoAggregate' : 'aiAssistantMongoRead';
+				try {
+					const directiveReplay = await callMethodAsUser(methodManager, idUser, replayMethod, directive.payload || {});
+					toolResult = buildSyntheticToolResultFromDirective(directive, directiveReplay);
+					toolSource = `synthetic_from_${directive.source}`;
+				}
+				catch (directiveError: any) {
+					result.errors.push(`Directive replay failed: ${String(directiveError?.message || directiveError || 'unknown error')}`);
+				}
+			}
+		}
+		if (toolResult?.input && toolResult?.output) {
+			const replayMethod = toolResult.type === 'mongo_agg' ? 'aiAssistantMongoAggregate' : 'aiAssistantMongoRead';
+			const replay = await callMethodAsUser(methodManager, idUser, replayMethod, toolResult.input || {});
+			const toolRowCount = Number(toolResult?.output?.rowCount || 0);
+			const replayRowCount = Array.isArray(replay?.documents)
+				? replay.documents.length
+				: Number(replay?.display?.rowCount || 0);
+			const replayErrors = compareReplayDisplay(toolResult?.output?.display, replay?.display, toolRowCount, replayRowCount);
+			if (replayErrors.length) {
+				result.errors.push(...replayErrors);
+			}
+			result.tool = {
+				type: normalizeOptionalString(toolResult?.type),
+				collection: normalizeOptionalString(toolResult?.output?.collection) || normalizeOptionalString(toolResult?.input?.collection),
+				rowCount: toolRowCount,
+				total: typeof toolResult?.output?.total === 'number' ? toolResult.output.total : null,
+				columns: Array.isArray(toolResult?.output?.columns) ? toolResult.output.columns : [],
+				source: toolSource
+			};
+		}
+		else if (classification.dataQuestion) {
+			result.errors.push('Data question did not produce a usable tool_result.');
+		}
+		result.pass = result.errors.length === 0;
+		return result;
+	}
+	catch (error: any) {
+		result.errors.push(String(error?.message || error || 'Unknown error'));
+		result.pass = false;
+		return result;
+	}
+}
+
+async function run(): Promise<void> {
+	const repoRoot = process.cwd();
+	const defaultServerDir = path.resolve(repoRoot, '../resolveio-all/geochem/server');
+	const serverDir = normalizeOptionalString(process.env.AI_ASSISTANT_CORPUS_SERVER_DIR) || defaultServerDir;
+	const settingsPath = path.join(serverDir, normalizeOptionalString(process.env.AI_ASSISTANT_CORPUS_SETTINGS) || 'settings.local.json');
+	const appId = normalizeOptionalString(process.env.AI_ASSISTANT_CORPUS_APP_ID) || path.basename(path.dirname(serverDir));
+	const corpusPath = path.join(repoRoot, normalizeOptionalString(process.env.AI_ASSISTANT_CORPUS_PATH) || 'tmp/ai-assistant-query-corpus.json');
+	const outputPath = path.join(repoRoot, normalizeOptionalString(process.env.AI_ASSISTANT_CORPUS_REPLAY_OUTPUT) || 'tmp/ai-assistant-corpus-replay-e2e-output.json');
+	const runLimit = parseInteger(process.env.AI_ASSISTANT_CORPUS_RUN_LIMIT, 40, 1, 500);
+	const dryRun = parseBoolean(process.env.AI_ASSISTANT_CORPUS_DRY_RUN);
+	const caseFilter = normalizeOptionalString(process.env.AI_ASSISTANT_CORPUS_CASES)
+		.split(',')
+		.map(value => value.trim())
+		.filter(Boolean);
+
+	if (!fs.existsSync(settingsPath)) {
+		throw new Error(`Settings file not found: ${settingsPath}`);
+	}
+	if (!fs.existsSync(corpusPath)) {
+		throw new Error(`Corpus file not found: ${corpusPath}`);
+	}
+
+	const corpus = JSON.parse(fs.readFileSync(corpusPath, 'utf8')) as CorpusFile;
+	const prompts = Array.isArray(corpus?.prompts) ? corpus.prompts : [];
+	const selected = prompts
+		.filter((entry) => normalizeOptionalString(entry?.prompt))
+		.filter((entry) => {
+			if (!caseFilter.length) {
+				return true;
+			}
+			const id = normalizeOptionalString(entry?.id);
+			return !!id && caseFilter.includes(id);
+		})
+		.sort((a, b) => Number(b?.score || 0) - Number(a?.score || 0))
+		.slice(0, runLimit);
+
+	if (!selected.length) {
+		throw new Error('No corpus prompts selected.');
+	}
+
+	if (dryRun) {
+		console.log(JSON.stringify({
+			mode: 'dry-run',
+			serverDir,
+			settingsPath,
+			appId,
+			corpusPath,
+			outputPath,
+			runLimit,
+			selected: selected.map(entry => normalizeOptionalString(entry.id) || normalizeOptionalString(entry.prompt))
+		}, null, 2));
+		return;
+	}
+
+	applyEnvFromServerDir(serverDir);
+	process.env.AI_ASSISTANT_WORKSPACE_ROOT = process.env.AI_ASSISTANT_WORKSPACE_ROOT || path.resolve(serverDir, '../..');
+	if (!process.env.OPENAI_API_KEY) {
+		throw new Error('OPENAI_API_KEY is required for corpus replay E2E run.');
+	}
+
+	const runtimeTmpDir = path.join(serverDir, 'tmp');
+	const runtimeSrcDir = path.join(serverDir, 'src');
+	const runtimeDir = fs.existsSync(path.join(runtimeTmpDir, 'server-app.js')) ? runtimeTmpDir : runtimeSrcDir;
+	const clientRoutesPath = path.join(runtimeDir, 'client-routes');
+	if (!fs.existsSync(clientRoutesPath) && !fs.existsSync(`${clientRoutesPath}.js`)) {
+		throw new Error(`client-routes module not found under runtime dir: ${runtimeDir}`);
+	}
+	const { CLIENT_ROUTES } = require(clientRoutesPath);
+	const serverConfig = JSON.parse(fs.readFileSync(settingsPath, 'utf8'));
+	const envDatabase = normalizeOptionalString(process.env.DATABASE);
+	const envMongoUrl = normalizeOptionalString(process.env.MONGO_URL);
+	const envOplogUrl = normalizeOptionalString(process.env.OPLOG_URL);
+	if (envDatabase) {
+		serverConfig.DATABASE = envDatabase;
+	}
+	if (envMongoUrl) {
+		serverConfig.MONGO_URL = envMongoUrl;
+	}
+	if (envOplogUrl) {
+		serverConfig.OPLOG_URL = envOplogUrl;
+	}
+
+	const codexHome = process.env.CODEX_HOME || path.join(repoRoot, 'tmp', '.codex-ai-corpus');
+	const codexSessions = path.join(codexHome, 'sessions');
+	const codexZdotdir = process.env.AI_DASHBOARD_CODEX_ZDOTDIR || path.join(repoRoot, 'tmp', '.codex-zdotdir-corpus');
+	fs.mkdirSync(codexSessions, { recursive: true });
+	fs.mkdirSync(codexZdotdir, { recursive: true });
+	process.env.CODEX_HOME = codexHome;
+	process.env.AI_DASHBOARD_CODEX_ZDOTDIR = codexZdotdir;
+	process.env.IS_WORKERS_ENABLED = 'false';
+	process.env.IS_WORKER_INSTANCE = 'false';
+	process.env.WORKER_INDEX = '';
+	process.env.NODE_APP_INSTANCE = process.env.NODE_APP_INSTANCE || '8';
+	process.env.AI_ASSISTANT_WORKER_DEBUG = process.env.AI_ASSISTANT_WORKER_DEBUG || 'false';
+
+	await ResolveIOServer.create(serverConfig, CLIENT_ROUTES(), appId.toUpperCase(), runtimeDir, false, false);
+	const mainServer = ResolveIOServer.getMainServer();
+	const methodManager = mainServer?.getMethodManager();
+	if (!methodManager) {
+		throw new Error('MethodManager unavailable after ResolveIOServer initialization.');
+	}
+	await methodManager.waitUntilReady(120000);
+
+	const dbName = normalizeOptionalString(serverConfig.DATABASE) || 'resolveio';
+	const db = ResolveIOServer.getMongoConnection().db(dbName);
+	const idUser = normalizeOptionalString(process.env.AI_ASSISTANT_CORPUS_USER_ID) || await findSuperAdminUserId(db);
+	const now = new Date();
+
+	const results: ReplayResult[] = [];
+	for (let index = 0; index < selected.length; index += 1) {
+		const entry = selected[index];
+		const replayResult = await runReplayCase(methodManager, db, idUser, appId, entry, index);
+		results.push(replayResult);
+		const status = replayResult.pass ? 'PASS' : 'FAIL';
+		const error = replayResult.errors.length ? `error=${replayResult.errors[0]}` : '';
+		console.log(`${status} ${replayResult.id} ${error}`.trim());
+	}
+
+	const passed = results.filter((result) => result.pass).length;
+	const failed = results.length - passed;
+	const output = {
+		now: now.toISOString(),
+		serverDir,
+		settingsPath,
+		appId,
+		database: dbName,
+		corpusPath,
+		total: results.length,
+		passed,
+		failed,
+		results
+	};
+	fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+	fs.writeFileSync(outputPath, JSON.stringify(output, null, 2));
+	console.log(`ai assistant corpus replay e2e: ${passed}/${results.length} passed`);
+	console.log(`wrote ${outputPath}`);
+	if (failed > 0) {
+		process.exitCode = 1;
+	}
+}
+
+run()
+	.then(() => {
+		process.exit(process.exitCode ?? 0);
+	})
+	.catch((error) => {
+		console.error(error);
+		process.exit(1);
+	});