@resolveio/server-lib 22.2.33 → 22.2.34

package/tests/ai-assistant-openai-e2e.ts

@@ -0,0 +1,1061 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { ResolveIOServer } from '../src/resolveio-server-app';
+
+type E2EUserProfile =
+	| 'super_admin'
+	| 'internal_invoice'
+	| 'internal_no_invoice'
+	| 'internal_blend_plural'
+	| 'internal_misc_invoice'
+	| 'customer_invoice'
+	| 'customer_no_invoice';
+
+type E2ECase = {
+	id: string;
+	prompt: string;
+	userProfile?: E2EUserProfile;
+	expect: {
+		hasTable?: boolean;
+		revenueMetricGrid?: boolean;
+		revenueCustomerGrid?: boolean;
+		timeGrain?: 'day' | 'week' | 'month' | 'quarter' | 'year';
+		dataResponse?: boolean;
+		requestType?: 'data' | 'navigation' | 'feature_info' | 'bug_issue' | 'mixed' | 'unknown';
+		dataQuestion?: boolean;
+		directiveType?: 'read' | 'aggregate';
+		fastPathType?: 'navigation' | 'change_history';
+		fastPathGeneric?: boolean;
+		allowFastPathMiss?: boolean;
+		mustIncludeRoute?: string;
+		mustNotIncludeRoute?: string;
+		mustIncludeAnyRoute?: string[];
+		mustNotIncludeAnyRoute?: string[];
+		mustContainAnyText?: string[];
+		mustNotContainAnyText?: string[];
+	};
+};
+
+type E2EResult = {
+	id: string;
+	attempt?: number;
+	prompt: string;
+	pass: boolean;
+	error?: string;
+	responsePreview?: string;
+	conversationId?: string;
+	classification?: {
+		type?: string;
+		dataQuestion?: boolean;
+		source?: string;
+	};
+	fastPath?: {
+		type?: string;
+		reason?: string;
+		generic?: boolean;
+		commitCount?: number;
+	};
+	userProfile?: E2EUserProfile;
+	tableHeader?: string[];
+};
+
+const E2E_PROMPTS: E2ECase[] = [
+	{
+		id: 'navigation_invoice_route_link_fast_path',
+		prompt: 'I just need the route link for invoice list.',
+		userProfile: 'super_admin',
+		expect: { fastPathType: 'navigation', mustIncludeRoute: '/invoice/list' }
+	},
+	{
+		id: 'navigation_invoice_route_link_customer_no_invoice',
+		prompt: 'I just need the route link for invoice list.',
+		userProfile: 'customer_no_invoice',
+		expect: { fastPathType: 'navigation', mustNotIncludeRoute: '/invoice/' }
+	},
+	{
+		id: 'navigation_blend_route_link_internal_blend_plural',
+		prompt: 'I just need the route link for chemical blend list.',
+		userProfile: 'internal_blend_plural',
+		expect: {
+			fastPathType: 'navigation',
+			mustIncludeAnyRoute: ['/chemical-blend/list', '/chemical-blends/list'],
+			mustNotIncludeRoute: '/invoice/'
+		}
+	},
+	{
+		id: 'navigation_blend_route_link_internal_no_blend',
+		prompt: 'I just need the route link for chemical blend list.',
+		userProfile: 'internal_no_invoice',
+		expect: {
+			fastPathType: 'navigation',
+			mustNotIncludeAnyRoute: ['/chemical-blend/', '/chemical-blends/']
+		}
+	},
+	{
+		id: 'navigation_order_alias_route_link_internal_no_invoice',
+		prompt: 'I just need the route link for /order/list.',
+		userProfile: 'internal_no_invoice',
+		expect: {
+			fastPathType: 'navigation',
+			mustIncludeAnyRoute: ['/order/list', '/work-order/list']
+		}
+	},
+	{
+		id: 'wo_last20_this_week_group_status',
+		prompt: 'Show me the last 20 work orders created this week, grouped by status.',
+		expect: { hasTable: true, dataResponse: true }
+	},
+	{
+		id: 'wo_last20_this_week_by_status_alias',
+		prompt: 'Show me the last 20 work orders created this week by status.',
+		expect: { dataResponse: true, directiveType: 'aggregate' }
+	},
+	{
+		id: 'wo_completed_per_day_last_30d',
+		prompt: 'For the last 30 days, how many work orders were completed per day?',
+		expect: { hasTable: true, dataResponse: true }
+	},
+	{
+		id: 'wo_top_customers_last_6m',
+		prompt: 'List the top 10 customers by number of work orders in the last 6 months.',
+		expect: { hasTable: true, dataResponse: true }
+	},
+	{
+		id: 'wo_this_week_customer_summary',
+		prompt: 'Summarize the work orders this week by customer',
+		expect: {
+			hasTable: true,
+			dataResponse: true,
+			mustNotContainAnyText: [
+				"i couldn't run the data query",
+				'positional operator in the middle of a path'
+			]
+		}
+	},
+	{
+		id: 'wo_top_customers_last_6m_internal_no_invoice',
+		prompt: 'List the top 10 customers by number of work orders in the last 6 months.',
+		userProfile: 'internal_no_invoice',
+		expect: { dataResponse: true, mustNotContainAnyText: ['does not have permission', 'access denied'] }
+	},
+	{
+		id: 'wo_open_missing_required_fields',
+		prompt: 'Show open work orders with missing required fields (customer, location, or scheduled date).',
+		expect: { hasTable: true, dataResponse: true }
+	},
+	{
+		id: 'blend_last10_summary',
+		prompt: 'For blending: summarize the last 10 blend tickets with product, total volume, and created date.',
+		expect: { hasTable: true, dataResponse: true }
+	},
+	{
+		id: 'blend_used_red_ultra_lift_last_30d',
+		prompt: 'Which blends used Chemical Red Ultra Lift in the last 30 days? Include lot/batch and volume.',
+		expect: { hasTable: true, dataResponse: true }
+	},
+	{
+		id: 'blend_throughput_by_day_last_2w',
+		prompt: 'Show blend throughput by day for the last 2 weeks.',
+		expect: { hasTable: true, dataResponse: true }
+	},
+	{
+		id: 'blend_throughput_by_day_last_2w_internal_blend_plural',
+		prompt: 'Show blend throughput by day for the last 2 weeks.',
+		userProfile: 'internal_blend_plural',
+		expect: { dataResponse: true, mustNotContainAnyText: ['does not have permission', 'access denied'] }
+	},
+	{
+		id: 'blend_throughput_by_day_last_2w_internal_no_blend',
+		prompt: 'Show blend throughput by day for the last 2 weeks.',
+		userProfile: 'internal_no_invoice',
+		expect: { mustContainAnyText: ['does not have permission', 'access denied', 'ask an admin'] }
+	},
+	{
+		id: 'invoice_top_customers_total_last_6m',
+		prompt: 'Show top 10 customers by invoice total in the last 6 months.',
+		expect: { hasTable: true, dataResponse: true }
+	},
+	{
+		id: 'invoice_top_customers_total_last_6m_internal_invoice',
+		prompt: 'Show top 10 customers by invoice total in the last 6 months.',
+		userProfile: 'internal_invoice',
+		expect: { dataResponse: true, mustNotContainAnyText: ['does not have permission', 'access denied'] }
+	},
+	{
+		id: 'invoice_top_customers_total_last_6m_internal_misc_invoice',
+		prompt: 'Show top 10 customers by invoice total in the last 6 months.',
+		userProfile: 'internal_misc_invoice',
+		expect: { dataResponse: true, mustNotContainAnyText: ['does not have permission', 'access denied'] }
+	},
+	{
+		id: 'invoice_top_customers_total_last_6m_internal_no_invoice_denied',
+		prompt: 'Show top 10 customers by invoice total in the last 6 months.',
+		userProfile: 'internal_no_invoice',
+		expect: { mustContainAnyText: ['does not have permission', 'access denied', 'ask an admin'] }
+	},
+	{
+		id: 'invoice_top_customers_total_last_6m_customer_invoice',
+		prompt: 'Show top 10 customers by invoice total in the last 6 months.',
+		userProfile: 'customer_invoice',
+		expect: { dataResponse: true, mustNotContainAnyText: ['does not have permission', 'access denied'] }
+	},
+	{
+		id: 'invoice_top_customers_total_last_6m_customer_no_invoice_denied',
+		prompt: 'Show top 10 customers by invoice total in the last 6 months.',
+		userProfile: 'customer_no_invoice',
+		expect: { mustContainAnyText: ['does not have permission', 'access denied', 'customer portal only', 'not available in this view', 'scope'] }
+	},
+	{
+		id: 'invoice_revenue_monthly_grid_last_6m',
+		prompt: 'Break down my total revenue over the last 6 month by month',
+		expect: { hasTable: true, dataResponse: true, revenueMetricGrid: true }
+	},
+	{
+		id: 'invoice_revenue_monthly_customer_grid_last_6m',
+		prompt: 'Break down my total revenue over the last 6 month by month by each customer',
+		expect: { hasTable: true, dataResponse: true, revenueCustomerGrid: true, directiveType: 'aggregate' }
+	},
+	{
+		id: 'invoice_revenue_monthly_customer_grid_last_6m_per_customer',
+		prompt: 'Break down my total revenue over the last 6 month per month per customer',
+		expect: { hasTable: true, dataResponse: true, revenueCustomerGrid: true, directiveType: 'aggregate' }
+	},
+	{
+		id: 'invoice_revenue_weekly_grid_last_12w',
+		prompt: 'Break down my total revenue over the last 12 weeks by week',
+		expect: { hasTable: true, dataResponse: true, timeGrain: 'week' }
+	},
+	{
+		id: 'invoice_revenue_daily_grid_last_30d',
+		prompt: 'Break down my total revenue over the last 30 days by day',
+		expect: { hasTable: true, dataResponse: true, timeGrain: 'day' }
+	},
+	{
+		id: 'invoice_revenue_quarterly_grid_last_8q',
+		prompt: 'Break down my total revenue over the last 8 quarters by quarter',
+		expect: { hasTable: true, dataResponse: true, timeGrain: 'quarter' }
+	},
+	{
+		id: 'invoice_revenue_yearly_grid_last_3y',
+		prompt: 'Break down my total revenue over the last 3 years by year',
+		expect: { hasTable: true, dataResponse: true, timeGrain: 'year' }
+	},
+	{
+		id: 'invoice_missing_invoiced_date_this_month',
+		prompt: 'List invoices missing an invoiced date but created this month.',
+		expect: { hasTable: true, dataResponse: true }
+	},
+	{
+		id: 'deliveries_per_driver_last_month',
+		prompt: 'Break down the number of deliveries per driver last month',
+		expect: { hasTable: true, dataResponse: true }
+	},
+	{
+		id: 'deliveries_per_driver_last_30d_top10',
+		prompt: 'Deliveries by driver for the last 30 days (top 10 drivers).',
+		expect: { hasTable: true, dataResponse: true, directiveType: 'aggregate' }
+	},
+	{
+		id: 'support_ticket_billable_hours_per_user_per_month_last_6m',
+		prompt: 'Break up the support tickets last 6 months, give me a total per user per month of the amount of hours they are billing for',
+		expect: { dataResponse: true, directiveType: 'aggregate' }
+	},
+	{
+		id: 'last20_deliveries_product_volume_destination',
+		prompt: 'List the last 20 deliveries with product, volume, and destination.',
+		expect: { hasTable: true, dataResponse: true }
+	},
+	{
+		id: 'jobs_highest_treatment_volume_last_90d',
+		prompt: 'Which jobs have the highest treatment volume in the last 90 days?',
+		expect: { hasTable: true, dataResponse: true }
+	},
+	{
+		id: 'change_history_recent_invoice_updates',
+		prompt: 'What changed recently in invoices? Keep it high-level and no code.',
+		userProfile: 'super_admin',
+		expect: { fastPathType: 'change_history', fastPathGeneric: false }
+	},
+	{
+		id: 'change_history_invoice_document_last_updated',
+		prompt: 'When was the last time the invoice document was updated?',
+		userProfile: 'super_admin',
+		expect: { fastPathType: 'change_history' }
+	},
+	{
+		id: 'change_history_dropdown_added_to_page',
+		prompt: 'When does this dropdown for XYZ get added to this page? What does it do?',
+		userProfile: 'super_admin',
+		expect: { fastPathType: 'change_history' }
+	},
+	{
+		id: 'change_history_this_page_updates',
+		prompt: 'What changed on this page?',
+		userProfile: 'super_admin',
+		expect: { fastPathType: 'change_history' }
+	},
+	{
+		id: 'change_history_invoice_page_updates',
+		prompt: 'What changed on the invoice page?',
+		userProfile: 'super_admin',
+		expect: { fastPathType: 'change_history' }
+	},
+	{
+		id: 'change_history_last_update_did_what',
+		prompt: 'What did the last update just do?',
+		userProfile: 'super_admin',
+		expect: { fastPathType: 'change_history' }
+	},
+	{
+		id: 'change_history_last_system_update_did_what',
+		prompt: 'What did the last system update do?',
+		userProfile: 'super_admin',
+		expect: { fastPathType: 'change_history', allowFastPathMiss: true }
+	},
+	{
+		id: 'change_history_that_last_system_update_did_what',
+		prompt: 'What did that last system update do?',
+		userProfile: 'super_admin',
+		expect: { fastPathType: 'change_history', allowFastPathMiss: true }
+	},
+	{
+		id: 'change_history_recent_invoice_updates_customer_invoice',
+		prompt: 'What changed recently in invoices? Keep it high-level and no code.',
+		userProfile: 'customer_invoice',
+		expect: { fastPathType: 'change_history', fastPathGeneric: false }
+	},
+	{
+		id: 'change_history_recent_invoice_updates_internal_invoice',
+		prompt: 'What changed recently in invoices? Keep it high-level and no code.',
+		userProfile: 'internal_invoice',
+		expect: { fastPathType: 'change_history', fastPathGeneric: false }
+	},
+	{
+		id: 'change_history_recent_invoice_updates_internal_misc_invoice',
+		prompt: 'What changed recently in invoices? Keep it high-level and no code.',
+		userProfile: 'internal_misc_invoice',
+		expect: { fastPathType: 'change_history', fastPathGeneric: false }
+	},
+	{
+		id: 'change_history_duration_work_orders',
+		prompt: 'How long has the current work order behavior been this way?',
+		userProfile: 'internal_no_invoice',
+		expect: { fastPathType: 'change_history' }
+	},
+	{
+		id: 'change_history_exception_list_been_around',
+		prompt: 'What is this new exception list? How long has it been around?',
+		userProfile: 'super_admin',
+		expect: { fastPathType: 'change_history' }
+	},
+	{
+		id: 'change_history_exception_list_on_this_page_duration',
+		prompt: 'How long has the exception list been on this page?',
+		userProfile: 'super_admin',
+		expect: { fastPathType: 'change_history' }
+	},
+	{
+		id: 'change_history_recent_invoice_updates_no_access',
+		prompt: 'What changed recently in invoices? Keep it high-level and no code.',
+		userProfile: 'customer_no_invoice',
+		expect: { fastPathType: 'change_history', fastPathGeneric: true }
+	},
+	{
+		id: 'change_history_recent_invoice_updates_internal_no_invoice',
+		prompt: 'What changed recently in invoices? Keep it high-level and no code.',
+		userProfile: 'internal_no_invoice',
+		expect: { fastPathType: 'change_history', fastPathGeneric: true }
+	},
+	{
+		id: 'bug_issue_missing_asset_in_dropdown',
+		prompt: "Why don't I see this asset in this dropdown?",
+		expect: { requestType: 'bug_issue', dataQuestion: false }
+	},
+	{
+		id: 'bug_issue_work_order_close_greyed_out',
+		prompt: "Why won't this work order let me close it, it's greyed out?",
+		expect: { requestType: 'bug_issue', dataQuestion: false }
+	},
+	{
+		id: 'bug_issue_submit_button_red',
+		prompt: 'The submit button is red and I filled out everything. What is likely wrong?',
+		expect: { requestType: 'bug_issue', dataQuestion: false }
+	},
+	{
+		id: 'last20_completed_wo_customer_product_delivered_returned',
+		prompt: 'Break down the last 20 completed work orders and tell me what customer, what product and how much we delivered or returned',
+		expect: { hasTable: true, dataResponse: true }
+	}
+];
+
+function parseDotEnvFile(filePath: string): Record<string, string> {
+	const parsed: Record<string, string> = {};
+	if (!fs.existsSync(filePath)) {
+		return parsed;
+	}
+	const content = fs.readFileSync(filePath, 'utf8');
+	content.split(/\r?\n/g).forEach((lineRaw) => {
+		const line = lineRaw.trim();
+		if (!line || line.startsWith('#')) {
+			return;
+		}
+		const index = line.indexOf('=');
+		if (index <= 0) {
+			return;
+		}
+		const key = line.slice(0, index).trim();
+		let value = line.slice(index + 1).trim();
+		if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith('\'') && value.endsWith('\''))) {
+			value = value.slice(1, -1);
+		}
+		if (key) {
+			parsed[key] = value;
+		}
+	});
+	return parsed;
+}
+
+function applyEnvFromDotEnv(filePath: string): void {
+	const parsed = parseDotEnvFile(filePath);
+	Object.keys(parsed).forEach((key) => {
+		if (!process.env[key]) {
+			process.env[key] = parsed[key];
+		}
+	});
+}
+
+function findMarkdownTableHeader(content: string): string[] {
+	const lines = String(content || '').split('\n');
+	for (let index = 0; index < lines.length - 1; index += 1) {
+		const header = lines[index]?.trim();
+		const separator = lines[index + 1]?.trim();
+		if (!header || !separator) {
+			continue;
+		}
+		if (!/^\|.+\|$/.test(header)) {
+			continue;
+		}
+		if (!/^\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)+\|?\s*$/.test(separator)) {
+			continue;
+		}
+		return header
+			.split('|')
+			.map(value => value.trim())
+			.filter(Boolean);
+	}
+	return [];
+}
+
+function isMonthHeader(value: string): boolean {
+	return /^\d{4}-\d{2}$/.test(String(value || '').trim());
+}
+
+function isDayHeader(value: string): boolean {
+	return /^\d{4}-\d{2}-\d{2}$/.test(String(value || '').trim());
+}
+
+function isWeekHeader(value: string): boolean {
+	const normalized = String(value || '').trim();
+	if (!normalized) {
+		return false;
+	}
+	return /^\d{4}-w\d{1,2}$/i.test(normalized)
+		|| /^w\d{1,2}\s+\d{4}$/i.test(normalized)
+		|| /^\d{4}-\d{2}-\d{2}\s+to\s+\d{4}-\d{2}-\d{2}$/i.test(normalized);
+}
+
+function isQuarterHeader(value: string): boolean {
+	const normalized = String(value || '').trim();
+	if (!normalized) {
+		return false;
+	}
+	return /^\d{4}-q[1-4]$/i.test(normalized)
+		|| /^q[1-4]\s+\d{4}$/i.test(normalized)
+		|| /^\d{4}\s+q[1-4]$/i.test(normalized);
+}
+
+function isYearHeader(value: string): boolean {
+	return /^\d{4}$/.test(String(value || '').trim());
+}
+
+function assertRevenueMetricGrid(header: string[]): string | null {
+	if (!header.length) {
+		return 'Expected markdown table header for monthly revenue grid.';
+	}
+	if (String(header[0] || '').toLowerCase() !== 'metric') {
+		return `Expected first column to be Metric, got ${header[0] || 'empty'}.`;
+	}
+	const monthColumns = header.slice(1).filter(isMonthHeader);
+	if (monthColumns.length < 3) {
+		return `Expected at least 3 month columns, got ${monthColumns.length}.`;
+	}
+	return null;
+}
+
+function assertRevenueCustomerGrid(header: string[]): string | null {
+	if (!header.length) {
+		return 'Expected markdown table header for customer monthly revenue grid.';
+	}
+	if (!String(header[0] || '').toLowerCase().includes('customer')) {
+		return `Expected first column to include Customer, got ${header[0] || 'empty'}.`;
+	}
+	const monthColumns = header.slice(1).filter(isMonthHeader);
+	if (monthColumns.length < 3) {
+		return `Expected at least 3 month columns, got ${monthColumns.length}.`;
+	}
+	return null;
+}
+
+function assertTimeGrainGrid(header: string[], grain: 'day' | 'week' | 'month' | 'quarter' | 'year'): string | null {
+	if (!header.length) {
+		return `Expected markdown table header for ${grain} report grid.`;
+	}
+	const first = String(header[0] || '').toLowerCase();
+	const lowered = header.map(value => String(value || '').toLowerCase());
+	const buckets = header.slice(1);
+	if (grain === 'day') {
+		if (lowered.some(column => column.includes('day') || column === 'date') || buckets.some(isDayHeader)) {
+			return null;
+		}
+		return `Expected day/date header for daily report, got ${header.join(', ')}.`;
+	}
+	if (grain === 'week') {
+		if (lowered.some(column => column.includes('week')) || buckets.some(isWeekHeader)) {
+			return null;
+		}
+		return `Expected week header for weekly report, got ${header.join(', ')}.`;
+	}
+	if (grain === 'month') {
+		if (lowered.some(column => column.includes('month')) || buckets.some(isMonthHeader)) {
+			return null;
+		}
+		return `Expected month header for monthly report, got ${header.join(', ')}.`;
+	}
+	if (grain === 'quarter') {
+		if (lowered.some(column => column.includes('quarter')) || buckets.some(isQuarterHeader)) {
+			return null;
+		}
+		return `Expected quarter header for quarterly report, got ${header.join(', ')}.`;
+	}
+	if (lowered.some(column => column.includes('year')) || buckets.some(isYearHeader)) {
+		return null;
+	}
+	return `Expected year header for yearly report, got ${header.join(', ')}.`;
+}
+
+async function sleep(ms: number): Promise<void> {
+	// eslint-disable-next-line no-restricted-syntax
+	return await new Promise(resolve => setTimeout(resolve, ms));
+}
+
+async function findSuperAdminUserId(db: any): Promise<string> {
+	const user = await db.collection('users').findOne(
+		{ 'roles.super_admin': true },
+		{ projection: { _id: 1, fullname: 1 }, sort: { updatedAt: -1, createdAt: -1 } }
+	);
+	if (!user?._id) {
+		throw new Error('Could not find super admin user in resolveio.users.');
+	}
+	return String(user._id);
+}
+
+async function upsertHarnessUser(db: any, params: {
+	id: string;
+	username: string;
+	fullname: string;
+	roles: Record<string, any>;
+	other?: Record<string, any>;
+}): Promise<string> {
+	const userId = String(params.id || '').trim();
+	if (!userId) {
+		throw new Error(`Harness user id is required for ${params.username}.`);
+	}
+	const now = new Date();
+	const users = db.collection('users');
+	let existing = await users.findOne(
+		{ username: params.username },
+		{ projection: { _id: 1 } }
+	);
+	if (existing?._id && String(existing._id) !== userId) {
+		await users.deleteOne({ _id: existing._id });
+		existing = null;
+	}
+	const setPayload = {
+		_id: userId,
+		username: params.username,
+		fullname: params.fullname,
+		email: `${params.username}@example.invalid`,
+		active: true,
+		roles: params.roles,
+		other: params.other || {},
+		salt: 'ai-e2e',
+		hash: 'ai-e2e',
+		attempts: 0,
+		updatedAt: now
+	};
+	if (existing?._id && String(existing._id) === userId) {
+		await users.updateOne(
+			{ _id: userId },
+			{ $set: setPayload }
+		);
+		return userId;
+	}
+	await users.updateOne(
+		{ _id: userId },
+		{
+			$set: setPayload,
+			$setOnInsert: {
+				createdAt: now
+			}
+		},
+		{ upsert: true }
+	);
+	return userId;
+}
+
+async function ensureHarnessUsers(db: any, superAdminId: string): Promise<Record<E2EUserProfile, string>> {
+	const users: Record<E2EUserProfile, string> = {
+		super_admin: superAdminId,
+		internal_invoice: '',
+		internal_no_invoice: '',
+		internal_blend_plural: '',
+		internal_misc_invoice: '',
+		customer_invoice: '',
+		customer_no_invoice: ''
+	};
+	users.internal_invoice = await upsertHarnessUser(db, {
+		id: 'ai-e2e-internal-invoice',
+		username: 'ai-e2e-internal-invoice',
+		fullname: 'AI E2E Internal Invoice',
+		roles: {
+			super_admin: false,
+			groups: [
+				{
+					name: 'E2E Internal Invoice',
+					views: ['/dashboard', '/invoice/list', '/work-order/list']
+				}
+			],
+			miscs: []
+		},
+		other: {
+			id_client: 'E2E-CLIENT-001'
+		}
+	});
+	users.internal_no_invoice = await upsertHarnessUser(db, {
+		id: 'ai-e2e-internal-no-invoice',
+		username: 'ai-e2e-internal-no-invoice',
+		fullname: 'AI E2E Internal No Invoice',
+		roles: {
+			super_admin: false,
+			groups: [
+				{
+					name: 'E2E Internal Field',
+					views: ['/dashboard', '/work-order/list', '/chemical/list']
+				}
+			],
+			miscs: []
+		},
+		other: {
+			id_client: 'E2E-CLIENT-001'
+		}
+	});
+	users.internal_blend_plural = await upsertHarnessUser(db, {
+		id: 'ai-e2e-internal-blend-plural',
+		username: 'ai-e2e-internal-blend-plural',
+		fullname: 'AI E2E Internal Blend Plural',
+		roles: {
+			super_admin: false,
+			groups: [
+				{
+					name: 'E2E Blend Ops',
+					views: ['/dashboard', '/chemical-blends/list', '/work-order/list']
+				}
+			],
+			miscs: []
+		},
+		other: {
+			id_client: 'E2E-CLIENT-001'
+		}
+	});
+	users.internal_misc_invoice = await upsertHarnessUser(db, {
+		id: 'ai-e2e-internal-misc-invoice',
+		username: 'ai-e2e-internal-misc-invoice',
+		fullname: 'AI E2E Internal Misc Invoice',
+		roles: {
+			super_admin: false,
+			groups: [
+				{
+					name: 'E2E Operations',
+					views: ['/dashboard', '/work-order/list']
+				}
+			],
+			miscs: ['Invoice Dashboard Access']
+		},
+		other: {
+			id_client: 'E2E-CLIENT-001'
+		}
+	});
+	users.customer_invoice = await upsertHarnessUser(db, {
+		id: 'ai-e2e-customer-invoice',
+		username: 'ai-e2e-customer-invoice',
+		fullname: 'AI E2E Customer Invoice',
+		roles: {
+			super_admin: false,
+			groups: [
+				{
+					name: 'E2E Customer Invoice',
+					views: ['/customer-portal/dashboard', '/customer-portal/invoice/list']
+				}
+			],
+			miscs: []
+		},
+		other: {
+			id_customer: 'E2E-CUST-INV'
+		}
+	});
+	users.customer_no_invoice = await upsertHarnessUser(db, {
+		id: 'ai-e2e-customer-no-invoice',
+		username: 'ai-e2e-customer-no-invoice',
+		fullname: 'AI E2E Customer No Invoice',
+		roles: {
+			super_admin: false,
+			groups: [
+				{
+					name: 'E2E Customer No Invoice',
+					views: ['/customer-portal/dashboard', '/customer-portal/order/list']
+				}
+			],
+			miscs: []
+		},
+		other: {
+			id_customer: 'E2E-CUST-NO-INV'
+		}
+	});
+	return users;
+}
+
+async function pollFinalAssistantMessage(db: any, idConversation: string, timeoutMs = 180000): Promise<any> {
+	const started = Date.now();
+	while (Date.now() - started < timeoutMs) {
+		const message = await db.collection('ai-terminal-messages').findOne(
+			{ id_conversation: idConversation, role: 'assistant' },
+			{ sort: { createdAt: -1 } }
+		);
+		if (message && message?.metadata?.pending !== true) {
+			return message;
+		}
+		await sleep(1200);
+	}
+	throw new Error(`Timed out waiting for assistant completion for conversation ${idConversation}`);
+}
+
+async function runPromptE2E(
+	methodManager: any,
+	db: any,
+	idUser: string,
+	testCase: E2ECase
+): Promise<E2EResult> {
+	const managerContext = Object.assign({}, methodManager, Object.getPrototypeOf(methodManager), {
+		id_user: idUser,
+		user: 'E2E Runner',
+		id_ws: 'ai-assistant-openai-e2e'
+	});
+	const payload = {
+		message: testCase.prompt,
+		id_app: 'snf',
+		max_history: 0,
+		config: {
+			model: process.env.AI_ASSISTANT_CODEX_MODEL || process.env.OPENAI_MODEL || 'gpt-5.3-codex'
+		}
+	};
+	const started = Date.now();
+	const response = await methodManager.callMethod.call(managerContext, 'aiCoderTerminalRunCodex', payload);
+	const idConversation = String(response?.conversation?._id || response?.conversation?.id_conversation || '');
+	if (!idConversation) {
+		throw new Error('No conversation id returned from aiCoderTerminalRunCodex.');
+	}
+	const finalMessage = await pollFinalAssistantMessage(db, idConversation);
+	const content = String(finalMessage?.content || '');
+	const loweredContent = content.toLowerCase();
+	const metadata = finalMessage?.metadata && typeof finalMessage.metadata === 'object' ? finalMessage.metadata : {};
+	const header = findMarkdownTableHeader(content);
+	const classificationType = String(metadata?.debug?.requestClassification?.type || '').toLowerCase();
+	const dataQuestion = metadata?.debug?.requestClassification?.dataQuestion === true;
+
+	let error: string | null = null;
+	if (testCase.expect.hasTable && !header.length) {
+		error = 'Expected markdown table in final response.';
+	}
+	if (!error && testCase.expect.requestType) {
+		if (classificationType !== testCase.expect.requestType) {
+			error = `Expected request type ${testCase.expect.requestType}, got ${classificationType || 'none'}.`;
+		}
+	}
+	if (!error && typeof testCase.expect.dataQuestion === 'boolean') {
+		if (dataQuestion !== testCase.expect.dataQuestion) {
+			error = `Expected dataQuestion=${testCase.expect.dataQuestion}, got ${dataQuestion}.`;
+		}
+	}
+	if (!error && testCase.expect.dataResponse) {
+		if (classificationType && classificationType !== 'data' && classificationType !== 'mixed') {
+			error = `Expected data/mixed classification, got ${classificationType}.`;
+		}
+		else if (classificationType && !dataQuestion) {
+			error = 'Expected dataQuestion=true in debug classification.';
+		}
+	}
+	if (!error && testCase.expect.directiveType) {
+		const directiveType = String(metadata?.debug?.directive?.type || '').toLowerCase();
+		if (directiveType !== testCase.expect.directiveType) {
+			error = `Expected directive type ${testCase.expect.directiveType}, got ${directiveType || 'none'}.`;
+		}
+	}
+	if (!error && testCase.expect.revenueMetricGrid) {
+		error = assertRevenueMetricGrid(header);
+	}
+	if (!error && testCase.expect.revenueCustomerGrid) {
+		error = assertRevenueCustomerGrid(header);
+	}
+	if (!error && testCase.expect.timeGrain) {
+		error = assertTimeGrainGrid(header, testCase.expect.timeGrain);
+	}
+	if (!error && testCase.expect.fastPathType) {
+		const fastPathType = String(metadata?.fast_path || '');
+		const allowFastPathMiss = testCase.expect.allowFastPathMiss === true;
+		const allowMissingFastPath = allowFastPathMiss && !fastPathType;
+		if (fastPathType !== testCase.expect.fastPathType && !allowMissingFastPath) {
+			error = `Expected fast_path=${testCase.expect.fastPathType}, got ${fastPathType || 'none'}.`;
+		}
+	}
+	if (!error && typeof testCase.expect.fastPathGeneric === 'boolean') {
+		const fastPathType = String(metadata?.fast_path || '');
+		if (testCase.expect.allowFastPathMiss === true && !fastPathType) {
+			// Optional fast-path checks are skipped when no fast path is present.
+		}
+		else {
+			const fastPathGeneric = metadata?.fast_path_change_generic === true;
+			if (fastPathGeneric !== testCase.expect.fastPathGeneric) {
+				error = `Expected fast_path_change_generic=${testCase.expect.fastPathGeneric}, got ${fastPathGeneric}.`;
+			}
+		}
+	}
+	if (!error && testCase.expect.mustIncludeRoute) {
+		if (!content.includes(testCase.expect.mustIncludeRoute)) {
+			error = `Expected response to include route ${testCase.expect.mustIncludeRoute}.`;
+		}
+	}
+	if (!error && Array.isArray(testCase.expect.mustIncludeAnyRoute) && testCase.expect.mustIncludeAnyRoute.length) {
+		const hasAny = testCase.expect.mustIncludeAnyRoute.some(route => content.includes(route));
+		if (!hasAny) {
+			error = `Expected response to include at least one route: ${testCase.expect.mustIncludeAnyRoute.join(', ')}.`;
+		}
+	}
+	if (!error && testCase.expect.mustNotIncludeRoute) {
+		if (content.includes(testCase.expect.mustNotIncludeRoute)) {
+			error = `Expected response to exclude route ${testCase.expect.mustNotIncludeRoute}.`;
+		}
+	}
+	if (!error && Array.isArray(testCase.expect.mustNotIncludeAnyRoute) && testCase.expect.mustNotIncludeAnyRoute.length) {
+		const matched = testCase.expect.mustNotIncludeAnyRoute.find(route => content.includes(route));
+		if (matched) {
+			error = `Expected response to exclude route ${matched}.`;
+		}
+	}
+	if (!error && Array.isArray(testCase.expect.mustContainAnyText) && testCase.expect.mustContainAnyText.length) {
+		const hasAny = testCase.expect.mustContainAnyText.some(token => loweredContent.includes(String(token || '').toLowerCase()));
+		if (!hasAny) {
+			error = `Expected response to include one of: ${testCase.expect.mustContainAnyText.join(', ')}.`;
+		}
+	}
+	if (!error && Array.isArray(testCase.expect.mustNotContainAnyText) && testCase.expect.mustNotContainAnyText.length) {
+		const matched = testCase.expect.mustNotContainAnyText.find(token => loweredContent.includes(String(token || '').toLowerCase()));
+		if (matched) {
+			error = `Expected response to exclude text "${matched}".`;
+		}
+	}
+
+		return {
+			id: testCase.id,
+			prompt: testCase.prompt,
+			pass: !error,
+		error: error || undefined,
+		conversationId: idConversation,
+		responsePreview: content.slice(0, 1400),
+			classification: {
+				type: metadata?.debug?.requestClassification?.type,
+				dataQuestion: metadata?.debug?.requestClassification?.dataQuestion,
+				source: metadata?.debug?.requestClassification?.source
+			},
+			fastPath: metadata?.fast_path ? {
+				type: metadata?.fast_path,
+				reason: metadata?.fast_path_reason,
+				generic: metadata?.fast_path_change_generic === true,
+				commitCount: Number(metadata?.fast_path_change_commit_count || 0)
+			} : undefined,
+			userProfile: testCase.userProfile || 'super_admin',
+			tableHeader: header.length ? header : undefined
+		};
+	}
+
+async function run(): Promise<void> {
+	const repoRoot = process.cwd();
+	const snfServerDir = path.resolve(repoRoot, '../snf/snf/server');
+	const snfTmpDir = path.join(snfServerDir, 'tmp');
+	const snfSrcDir = path.join(snfServerDir, 'src');
+	const snfRuntimeDir = fs.existsSync(path.join(snfTmpDir, 'server-app.js')) ? snfTmpDir : snfSrcDir;
+	const snfEnvPath = path.join(snfServerDir, '.env');
+	const settingsPathInput = String(process.env.AI_ASSISTANT_E2E_SETTINGS_PATH || '').trim();
+	const settingsPath = settingsPathInput
+		? (path.isAbsolute(settingsPathInput) ? settingsPathInput : path.resolve(snfServerDir, settingsPathInput))
+		: path.join(snfServerDir, 'settings.local.json');
+	const clientRoutesPath = path.join(snfRuntimeDir, 'client-routes');
+
+	if (!fs.existsSync(settingsPath)) {
+		throw new Error(`SNF settings not found at ${settingsPath}`);
+	}
+
+	applyEnvFromDotEnv(snfEnvPath);
+	process.env.AI_ASSISTANT_WORKSPACE_ROOT = process.env.AI_ASSISTANT_WORKSPACE_ROOT || path.resolve(snfServerDir, '../..');
+	const appHeuristicsPath = path.join(snfServerDir, 'src', 'ai', 'assistant-heuristics.json');
+	if (!process.env.AI_ASSISTANT_APP_HEURISTICS_FILE && fs.existsSync(appHeuristicsPath)) {
+		process.env.AI_ASSISTANT_APP_HEURISTICS_FILE = appHeuristicsPath;
+	}
+	if (!process.env.AI_ASSISTANT_APP_ID) {
+		process.env.AI_ASSISTANT_APP_ID = 'snf';
+	}
+	if (!process.env.OPENAI_API_KEY) {
+		throw new Error('OPENAI_API_KEY is required for true OpenAI end-to-end tests.');
+	}
+	const codexHome = process.env.CODEX_HOME || path.join(repoRoot, 'tmp', '.codex-openai-e2e');
+	const codexSessions = path.join(codexHome, 'sessions');
+	const codexZdotdir = process.env.AI_DASHBOARD_CODEX_ZDOTDIR || path.join(repoRoot, 'tmp', '.codex-zdotdir');
+	fs.mkdirSync(codexSessions, { recursive: true });
+	fs.mkdirSync(codexZdotdir, { recursive: true });
+	process.env.CODEX_HOME = codexHome;
+	process.env.AI_DASHBOARD_CODEX_ZDOTDIR = codexZdotdir;
+
+	process.env.IS_WORKERS_ENABLED = 'false';
+	process.env.IS_WORKER_INSTANCE = 'false';
+	process.env.WORKER_INDEX = '';
+	process.env.NODE_APP_INSTANCE = process.env.NODE_APP_INSTANCE || '7';
+	process.env.AI_ASSISTANT_WORKER_DEBUG = process.env.AI_ASSISTANT_WORKER_DEBUG || 'false';
+	process.env.RESOLVEIO_LOCAL_LOG_DIR = process.env.RESOLVEIO_LOCAL_LOG_DIR || path.join(repoRoot, 'tmp', 'logs');
+
+	const serverConfig = JSON.parse(fs.readFileSync(settingsPath, 'utf8'));
+	const configuredHeuristicsFile = String(serverConfig?.AI_ASSISTANT_APP_HEURISTICS_FILE || '').trim();
+	if (configuredHeuristicsFile && !path.isAbsolute(configuredHeuristicsFile)) {
+		serverConfig.AI_ASSISTANT_APP_HEURISTICS_FILE = path.resolve(snfServerDir, configuredHeuristicsFile);
+	}
+	if (!serverConfig?.AI_ASSISTANT_APP_HEURISTICS_FILE && process.env.AI_ASSISTANT_APP_HEURISTICS_FILE) {
+		serverConfig.AI_ASSISTANT_APP_HEURISTICS_FILE = process.env.AI_ASSISTANT_APP_HEURISTICS_FILE;
+	}
+	if (!serverConfig?.AI_ASSISTANT_APP_ID && process.env.AI_ASSISTANT_APP_ID) {
+		serverConfig.AI_ASSISTANT_APP_ID = process.env.AI_ASSISTANT_APP_ID;
+	}
+	const { CLIENT_ROUTES } = require(clientRoutesPath);
+	await ResolveIOServer.create(serverConfig, CLIENT_ROUTES(), 'SNF', snfRuntimeDir, false, false);
+
+	const mainServer = ResolveIOServer.getMainServer();
+	const methodManager = mainServer?.getMethodManager();
+	if (!methodManager) {
+		throw new Error('MethodManager not available after ResolveIOServer initialization.');
+	}
+	await methodManager.waitUntilReady(120000);
+
+	const db = ResolveIOServer.getMongoConnection().db(serverConfig.DATABASE || 'resolveio');
+	const superAdminId = await findSuperAdminUserId(db);
+	const harnessUsers = await ensureHarnessUsers(db, superAdminId);
+	const overrideUserId = String(process.env.AI_ASSISTANT_E2E_USER_ID || '').trim();
+	const caseFilter = String(process.env.AI_ASSISTANT_E2E_CASES || '')
+		.split(',')
+		.map(value => value.trim())
+		.filter(Boolean);
+	const repeatCountRaw = Number(process.env.AI_ASSISTANT_E2E_REPEAT || 1);
+	const repeatCount = Number.isFinite(repeatCountRaw)
+		? Math.min(Math.max(Math.round(repeatCountRaw), 1), 500)
+		: 1;
+	const selectedPrompts = caseFilter.length
+		? E2E_PROMPTS.filter(testCase => caseFilter.includes(testCase.id))
+		: E2E_PROMPTS;
+	if (!selectedPrompts.length) {
+		throw new Error('No matching E2E cases selected via AI_ASSISTANT_E2E_CASES.');
+	}
+
+	const results: E2EResult[] = [];
+	for (const testCase of selectedPrompts) {
+		for (let attempt = 1; attempt <= repeatCount; attempt += 1) {
+			try {
+				const profile = testCase.userProfile || 'super_admin';
+				const idUser = overrideUserId || harnessUsers[profile] || superAdminId;
+				const result = await runPromptE2E(methodManager, db, idUser, testCase);
+				const runResult = {
+					...result,
+					attempt
+				};
+				results.push(runResult);
+				const status = runResult.pass ? 'PASS' : 'FAIL';
+				console.log(`${status} ${testCase.id}#${attempt}/${repeatCount} ${runResult.error ? `error=${runResult.error}` : ''}`.trim());
+			}
+			catch (error: any) {
+				const message = String(error?.message || error || 'Unknown error');
+				results.push({
+					id: testCase.id,
+					attempt,
+					prompt: testCase.prompt,
+					pass: false,
+					error: message,
+					userProfile: testCase.userProfile || 'super_admin'
+				});
+				console.log(`FAIL ${testCase.id}#${attempt}/${repeatCount} error=${message}`);
+			}
+		}
+	}
+
+	const passCount = results.filter(result => result.pass).length;
+	const failCount = results.length - passCount;
+	const perCase = selectedPrompts.map((testCase) => {
+		const caseResults = results.filter(result => result.id === testCase.id);
+		const passed = caseResults.filter(result => result.pass).length;
+		const failed = caseResults.length - passed;
+		let maxConsecutivePass = 0;
+		let currentConsecutivePass = 0;
+		caseResults.forEach((result) => {
+			if (result.pass) {
+				currentConsecutivePass += 1;
+				if (currentConsecutivePass > maxConsecutivePass) {
+					maxConsecutivePass = currentConsecutivePass;
+				}
+			}
+			else {
+				currentConsecutivePass = 0;
+			}
+		});
+		return {
+			id: testCase.id,
+			runs: caseResults.length,
+			passed,
+			failed,
+			maxConsecutivePass
+		};
+	});
+	const output = {
+		now: new Date().toISOString(),
+		total: results.length,
+		passed: passCount,
+		failed: failCount,
+		repeatCount,
+		perCase,
+		results
+	};
+
+	const outputPath = path.join(repoRoot, 'tmp', 'ai-assistant-openai-e2e-output.json');
+	fs.writeFileSync(outputPath, JSON.stringify(output, null, 2));
+	console.log(`ai assistant openai e2e: ${passCount}/${results.length} passed`);
+	console.log(`wrote ${outputPath}`);
+	process.exit(failCount > 0 ? 1 : 0);
+}
+
+run().catch((error) => {
+	console.error(error);
+	process.exit(1);
+});