@resolveio/server-lib 22.3.42 → 22.3.44

package/tests/ai-assistant-openai-e2e.ts

@@ -1,1061 +0,0 @@
-import * as fs from 'fs';
-import * as path from 'path';
-import { ResolveIOServer } from '../src/resolveio-server-app';
-
-type E2EUserProfile =
-	| 'super_admin'
-	| 'internal_invoice'
-	| 'internal_no_invoice'
-	| 'internal_blend_plural'
-	| 'internal_misc_invoice'
-	| 'customer_invoice'
-	| 'customer_no_invoice';
-
-type E2ECase = {
-	id: string;
-	prompt: string;
-	userProfile?: E2EUserProfile;
-	expect: {
-		hasTable?: boolean;
-		revenueMetricGrid?: boolean;
-		revenueCustomerGrid?: boolean;
-		timeGrain?: 'day' | 'week' | 'month' | 'quarter' | 'year';
-		dataResponse?: boolean;
-		requestType?: 'data' | 'navigation' | 'feature_info' | 'bug_issue' | 'mixed' | 'unknown';
-		dataQuestion?: boolean;
-		directiveType?: 'read' | 'aggregate';
-		fastPathType?: 'navigation' | 'change_history';
-		fastPathGeneric?: boolean;
-		allowFastPathMiss?: boolean;
-		mustIncludeRoute?: string;
-		mustNotIncludeRoute?: string;
-		mustIncludeAnyRoute?: string[];
-		mustNotIncludeAnyRoute?: string[];
-		mustContainAnyText?: string[];
-		mustNotContainAnyText?: string[];
-	};
-};
-
-type E2EResult = {
-	id: string;
-	attempt?: number;
-	prompt: string;
-	pass: boolean;
-	error?: string;
-	responsePreview?: string;
-	conversationId?: string;
-	classification?: {
-		type?: string;
-		dataQuestion?: boolean;
-		source?: string;
-	};
-	fastPath?: {
-		type?: string;
-		reason?: string;
-		generic?: boolean;
-		commitCount?: number;
-	};
-	userProfile?: E2EUserProfile;
-	tableHeader?: string[];
-};
-
-const E2E_PROMPTS: E2ECase[] = [
-	{
-		id: 'navigation_invoice_route_link_fast_path',
-		prompt: 'I just need the route link for invoice list.',
-		userProfile: 'super_admin',
-		expect: { fastPathType: 'navigation', mustIncludeRoute: '/invoice/list' }
-	},
-	{
-		id: 'navigation_invoice_route_link_customer_no_invoice',
-		prompt: 'I just need the route link for invoice list.',
-		userProfile: 'customer_no_invoice',
-		expect: { fastPathType: 'navigation', mustNotIncludeRoute: '/invoice/' }
-	},
-	{
-		id: 'navigation_blend_route_link_internal_blend_plural',
-		prompt: 'I just need the route link for chemical blend list.',
-		userProfile: 'internal_blend_plural',
-		expect: {
-			fastPathType: 'navigation',
-			mustIncludeAnyRoute: ['/chemical-blend/list', '/chemical-blends/list'],
-			mustNotIncludeRoute: '/invoice/'
-		}
-	},
-	{
-		id: 'navigation_blend_route_link_internal_no_blend',
-		prompt: 'I just need the route link for chemical blend list.',
-		userProfile: 'internal_no_invoice',
-		expect: {
-			fastPathType: 'navigation',
-			mustNotIncludeAnyRoute: ['/chemical-blend/', '/chemical-blends/']
-		}
-	},
-	{
-		id: 'navigation_order_alias_route_link_internal_no_invoice',
-		prompt: 'I just need the route link for /order/list.',
-		userProfile: 'internal_no_invoice',
-		expect: {
-			fastPathType: 'navigation',
-			mustIncludeAnyRoute: ['/order/list', '/work-order/list']
-		}
-	},
-	{
-		id: 'wo_last20_this_week_group_status',
-		prompt: 'Show me the last 20 work orders created this week, grouped by status.',
-		expect: { hasTable: true, dataResponse: true }
-	},
-	{
-		id: 'wo_last20_this_week_by_status_alias',
-		prompt: 'Show me the last 20 work orders created this week by status.',
-		expect: { dataResponse: true, directiveType: 'aggregate' }
-	},
-	{
-		id: 'wo_completed_per_day_last_30d',
-		prompt: 'For the last 30 days, how many work orders were completed per day?',
-		expect: { hasTable: true, dataResponse: true }
-	},
-	{
-		id: 'wo_top_customers_last_6m',
-		prompt: 'List the top 10 customers by number of work orders in the last 6 months.',
-		expect: { hasTable: true, dataResponse: true }
-	},
-	{
-		id: 'wo_this_week_customer_summary',
-		prompt: 'Summarize the work orders this week by customer',
-		expect: {
-			dataResponse: true,
-			mustNotContainAnyText: [
-				"i couldn't run the data query",
-				'positional operator in the middle of a path'
-			]
-		}
-	},
-	{
-		id: 'wo_top_customers_last_6m_internal_no_invoice',
-		prompt: 'List the top 10 customers by number of work orders in the last 6 months.',
-		userProfile: 'internal_no_invoice',
-		expect: { dataResponse: true, mustNotContainAnyText: ['does not have permission', 'access denied'] }
-	},
-	{
-		id: 'wo_open_missing_required_fields',
-		prompt: 'Show open work orders with missing required fields (customer, location, or scheduled date).',
-		expect: { hasTable: true, dataResponse: true }
-	},
-	{
-		id: 'blend_last10_summary',
-		prompt: 'For blending: summarize the last 10 blend tickets with product, total volume, and created date.',
-		expect: { hasTable: true, dataResponse: true }
-	},
-	{
-		id: 'blend_used_red_ultra_lift_last_30d',
-		prompt: 'Which blends used Chemical Red Ultra Lift in the last 30 days? Include lot/batch and volume.',
-		expect: { hasTable: true, dataResponse: true }
-	},
-	{
-		id: 'blend_throughput_by_day_last_2w',
-		prompt: 'Show blend throughput by day for the last 2 weeks.',
-		expect: { hasTable: true, dataResponse: true }
-	},
-	{
-		id: 'blend_throughput_by_day_last_2w_internal_blend_plural',
-		prompt: 'Show blend throughput by day for the last 2 weeks.',
-		userProfile: 'internal_blend_plural',
-		expect: { dataResponse: true, mustNotContainAnyText: ['does not have permission', 'access denied'] }
-	},
-	{
-		id: 'blend_throughput_by_day_last_2w_internal_no_blend',
-		prompt: 'Show blend throughput by day for the last 2 weeks.',
-		userProfile: 'internal_no_invoice',
-		expect: { mustContainAnyText: ['does not have permission', 'access denied', 'ask an admin'] }
-	},
-	{
-		id: 'invoice_top_customers_total_last_6m',
-		prompt: 'Show top 10 customers by invoice total in the last 6 months.',
-		expect: { hasTable: true, dataResponse: true }
-	},
-	{
-		id: 'invoice_top_customers_total_last_6m_internal_invoice',
-		prompt: 'Show top 10 customers by invoice total in the last 6 months.',
-		userProfile: 'internal_invoice',
-		expect: { dataResponse: true, mustNotContainAnyText: ['does not have permission', 'access denied'] }
-	},
-	{
-		id: 'invoice_top_customers_total_last_6m_internal_misc_invoice',
-		prompt: 'Show top 10 customers by invoice total in the last 6 months.',
-		userProfile: 'internal_misc_invoice',
-		expect: { dataResponse: true, mustNotContainAnyText: ['does not have permission', 'access denied'] }
-	},
-	{
-		id: 'invoice_top_customers_total_last_6m_internal_no_invoice_denied',
-		prompt: 'Show top 10 customers by invoice total in the last 6 months.',
-		userProfile: 'internal_no_invoice',
-		expect: { mustContainAnyText: ['does not have permission', 'access denied', 'ask an admin'] }
-	},
-	{
-		id: 'invoice_top_customers_total_last_6m_customer_invoice',
-		prompt: 'Show top 10 customers by invoice total in the last 6 months.',
-		userProfile: 'customer_invoice',
-		expect: { dataResponse: true, mustNotContainAnyText: ['does not have permission', 'access denied'] }
-	},
-	{
-		id: 'invoice_top_customers_total_last_6m_customer_no_invoice_denied',
-		prompt: 'Show top 10 customers by invoice total in the last 6 months.',
-		userProfile: 'customer_no_invoice',
-		expect: { mustContainAnyText: ['does not have permission', 'access denied', 'customer portal only', 'not available in this view', 'scope'] }
-	},
-	{
-		id: 'invoice_revenue_monthly_grid_last_6m',
-		prompt: 'Break down my total revenue over the last 6 month by month',
-		expect: { hasTable: true, dataResponse: true, revenueMetricGrid: true }
-	},
-	{
-		id: 'invoice_revenue_monthly_customer_grid_last_6m',
-		prompt: 'Break down my total revenue over the last 6 month by month by each customer',
-		expect: { hasTable: true, dataResponse: true, revenueCustomerGrid: true, directiveType: 'aggregate' }
-	},
-	{
-		id: 'invoice_revenue_monthly_customer_grid_last_6m_per_customer',
-		prompt: 'Break down my total revenue over the last 6 month per month per customer',
-		expect: { hasTable: true, dataResponse: true, revenueCustomerGrid: true, directiveType: 'aggregate' }
-	},
-	{
-		id: 'invoice_revenue_weekly_grid_last_12w',
-		prompt: 'Break down my total revenue over the last 12 weeks by week',
-		expect: { hasTable: true, dataResponse: true, timeGrain: 'week' }
-	},
-	{
-		id: 'invoice_revenue_daily_grid_last_30d',
-		prompt: 'Break down my total revenue over the last 30 days by day',
-		expect: { hasTable: true, dataResponse: true, timeGrain: 'day' }
-	},
-	{
-		id: 'invoice_revenue_quarterly_grid_last_8q',
-		prompt: 'Break down my total revenue over the last 8 quarters by quarter',
-		expect: { hasTable: true, dataResponse: true, timeGrain: 'quarter' }
-	},
-	{
-		id: 'invoice_revenue_yearly_grid_last_3y',
-		prompt: 'Break down my total revenue over the last 3 years by year',
-		expect: { hasTable: true, dataResponse: true, timeGrain: 'year' }
-	},
-	{
-		id: 'invoice_missing_invoiced_date_this_month',
-		prompt: 'List invoices missing an invoiced date but created this month.',
-		expect: { hasTable: true, dataResponse: true }
-	},
-	{
-		id: 'deliveries_per_driver_last_month',
-		prompt: 'Break down the number of deliveries per driver last month',
-		expect: { hasTable: true, dataResponse: true }
-	},
-	{
-		id: 'deliveries_per_driver_last_30d_top10',
-		prompt: 'Deliveries by driver for the last 30 days (top 10 drivers).',
-		expect: { hasTable: true, dataResponse: true, directiveType: 'aggregate' }
-	},
-	{
-		id: 'support_ticket_billable_hours_per_user_per_month_last_6m',
-		prompt: 'Break up the support tickets last 6 months, give me a total per user per month of the amount of hours they are billing for',
-		expect: { dataResponse: true, directiveType: 'aggregate' }
-	},
-	{
-		id: 'last20_deliveries_product_volume_destination',
-		prompt: 'List the last 20 deliveries with product, volume, and destination.',
-		expect: { hasTable: true, dataResponse: true }
-	},
-	{
-		id: 'jobs_highest_treatment_volume_last_90d',
-		prompt: 'Which jobs have the highest treatment volume in the last 90 days?',
-		expect: { hasTable: true, dataResponse: true }
-	},
-	{
-		id: 'change_history_recent_invoice_updates',
-		prompt: 'What changed recently in invoices? Keep it high-level and no code.',
-		userProfile: 'super_admin',
-		expect: { fastPathType: 'change_history', fastPathGeneric: false }
-	},
-	{
-		id: 'change_history_invoice_document_last_updated',
-		prompt: 'When was the last time the invoice document was updated?',
-		userProfile: 'super_admin',
-		expect: { fastPathType: 'change_history' }
-	},
-	{
-		id: 'change_history_dropdown_added_to_page',
-		prompt: 'When does this dropdown for XYZ get added to this page? What does it do?',
-		userProfile: 'super_admin',
-		expect: { fastPathType: 'change_history' }
-	},
-	{
-		id: 'change_history_this_page_updates',
-		prompt: 'What changed on this page?',
-		userProfile: 'super_admin',
-		expect: { fastPathType: 'change_history' }
-	},
-	{
-		id: 'change_history_invoice_page_updates',
-		prompt: 'What changed on the invoice page?',
-		userProfile: 'super_admin',
-		expect: { fastPathType: 'change_history' }
-	},
-	{
-		id: 'change_history_last_update_did_what',
-		prompt: 'What did the last update just do?',
-		userProfile: 'super_admin',
-		expect: { fastPathType: 'change_history' }
-	},
-	{
-		id: 'change_history_last_system_update_did_what',
-		prompt: 'What did the last system update do?',
-		userProfile: 'super_admin',
-		expect: { fastPathType: 'change_history', allowFastPathMiss: true }
-	},
-	{
-		id: 'change_history_that_last_system_update_did_what',
-		prompt: 'What did that last system update do?',
-		userProfile: 'super_admin',
-		expect: { fastPathType: 'change_history', allowFastPathMiss: true }
-	},
-	{
-		id: 'change_history_recent_invoice_updates_customer_invoice',
-		prompt: 'What changed recently in invoices? Keep it high-level and no code.',
-		userProfile: 'customer_invoice',
-		expect: { fastPathType: 'change_history', fastPathGeneric: false }
-	},
-	{
-		id: 'change_history_recent_invoice_updates_internal_invoice',
-		prompt: 'What changed recently in invoices? Keep it high-level and no code.',
-		userProfile: 'internal_invoice',
-		expect: { fastPathType: 'change_history', fastPathGeneric: false }
-	},
-	{
-		id: 'change_history_recent_invoice_updates_internal_misc_invoice',
-		prompt: 'What changed recently in invoices? Keep it high-level and no code.',
-		userProfile: 'internal_misc_invoice',
-		expect: { fastPathType: 'change_history', fastPathGeneric: false }
-	},
-	{
-		id: 'change_history_duration_work_orders',
-		prompt: 'How long has the current work order behavior been this way?',
-		userProfile: 'internal_no_invoice',
-		expect: { fastPathType: 'change_history' }
-	},
-	{
-		id: 'change_history_exception_list_been_around',
-		prompt: 'What is this new exception list? How long has it been around?',
-		userProfile: 'super_admin',
-		expect: { fastPathType: 'change_history' }
-	},
-	{
-		id: 'change_history_exception_list_on_this_page_duration',
-		prompt: 'How long has the exception list been on this page?',
-		userProfile: 'super_admin',
-		expect: { fastPathType: 'change_history' }
-	},
-	{
-		id: 'change_history_recent_invoice_updates_no_access',
-		prompt: 'What changed recently in invoices? Keep it high-level and no code.',
-		userProfile: 'customer_no_invoice',
-		expect: { fastPathType: 'change_history', fastPathGeneric: true }
-	},
-	{
-		id: 'change_history_recent_invoice_updates_internal_no_invoice',
-		prompt: 'What changed recently in invoices? Keep it high-level and no code.',
-		userProfile: 'internal_no_invoice',
-		expect: { fastPathType: 'change_history', fastPathGeneric: true }
-	},
-	{
-		id: 'bug_issue_missing_asset_in_dropdown',
-		prompt: "Why don't I see this asset in this dropdown?",
-		expect: { requestType: 'bug_issue', dataQuestion: false }
-	},
-	{
-		id: 'bug_issue_work_order_close_greyed_out',
-		prompt: "Why won't this work order let me close it, it's greyed out?",
-		expect: { requestType: 'bug_issue', dataQuestion: false }
-	},
-	{
-		id: 'bug_issue_submit_button_red',
-		prompt: 'The submit button is red and I filled out everything. What is likely wrong?',
-		expect: { requestType: 'bug_issue', dataQuestion: false }
-	},
-	{
-		id: 'last20_completed_wo_customer_product_delivered_returned',
-		prompt: 'Break down the last 20 completed work orders and tell me what customer, what product and how much we delivered or returned',
-		expect: { hasTable: true, dataResponse: true }
-	}
-];
-
-function parseDotEnvFile(filePath: string): Record<string, string> {
-	const parsed: Record<string, string> = {};
-	if (!fs.existsSync(filePath)) {
-		return parsed;
-	}
-	const content = fs.readFileSync(filePath, 'utf8');
-	content.split(/\r?\n/g).forEach((lineRaw) => {
-		const line = lineRaw.trim();
-		if (!line || line.startsWith('#')) {
-			return;
-		}
-		const index = line.indexOf('=');
-		if (index <= 0) {
-			return;
-		}
-		const key = line.slice(0, index).trim();
-		let value = line.slice(index + 1).trim();
-		if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith('\'') && value.endsWith('\''))) {
-			value = value.slice(1, -1);
-		}
-		if (key) {
-			parsed[key] = value;
-		}
-	});
-	return parsed;
-}
-
-function applyEnvFromDotEnv(filePath: string): void {
-	const parsed = parseDotEnvFile(filePath);
-	Object.keys(parsed).forEach((key) => {
-		if (!process.env[key]) {
-			process.env[key] = parsed[key];
-		}
-	});
-}
-
-function findMarkdownTableHeader(content: string): string[] {
-	const lines = String(content || '').split('\n');
-	for (let index = 0; index < lines.length - 1; index += 1) {
-		const header = lines[index]?.trim();
-		const separator = lines[index + 1]?.trim();
-		if (!header || !separator) {
-			continue;
-		}
-		if (!/^\|.+\|$/.test(header)) {
-			continue;
-		}
-		if (!/^\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)+\|?\s*$/.test(separator)) {
-			continue;
-		}
-		return header
-			.split('|')
-			.map(value => value.trim())
-			.filter(Boolean);
-	}
-	return [];
-}
-
-function isMonthHeader(value: string): boolean {
-	return /^\d{4}-\d{2}$/.test(String(value || '').trim());
-}
-
-function isDayHeader(value: string): boolean {
-	return /^\d{4}-\d{2}-\d{2}$/.test(String(value || '').trim());
-}
-
-function isWeekHeader(value: string): boolean {
-	const normalized = String(value || '').trim();
-	if (!normalized) {
-		return false;
-	}
-	return /^\d{4}-w\d{1,2}$/i.test(normalized)
-		|| /^w\d{1,2}\s+\d{4}$/i.test(normalized)
-		|| /^\d{4}-\d{2}-\d{2}\s+to\s+\d{4}-\d{2}-\d{2}$/i.test(normalized);
-}
-
-function isQuarterHeader(value: string): boolean {
-	const normalized = String(value || '').trim();
-	if (!normalized) {
-		return false;
-	}
-	return /^\d{4}-q[1-4]$/i.test(normalized)
-		|| /^q[1-4]\s+\d{4}$/i.test(normalized)
-		|| /^\d{4}\s+q[1-4]$/i.test(normalized);
-}
-
-function isYearHeader(value: string): boolean {
-	return /^\d{4}$/.test(String(value || '').trim());
-}
-
-function assertRevenueMetricGrid(header: string[]): string | null {
-	if (!header.length) {
-		return 'Expected markdown table header for monthly revenue grid.';
-	}
-	if (String(header[0] || '').toLowerCase() !== 'metric') {
-		return `Expected first column to be Metric, got ${header[0] || 'empty'}.`;
-	}
-	const monthColumns = header.slice(1).filter(isMonthHeader);
-	if (monthColumns.length < 3) {
-		return `Expected at least 3 month columns, got ${monthColumns.length}.`;
-	}
-	return null;
-}
-
-function assertRevenueCustomerGrid(header: string[]): string | null {
-	if (!header.length) {
-		return 'Expected markdown table header for customer monthly revenue grid.';
-	}
-	if (!String(header[0] || '').toLowerCase().includes('customer')) {
-		return `Expected first column to include Customer, got ${header[0] || 'empty'}.`;
-	}
-	const monthColumns = header.slice(1).filter(isMonthHeader);
-	if (monthColumns.length < 3) {
-		return `Expected at least 3 month columns, got ${monthColumns.length}.`;
-	}
-	return null;
-}
-
-function assertTimeGrainGrid(header: string[], grain: 'day' | 'week' | 'month' | 'quarter' | 'year'): string | null {
-	if (!header.length) {
-		return `Expected markdown table header for ${grain} report grid.`;
-	}
-	const first = String(header[0] || '').toLowerCase();
-	const lowered = header.map(value => String(value || '').toLowerCase());
-	const buckets = header.slice(1);
-	if (grain === 'day') {
-		if (lowered.some(column => column.includes('day') || column === 'date') || buckets.some(isDayHeader)) {
-			return null;
-		}
-		return `Expected day/date header for daily report, got ${header.join(', ')}.`;
-	}
-	if (grain === 'week') {
-		if (lowered.some(column => column.includes('week')) || buckets.some(isWeekHeader)) {
-			return null;
-		}
-		return `Expected week header for weekly report, got ${header.join(', ')}.`;
-	}
-	if (grain === 'month') {
-		if (lowered.some(column => column.includes('month')) || buckets.some(isMonthHeader)) {
-			return null;
-		}
-		return `Expected month header for monthly report, got ${header.join(', ')}.`;
-	}
-	if (grain === 'quarter') {
-		if (lowered.some(column => column.includes('quarter')) || buckets.some(isQuarterHeader)) {
-			return null;
-		}
-		return `Expected quarter header for quarterly report, got ${header.join(', ')}.`;
-	}
-	if (lowered.some(column => column.includes('year')) || buckets.some(isYearHeader)) {
-		return null;
-	}
-	return `Expected year header for yearly report, got ${header.join(', ')}.`;
-}
-
-async function sleep(ms: number): Promise<void> {
-	// eslint-disable-next-line no-restricted-syntax
-	return await new Promise(resolve => setTimeout(resolve, ms));
-}
-
-async function findSuperAdminUserId(db: any): Promise<string> {
-	const user = await db.collection('users').findOne(
-		{ 'roles.super_admin': true },
-		{ projection: { _id: 1, fullname: 1 }, sort: { updatedAt: -1, createdAt: -1 } }
-	);
-	if (!user?._id) {
-		throw new Error('Could not find super admin user in resolveio.users.');
-	}
-	return String(user._id);
-}
-
-async function upsertHarnessUser(db: any, params: {
-	id: string;
-	username: string;
-	fullname: string;
-	roles: Record<string, any>;
-	other?: Record<string, any>;
-}): Promise<string> {
-	const userId = String(params.id || '').trim();
-	if (!userId) {
-		throw new Error(`Harness user id is required for ${params.username}.`);
-	}
-	const now = new Date();
-	const users = db.collection('users');
-	let existing = await users.findOne(
-		{ username: params.username },
-		{ projection: { _id: 1 } }
-	);
-	if (existing?._id && String(existing._id) !== userId) {
-		await users.deleteOne({ _id: existing._id });
-		existing = null;
-	}
-	const setPayload = {
-		_id: userId,
-		username: params.username,
-		fullname: params.fullname,
-		email: `${params.username}@example.invalid`,
-		active: true,
-		roles: params.roles,
-		other: params.other || {},
-		salt: 'ai-e2e',
-		hash: 'ai-e2e',
-		attempts: 0,
-		updatedAt: now
-	};
-	if (existing?._id && String(existing._id) === userId) {
-		await users.updateOne(
-			{ _id: userId },
-			{ $set: setPayload }
-		);
-		return userId;
-	}
-	await users.updateOne(
-		{ _id: userId },
-		{
-			$set: setPayload,
-			$setOnInsert: {
-				createdAt: now
-			}
-		},
-		{ upsert: true }
-	);
-	return userId;
-}
-
-async function ensureHarnessUsers(db: any, superAdminId: string): Promise<Record<E2EUserProfile, string>> {
-	const users: Record<E2EUserProfile, string> = {
-		super_admin: superAdminId,
-		internal_invoice: '',
-		internal_no_invoice: '',
-		internal_blend_plural: '',
-		internal_misc_invoice: '',
-		customer_invoice: '',
-		customer_no_invoice: ''
-	};
-	users.internal_invoice = await upsertHarnessUser(db, {
-		id: 'ai-e2e-internal-invoice',
-		username: 'ai-e2e-internal-invoice',
-		fullname: 'AI E2E Internal Invoice',
-		roles: {
-			super_admin: false,
-			groups: [
-				{
-					name: 'E2E Internal Invoice',
-					views: ['/dashboard', '/invoice/list', '/work-order/list']
-				}
-			],
-			miscs: []
-		},
-		other: {
-			id_client: 'E2E-CLIENT-001'
-		}
-	});
-	users.internal_no_invoice = await upsertHarnessUser(db, {
-		id: 'ai-e2e-internal-no-invoice',
-		username: 'ai-e2e-internal-no-invoice',
-		fullname: 'AI E2E Internal No Invoice',
-		roles: {
-			super_admin: false,
-			groups: [
-				{
-					name: 'E2E Internal Field',
-					views: ['/dashboard', '/work-order/list', '/chemical/list']
-				}
-			],
-			miscs: []
-		},
-		other: {
-			id_client: 'E2E-CLIENT-001'
-		}
-	});
-	users.internal_blend_plural = await upsertHarnessUser(db, {
-		id: 'ai-e2e-internal-blend-plural',
-		username: 'ai-e2e-internal-blend-plural',
-		fullname: 'AI E2E Internal Blend Plural',
-		roles: {
-			super_admin: false,
-			groups: [
-				{
-					name: 'E2E Blend Ops',
-					views: ['/dashboard', '/chemical-blends/list', '/work-order/list']
-				}
-			],
-			miscs: []
-		},
-		other: {
-			id_client: 'E2E-CLIENT-001'
-		}
-	});
-	users.internal_misc_invoice = await upsertHarnessUser(db, {
-		id: 'ai-e2e-internal-misc-invoice',
-		username: 'ai-e2e-internal-misc-invoice',
-		fullname: 'AI E2E Internal Misc Invoice',
-		roles: {
-			super_admin: false,
-			groups: [
-				{
-					name: 'E2E Operations',
-					views: ['/dashboard', '/work-order/list']
-				}
-			],
-			miscs: ['Invoice Dashboard Access']
-		},
-		other: {
-			id_client: 'E2E-CLIENT-001'
-		}
-	});
-	users.customer_invoice = await upsertHarnessUser(db, {
-		id: 'ai-e2e-customer-invoice',
-		username: 'ai-e2e-customer-invoice',
-		fullname: 'AI E2E Customer Invoice',
-		roles: {
-			super_admin: false,
-			groups: [
-				{
-					name: 'E2E Customer Invoice',
-					views: ['/customer-portal/dashboard', '/customer-portal/invoice/list']
-				}
-			],
-			miscs: []
-		},
-		other: {
-			id_customer: 'E2E-CUST-INV'
-		}
-	});
-	users.customer_no_invoice = await upsertHarnessUser(db, {
-		id: 'ai-e2e-customer-no-invoice',
-		username: 'ai-e2e-customer-no-invoice',
-		fullname: 'AI E2E Customer No Invoice',
-		roles: {
-			super_admin: false,
-			groups: [
-				{
-					name: 'E2E Customer No Invoice',
-					views: ['/customer-portal/dashboard', '/customer-portal/order/list']
-				}
-			],
-			miscs: []
-		},
-		other: {
-			id_customer: 'E2E-CUST-NO-INV'
-		}
-	});
-	return users;
-}
-
-async function pollFinalAssistantMessage(db: any, idConversation: string, timeoutMs = 180000): Promise<any> {
-	const started = Date.now();
-	while (Date.now() - started < timeoutMs) {
-		const message = await db.collection('ai-terminal-messages').findOne(
-			{ id_conversation: idConversation, role: 'assistant' },
-			{ sort: { createdAt: -1 } }
-		);
-		if (message && message?.metadata?.pending !== true) {
-			return message;
-		}
-		await sleep(1200);
-	}
-	throw new Error(`Timed out waiting for assistant completion for conversation ${idConversation}`);
-}
-
-async function runPromptE2E(
-	methodManager: any,
-	db: any,
-	idUser: string,
-	testCase: E2ECase
-): Promise<E2EResult> {
-	const managerContext = Object.assign({}, methodManager, Object.getPrototypeOf(methodManager), {
-		id_user: idUser,
-		user: 'E2E Runner',
-		id_ws: 'ai-assistant-openai-e2e'
-	});
-	const model = String(process.env.AI_ASSISTANT_CODEX_MODEL || process.env.OPENAI_MODEL || '').trim();
-	const payload = {
-		message: testCase.prompt,
-		id_app: 'snf',
-		max_history: 0,
-		config: {
-			...(model ? { model } : {})
-		}
-	};
-	const started = Date.now();
-	const response = await methodManager.callMethod.call(managerContext, 'aiCoderTerminalRunCodex', payload);
-	const idConversation = String(response?.conversation?._id || response?.conversation?.id_conversation || '');
-	if (!idConversation) {
-		throw new Error('No conversation id returned from aiCoderTerminalRunCodex.');
-	}
-	const finalMessage = await pollFinalAssistantMessage(db, idConversation);
-	const content = String(finalMessage?.content || '');
-	const loweredContent = content.toLowerCase();
-	const metadata = finalMessage?.metadata && typeof finalMessage.metadata === 'object' ? finalMessage.metadata : {};
-	const header = findMarkdownTableHeader(content);
-	const classificationType = String(metadata?.debug?.requestClassification?.type || '').toLowerCase();
-	const dataQuestion = metadata?.debug?.requestClassification?.dataQuestion === true;
-
-	let error: string | null = null;
-	if (testCase.expect.hasTable && !header.length) {
-		error = 'Expected markdown table in final response.';
-	}
-	if (!error && testCase.expect.requestType) {
-		if (classificationType !== testCase.expect.requestType) {
-			error = `Expected request type ${testCase.expect.requestType}, got ${classificationType || 'none'}.`;
-		}
-	}
-	if (!error && typeof testCase.expect.dataQuestion === 'boolean') {
-		if (dataQuestion !== testCase.expect.dataQuestion) {
-			error = `Expected dataQuestion=${testCase.expect.dataQuestion}, got ${dataQuestion}.`;
-		}
-	}
-	if (!error && testCase.expect.dataResponse) {
-		if (classificationType && classificationType !== 'data' && classificationType !== 'mixed') {
-			error = `Expected data/mixed classification, got ${classificationType}.`;
-		}
-		else if (classificationType && !dataQuestion) {
-			error = 'Expected dataQuestion=true in debug classification.';
-		}
-	}
-	if (!error && testCase.expect.directiveType) {
-		const directiveType = String(metadata?.debug?.directive?.type || '').toLowerCase();
-		if (directiveType !== testCase.expect.directiveType) {
-			error = `Expected directive type ${testCase.expect.directiveType}, got ${directiveType || 'none'}.`;
-		}
-	}
-	if (!error && testCase.expect.revenueMetricGrid) {
-		error = assertRevenueMetricGrid(header);
-	}
-	if (!error && testCase.expect.revenueCustomerGrid) {
-		error = assertRevenueCustomerGrid(header);
-	}
-	if (!error && testCase.expect.timeGrain) {
-		error = assertTimeGrainGrid(header, testCase.expect.timeGrain);
-	}
-	if (!error && testCase.expect.fastPathType) {
-		const fastPathType = String(metadata?.fast_path || '');
-		const allowFastPathMiss = testCase.expect.allowFastPathMiss === true;
-		const allowMissingFastPath = allowFastPathMiss && !fastPathType;
-		if (fastPathType !== testCase.expect.fastPathType && !allowMissingFastPath) {
-			error = `Expected fast_path=${testCase.expect.fastPathType}, got ${fastPathType || 'none'}.`;
-		}
-	}
-	if (!error && typeof testCase.expect.fastPathGeneric === 'boolean') {
-		const fastPathType = String(metadata?.fast_path || '');
-		if (testCase.expect.allowFastPathMiss === true && !fastPathType) {
-			// Optional fast-path checks are skipped when no fast path is present.
-		}
-		else {
-			const fastPathGeneric = metadata?.fast_path_change_generic === true;
-			if (fastPathGeneric !== testCase.expect.fastPathGeneric) {
-				error = `Expected fast_path_change_generic=${testCase.expect.fastPathGeneric}, got ${fastPathGeneric}.`;
-			}
-		}
-	}
-	if (!error && testCase.expect.mustIncludeRoute) {
-		if (!content.includes(testCase.expect.mustIncludeRoute)) {
-			error = `Expected response to include route ${testCase.expect.mustIncludeRoute}.`;
-		}
-	}
-	if (!error && Array.isArray(testCase.expect.mustIncludeAnyRoute) && testCase.expect.mustIncludeAnyRoute.length) {
-		const hasAny = testCase.expect.mustIncludeAnyRoute.some(route => content.includes(route));
-		if (!hasAny) {
-			error = `Expected response to include at least one route: ${testCase.expect.mustIncludeAnyRoute.join(', ')}.`;
-		}
-	}
-	if (!error && testCase.expect.mustNotIncludeRoute) {
-		if (content.includes(testCase.expect.mustNotIncludeRoute)) {
-			error = `Expected response to exclude route ${testCase.expect.mustNotIncludeRoute}.`;
-		}
-	}
-	if (!error && Array.isArray(testCase.expect.mustNotIncludeAnyRoute) && testCase.expect.mustNotIncludeAnyRoute.length) {
-		const matched = testCase.expect.mustNotIncludeAnyRoute.find(route => content.includes(route));
-		if (matched) {
-			error = `Expected response to exclude route ${matched}.`;
-		}
-	}
-	if (!error && Array.isArray(testCase.expect.mustContainAnyText) && testCase.expect.mustContainAnyText.length) {
-		const hasAny = testCase.expect.mustContainAnyText.some(token => loweredContent.includes(String(token || '').toLowerCase()));
-		if (!hasAny) {
-			error = `Expected response to include one of: ${testCase.expect.mustContainAnyText.join(', ')}.`;
-		}
-	}
-	if (!error && Array.isArray(testCase.expect.mustNotContainAnyText) && testCase.expect.mustNotContainAnyText.length) {
-		const matched = testCase.expect.mustNotContainAnyText.find(token => loweredContent.includes(String(token || '').toLowerCase()));
-		if (matched) {
-			error = `Expected response to exclude text "${matched}".`;
-		}
-	}
-
-		return {
-			id: testCase.id,
-			prompt: testCase.prompt,
-			pass: !error,
-		error: error || undefined,
-		conversationId: idConversation,
-		responsePreview: content.slice(0, 1400),
-			classification: {
-				type: metadata?.debug?.requestClassification?.type,
-				dataQuestion: metadata?.debug?.requestClassification?.dataQuestion,
-				source: metadata?.debug?.requestClassification?.source
-			},
-			fastPath: metadata?.fast_path ? {
-				type: metadata?.fast_path,
-				reason: metadata?.fast_path_reason,
-				generic: metadata?.fast_path_change_generic === true,
-				commitCount: Number(metadata?.fast_path_change_commit_count || 0)
-			} : undefined,
-			userProfile: testCase.userProfile || 'super_admin',
-			tableHeader: header.length ? header : undefined
-		};
-	}
-
-async function run(): Promise<void> {
-	const repoRoot = process.cwd();
-	const snfServerDir = path.resolve(repoRoot, '../snf/snf/server');
-	const snfTmpDir = path.join(snfServerDir, 'tmp');
-	const snfSrcDir = path.join(snfServerDir, 'src');
-	const snfRuntimeDir = fs.existsSync(path.join(snfTmpDir, 'server-app.js')) ? snfTmpDir : snfSrcDir;
-	const snfEnvPath = path.join(snfServerDir, '.env');
-	const settingsPathInput = String(process.env.AI_ASSISTANT_E2E_SETTINGS_PATH || '').trim();
-	const settingsPath = settingsPathInput
-		? (path.isAbsolute(settingsPathInput) ? settingsPathInput : path.resolve(snfServerDir, settingsPathInput))
-		: path.join(snfServerDir, 'settings.local.json');
-	const clientRoutesPath = path.join(snfRuntimeDir, 'client-routes');
-
-	if (!fs.existsSync(settingsPath)) {
-		throw new Error(`SNF settings not found at ${settingsPath}`);
-	}
-
-	applyEnvFromDotEnv(snfEnvPath);
-	process.env.AI_ASSISTANT_WORKSPACE_ROOT = process.env.AI_ASSISTANT_WORKSPACE_ROOT || path.resolve(snfServerDir, '../..');
-	const appHeuristicsPath = path.join(snfServerDir, 'src', 'ai', 'assistant-heuristics.json');
-	if (!process.env.AI_ASSISTANT_APP_HEURISTICS_FILE && fs.existsSync(appHeuristicsPath)) {
-		process.env.AI_ASSISTANT_APP_HEURISTICS_FILE = appHeuristicsPath;
-	}
-	if (!process.env.AI_ASSISTANT_APP_ID) {
-		process.env.AI_ASSISTANT_APP_ID = 'snf';
-	}
-	if (!process.env.OPENAI_API_KEY) {
-		throw new Error('OPENAI_API_KEY is required for true OpenAI end-to-end tests.');
-	}
-	const codexHome = process.env.CODEX_HOME || path.join(repoRoot, 'tmp', '.codex-openai-e2e');
-	const codexSessions = path.join(codexHome, 'sessions');
-	const codexZdotdir = process.env.AI_DASHBOARD_CODEX_ZDOTDIR || path.join(repoRoot, 'tmp', '.codex-zdotdir');
-	fs.mkdirSync(codexSessions, { recursive: true });
-	fs.mkdirSync(codexZdotdir, { recursive: true });
-	process.env.CODEX_HOME = codexHome;
-	process.env.AI_DASHBOARD_CODEX_ZDOTDIR = codexZdotdir;
-
-	process.env.IS_WORKERS_ENABLED = 'false';
-	process.env.IS_WORKER_INSTANCE = 'false';
-	process.env.WORKER_INDEX = '';
-	process.env.NODE_APP_INSTANCE = process.env.NODE_APP_INSTANCE || '7';
-	process.env.AI_ASSISTANT_WORKER_DEBUG = process.env.AI_ASSISTANT_WORKER_DEBUG || 'false';
-	process.env.RESOLVEIO_LOCAL_LOG_DIR = process.env.RESOLVEIO_LOCAL_LOG_DIR || path.join(repoRoot, 'tmp', 'logs');
-
-	const serverConfig = JSON.parse(fs.readFileSync(settingsPath, 'utf8'));
-	const configuredHeuristicsFile = String(serverConfig?.AI_ASSISTANT_APP_HEURISTICS_FILE || '').trim();
-	if (configuredHeuristicsFile && !path.isAbsolute(configuredHeuristicsFile)) {
-		serverConfig.AI_ASSISTANT_APP_HEURISTICS_FILE = path.resolve(snfServerDir, configuredHeuristicsFile);
-	}
-	if (!serverConfig?.AI_ASSISTANT_APP_HEURISTICS_FILE && process.env.AI_ASSISTANT_APP_HEURISTICS_FILE) {
-		serverConfig.AI_ASSISTANT_APP_HEURISTICS_FILE = process.env.AI_ASSISTANT_APP_HEURISTICS_FILE;
-	}
-	if (!serverConfig?.AI_ASSISTANT_APP_ID && process.env.AI_ASSISTANT_APP_ID) {
-		serverConfig.AI_ASSISTANT_APP_ID = process.env.AI_ASSISTANT_APP_ID;
-	}
-	const { CLIENT_ROUTES } = require(clientRoutesPath);
-	await ResolveIOServer.create(serverConfig, CLIENT_ROUTES(), 'SNF', snfRuntimeDir, false, false);
-
-	const mainServer = ResolveIOServer.getMainServer();
-	const methodManager = mainServer?.getMethodManager();
-	if (!methodManager) {
-		throw new Error('MethodManager not available after ResolveIOServer initialization.');
-	}
-	await methodManager.waitUntilReady(120000);
-
-	const db = ResolveIOServer.getMongoConnection().db(serverConfig.DATABASE || 'resolveio');
-	const superAdminId = await findSuperAdminUserId(db);
-	const harnessUsers = await ensureHarnessUsers(db, superAdminId);
-	const overrideUserId = String(process.env.AI_ASSISTANT_E2E_USER_ID || '').trim();
-	const caseFilter = String(process.env.AI_ASSISTANT_E2E_CASES || '')
-		.split(',')
-		.map(value => value.trim())
-		.filter(Boolean);
-	const repeatCountRaw = Number(process.env.AI_ASSISTANT_E2E_REPEAT || 1);
-	const repeatCount = Number.isFinite(repeatCountRaw)
-		? Math.min(Math.max(Math.round(repeatCountRaw), 1), 500)
-		: 1;
-	const selectedPrompts = caseFilter.length
-		? E2E_PROMPTS.filter(testCase => caseFilter.includes(testCase.id))
-		: E2E_PROMPTS;
-	if (!selectedPrompts.length) {
-		throw new Error('No matching E2E cases selected via AI_ASSISTANT_E2E_CASES.');
-	}
-
-	const results: E2EResult[] = [];
-	for (const testCase of selectedPrompts) {
-		for (let attempt = 1; attempt <= repeatCount; attempt += 1) {
-			try {
-				const profile = testCase.userProfile || 'super_admin';
-				const idUser = overrideUserId || harnessUsers[profile] || superAdminId;
-				const result = await runPromptE2E(methodManager, db, idUser, testCase);
-				const runResult = {
-					...result,
-					attempt
-				};
-				results.push(runResult);
-				const status = runResult.pass ? 'PASS' : 'FAIL';
-				console.log(`${status} ${testCase.id}#${attempt}/${repeatCount} ${runResult.error ? `error=${runResult.error}` : ''}`.trim());
-			}
-			catch (error: any) {
-				const message = String(error?.message || error || 'Unknown error');
-				results.push({
-					id: testCase.id,
-					attempt,
-					prompt: testCase.prompt,
-					pass: false,
-					error: message,
-					userProfile: testCase.userProfile || 'super_admin'
-				});
-				console.log(`FAIL ${testCase.id}#${attempt}/${repeatCount} error=${message}`);
-			}
-		}
-	}
-
-	const passCount = results.filter(result => result.pass).length;
-	const failCount = results.length - passCount;
-	const perCase = selectedPrompts.map((testCase) => {
-		const caseResults = results.filter(result => result.id === testCase.id);
-		const passed = caseResults.filter(result => result.pass).length;
-		const failed = caseResults.length - passed;
-		let maxConsecutivePass = 0;
-		let currentConsecutivePass = 0;
-		caseResults.forEach((result) => {
-			if (result.pass) {
-				currentConsecutivePass += 1;
-				if (currentConsecutivePass > maxConsecutivePass) {
-					maxConsecutivePass = currentConsecutivePass;
-				}
-			}
-			else {
-				currentConsecutivePass = 0;
-			}
-		});
-		return {
-			id: testCase.id,
-			runs: caseResults.length,
-			passed,
-			failed,
-			maxConsecutivePass
-		};
-	});
-	const output = {
-		now: new Date().toISOString(),
-		total: results.length,
-		passed: passCount,
-		failed: failCount,
-		repeatCount,
-		perCase,
-		results
-	};
-
-	const outputPath = path.join(repoRoot, 'tmp', 'ai-assistant-openai-e2e-output.json');
-	fs.writeFileSync(outputPath, JSON.stringify(output, null, 2));
-	console.log(`ai assistant openai e2e: ${passCount}/${results.length} passed`);
-	console.log(`wrote ${outputPath}`);
-	process.exit(failCount > 0 ? 1 : 0);
-}
-
-run().catch((error) => {
-	console.error(error);
-	process.exit(1);
-});