@resolveio/server-lib 22.3.54 → 22.3.55

package/tests/ai-assistant-routing-eval.test.ts

@@ -0,0 +1,560 @@
+import {
+	buildAssistantChangeHistorySummaryFromCommits,
+	classifyAssistantRequestType,
+	collectAssistantAllowedRoutesForUser,
+	evaluateAssistantNavigationFastPathWithRoutes,
+	resolveAssistantUserAccessTier,
+	shouldUseAssistantChangeHistoryFastPath,
+	userHasInvoiceAccess
+} from '../src/methods/ai-terminal';
+
+function assert(condition: boolean, message: string) {
+	if (!condition) {
+		throw new Error(message);
+	}
+}
+
+const ALL_ROUTES = [
+	'/dashboard',
+	'/invoice/list',
+	'/invoice/detail',
+	'/order/list',
+	'/work-order/list',
+	'/work-order/detail',
+	'/chemical/list',
+	'/chemical-blend/list',
+	'/customer-portal/dashboard',
+	'/customer-portal/invoice/list',
+	'/customer-portal/order/list'
+];
+
+const USERS = {
+	superAdmin: {
+		roles: {
+			super_admin: true,
+			groups: [],
+			miscs: []
+		},
+		other: {}
+	},
+	internalInvoice: {
+		roles: {
+			super_admin: false,
+			groups: [
+				{
+					name: 'Operations',
+					views: ['/dashboard', '/invoice/list', '/work-order/list']
+				}
+			],
+			miscs: []
+		},
+		other: {
+			id_client: 'CLIENT-A'
+		}
+	},
+	internalNoInvoice: {
+		roles: {
+			super_admin: false,
+			groups: [
+				{
+					name: 'Field',
+					views: ['/dashboard', '/work-order/list', '/chemical/list']
+				}
+			],
+			miscs: []
+		},
+		other: {
+			id_client: 'CLIENT-A'
+		}
+	},
+	internalBlendPlural: {
+		roles: {
+			super_admin: false,
+			groups: [
+				{
+					name: 'Blend Ops',
+					views: ['/dashboard', '/chemical-blends/list', '/work-order/list']
+				}
+			],
+			miscs: []
+		},
+		other: {
+			id_client: 'CLIENT-A'
+		}
+	},
+	internalMiscInvoice: {
+		roles: {
+			super_admin: false,
+			groups: [
+				{
+					name: 'Operations',
+					views: ['/dashboard', '/work-order/list']
+				}
+			],
+			miscs: ['Invoice Dashboard Access']
+		},
+		other: {
+			id_client: 'CLIENT-A'
+		}
+	},
+	customerWithInvoice: {
+		roles: {
+			super_admin: false,
+			groups: [
+				{
+					name: 'Customer Portal',
+					views: ['/customer-portal/dashboard', '/customer-portal/invoice/list']
+				}
+			],
+			miscs: []
+		},
+		other: {
+			id_customer: 'CUST-100'
+		}
+	},
+	customerNoInvoice: {
+		roles: {
+			super_admin: false,
+			groups: [
+				{
+					name: 'Customer Portal',
+					views: ['/customer-portal/dashboard', '/customer-portal/order/list']
+				}
+			],
+			miscs: []
+		},
+		other: {
+			id_customer: 'CUST-200'
+		}
+	}
+};
+
+function testPromptClassificationCoverage() {
+	const cases: Array<{ message: string; expected: 'navigation' | 'data' | 'mixed' | 'bug_issue' | 'feature_info'; expectedDataQuestion: boolean }> = [
+		{
+			message: 'What is the route link for invoice list?',
+			expected: 'navigation',
+			expectedDataQuestion: false
+		},
+		{
+			message: 'Where do I go to edit a work order?',
+			expected: 'navigation',
+			expectedDataQuestion: false
+		},
+		{
+			message: 'Where is work order list?',
+			expected: 'navigation',
+			expectedDataQuestion: false
+		},
+		{
+			message: 'How many invoices were paid this month?',
+			expected: 'data',
+			expectedDataQuestion: true
+		},
+		{
+			message: 'Break down my total revenue over the last 6 month by month',
+			expected: 'data',
+			expectedDataQuestion: true
+		},
+			{
+				message: 'Break down my total revenue over the last 6 month by month by each customer',
+				expected: 'data',
+				expectedDataQuestion: true
+			},
+			{
+				message: 'Break down my total revenue over the last 6 month per month per customer',
+				expected: 'data',
+				expectedDataQuestion: true
+			},
+			{
+				message: 'Total revenue by month for the last 6 months.',
+				expected: 'data',
+				expectedDataQuestion: true
+			},
+		{
+			message: 'Break up the support tickets last 6 months, give me a total per user per month of the amount of hours they are billing for',
+			expected: 'data',
+			expectedDataQuestion: true
+		},
+		{
+			message: 'Summarize the hazelwood cross job for pioneer',
+			expected: 'data',
+			expectedDataQuestion: true
+		},
+		{
+			message: 'Which blends used Chemical Red Ultra Lift in the last 30 days? Include lot/batch and volume.',
+			expected: 'data',
+			expectedDataQuestion: true
+		},
+		{
+			message: 'Where do I go and how many open invoices do we have?',
+			expected: 'mixed',
+			expectedDataQuestion: true
+		},
+		{
+			message: 'Invoice screen is not working and throws an exception.',
+			expected: 'bug_issue',
+			expectedDataQuestion: false
+		},
+		{
+			message: 'Why is this happening: the save button fails every time.',
+			expected: 'bug_issue',
+			expectedDataQuestion: false
+		},
+		{
+			message: "Why won't this work order let me close it, it's greyed out?",
+			expected: 'bug_issue',
+			expectedDataQuestion: false
+		},
+		{
+			message: "Why don't I see this asset in this dropdown?",
+			expected: 'bug_issue',
+			expectedDataQuestion: false
+		},
+		{
+			message: 'The submit button is red and I filled out everything.',
+			expected: 'bug_issue',
+			expectedDataQuestion: false
+		},
+		{
+			message: 'How does invoice approval workflow work?',
+			expected: 'feature_info',
+			expectedDataQuestion: false
+		},
+		{
+			message: 'What is the difference between quote and invoice?',
+			expected: 'feature_info',
+			expectedDataQuestion: false
+		},
+		{
+			message: 'What changed recently in invoices?',
+			expected: 'feature_info',
+			expectedDataQuestion: false
+		},
+		{
+			message: 'What is this new exception list? How long has it been around?',
+			expected: 'feature_info',
+			expectedDataQuestion: false
+		},
+		{
+			message: 'How long has the exception list been on this page?',
+			expected: 'feature_info',
+			expectedDataQuestion: false
+		},
+		{
+			message: 'What changed on this page?',
+			expected: 'feature_info',
+			expectedDataQuestion: false
+		},
+		{
+			message: 'What changed on the invoice page?',
+			expected: 'feature_info',
+			expectedDataQuestion: false
+		},
+		{
+			message: 'What did the last update just do?',
+			expected: 'feature_info',
+			expectedDataQuestion: false
+		},
+		{
+			message: 'What did the last system update do?',
+			expected: 'feature_info',
+			expectedDataQuestion: false
+		}
+	];
+
+	cases.forEach((entry) => {
+		const result = classifyAssistantRequestType(entry.message);
+		assert(
+			result.type === entry.expected,
+			`Classification mismatch for "${entry.message}". Expected ${entry.expected}, got ${result.type}`
+		);
+		assert(
+			result.dataQuestion === entry.expectedDataQuestion,
+			`dataQuestion mismatch for "${entry.message}". Expected ${entry.expectedDataQuestion}, got ${result.dataQuestion}`
+		);
+	});
+}
+
+function testUserAccessTiersAndInvoicePermissions() {
+	assert(resolveAssistantUserAccessTier(USERS.superAdmin) === 'super_admin', 'Expected super admin tier');
+	assert(resolveAssistantUserAccessTier(USERS.internalInvoice) === 'client_user', 'Expected client user tier');
+	assert(resolveAssistantUserAccessTier(USERS.customerWithInvoice) === 'customer_portal', 'Expected customer portal tier');
+
+	assert(userHasInvoiceAccess(USERS.superAdmin), 'Expected super admin to have invoice access');
+	assert(userHasInvoiceAccess(USERS.internalInvoice), 'Expected invoice user to have invoice access');
+	assert(userHasInvoiceAccess(USERS.internalMiscInvoice), 'Expected invoice misc-tagged user to have invoice access');
+	assert(!userHasInvoiceAccess(USERS.internalNoInvoice), 'Expected no-invoice user to have no invoice access');
+	assert(userHasInvoiceAccess(USERS.customerWithInvoice), 'Expected customer invoice user to have invoice access');
+	assert(!userHasInvoiceAccess(USERS.customerNoInvoice), 'Expected customer no-invoice user to have no invoice access');
+}
+
+function testAllowedRoutesByUser() {
+	const superAdminRoutes = collectAssistantAllowedRoutesForUser(USERS.superAdmin, ALL_ROUTES, true);
+	assert(superAdminRoutes.includes('/invoice/list'), 'Expected super admin route inventory to include invoice list');
+	assert(superAdminRoutes.includes('/customer-portal/invoice/list'), 'Expected super admin route inventory to include customer portal invoice list');
+
+	const internalNoInvoiceRoutes = collectAssistantAllowedRoutesForUser(USERS.internalNoInvoice, ALL_ROUTES, false);
+	assert(!internalNoInvoiceRoutes.includes('/invoice/list'), 'Expected non-invoice internal user to exclude invoice routes');
+	assert(internalNoInvoiceRoutes.includes('/work-order/list'), 'Expected internal user to include work order list');
+	assert(internalNoInvoiceRoutes.includes('/order/list'), 'Expected work-order permission to map to /order/list alias route');
+
+	const internalBlendPluralRoutes = collectAssistantAllowedRoutesForUser(USERS.internalBlendPlural, ALL_ROUTES, false);
+	assert(internalBlendPluralRoutes.includes('/chemical-blend/list'), 'Expected plural blend permission to map to canonical blend route');
+	assert(!internalBlendPluralRoutes.includes('/invoice/list'), 'Expected blend user without invoice permission to exclude invoice routes');
+
+	const customerRoutes = collectAssistantAllowedRoutesForUser(USERS.customerNoInvoice, ALL_ROUTES, false);
+	assert(customerRoutes.includes('/customer-portal/dashboard'), 'Expected customer to include customer dashboard');
+	assert(!customerRoutes.includes('/invoice/list'), 'Expected customer user to exclude internal invoice route');
+}
+
+function testNavigationFastPathPermissions() {
+	const invoicePrompt = 'Need the route link for invoice list';
+
+	const internalInvoiceRoutes = collectAssistantAllowedRoutesForUser(USERS.internalInvoice, ALL_ROUTES, false);
+	const internalInvoiceResult = evaluateAssistantNavigationFastPathWithRoutes({
+		message: invoicePrompt,
+		hasAttachments: false,
+		allowedRoutes: internalInvoiceRoutes
+	});
+	assert(!!internalInvoiceResult, 'Expected fast-path response for internal invoice user');
+	assert(
+		(internalInvoiceResult?.routes || []).some(route => route.includes('/invoice/list')),
+		'Expected invoice route for invoice-permitted user'
+	);
+
+	const internalNoInvoiceRoutes = collectAssistantAllowedRoutesForUser(USERS.internalNoInvoice, ALL_ROUTES, false);
+	const internalNoInvoiceResult = evaluateAssistantNavigationFastPathWithRoutes({
+		message: invoicePrompt,
+		hasAttachments: false,
+		allowedRoutes: internalNoInvoiceRoutes
+	});
+	assert(!!internalNoInvoiceResult, 'Expected fast-path response for no-invoice user');
+	assert(
+		!(internalNoInvoiceResult?.routes || []).some(route => route.includes('/invoice')),
+		'Expected no invoice routes for non-permitted user'
+	);
+	const wherePromptResult = evaluateAssistantNavigationFastPathWithRoutes({
+		message: 'Where is work order list?',
+		hasAttachments: false,
+		allowedRoutes: internalNoInvoiceRoutes
+	});
+	assert(!!wherePromptResult, 'Expected fast-path response for where/list route prompt');
+	assert(
+		(wherePromptResult?.routes || []).some(route => route.includes('/work-order/list')),
+		'Expected work-order list route for where/list prompt'
+	);
+
+	const customerWithInvoiceRoutes = collectAssistantAllowedRoutesForUser(USERS.customerWithInvoice, ALL_ROUTES, false);
+	const customerWithInvoiceResult = evaluateAssistantNavigationFastPathWithRoutes({
+		message: invoicePrompt,
+		hasAttachments: false,
+		allowedRoutes: customerWithInvoiceRoutes
+	});
+	assert(!!customerWithInvoiceResult, 'Expected fast-path response for customer invoice user');
+	assert(
+		(customerWithInvoiceResult?.routes || []).some(route => route.includes('/customer-portal/invoice/list')),
+		'Expected customer portal invoice route for permitted customer user'
+	);
+
+	const customerNoInvoiceRoutes = collectAssistantAllowedRoutesForUser(USERS.customerNoInvoice, ALL_ROUTES, false);
+	const customerNoInvoiceResult = evaluateAssistantNavigationFastPathWithRoutes({
+		message: invoicePrompt,
+		hasAttachments: false,
+		allowedRoutes: customerNoInvoiceRoutes
+	});
+	assert(!!customerNoInvoiceResult, 'Expected fast-path response for customer no-invoice user');
+	assert(
+		!(customerNoInvoiceResult?.routes || []).some(route => route.includes('/invoice')),
+		'Expected no invoice routes for customer without invoice permissions'
+	);
+	assert(
+		customerNoInvoiceResult?.reason === 'permission_denied' || customerNoInvoiceResult?.reason === 'ranked',
+		'Expected denied/ranked fallback reason for customer no-invoice prompt'
+	);
+}
+
+function testNavigationFastPathFuzzyRoutePermissions() {
+	const blendPrompt = 'Need the route link for chemical blend list';
+	const blendRoutes = collectAssistantAllowedRoutesForUser(USERS.internalBlendPlural, ALL_ROUTES, false);
+	const blendResult = evaluateAssistantNavigationFastPathWithRoutes({
+		message: blendPrompt,
+		hasAttachments: false,
+		allowedRoutes: blendRoutes
+	});
+	assert(!!blendResult, 'Expected fast-path response for blend-permitted user');
+	assert(
+		(blendResult?.routes || []).some(route => route.includes('/chemical-blend/list')),
+		'Expected canonical blend route for plural blend permission'
+	);
+
+	const noBlendRoutes = collectAssistantAllowedRoutesForUser(USERS.internalNoInvoice, ALL_ROUTES, false);
+	const noBlendResult = evaluateAssistantNavigationFastPathWithRoutes({
+		message: blendPrompt,
+		hasAttachments: false,
+		allowedRoutes: noBlendRoutes
+	});
+	assert(!!noBlendResult, 'Expected fast-path response for blend-denied user');
+	assert(
+		!(noBlendResult?.routes || []).some(route => route.includes('/chemical-blend')),
+		'Expected blend-denied user to avoid blend routes'
+	);
+	assert(
+		noBlendResult?.reason === 'permission_denied' || noBlendResult?.reason === 'ranked',
+		'Expected denied/ranked fallback reason for blend-denied user'
+	);
+
+	const orderAliasResult = evaluateAssistantNavigationFastPathWithRoutes({
+		message: 'I need the route link for /order/list',
+		hasAttachments: false,
+		allowedRoutes: noBlendRoutes
+	});
+	assert(!!orderAliasResult, 'Expected fast-path response for /order/list alias route');
+	assert(
+		(orderAliasResult?.routes || []).some(route => route.includes('/order/list') || route.includes('/work-order/list')),
+		'Expected order alias prompt to map to order/work-order route'
+	);
+}
+
+function testFastPathSkipsAttachmentAndDataPrompts() {
+	const routes = collectAssistantAllowedRoutesForUser(USERS.internalInvoice, ALL_ROUTES, false);
+	const attachmentResult = evaluateAssistantNavigationFastPathWithRoutes({
+		message: 'What route link for invoice list?',
+		hasAttachments: true,
+		allowedRoutes: routes
+	});
+	assert(!attachmentResult, 'Expected attachments to disable navigation fast path');
+
+	const dataResult = evaluateAssistantNavigationFastPathWithRoutes({
+		message: 'How many invoices were paid this month?',
+		hasAttachments: false,
+		allowedRoutes: routes
+	});
+	assert(!dataResult, 'Expected quantitative data prompt to skip navigation fast path');
+
+	const changeHistoryResult = evaluateAssistantNavigationFastPathWithRoutes({
+		message: 'What changed on this page?',
+		hasAttachments: false,
+		allowedRoutes: routes
+	});
+	assert(!changeHistoryResult, 'Expected change-history prompt to skip navigation fast path');
+	const durationOnPageResult = evaluateAssistantNavigationFastPathWithRoutes({
+		message: 'How long has the exception list been on this page?',
+		hasAttachments: false,
+		allowedRoutes: routes
+	});
+	assert(!durationOnPageResult, 'Expected "how long on this page" prompt to skip navigation fast path');
+
+	const bugIssueResult = evaluateAssistantNavigationFastPathWithRoutes({
+		message: 'Why don\'t I see this asset in this dropdown?',
+		hasAttachments: false,
+		allowedRoutes: routes
+	});
+	assert(!bugIssueResult, 'Expected bug/issue prompt to skip navigation fast path');
+}
+
+function testChangeHistoryFastPathAndPermissionSummary() {
+	assert(
+		shouldUseAssistantChangeHistoryFastPath('What changed recently in invoices?', false),
+		'Expected change-history question to use change-history fast path'
+	);
+	assert(
+		shouldUseAssistantChangeHistoryFastPath('When was the last time the invoice document was updated?', false),
+		'Expected "last updated" question to use change-history fast path'
+	);
+	assert(
+		shouldUseAssistantChangeHistoryFastPath('When does this dropdown for XYZ get added to this page?', false),
+		'Expected "dropdown added" question to use change-history fast path'
+	);
+	assert(
+		shouldUseAssistantChangeHistoryFastPath('What changed on this page?', false),
+		'Expected "what changed on this page" question to use change-history fast path'
+	);
+	assert(
+		shouldUseAssistantChangeHistoryFastPath('What changed on the invoice page?', false),
+		'Expected "what changed on invoice page" question to use change-history fast path'
+	);
+	assert(
+		shouldUseAssistantChangeHistoryFastPath('What did the last update just do?', false),
+		'Expected "last update did" question to use change-history fast path'
+	);
+	assert(
+		shouldUseAssistantChangeHistoryFastPath('What did the last system update do?', false),
+		'Expected "last system update did" question to use change-history fast path'
+	);
+	assert(
+		shouldUseAssistantChangeHistoryFastPath('What is this new exception list? How long has it been around?', false),
+		'Expected "been around" question to use change-history fast path'
+	);
+	assert(
+		shouldUseAssistantChangeHistoryFastPath('How long has the exception list been on this page?', false),
+		'Expected "been on this page" question to use change-history fast path'
+	);
+	assert(
+		!shouldUseAssistantChangeHistoryFastPath('What changed recently in invoices?', true),
+		'Expected attachments to disable change-history fast path'
+	);
+
+	const commits = [
+		{
+			date: '2026-02-05T12:00:00.000Z',
+			subject: 'Fix invoice approval validation edge case',
+			files: ['server/invoice/approval.ts'],
+			modules: ['invoice']
+		},
+		{
+			date: '2026-02-04T12:00:00.000Z',
+			subject: 'Improve work order dispatch retries',
+			files: ['server/work-order/dispatch.ts'],
+			modules: ['work_order']
+		}
+	] as any[];
+
+	const permitted = buildAssistantChangeHistorySummaryFromCommits({
+		message: 'What changed recently in invoices?',
+		user: USERS.internalInvoice,
+		isSuperAdmin: false,
+		commits,
+		branch: 'main',
+		now: new Date('2026-02-08T00:00:00.000Z')
+	});
+	assert(!permitted.generic, 'Expected invoice-permitted user to receive non-generic change summary');
+	assert(
+		String(permitted.response || '').includes('Fix invoice approval validation edge case'),
+		'Expected permitted summary to include invoice change detail'
+	);
+
+	const restricted = buildAssistantChangeHistorySummaryFromCommits({
+		message: 'What changed recently in invoices?',
+		user: USERS.internalNoInvoice,
+		isSuperAdmin: false,
+		commits,
+		branch: 'main',
+		now: new Date('2026-02-08T00:00:00.000Z')
+	});
+	assert(restricted.generic, 'Expected invoice-restricted user to receive generic change summary');
+	assert(
+		!String(restricted.response || '').includes('Fix invoice approval validation edge case'),
+		'Expected restricted summary to omit invoice detail'
+	);
+}
+
+function run() {
+	testPromptClassificationCoverage();
+	testUserAccessTiersAndInvoicePermissions();
+	testAllowedRoutesByUser();
+	testNavigationFastPathPermissions();
+	testNavigationFastPathFuzzyRoutePermissions();
+	testFastPathSkipsAttachmentAndDataPrompts();
+	testChangeHistoryFastPathAndPermissionSummary();
+	console.log('ai assistant routing eval tests passed');
+}
+
+try {
+	run();
+	process.exit(process.exitCode ?? 0);
+}
+catch (error) {
+	console.error(error);
+	process.exit(1);
+}