@resolveio/server-lib 22.3.19 → 22.3.20

package/tests/ai-assistant-routing-eval.test.ts

@@ -1,560 +0,0 @@
-import {
-	buildAssistantChangeHistorySummaryFromCommits,
-	classifyAssistantRequestType,
-	collectAssistantAllowedRoutesForUser,
-	evaluateAssistantNavigationFastPathWithRoutes,
-	resolveAssistantUserAccessTier,
-	shouldUseAssistantChangeHistoryFastPath,
-	userHasInvoiceAccess
-} from '../src/methods/ai-terminal';
-
-function assert(condition: boolean, message: string) {
-	if (!condition) {
-		throw new Error(message);
-	}
-}
-
-const ALL_ROUTES = [
-	'/dashboard',
-	'/invoice/list',
-	'/invoice/detail',
-	'/order/list',
-	'/work-order/list',
-	'/work-order/detail',
-	'/chemical/list',
-	'/chemical-blend/list',
-	'/customer-portal/dashboard',
-	'/customer-portal/invoice/list',
-	'/customer-portal/order/list'
-];
-
-const USERS = {
-	superAdmin: {
-		roles: {
-			super_admin: true,
-			groups: [],
-			miscs: []
-		},
-		other: {}
-	},
-	internalInvoice: {
-		roles: {
-			super_admin: false,
-			groups: [
-				{
-					name: 'Operations',
-					views: ['/dashboard', '/invoice/list', '/work-order/list']
-				}
-			],
-			miscs: []
-		},
-		other: {
-			id_client: 'CLIENT-A'
-		}
-	},
-	internalNoInvoice: {
-		roles: {
-			super_admin: false,
-			groups: [
-				{
-					name: 'Field',
-					views: ['/dashboard', '/work-order/list', '/chemical/list']
-				}
-			],
-			miscs: []
-		},
-		other: {
-			id_client: 'CLIENT-A'
-		}
-	},
-	internalBlendPlural: {
-		roles: {
-			super_admin: false,
-			groups: [
-				{
-					name: 'Blend Ops',
-					views: ['/dashboard', '/chemical-blends/list', '/work-order/list']
-				}
-			],
-			miscs: []
-		},
-		other: {
-			id_client: 'CLIENT-A'
-		}
-	},
-	internalMiscInvoice: {
-		roles: {
-			super_admin: false,
-			groups: [
-				{
-					name: 'Operations',
-					views: ['/dashboard', '/work-order/list']
-				}
-			],
-			miscs: ['Invoice Dashboard Access']
-		},
-		other: {
-			id_client: 'CLIENT-A'
-		}
-	},
-	customerWithInvoice: {
-		roles: {
-			super_admin: false,
-			groups: [
-				{
-					name: 'Customer Portal',
-					views: ['/customer-portal/dashboard', '/customer-portal/invoice/list']
-				}
-			],
-			miscs: []
-		},
-		other: {
-			id_customer: 'CUST-100'
-		}
-	},
-	customerNoInvoice: {
-		roles: {
-			super_admin: false,
-			groups: [
-				{
-					name: 'Customer Portal',
-					views: ['/customer-portal/dashboard', '/customer-portal/order/list']
-				}
-			],
-			miscs: []
-		},
-		other: {
-			id_customer: 'CUST-200'
-		}
-	}
-};
-
-function testPromptClassificationCoverage() {
-	const cases: Array<{ message: string; expected: 'navigation' | 'data' | 'mixed' | 'bug_issue' | 'feature_info'; expectedDataQuestion: boolean }> = [
-		{
-			message: 'What is the route link for invoice list?',
-			expected: 'navigation',
-			expectedDataQuestion: false
-		},
-		{
-			message: 'Where do I go to edit a work order?',
-			expected: 'navigation',
-			expectedDataQuestion: false
-		},
-		{
-			message: 'Where is work order list?',
-			expected: 'navigation',
-			expectedDataQuestion: false
-		},
-		{
-			message: 'How many invoices were paid this month?',
-			expected: 'data',
-			expectedDataQuestion: true
-		},
-		{
-			message: 'Break down my total revenue over the last 6 month by month',
-			expected: 'data',
-			expectedDataQuestion: true
-		},
-			{
-				message: 'Break down my total revenue over the last 6 month by month by each customer',
-				expected: 'data',
-				expectedDataQuestion: true
-			},
-			{
-				message: 'Break down my total revenue over the last 6 month per month per customer',
-				expected: 'data',
-				expectedDataQuestion: true
-			},
-			{
-				message: 'Total revenue by month for the last 6 months.',
-				expected: 'data',
-				expectedDataQuestion: true
-			},
-		{
-			message: 'Break up the support tickets last 6 months, give me a total per user per month of the amount of hours they are billing for',
-			expected: 'data',
-			expectedDataQuestion: true
-		},
-		{
-			message: 'Summarize the hazelwood cross job for pioneer',
-			expected: 'data',
-			expectedDataQuestion: true
-		},
-		{
-			message: 'Which blends used Chemical Red Ultra Lift in the last 30 days? Include lot/batch and volume.',
-			expected: 'data',
-			expectedDataQuestion: true
-		},
-		{
-			message: 'Where do I go and how many open invoices do we have?',
-			expected: 'mixed',
-			expectedDataQuestion: true
-		},
-		{
-			message: 'Invoice screen is not working and throws an exception.',
-			expected: 'bug_issue',
-			expectedDataQuestion: false
-		},
-		{
-			message: 'Why is this happening: the save button fails every time.',
-			expected: 'bug_issue',
-			expectedDataQuestion: false
-		},
-		{
-			message: "Why won't this work order let me close it, it's greyed out?",
-			expected: 'bug_issue',
-			expectedDataQuestion: false
-		},
-		{
-			message: "Why don't I see this asset in this dropdown?",
-			expected: 'bug_issue',
-			expectedDataQuestion: false
-		},
-		{
-			message: 'The submit button is red and I filled out everything.',
-			expected: 'bug_issue',
-			expectedDataQuestion: false
-		},
-		{
-			message: 'How does invoice approval workflow work?',
-			expected: 'feature_info',
-			expectedDataQuestion: false
-		},
-		{
-			message: 'What is the difference between quote and invoice?',
-			expected: 'feature_info',
-			expectedDataQuestion: false
-		},
-		{
-			message: 'What changed recently in invoices?',
-			expected: 'feature_info',
-			expectedDataQuestion: false
-		},
-		{
-			message: 'What is this new exception list? How long has it been around?',
-			expected: 'feature_info',
-			expectedDataQuestion: false
-		},
-		{
-			message: 'How long has the exception list been on this page?',
-			expected: 'feature_info',
-			expectedDataQuestion: false
-		},
-		{
-			message: 'What changed on this page?',
-			expected: 'feature_info',
-			expectedDataQuestion: false
-		},
-		{
-			message: 'What changed on the invoice page?',
-			expected: 'feature_info',
-			expectedDataQuestion: false
-		},
-		{
-			message: 'What did the last update just do?',
-			expected: 'feature_info',
-			expectedDataQuestion: false
-		},
-		{
-			message: 'What did the last system update do?',
-			expected: 'feature_info',
-			expectedDataQuestion: false
-		}
-	];
-
-	cases.forEach((entry) => {
-		const result = classifyAssistantRequestType(entry.message);
-		assert(
-			result.type === entry.expected,
-			`Classification mismatch for "${entry.message}". Expected ${entry.expected}, got ${result.type}`
-		);
-		assert(
-			result.dataQuestion === entry.expectedDataQuestion,
-			`dataQuestion mismatch for "${entry.message}". Expected ${entry.expectedDataQuestion}, got ${result.dataQuestion}`
-		);
-	});
-}
-
-function testUserAccessTiersAndInvoicePermissions() {
-	assert(resolveAssistantUserAccessTier(USERS.superAdmin) === 'super_admin', 'Expected super admin tier');
-	assert(resolveAssistantUserAccessTier(USERS.internalInvoice) === 'client_user', 'Expected client user tier');
-	assert(resolveAssistantUserAccessTier(USERS.customerWithInvoice) === 'customer_portal', 'Expected customer portal tier');
-
-	assert(userHasInvoiceAccess(USERS.superAdmin), 'Expected super admin to have invoice access');
-	assert(userHasInvoiceAccess(USERS.internalInvoice), 'Expected invoice user to have invoice access');
-	assert(userHasInvoiceAccess(USERS.internalMiscInvoice), 'Expected invoice misc-tagged user to have invoice access');
-	assert(!userHasInvoiceAccess(USERS.internalNoInvoice), 'Expected no-invoice user to have no invoice access');
-	assert(userHasInvoiceAccess(USERS.customerWithInvoice), 'Expected customer invoice user to have invoice access');
-	assert(!userHasInvoiceAccess(USERS.customerNoInvoice), 'Expected customer no-invoice user to have no invoice access');
-}
-
-function testAllowedRoutesByUser() {
-	const superAdminRoutes = collectAssistantAllowedRoutesForUser(USERS.superAdmin, ALL_ROUTES, true);
-	assert(superAdminRoutes.includes('/invoice/list'), 'Expected super admin route inventory to include invoice list');
-	assert(superAdminRoutes.includes('/customer-portal/invoice/list'), 'Expected super admin route inventory to include customer portal invoice list');
-
-	const internalNoInvoiceRoutes = collectAssistantAllowedRoutesForUser(USERS.internalNoInvoice, ALL_ROUTES, false);
-	assert(!internalNoInvoiceRoutes.includes('/invoice/list'), 'Expected non-invoice internal user to exclude invoice routes');
-	assert(internalNoInvoiceRoutes.includes('/work-order/list'), 'Expected internal user to include work order list');
-	assert(internalNoInvoiceRoutes.includes('/order/list'), 'Expected work-order permission to map to /order/list alias route');
-
-	const internalBlendPluralRoutes = collectAssistantAllowedRoutesForUser(USERS.internalBlendPlural, ALL_ROUTES, false);
-	assert(internalBlendPluralRoutes.includes('/chemical-blend/list'), 'Expected plural blend permission to map to canonical blend route');
-	assert(!internalBlendPluralRoutes.includes('/invoice/list'), 'Expected blend user without invoice permission to exclude invoice routes');
-
-	const customerRoutes = collectAssistantAllowedRoutesForUser(USERS.customerNoInvoice, ALL_ROUTES, false);
-	assert(customerRoutes.includes('/customer-portal/dashboard'), 'Expected customer to include customer dashboard');
-	assert(!customerRoutes.includes('/invoice/list'), 'Expected customer user to exclude internal invoice route');
-}
-
-function testNavigationFastPathPermissions() {
-	const invoicePrompt = 'Need the route link for invoice list';
-
-	const internalInvoiceRoutes = collectAssistantAllowedRoutesForUser(USERS.internalInvoice, ALL_ROUTES, false);
-	const internalInvoiceResult = evaluateAssistantNavigationFastPathWithRoutes({
-		message: invoicePrompt,
-		hasAttachments: false,
-		allowedRoutes: internalInvoiceRoutes
-	});
-	assert(!!internalInvoiceResult, 'Expected fast-path response for internal invoice user');
-	assert(
-		(internalInvoiceResult?.routes || []).some(route => route.includes('/invoice/list')),
-		'Expected invoice route for invoice-permitted user'
-	);
-
-	const internalNoInvoiceRoutes = collectAssistantAllowedRoutesForUser(USERS.internalNoInvoice, ALL_ROUTES, false);
-	const internalNoInvoiceResult = evaluateAssistantNavigationFastPathWithRoutes({
-		message: invoicePrompt,
-		hasAttachments: false,
-		allowedRoutes: internalNoInvoiceRoutes
-	});
-	assert(!!internalNoInvoiceResult, 'Expected fast-path response for no-invoice user');
-	assert(
-		!(internalNoInvoiceResult?.routes || []).some(route => route.includes('/invoice')),
-		'Expected no invoice routes for non-permitted user'
-	);
-	const wherePromptResult = evaluateAssistantNavigationFastPathWithRoutes({
-		message: 'Where is work order list?',
-		hasAttachments: false,
-		allowedRoutes: internalNoInvoiceRoutes
-	});
-	assert(!!wherePromptResult, 'Expected fast-path response for where/list route prompt');
-	assert(
-		(wherePromptResult?.routes || []).some(route => route.includes('/work-order/list')),
-		'Expected work-order list route for where/list prompt'
-	);
-
-	const customerWithInvoiceRoutes = collectAssistantAllowedRoutesForUser(USERS.customerWithInvoice, ALL_ROUTES, false);
-	const customerWithInvoiceResult = evaluateAssistantNavigationFastPathWithRoutes({
-		message: invoicePrompt,
-		hasAttachments: false,
-		allowedRoutes: customerWithInvoiceRoutes
-	});
-	assert(!!customerWithInvoiceResult, 'Expected fast-path response for customer invoice user');
-	assert(
-		(customerWithInvoiceResult?.routes || []).some(route => route.includes('/customer-portal/invoice/list')),
-		'Expected customer portal invoice route for permitted customer user'
-	);
-
-	const customerNoInvoiceRoutes = collectAssistantAllowedRoutesForUser(USERS.customerNoInvoice, ALL_ROUTES, false);
-	const customerNoInvoiceResult = evaluateAssistantNavigationFastPathWithRoutes({
-		message: invoicePrompt,
-		hasAttachments: false,
-		allowedRoutes: customerNoInvoiceRoutes
-	});
-	assert(!!customerNoInvoiceResult, 'Expected fast-path response for customer no-invoice user');
-	assert(
-		!(customerNoInvoiceResult?.routes || []).some(route => route.includes('/invoice')),
-		'Expected no invoice routes for customer without invoice permissions'
-	);
-	assert(
-		customerNoInvoiceResult?.reason === 'permission_denied' || customerNoInvoiceResult?.reason === 'ranked',
-		'Expected denied/ranked fallback reason for customer no-invoice prompt'
-	);
-}
-
-function testNavigationFastPathFuzzyRoutePermissions() {
-	const blendPrompt = 'Need the route link for chemical blend list';
-	const blendRoutes = collectAssistantAllowedRoutesForUser(USERS.internalBlendPlural, ALL_ROUTES, false);
-	const blendResult = evaluateAssistantNavigationFastPathWithRoutes({
-		message: blendPrompt,
-		hasAttachments: false,
-		allowedRoutes: blendRoutes
-	});
-	assert(!!blendResult, 'Expected fast-path response for blend-permitted user');
-	assert(
-		(blendResult?.routes || []).some(route => route.includes('/chemical-blend/list')),
-		'Expected canonical blend route for plural blend permission'
-	);
-
-	const noBlendRoutes = collectAssistantAllowedRoutesForUser(USERS.internalNoInvoice, ALL_ROUTES, false);
-	const noBlendResult = evaluateAssistantNavigationFastPathWithRoutes({
-		message: blendPrompt,
-		hasAttachments: false,
-		allowedRoutes: noBlendRoutes
-	});
-	assert(!!noBlendResult, 'Expected fast-path response for blend-denied user');
-	assert(
-		!(noBlendResult?.routes || []).some(route => route.includes('/chemical-blend')),
-		'Expected blend-denied user to avoid blend routes'
-	);
-	assert(
-		noBlendResult?.reason === 'permission_denied' || noBlendResult?.reason === 'ranked',
-		'Expected denied/ranked fallback reason for blend-denied user'
-	);
-
-	const orderAliasResult = evaluateAssistantNavigationFastPathWithRoutes({
-		message: 'I need the route link for /order/list',
-		hasAttachments: false,
-		allowedRoutes: noBlendRoutes
-	});
-	assert(!!orderAliasResult, 'Expected fast-path response for /order/list alias route');
-	assert(
-		(orderAliasResult?.routes || []).some(route => route.includes('/order/list') || route.includes('/work-order/list')),
-		'Expected order alias prompt to map to order/work-order route'
-	);
-}
-
-function testFastPathSkipsAttachmentAndDataPrompts() {
-	const routes = collectAssistantAllowedRoutesForUser(USERS.internalInvoice, ALL_ROUTES, false);
-	const attachmentResult = evaluateAssistantNavigationFastPathWithRoutes({
-		message: 'What route link for invoice list?',
-		hasAttachments: true,
-		allowedRoutes: routes
-	});
-	assert(!attachmentResult, 'Expected attachments to disable navigation fast path');
-
-	const dataResult = evaluateAssistantNavigationFastPathWithRoutes({
-		message: 'How many invoices were paid this month?',
-		hasAttachments: false,
-		allowedRoutes: routes
-	});
-	assert(!dataResult, 'Expected quantitative data prompt to skip navigation fast path');
-
-	const changeHistoryResult = evaluateAssistantNavigationFastPathWithRoutes({
-		message: 'What changed on this page?',
-		hasAttachments: false,
-		allowedRoutes: routes
-	});
-	assert(!changeHistoryResult, 'Expected change-history prompt to skip navigation fast path');
-	const durationOnPageResult = evaluateAssistantNavigationFastPathWithRoutes({
-		message: 'How long has the exception list been on this page?',
-		hasAttachments: false,
-		allowedRoutes: routes
-	});
-	assert(!durationOnPageResult, 'Expected "how long on this page" prompt to skip navigation fast path');
-
-	const bugIssueResult = evaluateAssistantNavigationFastPathWithRoutes({
-		message: 'Why don\'t I see this asset in this dropdown?',
-		hasAttachments: false,
-		allowedRoutes: routes
-	});
-	assert(!bugIssueResult, 'Expected bug/issue prompt to skip navigation fast path');
-}
-
-function testChangeHistoryFastPathAndPermissionSummary() {
-	assert(
-		shouldUseAssistantChangeHistoryFastPath('What changed recently in invoices?', false),
-		'Expected change-history question to use change-history fast path'
-	);
-	assert(
-		shouldUseAssistantChangeHistoryFastPath('When was the last time the invoice document was updated?', false),
-		'Expected "last updated" question to use change-history fast path'
-	);
-	assert(
-		shouldUseAssistantChangeHistoryFastPath('When does this dropdown for XYZ get added to this page?', false),
-		'Expected "dropdown added" question to use change-history fast path'
-	);
-	assert(
-		shouldUseAssistantChangeHistoryFastPath('What changed on this page?', false),
-		'Expected "what changed on this page" question to use change-history fast path'
-	);
-	assert(
-		shouldUseAssistantChangeHistoryFastPath('What changed on the invoice page?', false),
-		'Expected "what changed on invoice page" question to use change-history fast path'
-	);
-	assert(
-		shouldUseAssistantChangeHistoryFastPath('What did the last update just do?', false),
-		'Expected "last update did" question to use change-history fast path'
-	);
-	assert(
-		shouldUseAssistantChangeHistoryFastPath('What did the last system update do?', false),
-		'Expected "last system update did" question to use change-history fast path'
-	);
-	assert(
-		shouldUseAssistantChangeHistoryFastPath('What is this new exception list? How long has it been around?', false),
-		'Expected "been around" question to use change-history fast path'
-	);
-	assert(
-		shouldUseAssistantChangeHistoryFastPath('How long has the exception list been on this page?', false),
-		'Expected "been on this page" question to use change-history fast path'
-	);
-	assert(
-		!shouldUseAssistantChangeHistoryFastPath('What changed recently in invoices?', true),
-		'Expected attachments to disable change-history fast path'
-	);
-
-	const commits = [
-		{
-			date: '2026-02-05T12:00:00.000Z',
-			subject: 'Fix invoice approval validation edge case',
-			files: ['server/invoice/approval.ts'],
-			modules: ['invoice']
-		},
-		{
-			date: '2026-02-04T12:00:00.000Z',
-			subject: 'Improve work order dispatch retries',
-			files: ['server/work-order/dispatch.ts'],
-			modules: ['work_order']
-		}
-	] as any[];
-
-	const permitted = buildAssistantChangeHistorySummaryFromCommits({
-		message: 'What changed recently in invoices?',
-		user: USERS.internalInvoice,
-		isSuperAdmin: false,
-		commits,
-		branch: 'main',
-		now: new Date('2026-02-08T00:00:00.000Z')
-	});
-	assert(!permitted.generic, 'Expected invoice-permitted user to receive non-generic change summary');
-	assert(
-		String(permitted.response || '').includes('Fix invoice approval validation edge case'),
-		'Expected permitted summary to include invoice change detail'
-	);
-
-	const restricted = buildAssistantChangeHistorySummaryFromCommits({
-		message: 'What changed recently in invoices?',
-		user: USERS.internalNoInvoice,
-		isSuperAdmin: false,
-		commits,
-		branch: 'main',
-		now: new Date('2026-02-08T00:00:00.000Z')
-	});
-	assert(restricted.generic, 'Expected invoice-restricted user to receive generic change summary');
-	assert(
-		!String(restricted.response || '').includes('Fix invoice approval validation edge case'),
-		'Expected restricted summary to omit invoice detail'
-	);
-}
-
-function run() {
-	testPromptClassificationCoverage();
-	testUserAccessTiersAndInvoicePermissions();
-	testAllowedRoutesByUser();
-	testNavigationFastPathPermissions();
-	testNavigationFastPathFuzzyRoutePermissions();
-	testFastPathSkipsAttachmentAndDataPrompts();
-	testChangeHistoryFastPathAndPermissionSummary();
-	console.log('ai assistant routing eval tests passed');
-}
-
-try {
-	run();
-	process.exit(process.exitCode ?? 0);
-}
-catch (error) {
-	console.error(error);
-	process.exit(1);
-}