@resolveio/server-lib 22.3.219 → 22.3.221

package/tests/ai-assistant-openai-git-e2e.ts

@@ -0,0 +1,155 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { spawnSync } from 'child_process';
+
+type BaseResult = {
+	id: string;
+	pass: boolean;
+	error?: string;
+	responsePreview?: string;
+	fastPath?: {
+		type?: string;
+		reason?: string;
+		generic?: boolean;
+		commitCount?: number;
+	};
+};
+
+type GitHarnessCheck = {
+	id: string;
+	pass: boolean;
+	issues: string[];
+	fastPathType?: string;
+	commitCount?: number;
+};
+
+const DEFAULT_CASE_IDS = [
+	'change_history_last_system_update_did_what',
+	'change_history_that_last_system_update_did_what'
+];
+
+const FORBIDDEN_FALLBACK_SNIPPETS = [
+	'i could not locate a readable git repository from configured workspace paths',
+	'i could not read repository history in this environment right now',
+	'exact rollout dates are unavailable'
+];
+
+const CHANGE_BULLET_DATE_PATTERN = /-\s+\d{4}-\d{2}-\d{2}:/;
+
+function parseCaseIds(raw: string): string[] {
+	const ids = String(raw || '')
+		.split(',')
+		.map(value => value.trim())
+		.filter(Boolean);
+	return ids.length ? ids : DEFAULT_CASE_IDS;
+}
+
+function runBaseHarness(repoRoot: string, caseIds: string[]): void {
+	const env = {
+		...process.env,
+		AI_ASSISTANT_E2E_CASES: caseIds.join(',')
+	};
+	const run = spawnSync(
+		process.execPath,
+		['--require', 'ts-node/register', 'tests/ai-assistant-openai-e2e.ts'],
+		{
+			cwd: repoRoot,
+			env,
+			stdio: 'inherit'
+		}
+	);
+	if ((run.status || 0) !== 0) {
+		throw new Error(`Base OpenAI E2E harness failed with exit code ${run.status ?? 'unknown'}.`);
+	}
+}
+
+function validateResult(result: BaseResult | undefined, id: string): GitHarnessCheck {
+	if (!result) {
+		return {
+			id,
+			pass: false,
+			issues: ['Case result missing from base harness output.']
+		};
+	}
+	const issues: string[] = [];
+	if (!result.pass) {
+		issues.push(`Base harness marked case as failed: ${String(result.error || 'unknown error')}`);
+	}
+	const preview = String(result.responsePreview || '');
+	const loweredPreview = preview.toLowerCase();
+	for (const snippet of FORBIDDEN_FALLBACK_SNIPPETS) {
+		if (loweredPreview.includes(snippet)) {
+			issues.push(`Response used fallback text: "${snippet}".`);
+		}
+	}
+	if (!CHANGE_BULLET_DATE_PATTERN.test(preview)) {
+		issues.push('Response did not include a concrete dated change bullet (expected "- YYYY-MM-DD: ...").');
+	}
+	const fastPathType = String(result.fastPath?.type || '');
+	const commitCount = Number(result.fastPath?.commitCount || 0);
+	if (fastPathType === 'change_history') {
+		if (commitCount < 1) {
+			issues.push(`Expected fast-path commit count >= 1, got ${commitCount}.`);
+		}
+		if (result.fastPath?.generic === true) {
+			issues.push('Expected non-generic change-history response for this prompt.');
+		}
+	}
+	return {
+		id,
+		pass: issues.length === 0,
+		issues,
+		fastPathType: fastPathType || undefined,
+		commitCount: Number.isFinite(commitCount) ? commitCount : undefined
+	};
+}
+
+function run(): void {
+	const repoRoot = process.cwd();
+	const caseIds = parseCaseIds(process.env.AI_ASSISTANT_E2E_CASES || '');
+	runBaseHarness(repoRoot, caseIds);
+
+	const baseOutputPath = path.join(repoRoot, 'tmp', 'ai-assistant-openai-e2e-output.json');
+	if (!fs.existsSync(baseOutputPath)) {
+		throw new Error(`Base harness output not found at ${baseOutputPath}`);
+	}
+	const parsed = JSON.parse(fs.readFileSync(baseOutputPath, 'utf8'));
+	const results: BaseResult[] = Array.isArray(parsed?.results) ? parsed.results : [];
+	const resultById = new Map<string, BaseResult>();
+	results.forEach((entry) => {
+		if (entry?.id) {
+			resultById.set(entry.id, entry);
+		}
+	});
+
+	const checks = caseIds.map(id => validateResult(resultById.get(id), id));
+	const failed = checks.filter(check => !check.pass);
+	const report = {
+		now: new Date().toISOString(),
+		total: checks.length,
+		failed: failed.length,
+		passed: checks.length - failed.length,
+		caseIds,
+		checks
+	};
+	const reportPath = path.join(repoRoot, 'tmp', 'ai-assistant-openai-git-e2e-output.json');
+	fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
+
+	if (failed.length) {
+		failed.forEach((failure) => {
+			console.log(`FAIL ${failure.id}`);
+			failure.issues.forEach((issue) => console.log(`  - ${issue}`));
+		});
+		console.log(`ai assistant openai git e2e: ${report.passed}/${report.total} passed`);
+		console.log(`wrote ${reportPath}`);
+		process.exit(1);
+	}
+
+	checks.forEach((check) => {
+		console.log(`PASS ${check.id}`);
+	});
+	console.log(`ai assistant openai git e2e: ${report.passed}/${report.total} passed`);
+	console.log(`wrote ${reportPath}`);
+}
+
+run();

package/tests/ai-assistant-preflight-matrix.ts

@@ -0,0 +1,215 @@
+type AssistantAccessTier = 'super_admin' | 'client_user' | 'customer_portal';
+type AssistantExpectationType = 'data' | 'navigation' | 'bug_issue' | 'feature_info' | 'mixed';
+
+type AssistantPreflightScenario = {
+	id: string;
+	title: string;
+	accessTier: AssistantAccessTier;
+	prompt: string;
+	expectation: {
+		type: AssistantExpectationType;
+		requiresDataTool: boolean;
+		preferredCollectionFamilies?: string[];
+		requiresCustomerScope?: boolean;
+		requiresPermissionError?: boolean;
+		notes?: string;
+	};
+};
+
+function assert(condition: boolean, message: string) {
+	if (!condition) {
+		throw new Error(message);
+	}
+}
+
+const PRE_FLIGHT_MATRIX: AssistantPreflightScenario[] = [
+	{
+		id: 'rb-001',
+		title: 'Invoice monthly revenue trend',
+		accessTier: 'client_user',
+		prompt: 'Break down total paid invoice revenue over the last 6 full months by month.',
+		expectation: {
+			type: 'data',
+			requiresDataTool: true,
+			preferredCollectionFamilies: ['invoices', 'report-invoices'],
+			notes: 'Should select REPORT_BUILDER_AGG with calendar-period windows.'
+		}
+	},
+	{
+		id: 'rb-002',
+		title: 'Recent work orders list',
+		accessTier: 'client_user',
+		prompt: 'Show the last 20 work orders created this week with status and customer.',
+		expectation: {
+			type: 'data',
+			requiresDataTool: true,
+			preferredCollectionFamilies: ['work-orders', 'work-order-dynamics']
+		}
+	},
+	{
+		id: 'rb-003',
+		title: 'Name-based chemical lookup',
+		accessTier: 'client_user',
+		prompt: 'Which blends used Chemical Red Ultra Lift in the past 30 days?',
+		expectation: {
+			type: 'data',
+			requiresDataTool: true,
+			preferredCollectionFamilies: ['blends', 'chemicals'],
+			notes: 'Should use name matching and fallback ID lookup if needed.'
+		}
+	},
+	{
+		id: 'rb-004',
+		title: 'Customer portal scoped invoices',
+		accessTier: 'customer_portal',
+		prompt: 'How many invoices were paid last month?',
+		expectation: {
+			type: 'data',
+			requiresDataTool: true,
+			preferredCollectionFamilies: ['invoices'],
+			requiresCustomerScope: true
+		}
+	},
+	{
+		id: 'rb-005',
+		title: 'Customer portal cross-customer denial',
+		accessTier: 'customer_portal',
+		prompt: 'Show me all customers with unpaid invoices.',
+		expectation: {
+			type: 'data',
+			requiresDataTool: false,
+			requiresPermissionError: true,
+			notes: 'Should deny cross-customer scope expansion.'
+		}
+	},
+	{
+		id: 'rb-006',
+		title: 'Invoice permission denial',
+		accessTier: 'client_user',
+		prompt: 'List invoice totals by customer for last quarter.',
+		expectation: {
+			type: 'data',
+			requiresDataTool: false,
+			requiresPermissionError: true,
+			notes: 'Use when test user lacks invoice permission.'
+		}
+	},
+	{
+		id: 'rb-007',
+		title: 'Route/navigation request',
+		accessTier: 'client_user',
+		prompt: 'Where do I go to edit a work order?',
+		expectation: {
+			type: 'navigation',
+			requiresDataTool: false
+		}
+	},
+	{
+		id: 'rb-008',
+		title: 'Mixed route + data request',
+		accessTier: 'client_user',
+		prompt: 'Where is invoice list and how many invoices are unpaid right now?',
+		expectation: {
+			type: 'mixed',
+			requiresDataTool: true,
+			preferredCollectionFamilies: ['invoices']
+		}
+	},
+	{
+		id: 'rb-009',
+		title: 'Bug/issue question',
+		accessTier: 'client_user',
+		prompt: 'Invoice screen throws an exception when I click save.',
+		expectation: {
+			type: 'bug_issue',
+			requiresDataTool: false
+		}
+	},
+	{
+		id: 'rb-010',
+		title: 'Feature info / change-history style',
+		accessTier: 'client_user',
+		prompt: 'What changed recently on the invoice page?',
+		expectation: {
+			type: 'feature_info',
+			requiresDataTool: false
+		}
+	},
+	{
+		id: 'rb-011',
+		title: 'RB fallback to direct collection expected',
+		accessTier: 'super_admin',
+		prompt: 'Show top 10 support tickets by billed hours this month.',
+		expectation: {
+			type: 'data',
+			requiresDataTool: true,
+			preferredCollectionFamilies: ['support-tickets', 'time-entries'],
+			notes: 'Validates report-builder-first with direct collection fallback.'
+		}
+	},
+	{
+		id: 'rb-012',
+		title: 'Grouped aggregate by user',
+		accessTier: 'super_admin',
+		prompt: 'Break down total billed support-ticket hours per user for the last 6 months.',
+		expectation: {
+			type: 'data',
+			requiresDataTool: true,
+			preferredCollectionFamilies: ['support-tickets', 'users']
+		}
+	}
+];
+
+function validateMatrix(scenarios: AssistantPreflightScenario[]) {
+	assert(Array.isArray(scenarios) && scenarios.length > 0, 'Preflight matrix must include scenarios');
+	const ids = new Set<string>();
+	scenarios.forEach((scenario) => {
+		assert(!!scenario.id && scenario.id.trim().length >= 4, `Scenario id is invalid: ${scenario.id}`);
+		assert(!ids.has(scenario.id), `Duplicate scenario id: ${scenario.id}`);
+		ids.add(scenario.id);
+		assert(!!scenario.title, `Scenario title missing: ${scenario.id}`);
+		assert(!!scenario.prompt, `Scenario prompt missing: ${scenario.id}`);
+		assert(!!scenario.expectation?.type, `Scenario expectation type missing: ${scenario.id}`);
+		if (scenario.expectation.requiresPermissionError) {
+			assert(!scenario.expectation.requiresDataTool, `Permission-error scenario should not require data tool: ${scenario.id}`);
+		}
+	});
+}
+
+function renderMatrixMarkdown(scenarios: AssistantPreflightScenario[]): string {
+	const rows = scenarios.map((scenario) => {
+		const families = (scenario.expectation.preferredCollectionFamilies || []).join(', ');
+		const flags = [
+			scenario.expectation.requiresDataTool ? 'data-tool' : 'no-tool',
+			scenario.expectation.requiresCustomerScope ? 'customer-scope' : '',
+			scenario.expectation.requiresPermissionError ? 'permission-deny' : ''
+		].filter(Boolean).join(', ');
+		return `| ${scenario.id} | ${scenario.accessTier} | ${scenario.expectation.type} | ${flags || '-'} | ${families || '-'} | ${scenario.prompt} |`;
+	});
+	return [
+		'| id | access | type | flags | preferred collections | prompt |',
+		'| --- | --- | --- | --- | --- | --- |',
+		...rows
+	].join('\n');
+}
+
+function run() {
+	validateMatrix(PRE_FLIGHT_MATRIX);
+	if (process.argv.includes('--json')) {
+		console.log(JSON.stringify(PRE_FLIGHT_MATRIX, null, 2));
+		return;
+	}
+	console.log('AI assistant preflight matrix validated.');
+	console.log(`Scenario count: ${PRE_FLIGHT_MATRIX.length}`);
+	console.log(renderMatrixMarkdown(PRE_FLIGHT_MATRIX));
+}
+
+try {
+	run();
+	process.exit(process.exitCode ?? 0);
+}
+catch (error) {
+	console.error(error);
+	process.exit(1);
+}
+