@resolveio/server-lib 22.3.221 → 22.3.223

package/tests/ai-assistant-openai-git-e2e.ts

@@ -1,155 +0,0 @@
-import * as fs from 'fs';
-import * as path from 'path';
-import { spawnSync } from 'child_process';
-
-type BaseResult = {
-	id: string;
-	pass: boolean;
-	error?: string;
-	responsePreview?: string;
-	fastPath?: {
-		type?: string;
-		reason?: string;
-		generic?: boolean;
-		commitCount?: number;
-	};
-};
-
-type GitHarnessCheck = {
-	id: string;
-	pass: boolean;
-	issues: string[];
-	fastPathType?: string;
-	commitCount?: number;
-};
-
-const DEFAULT_CASE_IDS = [
-	'change_history_last_system_update_did_what',
-	'change_history_that_last_system_update_did_what'
-];
-
-const FORBIDDEN_FALLBACK_SNIPPETS = [
-	'i could not locate a readable git repository from configured workspace paths',
-	'i could not read repository history in this environment right now',
-	'exact rollout dates are unavailable'
-];
-
-const CHANGE_BULLET_DATE_PATTERN = /-\s+\d{4}-\d{2}-\d{2}:/;
-
-function parseCaseIds(raw: string): string[] {
-	const ids = String(raw || '')
-		.split(',')
-		.map(value => value.trim())
-		.filter(Boolean);
-	return ids.length ? ids : DEFAULT_CASE_IDS;
-}
-
-function runBaseHarness(repoRoot: string, caseIds: string[]): void {
-	const env = {
-		...process.env,
-		AI_ASSISTANT_E2E_CASES: caseIds.join(',')
-	};
-	const run = spawnSync(
-		process.execPath,
-		['--require', 'ts-node/register', 'tests/ai-assistant-openai-e2e.ts'],
-		{
-			cwd: repoRoot,
-			env,
-			stdio: 'inherit'
-		}
-	);
-	if ((run.status || 0) !== 0) {
-		throw new Error(`Base OpenAI E2E harness failed with exit code ${run.status ?? 'unknown'}.`);
-	}
-}
-
-function validateResult(result: BaseResult | undefined, id: string): GitHarnessCheck {
-	if (!result) {
-		return {
-			id,
-			pass: false,
-			issues: ['Case result missing from base harness output.']
-		};
-	}
-	const issues: string[] = [];
-	if (!result.pass) {
-		issues.push(`Base harness marked case as failed: ${String(result.error || 'unknown error')}`);
-	}
-	const preview = String(result.responsePreview || '');
-	const loweredPreview = preview.toLowerCase();
-	for (const snippet of FORBIDDEN_FALLBACK_SNIPPETS) {
-		if (loweredPreview.includes(snippet)) {
-			issues.push(`Response used fallback text: "${snippet}".`);
-		}
-	}
-	if (!CHANGE_BULLET_DATE_PATTERN.test(preview)) {
-		issues.push('Response did not include a concrete dated change bullet (expected "- YYYY-MM-DD: ...").');
-	}
-	const fastPathType = String(result.fastPath?.type || '');
-	const commitCount = Number(result.fastPath?.commitCount || 0);
-	if (fastPathType === 'change_history') {
-		if (commitCount < 1) {
-			issues.push(`Expected fast-path commit count >= 1, got ${commitCount}.`);
-		}
-		if (result.fastPath?.generic === true) {
-			issues.push('Expected non-generic change-history response for this prompt.');
-		}
-	}
-	return {
-		id,
-		pass: issues.length === 0,
-		issues,
-		fastPathType: fastPathType || undefined,
-		commitCount: Number.isFinite(commitCount) ? commitCount : undefined
-	};
-}
-
-function run(): void {
-	const repoRoot = process.cwd();
-	const caseIds = parseCaseIds(process.env.AI_ASSISTANT_E2E_CASES || '');
-	runBaseHarness(repoRoot, caseIds);
-
-	const baseOutputPath = path.join(repoRoot, 'tmp', 'ai-assistant-openai-e2e-output.json');
-	if (!fs.existsSync(baseOutputPath)) {
-		throw new Error(`Base harness output not found at ${baseOutputPath}`);
-	}
-	const parsed = JSON.parse(fs.readFileSync(baseOutputPath, 'utf8'));
-	const results: BaseResult[] = Array.isArray(parsed?.results) ? parsed.results : [];
-	const resultById = new Map<string, BaseResult>();
-	results.forEach((entry) => {
-		if (entry?.id) {
-			resultById.set(entry.id, entry);
-		}
-	});
-
-	const checks = caseIds.map(id => validateResult(resultById.get(id), id));
-	const failed = checks.filter(check => !check.pass);
-	const report = {
-		now: new Date().toISOString(),
-		total: checks.length,
-		failed: failed.length,
-		passed: checks.length - failed.length,
-		caseIds,
-		checks
-	};
-	const reportPath = path.join(repoRoot, 'tmp', 'ai-assistant-openai-git-e2e-output.json');
-	fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
-
-	if (failed.length) {
-		failed.forEach((failure) => {
-			console.log(`FAIL ${failure.id}`);
-			failure.issues.forEach((issue) => console.log(`  - ${issue}`));
-		});
-		console.log(`ai assistant openai git e2e: ${report.passed}/${report.total} passed`);
-		console.log(`wrote ${reportPath}`);
-		process.exit(1);
-	}
-
-	checks.forEach((check) => {
-		console.log(`PASS ${check.id}`);
-	});
-	console.log(`ai assistant openai git e2e: ${report.passed}/${report.total} passed`);
-	console.log(`wrote ${reportPath}`);
-}
-
-run();

package/tests/ai-assistant-preflight-matrix.ts

@@ -1,215 +0,0 @@
-type AssistantAccessTier = 'super_admin' | 'client_user' | 'customer_portal';
-type AssistantExpectationType = 'data' | 'navigation' | 'bug_issue' | 'feature_info' | 'mixed';
-
-type AssistantPreflightScenario = {
-	id: string;
-	title: string;
-	accessTier: AssistantAccessTier;
-	prompt: string;
-	expectation: {
-		type: AssistantExpectationType;
-		requiresDataTool: boolean;
-		preferredCollectionFamilies?: string[];
-		requiresCustomerScope?: boolean;
-		requiresPermissionError?: boolean;
-		notes?: string;
-	};
-};
-
-function assert(condition: boolean, message: string) {
-	if (!condition) {
-		throw new Error(message);
-	}
-}
-
-const PRE_FLIGHT_MATRIX: AssistantPreflightScenario[] = [
-	{
-		id: 'rb-001',
-		title: 'Invoice monthly revenue trend',
-		accessTier: 'client_user',
-		prompt: 'Break down total paid invoice revenue over the last 6 full months by month.',
-		expectation: {
-			type: 'data',
-			requiresDataTool: true,
-			preferredCollectionFamilies: ['invoices', 'report-invoices'],
-			notes: 'Should select REPORT_BUILDER_AGG with calendar-period windows.'
-		}
-	},
-	{
-		id: 'rb-002',
-		title: 'Recent work orders list',
-		accessTier: 'client_user',
-		prompt: 'Show the last 20 work orders created this week with status and customer.',
-		expectation: {
-			type: 'data',
-			requiresDataTool: true,
-			preferredCollectionFamilies: ['work-orders', 'work-order-dynamics']
-		}
-	},
-	{
-		id: 'rb-003',
-		title: 'Name-based chemical lookup',
-		accessTier: 'client_user',
-		prompt: 'Which blends used Chemical Red Ultra Lift in the past 30 days?',
-		expectation: {
-			type: 'data',
-			requiresDataTool: true,
-			preferredCollectionFamilies: ['blends', 'chemicals'],
-			notes: 'Should use name matching and fallback ID lookup if needed.'
-		}
-	},
-	{
-		id: 'rb-004',
-		title: 'Customer portal scoped invoices',
-		accessTier: 'customer_portal',
-		prompt: 'How many invoices were paid last month?',
-		expectation: {
-			type: 'data',
-			requiresDataTool: true,
-			preferredCollectionFamilies: ['invoices'],
-			requiresCustomerScope: true
-		}
-	},
-	{
-		id: 'rb-005',
-		title: 'Customer portal cross-customer denial',
-		accessTier: 'customer_portal',
-		prompt: 'Show me all customers with unpaid invoices.',
-		expectation: {
-			type: 'data',
-			requiresDataTool: false,
-			requiresPermissionError: true,
-			notes: 'Should deny cross-customer scope expansion.'
-		}
-	},
-	{
-		id: 'rb-006',
-		title: 'Invoice permission denial',
-		accessTier: 'client_user',
-		prompt: 'List invoice totals by customer for last quarter.',
-		expectation: {
-			type: 'data',
-			requiresDataTool: false,
-			requiresPermissionError: true,
-			notes: 'Use when test user lacks invoice permission.'
-		}
-	},
-	{
-		id: 'rb-007',
-		title: 'Route/navigation request',
-		accessTier: 'client_user',
-		prompt: 'Where do I go to edit a work order?',
-		expectation: {
-			type: 'navigation',
-			requiresDataTool: false
-		}
-	},
-	{
-		id: 'rb-008',
-		title: 'Mixed route + data request',
-		accessTier: 'client_user',
-		prompt: 'Where is invoice list and how many invoices are unpaid right now?',
-		expectation: {
-			type: 'mixed',
-			requiresDataTool: true,
-			preferredCollectionFamilies: ['invoices']
-		}
-	},
-	{
-		id: 'rb-009',
-		title: 'Bug/issue question',
-		accessTier: 'client_user',
-		prompt: 'Invoice screen throws an exception when I click save.',
-		expectation: {
-			type: 'bug_issue',
-			requiresDataTool: false
-		}
-	},
-	{
-		id: 'rb-010',
-		title: 'Feature info / change-history style',
-		accessTier: 'client_user',
-		prompt: 'What changed recently on the invoice page?',
-		expectation: {
-			type: 'feature_info',
-			requiresDataTool: false
-		}
-	},
-	{
-		id: 'rb-011',
-		title: 'RB fallback to direct collection expected',
-		accessTier: 'super_admin',
-		prompt: 'Show top 10 support tickets by billed hours this month.',
-		expectation: {
-			type: 'data',
-			requiresDataTool: true,
-			preferredCollectionFamilies: ['support-tickets', 'time-entries'],
-			notes: 'Validates report-builder-first with direct collection fallback.'
-		}
-	},
-	{
-		id: 'rb-012',
-		title: 'Grouped aggregate by user',
-		accessTier: 'super_admin',
-		prompt: 'Break down total billed support-ticket hours per user for the last 6 months.',
-		expectation: {
-			type: 'data',
-			requiresDataTool: true,
-			preferredCollectionFamilies: ['support-tickets', 'users']
-		}
-	}
-];
-
-function validateMatrix(scenarios: AssistantPreflightScenario[]) {
-	assert(Array.isArray(scenarios) && scenarios.length > 0, 'Preflight matrix must include scenarios');
-	const ids = new Set<string>();
-	scenarios.forEach((scenario) => {
-		assert(!!scenario.id && scenario.id.trim().length >= 4, `Scenario id is invalid: ${scenario.id}`);
-		assert(!ids.has(scenario.id), `Duplicate scenario id: ${scenario.id}`);
-		ids.add(scenario.id);
-		assert(!!scenario.title, `Scenario title missing: ${scenario.id}`);
-		assert(!!scenario.prompt, `Scenario prompt missing: ${scenario.id}`);
-		assert(!!scenario.expectation?.type, `Scenario expectation type missing: ${scenario.id}`);
-		if (scenario.expectation.requiresPermissionError) {
-			assert(!scenario.expectation.requiresDataTool, `Permission-error scenario should not require data tool: ${scenario.id}`);
-		}
-	});
-}
-
-function renderMatrixMarkdown(scenarios: AssistantPreflightScenario[]): string {
-	const rows = scenarios.map((scenario) => {
-		const families = (scenario.expectation.preferredCollectionFamilies || []).join(', ');
-		const flags = [
-			scenario.expectation.requiresDataTool ? 'data-tool' : 'no-tool',
-			scenario.expectation.requiresCustomerScope ? 'customer-scope' : '',
-			scenario.expectation.requiresPermissionError ? 'permission-deny' : ''
-		].filter(Boolean).join(', ');
-		return `| ${scenario.id} | ${scenario.accessTier} | ${scenario.expectation.type} | ${flags || '-'} | ${families || '-'} | ${scenario.prompt} |`;
-	});
-	return [
-		'| id | access | type | flags | preferred collections | prompt |',
-		'| --- | --- | --- | --- | --- | --- |',
-		...rows
-	].join('\n');
-}
-
-function run() {
-	validateMatrix(PRE_FLIGHT_MATRIX);
-	if (process.argv.includes('--json')) {
-		console.log(JSON.stringify(PRE_FLIGHT_MATRIX, null, 2));
-		return;
-	}
-	console.log('AI assistant preflight matrix validated.');
-	console.log(`Scenario count: ${PRE_FLIGHT_MATRIX.length}`);
-	console.log(renderMatrixMarkdown(PRE_FLIGHT_MATRIX));
-}
-
-try {
-	run();
-	process.exit(process.exitCode ?? 0);
-}
-catch (error) {
-	console.error(error);
-	process.exit(1);
-}
-