@yasserkhanorg/e2e-agents 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/plan.d.ts +2 -1
- package/dist/agent/plan.d.ts.map +1 -1
- package/dist/api.d.ts +4 -0
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +38 -0
- package/dist/cli.js +57 -14
- package/dist/engine/ai_enrichment.d.ts +43 -0
- package/dist/engine/ai_enrichment.d.ts.map +1 -0
- package/dist/engine/ai_enrichment.js +235 -0
- package/dist/engine/diff_loader.d.ts +11 -0
- package/dist/engine/diff_loader.d.ts.map +1 -0
- package/dist/engine/diff_loader.js +74 -0
- package/dist/engine/plan_builder.d.ts +2 -1
- package/dist/engine/plan_builder.d.ts.map +1 -1
- package/dist/engine/plan_builder.js +60 -15
- package/dist/esm/api.js +37 -0
- package/dist/esm/cli.js +58 -15
- package/dist/esm/engine/ai_enrichment.js +232 -0
- package/dist/esm/engine/diff_loader.js +70 -0
- package/dist/esm/engine/plan_builder.js +60 -15
- package/dist/esm/knowledge/route_families.js +2 -1
- package/dist/knowledge/route_families.d.ts.map +1 -1
- package/dist/knowledge/route_families.js +2 -1
- package/package.json +1 -1
|
@@ -199,7 +199,7 @@ function buildRecommendedTests(impact) {
|
|
|
199
199
|
}
|
|
200
200
|
return tests;
|
|
201
201
|
}
|
|
202
|
-
function buildPlanFromImpact(impact, policyOverride) {
|
|
202
|
+
function buildPlanFromImpact(impact, policyOverride, aiEnrichment) {
|
|
203
203
|
const policy = { ...DEFAULT_POLICY, ...(policyOverride || {}) };
|
|
204
204
|
const confidence = computeConfidence(impact);
|
|
205
205
|
const runSetResult = pickRunSet(impact, confidence, policy);
|
|
@@ -207,23 +207,61 @@ function buildPlanFromImpact(impact, policyOverride) {
|
|
|
207
207
|
const enforcement = evaluateEnforcement(decision, policy);
|
|
208
208
|
const gaps = (0, impact_engine_js_1.getGaps)(impact);
|
|
209
209
|
const partialGaps = (0, impact_engine_js_1.getPartialGaps)(impact);
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
210
|
+
// Build two separate lookup maps from aiEnrichment: one by featureId, one by familyId.
|
|
211
|
+
// The familyId map stores only the FIRST feature encountered to avoid last-write-wins collisions.
|
|
212
|
+
const aiFeatureByFeatureId = new Map();
|
|
213
|
+
const aiFeatureByFamilyId = new Map();
|
|
214
|
+
if (aiEnrichment) {
|
|
215
|
+
for (const ef of aiEnrichment.enrichedFeatures) {
|
|
216
|
+
if (ef.featureId) {
|
|
217
|
+
aiFeatureByFeatureId.set(ef.featureId, ef);
|
|
218
|
+
}
|
|
219
|
+
if (ef.familyId && !aiFeatureByFamilyId.has(ef.familyId)) {
|
|
220
|
+
aiFeatureByFamilyId.set(ef.familyId, ef);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
const gapDetails = gaps.map((f) => {
|
|
225
|
+
const label = featureLabel(f);
|
|
226
|
+
const aiFeature = f.featureId
|
|
227
|
+
? (aiFeatureByFeatureId.get(f.featureId) ?? aiFeatureByFamilyId.get(f.familyId))
|
|
228
|
+
: aiFeatureByFamilyId.get(f.familyId);
|
|
229
|
+
const baseReasons = [`No Playwright or Cypress tests found for ${label}`];
|
|
230
|
+
const reasons = aiFeature && aiFeature.aiReasons.length > 0
|
|
231
|
+
? [...baseReasons, ...aiFeature.aiReasons]
|
|
232
|
+
: baseReasons;
|
|
233
|
+
const missingScenarios = aiFeature && aiFeature.aiMissingScenarios.length > 0
|
|
234
|
+
? aiFeature.aiMissingScenarios
|
|
235
|
+
: (f.userFlows.length > 0 ? f.userFlows.slice(0, 5) : undefined);
|
|
236
|
+
return {
|
|
237
|
+
id: label,
|
|
238
|
+
name: label,
|
|
239
|
+
priority: f.priority,
|
|
240
|
+
reasons,
|
|
241
|
+
files: f.changedFiles.slice(0, 6),
|
|
242
|
+
missingScenarios,
|
|
243
|
+
source: aiFeature ? 'ai+deterministic' : 'deterministic',
|
|
244
|
+
};
|
|
245
|
+
});
|
|
218
246
|
// Add partial gaps as advisory info
|
|
219
247
|
for (const f of partialGaps) {
|
|
220
|
-
const
|
|
248
|
+
const coverageType = f.playwrightSpecs.length > 0 ? 'Cypress' : 'Playwright';
|
|
249
|
+
const hasOpposite = f.playwrightSpecs.length > 0 ? 'Playwright' : 'Cypress';
|
|
250
|
+
const label = featureLabel(f);
|
|
251
|
+
const aiFeature = f.featureId
|
|
252
|
+
? (aiFeatureByFeatureId.get(f.featureId) ?? aiFeatureByFamilyId.get(f.familyId))
|
|
253
|
+
: aiFeatureByFamilyId.get(f.familyId);
|
|
254
|
+
const baseReasons = [`Missing ${coverageType} tests for ${label} (has ${hasOpposite} only)`];
|
|
255
|
+
const reasons = aiFeature && aiFeature.aiReasons.length > 0
|
|
256
|
+
? [...baseReasons, ...aiFeature.aiReasons]
|
|
257
|
+
: baseReasons;
|
|
221
258
|
gapDetails.push({
|
|
222
|
-
id:
|
|
223
|
-
name: `${
|
|
259
|
+
id: label,
|
|
260
|
+
name: `${label} (partial)`,
|
|
224
261
|
priority: f.priority,
|
|
225
|
-
reasons
|
|
262
|
+
reasons,
|
|
226
263
|
files: f.changedFiles.slice(0, 6),
|
|
264
|
+
source: aiFeature ? 'ai+deterministic' : 'deterministic',
|
|
227
265
|
});
|
|
228
266
|
}
|
|
229
267
|
const coveredFlows = impact.impactedFeatures
|
|
@@ -243,11 +281,12 @@ function buildPlanFromImpact(impact, policyOverride) {
|
|
|
243
281
|
const p1 = impact.impactedFeatures.filter((f) => f.priority === 'P1').length;
|
|
244
282
|
const p2 = impact.impactedFeatures.filter((f) => f.priority === 'P2').length;
|
|
245
283
|
const runId = `plan-${Date.now().toString(36)}`;
|
|
284
|
+
const planSource = aiEnrichment ? 'ai+deterministic' : 'impact';
|
|
246
285
|
return {
|
|
247
286
|
schemaVersion: '1.0.0',
|
|
248
287
|
runId,
|
|
249
288
|
generatedAt: new Date().toISOString(),
|
|
250
|
-
source:
|
|
289
|
+
source: planSource,
|
|
251
290
|
runSet: runSetResult.runSet,
|
|
252
291
|
confidence,
|
|
253
292
|
reasons: runSetResult.reasons,
|
|
@@ -298,12 +337,18 @@ function renderCiSummaryMarkdown(plan) {
|
|
|
298
337
|
lines.push(`The following ${uncoveredP0P1Flows} feature(s) have no test coverage and must be covered before merge:`);
|
|
299
338
|
lines.push('');
|
|
300
339
|
for (const gap of plan.gapDetails.filter((g) => !g.name.includes('(partial)'))) {
|
|
301
|
-
|
|
340
|
+
const aiLabel = gap.source === 'ai+deterministic' ? ' ✦ AI-enriched' : '';
|
|
341
|
+
lines.push(`- **${gap.name}** [${gap.priority}]${aiLabel}`);
|
|
302
342
|
if (gap.missingScenarios && gap.missingScenarios.length > 0) {
|
|
303
343
|
for (const scenario of gap.missingScenarios) {
|
|
304
344
|
lines.push(` - ${scenario}`);
|
|
305
345
|
}
|
|
306
346
|
}
|
|
347
|
+
// Show AI-provided reasons (skip the first deterministic reason which is always included)
|
|
348
|
+
const aiReasons = gap.reasons.slice(1);
|
|
349
|
+
if (aiReasons.length > 0) {
|
|
350
|
+
lines.push(` - *AI insight*: ${aiReasons.join('; ')}`);
|
|
351
|
+
}
|
|
307
352
|
}
|
|
308
353
|
}
|
|
309
354
|
if (plan.coveredFlows.length > 0) {
|
package/dist/esm/api.js
CHANGED
|
@@ -5,6 +5,9 @@ import { appendPlanMetrics, } from './agent/plan.js';
|
|
|
5
5
|
import { analyzeImpact as analyzeImpactV2 } from './engine/impact_engine.js';
|
|
6
6
|
import { buildPlanFromImpact, renderCiSummaryMarkdown, writeCiSummary, writePlanReport, } from './engine/plan_builder.js';
|
|
7
7
|
import { getChangedFiles } from './agent/git.js';
|
|
8
|
+
import { loadDiffs } from './engine/diff_loader.js';
|
|
9
|
+
import { enrichImpactWithAI } from './engine/ai_enrichment.js';
|
|
10
|
+
import { AnthropicProvider } from './anthropic_provider.js';
|
|
8
11
|
import { finalizeGeneratedTests } from './agent/handoff.js';
|
|
9
12
|
import { ingestTraceabilityInput, } from './agent/traceability_ingest.js';
|
|
10
13
|
import { captureTraceabilityInput, } from './agent/traceability_capture.js';
|
|
@@ -56,6 +59,40 @@ export function recommendTestsDeterministic(options = {}) {
|
|
|
56
59
|
appendPlanMetrics(reportRoot, plan);
|
|
57
60
|
return { impact, plan, planPath, ciSummaryMarkdown, ciSummaryPath };
|
|
58
61
|
}
|
|
62
|
+
export async function recommendTestsAI(options = {}) {
|
|
63
|
+
const config = resolveAgent(options, 'impact');
|
|
64
|
+
const reportRoot = config.testsRoot || config.path;
|
|
65
|
+
const gitResult = getChangedFiles(config.path, config.git.since, { includeUncommitted: config.git.includeUncommitted });
|
|
66
|
+
const impact = analyzeImpactV2(gitResult.files, {
|
|
67
|
+
testsRoot: reportRoot,
|
|
68
|
+
routeFamilies: config.routeFamilies,
|
|
69
|
+
});
|
|
70
|
+
const apiKey = process.env.ANTHROPIC_API_KEY;
|
|
71
|
+
let aiEnrichment;
|
|
72
|
+
if (apiKey) {
|
|
73
|
+
const diffs = loadDiffs(config.path, config.git.since, gitResult.files);
|
|
74
|
+
const provider = new AnthropicProvider({ apiKey });
|
|
75
|
+
// Collect all known spec paths from impacted features
|
|
76
|
+
const specSet = new Set();
|
|
77
|
+
for (const feature of impact.impactedFeatures) {
|
|
78
|
+
for (const s of feature.playwrightSpecs) {
|
|
79
|
+
specSet.add(s);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
aiEnrichment = await enrichImpactWithAI({
|
|
83
|
+
deterministicImpact: impact,
|
|
84
|
+
diffs,
|
|
85
|
+
provider,
|
|
86
|
+
specList: [...specSet],
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
const plan = buildPlanFromImpact(impact, config.policy, aiEnrichment);
|
|
90
|
+
const planPath = writePlanReport(reportRoot, plan);
|
|
91
|
+
const ciSummaryMarkdown = renderCiSummaryMarkdown(plan);
|
|
92
|
+
const ciSummaryPath = writeCiSummary(reportRoot, ciSummaryMarkdown);
|
|
93
|
+
appendPlanMetrics(reportRoot, plan);
|
|
94
|
+
return { impact, plan, planPath, ciSummaryMarkdown, ciSummaryPath, aiEnrichment };
|
|
95
|
+
}
|
|
59
96
|
export function captureTraceability(options) {
|
|
60
97
|
const cwd = options.cwd || process.cwd();
|
|
61
98
|
const { config } = resolveConfig(cwd, options.configPath, {
|
package/dist/esm/cli.js
CHANGED
|
@@ -6,10 +6,10 @@ import { dirname, join, resolve } from 'path';
|
|
|
6
6
|
import { resolveConfig } from './agent/config.js';
|
|
7
7
|
import { AnthropicProvider } from './anthropic_provider.js';
|
|
8
8
|
import { LLMProviderError } from './provider_interface.js';
|
|
9
|
-
import { appendPlanMetrics } from './agent/plan.js';
|
|
10
9
|
import { analyzeImpact as analyzeImpactV2 } from './engine/impact_engine.js';
|
|
11
|
-
import {
|
|
10
|
+
import { writeCiSummary } from './engine/plan_builder.js';
|
|
12
11
|
import { getChangedFiles } from './agent/git.js';
|
|
12
|
+
import { recommendTestsAI, recommendTestsDeterministic } from './api.js';
|
|
13
13
|
import { appendFeedbackAndRecompute } from './agent/feedback.js';
|
|
14
14
|
import { finalizeGeneratedTests } from './agent/handoff.js';
|
|
15
15
|
import { ingestTraceabilityInput } from './agent/traceability_ingest.js';
|
|
@@ -493,6 +493,10 @@ function parseArgs(argv) {
|
|
|
493
493
|
i += 1;
|
|
494
494
|
continue;
|
|
495
495
|
}
|
|
496
|
+
if (arg === '--no-ai') {
|
|
497
|
+
parsed.noAi = true;
|
|
498
|
+
continue;
|
|
499
|
+
}
|
|
496
500
|
}
|
|
497
501
|
return parsed;
|
|
498
502
|
}
|
|
@@ -894,16 +898,55 @@ async function main() {
|
|
|
894
898
|
}
|
|
895
899
|
if (args.command === 'suggest' || args.command === 'plan') {
|
|
896
900
|
const reportRoot = config.testsRoot || config.path;
|
|
897
|
-
const
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
901
|
+
const apiOptions = {
|
|
902
|
+
cwd: process.cwd(),
|
|
903
|
+
configPath: autoConfig,
|
|
904
|
+
path: args.path,
|
|
905
|
+
profile: args.profile,
|
|
906
|
+
testsRoot: args.testsRoot,
|
|
907
|
+
gitSince: args.gitSince,
|
|
908
|
+
llmProvider: args.llmProvider,
|
|
909
|
+
policy: args.policyMinConfidence !== undefined ||
|
|
910
|
+
args.policySafeMergeConfidence !== undefined ||
|
|
911
|
+
args.policyWarningsThreshold !== undefined ||
|
|
912
|
+
(args.policyRiskyPatterns && args.policyRiskyPatterns.length > 0) ||
|
|
913
|
+
args.policyEnforcementMode !== undefined ||
|
|
914
|
+
(args.policyBlockActions && args.policyBlockActions.length > 0)
|
|
915
|
+
? {
|
|
916
|
+
minConfidenceForTargeted: args.policyMinConfidence,
|
|
917
|
+
safeMergeMinConfidence: args.policySafeMergeConfidence,
|
|
918
|
+
forceFullOnWarningsAtOrAbove: args.policyWarningsThreshold,
|
|
919
|
+
riskyFilePatterns: args.policyRiskyPatterns,
|
|
920
|
+
enforcementMode: args.policyEnforcementMode,
|
|
921
|
+
blockOnActions: args.policyBlockActions,
|
|
922
|
+
}
|
|
923
|
+
: undefined,
|
|
924
|
+
};
|
|
925
|
+
let result;
|
|
926
|
+
if (args.noAi) {
|
|
927
|
+
result = recommendTestsDeterministic(apiOptions);
|
|
928
|
+
}
|
|
929
|
+
else {
|
|
930
|
+
result = await recommendTestsAI(apiOptions);
|
|
931
|
+
if (result.aiEnrichment) {
|
|
932
|
+
const { aiEnrichment } = result;
|
|
933
|
+
// eslint-disable-next-line no-console
|
|
934
|
+
console.log(`AI enrichment: ${aiEnrichment.enrichedFeatures.length} features enriched (${aiEnrichment.tokenUsage.input + aiEnrichment.tokenUsage.output} tokens)`);
|
|
935
|
+
}
|
|
936
|
+
else if (!process.env.ANTHROPIC_API_KEY) {
|
|
937
|
+
// eslint-disable-next-line no-console
|
|
938
|
+
console.log('Tip: set ANTHROPIC_API_KEY to enable AI-powered enrichment');
|
|
939
|
+
}
|
|
940
|
+
}
|
|
941
|
+
const { plan, planPath, ciSummaryMarkdown, ciSummaryPath } = result;
|
|
942
|
+
// Write CI summary to an additional path if --ci-comment-path was specified
|
|
943
|
+
if (args.ciCommentPath) {
|
|
944
|
+
writeCiSummary(reportRoot, ciSummaryMarkdown, args.ciCommentPath);
|
|
945
|
+
}
|
|
946
|
+
const summaryPath = ciSummaryPath;
|
|
947
|
+
// Compute metrics paths (api already wrote metrics; derive paths for GHA output)
|
|
948
|
+
const metricsEventsPath = join(reportRoot, '.e2e-ai-agents/metrics.jsonl');
|
|
949
|
+
const metricsSummaryPath = join(reportRoot, '.e2e-ai-agents/metrics-summary.json');
|
|
907
950
|
const ghaOutput = args.githubOutputPath || process.env.GITHUB_OUTPUT;
|
|
908
951
|
if (ghaOutput) {
|
|
909
952
|
appendFileSync(ghaOutput, `run_set=${plan.runSet}\n`);
|
|
@@ -915,8 +958,8 @@ async function main() {
|
|
|
915
958
|
appendFileSync(ghaOutput, `required_new_tests_count=${plan.requiredNewTests.length}\n`);
|
|
916
959
|
appendFileSync(ghaOutput, `plan_path=${planPath}\n`);
|
|
917
960
|
appendFileSync(ghaOutput, `summary_path=${summaryPath}\n`);
|
|
918
|
-
appendFileSync(ghaOutput, `metrics_events_path=${
|
|
919
|
-
appendFileSync(ghaOutput, `metrics_summary_path=${
|
|
961
|
+
appendFileSync(ghaOutput, `metrics_events_path=${metricsEventsPath}\n`);
|
|
962
|
+
appendFileSync(ghaOutput, `metrics_summary_path=${metricsSummaryPath}\n`);
|
|
920
963
|
}
|
|
921
964
|
// eslint-disable-next-line no-console
|
|
922
965
|
console.log(`Suggested run set: ${plan.runSet} (confidence ${plan.confidence})`);
|
|
@@ -929,7 +972,7 @@ async function main() {
|
|
|
929
972
|
// eslint-disable-next-line no-console
|
|
930
973
|
console.log(`CI summary: ${summaryPath}`);
|
|
931
974
|
// eslint-disable-next-line no-console
|
|
932
|
-
console.log(`Plan metrics: ${
|
|
975
|
+
console.log(`Plan metrics: ${metricsSummaryPath}`);
|
|
933
976
|
const failOnLegacyFlag = args.failOnMustAddTests && plan.decision.action === 'must-add-tests';
|
|
934
977
|
if (failOnLegacyFlag || plan.enforcement.shouldFail) {
|
|
935
978
|
process.exit(2);
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
import { formatDiffsForPrompt } from './diff_loader.js';
|
|
4
|
+
const MAX_SPEC_LIST = 50;
|
|
5
|
+
function normalizePriority(value) {
|
|
6
|
+
if (value === 'P0' || value === 'P1' || value === 'P2') {
|
|
7
|
+
return value;
|
|
8
|
+
}
|
|
9
|
+
return 'P2';
|
|
10
|
+
}
|
|
11
|
+
function buildPrompt(options) {
|
|
12
|
+
const { deterministicImpact, diffs, specList, manifestSummary } = options;
|
|
13
|
+
const { changedFiles, impactedFeatures, unboundFiles } = deterministicImpact;
|
|
14
|
+
const lines = [];
|
|
15
|
+
// Optional manifest summary
|
|
16
|
+
if (manifestSummary) {
|
|
17
|
+
lines.push('## Application Overview');
|
|
18
|
+
lines.push(manifestSummary);
|
|
19
|
+
lines.push('');
|
|
20
|
+
}
|
|
21
|
+
// Changed files section
|
|
22
|
+
lines.push(`## Changed Files (${changedFiles.length} total)`);
|
|
23
|
+
for (const f of changedFiles) {
|
|
24
|
+
lines.push(`- ${f}`);
|
|
25
|
+
}
|
|
26
|
+
lines.push('');
|
|
27
|
+
// Diffs section
|
|
28
|
+
lines.push('## Code Diffs');
|
|
29
|
+
lines.push(formatDiffsForPrompt(diffs));
|
|
30
|
+
lines.push('');
|
|
31
|
+
// Deterministic features summary
|
|
32
|
+
lines.push('## Deterministic Impact Analysis');
|
|
33
|
+
lines.push('The following features/flows have been deterministically identified as impacted:');
|
|
34
|
+
lines.push('');
|
|
35
|
+
for (const feature of impactedFeatures) {
|
|
36
|
+
const featureIdPart = feature.featureId ? `featureId=${feature.featureId}` : 'featureId=undefined';
|
|
37
|
+
const specCount = feature.playwrightSpecs.length + feature.cypressSpecs.length;
|
|
38
|
+
const specList2 = [...feature.playwrightSpecs, ...feature.cypressSpecs];
|
|
39
|
+
const specsDisplay = specList2.length > 0 ? specList2.join(', ') : 'none';
|
|
40
|
+
lines.push(`- familyId=${feature.familyId} ${featureIdPart} (${feature.priority}): ${specCount} files, coverage=${feature.coverageStatus}, specs=[${specsDisplay}]`);
|
|
41
|
+
}
|
|
42
|
+
lines.push('');
|
|
43
|
+
// Unbound files
|
|
44
|
+
if (unboundFiles.length > 0) {
|
|
45
|
+
lines.push('## Unbound Files (not mapped to any feature)');
|
|
46
|
+
for (const f of unboundFiles) {
|
|
47
|
+
lines.push(`- ${f}`);
|
|
48
|
+
}
|
|
49
|
+
lines.push('');
|
|
50
|
+
}
|
|
51
|
+
// Spec list (capped at 50)
|
|
52
|
+
if (specList.length > 0) {
|
|
53
|
+
const cappedSpecs = specList.slice(0, MAX_SPEC_LIST);
|
|
54
|
+
lines.push(`## Available Test Specs (showing ${cappedSpecs.length} of ${specList.length})`);
|
|
55
|
+
for (const s of cappedSpecs) {
|
|
56
|
+
lines.push(`- ${s}`);
|
|
57
|
+
}
|
|
58
|
+
lines.push('');
|
|
59
|
+
}
|
|
60
|
+
// Instructions
|
|
61
|
+
lines.push('## Instructions');
|
|
62
|
+
lines.push('Return ONLY valid JSON (no markdown fences, no explanation) in this exact shape:');
|
|
63
|
+
lines.push('');
|
|
64
|
+
lines.push(JSON.stringify({
|
|
65
|
+
impactedFlows: [
|
|
66
|
+
{
|
|
67
|
+
id: '<featureId or familyId from the deterministic list above>',
|
|
68
|
+
name: '<human-readable flow name>',
|
|
69
|
+
priority: 'P0|P1|P2',
|
|
70
|
+
reasons: ['<specific reason why this flow is impacted by the changes>'],
|
|
71
|
+
coveredBy: ['<spec file paths that cover this flow>'],
|
|
72
|
+
missingScenarios: ['<specific test scenarios that are missing or should be added>'],
|
|
73
|
+
},
|
|
74
|
+
],
|
|
75
|
+
unboundFileAnalysis: [
|
|
76
|
+
{
|
|
77
|
+
file: '<path to unbound file>',
|
|
78
|
+
likelyFeature: '<best guess at which feature/family this affects>',
|
|
79
|
+
reason: '<why you think this file belongs to that feature>',
|
|
80
|
+
},
|
|
81
|
+
],
|
|
82
|
+
}, null, 2));
|
|
83
|
+
return lines.join('\n');
|
|
84
|
+
}
|
|
85
|
+
function extractJSON(text) {
|
|
86
|
+
// Try markdown fenced block first
|
|
87
|
+
const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/i);
|
|
88
|
+
const candidates = fenced ? [fenced[1], text] : [text];
|
|
89
|
+
for (const candidate of candidates) {
|
|
90
|
+
const start = candidate.indexOf('{');
|
|
91
|
+
const end = candidate.lastIndexOf('}');
|
|
92
|
+
if (start >= 0 && end > start) {
|
|
93
|
+
return candidate.slice(start, end + 1).trim();
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
// Fallback: return trimmed text
|
|
97
|
+
return text.trim();
|
|
98
|
+
}
|
|
99
|
+
function toEnrichedFeature(det, aiFlow) {
|
|
100
|
+
return {
|
|
101
|
+
familyId: det.familyId,
|
|
102
|
+
featureId: det.featureId,
|
|
103
|
+
priority: normalizePriority(det.priority),
|
|
104
|
+
changedFiles: det.changedFiles,
|
|
105
|
+
coverageStatus: det.coverageStatus,
|
|
106
|
+
playwrightSpecs: det.playwrightSpecs,
|
|
107
|
+
cypressSpecs: det.cypressSpecs,
|
|
108
|
+
userFlows: det.userFlows,
|
|
109
|
+
aiReasons: aiFlow?.reasons ?? [],
|
|
110
|
+
aiMissingScenarios: aiFlow?.missingScenarios ?? [],
|
|
111
|
+
aiCoveredBy: aiFlow?.coveredBy ?? [],
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Enriches a deterministic impact result with AI-generated reasons,
|
|
116
|
+
* missing test scenarios, and coverage insights.
|
|
117
|
+
*/
|
|
118
|
+
export async function enrichImpactWithAI(options) {
|
|
119
|
+
const { deterministicImpact, provider } = options;
|
|
120
|
+
const warnings = [];
|
|
121
|
+
let tokenUsage = { input: 0, output: 0 };
|
|
122
|
+
const prompt = buildPrompt(options);
|
|
123
|
+
let aiResponse = null;
|
|
124
|
+
let unboundFileInsights = [];
|
|
125
|
+
try {
|
|
126
|
+
const response = await provider.generateText(prompt, {
|
|
127
|
+
maxTokens: 4000,
|
|
128
|
+
temperature: 0,
|
|
129
|
+
timeout: 45000,
|
|
130
|
+
systemPrompt: 'You are an expert E2E test analyst. Return only valid JSON.',
|
|
131
|
+
});
|
|
132
|
+
tokenUsage = {
|
|
133
|
+
input: response.usage?.inputTokens ?? 0,
|
|
134
|
+
output: response.usage?.outputTokens ?? 0,
|
|
135
|
+
};
|
|
136
|
+
const rawJSON = extractJSON(response.text);
|
|
137
|
+
try {
|
|
138
|
+
const parsed = JSON.parse(rawJSON);
|
|
139
|
+
// Validate that impactedFlows is an array
|
|
140
|
+
if (!Array.isArray(parsed.impactedFlows)) {
|
|
141
|
+
warnings.push('AI response parsed but impactedFlows is not an array; returning empty enrichedFeatures');
|
|
142
|
+
return {
|
|
143
|
+
enrichedFeatures: [],
|
|
144
|
+
unboundFileInsights: [],
|
|
145
|
+
warnings,
|
|
146
|
+
providerName: provider.name,
|
|
147
|
+
tokenUsage,
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
aiResponse = parsed;
|
|
151
|
+
unboundFileInsights = (parsed.unboundFileAnalysis ?? []).map((item) => ({
|
|
152
|
+
file: item.file,
|
|
153
|
+
likelyFeature: item.likelyFeature,
|
|
154
|
+
reason: item.reason,
|
|
155
|
+
}));
|
|
156
|
+
}
|
|
157
|
+
catch (parseErr) {
|
|
158
|
+
warnings.push(`Failed to parse AI response as JSON: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`);
|
|
159
|
+
return {
|
|
160
|
+
enrichedFeatures: [],
|
|
161
|
+
unboundFileInsights: [],
|
|
162
|
+
warnings,
|
|
163
|
+
providerName: provider.name,
|
|
164
|
+
tokenUsage,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
catch (err) {
|
|
169
|
+
warnings.push(`AI provider error: ${err instanceof Error ? err.message : String(err)}`);
|
|
170
|
+
return {
|
|
171
|
+
enrichedFeatures: [],
|
|
172
|
+
unboundFileInsights: [],
|
|
173
|
+
warnings,
|
|
174
|
+
providerName: provider.name,
|
|
175
|
+
tokenUsage,
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
// Build a map of AI flows by id (featureId or familyId)
|
|
179
|
+
const aiFlowMap = new Map();
|
|
180
|
+
if (aiResponse?.impactedFlows) {
|
|
181
|
+
for (const flow of aiResponse.impactedFlows) {
|
|
182
|
+
aiFlowMap.set(flow.id, flow);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
// Build a set of all deterministic ids for unmatched-flow detection
|
|
186
|
+
const deterministicIds = new Set();
|
|
187
|
+
for (const det of deterministicImpact.impactedFeatures) {
|
|
188
|
+
if (det.featureId) {
|
|
189
|
+
deterministicIds.add(det.featureId);
|
|
190
|
+
}
|
|
191
|
+
deterministicIds.add(det.familyId);
|
|
192
|
+
}
|
|
193
|
+
// Warn on AI flows that don't match any deterministic feature
|
|
194
|
+
for (const flow of aiFlowMap.values()) {
|
|
195
|
+
if (!deterministicIds.has(flow.id)) {
|
|
196
|
+
warnings.push(`AI returned flow '${flow.id}' with no matching deterministic feature (using as-is)`);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
// Merge deterministic features with AI data
|
|
200
|
+
const enrichedFeatures = deterministicImpact.impactedFeatures.map((det) => {
|
|
201
|
+
// Match by featureId first, then by familyId
|
|
202
|
+
const aiFlow = det.featureId
|
|
203
|
+
? (aiFlowMap.get(det.featureId) ?? aiFlowMap.get(det.familyId))
|
|
204
|
+
: aiFlowMap.get(det.familyId);
|
|
205
|
+
return toEnrichedFeature(det, aiFlow);
|
|
206
|
+
});
|
|
207
|
+
// Include AI flows that had no deterministic match (as-is, with empty deterministic fields)
|
|
208
|
+
for (const flow of aiFlowMap.values()) {
|
|
209
|
+
if (!deterministicIds.has(flow.id)) {
|
|
210
|
+
enrichedFeatures.push({
|
|
211
|
+
familyId: flow.id,
|
|
212
|
+
featureId: undefined,
|
|
213
|
+
priority: normalizePriority(flow.priority),
|
|
214
|
+
changedFiles: [],
|
|
215
|
+
coverageStatus: 'uncovered',
|
|
216
|
+
playwrightSpecs: [],
|
|
217
|
+
cypressSpecs: [],
|
|
218
|
+
userFlows: [],
|
|
219
|
+
aiReasons: flow.reasons ?? [],
|
|
220
|
+
aiMissingScenarios: flow.missingScenarios ?? [],
|
|
221
|
+
aiCoveredBy: flow.coveredBy ?? [],
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
return {
|
|
226
|
+
enrichedFeatures,
|
|
227
|
+
unboundFileInsights,
|
|
228
|
+
warnings,
|
|
229
|
+
providerName: provider.name,
|
|
230
|
+
tokenUsage,
|
|
231
|
+
};
|
|
232
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
import { spawnSync } from 'child_process';
|
|
4
|
+
const MAX_DIFF_CHARS = 8000;
|
|
5
|
+
const MAX_TOTAL_CHARS = 60000;
|
|
6
|
+
const TRUNCATION_NOTICE = '\n... (diff truncated)';
|
|
7
|
+
function runGitRaw(args, cwd) {
|
|
8
|
+
const result = spawnSync('git', args, {
|
|
9
|
+
cwd,
|
|
10
|
+
encoding: 'utf-8',
|
|
11
|
+
timeout: 30000,
|
|
12
|
+
});
|
|
13
|
+
if (result.error || result.status !== 0) {
|
|
14
|
+
return null;
|
|
15
|
+
}
|
|
16
|
+
return result.stdout;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Loads git diffs for the given changed files relative to the given since ref.
|
|
20
|
+
* Uses `git merge-base` to find the accurate base ref first.
|
|
21
|
+
* Individual diffs are truncated at 8000 chars and total output is capped at 60000 chars.
|
|
22
|
+
*/
|
|
23
|
+
export function loadDiffs(appRoot, since, changedFiles) {
|
|
24
|
+
const result = new Map();
|
|
25
|
+
if (changedFiles.length === 0) {
|
|
26
|
+
return result;
|
|
27
|
+
}
|
|
28
|
+
// Try to get accurate merge base
|
|
29
|
+
let baseRef = since;
|
|
30
|
+
const mergeBaseOutput = runGitRaw(['merge-base', since, 'HEAD'], appRoot);
|
|
31
|
+
if (mergeBaseOutput) {
|
|
32
|
+
const candidate = mergeBaseOutput
|
|
33
|
+
.split('\n')
|
|
34
|
+
.map((line) => line.trim())
|
|
35
|
+
.find(Boolean);
|
|
36
|
+
if (candidate) {
|
|
37
|
+
baseRef = candidate;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
let totalChars = 0;
|
|
41
|
+
for (const file of changedFiles) {
|
|
42
|
+
if (totalChars >= MAX_TOTAL_CHARS) {
|
|
43
|
+
break;
|
|
44
|
+
}
|
|
45
|
+
const diffOutput = runGitRaw(['diff', `${baseRef}..HEAD`, '--', file], appRoot);
|
|
46
|
+
if (diffOutput === null) {
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
let diff = diffOutput;
|
|
50
|
+
if (diff.length > MAX_DIFF_CHARS) {
|
|
51
|
+
diff = diff.slice(0, MAX_DIFF_CHARS) + TRUNCATION_NOTICE;
|
|
52
|
+
}
|
|
53
|
+
result.set(file, diff);
|
|
54
|
+
totalChars += diff.length;
|
|
55
|
+
}
|
|
56
|
+
return result;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Formats a diffs map into a human-readable string suitable for an AI prompt.
|
|
60
|
+
*/
|
|
61
|
+
export function formatDiffsForPrompt(diffs) {
|
|
62
|
+
if (diffs.size === 0) {
|
|
63
|
+
return 'No diffs available.';
|
|
64
|
+
}
|
|
65
|
+
const sections = [];
|
|
66
|
+
for (const [file, diff] of diffs) {
|
|
67
|
+
sections.push(`--- ${file} ---\n${diff}`);
|
|
68
|
+
}
|
|
69
|
+
return sections.join('\n\n');
|
|
70
|
+
}
|