@yasserkhanorg/e2e-agents 0.7.6 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +13 -1
- package/dist/engine/ai_enrichment.d.ts +2 -1
- package/dist/engine/ai_enrichment.d.ts.map +1 -1
- package/dist/engine/ai_enrichment.js +28 -4
- package/dist/engine/impact_engine.d.ts +11 -0
- package/dist/engine/impact_engine.d.ts.map +1 -1
- package/dist/engine/impact_engine.js +50 -11
- package/dist/engine/plan_builder.d.ts.map +1 -1
- package/dist/engine/plan_builder.js +21 -16
- package/dist/esm/api.js +13 -1
- package/dist/esm/engine/ai_enrichment.js +28 -4
- package/dist/esm/engine/impact_engine.js +50 -12
- package/dist/esm/engine/plan_builder.js +21 -16
- package/dist/esm/index.js +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/package.json +1 -1
package/dist/api.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAIA,OAAO,EAAgB,KAAK,eAAe,EAAC,MAAM,mBAAmB,CAAC;AACtE,OAAO,EAEH,KAAK,UAAU,EAClB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAmC,KAAK,YAAY,EAAC,MAAM,2BAA2B,CAAC;AAS9F,OAAO,EAAqB,KAAK,kBAAkB,EAAC,MAAM,2BAA2B,CAAC;AAEtF,OAAO,EAAyB,KAAK,6BAA6B,EAAE,KAAK,4BAA4B,EAAC,MAAM,oBAAoB,CAAC;AACjI,OAAO,EAEH,KAAK,yBAAyB,EAC9B,KAAK,wBAAwB,EAChC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAGH,KAAK,yBAAyB,EACjC,MAAM,iCAAiC,CAAC;AAEzC,MAAM,WAAW,eAAgB,SAAQ,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC;IAClE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,aAAa,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,4BAA4B;IACzC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,yBAAyB,CAAC;CACvC;AAED,MAAM,WAAW,6BAA6B;IAC1C,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAcD,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,6BAA6B,GAAG,4BAA4B,CAE1G;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,4BAA4B,GAAG,wBAAwB,CASlG;AAED,MAAM,WAAW,sBAAsB;IACnC,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;CACzB;AAED,wBAAgB,0BAA0B,CAAC,OAAO,GAAE,eAAoB,GAAG,YAAY,CAQtF;AAED,wBAAgB,2BAA2B,CAAC,OAAO,GAAE,eAAoB,GAAG,sBAAsB,CAcjG;AAED,wBAAsB,gBAAgB,CAAC,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,sBAAsB,GAAG;IAAE,YAAY,CAAC,EAAE,kBAAkB,CAAA;CAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAIA,OAAO,EAAgB,KAAK,eAAe,EAAC,MAAM,mBAAmB,CAAC;AACtE,OAAO,EAEH,KAAK,UAAU,EAClB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAmC,KAAK,YAAY,EAAC,MAAM,2BAA2B,CAAC;AAS9F,OAAO,EAAqB,KAAK,kBAAkB,EAAC,MAAM,2BAA2B,CAAC;AAEtF,OAAO,EAAyB,KAAK,6BAA6B,EAAE,KAAK,4BAA4B,EAAC,MAAM,oBAAoB,CAAC;AACjI,OAAO,EAEH,KAAK,yBAAyB,EAC9B,KAAK,wBAAwB,EAChC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAGH,KAAK,yBAAyB,EACjC,MAAM,iCAAiC,CAAC;AAEzC,MAAM,WAAW,eAAgB,SAAQ,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC;IAClE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,aAAa,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,4BAA4B;IACzC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,yBAAyB,CAAC;CACvC;AAED,MAAM,WAAW,6BAA6B;IAC1C,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAcD,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,6BAA6B,GAAG,4BAA4B,CAE1G;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,4BAA4B,GAAG,wBAAwB,CASlG;AAED,MAAM,WAAW,sBAAsB;IACnC,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;CACzB;AAED,wBAAgB,0BAA0B,CAAC,OAAO,GAAE,eAAoB,GAAG,YAAY,CAQtF;AAED,wBAAgB,2BAA2B,CAAC,OAAO,GAAE,eAAoB,GAAG,sBAAsB,CAcjG;AAED,wBAAsB,gBAAgB,CAAC,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,sBAAsB,GAAG;IAAE,YAAY,CAAC,EAAE,kBAAkB,CAAA;CAAE,CAAC,CAgD7I;AAED,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,6BAA6B,GAAG,yBAAyB,CAkBrG"}
|
package/dist/api.js
CHANGED
|
@@ -80,18 +80,30 @@ async function recommendTestsAI(options = {}) {
|
|
|
80
80
|
if (apiKey) {
|
|
81
81
|
const diffs = (0, diff_loader_js_1.loadDiffs)(config.path, config.git.since, gitResult.files);
|
|
82
82
|
const provider = new anthropic_provider_js_1.AnthropicProvider({ apiKey });
|
|
83
|
-
// Collect all known spec paths from impacted features
|
|
83
|
+
// Collect all known spec paths and scenario details from impacted features
|
|
84
84
|
const specSet = new Set();
|
|
85
|
+
const specDetailsMap = new Map();
|
|
85
86
|
for (const feature of impact.impactedFeatures) {
|
|
86
87
|
for (const s of feature.playwrightSpecs) {
|
|
87
88
|
specSet.add(s);
|
|
88
89
|
}
|
|
90
|
+
for (const detail of feature.playwrightSpecDetails) {
|
|
91
|
+
if (!specDetailsMap.has(detail.file)) {
|
|
92
|
+
specDetailsMap.set(detail.file, detail);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
for (const detail of feature.cypressSpecDetails) {
|
|
96
|
+
if (!specDetailsMap.has(detail.file)) {
|
|
97
|
+
specDetailsMap.set(detail.file, detail);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
89
100
|
}
|
|
90
101
|
aiEnrichment = await (0, ai_enrichment_js_1.enrichImpactWithAI)({
|
|
91
102
|
deterministicImpact: impact,
|
|
92
103
|
diffs,
|
|
93
104
|
provider,
|
|
94
105
|
specList: [...specSet],
|
|
106
|
+
specDetails: [...specDetailsMap.values()],
|
|
95
107
|
});
|
|
96
108
|
}
|
|
97
109
|
const plan = (0, plan_builder_js_1.buildPlanFromImpact)(impact, config.policy, aiEnrichment);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { LLMProvider } from '../provider_interface.js';
|
|
2
|
-
import type { ImpactResult } from './impact_engine.js';
|
|
2
|
+
import type { ImpactResult, SpecWithScenarios } from './impact_engine.js';
|
|
3
3
|
import type { FeaturePriority } from '../knowledge/route_families.js';
|
|
4
4
|
export interface EnrichedFeature {
|
|
5
5
|
familyId: string;
|
|
@@ -33,6 +33,7 @@ export interface AIEnrichmentOptions {
|
|
|
33
33
|
diffs: Map<string, string>;
|
|
34
34
|
provider: LLMProvider;
|
|
35
35
|
specList: string[];
|
|
36
|
+
specDetails?: SpecWithScenarios[];
|
|
36
37
|
manifestSummary?: string;
|
|
37
38
|
}
|
|
38
39
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ai_enrichment.d.ts","sourceRoot":"","sources":["../../src/engine/ai_enrichment.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,EAAC,YAAY,
|
|
1
|
+
{"version":3,"file":"ai_enrichment.d.ts","sourceRoot":"","sources":["../../src/engine/ai_enrichment.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,EAAC,YAAY,EAAmB,iBAAiB,EAAC,MAAM,oBAAoB,CAAC;AACzF,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAGpE,MAAM,WAAW,eAAe;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,eAAe,CAAC;IAC1B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,WAAW,EAAE,MAAM,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,kBAAkB;IAC/B,gBAAgB,EAAE,eAAe,EAAE,CAAC;IACpC,mBAAmB,EAAE,KAAK,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAC,CAAC,CAAC;IAClF,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAC,CAAC;CAC/C;AAED,MAAM,WAAW,mBAAmB;IAChC,mBAAmB,EAAE,YAAY,CAAC;IAClC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3B,QAAQ,EAAE,WAAW,CAAC;IACtB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,WAAW,CAAC,EAAE,iBAAiB,EAAE,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAkLD;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAkIlG"}
|
|
@@ -11,8 +11,9 @@ function normalizePriority(value) {
|
|
|
11
11
|
}
|
|
12
12
|
return 'P2';
|
|
13
13
|
}
|
|
14
|
+
const MAX_SCENARIOS_PER_SPEC = 20;
|
|
14
15
|
function buildPrompt(options) {
|
|
15
|
-
const { deterministicImpact, diffs, specList, manifestSummary } = options;
|
|
16
|
+
const { deterministicImpact, diffs, specList, specDetails, manifestSummary } = options;
|
|
16
17
|
const { changedFiles, impactedFeatures, unboundFiles } = deterministicImpact;
|
|
17
18
|
const lines = [];
|
|
18
19
|
// Optional manifest summary
|
|
@@ -51,8 +52,26 @@ function buildPrompt(options) {
|
|
|
51
52
|
}
|
|
52
53
|
lines.push('');
|
|
53
54
|
}
|
|
54
|
-
// Spec
|
|
55
|
-
if (
|
|
55
|
+
// Spec coverage with scenario titles (when available) or bare file paths
|
|
56
|
+
if (specDetails && specDetails.length > 0) {
|
|
57
|
+
const cappedDetails = specDetails.slice(0, MAX_SPEC_LIST);
|
|
58
|
+
const totalScenarios = cappedDetails.reduce((sum, s) => sum + s.scenarios.length, 0);
|
|
59
|
+
lines.push(`## Existing Test Coverage (${cappedDetails.length} specs, ${totalScenarios} scenarios)`);
|
|
60
|
+
lines.push('Use this to avoid suggesting scenarios that already exist.');
|
|
61
|
+
lines.push('');
|
|
62
|
+
for (const spec of cappedDetails) {
|
|
63
|
+
lines.push(`- ${spec.file}`);
|
|
64
|
+
const cappedScenarios = spec.scenarios.slice(0, MAX_SCENARIOS_PER_SPEC);
|
|
65
|
+
for (const scenario of cappedScenarios) {
|
|
66
|
+
lines.push(` • "${scenario}"`);
|
|
67
|
+
}
|
|
68
|
+
if (spec.scenarios.length > MAX_SCENARIOS_PER_SPEC) {
|
|
69
|
+
lines.push(` • ... and ${spec.scenarios.length - MAX_SCENARIOS_PER_SPEC} more`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
lines.push('');
|
|
73
|
+
}
|
|
74
|
+
else if (specList.length > 0) {
|
|
56
75
|
const cappedSpecs = specList.slice(0, MAX_SPEC_LIST);
|
|
57
76
|
lines.push(`## Available Test Specs (showing ${cappedSpecs.length} of ${specList.length})`);
|
|
58
77
|
for (const s of cappedSpecs) {
|
|
@@ -64,9 +83,14 @@ function buildPrompt(options) {
|
|
|
64
83
|
lines.push('## Instructions');
|
|
65
84
|
lines.push('Return ONLY valid JSON (no markdown fences, no explanation) in this exact shape:');
|
|
66
85
|
lines.push('');
|
|
86
|
+
lines.push('Rules for coveredBy:');
|
|
87
|
+
lines.push('- Reference SPECIFIC scenario titles from the Existing Test Coverage section when possible.');
|
|
88
|
+
lines.push('- Format: "file.spec.ts → scenario title"');
|
|
89
|
+
lines.push('');
|
|
67
90
|
lines.push('Rules for missingScenarios:');
|
|
91
|
+
lines.push('- Cross-reference the scenario titles in Existing Test Coverage. If a scenario already exists that covers the behavior, do NOT suggest it — instead list it in coveredBy.');
|
|
68
92
|
lines.push('- For coverage=uncovered: list all scenarios the feature needs.');
|
|
69
|
-
lines.push('- For coverage=covered or coverage=partial: ONLY list scenarios introduced by THIS diff that
|
|
93
|
+
lines.push('- For coverage=covered or coverage=partial: ONLY list scenarios introduced by THIS diff that have NO matching scenario in existing coverage. If the diff adds no new user-visible behavior, return []. Do not pad with generic scenarios.');
|
|
70
94
|
lines.push('');
|
|
71
95
|
lines.push(JSON.stringify({
|
|
72
96
|
impactedFlows: [
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import type { FeaturePriority } from '../knowledge/route_families.js';
|
|
2
2
|
import type { RouteFamiliesConfig } from '../agent/config.js';
|
|
3
3
|
export type CoverageStatus = 'covered' | 'partial' | 'uncovered';
|
|
4
|
+
export interface SpecWithScenarios {
|
|
5
|
+
file: string;
|
|
6
|
+
scenarios: string[];
|
|
7
|
+
}
|
|
4
8
|
export interface ImpactedFeature {
|
|
5
9
|
familyId: string;
|
|
6
10
|
featureId?: string;
|
|
@@ -8,6 +12,8 @@ export interface ImpactedFeature {
|
|
|
8
12
|
changedFiles: string[];
|
|
9
13
|
playwrightSpecs: string[];
|
|
10
14
|
cypressSpecs: string[];
|
|
15
|
+
playwrightSpecDetails: SpecWithScenarios[];
|
|
16
|
+
cypressSpecDetails: SpecWithScenarios[];
|
|
11
17
|
userFlows: string[];
|
|
12
18
|
coverageStatus: CoverageStatus;
|
|
13
19
|
}
|
|
@@ -24,6 +30,11 @@ export interface ImpactEngineOptions {
|
|
|
24
30
|
routeFamilies?: RouteFamiliesConfig;
|
|
25
31
|
expandedFiles?: string[];
|
|
26
32
|
}
|
|
33
|
+
/**
|
|
34
|
+
* Extract describe/test/it titles from a spec file using regex.
|
|
35
|
+
* Returns an empty array if the file cannot be read.
|
|
36
|
+
*/
|
|
37
|
+
export declare function extractScenarios(filePath: string, framework: 'playwright' | 'cypress'): string[];
|
|
27
38
|
export declare function analyzeImpact(changedFiles: string[], options: ImpactEngineOptions): ImpactResult;
|
|
28
39
|
/**
|
|
29
40
|
* Get gaps: P0/P1 features with 'uncovered' status.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"impact_engine.d.ts","sourceRoot":"","sources":["../../src/engine/impact_engine.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAGR,eAAe,EAClB,MAAM,gCAAgC,CAAC;AASxC,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,oBAAoB,CAAC;AAE5D,MAAM,MAAM,cAAc,GAAG,SAAS,GAAG,SAAS,GAAG,WAAW,CAAC;AAEjE,MAAM,WAAW,eAAe;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,eAAe,CAAC;IAC1B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,cAAc,CAAC;CAClC;AAED,MAAM,WAAW,YAAY;IACzB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,gBAAgB,EAAE,eAAe,EAAE,CAAC;IACpC,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,mBAAmB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,mBAAmB,CAAC;IACpC,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B;
|
|
1
|
+
{"version":3,"file":"impact_engine.d.ts","sourceRoot":"","sources":["../../src/engine/impact_engine.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAGR,eAAe,EAClB,MAAM,gCAAgC,CAAC;AASxC,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,oBAAoB,CAAC;AAE5D,MAAM,MAAM,cAAc,GAAG,SAAS,GAAG,SAAS,GAAG,WAAW,CAAC;AAEjE,MAAM,WAAW,iBAAiB;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,eAAe,CAAC;IAC1B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,qBAAqB,EAAE,iBAAiB,EAAE,CAAC;IAC3C,kBAAkB,EAAE,iBAAiB,EAAE,CAAC;IACxC,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,cAAc,CAAC;CAClC;AAED,MAAM,WAAW,YAAY;IACzB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,gBAAgB,EAAE,eAAe,EAAE,CAAC;IACpC,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,mBAAmB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,mBAAmB,CAAC;IACpC,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B;AA+CD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,YAAY,GAAG,SAAS,GAAG,MAAM,EAAE,CAgBhG;AA0ED,wBAAgB,aAAa,CACzB,YAAY,EAAE,MAAM,EAAE,EACtB,OAAO,EAAE,mBAAmB,GAC7B,YAAY,CA6Ed;AAYD;;GAEG;AACH,wBAAgB,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,eAAe,EAAE,CAI/D;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,YAAY,GAAG,eAAe,EAAE,CAItE"}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
3
3
|
// See LICENSE.txt for license information.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
+
exports.extractScenarios = extractScenarios;
|
|
5
6
|
exports.analyzeImpact = analyzeImpact;
|
|
6
7
|
exports.getGaps = getGaps;
|
|
7
8
|
exports.getPartialGaps = getPartialGaps;
|
|
@@ -50,15 +51,48 @@ function scanDirForSpecsRecursive(dir, extension) {
|
|
|
50
51
|
}
|
|
51
52
|
return specs;
|
|
52
53
|
}
|
|
54
|
+
// Regex patterns for extracting test scenario titles from spec files.
|
|
55
|
+
// Playwright uses test() and test.describe(); Cypress uses describe(), context(), it().
|
|
56
|
+
const PLAYWRIGHT_SCENARIO_RE = /(?:test\.describe|test)\(\s*['"`]([^'"`]+)['"`]/g;
|
|
57
|
+
const CYPRESS_SCENARIO_RE = /(?:describe|context|it)\(\s*['"`]([^'"`]+)['"`]/g;
|
|
58
|
+
/**
|
|
59
|
+
* Extract describe/test/it titles from a spec file using regex.
|
|
60
|
+
* Returns an empty array if the file cannot be read.
|
|
61
|
+
*/
|
|
62
|
+
function extractScenarios(filePath, framework) {
|
|
63
|
+
let content;
|
|
64
|
+
try {
|
|
65
|
+
content = (0, fs_1.readFileSync)(filePath, 'utf-8');
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
return [];
|
|
69
|
+
}
|
|
70
|
+
const re = framework === 'playwright' ? PLAYWRIGHT_SCENARIO_RE : CYPRESS_SCENARIO_RE;
|
|
71
|
+
const scenarios = [];
|
|
72
|
+
let match;
|
|
73
|
+
// Reset lastIndex in case the regex was used before
|
|
74
|
+
re.lastIndex = 0;
|
|
75
|
+
while ((match = re.exec(content)) !== null) {
|
|
76
|
+
scenarios.push(match[1]);
|
|
77
|
+
}
|
|
78
|
+
return scenarios;
|
|
79
|
+
}
|
|
53
80
|
function resolvePlaywrightSpecs(testsRoot, specDirs) {
|
|
54
|
-
const
|
|
81
|
+
const paths = [];
|
|
82
|
+
const details = [];
|
|
55
83
|
for (const dir of specDirs) {
|
|
56
|
-
|
|
84
|
+
const found = scanDirForSpecs(testsRoot, dir, '.spec.ts');
|
|
85
|
+
for (const relPath of found) {
|
|
86
|
+
paths.push(relPath);
|
|
87
|
+
const absPath = (0, path_1.join)(testsRoot, relPath);
|
|
88
|
+
details.push({ file: relPath, scenarios: extractScenarios(absPath, 'playwright') });
|
|
89
|
+
}
|
|
57
90
|
}
|
|
58
|
-
return
|
|
91
|
+
return { paths, details };
|
|
59
92
|
}
|
|
60
93
|
function resolveCypressSpecs(cypressRoot, specDirs) {
|
|
61
|
-
const
|
|
94
|
+
const paths = [];
|
|
95
|
+
const details = [];
|
|
62
96
|
for (const dir of specDirs) {
|
|
63
97
|
// cypressSpecDirs are relative to testsRoot (e.g. ../cypress/tests/integration/channels/search/)
|
|
64
98
|
// Resolve them relative to the cypress root
|
|
@@ -68,9 +102,12 @@ function resolveCypressSpecs(cypressRoot, specDirs) {
|
|
|
68
102
|
}
|
|
69
103
|
const found = scanDirForSpecsRecursive(resolvedDir, '.js');
|
|
70
104
|
const tsFound = scanDirForSpecsRecursive(resolvedDir, '.ts');
|
|
71
|
-
|
|
105
|
+
for (const absPath of [...found, ...tsFound]) {
|
|
106
|
+
paths.push(absPath);
|
|
107
|
+
details.push({ file: absPath, scenarios: extractScenarios(absPath, 'cypress') });
|
|
108
|
+
}
|
|
72
109
|
}
|
|
73
|
-
return
|
|
110
|
+
return { paths, details };
|
|
74
111
|
}
|
|
75
112
|
function computeCoverageStatus(pwSpecs, cySpecs) {
|
|
76
113
|
// Playwright is the primary framework — having Playwright specs is sufficient for "covered".
|
|
@@ -143,16 +180,18 @@ function analyzeImpact(changedFiles, options) {
|
|
|
143
180
|
const cypressSpecDirs = (0, route_families_js_1.getCypressSpecDirsForBinding)(manifest, binding);
|
|
144
181
|
const priority = (0, route_families_js_1.getPriorityForBinding)(manifest, binding);
|
|
145
182
|
const userFlows = (0, route_families_js_1.getUserFlowsForBinding)(manifest, binding);
|
|
146
|
-
const
|
|
147
|
-
const
|
|
148
|
-
const coverageStatus = computeCoverageStatus(
|
|
183
|
+
const pw = resolvePlaywrightSpecs(testsRoot, specDirs);
|
|
184
|
+
const cy = cypressRoot ? resolveCypressSpecs(cypressRoot, cypressSpecDirs) : { paths: [], details: [] };
|
|
185
|
+
const coverageStatus = computeCoverageStatus(pw.paths, cy.paths);
|
|
149
186
|
impactedFeatures.push({
|
|
150
187
|
familyId: group.familyId,
|
|
151
188
|
featureId: group.featureId,
|
|
152
189
|
priority,
|
|
153
190
|
changedFiles: group.files,
|
|
154
|
-
playwrightSpecs,
|
|
155
|
-
cypressSpecs,
|
|
191
|
+
playwrightSpecs: pw.paths,
|
|
192
|
+
cypressSpecs: cy.paths,
|
|
193
|
+
playwrightSpecDetails: pw.details,
|
|
194
|
+
cypressSpecDetails: cy.details,
|
|
156
195
|
userFlows,
|
|
157
196
|
coverageStatus,
|
|
158
197
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"plan_builder.d.ts","sourceRoot":"","sources":["../../src/engine/plan_builder.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AACrD,OAAO,KAAK,EAAC,YAAY,EAAkB,MAAM,oBAAoB,CAAC;AAEtE,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,oBAAoB,CAAC;AAG3D,OAAO,KAAK,EACR,UAAU,EACV,SAAS,EACT,kBAAkB,EAIrB,MAAM,kBAAkB,CAAC;AAE1B,YAAY,EAAC,UAAU,EAAE,SAAS,EAAE,kBAAkB,EAAC,CAAC;AAqOxD,wBAAgB,mBAAmB,CAC/B,MAAM,EAAE,YAAY,EACpB,cAAc,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,EACtC,YAAY,CAAC,EAAE,kBAAkB,GAClC,UAAU,CA0IZ;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,MAAM,CAMzE;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,
|
|
1
|
+
{"version":3,"file":"plan_builder.d.ts","sourceRoot":"","sources":["../../src/engine/plan_builder.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AACrD,OAAO,KAAK,EAAC,YAAY,EAAkB,MAAM,oBAAoB,CAAC;AAEtE,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,oBAAoB,CAAC;AAG3D,OAAO,KAAK,EACR,UAAU,EACV,SAAS,EACT,kBAAkB,EAIrB,MAAM,kBAAkB,CAAC;AAE1B,YAAY,EAAC,UAAU,EAAE,SAAS,EAAE,kBAAkB,EAAC,CAAC;AAqOxD,wBAAgB,mBAAmB,CAC/B,MAAM,EAAE,YAAY,EACpB,cAAc,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,EACtC,YAAY,CAAC,EAAE,kBAAkB,GAClC,UAAU,CA0IZ;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,MAAM,CAMzE;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,CAuGhE;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,SAAiC,GAAG,MAAM,CAMvH"}
|
|
@@ -361,47 +361,52 @@ function renderCiSummaryMarkdown(plan) {
|
|
|
361
361
|
// ── Blocking gaps ──────────────────────────────────────────────────────────
|
|
362
362
|
if (mustAddTests && plan.requiredNewTests.length > 0) {
|
|
363
363
|
lines.push('');
|
|
364
|
-
lines.push(
|
|
365
|
-
lines.push('');
|
|
366
|
-
lines.push(`The following ${uncoveredP0P1Flows} feature(s) have no test coverage and must be covered before merge:`);
|
|
364
|
+
lines.push(`### ⚠️ Missing coverage for ${uncoveredP0P1Flows} P0/P1 flow(s)`);
|
|
367
365
|
lines.push('');
|
|
368
366
|
for (const gap of plan.gapDetails.filter((g) => !g.name.includes('(partial)'))) {
|
|
369
367
|
const aiLabel = gap.source === 'ai+deterministic' ? ' ✦ AI-enriched' : '';
|
|
368
|
+
// Warning box: name + priority + AI reason (always visible)
|
|
370
369
|
lines.push(`> [!WARNING]`);
|
|
371
370
|
lines.push(`> **${gap.name}** · ${gap.priority}${aiLabel}`);
|
|
372
|
-
// AI-provided reasons (skip the first generic deterministic reason)
|
|
373
371
|
const aiReasons = gap.reasons.slice(1);
|
|
374
372
|
if (aiReasons.length > 0) {
|
|
375
373
|
lines.push(`> ${aiReasons.join(' ')}`);
|
|
376
374
|
}
|
|
375
|
+
lines.push('');
|
|
376
|
+
// Scenarios: collapsible below the warning box
|
|
377
377
|
if (gap.missingScenarios && gap.missingScenarios.length > 0) {
|
|
378
|
-
lines.push(
|
|
379
|
-
lines.push(
|
|
378
|
+
lines.push(`<details><summary>📋 Suggested test scenarios (${gap.missingScenarios.length})</summary>`);
|
|
379
|
+
lines.push('');
|
|
380
380
|
for (const scenario of gap.missingScenarios) {
|
|
381
|
-
lines.push(
|
|
381
|
+
lines.push(`- [ ] ${scenario}`);
|
|
382
382
|
}
|
|
383
|
+
lines.push('');
|
|
384
|
+
lines.push('</details>');
|
|
385
|
+
lines.push('');
|
|
383
386
|
}
|
|
384
|
-
lines.push('');
|
|
385
387
|
}
|
|
386
388
|
}
|
|
387
|
-
// ── Advisory: covered flows with new behavior
|
|
389
|
+
// ── Advisory: covered flows with new behavior ─────────────────────────────
|
|
388
390
|
if (flowsWithAdvisory.length > 0) {
|
|
389
391
|
lines.push('');
|
|
390
|
-
lines.push(
|
|
391
|
-
lines.push(`<summary>💡 New behavior detected in ${flowsWithAdvisory.length} covered feature${flowsWithAdvisory.length !== 1 ? 's' : ''} — consider adding tests</summary>`);
|
|
392
|
-
lines.push('');
|
|
393
|
-
lines.push('These features already have E2E tests, but this PR introduces new behavior worth covering:');
|
|
392
|
+
lines.push(`### 💡 New behavior detected in ${flowsWithAdvisory.length} covered feature${flowsWithAdvisory.length !== 1 ? 's' : ''} — consider adding tests`);
|
|
394
393
|
lines.push('');
|
|
395
394
|
for (const flow of flowsWithAdvisory) {
|
|
396
|
-
|
|
397
|
-
lines.push(
|
|
395
|
+
// Green [!TIP] box: just the name (always visible, compact)
|
|
396
|
+
lines.push(`> [!TIP]`);
|
|
397
|
+
lines.push(`> **${flow.name}** · ${flow.priority}`);
|
|
398
|
+
lines.push('');
|
|
399
|
+
// Specs + scenarios: collapsible below
|
|
400
|
+
const coverageSummary = flow.coveredBy.join(', ');
|
|
401
|
+
lines.push(`<details><summary>${coverageSummary} — click to see suggested scenarios</summary>`);
|
|
398
402
|
lines.push('');
|
|
399
403
|
for (const s of flow.advisoryScenarios) {
|
|
400
404
|
lines.push(`- [ ] ${s}`);
|
|
401
405
|
}
|
|
402
406
|
lines.push('');
|
|
407
|
+
lines.push('</details>');
|
|
408
|
+
lines.push('');
|
|
403
409
|
}
|
|
404
|
-
lines.push('</details>');
|
|
405
410
|
}
|
|
406
411
|
// ── Clean covered flows (collapsed) ───────────────────────────────────────
|
|
407
412
|
if (cleanFlows.length > 0) {
|
package/dist/esm/api.js
CHANGED
|
@@ -72,18 +72,30 @@ export async function recommendTestsAI(options = {}) {
|
|
|
72
72
|
if (apiKey) {
|
|
73
73
|
const diffs = loadDiffs(config.path, config.git.since, gitResult.files);
|
|
74
74
|
const provider = new AnthropicProvider({ apiKey });
|
|
75
|
-
// Collect all known spec paths from impacted features
|
|
75
|
+
// Collect all known spec paths and scenario details from impacted features
|
|
76
76
|
const specSet = new Set();
|
|
77
|
+
const specDetailsMap = new Map();
|
|
77
78
|
for (const feature of impact.impactedFeatures) {
|
|
78
79
|
for (const s of feature.playwrightSpecs) {
|
|
79
80
|
specSet.add(s);
|
|
80
81
|
}
|
|
82
|
+
for (const detail of feature.playwrightSpecDetails) {
|
|
83
|
+
if (!specDetailsMap.has(detail.file)) {
|
|
84
|
+
specDetailsMap.set(detail.file, detail);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
for (const detail of feature.cypressSpecDetails) {
|
|
88
|
+
if (!specDetailsMap.has(detail.file)) {
|
|
89
|
+
specDetailsMap.set(detail.file, detail);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
81
92
|
}
|
|
82
93
|
aiEnrichment = await enrichImpactWithAI({
|
|
83
94
|
deterministicImpact: impact,
|
|
84
95
|
diffs,
|
|
85
96
|
provider,
|
|
86
97
|
specList: [...specSet],
|
|
98
|
+
specDetails: [...specDetailsMap.values()],
|
|
87
99
|
});
|
|
88
100
|
}
|
|
89
101
|
const plan = buildPlanFromImpact(impact, config.policy, aiEnrichment);
|
|
@@ -8,8 +8,9 @@ function normalizePriority(value) {
|
|
|
8
8
|
}
|
|
9
9
|
return 'P2';
|
|
10
10
|
}
|
|
11
|
+
const MAX_SCENARIOS_PER_SPEC = 20;
|
|
11
12
|
function buildPrompt(options) {
|
|
12
|
-
const { deterministicImpact, diffs, specList, manifestSummary } = options;
|
|
13
|
+
const { deterministicImpact, diffs, specList, specDetails, manifestSummary } = options;
|
|
13
14
|
const { changedFiles, impactedFeatures, unboundFiles } = deterministicImpact;
|
|
14
15
|
const lines = [];
|
|
15
16
|
// Optional manifest summary
|
|
@@ -48,8 +49,26 @@ function buildPrompt(options) {
|
|
|
48
49
|
}
|
|
49
50
|
lines.push('');
|
|
50
51
|
}
|
|
51
|
-
// Spec
|
|
52
|
-
if (
|
|
52
|
+
// Spec coverage with scenario titles (when available) or bare file paths
|
|
53
|
+
if (specDetails && specDetails.length > 0) {
|
|
54
|
+
const cappedDetails = specDetails.slice(0, MAX_SPEC_LIST);
|
|
55
|
+
const totalScenarios = cappedDetails.reduce((sum, s) => sum + s.scenarios.length, 0);
|
|
56
|
+
lines.push(`## Existing Test Coverage (${cappedDetails.length} specs, ${totalScenarios} scenarios)`);
|
|
57
|
+
lines.push('Use this to avoid suggesting scenarios that already exist.');
|
|
58
|
+
lines.push('');
|
|
59
|
+
for (const spec of cappedDetails) {
|
|
60
|
+
lines.push(`- ${spec.file}`);
|
|
61
|
+
const cappedScenarios = spec.scenarios.slice(0, MAX_SCENARIOS_PER_SPEC);
|
|
62
|
+
for (const scenario of cappedScenarios) {
|
|
63
|
+
lines.push(` • "${scenario}"`);
|
|
64
|
+
}
|
|
65
|
+
if (spec.scenarios.length > MAX_SCENARIOS_PER_SPEC) {
|
|
66
|
+
lines.push(` • ... and ${spec.scenarios.length - MAX_SCENARIOS_PER_SPEC} more`);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
lines.push('');
|
|
70
|
+
}
|
|
71
|
+
else if (specList.length > 0) {
|
|
53
72
|
const cappedSpecs = specList.slice(0, MAX_SPEC_LIST);
|
|
54
73
|
lines.push(`## Available Test Specs (showing ${cappedSpecs.length} of ${specList.length})`);
|
|
55
74
|
for (const s of cappedSpecs) {
|
|
@@ -61,9 +80,14 @@ function buildPrompt(options) {
|
|
|
61
80
|
lines.push('## Instructions');
|
|
62
81
|
lines.push('Return ONLY valid JSON (no markdown fences, no explanation) in this exact shape:');
|
|
63
82
|
lines.push('');
|
|
83
|
+
lines.push('Rules for coveredBy:');
|
|
84
|
+
lines.push('- Reference SPECIFIC scenario titles from the Existing Test Coverage section when possible.');
|
|
85
|
+
lines.push('- Format: "file.spec.ts → scenario title"');
|
|
86
|
+
lines.push('');
|
|
64
87
|
lines.push('Rules for missingScenarios:');
|
|
88
|
+
lines.push('- Cross-reference the scenario titles in Existing Test Coverage. If a scenario already exists that covers the behavior, do NOT suggest it — instead list it in coveredBy.');
|
|
65
89
|
lines.push('- For coverage=uncovered: list all scenarios the feature needs.');
|
|
66
|
-
lines.push('- For coverage=covered or coverage=partial: ONLY list scenarios introduced by THIS diff that
|
|
90
|
+
lines.push('- For coverage=covered or coverage=partial: ONLY list scenarios introduced by THIS diff that have NO matching scenario in existing coverage. If the diff adds no new user-visible behavior, return []. Do not pad with generic scenarios.');
|
|
67
91
|
lines.push('');
|
|
68
92
|
lines.push(JSON.stringify({
|
|
69
93
|
impactedFlows: [
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
2
|
// See LICENSE.txt for license information.
|
|
3
|
-
import { existsSync, readdirSync } from 'fs';
|
|
3
|
+
import { existsSync, readdirSync, readFileSync } from 'fs';
|
|
4
4
|
import { join } from 'path';
|
|
5
5
|
import { loadRouteFamilyManifest, bindFilesToFamilies, getSpecDirsForBinding, getCypressSpecDirsForBinding, getPriorityForBinding, getUserFlowsForBinding, } from '../knowledge/route_families.js';
|
|
6
6
|
function scanDirForSpecs(baseDir, specDir, extension) {
|
|
@@ -45,15 +45,48 @@ function scanDirForSpecsRecursive(dir, extension) {
|
|
|
45
45
|
}
|
|
46
46
|
return specs;
|
|
47
47
|
}
|
|
48
|
+
// Regex patterns for extracting test scenario titles from spec files.
|
|
49
|
+
// Playwright uses test() and test.describe(); Cypress uses describe(), context(), it().
|
|
50
|
+
const PLAYWRIGHT_SCENARIO_RE = /(?:test\.describe|test)\(\s*['"`]([^'"`]+)['"`]/g;
|
|
51
|
+
const CYPRESS_SCENARIO_RE = /(?:describe|context|it)\(\s*['"`]([^'"`]+)['"`]/g;
|
|
52
|
+
/**
|
|
53
|
+
* Extract describe/test/it titles from a spec file using regex.
|
|
54
|
+
* Returns an empty array if the file cannot be read.
|
|
55
|
+
*/
|
|
56
|
+
export function extractScenarios(filePath, framework) {
|
|
57
|
+
let content;
|
|
58
|
+
try {
|
|
59
|
+
content = readFileSync(filePath, 'utf-8');
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
return [];
|
|
63
|
+
}
|
|
64
|
+
const re = framework === 'playwright' ? PLAYWRIGHT_SCENARIO_RE : CYPRESS_SCENARIO_RE;
|
|
65
|
+
const scenarios = [];
|
|
66
|
+
let match;
|
|
67
|
+
// Reset lastIndex in case the regex was used before
|
|
68
|
+
re.lastIndex = 0;
|
|
69
|
+
while ((match = re.exec(content)) !== null) {
|
|
70
|
+
scenarios.push(match[1]);
|
|
71
|
+
}
|
|
72
|
+
return scenarios;
|
|
73
|
+
}
|
|
48
74
|
function resolvePlaywrightSpecs(testsRoot, specDirs) {
|
|
49
|
-
const
|
|
75
|
+
const paths = [];
|
|
76
|
+
const details = [];
|
|
50
77
|
for (const dir of specDirs) {
|
|
51
|
-
|
|
78
|
+
const found = scanDirForSpecs(testsRoot, dir, '.spec.ts');
|
|
79
|
+
for (const relPath of found) {
|
|
80
|
+
paths.push(relPath);
|
|
81
|
+
const absPath = join(testsRoot, relPath);
|
|
82
|
+
details.push({ file: relPath, scenarios: extractScenarios(absPath, 'playwright') });
|
|
83
|
+
}
|
|
52
84
|
}
|
|
53
|
-
return
|
|
85
|
+
return { paths, details };
|
|
54
86
|
}
|
|
55
87
|
function resolveCypressSpecs(cypressRoot, specDirs) {
|
|
56
|
-
const
|
|
88
|
+
const paths = [];
|
|
89
|
+
const details = [];
|
|
57
90
|
for (const dir of specDirs) {
|
|
58
91
|
// cypressSpecDirs are relative to testsRoot (e.g. ../cypress/tests/integration/channels/search/)
|
|
59
92
|
// Resolve them relative to the cypress root
|
|
@@ -63,9 +96,12 @@ function resolveCypressSpecs(cypressRoot, specDirs) {
|
|
|
63
96
|
}
|
|
64
97
|
const found = scanDirForSpecsRecursive(resolvedDir, '.js');
|
|
65
98
|
const tsFound = scanDirForSpecsRecursive(resolvedDir, '.ts');
|
|
66
|
-
|
|
99
|
+
for (const absPath of [...found, ...tsFound]) {
|
|
100
|
+
paths.push(absPath);
|
|
101
|
+
details.push({ file: absPath, scenarios: extractScenarios(absPath, 'cypress') });
|
|
102
|
+
}
|
|
67
103
|
}
|
|
68
|
-
return
|
|
104
|
+
return { paths, details };
|
|
69
105
|
}
|
|
70
106
|
function computeCoverageStatus(pwSpecs, cySpecs) {
|
|
71
107
|
// Playwright is the primary framework — having Playwright specs is sufficient for "covered".
|
|
@@ -138,16 +174,18 @@ export function analyzeImpact(changedFiles, options) {
|
|
|
138
174
|
const cypressSpecDirs = getCypressSpecDirsForBinding(manifest, binding);
|
|
139
175
|
const priority = getPriorityForBinding(manifest, binding);
|
|
140
176
|
const userFlows = getUserFlowsForBinding(manifest, binding);
|
|
141
|
-
const
|
|
142
|
-
const
|
|
143
|
-
const coverageStatus = computeCoverageStatus(
|
|
177
|
+
const pw = resolvePlaywrightSpecs(testsRoot, specDirs);
|
|
178
|
+
const cy = cypressRoot ? resolveCypressSpecs(cypressRoot, cypressSpecDirs) : { paths: [], details: [] };
|
|
179
|
+
const coverageStatus = computeCoverageStatus(pw.paths, cy.paths);
|
|
144
180
|
impactedFeatures.push({
|
|
145
181
|
familyId: group.familyId,
|
|
146
182
|
featureId: group.featureId,
|
|
147
183
|
priority,
|
|
148
184
|
changedFiles: group.files,
|
|
149
|
-
playwrightSpecs,
|
|
150
|
-
cypressSpecs,
|
|
185
|
+
playwrightSpecs: pw.paths,
|
|
186
|
+
cypressSpecs: cy.paths,
|
|
187
|
+
playwrightSpecDetails: pw.details,
|
|
188
|
+
cypressSpecDetails: cy.details,
|
|
151
189
|
userFlows,
|
|
152
190
|
coverageStatus,
|
|
153
191
|
});
|
|
@@ -355,47 +355,52 @@ export function renderCiSummaryMarkdown(plan) {
|
|
|
355
355
|
// ── Blocking gaps ──────────────────────────────────────────────────────────
|
|
356
356
|
if (mustAddTests && plan.requiredNewTests.length > 0) {
|
|
357
357
|
lines.push('');
|
|
358
|
-
lines.push(
|
|
359
|
-
lines.push('');
|
|
360
|
-
lines.push(`The following ${uncoveredP0P1Flows} feature(s) have no test coverage and must be covered before merge:`);
|
|
358
|
+
lines.push(`### ⚠️ Missing coverage for ${uncoveredP0P1Flows} P0/P1 flow(s)`);
|
|
361
359
|
lines.push('');
|
|
362
360
|
for (const gap of plan.gapDetails.filter((g) => !g.name.includes('(partial)'))) {
|
|
363
361
|
const aiLabel = gap.source === 'ai+deterministic' ? ' ✦ AI-enriched' : '';
|
|
362
|
+
// Warning box: name + priority + AI reason (always visible)
|
|
364
363
|
lines.push(`> [!WARNING]`);
|
|
365
364
|
lines.push(`> **${gap.name}** · ${gap.priority}${aiLabel}`);
|
|
366
|
-
// AI-provided reasons (skip the first generic deterministic reason)
|
|
367
365
|
const aiReasons = gap.reasons.slice(1);
|
|
368
366
|
if (aiReasons.length > 0) {
|
|
369
367
|
lines.push(`> ${aiReasons.join(' ')}`);
|
|
370
368
|
}
|
|
369
|
+
lines.push('');
|
|
370
|
+
// Scenarios: collapsible below the warning box
|
|
371
371
|
if (gap.missingScenarios && gap.missingScenarios.length > 0) {
|
|
372
|
-
lines.push(
|
|
373
|
-
lines.push(
|
|
372
|
+
lines.push(`<details><summary>📋 Suggested test scenarios (${gap.missingScenarios.length})</summary>`);
|
|
373
|
+
lines.push('');
|
|
374
374
|
for (const scenario of gap.missingScenarios) {
|
|
375
|
-
lines.push(
|
|
375
|
+
lines.push(`- [ ] ${scenario}`);
|
|
376
376
|
}
|
|
377
|
+
lines.push('');
|
|
378
|
+
lines.push('</details>');
|
|
379
|
+
lines.push('');
|
|
377
380
|
}
|
|
378
|
-
lines.push('');
|
|
379
381
|
}
|
|
380
382
|
}
|
|
381
|
-
// ── Advisory: covered flows with new behavior
|
|
383
|
+
// ── Advisory: covered flows with new behavior ─────────────────────────────
|
|
382
384
|
if (flowsWithAdvisory.length > 0) {
|
|
383
385
|
lines.push('');
|
|
384
|
-
lines.push(
|
|
385
|
-
lines.push(`<summary>💡 New behavior detected in ${flowsWithAdvisory.length} covered feature${flowsWithAdvisory.length !== 1 ? 's' : ''} — consider adding tests</summary>`);
|
|
386
|
-
lines.push('');
|
|
387
|
-
lines.push('These features already have E2E tests, but this PR introduces new behavior worth covering:');
|
|
386
|
+
lines.push(`### 💡 New behavior detected in ${flowsWithAdvisory.length} covered feature${flowsWithAdvisory.length !== 1 ? 's' : ''} — consider adding tests`);
|
|
388
387
|
lines.push('');
|
|
389
388
|
for (const flow of flowsWithAdvisory) {
|
|
390
|
-
|
|
391
|
-
lines.push(
|
|
389
|
+
// Green [!TIP] box: just the name (always visible, compact)
|
|
390
|
+
lines.push(`> [!TIP]`);
|
|
391
|
+
lines.push(`> **${flow.name}** · ${flow.priority}`);
|
|
392
|
+
lines.push('');
|
|
393
|
+
// Specs + scenarios: collapsible below
|
|
394
|
+
const coverageSummary = flow.coveredBy.join(', ');
|
|
395
|
+
lines.push(`<details><summary>${coverageSummary} — click to see suggested scenarios</summary>`);
|
|
392
396
|
lines.push('');
|
|
393
397
|
for (const s of flow.advisoryScenarios) {
|
|
394
398
|
lines.push(`- [ ] ${s}`);
|
|
395
399
|
}
|
|
396
400
|
lines.push('');
|
|
401
|
+
lines.push('</details>');
|
|
402
|
+
lines.push('');
|
|
397
403
|
}
|
|
398
|
-
lines.push('</details>');
|
|
399
404
|
}
|
|
400
405
|
// ── Clean covered flows (collapsed) ───────────────────────────────────────
|
|
401
406
|
if (cleanFlows.length > 0) {
|
package/dist/esm/index.js
CHANGED
|
@@ -12,6 +12,7 @@ export { LLMProviderFactory, validateProviderSetup } from './provider_factory.js
|
|
|
12
12
|
export { analyzeImpactDeterministic, recommendTestsDeterministic, handoffGeneratedTests, ingestTraceability, captureTraceability } from './api.js';
|
|
13
13
|
// V2 Engine (deterministic impact + plan)
|
|
14
14
|
export { analyzeImpact as analyzeImpactV2, getGaps, getPartialGaps } from './engine/impact_engine.js';
|
|
15
|
+
export { extractScenarios } from './engine/impact_engine.js';
|
|
15
16
|
export { buildPlanFromImpact } from './engine/plan_builder.js';
|
|
16
17
|
export { appendFeedbackAndRecompute, readCalibration } from './agent/feedback.js';
|
|
17
18
|
export { finalizeGeneratedTests } from './agent/handoff.js';
|
package/dist/index.d.ts
CHANGED
|
@@ -21,7 +21,8 @@ export type { HybridConfig } from './provider_factory.js';
|
|
|
21
21
|
export { analyzeImpactDeterministic, recommendTestsDeterministic, handoffGeneratedTests, ingestTraceability, captureTraceability } from './api.js';
|
|
22
22
|
export type { AgentApiOptions, RecommendTestsV2Result, TraceabilityIngestApiOptions, TraceabilityCaptureApiOptions, } from './api.js';
|
|
23
23
|
export { analyzeImpact as analyzeImpactV2, getGaps, getPartialGaps } from './engine/impact_engine.js';
|
|
24
|
-
export type { ImpactResult, ImpactedFeature, CoverageStatus, ImpactEngineOptions } from './engine/impact_engine.js';
|
|
24
|
+
export type { ImpactResult, ImpactedFeature, CoverageStatus, ImpactEngineOptions, SpecWithScenarios } from './engine/impact_engine.js';
|
|
25
|
+
export { extractScenarios } from './engine/impact_engine.js';
|
|
25
26
|
export { buildPlanFromImpact } from './engine/plan_builder.js';
|
|
26
27
|
export { appendFeedbackAndRecompute, readCalibration } from './agent/feedback.js';
|
|
27
28
|
export type { RecommendationFeedbackEntry, CalibrationSummary } from './agent/feedback.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;GAWG;AAGH,YAAY,EACR,WAAW,EACX,eAAe,EACf,UAAU,EACV,WAAW,EACX,UAAU,EACV,oBAAoB,EACpB,kBAAkB,EAClB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,YAAY,EACZ,YAAY,GACf,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAC,gBAAgB,EAAE,0BAA0B,EAAC,MAAM,yBAAyB,CAAC;AAGrF,OAAO,EAAC,iBAAiB,EAAE,mBAAmB,EAAC,MAAM,yBAAyB,CAAC;AAC/E,OAAO,EAAC,cAAc,EAAE,gBAAgB,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAC,cAAc,EAAE,gBAAgB,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AAGpD,OAAO,EAAC,kBAAkB,EAAE,qBAAqB,EAAC,MAAM,uBAAuB,CAAC;AAChF,YAAY,EAAC,YAAY,EAAC,MAAM,uBAAuB,CAAC;AAGxD,OAAO,EAAC,0BAA0B,EAAE,2BAA2B,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,mBAAmB,EAAC,MAAM,UAAU,CAAC;AACjJ,YAAY,EACR,eAAe,EACf,sBAAsB,EACtB,4BAA4B,EAC5B,6BAA6B,GAChC,MAAM,UAAU,CAAC;AAGlB,OAAO,EAAC,aAAa,IAAI,eAAe,EAAE,OAAO,EAAE,cAAc,EAAC,MAAM,2BAA2B,CAAC;AACpG,YAAY,EAAC,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,mBAAmB,EAAC,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;GAWG;AAGH,YAAY,EACR,WAAW,EACX,eAAe,EACf,UAAU,EACV,WAAW,EACX,UAAU,EACV,oBAAoB,EACpB,kBAAkB,EAClB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,YAAY,EACZ,YAAY,GACf,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAC,gBAAgB,EAAE,0BAA0B,EAAC,MAAM,yBAAyB,CAAC;AAGrF,OAAO,EAAC,iBAAiB,EAAE,mBAAmB,EAAC,MAAM,yBAAyB,CAAC;AAC/E,OAAO,EAAC,cAAc,EAAE,gBAAgB,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAC,cAAc,EAAE,gBAAgB,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AAGpD,OAAO,EAAC,kBAAkB,EAAE,qBAAqB,EAAC,MAAM,uBAAuB,CAAC;AAChF,YAAY,EAAC,YAAY,EAAC,MAAM,uBAAuB,CAAC;AAGxD,OAAO,EAAC,0BAA0B,EAAE,2BAA2B,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,mBAAmB,EAAC,MAAM,UAAU,CAAC;AACjJ,YAAY,EACR,eAAe,EACf,sBAAsB,EACtB,4BAA4B,EAC5B,6BAA6B,GAChC,MAAM,UAAU,CAAC;AAGlB,OAAO,EAAC,aAAa,IAAI,eAAe,EAAE,OAAO,EAAE,cAAc,EAAC,MAAM,2BAA2B,CAAC;AACpG,YAAY,EAAC,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,mBAAmB,EAAE,iBAAiB,EAAC,MAAM,2BAA2B,CAAC;AACrI,OAAO,EAAC,gBAAgB,EAAC,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAC,mBAAmB,EAAC,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAC,0BAA0B,EAAE,eAAe,EAAC,MAAM,qBAAqB,CAAC;AAChF,YAAY,EAAC,2BAA2B,EAAE,kBAAkB,EAAC,MAAM,qBAAqB,CAAC;AACzF,OAAO,EAAC,sBAAsB,EAAC,MAAM,oBAAoB,CAAC;AAC1D,YAAY,EAAC,6BAA6B,EAAE,4BAA4B,EAAC,MAAM,oBAAoB,CAAC;AACpG,OAAO,EAAC,uBAAuB,EAAC,MAAM,gCAAgC,CAAC;AACvE,YAAY,EAAC,yBAAyB,EAAE,wBAAwB,EAAE,uBAAuB,EAAC,MAAM,gCAAgC,CAAC;AACjI,OAAO,EAAC,wBAAwB,EAAC,MAAM,iCAAiC,CAAC;AACzE,YAAY,EAAC,0BAA0B,EAAE,yBAAyB,EAAC,MAAM,iCAAiC,CAAC;AAG3G,OAAO,EAAC,WAAW,EAAC,MAAM,4BAA4B,CAAC;AACvD,YAAY,EAAC,cAAc,EAAE,cAAc,EAAC,MAAM,4BAA4B,CAAC;AAC/E,YAAY,EAAC,YAAY,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,UAAU,EAAE,cAAc,EAAC,MAAM,+BAA+B,CAAC;AACrI,OAAO,EAAC,kBAAkB,EAAC,MAAM,iCAAiC,CAAC;AACnE,YAAY,EAAC,gBAAgB,EAAE,gBAAgB,EAAE,aAAa,EAAC,MAAM,iCAAiC,CAAC;AACvG,OAAO,EAAC,qBAAqB,EAAE,uBAAuB,EAAE,yBAAyB,EAAC,MAAM,yBAAyB,CAAC;AAClH,YAAY,EAAC,uBAAuB,EAAE,uBAAuB,EAAC,MAAM,yBAAyB,CAAC;AAC9F,OAAO,EAAC,YAAY,EAAE,cAAc,EAAE,kBAAkB,EAAE,kBAAkB,EAAC,MAAM,2BAA2B,CAAC;AAC/G,YAAY,EAAC,UAAU,EAAE,UAAU,EAAE,UAAU,EAAC,MAAM,2BAA2B,CAAC;AAClF,OAAO,EAAC,eAAe,EAAE,qBAAqB,EAAC,MAAM,mBAAmB,CAAC;AACzE,YAAY,EAAC,iBAAiB,EAAC,MAAM,mBAAmB,CAAC;AAGzD,OAAO,EAAC,uBAAuB,EAAE,mBAAmB,EAAE,4BAA4B,EAAE,qBAAqB,EAAE,sBAAsB,EAAC,MAAM,+BAA+B,CAAC;AACxK,YAAY,EAAC,WAAW,EAAE,YAAY,EAAE,mBAAmB,EAAE,WAAW,EAAE,eAAe,EAAC,MAAM,+BAA+B,CAAC;AAChI,OAAO,EAAC,eAAe,EAAE,qBAAqB,EAAC,MAAM,4BAA4B,CAAC;AAClF,YAAY,EAAC,iBAAiB,EAAE,iBAAiB,EAAC,MAAM,4BAA4B,CAAC;AACrF,OAAO,EAAC,cAAc,EAAE,iBAAiB,EAAC,MAAM,2BAA2B,CAAC;AAC5E,YAAY,EAAC,SAAS,EAAE,SAAS,EAAC,MAAM,2BAA2B,CAAC;AAGpE,YAAY,EAAC,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,OAAO,EAAE,WAAW,EAAC,MAAM,kBAAkB,CAAC;AACnG,YAAY,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
3
3
|
// See LICENSE.txt for license information.
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.getSpecsForFamily = exports.buildSpecIndex = exports.loadOrBuildApiSurface = exports.buildApiSurface = exports.getUserFlowsForBinding = exports.getPriorityForBinding = exports.getCypressSpecDirsForBinding = exports.bindFilesToFamilies = exports.loadRouteFamilyManifest = exports.buildQualityFixPrompt = exports.buildHealPrompt = exports.renderHealMarkdown = exports.resolveHealTargets = exports.healFromReport = exports.runHealStage = exports.detectHallucinatedMethods = exports.parseGenerationResponse = exports.buildGenerationPrompt = exports.runGenerationStage = exports.runPipeline = exports.captureTraceabilityInput = exports.ingestTraceabilityInput = exports.finalizeGeneratedTests = exports.readCalibration = exports.appendFeedbackAndRecompute = exports.buildPlanFromImpact = exports.getPartialGaps = exports.getGaps = exports.analyzeImpactV2 = exports.captureTraceability = exports.ingestTraceability = exports.handoffGeneratedTests = exports.recommendTestsDeterministic = exports.analyzeImpactDeterministic = exports.validateProviderSetup = exports.LLMProviderFactory = exports.CustomProvider = exports.checkOpenAISetup = exports.OpenAIProvider = exports.checkOllamaSetup = exports.OllamaProvider = exports.checkAnthropicSetup = exports.AnthropicProvider = exports.UnsupportedCapabilityError = exports.LLMProviderError = void 0;
|
|
5
|
+
exports.getSpecsForFamily = exports.buildSpecIndex = exports.loadOrBuildApiSurface = exports.buildApiSurface = exports.getUserFlowsForBinding = exports.getPriorityForBinding = exports.getCypressSpecDirsForBinding = exports.bindFilesToFamilies = exports.loadRouteFamilyManifest = exports.buildQualityFixPrompt = exports.buildHealPrompt = exports.renderHealMarkdown = exports.resolveHealTargets = exports.healFromReport = exports.runHealStage = exports.detectHallucinatedMethods = exports.parseGenerationResponse = exports.buildGenerationPrompt = exports.runGenerationStage = exports.runPipeline = exports.captureTraceabilityInput = exports.ingestTraceabilityInput = exports.finalizeGeneratedTests = exports.readCalibration = exports.appendFeedbackAndRecompute = exports.buildPlanFromImpact = exports.extractScenarios = exports.getPartialGaps = exports.getGaps = exports.analyzeImpactV2 = exports.captureTraceability = exports.ingestTraceability = exports.handoffGeneratedTests = exports.recommendTestsDeterministic = exports.analyzeImpactDeterministic = exports.validateProviderSetup = exports.LLMProviderFactory = exports.CustomProvider = exports.checkOpenAISetup = exports.OpenAIProvider = exports.checkOllamaSetup = exports.OllamaProvider = exports.checkAnthropicSetup = exports.AnthropicProvider = exports.UnsupportedCapabilityError = exports.LLMProviderError = void 0;
|
|
6
6
|
var provider_interface_js_1 = require("./provider_interface.js");
|
|
7
7
|
Object.defineProperty(exports, "LLMProviderError", { enumerable: true, get: function () { return provider_interface_js_1.LLMProviderError; } });
|
|
8
8
|
Object.defineProperty(exports, "UnsupportedCapabilityError", { enumerable: true, get: function () { return provider_interface_js_1.UnsupportedCapabilityError; } });
|
|
@@ -34,6 +34,8 @@ var impact_engine_js_1 = require("./engine/impact_engine.js");
|
|
|
34
34
|
Object.defineProperty(exports, "analyzeImpactV2", { enumerable: true, get: function () { return impact_engine_js_1.analyzeImpact; } });
|
|
35
35
|
Object.defineProperty(exports, "getGaps", { enumerable: true, get: function () { return impact_engine_js_1.getGaps; } });
|
|
36
36
|
Object.defineProperty(exports, "getPartialGaps", { enumerable: true, get: function () { return impact_engine_js_1.getPartialGaps; } });
|
|
37
|
+
var impact_engine_js_2 = require("./engine/impact_engine.js");
|
|
38
|
+
Object.defineProperty(exports, "extractScenarios", { enumerable: true, get: function () { return impact_engine_js_2.extractScenarios; } });
|
|
37
39
|
var plan_builder_js_1 = require("./engine/plan_builder.js");
|
|
38
40
|
Object.defineProperty(exports, "buildPlanFromImpact", { enumerable: true, get: function () { return plan_builder_js_1.buildPlanFromImpact; } });
|
|
39
41
|
var feedback_js_1 = require("./agent/feedback.js");
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@yasserkhanorg/e2e-agents",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.0",
|
|
4
4
|
"description": "Pluggable LLM provider library for AI-powered test automation. Use Claude, Ollama, or your own LLM. Integrate with Playwright, Jest, or any test framework. MCP server for test agents, cost tracking, and hybrid provider mode.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/esm/index.js",
|