@yasserkhanorg/e2e-agents 0.5.16 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/pipeline.d.ts +1 -1
- package/dist/agent/pipeline.d.ts.map +1 -1
- package/dist/agent/plan.d.ts +0 -12
- package/dist/agent/plan.d.ts.map +1 -1
- package/dist/agent/plan.js +0 -365
- package/dist/agent/types.d.ts +42 -0
- package/dist/agent/types.d.ts.map +1 -0
- package/dist/agent/types.js +4 -0
- package/dist/api.d.ts +10 -14
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +29 -59
- package/dist/cli.js +41 -174
- package/dist/engine/impact_engine.d.ts +36 -0
- package/dist/engine/impact_engine.d.ts.map +1 -0
- package/dist/engine/impact_engine.js +196 -0
- package/dist/engine/plan_builder.d.ts +9 -0
- package/dist/engine/plan_builder.d.ts.map +1 -0
- package/dist/engine/plan_builder.js +329 -0
- package/dist/esm/agent/plan.js +1 -360
- package/dist/esm/agent/types.js +3 -0
- package/dist/esm/api.js +27 -56
- package/dist/esm/cli.js +40 -173
- package/dist/esm/engine/impact_engine.js +191 -0
- package/dist/esm/engine/plan_builder.js +323 -0
- package/dist/esm/index.js +6 -3
- package/dist/esm/knowledge/route_families.js +57 -0
- package/dist/index.d.ts +9 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -5
- package/dist/knowledge/route_families.d.ts +19 -0
- package/dist/knowledge/route_families.d.ts.map +1 -1
- package/dist/knowledge/route_families.js +60 -0
- package/package.json +1 -1
- package/dist/agent/ai_flow_analysis.d.ts +0 -13
- package/dist/agent/ai_flow_analysis.d.ts.map +0 -1
- package/dist/agent/ai_flow_analysis.js +0 -334
- package/dist/agent/ai_mapping.d.ts +0 -14
- package/dist/agent/ai_mapping.d.ts.map +0 -1
- package/dist/agent/ai_mapping.js +0 -560
- package/dist/agent/analysis.d.ts +0 -64
- package/dist/agent/analysis.d.ts.map +0 -1
- package/dist/agent/analysis.js +0 -292
- package/dist/agent/blast_radius.d.ts +0 -4
- package/dist/agent/blast_radius.d.ts.map +0 -1
- package/dist/agent/blast_radius.js +0 -37
- package/dist/agent/dependency_graph.d.ts +0 -14
- package/dist/agent/dependency_graph.d.ts.map +0 -1
- package/dist/agent/dependency_graph.js +0 -227
- package/dist/agent/flags.d.ts +0 -23
- package/dist/agent/flags.d.ts.map +0 -1
- package/dist/agent/flags.js +0 -171
- package/dist/agent/flow_catalog.d.ts +0 -25
- package/dist/agent/flow_catalog.d.ts.map +0 -1
- package/dist/agent/flow_catalog.js +0 -115
- package/dist/agent/flow_mapping.d.ts +0 -10
- package/dist/agent/flow_mapping.d.ts.map +0 -1
- package/dist/agent/flow_mapping.js +0 -84
- package/dist/agent/framework.d.ts +0 -13
- package/dist/agent/framework.d.ts.map +0 -1
- package/dist/agent/framework.js +0 -149
- package/dist/agent/gap_suggestions.d.ts +0 -14
- package/dist/agent/gap_suggestions.d.ts.map +0 -1
- package/dist/agent/gap_suggestions.js +0 -101
- package/dist/agent/generator.d.ts +0 -10
- package/dist/agent/generator.d.ts.map +0 -1
- package/dist/agent/generator.js +0 -115
- package/dist/agent/operational_insights.d.ts +0 -41
- package/dist/agent/operational_insights.d.ts.map +0 -1
- package/dist/agent/operational_insights.js +0 -127
- package/dist/agent/report.d.ts +0 -97
- package/dist/agent/report.d.ts.map +0 -1
- package/dist/agent/report.js +0 -159
- package/dist/agent/runner.d.ts +0 -7
- package/dist/agent/runner.d.ts.map +0 -1
- package/dist/agent/runner.js +0 -898
- package/dist/agent/selectors.d.ts +0 -10
- package/dist/agent/selectors.d.ts.map +0 -1
- package/dist/agent/selectors.js +0 -75
- package/dist/agent/subsystem_risk.d.ts +0 -23
- package/dist/agent/subsystem_risk.d.ts.map +0 -1
- package/dist/agent/subsystem_risk.js +0 -207
- package/dist/agent/tests.d.ts +0 -19
- package/dist/agent/tests.d.ts.map +0 -1
- package/dist/agent/tests.js +0 -116
- package/dist/agent/traceability.d.ts +0 -22
- package/dist/agent/traceability.d.ts.map +0 -1
- package/dist/agent/traceability.js +0 -183
- package/dist/esm/agent/ai_flow_analysis.js +0 -331
- package/dist/esm/agent/ai_mapping.js +0 -557
- package/dist/esm/agent/analysis.js +0 -287
- package/dist/esm/agent/blast_radius.js +0 -34
- package/dist/esm/agent/dependency_graph.js +0 -224
- package/dist/esm/agent/flags.js +0 -160
- package/dist/esm/agent/flow_catalog.js +0 -112
- package/dist/esm/agent/flow_mapping.js +0 -81
- package/dist/esm/agent/framework.js +0 -145
- package/dist/esm/agent/gap_suggestions.js +0 -98
- package/dist/esm/agent/generator.js +0 -112
- package/dist/esm/agent/operational_insights.js +0 -124
- package/dist/esm/agent/report.js +0 -156
- package/dist/esm/agent/runner.js +0 -894
- package/dist/esm/agent/selectors.js +0 -71
- package/dist/esm/agent/subsystem_risk.js +0 -204
- package/dist/esm/agent/tests.js +0 -111
- package/dist/esm/agent/traceability.js +0 -180
package/dist/esm/api.js
CHANGED
|
@@ -1,21 +1,13 @@
|
|
|
1
1
|
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
2
|
// See LICENSE.txt for license information.
|
|
3
|
-
import { existsSync, readFileSync } from 'fs';
|
|
4
|
-
import { join } from 'path';
|
|
5
3
|
import { resolveConfig } from './agent/config.js';
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
4
|
+
import { appendPlanMetrics, } from './agent/plan.js';
|
|
5
|
+
import { analyzeImpact as analyzeImpactV2 } from './engine/impact_engine.js';
|
|
6
|
+
import { buildPlanFromImpact, renderCiSummaryMarkdown, writeCiSummary, writePlanReport, } from './engine/plan_builder.js';
|
|
7
|
+
import { getChangedFiles } from './agent/git.js';
|
|
9
8
|
import { finalizeGeneratedTests } from './agent/handoff.js';
|
|
10
9
|
import { ingestTraceabilityInput, } from './agent/traceability_ingest.js';
|
|
11
10
|
import { captureTraceabilityInput, } from './agent/traceability_capture.js';
|
|
12
|
-
function readReportJson(reportPath) {
|
|
13
|
-
if (!existsSync(reportPath)) {
|
|
14
|
-
throw new Error(`Expected report not found: ${reportPath}`);
|
|
15
|
-
}
|
|
16
|
-
const raw = readFileSync(reportPath, 'utf-8');
|
|
17
|
-
return JSON.parse(raw);
|
|
18
|
-
}
|
|
19
11
|
function resolveAgent(options, mode) {
|
|
20
12
|
const cwd = options.cwd || process.cwd();
|
|
21
13
|
const { config } = resolveConfig(cwd, options.configPath, {
|
|
@@ -27,63 +19,42 @@ function resolveAgent(options, mode) {
|
|
|
27
19
|
}
|
|
28
20
|
return config;
|
|
29
21
|
}
|
|
30
|
-
function
|
|
31
|
-
return
|
|
22
|
+
export function handoffGeneratedTests(options) {
|
|
23
|
+
return finalizeGeneratedTests(options);
|
|
32
24
|
}
|
|
33
|
-
export
|
|
34
|
-
const
|
|
35
|
-
|
|
25
|
+
export function ingestTraceability(options) {
|
|
26
|
+
const cwd = options.cwd || process.cwd();
|
|
27
|
+
const { config } = resolveConfig(cwd, options.configPath, {
|
|
28
|
+
path: options.path,
|
|
29
|
+
testsRoot: options.testsRoot,
|
|
30
|
+
mode: 'impact',
|
|
31
|
+
});
|
|
36
32
|
const reportRoot = config.testsRoot || config.path;
|
|
37
|
-
|
|
38
|
-
const report = readReportJson(reportPath);
|
|
39
|
-
return { report, reportPath };
|
|
33
|
+
return ingestTraceabilityInput(reportRoot, config.impact.traceability, options.payload, options.options);
|
|
40
34
|
}
|
|
41
|
-
export
|
|
42
|
-
const config = resolveAgent(options, '
|
|
43
|
-
await runGap(config, { apply: options.apply ?? false });
|
|
35
|
+
export function analyzeImpactDeterministic(options = {}) {
|
|
36
|
+
const config = resolveAgent(options, 'impact');
|
|
44
37
|
const reportRoot = config.testsRoot || config.path;
|
|
45
|
-
const
|
|
46
|
-
|
|
47
|
-
|
|
38
|
+
const gitResult = getChangedFiles(config.path, config.git.since, { includeUncommitted: config.git.includeUncommitted });
|
|
39
|
+
return analyzeImpactV2(gitResult.files, {
|
|
40
|
+
testsRoot: reportRoot,
|
|
41
|
+
routeFamilies: config.routeFamilies,
|
|
42
|
+
});
|
|
48
43
|
}
|
|
49
|
-
export
|
|
44
|
+
export function recommendTestsDeterministic(options = {}) {
|
|
50
45
|
const config = resolveAgent(options, 'impact');
|
|
51
|
-
await runImpact(config, { apply: options.apply ?? false });
|
|
52
46
|
const reportRoot = config.testsRoot || config.path;
|
|
53
|
-
const
|
|
54
|
-
const
|
|
55
|
-
const basePlan = buildPlanFromImpactReport(report, config.policy);
|
|
56
|
-
const withActions = attachDeveloperActions(basePlan, {
|
|
57
|
-
appPath: config.path,
|
|
47
|
+
const gitResult = getChangedFiles(config.path, config.git.since, { includeUncommitted: config.git.includeUncommitted });
|
|
48
|
+
const impact = analyzeImpactV2(gitResult.files, {
|
|
58
49
|
testsRoot: reportRoot,
|
|
59
|
-
|
|
50
|
+
routeFamilies: config.routeFamilies,
|
|
60
51
|
});
|
|
61
|
-
const plan =
|
|
52
|
+
const plan = buildPlanFromImpact(impact, config.policy);
|
|
62
53
|
const planPath = writePlanReport(reportRoot, plan);
|
|
63
54
|
const ciSummaryMarkdown = renderCiSummaryMarkdown(plan);
|
|
64
55
|
const ciSummaryPath = writeCiSummary(reportRoot, ciSummaryMarkdown);
|
|
65
56
|
appendPlanMetrics(reportRoot, plan);
|
|
66
|
-
return {
|
|
67
|
-
report,
|
|
68
|
-
reportPath: impactPath,
|
|
69
|
-
plan,
|
|
70
|
-
planPath,
|
|
71
|
-
ciSummaryMarkdown,
|
|
72
|
-
ciSummaryPath,
|
|
73
|
-
};
|
|
74
|
-
}
|
|
75
|
-
export function handoffGeneratedTests(options) {
|
|
76
|
-
return finalizeGeneratedTests(options);
|
|
77
|
-
}
|
|
78
|
-
export function ingestTraceability(options) {
|
|
79
|
-
const cwd = options.cwd || process.cwd();
|
|
80
|
-
const { config } = resolveConfig(cwd, options.configPath, {
|
|
81
|
-
path: options.path,
|
|
82
|
-
testsRoot: options.testsRoot,
|
|
83
|
-
mode: 'impact',
|
|
84
|
-
});
|
|
85
|
-
const reportRoot = config.testsRoot || config.path;
|
|
86
|
-
return ingestTraceabilityInput(reportRoot, config.impact.traceability, options.payload, options.options);
|
|
57
|
+
return { impact, plan, planPath, ciSummaryMarkdown, ciSummaryPath };
|
|
87
58
|
}
|
|
88
59
|
export function captureTraceability(options) {
|
|
89
60
|
const cwd = options.cwd || process.cwd();
|
package/dist/esm/cli.js
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
3
3
|
// See LICENSE.txt for license information.
|
|
4
|
-
import { appendFileSync, existsSync, readFileSync
|
|
4
|
+
import { appendFileSync, existsSync, readFileSync } from 'fs';
|
|
5
5
|
import { dirname, join, resolve } from 'path';
|
|
6
6
|
import { resolveConfig } from './agent/config.js';
|
|
7
7
|
import { AnthropicProvider } from './anthropic_provider.js';
|
|
8
8
|
import { LLMProviderError } from './provider_interface.js';
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
9
|
+
import { appendPlanMetrics } from './agent/plan.js';
|
|
10
|
+
import { analyzeImpact as analyzeImpactV2 } from './engine/impact_engine.js';
|
|
11
|
+
import { buildPlanFromImpact, renderCiSummaryMarkdown, writeCiSummary, writePlanReport, } from './engine/plan_builder.js';
|
|
12
|
+
import { getChangedFiles } from './agent/git.js';
|
|
12
13
|
import { appendFeedbackAndRecompute } from './agent/feedback.js';
|
|
13
14
|
import { finalizeGeneratedTests } from './agent/handoff.js';
|
|
14
15
|
import { ingestTraceabilityInput } from './agent/traceability_ingest.js';
|
|
@@ -59,13 +60,9 @@ function printUsage() {
|
|
|
59
60
|
console.log([
|
|
60
61
|
'Usage:',
|
|
61
62
|
' e2e-ai-agents impact --path <app-root> [options]',
|
|
62
|
-
' e2e-ai-agents gap --path <app-root> [options]',
|
|
63
63
|
' e2e-ai-agents plan --path <app-root> [options]',
|
|
64
|
-
' e2e-ai-agents generate --path <app-root> [options]',
|
|
65
|
-
' e2e-ai-agents heal --path <app-root> --traceability-report <json> [options]',
|
|
66
64
|
' e2e-ai-agents suggest --path <app-root> [options]',
|
|
67
|
-
' e2e-ai-agents
|
|
68
|
-
' e2e-ai-agents auto-heal-pr --path <app-root> [options]',
|
|
65
|
+
' e2e-ai-agents heal --path <app-root> --traceability-report <json> [options]',
|
|
69
66
|
' e2e-ai-agents finalize-generated-tests --path <app-root> [options]',
|
|
70
67
|
' e2e-ai-agents feedback --path <app-root> --feedback-input <json>',
|
|
71
68
|
' e2e-ai-agents traceability-capture --path <app-root> --traceability-report <json>',
|
|
@@ -144,13 +141,9 @@ function parseArgs(argv) {
|
|
|
144
141
|
}
|
|
145
142
|
const command = argv[0];
|
|
146
143
|
if (command === 'impact'
|
|
147
|
-
|| command === 'gap'
|
|
148
144
|
|| command === 'plan'
|
|
149
|
-
|| command === 'generate'
|
|
150
145
|
|| command === 'heal'
|
|
151
146
|
|| command === 'suggest'
|
|
152
|
-
|| command === 'approve-and-generate'
|
|
153
|
-
|| command === 'auto-heal-pr'
|
|
154
147
|
|| command === 'finalize-generated-tests'
|
|
155
148
|
|| command === 'feedback'
|
|
156
149
|
|| command === 'traceability-capture'
|
|
@@ -759,117 +752,6 @@ async function main() {
|
|
|
759
752
|
}
|
|
760
753
|
return;
|
|
761
754
|
}
|
|
762
|
-
if (args.command === 'auto-heal-pr') {
|
|
763
|
-
if (!args.path && !autoConfig) {
|
|
764
|
-
// eslint-disable-next-line no-console
|
|
765
|
-
console.error('Error: --path is required for auto-heal-pr command');
|
|
766
|
-
process.exit(1);
|
|
767
|
-
}
|
|
768
|
-
const { config } = resolveConfig(process.cwd(), autoConfig, {
|
|
769
|
-
path: args.path,
|
|
770
|
-
profile: args.profile,
|
|
771
|
-
testsRoot: args.testsRoot,
|
|
772
|
-
mode: 'gap',
|
|
773
|
-
framework: args.framework,
|
|
774
|
-
timeLimitMinutes: args.timeLimitMinutes,
|
|
775
|
-
budget: {
|
|
776
|
-
maxUSD: args.budgetUSD,
|
|
777
|
-
maxTokens: args.budgetTokens,
|
|
778
|
-
},
|
|
779
|
-
testPatterns: args.testPatterns,
|
|
780
|
-
flowPatterns: args.flowPatterns,
|
|
781
|
-
flowExclude: args.flowExclude,
|
|
782
|
-
flowCatalogPath: args.flowCatalogPath,
|
|
783
|
-
specPDF: args.specPDF,
|
|
784
|
-
gitSince: args.gitSince,
|
|
785
|
-
pipeline: {
|
|
786
|
-
enabled: true,
|
|
787
|
-
scenarios: args.pipelineScenarios,
|
|
788
|
-
outputDir: args.pipelineOutput,
|
|
789
|
-
baseUrl: args.pipelineBaseUrl,
|
|
790
|
-
browser: args.pipelineBrowser,
|
|
791
|
-
headless: args.pipelineHeadless,
|
|
792
|
-
project: args.pipelineProject,
|
|
793
|
-
parallel: args.pipelineParallel,
|
|
794
|
-
dryRun: args.pipelineDryRun,
|
|
795
|
-
mcp: args.pipelineMcp,
|
|
796
|
-
mcpAllowFallback: args.pipelineMcpAllowFallback,
|
|
797
|
-
mcpOnly: args.pipelineMcpOnly,
|
|
798
|
-
},
|
|
799
|
-
llmProvider: args.llmProvider,
|
|
800
|
-
});
|
|
801
|
-
if (args.allowFallback) {
|
|
802
|
-
config.impact.allowFallback = true;
|
|
803
|
-
}
|
|
804
|
-
await runGap(config, { apply: true });
|
|
805
|
-
const reportRoot = config.testsRoot || config.path;
|
|
806
|
-
if (args.traceabilityReportPath) {
|
|
807
|
-
const unstableSpecs = extractPlaywrightUnstableSpecs(args.traceabilityReportPath, [reportRoot, config.path]);
|
|
808
|
-
if (unstableSpecs.length > 0) {
|
|
809
|
-
const targetedSummary = runTargetedSpecHeal(reportRoot, unstableSpecs.map((spec) => ({
|
|
810
|
-
specPath: spec.specPath,
|
|
811
|
-
status: spec.status,
|
|
812
|
-
reason: `Playwright report: failingTests=${spec.failingTests}, flakyTests=${spec.flakyTests}`,
|
|
813
|
-
})), {
|
|
814
|
-
...config.pipeline,
|
|
815
|
-
enabled: true,
|
|
816
|
-
heal: true,
|
|
817
|
-
});
|
|
818
|
-
const healedCount = targetedSummary.results.filter((result) => result.healStatus === 'success').length;
|
|
819
|
-
// eslint-disable-next-line no-console
|
|
820
|
-
console.log(`Auto-heal targeted unstable specs: ${unstableSpecs.length} (healed=${healedCount})`);
|
|
821
|
-
if (targetedSummary.warnings.length > 0) {
|
|
822
|
-
// eslint-disable-next-line no-console
|
|
823
|
-
console.log(`Auto-heal warnings: ${targetedSummary.warnings.join(' | ')}`);
|
|
824
|
-
}
|
|
825
|
-
const gapPath = join(reportRoot, '.e2e-ai-agents', 'gap.json');
|
|
826
|
-
if (existsSync(gapPath)) {
|
|
827
|
-
const gap = JSON.parse(readFileSync(gapPath, 'utf-8'));
|
|
828
|
-
const existingResults = Array.isArray(gap.pipeline?.results) ? gap.pipeline?.results : [];
|
|
829
|
-
const existingWarnings = Array.isArray(gap.pipeline?.warnings) ? gap.pipeline?.warnings : [];
|
|
830
|
-
gap.pipeline = {
|
|
831
|
-
runner: gap.pipeline?.runner || targetedSummary.runner,
|
|
832
|
-
results: [...existingResults, ...targetedSummary.results],
|
|
833
|
-
warnings: Array.from(new Set([...(existingWarnings || []), ...targetedSummary.warnings])),
|
|
834
|
-
};
|
|
835
|
-
writeFileSync(gapPath, `${JSON.stringify(gap, null, 2)}\n`, 'utf-8');
|
|
836
|
-
}
|
|
837
|
-
}
|
|
838
|
-
else {
|
|
839
|
-
// eslint-disable-next-line no-console
|
|
840
|
-
console.log('Auto-heal targeted unstable specs: 0');
|
|
841
|
-
}
|
|
842
|
-
}
|
|
843
|
-
const branchSuffix = new Date().toISOString().replace(/[:.]/g, '-');
|
|
844
|
-
const result = finalizeGeneratedTests({
|
|
845
|
-
appPath: config.path,
|
|
846
|
-
testsRoot: reportRoot,
|
|
847
|
-
branch: args.branch || `auto-heal-${branchSuffix}`,
|
|
848
|
-
commitMessage: args.commitMessage || 'test(e2e): auto-heal generated specs',
|
|
849
|
-
createPr: true,
|
|
850
|
-
prTitle: args.prTitle || 'test(e2e): auto-heal generated specs',
|
|
851
|
-
prBody: args.prBody || 'Automated e2e-heal run generated by @yasserkhanorg/e2e-agents.',
|
|
852
|
-
baseBranch: args.prBase || 'master',
|
|
853
|
-
dryRun: args.dryRun,
|
|
854
|
-
});
|
|
855
|
-
// eslint-disable-next-line no-console
|
|
856
|
-
console.log(`Auto-heal repo root: ${result.repoRoot}`);
|
|
857
|
-
// eslint-disable-next-line no-console
|
|
858
|
-
console.log(`Auto-heal branch: ${result.branch}`);
|
|
859
|
-
// eslint-disable-next-line no-console
|
|
860
|
-
console.log(`Auto-heal staged paths: ${result.stagedPaths.join(', ') || 'none'}`);
|
|
861
|
-
// eslint-disable-next-line no-console
|
|
862
|
-
console.log(`Auto-heal commit: ${result.committed ? 'created' : 'skipped'}`);
|
|
863
|
-
if (result.commitSha) {
|
|
864
|
-
// eslint-disable-next-line no-console
|
|
865
|
-
console.log(`Auto-heal commit sha: ${result.commitSha}`);
|
|
866
|
-
}
|
|
867
|
-
if (result.prUrl) {
|
|
868
|
-
// eslint-disable-next-line no-console
|
|
869
|
-
console.log(`Auto-heal PR: ${result.prUrl}`);
|
|
870
|
-
}
|
|
871
|
-
return;
|
|
872
|
-
}
|
|
873
755
|
if (args.command === 'heal') {
|
|
874
756
|
if (!args.path && !autoConfig) {
|
|
875
757
|
// eslint-disable-next-line no-console
|
|
@@ -934,13 +816,11 @@ async function main() {
|
|
|
934
816
|
printUsage();
|
|
935
817
|
process.exit(1);
|
|
936
818
|
}
|
|
937
|
-
const
|
|
938
|
-
const forceAIPipelineFromApproval = args.command === 'approve-and-generate' || args.command === 'generate';
|
|
939
|
-
const { config, configPath } = resolveConfig(process.cwd(), autoConfig, {
|
|
819
|
+
const { config } = resolveConfig(process.cwd(), autoConfig, {
|
|
940
820
|
path: args.path,
|
|
941
821
|
profile: args.profile,
|
|
942
822
|
testsRoot: args.testsRoot,
|
|
943
|
-
mode:
|
|
823
|
+
mode: 'impact',
|
|
944
824
|
framework: args.framework,
|
|
945
825
|
timeLimitMinutes: args.timeLimitMinutes,
|
|
946
826
|
budget: {
|
|
@@ -954,7 +834,7 @@ async function main() {
|
|
|
954
834
|
specPDF: args.specPDF,
|
|
955
835
|
gitSince: args.gitSince,
|
|
956
836
|
llmProvider: args.llmProvider,
|
|
957
|
-
pipeline:
|
|
837
|
+
pipeline: args.pipeline
|
|
958
838
|
? {
|
|
959
839
|
enabled: true,
|
|
960
840
|
scenarios: args.pipelineScenarios,
|
|
@@ -965,9 +845,9 @@ async function main() {
|
|
|
965
845
|
project: args.pipelineProject,
|
|
966
846
|
parallel: args.pipelineParallel,
|
|
967
847
|
dryRun: args.pipelineDryRun,
|
|
968
|
-
mcp: args.pipelineMcp
|
|
848
|
+
mcp: args.pipelineMcp,
|
|
969
849
|
mcpAllowFallback: args.pipelineMcpAllowFallback,
|
|
970
|
-
mcpOnly: args.pipelineMcpOnly
|
|
850
|
+
mcpOnly: args.pipelineMcpOnly,
|
|
971
851
|
mcpCommandTimeoutMs: args.pipelineMcpTimeoutMs,
|
|
972
852
|
mcpRetries: args.pipelineMcpRetries,
|
|
973
853
|
}
|
|
@@ -988,42 +868,38 @@ async function main() {
|
|
|
988
868
|
}
|
|
989
869
|
: undefined,
|
|
990
870
|
});
|
|
991
|
-
if (args.allowFallback) {
|
|
992
|
-
config.impact.allowFallback = true;
|
|
993
|
-
}
|
|
994
871
|
if (args.command === 'impact') {
|
|
995
|
-
await runImpact(config, { apply: args.apply });
|
|
996
|
-
return;
|
|
997
|
-
}
|
|
998
|
-
if (args.command === 'suggest' || args.command === 'plan') {
|
|
999
872
|
const reportRoot = config.testsRoot || config.path;
|
|
1000
|
-
const
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
873
|
+
const gitResult = getChangedFiles(config.path, config.git.since, { includeUncommitted: config.git.includeUncommitted });
|
|
874
|
+
const impactResult = analyzeImpactV2(gitResult.files, {
|
|
875
|
+
testsRoot: reportRoot,
|
|
876
|
+
routeFamilies: config.routeFamilies,
|
|
877
|
+
});
|
|
878
|
+
// eslint-disable-next-line no-console
|
|
879
|
+
console.log(`Impact: ${impactResult.changedFiles.length} changed files → ${impactResult.impactedFeatures.length} features impacted`);
|
|
880
|
+
// eslint-disable-next-line no-console
|
|
881
|
+
console.log(`Unbound files: ${impactResult.unboundFiles.length}`);
|
|
882
|
+
for (const f of impactResult.impactedFeatures) {
|
|
883
|
+
const label = f.featureId || f.familyId;
|
|
884
|
+
// eslint-disable-next-line no-console
|
|
885
|
+
console.log(` [${f.priority}] ${label}: ${f.coverageStatus} (PW=${f.playwrightSpecs.length}, Cy=${f.cypressSpecs.length})`);
|
|
886
|
+
}
|
|
887
|
+
if (impactResult.warnings.length > 0) {
|
|
888
|
+
for (const w of impactResult.warnings) {
|
|
1008
889
|
// eslint-disable-next-line no-console
|
|
1009
|
-
console.warn(`
|
|
1010
|
-
}
|
|
1011
|
-
else {
|
|
1012
|
-
throw err;
|
|
890
|
+
console.warn(` Warning: ${w}`);
|
|
1013
891
|
}
|
|
1014
892
|
}
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
const
|
|
1019
|
-
const
|
|
1020
|
-
const
|
|
1021
|
-
appPath: config.path,
|
|
893
|
+
return;
|
|
894
|
+
}
|
|
895
|
+
if (args.command === 'suggest' || args.command === 'plan') {
|
|
896
|
+
const reportRoot = config.testsRoot || config.path;
|
|
897
|
+
const gitResult = getChangedFiles(config.path, config.git.since, { includeUncommitted: config.git.includeUncommitted });
|
|
898
|
+
const impactResult = analyzeImpactV2(gitResult.files, {
|
|
1022
899
|
testsRoot: reportRoot,
|
|
1023
|
-
|
|
1024
|
-
configPath,
|
|
900
|
+
routeFamilies: config.routeFamilies,
|
|
1025
901
|
});
|
|
1026
|
-
const plan =
|
|
902
|
+
const plan = buildPlanFromImpact(impactResult, config.policy);
|
|
1027
903
|
const planPath = writePlanReport(reportRoot, plan);
|
|
1028
904
|
const summaryMarkdown = renderCiSummaryMarkdown(plan);
|
|
1029
905
|
const summaryPath = writeCiSummary(reportRoot, summaryMarkdown, args.ciCommentPath);
|
|
@@ -1054,25 +930,16 @@ async function main() {
|
|
|
1054
930
|
console.log(`CI summary: ${summaryPath}`);
|
|
1055
931
|
// eslint-disable-next-line no-console
|
|
1056
932
|
console.log(`Plan metrics: ${metrics.summaryPath}`);
|
|
1057
|
-
if (plan.nextActions) {
|
|
1058
|
-
// eslint-disable-next-line no-console
|
|
1059
|
-
console.log(`Next action (run existing): ${plan.nextActions.runRecommendedTests || plan.nextActions.runSmokeSuite}`);
|
|
1060
|
-
// eslint-disable-next-line no-console
|
|
1061
|
-
console.log(`Next action (approve + generate): ${plan.nextActions.approveAndGenerate || plan.nextActions.generateMissingTests}`);
|
|
1062
|
-
// eslint-disable-next-line no-console
|
|
1063
|
-
console.log(`Next action (heal): ${plan.nextActions.healGeneratedTests}`);
|
|
1064
|
-
}
|
|
1065
933
|
const failOnLegacyFlag = args.failOnMustAddTests && plan.decision.action === 'must-add-tests';
|
|
1066
934
|
if (failOnLegacyFlag || plan.enforcement.shouldFail) {
|
|
1067
935
|
process.exit(2);
|
|
1068
936
|
}
|
|
1069
937
|
return;
|
|
1070
938
|
}
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
await runGap(config, { apply: args.apply });
|
|
939
|
+
// eslint-disable-next-line no-console
|
|
940
|
+
console.error(`Unknown command: ${args.command}`);
|
|
941
|
+
printUsage();
|
|
942
|
+
process.exit(1);
|
|
1076
943
|
}
|
|
1077
944
|
async function runLlmHealth() {
|
|
1078
945
|
if (!process.env.ANTHROPIC_API_KEY) {
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
import { existsSync, readdirSync } from 'fs';
|
|
4
|
+
import { join } from 'path';
|
|
5
|
+
import { loadRouteFamilyManifest, bindFilesToFamilies, getSpecDirsForBinding, getCypressSpecDirsForBinding, getPriorityForBinding, getUserFlowsForBinding, } from '../knowledge/route_families.js';
|
|
6
|
+
function scanDirForSpecs(baseDir, specDir, extension) {
|
|
7
|
+
const fullDir = join(baseDir, specDir);
|
|
8
|
+
if (!existsSync(fullDir)) {
|
|
9
|
+
return [];
|
|
10
|
+
}
|
|
11
|
+
const specs = [];
|
|
12
|
+
try {
|
|
13
|
+
const items = readdirSync(fullDir, { withFileTypes: true });
|
|
14
|
+
for (const item of items) {
|
|
15
|
+
const itemPath = join(fullDir, item.name);
|
|
16
|
+
if (item.isDirectory()) {
|
|
17
|
+
specs.push(...scanDirForSpecsRecursive(itemPath, extension));
|
|
18
|
+
}
|
|
19
|
+
else if (item.name.endsWith(extension)) {
|
|
20
|
+
specs.push(join(specDir, item.name));
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
// Directory not readable
|
|
26
|
+
}
|
|
27
|
+
return specs;
|
|
28
|
+
}
|
|
29
|
+
function scanDirForSpecsRecursive(dir, extension) {
|
|
30
|
+
const specs = [];
|
|
31
|
+
try {
|
|
32
|
+
const items = readdirSync(dir, { withFileTypes: true });
|
|
33
|
+
for (const item of items) {
|
|
34
|
+
const fullPath = join(dir, item.name);
|
|
35
|
+
if (item.isDirectory()) {
|
|
36
|
+
specs.push(...scanDirForSpecsRecursive(fullPath, extension));
|
|
37
|
+
}
|
|
38
|
+
else if (item.name.endsWith(extension)) {
|
|
39
|
+
specs.push(fullPath);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
// Directory not readable
|
|
45
|
+
}
|
|
46
|
+
return specs;
|
|
47
|
+
}
|
|
48
|
+
function resolvePlaywrightSpecs(testsRoot, specDirs) {
|
|
49
|
+
const specs = [];
|
|
50
|
+
for (const dir of specDirs) {
|
|
51
|
+
specs.push(...scanDirForSpecs(testsRoot, dir, '.spec.ts'));
|
|
52
|
+
}
|
|
53
|
+
return specs;
|
|
54
|
+
}
|
|
55
|
+
function resolveCypressSpecs(cypressRoot, specDirs) {
|
|
56
|
+
const specs = [];
|
|
57
|
+
for (const dir of specDirs) {
|
|
58
|
+
// cypressSpecDirs are relative to testsRoot (e.g. ../cypress/tests/integration/channels/search/)
|
|
59
|
+
// Resolve them relative to the cypress root
|
|
60
|
+
const resolvedDir = join(cypressRoot, dir.replace(/^\.\.\/cypress\//, ''));
|
|
61
|
+
if (!existsSync(resolvedDir)) {
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
const found = scanDirForSpecsRecursive(resolvedDir, '.js');
|
|
65
|
+
const tsFound = scanDirForSpecsRecursive(resolvedDir, '.ts');
|
|
66
|
+
specs.push(...found, ...tsFound);
|
|
67
|
+
}
|
|
68
|
+
return specs;
|
|
69
|
+
}
|
|
70
|
+
function computeCoverageStatus(pwSpecs, cySpecs) {
|
|
71
|
+
const hasPw = pwSpecs.length > 0;
|
|
72
|
+
const hasCy = cySpecs.length > 0;
|
|
73
|
+
if (hasPw && hasCy) {
|
|
74
|
+
return 'covered';
|
|
75
|
+
}
|
|
76
|
+
if (hasPw || hasCy) {
|
|
77
|
+
return 'partial';
|
|
78
|
+
}
|
|
79
|
+
return 'uncovered';
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Group file bindings into a deduplicated map of family/feature → changed files.
|
|
83
|
+
*/
|
|
84
|
+
function groupBindings(fileBindings) {
|
|
85
|
+
const groups = new Map();
|
|
86
|
+
for (const fb of fileBindings) {
|
|
87
|
+
for (const binding of fb.bindings) {
|
|
88
|
+
const key = binding.feature || binding.family;
|
|
89
|
+
const existing = groups.get(key);
|
|
90
|
+
if (existing) {
|
|
91
|
+
if (!existing.files.includes(fb.file)) {
|
|
92
|
+
existing.files.push(fb.file);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
else {
|
|
96
|
+
groups.set(key, {
|
|
97
|
+
familyId: binding.family,
|
|
98
|
+
featureId: binding.feature,
|
|
99
|
+
files: [fb.file],
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return groups;
|
|
105
|
+
}
|
|
106
|
+
export function analyzeImpact(changedFiles, options) {
|
|
107
|
+
const { testsRoot, routeFamilies } = options;
|
|
108
|
+
const warnings = [];
|
|
109
|
+
// Load manifest
|
|
110
|
+
const manifest = loadRouteFamilyManifest(testsRoot, routeFamilies);
|
|
111
|
+
if (!manifest) {
|
|
112
|
+
return {
|
|
113
|
+
changedFiles,
|
|
114
|
+
expandedFiles: options.expandedFiles || [],
|
|
115
|
+
impactedFeatures: [],
|
|
116
|
+
unboundFiles: [...changedFiles],
|
|
117
|
+
warnings: ['Route family manifest not found. All files are unbound.'],
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
// Combine original + expanded files
|
|
121
|
+
const allFiles = [...new Set([...changedFiles, ...(options.expandedFiles || [])])];
|
|
122
|
+
// Bind files to families
|
|
123
|
+
const fileBindings = bindFilesToFamilies(allFiles, manifest);
|
|
124
|
+
// Find unbound files
|
|
125
|
+
const unboundFiles = fileBindings
|
|
126
|
+
.filter((fb) => fb.bindings.length === 0)
|
|
127
|
+
.map((fb) => fb.file);
|
|
128
|
+
// Group bindings into features
|
|
129
|
+
const groups = groupBindings(fileBindings.filter((fb) => fb.bindings.length > 0));
|
|
130
|
+
// Determine cypress root
|
|
131
|
+
const cypressRoot = options.cypressRoot || inferCypressRoot(testsRoot);
|
|
132
|
+
// Resolve specs and compute coverage for each feature
|
|
133
|
+
const impactedFeatures = [];
|
|
134
|
+
for (const group of groups.values()) {
|
|
135
|
+
const binding = { family: group.familyId, feature: group.featureId };
|
|
136
|
+
const specDirs = getSpecDirsForBinding(manifest, binding);
|
|
137
|
+
const cypressSpecDirs = getCypressSpecDirsForBinding(manifest, binding);
|
|
138
|
+
const priority = getPriorityForBinding(manifest, binding);
|
|
139
|
+
const userFlows = getUserFlowsForBinding(manifest, binding);
|
|
140
|
+
const playwrightSpecs = resolvePlaywrightSpecs(testsRoot, specDirs);
|
|
141
|
+
const cypressSpecs = cypressRoot ? resolveCypressSpecs(cypressRoot, cypressSpecDirs) : [];
|
|
142
|
+
const coverageStatus = computeCoverageStatus(playwrightSpecs, cypressSpecs);
|
|
143
|
+
impactedFeatures.push({
|
|
144
|
+
familyId: group.familyId,
|
|
145
|
+
featureId: group.featureId,
|
|
146
|
+
priority,
|
|
147
|
+
changedFiles: group.files,
|
|
148
|
+
playwrightSpecs,
|
|
149
|
+
cypressSpecs,
|
|
150
|
+
userFlows,
|
|
151
|
+
coverageStatus,
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
// Sort by priority (P0 first, then P1, then P2)
|
|
155
|
+
const priorityOrder = { P0: 0, P1: 1, P2: 2 };
|
|
156
|
+
impactedFeatures.sort((a, b) => priorityOrder[a.priority] - priorityOrder[b.priority]);
|
|
157
|
+
if (unboundFiles.length > 0 && unboundFiles.length <= 5) {
|
|
158
|
+
warnings.push(`${unboundFiles.length} file(s) not mapped to any route family: ${unboundFiles.join(', ')}`);
|
|
159
|
+
}
|
|
160
|
+
else if (unboundFiles.length > 5) {
|
|
161
|
+
warnings.push(`${unboundFiles.length} file(s) not mapped to any route family`);
|
|
162
|
+
}
|
|
163
|
+
return {
|
|
164
|
+
changedFiles,
|
|
165
|
+
expandedFiles: options.expandedFiles || [],
|
|
166
|
+
impactedFeatures,
|
|
167
|
+
unboundFiles,
|
|
168
|
+
warnings,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
function inferCypressRoot(testsRoot) {
|
|
172
|
+
// testsRoot is typically the Playwright tests directory
|
|
173
|
+
// Cypress tests are at a sibling path: e2e-tests/cypress/tests/integration/channels/
|
|
174
|
+
const candidate = join(testsRoot, '..', 'cypress');
|
|
175
|
+
if (existsSync(candidate)) {
|
|
176
|
+
return candidate;
|
|
177
|
+
}
|
|
178
|
+
return undefined;
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Get gaps: P0/P1 features with 'uncovered' status.
|
|
182
|
+
*/
|
|
183
|
+
export function getGaps(result) {
|
|
184
|
+
return result.impactedFeatures.filter((f) => (f.priority === 'P0' || f.priority === 'P1') && f.coverageStatus === 'uncovered');
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Get partial gaps: P0/P1 features with 'partial' status (advisory).
|
|
188
|
+
*/
|
|
189
|
+
export function getPartialGaps(result) {
|
|
190
|
+
return result.impactedFeatures.filter((f) => (f.priority === 'P0' || f.priority === 'P1') && f.coverageStatus === 'partial');
|
|
191
|
+
}
|