@yasserkhanorg/e2e-agents 0.5.16 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/pipeline.d.ts +1 -1
- package/dist/agent/pipeline.d.ts.map +1 -1
- package/dist/agent/plan.d.ts +2 -13
- package/dist/agent/plan.d.ts.map +1 -1
- package/dist/agent/plan.js +0 -365
- package/dist/agent/types.d.ts +42 -0
- package/dist/agent/types.d.ts.map +1 -0
- package/dist/agent/types.js +4 -0
- package/dist/api.d.ts +14 -14
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +67 -59
- package/dist/cli.js +86 -176
- package/dist/engine/ai_enrichment.d.ts +43 -0
- package/dist/engine/ai_enrichment.d.ts.map +1 -0
- package/dist/engine/ai_enrichment.js +235 -0
- package/dist/engine/diff_loader.d.ts +11 -0
- package/dist/engine/diff_loader.d.ts.map +1 -0
- package/dist/engine/diff_loader.js +74 -0
- package/dist/engine/impact_engine.d.ts +36 -0
- package/dist/engine/impact_engine.d.ts.map +1 -0
- package/dist/engine/impact_engine.js +196 -0
- package/dist/engine/plan_builder.d.ts +10 -0
- package/dist/engine/plan_builder.d.ts.map +1 -0
- package/dist/engine/plan_builder.js +374 -0
- package/dist/esm/agent/plan.js +1 -360
- package/dist/esm/agent/types.js +3 -0
- package/dist/esm/api.js +62 -54
- package/dist/esm/cli.js +87 -177
- package/dist/esm/engine/ai_enrichment.js +232 -0
- package/dist/esm/engine/diff_loader.js +70 -0
- package/dist/esm/engine/impact_engine.js +191 -0
- package/dist/esm/engine/plan_builder.js +368 -0
- package/dist/esm/index.js +6 -3
- package/dist/esm/knowledge/route_families.js +59 -1
- package/dist/index.d.ts +9 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -5
- package/dist/knowledge/route_families.d.ts +19 -0
- package/dist/knowledge/route_families.d.ts.map +1 -1
- package/dist/knowledge/route_families.js +62 -1
- package/package.json +1 -1
- package/dist/agent/ai_flow_analysis.d.ts +0 -13
- package/dist/agent/ai_flow_analysis.d.ts.map +0 -1
- package/dist/agent/ai_flow_analysis.js +0 -334
- package/dist/agent/ai_mapping.d.ts +0 -14
- package/dist/agent/ai_mapping.d.ts.map +0 -1
- package/dist/agent/ai_mapping.js +0 -560
- package/dist/agent/analysis.d.ts +0 -64
- package/dist/agent/analysis.d.ts.map +0 -1
- package/dist/agent/analysis.js +0 -292
- package/dist/agent/blast_radius.d.ts +0 -4
- package/dist/agent/blast_radius.d.ts.map +0 -1
- package/dist/agent/blast_radius.js +0 -37
- package/dist/agent/dependency_graph.d.ts +0 -14
- package/dist/agent/dependency_graph.d.ts.map +0 -1
- package/dist/agent/dependency_graph.js +0 -227
- package/dist/agent/flags.d.ts +0 -23
- package/dist/agent/flags.d.ts.map +0 -1
- package/dist/agent/flags.js +0 -171
- package/dist/agent/flow_catalog.d.ts +0 -25
- package/dist/agent/flow_catalog.d.ts.map +0 -1
- package/dist/agent/flow_catalog.js +0 -115
- package/dist/agent/flow_mapping.d.ts +0 -10
- package/dist/agent/flow_mapping.d.ts.map +0 -1
- package/dist/agent/flow_mapping.js +0 -84
- package/dist/agent/framework.d.ts +0 -13
- package/dist/agent/framework.d.ts.map +0 -1
- package/dist/agent/framework.js +0 -149
- package/dist/agent/gap_suggestions.d.ts +0 -14
- package/dist/agent/gap_suggestions.d.ts.map +0 -1
- package/dist/agent/gap_suggestions.js +0 -101
- package/dist/agent/generator.d.ts +0 -10
- package/dist/agent/generator.d.ts.map +0 -1
- package/dist/agent/generator.js +0 -115
- package/dist/agent/operational_insights.d.ts +0 -41
- package/dist/agent/operational_insights.d.ts.map +0 -1
- package/dist/agent/operational_insights.js +0 -127
- package/dist/agent/report.d.ts +0 -97
- package/dist/agent/report.d.ts.map +0 -1
- package/dist/agent/report.js +0 -159
- package/dist/agent/runner.d.ts +0 -7
- package/dist/agent/runner.d.ts.map +0 -1
- package/dist/agent/runner.js +0 -898
- package/dist/agent/selectors.d.ts +0 -10
- package/dist/agent/selectors.d.ts.map +0 -1
- package/dist/agent/selectors.js +0 -75
- package/dist/agent/subsystem_risk.d.ts +0 -23
- package/dist/agent/subsystem_risk.d.ts.map +0 -1
- package/dist/agent/subsystem_risk.js +0 -207
- package/dist/agent/tests.d.ts +0 -19
- package/dist/agent/tests.d.ts.map +0 -1
- package/dist/agent/tests.js +0 -116
- package/dist/agent/traceability.d.ts +0 -22
- package/dist/agent/traceability.d.ts.map +0 -1
- package/dist/agent/traceability.js +0 -183
- package/dist/esm/agent/ai_flow_analysis.js +0 -331
- package/dist/esm/agent/ai_mapping.js +0 -557
- package/dist/esm/agent/analysis.js +0 -287
- package/dist/esm/agent/blast_radius.js +0 -34
- package/dist/esm/agent/dependency_graph.js +0 -224
- package/dist/esm/agent/flags.js +0 -160
- package/dist/esm/agent/flow_catalog.js +0 -112
- package/dist/esm/agent/flow_mapping.js +0 -81
- package/dist/esm/agent/framework.js +0 -145
- package/dist/esm/agent/gap_suggestions.js +0 -98
- package/dist/esm/agent/generator.js +0 -112
- package/dist/esm/agent/operational_insights.js +0 -124
- package/dist/esm/agent/report.js +0 -156
- package/dist/esm/agent/runner.js +0 -894
- package/dist/esm/agent/selectors.js +0 -71
- package/dist/esm/agent/subsystem_risk.js +0 -204
- package/dist/esm/agent/tests.js +0 -111
- package/dist/esm/agent/traceability.js +0 -180
package/dist/esm/cli.js
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
3
3
|
// See LICENSE.txt for license information.
|
|
4
|
-
import { appendFileSync, existsSync, readFileSync
|
|
4
|
+
import { appendFileSync, existsSync, readFileSync } from 'fs';
|
|
5
5
|
import { dirname, join, resolve } from 'path';
|
|
6
6
|
import { resolveConfig } from './agent/config.js';
|
|
7
7
|
import { AnthropicProvider } from './anthropic_provider.js';
|
|
8
8
|
import { LLMProviderError } from './provider_interface.js';
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
9
|
+
import { analyzeImpact as analyzeImpactV2 } from './engine/impact_engine.js';
|
|
10
|
+
import { writeCiSummary } from './engine/plan_builder.js';
|
|
11
|
+
import { getChangedFiles } from './agent/git.js';
|
|
12
|
+
import { recommendTestsAI, recommendTestsDeterministic } from './api.js';
|
|
12
13
|
import { appendFeedbackAndRecompute } from './agent/feedback.js';
|
|
13
14
|
import { finalizeGeneratedTests } from './agent/handoff.js';
|
|
14
15
|
import { ingestTraceabilityInput } from './agent/traceability_ingest.js';
|
|
@@ -59,13 +60,9 @@ function printUsage() {
|
|
|
59
60
|
console.log([
|
|
60
61
|
'Usage:',
|
|
61
62
|
' e2e-ai-agents impact --path <app-root> [options]',
|
|
62
|
-
' e2e-ai-agents gap --path <app-root> [options]',
|
|
63
63
|
' e2e-ai-agents plan --path <app-root> [options]',
|
|
64
|
-
' e2e-ai-agents generate --path <app-root> [options]',
|
|
65
|
-
' e2e-ai-agents heal --path <app-root> --traceability-report <json> [options]',
|
|
66
64
|
' e2e-ai-agents suggest --path <app-root> [options]',
|
|
67
|
-
' e2e-ai-agents
|
|
68
|
-
' e2e-ai-agents auto-heal-pr --path <app-root> [options]',
|
|
65
|
+
' e2e-ai-agents heal --path <app-root> --traceability-report <json> [options]',
|
|
69
66
|
' e2e-ai-agents finalize-generated-tests --path <app-root> [options]',
|
|
70
67
|
' e2e-ai-agents feedback --path <app-root> --feedback-input <json>',
|
|
71
68
|
' e2e-ai-agents traceability-capture --path <app-root> --traceability-report <json>',
|
|
@@ -144,13 +141,9 @@ function parseArgs(argv) {
|
|
|
144
141
|
}
|
|
145
142
|
const command = argv[0];
|
|
146
143
|
if (command === 'impact'
|
|
147
|
-
|| command === 'gap'
|
|
148
144
|
|| command === 'plan'
|
|
149
|
-
|| command === 'generate'
|
|
150
145
|
|| command === 'heal'
|
|
151
146
|
|| command === 'suggest'
|
|
152
|
-
|| command === 'approve-and-generate'
|
|
153
|
-
|| command === 'auto-heal-pr'
|
|
154
147
|
|| command === 'finalize-generated-tests'
|
|
155
148
|
|| command === 'feedback'
|
|
156
149
|
|| command === 'traceability-capture'
|
|
@@ -500,6 +493,10 @@ function parseArgs(argv) {
|
|
|
500
493
|
i += 1;
|
|
501
494
|
continue;
|
|
502
495
|
}
|
|
496
|
+
if (arg === '--no-ai') {
|
|
497
|
+
parsed.noAi = true;
|
|
498
|
+
continue;
|
|
499
|
+
}
|
|
503
500
|
}
|
|
504
501
|
return parsed;
|
|
505
502
|
}
|
|
@@ -759,117 +756,6 @@ async function main() {
|
|
|
759
756
|
}
|
|
760
757
|
return;
|
|
761
758
|
}
|
|
762
|
-
if (args.command === 'auto-heal-pr') {
|
|
763
|
-
if (!args.path && !autoConfig) {
|
|
764
|
-
// eslint-disable-next-line no-console
|
|
765
|
-
console.error('Error: --path is required for auto-heal-pr command');
|
|
766
|
-
process.exit(1);
|
|
767
|
-
}
|
|
768
|
-
const { config } = resolveConfig(process.cwd(), autoConfig, {
|
|
769
|
-
path: args.path,
|
|
770
|
-
profile: args.profile,
|
|
771
|
-
testsRoot: args.testsRoot,
|
|
772
|
-
mode: 'gap',
|
|
773
|
-
framework: args.framework,
|
|
774
|
-
timeLimitMinutes: args.timeLimitMinutes,
|
|
775
|
-
budget: {
|
|
776
|
-
maxUSD: args.budgetUSD,
|
|
777
|
-
maxTokens: args.budgetTokens,
|
|
778
|
-
},
|
|
779
|
-
testPatterns: args.testPatterns,
|
|
780
|
-
flowPatterns: args.flowPatterns,
|
|
781
|
-
flowExclude: args.flowExclude,
|
|
782
|
-
flowCatalogPath: args.flowCatalogPath,
|
|
783
|
-
specPDF: args.specPDF,
|
|
784
|
-
gitSince: args.gitSince,
|
|
785
|
-
pipeline: {
|
|
786
|
-
enabled: true,
|
|
787
|
-
scenarios: args.pipelineScenarios,
|
|
788
|
-
outputDir: args.pipelineOutput,
|
|
789
|
-
baseUrl: args.pipelineBaseUrl,
|
|
790
|
-
browser: args.pipelineBrowser,
|
|
791
|
-
headless: args.pipelineHeadless,
|
|
792
|
-
project: args.pipelineProject,
|
|
793
|
-
parallel: args.pipelineParallel,
|
|
794
|
-
dryRun: args.pipelineDryRun,
|
|
795
|
-
mcp: args.pipelineMcp,
|
|
796
|
-
mcpAllowFallback: args.pipelineMcpAllowFallback,
|
|
797
|
-
mcpOnly: args.pipelineMcpOnly,
|
|
798
|
-
},
|
|
799
|
-
llmProvider: args.llmProvider,
|
|
800
|
-
});
|
|
801
|
-
if (args.allowFallback) {
|
|
802
|
-
config.impact.allowFallback = true;
|
|
803
|
-
}
|
|
804
|
-
await runGap(config, { apply: true });
|
|
805
|
-
const reportRoot = config.testsRoot || config.path;
|
|
806
|
-
if (args.traceabilityReportPath) {
|
|
807
|
-
const unstableSpecs = extractPlaywrightUnstableSpecs(args.traceabilityReportPath, [reportRoot, config.path]);
|
|
808
|
-
if (unstableSpecs.length > 0) {
|
|
809
|
-
const targetedSummary = runTargetedSpecHeal(reportRoot, unstableSpecs.map((spec) => ({
|
|
810
|
-
specPath: spec.specPath,
|
|
811
|
-
status: spec.status,
|
|
812
|
-
reason: `Playwright report: failingTests=${spec.failingTests}, flakyTests=${spec.flakyTests}`,
|
|
813
|
-
})), {
|
|
814
|
-
...config.pipeline,
|
|
815
|
-
enabled: true,
|
|
816
|
-
heal: true,
|
|
817
|
-
});
|
|
818
|
-
const healedCount = targetedSummary.results.filter((result) => result.healStatus === 'success').length;
|
|
819
|
-
// eslint-disable-next-line no-console
|
|
820
|
-
console.log(`Auto-heal targeted unstable specs: ${unstableSpecs.length} (healed=${healedCount})`);
|
|
821
|
-
if (targetedSummary.warnings.length > 0) {
|
|
822
|
-
// eslint-disable-next-line no-console
|
|
823
|
-
console.log(`Auto-heal warnings: ${targetedSummary.warnings.join(' | ')}`);
|
|
824
|
-
}
|
|
825
|
-
const gapPath = join(reportRoot, '.e2e-ai-agents', 'gap.json');
|
|
826
|
-
if (existsSync(gapPath)) {
|
|
827
|
-
const gap = JSON.parse(readFileSync(gapPath, 'utf-8'));
|
|
828
|
-
const existingResults = Array.isArray(gap.pipeline?.results) ? gap.pipeline?.results : [];
|
|
829
|
-
const existingWarnings = Array.isArray(gap.pipeline?.warnings) ? gap.pipeline?.warnings : [];
|
|
830
|
-
gap.pipeline = {
|
|
831
|
-
runner: gap.pipeline?.runner || targetedSummary.runner,
|
|
832
|
-
results: [...existingResults, ...targetedSummary.results],
|
|
833
|
-
warnings: Array.from(new Set([...(existingWarnings || []), ...targetedSummary.warnings])),
|
|
834
|
-
};
|
|
835
|
-
writeFileSync(gapPath, `${JSON.stringify(gap, null, 2)}\n`, 'utf-8');
|
|
836
|
-
}
|
|
837
|
-
}
|
|
838
|
-
else {
|
|
839
|
-
// eslint-disable-next-line no-console
|
|
840
|
-
console.log('Auto-heal targeted unstable specs: 0');
|
|
841
|
-
}
|
|
842
|
-
}
|
|
843
|
-
const branchSuffix = new Date().toISOString().replace(/[:.]/g, '-');
|
|
844
|
-
const result = finalizeGeneratedTests({
|
|
845
|
-
appPath: config.path,
|
|
846
|
-
testsRoot: reportRoot,
|
|
847
|
-
branch: args.branch || `auto-heal-${branchSuffix}`,
|
|
848
|
-
commitMessage: args.commitMessage || 'test(e2e): auto-heal generated specs',
|
|
849
|
-
createPr: true,
|
|
850
|
-
prTitle: args.prTitle || 'test(e2e): auto-heal generated specs',
|
|
851
|
-
prBody: args.prBody || 'Automated e2e-heal run generated by @yasserkhanorg/e2e-agents.',
|
|
852
|
-
baseBranch: args.prBase || 'master',
|
|
853
|
-
dryRun: args.dryRun,
|
|
854
|
-
});
|
|
855
|
-
// eslint-disable-next-line no-console
|
|
856
|
-
console.log(`Auto-heal repo root: ${result.repoRoot}`);
|
|
857
|
-
// eslint-disable-next-line no-console
|
|
858
|
-
console.log(`Auto-heal branch: ${result.branch}`);
|
|
859
|
-
// eslint-disable-next-line no-console
|
|
860
|
-
console.log(`Auto-heal staged paths: ${result.stagedPaths.join(', ') || 'none'}`);
|
|
861
|
-
// eslint-disable-next-line no-console
|
|
862
|
-
console.log(`Auto-heal commit: ${result.committed ? 'created' : 'skipped'}`);
|
|
863
|
-
if (result.commitSha) {
|
|
864
|
-
// eslint-disable-next-line no-console
|
|
865
|
-
console.log(`Auto-heal commit sha: ${result.commitSha}`);
|
|
866
|
-
}
|
|
867
|
-
if (result.prUrl) {
|
|
868
|
-
// eslint-disable-next-line no-console
|
|
869
|
-
console.log(`Auto-heal PR: ${result.prUrl}`);
|
|
870
|
-
}
|
|
871
|
-
return;
|
|
872
|
-
}
|
|
873
759
|
if (args.command === 'heal') {
|
|
874
760
|
if (!args.path && !autoConfig) {
|
|
875
761
|
// eslint-disable-next-line no-console
|
|
@@ -934,13 +820,11 @@ async function main() {
|
|
|
934
820
|
printUsage();
|
|
935
821
|
process.exit(1);
|
|
936
822
|
}
|
|
937
|
-
const
|
|
938
|
-
const forceAIPipelineFromApproval = args.command === 'approve-and-generate' || args.command === 'generate';
|
|
939
|
-
const { config, configPath } = resolveConfig(process.cwd(), autoConfig, {
|
|
823
|
+
const { config } = resolveConfig(process.cwd(), autoConfig, {
|
|
940
824
|
path: args.path,
|
|
941
825
|
profile: args.profile,
|
|
942
826
|
testsRoot: args.testsRoot,
|
|
943
|
-
mode:
|
|
827
|
+
mode: 'impact',
|
|
944
828
|
framework: args.framework,
|
|
945
829
|
timeLimitMinutes: args.timeLimitMinutes,
|
|
946
830
|
budget: {
|
|
@@ -954,7 +838,7 @@ async function main() {
|
|
|
954
838
|
specPDF: args.specPDF,
|
|
955
839
|
gitSince: args.gitSince,
|
|
956
840
|
llmProvider: args.llmProvider,
|
|
957
|
-
pipeline:
|
|
841
|
+
pipeline: args.pipeline
|
|
958
842
|
? {
|
|
959
843
|
enabled: true,
|
|
960
844
|
scenarios: args.pipelineScenarios,
|
|
@@ -965,9 +849,9 @@ async function main() {
|
|
|
965
849
|
project: args.pipelineProject,
|
|
966
850
|
parallel: args.pipelineParallel,
|
|
967
851
|
dryRun: args.pipelineDryRun,
|
|
968
|
-
mcp: args.pipelineMcp
|
|
852
|
+
mcp: args.pipelineMcp,
|
|
969
853
|
mcpAllowFallback: args.pipelineMcpAllowFallback,
|
|
970
|
-
mcpOnly: args.pipelineMcpOnly
|
|
854
|
+
mcpOnly: args.pipelineMcpOnly,
|
|
971
855
|
mcpCommandTimeoutMs: args.pipelineMcpTimeoutMs,
|
|
972
856
|
mcpRetries: args.pipelineMcpRetries,
|
|
973
857
|
}
|
|
@@ -988,46 +872,81 @@ async function main() {
|
|
|
988
872
|
}
|
|
989
873
|
: undefined,
|
|
990
874
|
});
|
|
991
|
-
if (args.allowFallback) {
|
|
992
|
-
config.impact.allowFallback = true;
|
|
993
|
-
}
|
|
994
875
|
if (args.command === 'impact') {
|
|
995
|
-
|
|
876
|
+
const reportRoot = config.testsRoot || config.path;
|
|
877
|
+
const gitResult = getChangedFiles(config.path, config.git.since, { includeUncommitted: config.git.includeUncommitted });
|
|
878
|
+
const impactResult = analyzeImpactV2(gitResult.files, {
|
|
879
|
+
testsRoot: reportRoot,
|
|
880
|
+
routeFamilies: config.routeFamilies,
|
|
881
|
+
});
|
|
882
|
+
// eslint-disable-next-line no-console
|
|
883
|
+
console.log(`Impact: ${impactResult.changedFiles.length} changed files → ${impactResult.impactedFeatures.length} features impacted`);
|
|
884
|
+
// eslint-disable-next-line no-console
|
|
885
|
+
console.log(`Unbound files: ${impactResult.unboundFiles.length}`);
|
|
886
|
+
for (const f of impactResult.impactedFeatures) {
|
|
887
|
+
const label = f.featureId || f.familyId;
|
|
888
|
+
// eslint-disable-next-line no-console
|
|
889
|
+
console.log(` [${f.priority}] ${label}: ${f.coverageStatus} (PW=${f.playwrightSpecs.length}, Cy=${f.cypressSpecs.length})`);
|
|
890
|
+
}
|
|
891
|
+
if (impactResult.warnings.length > 0) {
|
|
892
|
+
for (const w of impactResult.warnings) {
|
|
893
|
+
// eslint-disable-next-line no-console
|
|
894
|
+
console.warn(` Warning: ${w}`);
|
|
895
|
+
}
|
|
896
|
+
}
|
|
996
897
|
return;
|
|
997
898
|
}
|
|
998
899
|
if (args.command === 'suggest' || args.command === 'plan') {
|
|
999
900
|
const reportRoot = config.testsRoot || config.path;
|
|
1000
|
-
const
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
901
|
+
const apiOptions = {
|
|
902
|
+
cwd: process.cwd(),
|
|
903
|
+
configPath: autoConfig,
|
|
904
|
+
path: args.path,
|
|
905
|
+
profile: args.profile,
|
|
906
|
+
testsRoot: args.testsRoot,
|
|
907
|
+
gitSince: args.gitSince,
|
|
908
|
+
llmProvider: args.llmProvider,
|
|
909
|
+
policy: args.policyMinConfidence !== undefined ||
|
|
910
|
+
args.policySafeMergeConfidence !== undefined ||
|
|
911
|
+
args.policyWarningsThreshold !== undefined ||
|
|
912
|
+
(args.policyRiskyPatterns && args.policyRiskyPatterns.length > 0) ||
|
|
913
|
+
args.policyEnforcementMode !== undefined ||
|
|
914
|
+
(args.policyBlockActions && args.policyBlockActions.length > 0)
|
|
915
|
+
? {
|
|
916
|
+
minConfidenceForTargeted: args.policyMinConfidence,
|
|
917
|
+
safeMergeMinConfidence: args.policySafeMergeConfidence,
|
|
918
|
+
forceFullOnWarningsAtOrAbove: args.policyWarningsThreshold,
|
|
919
|
+
riskyFilePatterns: args.policyRiskyPatterns,
|
|
920
|
+
enforcementMode: args.policyEnforcementMode,
|
|
921
|
+
blockOnActions: args.policyBlockActions,
|
|
922
|
+
}
|
|
923
|
+
: undefined,
|
|
924
|
+
};
|
|
925
|
+
let result;
|
|
926
|
+
if (args.noAi) {
|
|
927
|
+
result = recommendTestsDeterministic(apiOptions);
|
|
928
|
+
}
|
|
929
|
+
else {
|
|
930
|
+
result = await recommendTestsAI(apiOptions);
|
|
931
|
+
if (result.aiEnrichment) {
|
|
932
|
+
const { aiEnrichment } = result;
|
|
1008
933
|
// eslint-disable-next-line no-console
|
|
1009
|
-
console.
|
|
934
|
+
console.log(`AI enrichment: ${aiEnrichment.enrichedFeatures.length} features enriched (${aiEnrichment.tokenUsage.input + aiEnrichment.tokenUsage.output} tokens)`);
|
|
1010
935
|
}
|
|
1011
|
-
else {
|
|
1012
|
-
|
|
936
|
+
else if (!process.env.ANTHROPIC_API_KEY) {
|
|
937
|
+
// eslint-disable-next-line no-console
|
|
938
|
+
console.log('Tip: set ANTHROPIC_API_KEY to enable AI-powered enrichment');
|
|
1013
939
|
}
|
|
1014
940
|
}
|
|
1015
|
-
|
|
1016
|
-
|
|
941
|
+
const { plan, planPath, ciSummaryMarkdown, ciSummaryPath } = result;
|
|
942
|
+
// Write CI summary to an additional path if --ci-comment-path was specified
|
|
943
|
+
if (args.ciCommentPath) {
|
|
944
|
+
writeCiSummary(reportRoot, ciSummaryMarkdown, args.ciCommentPath);
|
|
1017
945
|
}
|
|
1018
|
-
const
|
|
1019
|
-
|
|
1020
|
-
const
|
|
1021
|
-
|
|
1022
|
-
testsRoot: reportRoot,
|
|
1023
|
-
sinceRef: config.git.since,
|
|
1024
|
-
configPath,
|
|
1025
|
-
});
|
|
1026
|
-
const plan = applyOperationalInsights(withActions, reportRoot);
|
|
1027
|
-
const planPath = writePlanReport(reportRoot, plan);
|
|
1028
|
-
const summaryMarkdown = renderCiSummaryMarkdown(plan);
|
|
1029
|
-
const summaryPath = writeCiSummary(reportRoot, summaryMarkdown, args.ciCommentPath);
|
|
1030
|
-
const metrics = appendPlanMetrics(reportRoot, plan);
|
|
946
|
+
const summaryPath = ciSummaryPath;
|
|
947
|
+
// Compute metrics paths (api already wrote metrics; derive paths for GHA output)
|
|
948
|
+
const metricsEventsPath = join(reportRoot, '.e2e-ai-agents/metrics.jsonl');
|
|
949
|
+
const metricsSummaryPath = join(reportRoot, '.e2e-ai-agents/metrics-summary.json');
|
|
1031
950
|
const ghaOutput = args.githubOutputPath || process.env.GITHUB_OUTPUT;
|
|
1032
951
|
if (ghaOutput) {
|
|
1033
952
|
appendFileSync(ghaOutput, `run_set=${plan.runSet}\n`);
|
|
@@ -1039,8 +958,8 @@ async function main() {
|
|
|
1039
958
|
appendFileSync(ghaOutput, `required_new_tests_count=${plan.requiredNewTests.length}\n`);
|
|
1040
959
|
appendFileSync(ghaOutput, `plan_path=${planPath}\n`);
|
|
1041
960
|
appendFileSync(ghaOutput, `summary_path=${summaryPath}\n`);
|
|
1042
|
-
appendFileSync(ghaOutput, `metrics_events_path=${
|
|
1043
|
-
appendFileSync(ghaOutput, `metrics_summary_path=${
|
|
961
|
+
appendFileSync(ghaOutput, `metrics_events_path=${metricsEventsPath}\n`);
|
|
962
|
+
appendFileSync(ghaOutput, `metrics_summary_path=${metricsSummaryPath}\n`);
|
|
1044
963
|
}
|
|
1045
964
|
// eslint-disable-next-line no-console
|
|
1046
965
|
console.log(`Suggested run set: ${plan.runSet} (confidence ${plan.confidence})`);
|
|
@@ -1053,26 +972,17 @@ async function main() {
|
|
|
1053
972
|
// eslint-disable-next-line no-console
|
|
1054
973
|
console.log(`CI summary: ${summaryPath}`);
|
|
1055
974
|
// eslint-disable-next-line no-console
|
|
1056
|
-
console.log(`Plan metrics: ${
|
|
1057
|
-
if (plan.nextActions) {
|
|
1058
|
-
// eslint-disable-next-line no-console
|
|
1059
|
-
console.log(`Next action (run existing): ${plan.nextActions.runRecommendedTests || plan.nextActions.runSmokeSuite}`);
|
|
1060
|
-
// eslint-disable-next-line no-console
|
|
1061
|
-
console.log(`Next action (approve + generate): ${plan.nextActions.approveAndGenerate || plan.nextActions.generateMissingTests}`);
|
|
1062
|
-
// eslint-disable-next-line no-console
|
|
1063
|
-
console.log(`Next action (heal): ${plan.nextActions.healGeneratedTests}`);
|
|
1064
|
-
}
|
|
975
|
+
console.log(`Plan metrics: ${metricsSummaryPath}`);
|
|
1065
976
|
const failOnLegacyFlag = args.failOnMustAddTests && plan.decision.action === 'must-add-tests';
|
|
1066
977
|
if (failOnLegacyFlag || plan.enforcement.shouldFail) {
|
|
1067
978
|
process.exit(2);
|
|
1068
979
|
}
|
|
1069
980
|
return;
|
|
1070
981
|
}
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
await runGap(config, { apply: args.apply });
|
|
982
|
+
// eslint-disable-next-line no-console
|
|
983
|
+
console.error(`Unknown command: ${args.command}`);
|
|
984
|
+
printUsage();
|
|
985
|
+
process.exit(1);
|
|
1076
986
|
}
|
|
1077
987
|
async function runLlmHealth() {
|
|
1078
988
|
if (!process.env.ANTHROPIC_API_KEY) {
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
import { formatDiffsForPrompt } from './diff_loader.js';
|
|
4
|
+
const MAX_SPEC_LIST = 50;
|
|
5
|
+
function normalizePriority(value) {
|
|
6
|
+
if (value === 'P0' || value === 'P1' || value === 'P2') {
|
|
7
|
+
return value;
|
|
8
|
+
}
|
|
9
|
+
return 'P2';
|
|
10
|
+
}
|
|
11
|
+
function buildPrompt(options) {
|
|
12
|
+
const { deterministicImpact, diffs, specList, manifestSummary } = options;
|
|
13
|
+
const { changedFiles, impactedFeatures, unboundFiles } = deterministicImpact;
|
|
14
|
+
const lines = [];
|
|
15
|
+
// Optional manifest summary
|
|
16
|
+
if (manifestSummary) {
|
|
17
|
+
lines.push('## Application Overview');
|
|
18
|
+
lines.push(manifestSummary);
|
|
19
|
+
lines.push('');
|
|
20
|
+
}
|
|
21
|
+
// Changed files section
|
|
22
|
+
lines.push(`## Changed Files (${changedFiles.length} total)`);
|
|
23
|
+
for (const f of changedFiles) {
|
|
24
|
+
lines.push(`- ${f}`);
|
|
25
|
+
}
|
|
26
|
+
lines.push('');
|
|
27
|
+
// Diffs section
|
|
28
|
+
lines.push('## Code Diffs');
|
|
29
|
+
lines.push(formatDiffsForPrompt(diffs));
|
|
30
|
+
lines.push('');
|
|
31
|
+
// Deterministic features summary
|
|
32
|
+
lines.push('## Deterministic Impact Analysis');
|
|
33
|
+
lines.push('The following features/flows have been deterministically identified as impacted:');
|
|
34
|
+
lines.push('');
|
|
35
|
+
for (const feature of impactedFeatures) {
|
|
36
|
+
const featureIdPart = feature.featureId ? `featureId=${feature.featureId}` : 'featureId=undefined';
|
|
37
|
+
const specCount = feature.playwrightSpecs.length + feature.cypressSpecs.length;
|
|
38
|
+
const specList2 = [...feature.playwrightSpecs, ...feature.cypressSpecs];
|
|
39
|
+
const specsDisplay = specList2.length > 0 ? specList2.join(', ') : 'none';
|
|
40
|
+
lines.push(`- familyId=${feature.familyId} ${featureIdPart} (${feature.priority}): ${specCount} files, coverage=${feature.coverageStatus}, specs=[${specsDisplay}]`);
|
|
41
|
+
}
|
|
42
|
+
lines.push('');
|
|
43
|
+
// Unbound files
|
|
44
|
+
if (unboundFiles.length > 0) {
|
|
45
|
+
lines.push('## Unbound Files (not mapped to any feature)');
|
|
46
|
+
for (const f of unboundFiles) {
|
|
47
|
+
lines.push(`- ${f}`);
|
|
48
|
+
}
|
|
49
|
+
lines.push('');
|
|
50
|
+
}
|
|
51
|
+
// Spec list (capped at 50)
|
|
52
|
+
if (specList.length > 0) {
|
|
53
|
+
const cappedSpecs = specList.slice(0, MAX_SPEC_LIST);
|
|
54
|
+
lines.push(`## Available Test Specs (showing ${cappedSpecs.length} of ${specList.length})`);
|
|
55
|
+
for (const s of cappedSpecs) {
|
|
56
|
+
lines.push(`- ${s}`);
|
|
57
|
+
}
|
|
58
|
+
lines.push('');
|
|
59
|
+
}
|
|
60
|
+
// Instructions
|
|
61
|
+
lines.push('## Instructions');
|
|
62
|
+
lines.push('Return ONLY valid JSON (no markdown fences, no explanation) in this exact shape:');
|
|
63
|
+
lines.push('');
|
|
64
|
+
lines.push(JSON.stringify({
|
|
65
|
+
impactedFlows: [
|
|
66
|
+
{
|
|
67
|
+
id: '<featureId or familyId from the deterministic list above>',
|
|
68
|
+
name: '<human-readable flow name>',
|
|
69
|
+
priority: 'P0|P1|P2',
|
|
70
|
+
reasons: ['<specific reason why this flow is impacted by the changes>'],
|
|
71
|
+
coveredBy: ['<spec file paths that cover this flow>'],
|
|
72
|
+
missingScenarios: ['<specific test scenarios that are missing or should be added>'],
|
|
73
|
+
},
|
|
74
|
+
],
|
|
75
|
+
unboundFileAnalysis: [
|
|
76
|
+
{
|
|
77
|
+
file: '<path to unbound file>',
|
|
78
|
+
likelyFeature: '<best guess at which feature/family this affects>',
|
|
79
|
+
reason: '<why you think this file belongs to that feature>',
|
|
80
|
+
},
|
|
81
|
+
],
|
|
82
|
+
}, null, 2));
|
|
83
|
+
return lines.join('\n');
|
|
84
|
+
}
|
|
85
|
+
function extractJSON(text) {
|
|
86
|
+
// Try markdown fenced block first
|
|
87
|
+
const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/i);
|
|
88
|
+
const candidates = fenced ? [fenced[1], text] : [text];
|
|
89
|
+
for (const candidate of candidates) {
|
|
90
|
+
const start = candidate.indexOf('{');
|
|
91
|
+
const end = candidate.lastIndexOf('}');
|
|
92
|
+
if (start >= 0 && end > start) {
|
|
93
|
+
return candidate.slice(start, end + 1).trim();
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
// Fallback: return trimmed text
|
|
97
|
+
return text.trim();
|
|
98
|
+
}
|
|
99
|
+
function toEnrichedFeature(det, aiFlow) {
|
|
100
|
+
return {
|
|
101
|
+
familyId: det.familyId,
|
|
102
|
+
featureId: det.featureId,
|
|
103
|
+
priority: normalizePriority(det.priority),
|
|
104
|
+
changedFiles: det.changedFiles,
|
|
105
|
+
coverageStatus: det.coverageStatus,
|
|
106
|
+
playwrightSpecs: det.playwrightSpecs,
|
|
107
|
+
cypressSpecs: det.cypressSpecs,
|
|
108
|
+
userFlows: det.userFlows,
|
|
109
|
+
aiReasons: aiFlow?.reasons ?? [],
|
|
110
|
+
aiMissingScenarios: aiFlow?.missingScenarios ?? [],
|
|
111
|
+
aiCoveredBy: aiFlow?.coveredBy ?? [],
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Enriches a deterministic impact result with AI-generated reasons,
|
|
116
|
+
* missing test scenarios, and coverage insights.
|
|
117
|
+
*/
|
|
118
|
+
export async function enrichImpactWithAI(options) {
|
|
119
|
+
const { deterministicImpact, provider } = options;
|
|
120
|
+
const warnings = [];
|
|
121
|
+
let tokenUsage = { input: 0, output: 0 };
|
|
122
|
+
const prompt = buildPrompt(options);
|
|
123
|
+
let aiResponse = null;
|
|
124
|
+
let unboundFileInsights = [];
|
|
125
|
+
try {
|
|
126
|
+
const response = await provider.generateText(prompt, {
|
|
127
|
+
maxTokens: 4000,
|
|
128
|
+
temperature: 0,
|
|
129
|
+
timeout: 45000,
|
|
130
|
+
systemPrompt: 'You are an expert E2E test analyst. Return only valid JSON.',
|
|
131
|
+
});
|
|
132
|
+
tokenUsage = {
|
|
133
|
+
input: response.usage?.inputTokens ?? 0,
|
|
134
|
+
output: response.usage?.outputTokens ?? 0,
|
|
135
|
+
};
|
|
136
|
+
const rawJSON = extractJSON(response.text);
|
|
137
|
+
try {
|
|
138
|
+
const parsed = JSON.parse(rawJSON);
|
|
139
|
+
// Validate that impactedFlows is an array
|
|
140
|
+
if (!Array.isArray(parsed.impactedFlows)) {
|
|
141
|
+
warnings.push('AI response parsed but impactedFlows is not an array; returning empty enrichedFeatures');
|
|
142
|
+
return {
|
|
143
|
+
enrichedFeatures: [],
|
|
144
|
+
unboundFileInsights: [],
|
|
145
|
+
warnings,
|
|
146
|
+
providerName: provider.name,
|
|
147
|
+
tokenUsage,
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
aiResponse = parsed;
|
|
151
|
+
unboundFileInsights = (parsed.unboundFileAnalysis ?? []).map((item) => ({
|
|
152
|
+
file: item.file,
|
|
153
|
+
likelyFeature: item.likelyFeature,
|
|
154
|
+
reason: item.reason,
|
|
155
|
+
}));
|
|
156
|
+
}
|
|
157
|
+
catch (parseErr) {
|
|
158
|
+
warnings.push(`Failed to parse AI response as JSON: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`);
|
|
159
|
+
return {
|
|
160
|
+
enrichedFeatures: [],
|
|
161
|
+
unboundFileInsights: [],
|
|
162
|
+
warnings,
|
|
163
|
+
providerName: provider.name,
|
|
164
|
+
tokenUsage,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
catch (err) {
|
|
169
|
+
warnings.push(`AI provider error: ${err instanceof Error ? err.message : String(err)}`);
|
|
170
|
+
return {
|
|
171
|
+
enrichedFeatures: [],
|
|
172
|
+
unboundFileInsights: [],
|
|
173
|
+
warnings,
|
|
174
|
+
providerName: provider.name,
|
|
175
|
+
tokenUsage,
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
// Build a map of AI flows by id (featureId or familyId)
|
|
179
|
+
const aiFlowMap = new Map();
|
|
180
|
+
if (aiResponse?.impactedFlows) {
|
|
181
|
+
for (const flow of aiResponse.impactedFlows) {
|
|
182
|
+
aiFlowMap.set(flow.id, flow);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
// Build a set of all deterministic ids for unmatched-flow detection
|
|
186
|
+
const deterministicIds = new Set();
|
|
187
|
+
for (const det of deterministicImpact.impactedFeatures) {
|
|
188
|
+
if (det.featureId) {
|
|
189
|
+
deterministicIds.add(det.featureId);
|
|
190
|
+
}
|
|
191
|
+
deterministicIds.add(det.familyId);
|
|
192
|
+
}
|
|
193
|
+
// Warn on AI flows that don't match any deterministic feature
|
|
194
|
+
for (const flow of aiFlowMap.values()) {
|
|
195
|
+
if (!deterministicIds.has(flow.id)) {
|
|
196
|
+
warnings.push(`AI returned flow '${flow.id}' with no matching deterministic feature (using as-is)`);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
// Merge deterministic features with AI data
|
|
200
|
+
const enrichedFeatures = deterministicImpact.impactedFeatures.map((det) => {
|
|
201
|
+
// Match by featureId first, then by familyId
|
|
202
|
+
const aiFlow = det.featureId
|
|
203
|
+
? (aiFlowMap.get(det.featureId) ?? aiFlowMap.get(det.familyId))
|
|
204
|
+
: aiFlowMap.get(det.familyId);
|
|
205
|
+
return toEnrichedFeature(det, aiFlow);
|
|
206
|
+
});
|
|
207
|
+
// Include AI flows that had no deterministic match (as-is, with empty deterministic fields)
|
|
208
|
+
for (const flow of aiFlowMap.values()) {
|
|
209
|
+
if (!deterministicIds.has(flow.id)) {
|
|
210
|
+
enrichedFeatures.push({
|
|
211
|
+
familyId: flow.id,
|
|
212
|
+
featureId: undefined,
|
|
213
|
+
priority: normalizePriority(flow.priority),
|
|
214
|
+
changedFiles: [],
|
|
215
|
+
coverageStatus: 'uncovered',
|
|
216
|
+
playwrightSpecs: [],
|
|
217
|
+
cypressSpecs: [],
|
|
218
|
+
userFlows: [],
|
|
219
|
+
aiReasons: flow.reasons ?? [],
|
|
220
|
+
aiMissingScenarios: flow.missingScenarios ?? [],
|
|
221
|
+
aiCoveredBy: flow.coveredBy ?? [],
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
return {
|
|
226
|
+
enrichedFeatures,
|
|
227
|
+
unboundFileInsights,
|
|
228
|
+
warnings,
|
|
229
|
+
providerName: provider.name,
|
|
230
|
+
tokenUsage,
|
|
231
|
+
};
|
|
232
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
import { spawnSync } from 'child_process';
|
|
4
|
+
const MAX_DIFF_CHARS = 8000;
|
|
5
|
+
const MAX_TOTAL_CHARS = 60000;
|
|
6
|
+
const TRUNCATION_NOTICE = '\n... (diff truncated)';
|
|
7
|
+
function runGitRaw(args, cwd) {
|
|
8
|
+
const result = spawnSync('git', args, {
|
|
9
|
+
cwd,
|
|
10
|
+
encoding: 'utf-8',
|
|
11
|
+
timeout: 30000,
|
|
12
|
+
});
|
|
13
|
+
if (result.error || result.status !== 0) {
|
|
14
|
+
return null;
|
|
15
|
+
}
|
|
16
|
+
return result.stdout;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Loads git diffs for the given changed files relative to the given since ref.
|
|
20
|
+
* Uses `git merge-base` to find the accurate base ref first.
|
|
21
|
+
* Individual diffs are truncated at 8000 chars and total output is capped at 60000 chars.
|
|
22
|
+
*/
|
|
23
|
+
export function loadDiffs(appRoot, since, changedFiles) {
|
|
24
|
+
const result = new Map();
|
|
25
|
+
if (changedFiles.length === 0) {
|
|
26
|
+
return result;
|
|
27
|
+
}
|
|
28
|
+
// Try to get accurate merge base
|
|
29
|
+
let baseRef = since;
|
|
30
|
+
const mergeBaseOutput = runGitRaw(['merge-base', since, 'HEAD'], appRoot);
|
|
31
|
+
if (mergeBaseOutput) {
|
|
32
|
+
const candidate = mergeBaseOutput
|
|
33
|
+
.split('\n')
|
|
34
|
+
.map((line) => line.trim())
|
|
35
|
+
.find(Boolean);
|
|
36
|
+
if (candidate) {
|
|
37
|
+
baseRef = candidate;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
let totalChars = 0;
|
|
41
|
+
for (const file of changedFiles) {
|
|
42
|
+
if (totalChars >= MAX_TOTAL_CHARS) {
|
|
43
|
+
break;
|
|
44
|
+
}
|
|
45
|
+
const diffOutput = runGitRaw(['diff', `${baseRef}..HEAD`, '--', file], appRoot);
|
|
46
|
+
if (diffOutput === null) {
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
let diff = diffOutput;
|
|
50
|
+
if (diff.length > MAX_DIFF_CHARS) {
|
|
51
|
+
diff = diff.slice(0, MAX_DIFF_CHARS) + TRUNCATION_NOTICE;
|
|
52
|
+
}
|
|
53
|
+
result.set(file, diff);
|
|
54
|
+
totalChars += diff.length;
|
|
55
|
+
}
|
|
56
|
+
return result;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Formats a diffs map into a human-readable string suitable for an AI prompt.
|
|
60
|
+
*/
|
|
61
|
+
export function formatDiffsForPrompt(diffs) {
|
|
62
|
+
if (diffs.size === 0) {
|
|
63
|
+
return 'No diffs available.';
|
|
64
|
+
}
|
|
65
|
+
const sections = [];
|
|
66
|
+
for (const [file, diff] of diffs) {
|
|
67
|
+
sections.push(`--- ${file} ---\n${diff}`);
|
|
68
|
+
}
|
|
69
|
+
return sections.join('\n\n');
|
|
70
|
+
}
|