@yasserkhanorg/e2e-agents 0.5.16 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/dist/agent/pipeline.d.ts +1 -1
  2. package/dist/agent/pipeline.d.ts.map +1 -1
  3. package/dist/agent/plan.d.ts +2 -13
  4. package/dist/agent/plan.d.ts.map +1 -1
  5. package/dist/agent/plan.js +0 -365
  6. package/dist/agent/types.d.ts +42 -0
  7. package/dist/agent/types.d.ts.map +1 -0
  8. package/dist/agent/types.js +4 -0
  9. package/dist/api.d.ts +14 -14
  10. package/dist/api.d.ts.map +1 -1
  11. package/dist/api.js +67 -59
  12. package/dist/cli.js +86 -176
  13. package/dist/engine/ai_enrichment.d.ts +43 -0
  14. package/dist/engine/ai_enrichment.d.ts.map +1 -0
  15. package/dist/engine/ai_enrichment.js +235 -0
  16. package/dist/engine/diff_loader.d.ts +11 -0
  17. package/dist/engine/diff_loader.d.ts.map +1 -0
  18. package/dist/engine/diff_loader.js +74 -0
  19. package/dist/engine/impact_engine.d.ts +36 -0
  20. package/dist/engine/impact_engine.d.ts.map +1 -0
  21. package/dist/engine/impact_engine.js +196 -0
  22. package/dist/engine/plan_builder.d.ts +10 -0
  23. package/dist/engine/plan_builder.d.ts.map +1 -0
  24. package/dist/engine/plan_builder.js +374 -0
  25. package/dist/esm/agent/plan.js +1 -360
  26. package/dist/esm/agent/types.js +3 -0
  27. package/dist/esm/api.js +62 -54
  28. package/dist/esm/cli.js +87 -177
  29. package/dist/esm/engine/ai_enrichment.js +232 -0
  30. package/dist/esm/engine/diff_loader.js +70 -0
  31. package/dist/esm/engine/impact_engine.js +191 -0
  32. package/dist/esm/engine/plan_builder.js +368 -0
  33. package/dist/esm/index.js +6 -3
  34. package/dist/esm/knowledge/route_families.js +59 -1
  35. package/dist/index.d.ts +9 -4
  36. package/dist/index.d.ts.map +1 -1
  37. package/dist/index.js +14 -5
  38. package/dist/knowledge/route_families.d.ts +19 -0
  39. package/dist/knowledge/route_families.d.ts.map +1 -1
  40. package/dist/knowledge/route_families.js +62 -1
  41. package/package.json +1 -1
  42. package/dist/agent/ai_flow_analysis.d.ts +0 -13
  43. package/dist/agent/ai_flow_analysis.d.ts.map +0 -1
  44. package/dist/agent/ai_flow_analysis.js +0 -334
  45. package/dist/agent/ai_mapping.d.ts +0 -14
  46. package/dist/agent/ai_mapping.d.ts.map +0 -1
  47. package/dist/agent/ai_mapping.js +0 -560
  48. package/dist/agent/analysis.d.ts +0 -64
  49. package/dist/agent/analysis.d.ts.map +0 -1
  50. package/dist/agent/analysis.js +0 -292
  51. package/dist/agent/blast_radius.d.ts +0 -4
  52. package/dist/agent/blast_radius.d.ts.map +0 -1
  53. package/dist/agent/blast_radius.js +0 -37
  54. package/dist/agent/dependency_graph.d.ts +0 -14
  55. package/dist/agent/dependency_graph.d.ts.map +0 -1
  56. package/dist/agent/dependency_graph.js +0 -227
  57. package/dist/agent/flags.d.ts +0 -23
  58. package/dist/agent/flags.d.ts.map +0 -1
  59. package/dist/agent/flags.js +0 -171
  60. package/dist/agent/flow_catalog.d.ts +0 -25
  61. package/dist/agent/flow_catalog.d.ts.map +0 -1
  62. package/dist/agent/flow_catalog.js +0 -115
  63. package/dist/agent/flow_mapping.d.ts +0 -10
  64. package/dist/agent/flow_mapping.d.ts.map +0 -1
  65. package/dist/agent/flow_mapping.js +0 -84
  66. package/dist/agent/framework.d.ts +0 -13
  67. package/dist/agent/framework.d.ts.map +0 -1
  68. package/dist/agent/framework.js +0 -149
  69. package/dist/agent/gap_suggestions.d.ts +0 -14
  70. package/dist/agent/gap_suggestions.d.ts.map +0 -1
  71. package/dist/agent/gap_suggestions.js +0 -101
  72. package/dist/agent/generator.d.ts +0 -10
  73. package/dist/agent/generator.d.ts.map +0 -1
  74. package/dist/agent/generator.js +0 -115
  75. package/dist/agent/operational_insights.d.ts +0 -41
  76. package/dist/agent/operational_insights.d.ts.map +0 -1
  77. package/dist/agent/operational_insights.js +0 -127
  78. package/dist/agent/report.d.ts +0 -97
  79. package/dist/agent/report.d.ts.map +0 -1
  80. package/dist/agent/report.js +0 -159
  81. package/dist/agent/runner.d.ts +0 -7
  82. package/dist/agent/runner.d.ts.map +0 -1
  83. package/dist/agent/runner.js +0 -898
  84. package/dist/agent/selectors.d.ts +0 -10
  85. package/dist/agent/selectors.d.ts.map +0 -1
  86. package/dist/agent/selectors.js +0 -75
  87. package/dist/agent/subsystem_risk.d.ts +0 -23
  88. package/dist/agent/subsystem_risk.d.ts.map +0 -1
  89. package/dist/agent/subsystem_risk.js +0 -207
  90. package/dist/agent/tests.d.ts +0 -19
  91. package/dist/agent/tests.d.ts.map +0 -1
  92. package/dist/agent/tests.js +0 -116
  93. package/dist/agent/traceability.d.ts +0 -22
  94. package/dist/agent/traceability.d.ts.map +0 -1
  95. package/dist/agent/traceability.js +0 -183
  96. package/dist/esm/agent/ai_flow_analysis.js +0 -331
  97. package/dist/esm/agent/ai_mapping.js +0 -557
  98. package/dist/esm/agent/analysis.js +0 -287
  99. package/dist/esm/agent/blast_radius.js +0 -34
  100. package/dist/esm/agent/dependency_graph.js +0 -224
  101. package/dist/esm/agent/flags.js +0 -160
  102. package/dist/esm/agent/flow_catalog.js +0 -112
  103. package/dist/esm/agent/flow_mapping.js +0 -81
  104. package/dist/esm/agent/framework.js +0 -145
  105. package/dist/esm/agent/gap_suggestions.js +0 -98
  106. package/dist/esm/agent/generator.js +0 -112
  107. package/dist/esm/agent/operational_insights.js +0 -124
  108. package/dist/esm/agent/report.js +0 -156
  109. package/dist/esm/agent/runner.js +0 -894
  110. package/dist/esm/agent/selectors.js +0 -71
  111. package/dist/esm/agent/subsystem_risk.js +0 -204
  112. package/dist/esm/agent/tests.js +0 -111
  113. package/dist/esm/agent/traceability.js +0 -180
package/dist/esm/cli.js CHANGED
@@ -1,14 +1,15 @@
1
1
  #!/usr/bin/env node
2
2
  // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
3
3
  // See LICENSE.txt for license information.
4
- import { appendFileSync, existsSync, readFileSync, writeFileSync } from 'fs';
4
+ import { appendFileSync, existsSync, readFileSync } from 'fs';
5
5
  import { dirname, join, resolve } from 'path';
6
6
  import { resolveConfig } from './agent/config.js';
7
7
  import { AnthropicProvider } from './anthropic_provider.js';
8
8
  import { LLMProviderError } from './provider_interface.js';
9
- import { runGap, runImpact } from './agent/runner.js';
10
- import { appendPlanMetrics, attachDeveloperActions, buildPlanFromImpactReport, renderCiSummaryMarkdown, writeCiSummary, writePlanReport, } from './agent/plan.js';
11
- import { applyOperationalInsights } from './agent/operational_insights.js';
9
+ import { analyzeImpact as analyzeImpactV2 } from './engine/impact_engine.js';
10
+ import { writeCiSummary } from './engine/plan_builder.js';
11
+ import { getChangedFiles } from './agent/git.js';
12
+ import { recommendTestsAI, recommendTestsDeterministic } from './api.js';
12
13
  import { appendFeedbackAndRecompute } from './agent/feedback.js';
13
14
  import { finalizeGeneratedTests } from './agent/handoff.js';
14
15
  import { ingestTraceabilityInput } from './agent/traceability_ingest.js';
@@ -59,13 +60,9 @@ function printUsage() {
59
60
  console.log([
60
61
  'Usage:',
61
62
  ' e2e-ai-agents impact --path <app-root> [options]',
62
- ' e2e-ai-agents gap --path <app-root> [options]',
63
63
  ' e2e-ai-agents plan --path <app-root> [options]',
64
- ' e2e-ai-agents generate --path <app-root> [options]',
65
- ' e2e-ai-agents heal --path <app-root> --traceability-report <json> [options]',
66
64
  ' e2e-ai-agents suggest --path <app-root> [options]',
67
- ' e2e-ai-agents approve-and-generate --path <app-root> [options]',
68
- ' e2e-ai-agents auto-heal-pr --path <app-root> [options]',
65
+ ' e2e-ai-agents heal --path <app-root> --traceability-report <json> [options]',
69
66
  ' e2e-ai-agents finalize-generated-tests --path <app-root> [options]',
70
67
  ' e2e-ai-agents feedback --path <app-root> --feedback-input <json>',
71
68
  ' e2e-ai-agents traceability-capture --path <app-root> --traceability-report <json>',
@@ -144,13 +141,9 @@ function parseArgs(argv) {
144
141
  }
145
142
  const command = argv[0];
146
143
  if (command === 'impact'
147
- || command === 'gap'
148
144
  || command === 'plan'
149
- || command === 'generate'
150
145
  || command === 'heal'
151
146
  || command === 'suggest'
152
- || command === 'approve-and-generate'
153
- || command === 'auto-heal-pr'
154
147
  || command === 'finalize-generated-tests'
155
148
  || command === 'feedback'
156
149
  || command === 'traceability-capture'
@@ -500,6 +493,10 @@ function parseArgs(argv) {
500
493
  i += 1;
501
494
  continue;
502
495
  }
496
+ if (arg === '--no-ai') {
497
+ parsed.noAi = true;
498
+ continue;
499
+ }
503
500
  }
504
501
  return parsed;
505
502
  }
@@ -759,117 +756,6 @@ async function main() {
759
756
  }
760
757
  return;
761
758
  }
762
- if (args.command === 'auto-heal-pr') {
763
- if (!args.path && !autoConfig) {
764
- // eslint-disable-next-line no-console
765
- console.error('Error: --path is required for auto-heal-pr command');
766
- process.exit(1);
767
- }
768
- const { config } = resolveConfig(process.cwd(), autoConfig, {
769
- path: args.path,
770
- profile: args.profile,
771
- testsRoot: args.testsRoot,
772
- mode: 'gap',
773
- framework: args.framework,
774
- timeLimitMinutes: args.timeLimitMinutes,
775
- budget: {
776
- maxUSD: args.budgetUSD,
777
- maxTokens: args.budgetTokens,
778
- },
779
- testPatterns: args.testPatterns,
780
- flowPatterns: args.flowPatterns,
781
- flowExclude: args.flowExclude,
782
- flowCatalogPath: args.flowCatalogPath,
783
- specPDF: args.specPDF,
784
- gitSince: args.gitSince,
785
- pipeline: {
786
- enabled: true,
787
- scenarios: args.pipelineScenarios,
788
- outputDir: args.pipelineOutput,
789
- baseUrl: args.pipelineBaseUrl,
790
- browser: args.pipelineBrowser,
791
- headless: args.pipelineHeadless,
792
- project: args.pipelineProject,
793
- parallel: args.pipelineParallel,
794
- dryRun: args.pipelineDryRun,
795
- mcp: args.pipelineMcp,
796
- mcpAllowFallback: args.pipelineMcpAllowFallback,
797
- mcpOnly: args.pipelineMcpOnly,
798
- },
799
- llmProvider: args.llmProvider,
800
- });
801
- if (args.allowFallback) {
802
- config.impact.allowFallback = true;
803
- }
804
- await runGap(config, { apply: true });
805
- const reportRoot = config.testsRoot || config.path;
806
- if (args.traceabilityReportPath) {
807
- const unstableSpecs = extractPlaywrightUnstableSpecs(args.traceabilityReportPath, [reportRoot, config.path]);
808
- if (unstableSpecs.length > 0) {
809
- const targetedSummary = runTargetedSpecHeal(reportRoot, unstableSpecs.map((spec) => ({
810
- specPath: spec.specPath,
811
- status: spec.status,
812
- reason: `Playwright report: failingTests=${spec.failingTests}, flakyTests=${spec.flakyTests}`,
813
- })), {
814
- ...config.pipeline,
815
- enabled: true,
816
- heal: true,
817
- });
818
- const healedCount = targetedSummary.results.filter((result) => result.healStatus === 'success').length;
819
- // eslint-disable-next-line no-console
820
- console.log(`Auto-heal targeted unstable specs: ${unstableSpecs.length} (healed=${healedCount})`);
821
- if (targetedSummary.warnings.length > 0) {
822
- // eslint-disable-next-line no-console
823
- console.log(`Auto-heal warnings: ${targetedSummary.warnings.join(' | ')}`);
824
- }
825
- const gapPath = join(reportRoot, '.e2e-ai-agents', 'gap.json');
826
- if (existsSync(gapPath)) {
827
- const gap = JSON.parse(readFileSync(gapPath, 'utf-8'));
828
- const existingResults = Array.isArray(gap.pipeline?.results) ? gap.pipeline?.results : [];
829
- const existingWarnings = Array.isArray(gap.pipeline?.warnings) ? gap.pipeline?.warnings : [];
830
- gap.pipeline = {
831
- runner: gap.pipeline?.runner || targetedSummary.runner,
832
- results: [...existingResults, ...targetedSummary.results],
833
- warnings: Array.from(new Set([...(existingWarnings || []), ...targetedSummary.warnings])),
834
- };
835
- writeFileSync(gapPath, `${JSON.stringify(gap, null, 2)}\n`, 'utf-8');
836
- }
837
- }
838
- else {
839
- // eslint-disable-next-line no-console
840
- console.log('Auto-heal targeted unstable specs: 0');
841
- }
842
- }
843
- const branchSuffix = new Date().toISOString().replace(/[:.]/g, '-');
844
- const result = finalizeGeneratedTests({
845
- appPath: config.path,
846
- testsRoot: reportRoot,
847
- branch: args.branch || `auto-heal-${branchSuffix}`,
848
- commitMessage: args.commitMessage || 'test(e2e): auto-heal generated specs',
849
- createPr: true,
850
- prTitle: args.prTitle || 'test(e2e): auto-heal generated specs',
851
- prBody: args.prBody || 'Automated e2e-heal run generated by @yasserkhanorg/e2e-agents.',
852
- baseBranch: args.prBase || 'master',
853
- dryRun: args.dryRun,
854
- });
855
- // eslint-disable-next-line no-console
856
- console.log(`Auto-heal repo root: ${result.repoRoot}`);
857
- // eslint-disable-next-line no-console
858
- console.log(`Auto-heal branch: ${result.branch}`);
859
- // eslint-disable-next-line no-console
860
- console.log(`Auto-heal staged paths: ${result.stagedPaths.join(', ') || 'none'}`);
861
- // eslint-disable-next-line no-console
862
- console.log(`Auto-heal commit: ${result.committed ? 'created' : 'skipped'}`);
863
- if (result.commitSha) {
864
- // eslint-disable-next-line no-console
865
- console.log(`Auto-heal commit sha: ${result.commitSha}`);
866
- }
867
- if (result.prUrl) {
868
- // eslint-disable-next-line no-console
869
- console.log(`Auto-heal PR: ${result.prUrl}`);
870
- }
871
- return;
872
- }
873
759
  if (args.command === 'heal') {
874
760
  if (!args.path && !autoConfig) {
875
761
  // eslint-disable-next-line no-console
@@ -934,13 +820,11 @@ async function main() {
934
820
  printUsage();
935
821
  process.exit(1);
936
822
  }
937
- const forcePipelineFromApproval = args.command === 'approve-and-generate' || args.command === 'generate';
938
- const forceAIPipelineFromApproval = args.command === 'approve-and-generate' || args.command === 'generate';
939
- const { config, configPath } = resolveConfig(process.cwd(), autoConfig, {
823
+ const { config } = resolveConfig(process.cwd(), autoConfig, {
940
824
  path: args.path,
941
825
  profile: args.profile,
942
826
  testsRoot: args.testsRoot,
943
- mode: (args.command === 'gap' || args.command === 'approve-and-generate' || args.command === 'generate') ? 'gap' : 'impact',
827
+ mode: 'impact',
944
828
  framework: args.framework,
945
829
  timeLimitMinutes: args.timeLimitMinutes,
946
830
  budget: {
@@ -954,7 +838,7 @@ async function main() {
954
838
  specPDF: args.specPDF,
955
839
  gitSince: args.gitSince,
956
840
  llmProvider: args.llmProvider,
957
- pipeline: (args.pipeline || forcePipelineFromApproval)
841
+ pipeline: args.pipeline
958
842
  ? {
959
843
  enabled: true,
960
844
  scenarios: args.pipelineScenarios,
@@ -965,9 +849,9 @@ async function main() {
965
849
  project: args.pipelineProject,
966
850
  parallel: args.pipelineParallel,
967
851
  dryRun: args.pipelineDryRun,
968
- mcp: args.pipelineMcp !== undefined ? args.pipelineMcp : forceAIPipelineFromApproval,
852
+ mcp: args.pipelineMcp,
969
853
  mcpAllowFallback: args.pipelineMcpAllowFallback,
970
- mcpOnly: args.pipelineMcpOnly !== undefined ? args.pipelineMcpOnly : forceAIPipelineFromApproval,
854
+ mcpOnly: args.pipelineMcpOnly,
971
855
  mcpCommandTimeoutMs: args.pipelineMcpTimeoutMs,
972
856
  mcpRetries: args.pipelineMcpRetries,
973
857
  }
@@ -988,46 +872,81 @@ async function main() {
988
872
  }
989
873
  : undefined,
990
874
  });
991
- if (args.allowFallback) {
992
- config.impact.allowFallback = true;
993
- }
994
875
  if (args.command === 'impact') {
995
- await runImpact(config, { apply: args.apply });
876
+ const reportRoot = config.testsRoot || config.path;
877
+ const gitResult = getChangedFiles(config.path, config.git.since, { includeUncommitted: config.git.includeUncommitted });
878
+ const impactResult = analyzeImpactV2(gitResult.files, {
879
+ testsRoot: reportRoot,
880
+ routeFamilies: config.routeFamilies,
881
+ });
882
+ // eslint-disable-next-line no-console
883
+ console.log(`Impact: ${impactResult.changedFiles.length} changed files → ${impactResult.impactedFeatures.length} features impacted`);
884
+ // eslint-disable-next-line no-console
885
+ console.log(`Unbound files: ${impactResult.unboundFiles.length}`);
886
+ for (const f of impactResult.impactedFeatures) {
887
+ const label = f.featureId || f.familyId;
888
+ // eslint-disable-next-line no-console
889
+ console.log(` [${f.priority}] ${label}: ${f.coverageStatus} (PW=${f.playwrightSpecs.length}, Cy=${f.cypressSpecs.length})`);
890
+ }
891
+ if (impactResult.warnings.length > 0) {
892
+ for (const w of impactResult.warnings) {
893
+ // eslint-disable-next-line no-console
894
+ console.warn(` Warning: ${w}`);
895
+ }
896
+ }
996
897
  return;
997
898
  }
998
899
  if (args.command === 'suggest' || args.command === 'plan') {
999
900
  const reportRoot = config.testsRoot || config.path;
1000
- const impactPath = join(reportRoot, '.e2e-ai-agents', 'impact.json');
1001
- try {
1002
- await runImpact(config, { apply: args.apply });
1003
- }
1004
- catch (err) {
1005
- // If impact analysis already ran (e.g. a prior CI step wrote impact.json),
1006
- // fall back to that data rather than failing the plan step.
1007
- if (existsSync(impactPath)) {
901
+ const apiOptions = {
902
+ cwd: process.cwd(),
903
+ configPath: autoConfig,
904
+ path: args.path,
905
+ profile: args.profile,
906
+ testsRoot: args.testsRoot,
907
+ gitSince: args.gitSince,
908
+ llmProvider: args.llmProvider,
909
+ policy: args.policyMinConfidence !== undefined ||
910
+ args.policySafeMergeConfidence !== undefined ||
911
+ args.policyWarningsThreshold !== undefined ||
912
+ (args.policyRiskyPatterns && args.policyRiskyPatterns.length > 0) ||
913
+ args.policyEnforcementMode !== undefined ||
914
+ (args.policyBlockActions && args.policyBlockActions.length > 0)
915
+ ? {
916
+ minConfidenceForTargeted: args.policyMinConfidence,
917
+ safeMergeMinConfidence: args.policySafeMergeConfidence,
918
+ forceFullOnWarningsAtOrAbove: args.policyWarningsThreshold,
919
+ riskyFilePatterns: args.policyRiskyPatterns,
920
+ enforcementMode: args.policyEnforcementMode,
921
+ blockOnActions: args.policyBlockActions,
922
+ }
923
+ : undefined,
924
+ };
925
+ let result;
926
+ if (args.noAi) {
927
+ result = recommendTestsDeterministic(apiOptions);
928
+ }
929
+ else {
930
+ result = await recommendTestsAI(apiOptions);
931
+ if (result.aiEnrichment) {
932
+ const { aiEnrichment } = result;
1008
933
  // eslint-disable-next-line no-console
1009
- console.warn(`Impact re-run failed (${err instanceof Error ? err.message : String(err)}); using existing impact.json.`);
934
+ console.log(`AI enrichment: ${aiEnrichment.enrichedFeatures.length} features enriched (${aiEnrichment.tokenUsage.input + aiEnrichment.tokenUsage.output} tokens)`);
1010
935
  }
1011
- else {
1012
- throw err;
936
+ else if (!process.env.ANTHROPIC_API_KEY) {
937
+ // eslint-disable-next-line no-console
938
+ console.log('Tip: set ANTHROPIC_API_KEY to enable AI-powered enrichment');
1013
939
  }
1014
940
  }
1015
- if (!existsSync(impactPath)) {
1016
- throw new Error(`Impact report not found at ${impactPath}`);
941
+ const { plan, planPath, ciSummaryMarkdown, ciSummaryPath } = result;
942
+ // Write CI summary to an additional path if --ci-comment-path was specified
943
+ if (args.ciCommentPath) {
944
+ writeCiSummary(reportRoot, ciSummaryMarkdown, args.ciCommentPath);
1017
945
  }
1018
- const impact = JSON.parse(readFileSync(impactPath, 'utf-8'));
1019
- const basePlan = buildPlanFromImpactReport(impact, config.policy);
1020
- const withActions = attachDeveloperActions(basePlan, {
1021
- appPath: config.path,
1022
- testsRoot: reportRoot,
1023
- sinceRef: config.git.since,
1024
- configPath,
1025
- });
1026
- const plan = applyOperationalInsights(withActions, reportRoot);
1027
- const planPath = writePlanReport(reportRoot, plan);
1028
- const summaryMarkdown = renderCiSummaryMarkdown(plan);
1029
- const summaryPath = writeCiSummary(reportRoot, summaryMarkdown, args.ciCommentPath);
1030
- const metrics = appendPlanMetrics(reportRoot, plan);
946
+ const summaryPath = ciSummaryPath;
947
+ // Compute metrics paths (api already wrote metrics; derive paths for GHA output)
948
+ const metricsEventsPath = join(reportRoot, '.e2e-ai-agents/metrics.jsonl');
949
+ const metricsSummaryPath = join(reportRoot, '.e2e-ai-agents/metrics-summary.json');
1031
950
  const ghaOutput = args.githubOutputPath || process.env.GITHUB_OUTPUT;
1032
951
  if (ghaOutput) {
1033
952
  appendFileSync(ghaOutput, `run_set=${plan.runSet}\n`);
@@ -1039,8 +958,8 @@ async function main() {
1039
958
  appendFileSync(ghaOutput, `required_new_tests_count=${plan.requiredNewTests.length}\n`);
1040
959
  appendFileSync(ghaOutput, `plan_path=${planPath}\n`);
1041
960
  appendFileSync(ghaOutput, `summary_path=${summaryPath}\n`);
1042
- appendFileSync(ghaOutput, `metrics_events_path=${metrics.eventsPath}\n`);
1043
- appendFileSync(ghaOutput, `metrics_summary_path=${metrics.summaryPath}\n`);
961
+ appendFileSync(ghaOutput, `metrics_events_path=${metricsEventsPath}\n`);
962
+ appendFileSync(ghaOutput, `metrics_summary_path=${metricsSummaryPath}\n`);
1044
963
  }
1045
964
  // eslint-disable-next-line no-console
1046
965
  console.log(`Suggested run set: ${plan.runSet} (confidence ${plan.confidence})`);
@@ -1053,26 +972,17 @@ async function main() {
1053
972
  // eslint-disable-next-line no-console
1054
973
  console.log(`CI summary: ${summaryPath}`);
1055
974
  // eslint-disable-next-line no-console
1056
- console.log(`Plan metrics: ${metrics.summaryPath}`);
1057
- if (plan.nextActions) {
1058
- // eslint-disable-next-line no-console
1059
- console.log(`Next action (run existing): ${plan.nextActions.runRecommendedTests || plan.nextActions.runSmokeSuite}`);
1060
- // eslint-disable-next-line no-console
1061
- console.log(`Next action (approve + generate): ${plan.nextActions.approveAndGenerate || plan.nextActions.generateMissingTests}`);
1062
- // eslint-disable-next-line no-console
1063
- console.log(`Next action (heal): ${plan.nextActions.healGeneratedTests}`);
1064
- }
975
+ console.log(`Plan metrics: ${metricsSummaryPath}`);
1065
976
  const failOnLegacyFlag = args.failOnMustAddTests && plan.decision.action === 'must-add-tests';
1066
977
  if (failOnLegacyFlag || plan.enforcement.shouldFail) {
1067
978
  process.exit(2);
1068
979
  }
1069
980
  return;
1070
981
  }
1071
- if (args.command === 'approve-and-generate' || args.command === 'generate') {
1072
- await runGap(config, { apply: args.apply });
1073
- return;
1074
- }
1075
- await runGap(config, { apply: args.apply });
982
+ // eslint-disable-next-line no-console
983
+ console.error(`Unknown command: ${args.command}`);
984
+ printUsage();
985
+ process.exit(1);
1076
986
  }
1077
987
  async function runLlmHealth() {
1078
988
  if (!process.env.ANTHROPIC_API_KEY) {
@@ -0,0 +1,232 @@
1
+ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
2
+ // See LICENSE.txt for license information.
3
+ import { formatDiffsForPrompt } from './diff_loader.js';
4
+ const MAX_SPEC_LIST = 50;
5
+ function normalizePriority(value) {
6
+ if (value === 'P0' || value === 'P1' || value === 'P2') {
7
+ return value;
8
+ }
9
+ return 'P2';
10
+ }
11
+ function buildPrompt(options) {
12
+ const { deterministicImpact, diffs, specList, manifestSummary } = options;
13
+ const { changedFiles, impactedFeatures, unboundFiles } = deterministicImpact;
14
+ const lines = [];
15
+ // Optional manifest summary
16
+ if (manifestSummary) {
17
+ lines.push('## Application Overview');
18
+ lines.push(manifestSummary);
19
+ lines.push('');
20
+ }
21
+ // Changed files section
22
+ lines.push(`## Changed Files (${changedFiles.length} total)`);
23
+ for (const f of changedFiles) {
24
+ lines.push(`- ${f}`);
25
+ }
26
+ lines.push('');
27
+ // Diffs section
28
+ lines.push('## Code Diffs');
29
+ lines.push(formatDiffsForPrompt(diffs));
30
+ lines.push('');
31
+ // Deterministic features summary
32
+ lines.push('## Deterministic Impact Analysis');
33
+ lines.push('The following features/flows have been deterministically identified as impacted:');
34
+ lines.push('');
35
+ for (const feature of impactedFeatures) {
36
+ const featureIdPart = feature.featureId ? `featureId=${feature.featureId}` : 'featureId=undefined';
37
+ const specCount = feature.playwrightSpecs.length + feature.cypressSpecs.length;
38
+ const specList2 = [...feature.playwrightSpecs, ...feature.cypressSpecs];
39
+ const specsDisplay = specList2.length > 0 ? specList2.join(', ') : 'none';
40
+ lines.push(`- familyId=${feature.familyId} ${featureIdPart} (${feature.priority}): ${specCount} files, coverage=${feature.coverageStatus}, specs=[${specsDisplay}]`);
41
+ }
42
+ lines.push('');
43
+ // Unbound files
44
+ if (unboundFiles.length > 0) {
45
+ lines.push('## Unbound Files (not mapped to any feature)');
46
+ for (const f of unboundFiles) {
47
+ lines.push(`- ${f}`);
48
+ }
49
+ lines.push('');
50
+ }
51
+ // Spec list (capped at 50)
52
+ if (specList.length > 0) {
53
+ const cappedSpecs = specList.slice(0, MAX_SPEC_LIST);
54
+ lines.push(`## Available Test Specs (showing ${cappedSpecs.length} of ${specList.length})`);
55
+ for (const s of cappedSpecs) {
56
+ lines.push(`- ${s}`);
57
+ }
58
+ lines.push('');
59
+ }
60
+ // Instructions
61
+ lines.push('## Instructions');
62
+ lines.push('Return ONLY valid JSON (no markdown fences, no explanation) in this exact shape:');
63
+ lines.push('');
64
+ lines.push(JSON.stringify({
65
+ impactedFlows: [
66
+ {
67
+ id: '<featureId or familyId from the deterministic list above>',
68
+ name: '<human-readable flow name>',
69
+ priority: 'P0|P1|P2',
70
+ reasons: ['<specific reason why this flow is impacted by the changes>'],
71
+ coveredBy: ['<spec file paths that cover this flow>'],
72
+ missingScenarios: ['<specific test scenarios that are missing or should be added>'],
73
+ },
74
+ ],
75
+ unboundFileAnalysis: [
76
+ {
77
+ file: '<path to unbound file>',
78
+ likelyFeature: '<best guess at which feature/family this affects>',
79
+ reason: '<why you think this file belongs to that feature>',
80
+ },
81
+ ],
82
+ }, null, 2));
83
+ return lines.join('\n');
84
+ }
85
+ function extractJSON(text) {
86
+ // Try markdown fenced block first
87
+ const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/i);
88
+ const candidates = fenced ? [fenced[1], text] : [text];
89
+ for (const candidate of candidates) {
90
+ const start = candidate.indexOf('{');
91
+ const end = candidate.lastIndexOf('}');
92
+ if (start >= 0 && end > start) {
93
+ return candidate.slice(start, end + 1).trim();
94
+ }
95
+ }
96
+ // Fallback: return trimmed text
97
+ return text.trim();
98
+ }
99
+ function toEnrichedFeature(det, aiFlow) {
100
+ return {
101
+ familyId: det.familyId,
102
+ featureId: det.featureId,
103
+ priority: normalizePriority(det.priority),
104
+ changedFiles: det.changedFiles,
105
+ coverageStatus: det.coverageStatus,
106
+ playwrightSpecs: det.playwrightSpecs,
107
+ cypressSpecs: det.cypressSpecs,
108
+ userFlows: det.userFlows,
109
+ aiReasons: aiFlow?.reasons ?? [],
110
+ aiMissingScenarios: aiFlow?.missingScenarios ?? [],
111
+ aiCoveredBy: aiFlow?.coveredBy ?? [],
112
+ };
113
+ }
114
+ /**
115
+ * Enriches a deterministic impact result with AI-generated reasons,
116
+ * missing test scenarios, and coverage insights.
117
+ */
118
+ export async function enrichImpactWithAI(options) {
119
+ const { deterministicImpact, provider } = options;
120
+ const warnings = [];
121
+ let tokenUsage = { input: 0, output: 0 };
122
+ const prompt = buildPrompt(options);
123
+ let aiResponse = null;
124
+ let unboundFileInsights = [];
125
+ try {
126
+ const response = await provider.generateText(prompt, {
127
+ maxTokens: 4000,
128
+ temperature: 0,
129
+ timeout: 45000,
130
+ systemPrompt: 'You are an expert E2E test analyst. Return only valid JSON.',
131
+ });
132
+ tokenUsage = {
133
+ input: response.usage?.inputTokens ?? 0,
134
+ output: response.usage?.outputTokens ?? 0,
135
+ };
136
+ const rawJSON = extractJSON(response.text);
137
+ try {
138
+ const parsed = JSON.parse(rawJSON);
139
+ // Validate that impactedFlows is an array
140
+ if (!Array.isArray(parsed.impactedFlows)) {
141
+ warnings.push('AI response parsed but impactedFlows is not an array; returning empty enrichedFeatures');
142
+ return {
143
+ enrichedFeatures: [],
144
+ unboundFileInsights: [],
145
+ warnings,
146
+ providerName: provider.name,
147
+ tokenUsage,
148
+ };
149
+ }
150
+ aiResponse = parsed;
151
+ unboundFileInsights = (parsed.unboundFileAnalysis ?? []).map((item) => ({
152
+ file: item.file,
153
+ likelyFeature: item.likelyFeature,
154
+ reason: item.reason,
155
+ }));
156
+ }
157
+ catch (parseErr) {
158
+ warnings.push(`Failed to parse AI response as JSON: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`);
159
+ return {
160
+ enrichedFeatures: [],
161
+ unboundFileInsights: [],
162
+ warnings,
163
+ providerName: provider.name,
164
+ tokenUsage,
165
+ };
166
+ }
167
+ }
168
+ catch (err) {
169
+ warnings.push(`AI provider error: ${err instanceof Error ? err.message : String(err)}`);
170
+ return {
171
+ enrichedFeatures: [],
172
+ unboundFileInsights: [],
173
+ warnings,
174
+ providerName: provider.name,
175
+ tokenUsage,
176
+ };
177
+ }
178
+ // Build a map of AI flows by id (featureId or familyId)
179
+ const aiFlowMap = new Map();
180
+ if (aiResponse?.impactedFlows) {
181
+ for (const flow of aiResponse.impactedFlows) {
182
+ aiFlowMap.set(flow.id, flow);
183
+ }
184
+ }
185
+ // Build a set of all deterministic ids for unmatched-flow detection
186
+ const deterministicIds = new Set();
187
+ for (const det of deterministicImpact.impactedFeatures) {
188
+ if (det.featureId) {
189
+ deterministicIds.add(det.featureId);
190
+ }
191
+ deterministicIds.add(det.familyId);
192
+ }
193
+ // Warn on AI flows that don't match any deterministic feature
194
+ for (const flow of aiFlowMap.values()) {
195
+ if (!deterministicIds.has(flow.id)) {
196
+ warnings.push(`AI returned flow '${flow.id}' with no matching deterministic feature (using as-is)`);
197
+ }
198
+ }
199
+ // Merge deterministic features with AI data
200
+ const enrichedFeatures = deterministicImpact.impactedFeatures.map((det) => {
201
+ // Match by featureId first, then by familyId
202
+ const aiFlow = det.featureId
203
+ ? (aiFlowMap.get(det.featureId) ?? aiFlowMap.get(det.familyId))
204
+ : aiFlowMap.get(det.familyId);
205
+ return toEnrichedFeature(det, aiFlow);
206
+ });
207
+ // Include AI flows that had no deterministic match (as-is, with empty deterministic fields)
208
+ for (const flow of aiFlowMap.values()) {
209
+ if (!deterministicIds.has(flow.id)) {
210
+ enrichedFeatures.push({
211
+ familyId: flow.id,
212
+ featureId: undefined,
213
+ priority: normalizePriority(flow.priority),
214
+ changedFiles: [],
215
+ coverageStatus: 'uncovered',
216
+ playwrightSpecs: [],
217
+ cypressSpecs: [],
218
+ userFlows: [],
219
+ aiReasons: flow.reasons ?? [],
220
+ aiMissingScenarios: flow.missingScenarios ?? [],
221
+ aiCoveredBy: flow.coveredBy ?? [],
222
+ });
223
+ }
224
+ }
225
+ return {
226
+ enrichedFeatures,
227
+ unboundFileInsights,
228
+ warnings,
229
+ providerName: provider.name,
230
+ tokenUsage,
231
+ };
232
+ }
@@ -0,0 +1,70 @@
1
+ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
2
+ // See LICENSE.txt for license information.
3
+ import { spawnSync } from 'child_process';
4
+ const MAX_DIFF_CHARS = 8000;
5
+ const MAX_TOTAL_CHARS = 60000;
6
+ const TRUNCATION_NOTICE = '\n... (diff truncated)';
7
+ function runGitRaw(args, cwd) {
8
+ const result = spawnSync('git', args, {
9
+ cwd,
10
+ encoding: 'utf-8',
11
+ timeout: 30000,
12
+ });
13
+ if (result.error || result.status !== 0) {
14
+ return null;
15
+ }
16
+ return result.stdout;
17
+ }
18
+ /**
19
+ * Loads git diffs for the given changed files relative to the given since ref.
20
+ * Uses `git merge-base` to find the accurate base ref first.
21
+ * Individual diffs are truncated at 8000 chars and total output is capped at 60000 chars.
22
+ */
23
+ export function loadDiffs(appRoot, since, changedFiles) {
24
+ const result = new Map();
25
+ if (changedFiles.length === 0) {
26
+ return result;
27
+ }
28
+ // Try to get accurate merge base
29
+ let baseRef = since;
30
+ const mergeBaseOutput = runGitRaw(['merge-base', since, 'HEAD'], appRoot);
31
+ if (mergeBaseOutput) {
32
+ const candidate = mergeBaseOutput
33
+ .split('\n')
34
+ .map((line) => line.trim())
35
+ .find(Boolean);
36
+ if (candidate) {
37
+ baseRef = candidate;
38
+ }
39
+ }
40
+ let totalChars = 0;
41
+ for (const file of changedFiles) {
42
+ if (totalChars >= MAX_TOTAL_CHARS) {
43
+ break;
44
+ }
45
+ const diffOutput = runGitRaw(['diff', `${baseRef}..HEAD`, '--', file], appRoot);
46
+ if (diffOutput === null) {
47
+ continue;
48
+ }
49
+ let diff = diffOutput;
50
+ if (diff.length > MAX_DIFF_CHARS) {
51
+ diff = diff.slice(0, MAX_DIFF_CHARS) + TRUNCATION_NOTICE;
52
+ }
53
+ result.set(file, diff);
54
+ totalChars += diff.length;
55
+ }
56
+ return result;
57
+ }
58
+ /**
59
+ * Formats a diffs map into a human-readable string suitable for an AI prompt.
60
+ */
61
+ export function formatDiffsForPrompt(diffs) {
62
+ if (diffs.size === 0) {
63
+ return 'No diffs available.';
64
+ }
65
+ const sections = [];
66
+ for (const [file, diff] of diffs) {
67
+ sections.push(`--- ${file} ---\n${diff}`);
68
+ }
69
+ return sections.join('\n\n');
70
+ }