@yasserkhanorg/e2e-agents 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -584
- package/dist/agent/api_catalog.d.ts +11 -0
- package/dist/agent/api_catalog.d.ts.map +1 -0
- package/dist/agent/api_catalog.js +210 -0
- package/dist/agent/llm_agents_flow.d.ts +15 -0
- package/dist/agent/llm_agents_flow.d.ts.map +1 -0
- package/dist/agent/llm_agents_flow.js +434 -0
- package/dist/agent/native_flow.d.ts +6 -0
- package/dist/agent/native_flow.d.ts.map +1 -0
- package/dist/agent/native_flow.js +179 -0
- package/dist/agent/pipeline.d.ts +2 -25
- package/dist/agent/pipeline.d.ts.map +1 -1
- package/dist/agent/pipeline.js +30 -1329
- package/dist/agent/pipeline_types.d.ts +54 -0
- package/dist/agent/pipeline_types.d.ts.map +1 -0
- package/dist/agent/pipeline_types.js +4 -0
- package/dist/agent/pipeline_utils.d.ts +12 -0
- package/dist/agent/pipeline_utils.d.ts.map +1 -0
- package/dist/agent/pipeline_utils.js +156 -0
- package/dist/agent/process_runner.d.ts +10 -0
- package/dist/agent/process_runner.d.ts.map +1 -0
- package/dist/agent/process_runner.js +92 -0
- package/dist/agent/spec_generator.d.ts +5 -0
- package/dist/agent/spec_generator.d.ts.map +1 -0
- package/dist/agent/spec_generator.js +253 -0
- package/dist/agent/validation_runner.d.ts +5 -0
- package/dist/agent/validation_runner.d.ts.map +1 -0
- package/dist/agent/validation_runner.js +77 -0
- package/dist/agentic/playwright_runner.js +1 -1
- package/dist/cli/commands/analyze.d.ts +3 -0
- package/dist/cli/commands/analyze.d.ts.map +1 -0
- package/dist/cli/commands/analyze.js +77 -0
- package/dist/cli/commands/feedback.d.ts +3 -0
- package/dist/cli/commands/feedback.d.ts.map +1 -0
- package/dist/cli/commands/feedback.js +39 -0
- package/dist/cli/commands/finalize.d.ts +3 -0
- package/dist/cli/commands/finalize.d.ts.map +1 -0
- package/dist/cli/commands/finalize.js +41 -0
- package/dist/cli/commands/generate.d.ts +4 -0
- package/dist/cli/commands/generate.d.ts.map +1 -0
- package/dist/cli/commands/generate.js +108 -0
- package/dist/cli/commands/heal.d.ts +3 -0
- package/dist/cli/commands/heal.d.ts.map +1 -0
- package/dist/cli/commands/heal.js +60 -0
- package/dist/cli/commands/impact.d.ts +4 -0
- package/dist/cli/commands/impact.d.ts.map +1 -0
- package/dist/cli/commands/impact.js +26 -0
- package/dist/cli/commands/llm_health.d.ts +2 -0
- package/dist/cli/commands/llm_health.d.ts.map +1 -0
- package/dist/cli/commands/llm_health.js +38 -0
- package/dist/cli/commands/plan.d.ts +4 -0
- package/dist/cli/commands/plan.d.ts.map +1 -0
- package/dist/cli/commands/plan.js +83 -0
- package/dist/cli/commands/traceability.d.ts +4 -0
- package/dist/cli/commands/traceability.d.ts.map +1 -0
- package/dist/cli/commands/traceability.js +77 -0
- package/dist/cli/parse_args.d.ts +6 -0
- package/dist/cli/parse_args.d.ts.map +1 -0
- package/dist/cli/parse_args.js +216 -0
- package/dist/cli/types.d.ts +70 -0
- package/dist/cli/types.d.ts.map +1 -0
- package/dist/cli/types.js +4 -0
- package/dist/cli/usage.d.ts +2 -0
- package/dist/cli/usage.d.ts.map +1 -0
- package/dist/cli/usage.js +86 -0
- package/dist/cli.js +26 -1057
- package/dist/esm/agent/api_catalog.js +199 -0
- package/dist/esm/agent/llm_agents_flow.js +421 -0
- package/dist/esm/agent/native_flow.js +175 -0
- package/dist/esm/agent/pipeline.js +8 -1307
- package/dist/esm/agent/pipeline_types.js +3 -0
- package/dist/esm/agent/pipeline_utils.js +146 -0
- package/dist/esm/agent/process_runner.js +83 -0
- package/dist/esm/agent/spec_generator.js +249 -0
- package/dist/esm/agent/validation_runner.js +73 -0
- package/dist/esm/agentic/playwright_runner.js +1 -1
- package/dist/esm/cli/commands/analyze.js +74 -0
- package/dist/esm/cli/commands/feedback.js +36 -0
- package/dist/esm/cli/commands/finalize.js +38 -0
- package/dist/esm/cli/commands/generate.js +105 -0
- package/dist/esm/cli/commands/heal.js +57 -0
- package/dist/esm/cli/commands/impact.js +23 -0
- package/dist/esm/cli/commands/llm_health.js +35 -0
- package/dist/esm/cli/commands/plan.js +80 -0
- package/dist/esm/cli/commands/traceability.js +73 -0
- package/dist/esm/cli/parse_args.js +210 -0
- package/dist/esm/cli/types.js +3 -0
- package/dist/esm/cli/usage.js +83 -0
- package/dist/esm/cli.js +20 -1051
- package/dist/esm/mcp-server.js +18 -1
- package/dist/mcp-server.d.ts.map +1 -1
- package/dist/mcp-server.js +17 -0
- package/package.json +2 -4
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
import { resolveConfig } from '../../agent/config.js';
|
|
4
|
+
import { finalizeGeneratedTests } from '../../agent/handoff.js';
|
|
5
|
+
export function runFinalizeCommand(args, autoConfig) {
|
|
6
|
+
if (!args.path && !autoConfig) {
|
|
7
|
+
console.error('Error: --path is required for finalize-generated-tests command');
|
|
8
|
+
process.exit(1);
|
|
9
|
+
}
|
|
10
|
+
const { config } = resolveConfig(process.cwd(), autoConfig, {
|
|
11
|
+
path: args.path,
|
|
12
|
+
profile: args.profile,
|
|
13
|
+
testsRoot: args.testsRoot,
|
|
14
|
+
mode: 'gap',
|
|
15
|
+
llmProvider: args.llmProvider,
|
|
16
|
+
});
|
|
17
|
+
const result = finalizeGeneratedTests({
|
|
18
|
+
appPath: config.path,
|
|
19
|
+
testsRoot: config.testsRoot || config.path,
|
|
20
|
+
branch: args.branch,
|
|
21
|
+
commitMessage: args.commitMessage,
|
|
22
|
+
createPr: args.createPr,
|
|
23
|
+
prTitle: args.prTitle,
|
|
24
|
+
prBody: args.prBody,
|
|
25
|
+
baseBranch: args.prBase,
|
|
26
|
+
dryRun: args.dryRun,
|
|
27
|
+
});
|
|
28
|
+
console.log(`Finalize repo root: ${result.repoRoot}`);
|
|
29
|
+
console.log(`Finalize branch: ${result.branch}`);
|
|
30
|
+
console.log(`Finalize staged paths: ${result.stagedPaths.join(', ') || 'none'}`);
|
|
31
|
+
console.log(`Finalize commit: ${result.committed ? 'created' : 'skipped'}`);
|
|
32
|
+
if (result.commitSha) {
|
|
33
|
+
console.log(`Finalize commit sha: ${result.commitSha}`);
|
|
34
|
+
}
|
|
35
|
+
if (result.prUrl) {
|
|
36
|
+
console.log(`Finalize PR: ${result.prUrl}`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
|
|
4
|
+
import { join } from 'path';
|
|
5
|
+
import { LLMProviderFactory } from '../../provider_factory.js';
|
|
6
|
+
import { runAgenticGeneration } from '../../agentic/runner.js';
|
|
7
|
+
import { loadOrBuildApiSurface } from '../../knowledge/api_surface.js';
|
|
8
|
+
export async function runGenerateCommand(args, config) {
|
|
9
|
+
const reportRoot = config.testsRoot || config.path;
|
|
10
|
+
// Load scenarios from --scenarios flag or plan-report.json
|
|
11
|
+
let scenarios = [];
|
|
12
|
+
if (args.generateScenarios) {
|
|
13
|
+
let raw;
|
|
14
|
+
if (existsSync(args.generateScenarios)) {
|
|
15
|
+
raw = JSON.parse(readFileSync(args.generateScenarios, 'utf-8'));
|
|
16
|
+
}
|
|
17
|
+
else {
|
|
18
|
+
raw = JSON.parse(args.generateScenarios);
|
|
19
|
+
}
|
|
20
|
+
if (!Array.isArray(raw)) {
|
|
21
|
+
console.error('--scenarios must be a JSON array of ScenarioInput objects.');
|
|
22
|
+
process.exit(1);
|
|
23
|
+
}
|
|
24
|
+
for (const item of raw) {
|
|
25
|
+
if (!item.id || !item.name || !Array.isArray(item.scenarios) || !item.routeFamily || !item.priority) {
|
|
26
|
+
console.error(`Invalid scenario: each must have id, name, scenarios[], routeFamily, priority.`);
|
|
27
|
+
process.exit(1);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
scenarios = raw;
|
|
31
|
+
}
|
|
32
|
+
else {
|
|
33
|
+
// Try plan.json first (written by plan/suggest command), then plan-report.json (legacy)
|
|
34
|
+
const planJsonPath = join(reportRoot, '.e2e-ai-agents', 'plan.json');
|
|
35
|
+
const planReportPath = join(reportRoot, '.e2e-ai-agents', 'plan-report.json');
|
|
36
|
+
const resolvedPlanPath = existsSync(planJsonPath) ? planJsonPath : existsSync(planReportPath) ? planReportPath : null;
|
|
37
|
+
if (!resolvedPlanPath) {
|
|
38
|
+
console.error('No plan report found. Run `plan` first or pass --scenarios.');
|
|
39
|
+
process.exit(1);
|
|
40
|
+
}
|
|
41
|
+
const planReport = JSON.parse(readFileSync(resolvedPlanPath, 'utf-8'));
|
|
42
|
+
scenarios = (planReport.gapDetails || []).map((gap) => ({
|
|
43
|
+
id: gap.id,
|
|
44
|
+
name: gap.id,
|
|
45
|
+
scenarios: gap.missingScenarios || gap.reasons || ['Verify core user flow'],
|
|
46
|
+
routeFamily: gap.id.split('.')[0] || gap.id,
|
|
47
|
+
priority: 'P1',
|
|
48
|
+
}));
|
|
49
|
+
}
|
|
50
|
+
if (scenarios.length === 0) {
|
|
51
|
+
console.log('No scenarios to generate tests for.');
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
let apiSurface;
|
|
55
|
+
try {
|
|
56
|
+
apiSurface = loadOrBuildApiSurface(reportRoot, config.apiSurface);
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
console.warn('Could not load API surface catalog. Generation will use generic selectors.');
|
|
60
|
+
}
|
|
61
|
+
const provider = await LLMProviderFactory.createFromEnv();
|
|
62
|
+
console.log(`Generating tests for ${scenarios.length} scenario(s)...`);
|
|
63
|
+
const summary = await runAgenticGeneration({
|
|
64
|
+
scenarios,
|
|
65
|
+
config: {
|
|
66
|
+
maxAttempts: args.maxAttempts || 3,
|
|
67
|
+
project: args.pipelineProject || 'chrome',
|
|
68
|
+
baseUrl: args.pipelineBaseUrl,
|
|
69
|
+
testTimeoutMs: 120000,
|
|
70
|
+
testsRoot: reportRoot,
|
|
71
|
+
dryRun: args.dryRun,
|
|
72
|
+
},
|
|
73
|
+
provider,
|
|
74
|
+
apiSurface,
|
|
75
|
+
});
|
|
76
|
+
console.log(`\nAgentic Generation Summary:`);
|
|
77
|
+
console.log(` Generated: ${summary.totalGenerated}`);
|
|
78
|
+
console.log(` Passed: ${summary.totalPassed}`);
|
|
79
|
+
console.log(` Failed: ${summary.totalFailed}`);
|
|
80
|
+
console.log(` Attempts: ${summary.totalAttempts}`);
|
|
81
|
+
console.log(` Duration: ${(summary.durationMs / 1000).toFixed(1)}s`);
|
|
82
|
+
for (const result of summary.results) {
|
|
83
|
+
const icon = result.status === 'passed' ? 'PASS' : result.status === 'skipped' ? 'SKIP' : 'FAIL';
|
|
84
|
+
console.log(` [${icon}] ${result.scenarioSource} (${result.attempts} attempts)`);
|
|
85
|
+
if (result.status === 'passed' || result.status === 'skipped') {
|
|
86
|
+
console.log(` ${result.specPath}`);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
if (summary.warnings.length > 0) {
|
|
90
|
+
console.log(`\nWarnings:`);
|
|
91
|
+
for (const w of summary.warnings) {
|
|
92
|
+
console.warn(` - ${w}`);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
const summaryDir = join(reportRoot, '.e2e-ai-agents');
|
|
96
|
+
if (!existsSync(summaryDir)) {
|
|
97
|
+
mkdirSync(summaryDir, { recursive: true });
|
|
98
|
+
}
|
|
99
|
+
const summaryPath = join(summaryDir, 'agentic-summary.json');
|
|
100
|
+
writeFileSync(summaryPath, JSON.stringify(summary, null, 2), 'utf-8');
|
|
101
|
+
console.log(`\nReport: ${summaryPath}`);
|
|
102
|
+
if (summary.totalFailed > 0) {
|
|
103
|
+
process.exit(1);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
import { resolveConfig } from '../../agent/config.js';
|
|
4
|
+
import { runTargetedSpecHeal } from '../../agent/pipeline.js';
|
|
5
|
+
import { extractPlaywrightUnstableSpecs } from '../../agent/playwright_report.js';
|
|
6
|
+
export function runHealCommand(args, autoConfig) {
|
|
7
|
+
if (!args.path && !autoConfig) {
|
|
8
|
+
console.error('Error: --path is required for heal command');
|
|
9
|
+
process.exit(1);
|
|
10
|
+
}
|
|
11
|
+
if (!args.traceabilityReportPath) {
|
|
12
|
+
console.error('Error: --traceability-report <path> is required for heal command');
|
|
13
|
+
process.exit(1);
|
|
14
|
+
}
|
|
15
|
+
const { config } = resolveConfig(process.cwd(), autoConfig, {
|
|
16
|
+
path: args.path,
|
|
17
|
+
profile: args.profile,
|
|
18
|
+
testsRoot: args.testsRoot,
|
|
19
|
+
mode: 'gap',
|
|
20
|
+
framework: args.framework,
|
|
21
|
+
pipeline: {
|
|
22
|
+
enabled: true,
|
|
23
|
+
scenarios: args.pipelineScenarios,
|
|
24
|
+
outputDir: args.pipelineOutput,
|
|
25
|
+
baseUrl: args.pipelineBaseUrl,
|
|
26
|
+
browser: args.pipelineBrowser,
|
|
27
|
+
headless: args.pipelineHeadless,
|
|
28
|
+
project: args.pipelineProject,
|
|
29
|
+
parallel: args.pipelineParallel,
|
|
30
|
+
dryRun: args.pipelineDryRun,
|
|
31
|
+
mcp: args.pipelineMcp,
|
|
32
|
+
mcpAllowFallback: args.pipelineMcpAllowFallback,
|
|
33
|
+
mcpOnly: args.pipelineMcpOnly,
|
|
34
|
+
},
|
|
35
|
+
llmProvider: args.llmProvider,
|
|
36
|
+
});
|
|
37
|
+
const reportRoot = config.testsRoot || config.path;
|
|
38
|
+
const unstableSpecs = extractPlaywrightUnstableSpecs(args.traceabilityReportPath, [reportRoot, config.path]);
|
|
39
|
+
if (unstableSpecs.length === 0) {
|
|
40
|
+
console.log('Heal targeted unstable specs: 0');
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
const targetedSummary = runTargetedSpecHeal(reportRoot, unstableSpecs.map((spec) => ({
|
|
44
|
+
specPath: spec.specPath,
|
|
45
|
+
status: spec.status,
|
|
46
|
+
reason: `Playwright report: failingTests=${spec.failingTests}, flakyTests=${spec.flakyTests}`,
|
|
47
|
+
})), {
|
|
48
|
+
...config.pipeline,
|
|
49
|
+
enabled: true,
|
|
50
|
+
heal: true,
|
|
51
|
+
});
|
|
52
|
+
const healedCount = targetedSummary.results.filter((result) => result.healStatus === 'success').length;
|
|
53
|
+
console.log(`Heal targeted unstable specs: ${unstableSpecs.length} (healed=${healedCount})`);
|
|
54
|
+
if (targetedSummary.warnings.length > 0) {
|
|
55
|
+
console.log(`Heal warnings: ${targetedSummary.warnings.join(' | ')}`);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
import { getChangedFiles } from '../../agent/git.js';
|
|
4
|
+
import { analyzeImpact as analyzeImpactV2 } from '../../engine/impact_engine.js';
|
|
5
|
+
export function runImpactCommand(args, config) {
|
|
6
|
+
const reportRoot = config.testsRoot || config.path;
|
|
7
|
+
const gitResult = getChangedFiles(config.path, config.git.since, { includeUncommitted: config.git.includeUncommitted });
|
|
8
|
+
const impactResult = analyzeImpactV2(gitResult.files, {
|
|
9
|
+
testsRoot: reportRoot,
|
|
10
|
+
routeFamilies: config.routeFamilies,
|
|
11
|
+
});
|
|
12
|
+
console.log(`Impact: ${impactResult.changedFiles.length} changed files → ${impactResult.impactedFeatures.length} features impacted`);
|
|
13
|
+
console.log(`Unbound files: ${impactResult.unboundFiles.length}`);
|
|
14
|
+
for (const f of impactResult.impactedFeatures) {
|
|
15
|
+
const label = f.featureId || f.familyId;
|
|
16
|
+
console.log(` [${f.priority}] ${label}: ${f.coverageStatus} (PW=${f.playwrightSpecs.length}, Cy=${f.cypressSpecs.length})`);
|
|
17
|
+
}
|
|
18
|
+
if (impactResult.warnings.length > 0) {
|
|
19
|
+
for (const w of impactResult.warnings) {
|
|
20
|
+
console.warn(` Warning: ${w}`);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
import { AnthropicProvider } from '../../anthropic_provider.js';
|
|
4
|
+
import { LLMProviderError } from '../../provider_interface.js';
|
|
5
|
+
export async function runLlmHealth() {
|
|
6
|
+
if (!process.env.ANTHROPIC_API_KEY) {
|
|
7
|
+
console.error('ANTHROPIC_API_KEY is required for llm-health.');
|
|
8
|
+
process.exit(1);
|
|
9
|
+
}
|
|
10
|
+
const model = process.env.ANTHROPIC_MODEL || 'claude-sonnet-4-5-20250929';
|
|
11
|
+
const provider = new AnthropicProvider({
|
|
12
|
+
apiKey: process.env.ANTHROPIC_API_KEY,
|
|
13
|
+
model,
|
|
14
|
+
});
|
|
15
|
+
try {
|
|
16
|
+
const response = await provider.generateText('Reply with OK.', { maxTokens: 8, timeout: 15000 });
|
|
17
|
+
const text = response.text.trim();
|
|
18
|
+
console.log(`Anthropic OK (${model}) -> ${text}`);
|
|
19
|
+
}
|
|
20
|
+
catch (error) {
|
|
21
|
+
if (error instanceof LLMProviderError) {
|
|
22
|
+
console.error(`Anthropic failed: ${error.message}`);
|
|
23
|
+
if (error.cause instanceof Error) {
|
|
24
|
+
console.error(`Cause: ${error.cause.message}`);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
else if (error instanceof Error) {
|
|
28
|
+
console.error(`Anthropic failed: ${error.message}`);
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
console.error(`Anthropic failed: ${String(error)}`);
|
|
32
|
+
}
|
|
33
|
+
process.exit(1);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
import { appendFileSync } from 'fs';
|
|
4
|
+
import { join } from 'path';
|
|
5
|
+
import { writeCiSummary } from '../../engine/plan_builder.js';
|
|
6
|
+
import { recommendTestsAI, recommendTestsDeterministic } from '../../api.js';
|
|
7
|
+
export async function runPlanCommand(args, autoConfig, config) {
|
|
8
|
+
const reportRoot = config.testsRoot || config.path;
|
|
9
|
+
const apiOptions = {
|
|
10
|
+
cwd: process.cwd(),
|
|
11
|
+
configPath: autoConfig,
|
|
12
|
+
path: args.path,
|
|
13
|
+
profile: args.profile,
|
|
14
|
+
testsRoot: args.testsRoot,
|
|
15
|
+
gitSince: args.gitSince,
|
|
16
|
+
llmProvider: args.llmProvider,
|
|
17
|
+
policy: args.policyMinConfidence !== undefined ||
|
|
18
|
+
args.policySafeMergeConfidence !== undefined ||
|
|
19
|
+
args.policyWarningsThreshold !== undefined ||
|
|
20
|
+
(args.policyRiskyPatterns && args.policyRiskyPatterns.length > 0) ||
|
|
21
|
+
args.policyEnforcementMode !== undefined ||
|
|
22
|
+
(args.policyBlockActions && args.policyBlockActions.length > 0)
|
|
23
|
+
? {
|
|
24
|
+
minConfidenceForTargeted: args.policyMinConfidence,
|
|
25
|
+
safeMergeMinConfidence: args.policySafeMergeConfidence,
|
|
26
|
+
forceFullOnWarningsAtOrAbove: args.policyWarningsThreshold,
|
|
27
|
+
riskyFilePatterns: args.policyRiskyPatterns,
|
|
28
|
+
enforcementMode: args.policyEnforcementMode,
|
|
29
|
+
blockOnActions: args.policyBlockActions,
|
|
30
|
+
}
|
|
31
|
+
: undefined,
|
|
32
|
+
};
|
|
33
|
+
let result;
|
|
34
|
+
if (args.noAi) {
|
|
35
|
+
result = recommendTestsDeterministic(apiOptions);
|
|
36
|
+
}
|
|
37
|
+
else {
|
|
38
|
+
result = await recommendTestsAI(apiOptions);
|
|
39
|
+
if (result.aiEnrichment) {
|
|
40
|
+
const { aiEnrichment } = result;
|
|
41
|
+
console.log(`AI enrichment: ${aiEnrichment.enrichedFeatures.length} features enriched (${aiEnrichment.tokenUsage.input + aiEnrichment.tokenUsage.output} tokens)`);
|
|
42
|
+
}
|
|
43
|
+
else if (!process.env.ANTHROPIC_API_KEY) {
|
|
44
|
+
console.log('Tip: set ANTHROPIC_API_KEY to enable AI-powered enrichment');
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
const { plan, planPath, ciSummaryMarkdown, ciSummaryPath } = result;
|
|
48
|
+
// Write CI summary to an additional path if --ci-comment-path was specified
|
|
49
|
+
if (args.ciCommentPath) {
|
|
50
|
+
writeCiSummary(reportRoot, ciSummaryMarkdown, args.ciCommentPath);
|
|
51
|
+
}
|
|
52
|
+
const summaryPath = ciSummaryPath;
|
|
53
|
+
// Compute metrics paths (api already wrote metrics; derive paths for GHA output)
|
|
54
|
+
const metricsEventsPath = join(reportRoot, '.e2e-ai-agents/metrics.jsonl');
|
|
55
|
+
const metricsSummaryPath = join(reportRoot, '.e2e-ai-agents/metrics-summary.json');
|
|
56
|
+
const ghaOutput = args.githubOutputPath || process.env.GITHUB_OUTPUT;
|
|
57
|
+
if (ghaOutput) {
|
|
58
|
+
appendFileSync(ghaOutput, `run_set=${plan.runSet}\n`);
|
|
59
|
+
appendFileSync(ghaOutput, `action=${plan.decision.action}\n`);
|
|
60
|
+
appendFileSync(ghaOutput, `confidence=${plan.confidence}\n`);
|
|
61
|
+
appendFileSync(ghaOutput, `enforcement_mode=${plan.enforcement.mode}\n`);
|
|
62
|
+
appendFileSync(ghaOutput, `enforcement_should_fail=${plan.enforcement.shouldFail}\n`);
|
|
63
|
+
appendFileSync(ghaOutput, `recommended_tests_count=${plan.recommendedTests.length}\n`);
|
|
64
|
+
appendFileSync(ghaOutput, `required_new_tests_count=${plan.requiredNewTests.length}\n`);
|
|
65
|
+
appendFileSync(ghaOutput, `plan_path=${planPath}\n`);
|
|
66
|
+
appendFileSync(ghaOutput, `summary_path=${summaryPath}\n`);
|
|
67
|
+
appendFileSync(ghaOutput, `metrics_events_path=${metricsEventsPath}\n`);
|
|
68
|
+
appendFileSync(ghaOutput, `metrics_summary_path=${metricsSummaryPath}\n`);
|
|
69
|
+
}
|
|
70
|
+
console.log(`Suggested run set: ${plan.runSet} (confidence ${plan.confidence})`);
|
|
71
|
+
console.log(`Decision: ${plan.decision.action} - ${plan.decision.summary}`);
|
|
72
|
+
console.log(`Enforcement: ${plan.enforcement.mode} (shouldFail=${plan.enforcement.shouldFail})`);
|
|
73
|
+
console.log(`Plan data: ${planPath}`);
|
|
74
|
+
console.log(`CI summary: ${summaryPath}`);
|
|
75
|
+
console.log(`Plan metrics: ${metricsSummaryPath}`);
|
|
76
|
+
const failOnLegacyFlag = args.failOnMustAddTests && plan.decision.action === 'must-add-tests';
|
|
77
|
+
if (failOnLegacyFlag || plan.enforcement.shouldFail) {
|
|
78
|
+
process.exit(2);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
import { readFileSync } from 'fs';
|
|
4
|
+
import { resolveConfig } from '../../agent/config.js';
|
|
5
|
+
import { captureTraceabilityInput } from '../../agent/traceability_capture.js';
|
|
6
|
+
import { ingestTraceabilityInput } from '../../agent/traceability_ingest.js';
|
|
7
|
+
export function runTraceabilityCaptureCommand(args, autoConfig) {
|
|
8
|
+
if (!args.path && !autoConfig) {
|
|
9
|
+
console.error('Error: --path is required for traceability-capture command');
|
|
10
|
+
process.exit(1);
|
|
11
|
+
}
|
|
12
|
+
if (!args.traceabilityReportPath) {
|
|
13
|
+
console.error('Error: --traceability-report <path> is required for traceability-capture command');
|
|
14
|
+
process.exit(1);
|
|
15
|
+
}
|
|
16
|
+
const { config } = resolveConfig(process.cwd(), autoConfig, {
|
|
17
|
+
path: args.path,
|
|
18
|
+
profile: args.profile,
|
|
19
|
+
testsRoot: args.testsRoot,
|
|
20
|
+
mode: 'impact',
|
|
21
|
+
gitSince: args.gitSince,
|
|
22
|
+
llmProvider: args.llmProvider,
|
|
23
|
+
});
|
|
24
|
+
const reportRoot = config.testsRoot || config.path;
|
|
25
|
+
const output = captureTraceabilityInput({
|
|
26
|
+
appPath: config.path,
|
|
27
|
+
testsRoot: reportRoot,
|
|
28
|
+
reportPath: args.traceabilityReportPath,
|
|
29
|
+
sinceRef: args.gitSince || config.git.since,
|
|
30
|
+
outputPath: args.traceabilityCaptureOutputPath,
|
|
31
|
+
coverageMapPath: args.traceabilityCoverageMapPath,
|
|
32
|
+
changedFilesPath: args.traceabilityChangedFilesPath,
|
|
33
|
+
});
|
|
34
|
+
console.log(`Traceability input: ${output.outputPath}`);
|
|
35
|
+
console.log(`Traceability tests seen: ${output.testsSeen}`);
|
|
36
|
+
console.log(`Traceability runs generated: ${output.runsGenerated}`);
|
|
37
|
+
console.log(`Traceability changed files used: ${output.changedFilesUsed}`);
|
|
38
|
+
if (output.warnings.length > 0) {
|
|
39
|
+
console.log(`Traceability warnings: ${output.warnings.join(' | ')}`);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
export function runTraceabilityIngestCommand(args, autoConfig) {
|
|
43
|
+
if (!args.path && !autoConfig) {
|
|
44
|
+
console.error('Error: --path is required for traceability-ingest command');
|
|
45
|
+
process.exit(1);
|
|
46
|
+
}
|
|
47
|
+
if (!args.traceabilityInputPath) {
|
|
48
|
+
console.error('Error: --traceability-input <path> is required for traceability-ingest command');
|
|
49
|
+
process.exit(1);
|
|
50
|
+
}
|
|
51
|
+
const { config } = resolveConfig(process.cwd(), autoConfig, {
|
|
52
|
+
path: args.path,
|
|
53
|
+
profile: args.profile,
|
|
54
|
+
testsRoot: args.testsRoot,
|
|
55
|
+
mode: 'impact',
|
|
56
|
+
llmProvider: args.llmProvider,
|
|
57
|
+
});
|
|
58
|
+
const reportRoot = config.testsRoot || config.path;
|
|
59
|
+
const raw = JSON.parse(readFileSync(args.traceabilityInputPath, 'utf-8'));
|
|
60
|
+
const output = ingestTraceabilityInput(reportRoot, config.impact.traceability, raw, {
|
|
61
|
+
minHits: args.traceabilityMinHits,
|
|
62
|
+
maxFilesPerTest: args.traceabilityMaxFilesPerTest,
|
|
63
|
+
maxAgeDays: args.traceabilityMaxAgeDays,
|
|
64
|
+
});
|
|
65
|
+
console.log(`Traceability manifest: ${output.manifestPath}`);
|
|
66
|
+
console.log(`Traceability state: ${output.statePath}`);
|
|
67
|
+
console.log(`Traceability ingested entries: ${output.entriesIngested}`);
|
|
68
|
+
console.log(`Traceability tracked tests: ${output.testsTracked}`);
|
|
69
|
+
console.log(`Traceability tracked edges: ${output.edgesTracked}`);
|
|
70
|
+
if (output.warnings.length > 0) {
|
|
71
|
+
console.log(`Traceability warnings: ${output.warnings.join(' | ')}`);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
import { existsSync } from 'fs';
|
|
4
|
+
import { dirname, join, resolve } from 'path';
|
|
5
|
+
export const CONFIG_CANDIDATES = ['e2e-ai-agents.config.json', '.e2e-ai-agents.config.json'];
|
|
6
|
+
export function findConfigUpwards(startDir) {
|
|
7
|
+
if (!startDir) {
|
|
8
|
+
return undefined;
|
|
9
|
+
}
|
|
10
|
+
let current = resolve(startDir);
|
|
11
|
+
while (true) {
|
|
12
|
+
for (const candidate of CONFIG_CANDIDATES) {
|
|
13
|
+
const fullPath = join(current, candidate);
|
|
14
|
+
if (existsSync(fullPath)) {
|
|
15
|
+
return fullPath;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
const parent = dirname(current);
|
|
19
|
+
if (parent === current) {
|
|
20
|
+
break;
|
|
21
|
+
}
|
|
22
|
+
current = parent;
|
|
23
|
+
}
|
|
24
|
+
return undefined;
|
|
25
|
+
}
|
|
26
|
+
export function resolveAutoConfig(args) {
|
|
27
|
+
if (args.configPath) {
|
|
28
|
+
return args.configPath;
|
|
29
|
+
}
|
|
30
|
+
const searchRoots = [
|
|
31
|
+
process.cwd(),
|
|
32
|
+
args.testsRoot,
|
|
33
|
+
args.path,
|
|
34
|
+
].filter(Boolean);
|
|
35
|
+
for (const root of searchRoots) {
|
|
36
|
+
const found = findConfigUpwards(root);
|
|
37
|
+
if (found) {
|
|
38
|
+
return found;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
return undefined;
|
|
42
|
+
}
|
|
43
|
+
const csvSplit = (v) => v.split(',').map((s) => s.trim()).filter(Boolean);
|
|
44
|
+
// prettier-ignore
|
|
45
|
+
const FLAGS = {
|
|
46
|
+
// -- boolean flags --
|
|
47
|
+
'--help': { key: 'help', type: 'boolean', aliases: ['-h'] },
|
|
48
|
+
'--apply': { key: 'apply', type: 'boolean' },
|
|
49
|
+
'--allow-fallback': { key: 'allowFallback', type: 'boolean' },
|
|
50
|
+
'--pipeline': { key: 'pipeline', type: 'boolean' },
|
|
51
|
+
'--pipeline-mcp': { key: 'pipelineMcp', type: 'boolean' },
|
|
52
|
+
'--pipeline-mcp-allow-fallback': { key: 'pipelineMcpAllowFallback', type: 'boolean' },
|
|
53
|
+
'--pipeline-mcp-only': { key: 'pipelineMcpOnly', type: 'boolean' },
|
|
54
|
+
'--pipeline-headless': { key: 'pipelineHeadless', type: 'boolean' },
|
|
55
|
+
'--pipeline-headed': { key: 'pipelineHeadless', type: 'boolean-false' },
|
|
56
|
+
'--pipeline-parallel': { key: 'pipelineParallel', type: 'boolean' },
|
|
57
|
+
'--pipeline-dry-run': { key: 'pipelineDryRun', type: 'boolean' },
|
|
58
|
+
'--fail-on-must-add-tests': { key: 'failOnMustAddTests', type: 'boolean' },
|
|
59
|
+
'--create-pr': { key: 'createPr', type: 'boolean' },
|
|
60
|
+
'--dry-run': { key: 'dryRun', type: 'boolean' },
|
|
61
|
+
'--generate': { key: 'analyzeGenerate', type: 'boolean' },
|
|
62
|
+
'--heal': { key: 'analyzeHeal', type: 'boolean' },
|
|
63
|
+
'--no-ai': { key: 'noAi', type: 'boolean' },
|
|
64
|
+
'--mattermost': { key: 'profile', type: 'boolean', transform: () => 'mattermost' },
|
|
65
|
+
// -- string flags --
|
|
66
|
+
'--config': { key: 'configPath', type: 'string' },
|
|
67
|
+
'--path': { key: 'path', type: 'string' },
|
|
68
|
+
'--tests-root': { key: 'testsRoot', type: 'string' },
|
|
69
|
+
'--framework': { key: 'framework', type: 'string', transform: (v) => v },
|
|
70
|
+
'--scenarios': { key: 'generateScenarios', type: 'string' },
|
|
71
|
+
'--pipeline-output': { key: 'pipelineOutput', type: 'string' },
|
|
72
|
+
'--pipeline-base-url': { key: 'pipelineBaseUrl', type: 'string' },
|
|
73
|
+
'--pipeline-project': { key: 'pipelineProject', type: 'string' },
|
|
74
|
+
'--spec': { key: 'specPDF', type: 'string' },
|
|
75
|
+
'--since': { key: 'gitSince', type: 'string' },
|
|
76
|
+
'--llm-provider': { key: 'llmProvider', type: 'string' },
|
|
77
|
+
'--ci-comment-path': { key: 'ciCommentPath', type: 'string' },
|
|
78
|
+
'--github-output': { key: 'githubOutputPath', type: 'string' },
|
|
79
|
+
'--feedback-input': { key: 'feedbackInputPath', type: 'string' },
|
|
80
|
+
'--traceability-report': { key: 'traceabilityReportPath', type: 'string' },
|
|
81
|
+
'--traceability-capture-output': { key: 'traceabilityCaptureOutputPath', type: 'string' },
|
|
82
|
+
'--traceability-coverage-map': { key: 'traceabilityCoverageMapPath', type: 'string' },
|
|
83
|
+
'--traceability-changed-files': { key: 'traceabilityChangedFilesPath', type: 'string' },
|
|
84
|
+
'--traceability-input': { key: 'traceabilityInputPath', type: 'string' },
|
|
85
|
+
'--branch': { key: 'branch', type: 'string' },
|
|
86
|
+
'--commit-message': { key: 'commitMessage', type: 'string' },
|
|
87
|
+
'--pr-title': { key: 'prTitle', type: 'string' },
|
|
88
|
+
'--pr-body': { key: 'prBody', type: 'string' },
|
|
89
|
+
'--pr-base': { key: 'prBase', type: 'string' },
|
|
90
|
+
'--generate-output': { key: 'analyzeGenerateOutputDir', type: 'string' },
|
|
91
|
+
'--heal-report': { key: 'analyzeHealReport', type: 'string' },
|
|
92
|
+
'--flow-catalog': { key: 'flowCatalogPath', type: 'string' },
|
|
93
|
+
// -- number flags (with isFinite guard) --
|
|
94
|
+
'--pipeline-scenarios': { key: 'pipelineScenarios', type: 'number' },
|
|
95
|
+
'--time': { key: 'timeLimitMinutes', type: 'number' },
|
|
96
|
+
'--budget-usd': { key: 'budgetUSD', type: 'number' },
|
|
97
|
+
'--budget-tokens': { key: 'budgetTokens', type: 'number' },
|
|
98
|
+
'--policy-min-confidence': { key: 'policyMinConfidence', type: 'number' },
|
|
99
|
+
'--policy-safe-merge-confidence': { key: 'policySafeMergeConfidence', type: 'number' },
|
|
100
|
+
'--policy-force-full-on-warnings': { key: 'policyWarningsThreshold', type: 'number' },
|
|
101
|
+
'--traceability-min-hits': { key: 'traceabilityMinHits', type: 'number' },
|
|
102
|
+
'--traceability-max-files-per-test': { key: 'traceabilityMaxFilesPerTest', type: 'number' },
|
|
103
|
+
'--traceability-max-age-days': { key: 'traceabilityMaxAgeDays', type: 'number' },
|
|
104
|
+
// -- number-raw flags (no isFinite guard, assigned directly via Number()) --
|
|
105
|
+
'--max-attempts': { key: 'maxAttempts', type: 'number-raw', transform: (v) => parseInt(v, 10) },
|
|
106
|
+
'--pipeline-mcp-timeout-ms': { key: 'pipelineMcpTimeoutMs', type: 'number-raw' },
|
|
107
|
+
'--pipeline-mcp-retries': { key: 'pipelineMcpRetries', type: 'number-raw' },
|
|
108
|
+
// -- enum flags --
|
|
109
|
+
'--profile': { key: 'profile', type: 'enum', enumValues: ['default', 'mattermost'] },
|
|
110
|
+
'--pipeline-browser': { key: 'pipelineBrowser', type: 'enum', enumValues: ['chrome', 'chromium', 'firefox', 'webkit'] },
|
|
111
|
+
'--policy-enforcement-mode': { key: 'policyEnforcementMode', type: 'enum', enumValues: ['advisory', 'warn', 'block'] },
|
|
112
|
+
// -- csv flags --
|
|
113
|
+
'--patterns': { key: 'testPatterns', type: 'csv' },
|
|
114
|
+
'--flow-patterns': { key: 'flowPatterns', type: 'csv' },
|
|
115
|
+
'--flow-exclude': { key: 'flowExclude', type: 'csv' },
|
|
116
|
+
'--policy-risky-patterns': { key: 'policyRiskyPatterns', type: 'csv' },
|
|
117
|
+
'--policy-block-actions': {
|
|
118
|
+
key: 'policyBlockActions',
|
|
119
|
+
type: 'csv',
|
|
120
|
+
transform: (v) => csvSplit(v).filter((s) => s === 'run-now' || s === 'must-add-tests' || s === 'safe-to-merge'),
|
|
121
|
+
},
|
|
122
|
+
};
|
|
123
|
+
// Build a lookup from alias -> canonical flag name
|
|
124
|
+
const ALIAS_MAP = {};
|
|
125
|
+
for (const [flag, def] of Object.entries(FLAGS)) {
|
|
126
|
+
ALIAS_MAP[flag] = flag;
|
|
127
|
+
if (def.aliases) {
|
|
128
|
+
for (const alias of def.aliases) {
|
|
129
|
+
ALIAS_MAP[alias] = flag;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
const COMMANDS = new Set([
|
|
134
|
+
'impact', 'plan', 'heal', 'suggest', 'generate',
|
|
135
|
+
'finalize-generated-tests', 'feedback',
|
|
136
|
+
'traceability-capture', 'traceability-ingest',
|
|
137
|
+
'analyze', 'llm-health',
|
|
138
|
+
]);
|
|
139
|
+
// ---------------------------------------------------------------------------
|
|
140
|
+
// Parser
|
|
141
|
+
// ---------------------------------------------------------------------------
|
|
142
|
+
function setField(obj, key, value) {
|
|
143
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
144
|
+
obj[key] = value;
|
|
145
|
+
}
|
|
146
|
+
export function parseArgs(argv) {
|
|
147
|
+
const parsed = { apply: false, help: false };
|
|
148
|
+
if (argv.length === 0) {
|
|
149
|
+
return parsed;
|
|
150
|
+
}
|
|
151
|
+
const command = argv[0];
|
|
152
|
+
if (COMMANDS.has(command)) {
|
|
153
|
+
parsed.command = command;
|
|
154
|
+
}
|
|
155
|
+
for (let i = 1; i < argv.length; i += 1) {
|
|
156
|
+
const arg = argv[i];
|
|
157
|
+
const canonical = ALIAS_MAP[arg];
|
|
158
|
+
if (!canonical) {
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
161
|
+
const def = FLAGS[canonical];
|
|
162
|
+
const next = argv[i + 1];
|
|
163
|
+
switch (def.type) {
|
|
164
|
+
case 'boolean':
|
|
165
|
+
setField(parsed, def.key, def.transform ? def.transform('') : true);
|
|
166
|
+
break;
|
|
167
|
+
case 'boolean-false':
|
|
168
|
+
setField(parsed, def.key, false);
|
|
169
|
+
break;
|
|
170
|
+
case 'string':
|
|
171
|
+
if (next) {
|
|
172
|
+
setField(parsed, def.key, def.transform ? def.transform(next) : next);
|
|
173
|
+
i += 1;
|
|
174
|
+
}
|
|
175
|
+
break;
|
|
176
|
+
case 'number':
|
|
177
|
+
if (next) {
|
|
178
|
+
const value = Number(next);
|
|
179
|
+
if (Number.isFinite(value)) {
|
|
180
|
+
setField(parsed, def.key, value);
|
|
181
|
+
}
|
|
182
|
+
i += 1;
|
|
183
|
+
}
|
|
184
|
+
break;
|
|
185
|
+
case 'number-raw':
|
|
186
|
+
if (next) {
|
|
187
|
+
setField(parsed, def.key, def.transform ? def.transform(next) : Number(next));
|
|
188
|
+
i += 1;
|
|
189
|
+
}
|
|
190
|
+
break;
|
|
191
|
+
case 'csv':
|
|
192
|
+
if (next) {
|
|
193
|
+
setField(parsed, def.key, def.transform ? def.transform(next) : csvSplit(next));
|
|
194
|
+
i += 1;
|
|
195
|
+
}
|
|
196
|
+
break;
|
|
197
|
+
case 'enum':
|
|
198
|
+
if (next) {
|
|
199
|
+
if (def.enumValues.includes(next)) {
|
|
200
|
+
setField(parsed, def.key, next);
|
|
201
|
+
}
|
|
202
|
+
i += 1;
|
|
203
|
+
}
|
|
204
|
+
break;
|
|
205
|
+
default:
|
|
206
|
+
break;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
return parsed;
|
|
210
|
+
}
|