@yasserkhanorg/e2e-agents 1.7.6 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/git.d.ts +5 -0
- package/dist/agent/git.d.ts.map +1 -1
- package/dist/agent/git.js +49 -0
- package/dist/agents/coverage-evaluator.d.ts +8 -0
- package/dist/agents/coverage-evaluator.d.ts.map +1 -0
- package/dist/agents/coverage-evaluator.js +41 -0
- package/dist/agents/cross-impact.d.ts +13 -0
- package/dist/agents/cross-impact.d.ts.map +1 -0
- package/dist/agents/cross-impact.js +135 -0
- package/dist/agents/executor.d.ts +8 -0
- package/dist/agents/executor.d.ts.map +1 -0
- package/dist/agents/executor.js +70 -0
- package/dist/agents/explorer.d.ts +12 -0
- package/dist/agents/explorer.d.ts.map +1 -0
- package/dist/agents/explorer.js +43 -0
- package/dist/agents/generator.d.ts +8 -0
- package/dist/agents/generator.d.ts.map +1 -0
- package/dist/agents/generator.js +77 -0
- package/dist/agents/healer.d.ts +8 -0
- package/dist/agents/healer.d.ts.map +1 -0
- package/dist/agents/healer.js +31 -0
- package/dist/agents/impact-analyst.d.ts +8 -0
- package/dist/agents/impact-analyst.d.ts.map +1 -0
- package/dist/agents/impact-analyst.js +38 -0
- package/dist/agents/regression-advisor.d.ts +8 -0
- package/dist/agents/regression-advisor.d.ts.map +1 -0
- package/dist/agents/regression-advisor.js +116 -0
- package/dist/agents/strategist.d.ts +9 -0
- package/dist/agents/strategist.d.ts.map +1 -0
- package/dist/agents/strategist.js +87 -0
- package/dist/agents/test-designer.d.ts +8 -0
- package/dist/agents/test-designer.d.ts.map +1 -0
- package/dist/agents/test-designer.js +106 -0
- package/dist/cli/commands/crew.d.ts +3 -0
- package/dist/cli/commands/crew.d.ts.map +1 -0
- package/dist/cli/commands/crew.js +137 -0
- package/dist/cli/parse_args.d.ts.map +1 -1
- package/dist/cli/parse_args.js +2 -1
- package/dist/cli/types.d.ts +2 -1
- package/dist/cli/types.d.ts.map +1 -1
- package/dist/cli.js +5 -0
- package/dist/crew/context.d.ts +40 -0
- package/dist/crew/context.d.ts.map +1 -0
- package/dist/crew/context.js +36 -0
- package/dist/crew/orchestrator.d.ts +36 -0
- package/dist/crew/orchestrator.d.ts.map +1 -0
- package/dist/crew/orchestrator.js +171 -0
- package/dist/crew/protocol.d.ts +33 -0
- package/dist/crew/protocol.d.ts.map +1 -0
- package/dist/crew/protocol.js +4 -0
- package/dist/crew/provider.d.ts +3 -0
- package/dist/crew/provider.d.ts.map +1 -0
- package/dist/crew/provider.js +16 -0
- package/dist/crew/sanitize.d.ts +3 -0
- package/dist/crew/sanitize.d.ts.map +1 -0
- package/dist/crew/sanitize.js +31 -0
- package/dist/crew/types.d.ts +52 -0
- package/dist/crew/types.d.ts.map +1 -0
- package/dist/crew/types.js +4 -0
- package/dist/crew/workflows.d.ts +52 -0
- package/dist/crew/workflows.d.ts.map +1 -0
- package/dist/crew/workflows.js +36 -0
- package/dist/esm/agent/git.js +48 -0
- package/dist/esm/agents/coverage-evaluator.js +37 -0
- package/dist/esm/agents/cross-impact.js +131 -0
- package/dist/esm/agents/executor.js +66 -0
- package/dist/esm/agents/explorer.js +39 -0
- package/dist/esm/agents/generator.js +73 -0
- package/dist/esm/agents/healer.js +27 -0
- package/dist/esm/agents/impact-analyst.js +34 -0
- package/dist/esm/agents/regression-advisor.js +112 -0
- package/dist/esm/agents/strategist.js +83 -0
- package/dist/esm/agents/test-designer.js +102 -0
- package/dist/esm/cli/commands/crew.js +134 -0
- package/dist/esm/cli/parse_args.js +2 -1
- package/dist/esm/cli.js +5 -0
- package/dist/esm/crew/context.js +32 -0
- package/dist/esm/crew/orchestrator.js +167 -0
- package/dist/esm/crew/protocol.js +3 -0
- package/dist/esm/crew/provider.js +13 -0
- package/dist/esm/crew/sanitize.js +27 -0
- package/dist/esm/crew/types.js +3 -0
- package/dist/esm/crew/workflows.js +33 -0
- package/dist/esm/index.js +14 -0
- package/dist/esm/prompts/cross-impact.js +71 -0
- package/dist/esm/prompts/strategist.js +79 -0
- package/dist/esm/prompts/test-designer.js +107 -0
- package/dist/index.d.ts +17 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +27 -1
- package/dist/prompts/cross-impact.d.ts +22 -0
- package/dist/prompts/cross-impact.d.ts.map +1 -0
- package/dist/prompts/cross-impact.js +75 -0
- package/dist/prompts/strategist.d.ts +25 -0
- package/dist/prompts/strategist.d.ts.map +1 -0
- package/dist/prompts/strategist.js +83 -0
- package/dist/prompts/test-designer.d.ts +33 -0
- package/dist/prompts/test-designer.d.ts.map +1 -0
- package/dist/prompts/test-designer.js +111 -0
- package/package.json +1 -1
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
/**
|
|
4
|
+
* CLI command: crew — runs multi-agent QA analysis workflows.
|
|
5
|
+
*/
|
|
6
|
+
import { resolveConfig } from '../../agent/config.js';
|
|
7
|
+
import { CrewOrchestrator } from '../../crew/orchestrator.js';
|
|
8
|
+
import { ImpactAnalystAgent } from '../../agents/impact-analyst.js';
|
|
9
|
+
import { GeneratorAgent } from '../../agents/generator.js';
|
|
10
|
+
import { ExecutorAgent } from '../../agents/executor.js';
|
|
11
|
+
import { HealerAgent } from '../../agents/healer.js';
|
|
12
|
+
import { StrategistAgent } from '../../agents/strategist.js';
|
|
13
|
+
import { TestDesignerAgent } from '../../agents/test-designer.js';
|
|
14
|
+
import { CrossImpactAgent } from '../../agents/cross-impact.js';
|
|
15
|
+
import { RegressionAdvisorAgent } from '../../agents/regression-advisor.js';
|
|
16
|
+
const VALID_WORKFLOWS = ['full-qa', 'quick-check', 'design-only'];
|
|
17
|
+
export async function runCrewCommand(args, autoConfig) {
|
|
18
|
+
if (!args.path && !autoConfig) {
|
|
19
|
+
console.error('Error: --path is required for crew command');
|
|
20
|
+
process.exit(1);
|
|
21
|
+
}
|
|
22
|
+
const { config } = resolveConfig(process.cwd(), autoConfig, {
|
|
23
|
+
path: args.path,
|
|
24
|
+
profile: args.profile,
|
|
25
|
+
testsRoot: args.testsRoot,
|
|
26
|
+
mode: 'impact',
|
|
27
|
+
gitSince: args.gitSince,
|
|
28
|
+
llmProvider: args.llmProvider,
|
|
29
|
+
});
|
|
30
|
+
const testsRoot = config.testsRoot || config.path;
|
|
31
|
+
const rawWorkflow = args.crewWorkflow || 'full-qa';
|
|
32
|
+
if (!VALID_WORKFLOWS.includes(rawWorkflow)) {
|
|
33
|
+
console.error(`Error: invalid workflow '${rawWorkflow}'. Valid: ${VALID_WORKFLOWS.join(', ')}`);
|
|
34
|
+
process.exit(1);
|
|
35
|
+
}
|
|
36
|
+
const workflowName = rawWorkflow;
|
|
37
|
+
const crewConfig = {
|
|
38
|
+
appPath: config.path,
|
|
39
|
+
testsRoot,
|
|
40
|
+
gitSince: args.gitSince || config.git.since,
|
|
41
|
+
routeFamilies: config.routeFamilies,
|
|
42
|
+
apiSurface: config.apiSurface,
|
|
43
|
+
workflow: workflowName,
|
|
44
|
+
providerOverride: args.llmProvider,
|
|
45
|
+
budgetUSD: args.budgetUSD,
|
|
46
|
+
dryRun: args.dryRun,
|
|
47
|
+
};
|
|
48
|
+
// Create orchestrator and register all agents
|
|
49
|
+
const orchestrator = new CrewOrchestrator();
|
|
50
|
+
orchestrator.registerAgent(new ImpactAnalystAgent());
|
|
51
|
+
orchestrator.registerAgent(new GeneratorAgent());
|
|
52
|
+
orchestrator.registerAgent(new ExecutorAgent());
|
|
53
|
+
orchestrator.registerAgent(new HealerAgent());
|
|
54
|
+
orchestrator.registerAgent(new StrategistAgent());
|
|
55
|
+
orchestrator.registerAgent(new TestDesignerAgent());
|
|
56
|
+
orchestrator.registerAgent(new CrossImpactAgent());
|
|
57
|
+
orchestrator.registerAgent(new RegressionAdvisorAgent());
|
|
58
|
+
let result;
|
|
59
|
+
try {
|
|
60
|
+
result = await orchestrator.run(crewConfig);
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
64
|
+
console.error(`Crew workflow failed: ${message}`);
|
|
65
|
+
process.exit(1);
|
|
66
|
+
}
|
|
67
|
+
const ctx = result.context;
|
|
68
|
+
// JSON output mode
|
|
69
|
+
if (args.jsonOutput) {
|
|
70
|
+
const jsonReport = {
|
|
71
|
+
workflow: workflowName,
|
|
72
|
+
changedFiles: ctx.changedFiles.length,
|
|
73
|
+
impactedFlows: ctx.impactedFlows,
|
|
74
|
+
strategyEntries: ctx.strategyEntries,
|
|
75
|
+
testDesigns: ctx.testDesigns,
|
|
76
|
+
crossImpacts: ctx.crossImpacts,
|
|
77
|
+
regressionRisks: ctx.regressionRisks,
|
|
78
|
+
findings: ctx.findings,
|
|
79
|
+
generatedSpecs: ctx.generatedSpecs.map((s) => ({ flowId: s.flowId, specPath: s.specPath, mode: s.mode, written: s.written })),
|
|
80
|
+
usage: { cost: ctx.usage.totalCost, requests: ctx.usage.requestCount, tokens: ctx.usage.totalTokens },
|
|
81
|
+
timings: result.timings,
|
|
82
|
+
warnings: result.warnings,
|
|
83
|
+
};
|
|
84
|
+
console.log(JSON.stringify(jsonReport, null, 2));
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
// Human-readable output
|
|
88
|
+
console.log(`Crew workflow: ${workflowName}`);
|
|
89
|
+
console.log(`Changed files: ${ctx.changedFiles.length}`);
|
|
90
|
+
console.log(`Impacted flows: ${ctx.impactedFlows.length}`);
|
|
91
|
+
console.log(`Strategy entries: ${ctx.strategyEntries.length}`);
|
|
92
|
+
console.log(`Test designs: ${ctx.testDesigns.length} (${ctx.testDesigns.reduce((sum, td) => sum + td.testCases.length, 0)} test cases)`);
|
|
93
|
+
console.log(`Cross-impacts: ${ctx.crossImpacts.length}`);
|
|
94
|
+
console.log(`Regression risks: ${ctx.regressionRisks.length}`);
|
|
95
|
+
console.log(`Findings: ${ctx.findings.length}`);
|
|
96
|
+
console.log(`Generated specs: ${ctx.generatedSpecs.length}`);
|
|
97
|
+
console.log(`Cost: $${ctx.usage.totalCost.toFixed(4)}`);
|
|
98
|
+
if (ctx.strategyEntries.length > 0) {
|
|
99
|
+
console.log('\nTest Strategy:');
|
|
100
|
+
for (const entry of ctx.strategyEntries) {
|
|
101
|
+
console.log(` ${entry.priority} ${entry.flowName} → ${entry.approach} [${entry.testCategories.join(', ')}]`);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
if (ctx.testDesigns.length > 0) {
|
|
105
|
+
console.log('\nTest Designs:');
|
|
106
|
+
for (const design of ctx.testDesigns) {
|
|
107
|
+
console.log(` ${design.flowName}: ${design.testCases.length} test cases`);
|
|
108
|
+
for (const tc of design.testCases) {
|
|
109
|
+
console.log(` [${tc.type}] ${tc.name} (${tc.priority})`);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
if (ctx.crossImpacts.length > 0) {
|
|
114
|
+
console.log('\nCross-Family Impacts:');
|
|
115
|
+
for (const ci of ctx.crossImpacts) {
|
|
116
|
+
console.log(` ${ci.sourceFamily} → ${ci.affectedFamily} (${ci.riskLevel}): ${ci.sharedDependency}`);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
if (result.timings && Object.keys(result.timings).length > 0) {
|
|
120
|
+
console.log('\nPhase timings:');
|
|
121
|
+
for (const [phase, ms] of Object.entries(result.timings)) {
|
|
122
|
+
console.log(` ${phase}: ${ms}ms`);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
if (result.warnings.length > 0) {
|
|
126
|
+
console.log(`\nWarnings: ${result.warnings.length}`);
|
|
127
|
+
for (const w of result.warnings.slice(0, 10)) {
|
|
128
|
+
console.log(` - ${w}`);
|
|
129
|
+
}
|
|
130
|
+
if (result.warnings.length > 10) {
|
|
131
|
+
console.log(` ... and ${result.warnings.length - 10} more`);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
@@ -98,6 +98,7 @@ const FLAGS = {
|
|
|
98
98
|
'--flow-catalog': { key: 'flowCatalogPath', type: 'string' },
|
|
99
99
|
'--output': { key: 'trainOutput', type: 'string' },
|
|
100
100
|
'--server-path': { key: 'serverPath', type: 'string' },
|
|
101
|
+
'--workflow': { key: 'crewWorkflow', type: 'string' },
|
|
101
102
|
// -- number flags (with isFinite guard) --
|
|
102
103
|
'--pipeline-scenarios': { key: 'pipelineScenarios', type: 'number' },
|
|
103
104
|
'--time': { key: 'timeLimitMinutes', type: 'number' },
|
|
@@ -143,7 +144,7 @@ const COMMANDS = new Set([
|
|
|
143
144
|
'init', 'impact', 'plan', 'heal', 'suggest', 'generate',
|
|
144
145
|
'finalize-generated-tests', 'feedback',
|
|
145
146
|
'traceability-capture', 'traceability-ingest',
|
|
146
|
-
'analyze', 'llm-health', 'train',
|
|
147
|
+
'analyze', 'llm-health', 'train', 'crew',
|
|
147
148
|
]);
|
|
148
149
|
// ---------------------------------------------------------------------------
|
|
149
150
|
// Parser
|
package/dist/esm/cli.js
CHANGED
|
@@ -15,6 +15,7 @@ import { runPlanCommand } from './cli/commands/plan.js';
|
|
|
15
15
|
import { runGenerateCommand } from './cli/commands/generate.js';
|
|
16
16
|
import { runInitCommand } from './cli/commands/init.js';
|
|
17
17
|
import { runTrainCommand } from './cli/commands/train.js';
|
|
18
|
+
import { runCrewCommand } from './cli/commands/crew.js';
|
|
18
19
|
async function main() {
|
|
19
20
|
const args = parseArgs(process.argv.slice(2));
|
|
20
21
|
const autoConfig = resolveAutoConfig(args);
|
|
@@ -59,6 +60,10 @@ async function main() {
|
|
|
59
60
|
runHealCommand(args, autoConfig);
|
|
60
61
|
return;
|
|
61
62
|
}
|
|
63
|
+
if (args.command === 'crew') {
|
|
64
|
+
await runCrewCommand(args, autoConfig);
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
62
67
|
if (!args.path && !autoConfig) {
|
|
63
68
|
console.error('Error: --path is required (or provide a config file with path set)');
|
|
64
69
|
printUsage();
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
export function createEmptyUsageStats() {
|
|
4
|
+
const now = new Date();
|
|
5
|
+
return {
|
|
6
|
+
requestCount: 0,
|
|
7
|
+
totalInputTokens: 0,
|
|
8
|
+
totalOutputTokens: 0,
|
|
9
|
+
totalTokens: 0,
|
|
10
|
+
totalCost: 0,
|
|
11
|
+
averageResponseTimeMs: 0,
|
|
12
|
+
failedRequests: 0,
|
|
13
|
+
startTime: now,
|
|
14
|
+
lastUpdated: now,
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
export function mergeUsageStats(target, source) {
|
|
18
|
+
const prevRequestCount = target.requestCount;
|
|
19
|
+
target.requestCount += source.requestCount;
|
|
20
|
+
target.totalInputTokens += source.totalInputTokens;
|
|
21
|
+
target.totalOutputTokens += source.totalOutputTokens;
|
|
22
|
+
target.totalTokens += source.totalTokens;
|
|
23
|
+
target.totalCost += source.totalCost;
|
|
24
|
+
target.failedRequests += source.failedRequests;
|
|
25
|
+
if (source.requestCount > 0 && target.requestCount > 0) {
|
|
26
|
+
const prevWeight = prevRequestCount / target.requestCount;
|
|
27
|
+
const newWeight = source.requestCount / target.requestCount;
|
|
28
|
+
target.averageResponseTimeMs =
|
|
29
|
+
target.averageResponseTimeMs * prevWeight + source.averageResponseTimeMs * newWeight;
|
|
30
|
+
}
|
|
31
|
+
target.lastUpdated = new Date();
|
|
32
|
+
}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
/**
|
|
4
|
+
* Crew Orchestrator — executes workflow definitions by dispatching to agents.
|
|
5
|
+
*/
|
|
6
|
+
import { getChangedFiles, isTestFile } from '../agent/git.js';
|
|
7
|
+
import { preprocess } from '../pipeline/stage0_preprocess.js';
|
|
8
|
+
import { logger } from '../logger.js';
|
|
9
|
+
import { createEmptyUsageStats, mergeUsageStats } from './context.js';
|
|
10
|
+
import { WORKFLOWS } from './workflows.js';
|
|
11
|
+
export class CrewOrchestrator {
|
|
12
|
+
constructor() {
|
|
13
|
+
this.agents = new Map();
|
|
14
|
+
}
|
|
15
|
+
registerAgent(agent) {
|
|
16
|
+
this.agents.set(agent.role, agent);
|
|
17
|
+
}
|
|
18
|
+
async run(config) {
|
|
19
|
+
const workflow = WORKFLOWS[config.workflow || 'full-qa'];
|
|
20
|
+
const timings = {};
|
|
21
|
+
const warnings = [];
|
|
22
|
+
// Step 1: Get changed files
|
|
23
|
+
const gitResult = getChangedFiles(config.appPath, config.gitSince, {
|
|
24
|
+
includeUncommitted: config.gitIncludeUncommitted,
|
|
25
|
+
});
|
|
26
|
+
if (gitResult.error) {
|
|
27
|
+
warnings.push(`Git diff warning: ${gitResult.error}`);
|
|
28
|
+
}
|
|
29
|
+
const changedFiles = gitResult.files
|
|
30
|
+
.map((f) => f.replace(/\\/g, '/'))
|
|
31
|
+
.filter((f) => !isTestFile(f));
|
|
32
|
+
if (changedFiles.length === 0) {
|
|
33
|
+
warnings.push('No changed application files detected.');
|
|
34
|
+
}
|
|
35
|
+
// Initialize context (will be populated during preprocess phase)
|
|
36
|
+
const ctx = {
|
|
37
|
+
changedFiles,
|
|
38
|
+
routeFamilies: [],
|
|
39
|
+
manifest: null,
|
|
40
|
+
apiSurface: { pageObjects: [], generatedAt: '' },
|
|
41
|
+
specIndex: { specs: [], indexedAt: '' },
|
|
42
|
+
context: { documents: [], warnings: [] },
|
|
43
|
+
familyGroups: [],
|
|
44
|
+
preprocessResult: null,
|
|
45
|
+
appPath: config.appPath,
|
|
46
|
+
testsRoot: config.testsRoot,
|
|
47
|
+
gitSince: config.gitSince,
|
|
48
|
+
providerOverride: config.providerOverride,
|
|
49
|
+
impactedFlows: [],
|
|
50
|
+
strategyEntries: [],
|
|
51
|
+
testDesigns: [],
|
|
52
|
+
crossImpacts: [],
|
|
53
|
+
regressionRisks: [],
|
|
54
|
+
findings: [],
|
|
55
|
+
generatedSpecs: [],
|
|
56
|
+
usage: createEmptyUsageStats(),
|
|
57
|
+
messages: [],
|
|
58
|
+
warnings,
|
|
59
|
+
};
|
|
60
|
+
// Execute each phase
|
|
61
|
+
for (const phase of workflow.phases) {
|
|
62
|
+
const timer = logger.timer(`crew:${phase.name}`);
|
|
63
|
+
if (phase.handler === 'built-in') {
|
|
64
|
+
await this.runBuiltInPhase(phase.name, ctx, config);
|
|
65
|
+
}
|
|
66
|
+
else if (phase.parallel && phase.parallel.length > 0) {
|
|
67
|
+
await this.runParallel(phase.parallel, phase.name, ctx);
|
|
68
|
+
}
|
|
69
|
+
else if (phase.sequential && phase.sequential.length > 0) {
|
|
70
|
+
await this.runSequential(phase.sequential, phase.name, ctx);
|
|
71
|
+
}
|
|
72
|
+
else {
|
|
73
|
+
warnings.push(`Phase '${phase.name}' has no handler, parallel, or sequential agents — skipped.`);
|
|
74
|
+
}
|
|
75
|
+
timings[phase.name] = timer.end();
|
|
76
|
+
// Budget check
|
|
77
|
+
if (config.budgetUSD && ctx.usage.totalCost >= config.budgetUSD) {
|
|
78
|
+
warnings.push(`Budget limit reached ($${ctx.usage.totalCost.toFixed(4)} >= $${config.budgetUSD}). Stopping workflow.`);
|
|
79
|
+
break;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return { context: ctx, warnings, timings };
|
|
83
|
+
}
|
|
84
|
+
async dispatch(role, action, ctx) {
|
|
85
|
+
const agent = this.agents.get(role);
|
|
86
|
+
if (!agent) {
|
|
87
|
+
return {
|
|
88
|
+
role,
|
|
89
|
+
status: 'failed',
|
|
90
|
+
output: null,
|
|
91
|
+
warnings: [`Agent '${role}' is not registered.`],
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
const task = { role, action, input: null };
|
|
95
|
+
try {
|
|
96
|
+
const result = await agent.execute(task, ctx);
|
|
97
|
+
if (result.usage) {
|
|
98
|
+
mergeUsageStats(ctx.usage, result.usage);
|
|
99
|
+
}
|
|
100
|
+
if (result.warnings && result.warnings.length > 0) {
|
|
101
|
+
ctx.warnings.push(...result.warnings);
|
|
102
|
+
}
|
|
103
|
+
return result;
|
|
104
|
+
}
|
|
105
|
+
catch (error) {
|
|
106
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
107
|
+
ctx.warnings.push(`Agent '${role}' failed: ${message}`);
|
|
108
|
+
return { role, status: 'failed', output: null, warnings: [message] };
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
async parallel(roles, action, ctx) {
|
|
112
|
+
const promises = roles.map((role) => this.dispatch(role, action, ctx));
|
|
113
|
+
return Promise.all(promises);
|
|
114
|
+
}
|
|
115
|
+
async broadcast(msg, ctx) {
|
|
116
|
+
ctx.messages.push(msg);
|
|
117
|
+
const promises = [];
|
|
118
|
+
for (const agent of this.agents.values()) {
|
|
119
|
+
if (agent.onMessage && agent.role !== msg.from) {
|
|
120
|
+
promises.push(agent.onMessage(msg).catch((err) => {
|
|
121
|
+
ctx.warnings.push(`Broadcast to ${agent.role} failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
122
|
+
}));
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
await Promise.all(promises);
|
|
126
|
+
}
|
|
127
|
+
async runBuiltInPhase(name, ctx, config) {
|
|
128
|
+
if (name === 'preprocess') {
|
|
129
|
+
if (ctx.changedFiles.length === 0) {
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
const result = preprocess(ctx.changedFiles, {
|
|
133
|
+
appPath: config.appPath,
|
|
134
|
+
testsRoot: config.testsRoot,
|
|
135
|
+
routeFamilies: config.routeFamilies,
|
|
136
|
+
apiSurface: config.apiSurface,
|
|
137
|
+
});
|
|
138
|
+
ctx.preprocessResult = result;
|
|
139
|
+
ctx.manifest = result.manifest;
|
|
140
|
+
ctx.routeFamilies = result.manifest?.families || [];
|
|
141
|
+
ctx.apiSurface = result.apiSurface;
|
|
142
|
+
ctx.specIndex = result.specIndex;
|
|
143
|
+
ctx.context = result.context;
|
|
144
|
+
ctx.familyGroups = result.familyGroups;
|
|
145
|
+
ctx.warnings.push(...result.warnings);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
async runParallel(roles, phaseName, ctx) {
|
|
149
|
+
logger.info(`Crew phase '${phaseName}': running ${roles.join(', ')} in parallel`);
|
|
150
|
+
const results = await this.parallel(roles, phaseName, ctx);
|
|
151
|
+
this.checkPhaseResults(phaseName, results, ctx);
|
|
152
|
+
}
|
|
153
|
+
async runSequential(roles, phaseName, ctx) {
|
|
154
|
+
logger.info(`Crew phase '${phaseName}': running ${roles.join(' → ')} sequentially`);
|
|
155
|
+
const results = [];
|
|
156
|
+
for (const role of roles) {
|
|
157
|
+
results.push(await this.dispatch(role, phaseName, ctx));
|
|
158
|
+
}
|
|
159
|
+
this.checkPhaseResults(phaseName, results, ctx);
|
|
160
|
+
}
|
|
161
|
+
checkPhaseResults(phaseName, results, ctx) {
|
|
162
|
+
const allFailed = results.length > 0 && results.every((r) => r.status === 'failed');
|
|
163
|
+
if (allFailed) {
|
|
164
|
+
ctx.warnings.push(`Phase '${phaseName}': all ${results.length} agent(s) failed. Downstream phases may produce empty results.`);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
/**
|
|
4
|
+
* Shared provider creation for crew agents — ensures consistent provider
|
|
5
|
+
* instantiation and prevents usage stats fragmentation.
|
|
6
|
+
*/
|
|
7
|
+
import { LLMProviderFactory } from '../provider_factory.js';
|
|
8
|
+
export async function getCrewProvider(providerOverride) {
|
|
9
|
+
if (providerOverride) {
|
|
10
|
+
return LLMProviderFactory.createFromString(providerOverride);
|
|
11
|
+
}
|
|
12
|
+
return LLMProviderFactory.createFromEnv();
|
|
13
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
/**
|
|
4
|
+
* Sanitize strings before interpolating into LLM prompts.
|
|
5
|
+
* Strips common prompt injection patterns while preserving useful content.
|
|
6
|
+
*/
|
|
7
|
+
const INJECTION_PATTERNS = [
|
|
8
|
+
/ignore\s+(all\s+)?previous\s+instructions/gi,
|
|
9
|
+
/disregard\s+(all\s+)?(above|prior|previous)/gi,
|
|
10
|
+
/system\s*:\s*/gi,
|
|
11
|
+
/\[INST\]/gi,
|
|
12
|
+
/<<SYS>>/gi,
|
|
13
|
+
/<\|im_start\|>/gi,
|
|
14
|
+
/\bHuman\s*:\s*/gi,
|
|
15
|
+
/\bAssistant\s*:\s*/gi,
|
|
16
|
+
];
|
|
17
|
+
const MAX_FIELD_LENGTH = 2000;
|
|
18
|
+
export function sanitizeForPrompt(value) {
|
|
19
|
+
let sanitized = value.slice(0, MAX_FIELD_LENGTH);
|
|
20
|
+
for (const pattern of INJECTION_PATTERNS) {
|
|
21
|
+
sanitized = sanitized.replace(pattern, '[filtered]');
|
|
22
|
+
}
|
|
23
|
+
return sanitized;
|
|
24
|
+
}
|
|
25
|
+
export function sanitizeArray(values) {
|
|
26
|
+
return values.map((v) => sanitizeForPrompt(v));
|
|
27
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
export const WORKFLOWS = {
|
|
4
|
+
'full-qa': {
|
|
5
|
+
name: 'full-qa',
|
|
6
|
+
description: 'Full multi-agent QA analysis: understand → strategize → execute → validate',
|
|
7
|
+
phases: [
|
|
8
|
+
{ name: 'preprocess', handler: 'built-in' },
|
|
9
|
+
{ name: 'understand', parallel: ['impact-analyst', 'cross-impact', 'regression-advisor'] },
|
|
10
|
+
{ name: 'strategize', sequential: ['strategist', 'test-designer'] },
|
|
11
|
+
{ name: 'execute', parallel: ['generator'] },
|
|
12
|
+
{ name: 'validate', sequential: ['executor', 'healer'] },
|
|
13
|
+
],
|
|
14
|
+
},
|
|
15
|
+
'quick-check': {
|
|
16
|
+
name: 'quick-check',
|
|
17
|
+
description: 'Quick impact analysis with strategy recommendations',
|
|
18
|
+
phases: [
|
|
19
|
+
{ name: 'preprocess', handler: 'built-in' },
|
|
20
|
+
{ name: 'understand', parallel: ['impact-analyst'] },
|
|
21
|
+
{ name: 'strategize', sequential: ['strategist'] },
|
|
22
|
+
],
|
|
23
|
+
},
|
|
24
|
+
'design-only': {
|
|
25
|
+
name: 'design-only',
|
|
26
|
+
description: 'Impact analysis through test design — no generation or execution',
|
|
27
|
+
phases: [
|
|
28
|
+
{ name: 'preprocess', handler: 'built-in' },
|
|
29
|
+
{ name: 'understand', parallel: ['impact-analyst', 'cross-impact'] },
|
|
30
|
+
{ name: 'strategize', sequential: ['strategist', 'test-designer'] },
|
|
31
|
+
],
|
|
32
|
+
},
|
|
33
|
+
};
|
package/dist/esm/index.js
CHANGED
|
@@ -30,6 +30,20 @@ export { buildApiSurface, loadOrBuildApiSurface } from './knowledge/api_surface.
|
|
|
30
30
|
export { buildSpecIndex, getSpecsForFamily } from './knowledge/spec_index.js';
|
|
31
31
|
// Agentic generation
|
|
32
32
|
export { runAgenticGeneration } from './agentic/runner.js';
|
|
33
|
+
// Crew (multi-agent QA workflows)
|
|
34
|
+
export { CrewOrchestrator } from './crew/orchestrator.js';
|
|
35
|
+
export { WORKFLOWS } from './crew/workflows.js';
|
|
36
|
+
// Crew agents
|
|
37
|
+
export { ImpactAnalystAgent } from './agents/impact-analyst.js';
|
|
38
|
+
export { CoverageEvaluatorAgent } from './agents/coverage-evaluator.js';
|
|
39
|
+
export { GeneratorAgent } from './agents/generator.js';
|
|
40
|
+
export { ExecutorAgent } from './agents/executor.js';
|
|
41
|
+
export { HealerAgent } from './agents/healer.js';
|
|
42
|
+
export { ExplorerAgent } from './agents/explorer.js';
|
|
43
|
+
export { StrategistAgent } from './agents/strategist.js';
|
|
44
|
+
export { TestDesignerAgent } from './agents/test-designer.js';
|
|
45
|
+
export { CrossImpactAgent } from './agents/cross-impact.js';
|
|
46
|
+
export { RegressionAdvisorAgent } from './agents/regression-advisor.js';
|
|
33
47
|
// Training (route-families bootstrap and maintenance)
|
|
34
48
|
export { scanProject } from './training/scanner.js';
|
|
35
49
|
export { mergeFamilies, detectStaleFamilies } from './training/merger.js';
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
import { sanitizeForPrompt } from '../crew/sanitize.js';
|
|
4
|
+
export function buildCrossImpactPrompt(ctx) {
|
|
5
|
+
const familiesBlock = ctx.families
|
|
6
|
+
.map((f) => {
|
|
7
|
+
const paths = [
|
|
8
|
+
...(f.webappPaths || []),
|
|
9
|
+
...(f.serverPaths || []),
|
|
10
|
+
...(f.components || []),
|
|
11
|
+
];
|
|
12
|
+
return `- ${f.id}: routes=[${f.routes.join(', ')}] paths=[${paths.join(', ')}] pageObjects=[${(f.pageObjects || []).join(', ')}]`;
|
|
13
|
+
})
|
|
14
|
+
.join('\n');
|
|
15
|
+
const changedBlock = ctx.changedFiles.map((f) => sanitizeForPrompt(f)).join('\n');
|
|
16
|
+
return [
|
|
17
|
+
'You are analyzing code changes in Mattermost to identify cross-family ripple effects.',
|
|
18
|
+
'When a change in one route family could affect another family through shared dependencies,',
|
|
19
|
+
'that is a cross-impact.',
|
|
20
|
+
'',
|
|
21
|
+
`CHANGED FILES (${ctx.changedFiles.length}):`,
|
|
22
|
+
changedBlock,
|
|
23
|
+
'',
|
|
24
|
+
`DIRECTLY IMPACTED FAMILIES: ${ctx.directlyImpactedFamilyIds.join(', ')}`,
|
|
25
|
+
'',
|
|
26
|
+
`ALL ROUTE FAMILIES (${ctx.families.length}):`,
|
|
27
|
+
familiesBlock,
|
|
28
|
+
'',
|
|
29
|
+
'TASK: Identify cross-family impacts. For each pair, explain the shared dependency.',
|
|
30
|
+
'',
|
|
31
|
+
'Look for:',
|
|
32
|
+
'1. Shared webapp paths (same component used by multiple families)',
|
|
33
|
+
'2. Shared page objects (same PO class referenced by multiple families)',
|
|
34
|
+
'3. Shared API endpoints (changes affecting data used by multiple families)',
|
|
35
|
+
'4. Shared components (React components imported across family boundaries)',
|
|
36
|
+
'5. Shared state management (Redux stores, contexts used across families)',
|
|
37
|
+
'',
|
|
38
|
+
'Return strict JSON only with this shape:',
|
|
39
|
+
'{"crossImpacts":[{"sourceFamily":"<directly impacted family>","affectedFamily":"<indirectly affected family>","sharedDependency":"<what connects them>","riskLevel":"high|medium|low","evidence":"<specific file/component/API that creates the dependency>"}]}',
|
|
40
|
+
'',
|
|
41
|
+
'Rules:',
|
|
42
|
+
'- Only report cross-impacts where both families are in the manifest.',
|
|
43
|
+
'- sourceFamily must be one of the directly impacted families.',
|
|
44
|
+
'- affectedFamily must be DIFFERENT from sourceFamily.',
|
|
45
|
+
'- Risk levels: high = shared data model/state, medium = shared UI component, low = shared utility.',
|
|
46
|
+
'- Evidence must cite specific files, components, or API paths.',
|
|
47
|
+
'- Return empty array if no cross-impacts are found.',
|
|
48
|
+
].join('\n');
|
|
49
|
+
}
|
|
50
|
+
export function parseCrossImpactResponse(text) {
|
|
51
|
+
const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/i);
|
|
52
|
+
const candidates = fenced ? [fenced[1], text] : [text];
|
|
53
|
+
for (const candidate of candidates) {
|
|
54
|
+
const start = candidate.indexOf('{');
|
|
55
|
+
const end = candidate.lastIndexOf('}');
|
|
56
|
+
if (start < 0 || end <= start) {
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
const raw = candidate.slice(start, end + 1);
|
|
60
|
+
try {
|
|
61
|
+
const parsed = JSON.parse(raw);
|
|
62
|
+
if (parsed && Array.isArray(parsed.crossImpacts)) {
|
|
63
|
+
return parsed;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
import { sanitizeForPrompt } from '../crew/sanitize.js';
|
|
4
|
+
export function buildStrategistPrompt(ctx) {
|
|
5
|
+
const flowsBlock = ctx.impactedFlows
|
|
6
|
+
.map((f) => {
|
|
7
|
+
const specs = f.existingSpecs.map((s) => `${s.path} (${s.coverageLevel})`).join(', ') || 'none';
|
|
8
|
+
return [
|
|
9
|
+
`- ${f.flowId} (${f.priority}): ${f.flowName}`,
|
|
10
|
+
` Route Family: ${f.routeFamily}`,
|
|
11
|
+
` Action: ${f.action}`,
|
|
12
|
+
` Confidence: ${f.confidence}%`,
|
|
13
|
+
` Existing Coverage: ${specs}`,
|
|
14
|
+
` User Actions: ${sanitizeForPrompt(f.userActions.join('; ') || 'unknown')}`,
|
|
15
|
+
` Changed Files: ${f.changedFiles.join(', ')}`,
|
|
16
|
+
].join('\n');
|
|
17
|
+
})
|
|
18
|
+
.join('\n\n');
|
|
19
|
+
const crossImpactBlock = ctx.crossImpacts.length > 0
|
|
20
|
+
? ctx.crossImpacts.map((ci) => `- ${ci.sourceFamily} → ${ci.affectedFamily} (${ci.riskLevel}): ${ci.sharedDependency} — ${ci.evidence}`).join('\n')
|
|
21
|
+
: 'No cross-family impacts detected.';
|
|
22
|
+
const regressionBlock = ctx.regressionRisks.length > 0
|
|
23
|
+
? ctx.regressionRisks.map((r) => `- ${r.familyId} (risk=${r.riskScore}): ${r.reason}`).join('\n')
|
|
24
|
+
: 'No regression risk data available.';
|
|
25
|
+
return [
|
|
26
|
+
'You are a senior QA strategist designing the overall test strategy for a code change.',
|
|
27
|
+
'',
|
|
28
|
+
`IMPACTED FLOWS (${ctx.impactedFlows.length}):`,
|
|
29
|
+
flowsBlock,
|
|
30
|
+
'',
|
|
31
|
+
'CROSS-FAMILY IMPACTS:',
|
|
32
|
+
crossImpactBlock,
|
|
33
|
+
'',
|
|
34
|
+
'REGRESSION RISK:',
|
|
35
|
+
regressionBlock,
|
|
36
|
+
'',
|
|
37
|
+
'TASK: Design a prioritized test strategy for each impacted flow.',
|
|
38
|
+
'',
|
|
39
|
+
'For each flow, decide:',
|
|
40
|
+
'1. Approach: full-test (comprehensive), smoke-test (critical path only), skip, or manual-review',
|
|
41
|
+
'2. Priority: P0 (critical path), P1 (important), P2 (nice to have)',
|
|
42
|
+
'3. Test categories to cover (from: happy-path, edge-case, boundary, negative, state-transition, race-condition, permission, accessibility, performance)',
|
|
43
|
+
'4. Cross-impact risk level based on shared dependencies',
|
|
44
|
+
'',
|
|
45
|
+
'Return strict JSON only with this shape:',
|
|
46
|
+
'{"strategy":[{"flowId":"<id>","flowName":"<name>","priority":"P0|P1|P2","approach":"full-test|smoke-test|skip|manual-review","rationale":"<why this approach>","testCategories":["happy-path","edge-case",...],"crossImpactRisk":"high|medium|low|none"}]}',
|
|
47
|
+
'',
|
|
48
|
+
'Rules:',
|
|
49
|
+
'- P0 flows with create_spec or add_scenarios action should always get full-test.',
|
|
50
|
+
'- Flows with high cross-impact risk should be promoted to at least P1.',
|
|
51
|
+
'- Flows with high regression risk should include edge-case and boundary categories.',
|
|
52
|
+
'- Skip flows only if confidence < 30 AND no cross-impact risk.',
|
|
53
|
+
'- Include accessibility category for any flow involving interactive UI elements.',
|
|
54
|
+
'- Include permission category for any flow involving role-based features.',
|
|
55
|
+
'- Keep rationale concise (1-2 sentences) explaining why this approach was chosen.',
|
|
56
|
+
].join('\n');
|
|
57
|
+
}
|
|
58
|
+
export function parseStrategistResponse(text) {
|
|
59
|
+
const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/i);
|
|
60
|
+
const candidates = fenced ? [fenced[1], text] : [text];
|
|
61
|
+
for (const candidate of candidates) {
|
|
62
|
+
const start = candidate.indexOf('{');
|
|
63
|
+
const end = candidate.lastIndexOf('}');
|
|
64
|
+
if (start < 0 || end <= start) {
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
const raw = candidate.slice(start, end + 1);
|
|
68
|
+
try {
|
|
69
|
+
const parsed = JSON.parse(raw);
|
|
70
|
+
if (parsed && Array.isArray(parsed.strategy)) {
|
|
71
|
+
return parsed;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return null;
|
|
79
|
+
}
|