@yasserkhanorg/e2e-agents 1.8.4 → 1.9.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +95 -8
- package/dist/adapters/cypress.d.ts +10 -0
- package/dist/adapters/cypress.d.ts.map +1 -0
- package/dist/adapters/cypress.js +86 -0
- package/dist/adapters/framework_adapter.d.ts +41 -0
- package/dist/adapters/framework_adapter.d.ts.map +1 -0
- package/dist/adapters/framework_adapter.js +152 -0
- package/dist/adapters/playwright.d.ts +10 -0
- package/dist/adapters/playwright.d.ts.map +1 -0
- package/dist/adapters/playwright.js +86 -0
- package/dist/adapters/pytest.d.ts +10 -0
- package/dist/adapters/pytest.d.ts.map +1 -0
- package/dist/adapters/pytest.js +96 -0
- package/dist/adapters/supertest.d.ts +12 -0
- package/dist/adapters/supertest.d.ts.map +1 -0
- package/dist/adapters/supertest.js +85 -0
- package/dist/agent/config.d.ts +1 -1
- package/dist/agent/config.d.ts.map +1 -1
- package/dist/agent/git.d.ts +1 -0
- package/dist/agent/git.d.ts.map +1 -1
- package/dist/agent/git.js +3 -0
- package/dist/agentic/fix_loop.d.ts.map +1 -1
- package/dist/agentic/fix_loop.js +5 -4
- package/dist/agentic/runner.d.ts +2 -0
- package/dist/agentic/runner.d.ts.map +1 -1
- package/dist/agentic/runner.js +15 -12
- package/dist/agents/cross-impact.d.ts.map +1 -1
- package/dist/agents/cross-impact.js +6 -1
- package/dist/agents/executor.d.ts.map +1 -1
- package/dist/agents/executor.js +6 -1
- package/dist/agents/strategist.d.ts.map +1 -1
- package/dist/agents/strategist.js +6 -1
- package/dist/agents/test-designer.d.ts.map +1 -1
- package/dist/agents/test-designer.js +6 -1
- package/dist/anthropic_provider.d.ts.map +1 -1
- package/dist/anthropic_provider.js +1 -0
- package/dist/base_provider.d.ts +56 -0
- package/dist/base_provider.d.ts.map +1 -1
- package/dist/base_provider.js +123 -1
- package/dist/budget_ledger.d.ts +28 -0
- package/dist/budget_ledger.d.ts.map +1 -0
- package/dist/budget_ledger.js +62 -0
- package/dist/cache/cached_provider.d.ts +45 -0
- package/dist/cache/cached_provider.d.ts.map +1 -0
- package/dist/cache/cached_provider.js +88 -0
- package/dist/cache/response_cache.d.ts +79 -0
- package/dist/cache/response_cache.d.ts.map +1 -0
- package/dist/cache/response_cache.js +177 -0
- package/dist/cli/commands/bootstrap.d.ts +3 -0
- package/dist/cli/commands/bootstrap.d.ts.map +1 -0
- package/dist/cli/commands/bootstrap.js +109 -0
- package/dist/cli/commands/cost_report.d.ts +3 -0
- package/dist/cli/commands/cost_report.d.ts.map +1 -0
- package/dist/cli/commands/cost_report.js +115 -0
- package/dist/cli/commands/crew.d.ts.map +1 -1
- package/dist/cli/commands/crew.js +118 -1
- package/dist/cli/commands/gate.d.ts +3 -0
- package/dist/cli/commands/gate.d.ts.map +1 -0
- package/dist/cli/commands/gate.js +86 -0
- package/dist/cli/commands/init.d.ts.map +1 -1
- package/dist/cli/commands/init.js +7 -62
- package/dist/cli/commands/plan_crew.d.ts.map +1 -1
- package/dist/cli/commands/plan_crew.js +33 -21
- package/dist/cli/commands/train.d.ts.map +1 -1
- package/dist/cli/commands/train.js +16 -21
- package/dist/cli/defaults.d.ts +35 -0
- package/dist/cli/defaults.d.ts.map +1 -0
- package/dist/cli/defaults.js +125 -0
- package/dist/cli/errors.d.ts +27 -0
- package/dist/cli/errors.d.ts.map +1 -0
- package/dist/cli/errors.js +57 -0
- package/dist/cli/parse_args.d.ts.map +1 -1
- package/dist/cli/parse_args.js +24 -2
- package/dist/cli/types.d.ts +7 -1
- package/dist/cli/types.d.ts.map +1 -1
- package/dist/cli.js +47 -2
- package/dist/crew/context.d.ts +15 -0
- package/dist/crew/context.d.ts.map +1 -1
- package/dist/crew/orchestrator.d.ts +14 -0
- package/dist/crew/orchestrator.d.ts.map +1 -1
- package/dist/crew/orchestrator.js +162 -4
- package/dist/crew/protocol.d.ts +13 -0
- package/dist/crew/protocol.d.ts.map +1 -1
- package/dist/crew/provider.d.ts +15 -1
- package/dist/crew/provider.d.ts.map +1 -1
- package/dist/crew/provider.js +24 -4
- package/dist/custom_provider.d.ts.map +1 -1
- package/dist/custom_provider.js +1 -0
- package/dist/engine/diff_loader.d.ts.map +1 -1
- package/dist/engine/diff_loader.js +3 -14
- package/dist/engine/impact_engine.d.ts.map +1 -1
- package/dist/engine/impact_engine.js +9 -23
- package/dist/esm/adapters/cypress.js +49 -0
- package/dist/esm/adapters/framework_adapter.js +114 -0
- package/dist/esm/adapters/playwright.js +49 -0
- package/dist/esm/adapters/pytest.js +59 -0
- package/dist/esm/adapters/supertest.js +48 -0
- package/dist/esm/agent/git.js +3 -1
- package/dist/esm/agentic/fix_loop.js +5 -4
- package/dist/esm/agentic/runner.js +15 -12
- package/dist/esm/agents/cross-impact.js +6 -1
- package/dist/esm/agents/executor.js +6 -1
- package/dist/esm/agents/strategist.js +6 -1
- package/dist/esm/agents/test-designer.js +6 -1
- package/dist/esm/anthropic_provider.js +1 -0
- package/dist/esm/base_provider.js +121 -0
- package/dist/esm/budget_ledger.js +58 -0
- package/dist/esm/cache/cached_provider.js +82 -0
- package/dist/esm/cache/response_cache.js +140 -0
- package/dist/esm/cli/commands/bootstrap.js +106 -0
- package/dist/esm/cli/commands/cost_report.js +112 -0
- package/dist/esm/cli/commands/crew.js +118 -1
- package/dist/esm/cli/commands/gate.js +83 -0
- package/dist/esm/cli/commands/init.js +3 -58
- package/dist/esm/cli/commands/plan_crew.js +33 -21
- package/dist/esm/cli/commands/train.js +16 -21
- package/dist/esm/cli/defaults.js +118 -0
- package/dist/esm/cli/errors.js +52 -0
- package/dist/esm/cli/parse_args.js +24 -2
- package/dist/esm/cli.js +47 -2
- package/dist/esm/crew/orchestrator.js +162 -4
- package/dist/esm/crew/provider.js +24 -4
- package/dist/esm/custom_provider.js +1 -0
- package/dist/esm/engine/diff_loader.js +1 -12
- package/dist/esm/engine/impact_engine.js +9 -23
- package/dist/esm/index.js +21 -0
- package/dist/esm/knowledge/cluster_utils.js +60 -0
- package/dist/esm/knowledge/kg_bridge.js +381 -0
- package/dist/esm/knowledge/kg_types.js +3 -0
- package/dist/esm/knowledge/route_families.js +89 -0
- package/dist/esm/mcp-server.js +2 -4
- package/dist/esm/metrics/prometheus.js +149 -0
- package/dist/esm/model_router.js +59 -0
- package/dist/esm/ollama_provider.js +1 -0
- package/dist/esm/openai_provider.js +1 -0
- package/dist/esm/pipeline/orchestrator.js +6 -12
- package/dist/esm/pipeline/stage0_preprocess.js +12 -19
- package/dist/esm/pipeline/stage2_coverage.js +1 -0
- package/dist/esm/pipeline/stage3_generation.js +1 -0
- package/dist/esm/progress.js +112 -0
- package/dist/esm/prompts/coverage.js +7 -24
- package/dist/esm/prompts/cross-impact.js +3 -21
- package/dist/esm/prompts/generation.js +158 -36
- package/dist/esm/prompts/generation_profile.js +147 -0
- package/dist/esm/prompts/heal.js +33 -15
- package/dist/esm/prompts/impact.js +3 -22
- package/dist/esm/prompts/json_extract.js +36 -0
- package/dist/esm/prompts/strategist.js +2 -20
- package/dist/esm/prompts/test-designer.js +6 -21
- package/dist/esm/provider_factory.js +6 -4
- package/dist/esm/reporters/junit.js +86 -0
- package/dist/esm/reporters/reporter.js +3 -0
- package/dist/esm/reporters/sarif.js +131 -0
- package/dist/esm/resilience/circuit_breaker.js +78 -0
- package/dist/esm/resilience/retry.js +56 -0
- package/dist/esm/sanitize.js +66 -0
- package/dist/esm/training/kg_scanner.js +115 -0
- package/dist/esm/training/scanner.js +27 -34
- package/dist/esm/version.js +33 -0
- package/dist/index.d.ts +21 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +45 -1
- package/dist/knowledge/cluster_utils.d.ts +28 -0
- package/dist/knowledge/cluster_utils.d.ts.map +1 -0
- package/dist/knowledge/cluster_utils.js +67 -0
- package/dist/knowledge/kg_bridge.d.ts +31 -0
- package/dist/knowledge/kg_bridge.d.ts.map +1 -0
- package/dist/knowledge/kg_bridge.js +388 -0
- package/dist/knowledge/kg_types.d.ts +75 -0
- package/dist/knowledge/kg_types.d.ts.map +1 -0
- package/dist/knowledge/kg_types.js +4 -0
- package/dist/knowledge/route_families.d.ts +18 -0
- package/dist/knowledge/route_families.d.ts.map +1 -1
- package/dist/knowledge/route_families.js +91 -0
- package/dist/mcp-server.d.ts.map +1 -1
- package/dist/mcp-server.js +2 -4
- package/dist/metrics/prometheus.d.ts +37 -0
- package/dist/metrics/prometheus.d.ts.map +1 -0
- package/dist/metrics/prometheus.js +153 -0
- package/dist/model_router.d.ts +28 -0
- package/dist/model_router.d.ts.map +1 -0
- package/dist/model_router.js +63 -0
- package/dist/ollama_provider.d.ts.map +1 -1
- package/dist/ollama_provider.js +1 -0
- package/dist/openai_provider.d.ts.map +1 -1
- package/dist/openai_provider.js +1 -0
- package/dist/pipeline/orchestrator.d.ts +2 -0
- package/dist/pipeline/orchestrator.d.ts.map +1 -1
- package/dist/pipeline/orchestrator.js +6 -12
- package/dist/pipeline/stage0_preprocess.d.ts.map +1 -1
- package/dist/pipeline/stage0_preprocess.js +11 -18
- package/dist/pipeline/stage2_coverage.d.ts +2 -0
- package/dist/pipeline/stage2_coverage.d.ts.map +1 -1
- package/dist/pipeline/stage2_coverage.js +1 -0
- package/dist/pipeline/stage3_generation.d.ts +2 -0
- package/dist/pipeline/stage3_generation.d.ts.map +1 -1
- package/dist/pipeline/stage3_generation.js +1 -0
- package/dist/pipeline/stage4_heal.d.ts +2 -0
- package/dist/pipeline/stage4_heal.d.ts.map +1 -1
- package/dist/progress.d.ts +22 -0
- package/dist/progress.d.ts.map +1 -0
- package/dist/progress.js +116 -0
- package/dist/prompts/coverage.d.ts +2 -0
- package/dist/prompts/coverage.d.ts.map +1 -1
- package/dist/prompts/coverage.js +7 -24
- package/dist/prompts/cross-impact.d.ts +1 -0
- package/dist/prompts/cross-impact.d.ts.map +1 -1
- package/dist/prompts/cross-impact.js +3 -21
- package/dist/prompts/generation.d.ts +3 -1
- package/dist/prompts/generation.d.ts.map +1 -1
- package/dist/prompts/generation.js +158 -36
- package/dist/prompts/generation_profile.d.ts +29 -0
- package/dist/prompts/generation_profile.d.ts.map +1 -0
- package/dist/prompts/generation_profile.js +151 -0
- package/dist/prompts/heal.d.ts +3 -1
- package/dist/prompts/heal.d.ts.map +1 -1
- package/dist/prompts/heal.js +33 -15
- package/dist/prompts/impact.d.ts +1 -0
- package/dist/prompts/impact.d.ts.map +1 -1
- package/dist/prompts/impact.js +3 -22
- package/dist/prompts/json_extract.d.ts +14 -0
- package/dist/prompts/json_extract.d.ts.map +1 -0
- package/dist/prompts/json_extract.js +39 -0
- package/dist/prompts/strategist.d.ts.map +1 -1
- package/dist/prompts/strategist.js +2 -20
- package/dist/prompts/test-designer.d.ts +2 -0
- package/dist/prompts/test-designer.d.ts.map +1 -1
- package/dist/prompts/test-designer.js +6 -21
- package/dist/provider_factory.d.ts.map +1 -1
- package/dist/provider_factory.js +6 -4
- package/dist/reporters/junit.d.ts +6 -0
- package/dist/reporters/junit.d.ts.map +1 -0
- package/dist/reporters/junit.js +89 -0
- package/dist/reporters/reporter.d.ts +42 -0
- package/dist/reporters/reporter.d.ts.map +1 -0
- package/dist/reporters/reporter.js +4 -0
- package/dist/reporters/sarif.d.ts +7 -0
- package/dist/reporters/sarif.d.ts.map +1 -0
- package/dist/reporters/sarif.js +134 -0
- package/dist/resilience/circuit_breaker.d.ts +36 -0
- package/dist/resilience/circuit_breaker.d.ts.map +1 -0
- package/dist/resilience/circuit_breaker.js +82 -0
- package/dist/resilience/retry.d.ts +11 -0
- package/dist/resilience/retry.d.ts.map +1 -0
- package/dist/resilience/retry.js +59 -0
- package/dist/sanitize.d.ts +15 -0
- package/dist/sanitize.d.ts.map +1 -0
- package/dist/sanitize.js +71 -0
- package/dist/training/kg_scanner.d.ts +13 -0
- package/dist/training/kg_scanner.d.ts.map +1 -0
- package/dist/training/kg_scanner.js +118 -0
- package/dist/training/scanner.d.ts +7 -2
- package/dist/training/scanner.d.ts.map +1 -1
- package/dist/training/scanner.js +27 -34
- package/dist/version.d.ts +6 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +36 -0
- package/package.json +7 -2
- package/schemas/route-families.schema.json +31 -1
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
import { ResponseCache, TTL } from './response_cache.js';
|
|
4
|
+
/**
|
|
5
|
+
* Decorator that adds transparent response caching to any LLMProvider.
|
|
6
|
+
*
|
|
7
|
+
* - `generateText()` checks the cache first and returns a cached response on hit.
|
|
8
|
+
* On a miss it delegates to the inner provider, stores the result, and returns it.
|
|
9
|
+
* - All other methods (analyzeImage, streamText, capabilities, usage stats)
|
|
10
|
+
* delegate directly to the wrapped provider.
|
|
11
|
+
*
|
|
12
|
+
* The TTL is selected based on the agent role: agents whose name contains
|
|
13
|
+
* "generat" use the shorter GENERATION TTL; all others use ANALYSIS.
|
|
14
|
+
*/
|
|
15
|
+
export class CachedProvider {
|
|
16
|
+
constructor(inner, cache, cacheContext) {
|
|
17
|
+
this.inner = inner;
|
|
18
|
+
this.cache = cache;
|
|
19
|
+
this.ctx = cacheContext;
|
|
20
|
+
this.name = inner.name;
|
|
21
|
+
this.capabilities = inner.capabilities;
|
|
22
|
+
// Pick TTL based on agent role
|
|
23
|
+
this.ttlMs = cacheContext.agent.toLowerCase().includes('generat')
|
|
24
|
+
? TTL.GENERATION
|
|
25
|
+
: TTL.ANALYSIS;
|
|
26
|
+
// Wire optional methods only when the inner provider supports them
|
|
27
|
+
if (inner.analyzeImage) {
|
|
28
|
+
this.analyzeImage = (images, prompt, options) => inner.analyzeImage(images, prompt, options);
|
|
29
|
+
}
|
|
30
|
+
if (inner.streamText) {
|
|
31
|
+
this.streamText = (prompt, options) => inner.streamText(prompt, options);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Generate text with cache-through semantics.
|
|
36
|
+
* On a cache hit the inner provider is never called, saving tokens and latency.
|
|
37
|
+
*/
|
|
38
|
+
async generateText(prompt, options) {
|
|
39
|
+
const { agent, family, fileHashes } = this.ctx;
|
|
40
|
+
const model = this.inner.name;
|
|
41
|
+
// Check cache
|
|
42
|
+
const cached = this.cache.get(agent, family, fileHashes, model);
|
|
43
|
+
if (cached) {
|
|
44
|
+
return {
|
|
45
|
+
text: cached.response,
|
|
46
|
+
usage: {
|
|
47
|
+
inputTokens: cached.usage.inputTokens,
|
|
48
|
+
outputTokens: cached.usage.outputTokens,
|
|
49
|
+
totalTokens: cached.usage.inputTokens + cached.usage.outputTokens,
|
|
50
|
+
cachedTokens: cached.usage.inputTokens,
|
|
51
|
+
},
|
|
52
|
+
cost: 0, // No cost on cache hit
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
// Cache miss - call inner provider
|
|
56
|
+
const response = await this.inner.generateText(prompt, options);
|
|
57
|
+
// Store in cache
|
|
58
|
+
const key = ResponseCache.buildKey({ agent, family, fileHashes, model });
|
|
59
|
+
const entry = {
|
|
60
|
+
key,
|
|
61
|
+
family,
|
|
62
|
+
response: response.text,
|
|
63
|
+
usage: {
|
|
64
|
+
inputTokens: response.usage.inputTokens,
|
|
65
|
+
outputTokens: response.usage.outputTokens,
|
|
66
|
+
cost: response.cost,
|
|
67
|
+
},
|
|
68
|
+
createdAt: new Date().toISOString(),
|
|
69
|
+
ttlMs: this.ttlMs,
|
|
70
|
+
};
|
|
71
|
+
this.cache.set(entry);
|
|
72
|
+
return response;
|
|
73
|
+
}
|
|
74
|
+
getUsageStats() {
|
|
75
|
+
return this.inner.getUsageStats();
|
|
76
|
+
}
|
|
77
|
+
resetUsageStats() {
|
|
78
|
+
this.inner.resetUsageStats();
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
// Re-export for convenience
|
|
82
|
+
export { ResponseCache, TTL } from './response_cache.js';
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
import * as fs from 'node:fs';
|
|
4
|
+
import * as path from 'node:path';
|
|
5
|
+
import * as crypto from 'node:crypto';
|
|
6
|
+
/**
|
|
7
|
+
* TTL presets for different cache entry types.
|
|
8
|
+
*/
|
|
9
|
+
export const TTL = {
|
|
10
|
+
/** 24 hours - for analysis results that change infrequently */
|
|
11
|
+
ANALYSIS: 24 * 60 * 60 * 1000,
|
|
12
|
+
/** 1 hour - for generated content that may need fresher context */
|
|
13
|
+
GENERATION: 1 * 60 * 60 * 1000,
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* Cross-run LLM response cache backed by JSON files.
|
|
17
|
+
*
|
|
18
|
+
* Stores entries as `{cacheDir}/{sha256}.json` using a content-addressed key
|
|
19
|
+
* derived from (agentRole + familyName + sorted file hashes + model).
|
|
20
|
+
*/
|
|
21
|
+
export class ResponseCache {
|
|
22
|
+
constructor(workspaceRoot) {
|
|
23
|
+
this.cacheDir = path.join(workspaceRoot, '.e2e-ai-agents', 'cache');
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Build a deterministic SHA-256 cache key from the provided parameters.
|
|
27
|
+
*/
|
|
28
|
+
static buildKey(params) {
|
|
29
|
+
const sorted = [...params.fileHashes].sort();
|
|
30
|
+
const payload = params.agent + params.family + JSON.stringify(sorted) + params.model;
|
|
31
|
+
return crypto.createHash('sha256').update(payload).digest('hex');
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Retrieve a cached response if it exists and has not expired.
|
|
35
|
+
* Returns `null` on cache miss or expiry.
|
|
36
|
+
*/
|
|
37
|
+
get(agent, family, fileHashes, model) {
|
|
38
|
+
const key = ResponseCache.buildKey({ agent, family, fileHashes, model });
|
|
39
|
+
const filePath = path.join(this.cacheDir, `${key}.json`);
|
|
40
|
+
try {
|
|
41
|
+
if (!fs.existsSync(filePath)) {
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
const raw = fs.readFileSync(filePath, 'utf-8');
|
|
45
|
+
const entry = JSON.parse(raw);
|
|
46
|
+
const age = Date.now() - new Date(entry.createdAt).getTime();
|
|
47
|
+
if (age > entry.ttlMs) {
|
|
48
|
+
// Expired - clean up eagerly
|
|
49
|
+
fs.unlinkSync(filePath);
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
return entry;
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
// Corrupted or unreadable - treat as miss
|
|
56
|
+
return null;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Write a cache entry to disk.
|
|
61
|
+
* Creates the cache directory if it does not yet exist.
|
|
62
|
+
*/
|
|
63
|
+
set(entry) {
|
|
64
|
+
this.ensureCacheDir();
|
|
65
|
+
const filePath = path.join(this.cacheDir, `${entry.key}.json`);
|
|
66
|
+
fs.writeFileSync(filePath, JSON.stringify(entry, null, 2), 'utf-8');
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Remove all cache entries belonging to the given family.
|
|
70
|
+
*
|
|
71
|
+
* Because the cache key is a one-way SHA-256 hash, we scan each file and
|
|
72
|
+
* check its stored `family` field. The directory is scoped to a single
|
|
73
|
+
* workspace so the scan is bounded.
|
|
74
|
+
*/
|
|
75
|
+
invalidateFamily(familyName) {
|
|
76
|
+
if (!fs.existsSync(this.cacheDir)) {
|
|
77
|
+
return 0;
|
|
78
|
+
}
|
|
79
|
+
let deleted = 0;
|
|
80
|
+
const files = fs.readdirSync(this.cacheDir).filter((f) => f.endsWith('.json'));
|
|
81
|
+
for (const file of files) {
|
|
82
|
+
const filePath = path.join(this.cacheDir, file);
|
|
83
|
+
try {
|
|
84
|
+
const raw = fs.readFileSync(filePath, 'utf-8');
|
|
85
|
+
const entry = JSON.parse(raw);
|
|
86
|
+
if (entry.family === familyName) {
|
|
87
|
+
fs.unlinkSync(filePath);
|
|
88
|
+
deleted++;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
// Skip unreadable files
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return deleted;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Remove all expired entries from the cache directory.
|
|
99
|
+
* Returns the number of entries deleted.
|
|
100
|
+
*/
|
|
101
|
+
prune() {
|
|
102
|
+
if (!fs.existsSync(this.cacheDir)) {
|
|
103
|
+
return 0;
|
|
104
|
+
}
|
|
105
|
+
let deleted = 0;
|
|
106
|
+
const now = Date.now();
|
|
107
|
+
const files = fs.readdirSync(this.cacheDir).filter((f) => f.endsWith('.json'));
|
|
108
|
+
for (const file of files) {
|
|
109
|
+
const filePath = path.join(this.cacheDir, file);
|
|
110
|
+
try {
|
|
111
|
+
const raw = fs.readFileSync(filePath, 'utf-8');
|
|
112
|
+
const entry = JSON.parse(raw);
|
|
113
|
+
const age = now - new Date(entry.createdAt).getTime();
|
|
114
|
+
if (age > entry.ttlMs) {
|
|
115
|
+
fs.unlinkSync(filePath);
|
|
116
|
+
deleted++;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
catch {
|
|
120
|
+
// Corrupted file - remove it
|
|
121
|
+
try {
|
|
122
|
+
fs.unlinkSync(filePath);
|
|
123
|
+
deleted++;
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
// Ignore if removal also fails
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
return deleted;
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Ensure the cache directory exists on disk.
|
|
134
|
+
*/
|
|
135
|
+
ensureCacheDir() {
|
|
136
|
+
if (!fs.existsSync(this.cacheDir)) {
|
|
137
|
+
fs.mkdirSync(this.cacheDir, { recursive: true });
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
/**
|
|
4
|
+
* Bootstrap command — takes a project with an Understand-Anything knowledge graph
|
|
5
|
+
* and generates route-families.json + initial test stubs.
|
|
6
|
+
*/
|
|
7
|
+
import { existsSync, mkdirSync, writeFileSync } from 'fs';
|
|
8
|
+
import { join, resolve } from 'path';
|
|
9
|
+
import { logger, LogLevel } from '../../logger.js';
|
|
10
|
+
import { loadKnowledgeGraph, classifyProjectType, transformKGToFamilies } from '../../knowledge/kg_bridge.js';
|
|
11
|
+
import { serializeManifest } from '../../knowledge/route_families.js';
|
|
12
|
+
import { detectFramework, detectTestMode } from '../../adapters/framework_adapter.js';
|
|
13
|
+
import { resolveGenerationProfile } from '../../prompts/generation_profile.js';
|
|
14
|
+
class BootstrapError extends Error {
|
|
15
|
+
constructor(message) {
|
|
16
|
+
super(message);
|
|
17
|
+
this.name = 'BootstrapError';
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
export async function runBootstrapCommand(args) {
|
|
21
|
+
const projectRoot = resolve(args.path || '.');
|
|
22
|
+
if (args.verbose)
|
|
23
|
+
logger.setLevel(LogLevel.DEBUG);
|
|
24
|
+
if (args.jsonOutput)
|
|
25
|
+
logger.setJsonMode(true);
|
|
26
|
+
logger.info('e2e-ai-agents bootstrap');
|
|
27
|
+
logger.info('=======================');
|
|
28
|
+
// ---------- Step 1: Check for knowledge graph ----------
|
|
29
|
+
const kgPath = args.bootstrapKgPath
|
|
30
|
+
? resolve(args.bootstrapKgPath)
|
|
31
|
+
: join(projectRoot, '.understand-anything', 'knowledge-graph.json');
|
|
32
|
+
if (!existsSync(kgPath)) {
|
|
33
|
+
throw new BootstrapError(`Knowledge graph not found at: ${kgPath}\n\n` +
|
|
34
|
+
'To bootstrap, first generate a knowledge graph for your project:\n' +
|
|
35
|
+
' 1. Install Understand-Anything: npm install -g understand-anything\n' +
|
|
36
|
+
' 2. Run: understand-anything analyze .\n' +
|
|
37
|
+
' 3. Then run: e2e-ai-agents bootstrap\n\n' +
|
|
38
|
+
'Or provide a path: e2e-ai-agents bootstrap --kg-path /path/to/knowledge-graph.json');
|
|
39
|
+
}
|
|
40
|
+
// ---------- Step 2: Load KG and classify ----------
|
|
41
|
+
logger.info('Loading knowledge graph...');
|
|
42
|
+
const kg = loadKnowledgeGraph(projectRoot, args.bootstrapKgPath ? kgPath : undefined);
|
|
43
|
+
if (!kg) {
|
|
44
|
+
throw new BootstrapError('Failed to load knowledge graph. Ensure it is valid JSON with nodes and edges arrays.');
|
|
45
|
+
}
|
|
46
|
+
const projectType = classifyProjectType(kg);
|
|
47
|
+
logger.info(`Project: ${kg.project.name || '(unnamed)'}`);
|
|
48
|
+
logger.info(`Type: ${projectType}`);
|
|
49
|
+
logger.info(`Frameworks: ${kg.project.frameworks.join(', ')}`);
|
|
50
|
+
logger.info(`Languages: ${kg.project.languages.join(', ')}`);
|
|
51
|
+
logger.info(`Nodes: ${kg.nodes.length}, Edges: ${kg.edges.length}`);
|
|
52
|
+
// ---------- Step 3: Transform KG to route families ----------
|
|
53
|
+
logger.info('');
|
|
54
|
+
logger.info('Generating route families from knowledge graph...');
|
|
55
|
+
const manifest = transformKGToFamilies(kg);
|
|
56
|
+
const maxFamilies = args.bootstrapMaxFamilies || 50;
|
|
57
|
+
if (manifest.families.length > maxFamilies) {
|
|
58
|
+
logger.info(`Limiting to top ${maxFamilies} families (of ${manifest.families.length} discovered). Use --max-families to adjust.`);
|
|
59
|
+
manifest.families = manifest.families.slice(0, maxFamilies);
|
|
60
|
+
}
|
|
61
|
+
const p0Count = manifest.families.filter((f) => f.priority === 'P0').length;
|
|
62
|
+
const p1Count = manifest.families.filter((f) => f.priority === 'P1').length;
|
|
63
|
+
const p2Count = manifest.families.filter((f) => f.priority === 'P2').length;
|
|
64
|
+
logger.info(`Discovered ${manifest.families.length} families: ${p0Count} P0, ${p1Count} P1, ${p2Count} P2`);
|
|
65
|
+
// ---------- Step 4: Detect/scaffold test framework ----------
|
|
66
|
+
const framework = detectFramework(projectRoot);
|
|
67
|
+
const testMode = args.bootstrapTestMode || detectTestMode(projectRoot, kg);
|
|
68
|
+
const profile = resolveGenerationProfile({ profile: args.profile, testMode }, kg);
|
|
69
|
+
logger.info(`Test framework: ${framework.name}`);
|
|
70
|
+
logger.info(`Test mode: ${testMode}`);
|
|
71
|
+
logger.info(`Generation profile: ${profile.projectName} (${profile.testFramework})`);
|
|
72
|
+
// ---------- Step 5: Write route-families.json ----------
|
|
73
|
+
const outputDir = join(projectRoot, '.e2e-ai-agents');
|
|
74
|
+
const outputPath = join(outputDir, 'route-families.json');
|
|
75
|
+
if (args.dryRun) {
|
|
76
|
+
logger.info('');
|
|
77
|
+
logger.info('Dry run — proposed manifest:');
|
|
78
|
+
console.log(serializeManifest(manifest));
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
if (!existsSync(outputDir)) {
|
|
82
|
+
mkdirSync(outputDir, { recursive: true });
|
|
83
|
+
}
|
|
84
|
+
writeFileSync(outputPath, serializeManifest(manifest), 'utf-8');
|
|
85
|
+
logger.info(`Wrote ${outputPath}`);
|
|
86
|
+
}
|
|
87
|
+
// ---------- Step 6: Summary and next steps ----------
|
|
88
|
+
logger.info('');
|
|
89
|
+
logger.info('Bootstrap complete!');
|
|
90
|
+
logger.info('');
|
|
91
|
+
logger.info('Route families summary:');
|
|
92
|
+
for (const family of manifest.families.slice(0, 15)) {
|
|
93
|
+
const endpoints = family.apiEndpoints?.length || 0;
|
|
94
|
+
const endpointSuffix = endpoints > 0 ? ` (${endpoints} API endpoints)` : '';
|
|
95
|
+
logger.info(` ${family.priority || 'P2'} ${family.id}: ${family.routes.join(', ')}${endpointSuffix}`);
|
|
96
|
+
}
|
|
97
|
+
if (manifest.families.length > 15) {
|
|
98
|
+
logger.info(` ... and ${manifest.families.length - 15} more`);
|
|
99
|
+
}
|
|
100
|
+
logger.info('');
|
|
101
|
+
logger.info('Next steps:');
|
|
102
|
+
logger.info(' 1. Review and refine .e2e-ai-agents/route-families.json');
|
|
103
|
+
logger.info(' 2. Run `e2e-ai-agents train --enrich` to add LLM-enriched metadata');
|
|
104
|
+
logger.info(' 3. Run `e2e-ai-agents plan` to see what tests are needed');
|
|
105
|
+
logger.info(' 4. Run `e2e-ai-agents generate` to create test stubs');
|
|
106
|
+
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
/**
|
|
4
|
+
* CLI command: cost-report — displays LLM cost breakdown from metrics.
|
|
5
|
+
*/
|
|
6
|
+
import { existsSync, readFileSync } from 'fs';
|
|
7
|
+
import { join } from 'path';
|
|
8
|
+
function parseMetricsFile(filePath) {
|
|
9
|
+
if (!existsSync(filePath)) {
|
|
10
|
+
return [];
|
|
11
|
+
}
|
|
12
|
+
const lines = readFileSync(filePath, 'utf-8').split('\n');
|
|
13
|
+
const events = [];
|
|
14
|
+
for (const line of lines) {
|
|
15
|
+
const trimmed = line.trim();
|
|
16
|
+
if (!trimmed)
|
|
17
|
+
continue;
|
|
18
|
+
try {
|
|
19
|
+
const parsed = JSON.parse(trimmed);
|
|
20
|
+
if (parsed.type === 'crew-run') {
|
|
21
|
+
events.push(parsed);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
continue;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
return events;
|
|
29
|
+
}
|
|
30
|
+
function filterByDays(events, days) {
|
|
31
|
+
const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
|
|
32
|
+
return events.filter((e) => new Date(e.timestamp).getTime() >= cutoff);
|
|
33
|
+
}
|
|
34
|
+
export function runCostReportCommand(args) {
|
|
35
|
+
const reportRoot = args.path || args.testsRoot || process.cwd();
|
|
36
|
+
const metricsPath = join(reportRoot, '.e2e-ai-agents', 'metrics.jsonl');
|
|
37
|
+
const days = 30; // Default; could be added as a CLI flag later
|
|
38
|
+
const allEvents = parseMetricsFile(metricsPath);
|
|
39
|
+
const events = filterByDays(allEvents, days);
|
|
40
|
+
if (events.length === 0) {
|
|
41
|
+
console.log('No crew metrics found.');
|
|
42
|
+
if (!existsSync(metricsPath)) {
|
|
43
|
+
console.log(`Metrics file not found at: ${metricsPath}`);
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
console.log('Run `e2e-ai-agents crew` to generate cost data.');
|
|
47
|
+
}
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
// JSON output
|
|
51
|
+
if (args.jsonOutput) {
|
|
52
|
+
const report = buildReport(events, days);
|
|
53
|
+
console.log(JSON.stringify(report, null, 2));
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
// Human-readable output
|
|
57
|
+
const totalCost = events.reduce((sum, e) => sum + e.totalCost, 0);
|
|
58
|
+
const totalRuns = events.length;
|
|
59
|
+
console.log(`E2E Agents Cost Report (last ${days} days)`);
|
|
60
|
+
console.log('='.repeat(45));
|
|
61
|
+
console.log(`\nTotal: $${totalCost.toFixed(2)} across ${totalRuns} runs\n`);
|
|
62
|
+
// By workflow
|
|
63
|
+
const byWorkflow = new Map();
|
|
64
|
+
for (const e of events) {
|
|
65
|
+
const entry = byWorkflow.get(e.workflow) || { runs: 0, cost: 0 };
|
|
66
|
+
entry.runs++;
|
|
67
|
+
entry.cost += e.totalCost;
|
|
68
|
+
byWorkflow.set(e.workflow, entry);
|
|
69
|
+
}
|
|
70
|
+
console.log('By workflow:');
|
|
71
|
+
for (const [workflow, data] of [...byWorkflow.entries()].sort((a, b) => b[1].cost - a[1].cost)) {
|
|
72
|
+
const avg = data.cost / data.runs;
|
|
73
|
+
console.log(` ${workflow.padEnd(14)} | ${String(data.runs).padStart(3)} runs | $${data.cost.toFixed(2).padStart(6)} | avg $${avg.toFixed(2)}/run`);
|
|
74
|
+
}
|
|
75
|
+
// By agent (top 5)
|
|
76
|
+
const byAgent = new Map();
|
|
77
|
+
for (const e of events) {
|
|
78
|
+
for (const au of e.agentUsage) {
|
|
79
|
+
byAgent.set(au.agent, (byAgent.get(au.agent) || 0) + au.cost);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
const sortedAgents = [...byAgent.entries()].sort((a, b) => b[1] - a[1]).slice(0, 5);
|
|
83
|
+
if (sortedAgents.length > 0) {
|
|
84
|
+
console.log('\nBy agent (top 5):');
|
|
85
|
+
for (const [agent, cost] of sortedAgents) {
|
|
86
|
+
const pct = totalCost > 0 ? ((cost / totalCost) * 100).toFixed(0) : '0';
|
|
87
|
+
console.log(` ${agent.padEnd(20)} | $${cost.toFixed(2).padStart(6)} | ${pct.padStart(3)}%`);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
function buildReport(events, days) {
|
|
92
|
+
const totalCost = events.reduce((sum, e) => sum + e.totalCost, 0);
|
|
93
|
+
const byWorkflow = {};
|
|
94
|
+
const byAgent = {};
|
|
95
|
+
for (const e of events) {
|
|
96
|
+
if (!byWorkflow[e.workflow]) {
|
|
97
|
+
byWorkflow[e.workflow] = { runs: 0, cost: 0 };
|
|
98
|
+
}
|
|
99
|
+
byWorkflow[e.workflow].runs++;
|
|
100
|
+
byWorkflow[e.workflow].cost += e.totalCost;
|
|
101
|
+
for (const au of e.agentUsage) {
|
|
102
|
+
byAgent[au.agent] = (byAgent[au.agent] || 0) + au.cost;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return {
|
|
106
|
+
days,
|
|
107
|
+
totalRuns: events.length,
|
|
108
|
+
totalCost,
|
|
109
|
+
byWorkflow,
|
|
110
|
+
byAgent,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
@@ -3,8 +3,12 @@
|
|
|
3
3
|
/**
|
|
4
4
|
* CLI command: crew — runs multi-agent QA analysis workflows.
|
|
5
5
|
*/
|
|
6
|
+
import { appendFileSync, mkdirSync } from 'fs';
|
|
7
|
+
import { join } from 'path';
|
|
6
8
|
import { resolveConfig } from '../../agent/config.js';
|
|
7
9
|
import { CrewOrchestrator } from '../../crew/orchestrator.js';
|
|
10
|
+
import { ResponseCache } from '../../cache/response_cache.js';
|
|
11
|
+
import { WORKFLOWS } from '../../crew/workflows.js';
|
|
8
12
|
import { ImpactAnalystAgent } from '../../agents/impact-analyst.js';
|
|
9
13
|
import { GeneratorAgent } from '../../agents/generator.js';
|
|
10
14
|
import { ExecutorAgent } from '../../agents/executor.js';
|
|
@@ -34,6 +38,23 @@ export async function runCrewCommand(args, autoConfig) {
|
|
|
34
38
|
process.exit(1);
|
|
35
39
|
}
|
|
36
40
|
const workflowName = rawWorkflow;
|
|
41
|
+
// Degraded mode: skip all AI features, deterministic analysis only
|
|
42
|
+
const degraded = args.degradedMode || process.env.E2E_AGENTS_DEGRADED === 'true';
|
|
43
|
+
if (degraded) {
|
|
44
|
+
console.log('Running in degraded mode — deterministic analysis only, no LLM calls.');
|
|
45
|
+
}
|
|
46
|
+
// Prune expired cache entries to prevent unbounded growth on CI
|
|
47
|
+
try {
|
|
48
|
+
const cache = new ResponseCache(testsRoot);
|
|
49
|
+
const pruned = cache.prune();
|
|
50
|
+
if (pruned > 0) {
|
|
51
|
+
console.log(`Cache: pruned ${pruned} expired entries.`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
catch (err) {
|
|
55
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
56
|
+
console.error(`Cache prune warning: ${msg}`);
|
|
57
|
+
}
|
|
37
58
|
const crewConfig = {
|
|
38
59
|
appPath: config.path,
|
|
39
60
|
testsRoot,
|
|
@@ -43,7 +64,7 @@ export async function runCrewCommand(args, autoConfig) {
|
|
|
43
64
|
workflow: workflowName,
|
|
44
65
|
providerOverride: args.llmProvider,
|
|
45
66
|
budgetUSD: args.budgetUSD,
|
|
46
|
-
dryRun: args.dryRun,
|
|
67
|
+
dryRun: degraded || args.dryRun,
|
|
47
68
|
};
|
|
48
69
|
// Create orchestrator and register all agents
|
|
49
70
|
const orchestrator = new CrewOrchestrator();
|
|
@@ -65,6 +86,33 @@ export async function runCrewCommand(args, autoConfig) {
|
|
|
65
86
|
process.exit(1);
|
|
66
87
|
}
|
|
67
88
|
const ctx = result.context;
|
|
89
|
+
// Dry-run output
|
|
90
|
+
if (result.dryRun) {
|
|
91
|
+
printDryRunOutput(result, workflowName, args.jsonOutput);
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
// Write crew metrics to metrics.jsonl for cost-report
|
|
95
|
+
if (ctx.usage.requestCount > 0) {
|
|
96
|
+
try {
|
|
97
|
+
const baseDir = join(testsRoot, '.e2e-ai-agents');
|
|
98
|
+
mkdirSync(baseDir, { recursive: true });
|
|
99
|
+
const metricsPath = join(baseDir, 'metrics.jsonl');
|
|
100
|
+
const crewMetric = {
|
|
101
|
+
type: 'crew-run',
|
|
102
|
+
timestamp: new Date().toISOString(),
|
|
103
|
+
workflow: workflowName,
|
|
104
|
+
totalCost: ctx.usage.totalCost,
|
|
105
|
+
totalTokens: ctx.usage.totalTokens,
|
|
106
|
+
totalInputTokens: ctx.usage.totalInputTokens,
|
|
107
|
+
totalOutputTokens: ctx.usage.totalOutputTokens,
|
|
108
|
+
agentUsage: ctx.agentUsage,
|
|
109
|
+
};
|
|
110
|
+
appendFileSync(metricsPath, `${JSON.stringify(crewMetric)}\n`, 'utf-8');
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
// Non-fatal: metrics writing should not break the workflow
|
|
114
|
+
}
|
|
115
|
+
}
|
|
68
116
|
// JSON output mode
|
|
69
117
|
if (args.jsonOutput) {
|
|
70
118
|
const jsonReport = {
|
|
@@ -132,3 +180,72 @@ export async function runCrewCommand(args, autoConfig) {
|
|
|
132
180
|
}
|
|
133
181
|
}
|
|
134
182
|
}
|
|
183
|
+
function printDryRunOutput(result, workflowName, jsonOutput) {
|
|
184
|
+
const ctx = result.context;
|
|
185
|
+
const workflow = WORKFLOWS[workflowName];
|
|
186
|
+
if (jsonOutput) {
|
|
187
|
+
console.log(JSON.stringify({
|
|
188
|
+
dryRun: true,
|
|
189
|
+
workflow: workflowName,
|
|
190
|
+
changedFiles: ctx.changedFiles,
|
|
191
|
+
familyGroups: ctx.familyGroups.map((fg) => ({
|
|
192
|
+
familyId: fg.familyId,
|
|
193
|
+
featureId: fg.featureId,
|
|
194
|
+
files: fg.files,
|
|
195
|
+
})),
|
|
196
|
+
phases: workflow.phases.map((p) => ({
|
|
197
|
+
name: p.name,
|
|
198
|
+
agents: p.parallel || p.sequential || [],
|
|
199
|
+
})),
|
|
200
|
+
manifestSource: ctx.manifest?.source || 'none',
|
|
201
|
+
warnings: result.warnings,
|
|
202
|
+
}, null, 2));
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
console.log('Dry run — no LLM calls will be made.\n');
|
|
206
|
+
console.log(`Changed files (${ctx.changedFiles.length}):`);
|
|
207
|
+
for (const f of ctx.changedFiles.slice(0, 20)) {
|
|
208
|
+
console.log(` ${f}`);
|
|
209
|
+
}
|
|
210
|
+
if (ctx.changedFiles.length > 20) {
|
|
211
|
+
console.log(` ... and ${ctx.changedFiles.length - 20} more`);
|
|
212
|
+
}
|
|
213
|
+
console.log(`\nAffected families (${ctx.familyGroups.length}):`);
|
|
214
|
+
for (const fg of ctx.familyGroups) {
|
|
215
|
+
const label = fg.featureId ? `${fg.familyId}/${fg.featureId}` : fg.familyId;
|
|
216
|
+
console.log(` ${label} (${fg.files.length} files)`);
|
|
217
|
+
}
|
|
218
|
+
if (ctx.manifest?.source === 'heuristic') {
|
|
219
|
+
console.log('\n Note: Using directory-based heuristics. Run `e2e-ai-agents train` for better accuracy.');
|
|
220
|
+
}
|
|
221
|
+
console.log(`\nWorkflow: ${workflowName}`);
|
|
222
|
+
const phaseNames = workflow.phases
|
|
223
|
+
.map((p) => {
|
|
224
|
+
const agents = p.parallel || p.sequential || [];
|
|
225
|
+
return agents.length > 0 ? `${p.name} (${agents.join(', ')})` : p.name;
|
|
226
|
+
})
|
|
227
|
+
.join(' → ');
|
|
228
|
+
console.log(`Phases: ${phaseNames}`);
|
|
229
|
+
// Cost estimation based on workflow and family count
|
|
230
|
+
const familyCount = Math.max(ctx.familyGroups.length, 1);
|
|
231
|
+
const agentCount = workflow.phases.reduce((sum, p) => sum + (p.parallel?.length || 0) + (p.sequential?.length || 0), 0);
|
|
232
|
+
const costEstimate = estimateCost(workflowName, familyCount, agentCount);
|
|
233
|
+
console.log(`\nEstimated cost: $${costEstimate.low.toFixed(2)}-$${costEstimate.high.toFixed(2)}`);
|
|
234
|
+
if (ctx.modelRoutingProviderType) {
|
|
235
|
+
console.log(` With model routing: $${(costEstimate.low * 0.5).toFixed(2)}-$${(costEstimate.high * 0.5).toFixed(2)} (Haiku for classification)`);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
/** Rough cost estimation based on observed averages per workflow type */
|
|
239
|
+
function estimateCost(workflow, families, _agents) {
|
|
240
|
+
// Per-family cost ranges by workflow (based on typical Sonnet pricing)
|
|
241
|
+
const ranges = {
|
|
242
|
+
'quick-check': { low: 0.03, high: 0.10 },
|
|
243
|
+
'design-only': { low: 0.10, high: 0.40 },
|
|
244
|
+
'full-qa': { low: 0.30, high: 1.00 },
|
|
245
|
+
};
|
|
246
|
+
const range = ranges[workflow] || ranges['full-qa'];
|
|
247
|
+
return {
|
|
248
|
+
low: range.low * families,
|
|
249
|
+
high: range.high * families,
|
|
250
|
+
};
|
|
251
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
2
|
+
// See LICENSE.txt for license information.
|
|
3
|
+
/**
|
|
4
|
+
* CLI command: gate — CI coverage gate that exits 1 if coverage is below threshold.
|
|
5
|
+
*
|
|
6
|
+
* Runs deterministic impact analysis (no LLM required) and checks what
|
|
7
|
+
* percentage of impacted features have test coverage.
|
|
8
|
+
*
|
|
9
|
+
* Usage:
|
|
10
|
+
* e2e-ai-agents gate --threshold 80 --path . --since origin/main
|
|
11
|
+
*/
|
|
12
|
+
import { resolveConfig } from '../../agent/config.js';
|
|
13
|
+
import { getChangedFiles } from '../../agent/git.js';
|
|
14
|
+
import { analyzeImpact } from '../../engine/impact_engine.js';
|
|
15
|
+
export async function runGateCommand(args, autoConfig) {
|
|
16
|
+
if (!args.path && !autoConfig) {
|
|
17
|
+
console.error('Error: --path is required for gate command');
|
|
18
|
+
process.exit(1);
|
|
19
|
+
}
|
|
20
|
+
const threshold = args.gateThreshold ?? 80;
|
|
21
|
+
const { config } = resolveConfig(process.cwd(), autoConfig, {
|
|
22
|
+
path: args.path,
|
|
23
|
+
profile: args.profile,
|
|
24
|
+
testsRoot: args.testsRoot,
|
|
25
|
+
mode: 'impact',
|
|
26
|
+
gitSince: args.gitSince,
|
|
27
|
+
});
|
|
28
|
+
const testsRoot = config.testsRoot || config.path;
|
|
29
|
+
const gitSince = args.gitSince || config.git.since;
|
|
30
|
+
// Get changed files
|
|
31
|
+
const result = await getChangedFiles(config.path, gitSince);
|
|
32
|
+
const changedFiles = result.files;
|
|
33
|
+
if (changedFiles.length === 0) {
|
|
34
|
+
console.log('No changed files detected. Gate passes.');
|
|
35
|
+
process.exit(0);
|
|
36
|
+
}
|
|
37
|
+
// Run deterministic impact analysis
|
|
38
|
+
const impact = analyzeImpact(changedFiles, {
|
|
39
|
+
testsRoot,
|
|
40
|
+
routeFamilies: config.routeFamilies,
|
|
41
|
+
});
|
|
42
|
+
const totalFeatures = impact.impactedFeatures.length;
|
|
43
|
+
if (totalFeatures === 0) {
|
|
44
|
+
console.log('No impacted features detected. Gate passes.');
|
|
45
|
+
process.exit(0);
|
|
46
|
+
}
|
|
47
|
+
const coveredFeatures = impact.impactedFeatures.filter((f) => f.coverageStatus === 'covered' || f.coverageStatus === 'partial').length;
|
|
48
|
+
const coveragePercent = Math.round((coveredFeatures / totalFeatures) * 100);
|
|
49
|
+
// Output
|
|
50
|
+
if (args.jsonOutput) {
|
|
51
|
+
console.log(JSON.stringify({
|
|
52
|
+
threshold,
|
|
53
|
+
coveragePercent,
|
|
54
|
+
totalFeatures,
|
|
55
|
+
coveredFeatures,
|
|
56
|
+
passed: coveragePercent >= threshold,
|
|
57
|
+
uncoveredFeatures: impact.impactedFeatures
|
|
58
|
+
.filter((f) => f.coverageStatus === 'uncovered')
|
|
59
|
+
.map((f) => ({ id: f.featureId || f.familyId, priority: f.priority })),
|
|
60
|
+
}, null, 2));
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
console.log(`Coverage gate: ${coveragePercent}% (${coveredFeatures}/${totalFeatures} features covered)`);
|
|
64
|
+
console.log(`Threshold: ${threshold}%`);
|
|
65
|
+
if (coveragePercent < threshold) {
|
|
66
|
+
console.log(`\nFAILED — coverage ${coveragePercent}% is below ${threshold}% threshold`);
|
|
67
|
+
const uncovered = impact.impactedFeatures.filter((f) => f.coverageStatus === 'uncovered');
|
|
68
|
+
if (uncovered.length > 0) {
|
|
69
|
+
console.log('\nUncovered features:');
|
|
70
|
+
for (const f of uncovered.slice(0, 10)) {
|
|
71
|
+
console.log(` ${f.priority || 'P2'} ${f.featureId || f.familyId}`);
|
|
72
|
+
}
|
|
73
|
+
if (uncovered.length > 10) {
|
|
74
|
+
console.log(` ... and ${uncovered.length - 10} more`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
else {
|
|
79
|
+
console.log('\nPASSED');
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
process.exit(coveragePercent >= threshold ? 0 : 1);
|
|
83
|
+
}
|