@avi770/testteam 2.0.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -1
- package/README.md +20 -2
- package/agents/15-regression-sentinel.ts +3 -2
- package/agents/base-agent.ts +8 -0
- package/agents/registry.ts +72 -70
- package/core/cli.ts +128 -6
- package/core/orchestrator.ts +72 -1
- package/dist/agents/15-regression-sentinel.d.ts +2 -1
- package/dist/agents/15-regression-sentinel.d.ts.map +1 -1
- package/dist/agents/15-regression-sentinel.js +2 -2
- package/dist/agents/15-regression-sentinel.js.map +1 -1
- package/dist/agents/base-agent.d.ts +3 -1
- package/dist/agents/base-agent.d.ts.map +1 -1
- package/dist/agents/base-agent.js +7 -1
- package/dist/agents/base-agent.js.map +1 -1
- package/dist/agents/registry.d.ts +2 -1
- package/dist/agents/registry.d.ts.map +1 -1
- package/dist/agents/registry.js +71 -71
- package/dist/agents/registry.js.map +1 -1
- package/dist/clients/agent-mvp.d.ts +66 -0
- package/dist/clients/agent-mvp.d.ts.map +1 -0
- package/dist/clients/agent-mvp.js +91 -0
- package/dist/clients/agent-mvp.js.map +1 -0
- package/dist/clients/total-recall.d.ts +37 -0
- package/dist/clients/total-recall.d.ts.map +1 -0
- package/dist/clients/total-recall.js +224 -0
- package/dist/clients/total-recall.js.map +1 -0
- package/dist/core/cli.d.ts +3 -0
- package/dist/core/cli.d.ts.map +1 -1
- package/dist/core/cli.js +122 -6
- package/dist/core/cli.js.map +1 -1
- package/dist/core/orchestrator.d.ts +10 -1
- package/dist/core/orchestrator.d.ts.map +1 -1
- package/dist/core/orchestrator.js +60 -1
- package/dist/core/orchestrator.js.map +1 -1
- package/package.json +2 -2
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,29 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [3.0.0] - 2026-03-30
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- **Total Recall integration** — persistent memory for findings history and corrections learning across runs
|
|
12
|
+
- **Agent-MVP integration** — reputation scoring, tier progression, coaching flags for all 70 agents
|
|
13
|
+
- **MAH integration** — Multi-Agent Hub support with 100-agent limit, CustomExecutor, code-execution skills
|
|
14
|
+
- **Execution Bridge** — new `bridge/` module with pluggable executors:
|
|
15
|
+
- `PlaywrightPool` — reusable browser instance pool (max 3 concurrent)
|
|
16
|
+
- `ShellExecutor` — sandboxed child process execution with timeout enforcement
|
|
17
|
+
- `FileScanner` — regex-based source file scanning with glob filtering
|
|
18
|
+
- `HttpClient` — native fetch wrapper with concurrent request support
|
|
19
|
+
- `BridgeRouter` — task routing to correct executor, including hybrid multi-step pipelines
|
|
20
|
+
- `--coaching <agentId>` CLI flag to query Agent-MVP coaching flags
|
|
21
|
+
- `--recall <query>` CLI flag to query Total Recall memory
|
|
22
|
+
- `--reputation` CLI flag to display agent leaderboard
|
|
23
|
+
- Startup banner showing version and connection status for all 3 services (TR, Agent-MVP, MAH)
|
|
24
|
+
- Docker Compose stack for full local development (Total Recall, Agent-MVP, PostgreSQL, Redis)
|
|
25
|
+
|
|
26
|
+
### Changed
|
|
27
|
+
- Version bumped to 3.0.0 (major: learning/reputation system is a fundamental capability change)
|
|
28
|
+
- CLI startup now prints "TestTeam v3.0.0 — Learning Agents" banner
|
|
29
|
+
- Package description updated to reflect learning + reputation capabilities
|
|
30
|
+
|
|
8
31
|
## [2.0.0] - 2026-03-30
|
|
9
32
|
|
|
10
33
|
### Added
|
|
@@ -77,6 +100,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
77
100
|
- 595 unit tests
|
|
78
101
|
- TypeScript strict mode throughout with full type safety
|
|
79
102
|
|
|
80
|
-
[
|
|
103
|
+
[3.0.0]: https://github.com/BAS-More/E2E-Tester/compare/v2.0.0...HEAD
|
|
104
|
+
[2.0.0]: https://github.com/BAS-More/E2E-Tester/compare/v1.1.0...v2.0.0
|
|
81
105
|
[1.1.0]: https://github.com/BAS-More/E2E-Tester/compare/v1.0.0...v1.1.0
|
|
82
106
|
[1.0.0]: https://github.com/BAS-More/E2E-Tester/releases/tag/v1.0.0
|
package/README.md
CHANGED
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
|
|
19
19
|
---
|
|
20
20
|
|
|
21
|
-
Autonomous multi-phase testing pipeline with 70
|
|
21
|
+
Autonomous multi-phase testing pipeline with 70 learning agents. Runs Alpha through Release with quality gates, self-healing fix loops, reputation scoring, persistent memory, and zero manual intervention.
|
|
22
22
|
|
|
23
23
|
## Quick Start
|
|
24
24
|
|
|
@@ -119,15 +119,30 @@ The init wizard auto-detects your framework, test runner, and scaffolds configur
|
|
|
119
119
|
| 69 | Type Safety Auditor | Alpha, Beta, UAT | Pro |
|
|
120
120
|
| 70 | Complexity Analyzer | Alpha, Beta, UAT | Pro |
|
|
121
121
|
|
|
122
|
+
## v3.0.0 — Learning Agents
|
|
123
|
+
|
|
124
|
+
TestTeam v3.0.0 introduces a learning layer that makes agents smarter over time:
|
|
125
|
+
|
|
126
|
+
- **Total Recall** — persistent memory stores findings, corrections, and context across runs. Agents recall past issues and avoid false positives.
|
|
127
|
+
- **Agent-MVP** — reputation scoring with tier progression (Rookie through Legend). High-performing agents earn trust; underperformers get coaching flags.
|
|
128
|
+
- **MAH (Multi-Agent Hub)** — orchestration layer supporting up to 100 concurrent agents with CustomExecutor and code-execution skills.
|
|
129
|
+
- **Execution Bridge** — pluggable executor system with PlaywrightPool (browser reuse), ShellExecutor (sandboxed commands), FileScanner (regex source scanning), and HttpClient (concurrent API testing).
|
|
130
|
+
|
|
122
131
|
## Architecture
|
|
123
132
|
|
|
124
133
|
```
|
|
134
|
+
Total Recall (Memory)
|
|
135
|
+
|
|
|
125
136
|
Alpha --> [gate] --> Beta --> [gate] --> UAT --> [gate] --> Release
|
|
126
137
|
| | |
|
|
127
138
|
+------- Fix Loop (Healer + Fixer) ---+
|
|
139
|
+
| |
|
|
140
|
+
+---- Execution Bridge (Pool/Shell/HTTP/Scan) ----+
|
|
141
|
+
| |
|
|
142
|
+
+------------ Agent-MVP (Reputation) -------------+
|
|
128
143
|
```
|
|
129
144
|
|
|
130
|
-
Each phase dispatches its agents, evaluates findings against blocking/advisory classification, and promotes to the next phase only if the gate passes. The fix loop runs automatically when agents detect issues.
|
|
145
|
+
Each phase dispatches its agents, evaluates findings against blocking/advisory classification, and promotes to the next phase only if the gate passes. The fix loop runs automatically when agents detect issues. Agent performance is tracked by Agent-MVP and findings are persisted in Total Recall.
|
|
131
146
|
|
|
132
147
|
## Pricing
|
|
133
148
|
|
|
@@ -176,6 +191,9 @@ Options:
|
|
|
176
191
|
--dry-run Show dispatch plan without executing
|
|
177
192
|
--agent <id> Run a single agent by ID
|
|
178
193
|
--config <path> Path to config file
|
|
194
|
+
--recall <query> Query Total Recall memory
|
|
195
|
+
--reputation Show agent reputation leaderboard
|
|
196
|
+
--coaching <agentId> Show coaching flags for an agent
|
|
179
197
|
--help Show help
|
|
180
198
|
```
|
|
181
199
|
|
|
@@ -2,6 +2,7 @@ import { mkdir, readdir, readFile, writeFile, stat } from 'node:fs/promises';
|
|
|
2
2
|
import { join } from 'path';
|
|
3
3
|
import type { Finding, Phase } from '../core/types';
|
|
4
4
|
import type { ValidatedConfig } from '../core/config';
|
|
5
|
+
import type { TotalRecallClient } from '../clients/total-recall';
|
|
5
6
|
import { BaseAgent } from './base-agent';
|
|
6
7
|
import { compareScreenshots, updateBaseline } from '../helpers/screenshot';
|
|
7
8
|
|
|
@@ -27,8 +28,8 @@ export class RegressionSentinelAgent extends BaseAgent {
|
|
|
27
28
|
/** Absolute path to the baselines directory (sibling of runDir). */
|
|
28
29
|
private readonly baselinesDir: string;
|
|
29
30
|
|
|
30
|
-
constructor(config: ValidatedConfig, phase: Phase, runDir: string) {
|
|
31
|
-
super(config, phase, runDir);
|
|
31
|
+
constructor(config: ValidatedConfig, phase: Phase, runDir: string, totalRecall?: TotalRecallClient) {
|
|
32
|
+
super(config, phase, runDir, totalRecall);
|
|
32
33
|
// Place baselines as a sibling directory to runDir
|
|
33
34
|
const parentDir = join(runDir, '..');
|
|
34
35
|
this.baselinesDir = join(parentDir, BASELINES_DIR_NAME);
|
package/agents/base-agent.ts
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import type { AgentResult, Finding, Phase, EvidenceItem } from '../core/types';
|
|
2
2
|
import type { ValidatedConfig } from '../core/config';
|
|
3
|
+
import type { TotalRecallClient } from '../clients/total-recall';
|
|
3
4
|
|
|
4
5
|
export abstract class BaseAgent {
|
|
5
6
|
constructor(
|
|
6
7
|
protected readonly config: ValidatedConfig,
|
|
7
8
|
protected readonly phase: Phase,
|
|
8
9
|
protected readonly runDir: string,
|
|
10
|
+
protected readonly totalRecall?: TotalRecallClient,
|
|
9
11
|
) {}
|
|
10
12
|
|
|
11
13
|
abstract readonly agentId: number;
|
|
@@ -16,6 +18,12 @@ export abstract class BaseAgent {
|
|
|
16
18
|
try {
|
|
17
19
|
await this.preFlight();
|
|
18
20
|
const findings = await this.execute();
|
|
21
|
+
|
|
22
|
+
// Store findings in Total Recall if available (fire-and-forget)
|
|
23
|
+
if (this.totalRecall && findings.length > 0) {
|
|
24
|
+
await this.totalRecall.storeFindings(this.agentId, this.agentName, findings).catch(() => {});
|
|
25
|
+
}
|
|
26
|
+
|
|
19
27
|
const completedAt = new Date().toISOString();
|
|
20
28
|
return {
|
|
21
29
|
agentId: this.agentId,
|
package/agents/registry.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { ValidatedConfig } from '../core/config';
|
|
2
2
|
import type { Phase } from '../core/types';
|
|
3
3
|
import type { Orchestrator } from '../core/orchestrator';
|
|
4
|
+
import type { TotalRecallClient } from '../clients/total-recall';
|
|
4
5
|
|
|
5
6
|
/** License tier type used for agent gating */
|
|
6
7
|
export type LicenseTier = 'free' | 'pro' | 'enterprise';
|
|
@@ -197,78 +198,79 @@ export function registerAllAgents(
|
|
|
197
198
|
config: ValidatedConfig,
|
|
198
199
|
phase: Phase,
|
|
199
200
|
runDir: string,
|
|
201
|
+
totalRecall?: TotalRecallClient,
|
|
200
202
|
): void {
|
|
201
203
|
const agents = [
|
|
202
|
-
new AnalystAgent(config, phase, runDir),
|
|
203
|
-
new SeedArchitectAgent(config, phase, runDir),
|
|
204
|
-
new TestGeneratorAgent(config, phase, runDir),
|
|
205
|
-
new UnitRunnerAgent(config, phase, runDir),
|
|
206
|
-
new BrowserCrawlerAgent(config, phase, runDir),
|
|
207
|
-
new ApiExerciserAgent(config, phase, runDir),
|
|
208
|
-
new SecurityScoutAgent(config, phase, runDir),
|
|
209
|
-
new A11yGuardianAgent(config, phase, runDir),
|
|
210
|
-
new HealerAgent(config, phase, runDir),
|
|
211
|
-
new ReporterAgent(config, phase, runDir),
|
|
212
|
-
new FixerAgent(config, phase, runDir),
|
|
213
|
-
new UxInspectorAgent(config, phase, runDir),
|
|
214
|
-
new PerformanceProfilerAgent(config, phase, runDir),
|
|
215
|
-
new DataIntegrityAuditorAgent(config, phase, runDir),
|
|
216
|
-
new RegressionSentinelAgent(config, phase, runDir),
|
|
217
|
-
new ChaosAgent(config, phase, runDir),
|
|
218
|
-
new DocumentationValidatorAgent(config, phase, runDir),
|
|
219
|
-
new IntegrationWatchdogAgent(config, phase, runDir),
|
|
220
|
-
new TenantIsolationAuditorAgent(config, phase, runDir),
|
|
221
|
-
new WorkflowCompletionTesterAgent(config, phase, runDir),
|
|
222
|
-
new StateSessionTesterAgent(config, phase, runDir),
|
|
223
|
-
new EmailNotificationVerifierAgent(config, phase, runDir),
|
|
224
|
-
new MigrationTesterAgent(config, phase, runDir),
|
|
225
|
-
new SignupOnboardingTesterAgent(config, phase, runDir),
|
|
226
|
-
new CrudFlowTesterAgent(config, phase, runDir),
|
|
227
|
-
new FormValidatorAgent(config, phase, runDir),
|
|
228
|
-
new SearchFilterTesterAgent(config, phase, runDir),
|
|
229
|
-
new NavigationRoutingTesterAgent(config, phase, runDir),
|
|
230
|
-
new ResponsiveInteractionTesterAgent(config, phase, runDir),
|
|
231
|
-
new MultiUserScenarioTesterAgent(config, phase, runDir),
|
|
232
|
-
new LoadTesterAgent(config, phase, runDir),
|
|
233
|
-
new MemoryLeakDetectorAgent(config, phase, runDir),
|
|
234
|
-
new BundleAnalyzerAgent(config, phase, runDir),
|
|
235
|
-
new XssScannerAgent(config, phase, runDir),
|
|
236
|
-
new CsrfTesterAgent(config, phase, runDir),
|
|
237
|
-
new AuthFuzzerAgent(config, phase, runDir),
|
|
238
|
-
new DependencyScannerAgent(config, phase, runDir),
|
|
239
|
-
new SecretsScannerAgent(config, phase, runDir),
|
|
240
|
-
new ApiContractTesterAgent(config, phase, runDir),
|
|
241
|
-
new RateLimitTesterAgent(config, phase, runDir),
|
|
242
|
-
new ApiPaginationTesterAgent(config, phase, runDir),
|
|
243
|
-
new GraphqlTesterAgent(config, phase, runDir),
|
|
244
|
-
new DataConsistencyCheckerAgent(config, phase, runDir),
|
|
245
|
-
new BackupRecoveryTesterAgent(config, phase, runDir),
|
|
246
|
-
new DataPrivacyScannerAgent(config, phase, runDir),
|
|
247
|
-
new SeoAuditorAgent(config, phase, runDir),
|
|
248
|
-
new SocialPreviewTesterAgent(config, phase, runDir),
|
|
249
|
-
new LighthouseAuditorAgent(config, phase, runDir),
|
|
250
|
-
new I18nTesterAgent(config, phase, runDir),
|
|
251
|
-
new TimezoneTesterAgent(config, phase, runDir),
|
|
252
|
-
new ErrorRecoveryTesterAgent(config, phase, runDir),
|
|
253
|
-
new OfflineModeTesterAgent(config, phase, runDir),
|
|
254
|
-
new GracefulDegradationTesterAgent(config, phase, runDir),
|
|
255
|
-
new WebSocketTesterAgent(config, phase, runDir),
|
|
256
|
-
new RealtimeSyncTesterAgent(config, phase, runDir),
|
|
257
|
-
new FileUploadTesterAgent(config, phase, runDir),
|
|
258
|
-
new ExportTesterAgent(config, phase, runDir),
|
|
259
|
-
new PaymentFlowTesterAgent(config, phase, runDir),
|
|
260
|
-
new SslTlsAuditorAgent(config, phase, runDir),
|
|
261
|
-
new DnsCdnTesterAgent(config, phase, runDir),
|
|
262
|
-
new DockerHealthCheckerAgent(config, phase, runDir),
|
|
263
|
-
new EnvConfigValidatorAgent(config, phase, runDir),
|
|
264
|
-
new LogQualityAuditorAgent(config, phase, runDir),
|
|
265
|
-
new AnalyticsTrackerTesterAgent(config, phase, runDir),
|
|
266
|
-
new GdprComplianceTesterAgent(config, phase, runDir),
|
|
267
|
-
new Soc2ControlValidatorAgent(config, phase, runDir),
|
|
268
|
-
new WcagAaaTesterAgent(config, phase, runDir),
|
|
269
|
-
new DeadCodeDetectorAgent(config, phase, runDir),
|
|
270
|
-
new TypeSafetyAuditorAgent(config, phase, runDir),
|
|
271
|
-
new ComplexityAnalyzerAgent(config, phase, runDir),
|
|
204
|
+
new AnalystAgent(config, phase, runDir, totalRecall),
|
|
205
|
+
new SeedArchitectAgent(config, phase, runDir, totalRecall),
|
|
206
|
+
new TestGeneratorAgent(config, phase, runDir, totalRecall),
|
|
207
|
+
new UnitRunnerAgent(config, phase, runDir, totalRecall),
|
|
208
|
+
new BrowserCrawlerAgent(config, phase, runDir, totalRecall),
|
|
209
|
+
new ApiExerciserAgent(config, phase, runDir, totalRecall),
|
|
210
|
+
new SecurityScoutAgent(config, phase, runDir, totalRecall),
|
|
211
|
+
new A11yGuardianAgent(config, phase, runDir, totalRecall),
|
|
212
|
+
new HealerAgent(config, phase, runDir, totalRecall),
|
|
213
|
+
new ReporterAgent(config, phase, runDir, totalRecall),
|
|
214
|
+
new FixerAgent(config, phase, runDir, totalRecall),
|
|
215
|
+
new UxInspectorAgent(config, phase, runDir, totalRecall),
|
|
216
|
+
new PerformanceProfilerAgent(config, phase, runDir, totalRecall),
|
|
217
|
+
new DataIntegrityAuditorAgent(config, phase, runDir, totalRecall),
|
|
218
|
+
new RegressionSentinelAgent(config, phase, runDir, totalRecall),
|
|
219
|
+
new ChaosAgent(config, phase, runDir, totalRecall),
|
|
220
|
+
new DocumentationValidatorAgent(config, phase, runDir, totalRecall),
|
|
221
|
+
new IntegrationWatchdogAgent(config, phase, runDir, totalRecall),
|
|
222
|
+
new TenantIsolationAuditorAgent(config, phase, runDir, totalRecall),
|
|
223
|
+
new WorkflowCompletionTesterAgent(config, phase, runDir, totalRecall),
|
|
224
|
+
new StateSessionTesterAgent(config, phase, runDir, totalRecall),
|
|
225
|
+
new EmailNotificationVerifierAgent(config, phase, runDir, totalRecall),
|
|
226
|
+
new MigrationTesterAgent(config, phase, runDir, totalRecall),
|
|
227
|
+
new SignupOnboardingTesterAgent(config, phase, runDir, totalRecall),
|
|
228
|
+
new CrudFlowTesterAgent(config, phase, runDir, totalRecall),
|
|
229
|
+
new FormValidatorAgent(config, phase, runDir, totalRecall),
|
|
230
|
+
new SearchFilterTesterAgent(config, phase, runDir, totalRecall),
|
|
231
|
+
new NavigationRoutingTesterAgent(config, phase, runDir, totalRecall),
|
|
232
|
+
new ResponsiveInteractionTesterAgent(config, phase, runDir, totalRecall),
|
|
233
|
+
new MultiUserScenarioTesterAgent(config, phase, runDir, totalRecall),
|
|
234
|
+
new LoadTesterAgent(config, phase, runDir, totalRecall),
|
|
235
|
+
new MemoryLeakDetectorAgent(config, phase, runDir, totalRecall),
|
|
236
|
+
new BundleAnalyzerAgent(config, phase, runDir, totalRecall),
|
|
237
|
+
new XssScannerAgent(config, phase, runDir, totalRecall),
|
|
238
|
+
new CsrfTesterAgent(config, phase, runDir, totalRecall),
|
|
239
|
+
new AuthFuzzerAgent(config, phase, runDir, totalRecall),
|
|
240
|
+
new DependencyScannerAgent(config, phase, runDir, totalRecall),
|
|
241
|
+
new SecretsScannerAgent(config, phase, runDir, totalRecall),
|
|
242
|
+
new ApiContractTesterAgent(config, phase, runDir, totalRecall),
|
|
243
|
+
new RateLimitTesterAgent(config, phase, runDir, totalRecall),
|
|
244
|
+
new ApiPaginationTesterAgent(config, phase, runDir, totalRecall),
|
|
245
|
+
new GraphqlTesterAgent(config, phase, runDir, totalRecall),
|
|
246
|
+
new DataConsistencyCheckerAgent(config, phase, runDir, totalRecall),
|
|
247
|
+
new BackupRecoveryTesterAgent(config, phase, runDir, totalRecall),
|
|
248
|
+
new DataPrivacyScannerAgent(config, phase, runDir, totalRecall),
|
|
249
|
+
new SeoAuditorAgent(config, phase, runDir, totalRecall),
|
|
250
|
+
new SocialPreviewTesterAgent(config, phase, runDir, totalRecall),
|
|
251
|
+
new LighthouseAuditorAgent(config, phase, runDir, totalRecall),
|
|
252
|
+
new I18nTesterAgent(config, phase, runDir, totalRecall),
|
|
253
|
+
new TimezoneTesterAgent(config, phase, runDir, totalRecall),
|
|
254
|
+
new ErrorRecoveryTesterAgent(config, phase, runDir, totalRecall),
|
|
255
|
+
new OfflineModeTesterAgent(config, phase, runDir, totalRecall),
|
|
256
|
+
new GracefulDegradationTesterAgent(config, phase, runDir, totalRecall),
|
|
257
|
+
new WebSocketTesterAgent(config, phase, runDir, totalRecall),
|
|
258
|
+
new RealtimeSyncTesterAgent(config, phase, runDir, totalRecall),
|
|
259
|
+
new FileUploadTesterAgent(config, phase, runDir, totalRecall),
|
|
260
|
+
new ExportTesterAgent(config, phase, runDir, totalRecall),
|
|
261
|
+
new PaymentFlowTesterAgent(config, phase, runDir, totalRecall),
|
|
262
|
+
new SslTlsAuditorAgent(config, phase, runDir, totalRecall),
|
|
263
|
+
new DnsCdnTesterAgent(config, phase, runDir, totalRecall),
|
|
264
|
+
new DockerHealthCheckerAgent(config, phase, runDir, totalRecall),
|
|
265
|
+
new EnvConfigValidatorAgent(config, phase, runDir, totalRecall),
|
|
266
|
+
new LogQualityAuditorAgent(config, phase, runDir, totalRecall),
|
|
267
|
+
new AnalyticsTrackerTesterAgent(config, phase, runDir, totalRecall),
|
|
268
|
+
new GdprComplianceTesterAgent(config, phase, runDir, totalRecall),
|
|
269
|
+
new Soc2ControlValidatorAgent(config, phase, runDir, totalRecall),
|
|
270
|
+
new WcagAaaTesterAgent(config, phase, runDir, totalRecall),
|
|
271
|
+
new DeadCodeDetectorAgent(config, phase, runDir, totalRecall),
|
|
272
|
+
new TypeSafetyAuditorAgent(config, phase, runDir, totalRecall),
|
|
273
|
+
new ComplexityAnalyzerAgent(config, phase, runDir, totalRecall),
|
|
272
274
|
];
|
|
273
275
|
|
|
274
276
|
for (const agent of agents) {
|
package/core/cli.ts
CHANGED
|
@@ -6,6 +6,8 @@ import { registerAllAgents } from '../agents/registry';
|
|
|
6
6
|
import { runInitWizard } from './init';
|
|
7
7
|
import { determineExitCode, writeCISummary, detectCIEnvironment } from './ci-output';
|
|
8
8
|
import type { ExitCode } from './ci-output';
|
|
9
|
+
import { TotalRecallClient } from '../clients/total-recall';
|
|
10
|
+
import { AgentMvpClient } from '../clients/agent-mvp';
|
|
9
11
|
import * as path from 'path';
|
|
10
12
|
|
|
11
13
|
const PHASES: readonly Phase[] = ['alpha', 'beta', 'uat', 'release'];
|
|
@@ -15,6 +17,9 @@ interface ParsedArgs {
|
|
|
15
17
|
dryRun: boolean;
|
|
16
18
|
agent: number | null;
|
|
17
19
|
configPath: string;
|
|
20
|
+
recallQuery: string | null;
|
|
21
|
+
reputation: boolean;
|
|
22
|
+
coaching: string | null;
|
|
18
23
|
}
|
|
19
24
|
|
|
20
25
|
function printUsage(): void {
|
|
@@ -27,6 +32,9 @@ Options:
|
|
|
27
32
|
--dry-run Show dispatch plan without executing
|
|
28
33
|
--agent <id> Run a single agent by ID
|
|
29
34
|
--config <path> Path to config file (default: testteam.config.ts)
|
|
35
|
+
--recall <query> Query Total Recall memory and print results
|
|
36
|
+
--reputation Query agent reputations and print a table
|
|
37
|
+
--coaching <agentId> Query coaching flags for an agent
|
|
30
38
|
--help Show this help message
|
|
31
39
|
`.trim());
|
|
32
40
|
}
|
|
@@ -37,6 +45,9 @@ export function parseArgs(argv: string[]): ParsedArgs {
|
|
|
37
45
|
let dryRun = false;
|
|
38
46
|
let agent: number | null = null;
|
|
39
47
|
let configPath = 'testteam.config.ts';
|
|
48
|
+
let recallQuery: string | null = null;
|
|
49
|
+
let reputation = false;
|
|
50
|
+
let coaching: string | null = null;
|
|
40
51
|
|
|
41
52
|
for (let i = 0; i < args.length; i++) {
|
|
42
53
|
const arg = args[i];
|
|
@@ -67,6 +78,25 @@ export function parseArgs(argv: string[]): ParsedArgs {
|
|
|
67
78
|
throw new Error('--config requires a path argument');
|
|
68
79
|
}
|
|
69
80
|
break;
|
|
81
|
+
case '--recall': {
|
|
82
|
+
const value = args[++i];
|
|
83
|
+
if (!value) {
|
|
84
|
+
throw new Error('--recall requires a query argument');
|
|
85
|
+
}
|
|
86
|
+
recallQuery = value;
|
|
87
|
+
break;
|
|
88
|
+
}
|
|
89
|
+
case '--reputation':
|
|
90
|
+
reputation = true;
|
|
91
|
+
break;
|
|
92
|
+
case '--coaching': {
|
|
93
|
+
const value = args[++i];
|
|
94
|
+
if (!value) {
|
|
95
|
+
throw new Error('--coaching requires an agent ID argument');
|
|
96
|
+
}
|
|
97
|
+
coaching = value;
|
|
98
|
+
break;
|
|
99
|
+
}
|
|
70
100
|
case '--help':
|
|
71
101
|
printUsage();
|
|
72
102
|
process.exit(0);
|
|
@@ -76,7 +106,16 @@ export function parseArgs(argv: string[]): ParsedArgs {
|
|
|
76
106
|
}
|
|
77
107
|
}
|
|
78
108
|
|
|
79
|
-
return { phase, dryRun, agent, configPath };
|
|
109
|
+
return { phase, dryRun, agent, configPath, recallQuery, reputation, coaching };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
async function checkMahAvailability(baseUrl = 'http://localhost:3300'): Promise<boolean> {
|
|
113
|
+
try {
|
|
114
|
+
const res = await fetch(`${baseUrl}/health`);
|
|
115
|
+
return res.ok;
|
|
116
|
+
} catch {
|
|
117
|
+
return false;
|
|
118
|
+
}
|
|
80
119
|
}
|
|
81
120
|
|
|
82
121
|
async function loadConfig(configPath: string): Promise<TestTeamConfig> {
|
|
@@ -112,7 +151,54 @@ export async function main(argv: string[] = process.argv): Promise<number> {
|
|
|
112
151
|
return 1;
|
|
113
152
|
}
|
|
114
153
|
|
|
115
|
-
const { phase, dryRun, configPath } = parsed;
|
|
154
|
+
const { phase, dryRun, configPath, recallQuery, reputation, coaching } = parsed;
|
|
155
|
+
|
|
156
|
+
// Handle --recall: query Total Recall and exit
|
|
157
|
+
if (recallQuery) {
|
|
158
|
+
const tr = new TotalRecallClient();
|
|
159
|
+
const available = await tr.isAvailable();
|
|
160
|
+
if (!available) {
|
|
161
|
+
console.error('Total Recall is not running at http://localhost:3777');
|
|
162
|
+
return 1;
|
|
163
|
+
}
|
|
164
|
+
const results = await tr.recallFindings(recallQuery);
|
|
165
|
+
console.log(JSON.stringify(results, null, 2));
|
|
166
|
+
return 0;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// Handle --coaching: query Agent-MVP coaching flags for a specific agent and exit
|
|
170
|
+
if (coaching) {
|
|
171
|
+
const mvp = new AgentMvpClient();
|
|
172
|
+
const mvpAvailable = await mvp.isAvailable();
|
|
173
|
+
if (!mvpAvailable) {
|
|
174
|
+
console.error('Agent-MVP is not running at http://localhost:3200');
|
|
175
|
+
return 1;
|
|
176
|
+
}
|
|
177
|
+
const flags = await mvp.getCoachingFlags(coaching);
|
|
178
|
+
if (flags.length > 0) {
|
|
179
|
+
console.log(JSON.stringify(flags, null, 2));
|
|
180
|
+
} else {
|
|
181
|
+
console.log(`No coaching flags for agent "${coaching}".`);
|
|
182
|
+
}
|
|
183
|
+
return 0;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// Handle --reputation: query Agent-MVP for all agent reputations and exit
|
|
187
|
+
if (reputation) {
|
|
188
|
+
const mvp = new AgentMvpClient();
|
|
189
|
+
const mvpAvailable = await mvp.isAvailable();
|
|
190
|
+
if (!mvpAvailable) {
|
|
191
|
+
console.error('Agent-MVP is not running at http://localhost:3200');
|
|
192
|
+
return 1;
|
|
193
|
+
}
|
|
194
|
+
const leaderboard = await mvp.getReputation('leaderboard');
|
|
195
|
+
if (leaderboard) {
|
|
196
|
+
console.log(JSON.stringify(leaderboard, null, 2));
|
|
197
|
+
} else {
|
|
198
|
+
console.log('No reputation data available.');
|
|
199
|
+
}
|
|
200
|
+
return 0;
|
|
201
|
+
}
|
|
116
202
|
|
|
117
203
|
if (!dryRun && !phase) {
|
|
118
204
|
console.error('--phase is required unless using --dry-run');
|
|
@@ -120,6 +206,10 @@ export async function main(argv: string[] = process.argv): Promise<number> {
|
|
|
120
206
|
return 1;
|
|
121
207
|
}
|
|
122
208
|
|
|
209
|
+
// Startup banner
|
|
210
|
+
console.log('TestTeam v3.0.0 — Learning Agents');
|
|
211
|
+
console.log('─'.repeat(40));
|
|
212
|
+
|
|
123
213
|
// Load and validate config
|
|
124
214
|
let rawConfig: TestTeamConfig;
|
|
125
215
|
try {
|
|
@@ -131,6 +221,33 @@ export async function main(argv: string[] = process.argv): Promise<number> {
|
|
|
131
221
|
|
|
132
222
|
const config = validateConfig(rawConfig);
|
|
133
223
|
|
|
224
|
+
// Initialize Total Recall (optional — continues normally if unavailable)
|
|
225
|
+
const totalRecall = new TotalRecallClient();
|
|
226
|
+
const trAvailable = await totalRecall.isAvailable();
|
|
227
|
+
if (trAvailable) {
|
|
228
|
+
await totalRecall.startSession();
|
|
229
|
+
console.log('[Total Recall] Connected — findings will be stored in memory');
|
|
230
|
+
} else {
|
|
231
|
+
console.log('[Total Recall] Not available at http://localhost:3777 — running without memory');
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// Initialize Agent-MVP client (optional ��� continues normally if unavailable)
|
|
235
|
+
const agentMvp = new AgentMvpClient();
|
|
236
|
+
const mvpAvailable = await agentMvp.isAvailable();
|
|
237
|
+
if (mvpAvailable) {
|
|
238
|
+
console.log('[Agent-MVP] Connected — agent scores will be reported');
|
|
239
|
+
} else {
|
|
240
|
+
console.log('[Agent-MVP] Not available at http://localhost:3200 — running without scoring');
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// Check MAH availability (optional — informational only)
|
|
244
|
+
const mahAvailable = await checkMahAvailability();
|
|
245
|
+
if (mahAvailable) {
|
|
246
|
+
console.log('[MAH] Connected — multi-agent hub active');
|
|
247
|
+
} else {
|
|
248
|
+
console.log('[MAH] Not available at http://localhost:3300 — running without hub');
|
|
249
|
+
}
|
|
250
|
+
|
|
134
251
|
// Shared run directory so all phases write results to the same location.
|
|
135
252
|
// This ensures the Reporter (Agent 10, release phase) can read results
|
|
136
253
|
// from prior phases (alpha, beta, uat).
|
|
@@ -142,8 +259,8 @@ export async function main(argv: string[] = process.argv): Promise<number> {
|
|
|
142
259
|
: [phase];
|
|
143
260
|
|
|
144
261
|
for (const p of targetPhases) {
|
|
145
|
-
const phaseOrchestrator = new Orchestrator(config, sharedRunDir);
|
|
146
|
-
registerAllAgents(phaseOrchestrator, config, p, sharedRunDir);
|
|
262
|
+
const phaseOrchestrator = new Orchestrator(config, sharedRunDir, mvpAvailable ? agentMvp : undefined);
|
|
263
|
+
registerAllAgents(phaseOrchestrator, config, p, sharedRunDir, trAvailable ? totalRecall : undefined);
|
|
147
264
|
const plan = phaseOrchestrator.dryRun(p);
|
|
148
265
|
console.log(`\nPhase: ${p}`);
|
|
149
266
|
for (const entry of plan) {
|
|
@@ -166,8 +283,8 @@ export async function main(argv: string[] = process.argv): Promise<number> {
|
|
|
166
283
|
for (const p of targetPhases) {
|
|
167
284
|
console.log(`\nRunning phase: ${p}`);
|
|
168
285
|
try {
|
|
169
|
-
const phaseOrchestrator = new Orchestrator(config, sharedRunDir);
|
|
170
|
-
registerAllAgents(phaseOrchestrator, config, p, sharedRunDir);
|
|
286
|
+
const phaseOrchestrator = new Orchestrator(config, sharedRunDir, mvpAvailable ? agentMvp : undefined);
|
|
287
|
+
registerAllAgents(phaseOrchestrator, config, p, sharedRunDir, trAvailable ? totalRecall : undefined);
|
|
171
288
|
const result = await phaseOrchestrator.runPhase(p);
|
|
172
289
|
|
|
173
290
|
if (result.skippedAgents.length > 0) {
|
|
@@ -195,6 +312,11 @@ export async function main(argv: string[] = process.argv): Promise<number> {
|
|
|
195
312
|
}
|
|
196
313
|
}
|
|
197
314
|
|
|
315
|
+
// Close Total Recall session
|
|
316
|
+
if (trAvailable) {
|
|
317
|
+
await totalRecall.endSession().catch(() => {});
|
|
318
|
+
}
|
|
319
|
+
|
|
198
320
|
// Determine structured exit code:
|
|
199
321
|
// - 0: All gates passed, no advisory failures
|
|
200
322
|
// - 1: Blocking agent failures (or runtime errors)
|
package/core/orchestrator.ts
CHANGED
|
@@ -11,6 +11,7 @@ import { getLicenseTier } from './license';
|
|
|
11
11
|
import type { LicenseTier } from './license';
|
|
12
12
|
import { isAgentAvailable, getAgentTier } from '../agents/registry';
|
|
13
13
|
import { agentGatedMessage, phaseGatedMessage, getAgentDisplayName } from './messages';
|
|
14
|
+
import type { AgentMvpClient, AgentScoreEvent } from '../clients/agent-mvp';
|
|
14
15
|
import * as path from 'path';
|
|
15
16
|
|
|
16
17
|
/** Agent IDs to dispatch per phase, in execution order. */
|
|
@@ -66,6 +67,7 @@ export class Orchestrator {
|
|
|
66
67
|
private readonly agents: Map<number, BaseAgent> = new Map();
|
|
67
68
|
private readonly healthChecks: HealthCheck[] = [];
|
|
68
69
|
private runDir: string;
|
|
70
|
+
private readonly agentMvpClient: AgentMvpClient | null;
|
|
69
71
|
|
|
70
72
|
/**
|
|
71
73
|
* License tier resolver — injectable for testing.
|
|
@@ -73,9 +75,10 @@ export class Orchestrator {
|
|
|
73
75
|
*/
|
|
74
76
|
_getLicenseTier: () => LicenseTier = () => getLicenseTier();
|
|
75
77
|
|
|
76
|
-
constructor(config: ValidatedConfig, runDir?: string) {
|
|
78
|
+
constructor(config: ValidatedConfig, runDir?: string, agentMvpClient?: AgentMvpClient) {
|
|
77
79
|
this.config = config;
|
|
78
80
|
this.runDir = runDir ?? path.join(process.cwd(), '.run', `run-${Date.now()}`);
|
|
81
|
+
this.agentMvpClient = agentMvpClient ?? null;
|
|
79
82
|
}
|
|
80
83
|
|
|
81
84
|
/** Register an agent instance for a given agent ID. */
|
|
@@ -282,6 +285,9 @@ export class Orchestrator {
|
|
|
282
285
|
scheduledForPhase,
|
|
283
286
|
);
|
|
284
287
|
|
|
288
|
+
// Step 7: Score agents in Agent-MVP (if connected)
|
|
289
|
+
await this.scoreAgentsInMvp(phase, agentResults);
|
|
290
|
+
|
|
285
291
|
return { gate, agentResults, skippedAgents, costTracker };
|
|
286
292
|
}
|
|
287
293
|
|
|
@@ -349,6 +355,71 @@ export class Orchestrator {
|
|
|
349
355
|
}
|
|
350
356
|
}
|
|
351
357
|
|
|
358
|
+
/**
|
|
359
|
+
* After a phase completes, send scoring events to Agent-MVP for each agent
|
|
360
|
+
* that produced results. Passed agents receive +1.0, failed agents receive -0.5.
|
|
361
|
+
*/
|
|
362
|
+
private async scoreAgentsInMvp(
|
|
363
|
+
phase: Phase,
|
|
364
|
+
agentResults: Record<number, AgentResult>,
|
|
365
|
+
): Promise<void> {
|
|
366
|
+
if (!this.agentMvpClient) return;
|
|
367
|
+
|
|
368
|
+
const events: AgentScoreEvent[] = [];
|
|
369
|
+
|
|
370
|
+
for (const [agentId, result] of Object.entries(agentResults)) {
|
|
371
|
+
if (result.status === 'skipped') continue;
|
|
372
|
+
|
|
373
|
+
const isPassed = result.status === 'passed';
|
|
374
|
+
const findingsCount = result.findings.length;
|
|
375
|
+
const worstSeverity = this.pickWorstSeverity(result.findings);
|
|
376
|
+
|
|
377
|
+
events.push({
|
|
378
|
+
agent_id: `agent-${agentId}`,
|
|
379
|
+
task_type: result.agentName,
|
|
380
|
+
outcome: isPassed ? 'success' : 'failure',
|
|
381
|
+
score_delta: isPassed ? 1.0 : -0.5,
|
|
382
|
+
phase,
|
|
383
|
+
findings_count: findingsCount > 0 ? findingsCount : undefined,
|
|
384
|
+
finding_severity: worstSeverity ?? undefined,
|
|
385
|
+
});
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
if (events.length > 0) {
|
|
389
|
+
try {
|
|
390
|
+
await this.agentMvpClient.scoreAgents(events);
|
|
391
|
+
} catch {
|
|
392
|
+
// Graceful -- Agent-MVP scoring is non-critical
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
/** Determine the worst (most severe) finding severity from a list. */
|
|
398
|
+
private pickWorstSeverity(findings: Finding[]): string | null {
|
|
399
|
+
if (findings.length === 0) return null;
|
|
400
|
+
|
|
401
|
+
const rank: Record<string, number> = {
|
|
402
|
+
critical: 5,
|
|
403
|
+
high: 4,
|
|
404
|
+
medium: 3,
|
|
405
|
+
low: 2,
|
|
406
|
+
info: 1,
|
|
407
|
+
};
|
|
408
|
+
|
|
409
|
+
let worst: string | null = null;
|
|
410
|
+
let worstRank = 0;
|
|
411
|
+
|
|
412
|
+
for (const finding of findings) {
|
|
413
|
+
const currentRank = rank[finding.severity] ?? 0;
|
|
414
|
+
if (currentRank > worstRank) {
|
|
415
|
+
worstRank = currentRank;
|
|
416
|
+
worst = finding.severity;
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
return worst;
|
|
421
|
+
}
|
|
422
|
+
|
|
352
423
|
/** Get the run directory path. */
|
|
353
424
|
getRunDir(): string {
|
|
354
425
|
return this.runDir;
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import type { Finding, Phase } from '../core/types';
|
|
2
2
|
import type { ValidatedConfig } from '../core/config';
|
|
3
|
+
import type { TotalRecallClient } from '../clients/total-recall';
|
|
3
4
|
import { BaseAgent } from './base-agent';
|
|
4
5
|
export declare class RegressionSentinelAgent extends BaseAgent {
|
|
5
6
|
readonly agentId = 15;
|
|
6
7
|
readonly agentName = "Regression Sentinel";
|
|
7
8
|
/** Absolute path to the baselines directory (sibling of runDir). */
|
|
8
9
|
private readonly baselinesDir;
|
|
9
|
-
constructor(config: ValidatedConfig, phase: Phase, runDir: string);
|
|
10
|
+
constructor(config: ValidatedConfig, phase: Phase, runDir: string, totalRecall?: TotalRecallClient);
|
|
10
11
|
protected preFlight(): Promise<void>;
|
|
11
12
|
protected execute(): Promise<Finding[]>;
|
|
12
13
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"15-regression-sentinel.d.ts","sourceRoot":"","sources":["../../agents/15-regression-sentinel.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAkBzC,qBAAa,uBAAwB,SAAQ,SAAS;IACpD,QAAQ,CAAC,OAAO,MAAM;IACtB,QAAQ,CAAC,SAAS,yBAAyB;IAE3C,oEAAoE;IACpE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM;
|
|
1
|
+
{"version":3,"file":"15-regression-sentinel.d.ts","sourceRoot":"","sources":["../../agents/15-regression-sentinel.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAkBzC,qBAAa,uBAAwB,SAAQ,SAAS;IACpD,QAAQ,CAAC,OAAO,MAAM;IACtB,QAAQ,CAAC,SAAS,yBAAyB;IAE3C,oEAAoE;IACpE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,iBAAiB;cAOlF,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;cAa1B,OAAO,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IAuI7C;;;;;;;OAOG;YACW,UAAU;IAiBxB,iFAAiF;IACjF,OAAO,CAAC,WAAW;YAOL,iBAAiB;CAyFhC"}
|
|
@@ -20,8 +20,8 @@ export class RegressionSentinelAgent extends BaseAgent {
|
|
|
20
20
|
agentName = 'Regression Sentinel';
|
|
21
21
|
/** Absolute path to the baselines directory (sibling of runDir). */
|
|
22
22
|
baselinesDir;
|
|
23
|
-
constructor(config, phase, runDir) {
|
|
24
|
-
super(config, phase, runDir);
|
|
23
|
+
constructor(config, phase, runDir, totalRecall) {
|
|
24
|
+
super(config, phase, runDir, totalRecall);
|
|
25
25
|
// Place baselines as a sibling directory to runDir
|
|
26
26
|
const parentDir = join(runDir, '..');
|
|
27
27
|
this.baselinesDir = join(parentDir, BASELINES_DIR_NAME);
|