modestbench 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/LICENSE.md +55 -0
- package/README.md +699 -0
- package/dist/bootstrap.cjs +37 -0
- package/dist/bootstrap.cjs.map +1 -0
- package/dist/bootstrap.d.cts +17 -0
- package/dist/bootstrap.d.cts.map +1 -0
- package/dist/bootstrap.d.ts +17 -0
- package/dist/bootstrap.d.ts.map +1 -0
- package/dist/bootstrap.js +33 -0
- package/dist/bootstrap.js.map +1 -0
- package/dist/cli/commands/history.cjs +459 -0
- package/dist/cli/commands/history.cjs.map +1 -0
- package/dist/cli/commands/history.d.cts +34 -0
- package/dist/cli/commands/history.d.cts.map +1 -0
- package/dist/cli/commands/history.d.ts +34 -0
- package/dist/cli/commands/history.d.ts.map +1 -0
- package/dist/cli/commands/history.js +422 -0
- package/dist/cli/commands/history.js.map +1 -0
- package/dist/cli/commands/init.cjs +566 -0
- package/dist/cli/commands/init.cjs.map +1 -0
- package/dist/cli/commands/init.d.cts +26 -0
- package/dist/cli/commands/init.d.cts.map +1 -0
- package/dist/cli/commands/init.d.ts +26 -0
- package/dist/cli/commands/init.d.ts.map +1 -0
- package/dist/cli/commands/init.js +562 -0
- package/dist/cli/commands/init.js.map +1 -0
- package/dist/cli/commands/run.cjs +285 -0
- package/dist/cli/commands/run.cjs.map +1 -0
- package/dist/cli/commands/run.d.cts +37 -0
- package/dist/cli/commands/run.d.cts.map +1 -0
- package/dist/cli/commands/run.d.ts +37 -0
- package/dist/cli/commands/run.d.ts.map +1 -0
- package/dist/cli/commands/run.js +248 -0
- package/dist/cli/commands/run.js.map +1 -0
- package/dist/cli/index.cjs +523 -0
- package/dist/cli/index.cjs.map +1 -0
- package/dist/cli/index.d.cts +58 -0
- package/dist/cli/index.d.cts.map +1 -0
- package/dist/cli/index.d.ts +58 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +515 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/config/manager.cjs +370 -0
- package/dist/config/manager.cjs.map +1 -0
- package/dist/config/manager.d.cts +46 -0
- package/dist/config/manager.d.cts.map +1 -0
- package/dist/config/manager.d.ts +46 -0
- package/dist/config/manager.d.ts.map +1 -0
- package/dist/config/manager.js +333 -0
- package/dist/config/manager.js.map +1 -0
- package/dist/config/schema.cjs +182 -0
- package/dist/config/schema.cjs.map +1 -0
- package/dist/config/schema.d.cts +51 -0
- package/dist/config/schema.d.cts.map +1 -0
- package/dist/config/schema.d.ts +51 -0
- package/dist/config/schema.d.ts.map +1 -0
- package/dist/config/schema.js +145 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/constants.cjs +22 -0
- package/dist/constants.cjs.map +1 -0
- package/dist/constants.d.cts +10 -0
- package/dist/constants.d.cts.map +1 -0
- package/dist/constants.d.ts +10 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/constants.js +19 -0
- package/dist/constants.js.map +1 -0
- package/dist/core/benchmark-schema.cjs +135 -0
- package/dist/core/benchmark-schema.cjs.map +1 -0
- package/dist/core/benchmark-schema.d.cts +139 -0
- package/dist/core/benchmark-schema.d.cts.map +1 -0
- package/dist/core/benchmark-schema.d.ts +139 -0
- package/dist/core/benchmark-schema.d.ts.map +1 -0
- package/dist/core/benchmark-schema.js +132 -0
- package/dist/core/benchmark-schema.js.map +1 -0
- package/dist/core/engine.cjs +669 -0
- package/dist/core/engine.cjs.map +1 -0
- package/dist/core/engine.d.cts +128 -0
- package/dist/core/engine.d.cts.map +1 -0
- package/dist/core/engine.d.ts +128 -0
- package/dist/core/engine.d.ts.map +1 -0
- package/dist/core/engine.js +632 -0
- package/dist/core/engine.js.map +1 -0
- package/dist/core/engines/accurate-engine.cjs +292 -0
- package/dist/core/engines/accurate-engine.cjs.map +1 -0
- package/dist/core/engines/accurate-engine.d.cts +63 -0
- package/dist/core/engines/accurate-engine.d.cts.map +1 -0
- package/dist/core/engines/accurate-engine.d.ts +63 -0
- package/dist/core/engines/accurate-engine.d.ts.map +1 -0
- package/dist/core/engines/accurate-engine.js +288 -0
- package/dist/core/engines/accurate-engine.js.map +1 -0
- package/dist/core/engines/index.cjs +21 -0
- package/dist/core/engines/index.cjs.map +1 -0
- package/dist/core/engines/index.d.cts +16 -0
- package/dist/core/engines/index.d.cts.map +1 -0
- package/dist/core/engines/index.d.ts +16 -0
- package/dist/core/engines/index.d.ts.map +1 -0
- package/dist/core/engines/index.js +16 -0
- package/dist/core/engines/index.js.map +1 -0
- package/dist/core/engines/tinybench-engine.cjs +286 -0
- package/dist/core/engines/tinybench-engine.cjs.map +1 -0
- package/dist/core/engines/tinybench-engine.d.cts +18 -0
- package/dist/core/engines/tinybench-engine.d.cts.map +1 -0
- package/dist/core/engines/tinybench-engine.d.ts +18 -0
- package/dist/core/engines/tinybench-engine.d.ts.map +1 -0
- package/dist/core/engines/tinybench-engine.js +282 -0
- package/dist/core/engines/tinybench-engine.js.map +1 -0
- package/dist/core/error-manager.cjs +303 -0
- package/dist/core/error-manager.cjs.map +1 -0
- package/dist/core/error-manager.d.cts +77 -0
- package/dist/core/error-manager.d.cts.map +1 -0
- package/dist/core/error-manager.d.ts +77 -0
- package/dist/core/error-manager.d.ts.map +1 -0
- package/dist/core/error-manager.js +299 -0
- package/dist/core/error-manager.js.map +1 -0
- package/dist/core/loader.cjs +287 -0
- package/dist/core/loader.cjs.map +1 -0
- package/dist/core/loader.d.cts +55 -0
- package/dist/core/loader.d.cts.map +1 -0
- package/dist/core/loader.d.ts +55 -0
- package/dist/core/loader.d.ts.map +1 -0
- package/dist/core/loader.js +250 -0
- package/dist/core/loader.js.map +1 -0
- package/dist/core/stats-utils.cjs +99 -0
- package/dist/core/stats-utils.cjs.map +1 -0
- package/dist/core/stats-utils.d.cts +50 -0
- package/dist/core/stats-utils.d.cts.map +1 -0
- package/dist/core/stats-utils.d.ts +50 -0
- package/dist/core/stats-utils.d.ts.map +1 -0
- package/dist/core/stats-utils.js +94 -0
- package/dist/core/stats-utils.js.map +1 -0
- package/dist/index.cjs +64 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +22 -0
- package/dist/index.d.cts.map +1 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +30 -0
- package/dist/index.js.map +1 -0
- package/dist/progress/manager.cjs +325 -0
- package/dist/progress/manager.cjs.map +1 -0
- package/dist/progress/manager.d.cts +125 -0
- package/dist/progress/manager.d.cts.map +1 -0
- package/dist/progress/manager.d.ts +125 -0
- package/dist/progress/manager.d.ts.map +1 -0
- package/dist/progress/manager.js +321 -0
- package/dist/progress/manager.js.map +1 -0
- package/dist/reporters/csv.cjs +250 -0
- package/dist/reporters/csv.cjs.map +1 -0
- package/dist/reporters/csv.d.cts +92 -0
- package/dist/reporters/csv.d.cts.map +1 -0
- package/dist/reporters/csv.d.ts +92 -0
- package/dist/reporters/csv.d.ts.map +1 -0
- package/dist/reporters/csv.js +246 -0
- package/dist/reporters/csv.js.map +1 -0
- package/dist/reporters/human.cjs +516 -0
- package/dist/reporters/human.cjs.map +1 -0
- package/dist/reporters/human.d.cts +86 -0
- package/dist/reporters/human.d.cts.map +1 -0
- package/dist/reporters/human.d.ts +86 -0
- package/dist/reporters/human.d.ts.map +1 -0
- package/dist/reporters/human.js +509 -0
- package/dist/reporters/human.js.map +1 -0
- package/dist/reporters/index.cjs +17 -0
- package/dist/reporters/index.cjs.map +1 -0
- package/dist/reporters/index.d.cts +10 -0
- package/dist/reporters/index.d.cts.map +1 -0
- package/dist/reporters/index.d.ts +10 -0
- package/dist/reporters/index.d.ts.map +1 -0
- package/dist/reporters/index.js +10 -0
- package/dist/reporters/index.js.map +1 -0
- package/dist/reporters/json.cjs +215 -0
- package/dist/reporters/json.cjs.map +1 -0
- package/dist/reporters/json.d.cts +79 -0
- package/dist/reporters/json.d.cts.map +1 -0
- package/dist/reporters/json.d.ts +79 -0
- package/dist/reporters/json.d.ts.map +1 -0
- package/dist/reporters/json.js +211 -0
- package/dist/reporters/json.js.map +1 -0
- package/dist/reporters/registry.cjs +255 -0
- package/dist/reporters/registry.cjs.map +1 -0
- package/dist/reporters/registry.d.cts +155 -0
- package/dist/reporters/registry.d.cts.map +1 -0
- package/dist/reporters/registry.d.ts +155 -0
- package/dist/reporters/registry.d.ts.map +1 -0
- package/dist/reporters/registry.js +249 -0
- package/dist/reporters/registry.js.map +1 -0
- package/dist/reporters/simple.cjs +328 -0
- package/dist/reporters/simple.cjs.map +1 -0
- package/dist/reporters/simple.d.cts +51 -0
- package/dist/reporters/simple.d.cts.map +1 -0
- package/dist/reporters/simple.d.ts +51 -0
- package/dist/reporters/simple.d.ts.map +1 -0
- package/dist/reporters/simple.js +321 -0
- package/dist/reporters/simple.js.map +1 -0
- package/dist/schema/modestbench-config.schema.json +162 -0
- package/dist/storage/history.cjs +456 -0
- package/dist/storage/history.cjs.map +1 -0
- package/dist/storage/history.d.cts +99 -0
- package/dist/storage/history.d.cts.map +1 -0
- package/dist/storage/history.d.ts +99 -0
- package/dist/storage/history.d.ts.map +1 -0
- package/dist/storage/history.js +452 -0
- package/dist/storage/history.js.map +1 -0
- package/dist/types/cli.cjs +21 -0
- package/dist/types/cli.cjs.map +1 -0
- package/dist/types/cli.d.cts +296 -0
- package/dist/types/cli.d.cts.map +1 -0
- package/dist/types/cli.d.ts +296 -0
- package/dist/types/cli.d.ts.map +1 -0
- package/dist/types/cli.js +18 -0
- package/dist/types/cli.js.map +1 -0
- package/dist/types/core.cjs +14 -0
- package/dist/types/core.cjs.map +1 -0
- package/dist/types/core.d.cts +380 -0
- package/dist/types/core.d.cts.map +1 -0
- package/dist/types/core.d.ts +380 -0
- package/dist/types/core.d.ts.map +1 -0
- package/dist/types/core.js +13 -0
- package/dist/types/core.js.map +1 -0
- package/dist/types/index.cjs +27 -0
- package/dist/types/index.cjs.map +1 -0
- package/dist/types/index.d.cts +11 -0
- package/dist/types/index.d.cts.map +1 -0
- package/dist/types/index.d.ts +11 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +11 -0
- package/dist/types/index.js.map +1 -0
- package/dist/types/interfaces.cjs +10 -0
- package/dist/types/interfaces.cjs.map +1 -0
- package/dist/types/interfaces.d.cts +381 -0
- package/dist/types/interfaces.d.cts.map +1 -0
- package/dist/types/interfaces.d.ts +381 -0
- package/dist/types/interfaces.d.ts.map +1 -0
- package/dist/types/interfaces.js +9 -0
- package/dist/types/interfaces.js.map +1 -0
- package/dist/types/utility.cjs +92 -0
- package/dist/types/utility.cjs.map +1 -0
- package/dist/types/utility.d.cts +330 -0
- package/dist/types/utility.d.cts.map +1 -0
- package/dist/types/utility.d.ts +330 -0
- package/dist/types/utility.d.ts.map +1 -0
- package/dist/types/utility.js +78 -0
- package/dist/types/utility.js.map +1 -0
- package/package.json +211 -0
- package/src/bootstrap.ts +35 -0
- package/src/cli/commands/history.ts +569 -0
- package/src/cli/commands/init.ts +658 -0
- package/src/cli/commands/run.ts +346 -0
- package/src/cli/index.ts +642 -0
- package/src/config/manager.ts +387 -0
- package/src/config/schema.ts +188 -0
- package/src/constants.ts +21 -0
- package/src/core/benchmark-schema.ts +185 -0
- package/src/core/engine.ts +888 -0
- package/src/core/engines/accurate-engine.ts +408 -0
- package/src/core/engines/index.ts +16 -0
- package/src/core/engines/tinybench-engine.ts +335 -0
- package/src/core/error-manager.ts +372 -0
- package/src/core/loader.ts +324 -0
- package/src/core/stats-utils.ts +135 -0
- package/src/index.ts +46 -0
- package/src/progress/manager.ts +415 -0
- package/src/reporters/csv.ts +368 -0
- package/src/reporters/human.ts +707 -0
- package/src/reporters/index.ts +10 -0
- package/src/reporters/json.ts +302 -0
- package/src/reporters/registry.ts +349 -0
- package/src/reporters/simple.ts +459 -0
- package/src/storage/history.ts +600 -0
- package/src/types/cli.ts +312 -0
- package/src/types/core.ts +414 -0
- package/src/types/index.ts +18 -0
- package/src/types/interfaces.ts +451 -0
- package/src/types/utility.ts +446 -0
|
@@ -0,0 +1,888 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ModestBench Core Engine
|
|
3
|
+
*
|
|
4
|
+
* Main orchestrator for benchmark discovery, validation, and execution.
|
|
5
|
+
* Implements the BenchmarkEngine interface with dependency injection
|
|
6
|
+
* architecture.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import type {
|
|
10
|
+
BenchmarkDefinition,
|
|
11
|
+
BenchmarkEngine,
|
|
12
|
+
BenchmarkRun,
|
|
13
|
+
BenchmarkSuite,
|
|
14
|
+
BenchmarkTask,
|
|
15
|
+
CiInfo,
|
|
16
|
+
ConfigurationManager,
|
|
17
|
+
EnvironmentInfo,
|
|
18
|
+
ErrorManager,
|
|
19
|
+
ExecutionPhase,
|
|
20
|
+
FileLoader,
|
|
21
|
+
FileResult,
|
|
22
|
+
GitInfo,
|
|
23
|
+
HistoryStorage,
|
|
24
|
+
ModestBenchConfig,
|
|
25
|
+
ProgressManager,
|
|
26
|
+
Reporter,
|
|
27
|
+
ReporterRegistry,
|
|
28
|
+
RunConfiguration,
|
|
29
|
+
SuiteResult,
|
|
30
|
+
TaskResult,
|
|
31
|
+
ValidationError,
|
|
32
|
+
ValidationResult,
|
|
33
|
+
ValidationWarning,
|
|
34
|
+
} from '../types/index.js';
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Dependencies required by the BenchmarkEngine
|
|
38
|
+
*/
|
|
39
|
+
interface EngineDependencies {
|
|
40
|
+
readonly configManager: ConfigurationManager;
|
|
41
|
+
readonly errorManager: ErrorManager;
|
|
42
|
+
readonly fileLoader: FileLoader;
|
|
43
|
+
readonly historyStorage: HistoryStorage;
|
|
44
|
+
readonly progressManager: ProgressManager;
|
|
45
|
+
readonly reporterRegistry: ReporterRegistry;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Abstract benchmark execution engine with dependency injection
|
|
50
|
+
*
|
|
51
|
+
* Provides generic orchestration logic for benchmark discovery, validation, and
|
|
52
|
+
* execution. Concrete implementations must provide the task execution logic via
|
|
53
|
+
* the executeBenchmarkTask method.
|
|
54
|
+
*/
|
|
55
|
+
export abstract class ModestBenchEngine implements BenchmarkEngine {
|
|
56
|
+
public readonly configManager: ConfigurationManager;
|
|
57
|
+
|
|
58
|
+
public readonly errorManager: ErrorManager;
|
|
59
|
+
|
|
60
|
+
public readonly fileLoader: FileLoader;
|
|
61
|
+
|
|
62
|
+
public readonly historyStorage: HistoryStorage;
|
|
63
|
+
|
|
64
|
+
public readonly progressManager: ProgressManager;
|
|
65
|
+
|
|
66
|
+
public readonly reporterRegistry: ReporterRegistry;
|
|
67
|
+
|
|
68
|
+
constructor(dependencies: EngineDependencies) {
|
|
69
|
+
this.configManager = dependencies.configManager;
|
|
70
|
+
this.fileLoader = dependencies.fileLoader;
|
|
71
|
+
this.reporterRegistry = dependencies.reporterRegistry;
|
|
72
|
+
this.historyStorage = dependencies.historyStorage;
|
|
73
|
+
this.progressManager = dependencies.progressManager;
|
|
74
|
+
this.errorManager = dependencies.errorManager;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Discover benchmark files matching the pattern(s)
|
|
79
|
+
*/
|
|
80
|
+
async discover(
|
|
81
|
+
pattern: string | string[],
|
|
82
|
+
exclude?: string[],
|
|
83
|
+
): Promise<string[]> {
|
|
84
|
+
try {
|
|
85
|
+
return await this.fileLoader.discover(pattern, exclude);
|
|
86
|
+
} catch (error) {
|
|
87
|
+
const discoveryError =
|
|
88
|
+
error instanceof Error ? error : new Error(String(error));
|
|
89
|
+
this.errorManager.handleError(discoveryError, {
|
|
90
|
+
metadata: { exclude, pattern },
|
|
91
|
+
phase: 'discovery',
|
|
92
|
+
timestamp: new Date(),
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
throw new Error(`File discovery failed: ${discoveryError.message}`);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Execute benchmarks with the given configuration
|
|
101
|
+
*/
|
|
102
|
+
async execute(
|
|
103
|
+
config: RunConfiguration,
|
|
104
|
+
reporters: Reporter[] = [],
|
|
105
|
+
signal?: AbortSignal,
|
|
106
|
+
): Promise<BenchmarkRun> {
|
|
107
|
+
const startTime = new Date();
|
|
108
|
+
let currentPhase: ExecutionPhase = 'discovery';
|
|
109
|
+
|
|
110
|
+
try {
|
|
111
|
+
// 1. Merge configuration with defaults
|
|
112
|
+
currentPhase = 'discovery';
|
|
113
|
+
const mergedConfig = await this.configManager.load(
|
|
114
|
+
undefined, // No specific config path for now
|
|
115
|
+
config as Record<string, unknown>,
|
|
116
|
+
);
|
|
117
|
+
|
|
118
|
+
// 2. Discover files if not explicitly provided
|
|
119
|
+
const files =
|
|
120
|
+
config.files ||
|
|
121
|
+
(await this.discover(mergedConfig.pattern, mergedConfig.exclude));
|
|
122
|
+
|
|
123
|
+
if (files.length === 0) {
|
|
124
|
+
const error = new Error(
|
|
125
|
+
'No benchmark files found matching the pattern',
|
|
126
|
+
);
|
|
127
|
+
this.errorManager.handleError(error, {
|
|
128
|
+
phase: currentPhase,
|
|
129
|
+
timestamp: new Date(),
|
|
130
|
+
});
|
|
131
|
+
throw error;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// 3. Validate files
|
|
135
|
+
currentPhase = 'validation';
|
|
136
|
+
const validationResult = await this.validate(files);
|
|
137
|
+
if (!validationResult.valid) {
|
|
138
|
+
const error = new Error(
|
|
139
|
+
`Validation failed: ${validationResult.errors.map((e) => e.message).join(', ')}`,
|
|
140
|
+
);
|
|
141
|
+
this.errorManager.handleError(error, {
|
|
142
|
+
phase: currentPhase,
|
|
143
|
+
timestamp: new Date(),
|
|
144
|
+
});
|
|
145
|
+
throw error;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// 4. Initialize progress tracking
|
|
149
|
+
currentPhase = 'setup';
|
|
150
|
+
const runId = this.generateRunId();
|
|
151
|
+
|
|
152
|
+
// Pre-calculate total tasks for progress tracking
|
|
153
|
+
let totalTasks = 0;
|
|
154
|
+
let totalSuites = 0;
|
|
155
|
+
|
|
156
|
+
for (const filePath of files) {
|
|
157
|
+
try {
|
|
158
|
+
const benchmarkFile = await this.fileLoader.load(filePath);
|
|
159
|
+
const benchmarkDef = benchmarkFile.exports as BenchmarkDefinition;
|
|
160
|
+
|
|
161
|
+
if (benchmarkDef?.suites && typeof benchmarkDef.suites === 'object') {
|
|
162
|
+
const fileTags = benchmarkDef.tags;
|
|
163
|
+
|
|
164
|
+
for (const [_suiteName, suiteData] of Object.entries(
|
|
165
|
+
benchmarkDef.suites,
|
|
166
|
+
)) {
|
|
167
|
+
// Use shared filtering logic
|
|
168
|
+
const { anyTaskMatches, suiteMatches, tasksToRun } =
|
|
169
|
+
this.getFilteredTasksForSuite(
|
|
170
|
+
suiteData,
|
|
171
|
+
fileTags,
|
|
172
|
+
mergedConfig.tags,
|
|
173
|
+
mergedConfig.excludeTags,
|
|
174
|
+
);
|
|
175
|
+
|
|
176
|
+
// Count suite only if it or any of its tasks match
|
|
177
|
+
if (suiteMatches || anyTaskMatches) {
|
|
178
|
+
totalSuites++;
|
|
179
|
+
totalTasks += tasksToRun.length;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
} catch (error) {
|
|
184
|
+
// If we can't load a file for counting, we'll handle it during execution
|
|
185
|
+
// Only show warning if not in quiet mode
|
|
186
|
+
if (!mergedConfig.quiet) {
|
|
187
|
+
console.warn(
|
|
188
|
+
`Warning: Could not pre-load ${filePath} for task counting:`,
|
|
189
|
+
error,
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// Create initial run structure for progress tracking
|
|
196
|
+
const gitInfo = await this.getGitInfo();
|
|
197
|
+
const ciInfo = await this.getCiInfo();
|
|
198
|
+
|
|
199
|
+
const initialRun: BenchmarkRun = {
|
|
200
|
+
config: mergedConfig,
|
|
201
|
+
duration: 0,
|
|
202
|
+
endTime: startTime,
|
|
203
|
+
environment: await this.getEnvironmentInfo(),
|
|
204
|
+
files: [],
|
|
205
|
+
id: runId,
|
|
206
|
+
startTime,
|
|
207
|
+
...(gitInfo && { git: gitInfo }),
|
|
208
|
+
...(ciInfo && { ci: ciInfo }),
|
|
209
|
+
summary: {
|
|
210
|
+
failedTasks: 0,
|
|
211
|
+
fastest: null,
|
|
212
|
+
overallMean: 0,
|
|
213
|
+
passedTasks: 0,
|
|
214
|
+
slowest: null,
|
|
215
|
+
totalFiles: files.length,
|
|
216
|
+
totalOperations: 0,
|
|
217
|
+
totalSuites,
|
|
218
|
+
totalTasks,
|
|
219
|
+
},
|
|
220
|
+
};
|
|
221
|
+
|
|
222
|
+
this.progressManager.initialize(initialRun);
|
|
223
|
+
|
|
224
|
+
// Register progress callbacks with reporters that support them
|
|
225
|
+
for (const reporter of reporters) {
|
|
226
|
+
if (typeof reporter.onProgress === 'function') {
|
|
227
|
+
this.progressManager.onProgress((state) => {
|
|
228
|
+
void reporter.onProgress(state);
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// 5. Call reporter onStart lifecycle method
|
|
234
|
+
await this.callReporters(reporters, 'onStart', initialRun);
|
|
235
|
+
|
|
236
|
+
// 6. Execute benchmark files
|
|
237
|
+
currentPhase = 'execution';
|
|
238
|
+
const fileResults: FileResult[] = [];
|
|
239
|
+
|
|
240
|
+
for (const filePath of files) {
|
|
241
|
+
try {
|
|
242
|
+
// Call reporter onFileStart
|
|
243
|
+
await this.callReporters(reporters, 'onFileStart', filePath);
|
|
244
|
+
|
|
245
|
+
const fileResult = await this.executeBenchmarkFile(
|
|
246
|
+
filePath,
|
|
247
|
+
mergedConfig,
|
|
248
|
+
reporters,
|
|
249
|
+
signal,
|
|
250
|
+
);
|
|
251
|
+
fileResults.push(fileResult);
|
|
252
|
+
|
|
253
|
+
// Call reporter onFileEnd
|
|
254
|
+
await this.callReporters(reporters, 'onFileEnd', fileResult);
|
|
255
|
+
|
|
256
|
+
// Update progress
|
|
257
|
+
this.progressManager.update({
|
|
258
|
+
currentFile: filePath,
|
|
259
|
+
filesCompleted: fileResults.length,
|
|
260
|
+
});
|
|
261
|
+
} catch (error) {
|
|
262
|
+
const fileError =
|
|
263
|
+
error instanceof Error ? error : new Error(String(error));
|
|
264
|
+
this.errorManager.handleError(fileError, {
|
|
265
|
+
file: filePath,
|
|
266
|
+
phase: currentPhase,
|
|
267
|
+
timestamp: new Date(),
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
// Call reporter onError
|
|
271
|
+
await this.callReporters(reporters, 'onError', fileError);
|
|
272
|
+
|
|
273
|
+
// Create error result for this file
|
|
274
|
+
const now = new Date();
|
|
275
|
+
const errorResult = {
|
|
276
|
+
duration: 0,
|
|
277
|
+
endTime: now,
|
|
278
|
+
error: fileError,
|
|
279
|
+
filePath,
|
|
280
|
+
startTime: now,
|
|
281
|
+
suites: [],
|
|
282
|
+
};
|
|
283
|
+
fileResults.push(errorResult);
|
|
284
|
+
|
|
285
|
+
// Call reporter onFileEnd for error case
|
|
286
|
+
await this.callReporters(reporters, 'onFileEnd', errorResult);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// Calculate summary statistics
|
|
291
|
+
const finalTotalSuites = fileResults.reduce(
|
|
292
|
+
(sum, file) => sum + file.suites.length,
|
|
293
|
+
0,
|
|
294
|
+
);
|
|
295
|
+
const allTasks = fileResults.flatMap((file) =>
|
|
296
|
+
file.suites.flatMap((suite: SuiteResult) => suite.tasks),
|
|
297
|
+
);
|
|
298
|
+
const finalTotalTasks = allTasks.length;
|
|
299
|
+
const failedTasks = allTasks.filter((task) => task.error).length;
|
|
300
|
+
const passedTasks = finalTotalTasks - failedTasks;
|
|
301
|
+
|
|
302
|
+
let fastest: null | TaskResult = null;
|
|
303
|
+
let slowest: null | TaskResult = null;
|
|
304
|
+
let totalOperations = 0;
|
|
305
|
+
let totalTime = 0;
|
|
306
|
+
|
|
307
|
+
for (const task of allTasks) {
|
|
308
|
+
if (!task.error) {
|
|
309
|
+
totalOperations += task.iterations;
|
|
310
|
+
totalTime += task.mean * task.iterations;
|
|
311
|
+
|
|
312
|
+
if (!fastest || task.mean < fastest.mean) {
|
|
313
|
+
fastest = task;
|
|
314
|
+
}
|
|
315
|
+
if (!slowest || task.mean > slowest.mean) {
|
|
316
|
+
slowest = task;
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
const overallMean = totalOperations > 0 ? totalTime / totalOperations : 0;
|
|
322
|
+
|
|
323
|
+
const endTime = new Date();
|
|
324
|
+
const finalRun: BenchmarkRun = {
|
|
325
|
+
...initialRun,
|
|
326
|
+
duration: endTime.getTime() - startTime.getTime(),
|
|
327
|
+
endTime,
|
|
328
|
+
files: fileResults,
|
|
329
|
+
summary: {
|
|
330
|
+
failedTasks,
|
|
331
|
+
fastest,
|
|
332
|
+
overallMean,
|
|
333
|
+
passedTasks,
|
|
334
|
+
slowest,
|
|
335
|
+
totalFiles: files.length,
|
|
336
|
+
totalOperations,
|
|
337
|
+
totalSuites: finalTotalSuites,
|
|
338
|
+
totalTasks: finalTotalTasks,
|
|
339
|
+
},
|
|
340
|
+
};
|
|
341
|
+
|
|
342
|
+
// 7. Save to history
|
|
343
|
+
await this.historyStorage.saveRun(finalRun);
|
|
344
|
+
|
|
345
|
+
// 8. Call reporter onEnd lifecycle method
|
|
346
|
+
await this.callReporters(reporters, 'onEnd', finalRun);
|
|
347
|
+
|
|
348
|
+
// 9. Return completed run
|
|
349
|
+
return finalRun;
|
|
350
|
+
} catch (error) {
|
|
351
|
+
const executionError =
|
|
352
|
+
error instanceof Error ? error : new Error(String(error));
|
|
353
|
+
const handledError = this.errorManager.handleError(executionError, {
|
|
354
|
+
phase: currentPhase,
|
|
355
|
+
timestamp: new Date(),
|
|
356
|
+
});
|
|
357
|
+
|
|
358
|
+
// Re-throw the original error with more context
|
|
359
|
+
throw new Error(`Benchmark execution failed: ${handledError.message}`);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* Get the configuration manager
|
|
365
|
+
*/
|
|
366
|
+
getConfigManager(): ConfigurationManager {
|
|
367
|
+
return this.configManager;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/**
|
|
371
|
+
* Get the file loader
|
|
372
|
+
*/
|
|
373
|
+
getFileLoader(): FileLoader {
|
|
374
|
+
return this.fileLoader;
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
/**
|
|
378
|
+
* Get the history storage
|
|
379
|
+
*/
|
|
380
|
+
getHistoryStorage(): HistoryStorage {
|
|
381
|
+
return this.historyStorage;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
/**
|
|
385
|
+
* Get the progress manager
|
|
386
|
+
*/
|
|
387
|
+
getProgressManager(): ProgressManager {
|
|
388
|
+
return this.progressManager;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/**
|
|
392
|
+
* Get all available reporters
|
|
393
|
+
*/
|
|
394
|
+
getReporters(): Record<string, Reporter> {
|
|
395
|
+
return this.reporterRegistry.getAll();
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/**
|
|
399
|
+
* Register a custom reporter
|
|
400
|
+
*/
|
|
401
|
+
registerReporter(name: string, reporter: Reporter): void {
|
|
402
|
+
this.reporterRegistry.register(name, reporter);
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
/**
|
|
406
|
+
* Validate benchmark files without executing them
|
|
407
|
+
*/
|
|
408
|
+
async validate(files: string[]): Promise<ValidationResult> {
|
|
409
|
+
try {
|
|
410
|
+
const errors: ValidationError[] = [];
|
|
411
|
+
const warnings: ValidationWarning[] = [];
|
|
412
|
+
const validatedFiles: string[] = [];
|
|
413
|
+
|
|
414
|
+
// Validate each file
|
|
415
|
+
for (const file of files) {
|
|
416
|
+
try {
|
|
417
|
+
const result = await this.fileLoader.validate(file);
|
|
418
|
+
validatedFiles.push(file);
|
|
419
|
+
errors.push(...result.errors);
|
|
420
|
+
warnings.push(...result.warnings);
|
|
421
|
+
} catch (error) {
|
|
422
|
+
const validationError =
|
|
423
|
+
error instanceof Error ? error : new Error(String(error));
|
|
424
|
+
this.errorManager.handleError(validationError, {
|
|
425
|
+
file,
|
|
426
|
+
phase: 'validation',
|
|
427
|
+
timestamp: new Date(),
|
|
428
|
+
});
|
|
429
|
+
|
|
430
|
+
errors.push({
|
|
431
|
+
code: 'FILE_VALIDATION_ERROR',
|
|
432
|
+
file,
|
|
433
|
+
message: `Failed to validate file: ${validationError.message}`,
|
|
434
|
+
severity: 'error' as const,
|
|
435
|
+
});
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
return {
|
|
440
|
+
errors,
|
|
441
|
+
files: validatedFiles,
|
|
442
|
+
valid: errors.length === 0,
|
|
443
|
+
warnings,
|
|
444
|
+
};
|
|
445
|
+
} catch (error) {
|
|
446
|
+
throw new Error(
|
|
447
|
+
`Validation failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
448
|
+
);
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/**
|
|
453
|
+
* Execute a single benchmark task
|
|
454
|
+
*
|
|
455
|
+
* This method must be implemented by concrete engine implementations to
|
|
456
|
+
* provide the actual benchmark execution logic.
|
|
457
|
+
*
|
|
458
|
+
* @param taskName - Name of the task being executed
|
|
459
|
+
* @param taskData - Task definition with function and metadata
|
|
460
|
+
* @param config - Benchmark configuration
|
|
461
|
+
* @param reporters - Array of active reporters
|
|
462
|
+
* @param signal - Optional abort signal for cancellation
|
|
463
|
+
* @returns Promise resolving to task execution result
|
|
464
|
+
*/
|
|
465
|
+
protected abstract executeBenchmarkTask(
|
|
466
|
+
taskName: string,
|
|
467
|
+
taskData: BenchmarkTask,
|
|
468
|
+
config: ModestBenchConfig,
|
|
469
|
+
reporters: Reporter[],
|
|
470
|
+
signal?: AbortSignal,
|
|
471
|
+
): Promise<TaskResult>;
|
|
472
|
+
|
|
473
|
+
/**
|
|
474
|
+
* Helper method to call a lifecycle method on all reporters
|
|
475
|
+
*/
|
|
476
|
+
private async callReporters(
|
|
477
|
+
reporters: Reporter[],
|
|
478
|
+
method: keyof Reporter,
|
|
479
|
+
...args: unknown[]
|
|
480
|
+
): Promise<void> {
|
|
481
|
+
for (const reporter of reporters) {
|
|
482
|
+
try {
|
|
483
|
+
const reporterMethod = reporter[method];
|
|
484
|
+
if (typeof reporterMethod === 'function') {
|
|
485
|
+
const result = (
|
|
486
|
+
reporterMethod as (...args: unknown[]) => unknown
|
|
487
|
+
).call(reporter, ...args);
|
|
488
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
|
489
|
+
if (result && typeof (result as any).then === 'function') {
|
|
490
|
+
await (result as Promise<void>);
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
} catch (error) {
|
|
494
|
+
// Log reporter errors but don't fail the benchmark run
|
|
495
|
+
console.error(`Reporter error in ${method}:`, error);
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
/**
|
|
501
|
+
* Execute a single benchmark file and return its results
|
|
502
|
+
*/
|
|
503
|
+
private async executeBenchmarkFile(
|
|
504
|
+
filePath: string,
|
|
505
|
+
config: ModestBenchConfig,
|
|
506
|
+
reporters: Reporter[] = [],
|
|
507
|
+
signal?: AbortSignal,
|
|
508
|
+
): Promise<FileResult> {
|
|
509
|
+
const startTime = new Date();
|
|
510
|
+
|
|
511
|
+
try {
|
|
512
|
+
// Load the benchmark file using the file loader
|
|
513
|
+
const benchmarkFile = await this.fileLoader.load(filePath);
|
|
514
|
+
const benchmarkDef = benchmarkFile.exports as BenchmarkDefinition;
|
|
515
|
+
|
|
516
|
+
if (!benchmarkDef || typeof benchmarkDef !== 'object') {
|
|
517
|
+
throw new Error(
|
|
518
|
+
'Benchmark file must export a default object with suites',
|
|
519
|
+
);
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
const suiteResults: SuiteResult[] = [];
|
|
523
|
+
const fileTags = benchmarkDef.tags;
|
|
524
|
+
|
|
525
|
+
// Process each suite in the file
|
|
526
|
+
if (benchmarkDef.suites && typeof benchmarkDef.suites === 'object') {
|
|
527
|
+
for (const [suiteName, suiteData] of Object.entries(
|
|
528
|
+
benchmarkDef.suites,
|
|
529
|
+
)) {
|
|
530
|
+
// Use shared filtering logic
|
|
531
|
+
const { anyTaskMatches, suiteMatches } =
|
|
532
|
+
this.getFilteredTasksForSuite(
|
|
533
|
+
suiteData,
|
|
534
|
+
fileTags,
|
|
535
|
+
config.tags,
|
|
536
|
+
config.excludeTags,
|
|
537
|
+
);
|
|
538
|
+
|
|
539
|
+
// Skip suite only if neither the suite nor any of its tasks match
|
|
540
|
+
if (!suiteMatches && !anyTaskMatches) {
|
|
541
|
+
continue;
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
await this.callReporters(reporters, 'onSuiteStart', suiteName);
|
|
545
|
+
const suiteResult = await this.executeBenchmarkSuite(
|
|
546
|
+
suiteName,
|
|
547
|
+
suiteData,
|
|
548
|
+
config,
|
|
549
|
+
reporters,
|
|
550
|
+
signal,
|
|
551
|
+
fileTags,
|
|
552
|
+
);
|
|
553
|
+
await this.callReporters(reporters, 'onSuiteEnd', suiteResult);
|
|
554
|
+
suiteResults.push(suiteResult);
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
const endTime = new Date();
|
|
559
|
+
|
|
560
|
+
return {
|
|
561
|
+
config: benchmarkDef.config,
|
|
562
|
+
duration: endTime.getTime() - startTime.getTime(),
|
|
563
|
+
endTime,
|
|
564
|
+
filePath,
|
|
565
|
+
startTime,
|
|
566
|
+
suites: suiteResults,
|
|
567
|
+
};
|
|
568
|
+
} catch (error) {
|
|
569
|
+
const endTime = new Date();
|
|
570
|
+
const executionError =
|
|
571
|
+
error instanceof Error ? error : new Error(String(error));
|
|
572
|
+
|
|
573
|
+
return {
|
|
574
|
+
duration: endTime.getTime() - startTime.getTime(),
|
|
575
|
+
endTime,
|
|
576
|
+
error: executionError,
|
|
577
|
+
filePath,
|
|
578
|
+
startTime,
|
|
579
|
+
suites: [],
|
|
580
|
+
};
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
/**
|
|
585
|
+
* Execute a single benchmark suite and return its results
|
|
586
|
+
*/
|
|
587
|
+
private async executeBenchmarkSuite(
|
|
588
|
+
suiteName: string,
|
|
589
|
+
suiteData: BenchmarkSuite,
|
|
590
|
+
config: ModestBenchConfig,
|
|
591
|
+
reporters: Reporter[] = [],
|
|
592
|
+
signal?: AbortSignal,
|
|
593
|
+
fileTags?: string[],
|
|
594
|
+
): Promise<SuiteResult> {
|
|
595
|
+
const startTime = new Date();
|
|
596
|
+
|
|
597
|
+
try {
|
|
598
|
+
const taskResults: TaskResult[] = [];
|
|
599
|
+
|
|
600
|
+
// Use shared filtering logic to determine which tasks will run
|
|
601
|
+
const { tasksToRun } = this.getFilteredTasksForSuite(
|
|
602
|
+
suiteData,
|
|
603
|
+
fileTags,
|
|
604
|
+
config.tags,
|
|
605
|
+
config.excludeTags,
|
|
606
|
+
);
|
|
607
|
+
|
|
608
|
+
// Only run setup/teardown if there are tasks to execute
|
|
609
|
+
if (tasksToRun.length === 0) {
|
|
610
|
+
// No tasks match the filters, return empty suite result
|
|
611
|
+
const endTime = new Date();
|
|
612
|
+
return {
|
|
613
|
+
duration: endTime.getTime() - startTime.getTime(),
|
|
614
|
+
endTime,
|
|
615
|
+
name: suiteName,
|
|
616
|
+
startTime,
|
|
617
|
+
tasks: [],
|
|
618
|
+
...(suiteData.config !== undefined && { config: suiteData.config }),
|
|
619
|
+
...(suiteData.metadata !== undefined && {
|
|
620
|
+
metadata: suiteData.metadata,
|
|
621
|
+
}),
|
|
622
|
+
...(suiteData.tags !== undefined && { tags: suiteData.tags }),
|
|
623
|
+
};
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
// Run suite setup if provided
|
|
627
|
+
if (suiteData.setup && typeof suiteData.setup === 'function') {
|
|
628
|
+
try {
|
|
629
|
+
await suiteData.setup();
|
|
630
|
+
} catch (error) {
|
|
631
|
+
const setupError =
|
|
632
|
+
error instanceof Error
|
|
633
|
+
? error
|
|
634
|
+
: new Error(`Setup failed: ${String(error)}`);
|
|
635
|
+
throw new Error(`Suite setup failed: ${setupError.message}`);
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
try {
|
|
640
|
+
// Process each task that passed filtering
|
|
641
|
+
for (const [taskName, taskData] of tasksToRun) {
|
|
642
|
+
await this.callReporters(reporters, 'onTaskStart', taskName);
|
|
643
|
+
|
|
644
|
+
// Mark task as in-progress (shows as 0.5 progress for current task)
|
|
645
|
+
const currentState = this.progressManager.getState();
|
|
646
|
+
this.progressManager.update({
|
|
647
|
+
tasksCompleted: currentState.tasksCompleted + 0.5,
|
|
648
|
+
});
|
|
649
|
+
|
|
650
|
+
const taskResult = await this.executeBenchmarkTask(
|
|
651
|
+
taskName,
|
|
652
|
+
taskData,
|
|
653
|
+
config,
|
|
654
|
+
reporters,
|
|
655
|
+
signal,
|
|
656
|
+
);
|
|
657
|
+
await this.callReporters(reporters, 'onTaskResult', taskResult);
|
|
658
|
+
taskResults.push(taskResult);
|
|
659
|
+
|
|
660
|
+
// Update task-level progress - task is now complete (remove the 0.5 and add 1)
|
|
661
|
+
this.progressManager.update({
|
|
662
|
+
tasksCompleted: currentState.tasksCompleted + 1,
|
|
663
|
+
});
|
|
664
|
+
}
|
|
665
|
+
} finally {
|
|
666
|
+
// Run suite teardown if provided (always runs, even if benchmarks fail)
|
|
667
|
+
if (suiteData.teardown && typeof suiteData.teardown === 'function') {
|
|
668
|
+
try {
|
|
669
|
+
await suiteData.teardown();
|
|
670
|
+
} catch (error) {
|
|
671
|
+
// Log teardown errors but don't fail the suite
|
|
672
|
+
const teardownError =
|
|
673
|
+
error instanceof Error ? error : new Error(String(error));
|
|
674
|
+
console.error(
|
|
675
|
+
`Warning: Suite teardown failed for "${suiteName}":`,
|
|
676
|
+
teardownError.message,
|
|
677
|
+
);
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
const endTime = new Date();
|
|
683
|
+
|
|
684
|
+
return {
|
|
685
|
+
duration: endTime.getTime() - startTime.getTime(),
|
|
686
|
+
endTime,
|
|
687
|
+
name: suiteName,
|
|
688
|
+
startTime,
|
|
689
|
+
tasks: taskResults,
|
|
690
|
+
...(suiteData.config !== undefined && { config: suiteData.config }),
|
|
691
|
+
...(suiteData.metadata !== undefined && {
|
|
692
|
+
metadata: suiteData.metadata,
|
|
693
|
+
}),
|
|
694
|
+
...(suiteData.tags !== undefined && { tags: suiteData.tags }),
|
|
695
|
+
};
|
|
696
|
+
} catch (error) {
|
|
697
|
+
const endTime = new Date();
|
|
698
|
+
const executionError =
|
|
699
|
+
error instanceof Error ? error : new Error(String(error));
|
|
700
|
+
|
|
701
|
+
return {
|
|
702
|
+
duration: endTime.getTime() - startTime.getTime(),
|
|
703
|
+
endTime,
|
|
704
|
+
error: executionError,
|
|
705
|
+
name: suiteName,
|
|
706
|
+
startTime,
|
|
707
|
+
tasks: [],
|
|
708
|
+
};
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
/**
|
|
713
|
+
* Generate a unique run ID
|
|
714
|
+
*/
|
|
715
|
+
private generateRunId(): string {
|
|
716
|
+
return `run-${Date.now()}-${Math.random().toString(36).substring(2, 15)}`;
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
/**
|
|
720
|
+
* Get CI/CD information if available
|
|
721
|
+
*/
|
|
722
|
+
private async getCiInfo(): Promise<CiInfo | undefined> {
|
|
723
|
+
const process = await import('node:process');
|
|
724
|
+
|
|
725
|
+
if (!process.env.CI) {
|
|
726
|
+
return undefined;
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
// Detect common CI providers
|
|
730
|
+
if (process.env.GITHUB_ACTIONS) {
|
|
731
|
+
return {
|
|
732
|
+
provider: 'GitHub Actions',
|
|
733
|
+
...(process.env.GITHUB_RUN_NUMBER && {
|
|
734
|
+
buildNumber: process.env.GITHUB_RUN_NUMBER,
|
|
735
|
+
}),
|
|
736
|
+
...(process.env.GITHUB_REPOSITORY &&
|
|
737
|
+
process.env.GITHUB_RUN_ID && {
|
|
738
|
+
buildUrl: `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`,
|
|
739
|
+
}),
|
|
740
|
+
...(process.env.GITHUB_EVENT_NAME === 'pull_request' &&
|
|
741
|
+
process.env.GITHUB_REF_NAME && {
|
|
742
|
+
pullRequest: process.env.GITHUB_REF_NAME,
|
|
743
|
+
}),
|
|
744
|
+
...(process.env.GITHUB_REF_NAME && {
|
|
745
|
+
branch: process.env.GITHUB_REF_NAME,
|
|
746
|
+
}),
|
|
747
|
+
...(process.env.GITHUB_SHA && { commit: process.env.GITHUB_SHA }),
|
|
748
|
+
};
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
// Default CI info
|
|
752
|
+
return {
|
|
753
|
+
provider: 'Unknown CI',
|
|
754
|
+
...(process.env.BRANCH && { branch: process.env.BRANCH }),
|
|
755
|
+
...(process.env.COMMIT && { commit: process.env.COMMIT }),
|
|
756
|
+
};
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
/**
|
|
760
|
+
* Get environment information
|
|
761
|
+
*/
|
|
762
|
+
private async getEnvironmentInfo(): Promise<EnvironmentInfo> {
|
|
763
|
+
const os = await import('node:os');
|
|
764
|
+
const process = await import('node:process');
|
|
765
|
+
|
|
766
|
+
return {
|
|
767
|
+
arch: process.arch,
|
|
768
|
+
availableMemory: os.freemem(),
|
|
769
|
+
cpu: {
|
|
770
|
+
cores: os.cpus().length,
|
|
771
|
+
model: os.cpus()[0]?.model || 'Unknown',
|
|
772
|
+
speed: os.cpus()[0]?.speed || 0,
|
|
773
|
+
},
|
|
774
|
+
env: {
|
|
775
|
+
CI: process.env.CI || 'false',
|
|
776
|
+
NODE_ENV: process.env.NODE_ENV || 'development',
|
|
777
|
+
},
|
|
778
|
+
hostname: os.hostname(),
|
|
779
|
+
memory: {
|
|
780
|
+
free: os.freemem(),
|
|
781
|
+
total: os.totalmem(),
|
|
782
|
+
used: os.totalmem() - os.freemem(),
|
|
783
|
+
},
|
|
784
|
+
nodeVersion: process.version,
|
|
785
|
+
platform: process.platform,
|
|
786
|
+
};
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
/**
|
|
790
|
+
* Get filtered tasks for a suite based on tag filtering Returns suite match
|
|
791
|
+
* status and list of tasks to run
|
|
792
|
+
*/
|
|
793
|
+
private getFilteredTasksForSuite(
|
|
794
|
+
suiteData: BenchmarkSuite,
|
|
795
|
+
fileTags: string[] | undefined,
|
|
796
|
+
includeTags: string[],
|
|
797
|
+
excludeTags: string[],
|
|
798
|
+
): {
|
|
799
|
+
anyTaskMatches: boolean;
|
|
800
|
+
suiteMatches: boolean;
|
|
801
|
+
tasksToRun: Array<[string, BenchmarkTask]>;
|
|
802
|
+
} {
|
|
803
|
+
// Check if suite itself matches filters
|
|
804
|
+
const mergedSuiteTags = this.mergeTags(fileTags, suiteData.tags);
|
|
805
|
+
const suiteMatches = this.matchesTags(
|
|
806
|
+
mergedSuiteTags,
|
|
807
|
+
includeTags,
|
|
808
|
+
excludeTags,
|
|
809
|
+
);
|
|
810
|
+
|
|
811
|
+
// Check which tasks match filters
|
|
812
|
+
const tasksToRun: Array<[string, BenchmarkTask]> = [];
|
|
813
|
+
if (suiteData.benchmarks && typeof suiteData.benchmarks === 'object') {
|
|
814
|
+
for (const [taskName, taskData] of Object.entries(suiteData.benchmarks)) {
|
|
815
|
+
// Merge task tags with suite and file tags (cascading)
|
|
816
|
+
const mergedTaskTags = this.mergeTags(mergedSuiteTags, taskData.tags);
|
|
817
|
+
|
|
818
|
+
// Check if task matches tag filters
|
|
819
|
+
if (this.matchesTags(mergedTaskTags, includeTags, excludeTags)) {
|
|
820
|
+
tasksToRun.push([taskName, taskData]);
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
return {
|
|
826
|
+
anyTaskMatches: tasksToRun.length > 0,
|
|
827
|
+
suiteMatches,
|
|
828
|
+
tasksToRun,
|
|
829
|
+
};
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
/**
|
|
833
|
+
* Get Git information if available
|
|
834
|
+
*/
|
|
835
|
+
private async getGitInfo(): Promise<GitInfo | undefined> {
|
|
836
|
+
// TODO: Implement Git information extraction
|
|
837
|
+
// This would use child_process to run git commands
|
|
838
|
+
return undefined;
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
/**
|
|
842
|
+
* Check if item tags match the filter criteria (OR logic)
|
|
843
|
+
*/
|
|
844
|
+
private matchesTags(
|
|
845
|
+
itemTags: string[] | undefined,
|
|
846
|
+
includeTags: string[],
|
|
847
|
+
excludeTags: string[],
|
|
848
|
+
): boolean {
|
|
849
|
+
const tags = itemTags || [];
|
|
850
|
+
|
|
851
|
+
// If exclude tags specified and any match, exclude this item
|
|
852
|
+
if (
|
|
853
|
+
excludeTags.length > 0 &&
|
|
854
|
+
excludeTags.some((tag) => tags.includes(tag))
|
|
855
|
+
) {
|
|
856
|
+
return false;
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
// If include tags specified, at least one must match
|
|
860
|
+
if (includeTags.length > 0) {
|
|
861
|
+
return includeTags.some((tag) => tags.includes(tag));
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
// No filters = include everything
|
|
865
|
+
return true;
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
/**
|
|
869
|
+
* Merge tags from parent to child (cascading)
|
|
870
|
+
*/
|
|
871
|
+
private mergeTags(
|
|
872
|
+
parentTags?: string[],
|
|
873
|
+
childTags?: string[],
|
|
874
|
+
): string[] | undefined {
|
|
875
|
+
const merged = new Set<string>();
|
|
876
|
+
if (parentTags) {
|
|
877
|
+
for (const tag of parentTags) {
|
|
878
|
+
merged.add(tag);
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
if (childTags) {
|
|
882
|
+
for (const tag of childTags) {
|
|
883
|
+
merged.add(tag);
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
return merged.size > 0 ? Array.from(merged) : undefined;
|
|
887
|
+
}
|
|
888
|
+
}
|