modestbench 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (275) hide show
  1. package/CHANGELOG.md +45 -0
  2. package/LICENSE.md +55 -0
  3. package/README.md +699 -0
  4. package/dist/bootstrap.cjs +37 -0
  5. package/dist/bootstrap.cjs.map +1 -0
  6. package/dist/bootstrap.d.cts +17 -0
  7. package/dist/bootstrap.d.cts.map +1 -0
  8. package/dist/bootstrap.d.ts +17 -0
  9. package/dist/bootstrap.d.ts.map +1 -0
  10. package/dist/bootstrap.js +33 -0
  11. package/dist/bootstrap.js.map +1 -0
  12. package/dist/cli/commands/history.cjs +459 -0
  13. package/dist/cli/commands/history.cjs.map +1 -0
  14. package/dist/cli/commands/history.d.cts +34 -0
  15. package/dist/cli/commands/history.d.cts.map +1 -0
  16. package/dist/cli/commands/history.d.ts +34 -0
  17. package/dist/cli/commands/history.d.ts.map +1 -0
  18. package/dist/cli/commands/history.js +422 -0
  19. package/dist/cli/commands/history.js.map +1 -0
  20. package/dist/cli/commands/init.cjs +566 -0
  21. package/dist/cli/commands/init.cjs.map +1 -0
  22. package/dist/cli/commands/init.d.cts +26 -0
  23. package/dist/cli/commands/init.d.cts.map +1 -0
  24. package/dist/cli/commands/init.d.ts +26 -0
  25. package/dist/cli/commands/init.d.ts.map +1 -0
  26. package/dist/cli/commands/init.js +562 -0
  27. package/dist/cli/commands/init.js.map +1 -0
  28. package/dist/cli/commands/run.cjs +285 -0
  29. package/dist/cli/commands/run.cjs.map +1 -0
  30. package/dist/cli/commands/run.d.cts +37 -0
  31. package/dist/cli/commands/run.d.cts.map +1 -0
  32. package/dist/cli/commands/run.d.ts +37 -0
  33. package/dist/cli/commands/run.d.ts.map +1 -0
  34. package/dist/cli/commands/run.js +248 -0
  35. package/dist/cli/commands/run.js.map +1 -0
  36. package/dist/cli/index.cjs +523 -0
  37. package/dist/cli/index.cjs.map +1 -0
  38. package/dist/cli/index.d.cts +58 -0
  39. package/dist/cli/index.d.cts.map +1 -0
  40. package/dist/cli/index.d.ts +58 -0
  41. package/dist/cli/index.d.ts.map +1 -0
  42. package/dist/cli/index.js +515 -0
  43. package/dist/cli/index.js.map +1 -0
  44. package/dist/config/manager.cjs +370 -0
  45. package/dist/config/manager.cjs.map +1 -0
  46. package/dist/config/manager.d.cts +46 -0
  47. package/dist/config/manager.d.cts.map +1 -0
  48. package/dist/config/manager.d.ts +46 -0
  49. package/dist/config/manager.d.ts.map +1 -0
  50. package/dist/config/manager.js +333 -0
  51. package/dist/config/manager.js.map +1 -0
  52. package/dist/config/schema.cjs +182 -0
  53. package/dist/config/schema.cjs.map +1 -0
  54. package/dist/config/schema.d.cts +51 -0
  55. package/dist/config/schema.d.cts.map +1 -0
  56. package/dist/config/schema.d.ts +51 -0
  57. package/dist/config/schema.d.ts.map +1 -0
  58. package/dist/config/schema.js +145 -0
  59. package/dist/config/schema.js.map +1 -0
  60. package/dist/constants.cjs +22 -0
  61. package/dist/constants.cjs.map +1 -0
  62. package/dist/constants.d.cts +10 -0
  63. package/dist/constants.d.cts.map +1 -0
  64. package/dist/constants.d.ts +10 -0
  65. package/dist/constants.d.ts.map +1 -0
  66. package/dist/constants.js +19 -0
  67. package/dist/constants.js.map +1 -0
  68. package/dist/core/benchmark-schema.cjs +135 -0
  69. package/dist/core/benchmark-schema.cjs.map +1 -0
  70. package/dist/core/benchmark-schema.d.cts +139 -0
  71. package/dist/core/benchmark-schema.d.cts.map +1 -0
  72. package/dist/core/benchmark-schema.d.ts +139 -0
  73. package/dist/core/benchmark-schema.d.ts.map +1 -0
  74. package/dist/core/benchmark-schema.js +132 -0
  75. package/dist/core/benchmark-schema.js.map +1 -0
  76. package/dist/core/engine.cjs +669 -0
  77. package/dist/core/engine.cjs.map +1 -0
  78. package/dist/core/engine.d.cts +128 -0
  79. package/dist/core/engine.d.cts.map +1 -0
  80. package/dist/core/engine.d.ts +128 -0
  81. package/dist/core/engine.d.ts.map +1 -0
  82. package/dist/core/engine.js +632 -0
  83. package/dist/core/engine.js.map +1 -0
  84. package/dist/core/engines/accurate-engine.cjs +292 -0
  85. package/dist/core/engines/accurate-engine.cjs.map +1 -0
  86. package/dist/core/engines/accurate-engine.d.cts +63 -0
  87. package/dist/core/engines/accurate-engine.d.cts.map +1 -0
  88. package/dist/core/engines/accurate-engine.d.ts +63 -0
  89. package/dist/core/engines/accurate-engine.d.ts.map +1 -0
  90. package/dist/core/engines/accurate-engine.js +288 -0
  91. package/dist/core/engines/accurate-engine.js.map +1 -0
  92. package/dist/core/engines/index.cjs +21 -0
  93. package/dist/core/engines/index.cjs.map +1 -0
  94. package/dist/core/engines/index.d.cts +16 -0
  95. package/dist/core/engines/index.d.cts.map +1 -0
  96. package/dist/core/engines/index.d.ts +16 -0
  97. package/dist/core/engines/index.d.ts.map +1 -0
  98. package/dist/core/engines/index.js +16 -0
  99. package/dist/core/engines/index.js.map +1 -0
  100. package/dist/core/engines/tinybench-engine.cjs +286 -0
  101. package/dist/core/engines/tinybench-engine.cjs.map +1 -0
  102. package/dist/core/engines/tinybench-engine.d.cts +18 -0
  103. package/dist/core/engines/tinybench-engine.d.cts.map +1 -0
  104. package/dist/core/engines/tinybench-engine.d.ts +18 -0
  105. package/dist/core/engines/tinybench-engine.d.ts.map +1 -0
  106. package/dist/core/engines/tinybench-engine.js +282 -0
  107. package/dist/core/engines/tinybench-engine.js.map +1 -0
  108. package/dist/core/error-manager.cjs +303 -0
  109. package/dist/core/error-manager.cjs.map +1 -0
  110. package/dist/core/error-manager.d.cts +77 -0
  111. package/dist/core/error-manager.d.cts.map +1 -0
  112. package/dist/core/error-manager.d.ts +77 -0
  113. package/dist/core/error-manager.d.ts.map +1 -0
  114. package/dist/core/error-manager.js +299 -0
  115. package/dist/core/error-manager.js.map +1 -0
  116. package/dist/core/loader.cjs +287 -0
  117. package/dist/core/loader.cjs.map +1 -0
  118. package/dist/core/loader.d.cts +55 -0
  119. package/dist/core/loader.d.cts.map +1 -0
  120. package/dist/core/loader.d.ts +55 -0
  121. package/dist/core/loader.d.ts.map +1 -0
  122. package/dist/core/loader.js +250 -0
  123. package/dist/core/loader.js.map +1 -0
  124. package/dist/core/stats-utils.cjs +99 -0
  125. package/dist/core/stats-utils.cjs.map +1 -0
  126. package/dist/core/stats-utils.d.cts +50 -0
  127. package/dist/core/stats-utils.d.cts.map +1 -0
  128. package/dist/core/stats-utils.d.ts +50 -0
  129. package/dist/core/stats-utils.d.ts.map +1 -0
  130. package/dist/core/stats-utils.js +94 -0
  131. package/dist/core/stats-utils.js.map +1 -0
  132. package/dist/index.cjs +64 -0
  133. package/dist/index.cjs.map +1 -0
  134. package/dist/index.d.cts +22 -0
  135. package/dist/index.d.cts.map +1 -0
  136. package/dist/index.d.ts +22 -0
  137. package/dist/index.d.ts.map +1 -0
  138. package/dist/index.js +30 -0
  139. package/dist/index.js.map +1 -0
  140. package/dist/progress/manager.cjs +325 -0
  141. package/dist/progress/manager.cjs.map +1 -0
  142. package/dist/progress/manager.d.cts +125 -0
  143. package/dist/progress/manager.d.cts.map +1 -0
  144. package/dist/progress/manager.d.ts +125 -0
  145. package/dist/progress/manager.d.ts.map +1 -0
  146. package/dist/progress/manager.js +321 -0
  147. package/dist/progress/manager.js.map +1 -0
  148. package/dist/reporters/csv.cjs +250 -0
  149. package/dist/reporters/csv.cjs.map +1 -0
  150. package/dist/reporters/csv.d.cts +92 -0
  151. package/dist/reporters/csv.d.cts.map +1 -0
  152. package/dist/reporters/csv.d.ts +92 -0
  153. package/dist/reporters/csv.d.ts.map +1 -0
  154. package/dist/reporters/csv.js +246 -0
  155. package/dist/reporters/csv.js.map +1 -0
  156. package/dist/reporters/human.cjs +516 -0
  157. package/dist/reporters/human.cjs.map +1 -0
  158. package/dist/reporters/human.d.cts +86 -0
  159. package/dist/reporters/human.d.cts.map +1 -0
  160. package/dist/reporters/human.d.ts +86 -0
  161. package/dist/reporters/human.d.ts.map +1 -0
  162. package/dist/reporters/human.js +509 -0
  163. package/dist/reporters/human.js.map +1 -0
  164. package/dist/reporters/index.cjs +17 -0
  165. package/dist/reporters/index.cjs.map +1 -0
  166. package/dist/reporters/index.d.cts +10 -0
  167. package/dist/reporters/index.d.cts.map +1 -0
  168. package/dist/reporters/index.d.ts +10 -0
  169. package/dist/reporters/index.d.ts.map +1 -0
  170. package/dist/reporters/index.js +10 -0
  171. package/dist/reporters/index.js.map +1 -0
  172. package/dist/reporters/json.cjs +215 -0
  173. package/dist/reporters/json.cjs.map +1 -0
  174. package/dist/reporters/json.d.cts +79 -0
  175. package/dist/reporters/json.d.cts.map +1 -0
  176. package/dist/reporters/json.d.ts +79 -0
  177. package/dist/reporters/json.d.ts.map +1 -0
  178. package/dist/reporters/json.js +211 -0
  179. package/dist/reporters/json.js.map +1 -0
  180. package/dist/reporters/registry.cjs +255 -0
  181. package/dist/reporters/registry.cjs.map +1 -0
  182. package/dist/reporters/registry.d.cts +155 -0
  183. package/dist/reporters/registry.d.cts.map +1 -0
  184. package/dist/reporters/registry.d.ts +155 -0
  185. package/dist/reporters/registry.d.ts.map +1 -0
  186. package/dist/reporters/registry.js +249 -0
  187. package/dist/reporters/registry.js.map +1 -0
  188. package/dist/reporters/simple.cjs +328 -0
  189. package/dist/reporters/simple.cjs.map +1 -0
  190. package/dist/reporters/simple.d.cts +51 -0
  191. package/dist/reporters/simple.d.cts.map +1 -0
  192. package/dist/reporters/simple.d.ts +51 -0
  193. package/dist/reporters/simple.d.ts.map +1 -0
  194. package/dist/reporters/simple.js +321 -0
  195. package/dist/reporters/simple.js.map +1 -0
  196. package/dist/schema/modestbench-config.schema.json +162 -0
  197. package/dist/storage/history.cjs +456 -0
  198. package/dist/storage/history.cjs.map +1 -0
  199. package/dist/storage/history.d.cts +99 -0
  200. package/dist/storage/history.d.cts.map +1 -0
  201. package/dist/storage/history.d.ts +99 -0
  202. package/dist/storage/history.d.ts.map +1 -0
  203. package/dist/storage/history.js +452 -0
  204. package/dist/storage/history.js.map +1 -0
  205. package/dist/types/cli.cjs +21 -0
  206. package/dist/types/cli.cjs.map +1 -0
  207. package/dist/types/cli.d.cts +296 -0
  208. package/dist/types/cli.d.cts.map +1 -0
  209. package/dist/types/cli.d.ts +296 -0
  210. package/dist/types/cli.d.ts.map +1 -0
  211. package/dist/types/cli.js +18 -0
  212. package/dist/types/cli.js.map +1 -0
  213. package/dist/types/core.cjs +14 -0
  214. package/dist/types/core.cjs.map +1 -0
  215. package/dist/types/core.d.cts +380 -0
  216. package/dist/types/core.d.cts.map +1 -0
  217. package/dist/types/core.d.ts +380 -0
  218. package/dist/types/core.d.ts.map +1 -0
  219. package/dist/types/core.js +13 -0
  220. package/dist/types/core.js.map +1 -0
  221. package/dist/types/index.cjs +27 -0
  222. package/dist/types/index.cjs.map +1 -0
  223. package/dist/types/index.d.cts +11 -0
  224. package/dist/types/index.d.cts.map +1 -0
  225. package/dist/types/index.d.ts +11 -0
  226. package/dist/types/index.d.ts.map +1 -0
  227. package/dist/types/index.js +11 -0
  228. package/dist/types/index.js.map +1 -0
  229. package/dist/types/interfaces.cjs +10 -0
  230. package/dist/types/interfaces.cjs.map +1 -0
  231. package/dist/types/interfaces.d.cts +381 -0
  232. package/dist/types/interfaces.d.cts.map +1 -0
  233. package/dist/types/interfaces.d.ts +381 -0
  234. package/dist/types/interfaces.d.ts.map +1 -0
  235. package/dist/types/interfaces.js +9 -0
  236. package/dist/types/interfaces.js.map +1 -0
  237. package/dist/types/utility.cjs +92 -0
  238. package/dist/types/utility.cjs.map +1 -0
  239. package/dist/types/utility.d.cts +330 -0
  240. package/dist/types/utility.d.cts.map +1 -0
  241. package/dist/types/utility.d.ts +330 -0
  242. package/dist/types/utility.d.ts.map +1 -0
  243. package/dist/types/utility.js +78 -0
  244. package/dist/types/utility.js.map +1 -0
  245. package/package.json +211 -0
  246. package/src/bootstrap.ts +35 -0
  247. package/src/cli/commands/history.ts +569 -0
  248. package/src/cli/commands/init.ts +658 -0
  249. package/src/cli/commands/run.ts +346 -0
  250. package/src/cli/index.ts +642 -0
  251. package/src/config/manager.ts +387 -0
  252. package/src/config/schema.ts +188 -0
  253. package/src/constants.ts +21 -0
  254. package/src/core/benchmark-schema.ts +185 -0
  255. package/src/core/engine.ts +888 -0
  256. package/src/core/engines/accurate-engine.ts +408 -0
  257. package/src/core/engines/index.ts +16 -0
  258. package/src/core/engines/tinybench-engine.ts +335 -0
  259. package/src/core/error-manager.ts +372 -0
  260. package/src/core/loader.ts +324 -0
  261. package/src/core/stats-utils.ts +135 -0
  262. package/src/index.ts +46 -0
  263. package/src/progress/manager.ts +415 -0
  264. package/src/reporters/csv.ts +368 -0
  265. package/src/reporters/human.ts +707 -0
  266. package/src/reporters/index.ts +10 -0
  267. package/src/reporters/json.ts +302 -0
  268. package/src/reporters/registry.ts +349 -0
  269. package/src/reporters/simple.ts +459 -0
  270. package/src/storage/history.ts +600 -0
  271. package/src/types/cli.ts +312 -0
  272. package/src/types/core.ts +414 -0
  273. package/src/types/index.ts +18 -0
  274. package/src/types/interfaces.ts +451 -0
  275. package/src/types/utility.ts +446 -0
@@ -0,0 +1,888 @@
1
+ /**
2
+ * ModestBench Core Engine
3
+ *
4
+ * Main orchestrator for benchmark discovery, validation, and execution.
5
+ * Implements the BenchmarkEngine interface with dependency injection
6
+ * architecture.
7
+ */
8
+
9
+ import type {
10
+ BenchmarkDefinition,
11
+ BenchmarkEngine,
12
+ BenchmarkRun,
13
+ BenchmarkSuite,
14
+ BenchmarkTask,
15
+ CiInfo,
16
+ ConfigurationManager,
17
+ EnvironmentInfo,
18
+ ErrorManager,
19
+ ExecutionPhase,
20
+ FileLoader,
21
+ FileResult,
22
+ GitInfo,
23
+ HistoryStorage,
24
+ ModestBenchConfig,
25
+ ProgressManager,
26
+ Reporter,
27
+ ReporterRegistry,
28
+ RunConfiguration,
29
+ SuiteResult,
30
+ TaskResult,
31
+ ValidationError,
32
+ ValidationResult,
33
+ ValidationWarning,
34
+ } from '../types/index.js';
35
+
36
+ /**
37
+ * Dependencies required by the BenchmarkEngine
38
+ */
39
+ interface EngineDependencies {
40
+ readonly configManager: ConfigurationManager;
41
+ readonly errorManager: ErrorManager;
42
+ readonly fileLoader: FileLoader;
43
+ readonly historyStorage: HistoryStorage;
44
+ readonly progressManager: ProgressManager;
45
+ readonly reporterRegistry: ReporterRegistry;
46
+ }
47
+
48
+ /**
49
+ * Abstract benchmark execution engine with dependency injection
50
+ *
51
+ * Provides generic orchestration logic for benchmark discovery, validation, and
52
+ * execution. Concrete implementations must provide the task execution logic via
53
+ * the executeBenchmarkTask method.
54
+ */
55
+ export abstract class ModestBenchEngine implements BenchmarkEngine {
56
+ public readonly configManager: ConfigurationManager;
57
+
58
+ public readonly errorManager: ErrorManager;
59
+
60
+ public readonly fileLoader: FileLoader;
61
+
62
+ public readonly historyStorage: HistoryStorage;
63
+
64
+ public readonly progressManager: ProgressManager;
65
+
66
+ public readonly reporterRegistry: ReporterRegistry;
67
+
68
+ constructor(dependencies: EngineDependencies) {
69
+ this.configManager = dependencies.configManager;
70
+ this.fileLoader = dependencies.fileLoader;
71
+ this.reporterRegistry = dependencies.reporterRegistry;
72
+ this.historyStorage = dependencies.historyStorage;
73
+ this.progressManager = dependencies.progressManager;
74
+ this.errorManager = dependencies.errorManager;
75
+ }
76
+
77
+ /**
78
+ * Discover benchmark files matching the pattern(s)
79
+ */
80
+ async discover(
81
+ pattern: string | string[],
82
+ exclude?: string[],
83
+ ): Promise<string[]> {
84
+ try {
85
+ return await this.fileLoader.discover(pattern, exclude);
86
+ } catch (error) {
87
+ const discoveryError =
88
+ error instanceof Error ? error : new Error(String(error));
89
+ this.errorManager.handleError(discoveryError, {
90
+ metadata: { exclude, pattern },
91
+ phase: 'discovery',
92
+ timestamp: new Date(),
93
+ });
94
+
95
+ throw new Error(`File discovery failed: ${discoveryError.message}`);
96
+ }
97
+ }
98
+
99
+ /**
100
+ * Execute benchmarks with the given configuration
101
+ */
102
+ async execute(
103
+ config: RunConfiguration,
104
+ reporters: Reporter[] = [],
105
+ signal?: AbortSignal,
106
+ ): Promise<BenchmarkRun> {
107
+ const startTime = new Date();
108
+ let currentPhase: ExecutionPhase = 'discovery';
109
+
110
+ try {
111
+ // 1. Merge configuration with defaults
112
+ currentPhase = 'discovery';
113
+ const mergedConfig = await this.configManager.load(
114
+ undefined, // No specific config path for now
115
+ config as Record<string, unknown>,
116
+ );
117
+
118
+ // 2. Discover files if not explicitly provided
119
+ const files =
120
+ config.files ||
121
+ (await this.discover(mergedConfig.pattern, mergedConfig.exclude));
122
+
123
+ if (files.length === 0) {
124
+ const error = new Error(
125
+ 'No benchmark files found matching the pattern',
126
+ );
127
+ this.errorManager.handleError(error, {
128
+ phase: currentPhase,
129
+ timestamp: new Date(),
130
+ });
131
+ throw error;
132
+ }
133
+
134
+ // 3. Validate files
135
+ currentPhase = 'validation';
136
+ const validationResult = await this.validate(files);
137
+ if (!validationResult.valid) {
138
+ const error = new Error(
139
+ `Validation failed: ${validationResult.errors.map((e) => e.message).join(', ')}`,
140
+ );
141
+ this.errorManager.handleError(error, {
142
+ phase: currentPhase,
143
+ timestamp: new Date(),
144
+ });
145
+ throw error;
146
+ }
147
+
148
+ // 4. Initialize progress tracking
149
+ currentPhase = 'setup';
150
+ const runId = this.generateRunId();
151
+
152
+ // Pre-calculate total tasks for progress tracking
153
+ let totalTasks = 0;
154
+ let totalSuites = 0;
155
+
156
+ for (const filePath of files) {
157
+ try {
158
+ const benchmarkFile = await this.fileLoader.load(filePath);
159
+ const benchmarkDef = benchmarkFile.exports as BenchmarkDefinition;
160
+
161
+ if (benchmarkDef?.suites && typeof benchmarkDef.suites === 'object') {
162
+ const fileTags = benchmarkDef.tags;
163
+
164
+ for (const [_suiteName, suiteData] of Object.entries(
165
+ benchmarkDef.suites,
166
+ )) {
167
+ // Use shared filtering logic
168
+ const { anyTaskMatches, suiteMatches, tasksToRun } =
169
+ this.getFilteredTasksForSuite(
170
+ suiteData,
171
+ fileTags,
172
+ mergedConfig.tags,
173
+ mergedConfig.excludeTags,
174
+ );
175
+
176
+ // Count suite only if it or any of its tasks match
177
+ if (suiteMatches || anyTaskMatches) {
178
+ totalSuites++;
179
+ totalTasks += tasksToRun.length;
180
+ }
181
+ }
182
+ }
183
+ } catch (error) {
184
+ // If we can't load a file for counting, we'll handle it during execution
185
+ // Only show warning if not in quiet mode
186
+ if (!mergedConfig.quiet) {
187
+ console.warn(
188
+ `Warning: Could not pre-load ${filePath} for task counting:`,
189
+ error,
190
+ );
191
+ }
192
+ }
193
+ }
194
+
195
+ // Create initial run structure for progress tracking
196
+ const gitInfo = await this.getGitInfo();
197
+ const ciInfo = await this.getCiInfo();
198
+
199
+ const initialRun: BenchmarkRun = {
200
+ config: mergedConfig,
201
+ duration: 0,
202
+ endTime: startTime,
203
+ environment: await this.getEnvironmentInfo(),
204
+ files: [],
205
+ id: runId,
206
+ startTime,
207
+ ...(gitInfo && { git: gitInfo }),
208
+ ...(ciInfo && { ci: ciInfo }),
209
+ summary: {
210
+ failedTasks: 0,
211
+ fastest: null,
212
+ overallMean: 0,
213
+ passedTasks: 0,
214
+ slowest: null,
215
+ totalFiles: files.length,
216
+ totalOperations: 0,
217
+ totalSuites,
218
+ totalTasks,
219
+ },
220
+ };
221
+
222
+ this.progressManager.initialize(initialRun);
223
+
224
+ // Register progress callbacks with reporters that support them
225
+ for (const reporter of reporters) {
226
+ if (typeof reporter.onProgress === 'function') {
227
+ this.progressManager.onProgress((state) => {
228
+ void reporter.onProgress(state);
229
+ });
230
+ }
231
+ }
232
+
233
+ // 5. Call reporter onStart lifecycle method
234
+ await this.callReporters(reporters, 'onStart', initialRun);
235
+
236
+ // 6. Execute benchmark files
237
+ currentPhase = 'execution';
238
+ const fileResults: FileResult[] = [];
239
+
240
+ for (const filePath of files) {
241
+ try {
242
+ // Call reporter onFileStart
243
+ await this.callReporters(reporters, 'onFileStart', filePath);
244
+
245
+ const fileResult = await this.executeBenchmarkFile(
246
+ filePath,
247
+ mergedConfig,
248
+ reporters,
249
+ signal,
250
+ );
251
+ fileResults.push(fileResult);
252
+
253
+ // Call reporter onFileEnd
254
+ await this.callReporters(reporters, 'onFileEnd', fileResult);
255
+
256
+ // Update progress
257
+ this.progressManager.update({
258
+ currentFile: filePath,
259
+ filesCompleted: fileResults.length,
260
+ });
261
+ } catch (error) {
262
+ const fileError =
263
+ error instanceof Error ? error : new Error(String(error));
264
+ this.errorManager.handleError(fileError, {
265
+ file: filePath,
266
+ phase: currentPhase,
267
+ timestamp: new Date(),
268
+ });
269
+
270
+ // Call reporter onError
271
+ await this.callReporters(reporters, 'onError', fileError);
272
+
273
+ // Create error result for this file
274
+ const now = new Date();
275
+ const errorResult = {
276
+ duration: 0,
277
+ endTime: now,
278
+ error: fileError,
279
+ filePath,
280
+ startTime: now,
281
+ suites: [],
282
+ };
283
+ fileResults.push(errorResult);
284
+
285
+ // Call reporter onFileEnd for error case
286
+ await this.callReporters(reporters, 'onFileEnd', errorResult);
287
+ }
288
+ }
289
+
290
+ // Calculate summary statistics
291
+ const finalTotalSuites = fileResults.reduce(
292
+ (sum, file) => sum + file.suites.length,
293
+ 0,
294
+ );
295
+ const allTasks = fileResults.flatMap((file) =>
296
+ file.suites.flatMap((suite: SuiteResult) => suite.tasks),
297
+ );
298
+ const finalTotalTasks = allTasks.length;
299
+ const failedTasks = allTasks.filter((task) => task.error).length;
300
+ const passedTasks = finalTotalTasks - failedTasks;
301
+
302
+ let fastest: null | TaskResult = null;
303
+ let slowest: null | TaskResult = null;
304
+ let totalOperations = 0;
305
+ let totalTime = 0;
306
+
307
+ for (const task of allTasks) {
308
+ if (!task.error) {
309
+ totalOperations += task.iterations;
310
+ totalTime += task.mean * task.iterations;
311
+
312
+ if (!fastest || task.mean < fastest.mean) {
313
+ fastest = task;
314
+ }
315
+ if (!slowest || task.mean > slowest.mean) {
316
+ slowest = task;
317
+ }
318
+ }
319
+ }
320
+
321
+ const overallMean = totalOperations > 0 ? totalTime / totalOperations : 0;
322
+
323
+ const endTime = new Date();
324
+ const finalRun: BenchmarkRun = {
325
+ ...initialRun,
326
+ duration: endTime.getTime() - startTime.getTime(),
327
+ endTime,
328
+ files: fileResults,
329
+ summary: {
330
+ failedTasks,
331
+ fastest,
332
+ overallMean,
333
+ passedTasks,
334
+ slowest,
335
+ totalFiles: files.length,
336
+ totalOperations,
337
+ totalSuites: finalTotalSuites,
338
+ totalTasks: finalTotalTasks,
339
+ },
340
+ };
341
+
342
+ // 7. Save to history
343
+ await this.historyStorage.saveRun(finalRun);
344
+
345
+ // 8. Call reporter onEnd lifecycle method
346
+ await this.callReporters(reporters, 'onEnd', finalRun);
347
+
348
+ // 9. Return completed run
349
+ return finalRun;
350
+ } catch (error) {
351
+ const executionError =
352
+ error instanceof Error ? error : new Error(String(error));
353
+ const handledError = this.errorManager.handleError(executionError, {
354
+ phase: currentPhase,
355
+ timestamp: new Date(),
356
+ });
357
+
358
+ // Re-throw the original error with more context
359
+ throw new Error(`Benchmark execution failed: ${handledError.message}`);
360
+ }
361
+ }
362
+
363
+ /**
364
+ * Get the configuration manager
365
+ */
366
+ getConfigManager(): ConfigurationManager {
367
+ return this.configManager;
368
+ }
369
+
370
+ /**
371
+ * Get the file loader
372
+ */
373
+ getFileLoader(): FileLoader {
374
+ return this.fileLoader;
375
+ }
376
+
377
+ /**
378
+ * Get the history storage
379
+ */
380
+ getHistoryStorage(): HistoryStorage {
381
+ return this.historyStorage;
382
+ }
383
+
384
+ /**
385
+ * Get the progress manager
386
+ */
387
+ getProgressManager(): ProgressManager {
388
+ return this.progressManager;
389
+ }
390
+
391
+ /**
392
+ * Get all available reporters
393
+ */
394
+ getReporters(): Record<string, Reporter> {
395
+ return this.reporterRegistry.getAll();
396
+ }
397
+
398
+ /**
399
+ * Register a custom reporter
400
+ */
401
+ registerReporter(name: string, reporter: Reporter): void {
402
+ this.reporterRegistry.register(name, reporter);
403
+ }
404
+
405
+ /**
406
+ * Validate benchmark files without executing them
407
+ */
408
+ async validate(files: string[]): Promise<ValidationResult> {
409
+ try {
410
+ const errors: ValidationError[] = [];
411
+ const warnings: ValidationWarning[] = [];
412
+ const validatedFiles: string[] = [];
413
+
414
+ // Validate each file
415
+ for (const file of files) {
416
+ try {
417
+ const result = await this.fileLoader.validate(file);
418
+ validatedFiles.push(file);
419
+ errors.push(...result.errors);
420
+ warnings.push(...result.warnings);
421
+ } catch (error) {
422
+ const validationError =
423
+ error instanceof Error ? error : new Error(String(error));
424
+ this.errorManager.handleError(validationError, {
425
+ file,
426
+ phase: 'validation',
427
+ timestamp: new Date(),
428
+ });
429
+
430
+ errors.push({
431
+ code: 'FILE_VALIDATION_ERROR',
432
+ file,
433
+ message: `Failed to validate file: ${validationError.message}`,
434
+ severity: 'error' as const,
435
+ });
436
+ }
437
+ }
438
+
439
+ return {
440
+ errors,
441
+ files: validatedFiles,
442
+ valid: errors.length === 0,
443
+ warnings,
444
+ };
445
+ } catch (error) {
446
+ throw new Error(
447
+ `Validation failed: ${error instanceof Error ? error.message : String(error)}`,
448
+ );
449
+ }
450
+ }
451
+
452
+ /**
453
+ * Execute a single benchmark task
454
+ *
455
+ * This method must be implemented by concrete engine implementations to
456
+ * provide the actual benchmark execution logic.
457
+ *
458
+ * @param taskName - Name of the task being executed
459
+ * @param taskData - Task definition with function and metadata
460
+ * @param config - Benchmark configuration
461
+ * @param reporters - Array of active reporters
462
+ * @param signal - Optional abort signal for cancellation
463
+ * @returns Promise resolving to task execution result
464
+ */
465
+ protected abstract executeBenchmarkTask(
466
+ taskName: string,
467
+ taskData: BenchmarkTask,
468
+ config: ModestBenchConfig,
469
+ reporters: Reporter[],
470
+ signal?: AbortSignal,
471
+ ): Promise<TaskResult>;
472
+
473
+ /**
474
+ * Helper method to call a lifecycle method on all reporters
475
+ */
476
+ private async callReporters(
477
+ reporters: Reporter[],
478
+ method: keyof Reporter,
479
+ ...args: unknown[]
480
+ ): Promise<void> {
481
+ for (const reporter of reporters) {
482
+ try {
483
+ const reporterMethod = reporter[method];
484
+ if (typeof reporterMethod === 'function') {
485
+ const result = (
486
+ reporterMethod as (...args: unknown[]) => unknown
487
+ ).call(reporter, ...args);
488
+ // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
489
+ if (result && typeof (result as any).then === 'function') {
490
+ await (result as Promise<void>);
491
+ }
492
+ }
493
+ } catch (error) {
494
+ // Log reporter errors but don't fail the benchmark run
495
+ console.error(`Reporter error in ${method}:`, error);
496
+ }
497
+ }
498
+ }
499
+
500
+ /**
501
+ * Execute a single benchmark file and return its results
502
+ */
503
+ private async executeBenchmarkFile(
504
+ filePath: string,
505
+ config: ModestBenchConfig,
506
+ reporters: Reporter[] = [],
507
+ signal?: AbortSignal,
508
+ ): Promise<FileResult> {
509
+ const startTime = new Date();
510
+
511
+ try {
512
+ // Load the benchmark file using the file loader
513
+ const benchmarkFile = await this.fileLoader.load(filePath);
514
+ const benchmarkDef = benchmarkFile.exports as BenchmarkDefinition;
515
+
516
+ if (!benchmarkDef || typeof benchmarkDef !== 'object') {
517
+ throw new Error(
518
+ 'Benchmark file must export a default object with suites',
519
+ );
520
+ }
521
+
522
+ const suiteResults: SuiteResult[] = [];
523
+ const fileTags = benchmarkDef.tags;
524
+
525
+ // Process each suite in the file
526
+ if (benchmarkDef.suites && typeof benchmarkDef.suites === 'object') {
527
+ for (const [suiteName, suiteData] of Object.entries(
528
+ benchmarkDef.suites,
529
+ )) {
530
+ // Use shared filtering logic
531
+ const { anyTaskMatches, suiteMatches } =
532
+ this.getFilteredTasksForSuite(
533
+ suiteData,
534
+ fileTags,
535
+ config.tags,
536
+ config.excludeTags,
537
+ );
538
+
539
+ // Skip suite only if neither the suite nor any of its tasks match
540
+ if (!suiteMatches && !anyTaskMatches) {
541
+ continue;
542
+ }
543
+
544
+ await this.callReporters(reporters, 'onSuiteStart', suiteName);
545
+ const suiteResult = await this.executeBenchmarkSuite(
546
+ suiteName,
547
+ suiteData,
548
+ config,
549
+ reporters,
550
+ signal,
551
+ fileTags,
552
+ );
553
+ await this.callReporters(reporters, 'onSuiteEnd', suiteResult);
554
+ suiteResults.push(suiteResult);
555
+ }
556
+ }
557
+
558
+ const endTime = new Date();
559
+
560
+ return {
561
+ config: benchmarkDef.config,
562
+ duration: endTime.getTime() - startTime.getTime(),
563
+ endTime,
564
+ filePath,
565
+ startTime,
566
+ suites: suiteResults,
567
+ };
568
+ } catch (error) {
569
+ const endTime = new Date();
570
+ const executionError =
571
+ error instanceof Error ? error : new Error(String(error));
572
+
573
+ return {
574
+ duration: endTime.getTime() - startTime.getTime(),
575
+ endTime,
576
+ error: executionError,
577
+ filePath,
578
+ startTime,
579
+ suites: [],
580
+ };
581
+ }
582
+ }
583
+
584
+ /**
585
+ * Execute a single benchmark suite and return its results
586
+ */
587
+ private async executeBenchmarkSuite(
588
+ suiteName: string,
589
+ suiteData: BenchmarkSuite,
590
+ config: ModestBenchConfig,
591
+ reporters: Reporter[] = [],
592
+ signal?: AbortSignal,
593
+ fileTags?: string[],
594
+ ): Promise<SuiteResult> {
595
+ const startTime = new Date();
596
+
597
+ try {
598
+ const taskResults: TaskResult[] = [];
599
+
600
+ // Use shared filtering logic to determine which tasks will run
601
+ const { tasksToRun } = this.getFilteredTasksForSuite(
602
+ suiteData,
603
+ fileTags,
604
+ config.tags,
605
+ config.excludeTags,
606
+ );
607
+
608
+ // Only run setup/teardown if there are tasks to execute
609
+ if (tasksToRun.length === 0) {
610
+ // No tasks match the filters, return empty suite result
611
+ const endTime = new Date();
612
+ return {
613
+ duration: endTime.getTime() - startTime.getTime(),
614
+ endTime,
615
+ name: suiteName,
616
+ startTime,
617
+ tasks: [],
618
+ ...(suiteData.config !== undefined && { config: suiteData.config }),
619
+ ...(suiteData.metadata !== undefined && {
620
+ metadata: suiteData.metadata,
621
+ }),
622
+ ...(suiteData.tags !== undefined && { tags: suiteData.tags }),
623
+ };
624
+ }
625
+
626
+ // Run suite setup if provided
627
+ if (suiteData.setup && typeof suiteData.setup === 'function') {
628
+ try {
629
+ await suiteData.setup();
630
+ } catch (error) {
631
+ const setupError =
632
+ error instanceof Error
633
+ ? error
634
+ : new Error(`Setup failed: ${String(error)}`);
635
+ throw new Error(`Suite setup failed: ${setupError.message}`);
636
+ }
637
+ }
638
+
639
+ try {
640
+ // Process each task that passed filtering
641
+ for (const [taskName, taskData] of tasksToRun) {
642
+ await this.callReporters(reporters, 'onTaskStart', taskName);
643
+
644
+ // Mark task as in-progress (shows as 0.5 progress for current task)
645
+ const currentState = this.progressManager.getState();
646
+ this.progressManager.update({
647
+ tasksCompleted: currentState.tasksCompleted + 0.5,
648
+ });
649
+
650
+ const taskResult = await this.executeBenchmarkTask(
651
+ taskName,
652
+ taskData,
653
+ config,
654
+ reporters,
655
+ signal,
656
+ );
657
+ await this.callReporters(reporters, 'onTaskResult', taskResult);
658
+ taskResults.push(taskResult);
659
+
660
+ // Update task-level progress - task is now complete (remove the 0.5 and add 1)
661
+ this.progressManager.update({
662
+ tasksCompleted: currentState.tasksCompleted + 1,
663
+ });
664
+ }
665
+ } finally {
666
+ // Run suite teardown if provided (always runs, even if benchmarks fail)
667
+ if (suiteData.teardown && typeof suiteData.teardown === 'function') {
668
+ try {
669
+ await suiteData.teardown();
670
+ } catch (error) {
671
+ // Log teardown errors but don't fail the suite
672
+ const teardownError =
673
+ error instanceof Error ? error : new Error(String(error));
674
+ console.error(
675
+ `Warning: Suite teardown failed for "${suiteName}":`,
676
+ teardownError.message,
677
+ );
678
+ }
679
+ }
680
+ }
681
+
682
+ const endTime = new Date();
683
+
684
+ return {
685
+ duration: endTime.getTime() - startTime.getTime(),
686
+ endTime,
687
+ name: suiteName,
688
+ startTime,
689
+ tasks: taskResults,
690
+ ...(suiteData.config !== undefined && { config: suiteData.config }),
691
+ ...(suiteData.metadata !== undefined && {
692
+ metadata: suiteData.metadata,
693
+ }),
694
+ ...(suiteData.tags !== undefined && { tags: suiteData.tags }),
695
+ };
696
+ } catch (error) {
697
+ const endTime = new Date();
698
+ const executionError =
699
+ error instanceof Error ? error : new Error(String(error));
700
+
701
+ return {
702
+ duration: endTime.getTime() - startTime.getTime(),
703
+ endTime,
704
+ error: executionError,
705
+ name: suiteName,
706
+ startTime,
707
+ tasks: [],
708
+ };
709
+ }
710
+ }
711
+
712
+ /**
713
+ * Generate a unique run ID
714
+ */
715
+ private generateRunId(): string {
716
+ return `run-${Date.now()}-${Math.random().toString(36).substring(2, 15)}`;
717
+ }
718
+
719
+ /**
720
+ * Get CI/CD information if available
721
+ */
722
+ private async getCiInfo(): Promise<CiInfo | undefined> {
723
+ const process = await import('node:process');
724
+
725
+ if (!process.env.CI) {
726
+ return undefined;
727
+ }
728
+
729
+ // Detect common CI providers
730
+ if (process.env.GITHUB_ACTIONS) {
731
+ return {
732
+ provider: 'GitHub Actions',
733
+ ...(process.env.GITHUB_RUN_NUMBER && {
734
+ buildNumber: process.env.GITHUB_RUN_NUMBER,
735
+ }),
736
+ ...(process.env.GITHUB_REPOSITORY &&
737
+ process.env.GITHUB_RUN_ID && {
738
+ buildUrl: `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`,
739
+ }),
740
+ ...(process.env.GITHUB_EVENT_NAME === 'pull_request' &&
741
+ process.env.GITHUB_REF_NAME && {
742
+ pullRequest: process.env.GITHUB_REF_NAME,
743
+ }),
744
+ ...(process.env.GITHUB_REF_NAME && {
745
+ branch: process.env.GITHUB_REF_NAME,
746
+ }),
747
+ ...(process.env.GITHUB_SHA && { commit: process.env.GITHUB_SHA }),
748
+ };
749
+ }
750
+
751
+ // Default CI info
752
+ return {
753
+ provider: 'Unknown CI',
754
+ ...(process.env.BRANCH && { branch: process.env.BRANCH }),
755
+ ...(process.env.COMMIT && { commit: process.env.COMMIT }),
756
+ };
757
+ }
758
+
759
+ /**
760
+ * Get environment information
761
+ */
762
+ private async getEnvironmentInfo(): Promise<EnvironmentInfo> {
763
+ const os = await import('node:os');
764
+ const process = await import('node:process');
765
+
766
+ return {
767
+ arch: process.arch,
768
+ availableMemory: os.freemem(),
769
+ cpu: {
770
+ cores: os.cpus().length,
771
+ model: os.cpus()[0]?.model || 'Unknown',
772
+ speed: os.cpus()[0]?.speed || 0,
773
+ },
774
+ env: {
775
+ CI: process.env.CI || 'false',
776
+ NODE_ENV: process.env.NODE_ENV || 'development',
777
+ },
778
+ hostname: os.hostname(),
779
+ memory: {
780
+ free: os.freemem(),
781
+ total: os.totalmem(),
782
+ used: os.totalmem() - os.freemem(),
783
+ },
784
+ nodeVersion: process.version,
785
+ platform: process.platform,
786
+ };
787
+ }
788
+
789
+ /**
790
+ * Get filtered tasks for a suite based on tag filtering Returns suite match
791
+ * status and list of tasks to run
792
+ */
793
+ private getFilteredTasksForSuite(
794
+ suiteData: BenchmarkSuite,
795
+ fileTags: string[] | undefined,
796
+ includeTags: string[],
797
+ excludeTags: string[],
798
+ ): {
799
+ anyTaskMatches: boolean;
800
+ suiteMatches: boolean;
801
+ tasksToRun: Array<[string, BenchmarkTask]>;
802
+ } {
803
+ // Check if suite itself matches filters
804
+ const mergedSuiteTags = this.mergeTags(fileTags, suiteData.tags);
805
+ const suiteMatches = this.matchesTags(
806
+ mergedSuiteTags,
807
+ includeTags,
808
+ excludeTags,
809
+ );
810
+
811
+ // Check which tasks match filters
812
+ const tasksToRun: Array<[string, BenchmarkTask]> = [];
813
+ if (suiteData.benchmarks && typeof suiteData.benchmarks === 'object') {
814
+ for (const [taskName, taskData] of Object.entries(suiteData.benchmarks)) {
815
+ // Merge task tags with suite and file tags (cascading)
816
+ const mergedTaskTags = this.mergeTags(mergedSuiteTags, taskData.tags);
817
+
818
+ // Check if task matches tag filters
819
+ if (this.matchesTags(mergedTaskTags, includeTags, excludeTags)) {
820
+ tasksToRun.push([taskName, taskData]);
821
+ }
822
+ }
823
+ }
824
+
825
+ return {
826
+ anyTaskMatches: tasksToRun.length > 0,
827
+ suiteMatches,
828
+ tasksToRun,
829
+ };
830
+ }
831
+
832
+ /**
833
+ * Get Git information if available
834
+ */
835
+ private async getGitInfo(): Promise<GitInfo | undefined> {
836
+ // TODO: Implement Git information extraction
837
+ // This would use child_process to run git commands
838
+ return undefined;
839
+ }
840
+
841
+ /**
842
+ * Check if item tags match the filter criteria (OR logic)
843
+ */
844
+ private matchesTags(
845
+ itemTags: string[] | undefined,
846
+ includeTags: string[],
847
+ excludeTags: string[],
848
+ ): boolean {
849
+ const tags = itemTags || [];
850
+
851
+ // If exclude tags specified and any match, exclude this item
852
+ if (
853
+ excludeTags.length > 0 &&
854
+ excludeTags.some((tag) => tags.includes(tag))
855
+ ) {
856
+ return false;
857
+ }
858
+
859
+ // If include tags specified, at least one must match
860
+ if (includeTags.length > 0) {
861
+ return includeTags.some((tag) => tags.includes(tag));
862
+ }
863
+
864
+ // No filters = include everything
865
+ return true;
866
+ }
867
+
868
+ /**
869
+ * Merge tags from parent to child (cascading)
870
+ */
871
+ private mergeTags(
872
+ parentTags?: string[],
873
+ childTags?: string[],
874
+ ): string[] | undefined {
875
+ const merged = new Set<string>();
876
+ if (parentTags) {
877
+ for (const tag of parentTags) {
878
+ merged.add(tag);
879
+ }
880
+ }
881
+ if (childTags) {
882
+ for (const tag of childTags) {
883
+ merged.add(tag);
884
+ }
885
+ }
886
+ return merged.size > 0 ? Array.from(merged) : undefined;
887
+ }
888
+ }