@fuzdev/fuz_util 0.42.0 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +19 -12
- package/dist/async.d.ts +2 -2
- package/dist/async.d.ts.map +1 -1
- package/dist/async.js +2 -2
- package/dist/benchmark.d.ts +179 -0
- package/dist/benchmark.d.ts.map +1 -0
- package/dist/benchmark.js +400 -0
- package/dist/benchmark_baseline.d.ts +195 -0
- package/dist/benchmark_baseline.d.ts.map +1 -0
- package/dist/benchmark_baseline.js +388 -0
- package/dist/benchmark_format.d.ts +87 -0
- package/dist/benchmark_format.d.ts.map +1 -0
- package/dist/benchmark_format.js +266 -0
- package/dist/benchmark_stats.d.ts +112 -0
- package/dist/benchmark_stats.d.ts.map +1 -0
- package/dist/benchmark_stats.js +219 -0
- package/dist/benchmark_types.d.ts +174 -0
- package/dist/benchmark_types.d.ts.map +1 -0
- package/dist/benchmark_types.js +1 -0
- package/dist/git.d.ts +12 -0
- package/dist/git.d.ts.map +1 -1
- package/dist/git.js +14 -0
- package/dist/library_json.d.ts +3 -3
- package/dist/library_json.d.ts.map +1 -1
- package/dist/library_json.js +1 -1
- package/dist/maths.d.ts +4 -0
- package/dist/maths.d.ts.map +1 -1
- package/dist/maths.js +8 -0
- package/dist/object.js +1 -1
- package/dist/source_json.d.ts +4 -4
- package/dist/stats.d.ts +180 -0
- package/dist/stats.d.ts.map +1 -0
- package/dist/stats.js +402 -0
- package/dist/string.d.ts +13 -0
- package/dist/string.d.ts.map +1 -1
- package/dist/string.js +58 -0
- package/dist/time.d.ts +165 -0
- package/dist/time.d.ts.map +1 -0
- package/dist/time.js +264 -0
- package/dist/timings.d.ts +1 -7
- package/dist/timings.d.ts.map +1 -1
- package/dist/timings.js +16 -16
- package/package.json +21 -19
- package/src/lib/async.ts +3 -3
- package/src/lib/benchmark.ts +498 -0
- package/src/lib/benchmark_baseline.ts +538 -0
- package/src/lib/benchmark_format.ts +314 -0
- package/src/lib/benchmark_stats.ts +311 -0
- package/src/lib/benchmark_types.ts +197 -0
- package/src/lib/git.ts +24 -0
- package/src/lib/library_json.ts +3 -3
- package/src/lib/maths.ts +8 -0
- package/src/lib/object.ts +1 -1
- package/src/lib/stats.ts +534 -0
- package/src/lib/string.ts +66 -0
- package/src/lib/time.ts +319 -0
- package/src/lib/timings.ts +17 -17
- package/src/lib/types.ts +2 -2
|
@@ -0,0 +1,400 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Benchmarking library.
|
|
3
|
+
*
|
|
4
|
+
* @example
|
|
5
|
+
* ```ts
|
|
6
|
+
* import {Benchmark} from './benchmark.js';
|
|
7
|
+
*
|
|
8
|
+
* const bench = new Benchmark({
|
|
9
|
+
* duration_ms: 5000,
|
|
10
|
+
* warmup_iterations: 5,
|
|
11
|
+
* });
|
|
12
|
+
*
|
|
13
|
+
* bench
|
|
14
|
+
* .add('slugify', () => slugify(title))
|
|
15
|
+
* .add('slugify_slower', () => slugify_slower(title));
|
|
16
|
+
*
|
|
17
|
+
* const results = await bench.run();
|
|
18
|
+
* console.log(bench.table());
|
|
19
|
+
* ```
|
|
20
|
+
*/
|
|
21
|
+
import { is_promise, wait } from './async.js';
|
|
22
|
+
import { BenchmarkStats } from './benchmark_stats.js';
|
|
23
|
+
import { timer_default, time_unit_detect_best, time_format } from './time.js';
|
|
24
|
+
import { benchmark_format_table, benchmark_format_table_grouped, benchmark_format_markdown, benchmark_format_json, benchmark_format_number, } from './benchmark_format.js';
|
|
25
|
+
// Default configuration values
|
|
26
|
+
const DEFAULT_DURATION_MS = 1000;
|
|
27
|
+
const DEFAULT_WARMUP_ITERATIONS = 10;
|
|
28
|
+
const DEFAULT_COOLDOWN_MS = 100;
|
|
29
|
+
const DEFAULT_MIN_ITERATIONS = 10;
|
|
30
|
+
const DEFAULT_MAX_ITERATIONS = 100_000;
|
|
31
|
+
/**
|
|
32
|
+
* Validate and normalize benchmark configuration.
|
|
33
|
+
* Throws if configuration is invalid.
|
|
34
|
+
*/
|
|
35
|
+
const validate_config = (config) => {
|
|
36
|
+
if (config.duration_ms !== undefined && config.duration_ms <= 0) {
|
|
37
|
+
throw new Error(`duration_ms must be positive, got ${config.duration_ms}`);
|
|
38
|
+
}
|
|
39
|
+
if (config.warmup_iterations !== undefined && config.warmup_iterations < 0) {
|
|
40
|
+
throw new Error(`warmup_iterations must be non-negative, got ${config.warmup_iterations}`);
|
|
41
|
+
}
|
|
42
|
+
if (config.cooldown_ms !== undefined && config.cooldown_ms < 0) {
|
|
43
|
+
throw new Error(`cooldown_ms must be non-negative, got ${config.cooldown_ms}`);
|
|
44
|
+
}
|
|
45
|
+
if (config.min_iterations !== undefined && config.min_iterations < 1) {
|
|
46
|
+
throw new Error(`min_iterations must be at least 1, got ${config.min_iterations}`);
|
|
47
|
+
}
|
|
48
|
+
if (config.max_iterations !== undefined && config.max_iterations < 1) {
|
|
49
|
+
throw new Error(`max_iterations must be at least 1, got ${config.max_iterations}`);
|
|
50
|
+
}
|
|
51
|
+
if (config.min_iterations !== undefined &&
|
|
52
|
+
config.max_iterations !== undefined &&
|
|
53
|
+
config.min_iterations > config.max_iterations) {
|
|
54
|
+
throw new Error(`min_iterations (${config.min_iterations}) cannot exceed max_iterations (${config.max_iterations})`);
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
/**
|
|
58
|
+
* Warmup function by running it multiple times.
|
|
59
|
+
* Detects whether the function is async based on return value.
|
|
60
|
+
*
|
|
61
|
+
* @param fn - Function to warmup (sync or async)
|
|
62
|
+
* @param iterations - Number of warmup iterations
|
|
63
|
+
* @param async_hint - If provided, use this instead of detecting
|
|
64
|
+
* @returns Whether the function is async
|
|
65
|
+
*
|
|
66
|
+
* @example
|
|
67
|
+
* ```ts
|
|
68
|
+
* const is_async = await benchmark_warmup(() => expensive_operation(), 10);
|
|
69
|
+
* ```
|
|
70
|
+
*/
|
|
71
|
+
export const benchmark_warmup = async (fn, iterations, async_hint) => {
|
|
72
|
+
// If we have an explicit hint, use it
|
|
73
|
+
if (async_hint !== undefined) {
|
|
74
|
+
// Still run warmup iterations for JIT
|
|
75
|
+
for (let i = 0; i < iterations; i++) {
|
|
76
|
+
const result = fn();
|
|
77
|
+
if (async_hint && is_promise(result)) {
|
|
78
|
+
await result; // eslint-disable-line no-await-in-loop
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return async_hint;
|
|
82
|
+
}
|
|
83
|
+
// Detect on first iteration
|
|
84
|
+
let detected_async = false;
|
|
85
|
+
for (let i = 0; i < iterations; i++) {
|
|
86
|
+
const result = fn();
|
|
87
|
+
if (i === 0) {
|
|
88
|
+
detected_async = is_promise(result);
|
|
89
|
+
}
|
|
90
|
+
if (detected_async && is_promise(result)) {
|
|
91
|
+
await result; // eslint-disable-line no-await-in-loop
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return detected_async;
|
|
95
|
+
};
|
|
96
|
+
/**
|
|
97
|
+
* Benchmark class for measuring and comparing function performance.
|
|
98
|
+
*/
|
|
99
|
+
export class Benchmark {
|
|
100
|
+
#config;
|
|
101
|
+
#tasks = [];
|
|
102
|
+
#results = [];
|
|
103
|
+
constructor(config = {}) {
|
|
104
|
+
validate_config(config);
|
|
105
|
+
this.#config = {
|
|
106
|
+
duration_ms: config.duration_ms ?? DEFAULT_DURATION_MS,
|
|
107
|
+
warmup_iterations: config.warmup_iterations ?? DEFAULT_WARMUP_ITERATIONS,
|
|
108
|
+
cooldown_ms: config.cooldown_ms ?? DEFAULT_COOLDOWN_MS,
|
|
109
|
+
min_iterations: config.min_iterations ?? DEFAULT_MIN_ITERATIONS,
|
|
110
|
+
max_iterations: config.max_iterations ?? DEFAULT_MAX_ITERATIONS,
|
|
111
|
+
timer: config.timer ?? timer_default,
|
|
112
|
+
on_iteration: config.on_iteration,
|
|
113
|
+
on_task_complete: config.on_task_complete,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
add(name_or_task, fn) {
|
|
117
|
+
const task_name = typeof name_or_task === 'string' ? name_or_task : name_or_task.name;
|
|
118
|
+
// Validate unique task names
|
|
119
|
+
if (this.#tasks.some((t) => t.name === task_name)) {
|
|
120
|
+
throw new Error(`Task "${task_name}" already exists`);
|
|
121
|
+
}
|
|
122
|
+
if (typeof name_or_task === 'string') {
|
|
123
|
+
if (!fn)
|
|
124
|
+
throw new Error('Function required when name is string');
|
|
125
|
+
this.#tasks.push({ name: name_or_task, fn });
|
|
126
|
+
}
|
|
127
|
+
else {
|
|
128
|
+
this.#tasks.push(name_or_task);
|
|
129
|
+
}
|
|
130
|
+
return this;
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Remove a benchmark task by name.
|
|
134
|
+
* @param name - Name of the task to remove
|
|
135
|
+
* @returns This Benchmark instance for chaining
|
|
136
|
+
* @throws Error if task with given name doesn't exist
|
|
137
|
+
*
|
|
138
|
+
* @example
|
|
139
|
+
* ```ts
|
|
140
|
+
* bench.add('task1', () => fn1());
|
|
141
|
+
* bench.add('task2', () => fn2());
|
|
142
|
+
* bench.remove('task1');
|
|
143
|
+
* // Only task2 remains
|
|
144
|
+
* ```
|
|
145
|
+
*/
|
|
146
|
+
remove(name) {
|
|
147
|
+
const index = this.#tasks.findIndex((t) => t.name === name);
|
|
148
|
+
if (index === -1) {
|
|
149
|
+
throw new Error(`Task "${name}" not found`);
|
|
150
|
+
}
|
|
151
|
+
this.#tasks.splice(index, 1);
|
|
152
|
+
return this;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Mark a task to be skipped during benchmark runs.
|
|
156
|
+
* @param name - Name of the task to skip
|
|
157
|
+
* @returns This Benchmark instance for chaining
|
|
158
|
+
* @throws Error if task with given name doesn't exist
|
|
159
|
+
*
|
|
160
|
+
* @example
|
|
161
|
+
* ```ts
|
|
162
|
+
* bench.add('task1', () => fn1());
|
|
163
|
+
* bench.add('task2', () => fn2());
|
|
164
|
+
* bench.skip('task1');
|
|
165
|
+
* // Only task2 will run
|
|
166
|
+
* ```
|
|
167
|
+
*/
|
|
168
|
+
skip(name) {
|
|
169
|
+
const task = this.#tasks.find((t) => t.name === name);
|
|
170
|
+
if (!task) {
|
|
171
|
+
throw new Error(`Task "${name}" not found`);
|
|
172
|
+
}
|
|
173
|
+
task.skip = true;
|
|
174
|
+
return this;
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Mark a task to run exclusively (along with other `only` tasks).
|
|
178
|
+
* @param name - Name of the task to run exclusively
|
|
179
|
+
* @returns This Benchmark instance for chaining
|
|
180
|
+
* @throws Error if task with given name doesn't exist
|
|
181
|
+
*
|
|
182
|
+
* @example
|
|
183
|
+
* ```ts
|
|
184
|
+
* bench.add('task1', () => fn1());
|
|
185
|
+
* bench.add('task2', () => fn2());
|
|
186
|
+
* bench.add('task3', () => fn3());
|
|
187
|
+
* bench.only('task2');
|
|
188
|
+
* // Only task2 will run
|
|
189
|
+
* ```
|
|
190
|
+
*/
|
|
191
|
+
only(name) {
|
|
192
|
+
const task = this.#tasks.find((t) => t.name === name);
|
|
193
|
+
if (!task) {
|
|
194
|
+
throw new Error(`Task "${name}" not found`);
|
|
195
|
+
}
|
|
196
|
+
task.only = true;
|
|
197
|
+
return this;
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Run all benchmark tasks.
|
|
201
|
+
* @returns Array of benchmark results
|
|
202
|
+
*/
|
|
203
|
+
async run() {
|
|
204
|
+
this.#results = [];
|
|
205
|
+
// Determine which tasks to run
|
|
206
|
+
const has_only = this.#tasks.some((t) => t.only);
|
|
207
|
+
const tasks_to_run = this.#tasks.filter((t) => {
|
|
208
|
+
if (t.skip)
|
|
209
|
+
return false;
|
|
210
|
+
if (has_only)
|
|
211
|
+
return t.only;
|
|
212
|
+
return true;
|
|
213
|
+
});
|
|
214
|
+
for (let i = 0; i < tasks_to_run.length; i++) {
|
|
215
|
+
const task = tasks_to_run[i];
|
|
216
|
+
const result = await this.#run_task(task); // eslint-disable-line no-await-in-loop
|
|
217
|
+
this.#results.push(result);
|
|
218
|
+
// Call on_task_complete callback
|
|
219
|
+
this.#config.on_task_complete?.(result, i, tasks_to_run.length);
|
|
220
|
+
// Cooldown between tasks (skip after last task)
|
|
221
|
+
if (this.#config.cooldown_ms > 0 && i < tasks_to_run.length - 1) {
|
|
222
|
+
await wait(this.#config.cooldown_ms); // eslint-disable-line no-await-in-loop
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
return this.#results;
|
|
226
|
+
}
|
|
227
|
+
/**
|
|
228
|
+
* Run a single benchmark task.
|
|
229
|
+
* Throws if the task fails during setup, warmup, or measurement.
|
|
230
|
+
*/
|
|
231
|
+
async #run_task(task) {
|
|
232
|
+
const suite_start_ns = this.#config.timer.now();
|
|
233
|
+
// Pre-allocate array to avoid GC pressure during measurement
|
|
234
|
+
const max_iterations = this.#config.max_iterations;
|
|
235
|
+
const timings_ns = new Array(max_iterations);
|
|
236
|
+
let timing_count = 0;
|
|
237
|
+
try {
|
|
238
|
+
// Setup
|
|
239
|
+
if (task.setup) {
|
|
240
|
+
await task.setup();
|
|
241
|
+
}
|
|
242
|
+
// Warmup and detect async
|
|
243
|
+
const is_async = await benchmark_warmup(task.fn, this.#config.warmup_iterations, task.async);
|
|
244
|
+
task.is_async = is_async;
|
|
245
|
+
// Measurement phase
|
|
246
|
+
const target_time_ns = this.#config.duration_ms * 1_000_000; // Convert ms to ns
|
|
247
|
+
const min_iterations = this.#config.min_iterations;
|
|
248
|
+
let aborted = false;
|
|
249
|
+
const abort = () => {
|
|
250
|
+
aborted = true;
|
|
251
|
+
};
|
|
252
|
+
const measurement_start_ns = this.#config.timer.now();
|
|
253
|
+
// Use separate code paths for sync vs async for better performance
|
|
254
|
+
if (is_async) {
|
|
255
|
+
// Async code path - await each iteration
|
|
256
|
+
// eslint-disable-next-line no-unmodified-loop-condition
|
|
257
|
+
while (timing_count < max_iterations && !aborted) {
|
|
258
|
+
const iter_start_ns = this.#config.timer.now();
|
|
259
|
+
await task.fn(); // eslint-disable-line no-await-in-loop
|
|
260
|
+
const iter_end_ns = this.#config.timer.now();
|
|
261
|
+
timings_ns[timing_count++] = iter_end_ns - iter_start_ns;
|
|
262
|
+
this.#config.on_iteration?.(task.name, timing_count, abort);
|
|
263
|
+
const total_elapsed_ns = iter_end_ns - measurement_start_ns;
|
|
264
|
+
if (timing_count >= min_iterations && total_elapsed_ns >= target_time_ns) {
|
|
265
|
+
break;
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
else {
|
|
270
|
+
// Sync code path - no promise checking overhead
|
|
271
|
+
// eslint-disable-next-line no-unmodified-loop-condition
|
|
272
|
+
while (timing_count < max_iterations && !aborted) {
|
|
273
|
+
const iter_start_ns = this.#config.timer.now();
|
|
274
|
+
task.fn();
|
|
275
|
+
const iter_end_ns = this.#config.timer.now();
|
|
276
|
+
timings_ns[timing_count++] = iter_end_ns - iter_start_ns;
|
|
277
|
+
this.#config.on_iteration?.(task.name, timing_count, abort);
|
|
278
|
+
const total_elapsed_ns = iter_end_ns - measurement_start_ns;
|
|
279
|
+
if (timing_count >= min_iterations && total_elapsed_ns >= target_time_ns) {
|
|
280
|
+
break;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
finally {
|
|
286
|
+
// Always run teardown
|
|
287
|
+
if (task.teardown) {
|
|
288
|
+
await task.teardown();
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
// Trim array to actual size
|
|
292
|
+
timings_ns.length = timing_count;
|
|
293
|
+
const suite_end_ns = this.#config.timer.now();
|
|
294
|
+
const total_time_ms = (suite_end_ns - suite_start_ns) / 1_000_000; // Convert back to ms for display
|
|
295
|
+
// Analyze results
|
|
296
|
+
const stats = new BenchmarkStats(timings_ns);
|
|
297
|
+
return {
|
|
298
|
+
name: task.name,
|
|
299
|
+
stats,
|
|
300
|
+
iterations: timing_count,
|
|
301
|
+
total_time_ms,
|
|
302
|
+
timings_ns,
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* Format results as an ASCII table with percentiles, min/max, and relative performance.
|
|
307
|
+
* @param options - Formatting options
|
|
308
|
+
* @returns Formatted table string
|
|
309
|
+
*
|
|
310
|
+
* @example
|
|
311
|
+
* ```ts
|
|
312
|
+
* // Standard table
|
|
313
|
+
* console.log(bench.table());
|
|
314
|
+
*
|
|
315
|
+
* // Grouped by category
|
|
316
|
+
* console.log(bench.table({
|
|
317
|
+
* groups: [
|
|
318
|
+
* { name: 'FAST PATHS', filter: (r) => r.name.includes('fast') },
|
|
319
|
+
* { name: 'SLOW PATHS', filter: (r) => r.name.includes('slow') },
|
|
320
|
+
* ]
|
|
321
|
+
* }));
|
|
322
|
+
* ```
|
|
323
|
+
*/
|
|
324
|
+
table(options) {
|
|
325
|
+
return options?.groups
|
|
326
|
+
? benchmark_format_table_grouped(this.#results, options.groups)
|
|
327
|
+
: benchmark_format_table(this.#results);
|
|
328
|
+
}
|
|
329
|
+
/**
|
|
330
|
+
* Format results as a Markdown table.
|
|
331
|
+
* @returns Formatted markdown string
|
|
332
|
+
*/
|
|
333
|
+
markdown() {
|
|
334
|
+
return benchmark_format_markdown(this.#results);
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* Format results as JSON.
|
|
338
|
+
* @param options - Formatting options (pretty, include_timings)
|
|
339
|
+
* @returns JSON string
|
|
340
|
+
*/
|
|
341
|
+
json(options) {
|
|
342
|
+
return benchmark_format_json(this.#results, options);
|
|
343
|
+
}
|
|
344
|
+
/**
|
|
345
|
+
* Get the benchmark results.
|
|
346
|
+
* Returns a shallow copy to prevent external mutation.
|
|
347
|
+
* @returns Array of benchmark results
|
|
348
|
+
*/
|
|
349
|
+
results() {
|
|
350
|
+
return [...this.#results];
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* Reset the benchmark results.
|
|
354
|
+
* Keeps tasks intact so benchmarks can be rerun.
|
|
355
|
+
* @returns This Benchmark instance for chaining
|
|
356
|
+
*/
|
|
357
|
+
reset() {
|
|
358
|
+
this.#results = [];
|
|
359
|
+
return this;
|
|
360
|
+
}
|
|
361
|
+
/**
|
|
362
|
+
* Clear everything (results and tasks).
|
|
363
|
+
* Use this to start fresh with a new set of benchmarks.
|
|
364
|
+
* @returns This Benchmark instance for chaining
|
|
365
|
+
*/
|
|
366
|
+
clear() {
|
|
367
|
+
this.#results = [];
|
|
368
|
+
this.#tasks.length = 0;
|
|
369
|
+
return this;
|
|
370
|
+
}
|
|
371
|
+
/**
|
|
372
|
+
* Get a quick text summary of the fastest task.
|
|
373
|
+
* @returns Human-readable summary string
|
|
374
|
+
*
|
|
375
|
+
* @example
|
|
376
|
+
* ```ts
|
|
377
|
+
* console.log(bench.summary());
|
|
378
|
+
* // "Fastest: slugify_v2 (1,285,515.00 ops/sec, 786.52ns per op)"
|
|
379
|
+
* // "Slowest: slugify (252,955.00 ops/sec, 3.95μs per op)"
|
|
380
|
+
* // "Speed difference: 5.08x"
|
|
381
|
+
* ```
|
|
382
|
+
*/
|
|
383
|
+
summary() {
|
|
384
|
+
if (this.#results.length === 0)
|
|
385
|
+
return 'No results';
|
|
386
|
+
const fastest = this.#results.reduce((a, b) => a.stats.ops_per_second > b.stats.ops_per_second ? a : b);
|
|
387
|
+
const slowest = this.#results.reduce((a, b) => a.stats.ops_per_second < b.stats.ops_per_second ? a : b);
|
|
388
|
+
const ratio = fastest.stats.ops_per_second / slowest.stats.ops_per_second;
|
|
389
|
+
// Detect best unit for consistent display
|
|
390
|
+
const mean_times = this.#results.map((r) => r.stats.mean_ns);
|
|
391
|
+
const unit = time_unit_detect_best(mean_times);
|
|
392
|
+
const lines = [];
|
|
393
|
+
lines.push(`Fastest: ${fastest.name} (${benchmark_format_number(fastest.stats.ops_per_second)} ops/sec, ${time_format(fastest.stats.mean_ns, unit)} per op)`);
|
|
394
|
+
if (this.#results.length > 1) {
|
|
395
|
+
lines.push(`Slowest: ${slowest.name} (${benchmark_format_number(slowest.stats.ops_per_second)} ops/sec, ${time_format(slowest.stats.mean_ns, unit)} per op)`);
|
|
396
|
+
lines.push(`Speed difference: ${ratio.toFixed(2)}x`);
|
|
397
|
+
}
|
|
398
|
+
return lines.join('\n');
|
|
399
|
+
}
|
|
400
|
+
}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Benchmark baseline storage and comparison utilities.
|
|
3
|
+
* Save benchmark results to disk and compare against baselines for regression detection.
|
|
4
|
+
*/
|
|
5
|
+
import { z } from 'zod';
|
|
6
|
+
import type { BenchmarkResult } from './benchmark_types.js';
|
|
7
|
+
import { type BenchmarkComparison } from './benchmark_stats.js';
|
|
8
|
+
/**
|
|
9
|
+
* Schema for a single benchmark entry in the baseline.
|
|
10
|
+
*/
|
|
11
|
+
export declare const BenchmarkBaselineEntry: z.ZodObject<{
|
|
12
|
+
name: z.ZodString;
|
|
13
|
+
mean_ns: z.ZodNumber;
|
|
14
|
+
median_ns: z.ZodNumber;
|
|
15
|
+
std_dev_ns: z.ZodNumber;
|
|
16
|
+
min_ns: z.ZodNumber;
|
|
17
|
+
max_ns: z.ZodNumber;
|
|
18
|
+
p75_ns: z.ZodNumber;
|
|
19
|
+
p90_ns: z.ZodNumber;
|
|
20
|
+
p95_ns: z.ZodNumber;
|
|
21
|
+
p99_ns: z.ZodNumber;
|
|
22
|
+
ops_per_second: z.ZodNumber;
|
|
23
|
+
sample_size: z.ZodNumber;
|
|
24
|
+
}, z.core.$strip>;
|
|
25
|
+
export type BenchmarkBaselineEntry = z.infer<typeof BenchmarkBaselineEntry>;
|
|
26
|
+
/**
|
|
27
|
+
* Schema for the complete baseline file.
|
|
28
|
+
*/
|
|
29
|
+
export declare const BenchmarkBaseline: z.ZodObject<{
|
|
30
|
+
version: z.ZodNumber;
|
|
31
|
+
timestamp: z.ZodString;
|
|
32
|
+
git_commit: z.ZodNullable<z.ZodString>;
|
|
33
|
+
git_branch: z.ZodNullable<z.ZodString>;
|
|
34
|
+
node_version: z.ZodString;
|
|
35
|
+
entries: z.ZodArray<z.ZodObject<{
|
|
36
|
+
name: z.ZodString;
|
|
37
|
+
mean_ns: z.ZodNumber;
|
|
38
|
+
median_ns: z.ZodNumber;
|
|
39
|
+
std_dev_ns: z.ZodNumber;
|
|
40
|
+
min_ns: z.ZodNumber;
|
|
41
|
+
max_ns: z.ZodNumber;
|
|
42
|
+
p75_ns: z.ZodNumber;
|
|
43
|
+
p90_ns: z.ZodNumber;
|
|
44
|
+
p95_ns: z.ZodNumber;
|
|
45
|
+
p99_ns: z.ZodNumber;
|
|
46
|
+
ops_per_second: z.ZodNumber;
|
|
47
|
+
sample_size: z.ZodNumber;
|
|
48
|
+
}, z.core.$strip>>;
|
|
49
|
+
}, z.core.$strip>;
|
|
50
|
+
export type BenchmarkBaseline = z.infer<typeof BenchmarkBaseline>;
|
|
51
|
+
/**
|
|
52
|
+
* Options for saving a baseline.
|
|
53
|
+
*/
|
|
54
|
+
export interface BenchmarkBaselineSaveOptions {
|
|
55
|
+
/** Directory to store baselines (default: '.gro/benchmarks') */
|
|
56
|
+
path?: string;
|
|
57
|
+
/** Git commit hash (auto-detected if not provided) */
|
|
58
|
+
git_commit?: string | null;
|
|
59
|
+
/** Git branch name (auto-detected if not provided) */
|
|
60
|
+
git_branch?: string | null;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Options for loading a baseline.
|
|
64
|
+
*/
|
|
65
|
+
export interface BenchmarkBaselineLoadOptions {
|
|
66
|
+
/** Directory to load baseline from (default: '.gro/benchmarks') */
|
|
67
|
+
path?: string;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Options for comparing against a baseline.
|
|
71
|
+
*/
|
|
72
|
+
export interface BenchmarkBaselineCompareOptions extends BenchmarkBaselineLoadOptions {
|
|
73
|
+
/**
|
|
74
|
+
* Minimum speedup ratio to consider a regression.
|
|
75
|
+
* For example, 1.05 means only flag regressions that are 5% or more slower.
|
|
76
|
+
* Default: 1.0 (any statistically significant slowdown is a regression)
|
|
77
|
+
*/
|
|
78
|
+
regression_threshold?: number;
|
|
79
|
+
/**
|
|
80
|
+
* Number of days after which to warn about stale baseline.
|
|
81
|
+
* Default: undefined (no staleness warning)
|
|
82
|
+
*/
|
|
83
|
+
staleness_warning_days?: number;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Result of comparing current results against a baseline.
|
|
87
|
+
*/
|
|
88
|
+
export interface BenchmarkBaselineComparisonResult {
|
|
89
|
+
/** Whether a baseline was found */
|
|
90
|
+
baseline_found: boolean;
|
|
91
|
+
/** Timestamp of the baseline */
|
|
92
|
+
baseline_timestamp: string | null;
|
|
93
|
+
/** Git commit of the baseline */
|
|
94
|
+
baseline_commit: string | null;
|
|
95
|
+
/** Age of the baseline in days */
|
|
96
|
+
baseline_age_days: number | null;
|
|
97
|
+
/** Whether the baseline is considered stale based on staleness_warning_days option */
|
|
98
|
+
baseline_stale: boolean;
|
|
99
|
+
/** Individual task comparisons */
|
|
100
|
+
comparisons: Array<BenchmarkBaselineTaskComparison>;
|
|
101
|
+
/** Tasks that regressed (slower with statistical significance), sorted by effect size (largest first) */
|
|
102
|
+
regressions: Array<BenchmarkBaselineTaskComparison>;
|
|
103
|
+
/** Tasks that improved (faster with statistical significance), sorted by effect size (largest first) */
|
|
104
|
+
improvements: Array<BenchmarkBaselineTaskComparison>;
|
|
105
|
+
/** Tasks with no significant change */
|
|
106
|
+
unchanged: Array<BenchmarkBaselineTaskComparison>;
|
|
107
|
+
/** Tasks in current run but not in baseline */
|
|
108
|
+
new_tasks: Array<string>;
|
|
109
|
+
/** Tasks in baseline but not in current run */
|
|
110
|
+
removed_tasks: Array<string>;
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Comparison result for a single task.
|
|
114
|
+
*/
|
|
115
|
+
export interface BenchmarkBaselineTaskComparison {
|
|
116
|
+
name: string;
|
|
117
|
+
baseline: BenchmarkBaselineEntry;
|
|
118
|
+
current: BenchmarkBaselineEntry;
|
|
119
|
+
comparison: BenchmarkComparison;
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Save benchmark results as the current baseline.
|
|
123
|
+
*
|
|
124
|
+
* @param results - Benchmark results to save
|
|
125
|
+
* @param options - Save options
|
|
126
|
+
*
|
|
127
|
+
* @example
|
|
128
|
+
* ```ts
|
|
129
|
+
* const bench = new Benchmark();
|
|
130
|
+
* bench.add('test', () => fn());
|
|
131
|
+
* await bench.run();
|
|
132
|
+
* await benchmark_baseline_save(bench.results());
|
|
133
|
+
* ```
|
|
134
|
+
*/
|
|
135
|
+
export declare const benchmark_baseline_save: (results: Array<BenchmarkResult>, options?: BenchmarkBaselineSaveOptions) => Promise<void>;
|
|
136
|
+
/**
|
|
137
|
+
* Load the current baseline from disk.
|
|
138
|
+
*
|
|
139
|
+
* @param options - Load options
|
|
140
|
+
* @returns The baseline, or null if not found or invalid
|
|
141
|
+
*
|
|
142
|
+
* @example
|
|
143
|
+
* ```ts
|
|
144
|
+
* const baseline = await benchmark_baseline_load();
|
|
145
|
+
* if (baseline) {
|
|
146
|
+
* console.log(`Baseline from ${baseline.timestamp}`);
|
|
147
|
+
* }
|
|
148
|
+
* ```
|
|
149
|
+
*/
|
|
150
|
+
export declare const benchmark_baseline_load: (options?: BenchmarkBaselineLoadOptions) => Promise<BenchmarkBaseline | null>;
|
|
151
|
+
/**
|
|
152
|
+
* Compare benchmark results against the stored baseline.
|
|
153
|
+
*
|
|
154
|
+
* @param results - Current benchmark results
|
|
155
|
+
* @param options - Comparison options including regression threshold and staleness warning
|
|
156
|
+
* @returns Comparison result with regressions, improvements, and unchanged tasks
|
|
157
|
+
*
|
|
158
|
+
* @example
|
|
159
|
+
* ```ts
|
|
160
|
+
* const bench = new Benchmark();
|
|
161
|
+
* bench.add('test', () => fn());
|
|
162
|
+
* await bench.run();
|
|
163
|
+
*
|
|
164
|
+
* const comparison = await benchmark_baseline_compare(bench.results(), {
|
|
165
|
+
* regression_threshold: 1.05, // Only flag regressions 5% or more slower
|
|
166
|
+
* staleness_warning_days: 7, // Warn if baseline is older than 7 days
|
|
167
|
+
* });
|
|
168
|
+
* if (comparison.regressions.length > 0) {
|
|
169
|
+
* console.log('Performance regressions detected!');
|
|
170
|
+
* for (const r of comparison.regressions) {
|
|
171
|
+
* console.log(` ${r.name}: ${r.comparison.speedup_ratio.toFixed(2)}x slower`);
|
|
172
|
+
* }
|
|
173
|
+
* process.exit(1);
|
|
174
|
+
* }
|
|
175
|
+
* ```
|
|
176
|
+
*/
|
|
177
|
+
export declare const benchmark_baseline_compare: (results: Array<BenchmarkResult>, options?: BenchmarkBaselineCompareOptions) => Promise<BenchmarkBaselineComparisonResult>;
|
|
178
|
+
/**
|
|
179
|
+
* Format a baseline comparison result as a human-readable string.
|
|
180
|
+
*
|
|
181
|
+
* @param result - Comparison result from benchmark_baseline_compare
|
|
182
|
+
* @returns Formatted string summary
|
|
183
|
+
*/
|
|
184
|
+
export declare const benchmark_baseline_format: (result: BenchmarkBaselineComparisonResult) => string;
|
|
185
|
+
/**
|
|
186
|
+
* Format a baseline comparison result as JSON for programmatic consumption.
|
|
187
|
+
*
|
|
188
|
+
* @param result - Comparison result from benchmark_baseline_compare
|
|
189
|
+
* @param options - Formatting options
|
|
190
|
+
* @returns JSON string
|
|
191
|
+
*/
|
|
192
|
+
export declare const benchmark_baseline_format_json: (result: BenchmarkBaselineComparisonResult, options?: {
|
|
193
|
+
pretty?: boolean;
|
|
194
|
+
}) => string;
|
|
195
|
+
//# sourceMappingURL=benchmark_baseline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"benchmark_baseline.d.ts","sourceRoot":"../src/lib/","sources":["../src/lib/benchmark_baseline.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EAAC,CAAC,EAAC,MAAM,KAAK,CAAC;AAItB,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAEN,KAAK,mBAAmB,EAExB,MAAM,sBAAsB,CAAC;AAM9B;;GAEG;AACH,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;iBAajC,CAAC;AACH,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAE5E;;GAEG;AACH,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;iBAO5B,CAAC;AACH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAElE;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC5C,gEAAgE;IAChE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,sDAAsD;IACtD,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC5C,mEAAmE;IACnE,IAAI,CAAC,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,+BAAgC,SAAQ,4BAA4B;IACpF;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;OAGG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,iCAAiC;IACjD,mCAAmC;IACnC,cAAc,EAAE,OAAO,CAAC;IACxB,gCAAgC;IAChC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,iCAAiC;IACjC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,kCAAkC;IAClC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,sFAAsF;IACtF,cAAc,EAAE,OAAO,CAAC;IACxB,kCAAkC;IAClC,WAAW,EAAE,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACpD,yGAAyG;IACzG,WAAW,EAAE,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACpD,wGAAwG;IACxG,YAAY,EAAE,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACrD,uCAAuC;IACvC,SAAS,EAAE,KAAK,CAAC,+BAA+B,CAAC,CAAC;IAClD,+CAA+C;IAC/C,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACzB,+CAA+C;IAC/C,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,+BAA+B;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,sBAAsB,CAAC;IACjC,OAAO,EAAE,sBAAsB,CAAC;IAChC,UAAU,EAAE,mBAAmB,CAAC;CAChC;AAyBD;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,uBAAuB,GACnC,SAAS,KAAK,CAAC,eAAe,CAAC,EAC/B,UAAS,4BAAiC,KACxC,OAAO,CAAC,IAAI,CAwBd,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,uBAAuB,GACnC,UAAS,4BAAiC,KACxC,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAiClC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,eAAO,MAAM,0BAA0B,GACtC,SAAS,KAAK,CAAC,eAAe,CAAC,EAC/B,UAAS,+BAAoC,KAC3C,OAAO,CAAC,iCAAiC,CAmI3C,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,yBAAyB,GAAI,QAAQ,iCAAiC,KAAG,MAwErF,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,8BAA8B,GAC1C,QAAQ,iCAAiC,EACzC,UAAS;IAAC,MAAM,CAAC,EAAE,OAAO,CAAA;CAAM,KAC9B,MAuCF,CAAC"}
|