nano-benchmark 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +29 -0
- package/README.md +99 -0
- package/bin/nano-bench.js +327 -0
- package/bin/nano-watch.js +200 -0
- package/package.json +52 -0
- package/src/bench/compare.js +35 -0
- package/src/bench/runner.js +173 -0
- package/src/median.js +39 -0
- package/src/significance/kstest.js +43 -0
- package/src/significance/kwtest.js +81 -0
- package/src/significance/mwtest.js +35 -0
- package/src/stats/beta-ppf.js +13 -0
- package/src/stats/beta.js +26 -0
- package/src/stats/chi-squared-ppf.js +10 -0
- package/src/stats/erf.js +23 -0
- package/src/stats/gamma.js +14 -0
- package/src/stats/normal-ppf.js +19 -0
- package/src/stats/normal.js +25 -0
- package/src/stats/ppf.js +18 -0
- package/src/stats/rank.js +41 -0
- package/src/stats/z-ppf.js +17 -0
- package/src/stats/z.js +7 -0
- package/src/stats/zeta.js +14 -0
- package/src/stats.js +116 -0
- package/src/stream-median.js +93 -0
- package/src/stream-stats.js +67 -0
- package/src/utils/bsearch.js +9 -0
- package/src/utils/rk.js +36 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2022, Eugene Lazutkin
|
|
4
|
+
All rights reserved.
|
|
5
|
+
|
|
6
|
+
Redistribution and use in source and binary forms, with or without
|
|
7
|
+
modification, are permitted provided that the following conditions are met:
|
|
8
|
+
|
|
9
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
10
|
+
list of conditions and the following disclaimer.
|
|
11
|
+
|
|
12
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
13
|
+
this list of conditions and the following disclaimer in the documentation
|
|
14
|
+
and/or other materials provided with the distribution.
|
|
15
|
+
|
|
16
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
17
|
+
contributors may be used to endorse or promote products derived from
|
|
18
|
+
this software without specific prior written permission.
|
|
19
|
+
|
|
20
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
21
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
22
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
23
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
24
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
25
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
26
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
27
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
28
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
29
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
package/README.md
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# nano-benchmark [![NPM version][npm-img]][npm-url]
|
|
2
|
+
|
|
3
|
+
[npm-img]: https://img.shields.io/npm/v/nano-benchmark.svg
|
|
4
|
+
[npm-url]: https://npmjs.org/package/nano-benchmark
|
|
5
|
+
|
|
6
|
+
`nano-benchmark` provides command-line utilities for benchmarking code and related statistical modules.
|
|
7
|
+
|
|
8
|
+
Two utilities are available:
|
|
9
|
+
|
|
10
|
+
* `nano-watch` — provides statistics in a streaming mode continuously running your code,
|
|
11
|
+
watching memory usage and updating the output.
|
|
12
|
+
* `nano-bench` — runs benchmark tests on your code, calculating statistics and
|
|
13
|
+
statistical significance, and presenting them in a tabular format.
|
|
14
|
+
|
|
15
|
+
The utilities are mostly used to measure performance of your code and compare it with other variants.
|
|
16
|
+
It is geared toward benchmarking and performance tuning of a small fast snippets of code, e.g.,
|
|
17
|
+
used in tight loops.
|
|
18
|
+
|
|
19
|
+
## Visual samples
|
|
20
|
+
|
|
21
|
+
### `nano-watch`
|
|
22
|
+
|
|
23
|
+

|
|
24
|
+
|
|
25
|
+
### `nano-bench`
|
|
26
|
+
|
|
27
|
+

|
|
28
|
+
|
|
29
|
+
## Installation
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
npm install --save nano-benchmark
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
### Deno and Bun support
|
|
36
|
+
|
|
37
|
+
Both [deno](https://deno.land/) and [bun](https://bun.sh/) are supported.
|
|
38
|
+
|
|
39
|
+
Don't forget to specify the appropriate permissions for Deno to run the benchmark scripts:
|
|
40
|
+
`--allow-read` (required) and `--allow-hrtime` (optional but recommended).
|
|
41
|
+
|
|
42
|
+
## Documentation
|
|
43
|
+
|
|
44
|
+
Both utilities are available by name if you installed `nano-benchmark` globally
|
|
45
|
+
(`npm install -g nano-benchmark`).
|
|
46
|
+
If it is installed as a dependency, you can use utilities by name in the `scripts` section of
|
|
47
|
+
your `package.json` file or from the command line by prefixing them with `npx`, e.g., `npx nano-watch`.
|
|
48
|
+
|
|
49
|
+
Utilities are self-documented — run them with `--help` flag to learn about arguments.
|
|
50
|
+
|
|
51
|
+
Both utilities import a module to benchmark using its (default) export.
|
|
52
|
+
`nano-bench` assumes that it is an object with functional properties,
|
|
53
|
+
which should be benchmarked and compared. `nano-watch` can use the same file format
|
|
54
|
+
as `nano-bench` or it can use a single function.
|
|
55
|
+
|
|
56
|
+
Example of a module for `nano-bench` called `bench-strings-concat.js`:
|
|
57
|
+
|
|
58
|
+
```js
|
|
59
|
+
export default {
|
|
60
|
+
strings: n => {
|
|
61
|
+
const a = 'a',
|
|
62
|
+
b = 'b';
|
|
63
|
+
for (let i = 0; i < n; ++i) {
|
|
64
|
+
const x = a + '-' + b;
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
backticks: n => {
|
|
68
|
+
const a = 'a',
|
|
69
|
+
b = 'b';
|
|
70
|
+
for (let i = 0; i < n; ++i) {
|
|
71
|
+
const x = `${a}-${b}`;
|
|
72
|
+
}
|
|
73
|
+
},
|
|
74
|
+
join: n => {
|
|
75
|
+
const a = 'a',
|
|
76
|
+
b = 'b';
|
|
77
|
+
for (let i = 0; i < n; ++i) {
|
|
78
|
+
const x = [a, b].join('-');
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
The way to use it:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
npx nano-bench bench-strings-concat.js
|
|
88
|
+
npx nano-watch bench-strings-concat.js backticks
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
See [wiki](https://github.com/uhop/nano-bench/wiki) for more details.
|
|
92
|
+
|
|
93
|
+
## License
|
|
94
|
+
|
|
95
|
+
BSD 3-Clause License
|
|
96
|
+
|
|
97
|
+
## Release history
|
|
98
|
+
|
|
99
|
+
- 1.0.0: *Initial release.*
|
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import process from 'node:process';
|
|
4
|
+
|
|
5
|
+
import {Option, program} from 'commander';
|
|
6
|
+
|
|
7
|
+
import {CURSOR_NORMAL, CURSOR_INVISIBLE, CLEAR_EOL} from 'console-toolkit/ansi';
|
|
8
|
+
import {infinity, minus, multiplication} from 'console-toolkit/symbols.js';
|
|
9
|
+
import {
|
|
10
|
+
abbrNumber,
|
|
11
|
+
compareDifference,
|
|
12
|
+
formatInteger,
|
|
13
|
+
formatNumber,
|
|
14
|
+
formatTime,
|
|
15
|
+
prepareTimeFormat
|
|
16
|
+
} from 'console-toolkit/alphanumeric/number-formatters.js';
|
|
17
|
+
import style, {c} from 'console-toolkit/style';
|
|
18
|
+
import makeTable from 'console-toolkit/table';
|
|
19
|
+
import lineTheme from 'console-toolkit/themes/lines/unicode-rounded.js';
|
|
20
|
+
import Writer from 'console-toolkit/output/writer.js';
|
|
21
|
+
import Updater from 'console-toolkit/output/updater.js';
|
|
22
|
+
|
|
23
|
+
import {findLevel, benchmarkSeries, benchmarkSeriesPar} from '../src/bench/runner.js';
|
|
24
|
+
import {bootstrap, getWeightedValue, mean} from '../src/stats.js';
|
|
25
|
+
import mwtest from '../src/significance/mwtest.js';
|
|
26
|
+
import kwtest from '../src/significance/kwtest.js';
|
|
27
|
+
|
|
28
|
+
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)),
|
|
29
|
+
toInt = value => parseInt(value),
|
|
30
|
+
toFloat = value => parseFloat(value);
|
|
31
|
+
|
|
32
|
+
program
|
|
33
|
+
.name('nano-bench')
|
|
34
|
+
.description('Small utility to benchmark and compare code.')
|
|
35
|
+
.version('1.0.0')
|
|
36
|
+
.argument('<file>', 'File to benchmark')
|
|
37
|
+
.option('-m, --ms <ms>', 'measurement time in milliseconds', toInt, 50)
|
|
38
|
+
.addOption(
|
|
39
|
+
new Option('-i, --iterations <iterations>', 'measurement iterations (overrides --ms)')
|
|
40
|
+
.conflicts('ms')
|
|
41
|
+
.argParser(toInt)
|
|
42
|
+
)
|
|
43
|
+
.option('--min-iterations <min-iterations>', 'minimum number of iterations', toInt, 1)
|
|
44
|
+
.option('-e, --export <name>', 'name of the export in the file', 'default')
|
|
45
|
+
.option('-a, --alpha <alpha>', 'significance level', toFloat, 0.05)
|
|
46
|
+
.option('-s, --samples <samples>', 'number of samples', toInt, 100)
|
|
47
|
+
.option('-p, --parallel', 'take samples in parallel asynchronously')
|
|
48
|
+
.option('-b, --bootstrap <bootstrap>', 'number of bootstrap samples', toInt, 1000)
|
|
49
|
+
.showHelpAfterError('(add --help to see available options)');
|
|
50
|
+
|
|
51
|
+
program.parse();
|
|
52
|
+
|
|
53
|
+
const options = program.opts(),
|
|
54
|
+
args = program.args;
|
|
55
|
+
|
|
56
|
+
// validate the options
|
|
57
|
+
|
|
58
|
+
if (options.minIterations < 1) program.error('The minimum number of iterations must be >= 1');
|
|
59
|
+
if (options.alpha <= 0 || options.alpha >= 1)
|
|
60
|
+
program.error('The significance level must be > 0 and < 1');
|
|
61
|
+
if (options.samples < 1) program.error('The number of samples must be >= 1');
|
|
62
|
+
if (options.bootstrap < 1) program.error('The number of bootstrap samples must be >= 1');
|
|
63
|
+
|
|
64
|
+
// open the file
|
|
65
|
+
|
|
66
|
+
const fileName = new URL(args[0], `file://${process.cwd()}/`);
|
|
67
|
+
|
|
68
|
+
let fns;
|
|
69
|
+
try {
|
|
70
|
+
const file = await import(fileName);
|
|
71
|
+
fns = file[options.export];
|
|
72
|
+
} catch (error) {
|
|
73
|
+
program.error(`File not found: ${args[0]} (${fileName})`);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
if (!fns) program.error(`Export not found: ${options.export}`);
|
|
77
|
+
|
|
78
|
+
const names = Object.keys(fns).filter(name => typeof fns[name] == 'function');
|
|
79
|
+
if (names.length < 1) {
|
|
80
|
+
program.error('The exported object has no functions to measure');
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// set up the writer and the updater
|
|
84
|
+
|
|
85
|
+
const writer = new Writer();
|
|
86
|
+
let updater;
|
|
87
|
+
|
|
88
|
+
process.once('exit', () => updater?.done());
|
|
89
|
+
process.once('SIGINT', async () => process.exit(130));
|
|
90
|
+
process.once('SIGTERM', () => process.exit(143));
|
|
91
|
+
|
|
92
|
+
// setup running the benchmark
|
|
93
|
+
|
|
94
|
+
const numericSortingAsc = (a, b) => a - b;
|
|
95
|
+
const getPercentile = weight => samples =>
|
|
96
|
+
getWeightedValue(samples.sort(numericSortingAsc), weight);
|
|
97
|
+
|
|
98
|
+
const getStats = samples => {
|
|
99
|
+
samples.sort(numericSortingAsc);
|
|
100
|
+
const median = getWeightedValue(samples, 0.5),
|
|
101
|
+
lo = getWeightedValue(samples, options.alpha / 2),
|
|
102
|
+
hi = getWeightedValue(samples, 1 - options.alpha / 2);
|
|
103
|
+
return {median, lo, hi, bootstrap: false};
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
const getBootstrapStats = samples => {
|
|
107
|
+
const median = mean(bootstrap(getPercentile(0.5), samples, options.bootstrap)),
|
|
108
|
+
lo = mean(bootstrap(getPercentile(options.alpha / 2), samples, options.bootstrap)),
|
|
109
|
+
hi = mean(bootstrap(getPercentile(1 - options.alpha / 2), samples, options.bootstrap));
|
|
110
|
+
return {median, lo, hi, bootstrap: true};
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
const normalizeSamples = (samples, batchSize) => {
|
|
114
|
+
for (let i = 0; i < samples.length; ++i) {
|
|
115
|
+
samples[i] /= batchSize;
|
|
116
|
+
}
|
|
117
|
+
return samples;
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
const benchSeries = options.parallel ? benchmarkSeriesPar : benchmarkSeries;
|
|
121
|
+
|
|
122
|
+
const bold = s => style.bold.text(s),
|
|
123
|
+
num = s => style.bright.yellow.text(s),
|
|
124
|
+
faster = s => style.bright.green.text(bold(s) + ' faster'),
|
|
125
|
+
slower = s => style.bright.red.text(bold(s) + ' slower');
|
|
126
|
+
|
|
127
|
+
const rabbit = '\u{1f407}',
|
|
128
|
+
turtle = '\u{1f422}';
|
|
129
|
+
|
|
130
|
+
let iterations = [];
|
|
131
|
+
if (options.iterations > 0) {
|
|
132
|
+
iterations = new Array(names.length).fill(Math.max(options.iterations, options.minIterations));
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
await writer.write([
|
|
136
|
+
c`{{bold.save.bright.cyan}}${program.name()}{{restore}} {{save.bright.yellow}}${program.version()}{{restore}}: ${program.description()}`,
|
|
137
|
+
'',
|
|
138
|
+
c`Confidence interval: {{save.bright.yellow}}${formatNumber(100 * (1 - options.alpha), {
|
|
139
|
+
precision: 2
|
|
140
|
+
})}%{{restore}}, samples: {{save.bright.yellow}}${formatInteger(
|
|
141
|
+
options.samples
|
|
142
|
+
)}{{restore}}, bootstrap samples: {{save.bright.yellow}}${formatInteger(
|
|
143
|
+
options.bootstrap
|
|
144
|
+
)}{{restore}}`,
|
|
145
|
+
iterations.length
|
|
146
|
+
? c`Measuring {{save.bright.yellow}}${formatInteger(
|
|
147
|
+
iterations[0]
|
|
148
|
+
)}{{restore}} iterations per sample ({{save.bright.yellow}}${formatInteger(
|
|
149
|
+
iterations[0] * options.samples
|
|
150
|
+
)}{{restore}} per function)`
|
|
151
|
+
: c`Measuring {{save.bright.yellow}}${formatTime(
|
|
152
|
+
options.ms,
|
|
153
|
+
prepareTimeFormat([options.ms], 1000)
|
|
154
|
+
)}{{restore}} per sample (~{{save.bright.yellow}}${formatTime(
|
|
155
|
+
options.ms * 2 * options.samples,
|
|
156
|
+
prepareTimeFormat([options.ms * 2 * options.samples], 1000)
|
|
157
|
+
)}{{restore}} per function)`,
|
|
158
|
+
''
|
|
159
|
+
]);
|
|
160
|
+
|
|
161
|
+
const results = [],
|
|
162
|
+
stats = [];
|
|
163
|
+
|
|
164
|
+
const tableHeader1 = [
|
|
165
|
+
{value: 'name', height: 2, align: 'dc'},
|
|
166
|
+
{value: 'time', width: 3, align: 'c'},
|
|
167
|
+
null,
|
|
168
|
+
null,
|
|
169
|
+
{value: 'op/s', height: 2, align: 'dc'},
|
|
170
|
+
{value: 'batch', height: 2, align: 'dc'}
|
|
171
|
+
].map(cell => (cell ? {...cell, value: bold(cell.value)} : null)),
|
|
172
|
+
tableHeader2 = [
|
|
173
|
+
null,
|
|
174
|
+
{value: 'median', align: 'c'},
|
|
175
|
+
{value: '+', align: 'c'},
|
|
176
|
+
{value: minus, align: 'c'},
|
|
177
|
+
null,
|
|
178
|
+
null
|
|
179
|
+
].map(cell => (cell ? {...cell, value: bold(cell.value)} : null));
|
|
180
|
+
|
|
181
|
+
const makeTableData = () => {
|
|
182
|
+
const tableData = [tableHeader1, tableHeader2];
|
|
183
|
+
for (let i = 0; i < names.length; ++i) {
|
|
184
|
+
const row = [bold(names[i])],
|
|
185
|
+
s = stats[i];
|
|
186
|
+
if (s) {
|
|
187
|
+
const format = prepareTimeFormat([s.median - s.lo, s.median, s.hi - s.median], 1000);
|
|
188
|
+
row.push(
|
|
189
|
+
{value: bold(num(formatTime(s.median, format))), align: 'r'},
|
|
190
|
+
{value: num('+' + formatTime(s.hi - s.median, format)), align: 'r'},
|
|
191
|
+
{value: num(minus + formatTime(s.median - s.lo, format)), align: 'r'},
|
|
192
|
+
{value: num(abbrNumber(1000 / s.median)), align: 'r'}
|
|
193
|
+
);
|
|
194
|
+
} else if (i == stats.length) {
|
|
195
|
+
row.push({value: 'measuring...', width: 4}, null, null, null);
|
|
196
|
+
} else {
|
|
197
|
+
row.push(null, null, null, null);
|
|
198
|
+
}
|
|
199
|
+
row.push(i < iterations.length ? {value: num(abbrNumber(iterations[i])), align: 'r'} : null);
|
|
200
|
+
tableData.push(row);
|
|
201
|
+
}
|
|
202
|
+
return tableData;
|
|
203
|
+
};
|
|
204
|
+
|
|
205
|
+
// find the level
|
|
206
|
+
|
|
207
|
+
const report = () => {
|
|
208
|
+
const tableData = makeTableData(),
|
|
209
|
+
table = makeTable(tableData, lineTheme);
|
|
210
|
+
table.vAxis[2] = 2;
|
|
211
|
+
return table.toStrings();
|
|
212
|
+
};
|
|
213
|
+
|
|
214
|
+
updater = new Updater(
|
|
215
|
+
report,
|
|
216
|
+
{prologue: CURSOR_INVISIBLE, epilogue: CURSOR_NORMAL, afterLine: CLEAR_EOL},
|
|
217
|
+
writer
|
|
218
|
+
);
|
|
219
|
+
|
|
220
|
+
while (iterations.length < names.length) {
|
|
221
|
+
const index = iterations.length,
|
|
222
|
+
fn = fns[names[index]];
|
|
223
|
+
|
|
224
|
+
iterations.push(0);
|
|
225
|
+
|
|
226
|
+
const batchSize = await findLevel(
|
|
227
|
+
fn,
|
|
228
|
+
{threshold: options.ms, startFrom: options.minIterations},
|
|
229
|
+
async (name, data) => {
|
|
230
|
+
if (name === 'finding-level-next') {
|
|
231
|
+
iterations[index] = data.n;
|
|
232
|
+
await updater.update();
|
|
233
|
+
await sleep(5);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
);
|
|
237
|
+
|
|
238
|
+
iterations[index] = batchSize;
|
|
239
|
+
await updater.update();
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// run the benchmark
|
|
243
|
+
|
|
244
|
+
for (let i = 0; i < iterations.length; ++i) {
|
|
245
|
+
const batchSize = iterations[i],
|
|
246
|
+
samples = await benchSeries(fns[names[i]], batchSize, {nSeries: options.samples});
|
|
247
|
+
normalizeSamples(samples, batchSize);
|
|
248
|
+
results.push(samples);
|
|
249
|
+
stats.push(getStats(samples));
|
|
250
|
+
await updater.update();
|
|
251
|
+
await sleep(5);
|
|
252
|
+
stats[i] = getBootstrapStats(samples);
|
|
253
|
+
await updater.update();
|
|
254
|
+
await sleep(5);
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
await updater.final();
|
|
258
|
+
updater = null;
|
|
259
|
+
|
|
260
|
+
// calculate significance
|
|
261
|
+
|
|
262
|
+
if (results.length > 1) {
|
|
263
|
+
let significance = null;
|
|
264
|
+
|
|
265
|
+
for (const samples of results) samples.sort(numericSortingAsc);
|
|
266
|
+
if (results.length == 2) {
|
|
267
|
+
const result = mwtest(results[0], results[1], options.alpha);
|
|
268
|
+
if (result.different)
|
|
269
|
+
significance = [
|
|
270
|
+
[false, result.different],
|
|
271
|
+
[result.different, false]
|
|
272
|
+
];
|
|
273
|
+
} else {
|
|
274
|
+
const result = kwtest(results, options.alpha);
|
|
275
|
+
if (result.different) significance = result.groupDifference;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
if (significance) {
|
|
279
|
+
const sortedStats = stats.slice().sort((a, b) => a.median - b.median),
|
|
280
|
+
tableData = [[' ', bold('#'), bold('name')]];
|
|
281
|
+
let rabbitIndex = -1,
|
|
282
|
+
turtleIndex = -1;
|
|
283
|
+
for (let i = 0; i < names.length; ++i) {
|
|
284
|
+
tableData[0].push({value: bold(formatInteger(i + 1)), align: 'c'});
|
|
285
|
+
const row = [null, formatInteger(i + 1), bold(names[i])],
|
|
286
|
+
signRow = significance[i];
|
|
287
|
+
for (let j = 0; j < signRow.length; ++j) {
|
|
288
|
+
if (signRow[j]) {
|
|
289
|
+
const result = compareDifference(stats[i].median, stats[j].median);
|
|
290
|
+
let text = '';
|
|
291
|
+
if (result.infinity) {
|
|
292
|
+
text = infinity;
|
|
293
|
+
} else if (result.percentage) {
|
|
294
|
+
text = result.percentage + '%';
|
|
295
|
+
} else if (result.ratio) {
|
|
296
|
+
text = result.ratio + multiplication;
|
|
297
|
+
}
|
|
298
|
+
if (text) {
|
|
299
|
+
text = result.less ? faster(text) : slower(text);
|
|
300
|
+
row.push({value: text, align: 'c'});
|
|
301
|
+
} else {
|
|
302
|
+
row.push(null);
|
|
303
|
+
}
|
|
304
|
+
} else {
|
|
305
|
+
row.push(null);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
if (stats[i] === sortedStats[0]) {
|
|
309
|
+
row[0] = {value: '\t1', align: 'c'};
|
|
310
|
+
} else if (stats[i] === sortedStats[sortedStats.length - 1]) {
|
|
311
|
+
row[0] = {value: '\t2', align: 'c'};
|
|
312
|
+
}
|
|
313
|
+
tableData.push(row);
|
|
314
|
+
}
|
|
315
|
+
const table = makeTable(tableData, lineTheme);
|
|
316
|
+
table.vAxis[1] = 2;
|
|
317
|
+
writer.writeString(
|
|
318
|
+
c`\n{{save.bright.cyan.bold}}The difference is statistically significant:{{restore}}\n\n`
|
|
319
|
+
);
|
|
320
|
+
const tableStrings = table
|
|
321
|
+
.toStrings()
|
|
322
|
+
.map(line => line.replace(/\t(1|2)/g, m => (m[1] == '2' ? turtle : rabbit)));
|
|
323
|
+
writer.write(tableStrings);
|
|
324
|
+
} else {
|
|
325
|
+
writer.writeString('\nThe difference is not statistically significant.\n');
|
|
326
|
+
}
|
|
327
|
+
}
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import process from 'node:process';
|
|
4
|
+
|
|
5
|
+
import {program} from 'commander';
|
|
6
|
+
|
|
7
|
+
import {CURSOR_NORMAL, CURSOR_INVISIBLE, CLEAR_EOL} from 'console-toolkit/ansi';
|
|
8
|
+
import {
|
|
9
|
+
abbrNumber,
|
|
10
|
+
formatInteger,
|
|
11
|
+
formatNumber,
|
|
12
|
+
formatTime,
|
|
13
|
+
prepareTimeFormat
|
|
14
|
+
} from 'console-toolkit/alphanumeric/number-formatters.js';
|
|
15
|
+
import style, {c} from 'console-toolkit/style';
|
|
16
|
+
import makeTable from 'console-toolkit/table';
|
|
17
|
+
import lineTheme from 'console-toolkit/themes/lines/unicode-rounded.js';
|
|
18
|
+
import Writer from 'console-toolkit/output/writer.js';
|
|
19
|
+
import Updater from 'console-toolkit/output/updater.js';
|
|
20
|
+
|
|
21
|
+
import {findLevel, benchmark} from '../src/bench/runner.js';
|
|
22
|
+
import {MedianCounter} from '../src/stream-median.js';
|
|
23
|
+
import {StatCounter} from '../src/stream-stats.js';
|
|
24
|
+
|
|
25
|
+
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
|
26
|
+
|
|
27
|
+
program
|
|
28
|
+
.name('nano-watch')
|
|
29
|
+
.description('Small utility to continuously benchmark code.')
|
|
30
|
+
.version('1.0.0')
|
|
31
|
+
.argument('<file>', 'File to benchmark')
|
|
32
|
+
.argument('[method]', 'Method name to benchmark')
|
|
33
|
+
.option('-m, --ms <ms>', 'milliseconds per iteration', value => parseInt(value), 500)
|
|
34
|
+
.option('-i, --iterations <number>', 'number of iterations (default: Infinity)', value =>
|
|
35
|
+
parseInt(value)
|
|
36
|
+
)
|
|
37
|
+
.option('-e, --export <name>', 'name of the export in the file', 'default')
|
|
38
|
+
.showHelpAfterError('(add --help to see available options)');
|
|
39
|
+
|
|
40
|
+
program.parse();
|
|
41
|
+
|
|
42
|
+
const options = program.opts(),
|
|
43
|
+
args = program.args;
|
|
44
|
+
|
|
45
|
+
const fileName = new URL(args[0], `file://${process.cwd()}/`);
|
|
46
|
+
|
|
47
|
+
let fn;
|
|
48
|
+
try {
|
|
49
|
+
const file = await import(fileName);
|
|
50
|
+
fn = file[options.export];
|
|
51
|
+
} catch (error) {
|
|
52
|
+
program.error(`File not found: ${args[0]} (${fileName})`);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
if (!fn) program.error(`Export not found: ${options.export}`);
|
|
56
|
+
if (args[1]) fn = fn[args[1]];
|
|
57
|
+
if (typeof fn != 'function')
|
|
58
|
+
program.error(
|
|
59
|
+
`Function not found (export: ${options.export}${args[1] ? `, method: ${args[1]}` : ''})`
|
|
60
|
+
);
|
|
61
|
+
|
|
62
|
+
const iterations =
|
|
63
|
+
isNaN(options.iterations) || options.iterations <= 0 ? Infinity : options.iterations;
|
|
64
|
+
|
|
65
|
+
const writer = new Writer();
|
|
66
|
+
|
|
67
|
+
await writer.writeString(
|
|
68
|
+
c`{{bold.save.bright.cyan}}${program.name()}{{restore}} {{save.bright.yellow}}${program.version()}{{restore}}: ${program.description()}\n\n`
|
|
69
|
+
);
|
|
70
|
+
|
|
71
|
+
let updater;
|
|
72
|
+
|
|
73
|
+
process.once('exit', () => updater?.done());
|
|
74
|
+
process.once('SIGINT', async () => {
|
|
75
|
+
if (typeof updater?.finalFrame == 'function') await updater.finalFrame();
|
|
76
|
+
process.exit(0);
|
|
77
|
+
});
|
|
78
|
+
process.once('SIGTERM', () => process.exit(143));
|
|
79
|
+
|
|
80
|
+
function reportFindLevel(state, level, time) {
|
|
81
|
+
switch (state) {
|
|
82
|
+
case 'active':
|
|
83
|
+
const format = prepareTimeFormat([time], 1000),
|
|
84
|
+
timeString = formatTime(time, format);
|
|
85
|
+
return c`Batch size: {{save.bright.cyan}}${abbrNumber(
|
|
86
|
+
level
|
|
87
|
+
)}{{restore}}, time: {{bright.cyan}}${timeString}`;
|
|
88
|
+
case 'finished':
|
|
89
|
+
return c`Batch size: {{bright.cyan}}${abbrNumber(
|
|
90
|
+
level
|
|
91
|
+
)}{{reset.all}} {{dim}}(use Ctrl+C to stop)\n`;
|
|
92
|
+
}
|
|
93
|
+
return [];
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
updater = new Updater(
|
|
97
|
+
reportFindLevel,
|
|
98
|
+
{prologue: CURSOR_INVISIBLE, epilogue: CURSOR_NORMAL, afterLine: CLEAR_EOL},
|
|
99
|
+
writer
|
|
100
|
+
);
|
|
101
|
+
|
|
102
|
+
const batchSize = await findLevel(fn, {threshold: options.ms}, async (name, data) => {
|
|
103
|
+
if (name === 'finding-level-next') {
|
|
104
|
+
await updater.update(undefined, data.n, data.time);
|
|
105
|
+
await sleep(5);
|
|
106
|
+
}
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
await updater.final(batchSize);
|
|
110
|
+
updater = null;
|
|
111
|
+
|
|
112
|
+
const medianCounter = new MedianCounter();
|
|
113
|
+
const statCounter = new StatCounter();
|
|
114
|
+
|
|
115
|
+
const showData = time => {
|
|
116
|
+
const m = process.memoryUsage(),
|
|
117
|
+
median = medianCounter.get(),
|
|
118
|
+
stdDev = Math.sqrt(statCounter.variance),
|
|
119
|
+
format = {
|
|
120
|
+
...prepareTimeFormat([statCounter.mean, stdDev, median], 1000),
|
|
121
|
+
keepFractionAsIs: true
|
|
122
|
+
},
|
|
123
|
+
tableData = [
|
|
124
|
+
['#', 'time', 'mean', 'stdDev', 'median', 'skewness', 'kurtosis'],
|
|
125
|
+
[style.bright.yellow.text(formatInteger(statCounter.count))]
|
|
126
|
+
.concat(
|
|
127
|
+
[time, statCounter.mean, stdDev, median]
|
|
128
|
+
.map(value => formatTime(value, format))
|
|
129
|
+
.map(value => style.bright.yellow.text(value))
|
|
130
|
+
)
|
|
131
|
+
.concat(
|
|
132
|
+
[statCounter.skewness, statCounter.kurtosis]
|
|
133
|
+
.map(value => formatNumber(value, {precision: 3}))
|
|
134
|
+
.map(value => style.bright.yellow.text(value))
|
|
135
|
+
),
|
|
136
|
+
[
|
|
137
|
+
style.bold.text('op/s'),
|
|
138
|
+
style.bright.yellow.text(abbrNumber(1000 / time)),
|
|
139
|
+
style.bright.yellow.text(abbrNumber(1000 / statCounter.mean)),
|
|
140
|
+
null,
|
|
141
|
+
style.bright.yellow.text(abbrNumber(1000 / median)),
|
|
142
|
+
null,
|
|
143
|
+
null
|
|
144
|
+
],
|
|
145
|
+
[
|
|
146
|
+
style.bold.text('memory'),
|
|
147
|
+
{
|
|
148
|
+
value: c`{{save.bold}}used:{{restore}} {{bright.cyan}}${abbrNumber(m.heapUsed)}`,
|
|
149
|
+
width: 2
|
|
150
|
+
},
|
|
151
|
+
null,
|
|
152
|
+
{
|
|
153
|
+
value: c`{{save.bold}}total:{{restore}} {{bright.cyan}}${abbrNumber(m.heapTotal)}`,
|
|
154
|
+
width: 2
|
|
155
|
+
},
|
|
156
|
+
null,
|
|
157
|
+
{
|
|
158
|
+
value: c`{{save.bold}}resident set size:{{restore}} {{bright.cyan}}${abbrNumber(m.rss)}`,
|
|
159
|
+
width: 2
|
|
160
|
+
},
|
|
161
|
+
null
|
|
162
|
+
]
|
|
163
|
+
],
|
|
164
|
+
table = makeTable(tableData, lineTheme, {hAlignDefault: 'r', states: {rowFirst: style.bold}});
|
|
165
|
+
|
|
166
|
+
table.vAxis[3] = 2;
|
|
167
|
+
|
|
168
|
+
return table.toStrings();
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
function reportBenchmark(state, time) {
|
|
172
|
+
switch (state) {
|
|
173
|
+
case 'active':
|
|
174
|
+
case 'finished': {
|
|
175
|
+
return showData(time);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return [];
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
updater = new Updater(
|
|
182
|
+
reportBenchmark,
|
|
183
|
+
{prologue: CURSOR_INVISIBLE, epilogue: CURSOR_NORMAL, afterLine: CLEAR_EOL},
|
|
184
|
+
writer
|
|
185
|
+
);
|
|
186
|
+
|
|
187
|
+
updater.finalFrame = () => updater.final(updater.data.time);
|
|
188
|
+
|
|
189
|
+
for (let i = 0; i < iterations; ++i) {
|
|
190
|
+
const time = (await benchmark(fn, batchSize)) / batchSize;
|
|
191
|
+
medianCounter.add(time);
|
|
192
|
+
statCounter.add(time);
|
|
193
|
+
|
|
194
|
+
updater.data = {time};
|
|
195
|
+
await updater.update(undefined, time);
|
|
196
|
+
await sleep(5);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
await updater.finalFrame();
|
|
200
|
+
updater = null;
|
package/package.json
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "nano-benchmark",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Small utilities to benchmark code with Node.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "src/index.js",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": "./src/index.js",
|
|
9
|
+
"./*": "./src/*"
|
|
10
|
+
},
|
|
11
|
+
"bin": {
|
|
12
|
+
"nano-bench": "./bin/nano-bench.js",
|
|
13
|
+
"nano-watch": "./bin/nano-watch.js"
|
|
14
|
+
},
|
|
15
|
+
"scripts": {
|
|
16
|
+
"test": "tape6 --flags FO"
|
|
17
|
+
},
|
|
18
|
+
"repository": {
|
|
19
|
+
"type": "git",
|
|
20
|
+
"url": "git+https://github.com/uhop/nano-bench.git"
|
|
21
|
+
},
|
|
22
|
+
"keywords": [
|
|
23
|
+
"benchmark",
|
|
24
|
+
"performance",
|
|
25
|
+
"statistics"
|
|
26
|
+
],
|
|
27
|
+
"author": "Eugene Lazutkin <eugene.lazutkin@gmail.com> (https://www.lazutkin.com/)",
|
|
28
|
+
"license": "BSD-3-Clause",
|
|
29
|
+
"funding": {
|
|
30
|
+
"type": "github",
|
|
31
|
+
"url": "https://github.com/sponsors/uhop"
|
|
32
|
+
},
|
|
33
|
+
"bugs": {
|
|
34
|
+
"url": "https://github.com/uhop/nano-bench/issues"
|
|
35
|
+
},
|
|
36
|
+
"homepage": "https://github.com/uhop/nano-bench#readme",
|
|
37
|
+
"files": [
|
|
38
|
+
"src"
|
|
39
|
+
],
|
|
40
|
+
"devDependencies": {
|
|
41
|
+
"tape-six": "^0.9.6"
|
|
42
|
+
},
|
|
43
|
+
"tape6": {
|
|
44
|
+
"tests": [
|
|
45
|
+
"tests/test-*.*js"
|
|
46
|
+
]
|
|
47
|
+
},
|
|
48
|
+
"dependencies": {
|
|
49
|
+
"commander": "^12.1.0",
|
|
50
|
+
"console-toolkit": "^1.2.0"
|
|
51
|
+
}
|
|
52
|
+
}
|