@vercel/devlow-bench 0.3.4 → 16.4.0-canary.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -33
- package/dist/browser.d.ts +1 -1
- package/dist/browser.js +52 -44
- package/dist/cli.js +114 -69
- package/dist/compare.d.ts +14 -0
- package/dist/compare.js +209 -0
- package/dist/describe.d.ts +1 -1
- package/dist/describe.js +18 -18
- package/dist/file.js +5 -5
- package/dist/index.d.ts +16 -3
- package/dist/index.js +3 -2
- package/dist/interfaces/compare.d.ts +4 -0
- package/dist/interfaces/compare.js +27 -0
- package/dist/interfaces/compose.d.ts +1 -1
- package/dist/interfaces/compose.js +1 -1
- package/dist/interfaces/console.d.ts +4 -2
- package/dist/interfaces/console.js +30 -8
- package/dist/interfaces/constants.js +7 -7
- package/dist/interfaces/datadog.d.ts +1 -1
- package/dist/interfaces/datadog.js +13 -13
- package/dist/interfaces/interactive.d.ts +1 -1
- package/dist/interfaces/interactive.js +8 -8
- package/dist/interfaces/json.d.ts +4 -2
- package/dist/interfaces/json.js +36 -17
- package/dist/interfaces/shell-test.d.ts +1 -0
- package/dist/interfaces/shell-test.js +56 -0
- package/dist/interfaces/snapshot.d.ts +6 -0
- package/dist/interfaces/snapshot.js +60 -0
- package/dist/interfaces/snowflake-test.js +3 -3
- package/dist/interfaces/snowflake.d.ts +1 -1
- package/dist/interfaces/snowflake.js +15 -13
- package/dist/runner.d.ts +5 -2
- package/dist/runner.js +132 -24
- package/dist/shell.d.ts +5 -3
- package/dist/shell.js +56 -28
- package/dist/snapshot.d.ts +18 -0
- package/dist/snapshot.js +162 -0
- package/dist/statistics-test.d.ts +1 -0
- package/dist/statistics-test.js +102 -0
- package/dist/statistics.d.ts +19 -0
- package/dist/statistics.js +123 -0
- package/dist/table.js +51 -51
- package/dist/units.js +6 -6
- package/dist/utils.d.ts +1 -0
- package/dist/utils.js +8 -6
- package/package.json +6 -5
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export declare const DEFAULT_SNAPSHOT_DIR = ".devlow-bench/snapshots";
|
|
2
|
+
export interface SnapshotRow {
|
|
3
|
+
timestamp: string;
|
|
4
|
+
scenario: string;
|
|
5
|
+
variant: string;
|
|
6
|
+
metric: string;
|
|
7
|
+
sample_idx: number;
|
|
8
|
+
value: number;
|
|
9
|
+
unit: string;
|
|
10
|
+
relative_to: string;
|
|
11
|
+
git_sha: string;
|
|
12
|
+
git_branch: string;
|
|
13
|
+
}
|
|
14
|
+
export declare function defaultSnapshotPath(dir?: string, now?: Date): string;
|
|
15
|
+
export declare function writeSnapshot(path: string, rows: SnapshotRow[]): Promise<void>;
|
|
16
|
+
export declare function readSnapshot(path: string): Promise<SnapshotRow[]>;
|
|
17
|
+
export declare function findMostRecentSnapshot(dir?: string, excludePath?: string): Promise<string | null>;
|
|
18
|
+
export declare function resolveCompareTarget(target: string | true | undefined, excludePath?: string): Promise<string | null>;
|
package/dist/snapshot.js
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import { readFile, readdir, mkdir, writeFile, stat } from 'node:fs/promises';
|
|
2
|
+
import { dirname, join, resolve } from 'node:path';
|
|
3
|
+
export const DEFAULT_SNAPSHOT_DIR = '.devlow-bench/snapshots';
|
|
4
|
+
const COLUMNS = [
|
|
5
|
+
'timestamp',
|
|
6
|
+
'scenario',
|
|
7
|
+
'variant',
|
|
8
|
+
'metric',
|
|
9
|
+
'sample_idx',
|
|
10
|
+
'value',
|
|
11
|
+
'unit',
|
|
12
|
+
'relative_to',
|
|
13
|
+
'git_sha',
|
|
14
|
+
'git_branch',
|
|
15
|
+
];
|
|
16
|
+
export function defaultSnapshotPath(dir = DEFAULT_SNAPSHOT_DIR, now = new Date()) {
|
|
17
|
+
// ISO-ish, filesystem-safe, lexicographic-sortable.
|
|
18
|
+
const stamp = now.toISOString().replace(/[:.]/g, '-').replace(/Z$/, 'Z');
|
|
19
|
+
return resolve(join(dir, `${stamp}.csv`));
|
|
20
|
+
}
|
|
21
|
+
export async function writeSnapshot(path, rows) {
|
|
22
|
+
await mkdir(dirname(path), { recursive: true });
|
|
23
|
+
const lines = [COLUMNS.join(',')];
|
|
24
|
+
for (const row of rows) {
|
|
25
|
+
lines.push(COLUMNS.map((c) => csvCell(row[c])).join(','));
|
|
26
|
+
}
|
|
27
|
+
await writeFile(path, lines.join('\n') + '\n');
|
|
28
|
+
}
|
|
29
|
+
export async function readSnapshot(path) {
|
|
30
|
+
const text = await readFile(path, 'utf-8');
|
|
31
|
+
const lines = text.split(/\r?\n/).filter((l) => l.length > 0);
|
|
32
|
+
if (lines.length === 0)
|
|
33
|
+
return [];
|
|
34
|
+
const header = parseCsvLine(lines[0]);
|
|
35
|
+
const idx = {};
|
|
36
|
+
for (let i = 0; i < header.length; i++) {
|
|
37
|
+
if (COLUMNS.includes(header[i])) {
|
|
38
|
+
idx[header[i]] = i;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
const rows = [];
|
|
42
|
+
for (let i = 1; i < lines.length; i++) {
|
|
43
|
+
const cells = parseCsvLine(lines[i]);
|
|
44
|
+
const get = (k) => {
|
|
45
|
+
const j = idx[k];
|
|
46
|
+
return j == null ? '' : (cells[j] ?? '');
|
|
47
|
+
};
|
|
48
|
+
rows.push({
|
|
49
|
+
timestamp: get('timestamp'),
|
|
50
|
+
scenario: get('scenario'),
|
|
51
|
+
variant: get('variant'),
|
|
52
|
+
metric: get('metric'),
|
|
53
|
+
sample_idx: Number(get('sample_idx')),
|
|
54
|
+
value: Number(get('value')),
|
|
55
|
+
unit: get('unit'),
|
|
56
|
+
relative_to: get('relative_to'),
|
|
57
|
+
git_sha: get('git_sha'),
|
|
58
|
+
git_branch: get('git_branch'),
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
return rows;
|
|
62
|
+
}
|
|
63
|
+
// Returns the newest .csv in the directory, excluding the optional exclude
|
|
64
|
+
// path. Returns null if the directory doesn't exist or has no matching files.
|
|
65
|
+
export async function findMostRecentSnapshot(dir = DEFAULT_SNAPSHOT_DIR, excludePath) {
|
|
66
|
+
let entries;
|
|
67
|
+
try {
|
|
68
|
+
entries = await readdir(dir);
|
|
69
|
+
}
|
|
70
|
+
catch (e) {
|
|
71
|
+
if (e.code === 'ENOENT')
|
|
72
|
+
return null;
|
|
73
|
+
throw e;
|
|
74
|
+
}
|
|
75
|
+
const candidates = entries.filter((f) => f.endsWith('.csv'));
|
|
76
|
+
const excludeResolved = excludePath ? resolve(excludePath) : null;
|
|
77
|
+
let best = null;
|
|
78
|
+
for (const f of candidates) {
|
|
79
|
+
const full = resolve(join(dir, f));
|
|
80
|
+
if (excludeResolved && full === excludeResolved)
|
|
81
|
+
continue;
|
|
82
|
+
const s = await stat(full);
|
|
83
|
+
if (!best || s.mtimeMs > best.mtime) {
|
|
84
|
+
best = { path: full, mtime: s.mtimeMs };
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return best?.path ?? null;
|
|
88
|
+
}
|
|
89
|
+
// Resolve --compare: if given a directory, pick the newest .csv inside;
|
|
90
|
+
// if given a file, use it; if null/undefined, fall back to DEFAULT_SNAPSHOT_DIR.
|
|
91
|
+
export async function resolveCompareTarget(target, excludePath) {
|
|
92
|
+
if (target == null)
|
|
93
|
+
return null;
|
|
94
|
+
let candidatePath = null;
|
|
95
|
+
if (target === true) {
|
|
96
|
+
candidatePath = await findMostRecentSnapshot(DEFAULT_SNAPSHOT_DIR, excludePath);
|
|
97
|
+
}
|
|
98
|
+
else {
|
|
99
|
+
let isDir = false;
|
|
100
|
+
try {
|
|
101
|
+
isDir = (await stat(target)).isDirectory();
|
|
102
|
+
}
|
|
103
|
+
catch {
|
|
104
|
+
// Path doesn't exist; let caller error out.
|
|
105
|
+
}
|
|
106
|
+
if (isDir) {
|
|
107
|
+
candidatePath = await findMostRecentSnapshot(target, excludePath);
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
candidatePath = resolve(target);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return candidatePath;
|
|
114
|
+
}
|
|
115
|
+
function csvCell(v) {
|
|
116
|
+
const s = String(v);
|
|
117
|
+
if (/[,"\n\r]/.test(s)) {
|
|
118
|
+
return `"${s.replace(/"/g, '""')}"`;
|
|
119
|
+
}
|
|
120
|
+
return s;
|
|
121
|
+
}
|
|
122
|
+
function parseCsvLine(line) {
|
|
123
|
+
const cells = [];
|
|
124
|
+
let i = 0;
|
|
125
|
+
while (i < line.length) {
|
|
126
|
+
if (line[i] === '"') {
|
|
127
|
+
let s = '';
|
|
128
|
+
i++; // skip opening quote
|
|
129
|
+
while (i < line.length) {
|
|
130
|
+
if (line[i] === '"') {
|
|
131
|
+
if (line[i + 1] === '"') {
|
|
132
|
+
s += '"';
|
|
133
|
+
i += 2;
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
i++; // closing quote
|
|
137
|
+
break;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
else {
|
|
141
|
+
s += line[i];
|
|
142
|
+
i++;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
cells.push(s);
|
|
146
|
+
if (line[i] === ',')
|
|
147
|
+
i++;
|
|
148
|
+
}
|
|
149
|
+
else {
|
|
150
|
+
const next = line.indexOf(',', i);
|
|
151
|
+
if (next === -1) {
|
|
152
|
+
cells.push(line.slice(i));
|
|
153
|
+
i = line.length;
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
cells.push(line.slice(i, next));
|
|
157
|
+
i = next + 1;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
return cells;
|
|
162
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { test } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { mean, quantile, welchsTTest, mannWhitneyU } from './statistics.js';
|
|
4
|
+
function near(actual, expected, tol = 1e-6, label = '') {
|
|
5
|
+
assert.ok(Math.abs(actual - expected) < tol, `${label}: expected ${expected}, got ${actual} (tol ${tol})`);
|
|
6
|
+
}
|
|
7
|
+
test('mean', () => {
|
|
8
|
+
near(mean([1, 2, 3, 4, 5]), 3);
|
|
9
|
+
near(mean([10]), 10);
|
|
10
|
+
});
|
|
11
|
+
test('quantile (linear interpolation, numpy/R type 7)', () => {
|
|
12
|
+
// [1,2,3,4,5] with q=0.5 → 3, q=0.9 → 4.6
|
|
13
|
+
near(quantile([1, 2, 3, 4, 5], 0.5), 3);
|
|
14
|
+
near(quantile([1, 2, 3, 4, 5], 0.9), 4.6);
|
|
15
|
+
// sorts internally
|
|
16
|
+
near(quantile([5, 3, 1, 2, 4], 0.5), 3);
|
|
17
|
+
// single sample
|
|
18
|
+
near(quantile([42], 0.9), 42);
|
|
19
|
+
});
|
|
20
|
+
test("Welch's t-test: small samples", () => {
|
|
21
|
+
// Hand-verified: means 2 and 3, both sample variances = 1.
|
|
22
|
+
// sea = seb = 1/3, denom = 2/3
|
|
23
|
+
// t = -1 / sqrt(2/3) = -1.2247448713915892
|
|
24
|
+
// df = (2/3)^2 / (2 * (1/3)^2 / 2) = (4/9) / (1/9) = 4
|
|
25
|
+
// p ≈ 0.288 (two-sided, df=4)
|
|
26
|
+
const r = welchsTTest([1, 2, 3], [2, 3, 4]);
|
|
27
|
+
near(r.t, -1.2247448713915892, 1e-12, 't');
|
|
28
|
+
near(r.df, 4, 1e-12, 'df');
|
|
29
|
+
near(r.p, 0.288, 1e-3, 'p');
|
|
30
|
+
});
|
|
31
|
+
test("Welch's t-test: large clearly-different samples", () => {
|
|
32
|
+
// mean_a=12, mean_b=22, var_a=var_b=2.5 → t=-10 exactly, df=8 exactly.
|
|
33
|
+
const a = [10, 11, 12, 13, 14];
|
|
34
|
+
const b = [20, 21, 22, 23, 24];
|
|
35
|
+
const r = welchsTTest(a, b);
|
|
36
|
+
near(r.t, -10, 1e-12, 't');
|
|
37
|
+
near(r.df, 8, 1e-12, 'df');
|
|
38
|
+
// P(|T|>10 | df=8) is on the order of 8e-6.
|
|
39
|
+
assert.ok(r.p > 1e-6 && r.p < 1e-5, `expected p in (1e-6, 1e-5), got ${r.p}`);
|
|
40
|
+
});
|
|
41
|
+
test("Welch's t-test: identical samples → t=0, p=1", () => {
|
|
42
|
+
const r = welchsTTest([1, 2, 3, 4, 5], [1, 2, 3, 4, 5]);
|
|
43
|
+
near(r.t, 0);
|
|
44
|
+
near(r.p, 1);
|
|
45
|
+
});
|
|
46
|
+
test('Welch handles unequal variance (small group variance)', () => {
|
|
47
|
+
// [1..5] vs [10, 10.1, 10.2, 10.3, 10.4]
|
|
48
|
+
// var_a = 2.5, var_b = 0.025 → very unequal.
|
|
49
|
+
// sea = 0.5, seb = 0.005, denom = 0.505
|
|
50
|
+
// t = -7.2 / sqrt(0.505) ≈ -10.1318
|
|
51
|
+
// df = 0.505^2 / (0.0625 + 6.25e-6) ≈ 4.080
|
|
52
|
+
const r = welchsTTest([1, 2, 3, 4, 5], [10, 10.1, 10.2, 10.3, 10.4]);
|
|
53
|
+
near(r.t, -10.131804644116203, 1e-9, 't');
|
|
54
|
+
near(r.df, 4.08, 1e-2, 'df');
|
|
55
|
+
// df is small (~4) so heavier tails than the df=8 case: p should be larger.
|
|
56
|
+
assert.ok(r.p < 1e-3, `expected p<1e-3, got ${r.p}`);
|
|
57
|
+
});
|
|
58
|
+
test('t-distribution CDF: known critical value', () => {
|
|
59
|
+
// df=10, |t|=2.228 → two-sided p ≈ 0.05 (standard table).
|
|
60
|
+
// We verify by constructing a sample with exactly that t-stat.
|
|
61
|
+
// Easier: use the fact that for df=∞ (large df), Welch's reduces to z-test.
|
|
62
|
+
// df=1000, t≈1.96 should give p≈0.05.
|
|
63
|
+
// Construct a=[0]*n, b shifted: we can't get exact df without specific values,
|
|
64
|
+
// so we just confirm the formula gives sane values across df range.
|
|
65
|
+
// Pair 1: df ≈ 10, t ≈ 2.228 — synthesize with two known groups.
|
|
66
|
+
// Simpler check: at very large df, p should match the normal-tail p for the same z.
|
|
67
|
+
const huge = Array.from({ length: 500 }, (_, i) => i);
|
|
68
|
+
const huger = Array.from({ length: 500 }, (_, i) => i + 0.2); // tiny shift
|
|
69
|
+
const r = welchsTTest(huge, huger);
|
|
70
|
+
// With this setup df is enormous (~1000) and t is small but well-defined.
|
|
71
|
+
// p should be between 0 and 1; finite.
|
|
72
|
+
assert.ok(Number.isFinite(r.p));
|
|
73
|
+
assert.ok(r.p >= 0 && r.p <= 1);
|
|
74
|
+
});
|
|
75
|
+
test('Mann–Whitney U: n=5 vs n=5, max separation', () => {
|
|
76
|
+
// [1..5] vs [6..10]: U=0. Exact two-sided p = 2/252 ≈ 0.00794;
|
|
77
|
+
// asymptotic with continuity correction ≈ 0.012. Both indicate a strong
|
|
78
|
+
// separation; we just verify U=0 and p is below 0.05.
|
|
79
|
+
const r = mannWhitneyU([1, 2, 3, 4, 5], [6, 7, 8, 9, 10]);
|
|
80
|
+
assert.equal(r.u, 0);
|
|
81
|
+
assert.ok(r.p < 0.05, `expected p<0.05, got ${r.p}`);
|
|
82
|
+
assert.ok(r.p > 0.005, `expected p>0.005, got ${r.p}`);
|
|
83
|
+
});
|
|
84
|
+
test('Mann–Whitney U: fully overlapping → no rejection', () => {
|
|
85
|
+
const r = mannWhitneyU([1, 2, 3], [1, 2, 3]);
|
|
86
|
+
assert.ok(r.p > 0.5, `expected p>0.5, got ${r.p}`);
|
|
87
|
+
});
|
|
88
|
+
test('Mann–Whitney U: 3 vs 3 distinct, no ties', () => {
|
|
89
|
+
// U=0 for group A. Exact two-sided p = 0.1; asymptotic ≈ 0.081.
|
|
90
|
+
const r = mannWhitneyU([1, 2, 3], [4, 5, 6]);
|
|
91
|
+
assert.equal(r.u, 0);
|
|
92
|
+
assert.ok(r.p > 0.05 && r.p < 0.15, `expected p in (0.05, 0.15), got ${r.p}`);
|
|
93
|
+
});
|
|
94
|
+
test('Mann–Whitney U: medium samples use normal approximation', () => {
|
|
95
|
+
// Two clearly separated groups of 12.
|
|
96
|
+
// scipy two-sided p with continuity correction is in the ~1e-5 range.
|
|
97
|
+
const a = Array.from({ length: 12 }, (_, i) => i + 1);
|
|
98
|
+
const b = Array.from({ length: 12 }, (_, i) => i + 100);
|
|
99
|
+
const r = mannWhitneyU(a, b);
|
|
100
|
+
assert.equal(r.u, 0);
|
|
101
|
+
assert.ok(r.p < 1e-4, `expected p<1e-4, got ${r.p}`);
|
|
102
|
+
});
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export interface Summary {
|
|
2
|
+
mean: number;
|
|
3
|
+
p50: number;
|
|
4
|
+
p90: number;
|
|
5
|
+
}
|
|
6
|
+
export declare function mean(xs: number[]): number;
|
|
7
|
+
export declare function quantile(xs: number[], q: number): number;
|
|
8
|
+
export declare function summary(samples: number[]): Summary;
|
|
9
|
+
export interface WelchsTResult {
|
|
10
|
+
t: number;
|
|
11
|
+
df: number;
|
|
12
|
+
p: number;
|
|
13
|
+
}
|
|
14
|
+
export declare function welchsTTest(a: number[], b: number[]): WelchsTResult;
|
|
15
|
+
export interface MannWhitneyResult {
|
|
16
|
+
u: number;
|
|
17
|
+
p: number;
|
|
18
|
+
}
|
|
19
|
+
export declare function mannWhitneyU(a: number[], b: number[]): MannWhitneyResult;
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
// Statistics helpers for repeated-sampling benchmarks.
|
|
2
|
+
//
|
|
3
|
+
// We use `jstat` for the underlying distribution CDFs (Student's t and
|
|
4
|
+
// standard normal). The rest — Welch's t-statistic, average-rank ranking,
|
|
5
|
+
// and the Mann–Whitney U statistic + asymptotic-tail p-value — is
|
|
6
|
+
// implemented here directly.
|
|
7
|
+
//
|
|
8
|
+
// For Mann–Whitney we use the normal approximation with continuity
|
|
9
|
+
// correction. For very small samples this approximation is less accurate
|
|
10
|
+
// (n=5 vs n=5 has a minimum exact two-sided p ≈ 0.008 vs the
|
|
11
|
+
// approximation's ≈ 0.012). Document this at the call site, not here.
|
|
12
|
+
import jstat from 'jstat';
|
|
13
|
+
export function mean(xs) {
|
|
14
|
+
if (xs.length === 0)
|
|
15
|
+
return NaN;
|
|
16
|
+
let s = 0;
|
|
17
|
+
for (const x of xs)
|
|
18
|
+
s += x;
|
|
19
|
+
return s / xs.length;
|
|
20
|
+
}
|
|
21
|
+
// Linear-interpolation quantile (numpy/R "type 7" default).
|
|
22
|
+
export function quantile(xs, q) {
|
|
23
|
+
if (xs.length === 0)
|
|
24
|
+
return NaN;
|
|
25
|
+
if (xs.length === 1)
|
|
26
|
+
return xs[0];
|
|
27
|
+
const sorted = [...xs].sort((a, b) => a - b);
|
|
28
|
+
const pos = q * (sorted.length - 1);
|
|
29
|
+
const lo = Math.floor(pos);
|
|
30
|
+
const hi = Math.ceil(pos);
|
|
31
|
+
if (lo === hi)
|
|
32
|
+
return sorted[lo];
|
|
33
|
+
return sorted[lo] + (pos - lo) * (sorted[hi] - sorted[lo]);
|
|
34
|
+
}
|
|
35
|
+
export function summary(samples) {
|
|
36
|
+
return {
|
|
37
|
+
mean: mean(samples),
|
|
38
|
+
p50: quantile(samples, 0.5),
|
|
39
|
+
p90: quantile(samples, 0.9),
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
// Two-sample Welch's t-test (unequal variances, two-sided).
|
|
43
|
+
export function welchsTTest(a, b) {
|
|
44
|
+
if (a.length < 2 || b.length < 2)
|
|
45
|
+
return { t: NaN, df: NaN, p: NaN };
|
|
46
|
+
if (allEqual(a) && allEqual(b)) {
|
|
47
|
+
const ma = a[0];
|
|
48
|
+
const mb = b[0];
|
|
49
|
+
return {
|
|
50
|
+
t: ma === mb ? 0 : ma > mb ? Infinity : -Infinity,
|
|
51
|
+
df: NaN,
|
|
52
|
+
p: ma === mb ? 1 : 0,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
const ma = mean(a);
|
|
56
|
+
const mb = mean(b);
|
|
57
|
+
const va = sampleVariance(a, ma);
|
|
58
|
+
const vb = sampleVariance(b, mb);
|
|
59
|
+
const na = a.length;
|
|
60
|
+
const nb = b.length;
|
|
61
|
+
const sea = va / na;
|
|
62
|
+
const seb = vb / nb;
|
|
63
|
+
const t = (ma - mb) / Math.sqrt(sea + seb);
|
|
64
|
+
// Welch–Satterthwaite degrees of freedom.
|
|
65
|
+
const df = (sea + seb) ** 2 / ((sea * sea) / (na - 1) + (seb * seb) / (nb - 1));
|
|
66
|
+
const p = 2 * (1 - jstat.studentt.cdf(Math.abs(t), df));
|
|
67
|
+
return { t, df, p };
|
|
68
|
+
}
|
|
69
|
+
// Two-sided Mann–Whitney U test, asymptotic with continuity correction.
|
|
70
|
+
export function mannWhitneyU(a, b) {
|
|
71
|
+
const na = a.length;
|
|
72
|
+
const nb = b.length;
|
|
73
|
+
if (na === 0 || nb === 0)
|
|
74
|
+
return { u: NaN, p: NaN };
|
|
75
|
+
const combined = [...a, ...b];
|
|
76
|
+
const r = averageRanks(combined);
|
|
77
|
+
let rankSumA = 0;
|
|
78
|
+
for (let i = 0; i < na; i++)
|
|
79
|
+
rankSumA += r[i];
|
|
80
|
+
const uA = rankSumA - (na * (na + 1)) / 2;
|
|
81
|
+
const uB = na * nb - uA;
|
|
82
|
+
const u = Math.min(uA, uB);
|
|
83
|
+
const meanU = (na * nb) / 2;
|
|
84
|
+
const sd = Math.sqrt((na * nb * (na + nb + 1)) / 12);
|
|
85
|
+
if (sd === 0)
|
|
86
|
+
return { u, p: 1 };
|
|
87
|
+
const z = Math.max(0, (Math.abs(uA - meanU) - 0.5) / sd);
|
|
88
|
+
const p = 2 * (1 - jstat.normal.cdf(z, 0, 1));
|
|
89
|
+
return { u, p: Math.min(1, Math.max(0, p)) };
|
|
90
|
+
}
|
|
91
|
+
// Average-rank ranking (R/scipy default tie correction): tied values share
|
|
92
|
+
// the mean of the ranks they would have received.
|
|
93
|
+
function averageRanks(xs) {
|
|
94
|
+
const n = xs.length;
|
|
95
|
+
const idx = xs.map((v, i) => [v, i]);
|
|
96
|
+
idx.sort((p, q) => p[0] - q[0]);
|
|
97
|
+
const r = new Array(n);
|
|
98
|
+
let i = 0;
|
|
99
|
+
while (i < n) {
|
|
100
|
+
let j = i;
|
|
101
|
+
while (j + 1 < n && idx[j + 1][0] === idx[i][0])
|
|
102
|
+
j++;
|
|
103
|
+
const avg = (i + j) / 2 + 1;
|
|
104
|
+
for (let k = i; k <= j; k++)
|
|
105
|
+
r[idx[k][1]] = avg;
|
|
106
|
+
i = j + 1;
|
|
107
|
+
}
|
|
108
|
+
return r;
|
|
109
|
+
}
|
|
110
|
+
function sampleVariance(xs, m) {
|
|
111
|
+
let s = 0;
|
|
112
|
+
for (const x of xs) {
|
|
113
|
+
const d = x - m;
|
|
114
|
+
s += d * d;
|
|
115
|
+
}
|
|
116
|
+
return s / (xs.length - 1);
|
|
117
|
+
}
|
|
118
|
+
function allEqual(xs) {
|
|
119
|
+
for (let i = 1; i < xs.length; i++)
|
|
120
|
+
if (xs[i] !== xs[0])
|
|
121
|
+
return false;
|
|
122
|
+
return true;
|
|
123
|
+
}
|
package/dist/table.js
CHANGED
|
@@ -1,45 +1,45 @@
|
|
|
1
|
-
import { readFile } from
|
|
2
|
-
import minimist from
|
|
1
|
+
import { readFile } from 'fs/promises';
|
|
2
|
+
import minimist from 'minimist';
|
|
3
3
|
(async () => {
|
|
4
4
|
const args = minimist(process.argv.slice(2), {
|
|
5
5
|
alias: {
|
|
6
|
-
r:
|
|
7
|
-
c:
|
|
8
|
-
|
|
9
|
-
h:
|
|
6
|
+
r: 'row',
|
|
7
|
+
c: 'column',
|
|
8
|
+
'?': 'help',
|
|
9
|
+
h: 'help',
|
|
10
10
|
},
|
|
11
11
|
});
|
|
12
|
-
const knownArgs = new Set([
|
|
12
|
+
const knownArgs = new Set(['row', 'r', 'column', 'c', 'help', 'h', '?', '_']);
|
|
13
13
|
if (args.help || (Object.keys(args).length === 1 && args._.length === 0)) {
|
|
14
|
-
console.log(
|
|
15
|
-
console.log(
|
|
16
|
-
console.log(
|
|
17
|
-
console.log(
|
|
18
|
-
console.log(
|
|
14
|
+
console.log('Usage: devlow-table <data.json>');
|
|
15
|
+
console.log(' --row=<name> Key to show as row');
|
|
16
|
+
console.log(' --column=<name> Key to show as column');
|
|
17
|
+
console.log(' --<prop>=<name> Filter values');
|
|
18
|
+
console.log(' --help, -h, -? Show this help');
|
|
19
19
|
}
|
|
20
|
-
let data = JSON.parse(await readFile(args._[0],
|
|
21
|
-
const getValue = (
|
|
22
|
-
if (name ===
|
|
23
|
-
return
|
|
20
|
+
let data = JSON.parse(await readFile(args._[0], 'utf-8'));
|
|
21
|
+
const getValue = (entry, name, includeKey) => {
|
|
22
|
+
if (name === 'value') {
|
|
23
|
+
return entry.text;
|
|
24
24
|
}
|
|
25
25
|
if (Array.isArray(name)) {
|
|
26
26
|
return name
|
|
27
|
-
.map((n) => getValue(
|
|
27
|
+
.map((n) => getValue(entry, n, true))
|
|
28
28
|
.filter((x) => x)
|
|
29
|
-
.join(
|
|
29
|
+
.join(' ');
|
|
30
30
|
}
|
|
31
|
-
const value =
|
|
31
|
+
const value = entry.key[name];
|
|
32
32
|
if (value === undefined)
|
|
33
|
-
return
|
|
33
|
+
return '';
|
|
34
34
|
if (value === true)
|
|
35
|
-
return includeKey ? name :
|
|
35
|
+
return includeKey ? name : 'true';
|
|
36
36
|
if (value === false)
|
|
37
|
-
return includeKey ?
|
|
37
|
+
return includeKey ? '' : 'false';
|
|
38
38
|
if (value === null)
|
|
39
|
-
return
|
|
39
|
+
return '';
|
|
40
40
|
if (includeKey)
|
|
41
41
|
return `${name}=${value}`;
|
|
42
|
-
return value +
|
|
42
|
+
return value + '';
|
|
43
43
|
};
|
|
44
44
|
for (const [key, value] of Object.entries(args)) {
|
|
45
45
|
if (knownArgs.has(key))
|
|
@@ -47,19 +47,19 @@ import minimist from "minimist";
|
|
|
47
47
|
const values = (Array.isArray(value) ? value : [value]).map((v) => v.toString());
|
|
48
48
|
data = data.filter((item) => {
|
|
49
49
|
const itemValue = getValue(item, key, false);
|
|
50
|
-
if (itemValue ===
|
|
50
|
+
if (itemValue === '')
|
|
51
51
|
return false;
|
|
52
52
|
return values.some((v) => itemValue === v);
|
|
53
53
|
});
|
|
54
54
|
}
|
|
55
55
|
if (data.length === 0) {
|
|
56
|
-
console.log(
|
|
56
|
+
console.log('No data');
|
|
57
57
|
return;
|
|
58
58
|
}
|
|
59
|
-
const
|
|
60
|
-
const
|
|
61
|
-
const getRow = (
|
|
62
|
-
const getColumn = (
|
|
59
|
+
const rowKey = args.row || 'name';
|
|
60
|
+
const columnKey = args.column || 'scenario';
|
|
61
|
+
const getRow = (entry) => getValue(entry, rowKey, false);
|
|
62
|
+
const getColumn = (entry) => getValue(entry, columnKey, false);
|
|
63
63
|
const allRows = new Set(data.map(getRow));
|
|
64
64
|
const allColumns = new Set(data.map(getColumn));
|
|
65
65
|
const table = [];
|
|
@@ -70,46 +70,46 @@ import minimist from "minimist";
|
|
|
70
70
|
for (const column of allColumns) {
|
|
71
71
|
let items = data
|
|
72
72
|
.filter((d) => getRow(d) === row && getColumn(d) === column)
|
|
73
|
-
.map((
|
|
74
|
-
rowData.push(items.join(
|
|
73
|
+
.map((item) => item.text);
|
|
74
|
+
rowData.push(items.join(', '));
|
|
75
75
|
columnSizes[i] = Math.max(columnSizes[i], rowData[i].length);
|
|
76
76
|
i++;
|
|
77
77
|
}
|
|
78
78
|
table.push(rowData);
|
|
79
79
|
}
|
|
80
80
|
const pad = (str, size) => {
|
|
81
|
-
return
|
|
81
|
+
return ' '.repeat(size - str.length) + str;
|
|
82
82
|
};
|
|
83
83
|
const firstColumnSize = Math.max(...[...allRows].map((r) => r.length));
|
|
84
84
|
// Header
|
|
85
85
|
{
|
|
86
|
-
let
|
|
87
|
-
let
|
|
88
|
-
|
|
89
|
-
|
|
86
|
+
let line = '| ';
|
|
87
|
+
let sepLine = '|:';
|
|
88
|
+
line += ' '.repeat(firstColumnSize);
|
|
89
|
+
sepLine += '-'.repeat(firstColumnSize);
|
|
90
90
|
const allColumnsArray = [...allColumns];
|
|
91
91
|
for (let i = 0; i < columnSizes.length; i++) {
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
92
|
+
line += ' | ';
|
|
93
|
+
line += pad(allColumnsArray[i], columnSizes[i]);
|
|
94
|
+
sepLine += ':|-';
|
|
95
|
+
sepLine += '-'.repeat(columnSizes[i]);
|
|
96
96
|
}
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
console.log(
|
|
100
|
-
console.log(
|
|
97
|
+
line += ' |';
|
|
98
|
+
sepLine += ':|';
|
|
99
|
+
console.log(line);
|
|
100
|
+
console.log(sepLine);
|
|
101
101
|
}
|
|
102
102
|
// Separator
|
|
103
103
|
let r = 0;
|
|
104
104
|
for (const rowName of allRows) {
|
|
105
|
-
let
|
|
106
|
-
|
|
105
|
+
let line = '| ';
|
|
106
|
+
line += pad(rowName, firstColumnSize);
|
|
107
107
|
for (let i = 0; i < columnSizes.length; i++) {
|
|
108
|
-
|
|
109
|
-
|
|
108
|
+
line += ' | ';
|
|
109
|
+
line += pad(table[r][i], columnSizes[i]);
|
|
110
110
|
}
|
|
111
|
-
|
|
112
|
-
console.log(
|
|
111
|
+
line += ' |';
|
|
112
|
+
console.log(line);
|
|
113
113
|
r++;
|
|
114
114
|
}
|
|
115
115
|
})().catch((e) => {
|
package/dist/units.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
const UNITS = {
|
|
2
2
|
ms: {
|
|
3
|
-
|
|
3
|
+
' s': 1000,
|
|
4
4
|
},
|
|
5
5
|
bytes: {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
6
|
+
' GB': 1024 * 1024 * 1024,
|
|
7
|
+
' MB': 1024 * 1024,
|
|
8
|
+
' KB': 1024,
|
|
9
9
|
},
|
|
10
10
|
requests: {
|
|
11
|
-
|
|
11
|
+
'K requests': 1000,
|
|
12
12
|
},
|
|
13
13
|
};
|
|
14
14
|
export function formatUnit(value, unit) {
|
|
@@ -20,5 +20,5 @@ export function formatUnit(value, unit) {
|
|
|
20
20
|
}
|
|
21
21
|
}
|
|
22
22
|
}
|
|
23
|
-
return `${value.toFixed(2).replace(/\.00$/,
|
|
23
|
+
return `${value.toFixed(2).replace(/\.00$/, '')} ${unit}`;
|
|
24
24
|
}
|
package/dist/utils.d.ts
CHANGED
package/dist/utils.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
export function formatVariant(scenario, props) {
|
|
2
|
-
const
|
|
2
|
+
const propsStr = formatVariantProps(props);
|
|
3
|
+
return propsStr ? `${scenario} ${propsStr}` : scenario;
|
|
4
|
+
}
|
|
5
|
+
// Just the props part (no scenario prefix). Stable key for snapshots/compare.
|
|
6
|
+
export function formatVariantProps(props) {
|
|
7
|
+
return Object.keys(props)
|
|
3
8
|
.filter((key) => props[key] !== false && props[key] !== null)
|
|
4
|
-
.map((key) => (props[key] === true ? key : `${key}=${props[key]}`))
|
|
5
|
-
|
|
6
|
-
return scenario;
|
|
7
|
-
}
|
|
8
|
-
return `${scenario} ${keys.join(" ")}`;
|
|
9
|
+
.map((key) => (props[key] === true ? key : `${key}=${props[key]}`))
|
|
10
|
+
.join(' ');
|
|
9
11
|
}
|