nano-benchmark 1.0.0

package/src/bench/compare.js

@@ -0,0 +1,35 @@
+import {measure, measurePar} from './runner.js';
+import kwtest from '../significance/kwtest.js';
+import mwtest from '../significance/mwtest.js';
+
+const ALPHA = 0.05;
+
+const compare = async (inputs, options = {}, report) => {
+  const keys = Object.keys(inputs);
+  if (keys.length < 2) throw new Error('The "inputs" is supposed to have 2 or more samples.');
+
+  options = Object.assign({alpha: ALPHA}, options);
+  const measureFn = options.usePar ? measurePar : measure;
+
+  const stats = new Array(keys.length);
+  for (let i = 0; i < keys.length; ++i) {
+    stats[i] = await measureFn(inputs[keys[i]], options, report);
+    stats[i].ensureSorted();
+  }
+  const reps = stats.map(stat => stat.reps);
+  stats.forEach(stat => stat.normalizeReps());
+
+  report?.('calculating-significance', {stats, options});
+  let results;
+  if (keys.length > 2) {
+    results = kwtest(stats.map(stat => stat.data), options.alpha);
+  } else {
+    results = mwtest(stats[0].data, stats[1].data, options.alpha);
+  }
+  results.data = stats.map(stat => stat.data);
+  results.reps = reps;
+  report?.('significance-results', results);
+  return results;
+};
+
+export default compare;

package/src/bench/runner.js

@@ -0,0 +1,173 @@
+import {performance} from 'node:perf_hooks';
+
+export const nextLevel = n => {
+  if (n < 1) return 1;
+  let exp = 0;
+  while (!(n % 10)) {
+    ++exp;
+    n = n / 10;
+  }
+  if (n < 5) {
+    n = n < 2 ? 2 : 5;
+  } else if (n < 10) {
+    n = 10;
+  } else {
+    n = n << 1;
+  }
+  while (exp--) {
+    n *= 10;
+  }
+  return n;
+};
+
+export const findLevel = (fn, {threshold = 20, startFrom = 1, timeout = 5} = {}, report) =>
+  new Promise((resolve, reject) => {
+    const bench = async n => {
+      report && (await report('finding-level', {n}));
+      try {
+        const start = performance.now(),
+          result = fn(n),
+          finish = performance.now();
+        if (result && typeof result.then == 'function') {
+          // thenable
+          result.then(async () => {
+            const finish = performance.now();
+            if (finish - start >= threshold) return resolve(n);
+            report && (await report('finding-level-next', {n, time: finish - start}));
+            setTimeout(bench, timeout, nextLevel(n));
+          }, reject);
+          return;
+        }
+        if (finish - start >= threshold) return resolve(n);
+        report && (await report('finding-level-next', {n, time: finish - start}));
+        setTimeout(bench, timeout, nextLevel(n));
+      } catch (error) {
+        reject(error);
+      }
+    };
+    bench(startFrom);
+  });
+
+export const benchmark = (fn, n) =>
+  new Promise((resolve, reject) => {
+    try {
+      const start = performance.now(),
+        result = fn(n),
+        finish = performance.now();
+      if (result && typeof result.then == 'function') {
+        // thenable
+        result.then(() => {
+          const finish = performance.now();
+          resolve(finish - start);
+        }, reject);
+        return;
+      }
+      resolve(finish - start);
+    } catch (error) {
+      reject(error);
+    }
+  });
+
+export const benchmarkSeries = async (
+  fn,
+  n,
+  {nSeries = 100, timeout = 5, DataArray = Array} = {}
+) => {
+  const data = new DataArray(nSeries);
+
+  const bench = async (nSeries, resolve, reject) => {
+    --nSeries;
+    try {
+      data[nSeries] = await benchmark(fn, n);
+      if (nSeries) {
+        setTimeout(bench, timeout, nSeries, resolve, reject);
+      } else {
+        resolve();
+      }
+    } catch (error) {
+      reject(error);
+    }
+  };
+
+  await new Promise((resolve, reject) => bench(nSeries, resolve, reject));
+
+  return data;
+};
+
+export const benchmarkSeriesPar = async (fn, n, {nSeries = 100, DataArray = Array} = {}) => {
+  const benchmarks = [];
+  for (; nSeries > 0; --nSeries) benchmarks.push(benchmark(fn, n));
+  const results = await Promise.all(benchmarks);
+  return DataArray === Array ? results : DataArray.from(results);
+};
+
+export class Stats {
+  constructor(object) {
+    Object.assign(this, object);
+  }
+
+  static sortNumbersAsc = (a, b) => a - b;
+
+  ensureSorted() {
+    if (!this.sorted) {
+      this.data.sort(Stats.sortNumbersAsc);
+      this.sorted = true;
+    }
+    return this;
+  }
+
+  normalizeReps() {
+    if (this.reps !== 1) {
+      const data = this.data,
+        reps = this.reps,
+        size = data.length;
+      for (let i = 0; i < size; ++i) {
+        data[i] /= reps;
+      }
+      this.reps = 1;
+    }
+    return this;
+  }
+
+  copyStats() {
+    return new Stats({...this, data: this.data.slice()});
+  }
+}
+
+export const measure = async (
+  fn,
+  {nSeries = 100, threshold = 20, startFrom = 1, timeout = 5, DataArray = Array} = {},
+  report
+) => {
+  report?.('finding-reps');
+  const reps = startFrom < 0 ? -startFrom : await findLevel(fn, {threshold, startFrom, timeout});
+  report?.('found-reps', {reps});
+  report?.('starting-benchmarks', {nSeries, reps});
+  const start = performance.now(),
+    data = await benchmarkSeries(fn, reps, {nSeries, timeout, DataArray}),
+    finish = performance.now(),
+    result = {data, reps, time: finish - start};
+  report?.('finished-benchmarks', {...result, nSeries});
+  return new Stats(result);
+};
+
+export const measurePar = async (
+  fn,
+  {nSeries = 100, threshold = 20, startFrom = 1, timeout = 5, DataArray = Array} = {},
+  report
+) => {
+  report?.('finding-reps');
+  const reps = startFrom < 0 ? -startFrom : await findLevel(fn, {threshold, startFrom, timeout});
+  report?.('found-reps', {reps});
+  report?.('starting-benchmarks', {nSeries, reps});
+  const start = performance.now(),
+    data = await benchmarkSeriesPar(fn, reps, {nSeries, DataArray}),
+    finish = performance.now(),
+    result = {data, reps, time: finish - start};
+  report?.('finished-benchmarks', {...result, nSeries});
+  return new Stats(result);
+};
+
+export const wrapper = fn => n => {
+  for (let i = 0; i < n; ++i) fn();
+};

package/src/median.js

@@ -0,0 +1,39 @@
+export const median = data => {
+  let step = 1,
+    n = Math.floor(data.length / 3);
+  for (;;) {
+    for (let i = 0; i < n; ++i) {
+      const ai = 3 * step * i,
+        bi = ai + step,
+        ci = bi + step;
+      let a = data[ai],
+        b = data[bi],
+        c = data[ci];
+      if (b < a) [a, b] = [b, a];
+      if (c < b) [b, c] = [c, b];
+      if (b < a) [a, b] = [b, a];
+      data[ai] = b;
+      data[bi] = a;
+      data[ci] = c;
+    }
+
+    const last = Math.ceil((data.length - 3 * step * n) / step);
+    if (last == 2) {
+      const ai = 3 * step * n,
+        bi = ai + step;
+      let a = data[ai],
+        b = data[bi];
+      if (b < a) [a, b] = [b, a];
+      data[ai] = b;
+      data[bi] = a;
+    }
+
+    step *= 3;
+    if (step >= data.length) break;
+    n = Math.floor(data.length / (3 * step));
+  }
+
+  return data[0];
+};
+
+export default median;

package/src/significance/kstest.js

@@ -0,0 +1,43 @@
+// Two sample Kolmogorov-Smirnov significance test
+// based on https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test
+
+// sup(F1(x) - F2(x))
+const sup = (sorted1, sorted2) => {
+  let size1 = sorted1.length,
+    size2 = sorted2.length,
+    i = 0,
+    j = 0,
+    result = 0;
+  while (i < size1 && j < size2) {
+    const a = sorted1[i],
+      b = sorted2[j];
+    if (a < b) {
+      for (++i; i < size1 && a === sorted1[i]; ++i);
+    } else if (b < a) {
+      for (++j; j < size2 && b === sorted2[j]; ++j);
+    } else {
+      for (++i; i < size1 && a === sorted1[i]; ++i);
+      for (++j; j < size2 && b === sorted2[j]; ++j);
+    }
+    result = Math.max(result, Math.abs(i / size1 - j / size2));
+  }
+  while (i < size1) {
+    const a = sorted1[i];
+    for (++i; i < size1 && a === sorted1[i]; ++i);
+    result = Math.max(result, Math.abs(i / size1 - 1));
+  }
+  while (j < size2) {
+    const b = sorted2[j];
+    for (++j; j < size2 && b === sorted2[j]; ++j);
+    result = Math.max(result, Math.abs(1 - j / size2));
+  }
+  return result;
+};
+
+export const kstest = (sorted1, sorted2, alpha = 0.05) => {
+  const d = sup(sorted1, sorted2),
+    limit = Math.sqrt((-Math.log(alpha / 2) / 2) * (1 / sorted1.length + 1 / sorted2.length));
+  return {value: d, alpha, limit, different: d > limit};
+};
+
+export default kstest;

package/src/significance/kwtest.js

@@ -0,0 +1,81 @@
+// Kruskal-Wallis significance test
+// based on https://en.wikipedia.org/wiki/Kruskal%E2%80%93Wallis_one-way_analysis_of_variance
+// beta approximation: https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.661.7863&rep=rep1&type=pdf
+
+import betaPpf from '../stats/beta-ppf.js';
+import chiSquaredPpf from '../stats/chi-squared-ppf.js';
+import rank, {getTotal} from '../stats/rank.js';
+
+export const getParameters = (groups, N = getTotal(groups)) => {
+  const k = groups.length,
+    mu = k - 1,
+    nu = (N * N * N - groups.reduce((acc, {length: n}) => acc + n * n * n, 0)) / N / (N + 1),
+    sigmaSquared =
+      2 * mu -
+      (0.4 * (3 * k * k - 6 * k + N * (2 * k * k - 6 * k + 1))) / N / (N + 1) -
+      1.2 * groups.reduce((acc, group) => acc + 1 / group.length, 0),
+    a = (mu * ((mu * (nu - mu)) / sigmaSquared - 1)) / nu,
+    b = a * (nu / mu - 1);
+  return {mu, nu, sigmaSquared, a, b, k, N};
+};
+
+export const rankData = groups => {
+  const {N, k, ranked: t, groupRank, avgGroupRank, avgRank} = rank(groups),
+    avgRankC = N * avgRank * avgRank;
+
+  // calculate required sums
+  let numerator = 0,
+    T = 0;
+  for (let i = 0; i < avgGroupRank.length; ++i) {
+    const x = avgGroupRank[i] - avgRank;
+    numerator += groups[i].length * x * x;
+    T += (groupRank[i] * groupRank[i]) / groups[i].length - avgRankC;
+  }
+
+  let denominator = 0,
+    S2 = 0;
+  for (let i = 0; i < t.length; ++i) {
+    const x = t[i].rank - avgRank;
+    denominator += x * x;
+    S2 = t[i].rank * t[i].rank - avgRankC;
+  }
+
+  S2 /= N - 1;
+  T /= S2;
+
+  // calculate and return H statistics
+  return {H: ((N - 1) * numerator) / denominator, T, S2, groupRank, avgGroupRank, avgRank, k, N};
+};
+
+export const kwtest = (sortedArrays, alpha = 0.05) => {
+  if (sortedArrays.length < 2) throw new Error('Two or more data arrays were expected');
+
+  const {a, b, nu, k, N} = getParameters(sortedArrays),
+    {H, T, S2, avgGroupRank} = rankData(sortedArrays),
+    limit = betaPpf(1 - alpha, a, b) * nu, // Hc
+    results = {value: H, alpha, limit, different: H > limit};
+
+  if (!results.different || k < 3) return results;
+
+  // post-hoc tests
+
+  const m = new Array(k),
+    C = chiSquaredPpf(1 - alpha / 2, N - k) * Math.sqrt((S2 * (N - 1 - T)) / (N - k));
+
+  for (let i = 0; i < k; ++i) {
+    m[i] = new Array(k);
+  }
+  for (let i = 0; i < k; ++i) {
+    m[i][i] = false;
+    for (let j = i + 1; j < k; ++j) {
+      m[i][j] = m[j][i] =
+        Math.abs(avgGroupRank[i] - avgGroupRank[j]) >
+        C * Math.sqrt(1 / sortedArrays[i].length + 1 / sortedArrays[j].length);
+    }
+  }
+
+  results.groupDifference = m;
+  return results;
+};
+
+export default kwtest;

package/src/significance/mwtest.js

@@ -0,0 +1,35 @@
+// Mann-Whitney U test
+// based on https://en.wikipedia.org/wiki/Mann%E2%80%93Whitney_U_test
+
+import rank from '../stats/rank.js';
+import zPpf from '../stats/z-ppf.js';
+
+export const mwtest = (sorted1, sorted2, alpha = 0.05) => {
+  const {groupRank, ranked: t} = rank([sorted1, sorted2]),
+    u1 = groupRank[0] - (sorted1.length * (sorted1.length + 1)) / 2,
+    u2 = groupRank[1] - (sorted2.length * (sorted2.length + 1)) / 2,
+    u = Math.min(u1, u2);
+
+  const m = (sorted1.length * sorted2.length) / 2;
+
+  let tiesC = 0;
+  for (let i = 0, size = t.length; i < size; ) {
+    const rank = t[i].rank;
+    let j = i + 1;
+    while (j < size && rank === t[j].rank) ++j;
+    const nt = j - i;
+    i = j;
+    if (nt > 1) {
+      tiesC += nt * nt * nt - nt;
+    }
+  }
+  const n = sorted1.length + sorted2.length,
+    s = Math.sqrt(((sorted1.length * sorted2.length) / 12) * (n + 1 - tiesC / n / (n - 1)));
+
+  const z = (u - m) / s,
+    zc = zPpf(alpha / 2);
+
+  return {value: z, alpha, limit: zc, different: z < zc || z > -zc};
+};
+
+export default mwtest;

package/src/stats/beta-ppf.js

@@ -0,0 +1,13 @@
+import ppf from './ppf.js';
+
+// percent point function
+const betaPpf = (z, a, b) => {
+  const fn = z => {
+    if (z == 0 || z == 1) return 0;
+    const result = Math.pow(z, a - 1) * Math.pow(1 - z, b - 1);
+    return isFinite(result) ? result : 0;
+  };
+  return ppf(fn, z);
+};
+
+export default betaPpf;

package/src/stats/beta.js

@@ -0,0 +1,26 @@
+const LIMIT = 1000;
+const EPSILON = 1e-30;
+
+export const incompleteBeta = (z, a, b) => {
+  let x = 1,
+    result = 1 / a;
+  for (let i = 1; i < LIMIT; ++i) {
+    x *= ((i - b) / i) * z;
+    const previous = result;
+    result += x / (a + i);
+    if (Math.abs(result - previous) < EPSILON * result) break;
+  }
+  return Math.pow(z, a) * result;
+};
+
+export const beta = (a, b) => {
+  let result = 1;
+  for (let i = 1; i < LIMIT; ++i) {
+    const previous = result;
+    result *= (1 + (a + b) / i) / (1 + a / i) / (1 + b / i);
+    if (Math.abs(result - previous) < EPSILON * result) break;
+  }
+  return ((a + b) * result) / a / b;
+};
+
+export default beta;

package/src/stats/chi-squared-ppf.js

@@ -0,0 +1,10 @@
+import ppf from './ppf.js';
+
+// percent point function
+const chiSquaredPpf = (z, k) => {
+  const C = k / 2 - 1,
+    fn = z => Math.exp(-z / 2) * Math.pow(z, C);
+  return ppf(fn, z);
+};
+
+export default chiSquaredPpf;

package/src/stats/erf.js

@@ -0,0 +1,23 @@
+const LIMIT = 1000;
+const EPSILON = 1e-30;
+
+const TWO_BY_SQRT_PI = 2 / Math.sqrt(Math.PI);
+
+export const erf = z => {
+  let x = z,
+    result = x;
+  for (let i = 1; i < LIMIT; ++i) {
+    const previous = result;
+    x *= (-z * z) / i;
+    result += x / (2 * i + 1);
+    if (
+      (Math.abs(result) < EPSILON ? Math.abs(result - previous) : Math.abs(1 - previous / result)) <
+      EPSILON
+    ) {
+      break;
+    }
+  }
+  return TWO_BY_SQRT_PI * result;
+};
+
+export default erf;

package/src/stats/gamma.js

@@ -0,0 +1,14 @@
+import zeta from './zeta.js';
+
+const LIMIT = 1000;
+const EPSILON = 1e-30;
+
+export const logGamma = z => {
+  let result = 0;
+  for (let i = 2; i < LIMIT; ++i) {
+    const previous = result;
+    result += ((i - 1) / i / (i + 1)) * zeta(i, z + 1);
+    if (Math.abs(result - previous) < EPSILON * result) break;
+  }
+  return (z - 0.5) * Math.log(z) - z + 0.5 * Math.log(2 * Math.PI) + 0.5 * result;
+};

package/src/stats/normal-ppf.js

@@ -0,0 +1,19 @@
+import {normalMakeCdf, normalMakePdf} from './normal.js';
+import ppf from './ppf.js';
+
+// percent point function
+const normalPpf = (z, mu = 0, sigma = 1) => {
+  // find the lower bound
+  const cdf = normalMakeCdf(mu, sigma);
+  let x = mu - 6 * sigma,
+    p = cdf(x);
+  while (p > z) {
+    x = mu - (mu - x) * 2;
+    p = cdf(x);
+  }
+
+  const fn = normalMakePdf(mu, sigma);
+  return ppf(fn, z, {a: x, b: 2 * mu - x, initialValue: p});
+};
+
+export default normalPpf;

package/src/stats/normal.js

@@ -0,0 +1,25 @@
+import erf from './erf.js';
+
+const LIMIT = 1000;
+const EPSILON = 1e-30;
+
+const SQRT_2 = Math.sqrt(2),
+  SQRT_2_PI = Math.sqrt(2 * Math.PI);
+
+export const normalCdf = (z, mu = 0, sigma = 1) => 0.5 * (1 + erf((z - mu) / sigma / SQRT_2));
+export const normalMakeCdf =
+  (mu = 0, sigma = 1) =>
+  z =>
+    0.5 * (1 + erf((z - mu) / sigma / SQRT_2));
+
+export const normalPdf = (z, mu = 0, sigma = 1) => {
+  const x = (z - mu) / sigma;
+  return Math.exp(-0.5 * x * x) / sigma / SQRT_2_PI;
+};
+
+export const normalMakePdf =
+  (mu = 0, sigma = 1) =>
+  z => {
+    const x = (z - mu) / sigma;
+    return Math.exp(-0.5 * x * x) / sigma / SQRT_2_PI;
+  };

package/src/stats/ppf.js

@@ -0,0 +1,18 @@
+import {rk23} from '../utils/rk.js';
+import bsearch from '../utils/bsearch.js';
+
+// percent point function
+const ppf = (fn, z, options) => {
+  const {ts, us, finalValue} = rk23(fn, options),
+    value = finalValue * z,
+    index = bsearch(us, x => x < value);
+  if (!index) return ts[0];
+  if (index >= us.length) return ts[ts.length - 1];
+  if (z == us[index]) return ts[index];
+  // linear interpolation
+  return (
+    ts[index] - ((us[index] - value) / (us[index] - us[index - 1])) * (ts[index] - ts[index - 1])
+  );
+};
+
+export default ppf;

package/src/stats/rank.js

@@ -0,0 +1,41 @@
+export const getTotal = groups => groups.reduce((acc, group) => acc + group.length, 0);
+
+export const rank = groups => {
+  const N = getTotal(groups),
+    k = groups.length,
+    t = new Array(N);
+
+  // put in one array preserving grouping
+  let o = 0;
+  for (let i = 0; i < k; ++i) {
+    const group = groups[i];
+    for (let j = 0; j < group.length; ++j) {
+      t[o++] = {value: group[j], group: i};
+    }
+  }
+
+  const groupRank = new Array(k);
+  groupRank.fill(0);
+
+  // sort and rank
+  t.sort((a, b) => a.value - b.value);
+  for (let i = 0; i < t.length; ) {
+    let ahead = i + 1;
+    const value = t[i].value;
+    while (ahead < t.length && value === t[ahead].value) ++ahead;
+    if (ahead - i === 1) {
+      groupRank[t[i].group] += t[i].rank = i + 1;
+    } else {
+      const rank = (i + 1 + ahead) / 2;
+      for (let j = i; j < ahead; ++j) {
+        groupRank[t[j].group] += t[j].rank = rank;
+      }
+    }
+    i = ahead;
+  }
+  const avgRank = (N + 1) / 2, avgGroupRank = groupRank.map((rank, i) => rank / groups[i].length);
+
+  return {ranked: t, N, k, avgRank, groupRank, avgGroupRank, groups};
+};
+
+export default rank;

package/src/stats/z-ppf.js

@@ -0,0 +1,17 @@
+import {zCdf, zPdf} from './z.js';
+import ppf from './ppf.js';
+
+// percent point function
+const zPpf = (z) => {
+  // find the lower bound
+  let x = -6,
+    p = zCdf(x);
+  while (p > z) {
+    x = 2 * x;
+    p = zCdf(x);
+  }
+
+  return ppf(zPdf, z, {a: x, b: -x, initialValue: p});
+};
+
+export default zPpf;

package/src/stats/z.js

@@ -0,0 +1,7 @@
+import erf from './erf.js';
+
+const SQRT_2 = Math.sqrt(2),
+  SQRT_2_PI = Math.sqrt(2 * Math.PI);
+
+export const zCdf = (z) => 0.5 * (1 + erf((z) / SQRT_2));
+export const zPdf = (z) => Math.exp(-0.5 * z * z) / SQRT_2_PI;

package/src/stats/zeta.js

@@ -0,0 +1,14 @@
+const LIMIT = 1000;
+const EPSILON = 1e-30;
+
+export const zeta = (s, a) => {
+  let result = 0;
+  for (let i = 0; i < LIMIT; ++i) {
+    const previous = result;
+    result += 1 / Math.pow(i + a, s);
+    if (Math.abs(result - previous) < EPSILON * result) break;
+  }
+  return result;
+};
+
+export default zeta;