benchforge 0.1.0

package/src/tests/RealDataValidation.test.ts

@@ -0,0 +1,177 @@
+import { test } from "vitest";
+import { checkConvergence } from "../runners/AdaptiveWrapper.ts";
+import {
+  coefficientOfVariation,
+  medianAbsoluteDeviation,
+  percentile,
+} from "../StatisticalUtils.ts";
+import { bevy30SamplesMs, bevy30SamplesNs } from "./fixtures/bevy30-samples.ts";
+
+test("bevy30 data characteristics", () => {
+  const sortedMs = [...bevy30SamplesMs].sort((a, b) => a - b);
+
+  const stats = {
+    count: sortedMs.length,
+    min: sortedMs[0],
+    p25: percentile(sortedMs, 0.25),
+    p50: percentile(sortedMs, 0.5),
+    p75: percentile(sortedMs, 0.75),
+    p95: percentile(sortedMs, 0.95),
+    p99: percentile(sortedMs, 0.99),
+    max: sortedMs[sortedMs.length - 1],
+    mean: sortedMs.reduce((a, b) => a + b, 0) / sortedMs.length,
+    cv: coefficientOfVariation(sortedMs),
+    mad: medianAbsoluteDeviation(sortedMs),
+  };
+
+  console.log("Bevy30 benchmark statistics:");
+  console.log(`  Samples: ${stats.count}`);
+  console.log(`  Min: ${stats.min.toFixed(2)}ms`);
+  console.log(`  P25: ${stats.p25.toFixed(2)}ms`);
+  console.log(`  P50: ${stats.p50.toFixed(2)}ms`);
+  console.log(`  P75: ${stats.p75.toFixed(2)}ms`);
+  console.log(`  P95: ${stats.p95.toFixed(2)}ms`);
+  console.log(`  P99: ${stats.p99.toFixed(2)}ms`);
+  console.log(`  Max: ${stats.max.toFixed(2)}ms`);
+  console.log(`  Mean: ${stats.mean.toFixed(2)}ms`);
+  console.log(`  CV: ${(stats.cv * 100).toFixed(1)}%`);
+  console.log(`  MAD: ${stats.mad.toFixed(2)}ms`);
+
+  if (stats.count !== 610) throw new Error("Expected 610 samples");
+  if (stats.min < 40 || stats.min > 100)
+    throw new Error("Unexpected min value");
+  if (stats.max < 50 || stats.max > 100)
+    throw new Error("Unexpected max value");
+
+  // Check for reasonable variation (not too noisy, not suspiciously stable)
+  if (stats.cv < 0.01) console.warn("Very low variation - may be synthetic");
+  if (stats.cv > 0.5) console.warn("Very high variation - may be unstable");
+});
+
+test("convergence at different time points matches CLI behavior", () => {
+  // Simulate 5-second run (approximately 100 samples at ~50ms each)
+  const samples5s = bevy30SamplesNs.slice(0, 100);
+  const result5s = checkConvergence(samples5s);
+
+  const msg5s = `5-second equivalent (100 samples): converged=${result5s.converged}, confidence=${result5s.confidence}%`;
+  console.log(msg5s);
+
+  // Should match what we saw in CLI: 100% convergence
+  if (result5s.confidence !== 100)
+    console.warn(
+      `Expected 100% convergence at 5s, got ${result5s.confidence}%`,
+    );
+
+  // Simulate very short run (0.5 seconds, ~10 samples)
+  const samples500ms = bevy30SamplesNs.slice(0, 10);
+  const result500ms = checkConvergence(samples500ms);
+
+  const msg500ms = `0.5-second equivalent (10 samples): converged=${result500ms.converged}, confidence=${result500ms.confidence}%`;
+  console.log(msg500ms);
+
+  // Should have low convergence like we saw in CLI
+  if (result500ms.confidence > 20)
+    console.warn(
+      `Expected low convergence at 0.5s, got ${result500ms.confidence}%`,
+    );
+});
+
+test("warm-up detection in real data", () => {
+  const windowSize = 20;
+  const windows: Array<{ start: number; median: number }> = [];
+
+  for (let i = 0; i <= bevy30SamplesMs.length - windowSize; i += windowSize) {
+    const window = bevy30SamplesMs.slice(i, i + windowSize);
+    const median = percentile(window, 0.5);
+    windows.push({ start: i, median });
+  }
+
+  // Find the window with highest median (likely during warm-up)
+  const maxWindow = windows.reduce((max, w) =>
+    w.median > max.median ? w : max,
+  );
+  const stableMedian = windows[windows.length - 1].median;
+
+  console.log(`Warm-up analysis:`);
+  const peak = `${maxWindow.median.toFixed(2)}ms at sample ${maxWindow.start}`;
+  console.log(`  Highest median: ${peak}`);
+  console.log(`  Stable median: ${stableMedian.toFixed(2)}ms`);
+  const overhead = ((maxWindow.median / stableMedian - 1) * 100).toFixed(1);
+  console.log(`  Warm-up overhead: ${overhead}%`);
+
+  if (maxWindow.start === 0) {
+    console.log("  Warm-up detected at start of benchmark");
+  }
+});
+
+test("convergence stability over sliding windows", () => {
+  const windowSize = 100;
+  const step = 50;
+  const history: Array<{ start: number; confidence: number }> = [];
+
+  for (let i = windowSize; i <= bevy30SamplesNs.length; i += step) {
+    const samples = bevy30SamplesNs.slice(0, i);
+    const result = checkConvergence(samples);
+    history.push({ start: i - windowSize, confidence: result.confidence });
+  }
+
+  const firstFull = history.findIndex(h => h.confidence === 100);
+
+  if (firstFull !== -1) {
+    const samplesNeeded = history[firstFull].start + windowSize;
+    console.log(`First 100% convergence at ${samplesNeeded} samples`);
+
+    const unstable = history.slice(firstFull).some(h => h.confidence < 100);
+    if (unstable) {
+      console.warn("Convergence became unstable after initially reaching 100%");
+    } else {
+      console.log(
+        "Convergence remained stable at 100% after initial achievement",
+      );
+    }
+  }
+});
+
+test("adaptive algorithm would stop at correct time", () => {
+  const target = 95;
+  const fallback = 80;
+  const minSamples = 50;
+
+  let stopAt = -1;
+  let stopConfidence = 0;
+
+  for (let i = minSamples; i <= bevy30SamplesNs.length; i++) {
+    const samples = bevy30SamplesNs.slice(0, i);
+    const result = checkConvergence(samples);
+
+    if (result.converged && result.confidence >= target) {
+      stopAt = i;
+      stopConfidence = result.confidence;
+      break;
+    }
+
+    // Check fallback condition (after minimum time)
+    if (i >= 100 && result.confidence >= fallback) {
+      stopAt = i;
+      stopConfidence = result.confidence;
+      break;
+    }
+  }
+
+  if (stopAt !== -1) {
+    const timeSec = ((stopAt * 50) / 1000).toFixed(1); // Approximate seconds
+    console.log(`Adaptive would stop at:`);
+    console.log(`  Samples: ${stopAt}`);
+    console.log(`  Confidence: ${stopConfidence}%`);
+    console.log(`  Estimated time: ${timeSec}s`);
+  } else {
+    console.log("Adaptive would run to maximum time");
+  }
+
+  // Should stop relatively quickly with this stable data
+  if (stopAt > 200) {
+    console.warn(
+      "Takes many samples to converge - may indicate initial instability",
+    );
+  }
+});

package/src/tests/fixtures/baseline/impl.ts

@@ -0,0 +1,4 @@
+export const run = () => {
+  let _x = 0;
+  for (let i = 0; i < 100; i++) _x++;
+};

package/src/tests/fixtures/bevy30-samples.ts

@@ -0,0 +1,158 @@
+export const bevy30SamplesMs = [
+  80.61558300000002, 66.37175000000002, 66.61120900000003, 59.746249999999975,
+  55.74012499999992, 52.160124999999994, 52.13995800000009, 54.286749999999984,
+  50.80370800000003, 59.12149999999997, 55.563625, 52.23887500000001,
+  52.77595900000006, 51.07529199999999, 50.427041000000145, 50.14474999999993,
+  48.28724999999986, 46.59700000000021, 52.06224999999995, 50.592083,
+  49.3754170000002, 48.68487500000015, 50.96391700000004, 62.87133299999982,
+  50.64754100000005, 49.78441699999985, 47.675250000000005, 48.32424999999989,
+  47.57083399999988, 49.08341599999994, 47.32058300000017, 46.605583000000024,
+  51.04104099999995, 58.430166999999756, 50.8732500000001, 50.563208999999915,
+  48.91212499999983, 48.50745800000004, 48.26179199999979, 50.23324999999977,
+  47.112125000000106, 46.432374999999865, 51.7100419999997, 50.23104199999989,
+  50.73958399999992, 49.152374999999665, 48.91470799999979, 47.4652500000002,
+  50.519874999999956, 48.05470800000012, 51.37241700000004, 47.42558299999973,
+  49.976624999999785, 50.045416999999816, 49.49662499999977, 48.76820799999996,
+  49.057042000000365, 45.995666000000256, 46.33241599999974, 48.73274999999967,
+  49.58787500000017, 48.81129099999998, 48.582708999999795, 47.78579200000013,
+  48.683750000000146, 58.49037500000031, 49.004333000000315, 48.63583299999982,
+  49.12008300000025, 49.317291999999725, 49.40095800000017, 46.12416699999994,
+  45.25195799999983, 45.45441699999992, 52.73383399999966, 50.02391699999953,
+  48.482333999999355, 48.20887499999935, 47.37054199999966, 47.55670800000007,
+  46.027916999999434, 45.494958000000224, 45.11879200000021, 49.45758300000034,
+  50.42462500000056, 49.037874999999985, 50.33866699999999, 54.52958300000046,
+  69.9647500000001, 49.59545900000012, 49.00279099999989, 47.60712499999954,
+  48.60533300000043, 45.83591699999943, 45.930459000000155, 45.48499999999967,
+  49.828042000000096, 49.24304200000006, 49.724583999999595, 53.80274999999983,
+  53.551124999999956, 46.535917000000154, 48.323333000000275, 49.24279200000001,
+  49.24879099999998, 48.51916600000004, 49.95087500000045, 45.488084000000526,
+  45.539458000000195, 45.34254199999941, 49.46349999999984, 49.354709000000184,
+  48.80279100000007, 54.84016699999938, 58.64879199999996, 49.04120899999998,
+  51.46087499999976, 48.319375000000036, 50.23133300000063, 49.4809580000001,
+  48.86879200000021, 46.242457999999715, 45.80670800000007, 45.17291699999987,
+  48.92499999999927, 48.98304199999984, 50.21979199999987, 55.287083000000166,
+  53.05920799999967, 46.0953329999993, 48.611291000000165, 49.58479100000022,
+  49.206874999999854, 48.37412500000028, 47.9027500000002, 47.0250419999993,
+  45.96295800000007, 46.65641700000015, 45.70629199999985, 48.69391700000051,
+  50.882416999999805, 48.422458000000006, 55.62487500000043, 53.483374999999796,
+  49.25241600000027, 49.792208999999275, 48.43920799999978, 49.772375000000466,
+  47.49924999999985, 47.568208000000595, 45.369832999999744, 45.570708999999624,
+  45.60504199999923, 48.647915999999896, 47.968957999999475, 48.74304200000006,
+  53.748333999999886, 52.548915999999736, 46.5418750000008, 49.49512499999946,
+  50.23383300000023, 49.32266600000003, 48.34254200000032, 47.87970900000073,
+  47.2142500000009, 45.91208299999926, 45.81825000000026, 45.34966600000007,
+  49.91004199999952, 49.10462499999994, 48.66849999999977, 53.483917000001384,
+  54.54808299999968, 45.9251660000009, 49.77758300000096, 49.1763749999991,
+  49.095666999999594, 48.957249999999476, 47.853290999999444, 47.53254200000083,
+  45.95770900000025, 46.21495900000082, 45.60704199999964, 49.116416999999274,
+  49.75299999999879, 48.92104199999994, 54.059792000000016, 52.9361659999995,
+  51.10912499999904, 49.26566599999933, 48.63745799999924, 48.949916999999914,
+  47.82220800000141, 47.57375000000138, 46.22849999999926, 45.18529200000012,
+  45.9122499999994, 49.942832999999155, 49.02745800000048, 48.77312499999971,
+  55.1904169999998, 54.38104199999907, 50.804040999999415, 48.82645800000137,
+  49.96941700000025, 48.79229200000009, 47.69579200000044, 47.33008299999892,
+  45.52220800000032, 45.705792000000656, 45.56195899999875, 49.780249999999796,
+  49.76762500000041, 48.413125000000946, 53.676542000001064, 53.817542000000685,
+  49.37212500000169, 49.303916999999274, 49.09791600000062, 49.281333000000814,
+  48.678916999999274, 48.69449999999961, 47.745334000001094, 47.116541999999754,
+  47.39574999999968, 51.91700000000128, 50.17708300000049, 50.595791000001554,
+  56.257958000000144, 54.78375000000051, 50.82316700000047, 49.921291000000565,
+  49.13329200000044, 48.78295799999978, 47.06054099999892, 47.66666699999951,
+  45.618749999999636, 45.70125000000007, 45.93916700000045, 49.99633300000096,
+  49.5435839999991, 48.83324999999968, 54.72229100000004, 52.958042000000205,
+  45.967249999999694, 48.97375000000102, 48.961583000000246, 49.52741700000115,
+  48.45679100000052, 49.45841599999949, 45.56870899999922, 45.78220799999872,
+  45.15870899999936, 48.222999999999956, 49.63791599999968, 49.90116699999999,
+  55.798375000000306, 53.571249999999054, 49.82495800000106, 49.107082999998966,
+  48.09837499999958, 48.91808299999866, 47.980875000001106, 47.301415999998426,
+  45.586332999999286, 45.82050000000163, 45.91820800000096, 49.95800000000054,
+  48.91974999999911, 49.33429200000137, 54.322291000000405, 54.163916999999856,
+  45.9068329999991, 49.55379199999879, 49.53358299999854, 48.059416000000056,
+  48.247417000000496, 49.68645800000013, 45.87512500000048, 45.10520799999904,
+  45.71270900000127, 48.5206249999992, 50.58295799999905, 48.3025000000016,
+  54.945708999999624, 54.39416600000004, 46.142250000000786, 50.030249999999796,
+  51.16112500000054, 48.798249999999825, 49.47741700000006, 47.373875000001135,
+  48.41849999999977, 45.97737500000039, 44.88337499999943, 45.64862499999981,
+  48.39933299999939, 51.11124999999993, 49.64616600000045, 53.925750000000335,
+  53.39754200000061, 46.21454100000119, 50.089500000000044, 50.36762499999895,
+  48.34858300000087, 48.51875000000109, 48.96274999999878, 45.74474999999984,
+  45.79920800000036, 45.58704100000068, 49.909083999998984, 49.55683299999873,
+  48.94958299999962, 57.076208000000406, 52.73583399999916, 46.15720799999872,
+  48.42695900000035, 49.7913750000007, 49.42625000000044, 48.889500000001135,
+  47.973917000001165, 47.287625000000844, 45.76283400000102, 46.195708000001105,
+  46.194124999999985, 50.33754100000078, 48.24095800000214, 49.14208399999916,
+  55.5870419999992, 53.15037500000108, 46.09875000000102, 50.084499999997206,
+  49.441209000000526, 48.692792000001646, 49.08374999999796, 47.79175000000032,
+  47.27362500000163, 45.38091699999859, 45.635250000003, 47.23349999999846,
+  49.77679100000023, 49.17045800000051, 48.45100000000093, 55.641208000000915,
+  54.024750000000495, 46.13704199999847, 49.809832999999344, 49.05558300000121,
+  47.957916999999725, 48.55262500000026, 48.66766600000119, 46.03325000000041,
+  45.76920799999789, 45.16666699999769, 49.53404199999932, 52.150707999997394,
+  49.74516700000095, 55.66187499999796, 53.50929199999882, 46.443083999998635,
+  50.69879199999923, 48.70245799999975, 48.95000000000073, 48.59899999999834,
+  50.081583000002865, 46.214625000000524, 45.804707999999664,
+  45.095792000000074, 48.98354100000142, 48.89466700000048, 48.812708000001294,
+  53.922207999999955, 52.70049999999901, 49.61370900000111, 50.798915999999736,
+  49.360875000002125, 49.354208000000654, 47.13891700000022, 47.816791999997804,
+  45.41125000000102, 45.98400000000038, 44.92933400000038, 48.742167000000336,
+  48.83304099999805, 49.36870899999849, 54.48670800000036, 52.75483300000269,
+  46.27441700000054, 48.46429199999693, 48.409333000003244, 48.59400000000096,
+  48.72195799999827, 48.66749999999956, 46.51574999999866, 45.342625000001135,
+  45.40983300000153, 51.87858299999789, 48.33650000000125, 48.89666699999725,
+  55.85283400000117, 53.97491699999955, 51.93662499999846, 49.21224999999686,
+  48.30145899999843, 48.45275000000038, 47.656707999998616, 48.81725000000006,
+  45.57354200000191, 45.459249999999884, 45.343958000001294, 49.432583999998315,
+  49.647124999999505, 48.851041999998415, 54.16299999999683, 53.54054100000212,
+  49.159208000000945, 48.99841599999854, 49.07083299999795, 48.51525000000038,
+  47.105374999999185, 48.29975000000195, 46.098667000002024, 46.108500000002095,
+  45.44562500000029, 49.38929100000314, 50.16675000000032, 48.86854200000016,
+  55.041166000002704, 54.83183300000019, 45.99804199999926, 50.307709000000614,
+  48.711708999999246, 48.71820800000205, 48.39937500000087, 49.33816599999773,
+  45.464874999997846, 47.00366700000086, 46.807958000001236, 50.48908300000039,
+  51.03966700000092, 50.539041, 55.56983300000138, 55.554917000001296,
+  47.528833000000304, 50.371125000001484, 51.22849999999744, 50.604583000000275,
+  49.44691700000112, 47.18741600000067, 47.74704200000269, 45.759665999998106,
+  45.92933300000004, 45.08745899999849, 49.758583999999246, 48.574875000002066,
+  48.469374999996944, 53.76145899999756, 53.07483299999876, 49.99329200000284,
+  48.59245799999917, 48.815458000000945, 48.628458999999566, 47.534708000002865,
+  47.15100000000166, 45.8171250000014, 46.22229199999856, 45.70787500000006,
+  50.791165999999066, 49.08420800000022, 49.10949999999866, 53.97541699999783,
+  53.11329200000182, 45.94574999999895, 50.71537499999977, 48.68037499999991,
+  49.84808300000077, 48.72179099999994, 48.80349999999817, 46.715250000001106,
+  45.805333000000246, 45.3507090000021, 50.67554199999722, 49.488375000000815,
+  49.021624999997584, 54.55595899999753, 53.0313750000023, 46.67766699999993,
+  48.7364579999994, 48.38183299999946, 48.359707999999955, 48.29820800000016,
+  49.10720799999763, 45.73804100000052, 45.88729200000307, 46.00895899999887,
+  49.11087499999849, 49.64783400000306, 48.246124999997846, 53.89704200000051,
+  54.30987499999901, 52.402375000001484, 49.89104199999929, 51.11099999999715,
+  50.291291999998066, 48.18249999999898, 47.65270800000144, 46.02037500000006,
+  45.74329099999886, 45.31195799999841, 49.5357499999991, 49.88429100000212,
+  49.181916999998066, 54.580792000000656, 53.722624999998516,
+  45.724874999999884, 49.99233300000196, 49.896624999997584, 48.12062499999956,
+  49.195457999998325, 48.91291700000147, 45.73216700000194, 45.643874999997934,
+  45.62666700000045, 49.03704199999993, 49.06529199999932, 48.76683299999786,
+  54.8761250000025, 52.949124999999185, 46.09779100000014, 49.21733400000085,
+  49.02845800000068, 48.11529199999859, 48.92212499999732, 46.77283400000306,
+  47.442416999998386, 45.066500000000815, 45.99537500000224, 45.02600000000166,
+  48.662875000001804, 48.97608399999808, 48.133874999999534, 53.967500000002474,
+  52.93541600000026, 47.48162500000035, 51.27733299999818, 48.93304200000057,
+  49.10966700000063, 49.105124999998225, 47.832791000000725, 47.55662500000108,
+  45.263958999999886, 45.89308300000266, 45.14274999999907, 50.08416700000089,
+  48.17337499999849, 49.68975000000137, 54.730875000001106, 53.85200000000259,
+  45.659583000000566, 49.74187499999971, 49.65020899999945, 48.14274999999907,
+  48.44183400000111, 49.54704100000163, 45.452666999997746, 45.54370800000106,
+  45.30216699999801, 48.969959000001836, 49.36304200000086, 48.47708299999795,
+  53.67975000000297, 53.69904099999985, 46.765542000001005, 50.87083300000086,
+  49.14904199999728, 48.34608300000036, 48.5568749999984, 49.42708300000231,
+  45.6563750000023, 45.61137499999677, 45.150540999999066, 49.74079200000051,
+  49.224124999997, 48.68712500000038, 54.10337499999878, 52.861957999997685,
+  45.97370800000135, 48.53074999999808, 48.85729199999696, 49.814375000001746,
+  48.98833400000149, 49.80070799999885, 50.524208999999246, 46.54212499999994,
+  46.037625000000844, 45.34429200000159, 49.72775000000183, 50.20683399999689,
+  49.39600000000064, 53.93683299999975, 53.79412500000035, 45.827917000002344,
+  48.4642079999976, 48.603000000002794, 49.95691699999952, 48.397916999998415,
+  48.471582999998645, 45.439125000000786, 45.06537500000195, 45.87945800000307,
+] as number[];
+
+/** Get samples in nanoseconds */
+export const bevy30SamplesNs = bevy30SamplesMs.map(s => s * 1_000_000);

package/src/tests/fixtures/cases/asyncCases.ts

@@ -0,0 +1,7 @@
+/** Test cases module with async loadCase */
+export const cases = ["alpha", "beta"];
+
+export async function loadCase(id: string) {
+  await Promise.resolve(); // simulate async
+  return { data: id.toUpperCase(), metadata: { original: id } };
+}

package/src/tests/fixtures/cases/cases.ts

@@ -0,0 +1,8 @@
+/** Test cases module for Phase 3 testing */
+export const cases = ["small", "large"];
+
+export function loadCase(id: string) {
+  const data =
+    id === "small" ? [1, 2, 3] : Array.from({ length: 100 }, (_, i) => i);
+  return { data, metadata: { size: data.length } };
+}

package/src/tests/fixtures/cases/variants/product.ts

@@ -0,0 +1,2 @@
+/** Variant that multiplies array elements */
+export const run = (arr: number[]) => arr.reduce((a, b) => a * b, 1);

package/src/tests/fixtures/cases/variants/sum.ts

@@ -0,0 +1,2 @@
+/** Variant that sums array elements */
+export const run = (arr: number[]) => arr.reduce((a, b) => a + b, 0);

package/src/tests/fixtures/discover/fast.ts

@@ -0,0 +1 @@
+export const run = () => {};

package/src/tests/fixtures/discover/slow.ts

@@ -0,0 +1,4 @@
+export const run = () => {
+  let _x = 0;
+  for (let i = 0; i < 100; i++) _x++;
+};

package/src/tests/fixtures/invalid/bad.ts

@@ -0,0 +1 @@
+export const notRun = () => {};

package/src/tests/fixtures/loader/fast.ts

@@ -0,0 +1 @@
+export const run = () => {};

package/src/tests/fixtures/loader/slow.ts

@@ -0,0 +1,4 @@
+export const run = () => {
+  let _x = 0;
+  for (let i = 0; i < 100; i++) _x++;
+};

package/src/tests/fixtures/loader/stateful.ts

@@ -0,0 +1,2 @@
+export const setup = data => ({ value: data });
+export const run = state => state.value;

package/src/tests/fixtures/stateful/stateful.ts

@@ -0,0 +1,2 @@
+export const setup = data => ({ value: data });
+export const run = state => state.value;

package/src/tests/fixtures/variants/extra.ts

@@ -0,0 +1 @@
+export const run = () => {};

package/src/tests/fixtures/variants/impl.ts

@@ -0,0 +1 @@
+export const run = () => {};

package/src/tests/fixtures/worker/fast.ts

@@ -0,0 +1 @@
+export const run = () => {};

package/src/tests/fixtures/worker/slow.ts

@@ -0,0 +1,4 @@
+export const run = () => {
+  let _x = 0;
+  for (let i = 0; i < 100; i++) _x++;
+};