@fuzdev/fuz_util 0.42.0 → 0.43.0

package/src/lib/benchmark_stats.ts

@@ -0,0 +1,448 @@
+/**
+ * Benchmark-specific statistical analysis.
+ * Uses the general stats utilities from stats.ts for timing/performance analysis.
+ * All timing values are in nanoseconds.
+ */
+
+import {TIME_NS_PER_SEC, time_format_adaptive} from './time.js';
+import {
+	stats_mean,
+	stats_median,
+	stats_std_dev,
+	stats_percentile,
+	stats_cv,
+	stats_min_max,
+	stats_confidence_interval,
+	stats_outliers_mad,
+} from './stats.js';
+
+/**
+ * Minimal stats interface for comparison.
+ * This allows comparing stats from different sources (e.g., loaded baselines).
+ */
+export interface BenchmarkStatsComparable {
+	mean_ns: number;
+	std_dev_ns: number;
+	sample_size: number;
+	confidence_interval_ns: [number, number];
+}
+
+/**
+ * Effect size magnitude interpretation (Cohen's d).
+ */
+export type EffectMagnitude = 'negligible' | 'small' | 'medium' | 'large';
+
+/**
+ * Result from comparing two benchmark stats.
+ */
+export interface BenchmarkComparison {
+	/** Which benchmark is faster ('a', 'b', or 'equal' if difference is negligible) */
+	faster: 'a' | 'b' | 'equal';
+	/** How much faster the winner is (e.g., 1.5 means 1.5x faster) */
+	speedup_ratio: number;
+	/** Whether the difference is statistically significant at the given alpha */
+	significant: boolean;
+	/** P-value from Welch's t-test (lower = more confident the difference is real) */
+	p_value: number;
+	/** Cohen's d effect size (magnitude of difference independent of sample size) */
+	effect_size: number;
+	/** Interpretation of effect size */
+	effect_magnitude: EffectMagnitude;
+	/** Whether the 95% confidence intervals overlap */
+	ci_overlap: boolean;
+	/** Human-readable interpretation of the comparison */
+	recommendation: string;
+}
+
+/**
+ * Options for benchmark comparison.
+ */
+export interface BenchmarkCompareOptions {
+	/** Significance level for hypothesis testing (default: 0.05) */
+	alpha?: number;
+}
+
+/**
+ * Complete statistical analysis of timing measurements.
+ * Includes outlier detection, descriptive statistics, and performance metrics.
+ * All timing values are in nanoseconds.
+ */
+export class BenchmarkStats {
+	/** Mean (average) time in nanoseconds */
+	readonly mean_ns: number;
+	/** Median time in nanoseconds */
+	readonly median_ns: number;
+	/** Standard deviation in nanoseconds */
+	readonly std_dev_ns: number;
+	/** Minimum time in nanoseconds */
+	readonly min_ns: number;
+	/** Maximum time in nanoseconds */
+	readonly max_ns: number;
+	/** 75th percentile in nanoseconds */
+	readonly p75_ns: number;
+	/** 90th percentile in nanoseconds */
+	readonly p90_ns: number;
+	/** 95th percentile in nanoseconds */
+	readonly p95_ns: number;
+	/** 99th percentile in nanoseconds */
+	readonly p99_ns: number;
+	/** Coefficient of variation (std_dev / mean) */
+	readonly cv: number;
+	/** 95% confidence interval for the mean in nanoseconds */
+	readonly confidence_interval_ns: [number, number];
+	/** Array of detected outlier values in nanoseconds */
+	readonly outliers_ns: Array<number>;
+	/** Ratio of outliers to total samples */
+	readonly outlier_ratio: number;
+	/** Number of samples after outlier removal */
+	readonly sample_size: number;
+	/** Original number of samples (before outlier removal) */
+	readonly raw_sample_size: number;
+	/** Operations per second (NS_PER_SEC / mean_ns) */
+	readonly ops_per_second: number;
+	/** Number of failed iterations (NaN, Infinity, or negative values) */
+	readonly failed_iterations: number;
+
+	constructor(timings_ns: Array<number>) {
+		// Filter out invalid values (NaN, Infinity, negative)
+		const valid_timings: Array<number> = [];
+		let failed_count = 0;
+
+		for (const t of timings_ns) {
+			if (!isNaN(t) && isFinite(t) && t > 0) {
+				valid_timings.push(t);
+			} else {
+				failed_count++;
+			}
+		}
+
+		this.failed_iterations = failed_count;
+		this.raw_sample_size = timings_ns.length;
+
+		// If no valid timings, return empty stats
+		if (valid_timings.length === 0) {
+			this.mean_ns = NaN;
+			this.median_ns = NaN;
+			this.std_dev_ns = NaN;
+			this.min_ns = NaN;
+			this.max_ns = NaN;
+			this.p75_ns = NaN;
+			this.p90_ns = NaN;
+			this.p95_ns = NaN;
+			this.p99_ns = NaN;
+			this.cv = NaN;
+			this.confidence_interval_ns = [NaN, NaN];
+			this.outliers_ns = [];
+			this.outlier_ratio = 0;
+			this.sample_size = 0;
+			this.ops_per_second = 0;
+			return;
+		}
+
+		// Detect and remove outliers
+		const {cleaned, outliers} = stats_outliers_mad(valid_timings);
+		const sorted_cleaned = [...cleaned].sort((a, b) => a - b);
+
+		this.outliers_ns = outliers;
+		this.outlier_ratio = outliers.length / valid_timings.length;
+		this.sample_size = cleaned.length;
+
+		// Calculate statistics on cleaned data
+		this.mean_ns = stats_mean(cleaned);
+		this.median_ns = stats_median(sorted_cleaned);
+		this.std_dev_ns = stats_std_dev(cleaned, this.mean_ns);
+
+		const {min, max} = stats_min_max(sorted_cleaned);
+		this.min_ns = min;
+		this.max_ns = max;
+
+		this.p75_ns = stats_percentile(sorted_cleaned, 0.75);
+		this.p90_ns = stats_percentile(sorted_cleaned, 0.9);
+		this.p95_ns = stats_percentile(sorted_cleaned, 0.95);
+		this.p99_ns = stats_percentile(sorted_cleaned, 0.99);
+
+		this.cv = stats_cv(this.mean_ns, this.std_dev_ns);
+		this.confidence_interval_ns = stats_confidence_interval(cleaned);
+
+		// Calculate throughput (operations per second)
+		this.ops_per_second = this.mean_ns > 0 ? TIME_NS_PER_SEC / this.mean_ns : 0;
+	}
+
+	/**
+	 * Format stats as a human-readable string.
+	 */
+	toString(): string {
+		return `BenchmarkStats(mean=${time_format_adaptive(this.mean_ns)}, ops/sec=${this.ops_per_second.toFixed(2)}, cv=${(this.cv * 100).toFixed(1)}%, samples=${this.sample_size})`;
+	}
+}
+
+/**
+ * Compare two benchmark results for statistical significance.
+ * Uses Welch's t-test (handles unequal variances) and Cohen's d effect size.
+ *
+ * @param a - First benchmark stats (or any object with required properties)
+ * @param b - Second benchmark stats (or any object with required properties)
+ * @param options - Comparison options
+ * @returns Comparison result with significance, effect size, and recommendation
+ *
+ * @example
+ * ```ts
+ * const comparison = benchmark_stats_compare(result_a.stats, result_b.stats);
+ * if (comparison.significant) {
+ *   console.log(`${comparison.faster} is ${comparison.speedup_ratio.toFixed(2)}x faster`);
+ * }
+ * ```
+ */
+export const benchmark_stats_compare = (
+	a: BenchmarkStatsComparable,
+	b: BenchmarkStatsComparable,
+	options?: BenchmarkCompareOptions,
+): BenchmarkComparison => {
+	const alpha = options?.alpha ?? 0.05;
+
+	// Handle edge cases
+	if (a.sample_size === 0 || b.sample_size === 0) {
+		return {
+			faster: 'equal',
+			speedup_ratio: 1,
+			significant: false,
+			p_value: 1,
+			effect_size: 0,
+			effect_magnitude: 'negligible',
+			ci_overlap: true,
+			recommendation: 'Insufficient data for comparison',
+		};
+	}
+
+	// Calculate speedup ratio (lower time = faster, so compare by time not ops/sec)
+	const speedup_ratio = a.mean_ns < b.mean_ns ? b.mean_ns / a.mean_ns : a.mean_ns / b.mean_ns;
+	const faster: 'a' | 'b' | 'equal' =
+		a.mean_ns < b.mean_ns ? 'a' : a.mean_ns > b.mean_ns ? 'b' : 'equal';
+
+	// Welch's t-test (handles unequal variances)
+	// Special case: if both have zero variance, t-test is undefined
+	let p_value: number;
+	if (a.std_dev_ns === 0 && b.std_dev_ns === 0) {
+		// When there's no variance, any difference is 100% reliable (p=0) or identical (p=1)
+		p_value = a.mean_ns === b.mean_ns ? 1 : 0;
+	} else {
+		const {t_statistic, degrees_of_freedom} = welch_t_test(
+			a.mean_ns,
+			a.std_dev_ns,
+			a.sample_size,
+			b.mean_ns,
+			b.std_dev_ns,
+			b.sample_size,
+		);
+		// Calculate two-tailed p-value using t-distribution approximation
+		p_value = t_distribution_p_value(Math.abs(t_statistic), degrees_of_freedom);
+	}
+
+	// Cohen's d effect size
+	const pooled_std_dev = Math.sqrt(
+		((a.sample_size - 1) * a.std_dev_ns ** 2 + (b.sample_size - 1) * b.std_dev_ns ** 2) /
+			(a.sample_size + b.sample_size - 2),
+	);
+
+	// When pooled_std_dev is 0 but means differ, effect is maximal (infinite)
+	// When means are equal, effect is 0
+	let effect_size: number;
+	let effect_magnitude: EffectMagnitude;
+
+	if (pooled_std_dev === 0) {
+		// Zero variance case - if means differ, it's a definitive difference
+		if (a.mean_ns === b.mean_ns) {
+			effect_size = 0;
+			effect_magnitude = 'negligible';
+		} else {
+			// Any difference is 100% reliable when there's no variance
+			effect_size = Infinity;
+			effect_magnitude = 'large';
+		}
+	} else {
+		effect_size = Math.abs(a.mean_ns - b.mean_ns) / pooled_std_dev;
+		// Interpret effect size (Cohen's conventions)
+		effect_magnitude =
+			effect_size < 0.2
+				? 'negligible'
+				: effect_size < 0.5
+					? 'small'
+					: effect_size < 0.8
+						? 'medium'
+						: 'large';
+	}
+
+	// Check confidence interval overlap
+	const ci_overlap =
+		a.confidence_interval_ns[0] <= b.confidence_interval_ns[1] &&
+		b.confidence_interval_ns[0] <= a.confidence_interval_ns[1];
+
+	// Determine significance
+	const significant = p_value < alpha;
+
+	// Generate recommendation
+	let recommendation: string;
+	if (!significant) {
+		recommendation =
+			effect_magnitude === 'negligible'
+				? 'No meaningful difference detected'
+				: `Difference not statistically significant (p=${p_value.toFixed(3)}), but effect size suggests ${effect_magnitude} practical difference`;
+	} else if (effect_magnitude === 'negligible') {
+		recommendation = `Statistically significant but negligible practical difference (${speedup_ratio.toFixed(2)}x)`;
+	} else {
+		recommendation = `${faster === 'a' ? 'First' : 'Second'} is ${speedup_ratio.toFixed(2)}x faster with ${effect_magnitude} effect size (p=${p_value.toFixed(3)})`;
+	}
+
+	// Adjust 'faster' to 'equal' if effect is negligible
+	const adjusted_faster = effect_magnitude === 'negligible' ? 'equal' : faster;
+
+	return {
+		faster: adjusted_faster,
+		speedup_ratio,
+		significant,
+		p_value,
+		effect_size,
+		effect_magnitude,
+		ci_overlap,
+		recommendation,
+	};
+};
+
+/**
+ * Calculate Welch's t-test statistic and degrees of freedom.
+ * Welch's t-test is more robust than Student's t-test when variances are unequal.
+ */
+const welch_t_test = (
+	mean1: number,
+	std1: number,
+	n1: number,
+	mean2: number,
+	std2: number,
+	n2: number,
+): {t_statistic: number; degrees_of_freedom: number} => {
+	const var1 = std1 ** 2;
+	const var2 = std2 ** 2;
+
+	const se1 = var1 / n1;
+	const se2 = var2 / n2;
+
+	const t_statistic = (mean1 - mean2) / Math.sqrt(se1 + se2);
+
+	// Welch-Satterthwaite degrees of freedom
+	const numerator = (se1 + se2) ** 2;
+	const denominator = se1 ** 2 / (n1 - 1) + se2 ** 2 / (n2 - 1);
+	const degrees_of_freedom = numerator / denominator;
+
+	return {t_statistic, degrees_of_freedom};
+};
+
+/**
+ * Approximate p-value from t-distribution using the approximation formula.
+ * This avoids requiring a full t-distribution table or library.
+ * For large df (>30), this approximation is very accurate.
+ */
+const t_distribution_p_value = (t: number, df: number): number => {
+	// Use normal approximation for large df
+	if (df > 100) {
+		// Standard normal CDF approximation
+		return 2 * (1 - normal_cdf(t));
+	}
+
+	// For smaller df, use a more accurate approximation
+	// Based on the incomplete beta function relationship
+	const x = df / (df + t * t);
+	const a = df / 2;
+	const b = 0.5;
+
+	// Approximation of regularized incomplete beta function
+	// This is accurate to about 4 decimal places for typical use cases
+	const beta_approx = incomplete_beta_approx(x, a, b);
+	return beta_approx;
+};
+
+/**
+ * Standard normal CDF approximation (Abramowitz and Stegun formula 7.1.26).
+ */
+const normal_cdf = (x: number): number => {
+	const t = 1 / (1 + 0.2316419 * Math.abs(x));
+	const d = 0.3989423 * Math.exp((-x * x) / 2);
+	const p =
+		d * t * (0.3193815 + t * (-0.3565638 + t * (1.781478 + t * (-1.821256 + t * 1.330274))));
+	return x > 0 ? 1 - p : p;
+};
+
+/**
+ * Approximate regularized incomplete beta function for p-value calculation.
+ * Uses continued fraction expansion for reasonable accuracy.
+ */
+const incomplete_beta_approx = (x: number, a: number, b: number): number => {
+	// Simple approximation using the relationship between beta and normal distributions
+	// For our use case (t-distribution p-values), this provides sufficient accuracy
+	if (x <= 0) return 0;
+	if (x >= 1) return 1;
+
+	// Use symmetry if needed
+	if (x > (a + 1) / (a + b + 2)) {
+		return 1 - incomplete_beta_approx(1 - x, b, a);
+	}
+
+	// Continued fraction approximation (first few terms)
+	const lnBeta = ln_gamma(a) + ln_gamma(b) - ln_gamma(a + b);
+	const front = Math.exp(Math.log(x) * a + Math.log(1 - x) * b - lnBeta) / a;
+
+	// Simple continued fraction (limited iterations for speed)
+	let f = 1;
+	let c = 1;
+	let d = 0;
+
+	for (let m = 1; m <= 100; m++) {
+		const m2 = 2 * m;
+
+		// Even step
+		let aa = (m * (b - m) * x) / ((a + m2 - 1) * (a + m2));
+		d = 1 + aa * d;
+		if (Math.abs(d) < 1e-30) d = 1e-30;
+		c = 1 + aa / c;
+		if (Math.abs(c) < 1e-30) c = 1e-30;
+		d = 1 / d;
+		f *= d * c;
+
+		// Odd step
+		aa = (-(a + m) * (a + b + m) * x) / ((a + m2) * (a + m2 + 1));
+		d = 1 + aa * d;
+		if (Math.abs(d) < 1e-30) d = 1e-30;
+		c = 1 + aa / c;
+		if (Math.abs(c) < 1e-30) c = 1e-30;
+		d = 1 / d;
+		const delta = d * c;
+		f *= delta;
+
+		if (Math.abs(delta - 1) < 1e-8) break;
+	}
+
+	return front * f;
+};
+
+/**
+ * Log gamma function approximation (Lanczos approximation).
+ */
+const ln_gamma = (z: number): number => {
+	const g = 7;
+	const c = [
+		0.99999999999980993, 676.5203681218851, -1259.1392167224028, 771.32342877765313,
+		-176.61502916214059, 12.507343278686905, -0.13857109526572012, 9.9843695780195716e-6,
+		1.5056327351493116e-7,
+	];
+
+	if (z < 0.5) {
+		return Math.log(Math.PI / Math.sin(Math.PI * z)) - ln_gamma(1 - z);
+	}
+
+	const z_adj = z - 1;
+	let x = c[0]!;
+	for (let i = 1; i < g + 2; i++) {
+		x += c[i]! / (z_adj + i);
+	}
+	const t = z_adj + g + 0.5;
+	return 0.5 * Math.log(2 * Math.PI) + (z_adj + 0.5) * Math.log(t) - t + Math.log(x);
+};

package/src/lib/benchmark_types.ts

@@ -0,0 +1,197 @@
+import type {BenchmarkStats} from './benchmark_stats.js';
+import type {Timer} from './time.js';
+
+/**
+ * Configuration options for a benchmark suite.
+ */
+export interface BenchmarkConfig {
+	/**
+	 * Target duration to run each benchmark task in milliseconds.
+	 * The benchmark will run until this duration is reached or max_iterations is hit.
+	 * Default: 1000ms
+	 */
+	duration_ms?: number;
+
+	/**
+	 * Number of warmup iterations before actual measurements.
+	 * Warmup helps stabilize JIT compilation and caches.
+	 * Default: 5
+	 */
+	warmup_iterations?: number;
+
+	/**
+	 * Cooldown time between tasks in milliseconds.
+	 * Helps prevent interference between benchmarks.
+	 * Default: 100ms
+	 */
+	cooldown_ms?: number;
+
+	/**
+	 * Minimum number of iterations to run.
+	 * Default: 10
+	 */
+	min_iterations?: number;
+
+	/**
+	 * Maximum number of iterations to run.
+	 * Prevents infinite loops if function is extremely fast.
+	 * Default: 100000
+	 */
+	max_iterations?: number;
+
+	/**
+	 * Custom timer to use for measurements.
+	 * Default: timer_default (auto-detects environment)
+	 */
+	timer?: Timer;
+
+	/**
+	 * Callback invoked after each iteration completes.
+	 * Useful for triggering garbage collection, logging progress, early termination,
+	 * or custom instrumentation.
+	 *
+	 * **Note**: The callback time is NOT included in iteration measurements - it runs
+	 * after the timing capture. However, frequent GC calls will slow overall benchmark
+	 * execution time.
+	 *
+	 * @param task_name - Name of the current task being benchmarked
+	 * @param iteration - Current iteration number (1-indexed)
+	 * @param abort - Call to stop the benchmark early for this task
+	 *
+	 * @example
+	 * ```ts
+	 * // Trigger GC between iterations (run node with --expose-gc)
+	 * new Benchmark({
+	 *   on_iteration: () => {
+	 *     if (globalThis.gc) globalThis.gc();
+	 *   }
+	 * })
+	 *
+	 * // Log progress for long-running benchmarks
+	 * new Benchmark({
+	 *   on_iteration: (name, iteration) => {
+	 *     if (iteration % 1000 === 0) {
+	 *       console.log(`${name}: ${iteration} iterations`);
+	 *     }
+	 *   }
+	 * })
+	 *
+	 * // Stop early when converged
+	 * new Benchmark({
+	 *   on_iteration: (name, iteration, abort) => {
+	 *     if (iteration > 1000 && has_stabilized()) abort();
+	 *   }
+	 * })
+	 * ```
+	 */
+	on_iteration?: (task_name: string, iteration: number, abort: () => void) => void;
+
+	/**
+	 * Callback invoked after each task completes.
+	 * Useful for logging progress during long benchmark runs.
+	 *
+	 * @param result - The completed benchmark result
+	 * @param index - Zero-based index of the completed task
+	 * @param total - Total number of tasks to run
+	 *
+	 * @example
+	 * ```ts
+	 * new Benchmark({
+	 *   on_task_complete: (result, index, total) => {
+	 *     console.log(`[${index + 1}/${total}] ${result.name}: ${result.stats.ops_per_second.toFixed(0)} ops/sec`);
+	 *   }
+	 * })
+	 * ```
+	 */
+	on_task_complete?: (result: BenchmarkResult, index: number, total: number) => void;
+}
+
+/**
+ * A benchmark task to execute.
+ */
+export interface BenchmarkTask {
+	/** Name of the task (for display) */
+	name: string;
+
+	/** Function to benchmark (sync or async). Return values are ignored. */
+	fn: () => unknown;
+
+	/**
+	 * Optional setup function run before benchmarking this task.
+	 * Not included in timing measurements.
+	 */
+	setup?: () => void | Promise<void>;
+
+	/**
+	 * Optional teardown function run after benchmarking this task.
+	 * Not included in timing measurements.
+	 */
+	teardown?: () => void | Promise<void>;
+
+	/**
+	 * If true, skip this task during benchmark runs.
+	 * Useful for temporarily disabling tasks during development.
+	 */
+	skip?: boolean;
+
+	/**
+	 * If true, run only this task (and other tasks marked `only`).
+	 * Useful for focusing on specific tasks during development.
+	 */
+	only?: boolean;
+
+	/**
+	 * Hint for whether the function is sync or async.
+	 * If not provided, automatically detected during warmup.
+	 * Setting this explicitly skips per-iteration promise checking for sync functions.
+	 */
+	async?: boolean;
+}
+
+/**
+ * Result from running a single benchmark task.
+ */
+export interface BenchmarkResult {
+	/** Task name */
+	name: string;
+
+	/** Statistical analysis of the benchmark */
+	stats: BenchmarkStats;
+
+	/** Number of iterations executed */
+	iterations: number;
+
+	/** Total time spent benchmarking (including warmup) in milliseconds */
+	total_time_ms: number;
+
+	/**
+	 * Raw timing data for each iteration in nanoseconds.
+	 * Useful for custom statistical analysis, histogram generation,
+	 * or exporting to external tools.
+	 */
+	timings_ns: Array<number>;
+}
+
+/**
+ * Options for table formatting.
+ */
+export interface BenchmarkFormatTableOptions {
+	/**
+	 * Group results by category using filter functions.
+	 */
+	groups?: Array<BenchmarkGroup>;
+}
+
+/**
+ * A group definition for organizing benchmark results.
+ */
+export interface BenchmarkGroup {
+	/** Display name for the group */
+	name: string;
+
+	/** Optional description shown below the group name */
+	description?: string;
+
+	/** Filter function to determine which results belong to this group */
+	filter: (result: BenchmarkResult) => boolean;
+}

package/src/lib/library_json.ts

@@ -13,11 +13,11 @@ import type {Url} from './url.js';
 export interface LibraryJson {
 	package_json: PackageJson;
 	source_json: SourceJson;
-	/** Package name, e.g. `@ryanatkn/fuz`. */
+	/** Package name, e.g. `@fuzdev/fuz_ui`. */
 	name: string;
 	/** Name without scope, e.g. `fuz`. */
 	repo_name: string;
-	/** GitHub repo URL, e.g. `https://github.com/ryanatkn/fuz`. */
+	/** GitHub repo URL, e.g. `https://github.com/fuzdev/fuz_ui`. */
 	repo_url: Url;
 	/** GitHub user/org, e.g. `ryanatkn`. */
 	owner_name: string | null;
@@ -95,7 +95,7 @@ export const library_json_parse = (
 };
 
 /**
- * Extracts repo name from a package name, e.g. `@ryanatkn/fuz` → `fuz`.
+ * Extracts repo name from a package name, e.g. `@fuzdev/fuz_ui` → `fuz`.
  */
 export const library_repo_name_parse = (name: string): string => {
 	if (name[0] === '@') {

package/src/lib/object.ts

@@ -85,7 +85,7 @@ export const reorder = <T extends Record<K, any>, K extends string | number>(
 /**
  * Frozen empty object with no properties, good for options default values.
  */
-export const EMPTY_OBJECT: Record<string | number | symbol, undefined> & object = Object.freeze({}); // eslint-disable-line @typescript-eslint/no-redundant-type-constituents
+export const EMPTY_OBJECT: Record<string | number | symbol, undefined> & object = Object.freeze({});
 
 /**
  * Performs a depth-first traversal of an object's enumerable properties,