benchforge 0.1.9 → 0.2.4

package/src/matrix/MatrixDirRunner.ts

@@ -0,0 +1,153 @@
+import type { RunnerOptions } from "../runners/BenchRunner.ts";
+import type { MeasuredResults } from "../runners/MeasuredResults.ts";
+import { runBatched } from "../runners/MergeBatches.ts";
+import { runMatrixVariant } from "../runners/RunnerOrchestrator.ts";
+import type {
+  BenchMatrix,
+  CaseResult,
+  RunMatrixOptions,
+  VariantResult,
+} from "./BenchMatrix.ts";
+import {
+  buildRunnerOptions,
+  computeDeltaPercent,
+  resolveCases,
+} from "./BenchMatrix.ts";
+import type { CasesModule } from "./CaseLoader.ts";
+import { loadCaseData } from "./CaseLoader.ts";
+import { discoverVariants } from "./VariantLoader.ts";
+
+type VariantArgs = Parameters<typeof runMatrixVariant>[0];
+
+/** Shared state for directory-based matrix execution */
+interface DirMatrixContext<T> {
+  matrix: BenchMatrix<T>;
+  casesModule?: CasesModule<T>;
+  baselineIds: string[];
+  caseIds: string[];
+  runnerOpts: RunnerOptions;
+  batches: number;
+  warmupBatch: boolean;
+  useWorker: boolean;
+}
+
+/** Run matrix using variant files from a directory, each in a worker process */
+export async function runMatrixWithDir<T>(
+  matrix: BenchMatrix<T>,
+  options: RunMatrixOptions,
+): Promise<{ name: string; variants: VariantResult[] }> {
+  const allVariantIds = await discoverVariants(matrix.variantDir!);
+  if (allVariantIds.length === 0) {
+    throw new Error(`No variants found in ${matrix.variantDir}`);
+  }
+  const variantIds = options.filteredVariants ?? allVariantIds;
+
+  const ctx = await createDirContext(matrix, options);
+  const variants = await runDirVariants(variantIds, ctx);
+  return { name: matrix.name, variants };
+}
+
+/** Create context for directory-based matrix execution */
+async function createDirContext<T>(
+  matrix: BenchMatrix<T>,
+  options: RunMatrixOptions,
+): Promise<DirMatrixContext<T>> {
+  const baselineIds = matrix.baselineDir
+    ? await discoverVariants(matrix.baselineDir)
+    : [];
+  const { casesModule, caseIds } = await resolveCases(matrix, options);
+  const runnerOpts = buildRunnerOptions(options);
+  const { batches = 1, warmupBatch = false, useWorker = true } = options;
+  return {
+    matrix,
+    casesModule,
+    baselineIds,
+    caseIds,
+    runnerOpts,
+    batches,
+    warmupBatch,
+    useWorker,
+  };
+}
+
+/** Run all variants sequentially, collecting per-case results */
+async function runDirVariants<T>(
+  variantIds: string[],
+  ctx: DirMatrixContext<T>,
+): Promise<VariantResult[]> {
+  const variants: VariantResult[] = [];
+  for (const id of variantIds) {
+    const cases = await runDirVariantCases(id, ctx);
+    variants.push({ id, cases });
+  }
+  return variants;
+}
+
+/** Run all cases for a single variant */
+async function runDirVariantCases<T>(
+  variantId: string,
+  ctx: DirMatrixContext<T>,
+): Promise<CaseResult[]> {
+  const { matrix, casesModule, caseIds, runnerOpts, batches } = ctx;
+  const cases: CaseResult[] = [];
+
+  for (const caseId of caseIds) {
+    const caseData = matrix.cases && !matrix.casesModule ? caseId : undefined;
+    const variantArgs: VariantArgs = {
+      variantDir: matrix.variantDir!,
+      variantId,
+      caseId,
+      caseData,
+      casesModule: matrix.casesModule,
+      runner: "timing" as const,
+      options: runnerOpts,
+      useWorker: ctx.useWorker,
+    };
+    const baselineArgs =
+      matrix.baselineDir && ctx.baselineIds.includes(variantId)
+        ? { ...variantArgs, variantDir: matrix.baselineDir! }
+        : undefined;
+
+    const { metadata } = await loadCaseData(casesModule, caseId);
+    const { measured, baseline } =
+      batches > 1
+        ? await runCaseBatched(variantArgs, baselineArgs, ctx)
+        : await runCaseSingle(variantArgs, baselineArgs);
+    const deltaPercent = baseline
+      ? computeDeltaPercent(baseline, measured)
+      : undefined;
+    cases.push({ caseId, measured, metadata, baseline, deltaPercent });
+  }
+  return cases;
+}
+
+/** Run a batched measurement for a case, alternating current/baseline order. */
+async function runCaseBatched<T>(
+  variantArgs: VariantArgs,
+  baselineArgs: VariantArgs | undefined,
+  ctx: DirMatrixContext<T>,
+): Promise<{ measured: MeasuredResults; baseline?: MeasuredResults }> {
+  const runCurrent = async () => (await runMatrixVariant(variantArgs))[0];
+  const runBase = baselineArgs
+    ? async () => (await runMatrixVariant(baselineArgs))[0]
+    : undefined;
+  const { results, baseline } = await runBatched(
+    [runCurrent],
+    runBase,
+    ctx.batches,
+    ctx.warmupBatch,
+  );
+  return { measured: results[0], baseline };
+}
+
+/** Run a single unbatched measurement for a case. */
+async function runCaseSingle(
+  variantArgs: VariantArgs,
+  baselineArgs: VariantArgs | undefined,
+): Promise<{ measured: MeasuredResults; baseline?: MeasuredResults }> {
+  const [measured] = await runMatrixVariant(variantArgs);
+  const baseline = baselineArgs
+    ? (await runMatrixVariant(baselineArgs))[0]
+    : undefined;
+  return { measured, baseline };
+}

package/src/matrix/MatrixFilter.ts

@@ -1,4 +1,4 @@
-import type { BenchMatrix } from "../BenchMatrix.ts";
+import type { BenchMatrix } from "./BenchMatrix.ts";
 import { loadCasesModule } from "./CaseLoader.ts";
 import { discoverVariants } from "./VariantLoader.ts";
 
@@ -8,24 +8,21 @@ export interface MatrixFilter {
   variant?: string;
 }
 
+/** Filtered matrix with explicit case and variant lists */
+export interface FilteredMatrix<T = unknown> extends BenchMatrix<T> {
+  filteredCases?: string[];
+  filteredVariants?: string[];
+}
+
 /** Parse filter string: "case/variant", "case/", "/variant", or "case" */
 export function parseMatrixFilter(filter: string): MatrixFilter {
   if (filter.includes("/")) {
-    const [casePart, variantPart] = filter.split("/", 2);
-    return {
-      case: casePart || undefined,
-      variant: variantPart || undefined,
-    };
+    const [casePart, varPart] = filter.split("/", 2);
+    return { case: casePart || undefined, variant: varPart || undefined };
   }
   return { case: filter };
 }
 
-/** Filtered matrix with explicit case and variant lists */
-export interface FilteredMatrix<T = unknown> extends BenchMatrix<T> {
-  filteredCases?: string[];
-  filteredVariants?: string[];
-}
-
 /** Apply filter to a matrix, merging with existing filters via intersection */
 export async function filterMatrix<T>(
   matrix: FilteredMatrix<T>,
@@ -36,68 +33,73 @@ export async function filterMatrix<T>(
   const caseList = await getFilteredCases(matrix, filter.case);
   const variantList = await getFilteredVariants(matrix, filter.variant);
 
-  const filteredCases =
-    caseList && matrix.filteredCases
-      ? caseList.filter(c => matrix.filteredCases!.includes(c))
-      : (caseList ?? matrix.filteredCases);
-
-  const filteredVariants =
-    variantList && matrix.filteredVariants
-      ? variantList.filter(v => matrix.filteredVariants!.includes(v))
-      : (variantList ?? matrix.filteredVariants);
+  const filteredCases = intersectFilters(caseList, matrix.filteredCases);
+  const filteredVariants = intersectFilters(
+    variantList,
+    matrix.filteredVariants,
+  );
 
   return { ...matrix, filteredCases, filteredVariants };
 }
 
-/** Get case IDs matching filter pattern */
+/** Collect all case IDs from either casesModule or inline cases */
+export async function resolveCaseIds<T>(
+  matrix: BenchMatrix<T>,
+): Promise<string[] | undefined> {
+  if (matrix.casesModule)
+    return (await loadCasesModule(matrix.casesModule)).cases;
+  return matrix.cases;
+}
+
+/** Collect all variant IDs from either inline variants or variantDir */
+export async function resolveVariantIds<T>(
+  matrix: BenchMatrix<T>,
+): Promise<string[]> {
+  if (matrix.variants) return Object.keys(matrix.variants);
+  if (matrix.variantDir) return discoverVariants(matrix.variantDir);
+  throw new Error("BenchMatrix requires 'variants' or 'variantDir'");
+}
+
+/** Return case IDs matching a substring pattern, or all if no pattern */
 async function getFilteredCases<T>(
   matrix: BenchMatrix<T>,
   casePattern?: string,
 ): Promise<string[] | undefined> {
   if (!casePattern) return undefined;
-
-  const caseIds = matrix.casesModule
-    ? (await loadCasesModule(matrix.casesModule)).cases
-    : matrix.cases;
+  const caseIds = await resolveCaseIds(matrix);
   if (!caseIds) return ["default"]; // implicit single case
-
-  const filtered = caseIds.filter(id => matchPattern(id, casePattern));
-  if (filtered.length === 0) {
-    throw new Error(`No cases match filter: "${casePattern}"`);
-  }
-  return filtered;
+  return filterByPattern(caseIds, casePattern, "cases");
 }
 
-/** Get variant IDs matching filter pattern */
+/** Return variant IDs matching a substring pattern, or all if no pattern */
 async function getFilteredVariants<T>(
   matrix: BenchMatrix<T>,
   variantPattern?: string,
 ): Promise<string[] | undefined> {
   if (!variantPattern) return undefined;
+  const allIds = await resolveVariantIds(matrix);
+  return filterByPattern(allIds, variantPattern, "variants");
+}
 
-  if (matrix.variants) {
-    const ids = Object.keys(matrix.variants).filter(id =>
-      matchPattern(id, variantPattern),
-    );
-    if (ids.length === 0) {
-      throw new Error(`No variants match filter: "${variantPattern}"`);
-    }
-    return ids;
-  }
-
-  if (matrix.variantDir) {
-    const allIds = await discoverVariants(matrix.variantDir);
-    const filtered = allIds.filter(id => matchPattern(id, variantPattern));
-    if (filtered.length === 0) {
-      throw new Error(`No variants match filter: "${variantPattern}"`);
-    }
-    return filtered;
-  }
+/** Intersect two optional filter lists: both present ==> intersection, otherwise the one that exists */
+function intersectFilters(a?: string[], b?: string[]): string[] | undefined {
+  if (a && b) return a.filter(v => b.includes(v));
+  return a ?? b;
+}
 
-  throw new Error("BenchMatrix requires 'variants' or 'variantDir'");
+/** Filter IDs by substring pattern, throwing if no matches */
+function filterByPattern(
+  ids: string[],
+  pattern: string,
+  label: string,
+): string[] {
+  const filtered = ids.filter(id => matchPattern(id, pattern));
+  if (filtered.length === 0)
+    throw new Error(`No ${label} match filter: "${pattern}"`);
+  return filtered;
 }
 
-/** Match id against pattern (case-insensitive substring) */
+/** Case-insensitive substring match */
 function matchPattern(id: string, pattern: string): boolean {
   return id.toLowerCase().includes(pattern.toLowerCase());
 }

package/src/matrix/MatrixInlineRunner.ts

@@ -0,0 +1,50 @@
+import { TimingRunner } from "../runners/TimingRunner.ts";
+import type {
+  BenchMatrix,
+  CaseResult,
+  RunMatrixOptions,
+  VariantResult,
+} from "./BenchMatrix.ts";
+import {
+  buildRunnerOptions,
+  prepareBenchFn,
+  resolveCases,
+} from "./BenchMatrix.ts";
+import { loadCaseData } from "./CaseLoader.ts";
+
+/** Run matrix with in-memory variant functions (no worker isolation) */
+export async function runMatrixInline<T>(
+  matrix: BenchMatrix<T>,
+  options: RunMatrixOptions,
+): Promise<{ name: string; variants: VariantResult[] }> {
+  if (matrix.baselineDir)
+    throw new Error(
+      "BenchMatrix with inline 'variants' cannot use 'baselineDir'. Use 'variantDir' instead.",
+    );
+
+  const { casesModule, caseIds } = await resolveCases(matrix, options);
+  const runner = new TimingRunner();
+  const runnerOpts = buildRunnerOptions(options);
+
+  const allEntries = Object.entries(matrix.variants!);
+  const { filteredVariants } = options;
+  const variantEntries = filteredVariants
+    ? allEntries.filter(([id]) => filteredVariants.includes(id))
+    : allEntries;
+
+  const variants: VariantResult[] = [];
+  for (const [variantId, variant] of variantEntries) {
+    const cases: CaseResult[] = [];
+    for (const caseId of caseIds) {
+      const loaded = await loadCaseData(casesModule, caseId);
+      const data = casesModule || matrix.cases ? loaded.data : (undefined as T);
+      const fn = await prepareBenchFn(variant, data);
+      const spec = { name: variantId, fn };
+      const [measured] = await runner.runBench(spec, runnerOpts);
+      cases.push({ caseId, measured, metadata: loaded.metadata });
+    }
+    variants.push({ id: variantId, cases });
+  }
+
+  return { name: matrix.name, variants };
+}