@patronage/factory-ci 0.2.1 → 1.0.0-alpha.13

package/src/vitest-profile.ts

@@ -0,0 +1,631 @@
+/**
+ * Vitest suite profiling — the measurement mechanics behind a CI runner
+ * comparison (#640, #647).
+ *
+ * A profile is a machine-readable record of N serial Vitest runs at one worker
+ * count, carrying per-file and per-test timings, aggregate duration statistics,
+ * and the hardware the samples actually ran on. The hardware capture is the
+ * point: the #640 Depot comparison only resolved because every sample recorded
+ * its `cpuModel`, which split otherwise-identical 4-CPU runs into two
+ * non-overlapping populations.
+ *
+ * Everything a repository decides stays with the repository: worker counts,
+ * sample counts, output paths, runner labels, and which suite to run at all.
+ * This module owns the schema, the report parsing, the environment capture, and
+ * the atomic write.
+ */
+
+import { execFileSync, spawn } from "node:child_process";
+import type { StdioOptions } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { once } from "node:events";
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { createRequire } from "node:module";
+import {
+  arch,
+  availableParallelism,
+  cpus,
+  platform,
+  release,
+  totalmem,
+} from "node:os";
+import path from "node:path";
+import { performance } from "node:perf_hooks";
+
+/** Schema version of the emitted profile document. */
+export const VITEST_PROFILE_SCHEMA_VERSION = 1;
+
+/** `tool` discriminator every emitted profile carries. */
+export const VITEST_PROFILE_TOOL = "factory-ci-vitest-profile";
+
+const RUN_TIMEOUT_MS = 15 * 60_000;
+const TERMINATION_GRACE_MS = 5000;
+
+/** Per-assertion status as Vitest's JSON reporter spells it. */
+export type VitestTestStatus =
+  | "disabled"
+  | "failed"
+  | "passed"
+  | "pending"
+  | "skipped"
+  | "todo";
+
+/** The subset of Vitest's `--reporter=json` document this module reads. */
+export interface VitestJsonReport {
+  numFailedTests: number;
+  numPassedTests: number;
+  numPendingTests: number;
+  numTodoTests: number;
+  numTotalTests: number;
+  numTotalTestSuites: number;
+  success: boolean;
+  testResults: {
+    assertionResults: {
+      duration?: number | null;
+      fullName: string;
+      status: VitestTestStatus;
+    }[];
+    endTime: number;
+    name: string;
+    startTime: number;
+    status: "failed" | "passed";
+  }[];
+}
+
+/** Duration statistics over one population of samples, in milliseconds. */
+export interface VitestProfileDurationSummary {
+  maximum: number;
+  mean: number;
+  median: number;
+  minimum: number;
+}
+
+/**
+ * The machine a sample ran on, plus the commit it measured. `cpuModel` is the
+ * field that makes two runs comparable at all: hosted runner pools mix silicon
+ * behind one label.
+ */
+export interface VitestProfileEnvironment {
+  arch: string;
+  availableParallelism: number;
+  cpuCount: number;
+  cpuModel: string | null;
+  gitDirty: boolean | null;
+  gitHead: string | null;
+  node: string;
+  osRelease: string;
+  platform: string;
+  totalMemoryBytes: number;
+  vitest: string;
+}
+
+/** One complete Vitest run inside a profile. */
+export interface VitestProfileSample {
+  counts: {
+    failed: number;
+    passed: number;
+    pending: number;
+    suites: number;
+    tests: number;
+    todo: number;
+  } | null;
+  durationMs: number;
+  endedAt: string;
+  exitCode: number;
+  failure: string | null;
+  files: {
+    durationMs: number;
+    path: string;
+    status: "failed" | "passed";
+  }[];
+  reportAvailable: boolean;
+  sample: number;
+  startedAt: string;
+  tests: {
+    durationMs: number;
+    file: string;
+    name: string;
+    status: VitestTestStatus;
+  }[];
+}
+
+/** The emitted profile document. */
+export interface VitestProfile {
+  command: string[];
+  endedAt: string;
+  environment: VitestProfileEnvironment;
+  options: {
+    maxWorkers: number;
+    samples: number;
+    slowLimit: number;
+  };
+  rawReportDirectory: string;
+  runs: VitestProfileSample[];
+  schemaVersion: typeof VITEST_PROFILE_SCHEMA_VERSION;
+  startedAt: string;
+  summary: {
+    durationMs: VitestProfileDurationSummary;
+    slowFiles: {
+      durationMs: VitestProfileDurationSummary;
+      path: string;
+      samples: number;
+    }[];
+    slowTests: {
+      durationMs: VitestProfileDurationSummary;
+      file: string;
+      name: string;
+      samples: number;
+    }[];
+  };
+  tool: typeof VITEST_PROFILE_TOOL;
+}
+
+/** What a caller must decide before a profile can run. */
+export interface VitestProfileOptions {
+  /** Directory Vitest runs in; file paths are recorded relative to it. */
+  cwd: string;
+  /** Directory whose Git state is recorded. Defaults to `cwd`. */
+  gitDirectory?: string;
+  /** Vitest `--maxWorkers` for every sample. */
+  maxWorkers: number;
+  /** Absolute path of the profile document to write. */
+  outputPath: string;
+  /** Called before each sample starts. */
+  onSampleStart?: (input: {
+    maxWorkers: number;
+    sample: number;
+    samples: number;
+  }) => void;
+  /** Called after each sample is normalized and persisted. */
+  onSampleComplete?: (input: {
+    result: VitestProfileSample;
+    sample: number;
+    samples: number;
+  }) => void;
+  /** How many serial runs to take. */
+  samples: number;
+  /** How many slow files and slow tests to keep in the summary. */
+  slowLimit: number;
+  /**
+   * `stdio` for the Vitest child. Defaults to `"inherit"`, which is what a
+   * caller printing progress to a terminal wants; a caller whose own stdout is
+   * structured passes `"ignore"` to silence the run.
+   */
+  stdio?: StdioOptions;
+}
+
+/** Outcome of one Vitest invocation, as `runSample` reports it. */
+export interface VitestProfileSampleExecution {
+  durationMs: number;
+  exitCode: number;
+  failure?: string | null;
+  report: VitestJsonReport | null;
+}
+
+/** Injectable seams; production passes none of them. */
+export interface VitestProfileDependencies {
+  now?: () => Date;
+  runSample?: (input: {
+    cwd: string;
+    maxWorkers: number;
+    reportPath: string;
+    sample: number;
+    stdio: StdioOptions;
+  }) => Promise<VitestProfileSampleExecution>;
+  writeResult?: (outputPath: string, profile: VitestProfile) => Promise<void>;
+}
+
+/**
+ * A sample failed. The partial profile is already on disk; `exitCode` is the
+ * status a caller should exit with.
+ */
+export class VitestProfileError extends Error {
+  readonly exitCode: number;
+
+  constructor(message: string, exitCode: number) {
+    super(message);
+    this.name = "VitestProfileError";
+    this.exitCode = exitCode;
+  }
+}
+
+const durationSummary = (durations: number[]): VitestProfileDurationSummary => {
+  const sorted = durations.toSorted((left, right) => left - right);
+  const middle = Math.floor(sorted.length / 2);
+  const median =
+    sorted.length % 2 === 0
+      ? ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2
+      : (sorted[middle] ?? 0);
+  return {
+    maximum: sorted.at(-1) ?? 0,
+    mean: durations.reduce((sum, value) => sum + value, 0) / durations.length,
+    median,
+    minimum: sorted[0] ?? 0,
+  };
+};
+
+const relativePath = (cwd: string, file: string): string =>
+  path.relative(cwd, file).split(path.sep).join("/");
+
+/**
+ * Fold one Vitest JSON report into a profile sample: file and test timings,
+ * both sorted slowest first, plus the run's counts. A missing report (crash,
+ * timeout, unwritable output) yields a sample with `reportAvailable: false`
+ * rather than nothing at all.
+ */
+export const normalizeVitestProfileSample = (
+  report: VitestJsonReport | null,
+  input: {
+    cwd: string;
+    durationMs: number;
+    endedAt: Date;
+    exitCode: number;
+    failure?: string | null;
+    sample: number;
+    startedAt: Date;
+  }
+): VitestProfileSample => {
+  const files = (report?.testResults ?? [])
+    .map((file) => ({
+      durationMs: Math.max(0, file.endTime - file.startTime),
+      path: relativePath(input.cwd, file.name),
+      status: file.status,
+    }))
+    .toSorted((left, right) => right.durationMs - left.durationMs);
+  const tests = (report?.testResults ?? [])
+    .flatMap((file) =>
+      file.assertionResults.map((test) => ({
+        durationMs: Math.max(0, test.duration ?? 0),
+        file: relativePath(input.cwd, file.name),
+        name: test.fullName,
+        status: test.status,
+      }))
+    )
+    .toSorted((left, right) => right.durationMs - left.durationMs);
+
+  return {
+    counts: report
+      ? {
+          failed: report.numFailedTests,
+          passed: report.numPassedTests,
+          pending: report.numPendingTests,
+          suites: report.numTotalTestSuites,
+          tests: report.numTotalTests,
+          todo: report.numTodoTests,
+        }
+      : null,
+    durationMs: input.durationMs,
+    endedAt: input.endedAt.toISOString(),
+    exitCode: input.exitCode,
+    failure: input.failure ?? null,
+    files,
+    reportAvailable: report !== null,
+    sample: input.sample,
+    startedAt: input.startedAt.toISOString(),
+    tests,
+  };
+};
+
+const aggregateSlowFiles = (
+  runs: VitestProfileSample[],
+  limit: number
+): VitestProfile["summary"]["slowFiles"] => {
+  const durations = new Map<string, number[]>();
+  for (const file of runs.flatMap((run) => run.files)) {
+    const recorded = durations.get(file.path) ?? [];
+    recorded.push(file.durationMs);
+    durations.set(file.path, recorded);
+  }
+  return [...durations]
+    .map(([filePath, values]) => ({
+      durationMs: durationSummary(values),
+      path: filePath,
+      samples: values.length,
+    }))
+    .toSorted((left, right) => right.durationMs.median - left.durationMs.median)
+    .slice(0, limit);
+};
+
+const aggregateSlowTests = (
+  runs: VitestProfileSample[],
+  limit: number
+): VitestProfile["summary"]["slowTests"] => {
+  const timings = new Map<
+    string,
+    { durations: number[]; file: string; name: string }
+  >();
+  for (const test of runs.flatMap((run) => run.tests)) {
+    const key = `${test.file}\0${test.name}`;
+    const recorded = timings.get(key) ?? {
+      durations: [],
+      file: test.file,
+      name: test.name,
+    };
+    recorded.durations.push(test.durationMs);
+    timings.set(key, recorded);
+  }
+  return [...timings.values()]
+    .map(({ durations, file, name }) => ({
+      durationMs: durationSummary(durations),
+      file,
+      name,
+      samples: durations.length,
+    }))
+    .toSorted((left, right) => right.durationMs.median - left.durationMs.median)
+    .slice(0, limit);
+};
+
+const gitValue = (cwd: string, args: string[]): string | null => {
+  try {
+    return execFileSync("git", args, {
+      cwd,
+      encoding: "utf-8",
+      stdio: ["ignore", "pipe", "ignore"],
+    }).trim();
+  } catch {
+    return null;
+  }
+};
+
+const resolveVitestPackage = (cwd: string): string =>
+  createRequire(path.join(path.resolve(cwd), "noop.js")).resolve(
+    "vitest/package.json"
+  );
+
+/**
+ * Record the machine and commit a profile was taken on. Vitest's version is
+ * resolved from `cwd`, so it is the consumer's Vitest and not this package's.
+ * Git failures degrade to `null` — an artifact from a tarball checkout is still
+ * a usable measurement.
+ */
+export const captureVitestProfileEnvironment = async (options: {
+  cwd: string;
+  gitDirectory?: string;
+}): Promise<VitestProfileEnvironment> => {
+  const gitDirectory = options.gitDirectory ?? options.cwd;
+  let vitestVersion = "unknown";
+  try {
+    const vitestPackage = JSON.parse(
+      await readFile(resolveVitestPackage(options.cwd), "utf-8")
+    ) as { version?: string };
+    vitestVersion = vitestPackage.version ?? "unknown";
+  } catch {
+    vitestVersion = "unknown";
+  }
+  const processors = cpus();
+  const status = gitValue(gitDirectory, ["status", "--porcelain"]);
+  return {
+    arch: arch(),
+    availableParallelism: availableParallelism(),
+    cpuCount: processors.length,
+    cpuModel: processors[0]?.model ?? null,
+    gitDirty: status === null ? null : status.length > 0,
+    gitHead: gitValue(gitDirectory, ["rev-parse", "HEAD"]),
+    node: process.version,
+    osRelease: release(),
+    platform: platform(),
+    totalMemoryBytes: totalmem(),
+    vitest: vitestVersion,
+  };
+};
+
+const spawnVitest = async ({
+  cwd,
+  maxWorkers,
+  reportPath,
+  stdio,
+}: {
+  cwd: string;
+  maxWorkers: number;
+  reportPath: string;
+  stdio: StdioOptions;
+}): Promise<VitestProfileSampleExecution> => {
+  const vitestCli = path.join(
+    path.dirname(resolveVitestPackage(cwd)),
+    "vitest.mjs"
+  );
+  const started = performance.now();
+  const child = spawn(
+    process.execPath,
+    [
+      vitestCli,
+      "run",
+      "--reporter=json",
+      `--outputFile=${reportPath}`,
+      `--maxWorkers=${maxWorkers}`,
+    ],
+    { cwd, stdio }
+  );
+
+  let timedOut = false;
+  let forceKillTimer: ReturnType<typeof setTimeout> | undefined;
+  // A timed-out sample is terminated rather than abandoned: SIGTERM, then
+  // SIGKILL after a grace period. Either way the child exits, so waiting on
+  // `exit` covers the timeout too.
+  const timeoutTimer = setTimeout(() => {
+    timedOut = true;
+    child.kill("SIGTERM");
+    forceKillTimer = setTimeout(() => {
+      child.kill("SIGKILL");
+    }, TERMINATION_GRACE_MS);
+  }, RUN_TIMEOUT_MS);
+  let exitCode = 0;
+  let failure: string | null = null;
+  try {
+    // `once` rejects if the child emits `error` first, which is how a Vitest
+    // that never started is told apart from one that ran and failed.
+    const [code, signal] = await once(child, "exit");
+    exitCode = (code as number | null) ?? (signal ? 1 : 0);
+  } catch (error) {
+    exitCode = 1;
+    failure = `Vitest could not start: ${
+      error instanceof Error ? error.message : String(error)
+    }`;
+  } finally {
+    clearTimeout(timeoutTimer);
+    if (forceKillTimer) {
+      clearTimeout(forceKillTimer);
+    }
+  }
+  if (timedOut) {
+    failure = `Vitest profile sample exceeded ${RUN_TIMEOUT_MS}ms and was terminated.`;
+  }
+  let report: VitestJsonReport | null = null;
+  try {
+    report = JSON.parse(
+      await readFile(reportPath, "utf-8")
+    ) as VitestJsonReport;
+  } catch (error) {
+    const reportFailure = `Vitest JSON report unavailable: ${
+      error instanceof Error ? error.message : String(error)
+    }`;
+    failure = failure ? `${failure} ${reportFailure}` : reportFailure;
+  }
+  return {
+    durationMs: Math.max(0, performance.now() - started),
+    exitCode: timedOut ? 1 : exitCode,
+    failure,
+    report,
+  };
+};
+
+/**
+ * Write a profile document atomically: a partial file must never be readable
+ * as a complete measurement, and the profile is rewritten after every sample.
+ */
+export const writeVitestProfile = async (
+  outputPath: string,
+  profile: VitestProfile
+): Promise<void> => {
+  await mkdir(path.dirname(outputPath), { recursive: true });
+  const temporaryPath = `${outputPath}.tmp`;
+  await writeFile(
+    temporaryPath,
+    `${JSON.stringify(profile, null, 2)}\n`,
+    "utf-8"
+  );
+  await rename(temporaryPath, outputPath);
+};
+
+/**
+ * Take `samples` serial Vitest runs at one worker count and persist the profile
+ * after each one. Samples never overlap: concurrent runs would measure CPU and
+ * I/O contention instead of the worker count under test. A failing sample
+ * throws `VitestProfileError` with the partial profile already written.
+ */
+export const runVitestProfile = async (
+  options: VitestProfileOptions,
+  dependencies: VitestProfileDependencies = {}
+): Promise<VitestProfile> => {
+  // A sample count below one would resolve green having measured nothing and
+  // written no artifact — the one input whose bad value looks like success.
+  if (!Number.isSafeInteger(options.samples) || options.samples < 1) {
+    throw new Error("runVitestProfile: samples must be a positive integer.");
+  }
+  const now = dependencies.now ?? (() => new Date());
+  const runSample = dependencies.runSample ?? spawnVitest;
+  const writeResult = dependencies.writeResult ?? writeVitestProfile;
+  const startedAt = now();
+  const rawRoot = path.join(`${options.outputPath}.raw`, randomUUID());
+  await mkdir(rawRoot, { recursive: true });
+  const profile: VitestProfile = {
+    command: [
+      "vitest",
+      "run",
+      "--reporter=json",
+      `--maxWorkers=${options.maxWorkers}`,
+    ],
+    endedAt: startedAt.toISOString(),
+    environment: await captureVitestProfileEnvironment({
+      cwd: options.cwd,
+      gitDirectory: options.gitDirectory,
+    }),
+    options: {
+      maxWorkers: options.maxWorkers,
+      samples: options.samples,
+      slowLimit: options.slowLimit,
+    },
+    rawReportDirectory: rawRoot,
+    runs: [],
+    schemaVersion: VITEST_PROFILE_SCHEMA_VERSION,
+    startedAt: startedAt.toISOString(),
+    summary: {
+      durationMs: { maximum: 0, mean: 0, median: 0, minimum: 0 },
+      slowFiles: [],
+      slowTests: [],
+    },
+    tool: VITEST_PROFILE_TOOL,
+  };
+
+  for (let sample = 1; sample <= options.samples; sample += 1) {
+    // Both reporting hooks are the caller's console, not controls: a throwing
+    // hook is captured and deferred so it can never decide whether the sample
+    // ran or why it failed. See the classification below.
+    let hookFailure: { error: unknown } | undefined;
+    try {
+      options.onSampleStart?.({
+        maxWorkers: options.maxWorkers,
+        sample,
+        samples: options.samples,
+      });
+    } catch (error) {
+      hookFailure = { error };
+    }
+    const sampleStartedAt = now();
+    const reportPath = path.join(rawRoot, `sample-${sample}.json`);
+    // Samples are intentionally serial: overlapping runs would measure CPU and
+    // I/O contention rather than the selected worker count.
+    // oxlint-disable-next-line no-await-in-loop
+    const execution = await runSample({
+      cwd: options.cwd,
+      maxWorkers: options.maxWorkers,
+      reportPath,
+      sample,
+      stdio: options.stdio ?? "inherit",
+    });
+    const sampleEndedAt = now();
+    const normalized = normalizeVitestProfileSample(execution.report, {
+      cwd: options.cwd,
+      durationMs: execution.durationMs,
+      endedAt: sampleEndedAt,
+      exitCode: execution.exitCode,
+      failure: execution.failure,
+      sample,
+      startedAt: sampleStartedAt,
+    });
+    profile.runs.push(normalized);
+    profile.endedAt = sampleEndedAt.toISOString();
+    profile.summary = {
+      durationMs: durationSummary(profile.runs.map((run) => run.durationMs)),
+      slowFiles: aggregateSlowFiles(profile.runs, options.slowLimit),
+      slowTests: aggregateSlowTests(profile.runs, options.slowLimit),
+    };
+    // Persist after every sample so a later red run still leaves useful data.
+    // oxlint-disable-next-line no-await-in-loop
+    await writeResult(options.outputPath, profile);
+
+    try {
+      options.onSampleComplete?.({
+        result: normalized,
+        sample,
+        samples: options.samples,
+      });
+    } catch (error) {
+      // An earlier hook failure in this sample is the one reported; both are
+      // deferred behind the classification either way.
+      hookFailure ??= { error };
+    }
+
+    if (execution.exitCode !== 0 || execution.report?.success !== true) {
+      throw new VitestProfileError(
+        `Vitest profile sample ${sample} failed; partial results saved.`,
+        execution.exitCode || 1
+      );
+    }
+    if (hookFailure) {
+      throw hookFailure.error;
+    }
+  }
+
+  return profile;
+};