@react-native-harness/jest 1.6.0 → 1.7.0

package/src/diagnostics/report.ts

@@ -1,11 +1,14 @@
+import fsSync from 'node:fs';
 import fs from 'node:fs/promises';
 import path from 'node:path';
-import type { DiagnosticEntry } from '@react-native-harness/tools';
-import { getHarnessRootPath } from '@react-native-harness/tools';
+import type { DiagnosticEntry, Diagnostics } from '@react-native-harness/tools';
+import { getHarnessRootPath, logger, unrefTimer } from '@react-native-harness/tools';
+
+const reportLogger = logger.child('diagnostics');
 
 type TraceEvent = {
   name: string;
-  ph: 'X' | 'i';
+  ph: 'X' | 'i' | 'C';
   ts: number;
   dur?: number;
   s?: 'g';
@@ -17,7 +20,20 @@ type TraceEvent = {
 const TRACE_PID = 1;
 const TRACE_TID = 1;
 
-const toTraceEvent = (entry: DiagnosticEntry): TraceEvent => {
+export const toTraceEvent = (entry: DiagnosticEntry): TraceEvent => {
+  if (entry.kind === 'gauge') {
+    // Counter events become graphed tracks in Perfetto / chrome://tracing,
+    // one per distinct name, so each gauge label is its own series.
+    return {
+      name: entry.label,
+      ph: 'C',
+      ts: entry.startTime * 1000,
+      pid: TRACE_PID,
+      tid: TRACE_TID,
+      args: { [entry.label]: entry.value ?? 0 },
+    };
+  }
+
   if (entry.kind === 'mark') {
     return {
       name: entry.label,
@@ -45,7 +61,7 @@ export const writeTraceFile = async (
   entries: DiagnosticEntry[],
   options: { projectRoot: string; runId: string }
 ): Promise<string> => {
-  const dir = path.join(getHarnessRootPath(options.projectRoot), 'diagnostics');
+  const dir = getDiagnosticsDir(options.projectRoot);
   await fs.mkdir(dir, { recursive: true });
 
   const filePath = path.join(dir, `${options.runId}.json`);
@@ -56,6 +72,303 @@ export const writeTraceFile = async (
   return filePath;
 };
 
+const getDiagnosticsDir = (projectRoot: string): string =>
+  path.join(getHarnessRootPath(projectRoot), 'diagnostics');
+
+export type DiagnosticsAppender = {
+  readonly filePath: string;
+  /** Synchronously writes everything not yet on disk. For `exit` handlers. */
+  flush(): void;
+  /** Waits for in-flight writes, then writes the rest and detaches. */
+  dispose(): Promise<void>;
+};
+
+/**
+ * Persists diagnostics entries as they are recorded, one JSON object per line
+ * in `.harness/diagnostics/<id>.jsonl`. The final trace is only written when a
+ * run finishes, so a job that is cancelled or killed mid-run (the case where
+ * the data matters most) would otherwise leave nothing behind.
+ *
+ * Entries are batched (every `flushIntervalMs` or `maxBufferedEntries`) and
+ * appended asynchronously, one write at a time, so recording an entry never
+ * blocks on disk, which matters most when the machine is overloaded. Only the
+ * `exit` handler writes synchronously: a process that is going away cannot
+ * wait for the event loop. The session already owns SIGINT/SIGTERM and leaves
+ * through `process.exit`, which emits `exit`; installing signal handlers here
+ * would depend on listener order and fight other handlers about who re-raises.
+ *
+ * Entries recorded before the appender was created are replayed, so the log
+ * starts at the beginning of the session.
+ */
+export const createDiagnosticsAppender = (
+  diagnostics: Diagnostics,
+  options: {
+    projectRoot: string;
+    id: string;
+    flushIntervalMs?: number;
+    maxBufferedEntries?: number;
+  }
+): DiagnosticsAppender => {
+  const filePath = path.join(
+    getDiagnosticsDir(options.projectRoot),
+    `${options.id}.jsonl`
+  );
+
+  if (!diagnostics.enabled) {
+    return {
+      filePath,
+      flush: () => undefined,
+      dispose: async () => undefined,
+    };
+  }
+
+  const maxBufferedEntries = options.maxBufferedEntries ?? 100;
+  let buffer: string[] = [];
+  // Chunks handed to the async writer and not yet confirmed on disk, kept so
+  // the `exit` handler can write them if the process leaves first. In the
+  // rare race where one was also completed by the writer, a chunk ends up in
+  // the file twice; losing it would be worse for a log whose purpose is
+  // surviving abrupt termination.
+  let pending: string[][] = [];
+  let writeChain: Promise<void> = Promise.resolve();
+  let directoryReady = false;
+
+  const render = (lines: string[]) => `${lines.join('\n')}\n`;
+
+  const flushAsync = () => {
+    if (buffer.length === 0) {
+      return;
+    }
+
+    const lines = buffer;
+    buffer = [];
+    pending.push(lines);
+    writeChain = writeChain.then(async () => {
+      try {
+        if (!directoryReady) {
+          await fs.mkdir(path.dirname(filePath), { recursive: true });
+          directoryReady = true;
+        }
+        await fs.appendFile(filePath, render(lines), 'utf-8');
+      } catch (error) {
+        reportLogger.debug('failed to append diagnostics: %s', error);
+      } finally {
+        pending = pending.filter((chunk) => chunk !== lines);
+      }
+    });
+  };
+
+  const flushSync = () => {
+    const lines = [...pending.flat(), ...buffer];
+    pending = [];
+    buffer = [];
+    if (lines.length === 0) {
+      return;
+    }
+
+    try {
+      fsSync.mkdirSync(path.dirname(filePath), { recursive: true });
+      fsSync.appendFileSync(filePath, render(lines), 'utf-8');
+    } catch (error) {
+      reportLogger.debug('failed to append diagnostics: %s', error);
+    }
+  };
+
+  const unsubscribe = diagnostics.subscribe(
+    (entry) => {
+      buffer.push(JSON.stringify(entry));
+      if (buffer.length >= maxBufferedEntries) {
+        flushAsync();
+      }
+    },
+    { replay: true }
+  );
+
+  const timer = setInterval(flushAsync, options.flushIntervalMs ?? 2000);
+  unrefTimer(timer);
+  process.on('exit', flushSync);
+
+  const dispose = async () => {
+    unsubscribe();
+    clearInterval(timer);
+    process.off('exit', flushSync);
+    flushAsync();
+    await writeChain;
+  };
+
+  return { filePath, flush: flushSync, dispose };
+};
+
+export type RunKpis = {
+  bootWaitMs: number | null;
+  timeToReadyMs: number | null;
+  peakHostCompressedMb: number | null;
+  peakLoad1: number | null;
+  minHostFreeMb: number | null;
+  /** Memory swapped out during the run, in MB. */
+  swapoutsDelta: number | null;
+  maxEventLoopLagMs: number | null;
+  testFileCount: number;
+  result: 'passed' | 'failed' | null;
+};
+
+const maxOf = (values: number[]): number | null =>
+  values.length > 0 ? Math.max(...values) : null;
+
+const minOf = (values: number[]): number | null =>
+  values.length > 0 ? Math.min(...values) : null;
+
+/**
+ * Reduces a run's entries to the handful of numbers worth comparing between
+ * runs. Anything that cannot be derived from the entries is null, never a
+ * guess, so a missing probe shows up as "n/a" rather than as a misleading 0.
+ */
+export const summarizeRunKpis = (entries: DiagnosticEntry[]): RunKpis => {
+  const gaugeValues = (label: string): number[] =>
+    entries
+      .filter((entry) => entry.kind === 'gauge' && entry.label === label)
+      .map((entry) => entry.value)
+      .filter((value): value is number => typeof value === 'number');
+
+  const spans = (label: string): DiagnosticEntry[] =>
+    entries.filter((entry) => entry.kind === 'span' && entry.label === label);
+
+  const bootSpans = entries.filter(
+    (entry) => entry.kind === 'span' && entry.label.endsWith('.waitForBoot')
+  );
+
+  const sessionStart = spans('session.config.load')[0]?.startTime;
+  const firstReady = spans('app.ready.total').find(
+    (entry) => entry.status === 'ok'
+  );
+
+  const swapouts = gaugeValues('host.mem.swapouts');
+  const runStatus = spans('run.total')[0]?.attrs?.status;
+
+  return {
+    bootWaitMs:
+      bootSpans.length > 0
+        ? bootSpans.reduce((total, entry) => total + entry.duration, 0)
+        : null,
+    timeToReadyMs:
+      sessionStart !== undefined && firstReady
+        ? firstReady.startTime + firstReady.duration - sessionStart
+        : null,
+    peakHostCompressedMb: maxOf(gaugeValues('host.mem.compressed')),
+    peakLoad1: maxOf(gaugeValues('host.load1')),
+    minHostFreeMb: minOf(gaugeValues('host.mem.free')),
+    swapoutsDelta:
+      swapouts.length >= 2
+        ? (swapouts[swapouts.length - 1] as number) - (swapouts[0] as number)
+        : null,
+    maxEventLoopLagMs: maxOf(gaugeValues('node.event_loop.max_ms')),
+    testFileCount: spans('run.file').length,
+    result:
+      runStatus === 'passed' || runStatus === 'failed' ? runStatus : null,
+  };
+};
+
+export const writeSummaryFile = async (
+  kpis: RunKpis,
+  options: { projectRoot: string; runId: string; sessionId?: string }
+): Promise<string> => {
+  const dir = getDiagnosticsDir(options.projectRoot);
+  await fs.mkdir(dir, { recursive: true });
+
+  const filePath = path.join(dir, `${options.runId}.summary.json`);
+  const summary = {
+    runId: options.runId,
+    ...(options.sessionId ? { sessionId: options.sessionId } : {}),
+    ...kpis,
+  };
+  await fs.writeFile(filePath, JSON.stringify(summary, null, 2), 'utf-8');
+
+  return filePath;
+};
+
+const formatNumber = (value: number | null, unit: string, digits = 0) =>
+  value === null ? 'n/a' : `${value.toFixed(digits)}${unit}`;
+
+const formatKpiDuration = (ms: number | null) =>
+  ms === null ? 'n/a' : formatDuration(ms);
+
+/** The KPIs as label/value pairs, in display order. */
+export const formatRunKpis = (kpis: RunKpis): Array<[string, string]> => [
+  ['Result', kpis.result ?? 'n/a'],
+  ['Test files', String(kpis.testFileCount)],
+  ['Boot wait', formatKpiDuration(kpis.bootWaitMs)],
+  ['Time to ready', formatKpiDuration(kpis.timeToReadyMs)],
+  ['Peak host load (1m)', formatNumber(kpis.peakLoad1, '', 1)],
+  ['Peak compressed memory', formatNumber(kpis.peakHostCompressedMb, ' MB')],
+  ['Min free memory', formatNumber(kpis.minHostFreeMb, ' MB')],
+  ['Swapped out during run', formatNumber(kpis.swapoutsDelta, ' MB')],
+  ['Max event loop lag', formatNumber(kpis.maxEventLoopLagMs, ' ms')],
+];
+
+/**
+ * Appends the KPIs as a markdown table to the GitHub Actions job summary when
+ * running there. Best effort: a failure here must never fail the run.
+ */
+export const appendStepSummary = async (
+  kpis: RunKpis,
+  env: NodeJS.ProcessEnv = process.env
+): Promise<void> => {
+  const summaryPath = env.GITHUB_STEP_SUMMARY;
+  if (!summaryPath) {
+    return;
+  }
+
+  const rows = formatRunKpis(kpis).map(([label, value]) => `| ${label} | ${value} |`);
+  const markdown = [
+    '### Harness run KPIs',
+    '',
+    '| KPI | Value |',
+    '| --- | --- |',
+    ...rows,
+    '',
+    '',
+  ].join('\n');
+
+  try {
+    await fs.appendFile(summaryPath, markdown, 'utf-8');
+  } catch (error) {
+    reportLogger.debug('failed to write GitHub step summary: %s', error);
+  }
+};
+
+/**
+ * Adds entries recorded after a run's trace was written (session teardown) to
+ * that trace, or writes a new trace if there is none yet.
+ */
+export const appendToTraceFile = async (
+  entries: DiagnosticEntry[],
+  options: { projectRoot: string; runId: string }
+): Promise<string> => {
+  const filePath = path.join(
+    getDiagnosticsDir(options.projectRoot),
+    `${options.runId}.json`
+  );
+
+  let existing: TraceEvent[] = [];
+  try {
+    const parsed = JSON.parse(await fs.readFile(filePath, 'utf-8')) as {
+      traceEvents?: TraceEvent[];
+    };
+    existing = parsed.traceEvents ?? [];
+  } catch {
+    // No trace yet (or an unreadable one): start a fresh file.
+  }
+
+  await fs.mkdir(path.dirname(filePath), { recursive: true });
+  await fs.writeFile(
+    filePath,
+    JSON.stringify({ traceEvents: [...existing, ...entries.map(toTraceEvent)] }),
+    'utf-8'
+  );
+
+  return filePath;
+};
+
 const formatDuration = (ms: number): string => {
   if (ms >= 1000) {
     return `${(ms / 1000).toFixed(1)}s`;
@@ -82,6 +395,10 @@ export const printSummary = (
 
   const statsByLabel = new Map<string, LabelStats>();
   for (const entry of entries) {
+    // Gauges are samples, not timings; they are summarised as KPIs below.
+    if (entry.kind === 'gauge') {
+      continue;
+    }
     const existing = statsByLabel.get(entry.label);
     if (existing) {
       existing.count += 1;
@@ -121,4 +438,8 @@ export const printSummary = (
       );
     }
   }
+  log('  KPIs');
+  for (const [label, value] of formatRunKpis(summarizeRunKpis(entries))) {
+    log(`    ${label}: ${value}`);
+  }
 };

package/src/diagnostics/telemetry.ts

@@ -0,0 +1,186 @@
+import { randomUUID } from 'node:crypto';
+import os from 'node:os';
+import {
+  logger,
+  createDefaultProbes,
+  createLifecycleEmitter,
+  DEFAULT_SAMPLE_INTERVAL_MS,
+  startTelemetryProbes,
+  type Diagnostics,
+  type DiagnosticAttrs,
+  type LifecyclePhase,
+  type HostPressure,
+} from '@react-native-harness/tools';
+import {
+  resolveDiagnosticsOptions,
+  type Config as HarnessConfig,
+} from '@react-native-harness/config';
+import type { HarnessPlatformRunner } from '@react-native-harness/platforms';
+import { appendToTraceFile, createDiagnosticsAppender } from './report.js';
+
+const telemetryLogger = logger.child('diagnostics');
+
+export type SessionTelemetry = {
+  /** Identifies the session; names the incremental `.jsonl` log. */
+  readonly sessionId: string;
+  /** Session code emits phases here; probes listen. */
+  readonly lifecycle: ReturnType<typeof createLifecycleEmitter>;
+  /** Emits a lifecycle phase. */
+  phase(phase: LifecyclePhase, attrs?: DiagnosticAttrs): void;
+  /** Makes the platform runner available to the platform collector probe. */
+  attachPlatform(platform: HarnessPlatformRunner): void;
+  /** Records the run that is current, so teardown data can join its trace. */
+  trackRun(runId: string): void;
+  /**
+   * Host pressure observed by the host-resources probe so far: peak load,
+   * mean idle CPU, lowest free memory and memory swapped out since the first
+   * sample. Null
+   * when no host sample was taken (diagnostics or the host probe are off).
+   */
+  hostPressure(): HostPressure | null;
+  /** Resolves once lifecycle driven collection started so far has finished. */
+  settled(): Promise<void>;
+  /**
+   * Stops probes and the incremental log. Entries recorded since the last
+   * run's report was written (the end-of-run snapshot, teardown spans) are
+   * added to that run's trace. Safe to call more than once.
+   */
+  dispose(): Promise<void>;
+};
+
+/**
+ * The single composition point for run telemetry: it wires the incremental
+ * log and the probes to the one diagnostics sink. Session code only emits
+ * lifecycle phases; it never calls a probe. Everything is a no-op, with no
+ * timers, listeners or spawned processes, when diagnostics are disabled.
+ */
+export const startSessionTelemetry = ({
+  diagnostics,
+  config,
+  projectRoot,
+}: {
+  diagnostics: Diagnostics;
+  config: Pick<HarnessConfig, 'diagnostics'>;
+  projectRoot: string;
+}): SessionTelemetry => {
+  const sessionId = randomUUID();
+  const lifecycle = createLifecycleEmitter();
+  const abortController = new AbortController();
+  const options = resolveDiagnosticsOptions(config);
+
+  let platform: HarnessPlatformRunner | undefined;
+  let lastRunId: string | undefined;
+
+  const appender = createDiagnosticsAppender(diagnostics, {
+    projectRoot,
+    id: sessionId,
+  });
+
+  const run = startTelemetryProbes(
+    createDefaultProbes({
+      host: options.probes.host,
+      node: options.probes.node,
+      snapshots: options.probes.snapshots,
+      platform: options.probes.platform
+        ? { getCollector: () => platform?.createTelemetryCollector?.() }
+        : undefined,
+    }),
+    {
+      diagnostics,
+      signal: abortController.signal,
+      lifecycle,
+      sampleIntervalMs: options.sampleIntervalMs ?? DEFAULT_SAMPLE_INTERVAL_MS,
+    }
+  );
+
+  // Running aggregates rather than stored samples: a watch-mode session can
+  // live for hours, and only the extremes are needed.
+  let peakLoad1: number | null = null;
+  let idlePctSum = 0;
+  let idlePctSamples = 0;
+  let minHostFreeMb: number | null = null;
+  let firstSwapoutsMb: number | null = null;
+  let lastSwapoutsMb: number | null = null;
+  const unsubscribePressure = diagnostics.subscribe(
+    ({ kind, label, value }) => {
+      if (kind !== 'gauge' || typeof value !== 'number') {
+        return;
+      }
+      if (label === 'host.load1') {
+        peakLoad1 = Math.max(peakLoad1 ?? value, value);
+      } else if (label === 'host.cpu.idle_pct') {
+        idlePctSum += value;
+        idlePctSamples += 1;
+      } else if (label === 'host.mem.free') {
+        minHostFreeMb = Math.min(minHostFreeMb ?? value, value);
+      } else if (label === 'host.mem.swapouts') {
+        firstSwapoutsMb ??= value;
+        lastSwapoutsMb = value;
+      }
+    },
+    { replay: true }
+  );
+
+  const hostPressure = (): HostPressure | null =>
+    peakLoad1 === null && minHostFreeMb === null && lastSwapoutsMb === null
+      ? null
+      : {
+          peakLoad1,
+          meanIdlePct: idlePctSamples > 0 ? idlePctSum / idlePctSamples : null,
+          minHostFreeMb,
+          swapoutsDeltaMb:
+            firstSwapoutsMb !== null && lastSwapoutsMb !== null
+              ? lastSwapoutsMb - firstSwapoutsMb
+              : null,
+          cpuCount: os.availableParallelism(),
+        };
+
+  const phase = (name: LifecyclePhase, attrs?: DiagnosticAttrs) => {
+    if (!diagnostics.enabled) {
+      return;
+    }
+    lifecycle.emit({ type: 'phase', phase: name, attrs });
+  };
+
+  let disposed: Promise<void> | null = null;
+
+  return {
+    sessionId,
+    lifecycle,
+    phase,
+    attachPlatform: (runner) => {
+      platform = runner;
+    },
+    trackRun: (runId) => {
+      lastRunId = runId;
+    },
+    hostPressure,
+    settled: run.settled,
+    dispose: () => {
+      disposed ??= (async () => {
+        unsubscribePressure();
+        try {
+          abortController.abort();
+          await run.dispose();
+          if (diagnostics.enabled && lastRunId) {
+            const leftover = diagnostics.flush();
+            if (leftover.length > 0) {
+              await appendToTraceFile(leftover, {
+                projectRoot,
+                runId: lastRunId,
+              });
+            }
+          }
+        } catch (error) {
+          telemetryLogger.debug(
+            'failed to finalize diagnostics trace: %s',
+            error
+          );
+        } finally {
+          await appender.dispose();
+        }
+      })();
+      return disposed;
+    },
+  };
+};

package/src/errors.ts

@@ -1,4 +1,10 @@
-import { formatPath, HarnessError } from '@react-native-harness/tools';
+import {
+  formatHostPressure,
+  formatPath,
+  HarnessError,
+  isHostStarved,
+  type HostPressure,
+} from '@react-native-harness/tools';
 import type { AppCrashDetails } from '@react-native-harness/platforms';
 export {
   StartupStallError,
@@ -27,13 +33,34 @@ export class InitializationTimeoutError extends HarnessError {
   }
 }
 
+const buildPlatformReadyTimeoutMessage = (
+  timeout: number,
+  hostPressure: HostPressure | null
+): string => {
+  const prefix = `The platform did not become ready within ${timeout}ms.`;
+
+  if (hostPressure && isHostStarved(hostPressure)) {
+    return `${prefix} The host was starved for resources during startup (${formatHostPressure(hostPressure)}). A larger "platformReadyTimeout" is unlikely to help; reduce other load on the host or use a larger machine.`;
+  }
+
+  const suggestion = `Increase "platformReadyTimeout" if your device, simulator, or emulator needs more time to start.`;
+  return hostPressure
+    ? `${prefix} ${suggestion} Host during startup: ${formatHostPressure(hostPressure)}.`
+    : `${prefix} ${suggestion}`;
+};
+
 export class PlatformReadyTimeoutError extends HarnessError {
-  constructor(public readonly timeout: number) {
-    super(
-      `The platform did not become ready within ${timeout}ms. Increase "platformReadyTimeout" if your device, simulator, or emulator needs more time to start.`
-    );
+  constructor(
+    public readonly timeout: number,
+    public readonly hostPressure: HostPressure | null = null
+  ) {
+    super(buildPlatformReadyTimeoutMessage(timeout, hostPressure));
     this.name = 'PlatformReadyTimeoutError';
   }
+
+  get hostStarved(): boolean {
+    return this.hostPressure !== null && isHostStarved(this.hostPressure);
+  }
 }
 
 export class MetroPortRangeExhaustedError extends HarnessError {
@@ -179,4 +206,14 @@ export class RuntimeDisconnectError extends HarnessError {
   }
 }
 
+// The runtime sends this name through the runTests RPC when its native host
+// surface is replaced while a test file is still running.
+export class HostSurfaceReplacedError extends HarnessError {
+  constructor(public readonly testFilePath: string, cause: Error) {
+    super(cause.message, { cause });
+    this.name = 'HostSurfaceReplacedError';
+    this.stack = cause.stack;
+  }
+}
+
 export type HarnessRuntimeFailure = NativeCrashError | RuntimeDisconnectError;