@react-native-harness/jest 1.4.0-rc.1 → 1.4.0

package/src/diagnostics/report.ts

@@ -0,0 +1,124 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import type { DiagnosticEntry } from '@react-native-harness/tools';
+import { getHarnessRootPath } from '@react-native-harness/tools';
+
+type TraceEvent = {
+  name: string;
+  ph: 'X' | 'i';
+  ts: number;
+  dur?: number;
+  s?: 'g';
+  pid: number;
+  tid: number;
+  args?: Record<string, unknown>;
+};
+
+const TRACE_PID = 1;
+const TRACE_TID = 1;
+
+const toTraceEvent = (entry: DiagnosticEntry): TraceEvent => {
+  if (entry.kind === 'mark') {
+    return {
+      name: entry.label,
+      ph: 'i',
+      s: 'g',
+      ts: entry.startTime * 1000,
+      pid: TRACE_PID,
+      tid: TRACE_TID,
+      args: entry.attrs,
+    };
+  }
+
+  return {
+    name: entry.label,
+    ph: 'X',
+    ts: entry.startTime * 1000,
+    dur: entry.duration * 1000,
+    pid: TRACE_PID,
+    tid: TRACE_TID,
+    args: entry.attrs,
+  };
+};
+
+export const writeTraceFile = async (
+  entries: DiagnosticEntry[],
+  options: { projectRoot: string; runId: string }
+): Promise<string> => {
+  const dir = path.join(getHarnessRootPath(options.projectRoot), 'diagnostics');
+  await fs.mkdir(dir, { recursive: true });
+
+  const filePath = path.join(dir, `${options.runId}.json`);
+  const trace = { traceEvents: entries.map(toTraceEvent) };
+
+  await fs.writeFile(filePath, JSON.stringify(trace), 'utf-8');
+
+  return filePath;
+};
+
+const formatDuration = (ms: number): string => {
+  if (ms >= 1000) {
+    return `${(ms / 1000).toFixed(1)}s`;
+  }
+  return `${Math.round(ms)}ms`;
+};
+
+const getNamespace = (label: string): string => label.split('.')[0] ?? label;
+
+type LabelStats = {
+  label: string;
+  count: number;
+  totalMs: number;
+  maxMs: number;
+};
+
+export const printSummary = (
+  entries: DiagnosticEntry[],
+  log: (line: string) => void
+): void => {
+  if (entries.length === 0) {
+    return;
+  }
+
+  const statsByLabel = new Map<string, LabelStats>();
+  for (const entry of entries) {
+    const existing = statsByLabel.get(entry.label);
+    if (existing) {
+      existing.count += 1;
+      existing.totalMs += entry.duration;
+      existing.maxMs = Math.max(existing.maxMs, entry.duration);
+    } else {
+      statsByLabel.set(entry.label, {
+        label: entry.label,
+        count: 1,
+        totalMs: entry.duration,
+        maxMs: entry.duration,
+      });
+    }
+  }
+
+  const byNamespace = new Map<string, LabelStats[]>();
+  for (const stats of statsByLabel.values()) {
+    const namespace = getNamespace(stats.label);
+    const group = byNamespace.get(namespace);
+    if (group) {
+      group.push(stats);
+    } else {
+      byNamespace.set(namespace, [stats]);
+    }
+  }
+
+  log('Diagnostics summary:');
+  const sortedNamespaces = [...byNamespace.entries()].sort(([a], [b]) =>
+    a.localeCompare(b)
+  );
+  for (const [namespace, group] of sortedNamespaces) {
+    log(`  ${namespace}`);
+    group.sort((a, b) => b.totalMs - a.totalMs);
+    for (const stats of group) {
+      log(
+        `    ${stats.label}: count=${stats.count} total=${formatDuration(stats.totalMs)} max=${formatDuration(stats.maxMs)}`
+      );
+    }
+  }
+};

package/src/environment-reset.ts

@@ -0,0 +1,138 @@
+import { getTimeoutSignal } from '@react-native-harness/tools';
+import type { Config as HarnessConfig } from '@react-native-harness/config';
+
+const ignorePromiseRejection = () => undefined;
+
+export type ResetStrategyKind = 'process' | 'runtime';
+
+export type EnvironmentResetStrategy = {
+  readonly kind: ResetStrategyKind;
+  // Post-condition: bridge connected, runtime ready, crash-monitor state consistent.
+  reset(ctx: { testFilePath: string }): Promise<void>;
+};
+
+/**
+ * Normalizes the `resetEnvironmentBetweenTestFiles` config value into a
+ * strategy kind, or `null` when no reset should happen between test files
+ * (i.e. the value is `false`).
+ */
+export const resolveResetStrategyKind = (
+  value: HarnessConfig['resetEnvironmentBetweenTestFiles']
+): ResetStrategyKind | null => {
+  if (value === false) {
+    return null;
+  }
+  if (value === true) {
+    return 'process';
+  }
+  return value;
+};
+
+export type ProcessResetStrategyDeps = {
+  // Shared with restartApp; killing and relaunching the app process.
+  restart: (testFilePath: string) => Promise<void>;
+};
+
+/**
+ * The 'process' strategy: kill and cold-restart the app process. This is
+ * the same implementation `restartApp` uses for crash/timeout recovery.
+ */
+export const createProcessResetStrategy = (
+  deps: ProcessResetStrategyDeps
+): EnvironmentResetStrategy => ({
+  kind: 'process',
+  reset: ({ testFilePath }) => deps.restart(testFilePath),
+});
+
+export type RuntimeResetConnection = {
+  resetEnvironment: () => Promise<void>;
+};
+
+export type RuntimeResetStrategyDeps = {
+  // Returns the active app connection, or null if there isn't one.
+  getConnection: () => RuntimeResetConnection | null;
+  // Opens a window that suppresses crash-monitor disconnect handling;
+  // returns a closer to end the window.
+  expectDisconnect: () => () => void;
+  // Resolves the next time the bridge reports a 'connected' event, or
+  // rejects when `signal` aborts.
+  waitForReconnect: (signal: AbortSignal) => Promise<void>;
+  // Identifies RPC rejections caused by the bridge disconnecting. Inside the
+  // expected-disconnect window these mean the reload tore the runtime down
+  // before the RPC ack flushed — the reload is in flight, not failed.
+  isDisconnectError: (error: unknown) => boolean;
+  // Reconnect timeout in ms (the configured bundleStartTimeout).
+  timeoutMs: number;
+};
+
+/**
+ * The 'runtime' strategy: reload the JS runtime in place (DevSettings.reload()
+ * on native, window.location.reload() on web) via the resetEnvironment RPC,
+ * and wait for the bridge to reconnect. The app process stays alive
+ * throughout, so the crash monitor's expected-disconnect window is used to
+ * avoid misclassifying the intentional disconnect as a runtime failure.
+ */
+export const createRuntimeResetStrategy = (
+  deps: RuntimeResetStrategyDeps
+): EnvironmentResetStrategy => ({
+  kind: 'runtime',
+  reset: async () => {
+    const connection = deps.getConnection();
+    if (!connection) {
+      throw new Error('No active app connection to reset the runtime on');
+    }
+
+    const closeWindow = deps.expectDisconnect();
+    try {
+      // Start listening for the reconnect before firing the RPC, so a fast
+      // reload can't reconnect before we're watching for it.
+      const reconnected = deps.waitForReconnect(getTimeoutSignal(deps.timeoutMs));
+      // Observe the rejection even when the RPC throws first, so the timeout
+      // abort firing later can't become an unhandled rejection. (.catch()
+      // returns a new promise; the original is what gets awaited below.)
+      reconnected.catch(ignorePromiseRejection);
+
+      try {
+        await connection.resetEnvironment();
+      } catch (error) {
+        // The reload can tear the runtime down before the RPC ack flushes.
+        // A disconnect-flavored rejection means the reload is in flight, so
+        // keep waiting for the reconnect; anything else is a real failure.
+        if (!deps.isDisconnectError(error)) {
+          throw error;
+        }
+      }
+
+      await reconnected;
+    } finally {
+      closeWindow();
+    }
+  },
+});
+
+export type EscalationOptions = {
+  // Invoked when the primary strategy fails and the fallback is about to run.
+  onEscalate?: (error: unknown) => void;
+};
+
+/**
+ * Wraps a primary strategy so that any failure (no active connection, RPC
+ * error, reconnect timeout, ...) falls back to running `fallback` instead of
+ * propagating the error. The composite reports the primary's `kind`, since
+ * that's the strategy that was requested.
+ */
+export const withEscalation = (
+  primary: EnvironmentResetStrategy,
+  fallback: EnvironmentResetStrategy,
+  options: EscalationOptions = {}
+): EnvironmentResetStrategy => ({
+  kind: primary.kind,
+  reset: async (ctx) => {
+    try {
+      await primary.reset(ctx);
+    } catch (error) {
+      options.onEscalate?.(error);
+      await fallback.reset(ctx);
+    }
+  },
+});

package/src/execute-run.ts

@@ -8,6 +8,7 @@ import { randomUUID } from 'node:crypto';
 import path from 'node:path';
 import { type HarnessSession, type HarnessRunState } from './harness-session.js';
 import { runHarnessTestFile } from './run.js';
+import { resolveResetStrategyKind } from './environment-reset.js';
 import {
   NativeCrashError,
   RuntimeDisconnectError,
@@ -22,11 +23,17 @@ import type {
   TestRunnerEvents,
   TestRunnerTestFinishedEvent,
   TestRunnerTestStartedEvent,
+  TestSuiteResult,
 } from '@react-native-harness/bridge';
 import {
   createPlatformSkippedTestResult,
   shouldRunHarnessTestFile,
 } from './test-file-platform-filter.js';
+import { formatHarnessErrorMessage } from './format-harness-error.js';
+import { logger } from '@react-native-harness/tools';
+import { printSummary, writeTraceFile } from './diagnostics/index.js';
+
+const diagnosticsLogger = logger.child('diagnostics');
 
 type EmitTestEvent = <Name extends keyof TestEvents>(
   eventName: Name,
@@ -86,8 +93,10 @@ const emitHarnessTestFinished = async (
   emitEvent: EmitTestEvent,
   event: TestRunnerTestFinishedEvent,
 ): Promise<void> => {
-  const failureMessage = event.error?.message;
   const codeFrame = event.error?.codeFrame;
+  const failureMessage = formatHarnessErrorMessage(event.error, {
+    testStartedAt: event.startedAt,
+  });
   const location = codeFrame?.location
     ? { column: codeFrame.location.column, line: codeFrame.location.row }
     : null;
@@ -95,9 +104,7 @@ const emitHarnessTestFinished = async (
     ancestorTitles: event.ancestorTitles,
     duration: event.duration,
     failureDetails: [],
-    failureMessages: failureMessage
-      ? [`${failureMessage}${codeFrame ? `\n\n${codeFrame.content}` : ''}`]
-      : [],
+    failureMessages: failureMessage ? [failureMessage] : [],
     fullName: event.fullName,
     location,
     numPassingAsserts: event.status === 'passed' ? 1 : 0,
@@ -114,6 +121,18 @@ const isHarnessCaseEvent = (
 ): event is TestRunnerTestStartedEvent | TestRunnerTestFinishedEvent =>
   event.type === 'test-started' || event.type === 'test-finished';
 
+const TIMEOUT_ERROR_NAMES = new Set([
+  'SuiteHookTimeoutError',
+  'TestCaseTimeoutError',
+]);
+
+const hasRuntimeTimeout = (result: TestSuiteResult): boolean =>
+  (result.error ? TIMEOUT_ERROR_NAMES.has(result.error.name) : false) ||
+  result.tests.some((test) =>
+    test.error ? TIMEOUT_ERROR_NAMES.has(test.error.name) : false,
+  ) ||
+  result.suites.some(hasRuntimeTimeout);
+
 export const executeRun = async (
   session: HarnessSession,
   tests: Array<Test>,
@@ -122,6 +141,8 @@ export const executeRun = async (
   globalConfig: Config.GlobalConfig,
 ): Promise<void> => {
   const runId = randomUUID();
+  const diagnostics = session.diagnostics.child({ runId });
+  const runSpan = diagnostics.start('run.total', { runId });
   const startTime = Date.now();
   const watchMode = globalConfig.watch || globalConfig.watchAll;
   const rootDir = globalConfig.rootDir ?? process.cwd();
@@ -145,6 +166,7 @@ export const executeRun = async (
       testFiles,
       watchMode,
       coverageEnabled: globalConfig.collectCoverage,
+      completed: false,
       summary,
       status: summary.failed > 0 ? 'failed' : 'passed',
       ...overrides,
@@ -168,12 +190,15 @@ export const executeRun = async (
     host: session.config.host?.trim() || undefined,
   });
 
-  const shouldResetEnv = session.config.resetEnvironmentBetweenTestFiles;
+  const resetStrategyKind = resolveResetStrategyKind(
+    session.config.resetEnvironmentBetweenTestFiles,
+  );
   const platformId = session.context.platform.platformId;
   const knownPlatformIds = new Set(
     session.config.runners.map((runner) => runner.platformId),
   );
   let isFirstTest = true;
+  let shouldRestartAfterTimeout = false;
   let runError: unknown;
 
   try {
@@ -200,22 +225,86 @@ export const executeRun = async (
       };
 
       await session.callHook('test-file:started', { runId, file: relativeTestPath });
+      const fileSpan = diagnostics.start('run.file', { file: relativeTestPath });
+      let fileStatus: 'passed' | 'failed' | 'skipped' | 'todo' = 'failed';
+
+      try {
+        if (
+          !shouldRunHarnessTestFile(test.path, platformId, knownPlatformIds)
+        ) {
+          try {
+            await emitEvent('test-file-start', test);
+            const skippedResult = createPlatformSkippedTestResult(test.path);
+            applyJestResultToSummary(summary, skippedResult);
+            updateRunState();
+            fileStatus = 'skipped';
+            await emitTestFileFinished({
+              status: 'skipped',
+              duration: Date.now() - fileStartedAt,
+              result: null,
+            });
+            await emitEvent('test-file-success', test, skippedResult);
+          } catch (err) {
+            fileStatus = 'failed';
+            if (!emittedTestFileFinished) {
+              await emitTestFileFinished({
+                status: 'failed',
+                duration: Date.now() - fileStartedAt,
+                result: null,
+              });
+            }
+            updateRunState({ error: err });
+            await emitEvent('test-file-failure', test, buildTestFailure(err));
+          }
+          continue;
+        }
 
-      if (
-        !shouldRunHarnessTestFile(test.path, platformId, knownPlatformIds)
-      ) {
         try {
+          if (shouldRestartAfterTimeout) {
+            // Recovery from a runtime timeout always uses the (stronger)
+            // process restart, never the configured reset strategy.
+            await session.restartApp(test.path);
+            shouldRestartAfterTimeout = false;
+          } else if (resetStrategyKind !== null && !isFirstTest) {
+            await session.resetEnvironment(test.path);
+          }
+          isFirstTest = false;
+
+          session.flushClientLogs();
           await emitEvent('test-file-start', test);
-          const skippedResult = createPlatformSkippedTestResult(test.path);
-          applyJestResultToSummary(summary, skippedResult);
+          await session.ensureAppReady(test.path);
+
+          // Crash detection is handled inside session.runTestFile; NativeCrashError
+          // propagates here if a crash wins the race.
+          const result = await runHarnessTestFile({
+            testPath: test.path,
+            session,
+            globalConfig,
+            projectConfig: test.context.config,
+          });
+
+          applyJestResultToSummary(summary, result.jestResult);
+          const clientLogs = session.flushClientLogs();
+          if (clientLogs.length > 0) {
+            result.jestResult.console = clientLogs;
+          }
           updateRunState();
+          fileStatus = result.harnessResult.status;
           await emitTestFileFinished({
-            status: 'skipped',
-            duration: Date.now() - fileStartedAt,
-            result: null,
+            status: result.harnessResult.status,
+            duration: result.duration,
+            result: result.harnessResult,
           });
-          await emitEvent('test-file-success', test, skippedResult);
+          const didRuntimeTimeout = hasRuntimeTimeout(result.harnessResult);
+          shouldRestartAfterTimeout = didRuntimeTimeout;
+          await caseEventChain;
+          await emitEvent('test-file-success', test, result.jestResult);
+          if (didRuntimeTimeout) {
+            await session.restartApp(test.path);
+            shouldRestartAfterTimeout = false;
+          }
         } catch (err) {
+          fileStatus = 'failed';
           if (!emittedTestFileFinished) {
             await emitTestFileFinished({
               status: 'failed',
@@ -223,72 +312,29 @@ export const executeRun = async (
               result: null,
             });
           }
-          updateRunState({ error: err });
-          await emitEvent('test-file-failure', test, buildTestFailure(err));
-        }
-        continue;
-      }
-
-      try {
-        if (shouldResetEnv && !isFirstTest) {
-          await session.restartApp(test.path);
-        }
-        isFirstTest = false;
-
-        session.flushClientLogs();
-        await emitEvent('test-file-start', test);
-        await session.ensureAppReady(test.path);
-
-        // Crash detection is handled inside session.runTestFile; NativeCrashError
-        // propagates here if a crash wins the race.
-        const result = await runHarnessTestFile({
-          testPath: test.path,
-          session,
-          globalConfig,
-          projectConfig: test.context.config,
-        });
 
-        applyJestResultToSummary(summary, result.jestResult);
-        const clientLogs = session.flushClientLogs();
-        if (clientLogs.length > 0) {
-          result.jestResult.console = clientLogs;
-        }
-        updateRunState();
-        await emitTestFileFinished({
-          status: result.harnessResult.status,
-          duration: result.duration,
-          result: result.harnessResult,
-        });
-        await caseEventChain;
-        await emitEvent('test-file-success', test, result.jestResult);
-      } catch (err) {
-        if (!emittedTestFileFinished) {
-          await emitTestFileFinished({
-            status: 'failed',
-            duration: Date.now() - fileStartedAt,
-            result: null,
-          });
-        }
+          const isRuntimeFailure =
+            err instanceof NativeCrashError ||
+            err instanceof RuntimeDisconnectError ||
+            err instanceof StartupStallError ||
+            err instanceof AppBridgeDisconnectedError ||
+            err instanceof DeviceNotRespondingError;
 
-        const isRuntimeFailure =
-          err instanceof NativeCrashError ||
-          err instanceof RuntimeDisconnectError ||
-          err instanceof StartupStallError ||
-          err instanceof AppBridgeDisconnectedError ||
-          err instanceof DeviceNotRespondingError;
+          if (isRuntimeFailure) {
+            summary.failed += 1;
+            updateRunState();
+          }
 
-        if (isRuntimeFailure) {
-          summary.failed += 1;
-          updateRunState();
-        }
+          if (err instanceof NativeCrashError || err instanceof RuntimeDisconnectError) {
+            session.resetCrashState();
+          }
 
-        if (err instanceof NativeCrashError || err instanceof RuntimeDisconnectError) {
-          session.resetCrashState();
+          updateRunState({ error: isRuntimeFailure ? undefined : err });
+          await caseEventChain;
+          await emitEvent('test-file-failure', test, buildTestFailure(err));
         }
-
-        updateRunState({ error: isRuntimeFailure ? undefined : err });
-        await caseEventChain;
-        await emitEvent('test-file-failure', test, buildTestFailure(err));
+      } finally {
+        fileSpan.end({ status: fileStatus });
       }
     }
   } catch (err) {
@@ -296,18 +342,31 @@ export const executeRun = async (
     if (!(err instanceof CancelRun)) throw err;
   } finally {
     const runState = updateRunState(
-      runError != null ? { error: runError, status: 'failed' } : {},
+      runError != null ? { completed: true, error: runError, status: 'failed' } : { completed: true },
     );
+    const finalStatus = runState.status ?? (runError != null ? 'failed' : 'passed');
     await session.callHook('run:finished', {
       runId,
       startTime,
       duration: Date.now() - startTime,
       testFiles,
       summary,
-      status: runState.status ?? (runError != null ? 'failed' : 'passed'),
+      status: finalStatus,
       ...(runError != null ? { error: runError } : {}),
     });
     await caseEventChain;
     unsubscribe();
+    runSpan.end({ runId, status: finalStatus });
+
+    if (session.diagnostics.enabled) {
+      const entries = session.diagnostics.flush();
+      try {
+        printSummary(entries, (line) => diagnosticsLogger.info(line));
+        const tracePath = await writeTraceFile(entries, { projectRoot: rootDir, runId });
+        diagnosticsLogger.info(`wrote diagnostics trace to ${tracePath}`);
+      } catch (reportError) {
+        diagnosticsLogger.warn('failed to write diagnostics report: %s', reportError);
+      }
+    }
   }
 };

package/src/format-harness-error.ts

@@ -0,0 +1,73 @@
+import type { SerializedError } from '@react-native-harness/bridge';
+
+const formatPendingPromiseStack = (stack: string): string =>
+  stack
+    .split('\n')
+    .map((line) => `  ${line}`)
+    .join('\n');
+
+const formatPendingPromises = (
+  pendingPromises: NonNullable<
+    NonNullable<SerializedError['diagnostics']>['pendingPromises']
+  >,
+  testStartedAt?: number,
+): string | null => {
+  if (pendingPromises.total === 0) {
+    return null;
+  }
+
+  const lines = [`Pending promises at timeout: ${pendingPromises.total}`];
+
+  if (pendingPromises.items.length < pendingPromises.total) {
+    lines.push(
+      `Showing ${pendingPromises.items.length} of ${pendingPromises.total} pending promises.`,
+    );
+  }
+
+  for (const promise of pendingPromises.items) {
+    const age =
+      testStartedAt !== undefined
+        ? `, created ${Math.max(0, promise.createdAt - testStartedAt)}ms after test start`
+        : '';
+
+    lines.push('');
+    lines.push(`Promise #${promise.id}${age}:`);
+
+    if (promise.stack) {
+      lines.push(formatPendingPromiseStack(promise.stack));
+    } else {
+      lines.push('  <stack unavailable>');
+    }
+  }
+
+  return lines.join('\n');
+};
+
+export const formatHarnessErrorMessage = (
+  error: SerializedError | undefined,
+  options: {
+    testStartedAt?: number;
+  } = {},
+): string | undefined => {
+  if (!error) {
+    return undefined;
+  }
+
+  const parts = [error.message];
+  const pendingPromiseDetails = error.diagnostics?.pendingPromises
+    ? formatPendingPromises(
+        error.diagnostics.pendingPromises,
+        options.testStartedAt,
+      )
+    : null;
+
+  if (pendingPromiseDetails) {
+    parts.push(pendingPromiseDetails);
+  }
+
+  if (error.codeFrame) {
+    parts.push(error.codeFrame.content);
+  }
+
+  return parts.join('\n\n');
+};