@remix-run/test 0.2.0 → 0.4.0

package/src/lib/reporters/files.ts

@@ -6,10 +6,17 @@ import type { Counts, TestResult, TestResults } from './results.ts'
 
 export class FilesReporter implements Reporter {
   #failures: { suiteName: string; name: string; error: TestResult['error'] }[] = []
+  #files = new Set<string>()
+  #suites = new Set<string>()
 
   onSectionStart(_label: string) {}
 
   onResult(results: TestResults, env?: string) {
+    for (let test of results.tests) {
+      if (test.filePath) this.#files.add(test.filePath)
+      if (test.suiteName) this.#suites.add(test.suiteName)
+    }
+
     let filePath = results.tests[0]?.filePath
     let fileName = filePath ? path.relative(process.cwd(), filePath) : '(unknown)'
     let envLabel = env ? ` ${colors.dim(`[${env}]`)}` : ''
@@ -66,6 +73,8 @@ export class FilesReporter implements Reporter {
     let { passed, failed, skipped, todo } = counts
     let info = colors.cyan('ℹ')
     console.log()
+    console.log(`${info} files ${this.#files.size}`)
+    console.log(`${info} suites ${this.#suites.size}`)
     console.log(`${info} tests ${passed + failed + skipped + todo}`)
     console.log(`${info} pass ${passed}`)
     console.log(`${info} fail ${failed}`)

package/src/lib/reporters/results.ts

@@ -1,4 +1,4 @@
-import type { V8CoverageEntry } from '../coverage'
+import type { V8CoverageEntry } from '../coverage.ts'
 
 export interface TestResult {
   name: string

package/src/lib/reporters/spec.ts

@@ -5,12 +5,19 @@ import type { Counts, TestResult, TestResults } from './results.ts'
 
 export class SpecReporter implements Reporter {
   #failures: { suiteName: string; name: string; error: TestResult['error'] }[] = []
+  #files = new Set<string>()
+  #suites = new Set<string>()
 
   onSectionStart(label: string) {
     console.log(label)
   }
 
   onResult(results: TestResults, env?: string) {
+    for (let test of results.tests) {
+      if (test.filePath) this.#files.add(test.filePath)
+      if (test.suiteName) this.#suites.add(test.suiteName)
+    }
+
     let suiteMap = new Map<string, TestResult[]>()
     for (let test of results.tests) {
       let suite = test.suiteName || 'Global'
@@ -163,6 +170,8 @@ export class SpecReporter implements Reporter {
     let { passed, failed, skipped, todo } = counts
     let info = colors.cyan('ℹ')
     console.log()
+    console.log(`${info} files ${this.#files.size}`)
+    console.log(`${info} suites ${this.#suites.size}`)
     console.log(`${info} tests ${passed + failed + skipped + todo}`)
     console.log(`${info} pass ${passed}`)
     console.log(`${info} fail ${failed}`)

package/src/lib/reporters/tap.ts

@@ -5,6 +5,8 @@ import type { Counts, TestResults } from './results.ts'
 export class TapReporter implements Reporter {
   #counter = 0
   #total = 0
+  #files = new Set<string>()
+  #suites = new Set<string>()
 
   onSectionStart(_label: string) {}
 
@@ -15,6 +17,11 @@ export class TapReporter implements Reporter {
 
     let envComment = env ? ` # ${env}` : ''
 
+    for (let test of results.tests) {
+      if (test.filePath) this.#files.add(test.filePath)
+      if (test.suiteName) this.#suites.add(test.suiteName)
+    }
+
     for (let test of results.tests) {
       this.#counter++
       this.#total++
@@ -48,6 +55,8 @@ export class TapReporter implements Reporter {
   onSummary(counts: Counts, durationMs: number) {
     let { passed, failed, skipped, todo } = counts
     console.log(`1..${this.#total}`)
+    console.log(`# files ${this.#files.size}`)
+    console.log(`# suites ${this.#suites.size}`)
     console.log(`# tests ${passed + failed + skipped + todo}`)
     console.log(`# pass ${passed}`)
     console.log(`# fail ${failed}`)

package/src/lib/runner-browser.ts

@@ -16,6 +16,8 @@ import {
 import type { Reporter } from './reporters/index.ts'
 import type { TestResults } from './reporters/results.ts'
 
+const BROWSER_TEST_FILE_TIMEOUT_MS = 90_000
+
 // The harness reports each test result with `filePath` set to the
 // `/scripts/<rel>` URL the iframe loaded. Reporters expect a real filesystem
 // path so they can compute `path.relative(cwd, ...)` cleanly; otherwise they
@@ -61,6 +63,10 @@ export async function runBrowserTests(options: TestRunOptions): Promise<{
       getPlaywrightLaunchOptions(options.playwrightUseOpts),
     )
     page = await browser.newPage(getPlaywrightPageOptions(options.playwrightUseOpts))
+    // Cap individual browser operations, then separately watch for per-file
+    // progress so large suites can run longer than this without hiding hangs.
+    page.setDefaultTimeout(BROWSER_TEST_FILE_TIMEOUT_MS)
+    page.setDefaultNavigationTimeout(BROWSER_TEST_FILE_TIMEOUT_MS)
 
     if (options.console) {
       page.on('console', (msg) => console.log(`${colors.dim('[browser console]')} ${msg.text()}`))
@@ -78,6 +84,32 @@ export async function runBrowserTests(options: TestRunOptions): Promise<{
     let totalSkipped = 0
     let totalTodo = 0
     let rootDir = getBrowserTestRootDir()
+    let completedFiles = 0
+    let totalFiles = options.testFiles?.length ?? 0
+    let progressTimeoutId: ReturnType<typeof setTimeout> | undefined
+    let rejectProgressTimeout: (error: Error) => void = () => {}
+    let progressTimeoutPromise = new Promise<never>((_, reject) => {
+      rejectProgressTimeout = reject
+    })
+
+    function clearProgressTimeout() {
+      if (progressTimeoutId !== undefined) {
+        clearTimeout(progressTimeoutId)
+        progressTimeoutId = undefined
+      }
+    }
+
+    function resetProgressTimeout() {
+      clearProgressTimeout()
+      progressTimeoutId = setTimeout(() => {
+        let progress = totalFiles > 0 ? ` (${completedFiles}/${totalFiles} files completed)` : ''
+        rejectProgressTimeout(
+          new Error(
+            `Timed out waiting ${BROWSER_TEST_FILE_TIMEOUT_MS}ms for browser test progress${progress}`,
+          ),
+        )
+      }, BROWSER_TEST_FILE_TIMEOUT_MS)
+    }
 
     await page.route('**/file-results', async (route) => {
       let results = route.request().postDataJSON() as TestResults
@@ -89,6 +121,8 @@ export async function runBrowserTests(options: TestRunOptions): Promise<{
       totalFailed += results.failed
       totalSkipped += results.skipped
       totalTodo += results.todo
+      completedFiles++
+      resetProgressTimeout()
       await route.fulfill({ status: 200 })
     })
 
@@ -111,9 +145,19 @@ export async function runBrowserTests(options: TestRunOptions): Promise<{
 
     // Prevent unhandled rejection if we fail before setting up the listener
     errorPromise.catch(() => {})
+    progressTimeoutPromise.catch(() => {})
 
-    await page.goto(options.baseUrl)
-    await Promise.race([page.waitForFunction('window.__testsDone'), errorPromise])
+    resetProgressTimeout()
+    try {
+      await page.goto(options.baseUrl)
+      await Promise.race([
+        page.waitForFunction('window.__testsDone', undefined, { timeout: 0 }),
+        errorPromise,
+        progressTimeoutPromise,
+      ])
+    } finally {
+      clearProgressTimeout()
+    }
 
     if (coverageEnabled) {
       let entries = (await page.coverage.stopJSCoverage()) as unknown as V8CoverageEntry[]

package/src/lib/runner.ts

@@ -1,8 +1,9 @@
+import { fork, type ChildProcess } from 'node:child_process'
 import * as fsp from 'node:fs/promises'
 import * as path from 'node:path'
-import { pathToFileURL } from 'node:url'
+import { fileURLToPath, pathToFileURL } from 'node:url'
 import { Worker } from 'node:worker_threads'
-import { IS_RUNNING_FROM_SRC } from './config.ts'
+import { IS_RUNNING_FROM_SRC, type RemixTestPool } from './config.ts'
 import {
   collectCoverageMapFromPlaywright,
   collectServerCoverageMap,
@@ -19,6 +20,22 @@ import type { Counts, TestResults } from './reporters/results.ts'
 const ext = IS_RUNNING_FROM_SRC ? '.ts' : '.js'
 const workerUrl = new URL(`./worker${ext}`, import.meta.url)
 const workerE2EUrl = new URL(`./worker-e2e${ext}`, import.meta.url)
+const workerProcessUrl = new URL(`./worker-process${ext}`, import.meta.url)
+const DEFAULT_WORKER_SHUTDOWN_TIMEOUT_MS = 10_000
+
+interface WorkerRun {
+  finished: Promise<void>
+  exited: Promise<number | null>
+  terminate(): Promise<boolean>
+}
+
+interface RunFileOptions {
+  cwd?: string
+  coverage?: CoverageConfig
+  open?: boolean
+  playwrightUseOpts?: PlaywrightUseOpts
+  pool?: RemixTestPool
+}
 
 export async function runServerTests(
   files: string[],
@@ -31,12 +48,15 @@ export async function runServerTests(
     playwrightUseOpts?: PlaywrightUseOpts
     projectName?: string
     coverage?: CoverageConfig
+    workerShutdownTimeoutMs?: number
+    pool?: RemixTestPool
   } = {},
 ): Promise<Counts & { coverageMap: CoverageMap | null }> {
   let counts: Counts = { passed: 0, failed: 0, skipped: 0, todo: 0 }
   let coverageMap: CoverageMap | null = null
   let cwd = options.cwd ?? process.cwd()
   let envLabel = options.projectName ? `${type}:${options.projectName}` : type
+  let pool = options.pool ?? 'forks'
 
   function accumulate(results: TestResults, file: string) {
     reporter.onResult(
@@ -56,7 +76,7 @@ export async function runServerTests(
       files,
       concurrency,
       (file) =>
-        runFileInWorker(
+        runFileInPool(
           file,
           type,
           (results) => {
@@ -67,10 +87,13 @@ export async function runServerTests(
           },
           {
             ...options,
+            pool,
             playwrightUseOpts: options.playwrightUseOpts,
           },
         ),
       () => counts.failed++,
+      !options.open,
+      options.workerShutdownTimeoutMs ?? DEFAULT_WORKER_SHUTDOWN_TIMEOUT_MS,
     )
 
     if (options.coverage && allBrowserCoverageEntries.length > 0) {
@@ -92,8 +115,14 @@ export async function runServerTests(
     await runInConcurrentWorkers(
       files,
       concurrency,
-      (file) => runFileInWorker(file, type, (results) => accumulate(results, file), options),
+      (file) =>
+        runFileInPool(file, type, (results) => accumulate(results, file), {
+          ...options,
+          pool,
+        }),
       () => counts.failed++,
+      true,
+      options.workerShutdownTimeoutMs ?? DEFAULT_WORKER_SHUTDOWN_TIMEOUT_MS,
     )
 
     if (coverageDataDir) {
@@ -109,8 +138,10 @@ export async function runServerTests(
 async function runInConcurrentWorkers(
   files: string[],
   concurrency: number,
-  runFile: (file: string) => Promise<void>,
+  runFile: (file: string) => WorkerRun,
   onError: () => void,
+  terminateWhenFinished: boolean,
+  workerShutdownTimeoutMs: number,
 ): Promise<void> {
   let index = 0
   let active = 0
@@ -122,22 +153,42 @@ async function runInConcurrentWorkers(
         index++
         active++
 
-        runFile(file).then(
-          () => {
-            active--
-            if (index < files.length) {
-              dispatch()
-            } else if (active === 0) {
-              resolve()
+        let run = runFile(file)
+
+        function complete() {
+          active--
+          if (index < files.length) {
+            dispatch()
+          } else if (active === 0) {
+            resolve()
+          }
+        }
+
+        run.finished.then(
+          async () => {
+            try {
+              if (terminateWhenFinished) {
+                let exited = await waitForWorkerExit(run.exited, workerShutdownTimeoutMs)
+                if (!exited) {
+                  let terminated = await run.terminate()
+                  if (!terminated) {
+                    onError()
+                  }
+                }
+              }
+            } finally {
+              complete()
             }
           },
-          (err) => {
-            console.error(`Error running ${file}:`, err.message)
-            console.error(err)
-            onError()
-            active--
-            if (active === 0 && index >= files.length) resolve()
-            else dispatch()
+          async (err) => {
+            try {
+              console.error(`Error running ${file}:`, err instanceof Error ? err.message : err)
+              console.error(err)
+              onError()
+              await run.terminate()
+            } finally {
+              complete()
+            }
           },
         )
       }
@@ -149,41 +200,193 @@ async function runInConcurrentWorkers(
   })
 }
 
-function runFileInWorker(
+function waitForWorkerExit(exited: Promise<number | null>, timeoutMs: number): Promise<boolean> {
+  return new Promise((resolve) => {
+    let timeout = setTimeout(() => resolve(false), timeoutMs)
+    exited.then(() => {
+      clearTimeout(timeout)
+      resolve(true)
+    })
+  })
+}
+
+function runFileInPool(
   file: string,
   type: 'server' | 'e2e',
   onResults: (results: TestResults) => void,
-  options: {
-    cwd?: string
-    coverage?: CoverageConfig
-    open?: boolean
-    playwrightUseOpts?: PlaywrightUseOpts
-  } = {},
-): Promise<void> {
-  return new Promise((resolve, reject) => {
-    let worker =
-      type === 'e2e'
-        ? new Worker(workerE2EUrl, {
-            workerData: {
-              file: pathToFileURL(file).href,
-              type,
-              coverage: options.coverage,
-              open: options.open,
-              playwrightUseOpts: options.playwrightUseOpts,
-            },
-          })
-        : new Worker(workerUrl, {
-            workerData: {
-              file: pathToFileURL(file).href,
-              type,
-              coverage: options.coverage,
-            },
-          })
-    worker.once('message', (msg: TestResults) => onResults(msg))
+  options: RunFileOptions,
+): WorkerRun {
+  return options.pool === 'threads'
+    ? runFileInWorker(file, type, onResults, options)
+    : runFileInProcess(file, type, onResults, options)
+}
+
+export function runFileInWorker(
+  file: string,
+  type: 'server' | 'e2e',
+  onResults: (results: TestResults) => void,
+  options: RunFileOptions = {},
+): WorkerRun {
+  let receivedResults = false
+  let worker =
+    type === 'e2e'
+      ? new Worker(workerE2EUrl, {
+          workerData: {
+            file: pathToFileURL(file).href,
+            type,
+            coverage: options.coverage,
+            open: options.open,
+            playwrightUseOpts: options.playwrightUseOpts,
+          },
+        })
+      : new Worker(workerUrl, {
+          workerData: {
+            file: pathToFileURL(file).href,
+            type,
+            coverage: options.coverage,
+          },
+        })
+
+  let exited = new Promise<number>((resolve) => {
+    worker.once('exit', (code) => resolve(code))
+  })
+
+  let finished = new Promise<void>((resolve, reject) => {
+    worker.once('message', (msg: TestResults) => {
+      receivedResults = true
+      try {
+        onResults(msg)
+      } catch (error) {
+        reject(error)
+        return
+      }
+      if (!options.open) {
+        resolve()
+      }
+    })
     worker.once('error', reject)
-    worker.once('exit', (code) => {
-      if (code !== 0) reject(new Error(`Worker exited with code ${code}`))
-      else resolve()
+    exited.then((code) => {
+      if (receivedResults || code === 0) {
+        resolve()
+      } else {
+        reject(new Error(`Worker exited with code ${code}`))
+      }
+    })
+  })
+
+  return {
+    finished,
+    exited,
+    async terminate() {
+      try {
+        await worker.terminate()
+        return true
+      } catch (err) {
+        console.error(
+          `Error terminating worker for ${file}:`,
+          err instanceof Error ? err.message : err,
+        )
+        console.error(err)
+        return false
+      }
+    },
+  }
+}
+
+function runFileInProcess(
+  file: string,
+  type: 'server' | 'e2e',
+  onResults: (results: TestResults) => void,
+  options: RunFileOptions = {},
+): WorkerRun {
+  let receivedResults = false
+  let child = fork(fileURLToPath(workerProcessUrl), [], {
+    serialization: 'advanced',
+    stdio: ['ignore', 'inherit', 'inherit', 'ipc'],
+  })
+
+  let exited = new Promise<number | null>((resolve) => {
+    child.once('exit', (code) => resolve(code))
+  })
+
+  let finished = new Promise<void>((resolve, reject) => {
+    child.once('message', (msg: unknown) => {
+      if (!isTestResults(msg)) {
+        reject(new Error('Test worker process sent invalid results'))
+        return
+      }
+
+      receivedResults = true
+      try {
+        onResults(msg)
+      } catch (error) {
+        reject(error)
+        return
+      }
+      if (!options.open) {
+        resolve()
+      }
     })
+    child.once('error', reject)
+    exited.then((code) => {
+      if (receivedResults || code === 0) {
+        resolve()
+      } else {
+        reject(new Error(`Worker process exited with code ${code}`))
+      }
+    })
+    child.send(
+      {
+        file: pathToFileURL(file).href,
+        type,
+        coverage: options.coverage,
+        open: options.open,
+        playwrightUseOpts: options.playwrightUseOpts,
+      },
+      (error) => {
+        if (error) {
+          reject(error)
+        }
+      },
+    )
   })
+
+  return {
+    finished,
+    exited,
+    terminate: () => terminateChildProcess(child, exited),
+  }
+}
+
+async function terminateChildProcess(
+  child: ChildProcess,
+  exited: Promise<number | null>,
+): Promise<boolean> {
+  if (child.exitCode !== null || child.signalCode !== null) {
+    return true
+  }
+
+  if (!child.kill()) {
+    return false
+  }
+
+  return await waitForWorkerExit(exited, 5_000)
+}
+
+function isTestResults(value: unknown): value is TestResults {
+  if (!isRecord(value)) {
+    return false
+  }
+
+  return (
+    typeof value.passed === 'number' &&
+    typeof value.failed === 'number' &&
+    typeof value.skipped === 'number' &&
+    typeof value.todo === 'number' &&
+    Array.isArray(value.tests)
+  )
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null
 }

package/src/lib/ts-transform.ts

@@ -1,4 +1,4 @@
-import { transform, type TsconfigRaw } from 'esbuild'
+import { transform } from 'esbuild'
 import { getTsconfig, type TsConfigResult } from 'get-tsconfig'
 import * as path from 'node:path'
 

package/src/lib/worker-e2e-file.ts

@@ -0,0 +1,98 @@
+import { runTests } from './executor.ts'
+import { importModule } from './import-module.ts'
+import {
+  getBrowserLauncher,
+  getPlaywrightLaunchOptions,
+  getPlaywrightPageOptions,
+  type PlaywrightUseOpts,
+} from './playwright.ts'
+import type { CoverageConfig } from './coverage.ts'
+import type { TestResults } from './reporters/results.ts'
+import { createFailedResults } from './worker-results.ts'
+import { isRecord, parseCoverageConfig } from './worker-server.ts'
+
+export interface E2ETestWorkerData {
+  file: string
+  coverage?: CoverageConfig
+  open?: boolean
+  playwrightUseOpts?: PlaywrightUseOpts
+}
+
+export async function runE2ETestFile(
+  value: unknown,
+  onOpenResults?: (results: TestResults) => void | Promise<void>,
+): Promise<TestResults | undefined> {
+  try {
+    let workerData = parseE2ETestWorkerData(value)
+
+    await importModule(workerData.file, import.meta)
+
+    let launcher = await getBrowserLauncher(workerData.playwrightUseOpts)
+    let opts = getPlaywrightLaunchOptions(workerData.playwrightUseOpts)
+    let browser = await launcher.launch(opts)
+    let browserClosed = false
+
+    try {
+      let results = await runTests({
+        browser,
+        open: workerData.open ?? false,
+        playwrightPageOptions: getPlaywrightPageOptions(workerData.playwrightUseOpts),
+        coverage: !!workerData.coverage,
+      })
+
+      if (workerData.open) {
+        await onOpenResults?.(results)
+        console.log('\nBrowser is open. Press Ctrl+C to close.')
+        await new Promise<void>((resolve) => browser.on('disconnected', () => resolve()))
+        return undefined
+      }
+
+      await browser.close()
+      browserClosed = true
+      return results
+    } finally {
+      if (!browserClosed) {
+        await browser.close()
+      }
+    }
+  } catch (error) {
+    return createFailedResults(error)
+  }
+}
+
+function parseE2ETestWorkerData(value: unknown): E2ETestWorkerData {
+  if (!isRecord(value) || typeof value.file !== 'string') {
+    throw new Error('Invalid E2E test worker data')
+  }
+
+  return {
+    file: value.file,
+    coverage: parseCoverageConfig(value.coverage),
+    open: parseBoolean(value.open, 'open'),
+    playwrightUseOpts: parsePlaywrightUseOpts(value.playwrightUseOpts),
+  }
+}
+
+function parseBoolean(value: unknown, name: string): boolean | undefined {
+  if (value === undefined) {
+    return undefined
+  }
+
+  if (typeof value !== 'boolean') {
+    throw new Error(`Invalid E2E test worker ${name}`)
+  }
+
+  return value
+}
+
+function parsePlaywrightUseOpts(value: unknown): PlaywrightUseOpts | undefined {
+  if (value === undefined) {
+    return undefined
+  }
+
+  if (!isRecord(value)) {
+    throw new Error('Invalid E2E test worker playwright options')
+  }
+
+  return value as PlaywrightUseOpts
+}