@remix-run/test 0.1.0 → 0.3.0

package/src/lib/reporters/spec.ts

@@ -1,15 +1,23 @@
-import { colors, normalizeLine, type Counts } from '../utils.ts'
-import type { TestResult, TestResults } from '../executor.ts'
+import { colors } from '../colors.ts'
+import { normalizeLine } from '../normalize.ts'
 import type { Reporter } from './index.ts'
+import type { Counts, TestResult, TestResults } from './results.ts'
 
 export class SpecReporter implements Reporter {
   #failures: { suiteName: string; name: string; error: TestResult['error'] }[] = []
+  #files = new Set<string>()
+  #suites = new Set<string>()
 
   onSectionStart(label: string) {
     console.log(label)
   }
 
   onResult(results: TestResults, env?: string) {
+    for (let test of results.tests) {
+      if (test.filePath) this.#files.add(test.filePath)
+      if (test.suiteName) this.#suites.add(test.suiteName)
+    }
+
     let suiteMap = new Map<string, TestResult[]>()
     for (let test of results.tests) {
       let suite = test.suiteName || 'Global'
@@ -162,6 +170,8 @@ export class SpecReporter implements Reporter {
     let { passed, failed, skipped, todo } = counts
     let info = colors.cyan('ℹ')
     console.log()
+    console.log(`${info} files ${this.#files.size}`)
+    console.log(`${info} suites ${this.#suites.size}`)
     console.log(`${info} tests ${passed + failed + skipped + todo}`)
     console.log(`${info} pass ${passed}`)
     console.log(`${info} fail ${failed}`)

package/src/lib/reporters/tap.ts

@@ -1,10 +1,12 @@
-import { normalizeLine, type Counts } from '../utils.ts'
-import type { TestResults } from '../executor.ts'
+import { normalizeLine } from '../normalize.ts'
 import type { Reporter } from './index.ts'
+import type { Counts, TestResults } from './results.ts'
 
 export class TapReporter implements Reporter {
   #counter = 0
   #total = 0
+  #files = new Set<string>()
+  #suites = new Set<string>()
 
   onSectionStart(_label: string) {}
 
@@ -15,6 +17,11 @@ export class TapReporter implements Reporter {
 
     let envComment = env ? ` # ${env}` : ''
 
+    for (let test of results.tests) {
+      if (test.filePath) this.#files.add(test.filePath)
+      if (test.suiteName) this.#suites.add(test.suiteName)
+    }
+
     for (let test of results.tests) {
       this.#counter++
       this.#total++
@@ -48,6 +55,8 @@ export class TapReporter implements Reporter {
   onSummary(counts: Counts, durationMs: number) {
     let { passed, failed, skipped, todo } = counts
     console.log(`1..${this.#total}`)
+    console.log(`# files ${this.#files.size}`)
+    console.log(`# suites ${this.#suites.size}`)
     console.log(`# tests ${passed + failed + skipped + todo}`)
     console.log(`# pass ${passed}`)
     console.log(`# fail ${failed}`)

package/src/lib/runner-browser.ts

@@ -0,0 +1,171 @@
+import * as path from 'node:path'
+import type { Browser, Page, Request } from 'playwright'
+import { colors } from './colors.ts'
+import { getBrowserTestRootDir } from './config.ts'
+import {
+  collectCoverageMapFromPlaywright,
+  type CoverageMap,
+  type V8CoverageEntry,
+} from './coverage.ts'
+import {
+  getBrowserLauncher,
+  getPlaywrightLaunchOptions,
+  getPlaywrightPageOptions,
+  type PlaywrightUseOpts,
+} from './playwright.ts'
+import type { Reporter } from './reporters/index.ts'
+import type { TestResults } from './reporters/results.ts'
+
+// The harness reports each test result with `filePath` set to the
+// `/scripts/<rel>` URL the iframe loaded. Reporters expect a real filesystem
+// path so they can compute `path.relative(cwd, ...)` cleanly; otherwise they
+// produce noisy `../../../scripts/...` strings.
+function urlPathToFilePath(urlPath: string, rootDir: string): string {
+  if (!urlPath.startsWith('/scripts/')) return urlPath
+  return path.resolve(rootDir, urlPath.slice('/scripts/'.length))
+}
+
+export interface TestRunOptions {
+  baseUrl: string
+  console?: boolean
+  coverage?: boolean
+  open?: boolean
+  playwrightUseOpts?: PlaywrightUseOpts
+  projectName?: string
+  reporter: Reporter
+  // Test file paths so coverage collection can skip them when mapping V8
+  // entries back to filesystem files.
+  testFiles?: string[]
+}
+
+export async function runBrowserTests(options: TestRunOptions): Promise<{
+  results: TestResults
+  coverageMap: CoverageMap | null
+  close: () => Promise<void>
+  disconnected: Promise<void>
+}> {
+  let envLabel = options.projectName ? `browser:${options.projectName}` : 'browser'
+  let browser: Browser | undefined
+  let page: Page | undefined
+  let close = async () => {
+    await page?.close()
+    await browser?.close()
+    browser = undefined
+    page = undefined
+  }
+  let results: TestResults
+  let coverageMap: CoverageMap | null = null
+
+  try {
+    browser = await getBrowserLauncher(options.playwrightUseOpts).launch(
+      getPlaywrightLaunchOptions(options.playwrightUseOpts),
+    )
+    page = await browser.newPage(getPlaywrightPageOptions(options.playwrightUseOpts))
+    // Cap how long we'll wait for a browser-test file to signal completion.
+    // Playwright's default is 30s; bumping to 60s buys headroom for slower
+    // suites without letting a hung test hide forever. Plumb this through
+    // config later if anyone needs to tune it.
+    page.setDefaultTimeout(90_000)
+    page.setDefaultNavigationTimeout(90_000)
+
+    if (options.console) {
+      page.on('console', (msg) => console.log(`${colors.dim('[browser console]')} ${msg.text()}`))
+    }
+
+    // Playwright's JS coverage is Chromium-only. Start before navigation so
+    // the harness scripts and test modules are instrumented from first parse.
+    let coverageEnabled = options.coverage && browser.browserType().name() === 'chromium'
+    if (coverageEnabled) {
+      await page.coverage.startJSCoverage({ resetOnNavigation: false })
+    }
+
+    let totalPassed = 0
+    let totalFailed = 0
+    let totalSkipped = 0
+    let totalTodo = 0
+    let rootDir = getBrowserTestRootDir()
+
+    await page.route('**/file-results', async (route) => {
+      let results = route.request().postDataJSON() as TestResults
+      for (let test of results.tests) {
+        if (test.filePath) test.filePath = urlPathToFilePath(test.filePath, rootDir)
+      }
+      options.reporter.onResult(results, envLabel)
+      totalPassed += results.passed
+      totalFailed += results.failed
+      totalSkipped += results.skipped
+      totalTodo += results.todo
+      await route.fulfill({ status: 200 })
+    })
+
+    // Fail the tests if any /scripts/ request fails (harness scripts, test
+    // modules, or their transitive imports — all served via the same prefix).
+    let errorPromise = new Promise((_, reject) => {
+      let isScriptRequest = (request: Request) =>
+        new URL(request.url()).pathname.startsWith('/scripts/')
+      page!.on('response', (response) => {
+        if (!response.ok() && isScriptRequest(response.request())) {
+          reject(new Error(`Failed to load script: ${response.request().url()}`))
+        }
+      })
+      page!.on('requestfailed', (request) => {
+        if (isScriptRequest(request)) {
+          reject(new Error(`Failed to load script: ${request.url()}`))
+        }
+      })
+    })
+
+    // Prevent unhandled rejection if we fail before setting up the listener
+    errorPromise.catch(() => {})
+
+    await page.goto(options.baseUrl)
+    await Promise.race([page.waitForFunction('window.__testsDone'), errorPromise])
+
+    if (coverageEnabled) {
+      let entries = (await page.coverage.stopJSCoverage()) as unknown as V8CoverageEntry[]
+      if (entries.length > 0) {
+        coverageMap = await collectCoverageMapFromPlaywright(
+          entries,
+          getBrowserTestRootDir(),
+          new Set(options.testFiles ?? []),
+          async (urlPath) =>
+            urlPath.startsWith('/scripts/') ? urlPath.slice('/scripts/'.length) : null,
+        )
+      }
+    }
+
+    results = {
+      passed: totalPassed,
+      failed: totalFailed,
+      skipped: totalSkipped,
+      todo: totalTodo,
+      tests: [],
+    }
+  } catch (error) {
+    console.error('Browser tests failed to run:', error)
+    results = {
+      passed: 0,
+      failed: 1,
+      skipped: 0,
+      todo: 0,
+      tests: [],
+    }
+  }
+
+  if (options.open) {
+    return {
+      results,
+      coverageMap,
+      close,
+      disconnected: new Promise((r) => browser!.on('disconnected', () => r())),
+    }
+  } else {
+    await close()
+    return {
+      results,
+      coverageMap,
+      close,
+      disconnected: Promise.resolve(),
+    }
+  }
+}

package/src/lib/runner.ts

@@ -1,14 +1,41 @@
+import { fork, type ChildProcess } from 'node:child_process'
+import * as fsp from 'node:fs/promises'
 import * as path from 'node:path'
-import { pathToFileURL } from 'node:url'
+import { fileURLToPath, pathToFileURL } from 'node:url'
 import { Worker } from 'node:worker_threads'
-import type { TestResults } from './executor.ts'
+import { IS_RUNNING_FROM_SRC, type RemixTestPool } from './config.ts'
+import {
+  collectCoverageMapFromPlaywright,
+  collectServerCoverageMap,
+  type CoverageConfig,
+  type CoverageMap,
+  type V8CoverageEntry,
+} from './coverage.ts'
 import { type PlaywrightUseOpts } from './playwright.ts'
 import type { Reporter } from './reporters/index.ts'
-import type { Counts } from './utils.ts'
+import type { Counts, TestResults } from './reporters/results.ts'
 
-const ext = path.extname(import.meta.url)
+// Ensure we load the right file whether we're running in the monorepo (TS) or
+// from a published package (JS)
+const ext = IS_RUNNING_FROM_SRC ? '.ts' : '.js'
 const workerUrl = new URL(`./worker${ext}`, import.meta.url)
 const workerE2EUrl = new URL(`./worker-e2e${ext}`, import.meta.url)
+const workerProcessUrl = new URL(`./worker-process${ext}`, import.meta.url)
+const DEFAULT_WORKER_SHUTDOWN_TIMEOUT_MS = 10_000
+
+interface WorkerRun {
+  finished: Promise<void>
+  exited: Promise<number | null>
+  terminate(): Promise<boolean>
+}
+
+interface RunFileOptions {
+  cwd?: string
+  coverage?: CoverageConfig
+  open?: boolean
+  playwrightUseOpts?: PlaywrightUseOpts
+  pool?: RemixTestPool
+}
 
 export async function runServerTests(
   files: string[],
@@ -16,13 +43,20 @@ export async function runServerTests(
   concurrency: number,
   type: 'server' | 'e2e',
   options: {
+    cwd?: string
     open?: boolean
     playwrightUseOpts?: PlaywrightUseOpts
     projectName?: string
+    coverage?: CoverageConfig
+    workerShutdownTimeoutMs?: number
+    pool?: RemixTestPool
   } = {},
-): Promise<Counts> {
+): Promise<Counts & { coverageMap: CoverageMap | null }> {
   let counts: Counts = { passed: 0, failed: 0, skipped: 0, todo: 0 }
+  let coverageMap: CoverageMap | null = null
+  let cwd = options.cwd ?? process.cwd()
   let envLabel = options.projectName ? `${type}:${options.projectName}` : type
+  let pool = options.pool ?? 'forks'
 
   function accumulate(results: TestResults, file: string) {
     reporter.onResult(
@@ -36,33 +70,78 @@ export async function runServerTests(
   }
 
   if (type === 'e2e') {
+    let allBrowserCoverageEntries: Array<{ entries: V8CoverageEntry[]; baseUrl: string }> = []
+
     await runInConcurrentWorkers(
       files,
       concurrency,
       (file) =>
-        runFileInWorker(file, type, (results) => accumulate(results, file), {
-          ...options,
-          playwrightUseOpts: options.playwrightUseOpts,
-        }),
+        runFileInPool(
+          file,
+          type,
+          (results) => {
+            accumulate(results, file)
+            if (results.e2eBrowserCoverageEntries) {
+              allBrowserCoverageEntries.push(...results.e2eBrowserCoverageEntries)
+            }
+          },
+          {
+            ...options,
+            pool,
+            playwrightUseOpts: options.playwrightUseOpts,
+          },
+        ),
       () => counts.failed++,
+      !options.open,
+      options.workerShutdownTimeoutMs ?? DEFAULT_WORKER_SHUTDOWN_TIMEOUT_MS,
     )
+
+    if (options.coverage && allBrowserCoverageEntries.length > 0) {
+      coverageMap = await collectCoverageMapFromPlaywright(
+        allBrowserCoverageEntries.flatMap((e) => e.entries),
+        cwd,
+        new Set(files),
+        async (urlPath) => (urlPath.startsWith('/') ? urlPath.slice(1) : urlPath),
+      )
+    }
   } else {
+    let coverageDataDir: string | undefined
+    if (options.coverage) {
+      coverageDataDir = path.resolve(cwd, options.coverage.dir)
+      await fsp.mkdir(coverageDataDir, { recursive: true })
+      process.env.NODE_V8_COVERAGE = coverageDataDir
+    }
+
     await runInConcurrentWorkers(
       files,
       concurrency,
-      (file) => runFileInWorker(file, type, (results) => accumulate(results, file)),
+      (file) =>
+        runFileInPool(file, type, (results) => accumulate(results, file), {
+          ...options,
+          pool,
+        }),
       () => counts.failed++,
+      true,
+      options.workerShutdownTimeoutMs ?? DEFAULT_WORKER_SHUTDOWN_TIMEOUT_MS,
     )
+
+    if (coverageDataDir) {
+      delete process.env.NODE_V8_COVERAGE
+      let serverMap = await collectServerCoverageMap(coverageDataDir, cwd, new Set(files))
+      coverageMap = serverMap
+    }
   }
 
-  return { ...counts }
+  return { ...counts, coverageMap }
 }
 
 async function runInConcurrentWorkers(
   files: string[],
   concurrency: number,
-  runFile: (file: string) => Promise<void>,
+  runFile: (file: string) => WorkerRun,
   onError: () => void,
+  terminateWhenFinished: boolean,
+  workerShutdownTimeoutMs: number,
 ): Promise<void> {
   let index = 0
   let active = 0
@@ -74,22 +153,42 @@ async function runInConcurrentWorkers(
         index++
         active++
 
-        runFile(file).then(
-          () => {
-            active--
-            if (index < files.length) {
-              dispatch()
-            } else if (active === 0) {
-              resolve()
+        let run = runFile(file)
+
+        function complete() {
+          active--
+          if (index < files.length) {
+            dispatch()
+          } else if (active === 0) {
+            resolve()
+          }
+        }
+
+        run.finished.then(
+          async () => {
+            try {
+              if (terminateWhenFinished) {
+                let exited = await waitForWorkerExit(run.exited, workerShutdownTimeoutMs)
+                if (!exited) {
+                  let terminated = await run.terminate()
+                  if (!terminated) {
+                    onError()
+                  }
+                }
+              }
+            } finally {
+              complete()
             }
           },
-          (err) => {
-            console.error(`Error running ${file}:`, err.message)
-            console.error(err)
-            onError()
-            active--
-            if (active === 0 && index >= files.length) resolve()
-            else dispatch()
+          async (err) => {
+            try {
+              console.error(`Error running ${file}:`, err instanceof Error ? err.message : err)
+              console.error(err)
+              onError()
+              await run.terminate()
+            } finally {
+              complete()
+            }
           },
         )
       }
@@ -101,37 +200,193 @@ async function runInConcurrentWorkers(
   })
 }
 
-function runFileInWorker(
+function waitForWorkerExit(exited: Promise<number | null>, timeoutMs: number): Promise<boolean> {
+  return new Promise((resolve) => {
+    let timeout = setTimeout(() => resolve(false), timeoutMs)
+    exited.then(() => {
+      clearTimeout(timeout)
+      resolve(true)
+    })
+  })
+}
+
+function runFileInPool(
   file: string,
   type: 'server' | 'e2e',
   onResults: (results: TestResults) => void,
-  options: {
-    open?: boolean
-    playwrightUseOpts?: PlaywrightUseOpts
-  } = {},
-): Promise<void> {
-  return new Promise((resolve, reject) => {
-    let worker =
-      type === 'e2e'
-        ? new Worker(workerE2EUrl, {
-            workerData: {
-              file: pathToFileURL(file).href,
-              type,
-              open: options.open,
-              playwrightUseOpts: options.playwrightUseOpts,
-            },
-          })
-        : new Worker(workerUrl, {
-            workerData: {
-              file: pathToFileURL(file).href,
-              type,
-            },
-          })
-    worker.once('message', (msg: TestResults) => onResults(msg))
+  options: RunFileOptions,
+): WorkerRun {
+  return options.pool === 'threads'
+    ? runFileInWorker(file, type, onResults, options)
+    : runFileInProcess(file, type, onResults, options)
+}
+
+export function runFileInWorker(
+  file: string,
+  type: 'server' | 'e2e',
+  onResults: (results: TestResults) => void,
+  options: RunFileOptions = {},
+): WorkerRun {
+  let receivedResults = false
+  let worker =
+    type === 'e2e'
+      ? new Worker(workerE2EUrl, {
+          workerData: {
+            file: pathToFileURL(file).href,
+            type,
+            coverage: options.coverage,
+            open: options.open,
+            playwrightUseOpts: options.playwrightUseOpts,
+          },
+        })
+      : new Worker(workerUrl, {
+          workerData: {
+            file: pathToFileURL(file).href,
+            type,
+            coverage: options.coverage,
+          },
+        })
+
+  let exited = new Promise<number>((resolve) => {
+    worker.once('exit', (code) => resolve(code))
+  })
+
+  let finished = new Promise<void>((resolve, reject) => {
+    worker.once('message', (msg: TestResults) => {
+      receivedResults = true
+      try {
+        onResults(msg)
+      } catch (error) {
+        reject(error)
+        return
+      }
+      if (!options.open) {
+        resolve()
+      }
+    })
     worker.once('error', reject)
-    worker.once('exit', (code) => {
-      if (code !== 0) reject(new Error(`Worker exited with code ${code}`))
-      else resolve()
+    exited.then((code) => {
+      if (receivedResults || code === 0) {
+        resolve()
+      } else {
+        reject(new Error(`Worker exited with code ${code}`))
+      }
+    })
+  })
+
+  return {
+    finished,
+    exited,
+    async terminate() {
+      try {
+        await worker.terminate()
+        return true
+      } catch (err) {
+        console.error(
+          `Error terminating worker for ${file}:`,
+          err instanceof Error ? err.message : err,
+        )
+        console.error(err)
+        return false
+      }
+    },
+  }
+}
+
+function runFileInProcess(
+  file: string,
+  type: 'server' | 'e2e',
+  onResults: (results: TestResults) => void,
+  options: RunFileOptions = {},
+): WorkerRun {
+  let receivedResults = false
+  let child = fork(fileURLToPath(workerProcessUrl), [], {
+    serialization: 'advanced',
+    stdio: ['ignore', 'inherit', 'inherit', 'ipc'],
+  })
+
+  let exited = new Promise<number | null>((resolve) => {
+    child.once('exit', (code) => resolve(code))
+  })
+
+  let finished = new Promise<void>((resolve, reject) => {
+    child.once('message', (msg: unknown) => {
+      if (!isTestResults(msg)) {
+        reject(new Error('Test worker process sent invalid results'))
+        return
+      }
+
+      receivedResults = true
+      try {
+        onResults(msg)
+      } catch (error) {
+        reject(error)
+        return
+      }
+      if (!options.open) {
+        resolve()
+      }
+    })
+    child.once('error', reject)
+    exited.then((code) => {
+      if (receivedResults || code === 0) {
+        resolve()
+      } else {
+        reject(new Error(`Worker process exited with code ${code}`))
+      }
     })
+    child.send(
+      {
+        file: pathToFileURL(file).href,
+        type,
+        coverage: options.coverage,
+        open: options.open,
+        playwrightUseOpts: options.playwrightUseOpts,
+      },
+      (error) => {
+        if (error) {
+          reject(error)
+        }
+      },
+    )
   })
+
+  return {
+    finished,
+    exited,
+    terminate: () => terminateChildProcess(child, exited),
+  }
+}
+
+async function terminateChildProcess(
+  child: ChildProcess,
+  exited: Promise<number | null>,
+): Promise<boolean> {
+  if (child.exitCode !== null || child.signalCode !== null) {
+    return true
+  }
+
+  if (!child.kill()) {
+    return false
+  }
+
+  return await waitForWorkerExit(exited, 5_000)
+}
+
+function isTestResults(value: unknown): value is TestResults {
+  if (!isRecord(value)) {
+    return false
+  }
+
+  return (
+    typeof value.passed === 'number' &&
+    typeof value.failed === 'number' &&
+    typeof value.skipped === 'number' &&
+    typeof value.todo === 'number' &&
+    Array.isArray(value.tests)
+  )
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null
 }

package/src/lib/runtime.ts

@@ -0,0 +1,2 @@
+// https://bun.com/docs/guides/util/detect-bun
+export const IS_BUN = typeof process.versions.bun === 'string'