@remix-run/test 0.0.0 → 0.2.0

package/src/lib/reporters/spec.ts

@@ -0,0 +1,174 @@
+import { colors } from '../colors.ts'
+import { normalizeLine } from '../normalize.ts'
+import type { Reporter } from './index.ts'
+import type { Counts, TestResult, TestResults } from './results.ts'
+
+export class SpecReporter implements Reporter {
+  #failures: { suiteName: string; name: string; error: TestResult['error'] }[] = []
+
+  onSectionStart(label: string) {
+    console.log(label)
+  }
+
+  onResult(results: TestResults, env?: string) {
+    let suiteMap = new Map<string, TestResult[]>()
+    for (let test of results.tests) {
+      let suite = test.suiteName || 'Global'
+      if (!suiteMap.has(suite)) suiteMap.set(suite, [])
+      suiteMap.get(suite)!.push(test)
+    }
+
+    let envLabel = env ? ` ${colors.dim(`[${env}]`)}` : ''
+    let lastParts: string[] = []
+
+    // Pre-compute aggregate test results for each path prefix so non-leaf
+    // suite headings can be colored the same way as leaf headings.
+    let prefixTests = new Map<string, TestResult[]>()
+    for (let [suiteName, tests] of suiteMap) {
+      let parts = suiteName.split(' > ')
+      for (let i = 0; i < parts.length; i++) {
+        let prefix = parts.slice(0, i + 1).join(' > ')
+        if (!prefixTests.has(prefix)) prefixTests.set(prefix, [])
+        prefixTests.get(prefix)!.push(...tests)
+      }
+    }
+
+    for (let [suiteName, suiteTests] of suiteMap) {
+      let parts = suiteName.split(' > ')
+
+      // Find where this path diverges from the last rendered path
+      let commonLen = 0
+      while (
+        commonLen < lastParts.length &&
+        commonLen < parts.length &&
+        lastParts[commonLen] === parts[commonLen]
+      ) {
+        commonLen++
+      }
+
+      // Print each new path component
+      for (let i = commonLen; i < parts.length; i++) {
+        let indent = '  '.repeat(i)
+        let isLeaf = i === parts.length - 1
+
+        if (isLeaf) {
+          let totalDuration = suiteTests.reduce((sum, t) => sum + t.duration, 0)
+          let suiteHasFailed = suiteTests.some((t) => t.status === 'failed')
+          let suiteAllSkipped = suiteTests.every((t) => t.status === 'skipped')
+          let suiteAllTodo = suiteTests.every((t) => t.status === 'todo')
+          let label = suiteHasFailed
+            ? colors.red(parts[i])
+            : suiteAllSkipped
+              ? colors.dim(parts[i])
+              : suiteAllTodo
+                ? colors.yellow(parts[i])
+                : colors.green(parts[i])
+          let suiteComment = suiteAllSkipped
+            ? colors.dim(' # skipped')
+            : suiteAllTodo
+              ? colors.yellow(' # todo')
+              : ''
+          let duration = suiteComment ? '' : ` (${totalDuration.toFixed(2)}ms)`
+          let label2 = envLabel
+          console.log(`${indent}${colors.dim('▶')} ${label}${duration}${suiteComment}${label2}`)
+        } else {
+          let prefix = parts.slice(0, i + 1).join(' > ')
+          let prefixTestList = prefixTests.get(prefix) ?? []
+          let prefixHasFailed = prefixTestList.some((t) => t.status === 'failed')
+          let prefixAllSkipped =
+            prefixTestList.length > 0 && prefixTestList.every((t) => t.status === 'skipped')
+          let prefixAllTodo =
+            prefixTestList.length > 0 && prefixTestList.every((t) => t.status === 'todo')
+          let nameColor = prefixHasFailed
+            ? colors.red
+            : prefixAllSkipped
+              ? colors.dim
+              : prefixAllTodo
+                ? colors.yellow
+                : colors.green
+          let prefixDuration = prefixTestList.reduce((sum, t) => sum + t.duration, 0)
+          let prefixComment = prefixAllSkipped
+            ? colors.dim(' # skipped')
+            : prefixAllTodo
+              ? colors.yellow(' # todo')
+              : ''
+          let prefixDurationStr = prefixComment ? '' : ` (${prefixDuration.toFixed(2)}ms)`
+          console.log(
+            `${indent}${colors.dim('▶')} ${nameColor(parts[i])}${prefixDurationStr}${prefixComment}${envLabel}`,
+          )
+        }
+      }
+
+      lastParts = parts
+
+      // Print tests indented to the suite's depth
+      let testIndent = '  '.repeat(parts.length)
+      for (let test of suiteTests) {
+        if (test.status === 'passed') {
+          console.log(
+            `${testIndent}${colors.green('✓')} ${test.name} (${test.duration.toFixed(2)}ms)`,
+          )
+        } else if (test.status === 'failed') {
+          console.log(
+            `${testIndent}${colors.red('✗')} ${test.name} (${test.duration.toFixed(2)}ms)`,
+          )
+          if (test.error) {
+            console.log(`${testIndent}  ${colors.red(`Error: ${test.error.message}`)}`)
+            if (test.error.stack) {
+              let stack = test.error.stack
+                .split('\n')
+                .map((line) => normalizeLine(line))
+                .join('\n')
+              console.log(
+                `${testIndent}    ${stack.split('\n').slice(1, 5).join(`\n${testIndent}    `)}`,
+              )
+            }
+          }
+          this.#failures.push({ suiteName: test.suiteName, name: test.name, error: test.error })
+        } else if (test.status === 'skipped') {
+          if (test.name)
+            console.log(`${testIndent}${colors.dim('↓')} ${colors.dim(`${test.name} # skipped`)}`)
+        } else if (test.status === 'todo') {
+          if (test.name)
+            console.log(
+              `${testIndent}${colors.yellow('…')} ${colors.yellow(`${test.name} # todo`)}`,
+            )
+        }
+      }
+    }
+  }
+
+  onSummary(counts: Counts, durationMs: number) {
+    if (this.#failures.length > 0) {
+      console.log()
+      console.log(colors.red('Failed tests:'))
+      for (let i = 0; i < this.#failures.length; i++) {
+        let { suiteName, name, error } = this.#failures[i]
+        let fullName = name ? `${suiteName} > ${name}` : suiteName
+        console.log(`\n  ${colors.red(`${i + 1})`)} ${fullName}`)
+        if (error) {
+          console.log(`     ${colors.red(error.message)}`)
+          if (error.stack) {
+            let frames = error.stack
+              .split('\n')
+              .slice(1, 4)
+              .map((l) => `     ${normalizeLine(l).trim()}`)
+              .join('\n')
+            console.log(frames)
+          }
+        }
+      }
+    }
+
+    let { passed, failed, skipped, todo } = counts
+    let info = colors.cyan('ℹ')
+    console.log()
+    console.log(`${info} tests ${passed + failed + skipped + todo}`)
+    console.log(`${info} pass ${passed}`)
+    console.log(`${info} fail ${failed}`)
+    if (skipped > 0) console.log(`${info} skipped ${skipped}`)
+    if (todo > 0) console.log(`${info} todo ${todo}`)
+    console.log(`${info} duration_ms ${durationMs.toFixed(5)}`)
+    console.log()
+  }
+}

package/src/lib/reporters/tap.ts

@@ -0,0 +1,58 @@
+import { normalizeLine } from '../normalize.ts'
+import type { Reporter } from './index.ts'
+import type { Counts, TestResults } from './results.ts'
+
+export class TapReporter implements Reporter {
+  #counter = 0
+  #total = 0
+
+  onSectionStart(_label: string) {}
+
+  onResult(results: TestResults, env?: string) {
+    if (this.#counter === 0) {
+      console.log('TAP version 14')
+    }
+
+    let envComment = env ? ` # ${env}` : ''
+
+    for (let test of results.tests) {
+      this.#counter++
+      this.#total++
+      let fullName = test.name
+        ? `${test.suiteName} > ${test.name}${envComment}`
+        : `${test.suiteName}${envComment}`
+
+      if (test.status === 'passed') {
+        console.log(`ok ${this.#counter} - ${fullName}`)
+      } else if (test.status === 'skipped') {
+        console.log(`ok ${this.#counter} - ${fullName} # SKIP`)
+      } else if (test.status === 'todo') {
+        console.log(`ok ${this.#counter} - ${fullName} # TODO`)
+      } else {
+        console.log(`not ok ${this.#counter} - ${fullName}`)
+        console.log('  ---')
+        console.log(`  message: ${test.error?.message ?? 'unknown error'}`)
+        if (test.error?.stack) {
+          let frames = test.error.stack
+            .split('\n')
+            .slice(1, 4)
+            .map((l) => normalizeLine(l).trim())
+            .join('\n    ')
+          console.log(`  stack: |\n    ${frames}`)
+        }
+        console.log('  ...')
+      }
+    }
+  }
+
+  onSummary(counts: Counts, durationMs: number) {
+    let { passed, failed, skipped, todo } = counts
+    console.log(`1..${this.#total}`)
+    console.log(`# tests ${passed + failed + skipped + todo}`)
+    console.log(`# pass ${passed}`)
+    console.log(`# fail ${failed}`)
+    if (skipped > 0) console.log(`# skipped ${skipped}`)
+    if (todo > 0) console.log(`# todo ${todo}`)
+    console.log(`# duration_ms ${durationMs.toFixed(5)}`)
+  }
+}

package/src/lib/runner-browser.ts

@@ -0,0 +1,165 @@
+import * as path from 'node:path'
+import type { Browser, Page, Request } from 'playwright'
+import { colors } from './colors.ts'
+import { getBrowserTestRootDir } from './config.ts'
+import {
+  collectCoverageMapFromPlaywright,
+  type CoverageMap,
+  type V8CoverageEntry,
+} from './coverage.ts'
+import {
+  getBrowserLauncher,
+  getPlaywrightLaunchOptions,
+  getPlaywrightPageOptions,
+  type PlaywrightUseOpts,
+} from './playwright.ts'
+import type { Reporter } from './reporters/index.ts'
+import type { TestResults } from './reporters/results.ts'
+
+// The harness reports each test result with `filePath` set to the
+// `/scripts/<rel>` URL the iframe loaded. Reporters expect a real filesystem
+// path so they can compute `path.relative(cwd, ...)` cleanly; otherwise they
+// produce noisy `../../../scripts/...` strings.
+function urlPathToFilePath(urlPath: string, rootDir: string): string {
+  if (!urlPath.startsWith('/scripts/')) return urlPath
+  return path.resolve(rootDir, urlPath.slice('/scripts/'.length))
+}
+
+export interface TestRunOptions {
+  baseUrl: string
+  console?: boolean
+  coverage?: boolean
+  open?: boolean
+  playwrightUseOpts?: PlaywrightUseOpts
+  projectName?: string
+  reporter: Reporter
+  // Test file paths so coverage collection can skip them when mapping V8
+  // entries back to filesystem files.
+  testFiles?: string[]
+}
+
+export async function runBrowserTests(options: TestRunOptions): Promise<{
+  results: TestResults
+  coverageMap: CoverageMap | null
+  close: () => Promise<void>
+  disconnected: Promise<void>
+}> {
+  let envLabel = options.projectName ? `browser:${options.projectName}` : 'browser'
+  let browser: Browser | undefined
+  let page: Page | undefined
+  let close = async () => {
+    await page?.close()
+    await browser?.close()
+    browser = undefined
+    page = undefined
+  }
+  let results: TestResults
+  let coverageMap: CoverageMap | null = null
+
+  try {
+    browser = await getBrowserLauncher(options.playwrightUseOpts).launch(
+      getPlaywrightLaunchOptions(options.playwrightUseOpts),
+    )
+    page = await browser.newPage(getPlaywrightPageOptions(options.playwrightUseOpts))
+
+    if (options.console) {
+      page.on('console', (msg) => console.log(`${colors.dim('[browser console]')} ${msg.text()}`))
+    }
+
+    // Playwright's JS coverage is Chromium-only. Start before navigation so
+    // the harness scripts and test modules are instrumented from first parse.
+    let coverageEnabled = options.coverage && browser.browserType().name() === 'chromium'
+    if (coverageEnabled) {
+      await page.coverage.startJSCoverage({ resetOnNavigation: false })
+    }
+
+    let totalPassed = 0
+    let totalFailed = 0
+    let totalSkipped = 0
+    let totalTodo = 0
+    let rootDir = getBrowserTestRootDir()
+
+    await page.route('**/file-results', async (route) => {
+      let results = route.request().postDataJSON() as TestResults
+      for (let test of results.tests) {
+        if (test.filePath) test.filePath = urlPathToFilePath(test.filePath, rootDir)
+      }
+      options.reporter.onResult(results, envLabel)
+      totalPassed += results.passed
+      totalFailed += results.failed
+      totalSkipped += results.skipped
+      totalTodo += results.todo
+      await route.fulfill({ status: 200 })
+    })
+
+    // Fail the tests if any /scripts/ request fails (harness scripts, test
+    // modules, or their transitive imports — all served via the same prefix).
+    let errorPromise = new Promise((_, reject) => {
+      let isScriptRequest = (request: Request) =>
+        new URL(request.url()).pathname.startsWith('/scripts/')
+      page!.on('response', (response) => {
+        if (!response.ok() && isScriptRequest(response.request())) {
+          reject(new Error(`Failed to load script: ${response.request().url()}`))
+        }
+      })
+      page!.on('requestfailed', (request) => {
+        if (isScriptRequest(request)) {
+          reject(new Error(`Failed to load script: ${request.url()}`))
+        }
+      })
+    })
+
+    // Prevent unhandled rejection if we fail before setting up the listener
+    errorPromise.catch(() => {})
+
+    await page.goto(options.baseUrl)
+    await Promise.race([page.waitForFunction('window.__testsDone'), errorPromise])
+
+    if (coverageEnabled) {
+      let entries = (await page.coverage.stopJSCoverage()) as unknown as V8CoverageEntry[]
+      if (entries.length > 0) {
+        coverageMap = await collectCoverageMapFromPlaywright(
+          entries,
+          getBrowserTestRootDir(),
+          new Set(options.testFiles ?? []),
+          async (urlPath) =>
+            urlPath.startsWith('/scripts/') ? urlPath.slice('/scripts/'.length) : null,
+        )
+      }
+    }
+
+    results = {
+      passed: totalPassed,
+      failed: totalFailed,
+      skipped: totalSkipped,
+      todo: totalTodo,
+      tests: [],
+    }
+  } catch (error) {
+    console.error('Browser tests failed to run:', error)
+    results = {
+      passed: 0,
+      failed: 1,
+      skipped: 0,
+      todo: 0,
+      tests: [],
+    }
+  }
+
+  if (options.open) {
+    return {
+      results,
+      coverageMap,
+      close,
+      disconnected: new Promise((r) => browser!.on('disconnected', () => r())),
+    }
+  } else {
+    await close()
+    return {
+      results,
+      coverageMap,
+      close,
+      disconnected: Promise.resolve(),
+    }
+  }
+}

package/src/lib/runner.ts

@@ -0,0 +1,189 @@
+import * as fsp from 'node:fs/promises'
+import * as path from 'node:path'
+import { pathToFileURL } from 'node:url'
+import { Worker } from 'node:worker_threads'
+import { IS_RUNNING_FROM_SRC } from './config.ts'
+import {
+  collectCoverageMapFromPlaywright,
+  collectServerCoverageMap,
+  type CoverageConfig,
+  type CoverageMap,
+  type V8CoverageEntry,
+} from './coverage.ts'
+import { type PlaywrightUseOpts } from './playwright.ts'
+import type { Reporter } from './reporters/index.ts'
+import type { Counts, TestResults } from './reporters/results.ts'
+
+// Ensure we load the right file whether we're running in the monorepo (TS) or
+// from a published package (JS)
+const ext = IS_RUNNING_FROM_SRC ? '.ts' : '.js'
+const workerUrl = new URL(`./worker${ext}`, import.meta.url)
+const workerE2EUrl = new URL(`./worker-e2e${ext}`, import.meta.url)
+
+export async function runServerTests(
+  files: string[],
+  reporter: Reporter,
+  concurrency: number,
+  type: 'server' | 'e2e',
+  options: {
+    cwd?: string
+    open?: boolean
+    playwrightUseOpts?: PlaywrightUseOpts
+    projectName?: string
+    coverage?: CoverageConfig
+  } = {},
+): Promise<Counts & { coverageMap: CoverageMap | null }> {
+  let counts: Counts = { passed: 0, failed: 0, skipped: 0, todo: 0 }
+  let coverageMap: CoverageMap | null = null
+  let cwd = options.cwd ?? process.cwd()
+  let envLabel = options.projectName ? `${type}:${options.projectName}` : type
+
+  function accumulate(results: TestResults, file: string) {
+    reporter.onResult(
+      { ...results, tests: results.tests.map((t) => ({ ...t, filePath: file })) },
+      envLabel,
+    )
+    counts.passed += results.passed
+    counts.failed += results.failed
+    counts.skipped += results.skipped
+    counts.todo += results.todo
+  }
+
+  if (type === 'e2e') {
+    let allBrowserCoverageEntries: Array<{ entries: V8CoverageEntry[]; baseUrl: string }> = []
+
+    await runInConcurrentWorkers(
+      files,
+      concurrency,
+      (file) =>
+        runFileInWorker(
+          file,
+          type,
+          (results) => {
+            accumulate(results, file)
+            if (results.e2eBrowserCoverageEntries) {
+              allBrowserCoverageEntries.push(...results.e2eBrowserCoverageEntries)
+            }
+          },
+          {
+            ...options,
+            playwrightUseOpts: options.playwrightUseOpts,
+          },
+        ),
+      () => counts.failed++,
+    )
+
+    if (options.coverage && allBrowserCoverageEntries.length > 0) {
+      coverageMap = await collectCoverageMapFromPlaywright(
+        allBrowserCoverageEntries.flatMap((e) => e.entries),
+        cwd,
+        new Set(files),
+        async (urlPath) => (urlPath.startsWith('/') ? urlPath.slice(1) : urlPath),
+      )
+    }
+  } else {
+    let coverageDataDir: string | undefined
+    if (options.coverage) {
+      coverageDataDir = path.resolve(cwd, options.coverage.dir)
+      await fsp.mkdir(coverageDataDir, { recursive: true })
+      process.env.NODE_V8_COVERAGE = coverageDataDir
+    }
+
+    await runInConcurrentWorkers(
+      files,
+      concurrency,
+      (file) => runFileInWorker(file, type, (results) => accumulate(results, file), options),
+      () => counts.failed++,
+    )
+
+    if (coverageDataDir) {
+      delete process.env.NODE_V8_COVERAGE
+      let serverMap = await collectServerCoverageMap(coverageDataDir, cwd, new Set(files))
+      coverageMap = serverMap
+    }
+  }
+
+  return { ...counts, coverageMap }
+}
+
+async function runInConcurrentWorkers(
+  files: string[],
+  concurrency: number,
+  runFile: (file: string) => Promise<void>,
+  onError: () => void,
+): Promise<void> {
+  let index = 0
+  let active = 0
+
+  await new Promise<void>((resolve) => {
+    function dispatch() {
+      while (active < concurrency && index < files.length) {
+        let file = files[index]
+        index++
+        active++
+
+        runFile(file).then(
+          () => {
+            active--
+            if (index < files.length) {
+              dispatch()
+            } else if (active === 0) {
+              resolve()
+            }
+          },
+          (err) => {
+            console.error(`Error running ${file}:`, err.message)
+            console.error(err)
+            onError()
+            active--
+            if (active === 0 && index >= files.length) resolve()
+            else dispatch()
+          },
+        )
+      }
+
+      if (index >= files.length && active === 0) resolve()
+    }
+
+    dispatch()
+  })
+}
+
+function runFileInWorker(
+  file: string,
+  type: 'server' | 'e2e',
+  onResults: (results: TestResults) => void,
+  options: {
+    cwd?: string
+    coverage?: CoverageConfig
+    open?: boolean
+    playwrightUseOpts?: PlaywrightUseOpts
+  } = {},
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    let worker =
+      type === 'e2e'
+        ? new Worker(workerE2EUrl, {
+            workerData: {
+              file: pathToFileURL(file).href,
+              type,
+              coverage: options.coverage,
+              open: options.open,
+              playwrightUseOpts: options.playwrightUseOpts,
+            },
+          })
+        : new Worker(workerUrl, {
+            workerData: {
+              file: pathToFileURL(file).href,
+              type,
+              coverage: options.coverage,
+            },
+          })
+    worker.once('message', (msg: TestResults) => onResults(msg))
+    worker.once('error', reject)
+    worker.once('exit', (code) => {
+      if (code !== 0) reject(new Error(`Worker exited with code ${code}`))
+      else resolve()
+    })
+  })
+}

package/src/lib/runtime.ts

@@ -0,0 +1,2 @@
+// https://bun.com/docs/guides/util/detect-bun
+export const IS_BUN = typeof process.versions.bun === 'string'

package/src/lib/ts-transform.ts

@@ -0,0 +1,36 @@
+import { transform, type TsconfigRaw } from 'esbuild'
+import { getTsconfig, type TsConfigResult } from 'get-tsconfig'
+import * as path from 'node:path'
+
+const tsconfigCache = new Map<string, TsConfigResult | null>()
+
+/*
+ * Transform a TypeScript file to JavaScript using esbuild with an inline
+ * source map and no minification. Used by the coverage ESM loader hook (so V8
+ * instruments readable JS), the coverage collector (so byte offsets can be
+ * re-derived and mapped back to TypeScript lines), and the browser harness
+ * server (so the bytes V8 sees in the browser match what the collector
+ * re-derives). Identical inputs must produce identical outputs across all
+ * call sites or coverage offsets won't line up.
+ *
+ * Compiler options (notably JSX) are taken from the nearest `tsconfig.json`
+ * walking up from the file's directory, so each project picks up its own
+ * `jsxImportSource` etc. Discovery results are cached by directory.
+ */
+export async function transformTypeScript(
+  source: string,
+  filePath: string,
+): Promise<{ code: string }> {
+  let loader: 'ts' | 'tsx' = filePath.endsWith('.tsx') ? 'tsx' : 'ts'
+
+  let tsConfig = getTsconfig(path.dirname(filePath), 'tsconfig.json', tsconfigCache)
+
+  let result = await transform(source, {
+    loader,
+    sourcemap: 'inline',
+    sourcesContent: true,
+    sourcefile: filePath,
+    tsconfigRaw: { compilerOptions: tsConfig?.config.compilerOptions ?? {} },
+  })
+  return { code: result.code }
+}

package/src/lib/watcher.ts

@@ -0,0 +1,46 @@
+import * as fs from 'node:fs'
+
+function getFileModTime(file: string): number {
+  try {
+    return fs.statSync(file).mtimeMs
+  } catch {
+    return 0
+  }
+}
+
+export function createWatcher(onChange: (file: string) => void) {
+  let watchers = new Set<fs.FSWatcher>()
+  let fileModTimes = new Map<string, number>()
+
+  function update(files: string[]) {
+    for (let watcher of watchers) {
+      watcher.close()
+    }
+    watchers.clear()
+
+    for (let file of files) {
+      fileModTimes.set(file, getFileModTime(file))
+      watchers.add(
+        fs.watch(file, () => {
+          // macOS FSEvents can fire multiple callbacks per save (e.g. write +
+          // metadata flush). Guard with mtime so only a real content change
+          // triggers a rerun instead of every duplicate event.
+          let mtime = getFileModTime(file)
+          if (mtime !== fileModTimes.get(file)) {
+            fileModTimes.set(file, mtime)
+            onChange(file)
+          }
+        }),
+      )
+    }
+  }
+
+  function close() {
+    for (let watcher of watchers) {
+      watcher.close()
+    }
+    watchers.clear()
+  }
+
+  return { update, close }
+}