@specbook/core 0.2.0 → 0.4.0

package/src/mock.ts

@@ -1,8 +1,8 @@
-// In-process mock transport: serves request-level mock overrides, falling back to generated examples.
+// In-process mock transport: answers from the generated examples, filled in from the request that was sent, and lets a
+// request's `withMock` hook change the answer.
 import { reflectRequest } from './mock-reflect'
 import type { PreparedRequest, RawResponse, Transport } from './runner/transport'
 import { deepMerge, render } from './template'
-import type { MockResponse } from './types'
 
 export interface MockExampleLoader {
   /** Statuses that have an example for this operation, in spec order. */
@@ -12,48 +12,35 @@ export interface MockExampleLoader {
 
 export interface MockTransportOptions {
   loader: MockExampleLoader
-  sleep?: (ms: number) => Promise<void>
+  /** Waits a mock's `delayMs`. The default can be ended early by the run's signal: a person pressing Stop. */
+  sleep?: (ms: number, signal?: AbortSignal) => Promise<void>
 }
 
-const isMockResponse = (v: unknown): v is MockResponse<unknown> =>
-  v !== null &&
-  typeof v === 'object' &&
-  ('data' in (v as object) ||
-    'init' in (v as object) ||
-    'headers' in (v as object) ||
-    'delayMs' in (v as object))
+const is2xx = (s: number) => s >= 200 && s < 300
 
-function evalInit(init: Record<string, unknown> | undefined) {
-  return Object.fromEntries(
-    Object.entries(init ?? {}).map(([k, v]) => [
-      k,
-      typeof v === 'function' ? (v as () => unknown)() : v,
-    ]),
-  )
-}
+/** The loader may hand out one object to every caller, and a hook edits what it is given. */
+const copy = <T>(v: T): T => (v === undefined ? v : (JSON.parse(JSON.stringify(v)) as T))
 
 export function createMockTransport(opts: MockTransportOptions): Transport {
-  const sleep = opts.sleep ?? ((ms) => new Promise<void>((r) => setTimeout(r, ms)))
-  return async (req: PreparedRequest): Promise<RawResponse> => {
+  const sleep =
+    opts.sleep ??
+    ((ms: number, signal?: AbortSignal) =>
+      new Promise<void>((done, stop) => {
+        if (signal?.aborted) return stop(new Error('Stopped before the API answered.'))
+        const timer = setTimeout(done, ms)
+        signal?.addEventListener(
+          'abort',
+          () => {
+            clearTimeout(timer)
+            stop(new Error('Stopped before the API answered.'))
+          },
+          { once: true },
+        )
+      }))
+  return async (req: PreparedRequest, signal?: AbortSignal): Promise<RawResponse> => {
     const hints = req.mock
-    const cfg = hints?.config
-    const known = await opts.loader.statuses(req.service, req.opId)
-    const overridden = Object.keys(cfg?.responses ?? {}).map(Number)
-    const all = [...new Set([...known, ...overridden])]
-
-    const fromWhen = cfg?.when?.({
-      vars: (hints?.vars ?? {}) as never,
-      body: req.body,
-      query: hints?.rawQuery,
-      path: hints?.pathParams,
-    })
-    const status =
-      req.mockStatus ??
-      fromWhen ??
-      cfg?.status ??
-      all.find((s) => s >= 200 && s < 300) ??
-      all[0] ??
-      200
+    const all = await opts.loader.statuses(req.service, req.opId)
+    const status = req.mockStatus ?? all.find(is2xx) ?? all[0] ?? 200
 
     // Secured operations answer 401 when no credential header was applied.
     if (
@@ -67,42 +54,65 @@ export function createMockTransport(opts: MockTransportOptions): Transport {
       return json(401, body)
     }
 
-    const override = (cfg?.responses as Record<number, unknown> | undefined)?.[status]
-    let delay = 0
-    let headers: Record<string, string> = {}
-    let scope = { ...(hints?.vars ?? {}), ...(hints?.pathParams ?? {}), ...(hints?.rawQuery ?? {}) }
-    let template: unknown = await opts.loader.example(req.service, req.opId, status)
+    const scope = {
+      ...(hints?.vars ?? {}),
+      ...(hints?.pathParams ?? {}),
+      ...(hints?.rawQuery ?? {}),
+    }
     // The generated example answers the request that was sent, so a created thing has the fields you posted.
-    if (status >= 200 && status < 300 && cfg?.reflect !== false)
-      template = reflectRequest(template, {
-        body: req.body,
-        path: hints?.pathParams,
-        query: hints?.rawQuery,
-      })
     // A declared status without a response schema (e.g. Petstore's 404) has no example: empty body.
-    if (override !== undefined) {
-      // Overrides are layered over the generated example (objects merge, arrays/scalars replace).
-      let data: unknown
-      if (isMockResponse(override)) {
-        data = await resolveSource(override.data)
-        scope = { ...scope, ...evalInit(override.init) }
-        headers = override.headers ? render(override.headers, scope) : {}
-        delay = override.delayMs ?? 0
-      } else {
-        data = await resolveSource(override)
-      }
-      if (data !== undefined) template = deepMerge(template, data)
+    const answer = async (s: number) => {
+      let template: unknown = copy(await opts.loader.example(req.service, req.opId, s))
+      if (is2xx(s))
+        template = reflectRequest(template, {
+          body: req.body,
+          path: hints?.pathParams,
+          query: hints?.rawQuery,
+        })
+      return template === undefined ? undefined : render(template, scope)
+    }
+
+    const res: {
+      status: number
+      data: unknown
+      headers: Record<string, string>
+      delayMs: number
+      respond(status: number, data?: unknown): void
+    } = {
+      status,
+      data: await answer(status),
+      headers: {},
+      delayMs: 0,
+      respond: (s, data) => {
+        pending = { status: s, data }
+      },
+    }
+    let pending: { status: number; data?: unknown } | undefined
+
+    if (hints?.hook) {
+      await hints.hook(
+        {
+          vars: (hints.vars ?? {}) as never,
+          body: req.body,
+          query: hints.rawQuery,
+          path: hints.pathParams,
+          headers: req.headers,
+        },
+        res as never,
+      )
+    }
+    if (pending) {
+      const base = await answer(pending.status)
+      res.status = pending.status
+      res.data = pending.data === undefined ? base : deepMerge(base, pending.data)
     }
-    if (delay) await sleep(delay)
-    const body = template === undefined ? undefined : render(template, scope)
-    return { ...json(status, body), headers: { 'content-type': 'application/json', ...headers } }
-  }
-}
 
-async function resolveSource(src: unknown): Promise<unknown> {
-  if (typeof src === 'function')
-    return (await (src as () => Promise<{ default: unknown }>)()).default
-  return src
+    if (res.delayMs) await sleep(res.delayMs, signal)
+    return {
+      ...json(res.status, res.data),
+      headers: { 'content-type': 'application/json', ...res.headers },
+    }
+  }
 }
 
 function json(status: number, body: unknown): RawResponse {

package/src/reporters/ci.ts

@@ -0,0 +1,53 @@
+// Where a run happens, from the environment a CI system sets, for the report's header and its links.
+
+export interface CiInfo {
+  provider: string
+  runUrl?: string
+  sha?: string
+  branch?: string
+}
+
+export function detectCi(env: Record<string, string | undefined>): CiInfo | undefined {
+  if (env.GITHUB_ACTIONS === 'true') {
+    const server = env.GITHUB_SERVER_URL ?? 'https://github.com'
+    return {
+      provider: 'github-actions',
+      runUrl:
+        env.GITHUB_REPOSITORY && env.GITHUB_RUN_ID
+          ? `${server}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}`
+          : undefined,
+      sha: env.GITHUB_SHA,
+      branch: env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME,
+    }
+  }
+  if (env.GITLAB_CI === 'true')
+    return {
+      provider: 'gitlab-ci',
+      runUrl: env.CI_PIPELINE_URL,
+      sha: env.CI_COMMIT_SHA,
+      branch: env.CI_COMMIT_REF_NAME,
+    }
+  if (env.CIRCLECI === 'true')
+    return {
+      provider: 'circleci',
+      runUrl: env.CIRCLE_BUILD_URL,
+      sha: env.CIRCLE_SHA1,
+      branch: env.CIRCLE_BRANCH,
+    }
+  if (env.BUILDKITE === 'true')
+    return {
+      provider: 'buildkite',
+      runUrl: env.BUILDKITE_BUILD_URL,
+      sha: env.BUILDKITE_COMMIT,
+      branch: env.BUILDKITE_BRANCH,
+    }
+  if (env.JENKINS_URL)
+    return {
+      provider: 'jenkins',
+      runUrl: env.BUILD_URL,
+      sha: env.GIT_COMMIT,
+      branch: env.GIT_BRANCH,
+    }
+  if (env.CI) return { provider: 'ci' }
+  return undefined
+}

package/src/reporters/color.ts

@@ -0,0 +1,34 @@
+// Terminal colors, only when they will be seen: a TTY or a CI log that renders them, not NO_COLOR, or FORCE_COLOR.
+
+export interface Colors {
+  red(s: string): string
+  green(s: string): string
+  yellow(s: string): string
+  cyan(s: string): string
+  dim(s: string): string
+  bold(s: string): string
+}
+
+/** CI systems whose logs render color though the stream is not a terminal. */
+const COLOR_CI = ['GITHUB_ACTIONS', 'GITLAB_CI', 'CIRCLECI', 'BUILDKITE']
+
+/** Whether to color output: FORCE_COLOR wins, then NO_COLOR, then an interactive terminal or a CI log that shows color. */
+export function colorEnabled(env: Record<string, string | undefined>, isTTY: boolean): boolean {
+  if (env.FORCE_COLOR && env.FORCE_COLOR !== '0') return true
+  if (env.NO_COLOR !== undefined && env.NO_COLOR !== '') return false
+  if (COLOR_CI.some((k) => env[k] && env[k] !== 'false')) return true
+  return isTTY && env.TERM !== 'dumb'
+}
+
+export function createColors(enabled: boolean): Colors {
+  const wrap = (open: number, close: number) => (s: string) =>
+    enabled ? `\u001b[${open}m${s}\u001b[${close}m` : s
+  return {
+    red: wrap(31, 39),
+    green: wrap(32, 39),
+    yellow: wrap(33, 39),
+    cyan: wrap(36, 39),
+    dim: wrap(2, 22),
+    bold: wrap(1, 22),
+  }
+}

package/src/reporters/envelope.ts

@@ -0,0 +1,63 @@
+// The JSON report: the results with what a monitoring system needs to know about the run around them.
+
+import type { RunResult } from '../types'
+import type { CiInfo } from './ci'
+
+export interface ReportMeta {
+  env: string
+  /** version of the specbook that ran it */
+  specbook?: string
+  node?: string
+  os?: string
+  git?: { sha?: string; branch?: string }
+  ci?: CiInfo
+  /** `bundle.json` details when run from a bundle */
+  bundle?: { sha?: string; createdAt?: string }
+  /** service → origin of the address its calls went to */
+  services?: Record<string, string>
+  startedAt: string
+  finishedAt: string
+}
+
+export interface ReportTotals {
+  flows: { passed: number; failed: number; errored: number; flaky: number }
+  steps: { passed: number; failed: number; skipped: number }
+  assertions: { passed: number; failed: number }
+}
+
+export interface RunReport extends ReportMeta {
+  schemaVersion: 2
+  durationMs: number
+  totals: ReportTotals
+  results: RunResult[]
+}
+
+export function reportTotals(results: RunResult[]): ReportTotals {
+  const t: ReportTotals = {
+    flows: { passed: 0, failed: 0, errored: 0, flaky: 0 },
+    steps: { passed: 0, failed: 0, skipped: 0 },
+    assertions: { passed: 0, failed: 0 },
+  }
+  for (const r of results) {
+    if (r.status === 'pass') t.flows.passed++
+    else if (r.status === 'error') t.flows.errored++
+    else t.flows.failed++
+    if (r.flaky) t.flows.flaky++
+    t.steps.passed += r.totals.passed
+    t.steps.failed += r.totals.failed
+    t.steps.skipped += r.totals.skipped
+    t.assertions.passed += r.totals.assertions.passed
+    t.assertions.failed += r.totals.assertions.failed
+  }
+  return t
+}
+
+export function buildReport(results: RunResult[], meta: ReportMeta): RunReport {
+  return {
+    schemaVersion: 2,
+    ...meta,
+    durationMs: Math.max(0, Date.parse(meta.finishedAt) - Date.parse(meta.startedAt)),
+    totals: reportTotals(results),
+    results,
+  }
+}

package/src/reporters/index.ts

@@ -1,5 +1,8 @@
 import { stripAnsi } from '../assert'
+import { diffAssertion } from '../diff'
 import type { RunResult, StepResult } from '../types'
+import { reportTotals } from './envelope'
+import { traceLines } from './trace'
 
 export interface Reporter {
   onEvent?(e: unknown): void
@@ -40,35 +43,74 @@ export function formatPretty(results: RunResult[]): string {
 
 const esc = (s: unknown) =>
   String(s).replace(
-    /[<>&"']/g,
-    (c) => ({ '<': '&lt;', '>': '&gt;', '&': '&amp;', '"': '&quot;', "'": '&apos;' })[c] as string,
+    // biome-ignore lint/suspicious/noControlCharactersInRegex: characters XML 1.0 forbids
+    /[<>&"']|[\u0000-\u0008\u000b\u000c\u000e-\u001f]/g,
+    (c) => ({ '<': '&lt;', '>': '&gt;', '&': '&amp;', '"': '&quot;', "'": '&apos;' })[c] ?? '',
   )
 
-export function toJUnit(results: RunResult[]): string {
+export interface JUnitOptions {
+  /** ISO time the run began; shown on each suite */
+  timestamp?: string
+  hostname?: string
+  /** shown as `<properties>` on each suite (env, git sha, seed…) */
+  properties?: Record<string, string | undefined>
+  /** service → the address its calls go to (origin and base path), for the trace in `<system-out>` */
+  origins?: Record<string, string>
+}
+
+const leafSteps = (steps: StepResult[]): StepResult[] =>
+  steps.flatMap((s) => (s.children?.length ? leafSteps(s.children) : [s]))
+
+/**
+ * JUnit XML: one suite per flow or request, one case per step (named `flow › step`, class = the project file), failures
+ * with the expected and received values, errors with their message, skipped steps, a flow that passed on a retry as a
+ * `<flakyFailure>`, and the redacted request/response trace in `<system-out>`.
+ */
+export function toJUnit(results: RunResult[], o: JUnitOptions = {}): string {
   const suites = results.map((r) => {
-    const flat: StepResult[] = []
-    const walk = (s: StepResult) => (s.children?.length ? s.children.forEach(walk) : flat.push(s))
-    r.steps.forEach(walk)
-    const failures = flat.filter((s) => s.status === 'fail' || s.status === 'error').length
+    const flat = leafSteps(r.steps)
+    const failures = flat.filter((s) => s.status === 'fail').length
+    const errors = flat.filter((s) => s.status === 'error').length
+    const skipped = flat.filter((s) => s.status === 'skipped').length
+    const props = Object.entries({ env: r.env, seed: r.seed, ...o.properties })
+      .filter(([, v]) => v !== undefined && v !== '')
+      .map(([k, v]) => `      <property name="${esc(k)}" value="${esc(v)}"/>`)
+      .join('\n')
     const cases = flat
       .map((s) => {
         const time = (s.durationMs / 1000).toFixed(3)
         const failed = s.assertions.filter((a) => !a.pass)
         let inner = ''
         if (s.status === 'skipped') inner = '<skipped/>'
-        else if (s.status === 'error') inner = `<error message="${esc(s.error ?? 'error')}"/>`
+        else if (s.status === 'error')
+          inner = `<error message="${esc(s.error ?? 'error')}" type="Error">${esc(s.error ?? 'error')}</error>`
         else if (failed.length) {
           inner = failed
-            .map(
-              (a) =>
-                `<failure message="${esc(a.label)}">${esc(`expected: ${JSON.stringify(a.expected)}\nactual: ${JSON.stringify(a.actual)}`)}</failure>`,
-            )
+            .map((a) => {
+              const lines = diffAssertion(a)
+                .filter((l) => l.kind !== 'skip')
+                .map(
+                  (l) =>
+                    `${l.kind === 'expected' ? '- ' : l.kind === 'received' ? '+ ' : '  '}${l.text}`,
+                )
+              const at = a.location
+                ? `\nat ${a.location.file}:${a.location.line}:${a.location.column ?? 1}`
+                : ''
+              return `<failure message="${esc(a.label)}" type="${esc(a.operator)}">${esc(`${lines.join('\n')}${at}`)}</failure>`
+            })
             .join('')
         }
-        return `    <testcase classname="${esc(r.id)}" name="${esc(s.name)}" time="${time}">${inner}</testcase>`
+        if (r.flaky && s.status === 'pass' && (r.attempts ?? 1) > 1)
+          inner += `<flakyFailure message="passed on attempt ${r.attempts}"/>`
+        const trace =
+          s.status === 'fail' || s.status === 'error'
+            ? `<system-out>${esc(traceLines(s, { origins: o.origins }).join('\n'))}</system-out>`
+            : ''
+        return `    <testcase classname="${esc(r.file ?? r.id)}" name="${esc(`${r.title} › ${s.name}`)}" time="${time}">${inner}${trace}</testcase>`
       })
       .join('\n')
-    return `  <testsuite name="${esc(r.title)}" tests="${flat.length}" failures="${failures}" time="${(r.durationMs / 1000).toFixed(3)}">\n${cases}\n  </testsuite>`
+    const stamp = r.startedAt || o.timestamp
+    return `  <testsuite name="${esc(r.title)}" tests="${flat.length}" failures="${failures}" errors="${errors}" skipped="${skipped}" time="${(r.durationMs / 1000).toFixed(3)}"${stamp ? ` timestamp="${esc(stamp)}"` : ''}${o.hostname ? ` hostname="${esc(o.hostname)}"` : ''}>\n    <properties>\n${props}\n    </properties>\n${cases}\n  </testsuite>`
   })
   return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites>\n${suites.join('\n')}\n</testsuites>\n`
 }
@@ -82,24 +124,96 @@ export function exitCode(results: RunResult[]): 0 | 1 | 2 {
   return 0
 }
 
-/** GitHub Actions workflow commands: one annotation per failing assertion/step. */
-export function formatGithub(results: RunResult[]): string {
-  const esc = (v: string) => v.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A')
+/** For `specbook test`: 0 when every flow passed (a flaky one counts), 1 when any failed or errored. */
+export function testExitCode(results: RunResult[]): 0 | 1 {
+  return results.some((r) => r.status !== 'pass') ? 1 : 0
+}
+
+export interface GithubOptions {
+  /** folder the paths are made relative to (the project or repository root) */
+  cwd?: string
+}
+
+/** A workflow command property: `%`, newlines, `:` and `,` are escaped. */
+const prop = (v: string) =>
+  v
+    .replace(/%/g, '%25')
+    .replace(/\r/g, '%0D')
+    .replace(/\n/g, '%0A')
+    .replace(/:/g, '%3A')
+    .replace(/,/g, '%2C')
+const data = (v: string) => v.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A')
+
+/** GitHub Actions workflow commands: one annotation per failing assertion/step, a warning for a flaky flow. */
+export function formatGithub(results: RunResult[], o: GithubOptions = {}): string {
+  const rel = (f: string) => (o.cwd && f.startsWith(`${o.cwd}/`) ? f.slice(o.cwd.length + 1) : f)
   const lines: string[] = []
   const visit = (r: RunResult, s: StepResult) => {
     for (const c of s.children ?? []) visit(r, c)
-    if (s.status === 'error' || s.status === 'skipped') {
-      if (s.status === 'error')
-        lines.push(`::error title=${esc(`${r.title}: ${s.name}`)}::${esc(s.error ?? 'error')}`)
-      return
-    }
+    if (s.children?.length) return
+    const title = prop(`${r.title}: ${s.name}`)
+    const file = r.file ? `file=${prop(r.file)},` : ''
+    if (s.status === 'error')
+      lines.push(`::error ${file}title=${title}::${data(s.error ?? 'error')}`)
+    if (s.status === 'skipped' || s.status === 'error') return
     for (const a of s.assertions.filter((x) => !x.pass)) {
-      const loc = a.location ? `file=${esc(a.location.file)},line=${a.location.line},` : ''
-      lines.push(`::error ${loc}title=${esc(`${r.title}: ${s.name}`)}::${esc(a.label)}`)
+      const loc = a.location ? `file=${prop(rel(a.location.file))},line=${a.location.line},` : file
+      lines.push(`::error ${loc}title=${title}::${data(a.label)}`)
     }
   }
-  for (const r of results) for (const s of r.steps) visit(r, s)
+  for (const r of results) {
+    for (const s of r.steps) visit(r, s)
+    if (r.flaky)
+      lines.push(
+        `::warning ${r.file ? `file=${prop(r.file)},` : ''}title=${prop(`${r.title}: flaky`)}::${data(`passed on attempt ${r.attempts}`)}`,
+      )
+  }
   const ok = results.filter((r) => r.status === 'pass').length
   lines.push(`${ok}/${results.length} runs passed`)
   return lines.join('\n')
 }
+
+/** Markdown for `$GITHUB_STEP_SUMMARY`: totals, a table of flows, and what failed. */
+export function githubSummary(
+  results: RunResult[],
+  meta: { env: string; runUrl?: string } = { env: '' },
+): string {
+  const t = reportTotals(results)
+  const mark = (r: RunResult) =>
+    r.status === 'pass' ? (r.flaky ? '⚠️ flaky' : '✅') : r.status === 'error' ? '❗ error' : '❌'
+  const cell = (s: string) => s.replace(/\|/g, '\\|')
+  const lines = [
+    `### specbook test${meta.env ? ` · ${meta.env}` : ''}`,
+    '',
+    `**${t.flows.passed}** passed · **${t.flows.failed + t.flows.errored}** failed · **${t.flows.flaky}** flaky · ${results.length} total`,
+    '',
+    '| | Flow | Steps | Assertions | Time |',
+    '| --- | --- | --- | --- | --- |',
+    ...results.map(
+      (r) =>
+        `| ${mark(r)} | ${cell(r.title)} | ${r.totals.passed}/${r.totals.steps} | ${r.totals.assertions.passed}/${r.totals.assertions.passed + r.totals.assertions.failed} | ${(r.durationMs / 1000).toFixed(2)}s |`,
+    ),
+  ]
+  const bad = results.flatMap((r) =>
+    leafSteps(r.steps)
+      .filter((s) => s.status === 'fail' || s.status === 'error')
+      .map((s) => ({ r, s })),
+  )
+  if (bad.length) {
+    lines.push('', '#### Failures', '')
+    for (const { r, s } of bad) {
+      lines.push(`- **${cell(r.title)} › ${cell(s.name)}**`)
+      if (s.error) lines.push(`  - ${s.error}`)
+      for (const a of s.assertions.filter((x) => !x.pass)) lines.push(`  - ${a.label}`)
+    }
+  }
+  if (meta.runUrl) lines.push('', `[Run](${meta.runUrl})`)
+  return `${lines.join('\n')}\n`
+}
+
+export * from './ci'
+export * from './color'
+export * from './envelope'
+export * from './list'
+export * from './redact'
+export * from './trace'