@specbook/core 0.2.0 → 0.4.0

package/src/reporters/list.ts

@@ -0,0 +1,254 @@
+// The default report: steps as they finish, then what failed (with the assertion, a source frame and the trace) and a summary.
+
+import { type DiffLine, diffAssertion } from '../diff'
+import type { RunEvent } from '../runner/run'
+import type { AssertionResult, RunResult, StepResult } from '../types'
+import { type Colors, createColors } from './color'
+import { reportTotals } from './envelope'
+import { type TraceOptions, traceLines } from './trace'
+
+export interface ListOptions {
+  colors?: Colors
+  /** only failures and the summary */
+  quiet?: boolean
+  /** show the request and response of a failed step (default), of every step, or not at all */
+  trace?: 'on-failure' | 'always' | 'off'
+  /** service → the address its calls go to (origin and base path), so a trace shows the URL that was called */
+  origins?: Record<string, string>
+  /** paths are shown relative to this folder */
+  cwd?: string
+  /** the text of a project file, for the source frame of a failed assertion */
+  readSource?: (file: string) => string | undefined
+  /** the command that runs just this flow again */
+  rerun?: (r: RunResult) => string | undefined
+  /** how long the whole run took, for the summary */
+  durationMs?: number
+}
+
+const MARKS = { pass: '✓', fail: '✗', error: '!', skipped: '–' } as const
+
+export const formatDuration = (ms: number): string =>
+  ms < 1000
+    ? `${Math.round(ms)}ms`
+    : ms < 60_000
+      ? `${(ms / 1000).toFixed(2)}s`
+      : `${Math.floor(ms / 60_000)}m ${Math.round((ms % 60_000) / 1000)}s`
+
+const leaves = (steps: StepResult[]): StepResult[] =>
+  steps.flatMap((s) => (s.children?.length ? leaves(s.children) : [s]))
+
+function paint(status: StepResult['status'], c: Colors, text: string): string {
+  return status === 'pass'
+    ? c.green(text)
+    : status === 'skipped'
+      ? c.dim(text)
+      : status === 'error'
+        ? c.yellow(text)
+        : c.red(text)
+}
+
+interface StepFacts {
+  status: StepResult['status']
+  flow: string
+  step: string
+  method?: string
+  url?: string
+  code?: number
+  ms?: number
+}
+
+/** `✓ Checkout › Create order  POST /orders  201  85ms` */
+export function stepLine(f: StepFacts, c: Colors, indent = '  '): string {
+  const detail = [
+    f.method && f.url ? `${f.method} ${f.url}` : '',
+    f.code ? String(f.code) : '',
+    f.ms !== undefined ? formatDuration(f.ms) : '',
+  ]
+    .filter(Boolean)
+    .join('  ')
+  // a request that is its own flow is named once
+  const name = f.flow === f.step ? f.step : `${f.flow} › ${f.step}`
+  return `${indent}${paint(f.status, c, MARKS[f.status])} ${name}${detail ? `  ${c.dim(detail)}` : ''}`
+}
+
+function diffText(lines: DiffLine[], c: Colors, indent: string): string[] {
+  return lines.map((l) =>
+    l.kind === 'expected'
+      ? `${indent}${c.green(`- ${l.text}`)}`
+      : l.kind === 'received'
+        ? `${indent}${c.red(`+ ${l.text}`)}`
+        : l.kind === 'skip'
+          ? `${indent}${c.dim(`  ${l.text}`)}`
+          : `${indent}${c.dim(`  ${l.text}`)}`,
+  )
+}
+
+/** The lines around `line` with a caret under `column`, from a file's text. */
+export function codeFrame(
+  source: string,
+  line: number,
+  column = 1,
+  c: Colors = createColors(false),
+  around = 2,
+): string[] {
+  const lines = source.split('\n')
+  const from = Math.max(1, line - around)
+  const to = Math.min(lines.length, line + around)
+  const width = String(to).length
+  const out: string[] = []
+  for (let n = from; n <= to; n++) {
+    const text = lines[n - 1] ?? ''
+    const mark = n === line ? c.red('>') : ' '
+    out.push(`${mark} ${c.dim(String(n).padStart(width))} ${c.dim('|')} ${text}`)
+    if (n === line)
+      out.push(
+        `  ${' '.repeat(width)} ${c.dim('|')} ${' '.repeat(Math.max(0, column - 1))}${c.red('^')}`,
+      )
+  }
+  return out
+}
+
+const rel = (file: string, cwd?: string) =>
+  cwd && file.startsWith(`${cwd}/`) ? file.slice(cwd.length + 1) : file
+
+function assertionBlock(a: AssertionResult, o: ListOptions, c: Colors): string[] {
+  const out = [`    ${c.red('✗')} ${a.label}`]
+  const diff = diffAssertion(a)
+  if (diff.length) {
+    out.push(`      ${c.green('- Expected')}  ${c.red('+ Received')}`)
+    out.push(...diffText(diff, c, '      '))
+  }
+  if (a.location) {
+    out.push(
+      `      ${c.dim(`at ${rel(a.location.file, o.cwd)}:${a.location.line}:${a.location.column ?? 1}`)}`,
+    )
+    const text = o.readSource?.(a.location.file)
+    if (text !== undefined)
+      for (const l of codeFrame(text, a.location.line, a.location.column, c))
+        out.push(`        ${l}`)
+  }
+  return out
+}
+
+/** One block per failed or errored step: the assertions that failed, why a step errored, and its trace. */
+export function formatFailures(results: RunResult[], o: ListOptions = {}): string {
+  const c = o.colors ?? createColors(false)
+  const out: string[] = []
+  for (const r of results) {
+    for (const s of leaves(r.steps)) {
+      if (s.status !== 'fail' && s.status !== 'error') continue
+      const where = r.file ? `${r.file} › ` : ''
+      out.push(
+        '',
+        `${c.bold(c.red(' FAIL '))} ${where}${r.title === s.name ? r.title : `${r.title} › ${s.name}`}  ${c.dim(`[${r.env}] ${formatDuration(s.durationMs)}`)}`,
+      )
+      if (s.status === 'error') out.push(`    ${c.yellow('!')} ${s.error ?? 'error'}`)
+      for (const a of s.assertions) if (!a.pass) out.push(...assertionBlock(a, o, c))
+      if ((o.trace ?? 'on-failure') !== 'off')
+        for (const l of traceLines(s, { origins: o.origins })) out.push(`    ${c.dim(l)}`)
+      const rerun = o.rerun?.(r)
+      if (rerun) out.push(`    ${c.dim(`Re-run: ${rerun}`)}`)
+    }
+  }
+  return out.join('\n')
+}
+
+/** `Flows 8 passed | 1 failed | 1 flaky`, steps, assertions, time and the slowest steps. */
+export function formatSummary(results: RunResult[], o: ListOptions = {}): string {
+  const c = o.colors ?? createColors(false)
+  const t = reportTotals(results)
+  const part = (n: number, word: string, paintFn: (s: string) => string) =>
+    n ? paintFn(`${n} ${word}`) : ''
+  const join = (...parts: string[]) => parts.filter(Boolean).join(c.dim(' | ')) || c.dim('none')
+  const lines = [
+    '',
+    `${c.bold(' Flows      ')} ${join(
+      part(t.flows.failed, 'failed', c.red),
+      part(t.flows.errored, 'errored', c.yellow),
+      part(t.flows.flaky, 'flaky', c.yellow),
+      part(t.flows.passed, 'passed', c.green),
+    )} ${c.dim(`(${results.length})`)}`,
+    `${c.bold(' Steps      ')} ${join(
+      part(t.steps.failed, 'failed', c.red),
+      part(t.steps.skipped, 'skipped', c.dim),
+      part(t.steps.passed, 'passed', c.green),
+    )}`,
+    `${c.bold(' Assertions ')} ${join(
+      part(t.assertions.failed, 'failed', c.red),
+      part(t.assertions.passed, 'passed', c.green),
+    )}`,
+  ]
+  const total = o.durationMs ?? results.reduce((n, r) => n + r.durationMs, 0)
+  lines.push(`${c.bold(' Duration   ')} ${formatDuration(total)}`)
+  const slow = results
+    .flatMap((r) =>
+      leaves(r.steps)
+        .filter((s) => s.status !== 'skipped')
+        // a request is a flow of one step: its name is said once
+        .map((s) => ({
+          name: r.title === s.name ? r.title : `${r.title} › ${s.name}`,
+          ms: s.durationMs,
+        })),
+    )
+    .sort((a, b) => b.ms - a.ms)
+    .slice(0, 5)
+  if (slow.length > 1) {
+    const times = slow.map((s) => formatDuration(s.ms))
+    const width = Math.max(...times.map((t) => t.length))
+    lines.push(c.bold(' Slowest'))
+    for (const [i, s] of slow.entries())
+      lines.push(`   ${c.dim(`${(times[i] as string).padStart(width)}  ${s.name}`)}`)
+  }
+  return lines.join('\n')
+}
+
+function stepFacts(r: RunResult, s: StepResult): StepFacts {
+  return {
+    status: s.status,
+    flow: r.title,
+    step: s.name,
+    method: s.request?.method,
+    url: s.request?.url,
+    code: s.response?.status,
+    ms: s.status === 'skipped' ? undefined : s.durationMs,
+  }
+}
+
+/** The whole report at once, from finished results (what the streaming reporter prints, in one string). */
+export function formatList(results: RunResult[], o: ListOptions = {}): string {
+  const c = o.colors ?? createColors(false)
+  const lines: string[] = []
+  if (!o.quiet)
+    for (const r of results)
+      for (const s of leaves(r.steps)) lines.push(stepLine(stepFacts(r, s), c))
+  const failures = formatFailures(results, o)
+  return [lines.join('\n'), failures, formatSummary(results, o)].filter((x) => x !== '').join('\n')
+}
+
+/** Prints each step as it finishes, from the run's events; at the end prints the failures and the summary. */
+export function createListReporter(o: ListOptions & { write(text: string): void }) {
+  const c = o.colors ?? createColors(false)
+  let flow = ''
+  const facts = new Map<string, Partial<StepFacts>>()
+  return {
+    onEvent(e: RunEvent) {
+      if (e.type === 'run:start') flow = e.title ?? e.id
+      else if (e.type === 'request')
+        facts.set(e.name, { ...facts.get(e.name), method: e.request.method, url: e.request.url })
+      else if (e.type === 'response')
+        facts.set(e.name, { ...facts.get(e.name), code: e.status, ms: e.durationMs })
+      else if (e.type === 'step:end') {
+        const f = facts.get(e.name)
+        facts.delete(e.name)
+        if (e.status === 'pass' && o.quiet) return
+        o.write(`${stepLine({ ...f, status: e.status, flow, step: e.name }, c)}\n`)
+      }
+    },
+    onEnd(results: RunResult[]) {
+      const rest = [formatFailures(results, o), formatSummary(results, o)].filter((x) => x !== '')
+      o.write(`${rest.join('\n')}\n`)
+    },
+  }
+}
+
+export type { TraceOptions }

package/src/reporters/redact.ts

@@ -0,0 +1,113 @@
+// What a report may show of a run. Resolved secret values are already masked by the runner; this removes what is secret
+// without being one of ours: credentials in headers, and tokens and passwords that an API returns or a request captures.
+
+import type { RunEvent } from '../runner/run'
+import type { AssertionResult, RunResult, StepResult } from '../types'
+
+export const REDACTED = '***'
+
+/** Headers whose values are never shown, whatever they hold. */
+const DENIED_HEADERS = new Set([
+  'authorization',
+  'proxy-authorization',
+  'cookie',
+  'set-cookie',
+  'x-api-key',
+  'x-auth-token',
+  'x-csrf-token',
+  'x-xsrf-token',
+])
+
+export const isDeniedHeader = (name: string) => DENIED_HEADERS.has(name.toLowerCase())
+
+/** `accessToken`, `access_token`, `client-secret` → words joined by `_`, lowercase. */
+const normalize = (key: string) =>
+  key
+    .replace(/([a-z0-9])([A-Z])/g, '$1_$2')
+    .replace(/[-.\s]+/g, '_')
+    .toLowerCase()
+
+const SENSITIVE =
+  /(^|_)(tokens?|secrets?|passwords?|passwd|authorization|credentials?|cookies?|api_?keys?|private_?keys?|access_?keys?|secret_?keys?)$/
+
+/** Whether a property named like this holds something that should not be printed (`access_token`, `password`, `apiKey`). */
+export const isSensitiveKey = (key: string) => SENSITIVE.test(normalize(key))
+
+function redactValue(v: unknown): unknown {
+  if (Array.isArray(v)) return v.map(redactValue)
+  if (v !== null && typeof v === 'object') {
+    return Object.fromEntries(
+      Object.entries(v).map(([k, x]) => [
+        k,
+        isSensitiveKey(k) && (typeof x === 'string' || typeof x === 'number')
+          ? REDACTED
+          : redactValue(x),
+      ]),
+    )
+  }
+  return v
+}
+
+export function redactHeaders(headers: Record<string, string>): Record<string, string> {
+  return Object.fromEntries(
+    Object.entries(headers).map(([k, v]) => [k, isDeniedHeader(k) ? REDACTED : v]),
+  )
+}
+
+function redactAssertion(a: AssertionResult): AssertionResult {
+  const hasValues = 'actual' in a || 'expected' in a
+  return {
+    ...a,
+    actual: redactValue(a.actual),
+    expected: redactValue(a.expected),
+    // a recorded diff is text made from the values; with the values present it is rebuilt from the clean ones
+    diff: hasValues ? undefined : a.diff,
+  }
+}
+
+function redactStep(s: StepResult): StepResult {
+  return {
+    ...s,
+    assertions: s.assertions.map(redactAssertion),
+    request: s.request
+      ? {
+          ...s.request,
+          headers: redactHeaders(s.request.headers),
+          body: redactValue(s.request.body),
+        }
+      : undefined,
+    response: s.response
+      ? {
+          ...s.response,
+          headers: redactHeaders(s.response.headers),
+          body: redactValue(s.response.body),
+        }
+      : undefined,
+    captured: redactValue(s.captured) as Record<string, unknown>,
+    children: s.children?.map(redactStep),
+  }
+}
+
+/** A copy of the results that is safe to print, attach to a build or send to a monitoring service. */
+export function redactResults(results: RunResult[]): RunResult[] {
+  return results.map((r) => ({ ...r, steps: r.steps.map(redactStep) }))
+}
+
+/** The same for one event of a run, before it is written to a log or a stream. */
+export function redactEvent(e: RunEvent): RunEvent {
+  if (e.type === 'request')
+    return {
+      ...e,
+      request: {
+        ...e.request,
+        headers: redactHeaders(e.request.headers),
+        body: redactValue(e.request.body),
+      },
+    }
+  if (e.type === 'capture')
+    return { ...e, captured: redactValue(e.captured) as Record<string, unknown> }
+  if (e.type === 'assert:pass' || e.type === 'assert:fail')
+    return { ...e, assertion: redactAssertion(e.assertion) }
+  if (e.type === 'done') return { ...e, result: redactResults([e.result])[0] as RunResult }
+  return e
+}

package/src/reporters/trace.ts

@@ -0,0 +1,70 @@
+// The request and response of a step as text, for a failure: what was sent, what came back, redacted and shortened.
+
+import { statusText } from '../diff'
+import type { StepResult } from '../types'
+import { isDeniedHeader, REDACTED } from './redact'
+
+/** Response headers worth showing: the rest is noise in a failure. */
+const SHOWN_RESPONSE_HEADERS = [
+  'content-type',
+  'x-request-id',
+  'x-correlation-id',
+  'x-amzn-trace-id',
+  'traceparent',
+  'retry-after',
+  'location',
+]
+
+export interface TraceOptions {
+  /** service → the address its calls go to (`https://api.example.com/v1`: origin and base path), to show the full URL */
+  origins?: Record<string, string>
+  /** characters of a body shown before it is cut (default 2048) */
+  maxBody?: number
+}
+
+function body(v: unknown, max: number): string[] {
+  if (v === undefined || v === null || v === '') return []
+  let text: string
+  try {
+    text = typeof v === 'string' ? v : (JSON.stringify(v, null, 2) ?? String(v))
+  } catch {
+    text = String(v)
+  }
+  const cut = text.length > max
+  const lines = (cut ? text.slice(0, max) : text).split('\n')
+  if (cut) lines.push(`… ${text.length - max} more characters`)
+  return lines
+}
+
+const query = (q: Record<string, string | string[]>) => {
+  const parts = Object.entries(q).flatMap(([k, v]) =>
+    (Array.isArray(v) ? v : [v]).map((x) => `${k}=${x}`),
+  )
+  return parts.length ? `?${parts.join('&')}` : ''
+}
+
+/** The trace of one step as lines (without indentation): `Request`, its headers and body, `Response`, and the same. */
+export function traceLines(step: StepResult, o: TraceOptions = {}): string[] {
+  const max = o.maxBody ?? 2048
+  const out: string[] = []
+  const r = step.request
+  if (r) {
+    const origin = o.origins?.[r.service] ?? ''
+    out.push(`Request   ${r.method} ${origin}${r.url}${query(r.query)}`)
+    for (const [k, v] of Object.entries(r.headers))
+      out.push(`  ${k}: ${isDeniedHeader(k) ? REDACTED : v}`)
+    for (const l of body(r.body, max)) out.push(`  ${l}`)
+  }
+  const res = step.response
+  if (res) {
+    out.push(
+      `Response  ${[res.status, statusText(res.status)].filter(Boolean).join(' ')}  ${res.durationMs}ms`,
+    )
+    for (const name of SHOWN_RESPONSE_HEADERS) {
+      const v = res.headers[name]
+      if (v !== undefined) out.push(`  ${name}: ${isDeniedHeader(name) ? REDACTED : v}`)
+    }
+    for (const l of body(res.body, max)) out.push(`  ${l}`)
+  }
+  return out
+}

package/src/request.ts

@@ -2,6 +2,7 @@ import { deepMerge } from './template'
 import type {
   AnyOp,
   Init,
+  MockHook,
   Operation,
   OpMeta,
   OpTypes,
@@ -15,6 +16,7 @@ function make<T extends OpTypes, I extends Init, C>(
   input: RequestInput<T, I, C>,
   dataPatches: readonly unknown[],
   varOverrides: Readonly<Record<string, unknown>> = {},
+  mockHook?: MockHook<T, I>,
 ): RequestDef<T, I, C> {
   return {
     kind: 'request',
@@ -22,6 +24,7 @@ function make<T extends OpTypes, I extends Init, C>(
     input,
     dataPatches,
     varOverrides,
+    mockHook,
     with<C2>(o: RequestOverrides<T, I, C2>) {
       const next: RequestInput<T, any, any> = { ...input }
       if (o.path) next.path = deepMerge(input.path as any, o.path) as any
@@ -29,16 +32,31 @@ function make<T extends OpTypes, I extends Init, C>(
       if (o.headers) next.headers = { ...(input.headers ?? {}), ...o.headers }
       if (o.capture) next.capture = { ...(input.capture ?? {}), ...o.capture } as any
       if (o.assert) next.assert = o.assert
-      if (o.mock) next.mock = { ...(input.mock ?? {}), ...o.mock } as any
       if (o.auth !== undefined) next.auth = o.auth
       if (o.assertMode) next.assertMode = o.assertMode
+      if (o.tags) next.tags = [...new Set([...(input.tags ?? []), ...o.tags])]
       const patches = o.data !== undefined ? [...dataPatches, o.data] : dataPatches
       const overrides = o.vars ? { ...varOverrides, ...o.vars } : varOverrides
-      return make(op, next, patches, overrides) as RequestDef<T, I, C & C2>
+      return make(op, next, patches, overrides, mockHook) as RequestDef<T, I, C & C2>
     },
   }
 }
 
+/**
+ * How the `mock` environment answers this request, when the generated example (filled from the request) is not enough.
+ * Rarely needed; `req` and `res` are typed from the operation. Returns the request, so it is the file's one export:
+ *
+ *     export default withMock(GetOrder.request({ ... }), (req, res) => {
+ *       if (res.status === 200) res.data.status = 'placed'
+ *     })
+ */
+export function withMock<T extends OpTypes, I extends Init, C>(
+  def: RequestDef<T, I, C>,
+  hook: MockHook<T, I>,
+): RequestDef<T, I, C> {
+  return make(def.op, def.input, def.dataPatches, def.varOverrides, hook)
+}
+
 /** Generated code calls this once per operation. */
 export function defineOperation<T extends OpTypes = AnyOp>(meta: OpMeta): Operation<T> {
   return {

package/src/runner/index.ts

@@ -1,3 +1,4 @@
 export * from './encode'
 export * from './run'
+export * from './suite'
 export * from './transport'