@specbook/core 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/assert.ts +17 -6
- package/src/config.ts +15 -2
- package/src/controls.ts +34 -9
- package/src/diff.ts +131 -0
- package/src/environment.ts +175 -0
- package/src/index.ts +5 -1
- package/src/mock.ts +80 -70
- package/src/reporters/ci.ts +53 -0
- package/src/reporters/color.ts +34 -0
- package/src/reporters/envelope.ts +63 -0
- package/src/reporters/index.ts +139 -25
- package/src/reporters/list.ts +254 -0
- package/src/reporters/redact.ts +113 -0
- package/src/reporters/trace.ts +70 -0
- package/src/request.ts +20 -2
- package/src/runner/index.ts +1 -0
- package/src/runner/run.ts +113 -27
- package/src/runner/suite.ts +122 -0
- package/src/runner/transport.ts +65 -5
- package/src/secrets.ts +31 -2
- package/src/sequence.ts +4 -4
- package/src/spec-config.ts +77 -11
- package/src/template.ts +13 -4
- package/src/types.ts +49 -32
package/src/runner/run.ts
CHANGED
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
isFakerName,
|
|
12
12
|
isPlainObject,
|
|
13
13
|
render,
|
|
14
|
+
rootName,
|
|
14
15
|
} from '../template'
|
|
15
16
|
import type {
|
|
16
17
|
ApiResponse,
|
|
@@ -23,7 +24,12 @@ import type {
|
|
|
23
24
|
VarUse,
|
|
24
25
|
} from '../types'
|
|
25
26
|
import { describeFiles, isBlob } from './encode'
|
|
26
|
-
import
|
|
27
|
+
import {
|
|
28
|
+
DEFAULT_TIMEOUT_MS,
|
|
29
|
+
type PreparedRequest,
|
|
30
|
+
type RawResponse,
|
|
31
|
+
type Transport,
|
|
32
|
+
} from './transport'
|
|
27
33
|
|
|
28
34
|
// ---------------- public options ----------------
|
|
29
35
|
|
|
@@ -41,12 +47,22 @@ export interface AuthProfile {
|
|
|
41
47
|
token?: (res: ApiResponse<number, any>) => Session
|
|
42
48
|
apply: { headers?: Record<string, string>; query?: Record<string, string> }
|
|
43
49
|
retryOn401?: boolean
|
|
44
|
-
/**
|
|
50
|
+
/**
|
|
51
|
+
* an interactive sign-in the UI can offer (authorization code with PKCE); the runner ignores it.
|
|
52
|
+
* `authorizationUrl`, `tokenUrl`, `openIdConnectUrl` and `clientId` may use environment variables
|
|
53
|
+
* (`'{{issuer}}/token'`).
|
|
54
|
+
*/
|
|
45
55
|
oauth?: OAuthSettings
|
|
46
56
|
}
|
|
47
57
|
|
|
48
58
|
export type RunEvent =
|
|
49
|
-
| {
|
|
59
|
+
| {
|
|
60
|
+
type: 'run:start'
|
|
61
|
+
id: string
|
|
62
|
+
title?: string
|
|
63
|
+
kind?: 'request' | 'sequence'
|
|
64
|
+
tags?: string[]
|
|
65
|
+
}
|
|
50
66
|
| { type: 'step:start'; name: string }
|
|
51
67
|
| { type: 'auth:login'; profile: string }
|
|
52
68
|
| { type: 'request'; name: string; request: RenderedRequest }
|
|
@@ -60,21 +76,31 @@ export type RunEvent =
|
|
|
60
76
|
|
|
61
77
|
export interface RunOptions {
|
|
62
78
|
env: string
|
|
63
|
-
/** config.vars
|
|
79
|
+
/** config.vars, then environments[env].vars, then the environment's file. */
|
|
64
80
|
vars?: Record<string, unknown>
|
|
65
81
|
/** Highest-precedence overrides (UI vars panel, CLI --var). */
|
|
66
82
|
overrides?: Record<string, unknown>
|
|
67
|
-
/**
|
|
68
|
-
|
|
83
|
+
/** Flow input values: the ones a person would set (UI controls, CLI --input). */
|
|
84
|
+
inputs?: Record<string, unknown>
|
|
69
85
|
transport: Transport
|
|
70
86
|
/** Resolved secret values, keyed by secret name. Not needed in the mock env. */
|
|
71
87
|
secrets?: Record<string, string>
|
|
88
|
+
/**
|
|
89
|
+
* Refuse to send a request that may change data (anything but GET, HEAD and OPTIONS, a GraphQL mutation, or a request
|
|
90
|
+
* not marked `safe`). Auth logins are exempt. For runs against real environments that must not write.
|
|
91
|
+
*/
|
|
92
|
+
readOnly?: boolean
|
|
93
|
+
/**
|
|
94
|
+
* A request with no `assert` of its own passes only on a 2xx response (recorded as the assertion `status is 2xx`);
|
|
95
|
+
* one with an `assert` is judged by that alone. Off, a request with no `assert` passes whatever the status.
|
|
96
|
+
*/
|
|
97
|
+
implicitStatusCheck?: boolean
|
|
72
98
|
/** Leave `{{secrets.*}}` placeholders in the wire request (a trusted server substitutes them). */
|
|
73
99
|
deferSecrets?: boolean
|
|
74
100
|
auth?: Record<string, AuthProfile>
|
|
75
101
|
/** service -> default auth profile name. */
|
|
76
102
|
serviceAuth?: Record<string, string>
|
|
77
|
-
/** Called at `input()` steps; omit for headless (
|
|
103
|
+
/** Called at `input()` steps; omit for headless (inputs given > default > first option). */
|
|
78
104
|
onInput?: (req: {
|
|
79
105
|
name: string
|
|
80
106
|
controls: ReturnType<typeof describeControls>
|
|
@@ -91,6 +117,13 @@ export interface RunOptions {
|
|
|
91
117
|
example?: ExampleLoader
|
|
92
118
|
/** Reads files named by upload fields (path relative to the example's `dir`). */
|
|
93
119
|
readFile?: FileReader
|
|
120
|
+
/**
|
|
121
|
+
* How long each request waits for an answer, in milliseconds, when the request does not say (`timeoutMs` on the request
|
|
122
|
+
* wins). 0 waits for as long as it takes. Default: 30 seconds.
|
|
123
|
+
*/
|
|
124
|
+
timeoutMs?: number
|
|
125
|
+
/** The `mock` environment answers with this status (the UI's "Respond with"). */
|
|
126
|
+
mockStatus?: number
|
|
94
127
|
}
|
|
95
128
|
|
|
96
129
|
// ---------------- internals ----------------
|
|
@@ -169,7 +202,7 @@ function usesOf(
|
|
|
169
202
|
const out: VarUse[] = []
|
|
170
203
|
for (const name of names) {
|
|
171
204
|
if (isDeferred(name) || isFakerName(name)) continue
|
|
172
|
-
const key = name
|
|
205
|
+
const key = rootName(name)
|
|
173
206
|
if (seen.has(key)) continue
|
|
174
207
|
seen.add(key)
|
|
175
208
|
for (let i = layers.length - 1; i >= 0; i--) {
|
|
@@ -222,8 +255,22 @@ function substitute<T>(value: T, rt: Runtime, profile?: string): T {
|
|
|
222
255
|
return walk(value) as T
|
|
223
256
|
}
|
|
224
257
|
|
|
258
|
+
/** Events are masked as they are emitted: a listener (a stream, a log) must never see a secret value. */
|
|
225
259
|
function emit(rt: Runtime, e: RunEvent) {
|
|
226
|
-
rt.o.onEvent?.(e)
|
|
260
|
+
rt.o.onEvent?.(rt.mask.size ? maskDeep(e, rt.mask) : e)
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
const READ_METHODS = new Set(['GET', 'HEAD', 'OPTIONS'])
|
|
264
|
+
|
|
265
|
+
/** True when the request cannot change data: a read method, a GraphQL query, or one marked `safe`. */
|
|
266
|
+
function isReadOnly(def: RequestDef<any, any, any>, prepared: PreparedRequest): boolean {
|
|
267
|
+
if (def.input.safe) return true
|
|
268
|
+
if (READ_METHODS.has(prepared.method)) return true
|
|
269
|
+
if (prepared.encoding === 'graphql') {
|
|
270
|
+
const body = prepared.body as { query?: unknown } | undefined
|
|
271
|
+
return typeof body?.query === 'string' && !/(^|[\s}])mutation\b/.test(body.query)
|
|
272
|
+
}
|
|
273
|
+
return false
|
|
227
274
|
}
|
|
228
275
|
|
|
229
276
|
function profileFor(rt: Runtime, def: RequestDef<any, any, any>): string | undefined {
|
|
@@ -289,13 +336,13 @@ async function ensureSession(
|
|
|
289
336
|
|
|
290
337
|
/** What earlier steps and the sequence hand to a request. */
|
|
291
338
|
interface Flow {
|
|
292
|
-
|
|
339
|
+
inputs: Record<string, unknown>
|
|
293
340
|
captured: Record<string, unknown>
|
|
294
341
|
/** capture name -> the step that produced it */
|
|
295
342
|
from: Record<string, string>
|
|
296
343
|
}
|
|
297
344
|
|
|
298
|
-
const NO_FLOW: Flow = {
|
|
345
|
+
const NO_FLOW: Flow = { inputs: {}, captured: {}, from: {} }
|
|
299
346
|
|
|
300
347
|
interface ExecOut {
|
|
301
348
|
result: StepResult
|
|
@@ -328,7 +375,7 @@ async function execute(
|
|
|
328
375
|
const layers: [VarUse['source'], Record<string, unknown>][] = [
|
|
329
376
|
['config', rt.o.vars ?? {}],
|
|
330
377
|
['init', evalInit(def.input.init)],
|
|
331
|
-
['
|
|
378
|
+
['input', flow.inputs],
|
|
332
379
|
['capture', flow.captured],
|
|
333
380
|
['override', evalInit(def.varOverrides)],
|
|
334
381
|
['override', rt.o.overrides ?? {}],
|
|
@@ -388,12 +435,12 @@ async function execute(
|
|
|
388
435
|
encoding,
|
|
389
436
|
mediaType: encoding === 'binary' ? def.op.mediaType : undefined,
|
|
390
437
|
formArrays: def.op.formArrays,
|
|
391
|
-
timeoutMs: def.input.timeoutMs,
|
|
392
|
-
mockStatus: rt.o.env === 'mock' ?
|
|
438
|
+
timeoutMs: def.input.timeoutMs ?? rt.o.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
439
|
+
mockStatus: rt.o.env === 'mock' ? rt.o.mockStatus : undefined,
|
|
393
440
|
mock:
|
|
394
441
|
rt.o.env === 'mock'
|
|
395
442
|
? {
|
|
396
|
-
|
|
443
|
+
hook: def.mockHook,
|
|
397
444
|
vars: scope,
|
|
398
445
|
pathParams,
|
|
399
446
|
rawQuery: query,
|
|
@@ -412,6 +459,10 @@ async function execute(
|
|
|
412
459
|
profile,
|
|
413
460
|
}
|
|
414
461
|
emit(rt, { type: 'request', name, request: result.request })
|
|
462
|
+
if (rt.o.readOnly && kind !== 'auth' && !isReadOnly(def, prepared))
|
|
463
|
+
throw new StepError(
|
|
464
|
+
`Refusing ${prepared.method} ${def.op.path} in "${rt.o.env}": this is a --safe run, which only reads.`,
|
|
465
|
+
)
|
|
415
466
|
|
|
416
467
|
const send = async () => {
|
|
417
468
|
if (profile) await ensureSession(rt, profile, authResults)
|
|
@@ -458,6 +509,17 @@ async function execute(
|
|
|
458
509
|
for (const r of a.results) {
|
|
459
510
|
emit(rt, { type: r.pass ? 'assert:pass' : 'assert:fail', name, assertion: r })
|
|
460
511
|
}
|
|
512
|
+
} else if (rt.o.implicitStatusCheck && kind === 'request') {
|
|
513
|
+
const pass = response.status >= 200 && response.status < 300
|
|
514
|
+
const implicit: AssertionResult = {
|
|
515
|
+
label: 'status is 2xx',
|
|
516
|
+
operator: 'status',
|
|
517
|
+
pass,
|
|
518
|
+
actual: response.status,
|
|
519
|
+
expected: '2xx',
|
|
520
|
+
}
|
|
521
|
+
result.assertions = [implicit]
|
|
522
|
+
emit(rt, { type: pass ? 'assert:pass' : 'assert:fail', name, assertion: implicit })
|
|
461
523
|
}
|
|
462
524
|
if (result.status === 'pass' && result.assertions.some((r) => !r.pass)) result.status = 'fail'
|
|
463
525
|
} catch (e: any) {
|
|
@@ -583,7 +645,13 @@ export async function collectSecretNames(
|
|
|
583
645
|
}
|
|
584
646
|
|
|
585
647
|
function newRuntime(o: RunOptions): Runtime {
|
|
586
|
-
|
|
648
|
+
// every resolved secret is masked from the start, not only once it has been substituted into a request
|
|
649
|
+
return {
|
|
650
|
+
o,
|
|
651
|
+
sessions: new Map(),
|
|
652
|
+
inflight: new Map(),
|
|
653
|
+
mask: new Set(Object.values(o.secrets ?? {})),
|
|
654
|
+
}
|
|
587
655
|
}
|
|
588
656
|
|
|
589
657
|
export async function runRequest(
|
|
@@ -592,10 +660,22 @@ export async function runRequest(
|
|
|
592
660
|
): Promise<RunResult> {
|
|
593
661
|
const rt = newRuntime(o)
|
|
594
662
|
const t0 = o.now?.() ?? now0()
|
|
595
|
-
emit(rt, {
|
|
663
|
+
emit(rt, {
|
|
664
|
+
type: 'run:start',
|
|
665
|
+
id: def.op.id,
|
|
666
|
+
title: def.input.name ?? def.op.id,
|
|
667
|
+
kind: 'request',
|
|
668
|
+
tags: def.input.tags,
|
|
669
|
+
})
|
|
596
670
|
const name = def.input.name ?? def.op.id
|
|
597
671
|
const { result, authResults } = await execute(rt, def, name, NO_FLOW)
|
|
598
|
-
return finish(
|
|
672
|
+
return finish(
|
|
673
|
+
rt,
|
|
674
|
+
'request',
|
|
675
|
+
{ id: def.op.id, title: name, tags: def.input.tags },
|
|
676
|
+
[...authResults, result],
|
|
677
|
+
t0,
|
|
678
|
+
)
|
|
599
679
|
}
|
|
600
680
|
|
|
601
681
|
export async function runSequence(seq: Sequence<any, any>, o: RunOptions): Promise<RunResult> {
|
|
@@ -603,7 +683,13 @@ export async function runSequence(seq: Sequence<any, any>, o: RunOptions): Promi
|
|
|
603
683
|
rt.seqAuth = seq.meta.auth
|
|
604
684
|
rt.seqAssertMode = seq.meta.assertMode
|
|
605
685
|
const t0 = o.now?.() ?? now0()
|
|
606
|
-
emit(rt, {
|
|
686
|
+
emit(rt, {
|
|
687
|
+
type: 'run:start',
|
|
688
|
+
id: seq.meta.id,
|
|
689
|
+
title: seq.meta.title,
|
|
690
|
+
kind: 'sequence',
|
|
691
|
+
tags: seq.meta.tags,
|
|
692
|
+
})
|
|
607
693
|
const steps: StepResult[] = []
|
|
608
694
|
const captured: Record<string, any> = {}
|
|
609
695
|
const from: Record<string, string> = {}
|
|
@@ -611,12 +697,12 @@ export async function runSequence(seq: Sequence<any, any>, o: RunOptions): Promi
|
|
|
611
697
|
Object.assign(captured, values)
|
|
612
698
|
for (const k of Object.keys(values)) from[k] = step
|
|
613
699
|
}
|
|
614
|
-
let
|
|
700
|
+
let inputs: Record<string, unknown>
|
|
615
701
|
try {
|
|
616
|
-
|
|
702
|
+
inputs = resolveControls(seq.meta.inputs ?? {}, o.inputs)
|
|
617
703
|
} catch (e: any) {
|
|
618
704
|
steps.push({
|
|
619
|
-
name: '
|
|
705
|
+
name: 'inputs',
|
|
620
706
|
kind: 'input',
|
|
621
707
|
status: 'error',
|
|
622
708
|
assertions: [],
|
|
@@ -628,7 +714,7 @@ export async function runSequence(seq: Sequence<any, any>, o: RunOptions): Promi
|
|
|
628
714
|
}
|
|
629
715
|
const vars = o.vars ?? {}
|
|
630
716
|
const ctx = (): StepCtx<any, any> => ({
|
|
631
|
-
|
|
717
|
+
inputs,
|
|
632
718
|
captured,
|
|
633
719
|
vars: { ...vars, ...(o.overrides ?? {}) },
|
|
634
720
|
env: o.env,
|
|
@@ -652,7 +738,7 @@ export async function runSequence(seq: Sequence<any, any>, o: RunOptions): Promi
|
|
|
652
738
|
if (node.kind === 'step') {
|
|
653
739
|
const def = typeof node.def === 'function' ? node.def(ctx()) : node.def
|
|
654
740
|
const { result, authResults } = await execute(rt, def, node.name, {
|
|
655
|
-
|
|
741
|
+
inputs,
|
|
656
742
|
captured,
|
|
657
743
|
from,
|
|
658
744
|
})
|
|
@@ -675,7 +761,7 @@ export async function runSequence(seq: Sequence<any, any>, o: RunOptions): Promi
|
|
|
675
761
|
for (let i = 0; i < items.length; i++) {
|
|
676
762
|
const def = node.def(items[i], ctx(), i)
|
|
677
763
|
const { result, authResults } = await execute(rt, def, `${node.name} #${i + 1}`, {
|
|
678
|
-
|
|
764
|
+
inputs,
|
|
679
765
|
captured,
|
|
680
766
|
from,
|
|
681
767
|
})
|
|
@@ -700,9 +786,9 @@ export async function runSequence(seq: Sequence<any, any>, o: RunOptions): Promi
|
|
|
700
786
|
emit(rt, { type: 'input:required', name: node.name, controls: described })
|
|
701
787
|
const given = o.onInput
|
|
702
788
|
? await o.onInput({ name: node.name, controls: described })
|
|
703
|
-
: (o.
|
|
789
|
+
: (o.inputs ?? {})
|
|
704
790
|
const values = resolveControls(controls, given)
|
|
705
|
-
|
|
791
|
+
inputs = { ...inputs, ...values }
|
|
706
792
|
steps.push({
|
|
707
793
|
name: node.name,
|
|
708
794
|
kind: 'input',
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
// Running a list of requests and flows the way `specbook test` does: in order, retrying a failure, stopping at a number of
|
|
2
|
+
// failures or at a global deadline. The CLI and the docs site's terminal share it, so what the page shows is what a
|
|
3
|
+
// terminal does. Nothing here touches files or the process.
|
|
4
|
+
|
|
5
|
+
import type { Colors } from '../reporters/color'
|
|
6
|
+
import type { Sequence } from '../sequence'
|
|
7
|
+
import type { RequestDef, RunResult } from '../types'
|
|
8
|
+
import { type RunOptions, runRequest, runSequence } from './run'
|
|
9
|
+
|
|
10
|
+
export interface SuiteTarget {
|
|
11
|
+
title: string
|
|
12
|
+
/** project file, shown in reports */
|
|
13
|
+
file?: string
|
|
14
|
+
tags: string[]
|
|
15
|
+
run: RequestDef<any, any, any> | Sequence<any, any>
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface SuiteOptions {
|
|
19
|
+
/** what every run is given; `signal` is the suite's own */
|
|
20
|
+
run: Omit<RunOptions, 'signal'>
|
|
21
|
+
/** extra tries for a failing target (0 = one try) */
|
|
22
|
+
retries?: number
|
|
23
|
+
/** stop after this many failed targets */
|
|
24
|
+
bail?: number
|
|
25
|
+
/** milliseconds after which no further target starts, and a running one is aborted */
|
|
26
|
+
globalTimeoutMs?: number
|
|
27
|
+
/** called before each try, e.g. to seed faker again */
|
|
28
|
+
beforeAttempt?: () => void
|
|
29
|
+
/** called when a failing target is about to be tried again */
|
|
30
|
+
onRetry?: (target: SuiteTarget, attempt: number, allowed: number) => void
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface SuiteResult {
|
|
34
|
+
results: RunResult[]
|
|
35
|
+
failures: number
|
|
36
|
+
/** targets that did not start */
|
|
37
|
+
notRun: number
|
|
38
|
+
/** why the suite stopped early; empty when it did not */
|
|
39
|
+
stopped: string
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export async function runSuite(targets: SuiteTarget[], o: SuiteOptions): Promise<SuiteResult> {
|
|
43
|
+
const controller = new AbortController()
|
|
44
|
+
const deadline = o.globalTimeoutMs
|
|
45
|
+
? setTimeout(() => controller.abort(), o.globalTimeoutMs)
|
|
46
|
+
: undefined
|
|
47
|
+
const results: RunResult[] = []
|
|
48
|
+
const allowed = 1 + Math.max(0, o.retries ?? 0)
|
|
49
|
+
let failures = 0
|
|
50
|
+
let notRun = 0
|
|
51
|
+
let stopped = ''
|
|
52
|
+
for (const [i, t] of targets.entries()) {
|
|
53
|
+
if (controller.signal.aborted) {
|
|
54
|
+
notRun = targets.length - i
|
|
55
|
+
stopped = `stopped: the global timeout of ${o.globalTimeoutMs}ms passed`
|
|
56
|
+
break
|
|
57
|
+
}
|
|
58
|
+
let result: RunResult | undefined
|
|
59
|
+
let attempt = 0
|
|
60
|
+
while (attempt < allowed) {
|
|
61
|
+
attempt++
|
|
62
|
+
o.beforeAttempt?.()
|
|
63
|
+
const opts = { ...o.run, signal: controller.signal }
|
|
64
|
+
result =
|
|
65
|
+
t.run.kind === 'sequence' ? await runSequence(t.run, opts) : await runRequest(t.run, opts)
|
|
66
|
+
if (result.status === 'pass' || controller.signal.aborted) break
|
|
67
|
+
if (attempt < allowed) o.onRetry?.(t, attempt + 1, allowed)
|
|
68
|
+
}
|
|
69
|
+
const final = result as RunResult
|
|
70
|
+
results.push({
|
|
71
|
+
...final,
|
|
72
|
+
file: t.file,
|
|
73
|
+
attempts: attempt,
|
|
74
|
+
flaky: final.status === 'pass' && attempt > 1,
|
|
75
|
+
tags: final.tags.length ? final.tags : t.tags,
|
|
76
|
+
})
|
|
77
|
+
if (final.status !== 'pass') failures++
|
|
78
|
+
if (o.bail && failures >= o.bail && i < targets.length - 1) {
|
|
79
|
+
notRun = targets.length - i - 1
|
|
80
|
+
stopped = `stopped after ${failures} failed ${failures === 1 ? 'flow' : 'flows'} (--bail)`
|
|
81
|
+
break
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
if (deadline) clearTimeout(deadline)
|
|
85
|
+
return { results, failures, notRun, stopped }
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** The line a run opens with: the tool and version, then what is being run and where. */
|
|
89
|
+
export function headerLine(
|
|
90
|
+
h: {
|
|
91
|
+
version: string
|
|
92
|
+
env: string
|
|
93
|
+
picked: number
|
|
94
|
+
total: number
|
|
95
|
+
/** `main@abc1234`, `bundle <sha> built <date>` */
|
|
96
|
+
where?: string[]
|
|
97
|
+
},
|
|
98
|
+
colors: Colors,
|
|
99
|
+
): string {
|
|
100
|
+
return `${colors.bold(`specbook test ${h.version}`)} ${colors.dim(
|
|
101
|
+
[
|
|
102
|
+
`env ${h.env}`,
|
|
103
|
+
`${h.picked}${h.picked < h.total ? `/${h.total}` : ''} tests`,
|
|
104
|
+
...(h.where ?? []),
|
|
105
|
+
].join(' · '),
|
|
106
|
+
)}\n\n`
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** What a retry looks like in a terminal. */
|
|
110
|
+
export const retryLine = (colors: Colors, title: string, attempt: number, allowed: number) =>
|
|
111
|
+
` ${colors.yellow('↻')} retrying ${title} (attempt ${attempt} of ${allowed})\n`
|
|
112
|
+
|
|
113
|
+
/** A bundle's assertions point into the bundled file, which tells a reader nothing. */
|
|
114
|
+
export function withoutLocations(results: RunResult[]): RunResult[] {
|
|
115
|
+
const strip = (steps: RunResult['steps']): RunResult['steps'] =>
|
|
116
|
+
steps.map((s) => ({
|
|
117
|
+
...s,
|
|
118
|
+
assertions: s.assertions.map((a) => ({ ...a, location: undefined })),
|
|
119
|
+
...(s.children ? { children: strip(s.children) } : {}),
|
|
120
|
+
}))
|
|
121
|
+
return results.map((r) => ({ ...r, steps: strip(r.steps) }))
|
|
122
|
+
}
|
package/src/runner/transport.ts
CHANGED
|
@@ -16,11 +16,11 @@ export interface PreparedRequest {
|
|
|
16
16
|
/** Array style per form field, from the spec's `encoding` object. */
|
|
17
17
|
formArrays?: Record<string, FormArrays>
|
|
18
18
|
timeoutMs?: number
|
|
19
|
-
/** Mock status the caller wants served (mock env only). */
|
|
19
|
+
/** Mock status the caller wants served, e.g. the UI's "Respond with" (mock env only). */
|
|
20
20
|
mockStatus?: number
|
|
21
21
|
/** In-process hints for the mock transport; never serialized or substituted. */
|
|
22
22
|
mock?: {
|
|
23
|
-
|
|
23
|
+
hook?: import('../types').MockHook<any, any>
|
|
24
24
|
vars: Record<string, unknown>
|
|
25
25
|
pathParams: Record<string, unknown>
|
|
26
26
|
rawQuery: Record<string, unknown>
|
|
@@ -46,6 +46,52 @@ export function buildUrl(base: string, req: PreparedRequest): string {
|
|
|
46
46
|
return `${base.replace(/\/+$/, '')}${req.url}${q ? `?${q}` : ''}`
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
+
/** A config `baseUrl` may be relative (`/api/v1`): it is resolved against the page. */
|
|
50
|
+
export function absoluteBase(baseUrl: string, pageHref?: string): string {
|
|
51
|
+
try {
|
|
52
|
+
return new URL(baseUrl, pageHref).toString()
|
|
53
|
+
} catch {
|
|
54
|
+
return baseUrl
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** How long a request waits for an answer when nothing says otherwise. */
|
|
59
|
+
export const DEFAULT_TIMEOUT_MS = 30_000
|
|
60
|
+
|
|
61
|
+
/** What node's fetch says in `error.cause.code`, in words a person can act on. */
|
|
62
|
+
const WHY: Record<string, string> = {
|
|
63
|
+
ECONNREFUSED: 'connection refused',
|
|
64
|
+
ENOTFOUND: 'host not found',
|
|
65
|
+
EAI_AGAIN: 'host not found',
|
|
66
|
+
ECONNRESET: 'connection reset',
|
|
67
|
+
ETIMEDOUT: 'connection timed out',
|
|
68
|
+
UND_ERR_CONNECT_TIMEOUT: 'connection timed out',
|
|
69
|
+
EHOSTUNREACH: 'host unreachable',
|
|
70
|
+
ENETUNREACH: 'network unreachable',
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** node's bare "fetch failed" says nothing; its cause says why. A browser gives no cause, so its error is left as it is. */
|
|
74
|
+
function unreachable(e: unknown, url: string): unknown {
|
|
75
|
+
const cause = (
|
|
76
|
+
e as { cause?: { code?: string; message?: string; errors?: { code?: string }[] } } | undefined
|
|
77
|
+
)?.cause
|
|
78
|
+
if (!(e instanceof TypeError) || !cause) return e
|
|
79
|
+
const code = cause.code ?? cause.errors?.find((x) => x.code)?.code
|
|
80
|
+
const tls =
|
|
81
|
+
code && /^CERT_|^ERR_TLS|SELF_SIGNED|UNABLE_TO_VERIFY/.test(code) ? 'certificate problem' : ''
|
|
82
|
+
// a reason there are words for, else the cause's own message ("bad port"), else nothing to add
|
|
83
|
+
const own = typeof cause.message === 'string' && cause.message.length <= 120 ? cause.message : ''
|
|
84
|
+
const why = (code && WHY[code]) || tls || own
|
|
85
|
+
if (!why) return e
|
|
86
|
+
let host = url
|
|
87
|
+
try {
|
|
88
|
+
host = new URL(url).host
|
|
89
|
+
} catch {
|
|
90
|
+
/* an address that is not a URL is shown as it is */
|
|
91
|
+
}
|
|
92
|
+
return new Error(`Could not reach ${host}: ${why}.`, { cause: e })
|
|
93
|
+
}
|
|
94
|
+
|
|
49
95
|
export interface FetchTransportOptions {
|
|
50
96
|
baseUrl: (service: string) => string
|
|
51
97
|
fetch?: typeof fetch
|
|
@@ -53,6 +99,8 @@ export interface FetchTransportOptions {
|
|
|
53
99
|
|
|
54
100
|
export function createFetchTransport(opts: FetchTransportOptions): Transport {
|
|
55
101
|
return async (req, signal) => {
|
|
102
|
+
// a run stopped before this request left: nothing is sent, and no 'abort' will ever be heard from a signal that has fired
|
|
103
|
+
if (signal?.aborted) throw new Error('Stopped before the API answered.')
|
|
56
104
|
const f = opts.fetch ?? fetch
|
|
57
105
|
const headers: Record<string, string> = { ...req.headers }
|
|
58
106
|
let body: BodyInit | undefined
|
|
@@ -61,13 +109,19 @@ export function createFetchTransport(opts: FetchTransportOptions): Transport {
|
|
|
61
109
|
body = enc.body
|
|
62
110
|
if (enc.contentType) headers['content-type'] ??= enc.contentType
|
|
63
111
|
}
|
|
64
|
-
if (req.mockStatus !== undefined) headers['x-specbook-mock-status'] = String(req.mockStatus)
|
|
65
112
|
|
|
66
113
|
const ctl = new AbortController()
|
|
67
|
-
|
|
114
|
+
let timedOut = false
|
|
115
|
+
const timer = req.timeoutMs
|
|
116
|
+
? setTimeout(() => {
|
|
117
|
+
timedOut = true
|
|
118
|
+
ctl.abort()
|
|
119
|
+
}, req.timeoutMs)
|
|
120
|
+
: undefined
|
|
68
121
|
signal?.addEventListener('abort', () => ctl.abort())
|
|
122
|
+
const url = buildUrl(opts.baseUrl(req.service), req)
|
|
69
123
|
try {
|
|
70
|
-
const res = await f(
|
|
124
|
+
const res = await f(url, {
|
|
71
125
|
method: req.method,
|
|
72
126
|
headers,
|
|
73
127
|
body,
|
|
@@ -88,6 +142,12 @@ export function createFetchTransport(opts: FetchTransportOptions): Transport {
|
|
|
88
142
|
}
|
|
89
143
|
}
|
|
90
144
|
return { status: res.status, headers: outHeaders, text, body: parsed }
|
|
145
|
+
} catch (e) {
|
|
146
|
+
// an abort that the request's own timeout caused says so, not "This operation was aborted"
|
|
147
|
+
if (timedOut) throw new Error(`Timed out after ${req.timeoutMs}ms: no answer from the API.`)
|
|
148
|
+
// a person pressing Stop, or the run's own deadline: say so, not "This operation was aborted"
|
|
149
|
+
if (signal?.aborted) throw new Error('Stopped before the API answered.')
|
|
150
|
+
throw unreachable(e, url)
|
|
91
151
|
} finally {
|
|
92
152
|
if (timer) clearTimeout(timer)
|
|
93
153
|
}
|
package/src/secrets.ts
CHANGED
|
@@ -67,9 +67,38 @@ export async function resolveSecrets(
|
|
|
67
67
|
return out
|
|
68
68
|
}
|
|
69
69
|
|
|
70
|
-
/**
|
|
70
|
+
/** Secrets shorter than this are not masked: they would blank out ordinary text. */
|
|
71
|
+
const MIN_MASKED = 3
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* A secret as it can appear in a URL, a JSON string or a Basic-auth style base64 value, besides itself: what a server
|
|
75
|
+
* echoes back or an error message quotes is not always the raw value.
|
|
76
|
+
*/
|
|
77
|
+
export function maskVariants(secret: string): string[] {
|
|
78
|
+
const out = new Set<string>([secret])
|
|
79
|
+
const add = (f: () => string) => {
|
|
80
|
+
try {
|
|
81
|
+
out.add(f())
|
|
82
|
+
} catch {
|
|
83
|
+
/* not encodable: the raw value is still masked */
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
add(() => encodeURIComponent(secret))
|
|
87
|
+
add(() => encodeURI(secret))
|
|
88
|
+
add(() => JSON.stringify(secret).slice(1, -1))
|
|
89
|
+
add(() => btoa(String.fromCharCode(...new TextEncoder().encode(secret))))
|
|
90
|
+
return [...out]
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Masks with `***`, longest first so a secret containing another is hidden whole. Every secret is masked in its
|
|
95
|
+
* raw, URL-encoded, JSON-escaped and base64 forms; values under 3 characters are skipped to avoid masking
|
|
96
|
+
* ordinary text.
|
|
97
|
+
*/
|
|
71
98
|
export function maskDeep<T>(value: T, secrets: Iterable<string>): T {
|
|
72
|
-
const list = [
|
|
99
|
+
const list = [
|
|
100
|
+
...new Set([...secrets].filter((s) => s.length >= MIN_MASKED).flatMap(maskVariants)),
|
|
101
|
+
].sort((a, b) => b.length - a.length)
|
|
73
102
|
if (list.length === 0) return value
|
|
74
103
|
const maskStr = (s: string) => list.reduce((acc, v) => acc.split(v).join('***'), s)
|
|
75
104
|
const walk = (v: unknown): unknown => {
|
package/src/sequence.ts
CHANGED
|
@@ -2,7 +2,7 @@ import type { Controls, InferControls } from './controls'
|
|
|
2
2
|
import type { Assert, AssertMode, AuthProfileName, RequestDef } from './types'
|
|
3
3
|
|
|
4
4
|
export interface StepCtx<P, C> {
|
|
5
|
-
|
|
5
|
+
inputs: P
|
|
6
6
|
captured: C
|
|
7
7
|
vars: Record<string, unknown>
|
|
8
8
|
env: string
|
|
@@ -34,7 +34,7 @@ export interface SequenceMeta {
|
|
|
34
34
|
tags?: string[]
|
|
35
35
|
/** Heading the flow is listed under in the catalog; flows without one are under "Default". */
|
|
36
36
|
group?: string
|
|
37
|
-
|
|
37
|
+
inputs?: Controls
|
|
38
38
|
auth?: AuthProfileName | false
|
|
39
39
|
assertMode?: AssertMode
|
|
40
40
|
continueOnFailure?: boolean
|
|
@@ -55,7 +55,7 @@ export interface Sequence<P = {}, C = {}> {
|
|
|
55
55
|
name: string,
|
|
56
56
|
fn: (ctx: StepCtx<P, C>, h: { assert: Assert }) => void | Promise<void>,
|
|
57
57
|
): Sequence<P, C>
|
|
58
|
-
/** Pause for user input; the chosen values merge into ctx.
|
|
58
|
+
/** Pause for user input; the chosen values merge into ctx.inputs. */
|
|
59
59
|
input<R extends Controls>(
|
|
60
60
|
name: string,
|
|
61
61
|
controls: (ctx: StepCtx<P, C>) => R,
|
|
@@ -77,7 +77,7 @@ function build(meta: SequenceMeta, nodes: readonly StepNode[]): Sequence<any, an
|
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
export function sequence<P extends Controls = {}>(
|
|
80
|
-
meta: Omit<SequenceMeta, '
|
|
80
|
+
meta: Omit<SequenceMeta, 'inputs'> & { inputs?: P },
|
|
81
81
|
): Sequence<InferControls<P>, {}> {
|
|
82
82
|
return build(meta, [])
|
|
83
83
|
}
|